From 6f2e96a3b03f61707fbab6f514d43b95e53aaaa8 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 23 Feb 2024 15:42:32 -0500 Subject: [PATCH 01/12] use ouranosinc/cookiecutter-pypackage template with cruft, update secrets --- .cruft.json | 29 +++ .editorconfig | 2 +- .flake8 | 30 +++ .github/ISSUE_TEMPLATE.md | 19 +- .../0001-GENERIC-ISSUE-TEMPLATE.yml | 34 +++ .github/ISSUE_TEMPLATE/0002-BUG-REPORT.yml | 44 ++++ .../ISSUE_TEMPLATE/0003-FEATURE-REQUEST.yml | 31 +++ .../ISSUE_TEMPLATE/0004-QUESTION-SUPPORT.yml | 23 ++ .github/ISSUE_TEMPLATE/config.yml | 1 + .github/PULL_REQUEST_TEMPLATE.md | 10 +- .github/dependabot.yml | 15 ++ .github/labeler.yml | 20 ++ .github/workflows/auto-bumpversion.yml | 43 ---- .github/workflows/bump-version.yml | 86 +++++++ .github/workflows/cache-cleaner.yml | 49 ++++ .github/workflows/dependency-review.yml | 31 +++ .github/workflows/first-pull-request.yml | 54 ++++ .github/workflows/label.yml | 37 +++ .github/workflows/main.yml | 112 +++++++-- .github/workflows/publish-pypi.yml | 30 ++- .github/workflows/scorecard.yml | 82 ++++++ .github/workflows/tag-testpypi.yml | 57 ++++- .github/workflows/workflow-warning.yml | 69 +++++ .gitignore | 114 +++++++-- .pre-commit-config.yaml | 44 ++-- .readthedocs.yml | 18 +- .secrets.baseline | 14 +- .yamllint.yaml | 2 +- .zenodo.json | 16 ++ HISTORY.rst => CHANGES.rst | 13 +- CONTRIBUTING.rst | 230 ++++++++++++++--- Makefile | 125 ++++++---- README.rst | 70 ++++-- docs/Makefile | 179 +------------ docs/changes.rst | 1 + docs/conf.py | 21 +- docs/history.rst | 1 - docs/index.rst | 19 +- docs/installation.rst | 37 ++- docs/make.bat | 236 ++---------------- docs/readme.rst | 1 + docs/usage.rst | 6 +- environment.yml => environment-dev.yml | 34 +-- environment-docs.yml | 16 ++ miranda/__init__.py | 5 +- miranda/archive/__init__.py | 1 + miranda/archive/_selection.py | 3 +- miranda/convert/__init__.py | 1 + miranda/convert/_aggregation.py | 5 +- miranda/convert/_data_corrections.py | 12 +- miranda/convert/_data_definitions.py | 9 +- miranda/convert/_reconstruction.py | 2 +- miranda/convert/deh.py | 13 +- miranda/convert/eccc.py | 1 + miranda/convert/eccc_rdrs.py | 3 +- miranda/convert/ecmwf.py | 1 + miranda/convert/hq.py | 1 + miranda/convert/melcc.py | 25 +- miranda/convert/utils.py | 3 +- miranda/cv.py | 1 + miranda/data.py | 1 + miranda/decode/__init__.py | 2 + miranda/decode/_decoder.py | 24 +- miranda/decode/_time.py | 3 +- miranda/eccc/__init__.py | 1 + miranda/eccc/_homogenized.py | 1 + miranda/eccc/_raw.py | 60 +++-- miranda/eccc/_summaries.py | 5 +- miranda/eccc/_utils.py | 6 +- miranda/ecmwf/__init__.py | 1 + miranda/ecmwf/_era5.py | 6 +- miranda/gis/__init__.py | 1 + miranda/gis/_domains.py | 6 +- miranda/io/__init__.py | 1 + miranda/io/_output.py | 5 +- miranda/io/_rechunk.py | 2 +- miranda/io/utils.py | 1 + miranda/ncar/__init__.py | 1 + miranda/remote/__init__.py | 1 + miranda/remote/archiver.py | 1 + miranda/remote/connect.py | 9 +- miranda/remote/ops.py | 21 +- miranda/remote/remove.py | 3 +- miranda/scripting.py | 1 + miranda/storage.py | 11 +- miranda/structure/__init__.py | 1 + miranda/structure/_structure.py | 12 +- miranda/units.py | 1 + miranda/utils.py | 5 +- miranda/validators.py | 1 + pyproject.toml | 194 ++++++++++++-- setup.cfg | 18 +- tests/test_miranda.py | 14 ++ tox.ini | 76 +++--- 94 files changed, 1814 insertions(+), 873 deletions(-) create mode 100644 .cruft.json create mode 100644 .flake8 create mode 100644 .github/ISSUE_TEMPLATE/0001-GENERIC-ISSUE-TEMPLATE.yml create mode 100644 .github/ISSUE_TEMPLATE/0002-BUG-REPORT.yml create mode 100644 .github/ISSUE_TEMPLATE/0003-FEATURE-REQUEST.yml create mode 100644 .github/ISSUE_TEMPLATE/0004-QUESTION-SUPPORT.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/dependabot.yml create mode 100644 .github/labeler.yml delete mode 100644 .github/workflows/auto-bumpversion.yml create mode 100644 .github/workflows/bump-version.yml create mode 100644 .github/workflows/cache-cleaner.yml create mode 100644 .github/workflows/dependency-review.yml create mode 100644 .github/workflows/first-pull-request.yml create mode 100644 .github/workflows/label.yml create mode 100644 .github/workflows/scorecard.yml create mode 100644 .github/workflows/workflow-warning.yml create mode 100644 .zenodo.json rename HISTORY.rst => CHANGES.rst (94%) create mode 100644 docs/changes.rst delete mode 100644 docs/history.rst create mode 100644 docs/readme.rst rename environment.yml => environment-dev.yml (53%) create mode 100644 environment-docs.yml diff --git a/.cruft.json b/.cruft.json new file mode 100644 index 00000000..a25f70e7 --- /dev/null +++ b/.cruft.json @@ -0,0 +1,29 @@ +{ + "template": "https://github.com/Ouranosinc/cookiecutter-pypackage", + "commit": "88b1bfe181d9f04c0905137be04150ea7f7b950f", + "checkout": null, + "context": { + "cookiecutter": { + "full_name": "Trevor James Smith", + "email": "smith.trevorj@ouranos.ca", + "github_username": "Zeitsperre", + "project_name": "miranda", + "project_slug": "miranda", + "project_short_description": "Python utilities for climate data collection, conversion, and management.", + "pypi_username": "Zeitsperre", + "version": "0.6.0", + "use_pytest": "y", + "use_black": "y", + "use_conda": "y", + "add_pyup_badge": "n", + "make_docs": "y", + "add_translations": "y", + "command_line_interface": "No command-line interface", + "create_author_file": "y", + "open_source_license": "Apache Software License 2.0", + "generated_with_cruft": "y", + "_template": "https://github.com/Ouranosinc/cookiecutter-pypackage" + } + }, + "directory": null +} diff --git a/.editorconfig b/.editorconfig index 5a2a23a6..70b87258 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,7 +10,7 @@ insert_final_newline = true charset = utf-8 end_of_line = lf -[*{.yaml,.yml}] +[*.{yaml,yml}] indent_size = 2 [*.bat] diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..1116575b --- /dev/null +++ b/.flake8 @@ -0,0 +1,30 @@ +[flake8] +exclude = + .eggs, + .git, + build, + docs, + tests +ignore = + AZ100, + AZ200, + AZ300, + C, + D, + E, + F, + W503 +per-file-ignores = +rst-roles = + doc, + mod, + py:attr, + py:attribute, + py:class, + py:const, + py:data, + py:func, + py:meth, + py:mod, + py:obj, + py:ref diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index f56bf782..7b013080 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -3,24 +3,13 @@ * Operating System: ### Description - +Describe what you were trying to get done. +Tell us what happened, what went wrong, and what you expected to happen. ### What I Did - -``` -$ pip install foo --bar -``` -### What I Received - ``` -Traceback (most recent call last): - File "/path/to/file/script.py", line 3326, in run_code - exec(code_obj, self.user_global_ns, self.user_ns) - File "", line 1, in - 1/0 -ZeroDivisionError: division by zero +Paste the command(s) you ran and the output. +If there was a crash, please include the traceback here. ``` diff --git a/.github/ISSUE_TEMPLATE/0001-GENERIC-ISSUE-TEMPLATE.yml b/.github/ISSUE_TEMPLATE/0001-GENERIC-ISSUE-TEMPLATE.yml new file mode 100644 index 00000000..85f4c364 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/0001-GENERIC-ISSUE-TEMPLATE.yml @@ -0,0 +1,34 @@ +name: Generic issue template +description: For detailing generic/uncategorized issues in miranda + +body: + - type: textarea + id: generic-issue + attributes: + label: Generic Issue + description: Please fill in the following information fields as needed. + value: | + * miranda version: + * Python version: + * Operating System: + + ### Description + + + ### What I Did + + ``` + $ pip install foo --bar + ``` + + ### What I Received + + ``` + Traceback (most recent call last): + File "/path/to/file/script.py", line 3326, in run_code + exec(code_obj, self.user_global_ns, self.user_ns) + File "", line 1, in + 1/0 + ZeroDivisionError: division by zero diff --git a/.github/ISSUE_TEMPLATE/0002-BUG-REPORT.yml b/.github/ISSUE_TEMPLATE/0002-BUG-REPORT.yml new file mode 100644 index 00000000..3b790f19 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/0002-BUG-REPORT.yml @@ -0,0 +1,44 @@ +name: Bug report +description: Help us improve miranda +labels: [ "bug" ] + +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + - type: textarea + id: setup-information + attributes: + label: Setup Information + description: | + What software versions are you running? Example: + - miranda version: 0.55.0-gamma + - Python version: 4.2 + - Operating System: Nutmeg Linux 12.34 | macOS 11.0 "Redmond" + value: | + - miranda version: + - Python version: + - Operating System: + - type: textarea + id: description + attributes: + label: Description + description: Describe what you were trying to get done. Tell us what happened, what went wrong, and what you expected to happen. + - type: textarea + id: steps-to-reproduce + attributes: + label: Steps To Reproduce + description: Paste the command(s) you ran and the output. If there was a crash, please include the traceback below. + - type: textarea + id: additional-context + attributes: + label: Additional context + description: Add any other context about the problem here. + - type: checkboxes + id: submit-pr + attributes: + label: Contribution + description: Do you intend to submit a fix for this bug? (The miranda developers will help with code compliance) + options: + - label: I would be willing/able to open a Pull Request to address this bug. diff --git a/.github/ISSUE_TEMPLATE/0003-FEATURE-REQUEST.yml b/.github/ISSUE_TEMPLATE/0003-FEATURE-REQUEST.yml new file mode 100644 index 00000000..ec027abc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/0003-FEATURE-REQUEST.yml @@ -0,0 +1,31 @@ +name: Feature request +description: Suggest an idea for miranda +labels: [ "enhancement" ] + +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this feature request! + - type: textarea + id: problem + attributes: + label: Addressing a Problem? + description: Is your feature request related to a problem? Please describe it. + - type: textarea + id: potential-solution + attributes: + label: Potential Solution + description: Describe the solution you'd like to see implemented. + - type: textarea + id: additional-context + attributes: + label: Additional context + description: Add any other context about the feature request here. + - type: checkboxes + id: submit-pr + attributes: + label: Contribution + description: Do you intend to submit a fix for this bug? (The miranda developers will help with code compliance) + options: + - label: I would be willing/able to open a Pull Request to contribute this feature. diff --git a/.github/ISSUE_TEMPLATE/0004-QUESTION-SUPPORT.yml b/.github/ISSUE_TEMPLATE/0004-QUESTION-SUPPORT.yml new file mode 100644 index 00000000..f52cd47f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/0004-QUESTION-SUPPORT.yml @@ -0,0 +1,23 @@ +name: Question/Support +description: Ask for help from the developers +labels: [ "support" ] + +body: + - type: textarea + id: setup-information + attributes: + label: Setup Information + description: | + What software versions are you running? Example: + - miranda version: 0.55.0-gamma + - Python version: 4.2 + - Operating System: Nutmeg Linux 12.34 | macOS 11.0 "Redmond" + value: | + - miranda version: + - Python version: + - Operating System: + - type: textarea + id: description + attributes: + label: Context + description: Describe what you were trying to get done. Tell us what happened, what went wrong, and what you expected to happen. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..0086358d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 010f7677..dc5f20e3 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,12 +1,12 @@ - + ### Pull Request Checklist: - [ ] This PR addresses an already opened issue (for bug fixes / features) - This PR fixes #xyz -- [ ] Tests for the changes have been added (for bug fixes / features) - - [ ] (If applicable) Documentation has been added / updated (for bug fixes / features) -- [ ] HISTORY.rst has been updated (with summary of main changes) - - [ ] Link to issue (:issue:`number`) and pull request (:pull:`number`) has been added +- [ ] (If applicable) Documentation has been added / updated (for bug fixes / features). +- [ ] (If applicable) Tests have been added. +- [ ] CHANGES.rst has been updated (with summary of main changes). + - [ ] Link to issue (:issue:`number`) and pull request (:pull:`number`) has been added. ### What kind of change does this PR introduce? diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..a9f28754 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: daily + time: '12:00' + open-pull-requests-limit: 10 + + - package-ecosystem: pip + directory: / + schedule: + interval: daily + time: '12:00' + open-pull-requests-limit: 10 diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 00000000..c88b1a73 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,20 @@ +# label rules used by .github/workflows/label.yml + +# label 'ci' all automation-related steps and files +# Since this repository is in itself an automation process to deploy a server instance, +# we refer here to CI as the 'meta' configuration files for managing the code and integrations with the repository, +# not configurations related to the deployment process itself. + +# Uncomment the following lines to enable the labeler (requires labels with the same name to exist in the repository) + +# label 'ci' all automation-related steps and files +#'CI': +# - changed-files: +# - any-glob-to-any-file: +# - '.editorconfig' +# - '.flake8' +# - '.pre-commit-config.yaml' +# - '.yamllint.yml' +# - '.github/workflows/*' +# - 'tox.ini' +# - 'Makefile' diff --git a/.github/workflows/auto-bumpversion.yml b/.github/workflows/auto-bumpversion.yml deleted file mode 100644 index 11f77c15..00000000 --- a/.github/workflows/auto-bumpversion.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: "Bump Patch Version" - -on: - push: - branches: - - main - paths-ignore: - - HISTORY.rst - - pyproject.toml - - setup.cfg - - tox.ini - - miranda/__init__.py - - .github/**.yml - - .pre-commit-config.yaml - -jobs: - bump_patch_version: - runs-on: ubuntu-latest - name: Bump patch version - steps: - - uses: actions/checkout@v3 - with: - persist-credentials: false - - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - name: Config Commit Bot - run: | - git config --local user.email "bumpversion[bot]@ouranos.ca" - git config --local user.name "bumpversion[bot]" - - name: Current Version - run: echo "current_version=$(grep -E '__version__' miranda/__init__.py | cut -d ' ' -f3)" - - name: Bump Patch Version - run: | - pip install bump2version - bump2version patch - echo "new_version=$(grep -E '__version__' miranda/__init__.py | cut -d ' ' -f3)" - - name: Push Changes - uses: ad-m/github-push-action@master - with: - force: false - github_token: ${{ secrets.BUMPVERSION_TOKEN }} - branch: ${{ github.ref }} diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml new file mode 100644 index 00000000..36e1247f --- /dev/null +++ b/.github/workflows/bump-version.yml @@ -0,0 +1,86 @@ +# This workflow requires a personal access token named `BUMP_VERSION_TOKEN` with the following privileges: +# - Contents: Read and Write +# - Metadata: Read-Only +# - Pull Requests: Read and Write + +name: "Bump Patch Version" + +on: + push: + branches: + - main + paths-ignore: + - .cruft.json + - .editorconfig + - .github/**.yml + - .gitignore + - .pre-commit-config.yaml + - .yamllint.yaml + - .zenodo.json + - AUTHORS.rst + - CHANGES.rst + - CONTRIBUTING.rst + - Makefile + - .readthedocs.yml + - docs/*.py + - docs/*.rst + - environment-docs.yml + - environment-dev.yml + - pyproject.toml + - tests/**.py + - tox.ini + - miranda/__init__.py + workflow_dispatch: + +permissions: + contents: read + +jobs: + bump_patch_version: + runs-on: ubuntu-latest + permissions: + actions: read + contents: write + steps: + - name: Harden Runner + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + files.pythonhosted.org:443 + github.com:443 + pypi.org:443 + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Config Commit Bot + run: | + git config --local user.email "bumpversion[bot]@ouranos.ca" + git config --local user.name "bumpversion[bot]" + - name: Install bump-my-version + run: | + python -m pip install "bump-my-version>=0.17.1" + - name: Current Version + run: | + bump-my-version show current_version + CURRENT_VERSION="$(grep -E '__version__' miranda/__init__.py | cut -d ' ' -f3)" + echo "CURRENT_VERSION=${CURRENT_VERSION}" >> $GITHUB_ENV + - name: Conditional Bump Version + run: | + if [[ ${{ env.CURRENT_VERSION }} =~ -dev(\.\d+)? ]]; then + echo "Development version (ends in 'dev(\.\d+)?'), bumping 'build' version" + bump-my-version show new_version --increment build + else + echo "Version is stable, bumping 'patch' version" + bump-my-version show new_version --increment patch + fi + - name: Push Changes + uses: ad-m/github-push-action@master + with: + force: false + github_token: ${{ secrets.BUMP_VERSION_TOKEN }} + branch: ${{ github.ref }} diff --git a/.github/workflows/cache-cleaner.yml b/.github/workflows/cache-cleaner.yml new file mode 100644 index 00000000..d48a7f68 --- /dev/null +++ b/.github/workflows/cache-cleaner.yml @@ -0,0 +1,49 @@ +# Example taken from https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#managing-caches +name: Cleanup Caches on Pull Request Merge +on: + pull_request: + types: + - closed + +permissions: # added using https://github.com/step-security/secure-repo + contents: read + +jobs: + cleanup: + name: Cleanup + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Harden Runner + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + objects.githubusercontent.com:443 + + - uses: actions/checkout@v4.1.1 + + - name: Cleanup + run: | + gh extension install actions/gh-actions-cache + + REPO=${{ github.repository }} + BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge" + + echo "Fetching list of cache key" + cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH -L 100 | cut -f 1 ) + + ## Setting this to not fail the workflow while deleting cache keys. + set +e + echo "Deleting caches..." + for cacheKey in $cacheKeysForPR + do + gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm + done + echo "Done" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 00000000..c977388d --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,31 @@ +# Dependency Review Action +# +# This Action will scan dependency manifest files that change as part of a Pull Request, surfacing known-vulnerable versions of the packages declared or updated in the PR. Once installed, if the workflow run is marked as required, PRs introducing known-vulnerable packages will be blocked from merging. +# +# Source repository: https://github.com/actions/dependency-review-action +# Public documentation: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement +name: 'Dependency Review' +on: + pull_request: + +permissions: + contents: read + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + + - name: 'Checkout Repository' + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: 'Dependency Review' + uses: actions/dependency-review-action@4901385134134e04cec5fbe5ddfe3b2c5bd5d976 diff --git a/.github/workflows/first-pull-request.yml b/.github/workflows/first-pull-request.yml new file mode 100644 index 00000000..dfaca22d --- /dev/null +++ b/.github/workflows/first-pull-request.yml @@ -0,0 +1,54 @@ +name: First Pull Request + +on: + pull_request_target: + types: + - opened + +jobs: + welcome: + name: Welcome + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + + - uses: actions/github-script@v6 + with: + script: | + // Get a list of all issues created by the PR opener + // See: https://octokit.github.io/rest.js/#pagination + const creator = context.payload.sender.login + const opts = github.rest.issues.listForRepo.endpoint.merge({ + ...context.issue, + creator, + state: 'all' + }) + const issues = await github.paginate(opts) + + for (const issue of issues) { + if (issue.number === context.issue.number) { + continue + } + + if (issue.pull_request) { + return // Creator is already a contributor. + } + } + + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `**Welcome**, new contributor! + + It appears that this is your first Pull Request. To give credit where it's due, we ask that you add your information to the \`AUTHORS.rst\` and \`.zenodo.json\`: + - [ ] The relevant author information has been added to \`AUTHORS.rst\` and \`.zenodo.json\`. + + Please make sure you've read our [contributing guide](CONTRIBUTING.rst). We look forward to reviewing your Pull Request shortly ✨` + }) diff --git a/.github/workflows/label.yml b/.github/workflows/label.yml new file mode 100644 index 00000000..00036d9c --- /dev/null +++ b/.github/workflows/label.yml @@ -0,0 +1,37 @@ +# This workflow will triage pull requests and apply a label based on the +# paths that are modified in the pull request. +# +# To use this workflow, you will need to set up a .github/labeler.yml +# file with configuration. For more information, see: +# https://github.com/actions/labeler/blob/master/README.md + +name: Labeler +on: + pull_request_target: + # Note: potential security risk from this action using pull_request_target. + # Do not add actions in here which need a checkout of the repo, and do not use any caching in here. + # See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target + +permissions: + contents: read + +jobs: + label: + name: Label + runs-on: ubuntu-latest + permissions: + checks: write + contents: read + pull-requests: write + steps: + - name: Harden Runner + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + + - uses: actions/labeler@v5.0.0 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 527c4ef6..4f010b9b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,16 +1,26 @@ -name: miranda +name: miranda Testing Suite on: push: branches: - main paths-ignore: - - HISTORY.rst + - .cruft.json + - CHANGES.rst + - README.rst - pyproject.toml - - setup.cfg + - tests/test_miranda.py - miranda/__init__.py pull_request: +concurrency: + # For a given workflow, if we push to the same branch, cancel all previous builds on that branch except on master. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +permissions: + contents: read + jobs: lint_and_docs: runs-on: ubuntu-latest @@ -21,42 +31,43 @@ jobs: - black - docs python-version: - - "3.8" + - "3.x" steps: - - name: Cancel previous runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - access_token: ${{ github.token }} - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - name: Set up Python${{ matrix.python-version }} + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install tox - run: pip install tox + run: | + python -m pip install tox - name: Run lint and docs testing suite - run: tox -e ${{ matrix.tox-env }} + run: | + python -m tox -e ${{ matrix.tox-env }} env: READTHEDOCS: 1 test: + name: Test with Python${{ matrix.python-version }} (Python${{ matrix.python-version }} + tox) needs: lint_and_docs runs-on: ubuntu-latest - name: Test-${{ matrix.tox-env }} strategy: matrix: include: - - tox-env: py37 + - tox-env: "py37" python-version: "3.7" - - tox-env: py38-pyston + - tox-env: "py38-pyston" python-version: "3.8" - - tox-env: py39-full + - tox-env: "py39-full" python-version: "3.9" - - tox-env: py310 + - tox-env: "py310" python-version: "3.10" - - tox-env: py311 + - tox-env: "py311" python-version: "3.11" + - tox-env: "py312" + python-version: "3.12" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -67,22 +78,75 @@ jobs: sudo apt-get update sudo apt-get install libgdal-dev - name: Install tox - run: pip install tox + run: | + python -m pip install tox - name: Test with tox - run: tox -e ${{ matrix.tox-env }} + run: | + python -m tox -e ${{ matrix.tox-env }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_FLAG_NAME: run-${{ matrix.tox-env }} COVERALLS_PARALLEL: true + COVERALLS_SERVICE_NAME: github + + test-conda: + name: Test with Python${{ matrix.python-version }} (Anaconda) + needs: lint + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + defaults: + run: + shell: bash -l {0} + steps: + - name: Harden Runner + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 + with: + egress-policy: audit + - uses: actions/checkout@v4 + - name: Setup Conda (Micromamba) with Python${{ matrix.python-version }} + uses: mamba-org/setup-micromamba@v1 + with: + cache-downloads: true + environment-file: environment-dev.yml + create-args: >- + mamba + python=${{ matrix.python-version }} + - name: Conda and Mamba versions + run: | + mamba --version + echo "micromamba $(micromamba --version)" + - name: Install miranda + run: | + python -m pip install --no-deps . + - name: Check versions + run: | + conda list + python -m pip check || true + - name: Test with pytest + run: | + python -m pytest --cov miranda + - name: Report coverage + run: | + python -m coveralls + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_FLAG_NAME: run-Python${{ matrix.python-version }}-conda + COVERALLS_PARALLEL: true + COVERALLS_SERVICE_NAME: github finish: - needs: test + needs: + - test-pypi + - test-conda runs-on: ubuntu-latest container: python:3-slim steps: - name: Coveralls Finished run: | - pip install --upgrade coveralls - coveralls --service=github --finish + python -m pip install --upgrade coveralls + python -m coveralls --finish env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_SERVICE_NAME: github diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index bf736bfb..f91b0180 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -1,28 +1,42 @@ -name: "Publish Python 🐍 distributions 📦 to PyPI" +name: Publish Python 🐍 distributions 📦 to PyPI on: release: types: - published +permissions: + contents: read + jobs: build-n-publish-pypi: name: Build and publish Python 🐍 distributions 📦 to PyPI runs-on: ubuntu-latest + environment: production + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write steps: - - uses: actions/checkout@v3 - - name: Set up Python + - name: Harden Runner + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + files.pythonhosted.org:443 + github.com:443 + pypi.org:443 + upload.pypi.org:443 + - uses: actions/checkout@v4 + - name: Set up Python3 uses: actions/setup-python@v4 with: python-version: "3.x" - name: Install packaging libraries run: | - pip install flit + python -m pip install flit - name: Build a binary wheel and a source tarball run: | - flit build + python -m flit build - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 00000000..cb4f4baf --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,82 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by separate terms of service, privacy +# policy, and support documentation. + +name: Scorecard supply-chain security +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + schedule: + - cron: '41 8 * * 4' + push: + branches: + - master + +# Declare default permissions as read only. +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + # Needed to publish results and get a badge (see publish_results below). + id-token: write + steps: + - name: Harden Runner + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + api.osv.dev:443 + api.securityscorecards.dev:443 + fulcio.sigstore.dev:443 + github.com:443 + oss-fuzz-build-logs.storage.googleapis.com:443 + rekor.sigstore.dev:443 + tuf-repo-cdn.sigstore.dev:443 + www.bestpractices.dev:443 + + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + persist-credentials: false + + - name: Run analysis + uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 + with: + results_file: results.sarif + results_format: sarif + # This job step requires a personal access token named `OPENSSF_SCORECARD_TOKEN` with the following privileges: + # - Administration: Read-Only + # - Metadata: Read-Only + # - Webhooks: Read-Only + # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. + repo_token: ${{ secrets.OPENSSF_SCORECARD_TOKEN }} + + # Public repositories: + # - Publish results to OpenSSF REST API for easy access by consumers + # - Allows the repository to include the Scorecard badge. + # - See https://github.com/ossf/scorecard-action#publishing-results. + publish_results: true + + # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF + # format to the repository Actions tab. + - name: Upload artifact + uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + # Upload the results to GitHub's code scanning dashboard. + - name: Upload to code-scanning + uses: github/codeql-action/upload-sarif@e5f05b81d5b6ff8cfa111c80c22c5fd02a384118 # 3.23.0 + with: + sarif_file: results.sarif diff --git a/.github/workflows/tag-testpypi.yml b/.github/workflows/tag-testpypi.yml index 83dbbd88..9033fb0c 100644 --- a/.github/workflows/tag-testpypi.yml +++ b/.github/workflows/tag-testpypi.yml @@ -1,30 +1,69 @@ -name: "Publish Python 🐍 distributions 📦 to TestPyPI" +name: Publish Python 🐍 distributions 📦 to TestPyPI on: push: tags: - - '*' + - 'v*.*' # Push events to matching v*, i.e. v1.0, v20.15.10 + +permissions: + contents: read jobs: - build-n-publish-testpypi: + release: + name: Create Release from tag + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '.0') + permissions: + contents: write + steps: + - name: Harden Runner + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 + with: + egress-policy: audit + - name: Checkout code + uses: actions/checkout@v4 + - name: Create Release + uses: softprops/action-gh-release@v1 + env: + # This token is provided by Actions, you do not need to create your own token + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref_name }} + name: Release ${{ github.ref_name }} + draft: true + prerelease: false + + deploy-testpypi: name: Build and publish Python 🐍 distributions 📦 to TestPyPI runs-on: ubuntu-latest + environment: staging + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write steps: - - uses: actions/checkout@v3 - - name: Set up Python + - name: Harden Runner + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + files.pythonhosted.org:443 + github.com:443 + pypi.org:443 + test.pypi.org:443 + - uses: actions/checkout@v4 + - name: Set up Python3 uses: actions/setup-python@v4 with: python-version: "3.x" - name: Install packaging libraries run: | - pip install flit + python -m pip install flit - name: Build a binary wheel and a source tarball run: | - flit build + python -m flit build - name: Publish distribution 📦 to Test PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: - user: __token__ - password: ${{ secrets.TEST_PYPI_API_TOKEN }} repository_url: https://test.pypi.org/legacy/ skip_existing: true diff --git a/.github/workflows/workflow-warning.yml b/.github/workflows/workflow-warning.yml new file mode 100644 index 00000000..433881bb --- /dev/null +++ b/.github/workflows/workflow-warning.yml @@ -0,0 +1,69 @@ +name: Workflow Changes Warnings + +on: + # Note: potential security risk from this action using pull_request_target. + # Do not add actions in here which need a checkout of the repo, and do not use any caching in here. + # See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target + pull_request_target: + types: + - opened + - reopened + - synchronize + paths: + - .github/workflows/*.yml + +permissions: + contents: read + +jobs: + comment-concerning-workflow-changes: + name: Comment Concerning Workflow Changes + runs-on: ubuntu-latest + if: | + (github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name) + permissions: + contents: read + pull-requests: write + steps: + - name: Harden Runner + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + - name: Find comment + uses: peter-evans/find-comment@a54c31d7fa095754bfef525c0c8e5e5674c4b4b1 # v2.4.0 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: | + This Pull Request modifies GitHub workflows and is coming from a fork. + - name: Create comment + if: | + (steps.fc.outputs.comment-id == '') && + (!contains(github.event.pull_request.labels.*.name, 'approved')) && + (github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name) + uses: peter-evans/create-or-update-comment@23ff15729ef2fc348714a3bb66d2f655ca9066f2 # v3.1.0 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: | + > **Warning** + > This Pull Request modifies GitHub Workflows and is coming from a fork. + **It is very important for the reviewer to ensure that the workflow changes are appropriate.** + edit-mode: replace + - name: Update comment + if: | + contains(github.event.pull_request.labels.*.name, 'approved') + uses: peter-evans/create-or-update-comment@23ff15729ef2fc348714a3bb66d2f655ca9066f2 # v3.1.0 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: | + > **Note** + > Changes have been approved by a maintainer. + reactions: | + hooray + edit-mode: append diff --git a/.gitignore b/.gitignore index ac5207b6..b8db2958 100644 --- a/.gitignore +++ b/.gitignore @@ -1,48 +1,110 @@ +# MIRANDA-specific + +# Byte-compiled / optimized / DLL files +__pycache__/ *.py[cod] +*$py.class # C extensions *.so -# Packages -*.egg -*.egg-info -dist -build -eggs -parts -bin -var -sdist -develop-eggs +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ .installed.cfg -lib -lib64 +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec # Installer logs pip-log.txt +pip-delete-this-directory.txt # Unit test / coverage reports +htmlcov/ +.tox/ .coverage -.tox +.coverage.* +.cache nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ # Translations *.mo +*.pot -# Mr Developer -.mr.developer.cfg -.project -.pydevproject +# Django stuff: +*.log +local_settings.py -# Complexity -output/*.html -output/*/index.html +# Flask stuff: +instance/ +.webassets-cache -# Sphinx -docs/_build +# Scrapy stuff: +.scrapy -# log files -*.log +# Sphinx documentation +docs/_build/ +docs/apidoc/modules.rst +docs/apidoc/miranda*.rst + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ -# JetBrains +# IDE settings +.vscode/ .idea/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8571ffdd..0a8787b1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,53 +11,53 @@ repos: rev: v4.5.0 hooks: - id: trailing-whitespace - exclude: setup.cfg - id: end-of-file-fixer - exclude: setup.cfg - id: check-json + - id: pretty-format-json + args: [ '--autofix', '--no-ensure-ascii', '--no-sort-keys' ] - id: check-toml - id: check-yaml args: [ '--allow-multiple-documents' ] - id: debug-statements - - id: pretty-format-json - args: [ '--autofix', '--no-ensure-ascii' ] - # exclude: .ipynb - repo: https://github.com/pappasam/toml-sort rev: v0.23.1 hooks: - id: toml-sort-fix + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: rst-inline-touching-normal - repo: https://github.com/adrienverge/yamllint.git rev: v1.32.0 hooks: - id: yamllint args: [ '--config-file=.yamllint.yaml' ] - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.9.1 + rev: 24.1.1 hooks: - id: black - args: [ '--target-version=py37' ] - - repo: https://github.com/pycqa/flake8 - rev: 6.1.0 - hooks: - - id: flake8 - args: [ '--config=setup.cfg' ] + exclude: ^docs/ - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort - - repo: https://github.com/pycqa/pydocstyle - rev: 6.3.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.14 + hooks: + - id: ruff + - repo: https://github.com/pycqa/flake8 + rev: 7.0.0 hooks: - - id: pydocstyle - args: [ '--config=setup.cfg' ] - exclude: templates|miranda/decode + - id: flake8 + additional_dependencies: [ 'flake8-alphabetize', 'flake8-rst-docstrings' ] + args: [ '--config=.flake8' ] - repo: https://github.com/keewis/blackdoc - rev: v0.3.8 + rev: v0.3.9 hooks: - id: blackdoc - additional_dependencies: [ 'black==23.9.1' ] + additional_dependencies: [ 'black==24.1.1' ] - repo: https://github.com/adrienverge/yamllint.git - rev: v1.32.0 + rev: v1.33.0 hooks: - id: yamllint args: ['--config-file=.yamllint.yaml'] @@ -67,7 +67,7 @@ repos: - id: detect-secrets args: [ '--baseline=.secrets.baseline' ] - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.27.0 + rev: 0.27.3 hooks: - id: check-github-workflows - id: check-readthedocs diff --git a/.readthedocs.yml b/.readthedocs.yml index 9e375057..57046f0e 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,17 +1,27 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + version: 2 +sphinx: + configuration: docs/conf.py + # fail_on_warning might generate hard to fix error, in this case it can be + # disabled but this also means those errors will fail silently, choose wisely. + fail_on_warning: true + build: os: ubuntu-22.04 tools: - python: "mambaforge-4.10" + python: "mambaforge-22.9" jobs: - post_install: - - sphinx-apidoc -o docs/apidoc --private --module-first miranda pre_build: + - sphinx-apidoc -o docs/apidoc --private --module-first miranda + - sphinx-build -M gettext docs docs/_build - git clone https://github.com/ES-DOC/pyessv-archive.git ~/.esdoc/pyessv-archive conda: - environment: environment.yml + environment: environment-docs.yml python: install: diff --git a/.secrets.baseline b/.secrets.baseline index 400ce79c..d0d57bc9 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -107,6 +107,16 @@ "path": "detect_secrets.filters.heuristic.is_templated_secret" } ], - "results": {}, - "generated_at": "2022-11-24T19:29:06Z" + "results": { + ".cruft.json": [ + { + "type": "Hex High Entropy String", + "filename": ".cruft.json", + "hashed_secret": "95ed94528595d4d669b5711a6401032d8d8e24ab", + "is_verified": true, + "line_number": 3 + } + ] + }, + "generated_at": "2024-02-23T20:41:08Z" } diff --git a/.yamllint.yaml b/.yamllint.yaml index 0964c110..2f3b4a3d 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -3,6 +3,6 @@ rules: document-start: disable line-length: - max: 88 + max: 120 level: warning truthy: disable diff --git a/.zenodo.json b/.zenodo.json new file mode 100644 index 00000000..b02cd93c --- /dev/null +++ b/.zenodo.json @@ -0,0 +1,16 @@ +{ + "title": "miranda", + "creators": [ + { + "name": "Smith, Trevor James" + } + ], + "keywords": [ + "miranda" + ], + "license": "Apache-2.0", + "language": "eng", + "communities": [], + "upload_type": "software", + "access_right": "open" +} diff --git a/HISTORY.rst b/CHANGES.rst similarity index 94% rename from HISTORY.rst rename to CHANGES.rst index 2ece7b4a..c958ef9b 100644 --- a/HISTORY.rst +++ b/CHANGES.rst @@ -1,8 +1,15 @@ .. :changelog: -======= -History -======= +========= +Changelog +========= + +v0.6.0 (unreleased) +------------------- + +Announcements +^^^^^^^^^^^^^ +* `miranda` boilerplate code is now versioned with `cruft `_ and the `Ouranosinc/cookiecutter-pypackage `_ template. v0.5.0 (2023-06-19) ------------------- diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index b9a76b93..3bbfe98a 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,9 +1,10 @@ +.. highlight:: shell + ============ Contributing ============ -Contributions are welcome, and they are greatly appreciated! Every -little bit helps, and credit will always be given. +Contributions are welcome, and they are greatly appreciated! Every little bit helps, and credit will always be given. You can contribute in many ways: @@ -13,7 +14,7 @@ Types of Contributions Report Bugs ~~~~~~~~~~~ -Report bugs at https://github.com/Ouranosinc/miranda/issues. +Report bugs at https://github.com/Zeitsperre/miranda/issues. If you are reporting a bug, please include: @@ -24,52 +25,82 @@ If you are reporting a bug, please include: Fix Bugs ~~~~~~~~ -Look through the GitHub issues for bugs. Anything tagged with "bug" -is open to whoever wants to implement it. +Look through the GitHub issues for bugs. Anything tagged with "bug" and "help wanted" is open to whoever wants to implement it. Implement Features ~~~~~~~~~~~~~~~~~~ -Look through the GitHub issues for features. Anything tagged with "feature" -is open to whoever wants to implement it. +Look through the GitHub issues for features. Anything tagged with "enhancement" and "help wanted" is open to whoever wants to implement it. Write Documentation ~~~~~~~~~~~~~~~~~~~ -Miranda could always use more documentation, whether as part of the -official Miranda docs, in docstrings, or even on the web in blog posts, -articles, and such. +miranda could always use more documentation, whether as part of the official miranda docs, in docstrings, or even on the web in blog posts, articles, and such. Submit Feedback ~~~~~~~~~~~~~~~ -The best way to send feedback is to file an issue at https://github.com/Ouranosinc/miranda/issues. +The best way to send feedback is to file an issue at https://github.com/Zeitsperre/miranda/issues. If you are proposing a feature: * Explain in detail how it would work. * Keep the scope as narrow as possible, to make it easier to implement. * Remember that this is a volunteer-driven project, and that contributions - are welcome :) + are welcome. :) Get Started! ------------ -Ready to contribute? Here's how to set up `miranda` for -local development. +.. note:: + + If you are new to using GitHub and `git`, please read `this guide `_ first. + +.. warning:: + + Anaconda Python users: Due to the complexity of some packages, the default dependency solver can take a long time to resolve the environment. Consider running the following commands in order to speed up the process:: + + $ conda install -n base conda-libmamba-solver + $ conda config --set solver libmamba -1. Fork_ the `miranda` repo on GitHub. -2. Clone your fork locally:: + For more information, please see the following link: https://www.anaconda.com/blog/a-faster-conda-for-a-growing-community + + Alternatively, you can use the `mamba `_ package manager, which is a drop-in replacement for ``conda``. If you are already using `mamba`, replace the following commands with ``mamba`` instead of ``conda``. + +Ready to contribute? Here's how to set up ``miranda`` for local development. + +#. Fork the ``miranda`` repo on GitHub. +#. Clone your fork locally:: $ git clone git@github.com:your_name_here/miranda.git -3. Create a branch for local development:: +#. Install your local copy into a development environment. You can create a new Anaconda development environment with:: + + $ conda env create -f environment-dev.yml + $ conda activate miranda + $ flit install --symlink + + This installs ``miranda`` in an "editable" state, meaning that changes to the code are immediately seen by the environment. + +#. To ensure a consistent coding style, install the ``pre-commit`` hooks to your local clone:: + + $ pre-commit install + + On commit, ``pre-commit`` will check that ``black``, ``blackdoc``, ``isort``, ``flake8``, and ``ruff`` checks are passing, perform automatic fixes if possible, and warn of violations that require intervention. If your commit fails the checks initially, simply fix the errors, re-add the files, and re-commit. + + You can also run the hooks manually with:: + + $ pre-commit run -a + + If you want to skip the ``pre-commit`` hooks temporarily, you can pass the ``--no-verify`` flag to `$ git commit`. + +#. Create a branch for local development:: $ git checkout -b name-of-your-bugfix-or-feature -Now you can make your changes locally. + Now you can make your changes locally. -4. Begin by installing a development build of your branch:: +#. Begin by installing a development build of your branch:: # To install miranda with its development environment dependencies $ pip install -e .[dev] @@ -80,40 +111,169 @@ Now you can make your changes locally. # To install miranda with its remote API dependencies $ pip install -e .[remote] -5. When you're done making changes, check that your changes pass style and unit - tests, including testing other Python versions with tox:: +#. When you're done making changes, we **strongly** suggest running the tests in your environment or with the help of ``tox``:: + $ python -m pytest + # Or, to run multiple build tests $ tox -To get tox, just pip install it. - -6. Commit your changes and push your branch to GitHub:: +#. Commit your changes and push your branch to GitHub:: $ git add . $ git commit -m "Your detailed description of your changes." $ git push origin name-of-your-bugfix-or-feature -7. Submit a pull request through the GitHub website. + If ``pre-commit`` hooks fail, try re-committing your changes (or, if need be, you can skip them with `$ git commit --no-verify`). -.. _Fork: https://github.com/Ouranosinc/miranda/fork +#. Submit a `Pull Request `_ through the GitHub website. + +#. When pushing your changes to your branch on GitHub, the documentation will automatically be tested to reflect the changes in your Pull Request. This build process can take several minutes at times. If you are actively making changes that affect the documentation and wish to save time, you can compile and test your changes beforehand locally with:: + + # To generate the html and open it in your browser + $ make docs + # To only generate the html + $ make autodoc + $ make -C docs html + # To simply test that the docs pass build checks + $ tox -e docs + +#. Once your Pull Request has been accepted and merged to the ``main`` branch, several automated workflows will be triggered: + + - The ``bump-version.yml`` workflow will automatically bump the patch version when pull requests are pushed to the ``main`` branch on GitHub. **It is not recommended to manually bump the version in your branch when merging (non-release) pull requests (this will cause the version to be bumped twice).** + - `ReadTheDocs` will automatically build the documentation and publish it to the `latest` branch of `miranda` documentation website. + - If your branch is not a fork (ie: you are a maintainer), your branch will be automatically deleted. + + You will have contributed your first changes to ``miranda``! Pull Request Guidelines ----------------------- Before you submit a pull request, check that it meets these guidelines: -#. The pull request should include tests. -#. If the pull request adds functionality, the docs should be updated. Put - your new functionality into a function with a docstring, and add the - feature to the list in README.rst. -#. The pull request should work for Python 3.7, 3.8, 3.9, 3.10, and 3.11. - Check https://github.com/Ouranosinc/miranda/actions - for active pull request builds or run the ``tox`` command and - make sure that the tests pass for all supported Python versions. +#. The pull request should include tests and should aim to provide `code coverage `_ for all new lines of code. You can use the ``--cov-report html --cov miranda`` flags during the call to ``pytest`` to generate an HTML report and analyse the current test coverage. + +#. If the pull request adds functionality, the docs should also be updated. Put your new functionality into a function with a docstring, and add the feature to the list in ``README.rst``. + +#. The pull request should work for Python 3.8, 3.9, 3.10, 3.11, and 3.12. Check that the tests pass for all supported Python versions. Tips ---- To run a subset of tests:: - $ pytest test/test_miranda.py + $ pytest tests.test_miranda + +To run specific code style checks:: + + $ black --check miranda tests + $ isort --check miranda tests + $ blackdoc --check miranda docs + $ ruff miranda tests + $ flake8 miranda tests + +To get ``black``, ``isort``, ``blackdoc``, ``ruff``, and ``flake8`` (with plugins ``flake8-alphabetize`` and ``flake8-rst-docstrings``) simply install them with `pip` (or `conda`) into your environment. + +Versioning/Tagging +------------------ + +A reminder for the **maintainers** on how to deploy. This section is only relevant when producing a new point release for the package. + +.. warning:: + + It is important to be aware that any changes to files found within the ``miranda`` folder (with the exception of ``miranda/__init__.py``) will trigger the ``bump-version.yml`` workflow. Be careful not to commit changes to files in this folder when preparing a new release. + +#. Create a new branch from `main` (e.g. `release-0.2.0`). +#. Update the `CHANGES.rst` file to change the `Unreleased` section to the current date. +#. Bump the version in your branch to the next version (e.g. `v0.1.0 -> v0.2.0`):: + + $ bump-my-version bump minor # In most cases, we will be releasing a minor version + $ git push + +#. Create a pull request from your branch to `main`. +#. Once the pull request is merged, create a new release on GitHub. On the main branch, run:: + + $ git tag v0.2.0 + $ git push --tags + + This will trigger a GitHub workflow to build the package and upload it to TestPyPI. At the same time, the GitHub workflow will create a draft release on GitHub. Assuming that the workflow passes, the final release can then be published on GitHub by finalizing the draft release. + +#. Once the release is published, the `publish-pypi.yml` workflow will go into an `awaiting approval` mode on Github Actions. Only authorized users may approve this workflow (notifications will be sent) to trigger the upload to PyPI. + +.. warning:: + + Uploads to PyPI can **never** be overwritten. If you make a mistake, you will need to bump the version and re-release the package. If the package uploaded to PyPI is broken, you should modify the GitHub release to mark the package as broken, as well as yank the package (mark the version "broken") on PyPI. + +Packaging +--------- + +When a new version has been minted (features have been successfully integrated test coverage and stability is adequate), maintainers should update the pip-installable package (wheel and source release) on PyPI as well as the binary on conda-forge. + +The simple approach +~~~~~~~~~~~~~~~~~~~ + +The simplest approach to packaging for general support (pip wheels) requires that ``flit`` be installed:: + + $ python -m pip install flit + +From the command line on your Linux distribution, simply run the following from the clone's main dev branch:: + + # To build the packages (sources and wheel) + $ python -m flit build + + # To upload to PyPI + $ python -m flit publish dist/* + +The new version based off of the version checked out will now be available via `pip` (`$ pip install miranda`). + +Releasing on conda-forge +~~~~~~~~~~~~~~~~~~~~~~~~ + +Initial Release +^^^^^^^^^^^^^^^ + +Before preparing an initial release on conda-forge, we *strongly* suggest consulting the following links: + * https://conda-forge.org/docs/maintainer/adding_pkgs.html + * https://github.com/conda-forge/staged-recipes + +In order to create a new conda build recipe, to be used when proposing packages to the conda-forge repository, we strongly suggest using the ``grayskull`` tool:: + + $ python -m pip install grayskull + $ grayskull pypi miranda + +For more information on ``grayskull``, please see the following link: https://github.com/conda/grayskull + +Before updating the main conda-forge recipe, we echo the conda-forge documentation and *strongly* suggest performing the following checks: + * Ensure that dependencies and dependency versions correspond with those of the tagged version, with open or pinned versions for the `host` requirements. + * If possible, configure tests within the conda-forge build CI (e.g. `imports: miranda`, `commands: pytest miranda`). + +Subsequent releases +^^^^^^^^^^^^^^^^^^^ + +If the conda-forge feedstock recipe is built from PyPI, then when a new release is published on PyPI, `regro-cf-autotick-bot` will open Pull Requests automatically on the conda-forge feedstock. It is up to the conda-forge feedstock maintainers to verify that the package is building properly before merging the Pull Request to the main branch. + +Building sources for wide support with `manylinux` image +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. warning:: + This section is for building source files that link to or provide links to C/C++ dependencies. + It is not necessary to perform the following when building pure Python packages. + +In order to do ensure best compatibility across architectures, we suggest building wheels using the `PyPA`'s `manylinux` +docker images (at time of writing, we endorse using `manylinux_2_24_x86_64`). + +With `docker` installed and running, begin by pulling the image:: + + $ sudo docker pull quay.io/pypa/manylinux_2_24_x86_64 + +From the miranda source folder we can enter into the docker container, providing access to the `miranda` source files by linking them to the running image:: + + $ sudo docker run --rm -ti -v $(pwd):/miranda -w /miranda quay.io/pypa/manylinux_2_24_x86_64 bash + +Finally, to build the wheel, we run it against the provided Python3.9 binary:: + + $ /opt/python/cp39-cp39m/bin/python -m build --sdist --wheel + +This will then place two files in `miranda/dist/` ("miranda-1.2.3-py3-none-any.whl" and "miranda-1.2.3.tar.gz"). +We can now leave our docker container (`$ exit`) and continue with uploading the files to PyPI:: + + $ twine upload dist/* diff --git a/Makefile b/Makefile index 183783f9..e6fed8a2 100644 --- a/Makefile +++ b/Makefile @@ -1,72 +1,113 @@ -.PHONY: help clean clean-pyc clean-build list test test-all coverage docs release sdist +.PHONY: clean clean-build clean-pyc clean-test coverage dist docs help install lint lint/flake8 lint/black +.DEFAULT_GOAL := help + +define BROWSER_PYSCRIPT +import os, webbrowser, sys + +from urllib.request import pathname2url + +webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) +endef +export BROWSER_PYSCRIPT + +define PRINT_HELP_PYSCRIPT +import re, sys + +for line in sys.stdin: + match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) + if match: + target, help = match.groups() + print("%-20s %s" % (target, help)) +endef +export PRINT_HELP_PYSCRIPT + +BROWSER := python -c "$$BROWSER_PYSCRIPT" +LOCALES := docs/locales help: - @echo "clean-build - remove build artifacts" - @echo "clean-pyc - remove Python file artifacts" - @echo "compliant - make changes to code to satisfy lint rules" - @echo "lint - check style with modified pep8 and black" - @echo "test - run tests quickly with the default Python" - @echo "test-all - run tests on every Python version with tox" - @echo "coverage - check code coverage quickly with the default Python" - @echo "docs - generate Sphinx HTML documentation, including API docs" - @echo "release - package and upload a release" - @echo "sdist - package" - @echo "install - install the library in developer mode" - -clean: clean-build clean-pyc ## remove all build, test, coverage and Python artifacts - -clean-build: + @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) + +clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts + +clean-build: ## remove build artifacts rm -fr build/ rm -fr dist/ - rm -fr *.egg-info + rm -fr .eggs/ + find . -name '*.egg-info' -exec rm -fr {} + + find . -name '*.egg' -exec rm -f {} + -clean-docs: +clean-docs: ## remove docs artifacts rm -f docs/apidoc/miranda*.rst rm -f docs/apidoc/modules.rst + rm -fr docs/locales/fr/LC_MESSAGES/*.mo $(MAKE) -C docs clean -clean-pyc: +clean-pyc: ## remove Python file artifacts find . -name '*.pyc' -exec rm -f {} + find . -name '*.pyo' -exec rm -f {} + find . -name '*~' -exec rm -f {} + + find . -name '__pycache__' -exec rm -fr {} + + +clean-test: ## remove test and coverage artifacts + rm -fr .tox/ + rm -f .coverage + rm -fr htmlcov/ + rm -fr .pytest_cache -compliant: - auotopep8 miranda tests - black miranda tests +lint/flake8: ## check style with flake8 + ruff miranda tests + flake8 --config=.flake8 miranda tests -lint: - black --check miranda tests - flake8 miranda tests +lint/black: ## check style with black + black --check miranda tests + blackdoc --check miranda docs + isort --check miranda tests -test: - pytest tests +lint: lint/flake8 lint/black ## check style -test-all: +test: ## run tests quickly with the default Python + python -m pytest + +test-all: ## run tests on every Python version with tox tox -coverage: - coverage run --source miranda setup.py test +coverage: ## check code coverage quickly with the default Python + coverage run --source miranda -m pytest coverage report -m coverage html - open htmlcov/index.html + $(BROWSER) htmlcov/index.html +initialize-translations: clean-docs ## initialize translations, ignoring autodoc-generated files + ${MAKE} -C docs gettext + sphinx-intl update -p docs/_build/gettext -d docs/locales -l fr -autodoc: clean-docs +autodoc: clean-docs ## create sphinx-apidoc files: sphinx-apidoc -o docs/apidoc --private --module-first miranda -docs: autodoc - $(MAKE) -C docs linkcheck html +linkcheck: autodoc ## run checks over all external links found throughout the documentation + $(MAKE) -C docs linkcheck + +docs: autodoc ## generate Sphinx HTML documentation, including API docs + $(MAKE) -C docs html BUILDDIR="_build/html/en" +ifneq ("$(wildcard $(LOCALES))","") + ${MAKE} -C docs gettext + $(MAKE) -C docs html BUILDDIR="_build/html/fr" SPHINXOPTS="-D language='fr'" +endif ifndef READTHEDOCS - xdg-open docs/_build/html/index.html + $(BROWSER) docs/_build/html/en/html/index.html endif -release: clean - python setup.py sdist upload - python setup.py bdist_wheel upload +servedocs: docs ## compile the docs watching for changes + watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . -sdist: clean - python setup.py sdist - python setup.py bdist_wheel upload +dist: clean ## builds source and wheel package + python -m flit build ls -l dist +release: dist ## package and upload a release + python -m flit publish dist/* + install: clean ## install the package to the active Python's site-packages - pip install -e . + python -m flit install + +dev: clean ## install the package to the active Python's site-packages + python -m flit install --symlink diff --git a/README.rst b/README.rst index cc7f8308..e60ded5c 100644 --- a/README.rst +++ b/README.rst @@ -2,12 +2,22 @@ Miranda |logo| ============== -|build| |coveralls| |black| - -Python utilities for climate data collection, conversion, and management - -* Documentation: |docs| -* Free Software: |license| ++----------------------------+-----------------------------------------------------+ +| Versions | |pypi| |versions| | ++----------------------------+-----------------------------------------------------+ +| Documentation and Support | |docs| | ++----------------------------+-----------------------------------------------------+ +| Open Source | |license| |ossf| | ++----------------------------+-----------------------------------------------------+ +| Coding Standards | |black| |ruff| |pre-commit| | ++----------------------------+-----------------------------------------------------+ +| Development Status | |status| |build| |coveralls| | ++----------------------------+-----------------------------------------------------+ + +Python utilities for climate data collection, conversion, and management. + +* Free software: Apache Software License 2.0 +* Documentation: https://miranda.readthedocs.io. Features -------- @@ -60,7 +70,6 @@ For more information about Anaconda/Miniconda/conda-forge: This library is optional for users who do not require validation checks, but enabling this feature requires additional installation steps:: - $ mkdir -p ~/.esdoc $ git clone git@github.com:ES-DOC/pyessv-archive.git ~/.esdoc/pyessv-archive @@ -68,26 +77,59 @@ Contributing ------------ See the contributing documentation: https://miranda.readthedocs.io/en/latest/contributing.html +Credits +------- + +This package was created with Cookiecutter_ and the `Ouranosinc/cookiecutter-pypackage`_ project template. + +.. _Cookiecutter: https://github.com/cookiecutter/cookiecutter +.. _`Ouranosinc/cookiecutter-pypackage`: https://github.com/Ouranosinc/cookiecutter-pypackage + + +.. |black| image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/psf/black + :alt: Python Black + .. |build| image:: https://github.com/Ouranosinc/miranda/actions/workflows/main.yml/badge.svg - :target: https://github.com/Ouranosinc/miranda/actions/workflows/main.yml + :target: https://github.com/Ouranosinc/miranda/actions :alt: Build Status .. |coveralls| image:: https://coveralls.io/repos/github/Ouranosinc/miranda/badge.svg :target: https://coveralls.io/github/Ouranosinc/miranda :alt: Coveralls -.. |docs| image:: https://readthedocs.org/projects/miranda/badge - :target: https://miranda.readthedocs.io/en/latest +.. |docs| image:: https://readthedocs.org/projects/miranda/badge/?version=latest + :target: https://miranda.readthedocs.io/en/latest/?version=latest :alt: Documentation Status .. |license| image:: https://img.shields.io/github/license/Ouranosinc/miranda.svg - :target: https://github.com/Ouranosinc/miranda/blob/master/LICENSE + :target: https://github.com/Ouranosinc/miranda/blob/main/LICENSE :alt: License -.. |black| image:: https://img.shields.io/badge/code%20style-black-000000.svg - :target: https://github.com/psf/black - :alt: Python Black +.. |ossf| image:: https://api.securityscorecards.dev/projects/github.com/Ouranosinc/miranda/badge + :target: https://securityscorecards.dev/viewer/?uri=github.com/Ouranosinc/miranda + :alt: OpenSSF Scorecard .. |logo| image:: https://raw.githubusercontent.com/Ouranosinc/miranda/main/docs/_static/images/miranda-logo-small.png :target: https://github.com/Ouranosinc/miranda :alt: Miranda + +.. |pre-commit| image:: https://results.pre-commit.ci/badge/github/Ouranosinc/miranda/main.svg + :target: https://results.pre-commit.ci/latest/github/Ouranosinc/miranda/main + :alt: pre-commit.ci status + +.. |pypi| image:: https://img.shields.io/pypi/v/miranda.svg + :target: https://pypi.python.org/pypi/miranda + :alt: PyPI + +.. |ruff| image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json + :target: https://github.com/astral-sh/ruff + :alt: Ruff + +.. |status| image:: https://www.repostatus.org/badges/latest/active.svg + :target: https://www.repostatus.org/#active + :alt: Project Status: Active – The project has reached a stable, usable state and is being actively developed. + +.. |versions| image:: https://img.shields.io/pypi/pyversions/miranda.svg + :target: https://pypi.python.org/pypi/miranda + :alt: Supported Python Versions diff --git a/docs/Makefile b/docs/Makefile index 0e35bee9..efdb44bb 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,177 +1,20 @@ -# Makefile for Sphinx documentation +# Minimal makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = +SPHINXBUILD = python -msphinx +SPHINXPROJ = miranda +SOURCEDIR = . BUILDDIR = _build -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - +# Put it first so that "make" without argument is like "make help". help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/complexity.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/complexity.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/complexity" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/complexity" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." +.PHONY: help Makefile -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/changes.rst b/docs/changes.rst new file mode 100644 index 00000000..d9e113ec --- /dev/null +++ b/docs/changes.rst @@ -0,0 +1 @@ +.. include:: ../CHANGES.rst diff --git a/docs/conf.py b/docs/conf.py index 9aee369a..50f3e2e8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -80,10 +80,12 @@ "scp", ] +# To ensure that underlined fields (e.g. `_field`) are shown in the docs. autodoc_default_options = { - "imported-members": True, + "members": True, + "undoc-members": True, "private-members": False, - "show-inheritance": False, + "special-members": False, } # Add any paths that contain templates here, relative to this directory. @@ -113,6 +115,10 @@ # for a list of supported languages. language = "en" +# Sphinx-intl configuration +locale_dirs = ['locales/'] +gettext_compact = False # optional + # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: # today = '' @@ -153,6 +159,9 @@ # If true, keep warnings as "system message" paragraphs in the built documents. # keep_warnings = False +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + # -- Options for HTML output --------------------------------------------------- @@ -256,7 +265,7 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ("index", "miranda.tex", "Miranda Documentation", "Trevor James Smith", "manual") + (master_doc, "miranda.tex", "Miranda Documentation", author, "manual") ] # The name of an image file (relative to this directory) to place at the top of @@ -283,7 +292,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [("index", "miranda", "Miranda Documentation", ["Trevor James Smith"], 1)] +man_pages = [(master_doc, "miranda", "Miranda Documentation", [author], 1)] # If true, show URL addresses after external links. # man_show_urls = False @@ -295,10 +304,10 @@ # dir menu entry, description, category) texinfo_documents = [ ( - "index", + master_doc, "miranda", "Miranda Documentation", - "Trevor James Smith", + author, "miranda", "Python utilities for climate data collection and management.", "Miscellaneous", diff --git a/docs/history.rst b/docs/history.rst deleted file mode 100644 index 25064996..00000000 --- a/docs/history.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../HISTORY.rst diff --git a/docs/index.rst b/docs/index.rst index ed126d61..ecc89d32 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,20 +1,20 @@ -.. include:: ../README.rst - -Contents: -========= +Welcome to miranda's documentation! +=================================== .. toctree:: :maxdepth: 2 + :caption: Contents: + readme installation usage contributing authors - history + changes license .. toctree:: - :maxdepth: 2 + :maxdepth: 1 :caption: All Modules apidoc/modules @@ -30,8 +30,7 @@ Contents: Feedback ======== -If you have any suggestions or questions about **Miranda** feel free to email me -at smith.trevorj@ouranos.ca. +If you have any suggestions or questions about **Miranda**, feel free to email `mailto:smith.trevorj@ouranos.ca`_. -If you encounter any errors or problems with **Miranda**, please let me know! -Open an Issue at the GitHub https://github.com/Ouranosinc/miranda main repository. +If you encounter any errors or problems with **Miranda**, please let us know! +Open an Issue at the `GitHub main repository `_. diff --git a/docs/installation.rst b/docs/installation.rst index b8c9d8df..658aeac8 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -2,9 +2,15 @@ Installation ============ -At the command line, enter:: - $ pip install miranda +Stable release +-------------- + +To install miranda, run this command in your terminal: + +.. code-block:: console + + $ python -m pip install miranda This is the preferred method to install miranda, as it will always install the most recent stable release. @@ -20,31 +26,36 @@ For better RAM usage when converting datasets, some additional/optional GIS libr If you don't have `pip`_ installed, this `Python installation guide`_ can guide you through the process. -.. _pip: https://pip.pypa.io/en/stable/ +.. _pip: https://pip.pypa.io .. _Python installation guide: https://docs.python-guide.org/starting/installation/ From sources ------------ + The sources for miranda can be downloaded from the `Github repo`_. -You can either clone the public repository:: +You can either clone the public repository: + +.. code-block:: console + + $ git clone git@github.com:Zeitsperre/miranda + +Or download the `tarball`_: - $ git clone git://github.com/Ouranosinc/miranda +.. code-block:: console -Or download the `tarball`_:: + $ curl -OJL https://github.com/Zeitsperre/miranda/tarball/main - $ curl -OL https://github.com/Ouranosinc/miranda/tarball/main +Once you have a copy of the source, you can install it with: -Once you have a copy of the source, you can install it with:: +.. code-block:: console - $ python setup.py install + $ python -m pip install . -Alternatively, you can also install a local copy via pip:: - $ pip install . +.. _Github repo: https://github.com/Zeitsperre/miranda +.. _tarball: https://github.com/Zeitsperre/miranda/tarball/main -.. _Github repo: https://github.com/Ouranosinc/miranda -.. _tarball: https://codeload.github.com/Ouranosinc/miranda/legacy.tar.gz/main Creating a Conda environment ---------------------------- diff --git a/docs/make.bat b/docs/make.bat index 2df9a8cb..7fec8f93 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -1,242 +1,36 @@ @ECHO OFF +pushd %~dp0 + REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build + set SPHINXBUILD=python -msphinx ) +set SOURCEDIR=. set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) +set SPHINXPROJ=miranda if "%1" == "" goto help -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. xml to make Docutils-native XML files - echo. pseudoxml to make pseudoxml-XML files for display purposes - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - - -%SPHINXBUILD% 2> nul +%SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. + echo.The Sphinx module was not found. Make sure you have Sphinx installed, + echo.then set the SPHINXBUILD environment variable to point to the full + echo.path of the 'sphinx-build' executable. Alternatively you may add the + echo.Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.http://sphinx-doc.org/ exit /b 1 ) -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\complexity.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\complexity.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdf" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdfja" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf-ja - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -if "%1" == "xml" ( - %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The XML files are in %BUILDDIR%/xml. - goto end -) - -if "%1" == "pseudoxml" ( - %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. - goto end -) +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% :end +popd diff --git a/docs/readme.rst b/docs/readme.rst new file mode 100644 index 00000000..72a33558 --- /dev/null +++ b/docs/readme.rst @@ -0,0 +1 @@ +.. include:: ../README.rst diff --git a/docs/usage.rst b/docs/usage.rst index cdbc65bc..36f0ff6b 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -2,6 +2,8 @@ Usage ===== -To use Miranda in a project:: +To use miranda in a project: - import miranda +.. code-block:: python + + import miranda diff --git a/environment.yml b/environment-dev.yml similarity index 53% rename from environment.yml rename to environment-dev.yml index 3e6b6e56..2464a925 100644 --- a/environment.yml +++ b/environment-dev.yml @@ -2,34 +2,24 @@ name: miranda channels: - conda-forge dependencies: - - python >=3.7,<3.12 - - black >=23.3.0 - - bump2version + - python >=3.9,<3.13 - cdsapi - cftime - clisops >=0.9 - - coverage - dask - distributed - ecmwf-api-client - fabric - - flake8 - - furo >=2023.07.26 - h5netcdf - intake - intake-esm - - ipython - - nbsphinx - netCDF4 - numba >=0.55.2 # There's a problem with numba on conda - numcodecs - numpy - pandas - - pandoc - paramiko >=2.6.0 - - pre-commit - pyproj - - pytest - pyupgrade - pyyaml - regionmask @@ -37,11 +27,23 @@ dependencies: - s3fs - schema - scp - - sphinx - - sphinx-codeautolink - - sphinx-copybutton - - tox >=4.0 - xarray - - xclim >=0.33.0 + - xclim >=0.47.0 - xesmf - zarr + # Dev tools and testing + - pip >=23.1.2 + - bump-my-version >=0.17.1 + - watchdog >=3.0.0 + - flake8 >=6.1.0 + - flake8-rst-docstrings >=0.3.0 + - flit + - tox >=4.5.1 + - coverage >=6.2.0,<7.0.0 + - coveralls >=3.3.1 + - pytest >=7.3.1 + - pytest-cov>=4.0.0 + - black ==24.1.1 + - blackdoc ==0.3.9 + - isort ==5.13.2 + - pre-commit >=3.3.2 diff --git a/environment-docs.yml b/environment-docs.yml new file mode 100644 index 00000000..44fcd965 --- /dev/null +++ b/environment-docs.yml @@ -0,0 +1,16 @@ +name: miranda +channels: + - conda-forge + - defaults +dependencies: + - sphinx >=7.1 + - pandoc + - furo >=2023.07.26 + - ipython + - ipykernel + - nbsphinx + - sphinx-autoapi + - sphinx-codeautolink + - sphinx-copybutton + - sphinx-intl + - sphinxcontrib-napoleon diff --git a/miranda/__init__.py b/miranda/__init__.py index dc47412c..1016ecff 100644 --- a/miranda/__init__.py +++ b/miranda/__init__.py @@ -13,11 +13,12 @@ See the License for the specific language governing permissions and limitations under the License. """ + from __future__ import annotations -__author__ = "Trevor James Smith" +__author__ = """Trevor James Smith""" __email__ = "smith.trevorj@ouranos.ca" -__version__ = "0.5.2-beta" +__version__ = "0.6.0" from . import ( diff --git a/miranda/archive/__init__.py b/miranda/archive/__init__.py index f13d3c57..5f961a15 100644 --- a/miranda/archive/__init__.py +++ b/miranda/archive/__init__.py @@ -1,4 +1,5 @@ """Archive module.""" + from __future__ import annotations from ._groupings import * diff --git a/miranda/archive/_selection.py b/miranda/archive/_selection.py index b69e057c..d3f4d00c 100644 --- a/miranda/archive/_selection.py +++ b/miranda/archive/_selection.py @@ -2,7 +2,6 @@ import logging from datetime import datetime -from logging import config from pathlib import Path from miranda.io import find_filepaths @@ -11,7 +10,7 @@ __all__ = ["select_by_date_modified"] -logging.config.dictConfig(LOGGING_CONFIG) +logging.config.dictConfig(LOGGING_CONFIG) # noqa def select_by_date_modified( diff --git a/miranda/convert/__init__.py b/miranda/convert/__init__.py index a38c34ee..71bcca04 100644 --- a/miranda/convert/__init__.py +++ b/miranda/convert/__init__.py @@ -1,4 +1,5 @@ """Data Conversion module.""" + from __future__ import annotations from . import deh, eccc, ecmwf, hq, melcc, utils diff --git a/miranda/convert/_aggregation.py b/miranda/convert/_aggregation.py index e4831bee..aa80e7ea 100644 --- a/miranda/convert/_aggregation.py +++ b/miranda/convert/_aggregation.py @@ -1,4 +1,5 @@ """Aggregation module.""" + from __future__ import annotations import logging.config @@ -11,7 +12,7 @@ logging.config.dictConfig(LOGGING_CONFIG) -__all__ = ["aggregations_possible", "aggregate"] +__all__ = ["aggregate", "aggregations_possible"] # There needs to be a better way (is there something in xclim?) _resampling_keys = dict() @@ -72,7 +73,7 @@ def aggregations_possible(ds: xr.Dataset, freq: str = "day") -> dict[str, set[st def aggregate(ds: xr.Dataset, freq: str = "day") -> dict[str, xr.Dataset]: - """ + """Aggregate a dataset to a specified frequency. Parameters ---------- diff --git a/miranda/convert/_data_corrections.py b/miranda/convert/_data_corrections.py index bac3e8b4..03c2697f 100644 --- a/miranda/convert/_data_corrections.py +++ b/miranda/convert/_data_corrections.py @@ -29,9 +29,9 @@ VERSION = datetime.datetime.now().strftime("%Y.%m.%d") __all__ = [ + "dataset_conversion", "dataset_corrections", "dims_conversion", - "dataset_conversion", "load_json_data_mappings", "metadata_conversion", "threshold_mask", @@ -247,9 +247,7 @@ def correct_time_entries( date = date_parser(Path(filename).stem.split(split)[location]) vals = np.arange(len(d[field])) days_since = f"days since {date}" - time = xr.coding.times.decode_cf_datetime( - vals, units=days_since, calendar="standard" - ) + time = times.decode_cf_datetime(vals, units=days_since, calendar="standard") d = d.assign_coords({field: time}) prev_history = d.attrs.get("history", "") @@ -262,7 +260,7 @@ def correct_time_entries( def correct_var_names(d: xr.Dataset, split: str = "_", location: int = 0) -> xr.Dataset: - """ + """Correct variable names. Parameters ---------- @@ -896,7 +894,7 @@ def dataset_conversion( preprocess: Callable | str | None = "auto", **xr_kwargs, ) -> xr.Dataset | xr.DataArray: - """Convert an existing Xarray-compatible dataset to another format with variable corrections applied. + r"""Convert an existing Xarray-compatible dataset to another format with variable corrections applied. Parameters ---------- @@ -919,7 +917,7 @@ def dataset_conversion( Preprocessing functions to perform over each Dataset. Default: "auto" - Run preprocessing fixes based on supplied fields from metadata definition. Callable - Runs function over Dataset (single) or supplied to `preprocess` (multifile dataset). - **xr_kwargs + \*\*xr_kwargs Arguments passed directly to xarray. Returns diff --git a/miranda/convert/_data_definitions.py b/miranda/convert/_data_definitions.py index 24fe4f2f..fa51a303 100644 --- a/miranda/convert/_data_definitions.py +++ b/miranda/convert/_data_definitions.py @@ -12,11 +12,12 @@ logging.config.dictConfig(LOGGING_CONFIG) __all__ = [ - "era5_variables", "eccc_rdrs_variables", + "era5_variables", "gather_agcfsr", "gather_agmerra", "gather_ecmwf", + "gather_emdna", "gather_grnch", "gather_nex", "gather_nrcan_gridded_obs", @@ -24,7 +25,6 @@ "gather_rdrs", "gather_sc_earth", "gather_wfdei_gem_capa", - "gather_emdna", "nasa_ag_variables", "nrcan_variables", "project_institutes", @@ -127,7 +127,7 @@ def gather_ecmwf( back_extension: bool = False, monthly_means: bool = False, ) -> dict[str, list[Path]]: - """ + """Gather ECMWF source data. Parameters ---------- @@ -244,8 +244,11 @@ def gather_rdrs( Parameters ---------- name : str + The variable to gather. path : str or os.PathLike + The location of the source data. suffix : str + The filename suffix. key : {"raw", "cf"} Indicating which variable name dictionary to search for. diff --git a/miranda/convert/_reconstruction.py b/miranda/convert/_reconstruction.py index 6cabf25d..ee5b552c 100644 --- a/miranda/convert/_reconstruction.py +++ b/miranda/convert/_reconstruction.py @@ -51,7 +51,7 @@ def reanalysis_processing( n_workers: int = 4, **dask_kwargs, ) -> None: - """ + """Reanalysis processing. Parameters ---------- diff --git a/miranda/convert/deh.py b/miranda/convert/deh.py index 6179d7fd..44b6ec2e 100644 --- a/miranda/convert/deh.py +++ b/miranda/convert/deh.py @@ -1,4 +1,5 @@ """DEH Hydrograph Conversion module.""" + from __future__ import annotations import json @@ -109,12 +110,12 @@ def parse_dms(coord): }, ) - ds.attrs[ - "institution" - ] = "Ministère de l'Environnement et de la Lutte contre les changements climatiques" - ds.attrs[ - "source" - ] = "Hydrometric data " + ds.attrs["institution"] = ( + "Ministère de l'Environnement et de la Lutte contre les changements climatiques" + ) + ds.attrs["source"] = ( + "Hydrometric data " + ) ds.attrs["redistribution"] = "Redistribution policy unknown. For internal use only." return ds diff --git a/miranda/convert/eccc.py b/miranda/convert/eccc.py index d3d9ac93..90b3296a 100644 --- a/miranda/convert/eccc.py +++ b/miranda/convert/eccc.py @@ -1,4 +1,5 @@ """Environment and Climate Change Canada Data Conversion module.""" + from __future__ import annotations from pathlib import Path diff --git a/miranda/convert/eccc_rdrs.py b/miranda/convert/eccc_rdrs.py index fa206746..628c216a 100644 --- a/miranda/convert/eccc_rdrs.py +++ b/miranda/convert/eccc_rdrs.py @@ -1,4 +1,5 @@ """Environment and Climate Change Canada RDRS conversion tools.""" + from __future__ import annotations import logging.config @@ -39,7 +40,7 @@ def convert_rdrs( overwrite: bool = False, **dask_kwargs, ) -> None: - r""" + r"""Convert RDRS dataset. Parameters ---------- diff --git a/miranda/convert/ecmwf.py b/miranda/convert/ecmwf.py index 21427378..434343da 100644 --- a/miranda/convert/ecmwf.py +++ b/miranda/convert/ecmwf.py @@ -1,4 +1,5 @@ """ECMWF TIGGE Conversion module.""" + from __future__ import annotations import itertools as it diff --git a/miranda/convert/hq.py b/miranda/convert/hq.py index 6425f0e1..58efaf1e 100644 --- a/miranda/convert/hq.py +++ b/miranda/convert/hq.py @@ -1,4 +1,5 @@ """Hydro Quebec Weather Station Data Conversion module.""" + from __future__ import annotations import csv diff --git a/miranda/convert/melcc.py b/miranda/convert/melcc.py index e7619848..53c82301 100644 --- a/miranda/convert/melcc.py +++ b/miranda/convert/melcc.py @@ -1,4 +1,5 @@ """MELCC (Québec) Weather Stations data conversion module.""" + from __future__ import annotations import datetime as dt @@ -44,15 +45,15 @@ } __all__ = [ - "parse_var_code", + "concat", + "convert_mdb", + "convert_melcc_obs", + "convert_snow_table", "list_tables", - "read_table", + "parse_var_code", "read_definitions", "read_stations", - "convert_mdb", - "convert_snow_table", - "concat", - "convert_melcc_obs", + "read_table", ] @@ -307,9 +308,9 @@ def convert_mdb( new_var_name = list( filter(lambda k: not k.endswith("_flag"), ds.data_vars.keys()) )[0] - ds.attrs[ - "history" - ] = f"[{dt.datetime.now():%Y-%m-%d %H:%M:%S}] Conversion from {database.name}:{tab} to netCDF." + ds.attrs["history"] = ( + f"[{dt.datetime.now():%Y-%m-%d %H:%M:%S}] Conversion from {database.name}:{tab} to netCDF." + ) date = "-".join(ds.indexes["time"][[0, -1]].strftime("%Y%m")) outs[(new_var_name, code)] = ( output / f"{new_var_name}_{code}_MELCC_{raw.attrs['frequency']}_{date}.nc" @@ -418,9 +419,9 @@ def concat( **{f"priority={i}": ds for i, ds in dss.items()}, ) instruments = [dss[p][vv].melcc_code for p in sorted(dss)] - ds_merged[vv].attrs[ - "melcc_code" - ] = "Merged sources in ascending priority : " + " ,".join(map(str, instruments)) + ds_merged[vv].attrs["melcc_code"] = ( + "Merged sources in ascending priority : " + " ,".join(map(str, instruments)) + ) ds_merged.attrs.update( source="info-climat-merged", diff --git a/miranda/convert/utils.py b/miranda/convert/utils.py index e1e3bd06..ba6cb221 100644 --- a/miranda/convert/utils.py +++ b/miranda/convert/utils.py @@ -1,4 +1,5 @@ """Conversion Utilities submodule.""" + from __future__ import annotations import hashlib @@ -15,7 +16,7 @@ logging.config.dictConfig(LOGGING_CONFIG) -__all__ = ["find_version_hash", "date_parser"] +__all__ = ["date_parser", "find_version_hash"] def find_version_hash(file: os.PathLike | str) -> dict: diff --git a/miranda/cv.py b/miranda/cv.py index 93d1f3c2..396885bd 100644 --- a/miranda/cv.py +++ b/miranda/cv.py @@ -1,4 +1,5 @@ """Controlled Vocabulary module.""" + from __future__ import annotations import warnings diff --git a/miranda/data.py b/miranda/data.py index f4e87778..912a55be 100644 --- a/miranda/data.py +++ b/miranda/data.py @@ -1,4 +1,5 @@ """Database Management module.""" + from __future__ import annotations import logging.config diff --git a/miranda/decode/__init__.py b/miranda/decode/__init__.py index a9e3d70c..c9fb51a0 100644 --- a/miranda/decode/__init__.py +++ b/miranda/decode/__init__.py @@ -1,3 +1,5 @@ +"""Module for decoding and encoding time data.""" + from __future__ import annotations from ._decoder import * diff --git a/miranda/decode/_decoder.py b/miranda/decode/_decoder.py index 088e0cc5..55d422ea 100644 --- a/miranda/decode/_decoder.py +++ b/miranda/decode/_decoder.py @@ -127,9 +127,11 @@ def decode( Parameters ---------- files : str or Path or list of str or Path or generator + The files to decode. chunks : int, optional The chunk size used when processing files. Not to be confused with xarray chunks for dimensions. raise_error : bool + Whether to raise an error if a file cannot be decoded. """ if isinstance(files, (str, os.PathLike)): files = [files] @@ -248,7 +250,7 @@ def _decode_primary_variable(file: Path) -> str: def _decode_hour_of_day_info( file: PathLike | str, ) -> dict: - """ + """Decode hour of day information. Parameters ---------- @@ -284,14 +286,14 @@ def _decode_hour_of_day_info( raise NotImplementedError() @staticmethod - def _decode_time_info( + def _decode_time_info( # noqa: C901 file: PathLike | str | list[str] | None = None, data: dict | None = None, term: str | None = None, *, field: str = None, ) -> str | NaTType: - """ + """Decode time information. Parameters ---------- @@ -454,12 +456,12 @@ def _decode_time_info( @staticmethod def _decode_version(file: PathLike | str, data: dict) -> dict: - """ + """Decode version information. Parameters ---------- - file: os.PathLike or str - data: dict + file : os.PathLike or str + data : dict Returns ------- @@ -488,6 +490,16 @@ def _decode_version(file: PathLike | str, data: dict) -> dict: @classmethod def decode_converted(cls, file: PathLike | str) -> dict: + """Decode converted data. + + Parameters + ---------- + file : os.PathLike or str + + Returns + ------- + dict + """ facets = dict() try: variable, date, data = cls._from_dataset(file=file) diff --git a/miranda/decode/_time.py b/miranda/decode/_time.py index ab455ac7..3cc359dc 100644 --- a/miranda/decode/_time.py +++ b/miranda/decode/_time.py @@ -1,13 +1,12 @@ from __future__ import annotations import logging -from logging import config from pandas._libs.tslibs import NaTType # noqa from miranda.scripting import LOGGING_CONFIG -logging.config.dictConfig(LOGGING_CONFIG) +logging.config.dictConfig(LOGGING_CONFIG) # noqa __all__ = [ "DecoderError", diff --git a/miranda/eccc/__init__.py b/miranda/eccc/__init__.py index 3fe8fdd3..4e05996d 100644 --- a/miranda/eccc/__init__.py +++ b/miranda/eccc/__init__.py @@ -1,4 +1,5 @@ """Environment and Climate Change Canada specialized conversion module.""" + from __future__ import annotations from ._homogenized import * diff --git a/miranda/eccc/_homogenized.py b/miranda/eccc/_homogenized.py index de4b04f0..f844be11 100644 --- a/miranda/eccc/_homogenized.py +++ b/miranda/eccc/_homogenized.py @@ -1,4 +1,5 @@ """Adjusted and Homogenized Canadian Clime Data module.""" + from __future__ import annotations import calendar diff --git a/miranda/eccc/_raw.py b/miranda/eccc/_raw.py index d4dc98ff..118ae860 100644 --- a/miranda/eccc/_raw.py +++ b/miranda/eccc/_raw.py @@ -165,7 +165,7 @@ def _convert_station_file( errored_files.append(data) return - except (UnicodeDecodeError, Exception) as e: + except UnicodeDecodeError as e: logging.error( f"File {data.name} was unable to be read. " f"This is probably an issue with the file: {e}" @@ -393,15 +393,15 @@ def convert_flat_files( mode: str = "hourly", n_workers: int = 4, ) -> None: - """ + """Convert flat formatted files. Parameters ---------- - source_files: str or Path - output_folder: str or Path - variables: str or List[str] - mode: {"hourly", "daily"} - n_workers: int + source_files : str or Path + output_folder : str or Path + variables : str or List[str] + mode : {"hourly", "daily"} + n_workers : int Returns ------- @@ -488,22 +488,28 @@ def aggregate_stations( temp_directory: str | os.PathLike | None = None, n_workers: int = 1, ) -> None: - """ + """Aggregate stations. Parameters ---------- - source_files: str or Path - output_folder: str or Path - variables: str or int or list of str or int, optional - time_step: {"hourly", "daily"} - include_flags: bool - groupings: int - The number of files in each group used for converting to multi-file Datasets. - mf_dataset_freq: str, optional - Resampling frequency for creating output multi-file Datasets. E.g. 'YS': 1 year per file, '5YS': 5 years per file. - temp_directory: str or Path, optional - Use another temporary directory location in case default location is not spacious enough. - n_workers: int + source_files : str or Path + Source files to be aggregated. + output_folder : str or Path + Output folder for the aggregated files. + variables : str or int or list of str or int, optional + The variable codes to be aggregated. + time_step : {"hourly", "daily"} + The time step to be used for aggregation. + include_flags : bool + Include flags in the output files. + groupings : int + The number of files in each group used for converting to multi-file Datasets. + mf_dataset_freq : str, optional + Resampling frequency for creating output multi-file Datasets. E.g. 'YS': 1 year per file, '5YS': 5 years per file. + temp_directory : str or Path, optional + Use another temporary directory location in case default location is not spacious enough. + n_workers : int + The number of workers to use. Returns ------- @@ -865,16 +871,16 @@ def merge_converted_variables( overwrite: bool = False, n_workers: int = 1, ) -> None: - """ + """Merge converted variables. Parameters ---------- - source_files: str, Path - output_folder: str, Path - variables: str or int or list of str or int, optional - station_metadata: str or Path, optional - overwrite: bool - n_workers: int + source_files : str, Path + output_folder : str, Path + variables : str or int or list of str or int, optional + station_metadata : str or Path, optional + overwrite : bool + n_workers : int Returns ------- diff --git a/miranda/eccc/_summaries.py b/miranda/eccc/_summaries.py index 8e0b42c1..e9cd1dfc 100755 --- a/miranda/eccc/_summaries.py +++ b/miranda/eccc/_summaries.py @@ -27,7 +27,8 @@ from miranda.scripting import LOGGING_CONFIG config.dictConfig(LOGGING_CONFIG) -__all__ = ["extract_daily_summaries", "daily_summaries_to_netcdf"] + +__all__ = ["daily_summaries_to_netcdf", "extract_daily_summaries"] eccc_metadata = json.load( open(Path(__file__).parent / "eccc_obs_summary_cf_attrs.json") @@ -192,7 +193,7 @@ def _read_multiple_daily_summaries( files: list[str | Path] | Generator[Path, None, None], rm_flags: bool = False, ) -> dict: - """ + """Read multiple daily summary files. Notes ----- diff --git a/miranda/eccc/_utils.py b/miranda/eccc/_utils.py index afb34770..dec5c856 100644 --- a/miranda/eccc/_utils.py +++ b/miranda/eccc/_utils.py @@ -6,13 +6,13 @@ from miranda.scripting import LOGGING_CONFIG -__all__ = ["cf_station_metadata", "cf_ahccd_metadata"] +__all__ = ["cf_ahccd_metadata", "cf_station_metadata"] logging.config.dictConfig(LOGGING_CONFIG) def cf_station_metadata(variable_code: int | str) -> Mapping[str, int | float | str]: - """ + """CF metadata for hourly station data. Parameters ---------- @@ -841,7 +841,7 @@ def cf_station_metadata(variable_code: int | str) -> Mapping[str, int | float | def cf_ahccd_metadata( code: str, gen: int ) -> (dict[str, int | float | str], dict, list[tuple[int, int]], int): - """ + """CF compliant metadata for ECCC Adjusted and Homogenized Climate Data (AHCCD). Parameters ---------- diff --git a/miranda/ecmwf/__init__.py b/miranda/ecmwf/__init__.py index 0037ab52..fd6d506b 100644 --- a/miranda/ecmwf/__init__.py +++ b/miranda/ecmwf/__init__.py @@ -1,4 +1,5 @@ """ECMWF Download module.""" + from __future__ import annotations from ._era5 import * diff --git a/miranda/ecmwf/_era5.py b/miranda/ecmwf/_era5.py index 7341db58..8b8ba4f7 100644 --- a/miranda/ecmwf/_era5.py +++ b/miranda/ecmwf/_era5.py @@ -30,7 +30,11 @@ logging.config.dictConfig(LOGGING_CONFIG) -__all__ = ["request_era5", "rename_era5_files", "ERA5_PROJECT_NAMES"] +__all__ = [ + "ERA5_PROJECT_NAMES", + "rename_era5_files", + "request_era5", +] ERA5_PROJECT_NAMES = [ diff --git a/miranda/gis/__init__.py b/miranda/gis/__init__.py index fe49f1c9..d3b5e40a 100644 --- a/miranda/gis/__init__.py +++ b/miranda/gis/__init__.py @@ -1,4 +1,5 @@ """GIS Operations module.""" + from __future__ import annotations from ._domains import * diff --git a/miranda/gis/_domains.py b/miranda/gis/_domains.py index cc3526f3..34d1405f 100644 --- a/miranda/gis/_domains.py +++ b/miranda/gis/_domains.py @@ -10,7 +10,11 @@ logging.config.dictConfig(LOGGING_CONFIG) -__all__ = ["subset_domain", "subsetting_domains", "add_ar6_regions"] +__all__ = [ + "add_ar6_regions", + "subset_domain", + "subsetting_domains", +] _gis_import_error_message = ( "`{}` requires installation of the miranda GIS libraries. These can be installed using the" diff --git a/miranda/io/__init__.py b/miranda/io/__init__.py index 3b247a51..e040ae10 100644 --- a/miranda/io/__init__.py +++ b/miranda/io/__init__.py @@ -1,4 +1,5 @@ """IO Utilities module.""" + from __future__ import annotations from . import utils diff --git a/miranda/io/_output.py b/miranda/io/_output.py index 015a5c88..af610cc0 100644 --- a/miranda/io/_output.py +++ b/miranda/io/_output.py @@ -1,4 +1,5 @@ """IO Output Operations module.""" + from __future__ import annotations import logging.config @@ -23,10 +24,10 @@ __all__ = [ - "write_dataset", - "write_dataset_dict", "concat_rechunk_zarr", "merge_rechunk_zarrs", + "write_dataset", + "write_dataset_dict", ] diff --git a/miranda/io/_rechunk.py b/miranda/io/_rechunk.py index 6c25d432..246dc25d 100644 --- a/miranda/io/_rechunk.py +++ b/miranda/io/_rechunk.py @@ -94,7 +94,7 @@ def fetch_chunk_config( dims: Sequence[str] | dict[str, int] | Frozen | tuple[Hashable], default_config: dict = chunk_configurations, ) -> dict[str, int]: - """ + """Fetch data chunking configuration. Parameters ---------- diff --git a/miranda/io/utils.py b/miranda/io/utils.py index 33da8643..4b97a5c8 100644 --- a/miranda/io/utils.py +++ b/miranda/io/utils.py @@ -1,4 +1,5 @@ """IO Utilities module.""" + from __future__ import annotations import json diff --git a/miranda/ncar/__init__.py b/miranda/ncar/__init__.py index 5c22b7db..1d0ef6d8 100644 --- a/miranda/ncar/__init__.py +++ b/miranda/ncar/__init__.py @@ -1,4 +1,5 @@ """NCAR Download module""" + from __future__ import annotations from ._aws_cordex import * diff --git a/miranda/remote/__init__.py b/miranda/remote/__init__.py index 2f59a55c..b4aa54da 100644 --- a/miranda/remote/__init__.py +++ b/miranda/remote/__init__.py @@ -1,4 +1,5 @@ """Remote Operations module.""" + from __future__ import annotations from .archiver import * diff --git a/miranda/remote/archiver.py b/miranda/remote/archiver.py index 65c3c39d..f149d708 100644 --- a/miranda/remote/archiver.py +++ b/miranda/remote/archiver.py @@ -1,4 +1,5 @@ """Archive Module.""" + from __future__ import annotations import logging.config diff --git a/miranda/remote/connect.py b/miranda/remote/connect.py index 39161e69..be0bb130 100644 --- a/miranda/remote/connect.py +++ b/miranda/remote/connect.py @@ -1,4 +1,5 @@ """Remote Connection Operations module.""" + from __future__ import annotations import logging.config @@ -60,10 +61,10 @@ def __call__(self, **kwargs): self.update(**kwargs) return self - def __str__(self): + def __str__(self): # noqa: D105 return f"Connection to {self.host} as {self.user}" - def __repr__(self): + def __repr__(self): # noqa: D105 return f"<{self.__class__.__module__}.{self.__class__.__name__} object at {hex(id(self))}>" def connect(self, **kwargs): @@ -92,8 +93,8 @@ def connect(self, **kwargs): except Exception as e: raise e - def __enter__(self, **kwargs): + def __enter__(self, **kwargs): # noqa: D105 return self.connect() - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, exc_type, exc_val, exc_tb): # noqa: D105 self.__c.close() diff --git a/miranda/remote/ops.py b/miranda/remote/ops.py index 73c71e94..47c64811 100644 --- a/miranda/remote/ops.py +++ b/miranda/remote/ops.py @@ -1,4 +1,5 @@ """Remote Operations module.""" + from __future__ import annotations import logging.config @@ -65,11 +66,9 @@ def create_remote_directory( def create_archive( source_files: list[str | os.PathLike], destination: str | os.PathLike, - transport: SCPClient - | SFTPClient - | fabric.Connection - | miranda.remote.Connection - | None = None, + transport: ( + SCPClient | SFTPClient | fabric.Connection | miranda.remote.Connection | None + ) = None, delete: bool = True, compression: bool = False, recursive: bool = True, @@ -114,11 +113,9 @@ def create_archive( def transfer_file( source_file: str | os.PathLike, destination_file: str | os.PathLike, - transport: SCPClient - | SFTPClient - | fabric.Connection - | miranda.remote.Connection - | None = None, + transport: ( + SCPClient | SFTPClient | fabric.Connection | miranda.remote.Connection | None + ) = None, ) -> bool: """Transfer file from one location (remote or local) to another. @@ -143,7 +140,7 @@ def transfer_file( f"Transferred { Path(destination_file).name} to {Path(destination_file).parent}" ) - except (SCPException, SSHException, OSError) as e: + except (OSError, SCPException, SSHException) as e: msg = f'File "{destination_file.name}" failed to be transferred: {e}.' logging.warning(msg) return False @@ -155,7 +152,7 @@ def transfer_file( else: try: destination_file.write_bytes(source_file.read_bytes()) - except (SCPException, SSHException, OSError) as e: + except (OSError, SCPException, SSHException) as e: msg = f'File "{source_file.name}" failed to be copied: {e}' logging.error(msg) return False diff --git a/miranda/remote/remove.py b/miranda/remote/remove.py index 993261ae..1df46de8 100644 --- a/miranda/remote/remove.py +++ b/miranda/remote/remove.py @@ -1,4 +1,5 @@ """Remote File Removal Operations module.""" + from __future__ import annotations import logging.config @@ -142,7 +143,7 @@ def delete_duplicates( pattern: str = None, delete_target_duplicates: bool = False, ) -> None: - """ + """Delete duplicate files. Parameters ---------- diff --git a/miranda/scripting.py b/miranda/scripting.py index 1fdb62ee..8533d664 100644 --- a/miranda/scripting.py +++ b/miranda/scripting.py @@ -1,4 +1,5 @@ """Scripting Helpers module.""" + from __future__ import annotations import pathlib diff --git a/miranda/storage.py b/miranda/storage.py index 0ee5fa71..9a299864 100644 --- a/miranda/storage.py +++ b/miranda/storage.py @@ -6,15 +6,16 @@ Classes: * DiskSpaceError - the exception raised on failure. - * :class:`FileMeta` - file and its size. - * :class:`StorageState` - storage capacity and availability of a medium. + * :py:class:`FileMeta` - file and its size. + * :py:class:`StorageState` - storage capacity and availability of a medium. Functions: - * :func:`total_size` - get total size of a list of files. - * :func:`size_division` - divide files based on number and size restrictions. + * :py:func:`total_size` - get total size of a list of files. + * :py:func:`size_division` - divide files based on number and size restrictions. """ + from __future__ import annotations import logging @@ -81,7 +82,7 @@ def __init__(self, path: str, size: int = -1): else: self.size = size - def __eq__(self, other): + def __eq__(self, other): # noqa: D105 if self.path == other.path: return True else: diff --git a/miranda/structure/__init__.py b/miranda/structure/__init__.py index e5d9c532..d6dfab95 100644 --- a/miranda/structure/__init__.py +++ b/miranda/structure/__init__.py @@ -1,4 +1,5 @@ """Data Structure module.""" + from __future__ import annotations from ._structure import * diff --git a/miranda/structure/_structure.py b/miranda/structure/_structure.py index cac8b53b..1804ce1f 100644 --- a/miranda/structure/_structure.py +++ b/miranda/structure/_structure.py @@ -23,8 +23,8 @@ logging.config.dictConfig(LOGGING_CONFIG) __all__ = [ - "create_version_hash_files", "build_path_from_schema", + "create_version_hash_files", "structure_datasets", ] @@ -113,11 +113,9 @@ def generate_hash_metadata( def create_version_hash_files( - input_files: str - | os.PathLike - | list[str | os.PathLike] - | GeneratorType - | None = None, + input_files: ( + str | os.PathLike | list[str | os.PathLike] | GeneratorType | None + ) = None, facet_dict: dict | None = None, verify_hash: bool = False, ) -> None: @@ -351,7 +349,7 @@ def structure_datasets( verify_hashes: bool = False, suffix: str = "nc", ) -> dict[Path, Path]: - """ + """Structure datasets. Parameters ---------- diff --git a/miranda/units.py b/miranda/units.py index 381ac063..e0d5096a 100644 --- a/miranda/units.py +++ b/miranda/units.py @@ -1,4 +1,5 @@ """Special Time Units-Handling submodule.""" + from __future__ import annotations import logging diff --git a/miranda/utils.py b/miranda/utils.py index 29252729..cc29cbba 100644 --- a/miranda/utils.py +++ b/miranda/utils.py @@ -1,4 +1,5 @@ """Miscellaneous Helper Utilities module.""" + from __future__ import annotations import gzip @@ -47,11 +48,11 @@ class HiddenPrints: CC-BY-SA 4.0 (https://creativecommons.org/licenses/by-sa/4.0/)- """ - def __enter__(self): + def __enter__(self): # noqa: D105 self._original_stdout = sys.stdout sys.stdout = open(os.devnull, "w") - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, exc_type, exc_val, exc_tb): # noqa: D105 sys.stdout.close() sys.stdout = self._original_stdout diff --git a/miranda/validators.py b/miranda/validators.py index 75551a9d..893864d9 100644 --- a/miranda/validators.py +++ b/miranda/validators.py @@ -1,4 +1,5 @@ """Data Validation module.""" + from __future__ import annotations import re diff --git a/pyproject.toml b/pyproject.toml index dd7bd2bb..1c4129c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,10 +11,9 @@ maintainers = [ {name = "Travis Logan", email = "logan.travis@ouranos.ca"}, {name = "Pascal Bourgault", email = "bourgault.pascal@ouranos.ca"} ] -description = "Python utilities for climate data collection, conversion, and management" readme = {file = "README.rst", content-type = "text/x-rst"} -requires-python = ">=3.7.0" -keywords = ["xarray", "climate", "meteorology", "hydrology", "archiving", "collection", "conversion"] +requires-python = ">=3.8.0" +keywords = ["xarray", "climate", "meteorology", "hydrology", "archiving", "collection", "conversion", "miranda"] license = {file = "LICENSE"} classifiers = [ "Development Status :: 3 - Alpha", @@ -24,14 +23,15 @@ classifiers = [ "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", "Topic :: Scientific/Engineering :: Atmospheric Science" ] -dynamic = ["version"] +dynamic = ["description", "version"] dependencies = [ "cftime", "dask", @@ -51,24 +51,37 @@ dependencies = [ [project.optional-dependencies] dev = [ # Dev tools and testing - "black>=23.3.0", - "bump2version", - "coverage[toml]", - "flake8", - "pre-commit", - "pytest", - "pytest-cov", - "tox" + "pip>=23.1.2", + "bump-my-version>=0.17.1", + "watchdog>=3.0.0", + "flake8>=6.1.0", + "flake8-alphabetize>=0.0.21", + "flake8-rst-docstrings>=0.3.0", + "flit>=3.9.0", + "tox>=4.5.1", + "coverage>=6.2.2,<7.0.0", + "coveralls>=3.3.1", + "pytest>=7.3.1", + "pytest-cov>=4.0.0", + "black==24.1.1", + "blackdoc==0.3.9", + "isort==5.13.2", + "ruff>=0.2.0", + "pre-commit>=3.3.2" ] docs = [ # Documentation and examples - "furo>=2023.07.26", - "ipykernel", - "nbsphinx", - "pandoc", "sphinx>=7.1", "sphinx_codeautolink", - "sphinx_copybutton" + "sphinx_copybutton", + "sphinx-intl", + "sphinxcontrib-napoleon", + "nbsphinx", + "pandoc", + "ipython", + "ipykernel", + "jupyter_client", + "furo>=2023.07.26" ] gis = [ # GIS library support @@ -91,44 +104,120 @@ remote = [ [project.urls] "Homepage" = "https://miranda.readthedocs.io/" +"Changelog" = "https://miranda.readthedocs.io/en/stable/changes.html" +"About Ouranos" = "https://www.ouranos.ca/en/" "Source" = "https://github.com/Ouranosinc/miranda/" -"Changelog" = "https://miranda.readthedocs.io/en/stable/history.html" "Issue tracker" = "https://github.com/Ouranosinc/miranda/issues" -"About Ouranos" = "https://www.ouranos.ca/en/" [tool] [tool.black] target-version = [ - "py37" + "py38", + "py39", + "py310", + "py311", + "py312" +] + +[tool.bumpversion] +current_version = "0.6.0" +commit = true +commit_args = "--no-verify" +tag = false +tag_name = "v{new_version}" +allow_dirty = false +parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)(\\-(?P[a-z]+)(\\.(?P\\d+)))?" +serialize = [ + "{major}.{minor}.{patch}-{release}.{build}", + "{major}.{minor}.{patch}" +] + +[[tool.bumpversion.files]] +filename = "miranda/__init__.py" +search = "__version__ = \"{current_version}\"" +replace = "__version__ = \"{new_version}\"" + +[[tool.bumpversion.files]] +filename = "tests/test_miranda.py" +search = "__version__ = \"{current_version}\"" +replace = "__version__ = \"{new_version}\"" + +[[tool.bumpversion.files]] +filename = ".cruft.json" +search = "\"version\": \"{current_version}\"" +replace = "\"version\": \"{new_version}\"" + +[tool.bumpversion.parts.build] +independent = false + +[tool.bumpversion.parts.release] +optional_value = "release" +values = [ + "dev", + "release" ] [tool.coverage.run] relative_files = true +include = ["miranda/*"] omit = ["tests/*.py"] [tool.flit.sdist] include = [ + ".zenodo.json", "AUTHORS.rst", + "CHANGES.rst", "CONTRIBUTING.rst", - "HISTORY.rst", "LICENSE", + "Makefile", "README.rst", - "docs/*.rst", + "environment-dev.yml", + "environment-docs.yml", + "docs/_static/_images/*.gif", + "docs/_static/_images/*.jpg", + "docs/_static/_images/*.png", + "docs/_static/_images/*.rst", "docs/Makefile", "docs/conf.py", "docs/make.bat", - "docs/static/images/*.png" + "tests/*.py", + "tox.ini", + "miranda" +] +exclude = [ + "*.py[co]", + "__pycache__", + ".coveralls.yml", + ".editorconfig", + ".flake8", + ".gitignore", + ".pre-commit-config.yaml", + ".readthedocs.yml", + ".yamllint.yaml", + "docs/_*", + "docs/apidoc/modules.rst", + "docs/apidoc/miranda*.rst", + "docs/locales" ] -exclude = [".*", "**/__pycache__", "**/*.py[co]", "docs/_build", "docs/apidoc/miranda*.rst", "docs/apidoc/modules.rst"] [tool.isort] profile = "black" -py_version = 37 +py_version = 38 append_only = true add_imports = "from __future__ import annotations" skip = ["templates", "docs/conf.py"] +[tool.mypy] +python_version = 3.8 +show_error_codes = true +warn_return_any = true +warn_unused_configs = true + +[[tool.mypy.overrides]] +module = [] +ignore_missing_imports = true + [tool.pytest.ini_options] addopts = [ "--verbose", @@ -137,3 +226,56 @@ addopts = [ "--cov-report=term-missing" ] filterwarnings = ["ignore::UserWarning"] +testpaths = "tests" + +[tool.ruff] +src = ["miranda"] +line-length = 150 +target-version = "py38" +exclude = [ + ".eggs", + ".git", + "build", + "docs", + "templates" +] + +[tool.ruff.format] +line-ending = "auto" + +[tool.ruff.lint] +ignore = [ + "D205", + "D400", + "D401" +] +select = [ + "C9", + "D", + "E", + "F", + "W" +] + +[tool.ruff.lint.flake8-bandit] +check-typed-exception = true + +[tool.ruff.lint.isort] +known-first-party = ["miranda"] +case-sensitive = true +detect-same-package = false +lines-after-imports = 1 +no-lines-before = ["future", "standard-library"] + +[tool.ruff.lint.mccabe] +max-complexity = 30 + +[tool.ruff.lint.per-file-ignores] +"miranda/**/__init__.py" = ["F401", "F403"] +"tests/**/*.py" = ["D100", "D101", "D102"] + +[tool.ruff.lint.pycodestyle] +max-doc-length = 180 + +[tool.ruff.lint.pydocstyle] +convention = "numpy" diff --git a/setup.cfg b/setup.cfg index 949ebd42..b7823584 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,13 +3,13 @@ current_version = 0.5.2-beta commit = True tag = False parse = (?P\d+)\.(?P\d+).(?P\d+)(\-(?P[a-z]+))? -serialize = +serialize = {major}.{minor}.{patch}-{release} {major}.{minor}.{patch} [bumpversion:part:release] optional_value = gamma -values = +values = beta gamma @@ -18,14 +18,14 @@ search = __version__ = "{current_version}" replace = __version__ = "{new_version}" [flake8] -exclude = +exclude = .git, docs, build, .eggs, max-line-length = 88 max-complexity = 12 -ignore = +ignore = C901 E203 E231 @@ -40,15 +40,15 @@ ignore = test = pytest [tool:pytest] -addopts = +addopts = --verbose --color=yes -filterwarnings = +filterwarnings = ignore::UserWarning [tool:pylint] ignore = docs,tests -disable = +disable = bad-continuation, invalid-name, line-too-long, @@ -58,7 +58,7 @@ disable = [pydocstyle] convention = numpy -add-ignore = +add-ignore = D105, D205, D400, @@ -69,7 +69,7 @@ match = ((?!(test_|conf)).)*\.py [pycodestyle] count = False exclude = tests -ignore = +ignore = E203, E226, E402, diff --git a/tests/test_miranda.py b/tests/test_miranda.py index cc4c72e7..c71ede45 100755 --- a/tests/test_miranda.py +++ b/tests/test_miranda.py @@ -1,5 +1,6 @@ from __future__ import annotations +import pkgutil from pathlib import Path import miranda @@ -57,3 +58,16 @@ def test_url_validator(self): assert db._url_validate(url) assert db._url_validate(short_url) assert not db._url_validate(not_url) + + +def test_package_metadata(): + """Test the package metadata.""" + project = pkgutil.get_loader("miranda").get_filename() + + metadata = Path(project).resolve().parent.joinpath("__init__.py") + + with open(metadata) as f: + contents = f.read() + assert """Trevor James Smith""" in contents + assert '__email__ = "smith.trevorj@ouranos.ca"' in contents + assert '__version__ = "0.6.0"' in contents diff --git a/tox.ini b/tox.ini index 44941155..b1a734b1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,22 +1,56 @@ [tox] min_version = 4.0 -envlist = py{37,38,39,310}, black, docs, full, pyston +envlist = + lint + py{38,39,310,311,312} + docs + coveralls requires = - pip >= 20.0 -opts = -v + flit + pip >= 23.3.0 +opts = + --verbose + +[testenv:lint] +skip_install = True +deps = + black ==24.1.1 + blackdoc ==0.3.9 + isort ==5.13.2 + flake8 + ruff >=0.2.0 +commands = + make lint +allowlist_externals = + make + +[testenv:docs] +extras = + docs +commands = + mkdir {envtmpdir}/.esdoc + git clone https://github.com/ES-DOC/pyessv-archive.git {envtmpdir}/.esdoc/pyessv-archive + make --directory=docs clean html +allowlist_externals = + git + make + mkdir [testenv] setenv = HOME = {envtmpdir} + PYTEST_ADDOPTS = "--color=yes" PYTHONPATH = {toxinidir} passenv = CI GITHUB_* COVERALLS_* READTHEDOCS -install_command = python -m pip install --no-user {opts} {packages} +extras = + dev + remote: remote download = True -extras = dev +install_command = python -m pip install --no-user {opts} {packages} deps = coveralls pytest-cov @@ -27,35 +61,9 @@ commands_pre = commands = mkdir {envtmpdir}/.esdoc git clone https://github.com/ES-DOC/pyessv-archive.git {envtmpdir}/.esdoc/pyessv-archive - pytest --cov=miranda --cov-report=term-missing - - coveralls --service=github -allowlist_externals = - git - mkdir - -[testenv:black] -skip_install = True -deps = - flake8 - black -commands = - black --check miranda tests - flake8 miranda tests - -[testenv:docs] -extras = docs -deps = - sphinx -commands = - mkdir {envtmpdir}/.esdoc - git clone https://github.com/ES-DOC/pyessv-archive.git {envtmpdir}/.esdoc/pyessv-archive - make docs + pytest --cov + # Coveralls requires access to a repo token set in .coveralls.yml in order to report stats + coveralls: - coveralls allowlist_externals = git - make mkdir - -[testenv:full] -extra = - dev - remote From de525d20df3cbef03c49b5268d46c57bad497103 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 23 Feb 2024 15:52:19 -0500 Subject: [PATCH 02/12] fix main.yml --- .github/workflows/main.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4f010b9b..c3b11e03 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,15 +47,13 @@ jobs: env: READTHEDOCS: 1 - test: + test-pypi: name: Test with Python${{ matrix.python-version }} (Python${{ matrix.python-version }} + tox) needs: lint_and_docs runs-on: ubuntu-latest strategy: matrix: include: - - tox-env: "py37" - python-version: "3.7" - tox-env: "py38-pyston" python-version: "3.8" - tox-env: "py39-full" From 791d4a1190d6e5b6a9022c98492b76eb34a3ed5b Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 23 Feb 2024 15:53:00 -0500 Subject: [PATCH 03/12] fix main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c3b11e03..2d42ab08 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -89,7 +89,7 @@ jobs: test-conda: name: Test with Python${{ matrix.python-version }} (Anaconda) - needs: lint + needs: lint_and_docs runs-on: ubuntu-latest strategy: matrix: From 38d48db8c1f110bb7d6f5ca91d3a140340be95d7 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 23 Feb 2024 17:11:57 -0500 Subject: [PATCH 04/12] workflow adjustments --- .github/workflows/main.yml | 2 +- tox.ini | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2d42ab08..8c82bd68 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,7 +28,7 @@ jobs: strategy: matrix: tox-env: - - black + - lint - docs python-version: - "3.x" diff --git a/tox.ini b/tox.ini index b1a734b1..7d570b1b 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,8 @@ deps = isort ==5.13.2 flake8 ruff >=0.2.0 +commands_pre = + pip list commands = make lint allowlist_externals = From ad586113e242fa86e185fcc03f60864ea0e38790 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 23 Feb 2024 17:20:27 -0500 Subject: [PATCH 05/12] add full recipe --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 7d570b1b..13cb628f 100644 --- a/tox.ini +++ b/tox.ini @@ -32,7 +32,7 @@ extras = commands = mkdir {envtmpdir}/.esdoc git clone https://github.com/ES-DOC/pyessv-archive.git {envtmpdir}/.esdoc/pyessv-archive - make --directory=docs clean html + make docs allowlist_externals = git make @@ -50,6 +50,7 @@ passenv = READTHEDOCS extras = dev + full: full remote: remote download = True install_command = python -m pip install --no-user {opts} {packages} From e8130b0b91a710adeccdca049e7eaa64d3bdbd0e Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 23 Feb 2024 17:20:39 -0500 Subject: [PATCH 06/12] fix mailto link --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index ecc89d32..4fac9f3f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,7 +30,7 @@ Welcome to miranda's documentation! Feedback ======== -If you have any suggestions or questions about **Miranda**, feel free to email `mailto:smith.trevorj@ouranos.ca`_. +If you have any suggestions or questions about **Miranda**, feel free to email `smith.trevorj@ouranos.ca `_. If you encounter any errors or problems with **Miranda**, please let us know! Open an Issue at the `GitHub main repository `_. From de791609192245a16f95793181e823930684ca46 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 23 Feb 2024 17:22:58 -0500 Subject: [PATCH 07/12] fix coverage for conda --- environment-dev.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index 2464a925..980cfa80 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -39,11 +39,16 @@ dependencies: - flake8-rst-docstrings >=0.3.0 - flit - tox >=4.5.1 - - coverage >=6.2.0,<7.0.0 - - coveralls >=3.3.1 +# - coverage >=6.2.0,<7.0.0 +# - coveralls >=3.3.1 - pytest >=7.3.1 - - pytest-cov>=4.0.0 +# - pytest-cov>=4.0.0 - black ==24.1.1 - blackdoc ==0.3.9 - isort ==5.13.2 - pre-commit >=3.3.2 + - pip + - pip: + - coverage >=6.2.0,<7.0.0 + - coveralls >=3.3.1 + - pytest-cov>=4.0.0 From 106a65b7cbbb7244178d4998bfa5459977ce07a7 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 23 Feb 2024 17:36:23 -0500 Subject: [PATCH 08/12] remove setup.cfg --- .github/workflows/codeql-analysis.yml | 3 +- environment-dev.yml | 1 + setup.cfg | 79 --------------------------- 3 files changed, 2 insertions(+), 81 deletions(-) delete mode 100644 setup.cfg diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 02f2e61a..8f23ff1c 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -15,9 +15,8 @@ on: push: branches: [ "main" ] paths-ignore: - - HISTORY.rst + - CHANGES.rst - pyproject.toml - - setup.cfg - tox.ini - miranda/__init__.py - .github/**.yml diff --git a/environment-dev.yml b/environment-dev.yml index 980cfa80..22a196ed 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -19,6 +19,7 @@ dependencies: - numpy - pandas - paramiko >=2.6.0 + - pyessv - pyproj - pyupgrade - pyyaml diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b7823584..00000000 --- a/setup.cfg +++ /dev/null @@ -1,79 +0,0 @@ -[bumpversion] -current_version = 0.5.2-beta -commit = True -tag = False -parse = (?P\d+)\.(?P\d+).(?P\d+)(\-(?P[a-z]+))? -serialize = - {major}.{minor}.{patch}-{release} - {major}.{minor}.{patch} - -[bumpversion:part:release] -optional_value = gamma -values = - beta - gamma - -[bumpversion:file:miranda/__init__.py] -search = __version__ = "{current_version}" -replace = __version__ = "{new_version}" - -[flake8] -exclude = - .git, - docs, - build, - .eggs, -max-line-length = 88 -max-complexity = 12 -ignore = - C901 - E203 - E231 - E266 - E501 - F401 - F403 - W503 - W504 - -[aliases] -test = pytest - -[tool:pytest] -addopts = - --verbose - --color=yes -filterwarnings = - ignore::UserWarning - -[tool:pylint] -ignore = docs,tests -disable = - bad-continuation, - invalid-name, - line-too-long, - protected-access, - too-few-public-methods, - too-many-arguments, - -[pydocstyle] -convention = numpy -add-ignore = - D105, - D205, - D400, - D401, - D403 -match = ((?!(test_|conf)).)*\.py - -[pycodestyle] -count = False -exclude = tests -ignore = - E203, - E226, - E402, - E501, - W503 -max-line-length = 120 -statistics = True From 0109a028c3ca71589aa158545a2c050df9828b3d Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 23 Feb 2024 17:39:18 -0500 Subject: [PATCH 09/12] =?UTF-8?q?Bump=20version:=200.6.0=20=E2=86=92=200.6?= =?UTF-8?q?.0-dev.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .cruft.json | 2 +- miranda/__init__.py | 2 +- pyproject.toml | 2 +- tests/test_miranda.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.cruft.json b/.cruft.json index a25f70e7..5c83dfad 100644 --- a/.cruft.json +++ b/.cruft.json @@ -11,7 +11,7 @@ "project_slug": "miranda", "project_short_description": "Python utilities for climate data collection, conversion, and management.", "pypi_username": "Zeitsperre", - "version": "0.6.0", + "version": "0.6.0-dev.0", "use_pytest": "y", "use_black": "y", "use_conda": "y", diff --git a/miranda/__init__.py b/miranda/__init__.py index 1016ecff..7bac747c 100644 --- a/miranda/__init__.py +++ b/miranda/__init__.py @@ -18,7 +18,7 @@ __author__ = """Trevor James Smith""" __email__ = "smith.trevorj@ouranos.ca" -__version__ = "0.6.0" +__version__ = "0.6.0-dev.0" from . import ( diff --git a/pyproject.toml b/pyproject.toml index 1c4129c5..3051a93a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,7 +121,7 @@ target-version = [ ] [tool.bumpversion] -current_version = "0.6.0" +current_version = "0.6.0-dev.0" commit = true commit_args = "--no-verify" tag = false diff --git a/tests/test_miranda.py b/tests/test_miranda.py index c71ede45..f64fba40 100755 --- a/tests/test_miranda.py +++ b/tests/test_miranda.py @@ -70,4 +70,4 @@ def test_package_metadata(): contents = f.read() assert """Trevor James Smith""" in contents assert '__email__ = "smith.trevorj@ouranos.ca"' in contents - assert '__version__ = "0.6.0"' in contents + assert '__version__ = "0.6.0-dev.0"' in contents From 566af5240667837fb82ad937d44341b7674ced0a Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 28 Feb 2024 15:25:14 -0500 Subject: [PATCH 10/12] fast-forward cookiecutter --- .cruft.json | 2 +- .github/workflows/bump-version.yml | 7 +++--- .github/workflows/tag-testpypi.yml | 4 +-- .pre-commit-config.yaml | 15 ++++++----- environment-dev.yml | 8 +++--- pyproject.toml | 40 +++++++++++++++--------------- tox.ini | 2 +- 7 files changed, 39 insertions(+), 39 deletions(-) diff --git a/.cruft.json b/.cruft.json index 5c83dfad..6e475e3d 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/Ouranosinc/cookiecutter-pypackage", - "commit": "88b1bfe181d9f04c0905137be04150ea7f7b950f", + "commit": "f391bbd6ee14ab2478c64a1f78b74bd9903cae81", "checkout": null, "context": { "cookiecutter": { diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml index 36e1247f..ea3e9569 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/bump-version.yml @@ -63,7 +63,7 @@ jobs: git config --local user.name "bumpversion[bot]" - name: Install bump-my-version run: | - python -m pip install "bump-my-version>=0.17.1" + python -m pip install "bump-my-version>=0.18.3" - name: Current Version run: | bump-my-version show current_version @@ -73,11 +73,12 @@ jobs: run: | if [[ ${{ env.CURRENT_VERSION }} =~ -dev(\.\d+)? ]]; then echo "Development version (ends in 'dev(\.\d+)?'), bumping 'build' version" - bump-my-version show new_version --increment build + bump-my-version bump build else echo "Version is stable, bumping 'patch' version" - bump-my-version show new_version --increment patch + bump-my-version bump patch fi + bump-my-version show-bump - name: Push Changes uses: ad-m/github-push-action@master with: diff --git a/.github/workflows/tag-testpypi.yml b/.github/workflows/tag-testpypi.yml index 9033fb0c..13487a3d 100644 --- a/.github/workflows/tag-testpypi.yml +++ b/.github/workflows/tag-testpypi.yml @@ -65,5 +65,5 @@ jobs: - name: Publish distribution 📦 to Test PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: - repository_url: https://test.pypi.org/legacy/ - skip_existing: true + repository-url: https://test.pypi.org/legacy/ + skip-existing: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0a8787b1..a09aa2f4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,14 +26,13 @@ repos: - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 hooks: + - id: python-check-blanket-noqa + - id: python-no-eval + - id: python-no-log-warn + - id: python-use-type-annotations - id: rst-inline-touching-normal - - repo: https://github.com/adrienverge/yamllint.git - rev: v1.32.0 - hooks: - - id: yamllint - args: [ '--config-file=.yamllint.yaml' ] - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.1.1 + rev: 24.2.0 hooks: - id: black exclude: ^docs/ @@ -55,12 +54,12 @@ repos: rev: v0.3.9 hooks: - id: blackdoc - additional_dependencies: [ 'black==24.1.1' ] + additional_dependencies: [ 'black==24.2.0' ] - repo: https://github.com/adrienverge/yamllint.git rev: v1.33.0 hooks: - id: yamllint - args: ['--config-file=.yamllint.yaml'] + args: [ '--config-file=.yamllint.yaml' ] - repo: https://github.com/Yelp/detect-secrets rev: v1.4.0 hooks: diff --git a/environment-dev.yml b/environment-dev.yml index 22a196ed..3127306d 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -34,7 +34,7 @@ dependencies: - zarr # Dev tools and testing - pip >=23.1.2 - - bump-my-version >=0.17.1 + - bump-my-version >=0.18.3 - watchdog >=3.0.0 - flake8 >=6.1.0 - flake8-rst-docstrings >=0.3.0 @@ -43,8 +43,8 @@ dependencies: # - coverage >=6.2.0,<7.0.0 # - coveralls >=3.3.1 - pytest >=7.3.1 -# - pytest-cov>=4.0.0 - - black ==24.1.1 +# - pytest-cov >=4.0.0 + - black ==24.2.0 - blackdoc ==0.3.9 - isort ==5.13.2 - pre-commit >=3.3.2 @@ -52,4 +52,4 @@ dependencies: - pip: - coverage >=6.2.0,<7.0.0 - coveralls >=3.3.1 - - pytest-cov>=4.0.0 + - pytest-cov >=4.0.0 diff --git a/pyproject.toml b/pyproject.toml index 3051a93a..a8a70ea5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,27 +51,27 @@ dependencies = [ [project.optional-dependencies] dev = [ # Dev tools and testing - "pip>=23.1.2", - "bump-my-version>=0.17.1", - "watchdog>=3.0.0", - "flake8>=6.1.0", - "flake8-alphabetize>=0.0.21", - "flake8-rst-docstrings>=0.3.0", - "flit>=3.9.0", - "tox>=4.5.1", - "coverage>=6.2.2,<7.0.0", - "coveralls>=3.3.1", - "pytest>=7.3.1", - "pytest-cov>=4.0.0", - "black==24.1.1", - "blackdoc==0.3.9", - "isort==5.13.2", - "ruff>=0.2.0", - "pre-commit>=3.3.2" + "pip >=23.1.2", + "bump-my-version >=0.18.3", + "watchdog >=3.0.0", + "flake8 >=6.1.0", + "flake8-alphabetize >=0.0.21", + "flake8-rst-docstrings >=0.3.0", + "flit >=3.9.0", + "tox >=4.5.1", + "coverage >=6.2.2,<7.0.0", + "coveralls >=3.3.1", + "pytest >=7.3.1", + "pytest-cov >=4.0.0", + "black ==24.2.0", + "blackdoc ==0.3.9", + "isort ==5.13.2", + "ruff >=0.2.0", + "pre-commit >=3.3.2" ] docs = [ # Documentation and examples - "sphinx>=7.1", + "sphinx >=7.1", "sphinx_codeautolink", "sphinx_copybutton", "sphinx-intl", @@ -81,7 +81,7 @@ docs = [ "ipython", "ipykernel", "jupyter_client", - "furo>=2023.07.26" + "furo >=2023.07.26" ] gis = [ # GIS library support @@ -97,7 +97,7 @@ remote = [ "intake", "intake-esm", "numcodecs", - "paramiko>=2.6.0", + "paramiko >=2.6.0", "s3fs", "scp" ] diff --git a/tox.ini b/tox.ini index 13cb628f..2d459865 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ opts = [testenv:lint] skip_install = True deps = - black ==24.1.1 + black ==24.2.0 blackdoc ==0.3.9 isort ==5.13.2 flake8 From 073ee5b1eb116095b3da0d7cc94fe9a35f1f6ac4 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 28 Feb 2024 15:34:13 -0500 Subject: [PATCH 11/12] exclude .cruft.json --- .secrets.baseline | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index d0d57bc9..51d02a13 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -105,18 +105,14 @@ }, { "path": "detect_secrets.filters.heuristic.is_templated_secret" + }, + { + "path": "detect_secrets.filters.regex.should_exclude_file", + "pattern": [ + ".cruft.json" + ] } ], - "results": { - ".cruft.json": [ - { - "type": "Hex High Entropy String", - "filename": ".cruft.json", - "hashed_secret": "95ed94528595d4d669b5711a6401032d8d8e24ab", - "is_verified": true, - "line_number": 3 - } - ] - }, - "generated_at": "2024-02-23T20:41:08Z" + "results": {}, + "generated_at": "2024-02-28T20:33:25Z" } From caecbc0fa2648e56584b209db1e8c8118daf893c Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 28 Feb 2024 15:34:46 -0500 Subject: [PATCH 12/12] ignore check-blanket-noqa for now --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a09aa2f4..a4464e55 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 hooks: - - id: python-check-blanket-noqa +# - id: python-check-blanket-noqa - id: python-no-eval - id: python-no-log-warn - id: python-use-type-annotations