From b526ff8902c42cc0c89f77f551de12b1e850c61b Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 2 May 2024 12:58:33 -0400 Subject: [PATCH 01/34] requirements: Add initial support for uploading PEP 740 attestations Signed-off-by: William Woodruff --- Dockerfile | 1 + README.md | 21 ++++++++ action.yml | 8 +++ attestations.py | 102 +++++++++++++++++++++++++++++++++++++++ requirements/runtime.in | 7 ++- requirements/runtime.txt | 75 +++++++++++++++++++++++----- twine-upload.sh | 37 ++++++++++++++ 7 files changed, 239 insertions(+), 12 deletions(-) create mode 100644 attestations.py diff --git a/Dockerfile b/Dockerfile index 72e1d22c..96c54f81 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,7 @@ COPY LICENSE.md . COPY twine-upload.sh . COPY print-hash.py . COPY oidc-exchange.py . +COPY attestations.py . RUN chmod +x twine-upload.sh ENTRYPOINT ["/app/twine-upload.sh"] diff --git a/README.md b/README.md index 32a95f32..f036a344 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,25 @@ for example. See [Creating & using secrets]. While still secure, [trusted publishing] is now encouraged over API tokens as a best practice on supported platforms (like GitHub). +### Generating and uploading attestations (EXPERIMENTAL) + +> [!NOTE] +> Support for generating and uploading [PEP 740 attestations] is currently +> experimental and limited only to Trusted Publishing flows using PyPI or TestPyPI. + +You can generate signed [PEP 740 attestations] for all the distribution files and +upload them all together by enabling the `attestations` setting: + +```yml + with: + attestations: true +``` + +This will use `sigstore` to create attestation objects for each distribution package, +signing them with the identity provided by the GitHub's OIDC token associated with the +current workflow. This means both the trusted publishing authentication and the +attestations are tied to the same identity. + ## License The Dockerfile and associated scripts and documentation in this project @@ -287,3 +306,5 @@ https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md [configured on PyPI]: https://docs.pypi.org/trusted-publishers/adding-a-publisher/ [how to specify username and password]: #specifying-a-different-username + +[PEP 740 attestations]: https://peps.python.org/pep-0740/ diff --git a/action.yml b/action.yml index 0c799494..40fed973 100644 --- a/action.yml +++ b/action.yml @@ -80,6 +80,13 @@ inputs: Use `print-hash` instead. required: false default: 'false' + attestations: + description: >- + [EXPERIMENTAL] + Enable experimental support for PEP 740 attestations. + Only works with PyPI and TestPyPI via Trusted Publishing. + required: false + default: 'false' branding: color: yellow icon: upload-cloud @@ -95,3 +102,4 @@ runs: - ${{ inputs.skip-existing }} - ${{ inputs.verbose }} - ${{ inputs.print-hash }} + - ${{ inputs.attestations }} diff --git a/attestations.py b/attestations.py new file mode 100644 index 00000000..faab9f9b --- /dev/null +++ b/attestations.py @@ -0,0 +1,102 @@ +import logging +import os +import sys +from pathlib import Path +from typing import NoReturn + +from pypi_attestation_models import AttestationPayload +from sigstore.oidc import IdentityError, IdentityToken, detect_credential +from sigstore.sign import Signer, SigningContext + +# Be very verbose. +sigstore_logger = logging.getLogger("sigstore") +sigstore_logger.setLevel(logging.DEBUG) +sigstore_logger.addHandler(logging.StreamHandler()) + +_GITHUB_STEP_SUMMARY = Path(os.getenv("GITHUB_STEP_SUMMARY")) + +# The top-level error message that gets rendered. +# This message wraps one of the other templates/messages defined below. +_ERROR_SUMMARY_MESSAGE = """ +Attestation generation failure: + +{message} + +You're seeing this because the action attempted to generated PEP 740 +attestations for its inputs, but failed to do so. +""" + +# Rendered if OIDC identity token retrieval fails for any reason. +_TOKEN_RETRIEVAL_FAILED_MESSAGE = """ +OpenID Connect token retrieval failed: {identity_error} + +This generally indicates a workflow configuration error, such as insufficient +permissions. Make sure that your workflow has `id-token: write` configured +at the job level, e.g.: + +```yaml +permissions: + id-token: write +``` + +Learn more at https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect#adding-permissions-settings. +""" + + +def die(msg: str) -> NoReturn: + with _GITHUB_STEP_SUMMARY.open("a", encoding="utf-8") as io: + print(_ERROR_SUMMARY_MESSAGE.format(message=msg), file=io) + + # HACK: GitHub Actions' annotations don't work across multiple lines naively; + # translating `\n` into `%0A` (i.e., HTML percent-encoding) is known to work. + # See: https://github.com/actions/toolkit/issues/193 + msg = msg.replace("\n", "%0A") + print(f"::error::Attestation generation failure: {msg}", file=sys.stderr) + sys.exit(1) + + +def debug(msg: str): + print(f"::debug::{msg}", file=sys.stderr) + + +# pylint: disable=redefined-outer-name +def attest_dist(dist: Path, signer: Signer) -> None: + # We are the publishing step, so there should be no pre-existing publish + # attestation. The presence of one indicates user confusion. + attestation_path = Path(f"{dist}.publish.attestation") + if attestation_path.is_file(): + die(f"{dist} already has a publish attestation: {attestation_path}") + + payload = AttestationPayload.from_dist(dist) + attestation = payload.sign(signer) + + attestation_path.write_text(attestation.model_dump_json(), encoding="utf-8") + debug(f"saved publish attestation: {dist=} {attestation_path=}") + + +packages_dir = Path(sys.argv[1]) + +try: + # NOTE: audience is always sigstore. + oidc_token = detect_credential() + identity = IdentityToken(oidc_token) +except IdentityError as identity_error: + # NOTE: We only perform attestations in trusted publishing flows, so we + # don't need to re-check for the "PR from fork" error mode, only + # generic token retrieval errors. + cause = _TOKEN_RETRIEVAL_FAILED_MESSAGE.format(identity_error=identity_error) + die(cause) + +# Collect all sdists and wheels. +dists = [sdist.absolute() for sdist in packages_dir.glob("*.tar.gz")] +dists.extend(whl.absolute() for whl in packages_dir.glob("*.whl")) + +with SigningContext.production().signer(identity, cache=True) as signer: + for dist in dists: + # This should never really happen, but some versions of GitHub's + # download-artifact will create a subdirectory with the same name + # as the artifact being downloaded, e.g. `dist/foo.whl/foo.whl`. + if not dist.is_file(): + die(f"Path looks like a distribution but is not a file: {dist}") + + attest_dist(dist, signer) diff --git a/requirements/runtime.in b/requirements/runtime.in index 57be3b99..4148e6ce 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -1,9 +1,14 @@ twine -# NOTE: Used to detect an ambient OIDC credential for OIDC publishing. +# NOTE: Used to detect an ambient OIDC credential for OIDC publishing, +# as well as PEP 740 attestations. id ~= 1.0 # NOTE: This is pulled in transitively through `twine`, but we also declare # NOTE: it explicitly here because `oidc-exchange.py` uses it. # Ref: https://github.com/di/id requests + +# NOTE: Used to generate attestations. +pypi-attestation-models == 0.0.2 +sigstore ~= 3.0.0 diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 57fde0c0..4a57cb04 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -6,6 +6,8 @@ # annotated-types==0.6.0 # via pydantic +betterproto==2.0.0b6 + # via sigstore-protobuf-specs certifi==2024.2.2 # via requests cffi==1.16.0 @@ -13,13 +15,32 @@ cffi==1.16.0 charset-normalizer==3.3.2 # via requests cryptography==42.0.7 - # via secretstorage + # via + # pyopenssl + # pypi-attestation-models + # sigstore +dnspython==2.6.1 + # via email-validator docutils==0.21.2 # via readme-renderer +email-validator==2.1.1 + # via pydantic +grpclib==0.4.7 + # via betterproto +h2==4.1.0 + # via grpclib +hpack==4.0.0 + # via h2 +hyperframe==6.0.1 + # via h2 id==1.4.0 - # via -r runtime.in + # via + # -r runtime.in + # sigstore idna==3.7 - # via requests + # via + # email-validator + # requests importlib-metadata==7.1.0 # via twine jaraco-classes==3.4.0 @@ -28,10 +49,6 @@ jaraco-context==5.3.0 # via keyring jaraco-functools==4.0.1 # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage keyring==25.2.1 # via twine markdown-it-py==3.0.0 @@ -42,20 +59,38 @@ more-itertools==10.2.0 # via # jaraco-classes # jaraco-functools +multidict==6.0.5 + # via grpclib nh3==0.2.17 # via readme-renderer pkginfo==1.10.0 # via twine +platformdirs==4.2.2 + # via sigstore +pyasn1==0.6.0 + # via sigstore pycparser==2.22 # via cffi pydantic==2.7.1 - # via id + # via + # id + # pypi-attestation-models + # sigstore + # sigstore-rekor-types pydantic-core==2.18.2 # via pydantic pygments==2.18.0 # via # readme-renderer # rich +pyjwt==2.8.0 + # via sigstore +pyopenssl==24.1.0 + # via sigstore +pypi-attestation-models==0.0.2 + # via -r runtime.in +python-dateutil==2.9.0.post0 + # via betterproto readme-renderer==43.0 # via twine requests==2.31.0 @@ -63,15 +98,33 @@ requests==2.31.0 # -r runtime.in # id # requests-toolbelt + # sigstore + # tuf # twine requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine +rfc8785==0.1.2 + # via sigstore rich==13.7.1 - # via twine -secretstorage==3.3.3 - # via keyring + # via + # sigstore + # twine +securesystemslib==1.0.0 + # via tuf +sigstore==3.0.0 + # via + # -r runtime.in + # pypi-attestation-models +sigstore-protobuf-specs==0.3.2 + # via sigstore +sigstore-rekor-types==0.0.13 + # via sigstore +six==1.16.0 + # via python-dateutil +tuf==5.0.0 + # via sigstore twine==5.1.0 # via -r runtime.in typing-extensions==4.11.0 diff --git a/twine-upload.sh b/twine-upload.sh index 9c608ade..8493825f 100755 --- a/twine-upload.sh +++ b/twine-upload.sh @@ -39,6 +39,7 @@ INPUT_PACKAGES_DIR="$(get-normalized-input 'packages-dir')" INPUT_VERIFY_METADATA="$(get-normalized-input 'verify-metadata')" INPUT_SKIP_EXISTING="$(get-normalized-input 'skip-existing')" INPUT_PRINT_HASH="$(get-normalized-input 'print-hash')" +INPUT_ATTESTATIONS="$(get-normalized-input 'attestations')" PASSWORD_DEPRECATION_NUDGE="::error title=Password-based uploads disabled::\ As of 2024, PyPI requires all users to enable Two-Factor \ @@ -53,6 +54,33 @@ environments like GitHub Actions without needing to use username/password \ combinations or API tokens to authenticate with PyPI. Read more: \ https://docs.pypi.org/trusted-publishers" +ATTESTATIONS_WITHOUT_TP_WARNING="::warning title=attestations setting ignored::\ +The workflow was run with 'attestations: true', but an explicit password was \ +also supplied, disabling Trusted Publishing. As a result, the attestations \ +setting is ignored." + +ATTESTATIONS_WRONG_INDEX_WARNING="::warning title=attestations setting ignored::\ +The workflow was run with 'attestations: true', but the specified repository URL \ +does not support PEP 740 attestations. As a result, the attestations setting \ +is ignored." + +if [[ "${INPUT_ATTESTATIONS}" != "false" ]] ; then + # Setting `attestations: true` and explicitly passing a password indicates + # user confusion, since attestations (currently) require Trusted Publishing. + if [[ -n "${INPUT_PASSWORD}" ]] ; then + echo "${ATTESTATIONS_WITHOUT_TP_WARNING}" + INPUT_ATTESTATIONS="false" + fi + + # Setting `attestations: true` with an index other than PyPI or TestPyPI + # indicates user confusion, since attestations are not supported on other + # indices presently. + if [[ ! "${INPUT_REPOSITORY_URL}" =~ pypi\.org ]] ; then + echo "${ATTESTATIONS_WRONG_INDEX_WARNING}" + INPUT_ATTESTATIONS="false" + fi +fi + if [[ "${INPUT_USER}" == "__token__" && -z "${INPUT_PASSWORD}" ]] ; then # No password supplied by the user implies that we're in the OIDC flow; # retrieve the OIDC credential and exchange it for a PyPI API token. @@ -130,6 +158,15 @@ if [[ ${INPUT_VERBOSE,,} != "false" ]] ; then TWINE_EXTRA_ARGS="--verbose $TWINE_EXTRA_ARGS" fi +if [[ ${INPUT_ATTESTATIONS,,} != "false" ]] ; then + # NOTE: Intentionally placed after `twine check`, to prevent attestation + # generation on distributions with invalid metadata. + echo "::debug::Generating and uploading PEP 740 attestations" + python /app/attestations.py "${INPUT_PACKAGES_DIR%%/}" + + TWINE_EXTRA_ARGS="--attestations $TWINE_EXTRA_ARGS" +fi + if [[ ${INPUT_PRINT_HASH,,} != "false" || ${INPUT_VERBOSE,,} != "false" ]] ; then python /app/print-hash.py ${INPUT_PACKAGES_DIR%%/} fi From f267787a9d9e677a5ceacde2b2fe540253c83ad5 Mon Sep 17 00:00:00 2001 From: Facundo Tuesca Date: Thu, 16 May 2024 19:43:17 +0200 Subject: [PATCH 02/34] Misc lint fixes Signed-off-by: Facundo Tuesca --- attestations.py | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/attestations.py b/attestations.py index faab9f9b..82577430 100644 --- a/attestations.py +++ b/attestations.py @@ -9,11 +9,11 @@ from sigstore.sign import Signer, SigningContext # Be very verbose. -sigstore_logger = logging.getLogger("sigstore") +sigstore_logger = logging.getLogger('sigstore') sigstore_logger.setLevel(logging.DEBUG) sigstore_logger.addHandler(logging.StreamHandler()) -_GITHUB_STEP_SUMMARY = Path(os.getenv("GITHUB_STEP_SUMMARY")) +_GITHUB_STEP_SUMMARY = Path(os.getenv('GITHUB_STEP_SUMMARY')) # The top-level error message that gets rendered. # This message wraps one of the other templates/messages defined below. @@ -40,46 +40,52 @@ ``` Learn more at https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect#adding-permissions-settings. -""" +""" # noqa: S105; not a password def die(msg: str) -> NoReturn: - with _GITHUB_STEP_SUMMARY.open("a", encoding="utf-8") as io: + with _GITHUB_STEP_SUMMARY.open('a', encoding='utf-8') as io: print(_ERROR_SUMMARY_MESSAGE.format(message=msg), file=io) # HACK: GitHub Actions' annotations don't work across multiple lines naively; # translating `\n` into `%0A` (i.e., HTML percent-encoding) is known to work. # See: https://github.com/actions/toolkit/issues/193 - msg = msg.replace("\n", "%0A") - print(f"::error::Attestation generation failure: {msg}", file=sys.stderr) + msg = msg.replace('\n', '%0A') + print(f'::error::Attestation generation failure: {msg}', file=sys.stderr) sys.exit(1) def debug(msg: str): - print(f"::debug::{msg}", file=sys.stderr) + print(f'::debug::{msg}', file=sys.stderr) # pylint: disable=redefined-outer-name def attest_dist(dist: Path, signer: Signer) -> None: # We are the publishing step, so there should be no pre-existing publish # attestation. The presence of one indicates user confusion. - attestation_path = Path(f"{dist}.publish.attestation") + attestation_path = Path(f'{dist}.publish.attestation') if attestation_path.is_file(): - die(f"{dist} already has a publish attestation: {attestation_path}") + die(f'{dist} already has a publish attestation: {attestation_path}') payload = AttestationPayload.from_dist(dist) attestation = payload.sign(signer) - attestation_path.write_text(attestation.model_dump_json(), encoding="utf-8") - debug(f"saved publish attestation: {dist=} {attestation_path=}") + attestation_path.write_text(attestation.model_dump_json(), encoding='utf-8') + debug(f'saved publish attestation: {dist=} {attestation_path=}') + + +def get_identity_token() -> IdentityToken: + # Will raise `sigstore.oidc.IdentityError` if it fails to get the token + # from the environment or if the token is malformed. + # NOTE: audience is always sigstore. + oidc_token = detect_credential() + return IdentityToken(oidc_token) packages_dir = Path(sys.argv[1]) try: - # NOTE: audience is always sigstore. - oidc_token = detect_credential() - identity = IdentityToken(oidc_token) + identity = get_identity_token() except IdentityError as identity_error: # NOTE: We only perform attestations in trusted publishing flows, so we # don't need to re-check for the "PR from fork" error mode, only @@ -88,8 +94,8 @@ def attest_dist(dist: Path, signer: Signer) -> None: die(cause) # Collect all sdists and wheels. -dists = [sdist.absolute() for sdist in packages_dir.glob("*.tar.gz")] -dists.extend(whl.absolute() for whl in packages_dir.glob("*.whl")) +dists = [sdist.absolute() for sdist in packages_dir.glob('*.tar.gz')] +dists.extend(whl.absolute() for whl in packages_dir.glob('*.whl')) with SigningContext.production().signer(identity, cache=True) as signer: for dist in dists: @@ -97,6 +103,6 @@ def attest_dist(dist: Path, signer: Signer) -> None: # download-artifact will create a subdirectory with the same name # as the artifact being downloaded, e.g. `dist/foo.whl/foo.whl`. if not dist.is_file(): - die(f"Path looks like a distribution but is not a file: {dist}") + die(f'Path looks like a distribution but is not a file: {dist}') attest_dist(dist, signer) From 1571a0dc4b7925b0b8317b71a031dcb1cf2edc69 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 11 Jun 2024 17:35:16 -0400 Subject: [PATCH 03/34] bump pypi_attestation_models, update usage Signed-off-by: William Woodruff --- attestations.py | 31 +++++++++++++++---------------- requirements/runtime.in | 2 +- requirements/runtime.txt | 8 ++++++-- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/attestations.py b/attestations.py index 82577430..af9e3980 100644 --- a/attestations.py +++ b/attestations.py @@ -4,16 +4,16 @@ from pathlib import Path from typing import NoReturn -from pypi_attestation_models import AttestationPayload +from pypi_attestation_models import Attestation from sigstore.oidc import IdentityError, IdentityToken, detect_credential from sigstore.sign import Signer, SigningContext # Be very verbose. -sigstore_logger = logging.getLogger('sigstore') +sigstore_logger = logging.getLogger("sigstore") sigstore_logger.setLevel(logging.DEBUG) sigstore_logger.addHandler(logging.StreamHandler()) -_GITHUB_STEP_SUMMARY = Path(os.getenv('GITHUB_STEP_SUMMARY')) +_GITHUB_STEP_SUMMARY = Path(os.getenv("GITHUB_STEP_SUMMARY")) # The top-level error message that gets rendered. # This message wraps one of the other templates/messages defined below. @@ -44,34 +44,33 @@ def die(msg: str) -> NoReturn: - with _GITHUB_STEP_SUMMARY.open('a', encoding='utf-8') as io: + with _GITHUB_STEP_SUMMARY.open("a", encoding="utf-8") as io: print(_ERROR_SUMMARY_MESSAGE.format(message=msg), file=io) # HACK: GitHub Actions' annotations don't work across multiple lines naively; # translating `\n` into `%0A` (i.e., HTML percent-encoding) is known to work. # See: https://github.com/actions/toolkit/issues/193 - msg = msg.replace('\n', '%0A') - print(f'::error::Attestation generation failure: {msg}', file=sys.stderr) + msg = msg.replace("\n", "%0A") + print(f"::error::Attestation generation failure: {msg}", file=sys.stderr) sys.exit(1) def debug(msg: str): - print(f'::debug::{msg}', file=sys.stderr) + print(f"::debug::{msg}", file=sys.stderr) # pylint: disable=redefined-outer-name def attest_dist(dist: Path, signer: Signer) -> None: # We are the publishing step, so there should be no pre-existing publish # attestation. The presence of one indicates user confusion. - attestation_path = Path(f'{dist}.publish.attestation') + attestation_path = Path(f"{dist}.publish.attestation") if attestation_path.is_file(): - die(f'{dist} already has a publish attestation: {attestation_path}') + die(f"{dist} already has a publish attestation: {attestation_path}") - payload = AttestationPayload.from_dist(dist) - attestation = payload.sign(signer) + attestation = Attestation.sign(signer, dist) - attestation_path.write_text(attestation.model_dump_json(), encoding='utf-8') - debug(f'saved publish attestation: {dist=} {attestation_path=}') + attestation_path.write_text(attestation.model_dump_json(), encoding="utf-8") + debug(f"saved publish attestation: {dist=} {attestation_path=}") def get_identity_token() -> IdentityToken: @@ -94,8 +93,8 @@ def get_identity_token() -> IdentityToken: die(cause) # Collect all sdists and wheels. -dists = [sdist.absolute() for sdist in packages_dir.glob('*.tar.gz')] -dists.extend(whl.absolute() for whl in packages_dir.glob('*.whl')) +dists = [sdist.absolute() for sdist in packages_dir.glob("*.tar.gz")] +dists.extend(whl.absolute() for whl in packages_dir.glob("*.whl")) with SigningContext.production().signer(identity, cache=True) as signer: for dist in dists: @@ -103,6 +102,6 @@ def get_identity_token() -> IdentityToken: # download-artifact will create a subdirectory with the same name # as the artifact being downloaded, e.g. `dist/foo.whl/foo.whl`. if not dist.is_file(): - die(f'Path looks like a distribution but is not a file: {dist}') + die(f"Path looks like a distribution but is not a file: {dist}") attest_dist(dist, signer) diff --git a/requirements/runtime.in b/requirements/runtime.in index 4148e6ce..8d051f75 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -10,5 +10,5 @@ id ~= 1.0 requests # NOTE: Used to generate attestations. -pypi-attestation-models == 0.0.2 +pypi-attestation-models == 0.0.4 sigstore ~= 3.0.0 diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 4a57cb04..16b30994 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -63,6 +63,8 @@ multidict==6.0.5 # via grpclib nh3==0.2.17 # via readme-renderer +packaging==24.1 + # via pypi-attestation-models pkginfo==1.10.0 # via twine platformdirs==4.2.2 @@ -87,7 +89,7 @@ pyjwt==2.8.0 # via sigstore pyopenssl==24.1.0 # via sigstore -pypi-attestation-models==0.0.2 +pypi-attestation-models==0.0.4 # via -r runtime.in python-dateutil==2.9.0.post0 # via betterproto @@ -118,7 +120,9 @@ sigstore==3.0.0 # -r runtime.in # pypi-attestation-models sigstore-protobuf-specs==0.3.2 - # via sigstore + # via + # pypi-attestation-models + # sigstore sigstore-rekor-types==0.0.13 # via sigstore six==1.16.0 From 27500cf1c30b666011e9ad881681e94107dd7dcb Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 11 Jun 2024 17:39:45 -0400 Subject: [PATCH 04/34] attestations: single quotes Signed-off-by: William Woodruff --- attestations.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/attestations.py b/attestations.py index af9e3980..4b304240 100644 --- a/attestations.py +++ b/attestations.py @@ -9,11 +9,11 @@ from sigstore.sign import Signer, SigningContext # Be very verbose. -sigstore_logger = logging.getLogger("sigstore") +sigstore_logger = logging.getLogger('sigstore') sigstore_logger.setLevel(logging.DEBUG) sigstore_logger.addHandler(logging.StreamHandler()) -_GITHUB_STEP_SUMMARY = Path(os.getenv("GITHUB_STEP_SUMMARY")) +_GITHUB_STEP_SUMMARY = Path(os.getenv('GITHUB_STEP_SUMMARY')) # The top-level error message that gets rendered. # This message wraps one of the other templates/messages defined below. @@ -44,33 +44,33 @@ def die(msg: str) -> NoReturn: - with _GITHUB_STEP_SUMMARY.open("a", encoding="utf-8") as io: + with _GITHUB_STEP_SUMMARY.open('a', encoding='utf-8') as io: print(_ERROR_SUMMARY_MESSAGE.format(message=msg), file=io) # HACK: GitHub Actions' annotations don't work across multiple lines naively; # translating `\n` into `%0A` (i.e., HTML percent-encoding) is known to work. # See: https://github.com/actions/toolkit/issues/193 - msg = msg.replace("\n", "%0A") - print(f"::error::Attestation generation failure: {msg}", file=sys.stderr) + msg = msg.replace('\n', '%0A') + print(f'::error::Attestation generation failure: {msg}', file=sys.stderr) sys.exit(1) def debug(msg: str): - print(f"::debug::{msg}", file=sys.stderr) + print(f'::debug::{msg}', file=sys.stderr) # pylint: disable=redefined-outer-name def attest_dist(dist: Path, signer: Signer) -> None: # We are the publishing step, so there should be no pre-existing publish # attestation. The presence of one indicates user confusion. - attestation_path = Path(f"{dist}.publish.attestation") + attestation_path = Path(f'{dist}.publish.attestation') if attestation_path.is_file(): - die(f"{dist} already has a publish attestation: {attestation_path}") + die(f'{dist} already has a publish attestation: {attestation_path}') attestation = Attestation.sign(signer, dist) - attestation_path.write_text(attestation.model_dump_json(), encoding="utf-8") - debug(f"saved publish attestation: {dist=} {attestation_path=}") + attestation_path.write_text(attestation.model_dump_json(), encoding='utf-8') + debug(f'saved publish attestation: {dist=} {attestation_path=}') def get_identity_token() -> IdentityToken: @@ -93,8 +93,8 @@ def get_identity_token() -> IdentityToken: die(cause) # Collect all sdists and wheels. -dists = [sdist.absolute() for sdist in packages_dir.glob("*.tar.gz")] -dists.extend(whl.absolute() for whl in packages_dir.glob("*.whl")) +dists = [sdist.absolute() for sdist in packages_dir.glob('*.tar.gz')] +dists.extend(whl.absolute() for whl in packages_dir.glob('*.whl')) with SigningContext.production().signer(identity, cache=True) as signer: for dist in dists: @@ -102,6 +102,6 @@ def get_identity_token() -> IdentityToken: # download-artifact will create a subdirectory with the same name # as the artifact being downloaded, e.g. `dist/foo.whl/foo.whl`. if not dist.is_file(): - die(f"Path looks like a distribution but is not a file: {dist}") + die(f'Path looks like a distribution but is not a file: {dist}') attest_dist(dist, signer) From e9c72dd8ce47b0f8e73dee72cd838f807a6b5ca4 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 11 Jun 2024 17:43:25 -0400 Subject: [PATCH 05/34] attestations: simplify err Signed-off-by: William Woodruff --- attestations.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/attestations.py b/attestations.py index 4b304240..b5ce9917 100644 --- a/attestations.py +++ b/attestations.py @@ -30,16 +30,8 @@ _TOKEN_RETRIEVAL_FAILED_MESSAGE = """ OpenID Connect token retrieval failed: {identity_error} -This generally indicates a workflow configuration error, such as insufficient -permissions. Make sure that your workflow has `id-token: write` configured -at the job level, e.g.: - -```yaml -permissions: - id-token: write -``` - -Learn more at https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect#adding-permissions-settings. +This failure occurred after a successful Trusted Publishing Flow, +suggesting a transient error. """ # noqa: S105; not a password @@ -88,7 +80,8 @@ def get_identity_token() -> IdentityToken: except IdentityError as identity_error: # NOTE: We only perform attestations in trusted publishing flows, so we # don't need to re-check for the "PR from fork" error mode, only - # generic token retrieval errors. + # generic token retrieval errors. We also render a simpler error, + # since permissions can't be to blame at this stage. cause = _TOKEN_RETRIEVAL_FAILED_MESSAGE.format(identity_error=identity_error) die(cause) From e7bd6eacb21d18f76f01160193b2818d55aeeda7 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 11 Jun 2024 17:55:37 -0400 Subject: [PATCH 06/34] README: add a link Signed-off-by: William Woodruff --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f036a344..1aa54f2e 100644 --- a/README.md +++ b/README.md @@ -260,10 +260,11 @@ upload them all together by enabling the `attestations` setting: attestations: true ``` -This will use `sigstore` to create attestation objects for each distribution package, -signing them with the identity provided by the GitHub's OIDC token associated with the -current workflow. This means both the trusted publishing authentication and the -attestations are tied to the same identity. +This will use [Sigstore](https://www.sigstore.dev/) to create attestation +objects for each distribution package, signing them with the identity provided +by the GitHub's OIDC token associated with the current workflow. This means +both the trusted publishing authentication and the attestations are tied to the +same identity. ## License From 5aa7e4119991eda8739ae483dc1ec68a10acf6d3 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 18 Jun 2024 10:04:22 -0400 Subject: [PATCH 07/34] runtime: constrain pypi-attestation-models with a range Signed-off-by: William Woodruff --- requirements/runtime.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/runtime.in b/requirements/runtime.in index 8d051f75..829ab31b 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -10,5 +10,5 @@ id ~= 1.0 requests # NOTE: Used to generate attestations. -pypi-attestation-models == 0.0.4 +pypi-attestation-models >= 0.0.4, < 0.1.0 sigstore ~= 3.0.0 From 0e2b9c963bc93d808bd7144c7738eefc040cee2e Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 20 Jun 2024 15:47:33 -0400 Subject: [PATCH 08/34] runtime: bump range for pypi-attestation-models Signed-off-by: William Woodruff --- requirements/runtime.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/runtime.in b/requirements/runtime.in index 829ab31b..7f169044 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -10,5 +10,5 @@ id ~= 1.0 requests # NOTE: Used to generate attestations. -pypi-attestation-models >= 0.0.4, < 0.1.0 +pypi-attestation-models >= 0.0.5, < 0.1.0 sigstore ~= 3.0.0 From 242d7e99db54e5000d0305b1e50cf0e465f67ad4 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Fri, 21 Jun 2024 15:01:17 -0400 Subject: [PATCH 09/34] requirements: refreeze Signed-off-by: William Woodruff --- requirements/runtime.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 05181419..77e4ccb8 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -89,7 +89,7 @@ pyjwt==2.8.0 # via sigstore pyopenssl==24.1.0 # via sigstore -pypi-attestation-models==0.0.4 +pypi-attestation-models==0.0.5 # via -r runtime.in python-dateutil==2.9.0.post0 # via betterproto From aa69903cc56099380141db6bf9440f04f7deed43 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Jun 2024 16:48:40 -0400 Subject: [PATCH 10/34] Update requirements/runtime.in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) --- requirements/runtime.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/runtime.in b/requirements/runtime.in index 7f169044..f8384d0d 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -10,5 +10,5 @@ id ~= 1.0 requests # NOTE: Used to generate attestations. -pypi-attestation-models >= 0.0.5, < 0.1.0 +pypi-attestation-models ~= 0.0.5 sigstore ~= 3.0.0 From 6b4d3716c2f30676f2230f595cfec75a36dca1da Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Jun 2024 17:03:31 -0400 Subject: [PATCH 11/34] attestations: pre-validate dists as files Signed-off-by: William Woodruff --- attestations.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/attestations.py b/attestations.py index b5ce9917..882f3e2d 100644 --- a/attestations.py +++ b/attestations.py @@ -89,12 +89,13 @@ def get_identity_token() -> IdentityToken: dists = [sdist.absolute() for sdist in packages_dir.glob('*.tar.gz')] dists.extend(whl.absolute() for whl in packages_dir.glob('*.whl')) +# Make sure everything that looks like a dist actually is one. +# We do this up-front to prevent partial signing. +for dist in dists: + if not dist.is_file(): + die(f'Path looks like a distribution but is not a file: {dist}') + + with SigningContext.production().signer(identity, cache=True) as signer: for dist in dists: - # This should never really happen, but some versions of GitHub's - # download-artifact will create a subdirectory with the same name - # as the artifact being downloaded, e.g. `dist/foo.whl/foo.whl`. - if not dist.is_file(): - die(f'Path looks like a distribution but is not a file: {dist}') - attest_dist(dist, signer) From 16aa3a2494ff3746488242ef2649fe485f75ca7d Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Jun 2024 17:08:39 -0400 Subject: [PATCH 12/34] README: relocate PEP 740 info Signed-off-by: William Woodruff --- README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 1aa54f2e..e82e4922 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,25 @@ filter to the job: if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') ``` +### Generating and uploading attestations + +> [!IMPORTANT] +> Support for generating and uploading [PEP 740 attestations] is currently +> experimental and limited only to Trusted Publishing flows using PyPI or TestPyPI. + +You can generate signed [PEP 740 attestations] for all the distribution files and +upload them all together by enabling the `attestations` setting: + +```yml + with: + attestations: true +``` + +This will use [Sigstore] to create attestation +objects for each distribution package, signing them with the identity provided +by the GitHub's OIDC token associated with the current workflow. This means +both the trusted publishing authentication and the attestations are tied to the +same identity. ## Non-goals @@ -246,26 +265,6 @@ for example. See [Creating & using secrets]. While still secure, [trusted publishing] is now encouraged over API tokens as a best practice on supported platforms (like GitHub). -### Generating and uploading attestations (EXPERIMENTAL) - -> [!NOTE] -> Support for generating and uploading [PEP 740 attestations] is currently -> experimental and limited only to Trusted Publishing flows using PyPI or TestPyPI. - -You can generate signed [PEP 740 attestations] for all the distribution files and -upload them all together by enabling the `attestations` setting: - -```yml - with: - attestations: true -``` - -This will use [Sigstore](https://www.sigstore.dev/) to create attestation -objects for each distribution package, signing them with the identity provided -by the GitHub's OIDC token associated with the current workflow. This means -both the trusted publishing authentication and the attestations are tied to the -same identity. - ## License The Dockerfile and associated scripts and documentation in this project @@ -309,3 +308,4 @@ https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md [how to specify username and password]: #specifying-a-different-username [PEP 740 attestations]: https://peps.python.org/pep-0740/ +[Sigstore]: https://www.sigstore.dev/ From 6dbccb5e73965faad649960dc2e1317f30f5f3c1 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Jun 2024 17:10:11 -0400 Subject: [PATCH 13/34] README: PEP 740 -> "digital attestations" Signed-off-by: William Woodruff --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e82e4922..913bea8b 100644 --- a/README.md +++ b/README.md @@ -102,10 +102,10 @@ filter to the job: ### Generating and uploading attestations > [!IMPORTANT] -> Support for generating and uploading [PEP 740 attestations] is currently +> Support for generating and uploading [digital attestations] is currently > experimental and limited only to Trusted Publishing flows using PyPI or TestPyPI. -You can generate signed [PEP 740 attestations] for all the distribution files and +You can generate signed [digital attestations] for all the distribution files and upload them all together by enabling the `attestations` setting: ```yml @@ -307,5 +307,5 @@ https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md [how to specify username and password]: #specifying-a-different-username -[PEP 740 attestations]: https://peps.python.org/pep-0740/ +[digital attestations]: https://peps.python.org/pep-0740/ [Sigstore]: https://www.sigstore.dev/ From 16b5dc1219fa6544b0e47e474879fdf19a60c224 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Jun 2024 17:14:52 -0400 Subject: [PATCH 14/34] README: explain that digital attestations require TP Signed-off-by: William Woodruff --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 913bea8b..f9a7eaf7 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,10 @@ filter to the job: > Support for generating and uploading [digital attestations] is currently > experimental and limited only to Trusted Publishing flows using PyPI or TestPyPI. +> [!NOTE] +> Generating and uploading digital attestations currently requires +> authentication with a [trusted publisher]. + You can generate signed [digital attestations] for all the distribution files and upload them all together by enabling the `attestations` setting: @@ -309,3 +313,4 @@ https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md [digital attestations]: https://peps.python.org/pep-0740/ [Sigstore]: https://www.sigstore.dev/ +[trusted publisher]: #trusted-publishing From 251402ef903e6ba6f663914258347657b1d437a3 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Jun 2024 17:15:59 -0400 Subject: [PATCH 15/34] attestations: fix pylint Signed-off-by: William Woodruff --- attestations.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/attestations.py b/attestations.py index 882f3e2d..9c3a4733 100644 --- a/attestations.py +++ b/attestations.py @@ -51,18 +51,17 @@ def debug(msg: str): print(f'::debug::{msg}', file=sys.stderr) -# pylint: disable=redefined-outer-name -def attest_dist(dist: Path, signer: Signer) -> None: +def attest_dist(dist_path: Path, signer: Signer) -> None: # We are the publishing step, so there should be no pre-existing publish # attestation. The presence of one indicates user confusion. - attestation_path = Path(f'{dist}.publish.attestation') + attestation_path = Path(f'{dist_path}.publish.attestation') if attestation_path.is_file(): - die(f'{dist} already has a publish attestation: {attestation_path}') + die(f'{dist_path} already has a publish attestation: {attestation_path}') - attestation = Attestation.sign(signer, dist) + attestation = Attestation.sign(signer, dist_path) attestation_path.write_text(attestation.model_dump_json(), encoding='utf-8') - debug(f'saved publish attestation: {dist=} {attestation_path=}') + debug(f'saved publish attestation: {dist_path=} {attestation_path=}') def get_identity_token() -> IdentityToken: @@ -96,6 +95,6 @@ def get_identity_token() -> IdentityToken: die(f'Path looks like a distribution but is not a file: {dist}') -with SigningContext.production().signer(identity, cache=True) as signer: +with SigningContext.production().signer(identity, cache=True) as s: for dist in dists: - attest_dist(dist, signer) + attest_dist(dist, s) From 1e91a3b91e355846990fb608a7b9c82ecc17d1f9 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Jun 2024 17:17:57 -0400 Subject: [PATCH 16/34] twine-upload: debug -> notice, rm PEP ref Signed-off-by: William Woodruff --- twine-upload.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/twine-upload.sh b/twine-upload.sh index 8493825f..c876bc3a 100755 --- a/twine-upload.sh +++ b/twine-upload.sh @@ -161,7 +161,7 @@ fi if [[ ${INPUT_ATTESTATIONS,,} != "false" ]] ; then # NOTE: Intentionally placed after `twine check`, to prevent attestation # generation on distributions with invalid metadata. - echo "::debug::Generating and uploading PEP 740 attestations" + echo "::notice::Generating and uploading digital attestations" python /app/attestations.py "${INPUT_PACKAGES_DIR%%/}" TWINE_EXTRA_ARGS="--attestations $TWINE_EXTRA_ARGS" From 835d65d20311bb96d0f274ba1b159cd5493b33bc Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Jun 2024 17:23:41 -0400 Subject: [PATCH 17/34] attestations: debug dists before signing Signed-off-by: William Woodruff --- attestations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/attestations.py b/attestations.py index 9c3a4733..afe2ccbc 100644 --- a/attestations.py +++ b/attestations.py @@ -96,5 +96,6 @@ def get_identity_token() -> IdentityToken: with SigningContext.production().signer(identity, cache=True) as s: + debug(f'attesting to dists: {dists}') for dist in dists: attest_dist(dist, s) From 95be6b900921358809053bfdc0a581d5508150c1 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Jun 2024 17:48:10 -0400 Subject: [PATCH 18/34] twine-upload: factor out TRUSTED_PUBLISHING Signed-off-by: William Woodruff --- twine-upload.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/twine-upload.sh b/twine-upload.sh index c876bc3a..00dd7982 100755 --- a/twine-upload.sh +++ b/twine-upload.sh @@ -64,10 +64,13 @@ The workflow was run with 'attestations: true', but the specified repository URL does not support PEP 740 attestations. As a result, the attestations setting \ is ignored." +[[ "${INPUT_USER}" == "__token__" && -z "${INPUT_PASSWORD}" ]] \ + && TRUSTED_PUBLISHING=true || TRUSTED_PUBLISHING=false + if [[ "${INPUT_ATTESTATIONS}" != "false" ]] ; then - # Setting `attestations: true` and explicitly passing a password indicates - # user confusion, since attestations (currently) require Trusted Publishing. - if [[ -n "${INPUT_PASSWORD}" ]] ; then + # Setting `attestations: true` without Trusted Publishing indicates + # user confusion, since attestations (currently) require it. + if ! "${TRUSTED_PUBLISHING}" ; then echo "${ATTESTATIONS_WITHOUT_TP_WARNING}" INPUT_ATTESTATIONS="false" fi @@ -81,7 +84,7 @@ if [[ "${INPUT_ATTESTATIONS}" != "false" ]] ; then fi fi -if [[ "${INPUT_USER}" == "__token__" && -z "${INPUT_PASSWORD}" ]] ; then +if "${TRUSTED_PUBLISHING}" ; then # No password supplied by the user implies that we're in the OIDC flow; # retrieve the OIDC credential and exchange it for a PyPI API token. echo "::debug::Authenticating to ${INPUT_REPOSITORY_URL} via Trusted Publishing" From 176c905e4e087a728378a9b356ba3c1d2632e44d Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Fri, 28 Jun 2024 13:09:52 -0400 Subject: [PATCH 19/34] pypi_attestation_models -> pypi_attestations Signed-off-by: William Woodruff --- attestations.py | 2 +- requirements/runtime.in | 2 +- requirements/runtime.txt | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/attestations.py b/attestations.py index afe2ccbc..ff9ea9c1 100644 --- a/attestations.py +++ b/attestations.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import NoReturn -from pypi_attestation_models import Attestation +from pypi_attestations import Attestation from sigstore.oidc import IdentityError, IdentityToken, detect_credential from sigstore.sign import Signer, SigningContext diff --git a/requirements/runtime.in b/requirements/runtime.in index f8384d0d..71cb4589 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -10,5 +10,5 @@ id ~= 1.0 requests # NOTE: Used to generate attestations. -pypi-attestation-models ~= 0.0.5 +pypi-attestations ~= 0.0.6 sigstore ~= 3.0.0 diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 77e4ccb8..e3be7494 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -17,7 +17,7 @@ charset-normalizer==3.3.2 cryptography==42.0.7 # via # pyopenssl - # pypi-attestation-models + # pypi-attestations # sigstore dnspython==2.6.1 # via email-validator @@ -64,7 +64,7 @@ multidict==6.0.5 nh3==0.2.17 # via readme-renderer packaging==24.1 - # via pypi-attestation-models + # via pypi-attestations pkginfo==1.10.0 # via twine platformdirs==4.2.2 @@ -76,7 +76,7 @@ pycparser==2.22 pydantic==2.7.1 # via # id - # pypi-attestation-models + # pypi-attestations # sigstore # sigstore-rekor-types pydantic-core==2.18.2 @@ -89,7 +89,7 @@ pyjwt==2.8.0 # via sigstore pyopenssl==24.1.0 # via sigstore -pypi-attestation-models==0.0.5 +pypi-attestations==0.0.6 # via -r runtime.in python-dateutil==2.9.0.post0 # via betterproto @@ -118,10 +118,10 @@ securesystemslib==1.0.0 sigstore==3.0.0 # via # -r runtime.in - # pypi-attestation-models + # pypi-attestations sigstore-protobuf-specs==0.3.2 # via - # pypi-attestation-models + # pypi-attestations # sigstore sigstore-rekor-types==0.0.13 # via sigstore From 6a808bfb7d8a430602592b1b2a5de946e79b437f Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 9 Jul 2024 15:35:58 -0400 Subject: [PATCH 20/34] runtime: bump constraints Signed-off-by: William Woodruff --- requirements/runtime.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/runtime.txt b/requirements/runtime.txt index e3be7494..74cb608d 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -95,7 +95,7 @@ python-dateutil==2.9.0.post0 # via betterproto readme-renderer==43.0 # via twine -requests==2.32.0 +requests==2.32.3 # via # -r runtime.in # id @@ -129,7 +129,7 @@ six==1.16.0 # via python-dateutil tuf==5.0.0 # via sigstore -twine==5.1.0 +twine==5.1.1 # via -r runtime.in typing-extensions==4.11.0 # via From 1bb6510a8c10470b8b1c0d1e50ab9ea83332ea96 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 10 Jul 2024 15:32:43 -0400 Subject: [PATCH 21/34] requirements: bump pypi-attestations Signed-off-by: William Woodruff --- requirements/runtime.in | 2 +- requirements/runtime.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/runtime.in b/requirements/runtime.in index 71cb4589..3b15e253 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -10,5 +10,5 @@ id ~= 1.0 requests # NOTE: Used to generate attestations. -pypi-attestations ~= 0.0.6 +pypi-attestations ~= 0.0.8 sigstore ~= 3.0.0 diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 74cb608d..dbad6ed1 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -89,7 +89,7 @@ pyjwt==2.8.0 # via sigstore pyopenssl==24.1.0 # via sigstore -pypi-attestations==0.0.6 +pypi-attestations==0.0.8 # via -r runtime.in python-dateutil==2.9.0.post0 # via betterproto From 8c640e3d80e7138ed7fbae7edaf5c7b8cf2daf04 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 17 Jul 2024 10:04:06 -0400 Subject: [PATCH 22/34] bump to pypi-attestations==0.0.9 Signed-off-by: William Woodruff --- attestations.py | 5 +++-- requirements/runtime.in | 2 +- requirements/runtime.txt | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/attestations.py b/attestations.py index ff9ea9c1..8db34c87 100644 --- a/attestations.py +++ b/attestations.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import NoReturn -from pypi_attestations import Attestation +from pypi_attestations import Attestation, Distribution from sigstore.oidc import IdentityError, IdentityToken, detect_credential from sigstore.sign import Signer, SigningContext @@ -58,7 +58,8 @@ def attest_dist(dist_path: Path, signer: Signer) -> None: if attestation_path.is_file(): die(f'{dist_path} already has a publish attestation: {attestation_path}') - attestation = Attestation.sign(signer, dist_path) + dist = Distribution.from_file(dist_path) + attestation = Attestation.sign(signer, dist) attestation_path.write_text(attestation.model_dump_json(), encoding='utf-8') debug(f'saved publish attestation: {dist_path=} {attestation_path=}') diff --git a/requirements/runtime.in b/requirements/runtime.in index 3b15e253..c7e99419 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -10,5 +10,5 @@ id ~= 1.0 requests # NOTE: Used to generate attestations. -pypi-attestations ~= 0.0.8 +pypi-attestations ~= 0.0.9 sigstore ~= 3.0.0 diff --git a/requirements/runtime.txt b/requirements/runtime.txt index dbad6ed1..c84988d1 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -89,7 +89,7 @@ pyjwt==2.8.0 # via sigstore pyopenssl==24.1.0 # via sigstore -pypi-attestations==0.0.8 +pypi-attestations==0.0.9 # via -r runtime.in python-dateutil==2.9.0.post0 # via betterproto From e6556abeb8e40b646ac6c1e1787f4b955290e3f6 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 17 Jul 2024 10:08:11 -0400 Subject: [PATCH 23/34] attestations: use __main__ scope Signed-off-by: William Woodruff --- attestations.py | 53 +++++++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/attestations.py b/attestations.py index 8db34c87..2ad7c761 100644 --- a/attestations.py +++ b/attestations.py @@ -73,30 +73,31 @@ def get_identity_token() -> IdentityToken: return IdentityToken(oidc_token) -packages_dir = Path(sys.argv[1]) - -try: - identity = get_identity_token() -except IdentityError as identity_error: - # NOTE: We only perform attestations in trusted publishing flows, so we - # don't need to re-check for the "PR from fork" error mode, only - # generic token retrieval errors. We also render a simpler error, - # since permissions can't be to blame at this stage. - cause = _TOKEN_RETRIEVAL_FAILED_MESSAGE.format(identity_error=identity_error) - die(cause) - -# Collect all sdists and wheels. -dists = [sdist.absolute() for sdist in packages_dir.glob('*.tar.gz')] -dists.extend(whl.absolute() for whl in packages_dir.glob('*.whl')) - -# Make sure everything that looks like a dist actually is one. -# We do this up-front to prevent partial signing. -for dist in dists: - if not dist.is_file(): - die(f'Path looks like a distribution but is not a file: {dist}') - - -with SigningContext.production().signer(identity, cache=True) as s: - debug(f'attesting to dists: {dists}') +if __name__ == '__main__': + packages_dir = Path(sys.argv[1]) + + try: + identity = get_identity_token() + except IdentityError as identity_error: + # NOTE: We only perform attestations in trusted publishing flows, so we + # don't need to re-check for the "PR from fork" error mode, only + # generic token retrieval errors. We also render a simpler error, + # since permissions can't be to blame at this stage. + cause = _TOKEN_RETRIEVAL_FAILED_MESSAGE.format(identity_error=identity_error) + die(cause) + + # Collect all sdists and wheels. + dists = [sdist.absolute() for sdist in packages_dir.glob('*.tar.gz')] + dists.extend(whl.absolute() for whl in packages_dir.glob('*.whl')) + + # Make sure everything that looks like a dist actually is one. + # We do this up-front to prevent partial signing. for dist in dists: - attest_dist(dist, s) + if not dist.is_file(): + die(f'Path looks like a distribution but is not a file: {dist}') + + + with SigningContext.production().signer(identity, cache=True) as s: + debug(f'attesting to dists: {dists}') + for dist in dists: + attest_dist(dist, s) From 8094cdf3ed4e291e7ecc25db06a95690f9b5cdef Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 17 Jul 2024 10:12:25 -0400 Subject: [PATCH 24/34] attestations: add main Signed-off-by: William Woodruff --- attestations.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/attestations.py b/attestations.py index 2ad7c761..a2533c1a 100644 --- a/attestations.py +++ b/attestations.py @@ -73,7 +73,7 @@ def get_identity_token() -> IdentityToken: return IdentityToken(oidc_token) -if __name__ == '__main__': +def main() -> None: packages_dir = Path(sys.argv[1]) try: @@ -87,17 +87,19 @@ def get_identity_token() -> IdentityToken: die(cause) # Collect all sdists and wheels. - dists = [sdist.absolute() for sdist in packages_dir.glob('*.tar.gz')] - dists.extend(whl.absolute() for whl in packages_dir.glob('*.whl')) + dist_paths = [sdist.absolute() for sdist in packages_dir.glob('*.tar.gz')] + dist_paths.extend(whl.absolute() for whl in packages_dir.glob('*.whl')) # Make sure everything that looks like a dist actually is one. # We do this up-front to prevent partial signing. - for dist in dists: - if not dist.is_file(): - die(f'Path looks like a distribution but is not a file: {dist}') - + for dist_path in dist_paths: + if not dist_path.is_file(): + die(f'Path looks like a distribution but is not a file: {dist_path}') with SigningContext.production().signer(identity, cache=True) as s: - debug(f'attesting to dists: {dists}') - for dist in dists: - attest_dist(dist, s) + debug(f'attesting to dists: {dist_paths}') + for dist_path in dist_paths: + attest_dist(dist_path, s) + +if __name__ == '__main__': + main() From 57dba074b8815ff40871c8d2f6c97081db4dda5c Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 17 Jul 2024 10:22:40 -0400 Subject: [PATCH 25/34] attestations: please the linter Signed-off-by: William Woodruff --- attestations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/attestations.py b/attestations.py index a2533c1a..3879c460 100644 --- a/attestations.py +++ b/attestations.py @@ -83,8 +83,7 @@ def main() -> None: # don't need to re-check for the "PR from fork" error mode, only # generic token retrieval errors. We also render a simpler error, # since permissions can't be to blame at this stage. - cause = _TOKEN_RETRIEVAL_FAILED_MESSAGE.format(identity_error=identity_error) - die(cause) + die(_TOKEN_RETRIEVAL_FAILED_MESSAGE.format(identity_error=identity_error)) # Collect all sdists and wheels. dist_paths = [sdist.absolute() for sdist in packages_dir.glob('*.tar.gz')] @@ -101,5 +100,6 @@ def main() -> None: for dist_path in dist_paths: attest_dist(dist_path, s) + if __name__ == '__main__': main() From af78f7afd60217d688a63a0b04e7f26d6377ef70 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 22 Jul 2024 10:45:20 -0400 Subject: [PATCH 26/34] README: emphasize beta Signed-off-by: William Woodruff --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f9a7eaf7..c0998c50 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,8 @@ filter to the job: > [!IMPORTANT] > Support for generating and uploading [digital attestations] is currently > experimental and limited only to Trusted Publishing flows using PyPI or TestPyPI. +> Support for this feature is not yet stable; the settings and behavior described +> below may change without prior notice. > [!NOTE] > Generating and uploading digital attestations currently requires From bcc935fca533b2d63eb6f1cf80090a5be6f4f970 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 22 Jul 2024 10:48:49 -0400 Subject: [PATCH 27/34] twine-upload: emphasize attestations is a setting Signed-off-by: William Woodruff --- twine-upload.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/twine-upload.sh b/twine-upload.sh index 00dd7982..3c2a40b9 100755 --- a/twine-upload.sh +++ b/twine-upload.sh @@ -55,14 +55,14 @@ combinations or API tokens to authenticate with PyPI. Read more: \ https://docs.pypi.org/trusted-publishers" ATTESTATIONS_WITHOUT_TP_WARNING="::warning title=attestations setting ignored::\ -The workflow was run with 'attestations: true', but an explicit password was \ -also supplied, disabling Trusted Publishing. As a result, the attestations \ -setting is ignored." +The workflow was run with the 'attestations: true' setting, but an explicit \ +password was also set, disabling Trusted Publishing. As a result, the \ +attestations setting is ignored." ATTESTATIONS_WRONG_INDEX_WARNING="::warning title=attestations setting ignored::\ -The workflow was run with 'attestations: true', but the specified repository URL \ -does not support PEP 740 attestations. As a result, the attestations setting \ -is ignored." +The workflow was run with 'attestations: true' setting, but the specified \ +repository URL does not support PEP 740 attestations. As a result, the \ +attestations setting is ignored." [[ "${INPUT_USER}" == "__token__" && -z "${INPUT_PASSWORD}" ]] \ && TRUSTED_PUBLISHING=true || TRUSTED_PUBLISHING=false From 66f02b6d7a98087105412c85f3c107239717aa00 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 22 Jul 2024 10:49:19 -0400 Subject: [PATCH 28/34] twine-upload: setting -> input Signed-off-by: William Woodruff --- twine-upload.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/twine-upload.sh b/twine-upload.sh index 3c2a40b9..31092c23 100755 --- a/twine-upload.sh +++ b/twine-upload.sh @@ -54,15 +54,15 @@ environments like GitHub Actions without needing to use username/password \ combinations or API tokens to authenticate with PyPI. Read more: \ https://docs.pypi.org/trusted-publishers" -ATTESTATIONS_WITHOUT_TP_WARNING="::warning title=attestations setting ignored::\ -The workflow was run with the 'attestations: true' setting, but an explicit \ +ATTESTATIONS_WITHOUT_TP_WARNING="::warning title=attestations input ignored::\ +The workflow was run with the 'attestations: true' input, but an explicit \ password was also set, disabling Trusted Publishing. As a result, the \ -attestations setting is ignored." +attestations input is ignored." -ATTESTATIONS_WRONG_INDEX_WARNING="::warning title=attestations setting ignored::\ -The workflow was run with 'attestations: true' setting, but the specified \ +ATTESTATIONS_WRONG_INDEX_WARNING="::warning title=attestations input ignored::\ +The workflow was run with 'attestations: true' input, but the specified \ repository URL does not support PEP 740 attestations. As a result, the \ -attestations setting is ignored." +attestations input is ignored." [[ "${INPUT_USER}" == "__token__" && -z "${INPUT_PASSWORD}" ]] \ && TRUSTED_PUBLISHING=true || TRUSTED_PUBLISHING=false From 28806ba79a0280fe523d300cf19122537ae185a9 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 31 Jul 2024 18:17:30 -0400 Subject: [PATCH 29/34] requirements: bump pypi-attestations, sigstore Signed-off-by: William Woodruff --- requirements/runtime.in | 4 ++-- requirements/runtime.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/runtime.in b/requirements/runtime.in index c7e99419..dd6ab073 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -10,5 +10,5 @@ id ~= 1.0 requests # NOTE: Used to generate attestations. -pypi-attestations ~= 0.0.9 -sigstore ~= 3.0.0 +pypi-attestations ~= 0.0.10 +sigstore ~= 3.1.0 diff --git a/requirements/runtime.txt b/requirements/runtime.txt index c84988d1..28b890c6 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -89,7 +89,7 @@ pyjwt==2.8.0 # via sigstore pyopenssl==24.1.0 # via sigstore -pypi-attestations==0.0.9 +pypi-attestations==0.0.10 # via -r runtime.in python-dateutil==2.9.0.post0 # via betterproto @@ -115,7 +115,7 @@ rich==13.7.1 # twine securesystemslib==1.0.0 # via tuf -sigstore==3.0.0 +sigstore==3.1.0 # via # -r runtime.in # pypi-attestations From fed8784281015dc00804a96c2a53d1e0c27b9db3 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 20 Aug 2024 13:37:06 -0400 Subject: [PATCH 30/34] requirements: bump sigstore, pypi-attestations Signed-off-by: William Woodruff --- requirements/runtime.in | 4 ++-- requirements/runtime.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/runtime.in b/requirements/runtime.in index dd6ab073..30519641 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -10,5 +10,5 @@ id ~= 1.0 requests # NOTE: Used to generate attestations. -pypi-attestations ~= 0.0.10 -sigstore ~= 3.1.0 +pypi-attestations ~= 0.0.11 +sigstore ~= 3.2.0 diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 28b890c6..d50cd5bd 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -89,7 +89,7 @@ pyjwt==2.8.0 # via sigstore pyopenssl==24.1.0 # via sigstore -pypi-attestations==0.0.10 +pypi-attestations==0.0.11 # via -r runtime.in python-dateutil==2.9.0.post0 # via betterproto @@ -115,7 +115,7 @@ rich==13.7.1 # twine securesystemslib==1.0.0 # via tuf -sigstore==3.1.0 +sigstore==3.2.0 # via # -r runtime.in # pypi-attestations From 61ffce1eb0421238d20f6e9bf2e4baeb3aea78d0 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 21 Aug 2024 11:55:20 -0400 Subject: [PATCH 31/34] Update attestations.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) --- attestations.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/attestations.py b/attestations.py index 3879c460..541becae 100644 --- a/attestations.py +++ b/attestations.py @@ -91,9 +91,12 @@ def main() -> None: # Make sure everything that looks like a dist actually is one. # We do this up-front to prevent partial signing. - for dist_path in dist_paths: - if not dist_path.is_file(): - die(f'Path looks like a distribution but is not a file: {dist_path}') + if (invalid_dists := [_path for _path in dist_paths if _path.is_file()]): + invalid_dist_list = ', '.join(map(str, invalid_dists)) + die( + 'The following paths look like distributions but ' + f'are not actually files: {invalid_dist_list}', + ) with SigningContext.production().signer(identity, cache=True) as s: debug(f'attesting to dists: {dist_paths}') From e1b63c320ce10500bc790fc37be60df595309ade Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 21 Aug 2024 11:55:40 -0400 Subject: [PATCH 32/34] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) --- requirements/runtime.in | 2 +- twine-upload.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/runtime.in b/requirements/runtime.in index 30519641..50f52b63 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -1,7 +1,7 @@ twine # NOTE: Used to detect an ambient OIDC credential for OIDC publishing, -# as well as PEP 740 attestations. +# NOTE: as well as PEP 740 attestations. id ~= 1.0 # NOTE: This is pulled in transitively through `twine`, but we also declare diff --git a/twine-upload.sh b/twine-upload.sh index 31092c23..12b57b28 100755 --- a/twine-upload.sh +++ b/twine-upload.sh @@ -163,7 +163,7 @@ fi if [[ ${INPUT_ATTESTATIONS,,} != "false" ]] ; then # NOTE: Intentionally placed after `twine check`, to prevent attestation - # generation on distributions with invalid metadata. + # NOTE: generation on distributions with invalid metadata. echo "::notice::Generating and uploading digital attestations" python /app/attestations.py "${INPUT_PACKAGES_DIR%%/}" From 15d937713f27d7ef2b4c68dcefeccfee8811a7fe Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 21 Aug 2024 12:01:07 -0400 Subject: [PATCH 33/34] attestations: use Path.resolve(), break out dist collection ...to make the linters happy. Signed-off-by: William Woodruff --- attestations.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/attestations.py b/attestations.py index 541becae..ac34daba 100644 --- a/attestations.py +++ b/attestations.py @@ -51,6 +51,23 @@ def debug(msg: str): print(f'::debug::{msg}', file=sys.stderr) +def collect_dists(packages_dir: Path) -> list[Path]: + # Collect all sdists and wheels. + dist_paths = [sdist.resolve() for sdist in packages_dir.glob('*.tar.gz')] + dist_paths.extend(whl.resolve() for whl in packages_dir.glob('*.whl')) + + # Make sure everything that looks like a dist actually is one. + # We do this up-front to prevent partial signing. + if (invalid_dists := [path for path in dist_paths if path.is_file()]): + invalid_dist_list = ', '.join(map(str, invalid_dists)) + die( + 'The following paths look like distributions but ' + f'are not actually files: {invalid_dist_list}', + ) + + return dist_paths + + def attest_dist(dist_path: Path, signer: Signer) -> None: # We are the publishing step, so there should be no pre-existing publish # attestation. The presence of one indicates user confusion. @@ -85,18 +102,7 @@ def main() -> None: # since permissions can't be to blame at this stage. die(_TOKEN_RETRIEVAL_FAILED_MESSAGE.format(identity_error=identity_error)) - # Collect all sdists and wheels. - dist_paths = [sdist.absolute() for sdist in packages_dir.glob('*.tar.gz')] - dist_paths.extend(whl.absolute() for whl in packages_dir.glob('*.whl')) - - # Make sure everything that looks like a dist actually is one. - # We do this up-front to prevent partial signing. - if (invalid_dists := [_path for _path in dist_paths if _path.is_file()]): - invalid_dist_list = ', '.join(map(str, invalid_dists)) - die( - 'The following paths look like distributions but ' - f'are not actually files: {invalid_dist_list}', - ) + dist_paths = collect_dists(packages_dir) with SigningContext.production().signer(identity, cache=True) as s: debug(f'attesting to dists: {dist_paths}') From 473ca48544171028af3c8cd19e6d2a946b5a3c6b Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 21 Aug 2024 12:02:06 -0400 Subject: [PATCH 34/34] attestations: use exists() instead of is_file() Signed-off-by: William Woodruff --- attestations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attestations.py b/attestations.py index ac34daba..6863c838 100644 --- a/attestations.py +++ b/attestations.py @@ -72,7 +72,7 @@ def attest_dist(dist_path: Path, signer: Signer) -> None: # We are the publishing step, so there should be no pre-existing publish # attestation. The presence of one indicates user confusion. attestation_path = Path(f'{dist_path}.publish.attestation') - if attestation_path.is_file(): + if attestation_path.exists(): die(f'{dist_path} already has a publish attestation: {attestation_path}') dist = Distribution.from_file(dist_path)