From 15185adb4c053cba451ff8a4686d9933c25dfcae Mon Sep 17 00:00:00 2001 From: adisbladis Date: Thu, 26 Oct 2023 16:39:13 +1300 Subject: [PATCH 1/3] vendor: bump pyproject.nix --- default.nix | 10 +- editable.nix | 2 +- mk-poetry-dep.nix | 6 +- pep425.nix | 2 +- vendor/pyproject.nix/default.nix | 27 +--- vendor/pyproject.nix/fetchers/default.nix | 122 ++++++++++++++++ .../fetchers/fetch-from-legacy.py | 137 ++++++++++++++++++ .../pyproject.nix/fetchers/fetch-from-pypi.sh | 27 ++++ vendor/pyproject.nix/lib/default.nix | 22 +++ vendor/pyproject.nix/{ => lib}/pep427.nix | 0 vendor/pyproject.nix/{ => lib}/pep440.nix | 3 +- vendor/pyproject.nix/{ => lib}/pep508.nix | 5 +- vendor/pyproject.nix/{ => lib}/pep518.nix | 0 vendor/pyproject.nix/{ => lib}/pep599.nix | 0 vendor/pyproject.nix/{ => lib}/pep600.nix | 0 vendor/pyproject.nix/{ => lib}/pep621.nix | 0 vendor/pyproject.nix/{ => lib}/pip.nix | 9 +- vendor/pyproject.nix/{ => lib}/poetry.nix | 111 +++++++------- vendor/pyproject.nix/{ => lib}/project.nix | 0 vendor/pyproject.nix/{ => lib}/pypa.nix | 0 vendor/pyproject.nix/{ => lib}/renderers.nix | 0 vendor/pyproject.nix/{ => lib}/util.nix | 10 +- vendor/pyproject.nix/{ => lib}/validators.nix | 0 23 files changed, 396 insertions(+), 97 deletions(-) create mode 100644 vendor/pyproject.nix/fetchers/default.nix create mode 100644 vendor/pyproject.nix/fetchers/fetch-from-legacy.py create mode 100644 vendor/pyproject.nix/fetchers/fetch-from-pypi.sh create mode 100644 vendor/pyproject.nix/lib/default.nix rename vendor/pyproject.nix/{ => lib}/pep427.nix (100%) rename vendor/pyproject.nix/{ => lib}/pep440.nix (99%) rename vendor/pyproject.nix/{ => lib}/pep508.nix (98%) rename vendor/pyproject.nix/{ => lib}/pep518.nix (100%) rename vendor/pyproject.nix/{ => lib}/pep599.nix (100%) rename vendor/pyproject.nix/{ => lib}/pep600.nix (100%) rename vendor/pyproject.nix/{ => lib}/pep621.nix (100%) rename vendor/pyproject.nix/{ => lib}/pip.nix (95%) rename vendor/pyproject.nix/{ => lib}/poetry.nix (89%) rename vendor/pyproject.nix/{ => lib}/project.nix (100%) rename vendor/pyproject.nix/{ => lib}/pypa.nix (100%) rename vendor/pyproject.nix/{ => lib}/renderers.nix (100%) rename vendor/pyproject.nix/{ => lib}/util.nix (59%) rename vendor/pyproject.nix/{ => lib}/validators.nix (100%) diff --git a/default.nix b/default.nix index ac38de969..4c17420f3 100644 --- a/default.nix +++ b/default.nix @@ -5,10 +5,10 @@ let inherit (poetryLib) isCompatible readTOML; - pyproject-nix = import ./vendor/pyproject.nix { inherit lib; }; + pyproject-nix = import ./vendor/pyproject.nix { inherit pkgs lib; }; # Name normalization - inherit (pyproject-nix.pypa) normalizePackageName; + inherit (pyproject-nix.lib.pypa) normalizePackageName; normalizePackageSet = lib.attrsets.mapAttrs' (name: value: lib.attrsets.nameValuePair (normalizePackageName name) value); # Map SPDX identifiers to license names @@ -172,7 +172,7 @@ lib.makeScope pkgs.newScope (self: { in lib.listToAttrs (lib.mapAttrsToList (n: v: { name = normalizePackageName n; value = v; }) lockfiles); - pep508Env = pyproject-nix.pep508.mkEnviron python; + pep508Env = pyproject-nix.lib.pep508.mkEnviron python; # Filter packages by their PEP508 markers & pyproject interpreter version partitions = @@ -181,9 +181,9 @@ lib.makeScope pkgs.newScope (self: { if pkgMeta ? marker then ( let - marker = pyproject-nix.pep508.parseMarkers pkgMeta.marker; + marker = pyproject-nix.lib.pep508.parseMarkers pkgMeta.marker; in - pyproject-nix.pep508.evalMarkers pep508Env marker + pyproject-nix.lib.pep508.evalMarkers pep508Env marker ) else true && isCompatible (poetryLib.getPythonVersion python) pkgMeta.python-versions; in lib.partition supportsPythonVersion poetryLock.package; diff --git a/editable.nix b/editable.nix index 5897212f9..a60aac020 100644 --- a/editable.nix +++ b/editable.nix @@ -6,7 +6,7 @@ , pyproject-nix }: let - name = pyproject-nix.pypa.normalizePackageName pyProject.tool.poetry.name; + name = pyproject-nix.lib.pypa.normalizePackageName pyProject.tool.poetry.name; # Just enough standard PKG-INFO fields for an editable installation pkgInfoFields = { diff --git a/mk-poetry-dep.nix b/mk-poetry-dep.nix index feab59fb8..d770144a7 100644 --- a/mk-poetry-dep.nix +++ b/mk-poetry-dep.nix @@ -29,7 +29,7 @@ pythonPackages.callPackage }@args: let inherit (python) stdenv; - inherit (pyproject-nix.pypa) normalizePackageName; + inherit (pyproject-nix.lib.pypa) normalizePackageName; inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromLegacy fetchFromPypi; inherit (import ./pep425.nix { @@ -145,7 +145,7 @@ pythonPackages.callPackage pep508Markers = v.markers or ""; in compat constraints && (if pep508Markers == "" then true else - (pyproject-nix.pep508.evalMarkers + (pyproject-nix.lib.pep508.evalMarkers (pep508Env // { extra = { # All extras are always enabled @@ -153,7 +153,7 @@ pythonPackages.callPackage value = lib.attrNames extras; }; }) - (pyproject-nix.pep508.parseMarkers pep508Markers))) + (pyproject-nix.lib.pep508.parseMarkers pep508Markers))) ) dependencies ); diff --git a/pep425.nix b/pep425.nix index f5935e140..27cb07cea 100644 --- a/pep425.nix +++ b/pep425.nix @@ -1,7 +1,7 @@ { lib, stdenv, python, pyproject-nix, isLinux ? stdenv.isLinux }: let inherit (lib.strings) escapeRegex hasPrefix hasSuffix hasInfix splitString removeSuffix; - targetMachine = pyproject-nix.pep599.manyLinuxTargetMachines.${stdenv.targetPlatform.parsed.cpu.name}; + targetMachine = pyproject-nix.lib.pep599.manyLinuxTargetMachines.${stdenv.targetPlatform.parsed.cpu.name}; pythonVer = let diff --git a/vendor/pyproject.nix/default.nix b/vendor/pyproject.nix/default.nix index bcc49993e..99f6f25e4 100644 --- a/vendor/pyproject.nix/default.nix +++ b/vendor/pyproject.nix/default.nix @@ -1,22 +1,5 @@ -{ lib }: -let - inherit (builtins) mapAttrs; - inherit (lib) fix; -in - -fix (self: mapAttrs (_: path: import path ({ inherit lib; } // self)) { - pip = ./pip.nix; - pypa = ./pypa.nix; - project = ./project.nix; - renderers = ./renderers.nix; - validators = ./validators.nix; - poetry = ./poetry.nix; - - pep427 = ./pep427.nix; - pep440 = ./pep440.nix; - pep508 = ./pep508.nix; - pep518 = ./pep518.nix; - pep599 = ./pep599.nix; - pep600 = ./pep600.nix; - pep621 = ./pep621.nix; -}) +{ pkgs, lib }: +{ + lib = import ./lib { inherit lib; }; + fetchers = import ./fetchers { inherit pkgs lib; }; +} diff --git a/vendor/pyproject.nix/fetchers/default.nix b/vendor/pyproject.nix/fetchers/default.nix new file mode 100644 index 000000000..1665a2e7c --- /dev/null +++ b/vendor/pyproject.nix/fetchers/default.nix @@ -0,0 +1,122 @@ +{ pkgs +, lib +, +}: +let + inherit (builtins) substring filter head nixPath; + inherit (lib) toLower; + + # Predict URL from the PyPI index. + # Args: + # pname: package name + # file: filename including extension + # hash: SRI hash + # kind: Language implementation and version tag + predictURLFromPypi = + { + # package name + pname + , # filename including extension + file + , # Language implementation and version tag + kind + , + }: "https://files.pythonhosted.org/packages/${kind}/${toLower (substring 0 1 file)}/${pname}/${file}"; +in +lib.mapAttrs (_: func: lib.makeOverridable func) { + /* + Fetch from the PyPI index. + + At first we try to fetch the predicated URL but if that fails we + will use the Pypi API to determine the correct URL. + + Type: fetchFromPypi :: AttrSet -> derivation + */ + fetchFromPypi = + { + # package name + pname + , # filename including extension + file + , # the version string of the dependency + version + , # SRI hash + hash + , # Language implementation and version tag + kind + , # Options to pass to `curl` + curlOpts ? "" + , + }: + let + predictedURL = predictURLFromPypi { inherit pname file kind; }; + in + pkgs.stdenvNoCC.mkDerivation { + name = file; + nativeBuildInputs = [ + pkgs.buildPackages.curl + pkgs.buildPackages.jq + ]; + isWheel = lib.strings.hasSuffix "whl" file; + system = "builtin"; + + preferLocalBuild = true; + impureEnvVars = + lib.fetchers.proxyImpureEnvVars + ++ [ + "NIX_CURL_FLAGS" + ]; + + inherit pname file version curlOpts predictedURL; + + builder = ./fetch-from-pypi.sh; + + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = hash; + + passthru = { + urls = [ predictedURL ]; # retain compatibility with nixpkgs' fetchurl + }; + }; + + /* + Fetch from the PyPI legacy API. + + Some repositories (such as Devpi) expose the Pypi legacy API (https://warehouse.pypa.io/api-reference/legacy.html). + + Type: fetchFromLegacy :: AttrSet -> derivation + */ + fetchFromLegacy = + { + # package name + pname + , # URL to package index + url + , # filename including extension + file + , # SRI hash + hash + , + }: + let + pathParts = filter ({ prefix, path }: "NETRC" == prefix) nixPath; # deadnix: skip + netrc_file = + if (pathParts != [ ]) + then (head pathParts).path + else ""; + in + pkgs.runCommand file + { + nativeBuildInputs = [ pkgs.buildPackages.python3 ]; + impureEnvVars = lib.fetchers.proxyImpureEnvVars; + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = hash; + NETRC = netrc_file; + passthru.isWheel = lib.strings.hasSuffix "whl" file; + } '' + python ${./fetch-from-legacy.py} ${url} ${pname} ${file} + mv ${file} $out + ''; +} diff --git a/vendor/pyproject.nix/fetchers/fetch-from-legacy.py b/vendor/pyproject.nix/fetchers/fetch-from-legacy.py new file mode 100644 index 000000000..790458bb1 --- /dev/null +++ b/vendor/pyproject.nix/fetchers/fetch-from-legacy.py @@ -0,0 +1,137 @@ +# Some repositories (such as Devpi) expose the Pypi legacy API +# (https://warehouse.pypa.io/api-reference/legacy.html). +# +# Note it is not possible to use pip +# https://discuss.python.org/t/pip-download-just-the-source-packages-no-building-no-metadata-etc/4651/12 + +import netrc +import os +import shutil +import ssl +import sys +import urllib.request +from html.parser import HTMLParser +from os.path import normpath +from typing import Optional +from urllib.parse import urlparse, urlunparse + + +# Parse the legacy index page to extract the href and package names +class Pep503(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.sources: dict[str, str] = {} + self.url: Optional[str] = None + self.name: Optional[str] = None + + def handle_data(self, data: str) -> None: + if self.url is not None: + self.name = data + + def handle_starttag(self, tag: str, attrs: list[tuple[str, Optional[str]]]) -> None: + if tag == "a": + for name, value in attrs: + if name == "href": + self.url = value + + def handle_endtag(self, tag: str) -> None: + if self.url is not None: + if not self.name: + raise ValueError("Name not set") + + self.sources[self.name] = self.url + self.url = None + + +url = sys.argv[1] +package_name = sys.argv[2] +index_url = url + "/" + package_name + "/" +package_filename = sys.argv[3] + +# Parse username and password for this host from the netrc file if given. +username: Optional[str] = None +password: Optional[str] = None +if os.environ["NETRC"]: + netrc_obj = netrc.netrc(os.environ["NETRC"]) + host = urlparse(index_url).netloc + # Strip port number if present + if ":" in host: + host = host.split(":")[0] + authenticators = netrc_obj.authenticators(host) + if authenticators: + username, _, password = authenticators + +print("Reading index %s" % index_url) + +context = ssl.create_default_context() +context.check_hostname = False +context.verify_mode = ssl.CERT_NONE + +# Extract out username/password from index_url, if present. +parsed_url = urlparse(index_url) +username = parsed_url.username or username +password = parsed_url.password or password +index_url = parsed_url._replace(netloc=parsed_url.netloc.rpartition("@")[-1]).geturl() + +req = urllib.request.Request(index_url) +if username and password: + import base64 + + password_b64 = base64.b64encode(":".join((username, password)).encode()).decode("utf-8") + req.add_header("Authorization", "Basic {}".format(password_b64)) +response = urllib.request.urlopen(req, context=context) +index = response.read() + +parser = Pep503() +parser.feed(str(index, "utf-8")) +if package_filename not in parser.sources: + print("The file %s has not be found in the index %s" % (package_filename, index_url)) + exit(1) + +package_file = open(package_filename, "wb") +# Sometimes the href is a relative or absolute path within the index's domain. +indicated_url = urlparse(parser.sources[package_filename]) +if indicated_url.netloc == "": + parsed_url = urlparse(index_url) + + if indicated_url.path.startswith("/"): + # An absolute path within the index's domain. + path = parser.sources[package_filename] + else: + # A relative path. + path = parsed_url.path + "/" + parser.sources[package_filename] + + package_url = urlunparse( + ( + parsed_url.scheme, + parsed_url.netloc, + path, + None, + None, + None, + ) + ) +else: + package_url = parser.sources[package_filename] + +# Handle urls containing "../" +parsed_url = urlparse(package_url) +real_package_url = urlunparse( + ( + parsed_url.scheme, + parsed_url.netloc, + normpath(parsed_url.path), + parsed_url.params, + parsed_url.query, + parsed_url.fragment, + ) +) +print("Downloading %s" % real_package_url) + +req = urllib.request.Request(real_package_url) +if username and password: + req.add_unredirected_header("Authorization", "Basic {}".format(password_b64)) +response = urllib.request.urlopen(req, context=context) + +with response as r: + shutil.copyfileobj(r, package_file) diff --git a/vendor/pyproject.nix/fetchers/fetch-from-pypi.sh b/vendor/pyproject.nix/fetchers/fetch-from-pypi.sh new file mode 100644 index 000000000..e4137c6cd --- /dev/null +++ b/vendor/pyproject.nix/fetchers/fetch-from-pypi.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC1091,SC2154 +source "$stdenv/setup" +set -euo pipefail + +curl="curl \ + --location \ + --max-redirs 20 \ + --retry 2 \ + --disable-epsv \ + --cookie-jar cookies \ + --insecure \ + --speed-time 5 \ + --progress-bar \ + --fail \ + $curlOpts \ + $NIX_CURL_FLAGS" + +echo "Trying to fetch with predicted URL: $predictedURL" + +$curl "$predictedURL" --output "$out" && exit 0 + +echo "Predicted URL '$predictedURL' failed, querying pypi.org" +$curl "https://pypi.org/pypi/$pname/json" | jq -r ".releases.\"$version\"[] | select(.filename == \"$file\") | .url" > url +url=$(cat url) +$curl "$url" --output "$out" diff --git a/vendor/pyproject.nix/lib/default.nix b/vendor/pyproject.nix/lib/default.nix new file mode 100644 index 000000000..bcc49993e --- /dev/null +++ b/vendor/pyproject.nix/lib/default.nix @@ -0,0 +1,22 @@ +{ lib }: +let + inherit (builtins) mapAttrs; + inherit (lib) fix; +in + +fix (self: mapAttrs (_: path: import path ({ inherit lib; } // self)) { + pip = ./pip.nix; + pypa = ./pypa.nix; + project = ./project.nix; + renderers = ./renderers.nix; + validators = ./validators.nix; + poetry = ./poetry.nix; + + pep427 = ./pep427.nix; + pep440 = ./pep440.nix; + pep508 = ./pep508.nix; + pep518 = ./pep518.nix; + pep599 = ./pep599.nix; + pep600 = ./pep600.nix; + pep621 = ./pep621.nix; +}) diff --git a/vendor/pyproject.nix/pep427.nix b/vendor/pyproject.nix/lib/pep427.nix similarity index 100% rename from vendor/pyproject.nix/pep427.nix rename to vendor/pyproject.nix/lib/pep427.nix diff --git a/vendor/pyproject.nix/pep440.nix b/vendor/pyproject.nix/lib/pep440.nix similarity index 99% rename from vendor/pyproject.nix/pep440.nix rename to vendor/pyproject.nix/lib/pep440.nix index 0b5f83428..3057ce43d 100644 --- a/vendor/pyproject.nix/pep440.nix +++ b/vendor/pyproject.nix/lib/pep440.nix @@ -151,7 +151,7 @@ fix (self: { */ parseVersionCond = cond: ( let - m = match " *([=>" = a: b: self.compareVersions a b > 0; "===" = throw "Arbitrary equality clause not supported"; + "" = _a: _b: true; }; }) diff --git a/vendor/pyproject.nix/pep508.nix b/vendor/pyproject.nix/lib/pep508.nix similarity index 98% rename from vendor/pyproject.nix/pep508.nix rename to vendor/pyproject.nix/lib/pep508.nix index 04b044771..40f75a18b 100644 --- a/vendor/pyproject.nix/pep508.nix +++ b/vendor/pyproject.nix/lib/pep508.nix @@ -3,7 +3,7 @@ let inherit (builtins) match elemAt split foldl' substring stringLength typeOf fromJSON isString head mapAttrs elem length; inherit (lib) stringToCharacters fix; - inherit (import ./util.nix { inherit lib; }) splitComma; + inherit (import ./util.nix { inherit lib; }) splitComma stripStr; re = { operators = "([=>= && < - else if c == "~" then [ - { - cond = ">="; - inherit version; - } - { - cond = "<"; - version = version // { - release = [ (head version.release + 1) ] ++ tail version.release; - }; - } - ] - # Desugar ^ into >= && < - else if c == "^" then [ - { - cond = ">="; - inherit version; - } - { - cond = "<"; - version = version // { - release = rewriteCaretRhs version.release; - }; - } - ] - # Versions without operators are exact matches, add operator according to PEP-440 - else [{ - cond = "=="; - inherit version; - }] - ); - # Normalized version of parseVersionCond' - parseVersionConds = s: flatten (map parseVersionCond' (splitComma s)); + parseVersionConds = s: flatten (map self.parseVersionCond (splitComma s)); dummyMarker = { type = "bool"; @@ -240,10 +195,64 @@ in build-systems = [ ]; # PEP-518 build-systems (List of parsed PEP-508 strings) } */ - # # Analogous to parseDependencies = pyproject: { dependencies = map parseDependency (normalizeDependendenciesToList (pyproject.tool.poetry.dependencies or { })); extras = mapAttrs (_: g: map parseDependency (normalizeDependendenciesToList g.dependencies)) pyproject.tool.poetry.group or { }; build-systems = pep518.parseBuildSystems pyproject; }; -} + + /* Parse a version conditional. + Supports additional non-standard operators `^` and `~` used by Poetry. + + Because some expressions desugar to multiple expressions parseVersionCond returns a list. + + Type: parseVersionCond :: string -> [ AttrSet ] + */ + parseVersionCond = cond: ( + let + m = match "^([~[:digit:]^])(.+)$" cond; + mAt = elemAt m; + c = mAt 0; + rest = mAt 1; + # Pad version before parsing as it's _much_ easier to reason about + # once they're the same length + version = pep440.parseVersion (lib.versions.pad 3 rest); + + # Count the number of segments in the input to use an an index in ~ rewriting + segments = length (filter (tok: typeOf tok == "string") (split "\\." rest)); + in + if m == null then [ (pep440.parseVersionCond cond) ] + # Desugar ~ into >= && < + else if c == "~" then [ + { + op = ">="; + inherit version; + } + { + op = "<"; + version = version // { + release = lib.imap0 (i: tok: if i >= segments - 1 then 0 else if i == segments - 2 then (tok + 1) else tok) version.release; + }; + } + ] + # Desugar ^ into >= && < + else if c == "^" then [ + { + op = ">="; + inherit version; + } + { + op = "<"; + version = version // { + release = rewriteCaretRhs version.release; + }; + } + ] + # Versions without operators are exact matches, add operator according to PEP-440 + else [{ + op = "=="; + inherit version; + }] + ); + +}) diff --git a/vendor/pyproject.nix/project.nix b/vendor/pyproject.nix/lib/project.nix similarity index 100% rename from vendor/pyproject.nix/project.nix rename to vendor/pyproject.nix/lib/project.nix diff --git a/vendor/pyproject.nix/pypa.nix b/vendor/pyproject.nix/lib/pypa.nix similarity index 100% rename from vendor/pyproject.nix/pypa.nix rename to vendor/pyproject.nix/lib/pypa.nix diff --git a/vendor/pyproject.nix/renderers.nix b/vendor/pyproject.nix/lib/renderers.nix similarity index 100% rename from vendor/pyproject.nix/renderers.nix rename to vendor/pyproject.nix/lib/renderers.nix diff --git a/vendor/pyproject.nix/util.nix b/vendor/pyproject.nix/lib/util.nix similarity index 59% rename from vendor/pyproject.nix/util.nix rename to vendor/pyproject.nix/lib/util.nix index 143d92338..2160d2f0a 100644 --- a/vendor/pyproject.nix/util.nix +++ b/vendor/pyproject.nix/lib/util.nix @@ -1,11 +1,19 @@ # Small utilities for internal reuse, not exposed externally { lib }: let - inherit (builtins) filter match split; + inherit (builtins) filter match split head; inherit (lib) isString; isEmptyStr = s: isString s && match " *" s == null; in { splitComma = s: if s == "" then [ ] else filter isEmptyStr (split " *, *" s); + + stripStr = s: + let + t = match "[\t ]*(.*[^\t ])[\t ]*" s; + in + if t == null + then "" + else head t; } diff --git a/vendor/pyproject.nix/validators.nix b/vendor/pyproject.nix/lib/validators.nix similarity index 100% rename from vendor/pyproject.nix/validators.nix rename to vendor/pyproject.nix/lib/validators.nix From beacdee91b7da690f1dfe1b7618b7860a8e9e64f Mon Sep 17 00:00:00 2001 From: adisbladis Date: Thu, 26 Oct 2023 16:39:29 +1300 Subject: [PATCH 2/3] vendor: Copy lib/fetchers directories separately --- vendor/update.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/vendor/update.py b/vendor/update.py index 5e3591174..c498351f9 100755 --- a/vendor/update.py +++ b/vendor/update.py @@ -1,4 +1,5 @@ -#!/usr/bin/env python3 +#!/usr/bin/env nix-shell +#! nix-shell -i python3 -p python3 import subprocess import shutil import json @@ -24,8 +25,14 @@ pass os.mkdir("pyproject.nix") + os.mkdir("pyproject.nix/lib") + shutil.copy(f"{store_path}/default.nix", f"pyproject.nix/default.nix") + + # Copy lib/ for filename in os.listdir(f"{store_path}/lib"): if filename.startswith("test") or not filename.endswith(".nix"): continue - shutil.copy(f"{store_path}/lib/{filename}", f"pyproject.nix/{filename}") + shutil.copy(f"{store_path}/lib/{filename}", f"pyproject.nix/lib/{filename}") + + shutil.copytree(f"{store_path}/fetchers", "pyproject.nix/fetchers") From 841aa10e1b56d69a524d4399718af0e8eecdaab9 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Thu, 26 Oct 2023 16:47:17 +1300 Subject: [PATCH 3/3] Use fetchers from pyproject.nix --- fetch-from-pypi.sh | 24 ---- fetch_from_legacy.py | 134 ------------------ lib.nix | 80 ----------- mk-poetry-dep.nix | 5 +- .../fetchers/fetch-from-legacy.py | 8 +- 5 files changed, 8 insertions(+), 243 deletions(-) delete mode 100644 fetch-from-pypi.sh delete mode 100644 fetch_from_legacy.py diff --git a/fetch-from-pypi.sh b/fetch-from-pypi.sh deleted file mode 100644 index e56dee684..000000000 --- a/fetch-from-pypi.sh +++ /dev/null @@ -1,24 +0,0 @@ -source $stdenv/setup -set -euo pipefail - -curl="curl \ - --location \ - --max-redirs 20 \ - --retry 2 \ - --disable-epsv \ - --cookie-jar cookies \ - --insecure \ - --speed-time 5 \ - --progress-bar \ - --fail \ - $curlOpts \ - $NIX_CURL_FLAGS" - -echo "Trying to fetch with predicted URL: $predictedURL" - -$curl $predictedURL --output $out && exit 0 - -echo "Predicted URL '$predictedURL' failed, querying pypi.org" -$curl "https://pypi.org/pypi/$pname/json" | jq -r ".releases.\"$version\"[] | select(.filename == \"$file\") | .url" > url -url=$(cat url) -$curl -k $url --output $out diff --git a/fetch_from_legacy.py b/fetch_from_legacy.py deleted file mode 100644 index 44ea092ac..000000000 --- a/fetch_from_legacy.py +++ /dev/null @@ -1,134 +0,0 @@ -# Some repositories (such as Devpi) expose the Pypi legacy API -# (https://warehouse.pypa.io/api-reference/legacy.html). -# -# Note it is not possible to use pip -# https://discuss.python.org/t/pip-download-just-the-source-packages-no-building-no-metadata-etc/4651/12 - -import os -import sys -import netrc -from urllib.parse import urlparse, urlunparse -from html.parser import HTMLParser -import urllib.request -import shutil -import ssl -from os.path import normpath - - -# Parse the legacy index page to extract the href and package names -class Pep503(HTMLParser): - def __init__(self): - super().__init__() - self.sources = {} - self.url = None - self.name = None - - def handle_data(self, data): - if self.url is not None: - self.name = data - - def handle_starttag(self, tag, attrs): - if tag == "a": - for name, value in attrs: - if name == "href": - self.url = value - - def handle_endtag(self, tag): - if self.url is not None: - self.sources[self.name] = self.url - self.url = None - - -url = sys.argv[1] -package_name = sys.argv[2] -index_url = url + "/" + package_name + "/" -package_filename = sys.argv[3] - -# Parse username and password for this host from the netrc file if given. -username, password = None, None -if os.environ["NETRC"]: - netrc_obj = netrc.netrc(os.environ["NETRC"]) - host = urlparse(index_url).netloc - # Strip port number if present - if ":" in host: - host = host.split(":")[0] - username, _, password = netrc_obj.authenticators(host) - -print("Reading index %s" % index_url) - -context = ssl.create_default_context() -context.check_hostname = False -context.verify_mode = ssl.CERT_NONE - -# Extract out username/password from index_url, if present. -parsed_url = urlparse(index_url) -username = parsed_url.username or username -password = parsed_url.password or password -index_url = parsed_url._replace(netloc=parsed_url.netloc.rpartition("@")[-1]).geturl() - -req = urllib.request.Request(index_url) -if username and password: - import base64 - - password_b64 = base64.b64encode(":".join((username, password)).encode()).decode( - "utf-8" - ) - req.add_header("Authorization", "Basic {}".format(password_b64)) -response = urllib.request.urlopen(req, context=context) -index = response.read() - -parser = Pep503() -parser.feed(str(index, "utf-8")) -if package_filename not in parser.sources: - print( - "The file %s has not be found in the index %s" % (package_filename, index_url) - ) - exit(1) - -package_file = open(package_filename, "wb") -# Sometimes the href is a relative or absolute path within the index's domain. -indicated_url = urlparse(parser.sources[package_filename]) -if indicated_url.netloc == "": - parsed_url = urlparse(index_url) - - if indicated_url.path.startswith("/"): - # An absolute path within the index's domain. - path = parser.sources[package_filename] - else: - # A relative path. - path = parsed_url.path + "/" + parser.sources[package_filename] - - package_url = urlunparse( - ( - parsed_url.scheme, - parsed_url.netloc, - path, - None, - None, - None, - ) - ) -else: - package_url = parser.sources[package_filename] - -# Handle urls containing "../" -parsed_url = urlparse(package_url) -real_package_url = urlunparse( - ( - parsed_url.scheme, - parsed_url.netloc, - normpath(parsed_url.path), - parsed_url.params, - parsed_url.query, - parsed_url.fragment, - ) -) -print("Downloading %s" % real_package_url) - -req = urllib.request.Request(real_package_url) -if username and password: - req.add_unredirected_header("Authorization", "Basic {}".format(password_b64)) -response = urllib.request.urlopen(req, context=context) - -with response as r: - shutil.copyfileobj(r, package_file) diff --git a/lib.nix b/lib.nix index 821e49ae7..2aea27ab9 100644 --- a/lib.nix +++ b/lib.nix @@ -79,84 +79,6 @@ let else if lib.strings.hasInfix "manylinux_" f then { pkg = [ ml.manylinux2014 ]; str = "pep600"; } else { pkg = [ ]; str = null; }; - # Predict URL from the PyPI index. - # Args: - # pname: package name - # file: filename including extension - # hash: SRI hash - # kind: Language implementation and version tag - predictURLFromPypi = lib.makeOverridable ( - { pname, file, kind }: - "https://files.pythonhosted.org/packages/${kind}/${lib.toLower (builtins.substring 0 1 file)}/${pname}/${file}" - ); - - - # Fetch from the PyPI index. - # At first we try to fetch the predicated URL but if that fails we - # will use the Pypi API to determine the correct URL. - # Args: - # pname: package name - # file: filename including extension - # version: the version string of the dependency - # hash: SRI hash - # kind: Language implementation and version tag - fetchFromPypi = lib.makeOverridable ( - { pname, file, version, hash, kind, curlOpts ? "" }: - let - predictedURL = predictURLFromPypi { inherit pname file kind; }; - in - pkgs.stdenvNoCC.mkDerivation { - name = file; - nativeBuildInputs = [ - pkgs.buildPackages.curl - pkgs.buildPackages.jq - ]; - isWheel = lib.strings.hasSuffix "whl" file; - system = "builtin"; - - preferLocalBuild = true; - impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ - "NIX_CURL_FLAGS" - ]; - - inherit pname file version curlOpts predictedURL; - - builder = ./fetch-from-pypi.sh; - - outputHashMode = "flat"; - outputHashAlgo = "sha256"; - outputHash = hash; - - passthru = { - urls = [ predictedURL ]; # retain compatibility with nixpkgs' fetchurl - }; - } - ); - - fetchFromLegacy = lib.makeOverridable ( - { python, pname, url, file, hash }: - let - pathParts = - builtins.filter - ({ prefix, path }: "NETRC" == prefix) - builtins.nixPath; - netrc_file = if (pathParts != [ ]) then (builtins.head pathParts).path else ""; - in - pkgs.runCommand file - { - nativeBuildInputs = [ python ]; - impureEnvVars = lib.fetchers.proxyImpureEnvVars; - outputHashMode = "flat"; - outputHashAlgo = "sha256"; - outputHash = hash; - NETRC = netrc_file; - passthru.isWheel = lib.strings.hasSuffix "whl" file; - } '' - python ${./fetch_from_legacy.py} ${url} ${pname} ${file} - mv ${file} $out - '' - ); - getBuildSystemPkgs = { pythonPackages , pyProject @@ -206,8 +128,6 @@ let in { inherit - fetchFromPypi - fetchFromLegacy getManyLinuxDeps isCompatible readTOML diff --git a/mk-poetry-dep.nix b/mk-poetry-dep.nix index d770144a7..47aa37e15 100644 --- a/mk-poetry-dep.nix +++ b/mk-poetry-dep.nix @@ -215,15 +215,14 @@ pythonPackages.callPackage else if isFile then localDepPath else if isLegacy then - fetchFromLegacy + pyproject-nix.fetchers.fetchFromLegacy { pname = name; - inherit python; inherit (fileInfo) file hash; inherit (source) url; } else - fetchFromPypi { + pyproject-nix.fetchers.fetchFromPypi { pname = name; inherit (fileInfo) file hash kind; inherit version; diff --git a/vendor/pyproject.nix/fetchers/fetch-from-legacy.py b/vendor/pyproject.nix/fetchers/fetch-from-legacy.py index 790458bb1..b39ef5df4 100644 --- a/vendor/pyproject.nix/fetchers/fetch-from-legacy.py +++ b/vendor/pyproject.nix/fetchers/fetch-from-legacy.py @@ -77,7 +77,9 @@ def handle_endtag(self, tag: str) -> None: if username and password: import base64 - password_b64 = base64.b64encode(":".join((username, password)).encode()).decode("utf-8") + password_b64 = base64.b64encode(":".join((username, password)).encode()).decode( + "utf-8" + ) req.add_header("Authorization", "Basic {}".format(password_b64)) response = urllib.request.urlopen(req, context=context) index = response.read() @@ -85,7 +87,9 @@ def handle_endtag(self, tag: str) -> None: parser = Pep503() parser.feed(str(index, "utf-8")) if package_filename not in parser.sources: - print("The file %s has not be found in the index %s" % (package_filename, index_url)) + print( + "The file %s has not be found in the index %s" % (package_filename, index_url) + ) exit(1) package_file = open(package_filename, "wb")