Skip to content

Commit

Permalink
Merge pull request #1363 from nix-community/pyproject-nix-fetchers
Browse files Browse the repository at this point in the history
Use pyproject.nix fetchers
  • Loading branch information
adisbladis authored Oct 26, 2023
2 parents 40192db + 841aa10 commit b6ddb11
Show file tree
Hide file tree
Showing 25 changed files with 270 additions and 199 deletions.
10 changes: 5 additions & 5 deletions default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@
let
inherit (poetryLib) isCompatible readTOML;

pyproject-nix = import ./vendor/pyproject.nix { inherit lib; };
pyproject-nix = import ./vendor/pyproject.nix { inherit pkgs lib; };

# Name normalization
inherit (pyproject-nix.pypa) normalizePackageName;
inherit (pyproject-nix.lib.pypa) normalizePackageName;
normalizePackageSet = lib.attrsets.mapAttrs' (name: value: lib.attrsets.nameValuePair (normalizePackageName name) value);

# Map SPDX identifiers to license names
Expand Down Expand Up @@ -172,7 +172,7 @@ lib.makeScope pkgs.newScope (self: {
in
lib.listToAttrs (lib.mapAttrsToList (n: v: { name = normalizePackageName n; value = v; }) lockfiles);

pep508Env = pyproject-nix.pep508.mkEnviron python;
pep508Env = pyproject-nix.lib.pep508.mkEnviron python;

# Filter packages by their PEP508 markers & pyproject interpreter version
partitions =
Expand All @@ -181,9 +181,9 @@ lib.makeScope pkgs.newScope (self: {
if pkgMeta ? marker then
(
let
marker = pyproject-nix.pep508.parseMarkers pkgMeta.marker;
marker = pyproject-nix.lib.pep508.parseMarkers pkgMeta.marker;
in
pyproject-nix.pep508.evalMarkers pep508Env marker
pyproject-nix.lib.pep508.evalMarkers pep508Env marker
) else true && isCompatible (poetryLib.getPythonVersion python) pkgMeta.python-versions;
in
lib.partition supportsPythonVersion poetryLock.package;
Expand Down
2 changes: 1 addition & 1 deletion editable.nix
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
, pyproject-nix
}:
let
name = pyproject-nix.pypa.normalizePackageName pyProject.tool.poetry.name;
name = pyproject-nix.lib.pypa.normalizePackageName pyProject.tool.poetry.name;

# Just enough standard PKG-INFO fields for an editable installation
pkgInfoFields = {
Expand Down
80 changes: 0 additions & 80 deletions lib.nix
Original file line number Diff line number Diff line change
Expand Up @@ -79,84 +79,6 @@ let
else if lib.strings.hasInfix "manylinux_" f then { pkg = [ ml.manylinux2014 ]; str = "pep600"; }
else { pkg = [ ]; str = null; };

# Predict URL from the PyPI index.
# Args:
# pname: package name
# file: filename including extension
# hash: SRI hash
# kind: Language implementation and version tag
predictURLFromPypi = lib.makeOverridable (
{ pname, file, kind }:
"https://files.pythonhosted.org/packages/${kind}/${lib.toLower (builtins.substring 0 1 file)}/${pname}/${file}"
);


# Fetch from the PyPI index.
# At first we try to fetch the predicated URL but if that fails we
# will use the Pypi API to determine the correct URL.
# Args:
# pname: package name
# file: filename including extension
# version: the version string of the dependency
# hash: SRI hash
# kind: Language implementation and version tag
fetchFromPypi = lib.makeOverridable (
{ pname, file, version, hash, kind, curlOpts ? "" }:
let
predictedURL = predictURLFromPypi { inherit pname file kind; };
in
pkgs.stdenvNoCC.mkDerivation {
name = file;
nativeBuildInputs = [
pkgs.buildPackages.curl
pkgs.buildPackages.jq
];
isWheel = lib.strings.hasSuffix "whl" file;
system = "builtin";

preferLocalBuild = true;
impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [
"NIX_CURL_FLAGS"
];

inherit pname file version curlOpts predictedURL;

builder = ./fetch-from-pypi.sh;

outputHashMode = "flat";
outputHashAlgo = "sha256";
outputHash = hash;

passthru = {
urls = [ predictedURL ]; # retain compatibility with nixpkgs' fetchurl
};
}
);

fetchFromLegacy = lib.makeOverridable (
{ python, pname, url, file, hash }:
let
pathParts =
builtins.filter
({ prefix, path }: "NETRC" == prefix)
builtins.nixPath;
netrc_file = if (pathParts != [ ]) then (builtins.head pathParts).path else "";
in
pkgs.runCommand file
{
nativeBuildInputs = [ python ];
impureEnvVars = lib.fetchers.proxyImpureEnvVars;
outputHashMode = "flat";
outputHashAlgo = "sha256";
outputHash = hash;
NETRC = netrc_file;
passthru.isWheel = lib.strings.hasSuffix "whl" file;
} ''
python ${./fetch_from_legacy.py} ${url} ${pname} ${file}
mv ${file} $out
''
);

getBuildSystemPkgs =
{ pythonPackages
, pyProject
Expand Down Expand Up @@ -206,8 +128,6 @@ let
in
{
inherit
fetchFromPypi
fetchFromLegacy
getManyLinuxDeps
isCompatible
readTOML
Expand Down
11 changes: 5 additions & 6 deletions mk-poetry-dep.nix
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ pythonPackages.callPackage
}@args:
let
inherit (python) stdenv;
inherit (pyproject-nix.pypa) normalizePackageName;
inherit (pyproject-nix.lib.pypa) normalizePackageName;
inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromLegacy fetchFromPypi;

inherit (import ./pep425.nix {
Expand Down Expand Up @@ -145,15 +145,15 @@ pythonPackages.callPackage
pep508Markers = v.markers or "";
in
compat constraints && (if pep508Markers == "" then true else
(pyproject-nix.pep508.evalMarkers
(pyproject-nix.lib.pep508.evalMarkers
(pep508Env // {
extra = {
# All extras are always enabled
type = "extra";
value = lib.attrNames extras;
};
})
(pyproject-nix.pep508.parseMarkers pep508Markers)))
(pyproject-nix.lib.pep508.parseMarkers pep508Markers)))
)
dependencies
);
Expand Down Expand Up @@ -215,15 +215,14 @@ pythonPackages.callPackage
else if isFile then
localDepPath
else if isLegacy then
fetchFromLegacy
pyproject-nix.fetchers.fetchFromLegacy
{
pname = name;
inherit python;
inherit (fileInfo) file hash;
inherit (source) url;
}
else
fetchFromPypi {
pyproject-nix.fetchers.fetchFromPypi {
pname = name;
inherit (fileInfo) file hash kind;
inherit version;
Expand Down
2 changes: 1 addition & 1 deletion pep425.nix
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{ lib, stdenv, python, pyproject-nix, isLinux ? stdenv.isLinux }:
let
inherit (lib.strings) escapeRegex hasPrefix hasSuffix hasInfix splitString removeSuffix;
targetMachine = pyproject-nix.pep599.manyLinuxTargetMachines.${stdenv.targetPlatform.parsed.cpu.name};
targetMachine = pyproject-nix.lib.pep599.manyLinuxTargetMachines.${stdenv.targetPlatform.parsed.cpu.name};

pythonVer =
let
Expand Down
27 changes: 5 additions & 22 deletions vendor/pyproject.nix/default.nix
Original file line number Diff line number Diff line change
@@ -1,22 +1,5 @@
{ lib }:
let
inherit (builtins) mapAttrs;
inherit (lib) fix;
in

fix (self: mapAttrs (_: path: import path ({ inherit lib; } // self)) {
pip = ./pip.nix;
pypa = ./pypa.nix;
project = ./project.nix;
renderers = ./renderers.nix;
validators = ./validators.nix;
poetry = ./poetry.nix;

pep427 = ./pep427.nix;
pep440 = ./pep440.nix;
pep508 = ./pep508.nix;
pep518 = ./pep518.nix;
pep599 = ./pep599.nix;
pep600 = ./pep600.nix;
pep621 = ./pep621.nix;
})
{ pkgs, lib }:
{
lib = import ./lib { inherit lib; };
fetchers = import ./fetchers { inherit pkgs lib; };
}
122 changes: 122 additions & 0 deletions vendor/pyproject.nix/fetchers/default.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
{ pkgs
, lib
,
}:
let
inherit (builtins) substring filter head nixPath;
inherit (lib) toLower;

# Predict URL from the PyPI index.
# Args:
# pname: package name
# file: filename including extension
# hash: SRI hash
# kind: Language implementation and version tag
predictURLFromPypi =
{
# package name
pname
, # filename including extension
file
, # Language implementation and version tag
kind
,
}: "https://files.pythonhosted.org/packages/${kind}/${toLower (substring 0 1 file)}/${pname}/${file}";
in
lib.mapAttrs (_: func: lib.makeOverridable func) {
/*
Fetch from the PyPI index.
At first we try to fetch the predicated URL but if that fails we
will use the Pypi API to determine the correct URL.
Type: fetchFromPypi :: AttrSet -> derivation
*/
fetchFromPypi =
{
# package name
pname
, # filename including extension
file
, # the version string of the dependency
version
, # SRI hash
hash
, # Language implementation and version tag
kind
, # Options to pass to `curl`
curlOpts ? ""
,
}:
let
predictedURL = predictURLFromPypi { inherit pname file kind; };
in
pkgs.stdenvNoCC.mkDerivation {
name = file;
nativeBuildInputs = [
pkgs.buildPackages.curl
pkgs.buildPackages.jq
];
isWheel = lib.strings.hasSuffix "whl" file;
system = "builtin";

preferLocalBuild = true;
impureEnvVars =
lib.fetchers.proxyImpureEnvVars
++ [
"NIX_CURL_FLAGS"
];

inherit pname file version curlOpts predictedURL;

builder = ./fetch-from-pypi.sh;

outputHashMode = "flat";
outputHashAlgo = "sha256";
outputHash = hash;

passthru = {
urls = [ predictedURL ]; # retain compatibility with nixpkgs' fetchurl
};
};

/*
Fetch from the PyPI legacy API.
Some repositories (such as Devpi) expose the Pypi legacy API (https://warehouse.pypa.io/api-reference/legacy.html).
Type: fetchFromLegacy :: AttrSet -> derivation
*/
fetchFromLegacy =
{
# package name
pname
, # URL to package index
url
, # filename including extension
file
, # SRI hash
hash
,
}:
let
pathParts = filter ({ prefix, path }: "NETRC" == prefix) nixPath; # deadnix: skip
netrc_file =
if (pathParts != [ ])
then (head pathParts).path
else "";
in
pkgs.runCommand file
{
nativeBuildInputs = [ pkgs.buildPackages.python3 ];
impureEnvVars = lib.fetchers.proxyImpureEnvVars;
outputHashMode = "flat";
outputHashAlgo = "sha256";
outputHash = hash;
NETRC = netrc_file;
passthru.isWheel = lib.strings.hasSuffix "whl" file;
} ''
python ${./fetch-from-legacy.py} ${url} ${pname} ${file}
mv ${file} $out
'';
}
Loading

0 comments on commit b6ddb11

Please sign in to comment.