diff --git a/.gitignore b/.gitignore index f118a34c..e019c6f9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ result result-* __pycache__ +.pytest_cache +.mypy_cache +.ruff_cache diff --git a/flake.lock b/flake.lock index edb64697..a60fc2f3 100644 --- a/flake.lock +++ b/flake.lock @@ -43,6 +43,21 @@ "type": "github" } }, + "flake-compat": { + "locked": { + "lastModified": 1688025799, + "narHash": "sha256-ktpB4dRtnksm9F5WawoIkEneh1nrEvuxb5lJFt1iOyw=", + "owner": "nix-community", + "repo": "flake-compat", + "rev": "8bf105319d44f6b9f0d764efa4fdef9f1cc9ba1c", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "flake-compat", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -95,6 +110,22 @@ "type": "github" } }, + "nixpkgs-unstable": { + "locked": { + "lastModified": 1716330097, + "narHash": "sha256-8BO3B7e3BiyIDsaKA0tY8O88rClYRTjvAp66y+VBUeU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "5710852ba686cc1fd0d3b8e22b3117d43ba374c2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs_2": { "locked": { "lastModified": 1711668574, @@ -114,7 +145,9 @@ "root": { "inputs": { "androidPkgs": "androidPkgs", - "nixpkgs": "nixpkgs_2" + "flake-compat": "flake-compat", + "nixpkgs": "nixpkgs_2", + "nixpkgs-unstable": "nixpkgs-unstable" } }, "systems": { diff --git a/flake.nix b/flake.nix index 368b9b48..84b2676e 100644 --- a/flake.nix +++ b/flake.nix @@ -3,10 +3,14 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11"; + nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixos-unstable"; + androidPkgs.url = "github:tadfisher/android-nixpkgs/stable"; + + flake-compat.url = "github:nix-community/flake-compat"; }; - outputs = { self, nixpkgs, androidPkgs, ... }@inputs: let + outputs = { self, nixpkgs, nixpkgs-unstable, androidPkgs, flake-compat, ... }@inputs: let pkgs = import ./pkgs/default.nix { inherit inputs; }; in { # robotnixSystem evaluates a robotnix configuration @@ -24,6 +28,7 @@ packages.x86_64-linux = { manual = (import ./docs { inherit pkgs; }).manual; + gitRepo = pkgs.gitRepo; }; devShell.x86_64-linux = pkgs.mkShell { diff --git a/flake/compat.nix b/flake/compat.nix new file mode 100644 index 00000000..c6fec7b2 --- /dev/null +++ b/flake/compat.nix @@ -0,0 +1,17 @@ +{ system ? builtins.currentSystem }: + let + lock = builtins.fromJSON (builtins.readFile ./../flake.lock); + flake-compat-entry = lock.nodes.root.inputs.flake-compat; + + inherit (lock.nodes."${ flake-compat-entry }".locked) owner repo narHash; + + flake-compat = builtins.fetchTarball { + url = "https://github.com/${ owner }/${ repo }/archive/${ lock.nodes.flake-compat.locked.rev }.tar.gz"; + sha256 = narHash; + }; + in + import flake-compat { + inherit system; + + src = ./..; + } diff --git a/flavors/lineageos/test_lineageos_updater.py b/flavors/lineageos/test_lineageos_updater.py deleted file mode 100644 index 7fb2bb4e..00000000 --- a/flavors/lineageos/test_lineageos_updater.py +++ /dev/null @@ -1,138 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Daniel Fullmer and robotnix contributors -# SPDX-License-Identifier: MIT - -from unittest.mock import patch -from typing import Any - -import update_device_metadata -import update_device_dirs - - -def test_fetch_metadata(tmp_path: Any) -> None: - lineage_build_targets = tmp_path / "lineage-build-targets" - lineage_build_targets.write_text( - '''# Ignore comments - crosshatch userdebug lineage-18.1 W - ''') - - devices_json = tmp_path / "devices.json" - devices_json.write_text( - '''[ - { "model": "crosshatch", "oem": "Google", "name": "Pixel 3 XL", "lineage_recovery": true} - ] - ''') - - metadata = update_device_metadata.fetch_metadata( - f"file://{lineage_build_targets}", f"file://{devices_json}" - ) - - assert metadata == { - "crosshatch": { - "branch": "lineage-18.1", - "lineage_recovery": True, - "name": "Pixel 3 XL", - "variant": "userdebug", - "vendor": "google" - }, - } - - -@patch("update_device_dirs.checkout_git") -@patch("update_device_dirs.ls_remote") -def test_fetch_device_dirs(ls_remote: Any, checkout_git: Any, tmpdir: Any) -> None: - baseurl = 'BASEURL' - - ls_remote_dict = { - f'{baseurl}/android_device_google_crosshatch': {'refs/heads/lineage-18.1': '12345'}, - f'{baseurl}/android_kernel_google_msm-4.9': {'refs/heads/lineage-18.1': '67890'} - } - ls_remote.side_effect = lambda url: ls_remote_dict[url] - - fake_dir = tmpdir.mkdir('android_device_google_crosshatch') - (fake_dir / "lineage.dependencies").write( - '''[ - { - "repository": "android_kernel_google_msm-4.9", - "target_path": "kernel/google/msm-4.9" - } - ]''' - ) - checkout_git_dict = { - (f'{baseurl}/android_device_google_crosshatch', 'refs/heads/lineage-18.1'): - { - "date": "2021-07-12T10:07:57-05:00", - "deepClone": False, - "deps": [ - "packages/apps/ElmyraService" - ], - "fetchSubmodules": False, - "leaveDotGit": False, - "path": fake_dir, - "rev": "0000000000ca33df352b45a4241c353b6a52ec7d", - "sha256": "00000000000aamp6w094ingaccr9cx6d6zi1wb9crmh4r7b19b1f", - "url": f"{baseurl}/android_device_google_crosshatch" - }, - (f'{baseurl}/android_kernel_google_msm-4.9', 'refs/heads/lineage-18.1'): - { - "date": "2021-07-12T10:07:57-05:00", - "deepClone": False, - "fetchSubmodules": False, - "leaveDotGit": False, - "path": tmpdir.mkdir('empty'), - "rev": "0000000000ca33df352b45a4241c353b6a52ec7d", - "sha256": "00000000000aamp6w094ingaccr9cx6d6zi1wb9crmh4r7b19b1f", - "url": f"{baseurl}/android_kernel_google_msm-4.9" - }, - } - checkout_git.side_effect = lambda url, rev: checkout_git_dict[url, rev] - - metadata = { - "crosshatch": { - "branch": "lineage-18.1", - "lineage_recovery": True, - "name": "Pixel 3 XL", - "variant": "userdebug", - "vendor": "google" - }, - } - dirs = update_device_dirs.fetch_device_dirs(metadata, baseurl, 'lineage-18.1') - - assert 'device/google/crosshatch' in dirs - - -@patch("update_device_dirs.checkout_git") -@patch("update_device_dirs.ls_remote") -def test_fetch_vendor_dirs(ls_remote: Any, checkout_git: Any, tmpdir: Any) -> None: - baseurl = 'BASEURL' - ls_remote_dict = { - 'BASEURL/proprietary_vendor_google': {'refs/heads/lineage-18.1': '12345'}, - } - ls_remote.side_effect = lambda url: ls_remote_dict[url] - - checkout_git_dict = { - ('BASEURL/proprietary_vendor_google', 'refs/heads/lineage-18.1'): - { - "date": "2021-07-12T10:07:57-05:00", - "deepClone": False, - "fetchSubmodules": False, - "leaveDotGit": False, - "path": 'foobar', - "rev": "0000000000ca33df352b45a4241c353b6a52ec7d", - "sha256": "00000000000aamp6w094ingaccr9cx6d6zi1wb9crmh4r7b19b1f", - "url": "BASEURL/proprietary_vendor_google" - }, - } - checkout_git.side_effect = lambda url, rev: checkout_git_dict[url, rev] - - metadata = { - "crosshatch": { - "branch": "lineage-18.1", - "lineage_recovery": True, - "name": "Pixel 3 XL", - "variant": "userdebug", - "vendor": "google" - }, - } - dirs = update_device_dirs.fetch_vendor_dirs(metadata, baseurl, 'lineage-18.1') - - assert 'vendor/google' in dirs diff --git a/modules/envpackages.nix b/modules/envpackages.nix index 25d93e94..23c09c1e 100644 --- a/modules/envpackages.nix +++ b/modules/envpackages.nix @@ -29,7 +29,15 @@ in nettools # Needed for "hostname" in build/soong/ui/build/sandbox_linux.go procps # Needed for "ps" in build/envsetup.sh ] - (mkIf (config.androidVersion >= 10) [ + (mkIf (config.androidVersion >= 12) [ + freetype # Needed by jdk9 prebuilt + fontconfig + + # Goldfish doesn't need py2 anymore in Android 12+! + # c.f. https://android.googlesource.com/device/generic/goldfish/+/605e6a14e44c99e87d48bf52507f8aa01633fb04 + python3 + ]) + (mkIf ((config.androidVersion >= 10) && (config.androidVersion <= 11)) [ freetype # Needed by jdk9 prebuilt fontconfig diff --git a/modules/release.nix b/modules/release.nix index e570d752..782677cb 100644 --- a/modules/release.nix +++ b/modules/release.nix @@ -12,7 +12,9 @@ let jre = if (config.androidVersion >= 11) then pkgs.jdk11_headless else pkgs.jre8_headless; deps = with pkgs; [ otaTools openssl jre zip unzip pkgs.getopt which toybox vboot_reference util-linux - python # ota_from_target_files invokes, brillo_update_payload which has "truncate_file" which invokes python + # ota_from_target_files invokes, brillo_update_payload which has "truncate_file" which invokes python + # c.f. https://android.googlesource.com/platform/system/update_engine/+/refs/heads/main/scripts/brillo_update_payload#338 + python3 ]; in '' export PATH=${lib.makeBinPath deps}:$PATH diff --git a/pkgs/default.nix b/pkgs/default.nix index 9851c663..d43ecb93 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -1,14 +1,11 @@ # SPDX-FileCopyrightText: 2020 Daniel Fullmer and robotnix contributors # SPDX-License-Identifier: MIT -{ inputs ? (import ( - fetchTarball { - url = "https://github.com/edolstra/flake-compat/archive/12c64ca55c1014cdc1b16ed5a804aa8576601ff2.tar.gz"; - sha256 = "0jm6nzb83wa6ai17ly9fzpqc40wg1viib8klq8lby54agpl213w5"; } - ) { - src = ../.; - }).defaultNix.inputs, - ... }@args: +{ + system ? builtins.currentSystem, + inputs ? (import ../flake/compat.nix { inherit system; }).defaultNix.inputs, + ... +}@args: let inherit (inputs) nixpkgs androidPkgs; @@ -17,5 +14,5 @@ in nixpkgs.legacyPackages.x86_64-linux.appendOverlays [ androidPkgs.packages = androidPkgs.packages.x86_64-linux; androidPkgs.sdk = androidPkgs.sdk.x86_64-linux; }) - (import ./overlay.nix) + (import ./overlay.nix { inherit inputs; }) ] diff --git a/pkgs/gitRepo/default.nix b/pkgs/gitRepo/default.nix new file mode 100644 index 00000000..0b57d244 --- /dev/null +++ b/pkgs/gitRepo/default.nix @@ -0,0 +1,70 @@ +{ lib, inputs, fetchFromGitHub, rsync, git, gnupg, less, openssh, ... }: +let + inherit (inputs) nixpkgs-unstable; + + unstablePkgs = nixpkgs-unstable.legacyPackages.x86_64-linux; +in + unstablePkgs.gitRepo.overrideAttrs(oldAttrs: rec { + version = "2.45"; + + src = fetchFromGitHub { + owner = "android"; + repo = "tools_repo"; + rev = "v${ version }"; + hash = "sha256-f765TcOHL8wdPa9qSmGegofjCXx1tF/K5bRQnYQcYVc="; + }; + + nativeBuildInputs = (oldAttrs.nativeBuildInputs or []) ++ [ rsync git ]; + + repo2nixPatches = ./patches; + + # NOTE: why `git apply` instead of relying on `patches`? For some reason when + # using `patches` the source `rsync`ed into `var/repo` is missing those changes + installPhase = '' + runHook preInstall + + mkdir -p var/repo + rsync -a $src/ var/repo/ + + ( + export GIT_CONFIG_GLOBAL=$TMPDIR/.gitconfig + export GIT_CONFIG_NOSYSTEM=true + + cd var/repo + + git config --global --add safe.directory "$PWD" + git config --global user.email "nemo@nix" + git config --global user.name "Nemo Nix" + + chmod +w -R . + + git init + git add -A + git commit -m "Upstream sources" + + git am $repo2nixPatches/*.patch + + git log -n 1 --format="%H" > ../../COMMITED_REPO_REV + ) + + mkdir -p $out/var/repo + mkdir -p $out/bin + + rsync -a var/repo/ $out/var/repo/ + + # Copying instead of symlinking to the above directory is necessary, because otherwise + # running `repo init` fails, as I assume the script gets confused by being located in + # a git repo itself + cp repo $out/bin/repo + + runHook postInstall + ''; + + # Specify the patched checkout as the default version of repo + postFixup = '' + wrapProgram "$out/bin/repo" \ + --set REPO_URL "file://$out/var/repo" \ + --set REPO_REV "$(cat ./COMMITED_REPO_REV)" \ + --prefix PATH ":" "${ lib.makeBinPath [ git gnupg less openssh ] }" + ''; + }) diff --git a/pkgs/gitRepo/patches/0001-add-initial-hacky-version-of-repo2nix.patch b/pkgs/gitRepo/patches/0001-add-initial-hacky-version-of-repo2nix.patch new file mode 100644 index 00000000..7b8e28d3 --- /dev/null +++ b/pkgs/gitRepo/patches/0001-add-initial-hacky-version-of-repo2nix.patch @@ -0,0 +1,137 @@ +From 8fe8a2efaa6bcd404c256a29d1dc5ed4ae241a08 Mon Sep 17 00:00:00 2001 +From: ajs124 +Date: Tue, 26 Feb 2019 04:48:13 +0100 +Subject: [PATCH 01/16] add initial (hacky) version of "repo2nix" + +now with less impurities +--- + project.py | 4 +- + subcmds/nix.py | 100 +++++++++++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 102 insertions(+), 2 deletions(-) + create mode 100644 subcmds/nix.py + +diff --git a/project.py b/project.py +index 1f5e4c3..c3c3384 100644 +--- a/project.py ++++ b/project.py +@@ -2887,8 +2887,8 @@ class Project: + ) + + def _LsRemote(self, refs): +- cmd = ["ls-remote", self.remote.name, refs] +- p = GitCommand(self, cmd, capture_stdout=True) ++ cmd = ["ls-remote", self.remote.url, refs] ++ p = GitCommand(self, cmd, cwd="/", capture_stdout=True) + if p.Wait() == 0: + return p.stdout + return None +diff --git a/subcmds/nix.py b/subcmds/nix.py +new file mode 100644 +index 0000000..f113ede +--- /dev/null ++++ b/subcmds/nix.py +@@ -0,0 +1,100 @@ ++# ++# Copyright (C) 2008 The Android Open Source Project ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++ ++from __future__ import print_function ++ ++from pyversion import is_python3 ++if is_python3(): ++ import http.cookiejar as cookielib ++ import urllib.error ++ import urllib.parse ++ import urllib.request ++ import xmlrpc.client ++else: ++ import imp ++ import urllib2 ++ import urlparse ++ import xmlrpclib ++ urllib = imp.new_module('urllib') ++ urllib.error = urllib2 ++ urllib.parse = urlparse ++ urllib.request = urllib2 ++ xmlrpc = imp.new_module('xmlrpc') ++ xmlrpc.client = xmlrpclib ++ ++try: ++ import threading as _threading ++except ImportError: ++ import dummy_threading as _threading ++ ++try: ++ import resource ++ def _rlimit_nofile(): ++ return resource.getrlimit(resource.RLIMIT_NOFILE) ++except ImportError: ++ def _rlimit_nofile(): ++ return (256, 256) ++ ++try: ++ import multiprocessing ++except ImportError: ++ multiprocessing = None ++ ++from command import Command, MirrorSafeCommand ++ ++class Nix(Command, MirrorSafeCommand): ++ common = True ++ helpSummary = "Export nix file with sources" ++ helpUsage = """ ++%prog [...] ++""" ++ helpDescription = """ ++""" ++ ++ def Execute(self, opt, args): ++ all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=False) ++ ++ oS = '{\n' ++ oS += "unpackPhase = ''\n" \ ++ 'echo "reassembling source tree from git source store paths"\n' \ ++ 'mkdir src; cd src\n' \ ++ 'for src in $srcs; do\n' \ ++ " dest_folder=$(stripHash $src); dest_folder=''${dest_folder//=//}\n" \ ++ ' echo "$src -> $dest_folder"\n' \ ++ ' mkdir -p "$dest_folder"\n' \ ++ ' cp --reflink=auto --no-preserve=ownership --no-dereference --preserve=links --recursive "$src/." "$dest_folder/"\n' \ ++ ' chmod -R u+w "$dest_folder"\n' \ ++ 'done\n' \ ++ 'echo "creating symlinks and copies as specified in repo manifest(s)"\n' ++ for p in all_projects: ++ for f in p.linkfiles: ++ oS += 'ln -s ' + f.src_rel_to_dest + ' ' + f.dest + '\n' ++ for c in p.copyfiles: ++ oS += 'cp --reflink=auto ' + p.relpath + '/' + c.src + ' ' + c.dest + '\n' ++ oS += "'';\n" ++ ++ oS += 'sources = [\n' ++ for p in all_projects: ++ oS += ' (builtins.fetchGit {\n' ++ oS += ' url = "' + p.remote.url + '";\n' ++ if 'refs/heads' in p.revisionExpr: ++ oS += ' ref = "' + p.revisionExpr.split('/')[-1] + '";\n' ++ else: ++ oS += ' ref = "' + p.revisionExpr + '";\n' ++ oS += ' rev = "' + p._LsRemote(p.revisionExpr).split('\t')[0] + '";\n' ++ oS += ' name = "' + p.relpath.replace('/', '=') + '";\n' ++ oS += ' })\n' ++ oS += '];\n}' ++ print(oS) +\ No newline at end of file +-- +2.44.0 + diff --git a/pkgs/gitRepo/patches/0002-repo2nix-output-just-json-do-more-processing-in-nix.patch b/pkgs/gitRepo/patches/0002-repo2nix-output-just-json-do-more-processing-in-nix.patch new file mode 100644 index 00000000..f37bc817 --- /dev/null +++ b/pkgs/gitRepo/patches/0002-repo2nix-output-just-json-do-more-processing-in-nix.patch @@ -0,0 +1,111 @@ +From 4550efa07636b497bfca0b8379c0656ed09770cb Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Mon, 15 Jul 2019 14:43:17 -0400 +Subject: [PATCH 02/16] repo2nix: output just json--do more processing in nix + +--- + subcmds/dumpjson.py | 92 +++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 92 insertions(+) + create mode 100644 subcmds/dumpjson.py + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +new file mode 100644 +index 0000000..a09885d +--- /dev/null ++++ b/subcmds/dumpjson.py +@@ -0,0 +1,92 @@ ++# ++# Copyright (C) 2008 The Android Open Source Project ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++ ++from __future__ import print_function ++ ++from pyversion import is_python3 ++if is_python3(): ++ import http.cookiejar as cookielib ++ import urllib.error ++ import urllib.parse ++ import urllib.request ++ import xmlrpc.client ++else: ++ import imp ++ import urllib2 ++ import urlparse ++ import xmlrpclib ++ urllib = imp.new_module('urllib') ++ urllib.error = urllib2 ++ urllib.parse = urlparse ++ urllib.request = urllib2 ++ xmlrpc = imp.new_module('xmlrpc') ++ xmlrpc.client = xmlrpclib ++ ++try: ++ import threading as _threading ++except ImportError: ++ import dummy_threading as _threading ++ ++try: ++ import resource ++ def _rlimit_nofile(): ++ return resource.getrlimit(resource.RLIMIT_NOFILE) ++except ImportError: ++ def _rlimit_nofile(): ++ return (256, 256) ++ ++try: ++ import multiprocessing ++except ImportError: ++ multiprocessing = None ++ ++from command import Command, MirrorSafeCommand ++ ++class Dumpjson(Command, MirrorSafeCommand): ++ common = True ++ helpSummary = "Export json file with sources" ++ helpUsage = """ ++%prog [...] ++""" ++ helpDescription = """ ++""" ++ ++ def Execute(self, opt, args): ++ all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=False) ++ ++ import json ++ data = { ++ p.name: { ++ "url": p.remote.url, ++ "relpath": p.relpath, ++ "groups": p.groups, ++ "revisionExpr": p.revisionExpr, ++ "rev": p._LsRemote(p.revisionExpr).split('\t')[0], ++ "linkfiles": [ ++ { "src_rel_to_dest": l.src_rel_to_dest, ++ "dest": l.dest, ++ } ++ for l in p.linkfiles ++ ], ++ "copyfiles": [ ++ { "src": c.src, ++ "dest": c.dest, ++ } ++ for c in p.copyfiles ++ ], ++ } ++ for p in all_projects ++ }; ++ print(json.dumps(data)) +-- +2.44.0 + diff --git a/pkgs/gitRepo/patches/0003-nix-Retry-20-times-if-the-http-request-fails.patch b/pkgs/gitRepo/patches/0003-nix-Retry-20-times-if-the-http-request-fails.patch new file mode 100644 index 00000000..11a0fb52 --- /dev/null +++ b/pkgs/gitRepo/patches/0003-nix-Retry-20-times-if-the-http-request-fails.patch @@ -0,0 +1,40 @@ +From 333fc9d324640adb94e2ec7b51ab24268a3084d4 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Janne=20He=C3=9F?= +Date: Sun, 27 Oct 2019 11:31:40 +0100 +Subject: [PATCH 03/16] nix: Retry 20 times if the http request fails + +--- + subcmds/nix.py | 9 +++++++-- + 1 file changed, 7 insertions(+), 2 deletions(-) + +diff --git a/subcmds/nix.py b/subcmds/nix.py +index f113ede..fd7c188 100644 +--- a/subcmds/nix.py ++++ b/subcmds/nix.py +@@ -53,6 +53,7 @@ except ImportError: + multiprocessing = None + + from command import Command, MirrorSafeCommand ++from itertools import repeat + + class Nix(Command, MirrorSafeCommand): + common = True +@@ -93,8 +94,12 @@ class Nix(Command, MirrorSafeCommand): + oS += ' ref = "' + p.revisionExpr.split('/')[-1] + '";\n' + else: + oS += ' ref = "' + p.revisionExpr + '";\n' +- oS += ' rev = "' + p._LsRemote(p.revisionExpr).split('\t')[0] + '";\n' ++ while repeat(None, 20): ++ raw_rev = p._LsRemote(p.revisionExpr) ++ if raw_rev != None: ++ break ++ oS += ' rev = "' + raw_rev.split('\t')[0] + '";\n' + oS += ' name = "' + p.relpath.replace('/', '=') + '";\n' + oS += ' })\n' + oS += '];\n}' +- print(oS) +\ No newline at end of file ++ print(oS) +-- +2.44.0 + diff --git a/pkgs/gitRepo/patches/0004-fml.patch b/pkgs/gitRepo/patches/0004-fml.patch new file mode 100644 index 00000000..0d6a208a --- /dev/null +++ b/pkgs/gitRepo/patches/0004-fml.patch @@ -0,0 +1,25 @@ +From 991502c597e3d3c9d5b25fcf08ecd216fdba107b Mon Sep 17 00:00:00 2001 +From: ajs124 +Date: Tue, 3 Dec 2019 20:23:01 +0100 +Subject: [PATCH 04/16] fml + +--- + subcmds/nix.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/subcmds/nix.py b/subcmds/nix.py +index fd7c188..ad1f19f 100644 +--- a/subcmds/nix.py ++++ b/subcmds/nix.py +@@ -98,7 +98,7 @@ class Nix(Command, MirrorSafeCommand): + raw_rev = p._LsRemote(p.revisionExpr) + if raw_rev != None: + break +- oS += ' rev = "' + raw_rev.split('\t')[0] + '";\n' ++ # oS += ' rev = "' + raw_rev.split('\t')[0] + '";\n' + oS += ' name = "' + p.relpath.replace('/', '=') + '";\n' + oS += ' })\n' + oS += '];\n}' +-- +2.44.0 + diff --git a/pkgs/gitRepo/patches/0005-dumpjson-Sort-json-output-by-keys-for-reproducibilit.patch b/pkgs/gitRepo/patches/0005-dumpjson-Sort-json-output-by-keys-for-reproducibilit.patch new file mode 100644 index 00000000..9117739c --- /dev/null +++ b/pkgs/gitRepo/patches/0005-dumpjson-Sort-json-output-by-keys-for-reproducibilit.patch @@ -0,0 +1,22 @@ +From 07d83a29279874566234b2a3aed6fc270e5f9b8b Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Tue, 3 Dec 2019 16:56:27 -0500 +Subject: [PATCH 05/16] dumpjson: Sort json output by keys for reproducibility + +--- + subcmds/dumpjson.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index a09885d..8997c27 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -89,4 +89,4 @@ class Dumpjson(Command, MirrorSafeCommand): + } + for p in all_projects + }; +- print(json.dumps(data)) ++ print(json.dumps(data, sort_keys=True)) +-- +2.44.0 + diff --git a/pkgs/gitRepo/patches/0006-dumpjson-parallelize.patch b/pkgs/gitRepo/patches/0006-dumpjson-parallelize.patch new file mode 100644 index 00000000..2ba4d3cd --- /dev/null +++ b/pkgs/gitRepo/patches/0006-dumpjson-parallelize.patch @@ -0,0 +1,57 @@ +From 40465bd1ec05fa648765a0bc5d3ed461ae8ce2e2 Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Sat, 7 Dec 2019 12:15:55 -0500 +Subject: [PATCH 06/16] dumpjson: parallelize + +--- + subcmds/dumpjson.py | 19 +++++++++++++++++-- + 1 file changed, 17 insertions(+), 2 deletions(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index 8997c27..18aca4c 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -52,8 +52,15 @@ try: + except ImportError: + multiprocessing = None + ++import sys ++import json ++ + from command import Command, MirrorSafeCommand + ++def _fetch_revs(p, sem): ++ with sem: ++ p.rev = p._LsRemote(p.revisionExpr).split('\t')[0] ++ + class Dumpjson(Command, MirrorSafeCommand): + common = True + helpSummary = "Export json file with sources" +@@ -66,14 +73,22 @@ class Dumpjson(Command, MirrorSafeCommand): + def Execute(self, opt, args): + all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=False) + +- import json ++ MAX_THREADS = 8 ++ sem = _threading.Semaphore(MAX_THREADS) ++ ++ threads = [ _threading.Thread(target=_fetch_revs, args=(p, sem)) for p in all_projects ] ++ for t in threads: ++ t.start() ++ for t in threads: ++ t.join() ++ + data = { + p.name: { + "url": p.remote.url, + "relpath": p.relpath, + "groups": p.groups, + "revisionExpr": p.revisionExpr, +- "rev": p._LsRemote(p.revisionExpr).split('\t')[0], ++ "rev": p.rev, + "linkfiles": [ + { "src_rel_to_dest": l.src_rel_to_dest, + "dest": l.dest, +-- +2.44.0 + diff --git a/pkgs/gitRepo/patches/0007-dumpjson-Sort-groups-list.patch b/pkgs/gitRepo/patches/0007-dumpjson-Sort-groups-list.patch new file mode 100644 index 00000000..2cf31a99 --- /dev/null +++ b/pkgs/gitRepo/patches/0007-dumpjson-Sort-groups-list.patch @@ -0,0 +1,25 @@ +From 92f28fed56d9195cf723aef22b3ce00b47742ed1 Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Sun, 8 Dec 2019 14:21:32 -0500 +Subject: [PATCH 07/16] dumpjson: Sort groups list + +--- + subcmds/dumpjson.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index 18aca4c..a1da03c 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -86,7 +86,7 @@ class Dumpjson(Command, MirrorSafeCommand): + p.name: { + "url": p.remote.url, + "relpath": p.relpath, +- "groups": p.groups, ++ "groups": sorted(p.groups), + "revisionExpr": p.revisionExpr, + "rev": p.rev, + "linkfiles": [ +-- +2.44.0 + diff --git a/pkgs/gitRepo/patches/0008-dumpjson-fix-explicitly-referencing-a-revision.patch b/pkgs/gitRepo/patches/0008-dumpjson-fix-explicitly-referencing-a-revision.patch new file mode 100644 index 00000000..c492ffd9 --- /dev/null +++ b/pkgs/gitRepo/patches/0008-dumpjson-fix-explicitly-referencing-a-revision.patch @@ -0,0 +1,39 @@ +From 35c3b4c59382e7db182b4eedfacba55c762295e5 Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Tue, 14 Apr 2020 19:47:47 -0400 +Subject: [PATCH 08/16] dumpjson: fix explicitly referencing a revision + +GrapheneOS does this as of QQ2A.200405.005.2020.04.13.21 +--- + subcmds/dumpjson.py | 11 +++++++++-- + 1 file changed, 9 insertions(+), 2 deletions(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index a1da03c..8ca9d08 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -54,12 +54,19 @@ except ImportError: + + import sys + import json ++import re + + from command import Command, MirrorSafeCommand + + def _fetch_revs(p, sem): +- with sem: +- p.rev = p._LsRemote(p.revisionExpr).split('\t')[0] ++ if re.match("[0-9a-f]{40}", p.revisionExpr): ++ # Use revisionExpr if it is already a SHA1 hash ++ p.rev = p.revisionExpr ++ else: ++ # Otherwise we need to grab the hash from the remote source ++ with sem: ++ p.rev = p._LsRemote(p.revisionExpr).split('\t')[0] ++ assert p.rev != "" + + class Dumpjson(Command, MirrorSafeCommand): + common = True +-- +2.44.0 + diff --git a/pkgs/gitRepo/patches/0009-dumpjson-fix-removed-LinkFile-attribute.patch b/pkgs/gitRepo/patches/0009-dumpjson-fix-removed-LinkFile-attribute.patch new file mode 100644 index 00000000..523a63fc --- /dev/null +++ b/pkgs/gitRepo/patches/0009-dumpjson-fix-removed-LinkFile-attribute.patch @@ -0,0 +1,26 @@ +From 35c373392754b6e79f1ef20a395b97d6a736ba92 Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Wed, 15 Apr 2020 13:05:14 -0400 +Subject: [PATCH 09/16] dumpjson: fix removed LinkFile attribute + +This unfortunately also changes the output schema for dumpjson. +--- + subcmds/dumpjson.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index 8ca9d08..726ccfd 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -97,7 +97,7 @@ class Dumpjson(Command, MirrorSafeCommand): + "revisionExpr": p.revisionExpr, + "rev": p.rev, + "linkfiles": [ +- { "src_rel_to_dest": l.src_rel_to_dest, ++ { "src": l.src, + "dest": l.dest, + } + for l in p.linkfiles +-- +2.44.0 + diff --git a/pkgs/gitRepo/patches/0010-dumpjson-filter-out-groups-with-redundant-informatio.patch b/pkgs/gitRepo/patches/0010-dumpjson-filter-out-groups-with-redundant-informatio.patch new file mode 100644 index 00000000..0589fae9 --- /dev/null +++ b/pkgs/gitRepo/patches/0010-dumpjson-filter-out-groups-with-redundant-informatio.patch @@ -0,0 +1,30 @@ +From 71a828d0123b6fe16c1ca5a869c49d55c8dc275b Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Wed, 15 Apr 2020 13:10:13 -0400 +Subject: [PATCH 10/16] dumpjson: filter out groups with redundant information + +--- + subcmds/dumpjson.py | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index 726ccfd..0bd2708 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -89,11 +89,12 @@ class Dumpjson(Command, MirrorSafeCommand): + for t in threads: + t.join() + ++ group_filter = lambda g: not (g == "all" or g.startswith("name:") or g.startswith("path:")) + data = { + p.name: { + "url": p.remote.url, + "relpath": p.relpath, +- "groups": sorted(p.groups), ++ "groups": sorted(filter(group_filter, p.groups)), + "revisionExpr": p.revisionExpr, + "rev": p.rev, + "linkfiles": [ +-- +2.44.0 + diff --git a/pkgs/gitRepo/patches/0011-dumpjson-conditionally-exclude-empty-attributes.patch b/pkgs/gitRepo/patches/0011-dumpjson-conditionally-exclude-empty-attributes.patch new file mode 100644 index 00000000..c3def6d7 --- /dev/null +++ b/pkgs/gitRepo/patches/0011-dumpjson-conditionally-exclude-empty-attributes.patch @@ -0,0 +1,55 @@ +From 0dde9323fa9480efcf68e2d5f0208e51a2e001c8 Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Wed, 15 Apr 2020 13:16:05 -0400 +Subject: [PATCH 11/16] dumpjson: conditionally exclude empty attributes + +--- + subcmds/dumpjson.py | 29 +++++++++++------------------ + 1 file changed, 11 insertions(+), 18 deletions(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index 0bd2708..176391c 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -89,27 +89,20 @@ class Dumpjson(Command, MirrorSafeCommand): + for t in threads: + t.join() + +- group_filter = lambda g: not (g == "all" or g.startswith("name:") or g.startswith("path:")) +- data = { +- p.name: { ++ data = {} ++ for p in all_projects: ++ data[p.name] = { + "url": p.remote.url, + "relpath": p.relpath, +- "groups": sorted(filter(group_filter, p.groups)), + "revisionExpr": p.revisionExpr, + "rev": p.rev, +- "linkfiles": [ +- { "src": l.src, +- "dest": l.dest, +- } +- for l in p.linkfiles +- ], +- "copyfiles": [ +- { "src": c.src, +- "dest": c.dest, +- } +- for c in p.copyfiles +- ], + } +- for p in all_projects +- }; ++ filtered_groups = filter(lambda g: not (g == "all" or g.startswith("name:") or g.startswith("path:")), p.groups) ++ if filtered_groups: ++ data[p.name]["groups"] = sorted(filtered_groups) ++ if p.linkfiles: ++ data[p.name]["linkfiles"] = [ { "src": l.src, "dest": l.dest } for l in p.linkfiles ] ++ if p.copyfiles: ++ data[p.name]["copyfiles"] = [ { "src": c.src, "dest": c.dest } for c in p.copyfiles ] ++ + print(json.dumps(data, sort_keys=True)) +-- +2.44.0 + diff --git a/pkgs/gitRepo/patches/0012-dumpjson-schema-change-to-use-relpath-as-keys.patch b/pkgs/gitRepo/patches/0012-dumpjson-schema-change-to-use-relpath-as-keys.patch new file mode 100644 index 00000000..12aa0081 --- /dev/null +++ b/pkgs/gitRepo/patches/0012-dumpjson-schema-change-to-use-relpath-as-keys.patch @@ -0,0 +1,39 @@ +From 5af943e5b9378220db65d5a2acc1f4b6f6fe409d Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Thu, 16 Apr 2020 14:15:52 -0400 +Subject: [PATCH 12/16] dumpjson: schema change to use relpath as keys + +--- + subcmds/dumpjson.py | 9 ++++----- + 1 file changed, 4 insertions(+), 5 deletions(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index 176391c..2105002 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -91,18 +91,17 @@ class Dumpjson(Command, MirrorSafeCommand): + + data = {} + for p in all_projects: +- data[p.name] = { ++ data[p.relpath] = { + "url": p.remote.url, +- "relpath": p.relpath, + "revisionExpr": p.revisionExpr, + "rev": p.rev, + } + filtered_groups = filter(lambda g: not (g == "all" or g.startswith("name:") or g.startswith("path:")), p.groups) + if filtered_groups: +- data[p.name]["groups"] = sorted(filtered_groups) ++ data[p.relpath]["groups"] = sorted(filtered_groups) + if p.linkfiles: +- data[p.name]["linkfiles"] = [ { "src": l.src, "dest": l.dest } for l in p.linkfiles ] ++ data[p.relpath]["linkfiles"] = [ { "src": l.src, "dest": l.dest } for l in p.linkfiles ] + if p.copyfiles: +- data[p.name]["copyfiles"] = [ { "src": c.src, "dest": c.dest } for c in p.copyfiles ] ++ data[p.relpath]["copyfiles"] = [ { "src": c.src, "dest": c.dest } for c in p.copyfiles ] + + print(json.dumps(data, sort_keys=True)) +-- +2.44.0 + diff --git a/pkgs/gitRepo/patches/0013-dumpjson-use-j-to-set-number-of-threads.patch b/pkgs/gitRepo/patches/0013-dumpjson-use-j-to-set-number-of-threads.patch new file mode 100644 index 00000000..863a683d --- /dev/null +++ b/pkgs/gitRepo/patches/0013-dumpjson-use-j-to-set-number-of-threads.patch @@ -0,0 +1,35 @@ +From 3edd5009f6f0d2b6aa64b81246ffeb1bfb7405d6 Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Sat, 11 Jul 2020 11:08:14 -0700 +Subject: [PATCH 13/16] dumpjson: use -j to set number of threads + +--- + subcmds/dumpjson.py | 9 ++++++--- + 1 file changed, 6 insertions(+), 3 deletions(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index 2105002..4648e10 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -77,12 +77,15 @@ class Dumpjson(Command, MirrorSafeCommand): + helpDescription = """ + """ + ++ def _Options(self, p): ++ p.add_option('-j', '--jobs', ++ dest='jobs', action='store', type='int', default=8, ++ help="number of projects to check simultaneously") ++ + def Execute(self, opt, args): + all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=False) + +- MAX_THREADS = 8 +- sem = _threading.Semaphore(MAX_THREADS) +- ++ sem = _threading.Semaphore(opt.jobs) + threads = [ _threading.Thread(target=_fetch_revs, args=(p, sem)) for p in all_projects ] + for t in threads: + t.start() +-- +2.44.0 + diff --git a/pkgs/gitRepo/patches/0014-dumpjson-add-a-local_only-option-for-offline-process.patch b/pkgs/gitRepo/patches/0014-dumpjson-add-a-local_only-option-for-offline-process.patch new file mode 100644 index 00000000..8b7d8fd0 --- /dev/null +++ b/pkgs/gitRepo/patches/0014-dumpjson-add-a-local_only-option-for-offline-process.patch @@ -0,0 +1,73 @@ +From 394b8dd50c263b18561386a85bf5294fda1fa5a2 Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Sat, 11 Jul 2020 11:25:38 -0700 +Subject: [PATCH 14/16] dumpjson: add a "local_only" option for offline + processing + +--- + subcmds/dumpjson.py | 38 ++++++++++++++++++++++++-------------- + 1 file changed, 24 insertions(+), 14 deletions(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index 4648e10..bbb4245 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -59,14 +59,9 @@ import re + from command import Command, MirrorSafeCommand + + def _fetch_revs(p, sem): +- if re.match("[0-9a-f]{40}", p.revisionExpr): +- # Use revisionExpr if it is already a SHA1 hash +- p.rev = p.revisionExpr +- else: +- # Otherwise we need to grab the hash from the remote source +- with sem: +- p.rev = p._LsRemote(p.revisionExpr).split('\t')[0] +- assert p.rev != "" ++ with sem: ++ p.rev = p._LsRemote(p.revisionExpr).split('\t')[0] ++ assert p.rev != "" + + class Dumpjson(Command, MirrorSafeCommand): + common = True +@@ -81,16 +76,31 @@ class Dumpjson(Command, MirrorSafeCommand): + p.add_option('-j', '--jobs', + dest='jobs', action='store', type='int', default=8, + help="number of projects to check simultaneously") ++ p.add_option('-l', '--local-only', ++ dest='local_only', action='store_true', ++ help="don't fetch project revisions even if they are missing") + + def Execute(self, opt, args): + all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=False) + +- sem = _threading.Semaphore(opt.jobs) +- threads = [ _threading.Thread(target=_fetch_revs, args=(p, sem)) for p in all_projects ] +- for t in threads: +- t.start() +- for t in threads: +- t.join() ++ # Fill out rev if we already have the information available ++ to_fetch = [] ++ for p in all_projects: ++ if re.match("[0-9a-f]{40}", p.revisionExpr): ++ # Use revisionExpr if it is already a SHA1 hash ++ p.rev = p.revisionExpr ++ else: ++ p.rev = None ++ to_fetch.append(p) ++ ++ if not opt.local_only: ++ # Fetch rev for projects we don't know yet ++ sem = _threading.Semaphore(opt.jobs) ++ threads = [ _threading.Thread(target=_fetch_revs, args=(p, sem)) for p in to_fetch ] ++ for t in threads: ++ t.start() ++ for t in threads: ++ t.join() + + data = {} + for p in all_projects: +-- +2.44.0 + diff --git a/pkgs/gitRepo/patches/0015-dumpjson-no-longers-output-rev-attribute.patch b/pkgs/gitRepo/patches/0015-dumpjson-no-longers-output-rev-attribute.patch new file mode 100644 index 00000000..8e044eb6 --- /dev/null +++ b/pkgs/gitRepo/patches/0015-dumpjson-no-longers-output-rev-attribute.patch @@ -0,0 +1,72 @@ +From 8224ebd83cfecde0736f5905889188cf652cff24 Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Wed, 3 Feb 2021 18:56:31 -0800 +Subject: [PATCH 15/16] dumpjson: no longers output rev attribute + +Moved logic into robotnix mk-repo-file.py script +--- + subcmds/dumpjson.py | 32 +------------------------------- + 1 file changed, 1 insertion(+), 31 deletions(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index bbb4245..870e269 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -58,11 +58,6 @@ import re + + from command import Command, MirrorSafeCommand + +-def _fetch_revs(p, sem): +- with sem: +- p.rev = p._LsRemote(p.revisionExpr).split('\t')[0] +- assert p.rev != "" +- + class Dumpjson(Command, MirrorSafeCommand): + common = True + helpSummary = "Export json file with sources" +@@ -73,41 +68,16 @@ class Dumpjson(Command, MirrorSafeCommand): + """ + + def _Options(self, p): +- p.add_option('-j', '--jobs', +- dest='jobs', action='store', type='int', default=8, +- help="number of projects to check simultaneously") +- p.add_option('-l', '--local-only', +- dest='local_only', action='store_true', +- help="don't fetch project revisions even if they are missing") ++ pass + + def Execute(self, opt, args): + all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=False) + +- # Fill out rev if we already have the information available +- to_fetch = [] +- for p in all_projects: +- if re.match("[0-9a-f]{40}", p.revisionExpr): +- # Use revisionExpr if it is already a SHA1 hash +- p.rev = p.revisionExpr +- else: +- p.rev = None +- to_fetch.append(p) +- +- if not opt.local_only: +- # Fetch rev for projects we don't know yet +- sem = _threading.Semaphore(opt.jobs) +- threads = [ _threading.Thread(target=_fetch_revs, args=(p, sem)) for p in to_fetch ] +- for t in threads: +- t.start() +- for t in threads: +- t.join() +- + data = {} + for p in all_projects: + data[p.relpath] = { + "url": p.remote.url, + "revisionExpr": p.revisionExpr, +- "rev": p.rev, + } + filtered_groups = filter(lambda g: not (g == "all" or g.startswith("name:") or g.startswith("path:")), p.groups) + if filtered_groups: +-- +2.44.0 + diff --git a/pkgs/gitRepo/patches/0016-Remove-python3-conditionals.patch b/pkgs/gitRepo/patches/0016-Remove-python3-conditionals.patch new file mode 100644 index 00000000..5b7b30e0 --- /dev/null +++ b/pkgs/gitRepo/patches/0016-Remove-python3-conditionals.patch @@ -0,0 +1,81 @@ +From dca531f6d6e9fdcf00aa9d18f0153bd66a2e32ea Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Tomek=20Ma=C5=84ko?= +Date: Fri, 24 May 2024 21:35:57 +0200 +Subject: [PATCH 16/16] Remove python3 conditionals + +--- + subcmds/dumpjson.py | 23 +++++------------------ + subcmds/nix.py | 23 +++++------------------ + 2 files changed, 10 insertions(+), 36 deletions(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index 870e269..7dfc465 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -15,24 +15,11 @@ + + from __future__ import print_function + +-from pyversion import is_python3 +-if is_python3(): +- import http.cookiejar as cookielib +- import urllib.error +- import urllib.parse +- import urllib.request +- import xmlrpc.client +-else: +- import imp +- import urllib2 +- import urlparse +- import xmlrpclib +- urllib = imp.new_module('urllib') +- urllib.error = urllib2 +- urllib.parse = urlparse +- urllib.request = urllib2 +- xmlrpc = imp.new_module('xmlrpc') +- xmlrpc.client = xmlrpclib ++import http.cookiejar as cookielib ++import urllib.error ++import urllib.parse ++import urllib.request ++import xmlrpc.client + + try: + import threading as _threading +diff --git a/subcmds/nix.py b/subcmds/nix.py +index ad1f19f..51be288 100644 +--- a/subcmds/nix.py ++++ b/subcmds/nix.py +@@ -15,24 +15,11 @@ + + from __future__ import print_function + +-from pyversion import is_python3 +-if is_python3(): +- import http.cookiejar as cookielib +- import urllib.error +- import urllib.parse +- import urllib.request +- import xmlrpc.client +-else: +- import imp +- import urllib2 +- import urlparse +- import xmlrpclib +- urllib = imp.new_module('urllib') +- urllib.error = urllib2 +- urllib.parse = urlparse +- urllib.request = urllib2 +- xmlrpc = imp.new_module('xmlrpc') +- xmlrpc.client = xmlrpclib ++import http.cookiejar as cookielib ++import urllib.error ++import urllib.parse ++import urllib.request ++import xmlrpc.client + + try: + import threading as _threading +-- +2.44.0 + diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix index c2aba7ad..d7db736d 100644 --- a/pkgs/overlay.nix +++ b/pkgs/overlay.nix @@ -1,3 +1,4 @@ +{ inputs }: self: super: { android-emulator = super.callPackage ./android-emulator {}; @@ -33,6 +34,8 @@ self: super: { }); nix-prefetch-git = super.callPackage ./fetchgit/nix-prefetch-git.nix {}; + gitRepo = super.callPackage ./gitRepo { inherit inputs; }; + ### # Robotnix helper derivations diff --git a/scripts/mk_repo_file.py b/scripts/mk_repo_file.py index 3f9813ba..4f12b619 100755 --- a/scripts/mk_repo_file.py +++ b/scripts/mk_repo_file.py @@ -21,8 +21,6 @@ REPO_FLAGS = [ "--quiet", - "--repo-url=https://github.com/danielfullmer/tools_repo", - "--repo-rev=9ecb9713ee5adba95120acbc0bfef1c77b02637f", "--no-repo-verify", "--depth=1", ] diff --git a/scripts/robotnix_common.py b/scripts/robotnix_common.py index 5b0d44d1..8f612b41 100644 --- a/scripts/robotnix_common.py +++ b/scripts/robotnix_common.py @@ -7,6 +7,7 @@ import os import subprocess import sys +from pathlib import Path ROBOTNIX_GIT_MIRRORS = os.environ.get('ROBOTNIX_GIT_MIRRORS', '') @@ -31,10 +32,20 @@ def save(filename: str, data: Any) -> None: def get_store_path(path): """Get actual path to a Nix store path; supports handling local remotes""" - prefix = os.getenv('NIX_REMOTE'); - if prefix and not prefix.startswith('/'): - raise Exception('Must be run on a local Nix store.') - return f"{prefix}/{path}" + prefix = os.getenv("NIX_REMOTE") + + if not prefix: + return path + + prefix = Path(prefix) + + if not prefix.is_absolute(): + raise Exception(f"Must be run on a local Nix store. Current Nix store: {prefix}") + + path = Path(path).resolve() + remote_path = prefix.resolve().joinpath(path.relative_to(f"{path.drive}{path.root}")) + + return str(remote_path) class GitCheckoutInfoDict(TypedDict): """Container for output from nix-prefetch-git""" diff --git a/scripts/test_mk_repo_file.py b/scripts/test_mk_repo_file.py index a1917892..f08d9b46 100644 --- a/scripts/test_mk_repo_file.py +++ b/scripts/test_mk_repo_file.py @@ -13,11 +13,11 @@ import mk_repo_file -def git_create(directory: str, tag: Optional[str] = "release") -> None: +def git_create(directory: str, tag: Optional[str] = "release", initial_branch: str = "main") -> None: """Turn a directory into a git repo""" cwd = os.getcwd() os.chdir(directory) - subprocess.check_call(["git", "init", "--initial-branch=master"]) + subprocess.check_call(["git", "init", f"--initial-branch={initial_branch}"]) subprocess.check_call(["git", "config", "--local", "user.name", "testenv"]) subprocess.check_call(["git", "config", "--local", "user.email", "testenv@example.com"]) subprocess.check_call(["git", "add", "."]) diff --git a/shell.nix b/shell.nix index f40c3c6f..80b0871c 100644 --- a/shell.nix +++ b/shell.nix @@ -1,7 +1,2 @@ -(import ( - fetchTarball { - url = "https://github.com/edolstra/flake-compat/archive/12c64ca55c1014cdc1b16ed5a804aa8576601ff2.tar.gz"; - sha256 = "0jm6nzb83wa6ai17ly9fzpqc40wg1viib8klq8lby54agpl213w5"; } -) { - src = ./.; -}).shellNix.default +{ system ? builtins.currentSystem }: + (import ./flake/compat.nix { inherit system; }).shellNix