diff --git a/flake.lock b/flake.lock index edb64697..a60fc2f3 100644 --- a/flake.lock +++ b/flake.lock @@ -43,6 +43,21 @@ "type": "github" } }, + "flake-compat": { + "locked": { + "lastModified": 1688025799, + "narHash": "sha256-ktpB4dRtnksm9F5WawoIkEneh1nrEvuxb5lJFt1iOyw=", + "owner": "nix-community", + "repo": "flake-compat", + "rev": "8bf105319d44f6b9f0d764efa4fdef9f1cc9ba1c", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "flake-compat", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -95,6 +110,22 @@ "type": "github" } }, + "nixpkgs-unstable": { + "locked": { + "lastModified": 1716330097, + "narHash": "sha256-8BO3B7e3BiyIDsaKA0tY8O88rClYRTjvAp66y+VBUeU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "5710852ba686cc1fd0d3b8e22b3117d43ba374c2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs_2": { "locked": { "lastModified": 1711668574, @@ -114,7 +145,9 @@ "root": { "inputs": { "androidPkgs": "androidPkgs", - "nixpkgs": "nixpkgs_2" + "flake-compat": "flake-compat", + "nixpkgs": "nixpkgs_2", + "nixpkgs-unstable": "nixpkgs-unstable" } }, "systems": { diff --git a/flake.nix b/flake.nix index 368b9b48..84b2676e 100644 --- a/flake.nix +++ b/flake.nix @@ -3,10 +3,14 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11"; + nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixos-unstable"; + androidPkgs.url = "github:tadfisher/android-nixpkgs/stable"; + + flake-compat.url = "github:nix-community/flake-compat"; }; - outputs = { self, nixpkgs, androidPkgs, ... }@inputs: let + outputs = { self, nixpkgs, nixpkgs-unstable, androidPkgs, flake-compat, ... }@inputs: let pkgs = import ./pkgs/default.nix { inherit inputs; }; in { # robotnixSystem evaluates a robotnix configuration @@ -24,6 +28,7 @@ packages.x86_64-linux = { manual = (import ./docs { inherit pkgs; }).manual; + gitRepo = pkgs.gitRepo; }; devShell.x86_64-linux = pkgs.mkShell { diff --git a/flake/compat.nix b/flake/compat.nix new file mode 100644 index 00000000..c6fec7b2 --- /dev/null +++ b/flake/compat.nix @@ -0,0 +1,17 @@ +{ system ? builtins.currentSystem }: + let + lock = builtins.fromJSON (builtins.readFile ./../flake.lock); + flake-compat-entry = lock.nodes.root.inputs.flake-compat; + + inherit (lock.nodes."${ flake-compat-entry }".locked) owner repo narHash; + + flake-compat = builtins.fetchTarball { + url = "https://github.com/${ owner }/${ repo }/archive/${ lock.nodes.flake-compat.locked.rev }.tar.gz"; + sha256 = narHash; + }; + in + import flake-compat { + inherit system; + + src = ./..; + } diff --git a/pkgs/default.nix b/pkgs/default.nix index 9851c663..906880a7 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -1,13 +1,7 @@ # SPDX-FileCopyrightText: 2020 Daniel Fullmer and robotnix contributors # SPDX-License-Identifier: MIT -{ inputs ? (import ( - fetchTarball { - url = "https://github.com/edolstra/flake-compat/archive/12c64ca55c1014cdc1b16ed5a804aa8576601ff2.tar.gz"; - sha256 = "0jm6nzb83wa6ai17ly9fzpqc40wg1viib8klq8lby54agpl213w5"; } - ) { - src = ../.; - }).defaultNix.inputs, +{ inputs ? (import ../flake/compat.nix).defaultNix.inputs, ... }@args: let @@ -17,5 +11,5 @@ in nixpkgs.legacyPackages.x86_64-linux.appendOverlays [ androidPkgs.packages = androidPkgs.packages.x86_64-linux; androidPkgs.sdk = androidPkgs.sdk.x86_64-linux; }) - (import ./overlay.nix) + (import ./overlay.nix { inherit inputs; }) ] diff --git a/pkgs/gitRepo/00001-add-support-for-repo2nix.patch b/pkgs/gitRepo/00001-add-support-for-repo2nix.patch new file mode 100644 index 00000000..bea2fa94 --- /dev/null +++ b/pkgs/gitRepo/00001-add-support-for-repo2nix.patch @@ -0,0 +1,882 @@ +From 8fe8a2efaa6bcd404c256a29d1dc5ed4ae241a08 Mon Sep 17 00:00:00 2001 +From: ajs124 +Date: Tue, 26 Feb 2019 04:48:13 +0100 +Subject: [PATCH 01/16] add initial (hacky) version of "repo2nix" + +now with less impurities +--- + project.py | 4 +- + subcmds/nix.py | 100 +++++++++++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 102 insertions(+), 2 deletions(-) + create mode 100644 subcmds/nix.py + +diff --git a/project.py b/project.py +index 1f5e4c3..c3c3384 100644 +--- a/project.py ++++ b/project.py +@@ -2887,8 +2887,8 @@ class Project: + ) + + def _LsRemote(self, refs): +- cmd = ["ls-remote", self.remote.name, refs] +- p = GitCommand(self, cmd, capture_stdout=True) ++ cmd = ["ls-remote", self.remote.url, refs] ++ p = GitCommand(self, cmd, cwd="/", capture_stdout=True) + if p.Wait() == 0: + return p.stdout + return None +diff --git a/subcmds/nix.py b/subcmds/nix.py +new file mode 100644 +index 0000000..f113ede +--- /dev/null ++++ b/subcmds/nix.py +@@ -0,0 +1,100 @@ ++# ++# Copyright (C) 2008 The Android Open Source Project ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++ ++from __future__ import print_function ++ ++from pyversion import is_python3 ++if is_python3(): ++ import http.cookiejar as cookielib ++ import urllib.error ++ import urllib.parse ++ import urllib.request ++ import xmlrpc.client ++else: ++ import imp ++ import urllib2 ++ import urlparse ++ import xmlrpclib ++ urllib = imp.new_module('urllib') ++ urllib.error = urllib2 ++ urllib.parse = urlparse ++ urllib.request = urllib2 ++ xmlrpc = imp.new_module('xmlrpc') ++ xmlrpc.client = xmlrpclib ++ ++try: ++ import threading as _threading ++except ImportError: ++ import dummy_threading as _threading ++ ++try: ++ import resource ++ def _rlimit_nofile(): ++ return resource.getrlimit(resource.RLIMIT_NOFILE) ++except ImportError: ++ def _rlimit_nofile(): ++ return (256, 256) ++ ++try: ++ import multiprocessing ++except ImportError: ++ multiprocessing = None ++ ++from command import Command, MirrorSafeCommand ++ ++class Nix(Command, MirrorSafeCommand): ++ common = True ++ helpSummary = "Export nix file with sources" ++ helpUsage = """ ++%prog [...] ++""" ++ helpDescription = """ ++""" ++ ++ def Execute(self, opt, args): ++ all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=False) ++ ++ oS = '{\n' ++ oS += "unpackPhase = ''\n" \ ++ 'echo "reassembling source tree from git source store paths"\n' \ ++ 'mkdir src; cd src\n' \ ++ 'for src in $srcs; do\n' \ ++ " dest_folder=$(stripHash $src); dest_folder=''${dest_folder//=//}\n" \ ++ ' echo "$src -> $dest_folder"\n' \ ++ ' mkdir -p "$dest_folder"\n' \ ++ ' cp --reflink=auto --no-preserve=ownership --no-dereference --preserve=links --recursive "$src/." "$dest_folder/"\n' \ ++ ' chmod -R u+w "$dest_folder"\n' \ ++ 'done\n' \ ++ 'echo "creating symlinks and copies as specified in repo manifest(s)"\n' ++ for p in all_projects: ++ for f in p.linkfiles: ++ oS += 'ln -s ' + f.src_rel_to_dest + ' ' + f.dest + '\n' ++ for c in p.copyfiles: ++ oS += 'cp --reflink=auto ' + p.relpath + '/' + c.src + ' ' + c.dest + '\n' ++ oS += "'';\n" ++ ++ oS += 'sources = [\n' ++ for p in all_projects: ++ oS += ' (builtins.fetchGit {\n' ++ oS += ' url = "' + p.remote.url + '";\n' ++ if 'refs/heads' in p.revisionExpr: ++ oS += ' ref = "' + p.revisionExpr.split('/')[-1] + '";\n' ++ else: ++ oS += ' ref = "' + p.revisionExpr + '";\n' ++ oS += ' rev = "' + p._LsRemote(p.revisionExpr).split('\t')[0] + '";\n' ++ oS += ' name = "' + p.relpath.replace('/', '=') + '";\n' ++ oS += ' })\n' ++ oS += '];\n}' ++ print(oS) +\ No newline at end of file +-- +2.44.0 + + +From 4550efa07636b497bfca0b8379c0656ed09770cb Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Mon, 15 Jul 2019 14:43:17 -0400 +Subject: [PATCH 02/16] repo2nix: output just json--do more processing in nix + +--- + subcmds/dumpjson.py | 92 +++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 92 insertions(+) + create mode 100644 subcmds/dumpjson.py + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +new file mode 100644 +index 0000000..a09885d +--- /dev/null ++++ b/subcmds/dumpjson.py +@@ -0,0 +1,92 @@ ++# ++# Copyright (C) 2008 The Android Open Source Project ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++ ++from __future__ import print_function ++ ++from pyversion import is_python3 ++if is_python3(): ++ import http.cookiejar as cookielib ++ import urllib.error ++ import urllib.parse ++ import urllib.request ++ import xmlrpc.client ++else: ++ import imp ++ import urllib2 ++ import urlparse ++ import xmlrpclib ++ urllib = imp.new_module('urllib') ++ urllib.error = urllib2 ++ urllib.parse = urlparse ++ urllib.request = urllib2 ++ xmlrpc = imp.new_module('xmlrpc') ++ xmlrpc.client = xmlrpclib ++ ++try: ++ import threading as _threading ++except ImportError: ++ import dummy_threading as _threading ++ ++try: ++ import resource ++ def _rlimit_nofile(): ++ return resource.getrlimit(resource.RLIMIT_NOFILE) ++except ImportError: ++ def _rlimit_nofile(): ++ return (256, 256) ++ ++try: ++ import multiprocessing ++except ImportError: ++ multiprocessing = None ++ ++from command import Command, MirrorSafeCommand ++ ++class Dumpjson(Command, MirrorSafeCommand): ++ common = True ++ helpSummary = "Export json file with sources" ++ helpUsage = """ ++%prog [...] ++""" ++ helpDescription = """ ++""" ++ ++ def Execute(self, opt, args): ++ all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=False) ++ ++ import json ++ data = { ++ p.name: { ++ "url": p.remote.url, ++ "relpath": p.relpath, ++ "groups": p.groups, ++ "revisionExpr": p.revisionExpr, ++ "rev": p._LsRemote(p.revisionExpr).split('\t')[0], ++ "linkfiles": [ ++ { "src_rel_to_dest": l.src_rel_to_dest, ++ "dest": l.dest, ++ } ++ for l in p.linkfiles ++ ], ++ "copyfiles": [ ++ { "src": c.src, ++ "dest": c.dest, ++ } ++ for c in p.copyfiles ++ ], ++ } ++ for p in all_projects ++ }; ++ print(json.dumps(data)) +-- +2.44.0 + + +From 333fc9d324640adb94e2ec7b51ab24268a3084d4 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Janne=20He=C3=9F?= +Date: Sun, 27 Oct 2019 11:31:40 +0100 +Subject: [PATCH 03/16] nix: Retry 20 times if the http request fails + +--- + subcmds/nix.py | 9 +++++++-- + 1 file changed, 7 insertions(+), 2 deletions(-) + +diff --git a/subcmds/nix.py b/subcmds/nix.py +index f113ede..fd7c188 100644 +--- a/subcmds/nix.py ++++ b/subcmds/nix.py +@@ -53,6 +53,7 @@ except ImportError: + multiprocessing = None + + from command import Command, MirrorSafeCommand ++from itertools import repeat + + class Nix(Command, MirrorSafeCommand): + common = True +@@ -93,8 +94,12 @@ class Nix(Command, MirrorSafeCommand): + oS += ' ref = "' + p.revisionExpr.split('/')[-1] + '";\n' + else: + oS += ' ref = "' + p.revisionExpr + '";\n' +- oS += ' rev = "' + p._LsRemote(p.revisionExpr).split('\t')[0] + '";\n' ++ while repeat(None, 20): ++ raw_rev = p._LsRemote(p.revisionExpr) ++ if raw_rev != None: ++ break ++ oS += ' rev = "' + raw_rev.split('\t')[0] + '";\n' + oS += ' name = "' + p.relpath.replace('/', '=') + '";\n' + oS += ' })\n' + oS += '];\n}' +- print(oS) +\ No newline at end of file ++ print(oS) +-- +2.44.0 + + +From 991502c597e3d3c9d5b25fcf08ecd216fdba107b Mon Sep 17 00:00:00 2001 +From: ajs124 +Date: Tue, 3 Dec 2019 20:23:01 +0100 +Subject: [PATCH 04/16] fml + +--- + subcmds/nix.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/subcmds/nix.py b/subcmds/nix.py +index fd7c188..ad1f19f 100644 +--- a/subcmds/nix.py ++++ b/subcmds/nix.py +@@ -98,7 +98,7 @@ class Nix(Command, MirrorSafeCommand): + raw_rev = p._LsRemote(p.revisionExpr) + if raw_rev != None: + break +- oS += ' rev = "' + raw_rev.split('\t')[0] + '";\n' ++ # oS += ' rev = "' + raw_rev.split('\t')[0] + '";\n' + oS += ' name = "' + p.relpath.replace('/', '=') + '";\n' + oS += ' })\n' + oS += '];\n}' +-- +2.44.0 + + +From 07d83a29279874566234b2a3aed6fc270e5f9b8b Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Tue, 3 Dec 2019 16:56:27 -0500 +Subject: [PATCH 05/16] dumpjson: Sort json output by keys for reproducibility + +--- + subcmds/dumpjson.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index a09885d..8997c27 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -89,4 +89,4 @@ class Dumpjson(Command, MirrorSafeCommand): + } + for p in all_projects + }; +- print(json.dumps(data)) ++ print(json.dumps(data, sort_keys=True)) +-- +2.44.0 + + +From 40465bd1ec05fa648765a0bc5d3ed461ae8ce2e2 Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Sat, 7 Dec 2019 12:15:55 -0500 +Subject: [PATCH 06/16] dumpjson: parallelize + +--- + subcmds/dumpjson.py | 19 +++++++++++++++++-- + 1 file changed, 17 insertions(+), 2 deletions(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index 8997c27..18aca4c 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -52,8 +52,15 @@ try: + except ImportError: + multiprocessing = None + ++import sys ++import json ++ + from command import Command, MirrorSafeCommand + ++def _fetch_revs(p, sem): ++ with sem: ++ p.rev = p._LsRemote(p.revisionExpr).split('\t')[0] ++ + class Dumpjson(Command, MirrorSafeCommand): + common = True + helpSummary = "Export json file with sources" +@@ -66,14 +73,22 @@ class Dumpjson(Command, MirrorSafeCommand): + def Execute(self, opt, args): + all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=False) + +- import json ++ MAX_THREADS = 8 ++ sem = _threading.Semaphore(MAX_THREADS) ++ ++ threads = [ _threading.Thread(target=_fetch_revs, args=(p, sem)) for p in all_projects ] ++ for t in threads: ++ t.start() ++ for t in threads: ++ t.join() ++ + data = { + p.name: { + "url": p.remote.url, + "relpath": p.relpath, + "groups": p.groups, + "revisionExpr": p.revisionExpr, +- "rev": p._LsRemote(p.revisionExpr).split('\t')[0], ++ "rev": p.rev, + "linkfiles": [ + { "src_rel_to_dest": l.src_rel_to_dest, + "dest": l.dest, +-- +2.44.0 + + +From 92f28fed56d9195cf723aef22b3ce00b47742ed1 Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Sun, 8 Dec 2019 14:21:32 -0500 +Subject: [PATCH 07/16] dumpjson: Sort groups list + +--- + subcmds/dumpjson.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index 18aca4c..a1da03c 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -86,7 +86,7 @@ class Dumpjson(Command, MirrorSafeCommand): + p.name: { + "url": p.remote.url, + "relpath": p.relpath, +- "groups": p.groups, ++ "groups": sorted(p.groups), + "revisionExpr": p.revisionExpr, + "rev": p.rev, + "linkfiles": [ +-- +2.44.0 + + +From 35c3b4c59382e7db182b4eedfacba55c762295e5 Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Tue, 14 Apr 2020 19:47:47 -0400 +Subject: [PATCH 08/16] dumpjson: fix explicitly referencing a revision + +GrapheneOS does this as of QQ2A.200405.005.2020.04.13.21 +--- + subcmds/dumpjson.py | 11 +++++++++-- + 1 file changed, 9 insertions(+), 2 deletions(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index a1da03c..8ca9d08 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -54,12 +54,19 @@ except ImportError: + + import sys + import json ++import re + + from command import Command, MirrorSafeCommand + + def _fetch_revs(p, sem): +- with sem: +- p.rev = p._LsRemote(p.revisionExpr).split('\t')[0] ++ if re.match("[0-9a-f]{40}", p.revisionExpr): ++ # Use revisionExpr if it is already a SHA1 hash ++ p.rev = p.revisionExpr ++ else: ++ # Otherwise we need to grab the hash from the remote source ++ with sem: ++ p.rev = p._LsRemote(p.revisionExpr).split('\t')[0] ++ assert p.rev != "" + + class Dumpjson(Command, MirrorSafeCommand): + common = True +-- +2.44.0 + + +From 35c373392754b6e79f1ef20a395b97d6a736ba92 Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Wed, 15 Apr 2020 13:05:14 -0400 +Subject: [PATCH 09/16] dumpjson: fix removed LinkFile attribute + +This unfortunately also changes the output schema for dumpjson. +--- + subcmds/dumpjson.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index 8ca9d08..726ccfd 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -97,7 +97,7 @@ class Dumpjson(Command, MirrorSafeCommand): + "revisionExpr": p.revisionExpr, + "rev": p.rev, + "linkfiles": [ +- { "src_rel_to_dest": l.src_rel_to_dest, ++ { "src": l.src, + "dest": l.dest, + } + for l in p.linkfiles +-- +2.44.0 + + +From 71a828d0123b6fe16c1ca5a869c49d55c8dc275b Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Wed, 15 Apr 2020 13:10:13 -0400 +Subject: [PATCH 10/16] dumpjson: filter out groups with redundant information + +--- + subcmds/dumpjson.py | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index 726ccfd..0bd2708 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -89,11 +89,12 @@ class Dumpjson(Command, MirrorSafeCommand): + for t in threads: + t.join() + ++ group_filter = lambda g: not (g == "all" or g.startswith("name:") or g.startswith("path:")) + data = { + p.name: { + "url": p.remote.url, + "relpath": p.relpath, +- "groups": sorted(p.groups), ++ "groups": sorted(filter(group_filter, p.groups)), + "revisionExpr": p.revisionExpr, + "rev": p.rev, + "linkfiles": [ +-- +2.44.0 + + +From 0dde9323fa9480efcf68e2d5f0208e51a2e001c8 Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Wed, 15 Apr 2020 13:16:05 -0400 +Subject: [PATCH 11/16] dumpjson: conditionally exclude empty attributes + +--- + subcmds/dumpjson.py | 29 +++++++++++------------------ + 1 file changed, 11 insertions(+), 18 deletions(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index 0bd2708..176391c 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -89,27 +89,20 @@ class Dumpjson(Command, MirrorSafeCommand): + for t in threads: + t.join() + +- group_filter = lambda g: not (g == "all" or g.startswith("name:") or g.startswith("path:")) +- data = { +- p.name: { ++ data = {} ++ for p in all_projects: ++ data[p.name] = { + "url": p.remote.url, + "relpath": p.relpath, +- "groups": sorted(filter(group_filter, p.groups)), + "revisionExpr": p.revisionExpr, + "rev": p.rev, +- "linkfiles": [ +- { "src": l.src, +- "dest": l.dest, +- } +- for l in p.linkfiles +- ], +- "copyfiles": [ +- { "src": c.src, +- "dest": c.dest, +- } +- for c in p.copyfiles +- ], + } +- for p in all_projects +- }; ++ filtered_groups = filter(lambda g: not (g == "all" or g.startswith("name:") or g.startswith("path:")), p.groups) ++ if filtered_groups: ++ data[p.name]["groups"] = sorted(filtered_groups) ++ if p.linkfiles: ++ data[p.name]["linkfiles"] = [ { "src": l.src, "dest": l.dest } for l in p.linkfiles ] ++ if p.copyfiles: ++ data[p.name]["copyfiles"] = [ { "src": c.src, "dest": c.dest } for c in p.copyfiles ] ++ + print(json.dumps(data, sort_keys=True)) +-- +2.44.0 + + +From 5af943e5b9378220db65d5a2acc1f4b6f6fe409d Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Thu, 16 Apr 2020 14:15:52 -0400 +Subject: [PATCH 12/16] dumpjson: schema change to use relpath as keys + +--- + subcmds/dumpjson.py | 9 ++++----- + 1 file changed, 4 insertions(+), 5 deletions(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index 176391c..2105002 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -91,18 +91,17 @@ class Dumpjson(Command, MirrorSafeCommand): + + data = {} + for p in all_projects: +- data[p.name] = { ++ data[p.relpath] = { + "url": p.remote.url, +- "relpath": p.relpath, + "revisionExpr": p.revisionExpr, + "rev": p.rev, + } + filtered_groups = filter(lambda g: not (g == "all" or g.startswith("name:") or g.startswith("path:")), p.groups) + if filtered_groups: +- data[p.name]["groups"] = sorted(filtered_groups) ++ data[p.relpath]["groups"] = sorted(filtered_groups) + if p.linkfiles: +- data[p.name]["linkfiles"] = [ { "src": l.src, "dest": l.dest } for l in p.linkfiles ] ++ data[p.relpath]["linkfiles"] = [ { "src": l.src, "dest": l.dest } for l in p.linkfiles ] + if p.copyfiles: +- data[p.name]["copyfiles"] = [ { "src": c.src, "dest": c.dest } for c in p.copyfiles ] ++ data[p.relpath]["copyfiles"] = [ { "src": c.src, "dest": c.dest } for c in p.copyfiles ] + + print(json.dumps(data, sort_keys=True)) +-- +2.44.0 + + +From 3edd5009f6f0d2b6aa64b81246ffeb1bfb7405d6 Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Sat, 11 Jul 2020 11:08:14 -0700 +Subject: [PATCH 13/16] dumpjson: use -j to set number of threads + +--- + subcmds/dumpjson.py | 9 ++++++--- + 1 file changed, 6 insertions(+), 3 deletions(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index 2105002..4648e10 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -77,12 +77,15 @@ class Dumpjson(Command, MirrorSafeCommand): + helpDescription = """ + """ + ++ def _Options(self, p): ++ p.add_option('-j', '--jobs', ++ dest='jobs', action='store', type='int', default=8, ++ help="number of projects to check simultaneously") ++ + def Execute(self, opt, args): + all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=False) + +- MAX_THREADS = 8 +- sem = _threading.Semaphore(MAX_THREADS) +- ++ sem = _threading.Semaphore(opt.jobs) + threads = [ _threading.Thread(target=_fetch_revs, args=(p, sem)) for p in all_projects ] + for t in threads: + t.start() +-- +2.44.0 + + +From 394b8dd50c263b18561386a85bf5294fda1fa5a2 Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Sat, 11 Jul 2020 11:25:38 -0700 +Subject: [PATCH 14/16] dumpjson: add a "local_only" option for offline + processing + +--- + subcmds/dumpjson.py | 38 ++++++++++++++++++++++++-------------- + 1 file changed, 24 insertions(+), 14 deletions(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index 4648e10..bbb4245 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -59,14 +59,9 @@ import re + from command import Command, MirrorSafeCommand + + def _fetch_revs(p, sem): +- if re.match("[0-9a-f]{40}", p.revisionExpr): +- # Use revisionExpr if it is already a SHA1 hash +- p.rev = p.revisionExpr +- else: +- # Otherwise we need to grab the hash from the remote source +- with sem: +- p.rev = p._LsRemote(p.revisionExpr).split('\t')[0] +- assert p.rev != "" ++ with sem: ++ p.rev = p._LsRemote(p.revisionExpr).split('\t')[0] ++ assert p.rev != "" + + class Dumpjson(Command, MirrorSafeCommand): + common = True +@@ -81,16 +76,31 @@ class Dumpjson(Command, MirrorSafeCommand): + p.add_option('-j', '--jobs', + dest='jobs', action='store', type='int', default=8, + help="number of projects to check simultaneously") ++ p.add_option('-l', '--local-only', ++ dest='local_only', action='store_true', ++ help="don't fetch project revisions even if they are missing") + + def Execute(self, opt, args): + all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=False) + +- sem = _threading.Semaphore(opt.jobs) +- threads = [ _threading.Thread(target=_fetch_revs, args=(p, sem)) for p in all_projects ] +- for t in threads: +- t.start() +- for t in threads: +- t.join() ++ # Fill out rev if we already have the information available ++ to_fetch = [] ++ for p in all_projects: ++ if re.match("[0-9a-f]{40}", p.revisionExpr): ++ # Use revisionExpr if it is already a SHA1 hash ++ p.rev = p.revisionExpr ++ else: ++ p.rev = None ++ to_fetch.append(p) ++ ++ if not opt.local_only: ++ # Fetch rev for projects we don't know yet ++ sem = _threading.Semaphore(opt.jobs) ++ threads = [ _threading.Thread(target=_fetch_revs, args=(p, sem)) for p in to_fetch ] ++ for t in threads: ++ t.start() ++ for t in threads: ++ t.join() + + data = {} + for p in all_projects: +-- +2.44.0 + + +From 8224ebd83cfecde0736f5905889188cf652cff24 Mon Sep 17 00:00:00 2001 +From: Daniel Fullmer +Date: Wed, 3 Feb 2021 18:56:31 -0800 +Subject: [PATCH 15/16] dumpjson: no longers output rev attribute + +Moved logic into robotnix mk-repo-file.py script +--- + subcmds/dumpjson.py | 32 +------------------------------- + 1 file changed, 1 insertion(+), 31 deletions(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index bbb4245..870e269 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -58,11 +58,6 @@ import re + + from command import Command, MirrorSafeCommand + +-def _fetch_revs(p, sem): +- with sem: +- p.rev = p._LsRemote(p.revisionExpr).split('\t')[0] +- assert p.rev != "" +- + class Dumpjson(Command, MirrorSafeCommand): + common = True + helpSummary = "Export json file with sources" +@@ -73,41 +68,16 @@ class Dumpjson(Command, MirrorSafeCommand): + """ + + def _Options(self, p): +- p.add_option('-j', '--jobs', +- dest='jobs', action='store', type='int', default=8, +- help="number of projects to check simultaneously") +- p.add_option('-l', '--local-only', +- dest='local_only', action='store_true', +- help="don't fetch project revisions even if they are missing") ++ pass + + def Execute(self, opt, args): + all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=False) + +- # Fill out rev if we already have the information available +- to_fetch = [] +- for p in all_projects: +- if re.match("[0-9a-f]{40}", p.revisionExpr): +- # Use revisionExpr if it is already a SHA1 hash +- p.rev = p.revisionExpr +- else: +- p.rev = None +- to_fetch.append(p) +- +- if not opt.local_only: +- # Fetch rev for projects we don't know yet +- sem = _threading.Semaphore(opt.jobs) +- threads = [ _threading.Thread(target=_fetch_revs, args=(p, sem)) for p in to_fetch ] +- for t in threads: +- t.start() +- for t in threads: +- t.join() +- + data = {} + for p in all_projects: + data[p.relpath] = { + "url": p.remote.url, + "revisionExpr": p.revisionExpr, +- "rev": p.rev, + } + filtered_groups = filter(lambda g: not (g == "all" or g.startswith("name:") or g.startswith("path:")), p.groups) + if filtered_groups: +-- +2.44.0 + + +From dca531f6d6e9fdcf00aa9d18f0153bd66a2e32ea Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Tomek=20Ma=C5=84ko?= +Date: Fri, 24 May 2024 21:35:57 +0200 +Subject: [PATCH 16/16] Remove python3 conditionals + +--- + subcmds/dumpjson.py | 23 +++++------------------ + subcmds/nix.py | 23 +++++------------------ + 2 files changed, 10 insertions(+), 36 deletions(-) + +diff --git a/subcmds/dumpjson.py b/subcmds/dumpjson.py +index 870e269..7dfc465 100644 +--- a/subcmds/dumpjson.py ++++ b/subcmds/dumpjson.py +@@ -15,24 +15,11 @@ + + from __future__ import print_function + +-from pyversion import is_python3 +-if is_python3(): +- import http.cookiejar as cookielib +- import urllib.error +- import urllib.parse +- import urllib.request +- import xmlrpc.client +-else: +- import imp +- import urllib2 +- import urlparse +- import xmlrpclib +- urllib = imp.new_module('urllib') +- urllib.error = urllib2 +- urllib.parse = urlparse +- urllib.request = urllib2 +- xmlrpc = imp.new_module('xmlrpc') +- xmlrpc.client = xmlrpclib ++import http.cookiejar as cookielib ++import urllib.error ++import urllib.parse ++import urllib.request ++import xmlrpc.client + + try: + import threading as _threading +diff --git a/subcmds/nix.py b/subcmds/nix.py +index ad1f19f..51be288 100644 +--- a/subcmds/nix.py ++++ b/subcmds/nix.py +@@ -15,24 +15,11 @@ + + from __future__ import print_function + +-from pyversion import is_python3 +-if is_python3(): +- import http.cookiejar as cookielib +- import urllib.error +- import urllib.parse +- import urllib.request +- import xmlrpc.client +-else: +- import imp +- import urllib2 +- import urlparse +- import xmlrpclib +- urllib = imp.new_module('urllib') +- urllib.error = urllib2 +- urllib.parse = urlparse +- urllib.request = urllib2 +- xmlrpc = imp.new_module('xmlrpc') +- xmlrpc.client = xmlrpclib ++import http.cookiejar as cookielib ++import urllib.error ++import urllib.parse ++import urllib.request ++import xmlrpc.client + + try: + import threading as _threading +-- +2.44.0 + diff --git a/pkgs/gitRepo/default.nix b/pkgs/gitRepo/default.nix new file mode 100644 index 00000000..2a3f529e --- /dev/null +++ b/pkgs/gitRepo/default.nix @@ -0,0 +1,70 @@ +{ lib, inputs, fetchFromGitHub, rsync, git, gnupg, less, openssh, ... }: +let + inherit (inputs) nixpkgs-unstable; + + unstablePkgs = nixpkgs-unstable.legacyPackages.x86_64-linux; +in + unstablePkgs.gitRepo.overrideAttrs(oldAttrs: rec { + version = "2.45"; + + src = fetchFromGitHub { + owner = "android"; + repo = "tools_repo"; + rev = "v${ version }"; + hash = "sha256-f765TcOHL8wdPa9qSmGegofjCXx1tF/K5bRQnYQcYVc="; + }; + + nativeBuildInputs = (oldAttrs.nativeBuildInputs or []) ++ [ rsync git ]; + + # NOTE: why `git apply` instead of relying on `patches`? For some reason when + # using `patches` the source `rsync`ed into `var/repo` is missing those changes + installPhase = '' + runHook preInstall + + mkdir -p var/repo + rsync -a $src/ var/repo/ + + ( + export GIT_CONFIG_GLOBAL=$TMPDIR/.gitconfig + export GIT_CONFIG_NOSYSTEM=true + + cd var/repo + + git config --global --add safe.directory "$PWD" + git config --global user.email "nemo@nix" + git config --global user.name "Nemo Nix" + + chmod +w -R . + + git init + git add -A + git commit -m "Upstream sources" + + git apply ${ ./00001-add-support-for-repo2nix.patch } + git add -A + git commit -m "Patch with repo2nix" + + git log -n 1 --format="%H" > ../../COMMITED_REPO_REV + ) + + mkdir -p $out/var/repo + mkdir -p $out/bin + + rsync -a var/repo/ $out/var/repo/ + + # Copying instead of symlinking to the above directory is necessary, because otherwise + # running `repo init` fails, as I assume the script gets confused by being located in + # a git repo itself + cp repo $out/bin/repo + + runHook postInstall + ''; + + # Specify the patched checkout as the default version of repo + postFixup = '' + wrapProgram "$out/bin/repo" \ + --set REPO_URL "file://$out/var/repo" \ + --set REPO_REV "$(cat COMMITED_REPO_REV)" \ + --prefix PATH ":" "${ lib.makeBinPath [ git gnupg less openssh ] }" + ''; + }) diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix index c2aba7ad..d7db736d 100644 --- a/pkgs/overlay.nix +++ b/pkgs/overlay.nix @@ -1,3 +1,4 @@ +{ inputs }: self: super: { android-emulator = super.callPackage ./android-emulator {}; @@ -33,6 +34,8 @@ self: super: { }); nix-prefetch-git = super.callPackage ./fetchgit/nix-prefetch-git.nix {}; + gitRepo = super.callPackage ./gitRepo { inherit inputs; }; + ### # Robotnix helper derivations diff --git a/scripts/mk_repo_file.py b/scripts/mk_repo_file.py index 3f9813ba..4f12b619 100755 --- a/scripts/mk_repo_file.py +++ b/scripts/mk_repo_file.py @@ -21,8 +21,6 @@ REPO_FLAGS = [ "--quiet", - "--repo-url=https://github.com/danielfullmer/tools_repo", - "--repo-rev=9ecb9713ee5adba95120acbc0bfef1c77b02637f", "--no-repo-verify", "--depth=1", ] diff --git a/shell.nix b/shell.nix index f40c3c6f..80b0871c 100644 --- a/shell.nix +++ b/shell.nix @@ -1,7 +1,2 @@ -(import ( - fetchTarball { - url = "https://github.com/edolstra/flake-compat/archive/12c64ca55c1014cdc1b16ed5a804aa8576601ff2.tar.gz"; - sha256 = "0jm6nzb83wa6ai17ly9fzpqc40wg1viib8klq8lby54agpl213w5"; } -) { - src = ./.; -}).shellNix.default +{ system ? builtins.currentSystem }: + (import ./flake/compat.nix { inherit system; }).shellNix