Skip to content

Commit

Permalink
Switch to python-dxf for registry access
Browse files Browse the repository at this point in the history 
Retrieves tag list through an API call that's not rate limited
  • Loading branch information
@jthiltges
jthiltges committed Mar 31, 2023
1 parent 040d0ca commit 531b3eb
Show file tree
Hide file tree
Showing 4 changed files with 67 additions and 1,081 deletions.
6 changes: 5 additions & 1 deletion .github/actions/docker-images-verification/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,11 @@ FROM python

RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
RUN python get-pip.py
RUN pip install requests furl sqlitedict
RUN pip install \
furl \
requests \
python-dxf \
sqlitedict
# Copies your code file from your action repository to the filesystem path `/` of the container

COPY entrypoint.sh /entrypoint.sh
Expand Down
106 changes: 61 additions & 45 deletions cvmfs-singularity-sync
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,15 @@ import argparse
import os
import errno
import fnmatch
import functools
import glob
import json
import urllib.request, urllib.error, urllib.parse
import hashlib
import traceback
import subprocess
import dockerhub
from dxf import DXF
import furl
import cleanup
import sqlitedict
import glob
Expand Down Expand Up @@ -129,9 +132,6 @@ def main():
singularity_rootfs = '/cvmfs/singularity.opensciencegrid.org'
singularity_rootfs = os.path.abspath(singularity_rootfs)

# Does the registry require a token?
doauth = not args.notoken

# Do we have a docker image specified?
if not args.docker and not (args.filelist or args.filelist_path):
print("No docker image or file list specified..", file=sys.stderr)
Expand All @@ -141,9 +141,9 @@ def main():
if args.docker:
image = args.docker
if not args.dryrun:
return publish_image(image, singularity_rootfs, args.registry, doauth, manifest_cache)
return publish_image(image, singularity_rootfs, args.registry, manifest_cache)
else:
return verify_image(image, args.registry, doauth, manifest_cache)
return verify_image(image, args.registry)
else:
final_retval = 0
failed_images = []
Expand All @@ -162,7 +162,7 @@ def main():

if '*' in repo_tag: # Treat wildcards as a glob
try:
tag_names = get_tags(namespace, repo_name, registry=registry, auth=doauth)
tag_names = get_tags(namespace, repo_name, registry=registry)
except Exception as ex:
image = '%s/%s/%s' % (registry, namespace, repo_name)
print("Failed to get tags for image: {}".format(image))
Expand Down Expand Up @@ -190,7 +190,7 @@ def main():
for i in range(tries):
if not args.dryrun:
try:
retval = publish_image(image, singularity_rootfs, registry, doauth, manifest_cache)
retval = publish_image(image, singularity_rootfs, registry, manifest_cache)
except Exception as ex:
if i < tries -1:
print("Failed to publish image: {}".format(image))
Expand All @@ -201,7 +201,7 @@ def main():
print("Tried {} times ".format(tries) + "for image {}".format(image) + ", giving up")
else:
try:
retval = verify_image(image, registry, doauth, manifest_cache)
retval = verify_image(image, registry)
except Exception as ex:
if i < tries -1:
print("Failed to verify image: {}".format(image))
Expand Down Expand Up @@ -254,21 +254,56 @@ def start_txn(singularity_rootfs):
if oe.errno != errno.EEXIST:
raise


def get_tags(username, repo, registry=None, auth=None):
if registry != "registry.hub.docker.com":
if "://" not in registry:
registry = "https://%s" % registry
auth = DOCKER_CREDS.get(registry, {})
hub = dockerhub.DockerHub(url=registry, namespace=username, repo=repo, **auth)
# REGISTRY -------------------------------------------------
# Reuse dxf object if possible. A token can be reused for access to all tags.
@functools.lru_cache(maxsize=None)
def get_dxf(registry, repo):
return DXF(registry, repo, docker_auth)

def docker_auth(dxf, response):
'''DXF auth handler, using DOCKER_CREDS global'''
origin = furl.furl(response.url).origin
authvars = DOCKER_CREDS.get(origin, {})
dxf.authenticate(response=response, **authvars)

def get_tags(namespace, repo_name, registry='registry.hub.docker.com'):
'''Retrieve tag list. This API call is uncounted.'''
repo = namespace + '/' + repo_name
#dxf = DXF(registry, repo, docker_auth)
dxf = get_dxf(registry, repo)
return dxf.list_aliases()

def get_manifest(namespace, repo_name, repo_tag, cache={}, registry='registry.hub.docker.com'):
'''Retrieve Docker manifest. If uncached, this counts as an API call.'''
repo = namespace + '/' + repo_name
#dxf = DXF(registry, repo, docker_auth)
dxf = get_dxf(registry, repo)
digest = dxf_get_digest(dxf, repo_tag)

if digest in cache:
return cache[digest], digest
else:
auth = DOCKER_CREDS.get('https://registry.hub.docker.com', {})
hub = dockerhub.DockerHub(**auth)
tag_names = []
for tag in hub.tags(username, repo):
tag_names.append(tag['name'])
return tag_names
manifest = dxf.get_manifest(repo_tag)
cache[digest] = manifest
return manifest

def get_digest(namespace, repo_name, repo_tag, registry='registry.hub.docker.com'):
'''Retrieve docker-content-digest of the manifest blob. This API call is uncounted.'''
repo = namespace + '/' + repo_name
#dxf = DXF(registry, repo, docker_auth)
dxf = get_dxf(registry, repo)
return dxf_get_digest(dxf, repo_tag)

def dxf_get_digest(dxf, repo_tag):
# Harbor returns 404 on HEAD of /v2/{repo_name}/manifests/{repo_tag}
# without the ACCEPT header
headers = {
'ACCEPT': 'application/vnd.oci.image.manifest.v1+json',
}
ret = dxf._request('head', 'manifests/' + repo_tag, headers=headers)
return ret.headers['docker-content-digest']

# ----------------------------------------------------------
def publish_txn():
global _in_txn
if _in_txn:
Expand Down Expand Up @@ -356,18 +391,7 @@ def parse_image(image):

return registry, namespace, repo_name, repo_tag

def get_manifest(hub, namespace, repo_name, repo_tag, manifest_cache):
metadata = hub.manifest(namespace, repo_name, repo_tag, head=True)
digest = metadata.headers['docker-content-digest']

if digest in manifest_cache:
return manifest_cache[digest]
else:
manifest = hub.manifest(namespace, repo_name, repo_tag)
manifest_cache[digest] = manifest
return manifest

def publish_image(image, singularity_rootfs, registry, doauth, manifest_cache):
def publish_image(image, singularity_rootfs, registry, manifest_cache):

# Tell the user the namespace, repo name and tag
registry, namespace, repo_name, repo_tag = parse_image(image)
Expand All @@ -383,8 +407,7 @@ def publish_image(image, singularity_rootfs, registry, doauth, manifest_cache):
if "://" not in registry:
registry = "https://%s" % registry
auth = DOCKER_CREDS.get(registry, {})
hub = dockerhub.DockerHub(url=registry, namespace=namespace, repo=repo_name, **auth)
manifest = get_manifest(hub, namespace, repo_name, repo_tag, manifest_cache)
manifest = get_manifest(namespace, repo_name, repo_tag, registry=registry, cache=manifest_cache)

# Calculate a unique hash across all layers. We'll use that as the identifier
# for the final image.
Expand Down Expand Up @@ -459,7 +482,7 @@ def publish_image(image, singularity_rootfs, registry, doauth, manifest_cache):
# Publish CVMFS as necessary.
return publish_txn()

def verify_image(image, registry, doauth, manifest_cache):
def verify_image(image, registry):

# Tell the user the namespace, repo name and tag
registry, namespace, repo_name, repo_tag = parse_image(image)
Expand All @@ -468,16 +491,9 @@ def verify_image(image, registry, doauth, manifest_cache):
# IMAGE METADATA -------------------------------------------
# Use Docker Registry API (version 2.0) to get images ids, manifest

# Get an image manifest - has image ids to parse, and will be
# used later to get Cmd
# Prepend "https://" to the registry
if "://" not in registry:
registry = "https://%s" % registry
auth = DOCKER_CREDS.get(registry, {})
hub = dockerhub.DockerHub(url=registry, namespace=namespace, repo=repo_name, **auth)
retval = 0
try:
hub.manifest(namespace, repo_name, repo_tag, head=True)
get_digest(namespace, repo_name, repo_tag, registry=registry)
print(repo_name + ":" + repo_tag + " manifest found")
retval = 0
except Exception as ex:
Expand Down
Loading

0 comments on commit 531b3eb

Please sign in to comment.