Skip to content

Commit

Permalink
httptransport: accept OCI manifests for indexing
Browse files Browse the repository at this point in the history
Signed-off-by: Hank Donnay <[email protected]>
  • Loading branch information
hdonnay committed Feb 4, 2022
1 parent 45fae9f commit 49dfb01
Show file tree
Hide file tree
Showing 4 changed files with 323 additions and 9 deletions.
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ require (
github.com/klauspost/compress v1.13.6
github.com/ldelossa/responserecorder v1.0.2-0.20210711162258-40bec93a9325
github.com/mattn/go-sqlite3 v1.11.0 // indirect
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.2
github.com/prometheus/client_golang v1.9.0
github.com/quay/clair/config v1.0.0
github.com/quay/claircore v1.2.0
Expand Down
6 changes: 2 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -720,8 +720,9 @@ github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
Expand Down Expand Up @@ -922,7 +923,6 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo=
github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
Expand Down Expand Up @@ -1106,7 +1106,6 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
Expand Down Expand Up @@ -1317,7 +1316,6 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down
93 changes: 88 additions & 5 deletions httptransport/indexer_v1.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@ import (
"fmt"
"net/http"
"path"
"strings"
"time"

"github.com/ldelossa/responserecorder"
oci "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/prometheus/client_golang/prometheus"
"github.com/quay/claircore"
"github.com/quay/zlog"
Expand Down Expand Up @@ -90,8 +92,8 @@ func (h *IndexerV1) indexReport(w http.ResponseWriter, r *http.Request) {
apiError(w, http.StatusInternalServerError, "could not retrieve indexer state: %v", err)
return
}
var m claircore.Manifest
if err := dec.Decode(&m); err != nil {
m, err := decodeManifest(ctx, r, dec)
if err != nil {
apiError(w, http.StatusBadRequest, "failed to deserialize manifest: %v", err)
return
}
Expand All @@ -109,9 +111,9 @@ func (h *IndexerV1) indexReport(w http.ResponseWriter, r *http.Request) {
return
}

// TODO Do we need some sort of background context embedded in the HTTP
// struct?
report, err := h.srv.Index(ctx, &m)
// TODO(hank) We should switch on the content-type header and not send
// back the report if we've received an OCI manifest.
report, err := h.srv.Index(ctx, m)
if err != nil {
apiError(w, http.StatusInternalServerError, "failed to start scan: %v", err)
return
Expand Down Expand Up @@ -342,3 +344,84 @@ var indexerv1wrapper = &wrapper{
[]string{"handler"},
),
}

const (
// Known manifest types we ingest.
typeOCIManifest = oci.MediaTypeImageManifest
typeNativeManifest = `application/vnd.projectquay.clair.mainfest.v1+json`
)

// DecodeManifest switches on the Request's Content-Type to consume the body.
//
// Defaults to expecting a native Claircore Manifest.
func decodeManifest(ctx context.Context, r *http.Request, dec *codec.Decoder) (*claircore.Manifest, error) {
var m claircore.Manifest

t := r.Header.Get("content-type")
if i := strings.IndexByte(t, ';'); i != -1 {
t = strings.TrimSpace(t[:i])
}
switch t {
case typeOCIManifest:
var om oci.Manifest
if err := dec.Decode(&om); err != nil {
return nil, err
}
if err := nativeFromOCI(&m, &om); err != nil {
return nil, err
}
case typeNativeManifest, "application/json", "":
if err := dec.Decode(&m); err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("unknown content-type %q", t)
}
return &m, nil
}

// These are the layer types we accept inside an OCI Manifest.
var ociLayerTypes = map[string]struct{}{
oci.MediaTypeImageLayer: {},
oci.MediaTypeImageLayerGzip: {},
oci.MediaTypeImageLayer + "+zstd": {}, // The specs package doesn't have zstd, oddly.
}

// NativeFromOCI populates the Manifest from the OCI Manifest, reporting an
// error if something is invalid.
func nativeFromOCI(m *claircore.Manifest, o *oci.Manifest) error {
const header = `header:`
var err error

m.Hash, err = claircore.ParseDigest(o.Config.Digest.String())
if err != nil {
return fmt.Errorf("unable to parse manifest digest %q: %w", o.Config.Digest, err)
}

for _, u := range o.Layers {
if len(u.URLs) == 0 {
// Manifest is missing URLs.
// They're optional in the spec, but we need them for obvious reasons.
return fmt.Errorf("missing URLs for layer %q", u.Digest)
}
if _, ok := ociLayerTypes[u.MediaType]; !ok {
return fmt.Errorf("invalid media type for layer %q", u.Digest)
}
l := claircore.Layer{
URI: u.URLs[0],
}
l.Hash, err = claircore.ParseDigest(u.Digest.String())
if err != nil {
return fmt.Errorf("unable to parse layer digest %q: %w", u.Digest, err)
}
for k, v := range u.Annotations {
if !strings.HasPrefix(k, header) {
continue
}
l.Headers[strings.TrimPrefix(k, header)] = []string{v}
}
m.Layers = append(m.Layers, &l)
}

return nil
}
Loading

0 comments on commit 49dfb01

Please sign in to comment.