From ba5d87c79ed4a3e89ac0061333a53aa4c1112897 Mon Sep 17 00:00:00 2001 From: zrdzn Date: Thu, 7 Nov 2024 20:38:32 +0100 Subject: [PATCH 1/8] Introduce oci plugin --- .../kotlin/com/reposilite/packages/.gitkeep | 0 .../reposilite/packages/PackageRepository.kt | 7 +++ .../com/reposilite/packages/Repository.kt | 4 -- .../com/reposilite/packages/oci/.gitkeep | 0 .../com/reposilite/packages/oci/OciFacade.kt | 43 +++++++++++++++++++ .../packages/oci/api/ManifestApi.kt | 43 +++++++++++++++++++ .../packages/oci/application/OciPlugin.kt | 35 +++++++++++++++ 7 files changed, 128 insertions(+), 4 deletions(-) delete mode 100644 reposilite-backend/src/main/kotlin/com/reposilite/packages/.gitkeep create mode 100644 reposilite-backend/src/main/kotlin/com/reposilite/packages/PackageRepository.kt delete mode 100644 reposilite-backend/src/main/kotlin/com/reposilite/packages/Repository.kt delete mode 100644 reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/.gitkeep create mode 100644 reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt create mode 100644 reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/ManifestApi.kt create mode 100644 reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/application/OciPlugin.kt diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/.gitkeep b/reposilite-backend/src/main/kotlin/com/reposilite/packages/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/PackageRepository.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/PackageRepository.kt new file mode 100644 index 000000000..56bd16bba --- /dev/null +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/PackageRepository.kt @@ -0,0 +1,7 @@ +package com.reposilite.packages + + +interface PackageRepository { + + +} diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/Repository.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/Repository.kt deleted file mode 100644 index fa544a6b6..000000000 --- a/reposilite-backend/src/main/kotlin/com/reposilite/packages/Repository.kt +++ /dev/null @@ -1,4 +0,0 @@ -package com.reposilite.packages - -interface Repository { -} \ No newline at end of file diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/.gitkeep b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt new file mode 100644 index 000000000..993ce161e --- /dev/null +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2023 dzikoysk + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.reposilite.packages.oci + +import com.reposilite.journalist.Journalist +import com.reposilite.journalist.Logger +import com.reposilite.packages.oci.api.ManifestResponse +import com.reposilite.packages.oci.api.SaveManifestRequest +import com.reposilite.plugin.api.Facade +import com.reposilite.shared.ErrorResponse +import com.reposilite.storage.StorageProvider +import com.reposilite.storage.api.toLocation +import panda.std.Result +import panda.std.asSuccess + +class OciFacade( + private val journalist: Journalist, + private val storageProvider: StorageProvider +) : Journalist, Facade { + + fun saveManifest(saveManifestRequest: SaveManifestRequest): Result { + storageProvider.putFile("oci/manifest.json".toLocation(), saveManifestRequest.toString().toByteArray().inputStream()) + return saveManifestRequest.let { ManifestResponse(it.schemaVersion, it.mediaType, it.config, it.layers) }.asSuccess() + } + + override fun getLogger(): Logger = + journalist.logger + +} diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/ManifestApi.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/ManifestApi.kt new file mode 100644 index 000000000..6f7e392b9 --- /dev/null +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/ManifestApi.kt @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2023 dzikoysk + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.reposilite.packages.oci.api + +data class SaveManifestRequest( + val schemaVersion: Int, + val mediaType: String, + val config: ManifestConfig, + val layers: List, +) + +data class ManifestResponse( + val schemaVersion: Int, + val mediaType: String, + val config: ManifestConfig, + val layers: List, +) + +data class ManifestConfig( + val mediaType: String, + val size: Int, + val digest: String, +) + +data class ManifestLayer( + val mediaType: String, + val size: Int, + val digest: String, +) diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/application/OciPlugin.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/application/OciPlugin.kt new file mode 100644 index 000000000..6eb267595 --- /dev/null +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/application/OciPlugin.kt @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 dzikoysk + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.reposilite.packages.oci.application + +import com.reposilite.packages.oci.OciFacade +import com.reposilite.plugin.api.Plugin +import com.reposilite.plugin.api.ReposilitePlugin + +@Plugin( + name = "oci", + dependencies = ["failure", "local-configuration", "shared-configuration", "statistics", "authentication", "access-token", "storage"] +) +internal class OciPlugin : ReposilitePlugin() { + + override fun initialize(): OciFacade { + val ociFacade = OciFacade(this) + + return ociFacade + } + +} From 66e7806a502acbb39921b2267441237a9f73570b Mon Sep 17 00:00:00 2001 From: zrdzn Date: Sat, 9 Nov 2024 17:57:06 +0100 Subject: [PATCH 2/8] Add some methods --- .../com/reposilite/packages/oci/OciFacade.kt | 55 ++++++++++++++++++- .../reposilite/packages/oci/api/BlobApi.kt | 9 +++ .../reposilite/packages/oci/api/UploadApi.kt | 9 +++ 3 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/BlobApi.kt create mode 100644 reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/UploadApi.kt diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt index 993ce161e..3bfe601e6 100644 --- a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt @@ -18,25 +18,76 @@ package com.reposilite.packages.oci import com.reposilite.journalist.Journalist import com.reposilite.journalist.Logger +import com.reposilite.packages.oci.api.BlobResponse import com.reposilite.packages.oci.api.ManifestResponse import com.reposilite.packages.oci.api.SaveManifestRequest +import com.reposilite.packages.oci.api.UploadState import com.reposilite.plugin.api.Facade import com.reposilite.shared.ErrorResponse import com.reposilite.storage.StorageProvider import com.reposilite.storage.api.toLocation import panda.std.Result import panda.std.asSuccess +import java.security.MessageDigest +import java.util.* class OciFacade( private val journalist: Journalist, private val storageProvider: StorageProvider ) : Journalist, Facade { - fun saveManifest(saveManifestRequest: SaveManifestRequest): Result { - storageProvider.putFile("oci/manifest.json".toLocation(), saveManifestRequest.toString().toByteArray().inputStream()) + private val sessions = mutableMapOf() + private val sha256Hash = MessageDigest.getInstance("SHA-256") + + fun saveManifest(namespace: String, digest: String, saveManifestRequest: SaveManifestRequest): Result { + storageProvider.putFile("manifests/${namespace}/${digest}".toLocation(), saveManifestRequest.toString().toByteArray().inputStream()) return saveManifestRequest.let { ManifestResponse(it.schemaVersion, it.mediaType, it.config, it.layers) }.asSuccess() } + fun saveTaggedManifest(namespace: String, tag: String, saveManifestRequest: SaveManifestRequest): Result { + val digest = sha256Hash.digest(saveManifestRequest.toString().toByteArray()).joinToString("") { "%02x".format(it) } + + storageProvider.putFile("manifests/${namespace}/${tag}/manifest".toLocation(), saveManifestRequest.toString().toByteArray().inputStream()) + storageProvider.putFile("manifests/${namespace}/${tag}/manifest.sha256".toLocation(), digest.toByteArray().inputStream()) + return saveManifestRequest.let { ManifestResponse(it.schemaVersion, it.mediaType, it.config, it.layers) }.asSuccess() + } + + fun retrieveUploadSessionId(namespace: String): Result { + val sessionId = UUID.randomUUID().toString() + + sessions[sessionId] = UploadState( + sessionId = sessionId, + name = namespace, + uploadedData = ByteArray(0), + bytesReceived = 0, + createdAt = System.currentTimeMillis().toString() + ) + + return sessionId.asSuccess() + } + + fun findBlobByDigest(namespace: String, digest: String): Result = + storageProvider.getFile("blobs/${namespace}/${digest}".toLocation()) + .map { + BlobResponse( + digest = digest, + length = it.available(), + content = it + ) + } + + fun findManifestChecksumByDigest(namespace: String, digest: String): Result { + val location = "manifests/${namespace}/${digest}".toLocation() + return storageProvider.getFile(location) + .map { it.readAllBytes().joinToString("") { "%02x".format(it) } } + } + + fun findManifestChecksumByTag(namespace: String, tag: String): Result { + val location = "manifests/${namespace}/${tag}/manifest.sha256".toLocation() + return storageProvider.getFile(location) + .map { it.readAllBytes().joinToString("") { "%02x".format(it) } } + } + override fun getLogger(): Logger = journalist.logger diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/BlobApi.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/BlobApi.kt new file mode 100644 index 000000000..84600057f --- /dev/null +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/BlobApi.kt @@ -0,0 +1,9 @@ +package com.reposilite.packages.oci.api + +import java.io.InputStream + +data class BlobResponse( + val length: Int, + val content: InputStream, + val digest: String, +) diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/UploadApi.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/UploadApi.kt new file mode 100644 index 000000000..fe5e4994b --- /dev/null +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/UploadApi.kt @@ -0,0 +1,9 @@ +package com.reposilite.packages.oci.api + +data class UploadState( + val sessionId: String, + val name: String, + val uploadedData: ByteArray, + val bytesReceived: Int, + val createdAt: String +) From f6e440ce04c18345e2af8149adf877a9214527ca Mon Sep 17 00:00:00 2001 From: zrdzn Date: Mon, 11 Nov 2024 18:08:30 +0100 Subject: [PATCH 3/8] Add first endpoint --- .../com/reposilite/packages/oci/OciFacade.kt | 21 ++++++- .../oci/api/{BlobApi.kt => OciBlobApi.kt} | 0 .../api/{ManifestApi.kt => OciManifestApi.kt} | 0 .../oci/api/{UploadApi.kt => OciUploadApi.kt} | 4 +- .../oci/infrastructure/OciEndpoints.kt | 58 +++++++++++++++++++ 5 files changed, 80 insertions(+), 3 deletions(-) rename reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/{BlobApi.kt => OciBlobApi.kt} (100%) rename reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/{ManifestApi.kt => OciManifestApi.kt} (100%) rename reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/{UploadApi.kt => OciUploadApi.kt} (69%) create mode 100644 reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt index 3bfe601e6..9b07b6d8d 100644 --- a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt @@ -24,6 +24,8 @@ import com.reposilite.packages.oci.api.SaveManifestRequest import com.reposilite.packages.oci.api.UploadState import com.reposilite.plugin.api.Facade import com.reposilite.shared.ErrorResponse +import com.reposilite.shared.badRequestError +import com.reposilite.shared.notFoundError import com.reposilite.storage.StorageProvider import com.reposilite.storage.api.toLocation import panda.std.Result @@ -52,7 +54,7 @@ class OciFacade( return saveManifestRequest.let { ManifestResponse(it.schemaVersion, it.mediaType, it.config, it.layers) }.asSuccess() } - fun retrieveUploadSessionId(namespace: String): Result { + fun retrieveBlobUploadSessionId(namespace: String): Result { val sessionId = UUID.randomUUID().toString() sessions[sessionId] = UploadState( @@ -66,6 +68,15 @@ class OciFacade( return sessionId.asSuccess() } + fun uploadBlobStreamPart(namespace: String, sessionId: String, part: ByteArray): Result { + val session = sessions[sessionId] ?: return notFoundError("Session not found") + + session.uploadedData += part + session.bytesReceived += part.size + + return session.asSuccess() + } + fun findBlobByDigest(namespace: String, digest: String): Result = storageProvider.getFile("blobs/${namespace}/${digest}".toLocation()) .map { @@ -88,6 +99,14 @@ class OciFacade( .map { it.readAllBytes().joinToString("") { "%02x".format(it) } } } + fun validateDigest(digest: String): Result { + if (!digest.startsWith("sha256:")) { + return badRequestError("Invalid digest format") + } + + return digest.asSuccess() + } + override fun getLogger(): Logger = journalist.logger diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/BlobApi.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/OciBlobApi.kt similarity index 100% rename from reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/BlobApi.kt rename to reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/OciBlobApi.kt diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/ManifestApi.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/OciManifestApi.kt similarity index 100% rename from reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/ManifestApi.kt rename to reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/OciManifestApi.kt diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/UploadApi.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/OciUploadApi.kt similarity index 69% rename from reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/UploadApi.kt rename to reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/OciUploadApi.kt index fe5e4994b..4277415c3 100644 --- a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/UploadApi.kt +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/api/OciUploadApi.kt @@ -3,7 +3,7 @@ package com.reposilite.packages.oci.api data class UploadState( val sessionId: String, val name: String, - val uploadedData: ByteArray, - val bytesReceived: Int, + var uploadedData: ByteArray, + var bytesReceived: Int, val createdAt: String ) diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt new file mode 100644 index 000000000..ba8deec64 --- /dev/null +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2023 dzikoysk + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.reposilite.packages.oci.infrastructure + +import com.reposilite.packages.oci.OciFacade +import com.reposilite.packages.oci.api.ManifestResponse +import com.reposilite.packages.oci.api.SaveManifestRequest +import com.reposilite.shared.badRequest +import com.reposilite.shared.extractFromHeader +import com.reposilite.shared.notFoundError +import com.reposilite.web.api.ReposiliteRoute +import com.reposilite.web.api.ReposiliteRoutes +import io.javalin.community.routing.Route.* +import io.javalin.http.bodyAsClass +import panda.std.Result.supplyThrowing + +internal class OciEndpoints( + private val ociFacade: OciFacade, + private val basePath: String, + private val compressionStrategy: String +) : ReposiliteRoutes() { + + fun saveManifest(namespace: String, reference: String) = + ReposiliteRoute("/api/oci/v2/$namespace/manifests/$reference", PUT) { + accessed { + val contentType = ctx.header("Content-Type") + if (contentType != "application/vnd.docker.distribution.manifest.v2+json") { + response = notFoundError("Invalid content type") + return@accessed + } + + response = supplyThrowing { ctx.bodyAsClass() } + .mapErr { badRequest("Request does not contain valid body") } + .flatMap { saveManifestRequest -> + ociFacade.validateDigest(reference) + .flatMap { ociFacade.saveManifest(namespace, reference, saveManifestRequest) } + .flatMapErr { ociFacade.saveTaggedManifest(namespace, reference, saveManifestRequest) } + } + } + } + + override val routes = routes(saveManifest()) + +} From fe3900dbbda540663719f191cef944474d8545ef Mon Sep 17 00:00:00 2001 From: zrdzn Date: Fri, 15 Nov 2024 21:37:54 +0100 Subject: [PATCH 4/8] Add more endpoints --- .../com/reposilite/packages/oci/OciFacade.kt | 18 +++- .../oci/infrastructure/OciEndpoints.kt | 85 ++++++++++++++++--- 2 files changed, 92 insertions(+), 11 deletions(-) diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt index 9b07b6d8d..9ccf1b458 100644 --- a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt @@ -25,6 +25,7 @@ import com.reposilite.packages.oci.api.UploadState import com.reposilite.plugin.api.Facade import com.reposilite.shared.ErrorResponse import com.reposilite.shared.badRequestError +import com.reposilite.shared.notFound import com.reposilite.shared.notFoundError import com.reposilite.storage.StorageProvider import com.reposilite.storage.api.toLocation @@ -68,7 +69,7 @@ class OciFacade( return sessionId.asSuccess() } - fun uploadBlobStreamPart(namespace: String, sessionId: String, part: ByteArray): Result { + fun uploadBlobStreamPart(sessionId: String, part: ByteArray): Result { val session = sessions[sessionId] ?: return notFoundError("Session not found") session.uploadedData += part @@ -86,6 +87,7 @@ class OciFacade( content = it ) } + .mapErr { notFound("Could not find blob with specified digest") } fun findManifestChecksumByDigest(namespace: String, digest: String): Result { val location = "manifests/${namespace}/${digest}".toLocation() @@ -95,10 +97,24 @@ class OciFacade( fun findManifestChecksumByTag(namespace: String, tag: String): Result { val location = "manifests/${namespace}/${tag}/manifest.sha256".toLocation() + return storageProvider.getFile(location) .map { it.readAllBytes().joinToString("") { "%02x".format(it) } } } + fun findManifestTagByDigest(namespace: String, digest: String): Result { + val tagsDirectory = "manifests/${namespace}".toLocation() + + // todo replace with exposed (digest to tag mapping) + return storageProvider.getFiles(tagsDirectory) + .flatMap { files -> + files + .map { storageProvider.getFile(it.resolve("manifest.sha256")) } + .map { it.map { it.readAllBytes().joinToString("") { "%02x".format(it) } } } + .first() + } + } + fun validateDigest(digest: String): Result { if (!digest.startsWith("sha256:")) { return badRequestError("Invalid digest format") diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt index ba8deec64..4a0daa2e9 100644 --- a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt @@ -20,29 +20,28 @@ import com.reposilite.packages.oci.OciFacade import com.reposilite.packages.oci.api.ManifestResponse import com.reposilite.packages.oci.api.SaveManifestRequest import com.reposilite.shared.badRequest -import com.reposilite.shared.extractFromHeader -import com.reposilite.shared.notFoundError +import com.reposilite.shared.badRequestError import com.reposilite.web.api.ReposiliteRoute -import com.reposilite.web.api.ReposiliteRoutes import io.javalin.community.routing.Route.* +import io.javalin.http.HandlerType import io.javalin.http.bodyAsClass import panda.std.Result.supplyThrowing internal class OciEndpoints( private val ociFacade: OciFacade, - private val basePath: String, - private val compressionStrategy: String -) : ReposiliteRoutes() { +) { - fun saveManifest(namespace: String, reference: String) = - ReposiliteRoute("/api/oci/v2/$namespace/manifests/$reference", PUT) { + fun saveManifest(namespace: String) = + ReposiliteRoute("/api/oci/v2/$namespace/manifests/{reference}", PUT) { accessed { val contentType = ctx.header("Content-Type") if (contentType != "application/vnd.docker.distribution.manifest.v2+json") { - response = notFoundError("Invalid content type") + response = badRequestError("Invalid content type") return@accessed } + val reference = parameter("reference") ?: return@accessed + response = supplyThrowing { ctx.bodyAsClass() } .mapErr { badRequest("Request does not contain valid body") } .flatMap { saveManifestRequest -> @@ -53,6 +52,72 @@ internal class OciEndpoints( } } - override val routes = routes(saveManifest()) + fun retrieveBlobUploadSessionId(namespace: String) = + ReposiliteRoute("/api/oci/v2/$namespace/blobs/uploads", POST) { + accessed { + val digest = queryParameter("digest") + if (digest == null) { + response = ociFacade.retrieveBlobUploadSessionId(namespace) + .map { + ctx.status(202) + ctx.header("Location", "/api/oci/v2/$namespace/blobs/uploads/$it") + } + + return@accessed + } + } + } + + fun uploadBlobStreamPart(namespace: String) = + ReposiliteRoute("/api/oci/v2/$namespace/blobs/uploads/{sessionId}", PATCH) { + accessed { + val contentType = ctx.header("Content-Type") + if (contentType != "application/octet-stream") { + response = badRequestError("Invalid content type") + return@accessed + } + + val sessionId = parameter("sessionId") ?: return@accessed + + response = supplyThrowing { ctx.bodyAsBytes() } + .mapErr { badRequest("Body does not contain any bytes") } + .flatMap { ociFacade.uploadBlobStreamPart(sessionId, it) } + .map { + ctx.status(202) + ctx.header("Location", "/api/oci/v2/$namespace/blobs/uploads/$sessionId") + ctx.header("Range", "0-${it.bytesReceived - 1}") + } + } + } + + fun findBlobByDigest(namespace: String) = + ReposiliteRoute("/api/oci/v2/$namespace/blobs/{digest}", GET, HEAD) { + accessed { + val digest = parameter("digest") ?: return@accessed + + response = ociFacade.findBlobByDigest(namespace, digest) + .peek { + ctx.header("Content-Length", it.length.toString()) + ctx.header("Docker-Content-Digest", it.digest) + } + .takeIf { ctx.method() == HandlerType.GET } + ?.map { it.content.readNBytes(it.length) } + } + } + + fun findManifestChecksumByReference(namespace: String) = + ReposiliteRoute("/api/oci/v2/$namespace/manifests/{reference}", HEAD) { + accessed { + val reference = parameter("reference") ?: return@accessed + + response = ociFacade.validateDigest(reference) + .flatMap { ociFacade.findManifestChecksumByDigest(namespace, reference) } + .flatMapErr { ociFacade.findManifestChecksumByTag(namespace, reference) } + .map { + ctx.status(200) + ctx.header("Docker-Content-Digest", it) + } + } + } } From 4144acce1ca231c4640a955882a2fa3351e3e00a Mon Sep 17 00:00:00 2001 From: zrdzn Date: Fri, 22 Nov 2024 21:34:54 +0100 Subject: [PATCH 5/8] Add blob finalizing --- .../com/reposilite/packages/oci/OciFacade.kt | 14 ++++++++++ .../oci/infrastructure/OciEndpoints.kt | 28 +++++++++++++++++-- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt index 9ccf1b458..f57fbb5fd 100644 --- a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt @@ -78,6 +78,20 @@ class OciFacade( return session.asSuccess() } + fun finalizeBlobUpload(namespace: String, digest: String, sessionId: String, lastPart: ByteArray?): Result { + val session = sessions[sessionId] ?: return notFoundError("Session not found") + + if (lastPart != null) { + session.bytesReceived += lastPart.size + } + + storageProvider.putFile("blobs/$namespace/$digest".toLocation(), session.uploadedData.inputStream()) + + sessions.remove(sessionId) + + return findBlobByDigest(namespace, digest) + } + fun findBlobByDigest(namespace: String, digest: String): Result = storageProvider.getFile("blobs/${namespace}/${digest}".toLocation()) .map { diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt index 4a0daa2e9..a67bedb30 100644 --- a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt @@ -90,6 +90,28 @@ internal class OciEndpoints( } } + fun finalizeBlobUpload(namespace: String) = + ReposiliteRoute("/api/oci/v2/$namespace/blobs/{sessionId}", PUT) { + accessed { + val sessionId = parameter("sessionId") ?: return@accessed + val digest = queryParameter("digest") + if (digest == null) { + response = badRequestError("No digest provided") + return@accessed + } + + response = supplyThrowing { ctx.bodyAsBytes() } + .fold( + { ociFacade.finalizeBlobUpload(namespace, digest, sessionId, it) }, + { ociFacade.finalizeBlobUpload(namespace, digest, sessionId, null) }, + ) + .map { + ctx.status(201) + ctx.header("Location", "/api/oci/v2/$namespace/blobs/${it.digest}") + } + } + } + fun findBlobByDigest(namespace: String) = ReposiliteRoute("/api/oci/v2/$namespace/blobs/{digest}", GET, HEAD) { accessed { @@ -111,8 +133,10 @@ internal class OciEndpoints( val reference = parameter("reference") ?: return@accessed response = ociFacade.validateDigest(reference) - .flatMap { ociFacade.findManifestChecksumByDigest(namespace, reference) } - .flatMapErr { ociFacade.findManifestChecksumByTag(namespace, reference) } + .fold( + { ociFacade.findManifestChecksumByDigest(namespace, reference) }, + { ociFacade.findManifestChecksumByTag(namespace, reference) } + ) .map { ctx.status(200) ctx.header("Docker-Content-Digest", it) From 18519254faea690dfaf074a5a7de1f6c5767be61 Mon Sep 17 00:00:00 2001 From: zrdzn Date: Fri, 22 Nov 2024 21:36:10 +0100 Subject: [PATCH 6/8] F0ld one more --- .../reposilite/packages/oci/infrastructure/OciEndpoints.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt index a67bedb30..1a294043f 100644 --- a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt @@ -46,8 +46,10 @@ internal class OciEndpoints( .mapErr { badRequest("Request does not contain valid body") } .flatMap { saveManifestRequest -> ociFacade.validateDigest(reference) - .flatMap { ociFacade.saveManifest(namespace, reference, saveManifestRequest) } - .flatMapErr { ociFacade.saveTaggedManifest(namespace, reference, saveManifestRequest) } + .fold( + { ociFacade.saveManifest(namespace, reference, saveManifestRequest) }, + { ociFacade.saveTaggedManifest(namespace, reference, saveManifestRequest) } + ) } } } From 8c238e8e231e248f9814b1a0d4a9715f991bc7e0 Mon Sep 17 00:00:00 2001 From: zrdzn Date: Fri, 22 Nov 2024 21:41:09 +0100 Subject: [PATCH 7/8] Rename namespace -> repository --- .../packages/oci/application/OciPlugin.kt | 6 ++- .../oci/infrastructure/OciEndpoints.kt | 46 +++++++++---------- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/application/OciPlugin.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/application/OciPlugin.kt index 6eb267595..963ac9704 100644 --- a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/application/OciPlugin.kt +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/application/OciPlugin.kt @@ -19,6 +19,7 @@ package com.reposilite.packages.oci.application import com.reposilite.packages.oci.OciFacade import com.reposilite.plugin.api.Plugin import com.reposilite.plugin.api.ReposilitePlugin +import com.reposilite.plugin.facade @Plugin( name = "oci", @@ -27,7 +28,10 @@ import com.reposilite.plugin.api.ReposilitePlugin internal class OciPlugin : ReposilitePlugin() { override fun initialize(): OciFacade { - val ociFacade = OciFacade(this) + val ociFacade = OciFacade( + journalist = this, + storageProvider = facade() + ) return ociFacade } diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt index 1a294043f..a11815ec6 100644 --- a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/infrastructure/OciEndpoints.kt @@ -31,8 +31,8 @@ internal class OciEndpoints( private val ociFacade: OciFacade, ) { - fun saveManifest(namespace: String) = - ReposiliteRoute("/api/oci/v2/$namespace/manifests/{reference}", PUT) { + fun saveManifest(repository: String) = + ReposiliteRoute("/api/oci/v2/$repository/manifests/{reference}", PUT) { accessed { val contentType = ctx.header("Content-Type") if (contentType != "application/vnd.docker.distribution.manifest.v2+json") { @@ -47,22 +47,22 @@ internal class OciEndpoints( .flatMap { saveManifestRequest -> ociFacade.validateDigest(reference) .fold( - { ociFacade.saveManifest(namespace, reference, saveManifestRequest) }, - { ociFacade.saveTaggedManifest(namespace, reference, saveManifestRequest) } + { ociFacade.saveManifest(repository, reference, saveManifestRequest) }, + { ociFacade.saveTaggedManifest(repository, reference, saveManifestRequest) } ) } } } - fun retrieveBlobUploadSessionId(namespace: String) = - ReposiliteRoute("/api/oci/v2/$namespace/blobs/uploads", POST) { + fun retrieveBlobUploadSessionId(repository: String) = + ReposiliteRoute("/api/oci/v2/$repository/blobs/uploads", POST) { accessed { val digest = queryParameter("digest") if (digest == null) { - response = ociFacade.retrieveBlobUploadSessionId(namespace) + response = ociFacade.retrieveBlobUploadSessionId(repository) .map { ctx.status(202) - ctx.header("Location", "/api/oci/v2/$namespace/blobs/uploads/$it") + ctx.header("Location", "/api/oci/v2/$repository/blobs/uploads/$it") } return@accessed @@ -70,8 +70,8 @@ internal class OciEndpoints( } } - fun uploadBlobStreamPart(namespace: String) = - ReposiliteRoute("/api/oci/v2/$namespace/blobs/uploads/{sessionId}", PATCH) { + fun uploadBlobStreamPart(repository: String) = + ReposiliteRoute("/api/oci/v2/$repository/blobs/uploads/{sessionId}", PATCH) { accessed { val contentType = ctx.header("Content-Type") if (contentType != "application/octet-stream") { @@ -86,14 +86,14 @@ internal class OciEndpoints( .flatMap { ociFacade.uploadBlobStreamPart(sessionId, it) } .map { ctx.status(202) - ctx.header("Location", "/api/oci/v2/$namespace/blobs/uploads/$sessionId") + ctx.header("Location", "/api/oci/v2/$repository/blobs/uploads/$sessionId") ctx.header("Range", "0-${it.bytesReceived - 1}") } } } - fun finalizeBlobUpload(namespace: String) = - ReposiliteRoute("/api/oci/v2/$namespace/blobs/{sessionId}", PUT) { + fun finalizeBlobUpload(repository: String) = + ReposiliteRoute("/api/oci/v2/$repository/blobs/{sessionId}", PUT) { accessed { val sessionId = parameter("sessionId") ?: return@accessed val digest = queryParameter("digest") @@ -104,22 +104,22 @@ internal class OciEndpoints( response = supplyThrowing { ctx.bodyAsBytes() } .fold( - { ociFacade.finalizeBlobUpload(namespace, digest, sessionId, it) }, - { ociFacade.finalizeBlobUpload(namespace, digest, sessionId, null) }, + { ociFacade.finalizeBlobUpload(repository, digest, sessionId, it) }, + { ociFacade.finalizeBlobUpload(repository, digest, sessionId, null) }, ) .map { ctx.status(201) - ctx.header("Location", "/api/oci/v2/$namespace/blobs/${it.digest}") + ctx.header("Location", "/api/oci/v2/$repository/blobs/${it.digest}") } } } - fun findBlobByDigest(namespace: String) = - ReposiliteRoute("/api/oci/v2/$namespace/blobs/{digest}", GET, HEAD) { + fun findBlobByDigest(repository: String) = + ReposiliteRoute("/api/oci/v2/$repository/blobs/{digest}", GET, HEAD) { accessed { val digest = parameter("digest") ?: return@accessed - response = ociFacade.findBlobByDigest(namespace, digest) + response = ociFacade.findBlobByDigest(repository, digest) .peek { ctx.header("Content-Length", it.length.toString()) ctx.header("Docker-Content-Digest", it.digest) @@ -129,15 +129,15 @@ internal class OciEndpoints( } } - fun findManifestChecksumByReference(namespace: String) = - ReposiliteRoute("/api/oci/v2/$namespace/manifests/{reference}", HEAD) { + fun findManifestChecksumByReference(repository: String) = + ReposiliteRoute("/api/oci/v2/$repository/manifests/{reference}", HEAD) { accessed { val reference = parameter("reference") ?: return@accessed response = ociFacade.validateDigest(reference) .fold( - { ociFacade.findManifestChecksumByDigest(namespace, reference) }, - { ociFacade.findManifestChecksumByTag(namespace, reference) } + { ociFacade.findManifestChecksumByDigest(repository, reference) }, + { ociFacade.findManifestChecksumByTag(repository, reference) } ) .map { ctx.status(200) From d140664bfb06c0720529f4975818797ce72e85fe Mon Sep 17 00:00:00 2001 From: zrdzn Date: Fri, 22 Nov 2024 23:15:22 +0100 Subject: [PATCH 8/8] Register endpoints --- .../com/reposilite/packages/oci/OciFacade.kt | 6 +++- .../reposilite/packages/oci/OciRepository.kt | 6 ++++ .../packages/oci/OciRepositoryProvider.kt | 9 ++++++ .../packages/oci/application/OciPlugin.kt | 28 ++++++++++++++++++- 4 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciRepository.kt create mode 100644 reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciRepositoryProvider.kt diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt index f57fbb5fd..f08b4a2ea 100644 --- a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciFacade.kt @@ -36,7 +36,8 @@ import java.util.* class OciFacade( private val journalist: Journalist, - private val storageProvider: StorageProvider + private val storageProvider: StorageProvider, + private val ociRepositoryProvider: OciRepositoryProvider ) : Journalist, Facade { private val sessions = mutableMapOf() @@ -137,6 +138,9 @@ class OciFacade( return digest.asSuccess() } + fun getRepositories(): Collection = + ociRepositoryProvider.getRepositories() + override fun getLogger(): Logger = journalist.logger diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciRepository.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciRepository.kt new file mode 100644 index 000000000..c6a93727f --- /dev/null +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciRepository.kt @@ -0,0 +1,6 @@ +package com.reposilite.packages.oci + +data class OciRepository( + val name: String, + val type: String, +) diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciRepositoryProvider.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciRepositoryProvider.kt new file mode 100644 index 000000000..38e435d96 --- /dev/null +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/OciRepositoryProvider.kt @@ -0,0 +1,9 @@ +package com.reposilite.packages.oci + +class OciRepositoryProvider { + + private val repositories = mutableMapOf() + + fun getRepositories(): Collection = repositories.values + +} \ No newline at end of file diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/application/OciPlugin.kt b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/application/OciPlugin.kt index 963ac9704..58e859109 100644 --- a/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/application/OciPlugin.kt +++ b/reposilite-backend/src/main/kotlin/com/reposilite/packages/oci/application/OciPlugin.kt @@ -17,9 +17,14 @@ package com.reposilite.packages.oci.application import com.reposilite.packages.oci.OciFacade +import com.reposilite.packages.oci.OciRepositoryProvider +import com.reposilite.packages.oci.infrastructure.OciEndpoints import com.reposilite.plugin.api.Plugin import com.reposilite.plugin.api.ReposilitePlugin +import com.reposilite.plugin.event import com.reposilite.plugin.facade +import com.reposilite.token.infrastructure.AccessTokenApiEndpoints +import com.reposilite.web.api.RoutingSetupEvent @Plugin( name = "oci", @@ -28,11 +33,32 @@ import com.reposilite.plugin.facade internal class OciPlugin : ReposilitePlugin() { override fun initialize(): OciFacade { + val ociRepositoryProvider = OciRepositoryProvider() + val ociFacade = OciFacade( journalist = this, - storageProvider = facade() + storageProvider = facade(), + ociRepositoryProvider = ociRepositoryProvider ) + // register endpoints + event { event: RoutingSetupEvent -> + ociFacade.getRepositories().forEach { + when (it.type) { + "oci" -> { + val ociEndpoints = OciEndpoints(ociFacade) + + event.register(ociEndpoints.saveManifest(it.name)) + event.register(ociEndpoints.retrieveBlobUploadSessionId(it.name)) + event.register(ociEndpoints.findManifestChecksumByReference(it.name)) + event.register(ociEndpoints.findBlobByDigest(it.name)) + event.register(ociEndpoints.uploadBlobStreamPart(it.name)) + event.register(ociEndpoints.finalizeBlobUpload(it.name)) + } + } + } + } + return ociFacade }