+ * For this implementation container name maps to AWS S3 bucket name as-is. + * + * @param containerName Container name to use as source AWS S3 bucket name. + */ + @Override + public void setContainerName(String containerName) { + this.containerName = containerName; + } +} diff --git a/src/main/java/no/entur/uttu/export/blob/S3BlobStoreRepositoryConfig.java b/src/main/java/no/entur/uttu/export/blob/S3BlobStoreRepositoryConfig.java new file mode 100644 index 00000000..8fe33300 --- /dev/null +++ b/src/main/java/no/entur/uttu/export/blob/S3BlobStoreRepositoryConfig.java @@ -0,0 +1,83 @@ +/* + * Licensed under the EUPL, Version 1.2 or – as soon they will be approved by + * the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * https://joinup.ec.europa.eu/software/page/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + * + */ + +package no.entur.uttu.export.blob; + +import java.net.URI; +import org.rutebanken.helper.storage.repository.BlobStoreRepository; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3CrtAsyncClientBuilder; + +@Configuration +@Profile("s3-blobstore") +public class S3BlobStoreRepositoryConfig { + + @Value("${blobstore.s3.region}") + private String region; + + @Value("${blobstore.s3.endpointOverride:#{null}}") + private String endpointOverride; + + @Bean + BlobStoreRepository blobStoreRepository( + @Value("${blobstore.s3.bucket}") String containerName, + S3AsyncClient s3AsyncClient + ) { + S3BlobStoreRepository s3BlobStoreRepository = new S3BlobStoreRepository( + s3AsyncClient + ); + s3BlobStoreRepository.setContainerName(containerName); + return s3BlobStoreRepository; + } + + @Profile("local | test") + @Bean + public AwsCredentialsProvider localCredentials( + @Value("blobstore.s3.accessKeyId") String accessKeyId, + @Value("blobstore.s3.secretKey") String secretKey + ) { + return StaticCredentialsProvider.create( + AwsBasicCredentials.create(accessKeyId, secretKey) + ); + } + + @Profile("!local & !test") + @Bean + public AwsCredentialsProvider cloudCredentials() { + return DefaultCredentialsProvider.create(); + } + + @Bean + public S3AsyncClient s3AsyncClient(AwsCredentialsProvider credentialsProvider) { + S3CrtAsyncClientBuilder b = S3AsyncClient + .crtBuilder() + .credentialsProvider(credentialsProvider) + .region(Region.of(region)); + if (endpointOverride != null) { + b = b.endpointOverride(URI.create(endpointOverride)); + } + return b.build(); + } +} diff --git a/src/test/groovy/no/entur/uttu/graphql/AbstractGraphQLResourceIntegrationTest.groovy b/src/test/groovy/no/entur/uttu/graphql/AbstractGraphQLResourceIntegrationTest.groovy index ea66aca4..a04f24c9 100644 --- a/src/test/groovy/no/entur/uttu/graphql/AbstractGraphQLResourceIntegrationTest.groovy +++ b/src/test/groovy/no/entur/uttu/graphql/AbstractGraphQLResourceIntegrationTest.groovy @@ -24,11 +24,13 @@ import no.entur.uttu.UttuIntegrationTest import no.entur.uttu.stubs.UserContextServiceStub import org.junit.Before import org.springframework.beans.factory.annotation.Autowired +import org.springframework.test.context.ActiveProfiles; import java.time.LocalDate import static io.restassured.RestAssured.given +@ActiveProfiles([ "in-memory-blobstore" ]) abstract class AbstractGraphQLResourceIntegrationTest extends UttuIntegrationTest { @Autowired diff --git a/src/test/java/no/entur/uttu/export/blob/S3BlobStoreRepositoryTest.java b/src/test/java/no/entur/uttu/export/blob/S3BlobStoreRepositoryTest.java new file mode 100644 index 00000000..4cb4b873 --- /dev/null +++ b/src/test/java/no/entur/uttu/export/blob/S3BlobStoreRepositoryTest.java @@ -0,0 +1,196 @@ +package no.entur.uttu.export.blob; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.ExecutionException; +import no.entur.uttu.UttuIntegrationTest; +import org.jetbrains.annotations.NotNull; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.rutebanken.helper.storage.BlobAlreadyExistsException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.containers.localstack.LocalStackContainer.Service; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; + +@Testcontainers +@ActiveProfiles({ "s3-blobstore" }) +public class S3BlobStoreRepositoryTest extends UttuIntegrationTest { + + private static final String TEST_BUCKET = "test-blobstore-exports"; + + private static LocalStackContainer localStack; + + @Autowired + private S3AsyncClient s3AsyncClient; + + @Autowired + private S3BlobStoreRepository blobStore; + + @DynamicPropertySource + static void blobStoreProperties(DynamicPropertyRegistry registry) { + registry.add( + "blobstore.s3.endpointOverride", + () -> localStack.getEndpointOverride(Service.S3) + ); + registry.add("blobstore.s3.region", () -> localStack.getRegion()); + registry.add("blobstore.s3.accessKeyId", () -> localStack.getAccessKey()); + registry.add("blobstore.s3.secretKey", () -> localStack.getSecretKey()); + registry.add("blobstore.s3.bucket", () -> TEST_BUCKET); + } + + private void createBucket(String bucketName) + throws ExecutionException, InterruptedException { + s3AsyncClient + .headBucket(request -> request.bucket(bucketName)) + .exceptionally(throwable -> { + if (throwable.getCause() instanceof NoSuchBucketException) { + s3AsyncClient + .createBucket(CreateBucketRequest.builder().bucket(bucketName).build()) + .join(); + } + return null; + }) + .get(); + } + + @BeforeClass + public static void init() { + localStack = + new LocalStackContainer(DockerImageName.parse("localstack/localstack:3.4.0")) + .withServices(Service.S3) + .withEnv("DEFAULT_REGION", Region.EU_NORTH_1.id()); + localStack.start(); + } + + @Before + public void setUp() throws Exception { + createBucket(TEST_BUCKET); + } + + @Test + public void canRoundtripAFile() throws Exception { + String original = "Hello, BlobStore!"; + assertBlobExists(TEST_BUCKET, "myblob", false); + blobStore.uploadBlob("myblob", new ByteArrayInputStream(original.getBytes())); + assertBlobExists(TEST_BUCKET, "myblob", true); + Assert.assertEquals(original, new String(blobStore.getBlob("myblob").readAllBytes())); + Assert.assertTrue(blobStore.delete("myblob")); + } + + @Test(expected = BlobAlreadyExistsException.class) + public void cannotOverWriteExistingObject() throws Exception { + String original = "another bytes the dust"; + assertBlobExists(TEST_BUCKET, "anotherblob", false); + blobStore.uploadNewBlob("anotherblob", new ByteArrayInputStream(original.getBytes())); + assertBlobExists(TEST_BUCKET, "anotherblob", true); + blobStore.uploadNewBlob( + "anotherblob", + new ByteArrayInputStream("something silly".getBytes()) + ); + } + + /** + * Implementation note: Content type can be set for S3, but it does not mean much when downloading. The header + * does persist in object metadata though. + */ + @Test + public void canSetContentTypeForUpload() throws Exception { + String contentType = "application/json"; + blobStore.uploadBlob( + "json", + new ByteArrayInputStream("{\"key\":false}".getBytes()), + contentType + ); + assertBlobExists(TEST_BUCKET, "json", true); + HeadObjectResponse response = s3AsyncClient + .headObject(request -> request.bucket(TEST_BUCKET).key("json")) + .join(); + Assert.assertEquals(contentType, response.contentType()); + } + + @Test + public void canCopyContentBetweenBuckets() throws Exception { + String targetBucket = "another-bucket"; + String content = "1"; + createBucket(targetBucket); + blobStore.uploadBlob("smallfile", asStream(content)); + blobStore.copyBlob(TEST_BUCKET, "smallfile", targetBucket, "tinyfile"); + blobStore.setContainerName(targetBucket); + Assert.assertEquals( + content, + new String(blobStore.getBlob("tinyfile").readAllBytes()) + ); + blobStore.setContainerName(TEST_BUCKET); + } + + /** + * Implementation note: Version is no-op with S3 as the current interface models it based on GCP's blob storage + * semantics. This is effectively the same as copying the blob normally. + * @throws Exception + */ + @Test + public void canCopyVersionedContentBetweenBuckets() throws Exception { + String targetBucket = "yet-another-bucket"; + String content = "a"; + createBucket(targetBucket); + blobStore.uploadBlob("minusculefile", asStream(content)); + blobStore.copyVersionedBlob( + TEST_BUCKET, + "minusculefile", + -1_000_000L, + targetBucket, + "barelyworthmentioningfile" + ); + blobStore.setContainerName(targetBucket); + Assert.assertEquals( + content, + new String(blobStore.getBlob("barelyworthmentioningfile").readAllBytes()) + ); + blobStore.setContainerName(TEST_BUCKET); + } + + @Test + public void canCopyAllBlobsWithSharedPrefix() throws Exception { + String targetBucket = "one-more-bucket"; + createBucket(targetBucket); + blobStore.uploadBlob("things/a", asStream("a")); + blobStore.uploadBlob("things/b", asStream("b")); + blobStore.uploadBlob("things/c", asStream("c")); + blobStore.uploadBlob("stuff/d", asStream("d")); + blobStore.copyAllBlobs(TEST_BUCKET, "things", targetBucket, "bits"); + assertBlobExists(targetBucket, "bits/a", true); + assertBlobExists(targetBucket, "bits/b", true); + assertBlobExists(targetBucket, "bits/c", true); + assertBlobExists(targetBucket, "stuff/d", false); + } + + private static @NotNull ByteArrayInputStream asStream(String source) { + return new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)); + } + + private void assertBlobExists(String bucket, String key, boolean exists) + throws ExecutionException, InterruptedException { + HeadObjectResponse response = s3AsyncClient + .headObject(request -> request.bucket(bucket).key(key)) + .exceptionally(throwable -> null) + .get(); + if (!exists && response != null) { + Assert.fail(bucket + " / " + key + " exists"); + } + if (exists && response == null) { + Assert.fail(bucket + " / " + key + " does not exist"); + } + } +}