Merge pull request #325 from esuomi/TIS-559/s3_blob_storage

AWS S3 blob storage

.gitignore

@@ -95,3 +95,6 @@ files
 dataset.zip
 
 **/.terraform/*
+
+# LocalStack default volume directory if override is not provided through environment variables
+./volume

README.md

@@ -10,7 +10,7 @@ pushing changes. You can also configure your IDE to reformat code when you save
 
 ## Security
 
-Running uttu with vanilla security features requires an Oauth2 issuer, which can be set with the following property:
+Running uttu with vanilla security features requires an OAuth2 issuer, which can be set with the following property:
 
 ```properties
 uttu.security.jwt.issuer-uri=https://my-jwt-issuer
@@ -75,35 +75,46 @@ If you don't use Google PubSub, sett this property:
 
 ## Running locally
 
+### Local Environment through Docker Compose
+
+Uttu has [docker-compose.yml](./docker-compose.yml) which contains all necessary dependent services for running uttu in
+various configurations. It is assumed this environment is always running when the service is being run locally 
+(see below).
+
+> **Note!** This uses the compose version included with modern versions of Docker, not the separately installable 
+> `docker-compose` command.
+
+All Docker Compose commands run in relation to the `docker-compose.yml` file located in the same directory in which the 
+command is executed.
+
+```shell
+# run with defaults - use ^C to shutdown containers
+docker compose up
+# run with additional profiles, e.g. with LocalStack based AWS simulator
+docker compose --profile aws up
+# run in background
+docker compose up -d # or --detach
+# shutdown containers
+docker compose down
+# shutdown containers included in specific profile
+docker compose --profile aws down
+```
+
+See [Docker Compose reference](https://docs.docker.com/compose/reference/) for more details.
+
 ### Build
 
 To build the project from source, you need Java 21 and Maven 3.
 
 ### Database
 
-#### Via Docker
-
-Install Docker.
+#### Via Docker Compose
 
+Ensure database is up with
 ```shell
-docker run \
-    --platform linux/amd64 \
-    --name=uttu \
-    -d \
-    -e POSTGRES_USER=uttu \
-    -e POSTGRES_PASSWORD=uttu \
-    -e POSTGRES_DB=uttu \
-    -p 5432:5432 \
-    -v db_local:/var/lib/postgresql \
-    --restart=always \
-    postgis/postgis:13-3.3
+docker compose up -d
 ```
 
-Now a Docker container is running in the background. Check its status with `docker ps`.
-
-To stop, find its ID from `docker ps`, and run `docker stop theid` (beginning of hash). To restart it, find the ID from 
-`docker container list` and run `docker restart theid`.
-
 Run the [database initialization script](./src/main/resources/db_init.sh).
 
 ```shell
@@ -129,14 +140,16 @@ Provider-specific GraphQL endpoint (replace {providerCode} with provider's codes
 
     /services/flexible-lines/{providerCode}/graphql
 
-## Netex Export
+## NeTEx Export
 
 Uttu exports (via provider specific GraphQL API) generated NeTEx file to a blobstore repository.
-Choose one of three implementations with profiles:
+Choose one from the available implementations with matching profile:
 
 - `in-memory-blobstore` - stores exports in memory, exports are lost on restarts, suitable for development and testing
 - `disk-blobstore` - stores exports on disk
 - `gcp-blobstore` - stores exports in Google Cloud Storage, requires additional configuration
+- `s3-blobstore` - stores exports in Amazon Web Services Simple Storage Service (AWS S3), requires additional 
+  configuration
 
 Alternatively, provide a
 [`BlobStoreRepository`](https://github.com/entur/rutebanken-helpers/blob/master/storage/src/main/java/org/rutebanken/helper/storage/repository/BlobStoreRepository.java)

docker-compose.yml

@@ -0,0 +1,46 @@
+name: 'uttu'
+
+services:
+  db:
+    container_name: '${COMPOSE_PROJECT_NAME}_postgis13'
+    image: postgis/postgis:13-3.3
+    platform: linux/amd64
+    restart: always
+    environment:
+      POSTGRES_USER: uttu
+      POSTGRES_PASSWORD: uttu
+      POSTGRES_DB: uttu
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres-data:/var/lib/postgresql
+    networks:
+      - uttu
+
+  localstack:
+    container_name: "${COMPOSE_PROJECT_NAME}_localstack"
+    profiles: ["aws"]
+    image: localstack/localstack:3.4.0
+    ports:
+      - "4566:4566"            # LocalStack Gateway
+      - "4510-4559:4510-4559"  # external services port range
+    environment:
+      - DEBUG=${DEBUG-}
+      - DOCKER_HOST=unix:///var/run/docker.sock
+      - DISABLE_EVENTS=1
+      - SERVICES=s3
+      - AWS_ACCESS_KEY_ID=localstack
+      - AWS_SECRET_ACCESS_KEY=localstack
+      - AWS_DEFAULT_REGION=eu-north-1
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock"
+      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
+      - "./scripts/init-localstack.sh:/etc/localstack/init/ready.d/init-localstack.sh"
+    networks:
+      - uttu
+
+volumes:
+  postgres-data:
+
+networks:
+  uttu:

pom.xml

@@ -62,6 +62,8 @@
         <prettier-java.version>2.1.0</prettier-java.version>
         <prettier-maven-plugin.version>0.22</prettier-maven-plugin.version>
         <plugin.prettier.goal>write</plugin.prettier.goal>
+
+        <awssdk.version>2.25.60</awssdk.version>
     </properties>
 
     <distributionManagement>
@@ -83,6 +85,18 @@
         </repository>
     </repositories>
 
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>software.amazon.awssdk</groupId>
+                <artifactId>bom</artifactId>
+                <version>${awssdk.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
     <dependencies>
 
         <!-- Entur -->
@@ -277,6 +291,26 @@
             <scope>test</scope>
         </dependency>
 
+        <!--
+        AWS integration
+        Including HTTP client implementations directly allows for configuring and customization of the clients.
+        See https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/http-configuration-apache.html
+        -->
+        <!-- asynchronous HTTP client for all service clients -->
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>netty-nio-client</artifactId>
+        </dependency>
+        <!-- CRT client is a specialization of the async client which allows for increased performance -->
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>aws-crt-client</artifactId>
+        </dependency>
+        <!-- AWS Simple Storage Service service client -->
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>s3</artifactId>
+        </dependency>
 
         <!-- Test-->
 
@@ -317,6 +351,12 @@
             <groupId>org.testcontainers</groupId>
             <artifactId>gcloud</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>localstack</artifactId>
+            <version>1.19.8</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.assertj</groupId>
             <artifactId>assertj-core</artifactId>

scripts/init-localstack.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# -- > Create S3 bucket for blob storage
+awslocal s3api create-bucket --bucket 'local-blobstore-exports' --create-bucket-configuration LocationConstraint=eu-north-1

src/ext-test/java/no/entur/uttu/ext/entur/export/messaging/EnturPubSubMessagingServiceTest.java

@@ -26,7 +26,12 @@
 import no.entur.uttu.UttuIntegrationTest;
 import no.entur.uttu.export.messaging.spi.MessagingService;
 import org.entur.pubsub.base.EnturGooglePubSubAdmin;
-import org.junit.*;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.test.context.TestConfiguration;
@@ -39,7 +44,7 @@
 import org.testcontainers.utility.DockerImageName;
 
 @Testcontainers
-@ActiveProfiles({ "test", "entur-pubsub-messaging-service" })
+@ActiveProfiles({ "in-memory-blobstore", "entur-pubsub-messaging-service" })
 public class EnturPubSubMessagingServiceTest extends UttuIntegrationTest {
 
   public static final String TEST_CODESPACE = "rut";

src/main/java/no/entur/uttu/export/blob/LocalDiskBlobStoreRepositoryConfig.java

@@ -24,7 +24,7 @@
 import org.springframework.context.annotation.Profile;
 
 @Configuration
-@Profile({ "local", "test", "local-disk-blobstore" })
+@Profile({ "local-disk-blobstore" })
 public class LocalDiskBlobStoreRepositoryConfig {
 
   @Bean