From a528b26bb1ca1a33dfb091bbaf32c2dc40487ced Mon Sep 17 00:00:00 2001 From: DenChaykovskiy <38767269+DenChaykovskiy@users.noreply.github.com> Date: Mon, 24 Apr 2023 17:27:09 +0200 Subject: [PATCH 01/79] Install-all WF: Add minio container for s3 testing --- .github/workflows/install_all.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/install_all.yml b/.github/workflows/install_all.yml index e29f3743f..91f29e443 100644 --- a/.github/workflows/install_all.yml +++ b/.github/workflows/install_all.yml @@ -62,6 +62,12 @@ jobs: docker tag openjdk:11 localhost:5000/openjdk:11 docker push localhost:5000/openjdk:11 + - name: Show current directory + run: pwd + + - name: Add minio container for s3 testing + run: docker compose -f ./storage-mgr/src/test/resources/docker-compose.yml up -d + - name: Show docker images run: docker images From 1d5925c31596013fb55badafd5c6b8cc63bb5fe8 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 16 Nov 2023 08:22:11 +0100 Subject: [PATCH 02/79] storage-mgr: renamed packages (without version2) --- .../FileLockedAfterMaxCyclesException.java | 2 +- .../{version2 => }/Exceptions/package-info.java | 2 +- .../{version2 => }/model/AtomicCommand.java | 2 +- .../model/DefaultRetryStrategy.java | 2 +- .../storagemgr/{version2 => }/model/Storage.java | 2 +- .../{version2 => }/model/StorageFile.java | 2 +- .../{version2 => }/model/StorageType.java | 2 +- .../{version2 => }/model/package-info.java | 2 +- .../{version2 => }/posix/PosixDAL.java | 2 +- .../{version2 => }/posix/PosixStorage.java | 8 ++++---- .../{version2 => }/posix/PosixStorageFile.java | 6 +++--- .../{version2 => }/posix/package-info.java | 2 +- .../storagemgr/rest/JobOrderControllerImpl.java | 2 +- .../storagemgr/rest/ProductControllerImpl.java | 6 +++--- .../rest/ProductfileControllerImpl.java | 6 +++--- .../{version2 => }/s3/S3AtomicBucketCreator.java | 4 ++-- .../{version2 => }/s3/S3AtomicBucketDeleter.java | 4 ++-- .../s3/S3AtomicBucketListGetter.java | 4 ++-- .../s3/S3AtomicFileContentGetter.java | 4 ++-- .../{version2 => }/s3/S3AtomicFileDeleter.java | 4 ++-- .../s3/S3AtomicFileDownloader.java | 4 ++-- .../s3/S3AtomicFileExistsGetter.java | 4 ++-- .../s3/S3AtomicFileListDeleter.java | 4 ++-- .../s3/S3AtomicFileListGetter.java | 4 ++-- .../s3/S3AtomicFileSizeGetter.java | 4 ++-- .../{version2 => }/s3/S3AtomicFileUploader.java | 4 ++-- .../s3/S3AtomicInputStreamGetter.java | 4 ++-- .../{version2 => }/s3/S3Configuration.java | 2 +- .../storagemgr/{version2 => }/s3/S3DAL.java | 6 +++--- .../storagemgr/{version2 => }/s3/S3Storage.java | 10 +++++----- .../{version2 => }/s3/S3StorageFile.java | 6 +++--- .../{version2 => }/s3/package-info.java | 2 +- .../storagemgr/version2/PathConverter.java | 2 +- .../storagemgr/version2/StorageFileLocker.java | 2 +- .../storagemgr/version2/StorageProvider.java | 16 ++++++++-------- .../proseo/storagemgr/BaseStorageTestUtils.java | 6 +++--- .../dlr/proseo/storagemgr/StorageTestUtils.java | 2 +- .../java/de/dlr/proseo/storagemgr/TestUtils.java | 6 +++--- .../ProductfileControllerImplTest_cache.java | 4 ++-- .../JobOrderControllerImplTest_download.java | 2 +- .../rest/JobOrderControllerImplTest_upload.java | 2 +- .../rest/ProductControllerImplTest_delete.java | 2 +- .../rest/ProductControllerImplTest_download.java | 6 +++--- .../rest/ProductControllerImplTest_get.java | 6 +++--- .../rest/ProductControllerImplTest_getAll.java | 2 +- .../rest/ProductControllerImplTest_upload.java | 4 ++-- .../ProductfileControllerImplTest_download.java | 4 ++-- .../ProductfileControllerImplTest_upload.java | 2 +- .../storagemgr/version2/StorageProviderTest.java | 6 +++--- .../storagemgr/version2/StorageUtilityTest.java | 4 ++-- .../storagemgr/version2/posix/PosixDALTest.java | 1 + .../version2/posix/PosixStorageTest.java | 4 ++-- .../storagemgr/version2/s3/S3StorageTest.java | 6 +++--- 53 files changed, 106 insertions(+), 105 deletions(-) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/Exceptions/FileLockedAfterMaxCyclesException.java (92%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/Exceptions/package-info.java (78%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/model/AtomicCommand.java (95%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/model/DefaultRetryStrategy.java (98%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/model/Storage.java (99%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/model/StorageFile.java (96%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/model/StorageType.java (85%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/model/package-info.java (84%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/posix/PosixDAL.java (99%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/posix/PosixStorage.java (98%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/posix/PosixStorageFile.java (95%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/posix/package-info.java (80%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicBucketCreator.java (96%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicBucketDeleter.java (95%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicBucketListGetter.java (96%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicFileContentGetter.java (96%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicFileDeleter.java (97%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicFileDownloader.java (98%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicFileExistsGetter.java (96%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicFileListDeleter.java (97%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicFileListGetter.java (97%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicFileSizeGetter.java (96%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicFileUploader.java (97%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicInputStreamGetter.java (97%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3Configuration.java (98%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3DAL.java (99%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3Storage.java (98%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3StorageFile.java (94%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/package-info.java (80%) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/Exceptions/FileLockedAfterMaxCyclesException.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/Exceptions/FileLockedAfterMaxCyclesException.java similarity index 92% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/Exceptions/FileLockedAfterMaxCyclesException.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/Exceptions/FileLockedAfterMaxCyclesException.java index 96d4c573a..f84c612da 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/Exceptions/FileLockedAfterMaxCyclesException.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/Exceptions/FileLockedAfterMaxCyclesException.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.Exceptions; +package de.dlr.proseo.storagemgr.Exceptions; import java.io.IOException; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/Exceptions/package-info.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/Exceptions/package-info.java similarity index 78% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/Exceptions/package-info.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/Exceptions/package-info.java index 5eb26a784..9f8c36686 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/Exceptions/package-info.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/Exceptions/package-info.java @@ -8,4 +8,4 @@ * * @author Denys Chaykovskiy */ -package de.dlr.proseo.storagemgr.version2.Exceptions; \ No newline at end of file +package de.dlr.proseo.storagemgr.Exceptions; \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/AtomicCommand.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/AtomicCommand.java similarity index 95% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/AtomicCommand.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/AtomicCommand.java index de44d1d1e..6610ebc74 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/AtomicCommand.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/AtomicCommand.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.model; +package de.dlr.proseo.storagemgr.model; import java.io.IOException; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/DefaultRetryStrategy.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/DefaultRetryStrategy.java similarity index 98% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/DefaultRetryStrategy.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/DefaultRetryStrategy.java index 3ddc839a1..c1bfd9055 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/DefaultRetryStrategy.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/DefaultRetryStrategy.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.model; +package de.dlr.proseo.storagemgr.model; import java.io.IOException; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/Storage.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/Storage.java similarity index 99% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/Storage.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/Storage.java index c23a9d855..72c9ce88a 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/Storage.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/Storage.java @@ -2,7 +2,7 @@ * Storage.java * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.model; +package de.dlr.proseo.storagemgr.model; import java.io.IOException; import java.io.InputStream; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/StorageFile.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/StorageFile.java similarity index 96% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/StorageFile.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/StorageFile.java index e0f6c1364..614be1098 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/StorageFile.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/StorageFile.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.model; +package de.dlr.proseo.storagemgr.model; /** * Storage File Structure. The file can exist or it can be just an abstract diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/StorageType.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/StorageType.java similarity index 85% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/StorageType.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/StorageType.java index d3d4cf619..ae2a4ffa9 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/StorageType.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/StorageType.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.model; +package de.dlr.proseo.storagemgr.model; /** * Storage types supported by the prosEO Storage Manager, currently S3 and diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/package-info.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/package-info.java similarity index 84% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/package-info.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/package-info.java index 1efd63c9d..d13c276a7 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/package-info.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/package-info.java @@ -9,4 +9,4 @@ * * @author Denys Chaykovskiy */ -package de.dlr.proseo.storagemgr.version2.model; \ No newline at end of file +package de.dlr.proseo.storagemgr.model; \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/PosixDAL.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java similarity index 99% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/PosixDAL.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java index 67e20269b..2b72fa439 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/PosixDAL.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.posix; +package de.dlr.proseo.storagemgr.posix; import java.io.File; import java.io.IOException; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorage.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorage.java similarity index 98% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorage.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorage.java index 76f3adf41..62720f424 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorage.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorage.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.posix; +package de.dlr.proseo.storagemgr.posix; import java.io.File; import java.io.FileInputStream; @@ -17,11 +17,11 @@ import de.dlr.proseo.logging.logger.ProseoLogger; import de.dlr.proseo.logging.messages.StorageMgrMessage; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.version2.FileUtils; import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; /** * An implementation of the Storage interface for a POSIX-based file system, diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageFile.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorageFile.java similarity index 95% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageFile.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorageFile.java index f1d91c688..8f8be2ed9 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageFile.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorageFile.java @@ -3,16 +3,16 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.posix; +package de.dlr.proseo.storagemgr.posix; import java.io.File; import org.apache.commons.io.FilenameUtils; import de.dlr.proseo.logging.logger.ProseoLogger; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; /** * Represents a file within a POSIX-based storage system. It implements the diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/package-info.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/package-info.java similarity index 80% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/package-info.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/package-info.java index 83f6139c7..3f38761ad 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/package-info.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/package-info.java @@ -8,4 +8,4 @@ * * @author Denys Chaykovskiy */ -package de.dlr.proseo.storagemgr.version2.posix; \ No newline at end of file +package de.dlr.proseo.storagemgr.posix; \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImpl.java index d84edc284..f543b3484 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImpl.java @@ -17,10 +17,10 @@ import org.springframework.stereotype.Component; import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; import de.dlr.proseo.logging.logger.ProseoLogger; import de.dlr.proseo.logging.messages.StorageMgrMessage; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; +import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.rest.model.RestJoborder; /** diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java index 92e7e091f..395563ca5 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java @@ -34,14 +34,14 @@ import de.dlr.proseo.storagemgr.version2.PathConverter; import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; import de.dlr.proseo.logging.http.HttpPrefix; import de.dlr.proseo.logging.http.ProseoHttp; import de.dlr.proseo.logging.logger.ProseoLogger; import de.dlr.proseo.logging.messages.StorageMgrMessage; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestProductFS; /** diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index fe4b2971b..0dfc06a4e 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -17,13 +17,13 @@ import de.dlr.proseo.logging.messages.StorageMgrMessage; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; +import de.dlr.proseo.storagemgr.Exceptions.FileLockedAfterMaxCyclesException; import de.dlr.proseo.storagemgr.cache.FileCache; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; import de.dlr.proseo.storagemgr.version2.StorageFileLocker; import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.Exceptions.FileLockedAfterMaxCyclesException; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; /** * Spring MVC controller for the prosEO Storage Manager; implements the services diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicBucketCreator.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicBucketCreator.java similarity index 96% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicBucketCreator.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicBucketCreator.java index 1285f459d..a33ea9f08 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicBucketCreator.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicBucketCreator.java @@ -3,14 +3,14 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import software.amazon.awssdk.core.waiters.WaiterResponse; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.CreateBucketRequest; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicBucketDeleter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicBucketDeleter.java similarity index 95% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicBucketDeleter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicBucketDeleter.java index eac1d3c5c..17a518936 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicBucketDeleter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicBucketDeleter.java @@ -3,14 +3,14 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicBucketListGetter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicBucketListGetter.java similarity index 96% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicBucketListGetter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicBucketListGetter.java index 009748c5d..91eb67a13 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicBucketListGetter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicBucketListGetter.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import java.util.ArrayList; @@ -12,7 +12,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.Bucket; import software.amazon.awssdk.services.s3.model.ListBucketsResponse; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileContentGetter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileContentGetter.java similarity index 96% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileContentGetter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileContentGetter.java index 0e5593a07..ed1e3a596 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileContentGetter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileContentGetter.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -11,7 +11,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.GetObjectRequest; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileDeleter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileDeleter.java similarity index 97% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileDeleter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileDeleter.java index 8b879a41a..6a18da3c5 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileDeleter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileDeleter.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import java.util.ArrayList; @@ -12,7 +12,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.Delete; import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileDownloader.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileDownloader.java similarity index 98% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileDownloader.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileDownloader.java index 4b43cd42d..c6eaa456b 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileDownloader.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileDownloader.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.File; import java.io.IOException; @@ -20,9 +20,9 @@ import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import de.dlr.proseo.storagemgr.version2.FileUtils; import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; /** * S3 Atomic File Downloader diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileExistsGetter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileExistsGetter.java similarity index 96% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileExistsGetter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileExistsGetter.java index 3a6801133..fca8c343d 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileExistsGetter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileExistsGetter.java @@ -3,14 +3,14 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileListDeleter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileListDeleter.java similarity index 97% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileListDeleter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileListDeleter.java index 28f26933e..7a97d5cd2 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileListDeleter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileListDeleter.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import java.util.ArrayList; @@ -13,7 +13,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.Delete; import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileListGetter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileListGetter.java similarity index 97% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileListGetter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileListGetter.java index 4f1414caf..e226719ce 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileListGetter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileListGetter.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import java.util.ArrayList; @@ -12,7 +12,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.ListObjectsRequest; import software.amazon.awssdk.services.s3.model.ListObjectsResponse; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileSizeGetter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileSizeGetter.java similarity index 96% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileSizeGetter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileSizeGetter.java index 0a2d9d3ad..5cc057d12 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileSizeGetter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileSizeGetter.java @@ -3,14 +3,14 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileUploader.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileUploader.java similarity index 97% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileUploader.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileUploader.java index f88607812..0f30d32fb 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileUploader.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileUploader.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.File; import java.io.IOException; @@ -16,8 +16,8 @@ import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; /** * S3 Atomic Uploader diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicInputStreamGetter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicInputStreamGetter.java similarity index 97% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicInputStreamGetter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicInputStreamGetter.java index bf0c3efc3..6a04c9bde 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicInputStreamGetter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicInputStreamGetter.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import java.io.InputStream; @@ -11,8 +11,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.GetObjectRequest; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3Configuration.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Configuration.java similarity index 98% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3Configuration.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Configuration.java index 0b9cfe4a1..4919a43fb 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3Configuration.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Configuration.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; /** * Holds the configuration settings required to connect to and interact with an diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3DAL.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3DAL.java similarity index 99% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3DAL.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3DAL.java index deb1716c2..0216e1249 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3DAL.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3DAL.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.File; import java.io.IOException; @@ -26,9 +26,9 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import de.dlr.proseo.storagemgr.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.DefaultRetryStrategy; import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; -import de.dlr.proseo.storagemgr.version2.model.DefaultRetryStrategy; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3Storage.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Storage.java similarity index 98% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3Storage.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Storage.java index 2fa49a06b..5856f32f7 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3Storage.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Storage.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.File; import java.io.IOException; @@ -13,12 +13,12 @@ import java.util.List; import de.dlr.proseo.logging.logger.ProseoLogger; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; +import de.dlr.proseo.storagemgr.posix.PosixStorageFile; import de.dlr.proseo.storagemgr.version2.FileUtils; import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; -import de.dlr.proseo.storagemgr.version2.posix.PosixStorageFile; /** * An implementation of the Storage interface for a S3-based file system, diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageFile.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3StorageFile.java similarity index 94% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageFile.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3StorageFile.java index 6eececeb4..804e43706 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageFile.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3StorageFile.java @@ -3,16 +3,16 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.File; import java.nio.file.Paths; import org.apache.commons.io.FilenameUtils; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; /** * Represents a file within a S3-based storage system. It implements the diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/package-info.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/package-info.java similarity index 80% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/package-info.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/package-info.java index 812dca519..c479a6000 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/package-info.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/package-info.java @@ -8,4 +8,4 @@ * * @author Denys Chaykovskiy */ -package de.dlr.proseo.storagemgr.version2.s3; \ No newline at end of file +package de.dlr.proseo.storagemgr.s3; \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/PathConverter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/PathConverter.java index 0ec17d274..b62ecdc58 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/PathConverter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/PathConverter.java @@ -11,7 +11,7 @@ import java.util.List; import de.dlr.proseo.logging.logger.ProseoLogger; -import de.dlr.proseo.storagemgr.version2.model.StorageType; +import de.dlr.proseo.storagemgr.model.StorageType; /** * A set of methods to convert and manipulate file paths based on different diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/StorageFileLocker.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/StorageFileLocker.java index db3e510d8..5ac8df142 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/StorageFileLocker.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/StorageFileLocker.java @@ -8,7 +8,7 @@ import java.util.concurrent.ConcurrentSkipListSet; import de.dlr.proseo.logging.logger.ProseoLogger; -import de.dlr.proseo.storagemgr.version2.Exceptions.FileLockedAfterMaxCyclesException; +import de.dlr.proseo.storagemgr.Exceptions.FileLockedAfterMaxCyclesException; /** * A mechanism to lock and unlock files during the download process. It helps diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/StorageProvider.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/StorageProvider.java index f98a8dea3..6c126e99a 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/StorageProvider.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/StorageProvider.java @@ -20,14 +20,14 @@ import de.dlr.proseo.logging.logger.ProseoLogger; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; -import de.dlr.proseo.storagemgr.version2.posix.PosixStorage; -import de.dlr.proseo.storagemgr.version2.posix.PosixStorageFile; -import de.dlr.proseo.storagemgr.version2.s3.S3Configuration; -import de.dlr.proseo.storagemgr.version2.s3.S3Storage; -import de.dlr.proseo.storagemgr.version2.s3.S3StorageFile; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; +import de.dlr.proseo.storagemgr.posix.PosixStorage; +import de.dlr.proseo.storagemgr.posix.PosixStorageFile; +import de.dlr.proseo.storagemgr.s3.S3Configuration; +import de.dlr.proseo.storagemgr.s3.S3Storage; +import de.dlr.proseo.storagemgr.s3.S3StorageFile; /** * A central manager for different types of storage in the application. It diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java index 42575d4df..868cd777d 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java @@ -6,10 +6,10 @@ import java.nio.file.Paths; import java.util.List; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.posix.PosixDAL; +import de.dlr.proseo.storagemgr.posix.PosixStorageFile; import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.posix.PosixDAL; -import de.dlr.proseo.storagemgr.version2.posix.PosixStorageFile; /** * @author Denys Chaykovskiy diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StorageTestUtils.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StorageTestUtils.java index 8353c1208..6aedf8f50 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StorageTestUtils.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StorageTestUtils.java @@ -10,7 +10,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import de.dlr.proseo.storagemgr.version2.model.Storage; +import de.dlr.proseo.storagemgr.model.Storage; /** * @author Denys Chaykovskiy diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java index c6ae4abd9..cfbe3b214 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java @@ -16,11 +16,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.version2.FileUtils; import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; /** * @author Denys Chaykovskiy diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java index 2f4d1403a..b2a24c82b 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java @@ -27,12 +27,12 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; import de.dlr.proseo.storagemgr.version2.FileUtils; import de.dlr.proseo.storagemgr.version2.PathConverter; import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java index 1a0b73cf5..6ba94da33 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java @@ -26,9 +26,9 @@ import de.dlr.proseo.storagemgr.StorageManagerConfiguration; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestJoborder; import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.StorageType; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java index 88819ad7e..df70763dd 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java @@ -27,9 +27,9 @@ import de.dlr.proseo.storagemgr.StorageManagerConfiguration; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestJoborder; import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.StorageType; /** * Job Order Upload test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java index 56e860208..8a609829d 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java @@ -24,9 +24,9 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.version2.PathConverter; import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.StorageType; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java index 7ace75f99..44460abff 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java @@ -21,11 +21,11 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.version2.PathConverter; import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java index 2afad9397..ef47c22e0 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java @@ -25,11 +25,11 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.version2.PathConverter; import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java index 2f0ff579a..d6564415e 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java @@ -21,8 +21,8 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.StorageType; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java index 2b063fed6..cfb2bb638 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java @@ -27,11 +27,11 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestProductFS; import de.dlr.proseo.storagemgr.version2.PathConverter; import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageType; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java index 4eb1e66a5..c4c2d5188 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java @@ -24,12 +24,12 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; import de.dlr.proseo.storagemgr.version2.FileUtils; import de.dlr.proseo.storagemgr.version2.PathConverter; import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java index d32d32a9b..f52b40eff 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java @@ -22,11 +22,11 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; import de.dlr.proseo.storagemgr.version2.FileUtils; import de.dlr.proseo.storagemgr.version2.PathConverter; import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.StorageType; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java index c03fcb59d..955c13f4f 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java @@ -20,9 +20,9 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java index 9791d9f64..ff7804b6b 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java @@ -17,8 +17,8 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; -import de.dlr.proseo.storagemgr.version2.model.StorageType; -import de.dlr.proseo.storagemgr.version2.model.Storage; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageType; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixDALTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixDALTest.java index 13e2491fb..906859238 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixDALTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixDALTest.java @@ -20,6 +20,7 @@ import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.UniqueStorageTestPaths; +import de.dlr.proseo.storagemgr.posix.PosixDAL; import de.dlr.proseo.storagemgr.version2.PathConverter; @RunWith(SpringJUnit4ClassRunner.class) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java index c82ee5a81..84b5eaf11 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java @@ -19,9 +19,9 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java index de43e2a1b..f936f7204 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java @@ -21,10 +21,10 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; -import de.dlr.proseo.storagemgr.version2.model.StorageType; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) From 0eeecb4d229057f4e8af110c244355452cdbe7b4 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 16 Nov 2023 08:25:54 +0100 Subject: [PATCH 03/79] storage-mgr: deleted StorageLogger, prosEO Logger is used --- .../storagemgr/utils/StorageLogger.java | 73 ------------------- 1 file changed, 73 deletions(-) delete mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageLogger.java diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageLogger.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageLogger.java deleted file mode 100644 index e49ba14e8..000000000 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageLogger.java +++ /dev/null @@ -1,73 +0,0 @@ -package de.dlr.proseo.storagemgr.utils; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.slf4j.Logger; - -/** - * General logger methods - * - * @author Denys Chaykovskiy - * - */ -public class StorageLogger { - - /** - * Create and log a formatted informational message - * - * @param loggger Logger object of the class - * @param messageFormat the message text with parameter placeholders in - * String.format() style - * @param messageId a (unique) message id - * @param messageParameters the message parameters (optional, depending on the - * message format) - * @return a formatted info message - */ - public static String logInfo(Logger logger, String messageFormat, int messageId, Object... messageParameters) { - - String message = createMessage(messageFormat, messageId, messageParameters); - logger.info(message); - - return message; - } - - /** - * Create and log a formatted error message - * - * @param loggger Logger object of the class - * @param messageFormat the message text with parameter placeholders in - * String.format() style - * @param messageId a (unique) message id - * @param messageParameters the message parameters (optional, depending on the - * message format) - * @return a formatted error message - */ - public static String logError(Logger logger, String messageFormat, int messageId, Object... messageParameters) { - - String message = createMessage(messageFormat, messageId, messageParameters); - logger.error(message); - - return message; - } - - /** - * Create a formatted log message - * - * @param messageFormat the message text with parameter placeholders in - * String.format() style - * @param messageId a (unique) message id - * @param messageParameters the message parameters (optional, depending on the - * message format) - * @return a formatted error message - */ - private static String createMessage(String messageFormat, int messageId, Object... messageParameters) { - - List messageParamList = new ArrayList<>(Arrays.asList(messageParameters)); - messageParamList.add(0, messageId); - - return String.format(messageFormat, messageParamList.toArray()); - } - -} From 5aeb49d21c18bb06b759c32692097175f51f73b0 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 16 Nov 2023 08:28:46 +0100 Subject: [PATCH 04/79] storage-mgr: Classes moved from version2 to utils --- .../de/dlr/proseo/storagemgr/{version2 => utils}/FileUtils.java | 0 .../dlr/proseo/storagemgr/{version2 => utils}/PathConverter.java | 0 .../proseo/storagemgr/{version2 => utils}/StorageFileLocker.java | 0 .../proseo/storagemgr/{version2 => utils}/StorageProvider.java | 0 .../dlr/proseo/storagemgr/{version2 => utils}/package-info.java | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => utils}/FileUtils.java (100%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => utils}/PathConverter.java (100%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => utils}/StorageFileLocker.java (100%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => utils}/StorageProvider.java (100%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => utils}/package-info.java (100%) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/FileUtils.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/FileUtils.java similarity index 100% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/FileUtils.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/FileUtils.java diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/PathConverter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/PathConverter.java similarity index 100% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/PathConverter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/PathConverter.java diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/StorageFileLocker.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java similarity index 100% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/StorageFileLocker.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/StorageProvider.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageProvider.java similarity index 100% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/StorageProvider.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageProvider.java diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/package-info.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/package-info.java similarity index 100% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/package-info.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/package-info.java From b396c8d52aec730ce04f47827503014649b6cea3 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 16 Nov 2023 08:30:20 +0100 Subject: [PATCH 05/79] storage-mgr: updated references for moved classes into utils --- .../main/java/de/dlr/proseo/storagemgr/cache/FileCache.java | 2 +- .../main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java | 4 ++-- .../java/de/dlr/proseo/storagemgr/posix/PosixStorage.java | 4 ++-- .../de/dlr/proseo/storagemgr/posix/PosixStorageFile.java | 2 +- .../dlr/proseo/storagemgr/rest/JobOrderControllerImpl.java | 2 +- .../dlr/proseo/storagemgr/rest/ProductControllerImpl.java | 4 ++-- .../proseo/storagemgr/rest/ProductfileControllerImpl.java | 4 ++-- .../de/dlr/proseo/storagemgr/s3/S3AtomicFileDownloader.java | 4 ++-- .../de/dlr/proseo/storagemgr/s3/S3AtomicFileUploader.java | 2 +- .../dlr/proseo/storagemgr/s3/S3AtomicInputStreamGetter.java | 2 +- .../src/main/java/de/dlr/proseo/storagemgr/s3/S3DAL.java | 2 +- .../main/java/de/dlr/proseo/storagemgr/s3/S3Storage.java | 4 ++-- .../java/de/dlr/proseo/storagemgr/s3/S3StorageFile.java | 2 +- .../main/java/de/dlr/proseo/storagemgr/utils/FileUtils.java | 2 +- .../java/de/dlr/proseo/storagemgr/utils/PathConverter.java | 2 +- .../de/dlr/proseo/storagemgr/utils/StorageFileLocker.java | 2 +- .../de/dlr/proseo/storagemgr/utils/StorageProvider.java | 2 +- .../java/de/dlr/proseo/storagemgr/utils/package-info.java | 2 +- .../java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java | 2 +- .../src/test/java/de/dlr/proseo/storagemgr/TestUtils.java | 4 ++-- .../de/dlr/proseo/storagemgr/UniqueStorageTestPaths.java | 2 +- .../java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java | 2 +- .../proseo/storagemgr/cache/FileSystemGeneratorTest.java | 2 +- .../java/de/dlr/proseo/storagemgr/cache/FileUtilsTest.java | 2 +- .../cache/ProductfileControllerImplTest_cache.java | 6 +++--- .../rest/JobOrderControllerImplTest_download.java | 2 +- .../storagemgr/rest/JobOrderControllerImplTest_upload.java | 2 +- .../storagemgr/rest/ProductControllerImplTest_delete.java | 4 ++-- .../storagemgr/rest/ProductControllerImplTest_download.java | 4 ++-- .../storagemgr/rest/ProductControllerImplTest_get.java | 4 ++-- .../storagemgr/rest/ProductControllerImplTest_getAll.java | 2 +- .../storagemgr/rest/ProductControllerImplTest_upload.java | 4 ++-- .../rest/ProductfileControllerImplTest_download.java | 6 +++--- .../rest/ProductfileControllerImplTest_upload.java | 6 +++--- .../dlr/proseo/storagemgr/version2/PathConverterTest.java | 2 ++ .../dlr/proseo/storagemgr/version2/StorageProviderTest.java | 1 + .../dlr/proseo/storagemgr/version2/StorageUtilityTest.java | 1 + .../dlr/proseo/storagemgr/version2/posix/PosixDALTest.java | 2 +- .../proseo/storagemgr/version2/posix/PosixStorageTest.java | 2 +- .../de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java | 2 +- 40 files changed, 58 insertions(+), 54 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java index 4c4b5f03f..3d22e4e7c 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java @@ -19,7 +19,7 @@ import de.dlr.proseo.logging.logger.ProseoLogger; import de.dlr.proseo.logging.messages.StorageMgrMessage; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; -import de.dlr.proseo.storagemgr.version2.FileUtils; +import de.dlr.proseo.storagemgr.utils.FileUtils; /** * File cache for managing files in cache storage. diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java index 2b72fa439..abe0ddada 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java @@ -17,8 +17,8 @@ import de.dlr.proseo.logging.logger.ProseoLogger; import de.dlr.proseo.logging.messages.StorageMgrMessage; -import de.dlr.proseo.storagemgr.version2.FileUtils; -import de.dlr.proseo.storagemgr.version2.PathConverter; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * A data access layer for interacting with a POSIX-based storage system. It diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorage.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorage.java index 62720f424..3763808ee 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorage.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorage.java @@ -20,8 +20,8 @@ import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; -import de.dlr.proseo.storagemgr.version2.FileUtils; -import de.dlr.proseo.storagemgr.version2.PathConverter; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * An implementation of the Storage interface for a POSIX-based file system, diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorageFile.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorageFile.java index 8f8be2ed9..6f4a2fa1e 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorageFile.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorageFile.java @@ -12,7 +12,7 @@ import de.dlr.proseo.logging.logger.ProseoLogger; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; -import de.dlr.proseo.storagemgr.version2.PathConverter; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * Represents a file within a POSIX-based storage system. It implements the diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImpl.java index f543b3484..fb0eac6b3 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImpl.java @@ -16,12 +16,12 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; -import de.dlr.proseo.storagemgr.version2.StorageProvider; import de.dlr.proseo.logging.logger.ProseoLogger; import de.dlr.proseo.logging.messages.StorageMgrMessage; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.rest.model.RestJoborder; +import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Spring MVC controller for the prosEO Storage Manager; implements the services diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java index 395563ca5..0a482801f 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java @@ -32,8 +32,6 @@ import com.nimbusds.jwt.JWTClaimsSet; import com.nimbusds.jwt.SignedJWT; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.StorageProvider; import de.dlr.proseo.logging.http.HttpPrefix; import de.dlr.proseo.logging.http.ProseoHttp; import de.dlr.proseo.logging.logger.ProseoLogger; @@ -43,6 +41,8 @@ import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestProductFS; +import de.dlr.proseo.storagemgr.utils.PathConverter; +import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Spring MVC controller for the prosEO Storage Manager; implements the services diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index 0dfc06a4e..addf6d411 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -22,8 +22,8 @@ import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; -import de.dlr.proseo.storagemgr.version2.StorageFileLocker; -import de.dlr.proseo.storagemgr.version2.StorageProvider; +import de.dlr.proseo.storagemgr.utils.StorageFileLocker; +import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Spring MVC controller for the prosEO Storage Manager; implements the services diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileDownloader.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileDownloader.java index c6eaa456b..9a3e72b8f 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileDownloader.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileDownloader.java @@ -21,8 +21,8 @@ import com.amazonaws.services.s3.transfer.TransferManagerBuilder; import de.dlr.proseo.storagemgr.model.AtomicCommand; -import de.dlr.proseo.storagemgr.version2.FileUtils; -import de.dlr.proseo.storagemgr.version2.PathConverter; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * S3 Atomic File Downloader diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileUploader.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileUploader.java index 0f30d32fb..90d69eb67 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileUploader.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileUploader.java @@ -17,7 +17,7 @@ import com.amazonaws.services.s3.transfer.TransferManagerBuilder; import de.dlr.proseo.storagemgr.model.AtomicCommand; -import de.dlr.proseo.storagemgr.version2.PathConverter; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * S3 Atomic Uploader diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicInputStreamGetter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicInputStreamGetter.java index 6a04c9bde..cc927c545 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicInputStreamGetter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicInputStreamGetter.java @@ -12,7 +12,7 @@ import org.slf4j.LoggerFactory; import de.dlr.proseo.storagemgr.model.AtomicCommand; -import de.dlr.proseo.storagemgr.version2.PathConverter; +import de.dlr.proseo.storagemgr.utils.PathConverter; import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.GetObjectRequest; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3DAL.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3DAL.java index 0216e1249..661145677 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3DAL.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3DAL.java @@ -28,7 +28,7 @@ import de.dlr.proseo.storagemgr.model.AtomicCommand; import de.dlr.proseo.storagemgr.model.DefaultRetryStrategy; -import de.dlr.proseo.storagemgr.version2.PathConverter; +import de.dlr.proseo.storagemgr.utils.PathConverter; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Storage.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Storage.java index 5856f32f7..209eb6a96 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Storage.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Storage.java @@ -17,8 +17,8 @@ import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.posix.PosixStorageFile; -import de.dlr.proseo.storagemgr.version2.FileUtils; -import de.dlr.proseo.storagemgr.version2.PathConverter; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * An implementation of the Storage interface for a S3-based file system, diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3StorageFile.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3StorageFile.java index 804e43706..fb25ba8ef 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3StorageFile.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3StorageFile.java @@ -12,7 +12,7 @@ import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; -import de.dlr.proseo.storagemgr.version2.PathConverter; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * Represents a file within a S3-based storage system. It implements the diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/FileUtils.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/FileUtils.java index d76d85613..dc417a2ab 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/FileUtils.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/FileUtils.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2; +package de.dlr.proseo.storagemgr.utils; import java.io.File; import java.io.FileWriter; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/PathConverter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/PathConverter.java index b62ecdc58..61ea2dcab 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/PathConverter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/PathConverter.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2; +package de.dlr.proseo.storagemgr.utils; import java.io.File; import java.nio.file.Paths; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java index 5ac8df142..888672e4c 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2; +package de.dlr.proseo.storagemgr.utils; import java.util.concurrent.ConcurrentSkipListSet; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageProvider.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageProvider.java index 6c126e99a..6d6865e3e 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageProvider.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageProvider.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2; +package de.dlr.proseo.storagemgr.utils; import java.io.IOException; import java.io.InputStream; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/package-info.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/package-info.java index 5cd76aeb4..0943b26bb 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/package-info.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/package-info.java @@ -8,4 +8,4 @@ * * @author Denys Chaykovskiy */ -package de.dlr.proseo.storagemgr.version2; \ No newline at end of file +package de.dlr.proseo.storagemgr.utils; \ No newline at end of file diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java index 868cd777d..cb325fb32 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java @@ -9,7 +9,7 @@ import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.posix.PosixDAL; import de.dlr.proseo.storagemgr.posix.PosixStorageFile; -import de.dlr.proseo.storagemgr.version2.PathConverter; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * @author Denys Chaykovskiy diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java index cfbe3b214..cefc5ea93 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java @@ -19,8 +19,8 @@ import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; -import de.dlr.proseo.storagemgr.version2.FileUtils; -import de.dlr.proseo.storagemgr.version2.StorageProvider; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * @author Denys Chaykovskiy diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/UniqueStorageTestPaths.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/UniqueStorageTestPaths.java index 2ba3c7a25..a1a2da79a 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/UniqueStorageTestPaths.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/UniqueStorageTestPaths.java @@ -4,7 +4,7 @@ import org.junit.rules.TestName; -import de.dlr.proseo.storagemgr.version2.PathConverter; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * @author Denys Chaykovskiy diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java index aaab28234..5b7e09642 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java @@ -20,7 +20,7 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.TestUtils; -import de.dlr.proseo.storagemgr.version2.FileUtils; +import de.dlr.proseo.storagemgr.utils.FileUtils; /** * @author Denys Chaykovskiy diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileSystemGeneratorTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileSystemGeneratorTest.java index 20ad2a2f9..d0cdd6c11 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileSystemGeneratorTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileSystemGeneratorTest.java @@ -1,6 +1,6 @@ package de.dlr.proseo.storagemgr.cache; -import de.dlr.proseo.storagemgr.version2.FileUtils; +import de.dlr.proseo.storagemgr.utils.FileUtils; /** * @author Denys Chaykovskiy diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileUtilsTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileUtilsTest.java index 5ac990198..4941f77e1 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileUtilsTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileUtilsTest.java @@ -18,7 +18,7 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.TestUtils; -import de.dlr.proseo.storagemgr.version2.FileUtils; +import de.dlr.proseo.storagemgr.utils.FileUtils; /** * @author Denys Chaykovskiy diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java index b2a24c82b..3f1cdd7c5 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java @@ -30,9 +30,9 @@ import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; -import de.dlr.proseo.storagemgr.version2.FileUtils; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.StorageProvider; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; +import de.dlr.proseo.storagemgr.utils.StorageProvider; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java index 6ba94da33..a5371ebe0 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java @@ -28,7 +28,7 @@ import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestJoborder; -import de.dlr.proseo.storagemgr.version2.StorageProvider; +import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java index df70763dd..d203839da 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java @@ -29,7 +29,7 @@ import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestJoborder; -import de.dlr.proseo.storagemgr.version2.StorageProvider; +import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Job Order Upload test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java index 8a609829d..bc8b19d8a 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java @@ -25,8 +25,8 @@ import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageType; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.StorageProvider; +import de.dlr.proseo.storagemgr.utils.PathConverter; +import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java index 44460abff..b2ac43077 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java @@ -24,8 +24,8 @@ import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.StorageProvider; +import de.dlr.proseo.storagemgr.utils.PathConverter; +import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java index ef47c22e0..e54af1057 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java @@ -28,8 +28,8 @@ import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.StorageProvider; +import de.dlr.proseo.storagemgr.utils.PathConverter; +import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java index d6564415e..d762f537e 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java @@ -22,7 +22,7 @@ import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageType; -import de.dlr.proseo.storagemgr.version2.StorageProvider; +import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java index cfb2bb638..698c13d16 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java @@ -30,8 +30,8 @@ import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestProductFS; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.StorageProvider; +import de.dlr.proseo.storagemgr.utils.PathConverter; +import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java index c4c2d5188..4a3ca6d62 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java @@ -27,9 +27,9 @@ import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; -import de.dlr.proseo.storagemgr.version2.FileUtils; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.StorageProvider; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; +import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java index f52b40eff..c5123f91c 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java @@ -24,9 +24,9 @@ import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; -import de.dlr.proseo.storagemgr.version2.FileUtils; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.StorageProvider; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; +import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/PathConverterTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/PathConverterTest.java index 12a39c661..ecf537d4c 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/PathConverterTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/PathConverterTest.java @@ -7,6 +7,8 @@ import org.junit.Test; +import de.dlr.proseo.storagemgr.utils.PathConverter; + public class PathConverterTest { @Test diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java index 955c13f4f..243e04a09 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java @@ -23,6 +23,7 @@ import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; +import de.dlr.proseo.storagemgr.utils.StorageProvider; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java index ff7804b6b..f4104526f 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java @@ -19,6 +19,7 @@ import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageType; +import de.dlr.proseo.storagemgr.utils.StorageProvider; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixDALTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixDALTest.java index 906859238..c10854df9 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixDALTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixDALTest.java @@ -21,7 +21,7 @@ import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.UniqueStorageTestPaths; import de.dlr.proseo.storagemgr.posix.PosixDAL; -import de.dlr.proseo.storagemgr.version2.PathConverter; +import de.dlr.proseo.storagemgr.utils.PathConverter; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java index 84b5eaf11..debd24987 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java @@ -21,7 +21,7 @@ import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; -import de.dlr.proseo.storagemgr.version2.StorageProvider; +import de.dlr.proseo.storagemgr.utils.StorageProvider; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java index f936f7204..e3add0ca5 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java @@ -24,7 +24,7 @@ import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; -import de.dlr.proseo.storagemgr.version2.StorageProvider; +import de.dlr.proseo.storagemgr.utils.StorageProvider; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) From b1e2bcd0e9ac2b23e86d1ec0bcd0ccea95b06b03 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 16 Nov 2023 08:42:03 +0100 Subject: [PATCH 06/79] storage-mgr: moved storage provider class to the highest SM level --- .../de/dlr/proseo/storagemgr/{utils => }/StorageProvider.java | 4 ++-- .../de/dlr/proseo/storagemgr/rest/JobOrderControllerImpl.java | 2 +- .../de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java | 2 +- .../dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java | 2 +- .../src/test/java/de/dlr/proseo/storagemgr/TestUtils.java | 1 - .../storagemgr/cache/ProductfileControllerImplTest_cache.java | 2 +- .../storagemgr/rest/JobOrderControllerImplTest_download.java | 2 +- .../storagemgr/rest/JobOrderControllerImplTest_upload.java | 2 +- .../storagemgr/rest/ProductControllerImplTest_delete.java | 2 +- .../storagemgr/rest/ProductControllerImplTest_download.java | 2 +- .../proseo/storagemgr/rest/ProductControllerImplTest_get.java | 2 +- .../storagemgr/rest/ProductControllerImplTest_getAll.java | 2 +- .../storagemgr/rest/ProductControllerImplTest_upload.java | 2 +- .../rest/ProductfileControllerImplTest_download.java | 2 +- .../storagemgr/rest/ProductfileControllerImplTest_upload.java | 2 +- .../dlr/proseo/storagemgr/version2/StorageProviderTest.java | 2 +- .../de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java | 2 +- .../proseo/storagemgr/version2/posix/PosixStorageTest.java | 2 +- .../de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java | 2 +- 19 files changed, 19 insertions(+), 20 deletions(-) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{utils => }/StorageProvider.java (99%) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageProvider.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageProvider.java similarity index 99% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageProvider.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageProvider.java index 6d6865e3e..844ee5ab7 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageProvider.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageProvider.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.utils; +package de.dlr.proseo.storagemgr; import java.io.IOException; import java.io.InputStream; @@ -19,7 +19,6 @@ import org.springframework.stereotype.Component; import de.dlr.proseo.logging.logger.ProseoLogger; -import de.dlr.proseo.storagemgr.StorageManagerConfiguration; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; @@ -28,6 +27,7 @@ import de.dlr.proseo.storagemgr.s3.S3Configuration; import de.dlr.proseo.storagemgr.s3.S3Storage; import de.dlr.proseo.storagemgr.s3.S3StorageFile; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * A central manager for different types of storage in the application. It diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImpl.java index fb0eac6b3..673487229 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImpl.java @@ -19,9 +19,9 @@ import de.dlr.proseo.logging.logger.ProseoLogger; import de.dlr.proseo.logging.messages.StorageMgrMessage; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; +import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.rest.model.RestJoborder; -import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Spring MVC controller for the prosEO Storage Manager; implements the services diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java index 0a482801f..411c6843d 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java @@ -37,12 +37,12 @@ import de.dlr.proseo.logging.logger.ProseoLogger; import de.dlr.proseo.logging.messages.StorageMgrMessage; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; +import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestProductFS; import de.dlr.proseo.storagemgr.utils.PathConverter; -import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Spring MVC controller for the prosEO Storage Manager; implements the services diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index addf6d411..043237996 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -17,13 +17,13 @@ import de.dlr.proseo.logging.messages.StorageMgrMessage; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; +import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.Exceptions.FileLockedAfterMaxCyclesException; import de.dlr.proseo.storagemgr.cache.FileCache; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; import de.dlr.proseo.storagemgr.utils.StorageFileLocker; -import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Spring MVC controller for the prosEO Storage Manager; implements the services diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java index cefc5ea93..88fe167d5 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java @@ -20,7 +20,6 @@ import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.utils.FileUtils; -import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * @author Denys Chaykovskiy diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java index 3f1cdd7c5..68bd3053d 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java @@ -25,6 +25,7 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; import de.dlr.proseo.storagemgr.StorageManager; +import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageFile; @@ -32,7 +33,6 @@ import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; import de.dlr.proseo.storagemgr.utils.FileUtils; import de.dlr.proseo.storagemgr.utils.PathConverter; -import de.dlr.proseo.storagemgr.utils.StorageProvider; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java index a5371ebe0..91166ba30 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java @@ -24,11 +24,11 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; +import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestJoborder; -import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java index d203839da..848839a52 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java @@ -25,11 +25,11 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; +import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestJoborder; -import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Job Order Upload test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java index bc8b19d8a..f0c05e882 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java @@ -22,11 +22,11 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; import de.dlr.proseo.storagemgr.StorageManager; +import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.utils.PathConverter; -import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java index b2ac43077..ffc22acbf 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java @@ -19,13 +19,13 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; import de.dlr.proseo.storagemgr.StorageManager; +import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.utils.PathConverter; -import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java index e54af1057..10088007f 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java @@ -23,13 +23,13 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; import de.dlr.proseo.storagemgr.StorageManager; +import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.utils.PathConverter; -import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java index d762f537e..d246fc390 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java @@ -19,10 +19,10 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; import de.dlr.proseo.storagemgr.StorageManager; +import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageType; -import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java index 698c13d16..6679fe7a7 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java @@ -25,13 +25,13 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; import de.dlr.proseo.storagemgr.StorageManager; +import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestProductFS; import de.dlr.proseo.storagemgr.utils.PathConverter; -import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java index 4a3ca6d62..5ad2e837d 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java @@ -22,6 +22,7 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; import de.dlr.proseo.storagemgr.StorageManager; +import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageFile; @@ -29,7 +30,6 @@ import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; import de.dlr.proseo.storagemgr.utils.FileUtils; import de.dlr.proseo.storagemgr.utils.PathConverter; -import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java index c5123f91c..f9dd9566e 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java @@ -20,13 +20,13 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; import de.dlr.proseo.storagemgr.StorageManager; +import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; import de.dlr.proseo.storagemgr.utils.FileUtils; import de.dlr.proseo.storagemgr.utils.PathConverter; -import de.dlr.proseo.storagemgr.utils.StorageProvider; /** * Mock Mvc test for Product Controller diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java index 243e04a09..3dfaa68aa 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java @@ -18,12 +18,12 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import de.dlr.proseo.storagemgr.StorageManager; +import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; -import de.dlr.proseo.storagemgr.utils.StorageProvider; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java index f4104526f..d2bb56d58 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java @@ -15,11 +15,11 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import de.dlr.proseo.storagemgr.StorageManager; +import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageType; -import de.dlr.proseo.storagemgr.utils.StorageProvider; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java index debd24987..8bce48203 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java @@ -17,11 +17,11 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import de.dlr.proseo.storagemgr.StorageManager; +import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; -import de.dlr.proseo.storagemgr.utils.StorageProvider; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java index e3add0ca5..3ea6f3a45 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java @@ -19,12 +19,12 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import de.dlr.proseo.storagemgr.StorageManager; +import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.StorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; -import de.dlr.proseo.storagemgr.utils.StorageProvider; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) From ead4581821e3528a3f8c6a65f25e683daa5cb6ca Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 16 Nov 2023 09:02:56 +0100 Subject: [PATCH 07/79] storage-mgr: added package-info to the cache package --- .../de/dlr/proseo/storagemgr/cache/package-info.java | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/package-info.java diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/package-info.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/package-info.java new file mode 100644 index 000000000..5bbecda96 --- /dev/null +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/package-info.java @@ -0,0 +1,11 @@ +/** + * package-info.java + * + * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH + */ +/** + * Cache classes for the prosEO Storage Manager component + * + * @author Denys Chaykovskiy + */ +package de.dlr.proseo.storagemgr.cache; \ No newline at end of file From ba1c469dfd1c8e87c7ba967833ad331d800b8565 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 16 Nov 2023 17:24:54 +0100 Subject: [PATCH 08/79] storage-mgr: added Cache file status enum --- .../storagemgr/cache/CacheFileStatus.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java new file mode 100644 index 000000000..611bfdb9b --- /dev/null +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java @@ -0,0 +1,20 @@ +/** + * CacheFileStatus.java + * + * (C) 2023 Dr. Bassler & Co. Managementberatung GmbH + */ +package de.dlr.proseo.storagemgr.cache; + +/** + * Cache File Status shows the current status of the file in the cache + * + * @author Denys Chaykovskiy + */ +public enum CacheFileStatus { + /** Status READY */ + READY, + /** Status UPLOADING */ + UPLOADING, + /** Status DOWNLOADING */ + DOWNLOADING, +} \ No newline at end of file From 142f0f75ec100519c8eae04605154a69a9d44188 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Mon, 20 Nov 2023 08:54:56 +0100 Subject: [PATCH 09/79] storage-mgr: added status functionality to the cache --- .../proseo/storagemgr/cache/FileCache.java | 165 ++++++++++++++++-- 1 file changed, 150 insertions(+), 15 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java index 3d22e4e7c..161742dbc 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java @@ -39,7 +39,10 @@ public class FileCache { /** Prefix for temporary files */ private static final String TEMPORARY_PREFIX = "temporary-"; - /** Cache Map for storing file pathes */ + /** Prefix for status files */ + private static final String STATUS_PREFIX = "status-"; + + /** Cache Map for storing file paths */ private MapCache mapCache; @Autowired @@ -94,6 +97,8 @@ public void put(String pathKey) { deleteLRU(); } + rewriteStatusPrefixFile(pathKey, CacheFileStatus.READY); + rewriteAccessedPrefixFile(pathKey); FileInfo fileInfo = new FileInfo(getFileAccessed(pathKey), getFileSize(pathKey)); @@ -138,7 +143,51 @@ public boolean containsKey(String pathKey) { public static String getTemporaryPrefix() { return TEMPORARY_PREFIX; } + + /** + * Returns a status of the cache file + * + * @param path The full path to the cache file + * @return a status of the cache file + */ + public CacheFileStatus getCacheFileStatus(String path) { + + if (logger.isTraceEnabled()) + logger.trace(">>> getCacheFileStatus({})", path); + + String status; + FileUtils fileUtils = new FileUtils(getStatusPath(path)); + + if (!hasStatus(path)) { + rewriteStatusPrefixFile(path, CacheFileStatus.READY); + } + + status = fileUtils.getFileContent(); + return CacheFileStatus.valueOf(status); + } + + /** + * Sets the status of a cache file + * + * @param pathKey File path as key + * @return a status of the cache file + */ + public void setCacheFileStatus(String pathKey, CacheFileStatus status) { + + if (logger.isTraceEnabled()) + logger.trace(">>> setCacheFileStatus({}, {})", pathKey, status); + + if (!mapCache.containsKey(pathKey)) { + if (!new File(pathKey).exists()) { + logger.log(StorageMgrMessage.CACHE_NO_FILE_FOR_PUTTING_TO_CACHE, pathKey); + } + return; + } + + rewriteStatusPrefixFile(pathKey, status); + } + /** * Initializes file cache with directory from Application.yml */ @@ -277,7 +326,7 @@ private double getRealUsage() { /** * Gets the path key from file cache * - * @param pathKey Path to file + * @param pathKey Path to the cache file * @return */ /* package */ FileInfo get(String pathKey) { @@ -286,7 +335,7 @@ private double getRealUsage() { } /** - * Removes cache element, file and accessed file + * Removes cache element, cache file and auxiliary files * * @param pathKey Path to file */ @@ -295,7 +344,7 @@ private double getRealUsage() { if (logger.isTraceEnabled()) logger.trace(">>> remove({}) - last accessed {}", pathKey, mapCache.get(pathKey).getAccessed()); - deleteFileAndAccessedPrefixFile(pathKey); + deleteCacheFileAndAuxPrefixFiles(pathKey); mapCache.remove(pathKey); } @@ -459,20 +508,24 @@ else if (file.isFile() && isCacheFile(file.getPath())) { } /** - * Deletes if exist file and logically connected accessed file from the disk + * Deletes a cache file and logically connected auxiliary files from the disk * - * @param path full path to the file + * @param path full path to the cache file */ - /* package */ void deleteFileAndAccessedPrefixFile(String path) { + /* package */ void deleteCacheFileAndAuxPrefixFiles(String path) { if (logger.isTraceEnabled()) - logger.trace(">>> deleteFileAndAccessed({})", path); + logger.trace(">>> deleteFileAndAuxPrefixFiles({})", path); - String accessedPath = getAccessedPath(path); String directory = new File(path).getParent(); deleteFile(path); - deleteFile(accessedPath); + + deleteFile(getAccessedPath(path)); + deleteFile(getStatusPath(path)); + deleteFile(getTemporaryPath(path)); + + deleteEmptyDirectoriesToTop(directory); } @@ -558,6 +611,40 @@ else if (file.isFile() && isCacheFile(file.getPath())) { return accessedPath; } + + /** + * Gets the path of the status file + * + * @param path The full path to the cache file + * @return the path to the status file of the cache file + */ + /* package */ String getStatusPath(String path) { + + if (logger.isTraceEnabled()) + logger.trace(">>> getStatusPath({})", path); + + File file = new File(path); + String statusPath = file.getParent() + "/" + STATUS_PREFIX + file.getName(); + + return statusPath; + } + + /** + * Gets the path of the temporary file + * + * @param path The full path to the cache file + * @return the path to the temporary file of the cache file + */ + /* package */ String getTemporaryPath(String path) { + + if (logger.isTraceEnabled()) + logger.trace(">>> getTemporaryPath({})", path); + + File file = new File(path); + String temporaryPath = file.getParent() + "/" + TEMPORARY_PREFIX + file.getName(); + + return temporaryPath; + } /** * Gets the path to file from accessed path @@ -575,7 +662,7 @@ else if (file.isFile() && isCacheFile(file.getPath())) { return path; } - + /** * Returns true if the file was accessed * @@ -588,16 +675,33 @@ private boolean wasAccessed(String path) { return f.isFile(); } + + /** + * Returns true if the cache file has a status + * + * @param path the full path to the cache file + * @return true if the file was accessed + */ + private boolean hasStatus(String path) { + + File f = new File(getStatusPath(path)); + + return f.isFile(); + } /** - * Returns true if the file is the cache file (not accessed, not temporary and - * not hidden file) + * Returns true if the file is the cache file (starts with the cache path, not accessed, not temporary, + * not hidden and not status file) * * @param path the full path to the file * @return true if the file is the cache file */ private boolean isCacheFile(String path) { + if (!path.startsWith(cachePath)) { + return false; + } + if (isAccessedPrefixFile(path)) { return false; } @@ -605,11 +709,14 @@ private boolean isCacheFile(String path) { if (isTemporaryPrefixFile(path)) { return false; } + + if (isStatusPrefixFile(path)) { + return false; + } if (isHiddenFile(path)) { return false; - - } + } return true; } @@ -635,6 +742,17 @@ private boolean isTemporaryPrefixFile(String path) { return hasFilePrefix(path, TEMPORARY_PREFIX); } + + /** + * Returns true if the file has the status prefix + * + * @param path the full path to the cache file + * @return true if the cache file path has the status prefix + */ + private boolean isStatusPrefixFile(String path) { + + return hasFilePrefix(path, STATUS_PREFIX); + } /** * Returns true if the file is a hidden file @@ -676,6 +794,23 @@ private void rewriteAccessedPrefixFile(String path) { fileUtils.createFile(timeStamp); } + + /** + * Rewrites status prefix file with the current status + * + * @param path The full path to the cache file + * @param status The + */ + private void rewriteStatusPrefixFile(String path, CacheFileStatus status) { + + if (logger.isTraceEnabled()) + logger.trace(">>> rewriteStatusPrefixFile({}, {})", path, status); + + String statusPath = getStatusPath(path); + FileUtils fileUtils = new FileUtils(statusPath); + + fileUtils.createFile(status.toString()); + } /** * Deletes file From 998f455d4eb6cc77cbbbbc2ef2b76ff67c585d2e Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Tue, 21 Nov 2023 16:05:19 +0100 Subject: [PATCH 10/79] storage-mgr: created a test for the file cache status --- .../storagemgr/cache/FileCacheTest.java | 294 +++++++++++------- 1 file changed, 187 insertions(+), 107 deletions(-) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java index 5b7e09642..be6e8f1e1 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java @@ -40,16 +40,16 @@ public class FileCacheTest { @Autowired private FileCache fileCache; - - String testCachePath; - String cachePath; - + + String testCachePath; + String cachePath; + @PostConstruct private void init() { testCachePath = testUtils.getTestCachePath(); cachePath = testUtils.getCachePath(); } - + /** * */ @@ -58,23 +58,28 @@ public void testDeleteEmptyDirectoriesToTop() { TestUtils.printMethodName(this, testName); TestUtils.createEmptyTestDirectories(); - + String emptyDirectories = testCachePath + "/d1/d2/d3"; - - TestUtils.createDirectory(emptyDirectories); - File f = new File(emptyDirectories); - - TestUtils.printDirectoryTree(testCachePath); + + // create test directories + + TestUtils.createDirectory(emptyDirectories); + + TestUtils.printDirectoryTree("Directories after creation (expectected: /d1/d2/d3 ):", testCachePath); assertTrue("Empty Directories were not created: " + emptyDirectories, f.exists()); + // delete test directories + fileCache.deleteEmptyDirectoriesToTop(emptyDirectories); - TestUtils.printDirectoryTree(cachePath); + TestUtils.printDirectoryTree("Directories after deletion (expectected: nothing)", cachePath); assertTrue("Empty Directories were not deleted: " + emptyDirectories, !f.exists()); - + + // clear + TestUtils.deleteTestDirectories(); } @@ -87,13 +92,13 @@ public void testGetLastAccessed() { TestUtils.printMethodName(this, testName); TestUtils.createEmptyTestDirectories(); fileCache.setPath(testCachePath); - + String testFile = "testLastAccessed.txt"; String path = testCachePath + "/" + testFile; FileUtils fileUtils = new FileUtils(path); fileUtils.createFile(""); - + Instant testInstant; testInstant = fileCache.getFileAccessed(path); @@ -103,7 +108,7 @@ public void testGetLastAccessed() { File f = new File(fileCache.getAccessedPath(path)); assertTrue("Last Accessed File not exists: " + f.getPath(), f.exists() && !f.isDirectory()); - + fileCache.clear(); TestUtils.deleteTestDirectories(); } @@ -127,7 +132,7 @@ public void testGetAccessedPath() { System.out.println("Accessed Path: " + accessedPath); System.out.println("Expected Accessed Path: " + expectedAccessedPath); - assertTrue("Accessed Path is wrong: " + accessedPath, accessedPath.equals(expectedAccessedPath)); + assertTrue("Accessed Path is wrong: " + accessedPath, accessedPath.equals(expectedAccessedPath)); } /** @@ -145,20 +150,19 @@ public void testGetPutContainsRemove() { String path3 = Paths.get(testCachePath + "/test1/test2/test3.txt").toString(); String path4 = Paths.get(testCachePath + "/test4.x").toString(); String path5 = Paths.get(testCachePath + "/test5.x").toString(); - - String pathNotExists = Paths.get(testCachePath + "/xxx/xxx/zzz.txt").toString(); - + + String pathNotExists = Paths.get(testCachePath + "/xxx/xxx/zzz.txt").toString(); + TestUtils.createFile(path1, ""); TestUtils.createFile(path2, ""); TestUtils.createFile(path3, ""); - + TestUtils.createFile(path4, ""); TestUtils.createFile(path5, ""); - + fileCache.putFilesToCache(testCachePath); - assertTrue("Cache does not contain 5 elements after dir init: " + fileCache.size(), - fileCache.size() == 5); + assertTrue("Cache does not contain 5 elements after dir init: " + fileCache.size(), fileCache.size() == 5); System.out.println("Before adding the element: " + path1); @@ -182,133 +186,209 @@ public void testGetPutContainsRemove() { MapCacheTest.printCache("Cache after deleting 1 element: " + path3, fileCache.getMapCache()); TestUtils.printDirectoryTree(testCachePath); - + fileCache.clear(); TestUtils.deleteTestDirectories(); } + + /** + * + */ + @Test + public void testStatus() { + + TestUtils.printMethodName(this, testName); + TestUtils.createEmptyTestDirectories(); + fileCache.setPath(testCachePath); + + String path1 = Paths.get(testCachePath + "/" + "testStatus1.txt").toString(); + String path2 = Paths.get(testCachePath + "/" + "testStatus2.txt").toString(); + + CacheFileStatus status1; + CacheFileStatus status2; + + // create 2 files for cache + + TestUtils.createFile(path1, ""); + TestUtils.createFile(path2, ""); + + // put 2 files to cache + + fileCache.putFilesToCache(testCachePath); + + assertTrue("Expected Cache size is 2 elements after put(). Exists: " + fileCache.size(), fileCache.size() == 2); + + // check status after put() + + status1 = fileCache.getCacheFileStatus(path1); + status2 = fileCache.getCacheFileStatus(path2); + + assertTrue("Expected cache file1 status after put() is Ready. Exists: " + status1.toString(), + status1 == CacheFileStatus.READY); + + assertTrue("Expected cache file2 status after put() is Ready. Exists: " + status2.toString(), + status2 == CacheFileStatus.READY); + + // changing status1 to Uploading + + fileCache.setCacheFileStatus(path1, CacheFileStatus.UPLOADING); + + status1 = fileCache.getCacheFileStatus(path1); + status2 = fileCache.getCacheFileStatus(path2); + + assertTrue("Expected cache file1 status after setStatus(Uploading) is Uploading. Exists:" + status1.toString(), + status1 == CacheFileStatus.UPLOADING); + + assertTrue("Expected cache file2 status after is Ready. Exists:" + status2.toString(), + status2 == CacheFileStatus.READY); + + // changing status2 to Downloading + + fileCache.setCacheFileStatus(path2, CacheFileStatus.DOWNLOADING); + + status1 = fileCache.getCacheFileStatus(path1); + status2 = fileCache.getCacheFileStatus(path2); + + assertTrue("Expected cache file1 status after is Uploading. Exists: " + status1.toString(), + status1 == CacheFileStatus.UPLOADING); + + assertTrue("Expected cache file2 status after setStatus(Downloading) is Downlading. Exists: " + status2.toString(), + status2 == CacheFileStatus.DOWNLOADING); + + // changing status1 to Ready + + fileCache.setCacheFileStatus(path1, CacheFileStatus.READY); + + status1 = fileCache.getCacheFileStatus(path1); + status2 = fileCache.getCacheFileStatus(path2); + + assertTrue("Expected cache file1 status after setStatus(Ready) is Ready. Exists: " + status1.toString(), + status1 == CacheFileStatus.READY); + + assertTrue("Expected cache file2 status after is Downloading. Exists: " + status2.toString(), + status2 == CacheFileStatus.DOWNLOADING); + // check cache after status changes + + assertTrue("Expected 2 elements in the cache. Exists: " + fileCache.size(), fileCache.size() == 2); + + // clear + + fileCache.clear(); + TestUtils.deleteTestDirectories(); + } + /** * */ @Test public void testInterface() { - + TestUtils.printMethodName(this, testName); TestUtils.createEmptyTestDirectories(); fileCache.setPath(testCachePath); - + String path1 = Paths.get(testCachePath + "/" + "test1.txt").toString(); String path2 = Paths.get(testCachePath + "/" + "test2.txt").toString(); String path3 = Paths.get(testCachePath + "/" + "test3.txt").toString(); String pathNotExists = Paths.get(testCachePath + "/xxx/xxx/" + " zzz.txt").toString(); Instant timeNotChanged; - Instant timeNotChanged2; - Instant timeChanged; - + Instant timeNotChanged2; + Instant timeChanged; + TestUtils.createFile(path1, ""); TestUtils.createFile(path2, ""); - + fileCache.putFilesToCache(testCachePath); - - assertTrue("Cache does not contain 2 elements after dir init: " + fileCache.size(), - fileCache.size() == 2); - - // check containsKey - contains and update accessed - + + assertTrue("Cache does not contain 2 elements after dir init: " + fileCache.size(), fileCache.size() == 2); + + // check containsKey - contains and update accessed + System.out.println("Subtest: check containsKey - contains and update accessed "); - + timeChanged = fileCache.getFileAccessed(path1); - timeNotChanged = fileCache.getFileAccessed(path2); - - assertTrue("Cache does not contain an elements after dir init: " + path1, - fileCache.containsKey(path1)); - + timeNotChanged = fileCache.getFileAccessed(path2); + + assertTrue("Cache does not contain an elements after dir init: " + path1, fileCache.containsKey(path1)); + System.out.println("Path1 Time: " + timeChanged); System.out.println("Path1 time after contains(changed): " + fileCache.getFileAccessed(path1)); - + System.out.println(); - - System.out.println("Path2 Time: " + timeNotChanged); - System.out.println("Path2 time (stable): " + fileCache.getFileAccessed(path2)); - - assertTrue("Last accessed was not updated: ", - fileCache.getFileAccessed(path1).compareTo(timeChanged) > 0); - - assertTrue("Last accessed must not be updated: ", + + System.out.println("Path2 Time: " + timeNotChanged); + System.out.println("Path2 time (stable): " + fileCache.getFileAccessed(path2)); + + assertTrue("Last accessed was not updated: ", fileCache.getFileAccessed(path1).compareTo(timeChanged) > 0); + + assertTrue("Last accessed must not be updated: ", fileCache.getFileAccessed(path2).compareTo(timeNotChanged) == 0); - - assertTrue("Cache does not contain 2 elements after contains: " + fileCache.size(), - fileCache.size() == 2); - - // check containsKey - not contains - + + assertTrue("Cache does not contain 2 elements after contains: " + fileCache.size(), fileCache.size() == 2); + + // check containsKey - not contains + System.out.println("Subtest: check containsKey - not contains"); - + timeNotChanged = fileCache.getFileAccessed(path1); - timeNotChanged2 = fileCache.getFileAccessed(path2); - - assertTrue("Cache contains an element, but must not: " + pathNotExists, - !fileCache.containsKey(pathNotExists)); - + timeNotChanged2 = fileCache.getFileAccessed(path2); + + assertTrue("Cache contains an element, but must not: " + pathNotExists, !fileCache.containsKey(pathNotExists)); + System.out.println("Path1 Time: " + timeNotChanged); System.out.println("Path1 time after contains(stable): " + fileCache.getFileAccessed(path1)); - + System.out.println(); - - System.out.println("Path2 Time: " + timeNotChanged2); - System.out.println("Path2 time after contains(stable): " + fileCache.getFileAccessed(path2)); - - assertTrue("path1 Last accessed must not be updated: ", + + System.out.println("Path2 Time: " + timeNotChanged2); + System.out.println("Path2 time after contains(stable): " + fileCache.getFileAccessed(path2)); + + assertTrue("path1 Last accessed must not be updated: ", fileCache.getFileAccessed(path1).compareTo(timeNotChanged) == 0); - - assertTrue("path2 Last accessed must not be updated: ", + + assertTrue("path2 Last accessed must not be updated: ", fileCache.getFileAccessed(path2).compareTo(timeNotChanged2) == 0); - - // check put - not contains - + + // check put - not contains + System.out.println("Subtest: check containsKey - not contains"); - - assertTrue("Cache does not contain 2 elements after contains: " + fileCache.size(), - fileCache.size() == 2); - - assertTrue("Cache contains an element before put: " + path3, - !fileCache.containsKey(path3)); - + + assertTrue("Cache does not contain 2 elements after contains: " + fileCache.size(), fileCache.size() == 2); + + assertTrue("Cache contains an element before put: " + path3, !fileCache.containsKey(path3)); + TestUtils.createFile(path3, ""); fileCache.put(path3); - - assertTrue("Cache does not contain 3 elements after contains: " + fileCache.size(), - fileCache.size() == 3); - - assertTrue("Cache does not contains an element after put: " + path3, - fileCache.containsKey(path3)); - + + assertTrue("Cache does not contain 3 elements after contains: " + fileCache.size(), fileCache.size() == 3); + + assertTrue("Cache does not contains an element after put: " + path3, fileCache.containsKey(path3)); + // check put - contains, update - + System.out.println("Subtest: check put - contains, update "); - + timeChanged = fileCache.getFileAccessed(path1); - timeNotChanged = fileCache.getFileAccessed(path2); - + timeNotChanged = fileCache.getFileAccessed(path2); + fileCache.put(path1); - + System.out.println("Path1 Time: " + timeChanged); System.out.println("Path1 time after contains(changed): " + fileCache.getFileAccessed(path1)); - + System.out.println(); - - System.out.println("Path2 Time: " + timeNotChanged); - System.out.println("Path2 time (stable): " + fileCache.getFileAccessed(path2)); - - assertTrue("Last accessed was not updated: ", - fileCache.getFileAccessed(path1).compareTo(timeChanged) > 0); - - assertTrue("Last accessed must not be updated: ", + + System.out.println("Path2 Time: " + timeNotChanged); + System.out.println("Path2 time (stable): " + fileCache.getFileAccessed(path2)); + + assertTrue("Last accessed was not updated: ", fileCache.getFileAccessed(path1).compareTo(timeChanged) > 0); + + assertTrue("Last accessed must not be updated: ", fileCache.getFileAccessed(path2).compareTo(timeNotChanged) == 0); - - assertTrue("Cache does not contain 3 elements after contains: " + fileCache.size(), - fileCache.size() == 3); - + + assertTrue("Cache does not contain 3 elements after contains: " + fileCache.size(), fileCache.size() == 3); + fileCache.clear(); TestUtils.deleteTestDirectories(); } From b2db66d8335bc1acf8484ae8d58c86bd2e813cb9 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Tue, 21 Nov 2023 16:06:23 +0100 Subject: [PATCH 11/79] storage-mgr: added printDirectoryTree with message to test utils --- .../de/dlr/proseo/storagemgr/TestUtils.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java index 88fe167d5..340027892 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java @@ -320,6 +320,18 @@ public static void printList(String message, List list) { } System.out.println(); } + + /** + * @param message + * @param directoryPath + */ + public static void printDirectoryTree(String message, String directoryPath) { + + System.out.println(); + System.out.println(message); + + printDirectoryTree(directoryPath); + } /** * @param directoryPath @@ -341,7 +353,7 @@ public static void printDirectoryTree(String directoryPath) { System.out.print(" Files: " + countFilesInDirectory(directoryPath)); System.out.println(" Folders: " + countDirectoriesInDirectory(directoryPath)); - printDirectoryTree(directoryPath, ""); + printDirectoryTreeWithDepth(directoryPath, ""); System.out.println(); } @@ -349,7 +361,7 @@ public static void printDirectoryTree(String directoryPath) { * @param directoryPath * @param depth */ - private static void printDirectoryTree(String directoryPath, String depth) { + private static void printDirectoryTreeWithDepth(String directoryPath, String depth) { File directory = new File(directoryPath); @@ -373,7 +385,7 @@ private static void printDirectoryTree(String directoryPath, String depth) { for (File file : files) { if (file.isDirectory()) { System.out.println(depth + file.getName() + " "); - printDirectoryTree(file.getPath(), OUTPUT_TAB + depth); + printDirectoryTreeWithDepth(file.getPath(), OUTPUT_TAB + depth); } } } From ab12f79dc923b8af0fa1583e367ebf4b35fe7899 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Wed, 22 Nov 2023 14:30:54 +0100 Subject: [PATCH 12/79] storage-mgr: removed obsolete error headers, improved comments --- .../rest/ProductfileControllerImpl.java | 35 ++++++------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index 043237996..d0c65e711 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -6,7 +6,6 @@ import java.nio.file.Paths; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; @@ -37,9 +36,6 @@ @Component public class ProductfileControllerImpl implements ProductfileController { - private static final String HTTP_HEADER_WARNING = "Warning"; - private static final String HTTP_MSG_PREFIX = "199 proseo-storage-mgr "; - /** A logger for this class */ private static ProseoLogger logger = new ProseoLogger(ProductControllerImpl.class); private static ProseoHttp http = new ProseoHttp(logger, HttpPrefix.STORAGE_MGR); @@ -92,21 +88,21 @@ public ResponseEntity getRestFileInfoByPathInfo(String pathInfo) { fileLocker.lock(); - // After lock the active thread downloaded the file and put it to the cache (see + // After lock() the active thread downloads the file and put it to the cache (see // below) - // After lock the passive thread did nothing, but the file has been downloaded + // After lock() the passive thread did nothing, but the file has been downloaded // and the cache has been updated - need to check if file contains in the cache // again if (!cache.containsKey(targetFile.getFullPath())) { - // download-thread + // active thread - downloads the file and puts it to the cache storageProvider.getStorage().downloadFile(sourceFile, targetFile); logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED, targetFile.getFullPath()); cache.put(targetFile.getFullPath()); } else { - // waiting-thread when the file downloaded and use it from cache + // passive thread - did nothing, just waited until the file has been downloaded and use it from cache logger.debug("... waiting-thread when the file downloaded and use it from cache: ", targetFile.getFullPath()); } @@ -154,9 +150,6 @@ public ResponseEntity getRestFileInfoByPathInfo(String pathInfo) { * @param productId Product id * @return Target file name */ - /** - * - */ @Override public ResponseEntity updateProductfiles(String pathInfo, Long productId, Long fileSize) { @@ -165,6 +158,10 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro // Storage Manager version 2: pathInfo absolute path, upload absolute file -> storage + // 1. download to cache absolute-file -> cache + // 2. add to cache cache.put() + // 3. upload to storage cache -> storage + if (pathInfo == null) { return new ResponseEntity(new RestFileInfo(), HttpStatus.BAD_REQUEST); } @@ -187,9 +184,9 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro return new ResponseEntity<>(restFileInfo, HttpStatus.CREATED); } catch (Exception e) { - + String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); - return new ResponseEntity<>(errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); + return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); } } @@ -213,16 +210,4 @@ private static RestFileInfo convertToRestFileInfo(StorageFile storageFile, long return restFileInfo; } - /** - * Create an HTTP "Warning" header with the given text message - * - * @param message the message text - * @return an HttpHeaders object with a warning message - */ - private HttpHeaders errorHeaders(String message) { - HttpHeaders responseHeaders = new HttpHeaders(); - responseHeaders.set(HTTP_HEADER_WARNING, - HTTP_MSG_PREFIX + (null == message ? "null" : message.replaceAll("\n", " "))); - return responseHeaders; - } } From d1e8297320dec3748aa7b6443b76338e57039415 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Wed, 22 Nov 2023 17:15:22 +0100 Subject: [PATCH 13/79] storage-mgr: added synchronized download methods to file controller --- .../rest/ProductfileControllerImpl.java | 189 ++++++++++++------ 1 file changed, 125 insertions(+), 64 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index d0c65e711..be5a0ac26 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -76,43 +76,8 @@ public ResponseEntity getRestFileInfoByPathInfo(String pathInfo) { cfg.getFileCheckMaxCycles()); try { - // relative path depends on path, not on actual storage - String relativePath = storageProvider.getRelativePath(pathInfo); - - StorageFile sourceFile = storageProvider.getStorageFile(relativePath); - StorageFile targetFile = storageProvider.getCacheFile(sourceFile.getRelativePath()); - - FileCache cache = FileCache.getInstance(); - - if (!cache.containsKey(targetFile.getFullPath())) { - - fileLocker.lock(); - - // After lock() the active thread downloads the file and put it to the cache (see - // below) - // After lock() the passive thread did nothing, but the file has been downloaded - // and the cache has been updated - need to check if file contains in the cache - // again - if (!cache.containsKey(targetFile.getFullPath())) { - - // active thread - downloads the file and puts it to the cache - storageProvider.getStorage().downloadFile(sourceFile, targetFile); - logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED, targetFile.getFullPath()); - - cache.put(targetFile.getFullPath()); - } else { - - // passive thread - did nothing, just waited until the file has been downloaded and use it from cache - logger.debug("... waiting-thread when the file downloaded and use it from cache: ", - targetFile.getFullPath()); - } - - } else { - logger.debug("... no download and no lock - the file is in cache: ", targetFile.getFullPath()); - } - - RestFileInfo restFileInfo = convertToRestFileInfo(targetFile, - storageProvider.getCacheFileSize(sourceFile.getRelativePath())); + + RestFileInfo restFileInfo = synchronizedDownloadFromStorageToCache(pathInfo, fileLocker); return new ResponseEntity<>(restFileInfo, HttpStatus.OK); @@ -138,7 +103,6 @@ public ResponseEntity getRestFileInfoByPathInfo(String pathInfo) { fileLocker.unlock(); logger.debug("... unlocked the file: ", pathInfo); - } } @@ -155,39 +119,40 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro if (logger.isTraceEnabled()) logger.trace(">>> updateProductfiles({}, {})", pathInfo, productId); - - // Storage Manager version 2: pathInfo absolute path, upload absolute file -> storage - - // 1. download to cache absolute-file -> cache - // 2. add to cache cache.put() - // 3. upload to storage cache -> storage - - if (pathInfo == null) { - return new ResponseEntity(new RestFileInfo(), HttpStatus.BAD_REQUEST); - } - try { - Storage storage = storageProvider.getStorage(); - String absolutePath = pathInfo; - String fileName = new File(pathInfo).getName(); - String productFolderWithFilename = Paths.get(String.valueOf(productId), fileName).toString(); + // Storage Manager version 2: pathInfo absolute path, upload absolute file -> + // storage + + // 1. download to cache absolute-file -> cache + // 2. add to cache cache.put() + // 3. upload to storage cache -> storage + + if (pathInfo == null) { + return new ResponseEntity(new RestFileInfo(), HttpStatus.BAD_REQUEST); + } + + try { + Storage storage = storageProvider.getStorage(); + String absolutePath = pathInfo; + String fileName = new File(pathInfo).getName(); + String productFolderWithFilename = Paths.get(String.valueOf(productId), fileName).toString(); + + StorageFile sourceFile = storageProvider.getAbsoluteFile(absolutePath); + StorageFile targetFile = storageProvider.getStorageFile(productFolderWithFilename); - StorageFile sourceFile = storageProvider.getAbsoluteFile(absolutePath); - StorageFile targetFile = storageProvider.getStorageFile(productFolderWithFilename); + storage.uploadFile(sourceFile, targetFile); - storage.uploadFile(sourceFile, targetFile); + RestFileInfo restFileInfo = convertToRestFileInfo(targetFile, storage.getFileSize(targetFile)); - RestFileInfo restFileInfo = convertToRestFileInfo(targetFile, storage.getFileSize(targetFile)); + logger.log(StorageMgrMessage.PRODUCT_FILE_UPLOADED, pathInfo, productId); - logger.log(StorageMgrMessage.PRODUCT_FILE_UPLOADED, pathInfo, productId); + return new ResponseEntity<>(restFileInfo, HttpStatus.CREATED); - return new ResponseEntity<>(restFileInfo, HttpStatus.CREATED); + } catch (Exception e) { - } catch (Exception e) { - - String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); - return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); - } + String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); + return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); + } } /** @@ -210,4 +175,100 @@ private static RestFileInfo convertToRestFileInfo(StorageFile storageFile, long return restFileInfo; } + private RestFileInfo synchronizedDownloadFromStorageToCache(String storageFilePath, StorageFileLocker fileLocker) + throws FileLockedAfterMaxCyclesException, IOException, Exception { + + // relative path depends on path, not on actual storage + String relativePath = storageProvider.getRelativePath(storageFilePath); + + StorageFile sourceFile = storageProvider.getStorageFile(relativePath); + StorageFile targetFile = storageProvider.getCacheFile(sourceFile.getRelativePath()); + + FileCache cache = FileCache.getInstance(); + + if (!cache.containsKey(targetFile.getFullPath())) { + + fileLocker.lock(); + + // After lock() the active thread downloads the file and put it to the cache + // (see + // below) + // After lock() the passive thread did nothing, but the file has been downloaded + // and the cache has been updated - need to check if file contains in the cache + // again + if (!cache.containsKey(targetFile.getFullPath())) { + + // active thread - downloads the file and puts it to the cache + storageProvider.getStorage().downloadFile(sourceFile, targetFile); + logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED, targetFile.getFullPath()); + + cache.put(targetFile.getFullPath()); + } else { + + // passive thread - did nothing, just waited until the file has been downloaded + // and use it from cache + logger.debug("... waiting-thread when the file downloaded and use it from cache: ", + targetFile.getFullPath()); + } + + } else { + logger.debug("... no download and no lock - the file is in cache: ", targetFile.getFullPath()); + } + + RestFileInfo restFileInfo = convertToRestFileInfo(targetFile, + storageProvider.getCacheFileSize(sourceFile.getRelativePath())); + + return restFileInfo; + } + + private RestFileInfo synchronizedDownloadFromAbsolutePathToCache(String absolutePath, Long productId, Long fileSize, StorageFileLocker fileLocker) + throws FileLockedAfterMaxCyclesException, IOException, Exception { + + StorageFile sourceFile = storageProvider.getAbsoluteFile(absolutePath); + + String fileName = new File(absolutePath).getName(); + String productFolderWithFilename = Paths.get(String.valueOf(productId), fileName).toString(); + StorageFile targetFile = storageProvider.getCacheFile(productFolderWithFilename); + + + FileCache cache = FileCache.getInstance(); + + if (!cache.containsKey(targetFile.getFullPath())) { + + fileLocker.lock(); + + // After lock() the active thread downloads the file and put it to the cache + // (see + // below) + // After lock() the passive thread did nothing, but the file has been downloaded + // and the cache has been updated - need to check if file contains in the cache + // again + if (!cache.containsKey(targetFile.getFullPath())) { + + // active thread - downloads the file and puts it to the cache + + // TODO: ADD FUNCTIONALITY DOWNLOAD TO CACHE + //storageProvider.getStorage().downloadFile(sourceFile, targetFile); + + logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED, targetFile.getFullPath()); + + cache.put(targetFile.getFullPath()); + } else { + + // passive thread - did nothing, just waited until the file has been downloaded + // and use it from cache + logger.debug("... waiting-thread when the file downloaded and use it from cache: ", + targetFile.getFullPath()); + } + + } else { + logger.debug("... no download and no lock - the file is in cache: ", targetFile.getFullPath()); + } + + RestFileInfo restFileInfo = convertToRestFileInfo(targetFile, + storageProvider.getCacheFileSize(sourceFile.getRelativePath())); + + return restFileInfo; + } + } From f83962823d62eb7f4c2793d5011146150bf8cc54 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 23 Nov 2023 15:44:27 +0100 Subject: [PATCH 14/79] storage-mgr: created posix atomic file loader for posix storage --- .../posix/PosixAtomicFileUploader.java | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileUploader.java diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileUploader.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileUploader.java new file mode 100644 index 000000000..812ca22cd --- /dev/null +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileUploader.java @@ -0,0 +1,131 @@ +/** + * PosixAtomicFileUploader.java + * + * (C) 2023 Dr. Bassler & Co. Managementberatung GmbH + */ +package de.dlr.proseo.storagemgr.posix; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.dlr.proseo.storagemgr.model.AtomicCommand; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; + +/** + * Posix Atomic Uploader + * + * @author Denys Chaykovskiy + * + */ +public class PosixAtomicFileUploader implements AtomicCommand { + + /** Info */ + private static final String INFO = "Posix ATOMIC File Uploader"; + + /** Completed Info */ + private static final String COMPLETED = "file UPLOADED"; + + /** Failed Info */ + private static final String FAILED = "file upload FAILED"; + + /** Logger for this class */ + private static Logger logger = LoggerFactory.getLogger(PosixAtomicFileUploader.class); + + /** source file */ + private String sourceFile; + + /** target file or dir */ + private String targetFileOrDir; + + /** + * Constructor + * + * @param sourceFile sourceFile + * @param targetFileOrDir target file or directory + */ + public PosixAtomicFileUploader(String sourceFile, String targetFileOrDir) { + + this.sourceFile = sourceFile; + this.targetFileOrDir = targetFileOrDir; + } + + /** + * Executes upload of the file to posix + * + * @return uploaded file name + */ + @Override + public String execute() throws IOException { + + if (logger.isTraceEnabled()) + logger.trace(">>> execute() - uploadFile({},{})", sourceFile, targetFileOrDir); + + String targetFile = targetFileOrDir; + + if (new PathConverter(targetFileOrDir).isDirectory()) { + targetFile = new PathConverter(targetFileOrDir, getFileName(sourceFile)).getPath(); + } + + new FileUtils(targetFile).createParentDirectories(); + + Path sourceFilePath = new File(sourceFile).toPath(); + Path targetFilePath = new File(targetFile).toPath(); + + try { + Path copiedPath = Files.copy(sourceFilePath, targetFilePath, StandardCopyOption.REPLACE_EXISTING); + return copiedPath.toString(); + + } catch (Exception e) { + if (logger.isTraceEnabled()) + logger.trace(getFailedInfo() + e.getMessage()); + throw new IOException(e); + } + } + + /** + * Gets information about atomic command (mostly for logs) + * + * @return information about atomic command + */ + @Override + public String getInfo() { + return INFO + " "; + } + + /** + * Gets information about completed atomic command (mostly for logs) + * + * @return information about completed atomic command + */ + @Override + public String getCompletedInfo() { + return INFO + ": " + COMPLETED + " "; + } + + /** + * Gets information about failed atomic command (mostly for logs) + * + * @return information about failed atomic command + */ + @Override + public String getFailedInfo() { + return INFO + ": " + FAILED + " "; + } + + /** + * Gets file name + * + * @param path path + * @return file name + */ + private String getFileName(String path) { + return new File(path).getName(); + } +} \ No newline at end of file From 7424af7c0d92eda3359bd5c8823f8b3fd27d3028 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 23 Nov 2023 15:45:52 +0100 Subject: [PATCH 15/79] storage-mgr: created posix configuration for the posix storage --- .../storagemgr/posix/PosixConfiguration.java | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixConfiguration.java diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixConfiguration.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixConfiguration.java new file mode 100644 index 000000000..92b855337 --- /dev/null +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixConfiguration.java @@ -0,0 +1,78 @@ +/** + * PosixConfiguration.java + * + * (C) 2023 Dr. Bassler & Co. Managementberatung GmbH + */ +package de.dlr.proseo.storagemgr.posix; + +/** + * Holds the configuration settings required to connect to and interact with an + * POSIX storage system. It allows for the customization of various aspects such as + * access credentials, bucket, paths, and behavior-related settings like + * request attempts and file transfer management. + * + * @author Denys Chaykovskiy + */ +public class PosixConfiguration { + + /** Bucket */ + private String bucket; + + /** base path */ + private String basePath; + + /** source path */ + private String sourcePath; + + /** max request attempts */ + private int maxRequestAttempts; + + /** wait time */ + private long fileCheckWaitTime; + + + public String getBucket() { + return bucket; + } + + public void setBucket(String bucket) { + this.bucket = bucket; + } + + public String getBasePath() { + return basePath; + } + + public void setBasePath(String basePath) { + this.basePath = basePath; + } + + public String getSourcePath() { + return sourcePath; + } + + public void setSourcePath(String sourcePath) { + this.sourcePath = sourcePath; + } + + public int getMaxRequestAttempts() { + return maxRequestAttempts; + } + + public void setMaxRequestAttempts(int maxRequestAttempts) { + this.maxRequestAttempts = maxRequestAttempts; + } + + public long getFileCheckWaitTime() { + return fileCheckWaitTime; + } + + public void setFileCheckWaitTime(Long fileCheckWaitTime) { + this.fileCheckWaitTime = fileCheckWaitTime; + } + + public void setFileCheckWaitTime(long fileCheckWaitTime) { + this.fileCheckWaitTime = fileCheckWaitTime; + } + +} \ No newline at end of file From 637a2ee695ca197f325528222798c14cde7dd3a7 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 23 Nov 2023 15:49:21 +0100 Subject: [PATCH 16/79] storage-mgr: merged functionalities of storage test utilities --- .../storagemgr/BaseStorageTestUtils.java | 75 ++++++++++++++++++- .../proseo/storagemgr/StorageTestUtils.java | 72 ------------------ 2 files changed, 73 insertions(+), 74 deletions(-) delete mode 100644 storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StorageTestUtils.java diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java index cb325fb32..49581f85c 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java @@ -6,6 +6,19 @@ import java.nio.file.Paths; import java.util.List; +import javax.annotation.PostConstruct; + +import org.junit.Rule; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; +import org.springframework.stereotype.Component; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.posix.PosixDAL; import de.dlr.proseo.storagemgr.posix.PosixStorageFile; @@ -15,11 +28,45 @@ * @author Denys Chaykovskiy * */ +/* +@RunWith(SpringJUnit4ClassRunner.class) +@SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) +@AutoConfigureMockMvc +*/ +@Component public class BaseStorageTestUtils { + + @Autowired + private StorageProvider storageProvider; + + @Autowired + private TestUtils testUtils; + + @Rule + public TestName testName = new TestName(); protected String sourcePath; protected String storagePath; protected String cachePath; + + @PostConstruct + private void init() { + + sourcePath = testUtils.getSourcePath(); + storagePath = testUtils.getStoragePath(); + cachePath = testUtils.getCachePath(); + + theTestUtils = this; + + // storageProvider = new StorageProvider(); + } + + private static BaseStorageTestUtils theTestUtils; + + public static BaseStorageTestUtils getInstance() { + + return theTestUtils; + } public StorageFile getSourceFile(String relativePath) { @@ -93,7 +140,7 @@ public String getAbsoluteSourcePath(String relativePath) { public void uploadToPosixStorage(String relativePath) { - PosixDAL posixDAL = new PosixDAL(); + PosixDAL posixDAL = new PosixDAL(storageProvider.getPosixConfigurationFromFile()); StorageFile sourceFile = getSourceFile(relativePath); StorageFile destFile = getStorageFile(relativePath); @@ -112,7 +159,7 @@ public void uploadToPosixStorage(String relativePath) { public void downloadFromPosixStorage(String relativePath) { - PosixDAL posixDAL = new PosixDAL(); + PosixDAL posixDAL = new PosixDAL(storageProvider.getPosixConfigurationFromFile()); StorageFile sourceFile = getStorageFile(relativePath); StorageFile destFile = getCacheFile(relativePath); @@ -156,4 +203,28 @@ public String getCachePath() { public String getSourcePath() { return sourcePath; } + + public static void printStorageFiles(String message, Storage storage) { + + List storageFiles; + try { + storageFiles = storage.getRelativeFiles(); + String storageType = storage.getStorageType().toString(); + TestUtils.printList(message + ". Storage " + storageType + " files || " + storage.getAbsoluteBasePath(), storageFiles); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public static void printStorageFilesWithPrefix(String message, Storage storage, String prefix) { + + List storageFiles; + try { + storageFiles = storage.getRelativeFiles(prefix); + String storageType = storage.getStorageType().toString(); + TestUtils.printList(message + ". Storage " + storageType + " files || Prefix: " + prefix + " || " + storage.getAbsoluteBasePath(), storageFiles); + } catch (IOException e) { + e.printStackTrace(); + } + } } diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StorageTestUtils.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StorageTestUtils.java deleted file mode 100644 index 6aedf8f50..000000000 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StorageTestUtils.java +++ /dev/null @@ -1,72 +0,0 @@ -package de.dlr.proseo.storagemgr; - -import java.io.IOException; -import java.util.List; - -import javax.annotation.PostConstruct; - -import org.junit.Rule; -import org.junit.rules.TestName; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import de.dlr.proseo.storagemgr.model.Storage; - -/** - * @author Denys Chaykovskiy - * - */ -@Component -public class StorageTestUtils extends BaseStorageTestUtils { - - - @Autowired - private TestUtils testUtils; - - @Rule - public TestName testName = new TestName(); - - @PostConstruct - private void init() { - - sourcePath = testUtils.getSourcePath(); - storagePath = testUtils.getStoragePath(); - cachePath = testUtils.getCachePath(); - - theTestUtils = this; - - // storageProvider = new StorageProvider(); - } - - private static StorageTestUtils theTestUtils; - - public static StorageTestUtils getInstance() { - - return theTestUtils; - } - - - public static void printStorageFiles(String message, Storage storage) { - - List storageFiles; - try { - storageFiles = storage.getRelativeFiles(); - String storageType = storage.getStorageType().toString(); - TestUtils.printList(message + ". Storage " + storageType + " files || " + storage.getAbsoluteBasePath(), storageFiles); - } catch (IOException e) { - e.printStackTrace(); - } - } - - public static void printStorageFilesWithPrefix(String message, Storage storage, String prefix) { - - List storageFiles; - try { - storageFiles = storage.getRelativeFiles(prefix); - String storageType = storage.getStorageType().toString(); - TestUtils.printList(message + ". Storage " + storageType + " files || Prefix: " + prefix + " || " + storage.getAbsoluteBasePath(), storageFiles); - } catch (IOException e) { - e.printStackTrace(); - } - } -} From a7bf0342609f7ea89bbe713671097e8e43c49ea8 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 23 Nov 2023 15:52:47 +0100 Subject: [PATCH 17/79] storage-mgr: integrated posix configuration, posix atomic uploader --- .../proseo/storagemgr/StorageProvider.java | 109 +++++++++++------- .../dlr/proseo/storagemgr/posix/PosixDAL.java | 54 +++++---- .../proseo/storagemgr/posix/PosixStorage.java | 81 ++++++------- .../dlr/proseo/storagemgr/s3/S3Storage.java | 2 +- 4 files changed, 134 insertions(+), 112 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageProvider.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageProvider.java index 844ee5ab7..94fc5c368 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageProvider.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageProvider.java @@ -22,11 +22,15 @@ import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; + import de.dlr.proseo.storagemgr.posix.PosixStorage; import de.dlr.proseo.storagemgr.posix.PosixStorageFile; +import de.dlr.proseo.storagemgr.posix.PosixConfiguration; + import de.dlr.proseo.storagemgr.s3.S3Configuration; import de.dlr.proseo.storagemgr.s3.S3Storage; import de.dlr.proseo.storagemgr.s3.S3StorageFile; + import de.dlr.proseo.storagemgr.utils.PathConverter; /** @@ -84,23 +88,6 @@ public StorageProvider() { // init(); } - /** - * Initializes storage(s) from Application.yml - * - * @throws IOException if an error occurs during initialization - */ - @PostConstruct - private void init() throws IOException { - theStorageProvider = this; - storage = createStorage(StorageType.valueOf(cfg.getDefaultStorageType()), cfg.getPosixBackendPath()); - - basePaths.add(storage.getBasePath()); - basePaths.add(cfg.getDefaultSourcePath()); - basePaths.add(cfg.getPosixCachePath()); - - loadDefaultPaths(); - } - /** * Load default source, storage, and cache paths */ @@ -265,31 +252,6 @@ public long getPosixFileSize(String absolutePath) throws IOException { return Files.size(path); } - /** - * Creates a storage instance based on the specified storage type and storage - * path. - * - * @param storageType the storage type (S3 or POSIX) - * @param storagePath the base path of the POSIX storage (used for temporary - * files in S3) - * @return the created storage instance - * @throws IOException if an error occurs during storage creation - */ - private Storage createStorage(StorageType storageType, String storagePath) throws IOException { - if (logger.isTraceEnabled()) - logger.trace(">>> createStorage({}, {})", storageType.toString(), storagePath); - - sourcePath = cfg.getDefaultSourcePath(); - - if (storageType == StorageType.POSIX) { - return new PosixStorage(storagePath, sourcePath); - } else if (storageType == StorageType.S3) { - return new S3Storage(getS3ConfigurationFromFile()); - } - - throw new IllegalArgumentException("Storage Type " + storageType.toString() + " is wrong"); - } - /** * Gets the cache file as a StorageFile from the relative path. The path can be * virtual. @@ -475,6 +437,27 @@ public StorageFile createStorageFile(String relativePath, String content) throws return storage.createStorageFile(relativePath, content); } + + /** + * Gets the POSIX configuration from the file. + * + * @return the POSIX configuration + */ + public PosixConfiguration getPosixConfigurationFromFile() { + PosixConfiguration posixConfiguration = new PosixConfiguration(); + + // FYI: No bucket configuration is used in prosEO for POSIX Storage + posixConfiguration.setBucket(StorageFile.NO_BUCKET); + + posixConfiguration.setBasePath(cfg.getPosixBackendPath()); + posixConfiguration.setSourcePath(cfg.getDefaultSourcePath()); + + posixConfiguration.setMaxRequestAttempts(cfg.getMaxRequestAttempts()); + + posixConfiguration.setFileCheckWaitTime(cfg.getFileCheckWaitTime()); + + return posixConfiguration; + } /** * Gets the S3 configuration from the file. @@ -501,4 +484,46 @@ public S3Configuration getS3ConfigurationFromFile() { return s3Configuration; } + + /** + * Initializes storage(s) from Application.yml + * + * @throws IOException if an error occurs during initialization + */ + @PostConstruct + private void init() throws IOException { + theStorageProvider = this; + storage = createStorage(StorageType.valueOf(cfg.getDefaultStorageType()), cfg.getPosixBackendPath()); + + basePaths.add(storage.getBasePath()); + basePaths.add(cfg.getDefaultSourcePath()); + basePaths.add(cfg.getPosixCachePath()); + + loadDefaultPaths(); + } + + /** + * Creates a storage instance based on the specified storage type and storage + * path. + * + * @param storageType the storage type (S3 or POSIX) + * @param storagePath the base path of the POSIX storage (used for temporary + * files in S3) + * @return the created storage instance + * @throws IOException if an error occurs during storage creation + */ + private Storage createStorage(StorageType storageType, String storagePath) throws IOException { + if (logger.isTraceEnabled()) + logger.trace(">>> createStorage({}, {})", storageType.toString(), storagePath); + + sourcePath = cfg.getDefaultSourcePath(); + + if (storageType == StorageType.POSIX) { + return new PosixStorage(getPosixConfigurationFromFile()); + } else if (storageType == StorageType.S3) { + return new S3Storage(getS3ConfigurationFromFile()); + } + + throw new IllegalArgumentException("Storage Type " + storageType.toString() + " is wrong"); + } } \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java index abe0ddada..aade87071 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java @@ -17,6 +17,8 @@ import de.dlr.proseo.logging.logger.ProseoLogger; import de.dlr.proseo.logging.messages.StorageMgrMessage; +import de.dlr.proseo.storagemgr.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.DefaultRetryStrategy; import de.dlr.proseo.storagemgr.utils.FileUtils; import de.dlr.proseo.storagemgr.utils.PathConverter; @@ -33,7 +35,30 @@ public class PosixDAL { /** Logger for this class */ private static ProseoLogger logger = new ProseoLogger(PosixDAL.class); + + /** POSIX configuration */ + private PosixConfiguration cfg; + + /** + * Constructor + * + * @param cfg POSIX Configuration + */ + public PosixDAL(PosixConfiguration cfg) { + + this.cfg = cfg; + } + /** + * Gets the POSIX configuration used by this PosixDAL instance. + * + * @return the PosixConfiguration object + */ + public PosixConfiguration getConfiguration() { + + return cfg; + } + /** * Retrieves a list of files that match a given path (or prefix). It recursively * scans directories and adds the absolute paths of files to the result @@ -235,30 +260,11 @@ public String downloadFile(String sourceFile, String targetFileOrDir) throws IOE if (logger.isTraceEnabled()) logger.trace(">>> downloadFile({},{})", sourceFile, targetFileOrDir); + + + AtomicCommand fileUploader = new PosixAtomicFileUploader(sourceFile, targetFileOrDir); - String targetFile = targetFileOrDir; - - if (new PathConverter(targetFileOrDir).isDirectory()) { - targetFile = new PathConverter(targetFileOrDir, getFileName(sourceFile)).getPath(); - } - - createParentDirectories(targetFile); - - Path sourceFilePath = new File(sourceFile).toPath(); - Path targetFilePath = new File(targetFile).toPath(); - - try { - Path copiedPath = Files.copy(sourceFilePath, targetFilePath, StandardCopyOption.REPLACE_EXISTING); - return copiedPath.toString(); - - } catch (Exception e) { - if (logger.isDebugEnabled()) { - logger.debug("An exception occurred. Cause: ", e); - } - if (logger.isTraceEnabled()) - logger.log(StorageMgrMessage.FILE_NOT_DOWNLOADED, sourceFile, targetFileOrDir, e.getMessage()); - throw e; - } + return new DefaultRetryStrategy<>(fileUploader, cfg.getMaxRequestAttempts(), cfg.getFileCheckWaitTime()).execute(); } /** @@ -356,7 +362,7 @@ public List delete(String sourceFileOrDir) throws IOException { private void createParentDirectories(String path) { if (logger.isTraceEnabled()) - logger.trace(">>> getFiles({})", path); + logger.trace(">>> createParentDirectories({})", path); File targetFile = new File(path); File parent = targetFile.getParentFile(); diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorage.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorage.java index 3763808ee..eb2645ce4 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorage.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorage.java @@ -29,47 +29,38 @@ * retrieving information about the storage, performing file operations, and * path conversion. * - * This class assumes a one-bucket concept for POSIX storage. + * This class assumes a no-bucket concept for POSIX storage. * * @author Denys Chaykovskiy */ public class PosixStorage implements Storage { - /** Base path */ - private String basePath; - - /** Source path */ - private String sourcePath; - /** Bucket */ private String bucket; + + /** POSIX data access layer object */ + private PosixDAL posixDAL; + + /** POSIX configuration */ + private PosixConfiguration cfg; /** Logger for this class */ private static ProseoLogger logger = new ProseoLogger(PosixStorage.class); - /** POSIX data access layer object */ - private PosixDAL posixDAL = new PosixDAL(); - - /** - * Default Constructor - */ - public PosixStorage() { - } - /** * No bucket constructor initializing the storage with a basePath and * sourcePath. It creates the necessary directories if they don't exist. * - * @param basePath base path - * @param sourcePath source path + * @param cfg POSIX configuration */ - public PosixStorage(String basePath, String sourcePath) { - this.basePath = basePath; - this.sourcePath = sourcePath; - this.bucket = StorageFile.NO_BUCKET; + public PosixStorage(PosixConfiguration cfg) { + this.bucket = cfg.getBucket(); // StorageFile.NO_BUCKET is used in prosEO in POSIX Storage; - new FileUtils(basePath).createDirectories(); - new FileUtils(sourcePath).createDirectories(); + new FileUtils(cfg.getBasePath()).createDirectories(); + new FileUtils(cfg.getSourcePath()).createDirectories(); + + this.cfg = cfg; + posixDAL = new PosixDAL(cfg); } /** @@ -89,7 +80,7 @@ public StorageType getStorageType() { */ @Override public String getBasePath() { - return basePath; + return cfg.getBasePath(); } /** @@ -111,7 +102,7 @@ public String getAbsoluteBasePath() { */ @Override public String getSourcePath() { - return sourcePath; + return cfg.getSourcePath(); } /** @@ -123,7 +114,7 @@ public String getSourcePath() { public void setBucket(String bucket) { this.bucket = bucket; - String bucketPath = Paths.get(basePath, bucket).toString(); + String bucketPath = Paths.get(cfg.getBasePath(), bucket).toString(); new FileUtils(bucketPath).createDirectories(); } @@ -182,7 +173,7 @@ public List getRelativeFiles(String relativePath) { if (logger.isTraceEnabled()) logger.trace(">>> getFiles({})", relativePath); - String path = new PathConverter(basePath, relativePath).getPath(); + String path = new PathConverter(cfg.getBasePath(), relativePath).getPath(); return getRelativePath(posixDAL.getFiles(path)); } @@ -194,7 +185,7 @@ public List getRelativeFiles(String relativePath) { */ @Override public List getRelativeFiles() { - return getRelativePath(posixDAL.getFiles(basePath)); + return getRelativePath(posixDAL.getFiles(cfg.getBasePath())); } /** @@ -207,7 +198,7 @@ public List getAbsoluteFiles(String relativePath) { if (logger.isTraceEnabled()) logger.trace(">>> getFiles({})", relativePath); - String path = new PathConverter(basePath, relativePath).getPath(); + String path = new PathConverter(cfg.getBasePath(), relativePath).getPath(); return posixDAL.getFiles(path); } @@ -218,7 +209,7 @@ public List getAbsoluteFiles(String relativePath) { * @return the list of all files from the storage */ public List getAbsoluteFiles() { - return posixDAL.getFiles(basePath); + return posixDAL.getFiles(cfg.getBasePath()); } /** @@ -233,8 +224,8 @@ public String getRelativePath(String absolutePath) { logger.trace(">>> getRelativePath({})", absolutePath); List basePaths = new ArrayList<>(); - basePaths.add(basePath); - basePaths.add(sourcePath); + basePaths.add(cfg.getBasePath()); + basePaths.add(cfg.getSourcePath()); return new PathConverter(absolutePath, basePaths).getRelativePath().getPath(); } @@ -250,10 +241,10 @@ public List getRelativePath(List absolutePaths) { if (logger.isTraceEnabled()) logger.trace(">>> getRelativePath({})", absolutePaths); - logger.trace("... basePath = {}, sourcePath = {}", basePath, sourcePath); + logger.trace("... basePath = {}, sourcePath = {}", cfg.getBasePath(), cfg.getSourcePath()); List basePaths = new ArrayList<>(); - basePaths.add(basePath); - basePaths.add(sourcePath); + basePaths.add(cfg.getBasePath()); + basePaths.add(cfg.getSourcePath()); List relativePaths = new ArrayList<>(); @@ -305,7 +296,7 @@ public List getAbsolutePath(List relativePaths) { */ @Override public StorageFile getStorageFile(String relativePath) { - return new PosixStorageFile(basePath, relativePath); + return new PosixStorageFile(cfg.getBasePath(), relativePath); } /** @@ -315,7 +306,7 @@ public StorageFile getStorageFile(String relativePath) { */ @Override public List getStorageFiles() { - List paths = posixDAL.getFiles(basePath); + List paths = posixDAL.getFiles(cfg.getBasePath()); List storageFiles = new ArrayList<>(); for (String path : paths) { @@ -453,7 +444,7 @@ public List upload(StorageFile sourceFileOrDir) throws IOException { if (logger.isTraceEnabled()) logger.trace(">>> upload({})", sourceFileOrDir.getFullPath()); - StorageFile targetFileOrDir = new PosixStorageFile(basePath, sourceFileOrDir.getRelativePath()); + StorageFile targetFileOrDir = new PosixStorageFile(cfg.getBasePath(), sourceFileOrDir.getRelativePath()); List uploadedAbsolutePaths = posixDAL.upload(sourceFileOrDir.getFullPath(), targetFileOrDir.getFullPath()); @@ -472,7 +463,7 @@ public String uploadFile(StorageFile sourceFile) throws IOException { if (logger.isTraceEnabled()) logger.trace(">>> uploadFile({})", sourceFile.getFullPath()); - StorageFile targetFile = new PosixStorageFile(basePath, sourceFile.getRelativePath()); + StorageFile targetFile = new PosixStorageFile(cfg.getBasePath(), sourceFile.getRelativePath()); String uploadedFile = posixDAL.uploadFile(sourceFile.getFullPath(), targetFile.getFullPath()); @@ -491,8 +482,8 @@ public List uploadSourceFileOrDir(String relativeSourceFileOrDir) throws if (logger.isTraceEnabled()) logger.trace(">>> upload({})", relativeSourceFileOrDir); - StorageFile sourceFileOrDir = new PosixStorageFile(sourcePath, relativeSourceFileOrDir); - StorageFile targetFileOrDir = new PosixStorageFile(basePath, relativeSourceFileOrDir); + StorageFile sourceFileOrDir = new PosixStorageFile(cfg.getSourcePath(), relativeSourceFileOrDir); + StorageFile targetFileOrDir = new PosixStorageFile(cfg.getBasePath(), relativeSourceFileOrDir); List uploadedAbsolutePaths = posixDAL.upload(sourceFileOrDir.getFullPath(), targetFileOrDir.getFullPath()); @@ -511,8 +502,8 @@ public String uploadSourceFile(String relativeSourceFile) throws IOException { if (logger.isTraceEnabled()) logger.trace(">>> uploadFile({})", relativeSourceFile); - StorageFile sourceFile = new PosixStorageFile(sourcePath, relativeSourceFile); - StorageFile targetFile = new PosixStorageFile(basePath, relativeSourceFile); + StorageFile sourceFile = new PosixStorageFile(cfg.getSourcePath(), relativeSourceFile); + StorageFile targetFile = new PosixStorageFile(cfg.getBasePath(), relativeSourceFile); String uploadedFile = posixDAL.uploadFile(sourceFile.getFullPath(), targetFile.getFullPath()); @@ -672,6 +663,6 @@ public InputStream getInputStream(StorageFile storageFile) throws IOException { * @return the full bucket path */ private String getFullBucketPath() { - return bucket.equals(StorageFile.NO_BUCKET) ? basePath : Paths.get(basePath, bucket).toString(); + return bucket.equals(StorageFile.NO_BUCKET) ? cfg.getBasePath() : Paths.get(cfg.getBasePath(), bucket).toString(); } } \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Storage.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Storage.java index 209eb6a96..f37076f6a 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Storage.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Storage.java @@ -37,7 +37,7 @@ public class S3Storage implements Storage { private static ProseoLogger logger = new ProseoLogger(S3Storage.class); /** - * Constructor with bucket, access keys, region and end point + * Constructor with the s3 configuration * * @param cfg s3 configuration * @throws IOException if an I/O exception occurs From 3fa43ed4c40f51133017d21b2ce162ed0792868e Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 23 Nov 2023 15:53:55 +0100 Subject: [PATCH 18/79] storage:mgr: updated tests with the merged test utility and posix cfg --- .../cache/ProductfileControllerImplTest_cache.java | 4 ++-- .../rest/JobOrderControllerImplTest_download.java | 4 ++-- .../rest/JobOrderControllerImplTest_upload.java | 6 +++--- .../rest/ProductControllerImplTest_delete.java | 10 +++++----- .../rest/ProductControllerImplTest_download.java | 8 ++++---- .../rest/ProductControllerImplTest_get.java | 8 ++++---- .../rest/ProductControllerImplTest_getAll.java | 4 ++-- .../rest/ProductControllerImplTest_upload.java | 6 +++--- .../rest/ProductfileControllerImplTest_download.java | 6 +++--- .../rest/ProductfileControllerImplTest_upload.java | 6 +++--- .../storagemgr/version2/StorageProviderTest.java | 6 +++--- .../storagemgr/version2/StorageUtilityTest.java | 12 ++++++------ .../storagemgr/version2/posix/PosixDALTest.java | 10 +++++++--- .../storagemgr/version2/posix/PosixStorageTest.java | 4 ++-- .../proseo/storagemgr/version2/s3/S3StorageTest.java | 4 ++-- 15 files changed, 51 insertions(+), 47 deletions(-) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java index 68bd3053d..3ee8a9092 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java @@ -26,7 +26,7 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageProvider; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; @@ -52,7 +52,7 @@ public class ProductfileControllerImplTest_cache { public TestName testName = new TestName(); @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; String cachePath; String storagePath; diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java index 91166ba30..2f7212724 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java @@ -25,7 +25,7 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; import de.dlr.proseo.storagemgr.StorageProvider; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestJoborder; @@ -123,7 +123,7 @@ private void downloadRestJobOrder(StorageType storageType) throws Exception { System.out.println("Created uploaded job order path: " + pathInfo); // show storage files - StorageTestUtils.printStorageFiles("After http-upload-call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("After http-upload-call", storageProvider.getStorage()); // Only 1 job order expected today, because we deleted all today-orders earlier int jobOrderCount = storageProvider.getStorage().getRelativeFiles(getJobOrderPrefixForToday()).size(); diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java index 848839a52..e322e41ac 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java @@ -26,7 +26,7 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; import de.dlr.proseo.storagemgr.StorageProvider; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestJoborder; @@ -136,7 +136,7 @@ private void uploadRestJobOrder(StorageType storageType) throws Exception { System.out.println("Created job order path: " + result.getPathInfo()); // show storage files - StorageTestUtils.printStorageFiles("After http-call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("After http-call", storageProvider.getStorage()); // Only 1 job order expected today, because we deleted all today-orders earlier int jobOrderCount = storageProvider.getStorage().getRelativeFiles(getJobOrderPrefixForToday()).size(); @@ -146,7 +146,7 @@ private void uploadRestJobOrder(StorageType storageType) throws Exception { storageProvider.getStorage().delete(getJobOrderPrefixForToday()); // show storage files - StorageTestUtils.printStorageFiles("After job order cleaning", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("After job order cleaning", storageProvider.getStorage()); } // creates prefix for today only. ignores hour and file name (because it is diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java index f0c05e882..5f4c635b3 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java @@ -23,7 +23,7 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageProvider; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.utils.PathConverter; @@ -46,7 +46,7 @@ public class ProductControllerImplTest_delete { private MockMvc mockMvc; @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; @Rule public TestName testName = new TestName(); @@ -116,10 +116,10 @@ private void delete(StorageProvider storageProvider) throws Exception { } // show storage files before http-delete-call - StorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); // show storage files with prefix before http-delete-call - StorageTestUtils.printStorageFilesWithPrefix("Before http-call", storageProvider.getStorage(), prefix); + BaseStorageTestUtils.printStorageFilesWithPrefix("Before http-call", storageProvider.getStorage(), prefix); // check count of uploaded prefix storage files int realStorageFileCount = storageProvider.getStorage().getRelativeFiles(prefix).size(); @@ -141,7 +141,7 @@ private void delete(StorageProvider storageProvider) throws Exception { TestUtils.printMvcResult(REQUEST_STRING, mvcResult); // show storage files with prefix before http-delete-call - StorageTestUtils.printStorageFilesWithPrefix("After http-call", storageProvider.getStorage(), prefix); + BaseStorageTestUtils.printStorageFilesWithPrefix("After http-call", storageProvider.getStorage(), prefix); // check files after delete (expected: 0) realStorageFileCount = storageProvider.getStorage().getRelativeFiles(prefix).size(); diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java index ffc22acbf..2dad0bffa 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java @@ -20,7 +20,7 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageProvider; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; @@ -48,7 +48,7 @@ public class ProductControllerImplTest_download { private MockMvc mockMvc; @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; @Autowired private StorageProvider storageProvider; @@ -151,7 +151,7 @@ private void downloadProductFiles(StorageType storageType) throws Exception { Long toByte = 7l; // show storage files - StorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); // TEST PARTIAL CONTENT // HTTP Download files (partial content) from storage @@ -204,6 +204,6 @@ private void downloadProductFiles(StorageType storageType) throws Exception { storageProvider.getStorage().delete(prefix); // show storage files after deletion - StorageTestUtils.printStorageFiles("After deletion", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("After deletion", storageProvider.getStorage()); } } \ No newline at end of file diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java index 10088007f..8698243a0 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java @@ -24,7 +24,7 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageProvider; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; @@ -49,7 +49,7 @@ public class ProductControllerImplTest_get { private MockMvc mockMvc; @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; @Autowired private StorageProvider storageProvider; @@ -140,7 +140,7 @@ private void getProductFiles(StorageType storageType) throws Exception { } // show storage files - StorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); // HTTP Get files from storage MockHttpServletRequestBuilder request = MockMvcRequestBuilders.get(REQUEST_STRING) @@ -175,6 +175,6 @@ private void getProductFiles(StorageType storageType) throws Exception { storageProvider.getStorage().delete(prefix); // show storage files after deletion - StorageTestUtils.printStorageFiles("After deletion", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("After deletion", storageProvider.getStorage()); } } \ No newline at end of file diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java index d246fc390..2177c4cd8 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java @@ -20,7 +20,7 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageProvider; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageType; @@ -112,7 +112,7 @@ private void getProductFiles(StorageType storageType) throws Exception { TestUtils.printMethodName(this, testName); // show storage files - StorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); // HTTP Get files from storage MockHttpServletRequestBuilder request = MockMvcRequestBuilders.get(REQUEST_STRING) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java index 6679fe7a7..07b09b759 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java @@ -26,7 +26,7 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageProvider; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageType; @@ -143,7 +143,7 @@ private void uploadRestProductFS() throws Exception { RestProductFS restProductFS = populateRestProductFS(productId, relativePaths); // show storage files - StorageTestUtils.printStorageFiles("Before http-upload call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("Before http-upload call", storageProvider.getStorage()); // http-upload call MockHttpServletRequestBuilder request = MockMvcRequestBuilders.post(REQUEST_STRING) @@ -155,7 +155,7 @@ private void uploadRestProductFS() throws Exception { TestUtils.printMvcResult(REQUEST_STRING, mvcResult); // show storage files after http-upload - StorageTestUtils.printStorageFiles("After http-upload call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("After http-upload call", storageProvider.getStorage()); // check real with expected absolute storage paths String json = mvcResult.getResponse().getContentAsString(); diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java index 5ad2e837d..2bbb9921a 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java @@ -23,7 +23,7 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageProvider; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; @@ -49,7 +49,7 @@ public class ProductfileControllerImplTest_download { private MockMvc mockMvc; @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; @Autowired private StorageProvider storageProvider; @@ -119,7 +119,7 @@ private void download(String testID) throws Exception { storageProvider.getStorage().upload(sourceFile, storageFile); // show storage files - StorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); // rest-download file from storage to cache String absoluteStoragePath = storageProvider.getStorage().getAbsolutePath(relativePath); diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java index f9dd9566e..abd14f994 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java @@ -21,7 +21,7 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageProvider; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; @@ -46,7 +46,7 @@ public class ProductfileControllerImplTest_upload { private MockMvc mockMvc; @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; @Autowired private StorageProvider storageProvider; @@ -124,7 +124,7 @@ private void upload() throws Exception { assertTrue("Expected path: " + realRelativeStoragePath + " Exists: " + relativePath, relativePath.equals(realRelativeStoragePath)); // show storage files - StorageTestUtils.printStorageFiles("After http-call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("After http-call", storageProvider.getStorage()); // delete files with empty folders new FileUtils(absoluteSourcePath).deleteFile(); // source diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java index 3dfaa68aa..d1aad5592 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java @@ -19,7 +19,7 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageProvider; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; @@ -34,7 +34,7 @@ public class StorageProviderTest { private TestUtils testUtils; @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; @Autowired private StorageProvider storageProvider; @@ -97,7 +97,7 @@ public void testPosixPosixProvider() throws IOException { assertTrue("File was not uploaded to storage: " + storageFilePath, TestUtils.fileExists(storageFilePath)); - StorageTestUtils.printStorageFileList("Storage Files (should be 1 file) ", storage.getStorageFiles()); + BaseStorageTestUtils.printStorageFileList("Storage Files (should be 1 file) ", storage.getStorageFiles()); // ----------------------- download -------------------------- diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java index d2bb56d58..e984a337e 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java @@ -16,7 +16,7 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageProvider; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageType; @@ -30,7 +30,7 @@ public class StorageUtilityTest { private TestUtils testUtils; @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; @Autowired private StorageProvider storageProvider; @@ -67,13 +67,13 @@ public void testUtility() throws IOException { TestUtils.printList("S3 Buckets:", s3Storage.getBuckets()); // show s3 storage files - StorageTestUtils.printStorageFiles("S3 before Action", s3Storage); + BaseStorageTestUtils.printStorageFiles("S3 before Action", s3Storage); // show posix storage files - StorageTestUtils.printStorageFiles("POSIX before Action", posixStorage); + BaseStorageTestUtils.printStorageFiles("POSIX before Action", posixStorage); // show default storage files - StorageTestUtils.printStorageFiles("Default Storage BEFORE Action", storage); + BaseStorageTestUtils.printStorageFiles("Default Storage BEFORE Action", storage); // ACTIONS // @@ -95,6 +95,6 @@ public void testUtility() throws IOException { // storage.delete(path); // show default storage files - StorageTestUtils.printStorageFiles("Default Storage AFTER Action", storage); + BaseStorageTestUtils.printStorageFiles("Default Storage AFTER Action", storage); } } diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixDALTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixDALTest.java index c10854df9..48a1377cb 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixDALTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixDALTest.java @@ -17,7 +17,8 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import de.dlr.proseo.storagemgr.StorageManager; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.UniqueStorageTestPaths; import de.dlr.proseo.storagemgr.posix.PosixDAL; @@ -32,7 +33,10 @@ public class PosixDALTest { public TestName testName = new TestName(); @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; + + @Autowired + private StorageProvider storageProvider; @Test public void test() { @@ -56,7 +60,7 @@ public void test() { String sourcePath = uniquePaths.getUniqueSourcePath(); String storagePath = uniquePaths.getUniqueStoragePath(); - PosixDAL posixDAL = new PosixDAL(); + PosixDAL posixDAL = new PosixDAL(storageProvider.getPosixConfigurationFromFile()); try { // print source files diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java index 8bce48203..7bbc6949a 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java @@ -18,7 +18,7 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageProvider; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; @@ -29,7 +29,7 @@ public class PosixStorageTest { @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; @Autowired private StorageProvider storageProvider; diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java index 3ea6f3a45..bbbdba7b3 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java @@ -20,7 +20,7 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageProvider; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; @@ -35,7 +35,7 @@ public class S3StorageTest { private TestUtils testUtils; @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; @Autowired private StorageProvider storageProvider; From b7d18d2270f1156a80cf10d980d68777abdd96a9 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Fri, 24 Nov 2023 15:15:09 +0100 Subject: [PATCH 19/79] storage-mgr: integrated posix atomic file uploader and downloader --- .../posix/PosixAtomicFileDownloader.java | 131 ++++++++++++++++++ .../posix/PosixAtomicFileUploader.java | 4 +- .../dlr/proseo/storagemgr/posix/PosixDAL.java | 31 +---- .../rest/ProductfileControllerImpl.java | 3 +- .../storagemgr/BaseStorageTestUtils.java | 10 -- .../ProductfileControllerImplTest_upload.java | 4 +- 6 files changed, 141 insertions(+), 42 deletions(-) create mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileDownloader.java diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileDownloader.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileDownloader.java new file mode 100644 index 000000000..a605f556d --- /dev/null +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileDownloader.java @@ -0,0 +1,131 @@ +/** + * PosixAtomicFileDownloader.java + * + * (C) 2023 Dr. Bassler & Co. Managementberatung GmbH + */ +package de.dlr.proseo.storagemgr.posix; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.dlr.proseo.storagemgr.model.AtomicCommand; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; + +/** + * Posix Atomic File Downloader from the POSIX Storage to some POSIX Path (normally, to the cache) + * + * @author Denys Chaykovskiy + * + */ +public class PosixAtomicFileDownloader implements AtomicCommand { + + /** Info */ + private static final String INFO = "Posix ATOMIC File Downloader"; + + /** Completed Info */ + private static final String COMPLETED = "file DOWNLOADED"; + + /** Failed Info */ + private static final String FAILED = "file download FAILED"; + + /** Logger for this class */ + private static Logger logger = LoggerFactory.getLogger(PosixAtomicFileDownloader.class); + + /** source file */ + private String sourceFile; + + /** target file or dir */ + private String targetFileOrDir; + + /** + * Constructor + * + * @param sourceFile sourceFile + * @param targetFileOrDir target file or directory + */ + public PosixAtomicFileDownloader(String sourceFile, String targetFileOrDir) { + + this.sourceFile = sourceFile; + this.targetFileOrDir = targetFileOrDir; + } + + /** + * Executes download of the file from the storage to posix path + * + * @return uploaded file name + */ + @Override + public String execute() throws IOException { + + if (logger.isTraceEnabled()) + logger.trace(">>> execute() - downloadFile({},{})", sourceFile, targetFileOrDir); + + String targetFile = targetFileOrDir; + + if (new PathConverter(targetFileOrDir).isDirectory()) { + targetFile = new PathConverter(targetFileOrDir, getFileName(sourceFile)).getPath(); + } + + new FileUtils(targetFile).createParentDirectories(); + + Path sourceFilePath = new File(sourceFile).toPath(); + Path targetFilePath = new File(targetFile).toPath(); + + try { + Path copiedPath = Files.copy(sourceFilePath, targetFilePath, StandardCopyOption.REPLACE_EXISTING); + return copiedPath.toString(); + + } catch (Exception e) { + if (logger.isTraceEnabled()) + logger.trace(getFailedInfo() + e.getMessage()); + throw new IOException(e); + } + } + + /** + * Gets information about atomic command (mostly for logs) + * + * @return information about atomic command + */ + @Override + public String getInfo() { + return INFO + " "; + } + + /** + * Gets information about completed atomic command (mostly for logs) + * + * @return information about completed atomic command + */ + @Override + public String getCompletedInfo() { + return INFO + ": " + COMPLETED + " "; + } + + /** + * Gets information about failed atomic command (mostly for logs) + * + * @return information about failed atomic command + */ + @Override + public String getFailedInfo() { + return INFO + ": " + FAILED + " "; + } + + /** + * Gets file name + * + * @param path path + * @return file name + */ + private String getFileName(String path) { + return new File(path).getName(); + } +} \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileUploader.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileUploader.java index 812ca22cd..f456e5ecd 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileUploader.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileUploader.java @@ -19,7 +19,7 @@ import de.dlr.proseo.storagemgr.utils.PathConverter; /** - * Posix Atomic Uploader + * Posix Atomic File Uploader from some external POSIX absolute path to the POSIX Storage * * @author Denys Chaykovskiy * @@ -57,7 +57,7 @@ public PosixAtomicFileUploader(String sourceFile, String targetFileOrDir) { } /** - * Executes upload of the file to posix + * Executes upload of the file to posix path * * @return uploaded file name */ diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java index aade87071..8a285e13b 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java @@ -170,30 +170,10 @@ public String uploadFile(String sourceFile, String targetFileOrDir) throws IOExc if (logger.isTraceEnabled()) logger.trace(">>> uploadFile({},{})", sourceFile, targetFileOrDir); + + AtomicCommand fileUploader = new PosixAtomicFileUploader(sourceFile, targetFileOrDir); - String targetFile = targetFileOrDir; - - if (new PathConverter(targetFileOrDir).isDirectory()) { - targetFile = new PathConverter(targetFileOrDir, getFileName(sourceFile)).getPath(); - } - - createParentDirectories(targetFile); - - Path sourceFilePath = new File(sourceFile).toPath(); - Path targetFilePath = new File(targetFile).toPath(); - - try { - Path copiedPath = Files.copy(sourceFilePath, targetFilePath, StandardCopyOption.REPLACE_EXISTING); - return copiedPath.toString(); - - } catch (IOException e) { - if (logger.isDebugEnabled()) { - logger.debug("An exception occurred. Cause: ", e); - } - if (logger.isTraceEnabled()) - logger.log(StorageMgrMessage.FILE_NOT_UPLOADED, sourceFile, targetFileOrDir, e.getMessage()); - throw e; - } + return new DefaultRetryStrategy<>(fileUploader, cfg.getMaxRequestAttempts(), cfg.getFileCheckWaitTime()).execute(); } /** @@ -261,10 +241,9 @@ public String downloadFile(String sourceFile, String targetFileOrDir) throws IOE if (logger.isTraceEnabled()) logger.trace(">>> downloadFile({},{})", sourceFile, targetFileOrDir); - - AtomicCommand fileUploader = new PosixAtomicFileUploader(sourceFile, targetFileOrDir); + AtomicCommand fileDownloader = new PosixAtomicFileDownloader(sourceFile, targetFileOrDir); - return new DefaultRetryStrategy<>(fileUploader, cfg.getMaxRequestAttempts(), cfg.getFileCheckWaitTime()).execute(); + return new DefaultRetryStrategy<>(fileDownloader, cfg.getMaxRequestAttempts(), cfg.getFileCheckWaitTime()).execute(); } /** diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index be5a0ac26..d82fa3e18 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -230,7 +230,6 @@ private RestFileInfo synchronizedDownloadFromAbsolutePathToCache(String absolute String productFolderWithFilename = Paths.get(String.valueOf(productId), fileName).toString(); StorageFile targetFile = storageProvider.getCacheFile(productFolderWithFilename); - FileCache cache = FileCache.getInstance(); if (!cache.containsKey(targetFile.getFullPath())) { @@ -248,7 +247,7 @@ private RestFileInfo synchronizedDownloadFromAbsolutePathToCache(String absolute // active thread - downloads the file and puts it to the cache // TODO: ADD FUNCTIONALITY DOWNLOAD TO CACHE - //storageProvider.getStorage().downloadFile(sourceFile, targetFile); + storageProvider.getStorage().downloadFile(sourceFile, targetFile); logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED, targetFile.getFullPath()); diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java index 49581f85c..cc5ebd543 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java @@ -10,13 +10,8 @@ import org.junit.Rule; import org.junit.rules.TestName; -import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; import org.springframework.stereotype.Component; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; @@ -28,11 +23,6 @@ * @author Denys Chaykovskiy * */ -/* -@RunWith(SpringJUnit4ClassRunner.class) -@SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) -@AutoConfigureMockMvc -*/ @Component public class BaseStorageTestUtils { diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java index abd14f994..4f3b951fc 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java @@ -57,7 +57,7 @@ public class ProductfileControllerImplTest_upload { private static final String REQUEST_STRING = "/proseo/storage-mgr/x/productfiles"; @Test - public void testUpload_v2Posix() throws Exception { + public void testUpload_Posix() throws Exception { StorageType storageType = StorageType.POSIX; storageProvider.setStorage(storageType); @@ -69,7 +69,7 @@ public void testUpload_v2Posix() throws Exception { } @Test - public void testUpload_v2S3() throws Exception { + public void testUpload_S3() throws Exception { StorageType storageType = StorageType.S3; storageProvider.setStorage(storageType); From a455f0f8c2ea41f1519b579bdf38ab4260676b17 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Mon, 4 Dec 2023 14:03:31 +0100 Subject: [PATCH 20/79] storage-mgr: created atomic file copier for posix fs --- .../posix/PosixAtomicFileCopier.java | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileCopier.java diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileCopier.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileCopier.java new file mode 100644 index 000000000..0ff624039 --- /dev/null +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileCopier.java @@ -0,0 +1,131 @@ +/** + * PosixAtomicFileCopier.java + * + * (C) 2023 Dr. Bassler & Co. Managementberatung GmbH + */ +package de.dlr.proseo.storagemgr.posix; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.dlr.proseo.storagemgr.model.AtomicCommand; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; + +/** + * Posix Atomic File Copier from some POSIX absolute path to another POSIX absolute path (normally, to cache) + * + * @author Denys Chaykovskiy + * + */ +public class PosixAtomicFileCopier implements AtomicCommand { + + /** Info */ + private static final String INFO = "Posix ATOMIC File Copier"; + + /** Completed Info */ + private static final String COMPLETED = "file COPIED"; + + /** Failed Info */ + private static final String FAILED = "file copy FAILED"; + + /** Logger for this class */ + private static Logger logger = LoggerFactory.getLogger(PosixAtomicFileCopier.class); + + /** source file */ + private String sourceFile; + + /** target file or dir */ + private String targetFileOrDir; + + /** + * Constructor + * + * @param sourceFile sourceFile + * @param targetFileOrDir target file or directory + */ + public PosixAtomicFileCopier(String sourceFile, String targetFileOrDir) { + + this.sourceFile = sourceFile; + this.targetFileOrDir = targetFileOrDir; + } + + /** + * Executes copy of the file from a posix path to a posix path (normally, to the cache) + * + * @return copied file name + */ + @Override + public String execute() throws IOException { + + if (logger.isTraceEnabled()) + logger.trace(">>> execute() - copyFile({},{})", sourceFile, targetFileOrDir); + + String targetFile = targetFileOrDir; + + if (new PathConverter(targetFileOrDir).isDirectory()) { + targetFile = new PathConverter(targetFileOrDir, getFileName(sourceFile)).getPath(); + } + + new FileUtils(targetFile).createParentDirectories(); + + Path sourceFilePath = new File(sourceFile).toPath(); + Path targetFilePath = new File(targetFile).toPath(); + + try { + Path copiedPath = Files.copy(sourceFilePath, targetFilePath, StandardCopyOption.REPLACE_EXISTING); + return copiedPath.toString(); + + } catch (Exception e) { + if (logger.isTraceEnabled()) + logger.trace(getFailedInfo() + e.getMessage()); + throw new IOException(e); + } + } + + /** + * Gets information about atomic command (mostly for logs) + * + * @return information about atomic command + */ + @Override + public String getInfo() { + return INFO + " "; + } + + /** + * Gets information about completed atomic command (mostly for logs) + * + * @return information about completed atomic command + */ + @Override + public String getCompletedInfo() { + return INFO + ": " + COMPLETED + " "; + } + + /** + * Gets information about failed atomic command (mostly for logs) + * + * @return information about failed atomic command + */ + @Override + public String getFailedInfo() { + return INFO + ": " + FAILED + " "; + } + + /** + * Gets file name + * + * @param path path + * @return file name + */ + private String getFileName(String path) { + return new File(path).getName(); + } +} \ No newline at end of file From 326f8b8c78b75663f53047f35947f3a9ac72a6fe Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Tue, 5 Dec 2023 16:33:23 +0100 Subject: [PATCH 21/79] storage-mgr: adding copy files posix-posix method to posixDAL --- .../dlr/proseo/storagemgr/posix/PosixDAL.java | 96 +++++++++++++++---- 1 file changed, 76 insertions(+), 20 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java index 8a285e13b..83b31f7d2 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java @@ -10,13 +10,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import de.dlr.proseo.logging.logger.ProseoLogger; -import de.dlr.proseo.logging.messages.StorageMgrMessage; import de.dlr.proseo.storagemgr.model.AtomicCommand; import de.dlr.proseo.storagemgr.model.DefaultRetryStrategy; import de.dlr.proseo.storagemgr.utils.FileUtils; @@ -302,6 +300,82 @@ public List download(String sourceFileOrDir, String targetFileOrDir) thr return downloadedFiles; } + + /** + * Copies a file from the POSIX file system to another posix file system (normally, to the cache) + * location + * + * @param sourceFile posix source file + * @param targetFileOrDir posix target file or directory + * @return path of downloaded file + * @throws IOException if file cannot be downloaded + */ + public String copyFile(String sourceFile, String targetFileOrDir) throws IOException { + + if (logger.isTraceEnabled()) + logger.trace(">>> copyFile({},{})", sourceFile, targetFileOrDir); + + AtomicCommand fileCopier = new PosixAtomicFileCopier(sourceFile, targetFileOrDir); + + return new DefaultRetryStrategy<>(fileCopier, cfg.getMaxRequestAttempts(), cfg.getFileCheckWaitTime()).execute(); + } + + /** + * Copies files or directories recursively from the posix file system to another posix file system (normally, to the cache) + * + * @param sourceFileOrDir posix source file or directory + * @param targetFileOrDir posix target file or directory + * @return path list of copied files + * @throws IOException true if file or directory cannot be copied + */ + public List copy(String sourceFileOrDir, String targetFileOrDir) throws IOException { + + if (logger.isTraceEnabled()) + logger.trace(">>> copy({},{})", sourceFileOrDir, targetFileOrDir); + + List copiedFiles = new ArrayList<>(); + + if (isFile(sourceFileOrDir)) { + String copiedFile = copyFile(sourceFileOrDir, targetFileOrDir); + copiedFiles.add(copiedFile); + return copiedFiles; + } + + String sourceDir = sourceFileOrDir; + String targetDir = targetFileOrDir; + targetDir = new PathConverter(targetDir).addSlashAtEnd().getPath(); + + File directory = new File(sourceDir); + File[] files = directory.listFiles(); + if (files == null) + return copiedFiles; + Arrays.sort(files); + + for (File file : files) { + if (file.isFile()) { + String sourceFile = file.getAbsolutePath(); + String copiedFile = downloadFile(sourceFile, targetDir); + copiedFiles.add(copiedFile); + } + } + + for (File file : files) { + if (file.isDirectory()) { + + String sourceSubDir = file.getAbsolutePath(); + String targetSubDir = Paths.get(targetDir, file.getName()).toString(); + targetSubDir = new PathConverter(targetSubDir).addSlashAtEnd().getPath(); + + // String path = new PathConverter(targetSubDirPath).addSlashAtEnd().getPath(); + // targetSubDir.setRelativePath(path); + + List subDirFiles = copy(sourceSubDir, targetSubDir); + copiedFiles.addAll(subDirFiles); + } + } + + return copiedFiles; + } /** * Deletes file in storage @@ -332,22 +406,4 @@ public List delete(String sourceFileOrDir) throws IOException { return new FileUtils(sourceFileOrDir).delete(); } - - /** - * Create parent directories for a given path if they do not exist - * - * @param path path - */ - private void createParentDirectories(String path) { - - if (logger.isTraceEnabled()) - logger.trace(">>> createParentDirectories({})", path); - - File targetFile = new File(path); - File parent = targetFile.getParentFile(); - - if (parent != null && !parent.exists() && !parent.mkdirs()) { - throw new IllegalStateException("Couldn't create dir: " + parent); - } - } } \ No newline at end of file From c3726a6b739455c9b1897e345984be60e7ad560f Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Tue, 5 Dec 2023 16:43:11 +0100 Subject: [PATCH 22/79] storage-mgr: copy files to cache from external storage in provider --- .../proseo/storagemgr/StorageProvider.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageProvider.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageProvider.java index 94fc5c368..a6e3612c1 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageProvider.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageProvider.java @@ -5,6 +5,7 @@ */ package de.dlr.proseo.storagemgr; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; @@ -26,6 +27,7 @@ import de.dlr.proseo.storagemgr.posix.PosixStorage; import de.dlr.proseo.storagemgr.posix.PosixStorageFile; import de.dlr.proseo.storagemgr.posix.PosixConfiguration; +import de.dlr.proseo.storagemgr.posix.PosixDAL; import de.dlr.proseo.storagemgr.s3.S3Configuration; import de.dlr.proseo.storagemgr.s3.S3Storage; @@ -485,6 +487,26 @@ public S3Configuration getS3ConfigurationFromFile() { return s3Configuration; } + /** + * @param sourceFileOrDir + * @return + * @throws IOException + */ + public List copyAbsoluteFilesToCache(String sourceFileOrDir, Long productId ) throws IOException { + + if (logger.isTraceEnabled()) + logger.trace(">>> copyAbsoluteFilesToCache({})", sourceFileOrDir); + + String fileName = new File(sourceFileOrDir).getName(); + String productFolderWithFilename = Paths.get(String.valueOf(productId), fileName).toString(); + StorageFile targetFile = getCacheFile(productFolderWithFilename); + + PosixDAL posixDAL = new PosixDAL(getPosixConfigurationFromFile()); + + return posixDAL.copy(sourceFileOrDir, targetFile.getFullPath()); + + } + /** * Initializes storage(s) from Application.yml * From b8aab5a641463733390714d5cb3339856c837b86 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Tue, 5 Dec 2023 16:45:50 +0100 Subject: [PATCH 23/79] storage-mgr: renamed lock to lockOrWaitUntilUnlocked in StrgFileLocker --- .../rest/ProductfileControllerImpl.java | 35 ++++++++++--------- .../storagemgr/utils/StorageFileLocker.java | 2 +- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index d82fa3e18..382733404 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -76,7 +76,7 @@ public ResponseEntity getRestFileInfoByPathInfo(String pathInfo) { cfg.getFileCheckMaxCycles()); try { - + RestFileInfo restFileInfo = synchronizedDownloadFromStorageToCache(pathInfo, fileLocker); return new ResponseEntity<>(restFileInfo, HttpStatus.OK); @@ -188,24 +188,25 @@ private RestFileInfo synchronizedDownloadFromStorageToCache(String storageFilePa if (!cache.containsKey(targetFile.getFullPath())) { - fileLocker.lock(); + fileLocker.lockOrWaitUntilUnlocked(); - // After lock() the active thread downloads the file and put it to the cache - // (see - // below) + // After lock() the active thread starts to download the file and put it to the cache + // (see below) // After lock() the passive thread did nothing, but the file has been downloaded // and the cache has been updated - need to check if file contains in the cache // again + if (!cache.containsKey(targetFile.getFullPath())) { - // active thread - downloads the file and puts it to the cache + // I am active thread - downloads the file and puts it to the cache storageProvider.getStorage().downloadFile(sourceFile, targetFile); logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED, targetFile.getFullPath()); cache.put(targetFile.getFullPath()); + } else { - // passive thread - did nothing, just waited until the file has been downloaded + // I am passive thread - did nothing, just waited until the file has been downloaded // and use it from cache logger.debug("... waiting-thread when the file downloaded and use it from cache: ", targetFile.getFullPath()); @@ -220,21 +221,21 @@ private RestFileInfo synchronizedDownloadFromStorageToCache(String storageFilePa return restFileInfo; } - - private RestFileInfo synchronizedDownloadFromAbsolutePathToCache(String absolutePath, Long productId, Long fileSize, StorageFileLocker fileLocker) - throws FileLockedAfterMaxCyclesException, IOException, Exception { + + private RestFileInfo synchronizedDownloadFromAbsolutePathToCache(String absolutePath, Long productId, Long fileSize, + StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { StorageFile sourceFile = storageProvider.getAbsoluteFile(absolutePath); - + String fileName = new File(absolutePath).getName(); String productFolderWithFilename = Paths.get(String.valueOf(productId), fileName).toString(); StorageFile targetFile = storageProvider.getCacheFile(productFolderWithFilename); - + FileCache cache = FileCache.getInstance(); if (!cache.containsKey(targetFile.getFullPath())) { - fileLocker.lock(); + fileLocker.lockOrWaitUntilUnlocked(); // After lock() the active thread downloads the file and put it to the cache // (see @@ -245,10 +246,10 @@ private RestFileInfo synchronizedDownloadFromAbsolutePathToCache(String absolute if (!cache.containsKey(targetFile.getFullPath())) { // active thread - downloads the file and puts it to the cache - - // TODO: ADD FUNCTIONALITY DOWNLOAD TO CACHE - storageProvider.getStorage().downloadFile(sourceFile, targetFile); - + + // TODO: WIP + storageProvider.copyAbsoluteFilesToCache(absolutePath, productId); + logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED, targetFile.getFullPath()); cache.put(targetFile.getFullPath()); diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java index 888672e4c..1c2160092 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java @@ -65,7 +65,7 @@ public StorageFileLocker(String path, long waitTime, long fileCheckMaxCycles) { * waiting for the concurrent access * to terminate */ - public void lock() throws FileLockedAfterMaxCyclesException, InterruptedException { + public void lockOrWaitUntilUnlocked() throws FileLockedAfterMaxCyclesException, InterruptedException { long i = 0; From 6af0a757b0d535ecd7727744742718dbfda8eebf Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Wed, 6 Dec 2023 15:49:39 +0100 Subject: [PATCH 24/79] storage-mgr: added synchronized uploading from cache to storage --- .../logging/messages/StorageMgrMessage.java | 6 + .../proseo/storagemgr/StorageProvider.java | 2 +- .../rest/ProductfileControllerImpl.java | 156 ++++++++++++++---- 3 files changed, 131 insertions(+), 33 deletions(-) diff --git a/logging/src/main/java/de/dlr/proseo/logging/messages/StorageMgrMessage.java b/logging/src/main/java/de/dlr/proseo/logging/messages/StorageMgrMessage.java index 73f28a6ed..4fcae3703 100644 --- a/logging/src/main/java/de/dlr/proseo/logging/messages/StorageMgrMessage.java +++ b/logging/src/main/java/de/dlr/proseo/logging/messages/StorageMgrMessage.java @@ -65,6 +65,12 @@ public enum StorageMgrMessage implements ProseoMessage { CACHE_CLEANUP_REPORT (5573, Level.INFO, true, "Cache cleanup removed {0} entries from file cache in {1} ms", ""), CACHE_NOT_ENOUGH_SPACE_AFTER_CLEANING (5574, Level.ERROR, false, "Disk usage {0} exceeds maximum usage {1} after emptying cache", ""), CACHE_FILE_NOT_DELETED (5575, Level.WARN, false, "Cache File was not deleted: {0}", ""), + + /** Downloaded = retrieved */ + PRODUCT_FILE_DOWNLOADED_FROM_EXTERNAL_TO_CACHE (5581, Level.INFO, true, "Product file downloaded from external storage to cache: {0}", ""), + /** Uploaded = registered */ + PRODUCT_FILE_UPLOADED_FROM_CACHE_TO_STORAGE (5552, Level.INFO, true, "Product file {0} uploaded from cache to storage for product {1}", ""), + ; private final int code; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageProvider.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageProvider.java index a6e3612c1..6aa5c8027 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageProvider.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageProvider.java @@ -492,7 +492,7 @@ public S3Configuration getS3ConfigurationFromFile() { * @return * @throws IOException */ - public List copyAbsoluteFilesToCache(String sourceFileOrDir, Long productId ) throws IOException { + public List copyAbsoluteFilesToCache(String sourceFileOrDir, Long productId) throws IOException { if (logger.isTraceEnabled()) logger.trace(">>> copyAbsoluteFilesToCache({})", sourceFileOrDir); diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index 382733404..5a31e2101 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -71,20 +71,32 @@ public ResponseEntity getRestFileInfoByPathInfo(String pathInfo) { // Storage Manager version 2: download Storage -> Cache // pathInfo is absolute path s3://bucket/.. or /storagePath/.. + + String absoluteStoragePath = pathInfo; + + StorageFile cacheFile; + try { + String relativePath = storageProvider.getRelativePath(absoluteStoragePath); + cacheFile = storageProvider.getCacheFile(relativePath); + + } catch (IOException e) { + String msg = logger.log(StorageMgrMessage.PRODUCT_FILE_CANNOT_BE_DOWNLOADED, e.getMessage()); + return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.BAD_REQUEST); + } - StorageFileLocker fileLocker = new StorageFileLocker(pathInfo, cfg.getFileCheckWaitTime(), + StorageFileLocker fileLocker = new StorageFileLocker(cacheFile.getFullPath(), cfg.getFileCheckWaitTime(), cfg.getFileCheckMaxCycles()); try { - RestFileInfo restFileInfo = synchronizedDownloadFromStorageToCache(pathInfo, fileLocker); + RestFileInfo restFileInfo = synchronizedDownloadFromStorageToCache(absoluteStoragePath, fileLocker); return new ResponseEntity<>(restFileInfo, HttpStatus.OK); } catch (FileLockedAfterMaxCyclesException e) { String time = String.valueOf(cfg.getFileCheckMaxCycles() * cfg.getFileCheckWaitTime() / 1000); - String msg = logger.log(StorageMgrMessage.READ_TIME_OUT, pathInfo, time, e.getLocalizedMessage()); + String msg = logger.log(StorageMgrMessage.READ_TIME_OUT, absoluteStoragePath, time, e.getLocalizedMessage()); return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.SERVICE_UNAVAILABLE); @@ -120,7 +132,7 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro if (logger.isTraceEnabled()) logger.trace(">>> updateProductfiles({}, {})", pathInfo, productId); - // Storage Manager version 2: pathInfo absolute path, upload absolute file -> + // Storage Manager version 2: pathInfo absolute external path, upload absolute file -> // storage // 1. download to cache absolute-file -> cache @@ -130,21 +142,39 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro if (pathInfo == null) { return new ResponseEntity(new RestFileInfo(), HttpStatus.BAD_REQUEST); } + + String absoluteExternalPath = pathInfo; + + String relativePath = getProductFolderWithFilename(absoluteExternalPath, productId); + StorageFile cacheFile = storageProvider.getCacheFile(relativePath); + + StorageFileLocker fileLocker = new StorageFileLocker(cacheFile.getFullPath(), cfg.getFileCheckWaitTime(), + cfg.getFileCheckMaxCycles()); try { + + RestFileInfo restFileInfo = synchronizedDownloadFromAbsolutePathToCache(absoluteExternalPath, productId, fileSize, fileLocker); + + fileLocker.unlock(); + + restFileInfo = synchronizedUploadFromCacheToStorage(relativePath, fileLocker); + + /* Storage storage = storageProvider.getStorage(); String absolutePath = pathInfo; - String fileName = new File(pathInfo).getName(); + String fileName = new File(absolutePath).getName(); String productFolderWithFilename = Paths.get(String.valueOf(productId), fileName).toString(); - StorageFile sourceFile = storageProvider.getAbsoluteFile(absolutePath); - StorageFile targetFile = storageProvider.getStorageFile(productFolderWithFilename); + StorageFile sourceExternalFile = storageProvider.getAbsoluteFile(absolutePath); + StorageFile targetStorageFile = storageProvider.getStorageFile(productFolderWithFilename); - storage.uploadFile(sourceFile, targetFile); + storage.uploadFile(sourceExternalFile, targetStorageFile); - RestFileInfo restFileInfo = convertToRestFileInfo(targetFile, storage.getFileSize(targetFile)); + RestFileInfo restFileInfo = convertToRestFileInfo(targetStorageFile, + storage.getFileSize(targetStorageFile)); + */ - logger.log(StorageMgrMessage.PRODUCT_FILE_UPLOADED, pathInfo, productId); + logger.log(StorageMgrMessage.PRODUCT_FILE_UPLOADED, absoluteExternalPath, productId); return new ResponseEntity<>(restFileInfo, HttpStatus.CREATED); @@ -153,6 +183,11 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); } + finally { + + fileLocker.unlock(); + logger.debug("... unlocked the file: ", absoluteExternalPath); + } } /** @@ -190,7 +225,8 @@ private RestFileInfo synchronizedDownloadFromStorageToCache(String storageFilePa fileLocker.lockOrWaitUntilUnlocked(); - // After lock() the active thread starts to download the file and put it to the cache + // After lock() the active thread starts to download the file and put it to the + // cache // (see below) // After lock() the passive thread did nothing, but the file has been downloaded // and the cache has been updated - need to check if file contains in the cache @@ -203,10 +239,11 @@ private RestFileInfo synchronizedDownloadFromStorageToCache(String storageFilePa logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED, targetFile.getFullPath()); cache.put(targetFile.getFullPath()); - + } else { - // I am passive thread - did nothing, just waited until the file has been downloaded + // I am passive thread - did nothing, just waited until the file has been + // downloaded // and use it from cache logger.debug("... waiting-thread when the file downloaded and use it from cache: ", targetFile.getFullPath()); @@ -222,53 +259,108 @@ private RestFileInfo synchronizedDownloadFromStorageToCache(String storageFilePa return restFileInfo; } - private RestFileInfo synchronizedDownloadFromAbsolutePathToCache(String absolutePath, Long productId, Long fileSize, + private RestFileInfo synchronizedDownloadFromAbsolutePathToCache(String absoluteExternalPath, Long productId, Long fileSize, StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { - StorageFile sourceFile = storageProvider.getAbsoluteFile(absolutePath); - - String fileName = new File(absolutePath).getName(); - String productFolderWithFilename = Paths.get(String.valueOf(productId), fileName).toString(); - StorageFile targetFile = storageProvider.getCacheFile(productFolderWithFilename); + String productFolderWithFilename = getProductFolderWithFilename(absoluteExternalPath, productId); + StorageFile targetCacheFile = storageProvider.getCacheFile(productFolderWithFilename); FileCache cache = FileCache.getInstance(); - if (!cache.containsKey(targetFile.getFullPath())) { + if (!cache.containsKey(targetCacheFile.getFullPath())) { fileLocker.lockOrWaitUntilUnlocked(); - // After lock() the active thread downloads the file and put it to the cache - // (see - // below) + // After lock() the active thread starts to download the file and put it to the + // cache + // (see below) // After lock() the passive thread did nothing, but the file has been downloaded // and the cache has been updated - need to check if file contains in the cache // again - if (!cache.containsKey(targetFile.getFullPath())) { - // active thread - downloads the file and puts it to the cache + if (!cache.containsKey(targetCacheFile.getFullPath())) { + + // I am active thread - downloads the file and puts it to the cache // TODO: WIP - storageProvider.copyAbsoluteFilesToCache(absolutePath, productId); + storageProvider.copyAbsoluteFilesToCache(absoluteExternalPath, productId); - logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED, targetFile.getFullPath()); + logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED_FROM_EXTERNAL_TO_CACHE, + targetCacheFile.getFullPath()); - cache.put(targetFile.getFullPath()); + cache.put(targetCacheFile.getFullPath()); } else { // passive thread - did nothing, just waited until the file has been downloaded // and use it from cache + logger.debug( + "... waiting-thread when the file downloaded to cache from external storage and use it from cache: ", + targetCacheFile.getFullPath()); + } + + } else { + logger.debug("... no download and no lock - the file is in cache: ", targetCacheFile.getFullPath()); + } + + RestFileInfo restFileInfo = convertToRestFileInfo(targetCacheFile, + storageProvider.getCacheFileSize(targetCacheFile.getRelativePath())); + + return restFileInfo; + } + + private RestFileInfo synchronizedUploadFromCacheToStorage(String relativeCachePath, StorageFileLocker fileLocker) + throws FileLockedAfterMaxCyclesException, IOException, Exception { + + Storage storage = storageProvider.getStorage(); + + StorageFile sourceCacheFile = storageProvider.getCacheFile(relativeCachePath); + StorageFile targetStorageFile = storageProvider.getStorageFile(relativeCachePath); + + FileCache cache = FileCache.getInstance(); + + if (!cache.containsKey(sourceCacheFile.getFullPath())) { + + fileLocker.lockOrWaitUntilUnlocked(); + + // After lock() the active thread starts to download the file and put it to the + // cache + // (see below) + // After lock() the passive thread did nothing, but the file has been downloaded + // and the cache has been updated - need to check if file contains in the cache + // again + + if (!cache.containsKey(sourceCacheFile.getFullPath())) { + + // I am active thread - uploads the file and puts it to the cache + + storage.uploadFile(sourceCacheFile, targetStorageFile); + + logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED_FROM_EXTERNAL_TO_CACHE, + targetStorageFile.getFullPath()); + + cache.put(targetStorageFile.getFullPath()); + } else { + + // passive thread - did nothing, just waited until the file has been uploaded + // and use it from cache logger.debug("... waiting-thread when the file downloaded and use it from cache: ", - targetFile.getFullPath()); + targetStorageFile.getFullPath()); } } else { - logger.debug("... no download and no lock - the file is in cache: ", targetFile.getFullPath()); + logger.debug("... no download and no lock - the file is in cache: ", targetStorageFile.getFullPath()); } - RestFileInfo restFileInfo = convertToRestFileInfo(targetFile, - storageProvider.getCacheFileSize(sourceFile.getRelativePath())); + RestFileInfo restFileInfo = convertToRestFileInfo(targetStorageFile, + storageProvider.getCacheFileSize(sourceCacheFile.getRelativePath())); return restFileInfo; } + + private String getProductFolderWithFilename(String absoluteExternalPath, Long productId) { + + String fileName = new File(absoluteExternalPath).getName(); + return Paths.get(String.valueOf(productId), fileName).toString(); + } } From 98a28b4b4314bf8a3dcd77ef0b28ac9461b75e9d Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Wed, 13 Dec 2023 13:50:04 +0100 Subject: [PATCH 25/79] storage-mgr: created synchro method for creating a file in utils --- .../proseo/storagemgr/utils/FileUtils.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/FileUtils.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/FileUtils.java index dc417a2ab..03470d1d2 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/FileUtils.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/FileUtils.java @@ -16,6 +16,7 @@ import de.dlr.proseo.logging.logger.ProseoLogger; import de.dlr.proseo.logging.messages.StorageMgrMessage; +import de.dlr.proseo.storagemgr.Exceptions.FileLockedAfterMaxCyclesException; /** * A utility class for common file operations. It provides methods to create, @@ -90,6 +91,47 @@ public boolean createFile(String content) { } } + /** + * Creates the file with the content in a synchro modus + * + * @param content Content of the file + * @param waitTime the wait time between each cycle of checking the + * file lock status + * @param fileCheckMaxCycles file check max cycles + * @return true if file was successfully created + */ + public boolean synchroCreateFile(String content, long waitTime, long fileCheckMaxCycles) { + + if (logger.isTraceEnabled()) + logger.trace(">>> synchroCreateFile({}, {}, {})", content, waitTime, fileCheckMaxCycles); + + StorageFileLocker fileLocker = new StorageFileLocker(path, waitTime, fileCheckMaxCycles); + boolean fileCreatedStatus; + + try { + + fileLocker.lockOrWaitUntilUnlockedAndLock(); + fileCreatedStatus = createFile(content); + + } catch (FileLockedAfterMaxCyclesException e) { + + logger.debug("... the file is locked after max check cycles: ", path, " ", e.getMessage()); + fileCreatedStatus = false; + + } catch (InterruptedException e) { + + logger.debug("... the file is locked after max check cycles: ", path, " ", e.getMessage()); + fileCreatedStatus = false; + + } finally { + + fileLocker.unlock(); + logger.debug("... unlocked the file: ", path); + } + + return fileCreatedStatus; + } + /** * Gets the file size * From 143f123caf00333f94a672ee3f24dc22a113f2a1 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Wed, 13 Dec 2023 14:00:53 +0100 Subject: [PATCH 26/79] storage-mgr: added additional logger info to the storage file locker --- .../storagemgr/rest/ProductfileControllerImpl.java | 6 +++--- .../dlr/proseo/storagemgr/utils/StorageFileLocker.java | 9 +++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index 5a31e2101..9ecbf268c 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -223,7 +223,7 @@ private RestFileInfo synchronizedDownloadFromStorageToCache(String storageFilePa if (!cache.containsKey(targetFile.getFullPath())) { - fileLocker.lockOrWaitUntilUnlocked(); + fileLocker.lockOrWaitUntilUnlockedAndLock(); // After lock() the active thread starts to download the file and put it to the // cache @@ -269,7 +269,7 @@ private RestFileInfo synchronizedDownloadFromAbsolutePathToCache(String absolute if (!cache.containsKey(targetCacheFile.getFullPath())) { - fileLocker.lockOrWaitUntilUnlocked(); + fileLocker.lockOrWaitUntilUnlockedAndLock(); // After lock() the active thread starts to download the file and put it to the // cache @@ -320,7 +320,7 @@ private RestFileInfo synchronizedUploadFromCacheToStorage(String relativeCachePa if (!cache.containsKey(sourceCacheFile.getFullPath())) { - fileLocker.lockOrWaitUntilUnlocked(); + fileLocker.lockOrWaitUntilUnlockedAndLock(); // After lock() the active thread starts to download the file and put it to the // cache diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java index 1c2160092..0f044fe26 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java @@ -65,19 +65,23 @@ public StorageFileLocker(String path, long waitTime, long fileCheckMaxCycles) { * waiting for the concurrent access * to terminate */ - public void lockOrWaitUntilUnlocked() throws FileLockedAfterMaxCyclesException, InterruptedException { + public void lockOrWaitUntilUnlockedAndLock() throws FileLockedAfterMaxCyclesException, InterruptedException { long i = 0; for (; i < fileCheckMaxCycles; ++i) { - synchronized (productLockSet) { + // lock the file if not locked (not in the set) + synchronized (productLockSet) { if (!productLockSet.contains(path)) { + productLockSet.add(path); + logger.debug("... the file {} is locked", path); break; } } + // wait, because file is locked (in the set) in another thread if (logger.isDebugEnabled()) logger.debug("... waiting for concurrent access to {} to terminate", path); Thread.sleep(waitTime); @@ -94,5 +98,6 @@ public void lockOrWaitUntilUnlocked() throws FileLockedAfterMaxCyclesException, */ public void unlock() { productLockSet.remove(path); + logger.debug("... the file {} is unlocked", path); } } \ No newline at end of file From e10833bebc48c8a38359d92888c5da7b961c6110 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Wed, 13 Dec 2023 14:05:31 +0100 Subject: [PATCH 27/79] storage-mgr: synchro update of accessed and status files in file cache --- .../main/java/de/dlr/proseo/storagemgr/cache/FileCache.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java index 161742dbc..235bce929 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java @@ -791,8 +791,8 @@ private void rewriteAccessedPrefixFile(String path) { String accessedPath = getAccessedPath(path); String timeStamp = Instant.now().toString(); FileUtils fileUtils = new FileUtils(accessedPath); - - fileUtils.createFile(timeStamp); + + fileUtils.synchroCreateFile(timeStamp, cfg.getFileCheckWaitTime(), cfg.getFileCheckMaxCycles()); } /** @@ -809,7 +809,7 @@ private void rewriteStatusPrefixFile(String path, CacheFileStatus status) { String statusPath = getStatusPath(path); FileUtils fileUtils = new FileUtils(statusPath); - fileUtils.createFile(status.toString()); + fileUtils.synchroCreateFile(status.toString(), cfg.getFileCheckWaitTime(), cfg.getFileCheckMaxCycles()); } /** From 727a2049200fb4ac7f6b704f6f1e943f8d4fae0b Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Wed, 13 Dec 2023 16:12:03 +0100 Subject: [PATCH 28/79] storage-mgr: created s3 atomic synchro file uploader --- .../s3/S3AtomicSynchroFileUploader.java | 211 ++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicSynchroFileUploader.java diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicSynchroFileUploader.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicSynchroFileUploader.java new file mode 100644 index 000000000..d89fb87fd --- /dev/null +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicSynchroFileUploader.java @@ -0,0 +1,211 @@ +/** + * S3AtomicSynchroFileUploader.java + * + * (C) 2023 Dr. Bassler & Co. Managementberatung GmbH + */ +package de.dlr.proseo.storagemgr.s3; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; + +import de.dlr.proseo.storagemgr.Exceptions.FileLockedAfterMaxCyclesException; +import de.dlr.proseo.storagemgr.model.AtomicCommand; +import de.dlr.proseo.storagemgr.utils.PathConverter; +import de.dlr.proseo.storagemgr.utils.StorageFileLocker; + +/** + * S3 Atomic Synchro Uploader + * + * @author Denys Chaykovskiy + * + */ +public class S3AtomicSynchroFileUploader implements AtomicCommand { + + /** Info */ + private static final String INFO = "S3 ATOMIC Synchro File Uploader"; + + /** Completed Info */ + private static final String COMPLETED = "file UPLOADED"; + + /** Failed Info */ + private static final String FAILED = "file upload FAILED"; + + /** Logger for this class */ + private static Logger logger = LoggerFactory.getLogger(S3AtomicSynchroFileUploader.class); + + /** Chunk size for uploads to S3 storage (128 MB) */ + private static final Long MULTIPART_UPLOAD_PARTSIZE_BYTES = (long) (128 * 1024 * 1024); + + /** source file */ + private String sourceFile; + + /** target file or dir */ + private String targetFileOrDir; + + /** S3 Client */ + private AmazonS3 s3ClientV1; + + /** Bucket */ + private String bucket; + + /** S3 Configuration */ + private S3Configuration cfg; + + + /** + * Constructor + * + * @param s3ClientV1 s3 client + * @param bucket bucket + * @param sourceFile sourceFile + * @param targetFileOrDir target file or directory + * @param cfg s3 Configuration + + */ + public S3AtomicSynchroFileUploader(AmazonS3 s3ClientV1, String bucket, String sourceFile, String targetFileOrDir, S3Configuration cfg) { + + this.s3ClientV1 = s3ClientV1; + this.bucket = bucket; + this.sourceFile = sourceFile; + this.targetFileOrDir = targetFileOrDir; + this.cfg = cfg; + } + + /** + * Executes synchro upload of the file to s3 + * + * @return uploaded file name + */ + @Override + public String execute() throws IOException { + + if (logger.isTraceEnabled()) + logger.trace(">>> execute() - synchroUploadFile({},{})", sourceFile, targetFileOrDir); + + String targetFile = targetFileOrDir; + + if (new PathConverter(targetFileOrDir).isDirectory()) { + targetFile = Paths.get(targetFileOrDir, getFileName(sourceFile)).toString(); + targetFile = new PathConverter(targetFile).posixToS3Path().convertToSlash().getPath(); + } + + File f = new File(sourceFile); + + if (f == null || !f.isFile()) { + throw new IOException("Cannot upload to s3, source file does not exist: " + sourceFile); + } + + StorageFileLocker fileLocker = new StorageFileLocker(targetFile, cfg.getFileCheckWaitTime(), + cfg.getMaxRequestAttempts()); + + try { + + fileLocker.lockOrWaitUntilUnlockedAndLock(); + return uploadWithTranferManager(sourceFile, targetFile); + + } catch (FileLockedAfterMaxCyclesException e) { + + logger.debug("... the file is locked after max check cycles: ", targetFile, " ", e.getMessage()); + throw new IOException(e); + + } catch (InterruptedException e) { + + logger.debug("... the file is locked after max check cycles: ", targetFile, " ", e.getMessage()); + throw new IOException(e); + + } finally { + + fileLocker.unlock(); + logger.debug("... unlocked the file: ", targetFile); + } + + } + + /** + * Gets information about atomic command (mostly for logs) + * + * @return information about atomic command + */ + @Override + public String getInfo() { + return INFO + " "; + } + + /** + * Gets information about completed atomic command (mostly for logs) + * + * @return information about completed atomic command + */ + @Override + public String getCompletedInfo() { + return INFO + ": " + COMPLETED + " "; + } + + /** + * Gets information about failed atomic command (mostly for logs) + * + * @return information about failed atomic command + */ + @Override + public String getFailedInfo() { + return INFO + ": " + FAILED + " "; + } + + /** + * Gets file name + * + * @param path path + * @return file name + */ + private String getFileName(String path) { + return new File(path).getName(); + } + + + /** + * @param sourceFile source file + * @param targetFile target file + * @return uploaded file (target file) + * @throws IOException if file uploading was not successful + */ + private String uploadWithTranferManager(String sourceFile, String targetFile) throws IOException { + + TransferManager transferManager; + + try { + transferManager = TransferManagerBuilder.standard() + .withMultipartCopyPartSize(MULTIPART_UPLOAD_PARTSIZE_BYTES) + .withS3Client(s3ClientV1) + .build(); + + } catch (Exception e) { + if (logger.isTraceEnabled()) + logger.trace(getFailedInfo() + e.getMessage()); + throw new IOException(e); + } + + try { + transferManager.upload(bucket, targetFile, new File(sourceFile)).waitForCompletion(); + + if (logger.isTraceEnabled()) + logger.trace("... " + getCompletedInfo() + " - " + targetFile); + + return targetFile; + + } catch (Exception e) { + if (logger.isTraceEnabled()) + logger.trace(getFailedInfo() + e.getMessage()); + throw new IOException(e); + } finally { + transferManager.shutdownNow(false); + } + } +} \ No newline at end of file From e03ecfd9643ed2d36eb0da632ec5bf63c02ce5cd Mon Sep 17 00:00:00 2001 From: Denys Chaykovskiy Date: Thu, 14 Dec 2023 15:07:15 +0100 Subject: [PATCH 29/79] storage-mgr: added/removed some comments in product controller --- .../rest/ProductfileControllerImpl.java | 81 +++++++------------ 1 file changed, 27 insertions(+), 54 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index 9ecbf268c..a5a12f10a 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -71,15 +71,17 @@ public ResponseEntity getRestFileInfoByPathInfo(String pathInfo) { // Storage Manager version 2: download Storage -> Cache // pathInfo is absolute path s3://bucket/.. or /storagePath/.. - - String absoluteStoragePath = pathInfo; - + + String absoluteStoragePath = pathInfo; + StorageFile cacheFile; try { + String relativePath = storageProvider.getRelativePath(absoluteStoragePath); cacheFile = storageProvider.getCacheFile(relativePath); } catch (IOException e) { + String msg = logger.log(StorageMgrMessage.PRODUCT_FILE_CANNOT_BE_DOWNLOADED, e.getMessage()); return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.BAD_REQUEST); } @@ -90,14 +92,13 @@ public ResponseEntity getRestFileInfoByPathInfo(String pathInfo) { try { RestFileInfo restFileInfo = synchronizedDownloadFromStorageToCache(absoluteStoragePath, fileLocker); - return new ResponseEntity<>(restFileInfo, HttpStatus.OK); } catch (FileLockedAfterMaxCyclesException e) { String time = String.valueOf(cfg.getFileCheckMaxCycles() * cfg.getFileCheckWaitTime() / 1000); - String msg = logger.log(StorageMgrMessage.READ_TIME_OUT, absoluteStoragePath, time, e.getLocalizedMessage()); - + String msg = logger.log(StorageMgrMessage.READ_TIME_OUT, absoluteStoragePath, time, + e.getLocalizedMessage()); return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.SERVICE_UNAVAILABLE); } catch (IOException e) { @@ -109,9 +110,8 @@ public ResponseEntity getRestFileInfoByPathInfo(String pathInfo) { String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); - } - finally { + } finally { fileLocker.unlock(); logger.debug("... unlocked the file: ", pathInfo); @@ -132,9 +132,7 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro if (logger.isTraceEnabled()) logger.trace(">>> updateProductfiles({}, {})", pathInfo, productId); - // Storage Manager version 2: pathInfo absolute external path, upload absolute file -> - // storage - + // uploads absolute external file -> cache -> storage // 1. download to cache absolute-file -> cache // 2. add to cache cache.put() // 3. upload to storage cache -> storage @@ -142,48 +140,31 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro if (pathInfo == null) { return new ResponseEntity(new RestFileInfo(), HttpStatus.BAD_REQUEST); } - - String absoluteExternalPath = pathInfo; - + + String absoluteExternalPath = pathInfo; + String relativePath = getProductFolderWithFilename(absoluteExternalPath, productId); StorageFile cacheFile = storageProvider.getCacheFile(relativePath); - + StorageFileLocker fileLocker = new StorageFileLocker(cacheFile.getFullPath(), cfg.getFileCheckWaitTime(), cfg.getFileCheckMaxCycles()); try { - - RestFileInfo restFileInfo = synchronizedDownloadFromAbsolutePathToCache(absoluteExternalPath, productId, fileSize, fileLocker); - - fileLocker.unlock(); + RestFileInfo restFileInfo = synchronizedDownloadFromAbsolutePathToCache(absoluteExternalPath, productId, + fileSize, fileLocker); + fileLocker.unlock(); restFileInfo = synchronizedUploadFromCacheToStorage(relativePath, fileLocker); - - /* - Storage storage = storageProvider.getStorage(); - String absolutePath = pathInfo; - String fileName = new File(absolutePath).getName(); - String productFolderWithFilename = Paths.get(String.valueOf(productId), fileName).toString(); - - StorageFile sourceExternalFile = storageProvider.getAbsoluteFile(absolutePath); - StorageFile targetStorageFile = storageProvider.getStorageFile(productFolderWithFilename); - - storage.uploadFile(sourceExternalFile, targetStorageFile); - - RestFileInfo restFileInfo = convertToRestFileInfo(targetStorageFile, - storage.getFileSize(targetStorageFile)); - */ logger.log(StorageMgrMessage.PRODUCT_FILE_UPLOADED, absoluteExternalPath, productId); - return new ResponseEntity<>(restFileInfo, HttpStatus.CREATED); } catch (Exception e) { String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); - } - finally { + + } finally { fileLocker.unlock(); logger.debug("... unlocked the file: ", absoluteExternalPath); @@ -225,16 +206,9 @@ private RestFileInfo synchronizedDownloadFromStorageToCache(String storageFilePa fileLocker.lockOrWaitUntilUnlockedAndLock(); - // After lock() the active thread starts to download the file and put it to the - // cache - // (see below) - // After lock() the passive thread did nothing, but the file has been downloaded - // and the cache has been updated - need to check if file contains in the cache - // again - if (!cache.containsKey(targetFile.getFullPath())) { - // I am active thread - downloads the file and puts it to the cache + // active thread - downloads the file and puts it to the cache storageProvider.getStorage().downloadFile(sourceFile, targetFile); logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED, targetFile.getFullPath()); @@ -242,9 +216,7 @@ private RestFileInfo synchronizedDownloadFromStorageToCache(String storageFilePa } else { - // I am passive thread - did nothing, just waited until the file has been - // downloaded - // and use it from cache + // passive thread - does nothing, waited for downloaded file and use it from cache logger.debug("... waiting-thread when the file downloaded and use it from cache: ", targetFile.getFullPath()); } @@ -259,8 +231,9 @@ private RestFileInfo synchronizedDownloadFromStorageToCache(String storageFilePa return restFileInfo; } - private RestFileInfo synchronizedDownloadFromAbsolutePathToCache(String absoluteExternalPath, Long productId, Long fileSize, - StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { + private RestFileInfo synchronizedDownloadFromAbsolutePathToCache(String absoluteExternalPath, Long productId, + Long fileSize, StorageFileLocker fileLocker) + throws FileLockedAfterMaxCyclesException, IOException, Exception { String productFolderWithFilename = getProductFolderWithFilename(absoluteExternalPath, productId); StorageFile targetCacheFile = storageProvider.getCacheFile(productFolderWithFilename); @@ -312,7 +285,7 @@ private RestFileInfo synchronizedUploadFromCacheToStorage(String relativeCachePa throws FileLockedAfterMaxCyclesException, IOException, Exception { Storage storage = storageProvider.getStorage(); - + StorageFile sourceCacheFile = storageProvider.getCacheFile(relativeCachePath); StorageFile targetStorageFile = storageProvider.getStorageFile(relativeCachePath); @@ -356,11 +329,11 @@ private RestFileInfo synchronizedUploadFromCacheToStorage(String relativeCachePa return restFileInfo; } - + private String getProductFolderWithFilename(String absoluteExternalPath, Long productId) { - + String fileName = new File(absoluteExternalPath).getName(); - return Paths.get(String.valueOf(productId), fileName).toString(); + return Paths.get(String.valueOf(productId), fileName).toString(); } } From 023901346df8a20be2e1c825cd9073f476812c50 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Wed, 3 Jan 2024 15:46:43 +0100 Subject: [PATCH 30/79] storage-mgr: changed info code for product file uploaded from cache --- .../java/de/dlr/proseo/logging/messages/StorageMgrMessage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logging/src/main/java/de/dlr/proseo/logging/messages/StorageMgrMessage.java b/logging/src/main/java/de/dlr/proseo/logging/messages/StorageMgrMessage.java index 4fcae3703..4dabccd6a 100644 --- a/logging/src/main/java/de/dlr/proseo/logging/messages/StorageMgrMessage.java +++ b/logging/src/main/java/de/dlr/proseo/logging/messages/StorageMgrMessage.java @@ -69,7 +69,7 @@ public enum StorageMgrMessage implements ProseoMessage { /** Downloaded = retrieved */ PRODUCT_FILE_DOWNLOADED_FROM_EXTERNAL_TO_CACHE (5581, Level.INFO, true, "Product file downloaded from external storage to cache: {0}", ""), /** Uploaded = registered */ - PRODUCT_FILE_UPLOADED_FROM_CACHE_TO_STORAGE (5552, Level.INFO, true, "Product file {0} uploaded from cache to storage for product {1}", ""), + PRODUCT_FILE_UPLOADED_FROM_CACHE_TO_STORAGE (5582, Level.INFO, true, "Product file {0} uploaded from cache to storage for product {1}", ""), ; From ae1b077220364af8a8d273f6342feab8cc859513 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Wed, 3 Jan 2024 16:10:40 +0100 Subject: [PATCH 31/79] storage-mgr: added "docker" directory to run minio test container --- storage-mgr/src/test/resources/docker/.gitignore | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 storage-mgr/src/test/resources/docker/.gitignore diff --git a/storage-mgr/src/test/resources/docker/.gitignore b/storage-mgr/src/test/resources/docker/.gitignore new file mode 100644 index 000000000..bd5136701 --- /dev/null +++ b/storage-mgr/src/test/resources/docker/.gitignore @@ -0,0 +1,5 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore +# This directory must exist to run the minio container for storage manager tests \ No newline at end of file From eafd9e116b32217049e99ab673a14bbc207a416d Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Fri, 5 Jan 2024 14:33:11 +0100 Subject: [PATCH 32/79] storage-mgr: refactoring of the sync methods in product file controller --- .../rest/ProductfileControllerImpl.java | 200 +++++++++--------- 1 file changed, 100 insertions(+), 100 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index a5a12f10a..ef5394232 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -69,7 +69,8 @@ public ResponseEntity getRestFileInfoByPathInfo(String pathInfo) { return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.BAD_REQUEST); } - // Storage Manager version 2: download Storage -> Cache + // 1. copy Storage -> Cache + // 2. add to cache cache.put(cache file) // pathInfo is absolute path s3://bucket/.. or /storagePath/.. String absoluteStoragePath = pathInfo; @@ -91,7 +92,7 @@ public ResponseEntity getRestFileInfoByPathInfo(String pathInfo) { try { - RestFileInfo restFileInfo = synchronizedDownloadFromStorageToCache(absoluteStoragePath, fileLocker); + RestFileInfo restFileInfo = storageToCacheFileCopy(absoluteStoragePath, fileLocker); return new ResponseEntity<>(restFileInfo, HttpStatus.OK); } catch (FileLockedAfterMaxCyclesException e) { @@ -132,18 +133,19 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro if (logger.isTraceEnabled()) logger.trace(">>> updateProductfiles({}, {})", pathInfo, productId); - // uploads absolute external file -> cache -> storage - // 1. download to cache absolute-file -> cache - // 2. add to cache cache.put() - // 3. upload to storage cache -> storage + // copies absolute external file -> cache file -> storage file + // 1. copy external -> cache + // 2. add to cache cache.put(cache file) + // 3. copy cache -> storage + // pathInfo is absolute external path if (pathInfo == null) { return new ResponseEntity(new RestFileInfo(), HttpStatus.BAD_REQUEST); } - String absoluteExternalPath = pathInfo; + String externalPath = pathInfo; - String relativePath = getProductFolderWithFilename(absoluteExternalPath, productId); + String relativePath = getProductFolderWithFilename(externalPath, productId); StorageFile cacheFile = storageProvider.getCacheFile(relativePath); StorageFileLocker fileLocker = new StorageFileLocker(cacheFile.getFullPath(), cfg.getFileCheckWaitTime(), @@ -151,189 +153,187 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro try { - RestFileInfo restFileInfo = synchronizedDownloadFromAbsolutePathToCache(absoluteExternalPath, productId, - fileSize, fileLocker); + RestFileInfo restFileInfo = externalToCacheFileCopy(externalPath, productId, fileSize, fileLocker); fileLocker.unlock(); - restFileInfo = synchronizedUploadFromCacheToStorage(relativePath, fileLocker); + restFileInfo = cacheToStorageFileCopy(relativePath, fileLocker); - logger.log(StorageMgrMessage.PRODUCT_FILE_UPLOADED, absoluteExternalPath, productId); + logger.log(StorageMgrMessage.PRODUCT_FILE_UPLOADED, externalPath, productId); return new ResponseEntity<>(restFileInfo, HttpStatus.CREATED); } catch (Exception e) { String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); - + } finally { fileLocker.unlock(); - logger.debug("... unlocked the file: ", absoluteExternalPath); + logger.debug("... unlocked the file: ", externalPath); } } - /** - * Converts storage file to rest file info (generated model) - * - * @param storageFile storage file - * @param fileSize file size - * - * @return rest file info - */ - private static RestFileInfo convertToRestFileInfo(StorageFile storageFile, long fileSize) { - - RestFileInfo restFileInfo = new RestFileInfo(); - - restFileInfo.setStorageType(storageFile.getStorageType().toString()); - restFileInfo.setFilePath(storageFile.getFullPath()); - restFileInfo.setFileName(storageFile.getFileName()); - restFileInfo.setFileSize(fileSize); - - return restFileInfo; - } - - private RestFileInfo synchronizedDownloadFromStorageToCache(String storageFilePath, StorageFileLocker fileLocker) + private RestFileInfo storageToCacheFileCopy(String storageFilePath, StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { // relative path depends on path, not on actual storage String relativePath = storageProvider.getRelativePath(storageFilePath); - StorageFile sourceFile = storageProvider.getStorageFile(relativePath); - StorageFile targetFile = storageProvider.getCacheFile(sourceFile.getRelativePath()); + StorageFile storageFile = storageProvider.getStorageFile(relativePath); + StorageFile cacheFile = storageProvider.getCacheFile(storageFile.getRelativePath()); FileCache cache = FileCache.getInstance(); - if (!cache.containsKey(targetFile.getFullPath())) { + if (!cache.containsKey(cacheFile.getFullPath())) { fileLocker.lockOrWaitUntilUnlockedAndLock(); - if (!cache.containsKey(targetFile.getFullPath())) { + // check again, the file could be copied to cache from another thread after lock + if (!cache.containsKey(cacheFile.getFullPath())) { - // active thread - downloads the file and puts it to the cache - storageProvider.getStorage().downloadFile(sourceFile, targetFile); - logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED, targetFile.getFullPath()); + // active thread - copies the file to the cache storage and puts it to the cache + storageProvider.getStorage().downloadFile(storageFile, cacheFile); + + logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED, cacheFile.getFullPath()); - cache.put(targetFile.getFullPath()); + cache.put(cacheFile.getFullPath()); } else { - // passive thread - does nothing, waited for downloaded file and use it from cache + // passive thread - did nothing, waited for copied file and use it from cache logger.debug("... waiting-thread when the file downloaded and use it from cache: ", - targetFile.getFullPath()); + cacheFile.getFullPath()); } } else { - logger.debug("... no download and no lock - the file is in cache: ", targetFile.getFullPath()); + + logger.debug("... no download and no lock - the file is in cache: ", cacheFile.getFullPath()); } - RestFileInfo restFileInfo = convertToRestFileInfo(targetFile, - storageProvider.getCacheFileSize(sourceFile.getRelativePath())); + RestFileInfo restFileInfo = convertToRestFileInfo(cacheFile, + storageProvider.getCacheFileSize(storageFile.getRelativePath())); return restFileInfo; } - private RestFileInfo synchronizedDownloadFromAbsolutePathToCache(String absoluteExternalPath, Long productId, - Long fileSize, StorageFileLocker fileLocker) - throws FileLockedAfterMaxCyclesException, IOException, Exception { + private RestFileInfo externalToCacheFileCopy(String externalPath, Long productId, Long fileSize, + StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { - String productFolderWithFilename = getProductFolderWithFilename(absoluteExternalPath, productId); - StorageFile targetCacheFile = storageProvider.getCacheFile(productFolderWithFilename); + String productFolderWithFilename = getProductFolderWithFilename(externalPath, productId); + StorageFile cacheFile = storageProvider.getCacheFile(productFolderWithFilename); FileCache cache = FileCache.getInstance(); - if (!cache.containsKey(targetCacheFile.getFullPath())) { + if (!cache.containsKey(cacheFile.getFullPath())) { fileLocker.lockOrWaitUntilUnlockedAndLock(); - // After lock() the active thread starts to download the file and put it to the - // cache - // (see below) - // After lock() the passive thread did nothing, but the file has been downloaded - // and the cache has been updated - need to check if file contains in the cache - // again - - if (!cache.containsKey(targetCacheFile.getFullPath())) { + // check again, the file could be copied to cache from another thread after lock + if (!cache.containsKey(cacheFile.getFullPath())) { - // I am active thread - downloads the file and puts it to the cache - - // TODO: WIP - storageProvider.copyAbsoluteFilesToCache(absoluteExternalPath, productId); + // active thread - copies the file to the cache storage and puts it to the cache + storageProvider.copyAbsoluteFilesToCache(externalPath, productId); logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED_FROM_EXTERNAL_TO_CACHE, - targetCacheFile.getFullPath()); + cacheFile.getFullPath()); - cache.put(targetCacheFile.getFullPath()); + cache.put(cacheFile.getFullPath()); + } else { - // passive thread - did nothing, just waited until the file has been downloaded - // and use it from cache + // passive thread - did nothing, waited for copied file and use it from cache logger.debug( "... waiting-thread when the file downloaded to cache from external storage and use it from cache: ", - targetCacheFile.getFullPath()); + cacheFile.getFullPath()); } } else { - logger.debug("... no download and no lock - the file is in cache: ", targetCacheFile.getFullPath()); + + logger.debug("... no download and no lock - the file is in cache: ", cacheFile.getFullPath()); } - RestFileInfo restFileInfo = convertToRestFileInfo(targetCacheFile, - storageProvider.getCacheFileSize(targetCacheFile.getRelativePath())); + RestFileInfo restFileInfo = convertToRestFileInfo(cacheFile, + storageProvider.getCacheFileSize(cacheFile.getRelativePath())); return restFileInfo; } - private RestFileInfo synchronizedUploadFromCacheToStorage(String relativeCachePath, StorageFileLocker fileLocker) + // TODO: WIP Special use case for cache recovery - file in cache, but not in storage + // TODO: WIP Special use case for cache state - not uploaded to storage + + private RestFileInfo cacheToStorageFileCopy(String relativeCachePath, StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { Storage storage = storageProvider.getStorage(); - StorageFile sourceCacheFile = storageProvider.getCacheFile(relativeCachePath); - StorageFile targetStorageFile = storageProvider.getStorageFile(relativeCachePath); + StorageFile cacheFile = storageProvider.getCacheFile(relativeCachePath); + StorageFile storageFile = storageProvider.getStorageFile(relativeCachePath); FileCache cache = FileCache.getInstance(); - if (!cache.containsKey(sourceCacheFile.getFullPath())) { + if (!cache.containsKey(cacheFile.getFullPath())) { fileLocker.lockOrWaitUntilUnlockedAndLock(); - // After lock() the active thread starts to download the file and put it to the - // cache - // (see below) - // After lock() the passive thread did nothing, but the file has been downloaded - // and the cache has been updated - need to check if file contains in the cache - // again + // check again, the file could be copied to storage from another thread after lock + if (!cache.containsKey(cacheFile.getFullPath())) { - if (!cache.containsKey(sourceCacheFile.getFullPath())) { - - // I am active thread - uploads the file and puts it to the cache - - storage.uploadFile(sourceCacheFile, targetStorageFile); + // active thread - copies the file to the storage and checks it as OK (WIP) + storage.uploadFile(cacheFile, storageFile); logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED_FROM_EXTERNAL_TO_CACHE, - targetStorageFile.getFullPath()); + storageFile.getFullPath()); - cache.put(targetStorageFile.getFullPath()); + cache.put(storageFile.getFullPath()); + } else { - // passive thread - did nothing, just waited until the file has been uploaded - // and use it from cache + // passive thread - did nothing, waited for copied file and use it from cache logger.debug("... waiting-thread when the file downloaded and use it from cache: ", - targetStorageFile.getFullPath()); + storageFile.getFullPath()); } } else { - logger.debug("... no download and no lock - the file is in cache: ", targetStorageFile.getFullPath()); + + logger.debug("... no download and no lock - the file is in cache: ", storageFile.getFullPath()); } - RestFileInfo restFileInfo = convertToRestFileInfo(targetStorageFile, - storageProvider.getCacheFileSize(sourceCacheFile.getRelativePath())); + RestFileInfo restFileInfo = convertToRestFileInfo(storageFile, + storageProvider.getCacheFileSize(cacheFile.getRelativePath())); return restFileInfo; } - private String getProductFolderWithFilename(String absoluteExternalPath, Long productId) { + /** + * Converts storage file to rest file info (generated model) + * + * @param storageFile storage file + * @param fileSize file size + * + * @return rest file info + */ + private static RestFileInfo convertToRestFileInfo(StorageFile storageFile, long fileSize) { - String fileName = new File(absoluteExternalPath).getName(); - return Paths.get(String.valueOf(productId), fileName).toString(); + RestFileInfo restFileInfo = new RestFileInfo(); + + restFileInfo.setStorageType(storageFile.getStorageType().toString()); + restFileInfo.setFilePath(storageFile.getFullPath()); + restFileInfo.setFileName(storageFile.getFileName()); + restFileInfo.setFileSize(fileSize); + + return restFileInfo; } + /** + * Gets a product folder with the file name from the given external path using + * product id + * + * @param externalPath absolute external path + * @param productId product id + * @return product folder with the file name + */ + private String getProductFolderWithFilename(String externalPath, Long productId) { + + String fileName = new File(externalPath).getName(); + return Paths.get(String.valueOf(productId), fileName).toString(); + } } From 04c8d35f7ca4decca396d8de0594f8794bc47ffe Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Mon, 29 Jan 2024 08:56:25 +0100 Subject: [PATCH 33/79] storage-mgr: added "not exists" status to the file cache --- .../logging/messages/StorageMgrMessage.java | 4 +- .../storagemgr/cache/CacheFileStatus.java | 6 +- .../proseo/storagemgr/cache/FileCache.java | 129 +++++++++++++----- .../rest/ProductControllerImpl.java | 2 +- .../rest/ProductfileControllerImpl.java | 9 +- .../storagemgr/cache/FileCacheTest.java | 36 +++-- 6 files changed, 132 insertions(+), 54 deletions(-) diff --git a/logging/src/main/java/de/dlr/proseo/logging/messages/StorageMgrMessage.java b/logging/src/main/java/de/dlr/proseo/logging/messages/StorageMgrMessage.java index 4dabccd6a..71bb073e4 100644 --- a/logging/src/main/java/de/dlr/proseo/logging/messages/StorageMgrMessage.java +++ b/logging/src/main/java/de/dlr/proseo/logging/messages/StorageMgrMessage.java @@ -48,9 +48,9 @@ public enum StorageMgrMessage implements ProseoMessage { /** Listed = got */ PRODUCT_FILES_LISTED (5552, Level.INFO, true, "Product files listed: {0}", ""), /** Downloaded = retrieved */ - PRODUCT_FILE_DOWNLOADED (5553, Level.INFO, true, "Product file downloaded: {0}", ""), + PRODUCT_FILE_DOWNLOADED_FROM_STORAGE (5553, Level.INFO, true, "Product file downloaded from storage: {0}", ""), /** Uploaded = registered */ - PRODUCT_FILE_UPLOADED (5554, Level.INFO, true, "Product file {0} uploaded for product {1}", ""), + PRODUCT_FILE_UPLOADED_TO_STORAGE (5554, Level.INFO, true, "Product file {0} uploaded to storage for product {1}", ""), PRODUCT_FILE_CANNOT_BE_DOWNLOADED (5555, Level.ERROR, false, "Product file cannot be downloaded: {0}", ""), PRODUCT_FILE_PARTIALLY_DOWNLOADED (5556, Level.INFO, true, "Product file {0} partially downloaded from byte {1} to byte {2}, {3} bytes transferred", ""), diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java index 611bfdb9b..e90b4c54f 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java @@ -14,7 +14,7 @@ public enum CacheFileStatus { /** Status READY */ READY, /** Status UPLOADING */ - UPLOADING, - /** Status DOWNLOADING */ - DOWNLOADING, + COPYING_FROM_CACHE, + /** Status Cache File does not exist in the cache */ + NOT_EXISTS, } \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java index 235bce929..26049790f 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java @@ -22,7 +22,25 @@ import de.dlr.proseo.storagemgr.utils.FileUtils; /** - * File cache for managing files in cache storage. + * File cache for managing files in the cache storage. Features: 1) saves the + * time stamp of the last access of the cache file; 2) contains the status of + * the cache file, which can be changed; 3) uses the possibility to save in the + * cache also "temporary" files (temporary file is a file, which has not yet + * been copied to the cache. After copying it will be renamed to the cache file. + * In case of unsuccessful copying a temporary file will be deleted) + * + * Information about some public methods: + * + * The method put(fullPath): - puts the file to the cache; - sets the file + * status "READY"; - sets/updates the last accessed record; - cleans the cache + * if there is not enough free space - deletes the file physically if it is a + * temporary file + * + * The method containsKey(fullPath): - checks if the path is in the cache. Only + * cache paths with the status "READY" are visible - removes the cache path from + * the cache if the physical file does not exist anymore - if the file is in the + * cache, updates the time stamp of the last access calling put() + * * * @author Denys Chaykovskiy * @@ -41,7 +59,7 @@ public class FileCache { /** Prefix for status files */ private static final String STATUS_PREFIX = "status-"; - + /** Cache Map for storing file paths */ private MapCache mapCache; @@ -68,7 +86,7 @@ public static FileCache getInstance() { * Puts the new element to map. If element exists, it will be overwritten. * Removes the file if it is temporary * - * @param pathKey File path as a key + * @param pathKey the full cache file path as a key */ public void put(String pathKey) { @@ -80,6 +98,8 @@ public void put(String pathKey) { logger.log(StorageMgrMessage.CACHE_NO_FILE_FOR_PUTTING_TO_CACHE, pathKey); return; } + + // pathKey is the full path, beginning with the cache path if (!pathKey.startsWith(cachePath)) { if (logger.isTraceEnabled()) logger.trace("... not adding {} to cache, because it is considered a backend file", pathKey); @@ -98,7 +118,7 @@ public void put(String pathKey) { } rewriteStatusPrefixFile(pathKey, CacheFileStatus.READY); - + rewriteAccessedPrefixFile(pathKey); FileInfo fileInfo = new FileInfo(getFileAccessed(pathKey), getFileSize(pathKey)); @@ -106,8 +126,10 @@ public void put(String pathKey) { } /** - * Checks if key available in map and updates the record in file cache if - * available + * Checks if the cache contains the path If not - returns false. If the physical + * file does not exits anymore - deletes the path from the cache. Returns true + * if the path is available in the cache and also updates the file record of the + * last access * * @param pathKey File path as key * @return true if pathkey is in file cache @@ -124,7 +146,7 @@ public boolean containsKey(String pathKey) { File file = new File(pathKey); - if (!file.exists() || !file.isFile()) { + if (!file.isFile()) { remove(pathKey); return false; @@ -135,6 +157,42 @@ public boolean containsKey(String pathKey) { return true; } + /** + * Puts the new cache file to the cache. The file does not exist yet in the + * cache - it is uploading. The put() method checks if the cache file exists, + * that's why there is a need in the special method The method is important for + * the recovery - if an uploading to the cache was not successful, the file must + * be deleted. + * + * @param pathKey the full cache file path as a key + */ + public void putNonExistingUploading(String pathKey) { + + if (logger.isTraceEnabled()) + logger.trace(">>> put({})", pathKey); + + // pathKey is the full path, beginning with the cache path + if (!pathKey.startsWith(cachePath)) { + + if (logger.isTraceEnabled()) + logger.trace("... not adding {} to cache, because it does not start with the cache path", pathKey); + return; + } + + if (isTemporaryPrefixFile(pathKey)) { + + deleteFile(pathKey); + logger.log(StorageMgrMessage.CACHE_TEMPORARY_FILE_DELETED, pathKey); + } + + rewriteStatusPrefixFile(pathKey, CacheFileStatus.NOT_EXISTS); + + rewriteAccessedPrefixFile(pathKey); + FileInfo fileInfo = new FileInfo(getFileAccessed(pathKey), 0); + + mapCache.put(pathKey, fileInfo); + } + /** * Gets temporary prefix of the file * @@ -143,7 +201,7 @@ public boolean containsKey(String pathKey) { public static String getTemporaryPrefix() { return TEMPORARY_PREFIX; } - + /** * Returns a status of the cache file * @@ -166,7 +224,7 @@ public CacheFileStatus getCacheFileStatus(String path) { return CacheFileStatus.valueOf(status); } - + /** * Sets the status of a cache file * @@ -178,16 +236,19 @@ public void setCacheFileStatus(String pathKey, CacheFileStatus status) { if (logger.isTraceEnabled()) logger.trace(">>> setCacheFileStatus({}, {})", pathKey, status); - if (!mapCache.containsKey(pathKey)) { - if (!new File(pathKey).exists()) { - logger.log(StorageMgrMessage.CACHE_NO_FILE_FOR_PUTTING_TO_CACHE, pathKey); + // There is no cache file for the status == not exists + if (status != CacheFileStatus.NOT_EXISTS) { + if (!mapCache.containsKey(pathKey)) { + if (!new File(pathKey).exists()) { + logger.log(StorageMgrMessage.CACHE_NO_FILE_FOR_PUTTING_TO_CACHE, pathKey); + } + return; } - return; } - + rewriteStatusPrefixFile(pathKey, status); } - + /** * Initializes file cache with directory from Application.yml */ @@ -520,13 +581,11 @@ else if (file.isFile() && isCacheFile(file.getPath())) { String directory = new File(path).getParent(); deleteFile(path); - + deleteFile(getAccessedPath(path)); deleteFile(getStatusPath(path)); deleteFile(getTemporaryPath(path)); - - deleteEmptyDirectoriesToTop(directory); } @@ -611,7 +670,7 @@ else if (file.isFile() && isCacheFile(file.getPath())) { return accessedPath; } - + /** * Gets the path of the status file * @@ -628,7 +687,7 @@ else if (file.isFile() && isCacheFile(file.getPath())) { return statusPath; } - + /** * Gets the path of the temporary file * @@ -662,7 +721,7 @@ else if (file.isFile() && isCacheFile(file.getPath())) { return path; } - + /** * Returns true if the file was accessed * @@ -675,7 +734,7 @@ private boolean wasAccessed(String path) { return f.isFile(); } - + /** * Returns true if the cache file has a status * @@ -690,8 +749,8 @@ private boolean hasStatus(String path) { } /** - * Returns true if the file is the cache file (starts with the cache path, not accessed, not temporary, - * not hidden and not status file) + * Returns true if the file is the cache file (starts with the cache path, not + * accessed, not temporary, not hidden and not status file) * * @param path the full path to the file * @return true if the file is the cache file @@ -699,9 +758,9 @@ private boolean hasStatus(String path) { private boolean isCacheFile(String path) { if (!path.startsWith(cachePath)) { - return false; + return false; } - + if (isAccessedPrefixFile(path)) { return false; } @@ -709,14 +768,14 @@ private boolean isCacheFile(String path) { if (isTemporaryPrefixFile(path)) { return false; } - + if (isStatusPrefixFile(path)) { return false; } if (isHiddenFile(path)) { return false; - } + } return true; } @@ -742,7 +801,7 @@ private boolean isTemporaryPrefixFile(String path) { return hasFilePrefix(path, TEMPORARY_PREFIX); } - + /** * Returns true if the file has the status prefix * @@ -791,25 +850,25 @@ private void rewriteAccessedPrefixFile(String path) { String accessedPath = getAccessedPath(path); String timeStamp = Instant.now().toString(); FileUtils fileUtils = new FileUtils(accessedPath); - + fileUtils.synchroCreateFile(timeStamp, cfg.getFileCheckWaitTime(), cfg.getFileCheckMaxCycles()); } - + /** * Rewrites status prefix file with the current status * - * @param path The full path to the cache file - * @param status The + * @param path The full path to the cache file + * @param status The */ private void rewriteStatusPrefixFile(String path, CacheFileStatus status) { if (logger.isTraceEnabled()) - logger.trace(">>> rewriteStatusPrefixFile({}, {})", path, status); + logger.trace(">>> rewriteStatusPrefixFile({}, {})", path, status); String statusPath = getStatusPath(path); FileUtils fileUtils = new FileUtils(statusPath); - fileUtils.synchroCreateFile(status.toString(), cfg.getFileCheckWaitTime(), cfg.getFileCheckMaxCycles()); + fileUtils.synchroCreateFile(status.toString(), cfg.getFileCheckWaitTime(), cfg.getFileCheckMaxCycles()); } /** diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java index 411c6843d..58507f6fa 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java @@ -256,7 +256,7 @@ public ResponseEntity getObject(String pathInfo, String token, Long fromByte, InputStreamResource fsr = new InputStreamResource(stream); if (null == fromByte || null == toByte) { - logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED, pathInfo); + logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED_FROM_STORAGE, pathInfo); } else { logger.log(StorageMgrMessage.PRODUCT_FILE_PARTIALLY_DOWNLOADED, pathInfo, Long.toString(fromByte), Long.toString(toByte), Long.toString(toByte - fromByte)); diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index ef5394232..a8ebdc001 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -157,7 +157,7 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro fileLocker.unlock(); restFileInfo = cacheToStorageFileCopy(relativePath, fileLocker); - logger.log(StorageMgrMessage.PRODUCT_FILE_UPLOADED, externalPath, productId); + logger.log(StorageMgrMessage.PRODUCT_FILE_UPLOADED_TO_STORAGE, externalPath, productId); return new ResponseEntity<>(restFileInfo, HttpStatus.CREATED); } catch (Exception e) { @@ -191,11 +191,14 @@ private RestFileInfo storageToCacheFileCopy(String storageFilePath, StorageFileL if (!cache.containsKey(cacheFile.getFullPath())) { // active thread - copies the file to the cache storage and puts it to the cache + + // cache.putNonExistingUploading(cacheFile.getFullPath()); // status = copying_to_cache + storageProvider.getStorage().downloadFile(storageFile, cacheFile); - logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED, cacheFile.getFullPath()); + logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED_FROM_STORAGE, cacheFile.getFullPath()); - cache.put(cacheFile.getFullPath()); + cache.put(cacheFile.getFullPath()); // status = READY } else { diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java index be6e8f1e1..71d618d84 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java @@ -203,9 +203,13 @@ public void testStatus() { String path1 = Paths.get(testCachePath + "/" + "testStatus1.txt").toString(); String path2 = Paths.get(testCachePath + "/" + "testStatus2.txt").toString(); + String path3 = Paths.get(testCachePath + "/" + "testStatus3.txt").toString(); + CacheFileStatus status1; CacheFileStatus status2; + CacheFileStatus status3; + // create 2 files for cache @@ -228,36 +232,48 @@ public void testStatus() { assertTrue("Expected cache file2 status after put() is Ready. Exists: " + status2.toString(), status2 == CacheFileStatus.READY); + + // check not exists status - file is not in cache, but has a status + + fileCache.setCacheFileStatus(path3, CacheFileStatus.NOT_EXISTS); + + status3 = fileCache.getCacheFileStatus(path3); + + assertTrue("Expected cache file3 does not exist in the cache. Exists:" + fileCache.containsKey(path3), + !fileCache.containsKey(path3)); + + assertTrue("Expected cache file3 status after setStatus(Not_exists) is Not_exists. Exists:" + status3.toString(), + status3 == CacheFileStatus.NOT_EXISTS); + + // changing status1 to copying from cache - // changing status1 to Uploading - - fileCache.setCacheFileStatus(path1, CacheFileStatus.UPLOADING); + fileCache.setCacheFileStatus(path1, CacheFileStatus.COPYING_FROM_CACHE); status1 = fileCache.getCacheFileStatus(path1); status2 = fileCache.getCacheFileStatus(path2); assertTrue("Expected cache file1 status after setStatus(Uploading) is Uploading. Exists:" + status1.toString(), - status1 == CacheFileStatus.UPLOADING); + status1 == CacheFileStatus.COPYING_FROM_CACHE); assertTrue("Expected cache file2 status after is Ready. Exists:" + status2.toString(), status2 == CacheFileStatus.READY); - // changing status2 to Downloading + // changing status2 to copying to cache - fileCache.setCacheFileStatus(path2, CacheFileStatus.DOWNLOADING); + fileCache.setCacheFileStatus(path2, CacheFileStatus.NOT_EXISTS); status1 = fileCache.getCacheFileStatus(path1); status2 = fileCache.getCacheFileStatus(path2); assertTrue("Expected cache file1 status after is Uploading. Exists: " + status1.toString(), - status1 == CacheFileStatus.UPLOADING); + status1 == CacheFileStatus.COPYING_FROM_CACHE); assertTrue("Expected cache file2 status after setStatus(Downloading) is Downlading. Exists: " + status2.toString(), - status2 == CacheFileStatus.DOWNLOADING); + status2 == CacheFileStatus.NOT_EXISTS); // changing status1 to Ready - fileCache.setCacheFileStatus(path1, CacheFileStatus.READY); + fileCache.put(path1); // status is set as ready status1 = fileCache.getCacheFileStatus(path1); status2 = fileCache.getCacheFileStatus(path2); @@ -266,7 +282,7 @@ public void testStatus() { status1 == CacheFileStatus.READY); assertTrue("Expected cache file2 status after is Downloading. Exists: " + status2.toString(), - status2 == CacheFileStatus.DOWNLOADING); + status2 == CacheFileStatus.NOT_EXISTS); // check cache after status changes From 22a5da03015b127a760818b0193029b63057787e Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Mon, 29 Jan 2024 09:17:22 +0100 Subject: [PATCH 34/79] storage-mgr: removed putNotExisting(), status "not exists" are used --- .../proseo/storagemgr/cache/FileCache.java | 39 +------------------ 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java index 26049790f..9af92bc10 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java @@ -157,42 +157,6 @@ public boolean containsKey(String pathKey) { return true; } - /** - * Puts the new cache file to the cache. The file does not exist yet in the - * cache - it is uploading. The put() method checks if the cache file exists, - * that's why there is a need in the special method The method is important for - * the recovery - if an uploading to the cache was not successful, the file must - * be deleted. - * - * @param pathKey the full cache file path as a key - */ - public void putNonExistingUploading(String pathKey) { - - if (logger.isTraceEnabled()) - logger.trace(">>> put({})", pathKey); - - // pathKey is the full path, beginning with the cache path - if (!pathKey.startsWith(cachePath)) { - - if (logger.isTraceEnabled()) - logger.trace("... not adding {} to cache, because it does not start with the cache path", pathKey); - return; - } - - if (isTemporaryPrefixFile(pathKey)) { - - deleteFile(pathKey); - logger.log(StorageMgrMessage.CACHE_TEMPORARY_FILE_DELETED, pathKey); - } - - rewriteStatusPrefixFile(pathKey, CacheFileStatus.NOT_EXISTS); - - rewriteAccessedPrefixFile(pathKey); - FileInfo fileInfo = new FileInfo(getFileAccessed(pathKey), 0); - - mapCache.put(pathKey, fileInfo); - } - /** * Gets temporary prefix of the file * @@ -226,7 +190,8 @@ public CacheFileStatus getCacheFileStatus(String path) { } /** - * Sets the status of a cache file + * Sets the status of a cache file. The pathKey must be an existing cache file, + * the exception is for "not exists" cache file status. * * @param pathKey File path as key * @return a status of the cache file From 04a7f4cded0edd5db1bdda5cb605d7bde207e6d7 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Mon, 29 Jan 2024 09:24:57 +0100 Subject: [PATCH 35/79] storage-mgr: integrated "not exists" in X-to-cache-copy rest methods --- .../storagemgr/rest/ProductfileControllerImpl.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index a8ebdc001..cbca2ebcc 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -18,6 +18,7 @@ import de.dlr.proseo.storagemgr.StorageManagerConfiguration; import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.Exceptions.FileLockedAfterMaxCyclesException; +import de.dlr.proseo.storagemgr.cache.CacheFileStatus; import de.dlr.proseo.storagemgr.cache.FileCache; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; @@ -192,13 +193,13 @@ private RestFileInfo storageToCacheFileCopy(String storageFilePath, StorageFileL // active thread - copies the file to the cache storage and puts it to the cache - // cache.putNonExistingUploading(cacheFile.getFullPath()); // status = copying_to_cache + cache.setCacheFileStatus(cacheFile.getFullPath(), CacheFileStatus.NOT_EXISTS); storageProvider.getStorage().downloadFile(storageFile, cacheFile); logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED_FROM_STORAGE, cacheFile.getFullPath()); - cache.put(cacheFile.getFullPath()); // status = READY + cache.put(cacheFile.getFullPath()); // cache file status = READY } else { @@ -234,12 +235,15 @@ private RestFileInfo externalToCacheFileCopy(String externalPath, Long productId if (!cache.containsKey(cacheFile.getFullPath())) { // active thread - copies the file to the cache storage and puts it to the cache + + cache.setCacheFileStatus(cacheFile.getFullPath(), CacheFileStatus.NOT_EXISTS); + storageProvider.copyAbsoluteFilesToCache(externalPath, productId); logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED_FROM_EXTERNAL_TO_CACHE, cacheFile.getFullPath()); - cache.put(cacheFile.getFullPath()); + cache.put(cacheFile.getFullPath()); // cache file status = READY } else { From 6859976e139a33a1c9158f991a967d13677c2462 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Mon, 29 Jan 2024 11:00:03 +0100 Subject: [PATCH 36/79] storage-mgr: added recovery for the cache files in status "not exists" --- .../proseo/storagemgr/cache/FileCache.java | 58 +++++++++++++++++-- .../rest/ProductfileControllerImpl.java | 2 + 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java index 9af92bc10..8df567b4d 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java @@ -146,7 +146,7 @@ public boolean containsKey(String pathKey) { File file = new File(pathKey); - if (!file.isFile()) { + if (!file.isFile() || hasNotExistsStatus(pathKey)) { remove(pathKey); return false; @@ -156,7 +156,7 @@ public boolean containsKey(String pathKey) { return true; } - + /** * Gets temporary prefix of the file * @@ -481,7 +481,7 @@ private double getRealUsage() { deleteFile(file.getPath()); } - // delete if accessed file alone without file + // delete if accessed file alone without cache file if (isAccessedPrefixFile(file.getPath()) && !Files.exists(Paths.get(getPathFromAccessed(file.getPath())))) { file.delete(); @@ -493,9 +493,30 @@ private double getRealUsage() { } continue; } + + // delete if status file alone without cache file + if (isStatusPrefixFile(file.getPath()) && !Files.exists(Paths.get(getPathFromStatus(file.getPath())))) { + + file.delete(); + + if (new FileUtils(path).isEmptyDirectory()) { + + deleteEmptyDirectoriesToTop(path); + return; + } + continue; + } // if cache file, adds to cache without update accessed prefix file else if (file.isFile() && isCacheFile(file.getPath())) { + + // delete cache file with status file if status = not exists + if (hasNotExistsStatus(file.getPath())) { + + deleteFile(file.getPath()); + deleteFile(getStatusPath(file.getPath())); + continue; + } putWithoutUpdateAccessedPrefixFile(file.getPath()); } @@ -546,7 +567,7 @@ else if (file.isFile() && isCacheFile(file.getPath())) { String directory = new File(path).getParent(); deleteFile(path); - + deleteFile(getAccessedPath(path)); deleteFile(getStatusPath(path)); deleteFile(getTemporaryPath(path)); @@ -686,6 +707,24 @@ else if (file.isFile() && isCacheFile(file.getPath())) { return path; } + + /** + * Gets a cache path from a status path + * + * @param status Path status Path to the file + * @return the full path to cache file + */ + /* package */ String getPathFromStatus(String statusPath) { + + if (logger.isTraceEnabled()) + logger.trace(">>> getPathFromStatus({})", statusPath); + + File file = new File(statusPath); + String path = file.getParent() + "/" + file.getName().replace(STATUS_PREFIX, ""); + + return path; + } + /** * Returns true if the file was accessed @@ -801,6 +840,17 @@ private boolean hasFilePrefix(String path, String prefix) { return fileName.startsWith(prefix) ? true : false; } + + /** + * Returns true if the cache file has "not exists" status + * + * @param cacheFile the full path to the cache file + * @return true if the file has "not exists" status + */ + private boolean hasNotExistsStatus(String cacheFile) { + + return getCacheFileStatus(cacheFile) == CacheFileStatus.NOT_EXISTS; + } /** * Rewrites accessed prefix file with the current time stamp diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index cbca2ebcc..1b081efdc 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -173,6 +173,7 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro } } + // x-to-cache-copy method, status "not exists is used" private RestFileInfo storageToCacheFileCopy(String storageFilePath, StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { @@ -219,6 +220,7 @@ private RestFileInfo storageToCacheFileCopy(String storageFilePath, StorageFileL return restFileInfo; } + // x-to-cache-copy method, status "not exists is used" private RestFileInfo externalToCacheFileCopy(String externalPath, Long productId, Long fileSize, StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { From 5a7e7225ef49e4108429e636bc397b40b91dccc8 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Tue, 30 Jan 2024 09:52:15 +0100 Subject: [PATCH 37/79] storage-mgr: added comments and renamed copy methods in prod file ctrl --- .../rest/ProductfileControllerImpl.java | 59 +++++++++++++++---- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index 1b081efdc..21d82268a 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -93,7 +93,7 @@ public ResponseEntity getRestFileInfoByPathInfo(String pathInfo) { try { - RestFileInfo restFileInfo = storageToCacheFileCopy(absoluteStoragePath, fileLocker); + RestFileInfo restFileInfo = copyFileStorageToCache(absoluteStoragePath, fileLocker); return new ResponseEntity<>(restFileInfo, HttpStatus.OK); } catch (FileLockedAfterMaxCyclesException e) { @@ -154,9 +154,9 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro try { - RestFileInfo restFileInfo = externalToCacheFileCopy(externalPath, productId, fileSize, fileLocker); + RestFileInfo restFileInfo = copyFileExternalToCache(externalPath, productId, fileSize, fileLocker); fileLocker.unlock(); - restFileInfo = cacheToStorageFileCopy(relativePath, fileLocker); + restFileInfo = copyFileCacheToStorage(relativePath, fileLocker); logger.log(StorageMgrMessage.PRODUCT_FILE_UPLOADED_TO_STORAGE, externalPath, productId); return new ResponseEntity<>(restFileInfo, HttpStatus.CREATED); @@ -172,10 +172,23 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro logger.debug("... unlocked the file: ", externalPath); } } - - // x-to-cache-copy method, status "not exists is used" - private RestFileInfo storageToCacheFileCopy(String storageFilePath, StorageFileLocker fileLocker) + + /** + * Copies the file from the storage to the cache using synchronization. + * During the copying to the cache, the status of the file will be "not exists", + * after the completion the status will be set to "ready" + * + * @param storageFilePath the file path in the storage + * @param fileLocker file locker is used for synchronization + * @return RestFileInfo + * @throws FileLockedAfterMaxCyclesException + * @throws IOException + * @throws Exception + */ + private RestFileInfo copyFileStorageToCache(String storageFilePath, StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { + + // x-to-cache-copy method, status "not exists" is used // relative path depends on path, not on actual storage String relativePath = storageProvider.getRelativePath(storageFilePath); @@ -220,10 +233,25 @@ private RestFileInfo storageToCacheFileCopy(String storageFilePath, StorageFileL return restFileInfo; } - // x-to-cache-copy method, status "not exists is used" - private RestFileInfo externalToCacheFileCopy(String externalPath, Long productId, Long fileSize, + /** + * Copies the file from the external source to the cache using synchronization. + * During the copying to the cache, the status of the file will be "not exists", + * after the completion the status will be set to "ready" + * + * @param externalPath external path of the file, which will be copied to the cache + * @param productId product id is used as a directory to store copied file in cache + * @param fileSize file size + * @param fileLocker file locker is used for synchronization + * @return Rest File Info + * @throws FileLockedAfterMaxCyclesException + * @throws IOException + * @throws Exception + */ + private RestFileInfo copyFileExternalToCache(String externalPath, Long productId, Long fileSize, StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { + // x-to-cache-copy method, status "not exists" is used + String productFolderWithFilename = getProductFolderWithFilename(externalPath, productId); StorageFile cacheFile = storageProvider.getCacheFile(productFolderWithFilename); @@ -269,7 +297,19 @@ private RestFileInfo externalToCacheFileCopy(String externalPath, Long productId // TODO: WIP Special use case for cache recovery - file in cache, but not in storage // TODO: WIP Special use case for cache state - not uploaded to storage - private RestFileInfo cacheToStorageFileCopy(String relativeCachePath, StorageFileLocker fileLocker) + /** + * Copies the file from the cache to the storage using synchronization. + * During the copying to the cache, the status of the file will be "not exists", + * after the completion the status will be set to "ready" + * + * @param relativeCachePath relative cache path + * @param fileLocker is used for synchronization + * @return RestFileInfo + * @throws FileLockedAfterMaxCyclesException + * @throws IOException + * @throws Exception + */ + private RestFileInfo copyFileCacheToStorage(String relativeCachePath, StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { Storage storage = storageProvider.getStorage(); @@ -317,7 +357,6 @@ private RestFileInfo cacheToStorageFileCopy(String relativeCachePath, StorageFil * * @param storageFile storage file * @param fileSize file size - * * @return rest file info */ private static RestFileInfo convertToRestFileInfo(StorageFile storageFile, long fileSize) { From a1e3af548e413490ed25c613aa5052d423491279 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Wed, 31 Jan 2024 11:03:53 +0100 Subject: [PATCH 38/79] storage-mgr: added a description for the status files in the file cache --- .../proseo/storagemgr/cache/FileCache.java | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java index 8df567b4d..c28def0eb 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java @@ -22,12 +22,20 @@ import de.dlr.proseo.storagemgr.utils.FileUtils; /** - * File cache for managing files in the cache storage. Features: 1) saves the - * time stamp of the last access of the cache file; 2) contains the status of - * the cache file, which can be changed; 3) uses the possibility to save in the - * cache also "temporary" files (temporary file is a file, which has not yet - * been copied to the cache. After copying it will be renamed to the cache file. - * In case of unsuccessful copying a temporary file will be deleted) + * File cache for managing files in the cache storage. Features: + * 1) saves the time stamp of the last access of the cache file; + * 2) contains the status of the cache file, which can be changed. + * The status file with the status "not exists" can exists alone or + * together with the cache file. The existence of status file "not exists" + * shows, that copying failed and the status file "not exists" together with + * the cache file (if exists) will be deleted after the new initialization + * of the cache. In case of new copy attempt they will be overwritten. + * In case of calling put() with the existing cache file, status will be + * changed to "ready" + * 3) uses the possibility to save in the cache also "temporary" files + * (temporary file is a file, which has not yet been copied to the cache. + * After copying it will be renamed to the cache file. + * In case of unsuccessful copying a temporary file will be deleted) * * Information about some public methods: * From 5474739f4bab0b8a1025cfc81b1afa2ac3fd9080 Mon Sep 17 00:00:00 2001 From: Thomas Bassler Date: Thu, 1 Feb 2024 11:24:10 +0100 Subject: [PATCH 39/79] Intermediate commit for merge: storage-mgr directory replaced --- storage-mgr/pom.xml | 4 +- .../FileLockedAfterMaxCyclesException.java | 2 +- .../Exceptions/package-info.java | 2 +- .../StorageManagerConfiguration.java | 11 - .../{version2 => }/StorageProvider.java | 191 +++-- .../storagemgr/cache/CacheFileStatus.java | 20 + .../proseo/storagemgr/cache/FileCache.java | 261 +++++- .../proseo/storagemgr/cache/package-info.java | 11 + .../de/dlr/proseo/storagemgr/fs/s3/S3Ops.java | 787 ------------------ .../{version2 => }/model/AtomicCommand.java | 2 +- .../model/DefaultRetryStrategy.java | 2 +- .../{version2 => }/model/Storage.java | 2 +- .../{version2 => }/model/StorageFile.java | 2 +- .../{version2 => }/model/StorageType.java | 2 +- .../{version2 => }/model/package-info.java | 2 +- .../posix/PosixAtomicFileCopier.java | 131 +++ .../posix/PosixAtomicFileDownloader.java | 131 +++ .../posix/PosixAtomicFileUploader.java | 131 +++ .../storagemgr/posix/PosixConfiguration.java | 78 ++ .../{version2 => }/posix/PosixDAL.java | 179 ++-- .../{version2 => }/posix/PosixStorage.java | 93 +-- .../posix/PosixStorageFile.java | 8 +- .../{version2 => }/posix/package-info.java | 2 +- .../rest/JobOrderControllerImpl.java | 198 +---- .../rest/ProductControllerImpl.java | 514 +++--------- .../rest/ProductfileControllerImpl.java | 509 ++++++----- .../s3/S3AtomicBucketCreator.java | 4 +- .../s3/S3AtomicBucketDeleter.java | 4 +- .../s3/S3AtomicBucketListGetter.java | 4 +- .../s3/S3AtomicFileContentGetter.java | 4 +- .../s3/S3AtomicFileDeleter.java | 4 +- .../s3/S3AtomicFileDownloader.java | 8 +- .../s3/S3AtomicFileExistsGetter.java | 4 +- .../s3/S3AtomicFileListDeleter.java | 4 +- .../s3/S3AtomicFileListGetter.java | 4 +- .../s3/S3AtomicFileSizeGetter.java | 4 +- .../s3/S3AtomicFileUploader.java | 6 +- .../s3/S3AtomicInputStreamGetter.java | 6 +- .../s3/S3AtomicSynchroFileUploader.java | 211 +++++ .../{version2 => }/s3/S3Configuration.java | 2 +- .../storagemgr/{version2 => }/s3/S3DAL.java | 8 +- .../{version2 => }/s3/S3Storage.java | 16 +- .../{version2 => }/s3/S3StorageFile.java | 8 +- .../{version2 => }/s3/package-info.java | 2 +- .../{version2 => utils}/FileUtils.java | 44 +- .../{version2 => utils}/PathConverter.java | 4 +- .../proseo/storagemgr/utils/ProseoFile.java | 380 --------- .../storagemgr/utils/ProseoFileAlluxio.java | 167 ---- .../storagemgr/utils/ProseoFilePosix.java | 391 --------- .../proseo/storagemgr/utils/ProseoFileS3.java | 445 ---------- .../StorageFileLocker.java | 13 +- .../storagemgr/utils/StorageLogger.java | 73 -- .../storagemgr/utils/StorageManagerUtils.java | 169 ---- .../proseo/storagemgr/utils/StorageType.java | 18 - .../{version2 => utils}/package-info.java | 2 +- .../src/main/resources/application.yml | 1 - .../storagemgr/BaseStorageTestUtils.java | 73 +- .../proseo/storagemgr/StorageTestUtils.java | 80 -- .../de/dlr/proseo/storagemgr/TestUtils.java | 27 +- .../storagemgr/UniqueStorageTestPaths.java | 2 +- .../storagemgr/cache/FileCacheTest.java | 312 ++++--- .../cache/FileSystemGeneratorTest.java | 2 +- .../storagemgr/cache/FileUtilsTest.java | 2 +- .../ProductfileControllerImplTest_cache.java | 56 +- .../storagemgr/cache/ProseoFilePosixTest.java | 121 --- .../JobOrderControllerImplTest_download.java | 45 +- .../JobOrderControllerImplTest_upload.java | 49 +- .../ProductControllerImplTest_delete.java | 66 +- .../ProductControllerImplTest_download.java | 72 +- .../rest/ProductControllerImplTest_get.java | 64 +- .../ProductControllerImplTest_getAll.java | 58 +- .../ProductControllerImplTest_upload.java | 65 +- ...roductfileControllerImplTest_download.java | 65 +- .../ProductfileControllerImplTest_upload.java | 50 +- .../version2/PathConverterTest.java | 2 + .../version2/StorageProviderTest.java | 20 +- .../version2/StorageUtilityTest.java | 18 +- .../version2/posix/PosixDALTest.java | 13 +- .../version2/posix/PosixStorageTest.java | 14 +- .../storagemgr/version2/s3/S3StorageTest.java | 16 +- .../src/test/resources/application.yml | 3 +- .../src/test/resources/docker/.gitignore | 5 + 82 files changed, 2128 insertions(+), 4457 deletions(-) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/Exceptions/FileLockedAfterMaxCyclesException.java (92%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/Exceptions/package-info.java (78%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/StorageProvider.java (88%) create mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java create mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/package-info.java delete mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/fs/s3/S3Ops.java rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/model/AtomicCommand.java (95%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/model/DefaultRetryStrategy.java (98%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/model/Storage.java (99%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/model/StorageFile.java (96%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/model/StorageType.java (85%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/model/package-info.java (84%) create mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileCopier.java create mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileDownloader.java create mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileUploader.java create mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixConfiguration.java rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/posix/PosixDAL.java (70%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/posix/PosixStorage.java (88%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/posix/PosixStorageFile.java (94%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/posix/package-info.java (80%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicBucketCreator.java (96%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicBucketDeleter.java (95%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicBucketListGetter.java (96%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicFileContentGetter.java (96%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicFileDeleter.java (97%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicFileDownloader.java (97%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicFileExistsGetter.java (96%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicFileListDeleter.java (97%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicFileListGetter.java (97%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicFileSizeGetter.java (96%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicFileUploader.java (95%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3AtomicInputStreamGetter.java (95%) create mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicSynchroFileUploader.java rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3Configuration.java (98%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3DAL.java (98%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3Storage.java (97%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/S3StorageFile.java (93%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => }/s3/package-info.java (80%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => utils}/FileUtils.java (83%) rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => utils}/PathConverter.java (99%) delete mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/ProseoFile.java delete mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/ProseoFileAlluxio.java delete mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/ProseoFilePosix.java delete mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/ProseoFileS3.java rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => utils}/StorageFileLocker.java (84%) delete mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageLogger.java delete mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageManagerUtils.java delete mode 100644 storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageType.java rename storage-mgr/src/main/java/de/dlr/proseo/storagemgr/{version2 => utils}/package-info.java (81%) delete mode 100644 storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StorageTestUtils.java delete mode 100644 storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProseoFilePosixTest.java create mode 100644 storage-mgr/src/test/resources/docker/.gitignore diff --git a/storage-mgr/pom.xml b/storage-mgr/pom.xml index 666b354d3..5deab5b92 100644 --- a/storage-mgr/pom.xml +++ b/storage-mgr/pom.xml @@ -6,7 +6,7 @@ de.dlr.proseo proseo - 0.9.5 + 0.9.5-SNAPSHOT @@ -20,7 +20,7 @@ - 0.9.5 + 0.9.5-SNAPSHOT ${basedir}/../.. src/main/resources/raml/storage-mgr-api.raml de.dlr.proseo.storagemgr.rest diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/Exceptions/FileLockedAfterMaxCyclesException.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/Exceptions/FileLockedAfterMaxCyclesException.java similarity index 92% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/Exceptions/FileLockedAfterMaxCyclesException.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/Exceptions/FileLockedAfterMaxCyclesException.java index 96d4c573a..f84c612da 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/Exceptions/FileLockedAfterMaxCyclesException.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/Exceptions/FileLockedAfterMaxCyclesException.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.Exceptions; +package de.dlr.proseo.storagemgr.Exceptions; import java.io.IOException; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/Exceptions/package-info.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/Exceptions/package-info.java similarity index 78% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/Exceptions/package-info.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/Exceptions/package-info.java index 5eb26a784..9f8c36686 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/Exceptions/package-info.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/Exceptions/package-info.java @@ -8,4 +8,4 @@ * * @author Denys Chaykovskiy */ -package de.dlr.proseo.storagemgr.version2.Exceptions; \ No newline at end of file +package de.dlr.proseo.storagemgr.Exceptions; \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageManagerConfiguration.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageManagerConfiguration.java index 38b7b328c..824680e7a 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageManagerConfiguration.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageManagerConfiguration.java @@ -30,10 +30,6 @@ public class StorageManagerConfiguration { @Value("${proseo.global.storageIdPrefix}") private String storageIdPrefix; - /** Flag to indicate whether V2 implementation shall be used or older V1 implementation (will be deleted after removal of V1) */ - @Value("${proseo.global.storageManagerVersion2}") - private String storageManagerVersion2; - /** Default mount point for files to upload (always POSIX) */ @Value("${proseo.global.sourcePath}") private String sourcePath; @@ -124,13 +120,6 @@ public static StorageManagerConfiguration getConfiguration() { return theConfiguration; } - /** - * @return the storageManagerVersion2 - */ - public String getStorageManagerVersion2() { - return storageManagerVersion2; - } - /** * @return the defaultStorageType from config file or mounted storage type */ diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/StorageProvider.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageProvider.java similarity index 88% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/StorageProvider.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageProvider.java index bd916a344..6aa5c8027 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/StorageProvider.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/StorageProvider.java @@ -3,8 +3,9 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2; +package de.dlr.proseo.storagemgr; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; @@ -19,15 +20,20 @@ import org.springframework.stereotype.Component; import de.dlr.proseo.logging.logger.ProseoLogger; -import de.dlr.proseo.storagemgr.StorageManagerConfiguration; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; -import de.dlr.proseo.storagemgr.version2.posix.PosixStorage; -import de.dlr.proseo.storagemgr.version2.posix.PosixStorageFile; -import de.dlr.proseo.storagemgr.version2.s3.S3Configuration; -import de.dlr.proseo.storagemgr.version2.s3.S3Storage; -import de.dlr.proseo.storagemgr.version2.s3.S3StorageFile; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; + +import de.dlr.proseo.storagemgr.posix.PosixStorage; +import de.dlr.proseo.storagemgr.posix.PosixStorageFile; +import de.dlr.proseo.storagemgr.posix.PosixConfiguration; +import de.dlr.proseo.storagemgr.posix.PosixDAL; + +import de.dlr.proseo.storagemgr.s3.S3Configuration; +import de.dlr.proseo.storagemgr.s3.S3Storage; +import de.dlr.proseo.storagemgr.s3.S3StorageFile; + +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * A central manager for different types of storage in the application. It @@ -52,9 +58,6 @@ public class StorageProvider { /** Base Paths are used to get relative path from absolute path */ List basePaths = new ArrayList<>(); - /** For smooth integration only, will be removed */ - private boolean version2; - /** Logger for this class */ private static ProseoLogger logger = new ProseoLogger(StorageProvider.class); @@ -87,25 +90,6 @@ public StorageProvider() { // init(); } - /** - * Initializes storage(s) from Application.yml - * - * @throws IOException if an error occurs during initialization - */ - @PostConstruct - private void init() throws IOException { - theStorageProvider = this; - storage = createStorage(StorageType.valueOf(cfg.getDefaultStorageType()), cfg.getPosixBackendPath()); - - version2 = cfg.getStorageManagerVersion2().equals("true") ? true : false; - - basePaths.add(storage.getBasePath()); - basePaths.add(cfg.getDefaultSourcePath()); - basePaths.add(cfg.getPosixCachePath()); - - loadDefaultPaths(); - } - /** * Load default source, storage, and cache paths */ @@ -227,41 +211,6 @@ public List getBasePaths() { return basePaths; } - // all ..version.. methods will be removed in release, for smooth integration - // only - - /** - * Loads version 1 of the storage manager (currently used). - */ - public void loadVersion1() { - version2 = false; - } - - /** - * Loads version 2 of the storage manager (new source code). - */ - public void loadVersion2() { - version2 = true; - } - - /** - * Checks if version 2 of the storage manager is currently used. - * - * @return true if version 2 is used, false otherwise - */ - public boolean isVersion2() { - return version2; - } - - /** - * Checks if version 1 of the storage manager is currently used. - * - * @return true if version 1 is used, false otherwise - */ - public boolean isVersion1() { - return !version2; - } - /** * Gets the file size of the cache file specified by the relative path. * @@ -305,31 +254,6 @@ public long getPosixFileSize(String absolutePath) throws IOException { return Files.size(path); } - /** - * Creates a storage instance based on the specified storage type and storage - * path. - * - * @param storageType the storage type (S3 or POSIX) - * @param storagePath the base path of the POSIX storage (used for temporary - * files in S3) - * @return the created storage instance - * @throws IOException if an error occurs during storage creation - */ - private Storage createStorage(StorageType storageType, String storagePath) throws IOException { - if (logger.isTraceEnabled()) - logger.trace(">>> createStorage({}, {})", storageType.toString(), storagePath); - - sourcePath = cfg.getDefaultSourcePath(); - - if (storageType == StorageType.POSIX) { - return new PosixStorage(storagePath, sourcePath); - } else if (storageType == StorageType.S3) { - return new S3Storage(getS3ConfigurationFromFile()); - } - - throw new IllegalArgumentException("Storage Type " + storageType.toString() + " is wrong"); - } - /** * Gets the cache file as a StorageFile from the relative path. The path can be * virtual. @@ -515,6 +439,27 @@ public StorageFile createStorageFile(String relativePath, String content) throws return storage.createStorageFile(relativePath, content); } + + /** + * Gets the POSIX configuration from the file. + * + * @return the POSIX configuration + */ + public PosixConfiguration getPosixConfigurationFromFile() { + PosixConfiguration posixConfiguration = new PosixConfiguration(); + + // FYI: No bucket configuration is used in prosEO for POSIX Storage + posixConfiguration.setBucket(StorageFile.NO_BUCKET); + + posixConfiguration.setBasePath(cfg.getPosixBackendPath()); + posixConfiguration.setSourcePath(cfg.getDefaultSourcePath()); + + posixConfiguration.setMaxRequestAttempts(cfg.getMaxRequestAttempts()); + + posixConfiguration.setFileCheckWaitTime(cfg.getFileCheckWaitTime()); + + return posixConfiguration; + } /** * Gets the S3 configuration from the file. @@ -541,4 +486,66 @@ public S3Configuration getS3ConfigurationFromFile() { return s3Configuration; } + + /** + * @param sourceFileOrDir + * @return + * @throws IOException + */ + public List copyAbsoluteFilesToCache(String sourceFileOrDir, Long productId) throws IOException { + + if (logger.isTraceEnabled()) + logger.trace(">>> copyAbsoluteFilesToCache({})", sourceFileOrDir); + + String fileName = new File(sourceFileOrDir).getName(); + String productFolderWithFilename = Paths.get(String.valueOf(productId), fileName).toString(); + StorageFile targetFile = getCacheFile(productFolderWithFilename); + + PosixDAL posixDAL = new PosixDAL(getPosixConfigurationFromFile()); + + return posixDAL.copy(sourceFileOrDir, targetFile.getFullPath()); + + } + + /** + * Initializes storage(s) from Application.yml + * + * @throws IOException if an error occurs during initialization + */ + @PostConstruct + private void init() throws IOException { + theStorageProvider = this; + storage = createStorage(StorageType.valueOf(cfg.getDefaultStorageType()), cfg.getPosixBackendPath()); + + basePaths.add(storage.getBasePath()); + basePaths.add(cfg.getDefaultSourcePath()); + basePaths.add(cfg.getPosixCachePath()); + + loadDefaultPaths(); + } + + /** + * Creates a storage instance based on the specified storage type and storage + * path. + * + * @param storageType the storage type (S3 or POSIX) + * @param storagePath the base path of the POSIX storage (used for temporary + * files in S3) + * @return the created storage instance + * @throws IOException if an error occurs during storage creation + */ + private Storage createStorage(StorageType storageType, String storagePath) throws IOException { + if (logger.isTraceEnabled()) + logger.trace(">>> createStorage({}, {})", storageType.toString(), storagePath); + + sourcePath = cfg.getDefaultSourcePath(); + + if (storageType == StorageType.POSIX) { + return new PosixStorage(getPosixConfigurationFromFile()); + } else if (storageType == StorageType.S3) { + return new S3Storage(getS3ConfigurationFromFile()); + } + + throw new IllegalArgumentException("Storage Type " + storageType.toString() + " is wrong"); + } } \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java new file mode 100644 index 000000000..e90b4c54f --- /dev/null +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java @@ -0,0 +1,20 @@ +/** + * CacheFileStatus.java + * + * (C) 2023 Dr. Bassler & Co. Managementberatung GmbH + */ +package de.dlr.proseo.storagemgr.cache; + +/** + * Cache File Status shows the current status of the file in the cache + * + * @author Denys Chaykovskiy + */ +public enum CacheFileStatus { + /** Status READY */ + READY, + /** Status UPLOADING */ + COPYING_FROM_CACHE, + /** Status Cache File does not exist in the cache */ + NOT_EXISTS, +} \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java index 4c4b5f03f..c28def0eb 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java @@ -19,10 +19,36 @@ import de.dlr.proseo.logging.logger.ProseoLogger; import de.dlr.proseo.logging.messages.StorageMgrMessage; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; -import de.dlr.proseo.storagemgr.version2.FileUtils; +import de.dlr.proseo.storagemgr.utils.FileUtils; /** - * File cache for managing files in cache storage. + * File cache for managing files in the cache storage. Features: + * 1) saves the time stamp of the last access of the cache file; + * 2) contains the status of the cache file, which can be changed. + * The status file with the status "not exists" can exists alone or + * together with the cache file. The existence of status file "not exists" + * shows, that copying failed and the status file "not exists" together with + * the cache file (if exists) will be deleted after the new initialization + * of the cache. In case of new copy attempt they will be overwritten. + * In case of calling put() with the existing cache file, status will be + * changed to "ready" + * 3) uses the possibility to save in the cache also "temporary" files + * (temporary file is a file, which has not yet been copied to the cache. + * After copying it will be renamed to the cache file. + * In case of unsuccessful copying a temporary file will be deleted) + * + * Information about some public methods: + * + * The method put(fullPath): - puts the file to the cache; - sets the file + * status "READY"; - sets/updates the last accessed record; - cleans the cache + * if there is not enough free space - deletes the file physically if it is a + * temporary file + * + * The method containsKey(fullPath): - checks if the path is in the cache. Only + * cache paths with the status "READY" are visible - removes the cache path from + * the cache if the physical file does not exist anymore - if the file is in the + * cache, updates the time stamp of the last access calling put() + * * * @author Denys Chaykovskiy * @@ -39,7 +65,10 @@ public class FileCache { /** Prefix for temporary files */ private static final String TEMPORARY_PREFIX = "temporary-"; - /** Cache Map for storing file pathes */ + /** Prefix for status files */ + private static final String STATUS_PREFIX = "status-"; + + /** Cache Map for storing file paths */ private MapCache mapCache; @Autowired @@ -65,7 +94,7 @@ public static FileCache getInstance() { * Puts the new element to map. If element exists, it will be overwritten. * Removes the file if it is temporary * - * @param pathKey File path as a key + * @param pathKey the full cache file path as a key */ public void put(String pathKey) { @@ -77,6 +106,8 @@ public void put(String pathKey) { logger.log(StorageMgrMessage.CACHE_NO_FILE_FOR_PUTTING_TO_CACHE, pathKey); return; } + + // pathKey is the full path, beginning with the cache path if (!pathKey.startsWith(cachePath)) { if (logger.isTraceEnabled()) logger.trace("... not adding {} to cache, because it is considered a backend file", pathKey); @@ -94,6 +125,8 @@ public void put(String pathKey) { deleteLRU(); } + rewriteStatusPrefixFile(pathKey, CacheFileStatus.READY); + rewriteAccessedPrefixFile(pathKey); FileInfo fileInfo = new FileInfo(getFileAccessed(pathKey), getFileSize(pathKey)); @@ -101,8 +134,10 @@ public void put(String pathKey) { } /** - * Checks if key available in map and updates the record in file cache if - * available + * Checks if the cache contains the path If not - returns false. If the physical + * file does not exits anymore - deletes the path from the cache. Returns true + * if the path is available in the cache and also updates the file record of the + * last access * * @param pathKey File path as key * @return true if pathkey is in file cache @@ -119,7 +154,7 @@ public boolean containsKey(String pathKey) { File file = new File(pathKey); - if (!file.exists() || !file.isFile()) { + if (!file.isFile() || hasNotExistsStatus(pathKey)) { remove(pathKey); return false; @@ -129,7 +164,7 @@ public boolean containsKey(String pathKey) { return true; } - + /** * Gets temporary prefix of the file * @@ -139,6 +174,54 @@ public static String getTemporaryPrefix() { return TEMPORARY_PREFIX; } + /** + * Returns a status of the cache file + * + * @param path The full path to the cache file + * @return a status of the cache file + */ + public CacheFileStatus getCacheFileStatus(String path) { + + if (logger.isTraceEnabled()) + logger.trace(">>> getCacheFileStatus({})", path); + + String status; + FileUtils fileUtils = new FileUtils(getStatusPath(path)); + + if (!hasStatus(path)) { + rewriteStatusPrefixFile(path, CacheFileStatus.READY); + } + + status = fileUtils.getFileContent(); + + return CacheFileStatus.valueOf(status); + } + + /** + * Sets the status of a cache file. The pathKey must be an existing cache file, + * the exception is for "not exists" cache file status. + * + * @param pathKey File path as key + * @return a status of the cache file + */ + public void setCacheFileStatus(String pathKey, CacheFileStatus status) { + + if (logger.isTraceEnabled()) + logger.trace(">>> setCacheFileStatus({}, {})", pathKey, status); + + // There is no cache file for the status == not exists + if (status != CacheFileStatus.NOT_EXISTS) { + if (!mapCache.containsKey(pathKey)) { + if (!new File(pathKey).exists()) { + logger.log(StorageMgrMessage.CACHE_NO_FILE_FOR_PUTTING_TO_CACHE, pathKey); + } + return; + } + } + + rewriteStatusPrefixFile(pathKey, status); + } + /** * Initializes file cache with directory from Application.yml */ @@ -277,7 +360,7 @@ private double getRealUsage() { /** * Gets the path key from file cache * - * @param pathKey Path to file + * @param pathKey Path to the cache file * @return */ /* package */ FileInfo get(String pathKey) { @@ -286,7 +369,7 @@ private double getRealUsage() { } /** - * Removes cache element, file and accessed file + * Removes cache element, cache file and auxiliary files * * @param pathKey Path to file */ @@ -295,7 +378,7 @@ private double getRealUsage() { if (logger.isTraceEnabled()) logger.trace(">>> remove({}) - last accessed {}", pathKey, mapCache.get(pathKey).getAccessed()); - deleteFileAndAccessedPrefixFile(pathKey); + deleteCacheFileAndAuxPrefixFiles(pathKey); mapCache.remove(pathKey); } @@ -406,7 +489,7 @@ private double getRealUsage() { deleteFile(file.getPath()); } - // delete if accessed file alone without file + // delete if accessed file alone without cache file if (isAccessedPrefixFile(file.getPath()) && !Files.exists(Paths.get(getPathFromAccessed(file.getPath())))) { file.delete(); @@ -418,9 +501,30 @@ private double getRealUsage() { } continue; } + + // delete if status file alone without cache file + if (isStatusPrefixFile(file.getPath()) && !Files.exists(Paths.get(getPathFromStatus(file.getPath())))) { + + file.delete(); + + if (new FileUtils(path).isEmptyDirectory()) { + + deleteEmptyDirectoriesToTop(path); + return; + } + continue; + } // if cache file, adds to cache without update accessed prefix file else if (file.isFile() && isCacheFile(file.getPath())) { + + // delete cache file with status file if status = not exists + if (hasNotExistsStatus(file.getPath())) { + + deleteFile(file.getPath()); + deleteFile(getStatusPath(file.getPath())); + continue; + } putWithoutUpdateAccessedPrefixFile(file.getPath()); } @@ -459,20 +563,22 @@ else if (file.isFile() && isCacheFile(file.getPath())) { } /** - * Deletes if exist file and logically connected accessed file from the disk + * Deletes a cache file and logically connected auxiliary files from the disk * - * @param path full path to the file + * @param path full path to the cache file */ - /* package */ void deleteFileAndAccessedPrefixFile(String path) { + /* package */ void deleteCacheFileAndAuxPrefixFiles(String path) { if (logger.isTraceEnabled()) - logger.trace(">>> deleteFileAndAccessed({})", path); + logger.trace(">>> deleteFileAndAuxPrefixFiles({})", path); - String accessedPath = getAccessedPath(path); String directory = new File(path).getParent(); deleteFile(path); - deleteFile(accessedPath); + + deleteFile(getAccessedPath(path)); + deleteFile(getStatusPath(path)); + deleteFile(getTemporaryPath(path)); deleteEmptyDirectoriesToTop(directory); } @@ -559,6 +665,40 @@ else if (file.isFile() && isCacheFile(file.getPath())) { return accessedPath; } + /** + * Gets the path of the status file + * + * @param path The full path to the cache file + * @return the path to the status file of the cache file + */ + /* package */ String getStatusPath(String path) { + + if (logger.isTraceEnabled()) + logger.trace(">>> getStatusPath({})", path); + + File file = new File(path); + String statusPath = file.getParent() + "/" + STATUS_PREFIX + file.getName(); + + return statusPath; + } + + /** + * Gets the path of the temporary file + * + * @param path The full path to the cache file + * @return the path to the temporary file of the cache file + */ + /* package */ String getTemporaryPath(String path) { + + if (logger.isTraceEnabled()) + logger.trace(">>> getTemporaryPath({})", path); + + File file = new File(path); + String temporaryPath = file.getParent() + "/" + TEMPORARY_PREFIX + file.getName(); + + return temporaryPath; + } + /** * Gets the path to file from accessed path * @@ -575,6 +715,24 @@ else if (file.isFile() && isCacheFile(file.getPath())) { return path; } + + /** + * Gets a cache path from a status path + * + * @param status Path status Path to the file + * @return the full path to cache file + */ + /* package */ String getPathFromStatus(String statusPath) { + + if (logger.isTraceEnabled()) + logger.trace(">>> getPathFromStatus({})", statusPath); + + File file = new File(statusPath); + String path = file.getParent() + "/" + file.getName().replace(STATUS_PREFIX, ""); + + return path; + } + /** * Returns true if the file was accessed @@ -590,14 +748,31 @@ private boolean wasAccessed(String path) { } /** - * Returns true if the file is the cache file (not accessed, not temporary and - * not hidden file) + * Returns true if the cache file has a status + * + * @param path the full path to the cache file + * @return true if the file was accessed + */ + private boolean hasStatus(String path) { + + File f = new File(getStatusPath(path)); + + return f.isFile(); + } + + /** + * Returns true if the file is the cache file (starts with the cache path, not + * accessed, not temporary, not hidden and not status file) * * @param path the full path to the file * @return true if the file is the cache file */ private boolean isCacheFile(String path) { + if (!path.startsWith(cachePath)) { + return false; + } + if (isAccessedPrefixFile(path)) { return false; } @@ -606,9 +781,12 @@ private boolean isCacheFile(String path) { return false; } - if (isHiddenFile(path)) { + if (isStatusPrefixFile(path)) { return false; + } + if (isHiddenFile(path)) { + return false; } return true; @@ -636,6 +814,17 @@ private boolean isTemporaryPrefixFile(String path) { return hasFilePrefix(path, TEMPORARY_PREFIX); } + /** + * Returns true if the file has the status prefix + * + * @param path the full path to the cache file + * @return true if the cache file path has the status prefix + */ + private boolean isStatusPrefixFile(String path) { + + return hasFilePrefix(path, STATUS_PREFIX); + } + /** * Returns true if the file is a hidden file * @@ -659,6 +848,17 @@ private boolean hasFilePrefix(String path, String prefix) { return fileName.startsWith(prefix) ? true : false; } + + /** + * Returns true if the cache file has "not exists" status + * + * @param cacheFile the full path to the cache file + * @return true if the file has "not exists" status + */ + private boolean hasNotExistsStatus(String cacheFile) { + + return getCacheFileStatus(cacheFile) == CacheFileStatus.NOT_EXISTS; + } /** * Rewrites accessed prefix file with the current time stamp @@ -674,7 +874,24 @@ private void rewriteAccessedPrefixFile(String path) { String timeStamp = Instant.now().toString(); FileUtils fileUtils = new FileUtils(accessedPath); - fileUtils.createFile(timeStamp); + fileUtils.synchroCreateFile(timeStamp, cfg.getFileCheckWaitTime(), cfg.getFileCheckMaxCycles()); + } + + /** + * Rewrites status prefix file with the current status + * + * @param path The full path to the cache file + * @param status The + */ + private void rewriteStatusPrefixFile(String path, CacheFileStatus status) { + + if (logger.isTraceEnabled()) + logger.trace(">>> rewriteStatusPrefixFile({}, {})", path, status); + + String statusPath = getStatusPath(path); + FileUtils fileUtils = new FileUtils(statusPath); + + fileUtils.synchroCreateFile(status.toString(), cfg.getFileCheckWaitTime(), cfg.getFileCheckMaxCycles()); } /** diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/package-info.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/package-info.java new file mode 100644 index 000000000..5bbecda96 --- /dev/null +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/package-info.java @@ -0,0 +1,11 @@ +/** + * package-info.java + * + * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH + */ +/** + * Cache classes for the prosEO Storage Manager component + * + * @author Denys Chaykovskiy + */ +package de.dlr.proseo.storagemgr.cache; \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/fs/s3/S3Ops.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/fs/s3/S3Ops.java deleted file mode 100644 index 164d85837..000000000 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/fs/s3/S3Ops.java +++ /dev/null @@ -1,787 +0,0 @@ -package de.dlr.proseo.storagemgr.fs.s3; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.AmazonS3URI; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.amazonaws.services.s3.transfer.Download; -import com.amazonaws.services.s3.transfer.TransferManager; -import com.amazonaws.services.s3.transfer.TransferManagerBuilder; -import de.dlr.proseo.storagemgr.StorageManagerConfiguration; -import de.dlr.proseo.storagemgr.version2.StorageProvider; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.awscore.exception.AwsServiceException; -import software.amazon.awssdk.core.ResponseInputStream; -import software.amazon.awssdk.core.exception.SdkClientException; -import software.amazon.awssdk.core.exception.SdkException; -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.core.sync.ResponseTransformer; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration; -import software.amazon.awssdk.services.s3.model.CreateBucketRequest; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.ListBucketsRequest; -import software.amazon.awssdk.services.s3.model.ListBucketsResponse; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.S3Exception; - -/** - * @author Hubert Asamer - * - */ -public class S3Ops { - - /** Logger for this class */ - private static Logger logger = LoggerFactory.getLogger(S3Ops.class); - - /** Chunk size for uploads to S3 storage (128 MB) */ - private static final Long MULTIPART_UPLOAD_PARTSIZE_BYTES = (long) (128 * 1024 * 1024); - - /** Maximum number of retries for data uploads to S3 storage */ - private static final int MAX_UPLOAD_RETRIES = 3; - - /** - * Creates the empty key - * - * @param s3 a given instantiated S3Client - * @param bucketName Bucket name to which the PUT operation was initiated - * @param key Object key for which the PUT operation was initiated - * @param manifestMsg String to send to the service - * @return returns the created empty key - */ - public static String createEmptyKey(S3Client s3, String bucketName, String key, String manifestMsg) { - - if (logger.isTraceEnabled()) logger.trace(">>> createEmptyKey({}, {}, {}, {})", s3, bucketName, key, manifestMsg); - - try { - s3.putObject(PutObjectRequest.builder().bucket(bucketName).key(key).build(), - RequestBody.fromString(manifestMsg)); - return key; - - } catch (AwsServiceException | SdkClientException e) { - logger.error(e.getMessage()); - return null; - } - } - - /** - * List keys in bucket based on prefix - * - * @param s3 the V1 S3 client to use - * @param bucketName the bucket name - * @param prefix the bucket prefix - * @return the keys contained in the bucket - * @throws SdkClientException if any error occurred in the communication with the S3 backend storage - * @throws IOException - */ - public static List listObjectsInBucket(AmazonS3 s3, String bucketName, String prefix) - throws SdkClientException, IOException { - - if (logger.isTraceEnabled()) logger.trace(">>> listObjectsInBucket({}, {}, {})", s3, bucketName, prefix); - - Boolean isTopLevel = false; - String delimiter = "/"; - if (prefix == "" || prefix == "/") { - isTopLevel = true; - } - if (!prefix.endsWith(delimiter)) { - prefix += delimiter; - } - ListObjectsRequest listObjectsRequest = null; - if (isTopLevel) { - listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); - } else { - listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix); - } - List folderLike = new ArrayList(); - ObjectListing objects = null; - objects = s3.listObjects(listObjectsRequest); - - for (S3ObjectSummary f : objects.getObjectSummaries()) { - folderLike.add("s3://" + f.getBucketName() + "/" + f.getKey()); - } - while (objects.isTruncated()) { - objects = s3.listNextBatchOfObjects(objects); - for (S3ObjectSummary f : objects.getObjectSummaries()) { - folderLike.add("s3://" + f.getBucketName() + "/" + f.getKey()); - } - } - return folderLike; - } - - /** - * List all buckets; passes all exceptions on to the caller - * - * @param s3 the S3 client to use - * @return a list of buckets - */ - public static ArrayList listBuckets(S3Client s3) { - - if (logger.isTraceEnabled()) logger.trace(">>> listBuckets({})", s3); - - ArrayList buckets = new ArrayList(); - ListBucketsRequest listBucketsRequest = ListBucketsRequest.builder().build(); - ListBucketsResponse listBucketsResponse = null; - try { - listBucketsResponse = s3.listBuckets(listBucketsRequest); - } catch (Exception e) { - logger.error(e.getMessage()); - throw e; - } - listBucketsResponse.buckets().stream().forEach(x -> buckets.add(x.name())); - return buckets; - } - - /** - * Creates a new S3 bucket - * - * @param s3 the S3 client to use - * @param bucketName the name of the new bucket - * @param region the region, in which the bucket shall be stored - * @return the new bucket name or null, if the operation failed - */ - public static String createBucket(S3Client s3, String bucketName, String region) { - - if (logger.isTraceEnabled()) logger.trace(">>> createBucket({}, {}, {})", s3, bucketName, region); - - try { - CreateBucketRequest createBucketRequest = CreateBucketRequest.builder().bucket(bucketName) - .createBucketConfiguration(CreateBucketConfiguration.builder().locationConstraint(region).build()) - .build(); - s3.createBucket(createBucketRequest); - return createBucketRequest.bucket(); - } catch (Exception e) { - logger.error(e.getMessage()); - return null; - } - } - - /** - * Create a base V2 S3 client - * - * @param s3AccessKey the access key for the client - * @param secretAccessKey the secret access key for the client - * @param s3Endpoint the S3 endpoint to connect to - * @param region the region, on which the client shall operate - * @return a configured S3 client or null, if an error occurred - */ - public static S3Client v2S3Client(String s3AccessKey, String secretAccessKey, String s3Endpoint, String region) { - - if (logger.isTraceEnabled()) logger.trace(">>> v2S3Client({}, {}, {}, {}))", "***", "***", s3Endpoint, region); - - try { - AwsBasicCredentials creds = AwsBasicCredentials.create(s3AccessKey, secretAccessKey); - - S3Client s3; - - if (StorageProvider.getInstance().getS3ConfigurationFromFile().isDefaultEndPoint()) { - s3 = S3Client.builder().region(Region.of(region)) - .credentialsProvider(StaticCredentialsProvider.create(creds)).build(); - } - else { - s3 = S3Client.builder().region(Region.of(region)).endpointOverride(URI.create(s3Endpoint)) - .credentialsProvider(StaticCredentialsProvider.create(creds)).build(); - } - - return s3; - - } catch (software.amazon.awssdk.core.exception.SdkClientException e) { - logger.error(e.getMessage()); - return null; - - } catch (java.lang.NullPointerException e1) { - logger.error(e1.getMessage()); - return null; - } - } - - /** - * Create a base V1 S3 client - * - * @param s3AccessKey the access key for the client - * @param secretAccessKey the secret access key for the client - * @param s3Endpoint the S3 endpoint to connect to - * @param region the region, on which the client shall operate - * @return a configured S3 client or null, if an error occurred - */ - public static AmazonS3 v1S3Client(String s3AccessKey, String secretAccessKey, String s3Endpoint, String region) { - - if (logger.isTraceEnabled()) logger.trace(">>> v1S3Client({}, {}, {}, {}))", "***", "***", s3Endpoint, region); - - try { - BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3AccessKey, secretAccessKey); - ClientConfiguration clientConfiguration = new ClientConfiguration(); - clientConfiguration.setSignerOverride("AWSS3V4SignerType"); - - AmazonS3 amazonS3; - - if (StorageProvider.getInstance().getS3ConfigurationFromFile().isDefaultEndPoint()) { - - amazonS3 = AmazonS3ClientBuilder.standard().withCredentials(new AWSStaticCredentialsProvider(awsCreds)) - .withRegion(region).build(); - } else { - - amazonS3 = AmazonS3ClientBuilder.standard() - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(s3Endpoint, Region.of(region).id())) - .withPathStyleAccessEnabled(true).withClientConfiguration(clientConfiguration) - .withCredentials(new AWSStaticCredentialsProvider(awsCreds)).build(); - } - - return amazonS3; - - } catch (AmazonServiceException e) { - logger.error(e.getMessage()); - return null; - } catch (AmazonClientException e) { - logger.error(e.getMessage()); - return null; - } catch (java.lang.NullPointerException e) { - logger.error(e.getMessage()); - return null; - } - - } - - /** - * Fetch file from S3 to local file - * - * @param s3 a given instantiated S3Client - * @param s3Object URI of S3-Object (e.g. s3://bucket/path/to/some/file) - * @param containerPath local target filePath - * @return true, if the operation succeeded, false otherwise - */ - public static Boolean v2FetchFile(S3Client s3, String s3Object, String containerPath) { - - if (logger.isTraceEnabled()) logger.trace(">>> v2FetchFile({}, {}, {})", - (null == s3 ? "MISSING" : s3.serviceName()), s3Object, containerPath); - - try { - Path targetPath = Paths.get(containerPath); - File subdirs = targetPath.getParent().toFile(); - subdirs.mkdirs(); - - AmazonS3URI s3uri = new AmazonS3URI(s3Object); - - ResponseInputStream is = s3.getObject(GetObjectRequest.builder().bucket(s3uri.getBucket()).key(s3uri.getKey()).build()); - if (null == is) { - logger.error("Failed accessing S3 object {} (received 'null' response)", s3Object); - return false; - } else { - try (is) { - Files.copy(is, targetPath, StandardCopyOption.REPLACE_EXISTING); - // Unfortunately returning from Files.copy() does not mean the file is fully written to disk! - Long contentLength = is.response().contentLength(); - int i = 0; - long maxCycles = StorageManagerConfiguration.getConfiguration().getFileCheckMaxCycles(); - long waitTime = StorageManagerConfiguration.getConfiguration().getFileCheckWaitTime(); - while (Files.size(targetPath) < contentLength && i < maxCycles) { - logger.info("... waiting to complete writing of {}", containerPath); - Thread.sleep(waitTime); - } - if (maxCycles <= i) { - throw new IOException("Read timed out after " + (maxCycles * waitTime) + " ms"); - } - } catch (IOException e) { - logger.error("Failed to copy S3 object {} to file {} (cause: {})", s3Object, containerPath, e.getMessage()); - return false; - } catch (InterruptedException e) { - logger.error("Interrupted while copying S3 object {} to file {} (cause: {})", s3Object, containerPath, e.getMessage()); - return false; - } - } - logger.info("Copied S3 object {} to file {}", s3Object, containerPath); - return true; - } catch (SdkClientException e) { - try { - if (e.getCause().getCause().getCause().getClass().equals(FileAlreadyExistsException.class)) { - return true; - } - } catch (Exception ee) { - ee.printStackTrace(); - logger.error(ee.getMessage()); - } - logger.error("Failed accessing S3 object {} (cause: {}: {})", s3Object, e.getClass().getName(), e.getMessage()); - return false; - } catch (S3Exception e) { - logger.error("Failed accessing S3 object {} (cause: {}: {}, details {})", s3Object, e.getClass().getName(), e.getMessage(), e.awsErrorDetails()); - return false; - } catch (SecurityException e) { - logger.error("Security exception accessing S3 object {} (cause: {})", s3Object, e.getMessage()); - return false; - } catch (Exception e) { - logger.error(e.getMessage(), e); - return false; - } - } - - /** - * Fetch file from S3 to local file using AWS S3 SDK V1 TransferManager - * - * @param s3Client a given instantiated V1 S3 client - * @param s3Bucket S3 bucket name - * @param s3Key S3 object key (without bucket) - * @param targetFile local (POSIX) target file path - * @return true, if the operation succeeded, false otherwise - */ - public static Boolean v1FetchFile(AmazonS3 s3Client, String s3Bucket, String s3Key, File targetFile) { - - if (logger.isTraceEnabled()) - logger.trace(">>> v1FetchFile({}, {}, {}, {})", s3Client, s3Bucket, s3Key, targetFile); - - // Download using TransferManager as per - // https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/examples-s3-transfermanager.html#transfermanager-downloading - TransferManager transferManager; - try { - // Make sure target file is not already present - if (targetFile.exists()) { - targetFile.delete(); - } - - transferManager = TransferManagerBuilder.standard() - .withMultipartCopyPartSize(MULTIPART_UPLOAD_PARTSIZE_BYTES).withS3Client(s3Client).build(); - if (null == transferManager) { - logger.error("Unable to create Transfer Manager"); - return false; - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - return false; - } - - try { - Download download = transferManager.download(s3Bucket, s3Key, targetFile); - - download.waitForCompletion(); - - // TODO This may not apply to the TransferManager any more (it did for V2 getObject() and Files.copy()) - // Unfortunately returning from waitForCompletion() may not mean the file is fully written to disk! - Long contentLength = download.getObjectMetadata().getContentLength(); - int i = 0; - long maxCycles = StorageManagerConfiguration.getConfiguration().getFileCheckMaxCycles(); - long waitTime = StorageManagerConfiguration.getConfiguration().getFileCheckWaitTime(); - while (Files.size(targetFile.toPath()) < contentLength && i < maxCycles) { - logger.info("... waiting to complete writing of {}", targetFile); - Thread.sleep(waitTime); - } - if (maxCycles <= i) { - throw new IOException("Read timed out after " + (maxCycles * waitTime) + " ms"); - } - - return true; - } catch (InterruptedException e) { - logger.error("Interrupted while copying S3 object s3:/{}/{} to file {} (cause: {})", s3Bucket, s3Key, targetFile, e.getMessage()); - return false; - } catch (AmazonServiceException e) { - logger.error("Failed to copy S3 object s3:/{}/{} to file {} (cause: {})", s3Bucket, s3Key, targetFile, e.getErrorMessage()); - return false; - } catch (IOException | AmazonClientException e) { - logger.error("Failed to copy S3 object s3:/{}/{} to file {} (cause: {})", s3Bucket, s3Key, targetFile, e.getMessage()); - return false; - } catch (Exception e) { - logger.error(e.getMessage(), e); - return false; - } finally { - transferManager.shutdownNow(false); - } - } - - /** - * Fetch file from S3 as input stream - * - * @param s3 a given instantiated S3Client - * @param s3Object URI of S3-Object (e.g. s3://bucket/path/to/some/file) - * @return the file content - */ - public static InputStream v2FetchStream(S3Client s3, String s3Object) { - - if (logger.isTraceEnabled()) - logger.trace(">>> v2FetchStream({}, {})", (null == s3 ? "MISSING" : s3.serviceName()), s3Object); - - InputStream stream = null; - try { - AmazonS3URI s3uri = new AmazonS3URI(s3Object); - stream = s3.getObject(GetObjectRequest.builder().bucket(s3uri.getBucket()).key(s3uri.getKey()).build(), - ResponseTransformer.toInputStream()); - } catch (software.amazon.awssdk.core.exception.SdkClientException e) { - logger.error(e.getMessage()); - } catch (software.amazon.awssdk.services.s3.model.NoSuchKeyException e) { - logger.error(s3Object + " --> " + e.getMessage()); - } catch (software.amazon.awssdk.services.s3.model.NoSuchBucketException e) { - logger.error(s3Object + " --> " + e.getMessage()); - } catch (software.amazon.awssdk.services.s3.model.S3Exception e) { - logger.error(e.getMessage()); - } catch (SecurityException e) { - logger.error(e.getMessage()); - } - return stream; - } - - /** - * Upload a directory to S3 using multipart uploads - * - * @param v1S3Client the S3 V1 client to use - * @param sourceDirPath path to the directory to upload - * @param targetBucketName the name of the target bucket - * @param targetKeyPrefix the key prefix to set for the target bucket - * @param recursive true, if subdirectories shall be copied, too, false otherwise - * @param pause (not used) - * @return a list of uploaded keys or null, if the operation failed - * @throws AmazonClientException if an error occurred during communication with the S3 backend storage - * @throws InterruptedException if the wait for the upload completion was interrupted - */ - public static ArrayList v1UploadDir(AmazonS3 v1S3Client, String sourceDirPath, String targetBucketName, String targetKeyPrefix, - boolean recursive, boolean pause) throws AmazonClientException, InterruptedException { - - if (logger.isTraceEnabled()) logger.trace(">>> v1UploadDir({}, {}, {}, {}, {})", - v1S3Client, sourceDirPath, targetBucketName, targetKeyPrefix, recursive, pause); - - ArrayList response = new ArrayList(); - TransferManager transferManager = TransferManagerBuilder.standard() - .withMultipartCopyPartSize(MULTIPART_UPLOAD_PARTSIZE_BYTES).withS3Client(v1S3Client).build(); - AmazonS3URI s3uri = new AmazonS3URI(targetBucketName); - String bucket = s3uri.getBucket(); - - for (int i = 1; i <= MAX_UPLOAD_RETRIES; ++i) { - try { - transferManager - .uploadDirectory(bucket, targetKeyPrefix, new File(sourceDirPath), recursive) - .waitForCompletion(); - // Success, so no retry required - break; - } catch (Exception e) { - if (i >= MAX_UPLOAD_RETRIES) { // fail at the last try - transferManager.shutdownNow(false); - throw e; - } else { - logger.warn("Uploading directory {} failed (cause: {}), retrying after 100 ms ...", sourceDirPath, e.getMessage()); - Thread.sleep(100); - } - } - } - - transferManager.shutdownNow(false); - - // check files in s3 & add to response - List list = v1S3Client.listObjectsV2(bucket, targetKeyPrefix).getObjectSummaries(); - for (S3ObjectSummary o : list) { - response.add(o.getKey()); - } - logger.info("Copied dir://{} to {}/{}", sourceDirPath, targetBucketName, targetKeyPrefix); - - return response; - } - - /** - * Upload file to S3 using multipart uploads - * - * The source file is uploaded to a S3 storage. The target key in the storage is build as: - * s3://<bucket name>/<target key prefix>/<source file name> - * - * @param v1S3Client the S3 V1 client to use - * @param sourceFilePath The path of source file - * @param targetBucketName The S3 bucket to store the file - * @param targetKeyPrefix The key in the bucket to store the file - * @param pause (not used) - * @return a single-element list of keys uploaded or null, if the operation failed - * @throws AmazonClientException if an error occurred during communication with the S3 backend storage - * @throws InterruptedException if the wait for the upload completion was interrupted - */ - public static ArrayList v1UploadFile(AmazonS3 v1S3Client, String sourceFilePath, String targetBucketName, - String targetKeyPrefix, boolean pause) throws Exception { - - if (logger.isTraceEnabled()) logger.trace(">>> v1UploadFile({}, {}, {}, {}, {})", - v1S3Client, sourceFilePath, targetBucketName, targetKeyPrefix, pause); - - ArrayList response = new ArrayList(); - String targetKeyName = null; - File f = new File(sourceFilePath); - if (f != null && f.isFile()) { - String fn = f.getName(); - if (targetKeyPrefix != null && !targetKeyPrefix.isEmpty()) { - targetKeyName = targetKeyPrefix + (targetKeyPrefix.endsWith("/") ? "" : "/") + fn; - } else { - targetKeyName = fn; - } - AmazonS3URI s3uri = new AmazonS3URI(targetBucketName); - String bucket = s3uri.getBucket(); - TransferManager transferManager = TransferManagerBuilder.standard() - .withMultipartCopyPartSize(MULTIPART_UPLOAD_PARTSIZE_BYTES).withS3Client(v1S3Client).build(); - - for (int i = 1; i <= MAX_UPLOAD_RETRIES; ++i) { - try { - transferManager.upload(bucket, targetKeyName, f).waitForCompletion(); - - // Success, so no retry required - break; - } catch (Exception e) { - if (i >= MAX_UPLOAD_RETRIES) { // fail at the last try - transferManager.shutdownNow(false); - throw e; - } else { - logger.warn("Uploading file {} failed (cause: {}), retrying after 100 ms ...", sourceFilePath, e.getMessage()); - Thread.sleep(100); - } - } - } - - String result = "s3://" + bucket + (targetKeyName.startsWith("/") ? "" : "/") + targetKeyName; - response.add(result); - logger.info("Copied file://{} to {}", sourceFilePath, result); - transferManager.shutdownNow(false); - return response; - } else { - return null; - } - } - - /** - * Upload files or directories to S3 using Multipart-Uploads - * - * @param v1S3Client the S3 V1 client to use - * @param sourcePath path to the file or directory to upload - * @param targetBucketName the name of the target bucket - * @param targetPathPrefix the key prefix to set for the target bucket - * @param pause (not used) - * @return a list of uploaded keys or null, if the operation failed - * @throws AmazonClientException if an error occurred during communication with the S3 backend storage - * @throws InterruptedException if the wait for the upload completion was interrupted - * @throws Exception on an unspecific error condition during a file upload - */ - public static ArrayList v1Upload(AmazonS3 v1S3Client, String sourcePath, String targetBucketName, String targetPathPrefix, - boolean pause) throws AmazonClientException, InterruptedException, Exception { - - if (logger.isTraceEnabled()) logger.trace(">>> v1Upload({}, {}, {}, {}, {})", - v1S3Client, sourcePath, targetBucketName, targetPathPrefix, pause); - - String s3Prefix = "s3://"; - if (!targetBucketName.startsWith(s3Prefix)) { - targetBucketName = s3Prefix + targetBucketName; - } - File f = new File(sourcePath); - - ArrayList response = null; - try { - if (f.isFile()) { - response = v1UploadFile(v1S3Client, sourcePath, targetBucketName, targetPathPrefix, false); - } - if (f.isDirectory()) { - response = v1UploadDir(v1S3Client, sourcePath, targetBucketName, targetPathPrefix, true, false); - } - } catch (AmazonServiceException e) { - logger.error("Amazon service error: " + e.getMessage()); - throw e; - } catch (AmazonClientException e) { - logger.error("Amazon client error: " + e.getMessage()); - throw e; - } catch (InterruptedException e) { - logger.error("Transfer interrupted: " + e.getMessage()); - throw e; - } catch (Exception e) { - logger.error("Unspecific exception: " + e.getClass().getName() + " / " + e.getMessage()); - throw e; - } - return response; - } - - /** - * Copy objects between S3 buckets or inside S3 buckets - * - * @param s3Client the S3 V1 client to use - * @param sourceBucketName the bucket to copy from - * @param sourceObjectKeyPrefix the object key to copy from - * @param destBucketName the bucket to copy to - * @param destObjectPrefix the object key to copy to - * @return a list of copied keys or null, if the operation failed - * @throws AmazonClientException if an error occurred during communication with the S3 backend storage - * @throws InterruptedException if the wait for the upload completion was interrupted - */ - public static ArrayList v1Copy(AmazonS3 s3Client, String sourceBucketName, String sourceObjectKeyPrefix, - String destBucketName, String destObjectPrefix) throws AmazonClientException, InterruptedException { - - if (logger.isTraceEnabled()) logger.trace(">>> v1Copy({}, {}, {}, {}, {})", - s3Client, sourceBucketName, sourceObjectKeyPrefix, destBucketName, destObjectPrefix); - - String separator = "/"; - - if (destObjectPrefix.endsWith(separator)) { - destObjectPrefix = destObjectPrefix.substring(0, destObjectPrefix.length() - 1); - } - - ArrayList response = new ArrayList(); - TransferManager transferManager = TransferManagerBuilder.standard() - .withMultipartCopyPartSize(MULTIPART_UPLOAD_PARTSIZE_BYTES).withS3Client(s3Client).build(); - try { - - // list objects under sourceObjectKey (is 1:1 if single file, is 1:n if "folder") - List list = s3Client.listObjectsV2(sourceBucketName, sourceObjectKeyPrefix).getObjectSummaries(); - - - for (S3ObjectSummary o : list) { - // destinationKey is built using this pattern: / - String key = o.getKey(); - - for (int i = 1; i <= MAX_UPLOAD_RETRIES; ++i) { - try { - transferManager - .copy(sourceBucketName, key, destBucketName, destObjectPrefix+separator+key) - .waitForCompletion(); - // Success, so no retry required - break; - } catch (Exception e) { - if (i >= MAX_UPLOAD_RETRIES) { // fail at the last try - transferManager.shutdownNow(false); - throw e; - } else { - logger.warn("Copying s3://{}/{} failed (cause: {}), retrying after 100 ms ...", sourceBucketName, key, e.getMessage()); - Thread.sleep(100); - } - } - } - - response.add("s3://" + destBucketName + "/" + destObjectPrefix + "/" + key); - logger.info("Copied s3://{}/{} to s3://{}/{}/{}", sourceBucketName, key, destBucketName, destObjectPrefix,key); - } - - transferManager.shutdownNow(false); - return response; - } catch (AmazonServiceException e) { - logger.error(e.getErrorMessage()); - return null; - } - - } - - /** - * Create a folder-like object in repository. This method passes low-level S3 exceptions on. - * - * @param client the S3 V2 client to use - * @param bucketName the name of the bucket to create the folder in - * @param folderName the name of the folder to create - * @throws SdkException Base class for all exceptions that can be thrown by the SDK (both service and client). Can be used for catch all scenarios. - * @throws SdkClientException If any client side error occurs such as an IO related failure, failure to get credentials, etc. - * @throws S3Exception Base class for all service exceptions. Unknown exceptions will be thrown as an instance of this type. - * @throws AwsServiceException if an error in the S3 object storage service occurs - */ - public static void createFolder(S3Client client, String bucketName, String folderName) - throws SdkException, SdkClientException, S3Exception, AwsServiceException { - - if (logger.isTraceEnabled()) logger.trace(">>> createFolder({}, {}, {})", - (null == client ? "MISSING" : client.serviceName()), bucketName, folderName); - - // create meta-data for your folder and set content-length to 0 - String key = folderName; - if (!key.endsWith("/")) { - key += "/"; - } - PutObjectRequest putRequest = PutObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build(); - try { - client.putObject(putRequest, RequestBody.empty()); - } catch (S3Exception e) { - logger.error(e.getMessage()); - throw e; - } catch (AwsServiceException e) { - logger.error(e.getMessage()); - throw e; - } catch (SdkClientException e) { - logger.error(e.getMessage()); - throw e; - } - } - - /** - * Delete object(s) in repository. - * The prefix is either an object key or like a directory path - * - * @param client the S3 V1 client to use - * @param bucketName the name of the bucket to delete the object(s) in - * @param prefix the object prefix - */ - public static void deleteDirectory(AmazonS3 client, String bucketName, String prefix) { - - if (logger.isTraceEnabled()) logger.trace(">>> deleteDirectory({}, {}, {})", client, bucketName, prefix); - - try { - ObjectListing objectList = client.listObjects(bucketName, prefix); - List objectSummeryList = objectList.getObjectSummaries(); - while (objectList.isTruncated()) { - objectList = client.listNextBatchOfObjects(objectList); - objectSummeryList.addAll(objectList.getObjectSummaries()); - } - - String[] keysList = new String[objectSummeryList.size()]; - int count = 0; - for (S3ObjectSummary summery : objectSummeryList) { - keysList[count++] = summery.getKey(); - } - DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucketName).withKeys(keysList); - client.deleteObjects(deleteObjectsRequest); - } catch (AmazonServiceException e) { - logger.error(e.getMessage()); - throw e; - } catch (com.amazonaws.SdkClientException e) { - logger.error(e.getMessage()); - throw e; - } - } - - /** - * Get the object length (file size) - * - * @param client the S3 V1 client to use - * @param bucketName the name of the bucket, in which the object is stored - * @param key the object key - * @return the length of the object or zero, if no object metadata could be retrieved - */ - public static long getLength(AmazonS3 client, String bucketName, String key) { - - if (logger.isTraceEnabled()) logger.trace(">>> getLength({}, {}, {})", client, bucketName, key); - - ObjectMetadata md; - try { - md = client.getObjectMetadata(bucketName, key); - } catch (AmazonServiceException e) { - logger.error(e.getMessage()); - throw e; - } catch (com.amazonaws.SdkClientException e) { - logger.error(e.getMessage()); - throw e; - } - if (md != null) { - return md.getContentLength(); - } - return 0; - } - -} diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/AtomicCommand.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/AtomicCommand.java similarity index 95% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/AtomicCommand.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/AtomicCommand.java index de44d1d1e..6610ebc74 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/AtomicCommand.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/AtomicCommand.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.model; +package de.dlr.proseo.storagemgr.model; import java.io.IOException; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/DefaultRetryStrategy.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/DefaultRetryStrategy.java similarity index 98% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/DefaultRetryStrategy.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/DefaultRetryStrategy.java index 3ddc839a1..c1bfd9055 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/DefaultRetryStrategy.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/DefaultRetryStrategy.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.model; +package de.dlr.proseo.storagemgr.model; import java.io.IOException; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/Storage.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/Storage.java similarity index 99% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/Storage.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/Storage.java index c23a9d855..72c9ce88a 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/Storage.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/Storage.java @@ -2,7 +2,7 @@ * Storage.java * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.model; +package de.dlr.proseo.storagemgr.model; import java.io.IOException; import java.io.InputStream; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/StorageFile.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/StorageFile.java similarity index 96% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/StorageFile.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/StorageFile.java index e0f6c1364..614be1098 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/StorageFile.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/StorageFile.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.model; +package de.dlr.proseo.storagemgr.model; /** * Storage File Structure. The file can exist or it can be just an abstract diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/StorageType.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/StorageType.java similarity index 85% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/StorageType.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/StorageType.java index d3d4cf619..ae2a4ffa9 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/StorageType.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/StorageType.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.model; +package de.dlr.proseo.storagemgr.model; /** * Storage types supported by the prosEO Storage Manager, currently S3 and diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/package-info.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/package-info.java similarity index 84% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/package-info.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/package-info.java index 1efd63c9d..d13c276a7 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/model/package-info.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/model/package-info.java @@ -9,4 +9,4 @@ * * @author Denys Chaykovskiy */ -package de.dlr.proseo.storagemgr.version2.model; \ No newline at end of file +package de.dlr.proseo.storagemgr.model; \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileCopier.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileCopier.java new file mode 100644 index 000000000..0ff624039 --- /dev/null +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileCopier.java @@ -0,0 +1,131 @@ +/** + * PosixAtomicFileCopier.java + * + * (C) 2023 Dr. Bassler & Co. Managementberatung GmbH + */ +package de.dlr.proseo.storagemgr.posix; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.dlr.proseo.storagemgr.model.AtomicCommand; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; + +/** + * Posix Atomic File Copier from some POSIX absolute path to another POSIX absolute path (normally, to cache) + * + * @author Denys Chaykovskiy + * + */ +public class PosixAtomicFileCopier implements AtomicCommand { + + /** Info */ + private static final String INFO = "Posix ATOMIC File Copier"; + + /** Completed Info */ + private static final String COMPLETED = "file COPIED"; + + /** Failed Info */ + private static final String FAILED = "file copy FAILED"; + + /** Logger for this class */ + private static Logger logger = LoggerFactory.getLogger(PosixAtomicFileCopier.class); + + /** source file */ + private String sourceFile; + + /** target file or dir */ + private String targetFileOrDir; + + /** + * Constructor + * + * @param sourceFile sourceFile + * @param targetFileOrDir target file or directory + */ + public PosixAtomicFileCopier(String sourceFile, String targetFileOrDir) { + + this.sourceFile = sourceFile; + this.targetFileOrDir = targetFileOrDir; + } + + /** + * Executes copy of the file from a posix path to a posix path (normally, to the cache) + * + * @return copied file name + */ + @Override + public String execute() throws IOException { + + if (logger.isTraceEnabled()) + logger.trace(">>> execute() - copyFile({},{})", sourceFile, targetFileOrDir); + + String targetFile = targetFileOrDir; + + if (new PathConverter(targetFileOrDir).isDirectory()) { + targetFile = new PathConverter(targetFileOrDir, getFileName(sourceFile)).getPath(); + } + + new FileUtils(targetFile).createParentDirectories(); + + Path sourceFilePath = new File(sourceFile).toPath(); + Path targetFilePath = new File(targetFile).toPath(); + + try { + Path copiedPath = Files.copy(sourceFilePath, targetFilePath, StandardCopyOption.REPLACE_EXISTING); + return copiedPath.toString(); + + } catch (Exception e) { + if (logger.isTraceEnabled()) + logger.trace(getFailedInfo() + e.getMessage()); + throw new IOException(e); + } + } + + /** + * Gets information about atomic command (mostly for logs) + * + * @return information about atomic command + */ + @Override + public String getInfo() { + return INFO + " "; + } + + /** + * Gets information about completed atomic command (mostly for logs) + * + * @return information about completed atomic command + */ + @Override + public String getCompletedInfo() { + return INFO + ": " + COMPLETED + " "; + } + + /** + * Gets information about failed atomic command (mostly for logs) + * + * @return information about failed atomic command + */ + @Override + public String getFailedInfo() { + return INFO + ": " + FAILED + " "; + } + + /** + * Gets file name + * + * @param path path + * @return file name + */ + private String getFileName(String path) { + return new File(path).getName(); + } +} \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileDownloader.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileDownloader.java new file mode 100644 index 000000000..a605f556d --- /dev/null +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileDownloader.java @@ -0,0 +1,131 @@ +/** + * PosixAtomicFileDownloader.java + * + * (C) 2023 Dr. Bassler & Co. Managementberatung GmbH + */ +package de.dlr.proseo.storagemgr.posix; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.dlr.proseo.storagemgr.model.AtomicCommand; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; + +/** + * Posix Atomic File Downloader from the POSIX Storage to some POSIX Path (normally, to the cache) + * + * @author Denys Chaykovskiy + * + */ +public class PosixAtomicFileDownloader implements AtomicCommand { + + /** Info */ + private static final String INFO = "Posix ATOMIC File Downloader"; + + /** Completed Info */ + private static final String COMPLETED = "file DOWNLOADED"; + + /** Failed Info */ + private static final String FAILED = "file download FAILED"; + + /** Logger for this class */ + private static Logger logger = LoggerFactory.getLogger(PosixAtomicFileDownloader.class); + + /** source file */ + private String sourceFile; + + /** target file or dir */ + private String targetFileOrDir; + + /** + * Constructor + * + * @param sourceFile sourceFile + * @param targetFileOrDir target file or directory + */ + public PosixAtomicFileDownloader(String sourceFile, String targetFileOrDir) { + + this.sourceFile = sourceFile; + this.targetFileOrDir = targetFileOrDir; + } + + /** + * Executes download of the file from the storage to posix path + * + * @return uploaded file name + */ + @Override + public String execute() throws IOException { + + if (logger.isTraceEnabled()) + logger.trace(">>> execute() - downloadFile({},{})", sourceFile, targetFileOrDir); + + String targetFile = targetFileOrDir; + + if (new PathConverter(targetFileOrDir).isDirectory()) { + targetFile = new PathConverter(targetFileOrDir, getFileName(sourceFile)).getPath(); + } + + new FileUtils(targetFile).createParentDirectories(); + + Path sourceFilePath = new File(sourceFile).toPath(); + Path targetFilePath = new File(targetFile).toPath(); + + try { + Path copiedPath = Files.copy(sourceFilePath, targetFilePath, StandardCopyOption.REPLACE_EXISTING); + return copiedPath.toString(); + + } catch (Exception e) { + if (logger.isTraceEnabled()) + logger.trace(getFailedInfo() + e.getMessage()); + throw new IOException(e); + } + } + + /** + * Gets information about atomic command (mostly for logs) + * + * @return information about atomic command + */ + @Override + public String getInfo() { + return INFO + " "; + } + + /** + * Gets information about completed atomic command (mostly for logs) + * + * @return information about completed atomic command + */ + @Override + public String getCompletedInfo() { + return INFO + ": " + COMPLETED + " "; + } + + /** + * Gets information about failed atomic command (mostly for logs) + * + * @return information about failed atomic command + */ + @Override + public String getFailedInfo() { + return INFO + ": " + FAILED + " "; + } + + /** + * Gets file name + * + * @param path path + * @return file name + */ + private String getFileName(String path) { + return new File(path).getName(); + } +} \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileUploader.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileUploader.java new file mode 100644 index 000000000..f456e5ecd --- /dev/null +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixAtomicFileUploader.java @@ -0,0 +1,131 @@ +/** + * PosixAtomicFileUploader.java + * + * (C) 2023 Dr. Bassler & Co. Managementberatung GmbH + */ +package de.dlr.proseo.storagemgr.posix; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.dlr.proseo.storagemgr.model.AtomicCommand; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; + +/** + * Posix Atomic File Uploader from some external POSIX absolute path to the POSIX Storage + * + * @author Denys Chaykovskiy + * + */ +public class PosixAtomicFileUploader implements AtomicCommand { + + /** Info */ + private static final String INFO = "Posix ATOMIC File Uploader"; + + /** Completed Info */ + private static final String COMPLETED = "file UPLOADED"; + + /** Failed Info */ + private static final String FAILED = "file upload FAILED"; + + /** Logger for this class */ + private static Logger logger = LoggerFactory.getLogger(PosixAtomicFileUploader.class); + + /** source file */ + private String sourceFile; + + /** target file or dir */ + private String targetFileOrDir; + + /** + * Constructor + * + * @param sourceFile sourceFile + * @param targetFileOrDir target file or directory + */ + public PosixAtomicFileUploader(String sourceFile, String targetFileOrDir) { + + this.sourceFile = sourceFile; + this.targetFileOrDir = targetFileOrDir; + } + + /** + * Executes upload of the file to posix path + * + * @return uploaded file name + */ + @Override + public String execute() throws IOException { + + if (logger.isTraceEnabled()) + logger.trace(">>> execute() - uploadFile({},{})", sourceFile, targetFileOrDir); + + String targetFile = targetFileOrDir; + + if (new PathConverter(targetFileOrDir).isDirectory()) { + targetFile = new PathConverter(targetFileOrDir, getFileName(sourceFile)).getPath(); + } + + new FileUtils(targetFile).createParentDirectories(); + + Path sourceFilePath = new File(sourceFile).toPath(); + Path targetFilePath = new File(targetFile).toPath(); + + try { + Path copiedPath = Files.copy(sourceFilePath, targetFilePath, StandardCopyOption.REPLACE_EXISTING); + return copiedPath.toString(); + + } catch (Exception e) { + if (logger.isTraceEnabled()) + logger.trace(getFailedInfo() + e.getMessage()); + throw new IOException(e); + } + } + + /** + * Gets information about atomic command (mostly for logs) + * + * @return information about atomic command + */ + @Override + public String getInfo() { + return INFO + " "; + } + + /** + * Gets information about completed atomic command (mostly for logs) + * + * @return information about completed atomic command + */ + @Override + public String getCompletedInfo() { + return INFO + ": " + COMPLETED + " "; + } + + /** + * Gets information about failed atomic command (mostly for logs) + * + * @return information about failed atomic command + */ + @Override + public String getFailedInfo() { + return INFO + ": " + FAILED + " "; + } + + /** + * Gets file name + * + * @param path path + * @return file name + */ + private String getFileName(String path) { + return new File(path).getName(); + } +} \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixConfiguration.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixConfiguration.java new file mode 100644 index 000000000..92b855337 --- /dev/null +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixConfiguration.java @@ -0,0 +1,78 @@ +/** + * PosixConfiguration.java + * + * (C) 2023 Dr. Bassler & Co. Managementberatung GmbH + */ +package de.dlr.proseo.storagemgr.posix; + +/** + * Holds the configuration settings required to connect to and interact with an + * POSIX storage system. It allows for the customization of various aspects such as + * access credentials, bucket, paths, and behavior-related settings like + * request attempts and file transfer management. + * + * @author Denys Chaykovskiy + */ +public class PosixConfiguration { + + /** Bucket */ + private String bucket; + + /** base path */ + private String basePath; + + /** source path */ + private String sourcePath; + + /** max request attempts */ + private int maxRequestAttempts; + + /** wait time */ + private long fileCheckWaitTime; + + + public String getBucket() { + return bucket; + } + + public void setBucket(String bucket) { + this.bucket = bucket; + } + + public String getBasePath() { + return basePath; + } + + public void setBasePath(String basePath) { + this.basePath = basePath; + } + + public String getSourcePath() { + return sourcePath; + } + + public void setSourcePath(String sourcePath) { + this.sourcePath = sourcePath; + } + + public int getMaxRequestAttempts() { + return maxRequestAttempts; + } + + public void setMaxRequestAttempts(int maxRequestAttempts) { + this.maxRequestAttempts = maxRequestAttempts; + } + + public long getFileCheckWaitTime() { + return fileCheckWaitTime; + } + + public void setFileCheckWaitTime(Long fileCheckWaitTime) { + this.fileCheckWaitTime = fileCheckWaitTime; + } + + public void setFileCheckWaitTime(long fileCheckWaitTime) { + this.fileCheckWaitTime = fileCheckWaitTime; + } + +} \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/PosixDAL.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java similarity index 70% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/PosixDAL.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java index 67e20269b..83b31f7d2 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/PosixDAL.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixDAL.java @@ -3,22 +3,22 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.posix; +package de.dlr.proseo.storagemgr.posix; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import de.dlr.proseo.logging.logger.ProseoLogger; -import de.dlr.proseo.logging.messages.StorageMgrMessage; -import de.dlr.proseo.storagemgr.version2.FileUtils; -import de.dlr.proseo.storagemgr.version2.PathConverter; +import de.dlr.proseo.storagemgr.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.DefaultRetryStrategy; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * A data access layer for interacting with a POSIX-based storage system. It @@ -33,7 +33,30 @@ public class PosixDAL { /** Logger for this class */ private static ProseoLogger logger = new ProseoLogger(PosixDAL.class); + + /** POSIX configuration */ + private PosixConfiguration cfg; + + /** + * Constructor + * + * @param cfg POSIX Configuration + */ + public PosixDAL(PosixConfiguration cfg) { + + this.cfg = cfg; + } + /** + * Gets the POSIX configuration used by this PosixDAL instance. + * + * @return the PosixConfiguration object + */ + public PosixConfiguration getConfiguration() { + + return cfg; + } + /** * Retrieves a list of files that match a given path (or prefix). It recursively * scans directories and adds the absolute paths of files to the result @@ -145,30 +168,10 @@ public String uploadFile(String sourceFile, String targetFileOrDir) throws IOExc if (logger.isTraceEnabled()) logger.trace(">>> uploadFile({},{})", sourceFile, targetFileOrDir); + + AtomicCommand fileUploader = new PosixAtomicFileUploader(sourceFile, targetFileOrDir); - String targetFile = targetFileOrDir; - - if (new PathConverter(targetFileOrDir).isDirectory()) { - targetFile = new PathConverter(targetFileOrDir, getFileName(sourceFile)).getPath(); - } - - createParentDirectories(targetFile); - - Path sourceFilePath = new File(sourceFile).toPath(); - Path targetFilePath = new File(targetFile).toPath(); - - try { - Path copiedPath = Files.copy(sourceFilePath, targetFilePath, StandardCopyOption.REPLACE_EXISTING); - return copiedPath.toString(); - - } catch (IOException e) { - if (logger.isDebugEnabled()) { - logger.debug("An exception occurred. Cause: ", e); - } - if (logger.isTraceEnabled()) - logger.log(StorageMgrMessage.FILE_NOT_UPLOADED, sourceFile, targetFileOrDir, e.getMessage()); - throw e; - } + return new DefaultRetryStrategy<>(fileUploader, cfg.getMaxRequestAttempts(), cfg.getFileCheckWaitTime()).execute(); } /** @@ -235,30 +238,10 @@ public String downloadFile(String sourceFile, String targetFileOrDir) throws IOE if (logger.isTraceEnabled()) logger.trace(">>> downloadFile({},{})", sourceFile, targetFileOrDir); + + AtomicCommand fileDownloader = new PosixAtomicFileDownloader(sourceFile, targetFileOrDir); - String targetFile = targetFileOrDir; - - if (new PathConverter(targetFileOrDir).isDirectory()) { - targetFile = new PathConverter(targetFileOrDir, getFileName(sourceFile)).getPath(); - } - - createParentDirectories(targetFile); - - Path sourceFilePath = new File(sourceFile).toPath(); - Path targetFilePath = new File(targetFile).toPath(); - - try { - Path copiedPath = Files.copy(sourceFilePath, targetFilePath, StandardCopyOption.REPLACE_EXISTING); - return copiedPath.toString(); - - } catch (Exception e) { - if (logger.isDebugEnabled()) { - logger.debug("An exception occurred. Cause: ", e); - } - if (logger.isTraceEnabled()) - logger.log(StorageMgrMessage.FILE_NOT_DOWNLOADED, sourceFile, targetFileOrDir, e.getMessage()); - throw e; - } + return new DefaultRetryStrategy<>(fileDownloader, cfg.getMaxRequestAttempts(), cfg.getFileCheckWaitTime()).execute(); } /** @@ -317,6 +300,82 @@ public List download(String sourceFileOrDir, String targetFileOrDir) thr return downloadedFiles; } + + /** + * Copies a file from the POSIX file system to another posix file system (normally, to the cache) + * location + * + * @param sourceFile posix source file + * @param targetFileOrDir posix target file or directory + * @return path of downloaded file + * @throws IOException if file cannot be downloaded + */ + public String copyFile(String sourceFile, String targetFileOrDir) throws IOException { + + if (logger.isTraceEnabled()) + logger.trace(">>> copyFile({},{})", sourceFile, targetFileOrDir); + + AtomicCommand fileCopier = new PosixAtomicFileCopier(sourceFile, targetFileOrDir); + + return new DefaultRetryStrategy<>(fileCopier, cfg.getMaxRequestAttempts(), cfg.getFileCheckWaitTime()).execute(); + } + + /** + * Copies files or directories recursively from the posix file system to another posix file system (normally, to the cache) + * + * @param sourceFileOrDir posix source file or directory + * @param targetFileOrDir posix target file or directory + * @return path list of copied files + * @throws IOException true if file or directory cannot be copied + */ + public List copy(String sourceFileOrDir, String targetFileOrDir) throws IOException { + + if (logger.isTraceEnabled()) + logger.trace(">>> copy({},{})", sourceFileOrDir, targetFileOrDir); + + List copiedFiles = new ArrayList<>(); + + if (isFile(sourceFileOrDir)) { + String copiedFile = copyFile(sourceFileOrDir, targetFileOrDir); + copiedFiles.add(copiedFile); + return copiedFiles; + } + + String sourceDir = sourceFileOrDir; + String targetDir = targetFileOrDir; + targetDir = new PathConverter(targetDir).addSlashAtEnd().getPath(); + + File directory = new File(sourceDir); + File[] files = directory.listFiles(); + if (files == null) + return copiedFiles; + Arrays.sort(files); + + for (File file : files) { + if (file.isFile()) { + String sourceFile = file.getAbsolutePath(); + String copiedFile = downloadFile(sourceFile, targetDir); + copiedFiles.add(copiedFile); + } + } + + for (File file : files) { + if (file.isDirectory()) { + + String sourceSubDir = file.getAbsolutePath(); + String targetSubDir = Paths.get(targetDir, file.getName()).toString(); + targetSubDir = new PathConverter(targetSubDir).addSlashAtEnd().getPath(); + + // String path = new PathConverter(targetSubDirPath).addSlashAtEnd().getPath(); + // targetSubDir.setRelativePath(path); + + List subDirFiles = copy(sourceSubDir, targetSubDir); + copiedFiles.addAll(subDirFiles); + } + } + + return copiedFiles; + } /** * Deletes file in storage @@ -347,22 +406,4 @@ public List delete(String sourceFileOrDir) throws IOException { return new FileUtils(sourceFileOrDir).delete(); } - - /** - * Create parent directories for a given path if they do not exist - * - * @param path path - */ - private void createParentDirectories(String path) { - - if (logger.isTraceEnabled()) - logger.trace(">>> getFiles({})", path); - - File targetFile = new File(path); - File parent = targetFile.getParentFile(); - - if (parent != null && !parent.exists() && !parent.mkdirs()) { - throw new IllegalStateException("Couldn't create dir: " + parent); - } - } } \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorage.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorage.java similarity index 88% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorage.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorage.java index 76f3adf41..eb2645ce4 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorage.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorage.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.posix; +package de.dlr.proseo.storagemgr.posix; import java.io.File; import java.io.FileInputStream; @@ -17,11 +17,11 @@ import de.dlr.proseo.logging.logger.ProseoLogger; import de.dlr.proseo.logging.messages.StorageMgrMessage; -import de.dlr.proseo.storagemgr.version2.FileUtils; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * An implementation of the Storage interface for a POSIX-based file system, @@ -29,47 +29,38 @@ * retrieving information about the storage, performing file operations, and * path conversion. * - * This class assumes a one-bucket concept for POSIX storage. + * This class assumes a no-bucket concept for POSIX storage. * * @author Denys Chaykovskiy */ public class PosixStorage implements Storage { - /** Base path */ - private String basePath; - - /** Source path */ - private String sourcePath; - /** Bucket */ private String bucket; + + /** POSIX data access layer object */ + private PosixDAL posixDAL; + + /** POSIX configuration */ + private PosixConfiguration cfg; /** Logger for this class */ private static ProseoLogger logger = new ProseoLogger(PosixStorage.class); - /** POSIX data access layer object */ - private PosixDAL posixDAL = new PosixDAL(); - - /** - * Default Constructor - */ - public PosixStorage() { - } - /** * No bucket constructor initializing the storage with a basePath and * sourcePath. It creates the necessary directories if they don't exist. * - * @param basePath base path - * @param sourcePath source path + * @param cfg POSIX configuration */ - public PosixStorage(String basePath, String sourcePath) { - this.basePath = basePath; - this.sourcePath = sourcePath; - this.bucket = StorageFile.NO_BUCKET; + public PosixStorage(PosixConfiguration cfg) { + this.bucket = cfg.getBucket(); // StorageFile.NO_BUCKET is used in prosEO in POSIX Storage; - new FileUtils(basePath).createDirectories(); - new FileUtils(sourcePath).createDirectories(); + new FileUtils(cfg.getBasePath()).createDirectories(); + new FileUtils(cfg.getSourcePath()).createDirectories(); + + this.cfg = cfg; + posixDAL = new PosixDAL(cfg); } /** @@ -89,7 +80,7 @@ public StorageType getStorageType() { */ @Override public String getBasePath() { - return basePath; + return cfg.getBasePath(); } /** @@ -111,7 +102,7 @@ public String getAbsoluteBasePath() { */ @Override public String getSourcePath() { - return sourcePath; + return cfg.getSourcePath(); } /** @@ -123,7 +114,7 @@ public String getSourcePath() { public void setBucket(String bucket) { this.bucket = bucket; - String bucketPath = Paths.get(basePath, bucket).toString(); + String bucketPath = Paths.get(cfg.getBasePath(), bucket).toString(); new FileUtils(bucketPath).createDirectories(); } @@ -182,7 +173,7 @@ public List getRelativeFiles(String relativePath) { if (logger.isTraceEnabled()) logger.trace(">>> getFiles({})", relativePath); - String path = new PathConverter(basePath, relativePath).getPath(); + String path = new PathConverter(cfg.getBasePath(), relativePath).getPath(); return getRelativePath(posixDAL.getFiles(path)); } @@ -194,7 +185,7 @@ public List getRelativeFiles(String relativePath) { */ @Override public List getRelativeFiles() { - return getRelativePath(posixDAL.getFiles(basePath)); + return getRelativePath(posixDAL.getFiles(cfg.getBasePath())); } /** @@ -207,7 +198,7 @@ public List getAbsoluteFiles(String relativePath) { if (logger.isTraceEnabled()) logger.trace(">>> getFiles({})", relativePath); - String path = new PathConverter(basePath, relativePath).getPath(); + String path = new PathConverter(cfg.getBasePath(), relativePath).getPath(); return posixDAL.getFiles(path); } @@ -218,7 +209,7 @@ public List getAbsoluteFiles(String relativePath) { * @return the list of all files from the storage */ public List getAbsoluteFiles() { - return posixDAL.getFiles(basePath); + return posixDAL.getFiles(cfg.getBasePath()); } /** @@ -233,8 +224,8 @@ public String getRelativePath(String absolutePath) { logger.trace(">>> getRelativePath({})", absolutePath); List basePaths = new ArrayList<>(); - basePaths.add(basePath); - basePaths.add(sourcePath); + basePaths.add(cfg.getBasePath()); + basePaths.add(cfg.getSourcePath()); return new PathConverter(absolutePath, basePaths).getRelativePath().getPath(); } @@ -250,10 +241,10 @@ public List getRelativePath(List absolutePaths) { if (logger.isTraceEnabled()) logger.trace(">>> getRelativePath({})", absolutePaths); - logger.trace("... basePath = {}, sourcePath = {}", basePath, sourcePath); + logger.trace("... basePath = {}, sourcePath = {}", cfg.getBasePath(), cfg.getSourcePath()); List basePaths = new ArrayList<>(); - basePaths.add(basePath); - basePaths.add(sourcePath); + basePaths.add(cfg.getBasePath()); + basePaths.add(cfg.getSourcePath()); List relativePaths = new ArrayList<>(); @@ -305,7 +296,7 @@ public List getAbsolutePath(List relativePaths) { */ @Override public StorageFile getStorageFile(String relativePath) { - return new PosixStorageFile(basePath, relativePath); + return new PosixStorageFile(cfg.getBasePath(), relativePath); } /** @@ -315,7 +306,7 @@ public StorageFile getStorageFile(String relativePath) { */ @Override public List getStorageFiles() { - List paths = posixDAL.getFiles(basePath); + List paths = posixDAL.getFiles(cfg.getBasePath()); List storageFiles = new ArrayList<>(); for (String path : paths) { @@ -453,7 +444,7 @@ public List upload(StorageFile sourceFileOrDir) throws IOException { if (logger.isTraceEnabled()) logger.trace(">>> upload({})", sourceFileOrDir.getFullPath()); - StorageFile targetFileOrDir = new PosixStorageFile(basePath, sourceFileOrDir.getRelativePath()); + StorageFile targetFileOrDir = new PosixStorageFile(cfg.getBasePath(), sourceFileOrDir.getRelativePath()); List uploadedAbsolutePaths = posixDAL.upload(sourceFileOrDir.getFullPath(), targetFileOrDir.getFullPath()); @@ -472,7 +463,7 @@ public String uploadFile(StorageFile sourceFile) throws IOException { if (logger.isTraceEnabled()) logger.trace(">>> uploadFile({})", sourceFile.getFullPath()); - StorageFile targetFile = new PosixStorageFile(basePath, sourceFile.getRelativePath()); + StorageFile targetFile = new PosixStorageFile(cfg.getBasePath(), sourceFile.getRelativePath()); String uploadedFile = posixDAL.uploadFile(sourceFile.getFullPath(), targetFile.getFullPath()); @@ -491,8 +482,8 @@ public List uploadSourceFileOrDir(String relativeSourceFileOrDir) throws if (logger.isTraceEnabled()) logger.trace(">>> upload({})", relativeSourceFileOrDir); - StorageFile sourceFileOrDir = new PosixStorageFile(sourcePath, relativeSourceFileOrDir); - StorageFile targetFileOrDir = new PosixStorageFile(basePath, relativeSourceFileOrDir); + StorageFile sourceFileOrDir = new PosixStorageFile(cfg.getSourcePath(), relativeSourceFileOrDir); + StorageFile targetFileOrDir = new PosixStorageFile(cfg.getBasePath(), relativeSourceFileOrDir); List uploadedAbsolutePaths = posixDAL.upload(sourceFileOrDir.getFullPath(), targetFileOrDir.getFullPath()); @@ -511,8 +502,8 @@ public String uploadSourceFile(String relativeSourceFile) throws IOException { if (logger.isTraceEnabled()) logger.trace(">>> uploadFile({})", relativeSourceFile); - StorageFile sourceFile = new PosixStorageFile(sourcePath, relativeSourceFile); - StorageFile targetFile = new PosixStorageFile(basePath, relativeSourceFile); + StorageFile sourceFile = new PosixStorageFile(cfg.getSourcePath(), relativeSourceFile); + StorageFile targetFile = new PosixStorageFile(cfg.getBasePath(), relativeSourceFile); String uploadedFile = posixDAL.uploadFile(sourceFile.getFullPath(), targetFile.getFullPath()); @@ -672,6 +663,6 @@ public InputStream getInputStream(StorageFile storageFile) throws IOException { * @return the full bucket path */ private String getFullBucketPath() { - return bucket.equals(StorageFile.NO_BUCKET) ? basePath : Paths.get(basePath, bucket).toString(); + return bucket.equals(StorageFile.NO_BUCKET) ? cfg.getBasePath() : Paths.get(cfg.getBasePath(), bucket).toString(); } } \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageFile.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorageFile.java similarity index 94% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageFile.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorageFile.java index f1d91c688..6f4a2fa1e 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageFile.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/PosixStorageFile.java @@ -3,16 +3,16 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.posix; +package de.dlr.proseo.storagemgr.posix; import java.io.File; import org.apache.commons.io.FilenameUtils; import de.dlr.proseo.logging.logger.ProseoLogger; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * Represents a file within a POSIX-based storage system. It implements the diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/package-info.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/package-info.java similarity index 80% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/package-info.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/package-info.java index 83f6139c7..3f38761ad 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/posix/package-info.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/posix/package-info.java @@ -8,4 +8,4 @@ * * @author Denys Chaykovskiy */ -package de.dlr.proseo.storagemgr.version2.posix; \ No newline at end of file +package de.dlr.proseo.storagemgr.posix; \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImpl.java index 03e9ebfe0..673487229 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImpl.java @@ -5,57 +5,35 @@ */ package de.dlr.proseo.storagemgr.rest; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; import java.util.UUID; import javax.validation.Valid; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; -import de.dlr.proseo.storagemgr.utils.StorageType; -import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; import de.dlr.proseo.logging.logger.ProseoLogger; import de.dlr.proseo.logging.messages.StorageMgrMessage; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.rest.model.RestJoborder; -import de.dlr.proseo.storagemgr.utils.ProseoFile; -import de.dlr.proseo.storagemgr.utils.StorageManagerUtils; /** * Spring MVC controller for the prosEO Storage Manager; implements the services * required to manage Joborders * - * @author Hubert Asamer * @author Denys Chaykovskiy + * @author Hubert Asamer * */ @Component public class JobOrderControllerImpl implements JoborderController { - private static final Charset JOF_CHARSET = StandardCharsets.UTF_8; - private static final String HTTP_HEADER_WARNING = "Warning"; - private static final String HTTP_MSG_PREFIX = "4000 proseo-storage-mgr "; - private static final String MSG_EXCEPTION_THROWN = "(E%d) Exception thrown: %s"; - private static final int MSG_ID_EXCEPTION_THROWN = 9001; - - private static Logger loggerLegacy = LoggerFactory.getLogger(JoborderController.class); - /** A logger for this class */ private static ProseoLogger logger = new ProseoLogger(JobOrderControllerImpl.class); @@ -78,79 +56,29 @@ public ResponseEntity createRestJoborder(@Valid RestJoborder jobor if (logger.isTraceEnabled()) logger.trace(">>> createRestJoborder({})", (null == joborder ? "MISSING" : joborder.getMessage())); - if (storageProvider.isVersion2()) { // begin version 2 String -> StorageFile + // Storage Manager version 2: String -> StorageFile - String jobOrder64 = joborder.getJobOrderStringBase64(); + String jobOrder64 = joborder.getJobOrderStringBase64(); - if (!isStringBase64(jobOrder64)) { + if (!isStringBase64(jobOrder64)) { - String msg = logger.log(StorageMgrMessage.STRING_NOT_BASE64_ENCODED); - return new ResponseEntity<>(createBadResponse(msg, jobOrder64), HttpStatus.FORBIDDEN); - } - - String relativePath = getJobOrderRelativePath(cfg.getJoborderPrefix()); + String msg = logger.log(StorageMgrMessage.STRING_NOT_BASE64_ENCODED); + return new ResponseEntity<>(createBadResponse(msg, jobOrder64), HttpStatus.FORBIDDEN); + } - try { - StorageFile targetFile = storageProvider.createStorageFile(relativePath, jobOrder64); - logger.log(StorageMgrMessage.JOB_ORDER_FILE_UPLOADED, targetFile); - return new ResponseEntity<>(createOkResponse(targetFile, jobOrder64), HttpStatus.CREATED); + String relativePath = getJobOrderRelativePath(cfg.getJoborderPrefix()); - } catch (Exception e) { + try { + StorageFile targetFile = storageProvider.createStorageFile(relativePath, jobOrder64); + logger.log(StorageMgrMessage.JOB_ORDER_FILE_UPLOADED, targetFile); + return new ResponseEntity<>(createOkResponse(targetFile, jobOrder64), HttpStatus.CREATED); - String msg = logger.log(StorageMgrMessage.JOB_ORDER_CREATION_ERROR, jobOrder64 + " " + e.getMessage()); + } catch (Exception e) { - return new ResponseEntity<>(createBadResponse(msg, jobOrder64), HttpStatus.INTERNAL_SERVER_ERROR); - } - } // end version 2 + String msg = logger.log(StorageMgrMessage.JOB_ORDER_CREATION_ERROR, jobOrder64 + " " + e.getMessage()); - RestJoborder response = new RestJoborder(); - String separator = "/"; - try { - // check if we have a Base64 encoded string & if we have valid XML - if (!joborder.getJobOrderStringBase64() - .matches("^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)?$")) { - - response.setUploaded(false); - response.setJobOrderStringBase64(joborder.getJobOrderStringBase64()); - response.setPathInfo("n/a"); - response.setMessage("Attribute jobOrderStringBase64 is not Base64-encoded..."); - return new ResponseEntity<>(response, HttpStatus.FORBIDDEN); - } - - DateTime tstamp = DateTime.now(DateTimeZone.UTC); - String objKey = cfg.getJoborderPrefix() + separator + tstamp.getYear() + separator + tstamp.getMonthOfYear() - + separator + tstamp.getDayOfMonth() + separator + tstamp.getHourOfDay() + separator - + UUID.randomUUID().toString() + ".xml"; - - String base64String = joborder.getJobOrderStringBase64(); - byte[] bytes = java.util.Base64.getDecoder().decode(base64String); - if (!StorageManagerUtils - .checkXml(StorageManagerUtils.inputStreamToString(new ByteArrayInputStream(bytes), JOF_CHARSET))) { - response.setUploaded(false); - response.setJobOrderStringBase64(joborder.getJobOrderStringBase64()); - response.setPathInfo("n/a"); - response.setMessage("XML Doc parsed from attribute jobOrderStringBase64 is not valid..."); - return new ResponseEntity<>(response, HttpStatus.FORBIDDEN); - } - // TODO - Change to ProseoFile.fromTypeAndBucket(..., cfg.getJoborderBucket(), - // ...); - ProseoFile proFile = ProseoFile.fromType(StorageType.valueOf(joborder.getFsType()), objKey, cfg); - if (proFile != null) { - - if (proFile.writeBytes(bytes)) { - response.setFsType(proFile.getFsType().toString()); - response.setPathInfo(proFile.getFullPath()); - response.setUploaded(true); - response.setJobOrderStringBase64(joborder.getJobOrderStringBase64()); - loggerLegacy.info("Received & Uploaded joborder-file: {}", response.getPathInfo()); - return new ResponseEntity<>(response, HttpStatus.CREATED); - } - } - } catch (Exception e) { - return new ResponseEntity<>(errorHeaders(MSG_EXCEPTION_THROWN, MSG_ID_EXCEPTION_THROWN, - e.getClass().toString() + ": " + e.getMessage()), HttpStatus.INTERNAL_SERVER_ERROR); + return new ResponseEntity<>(createBadResponse(msg, jobOrder64), HttpStatus.INTERNAL_SERVER_ERROR); } - return new ResponseEntity<>(response, HttpStatus.NOT_IMPLEMENTED); } /** @@ -165,67 +93,34 @@ public ResponseEntity getObjectByPathInfo(String pathInfo) { if (logger.isTraceEnabled()) logger.trace(">>> getObjectByPathInfo({})", pathInfo); - if (storageProvider.isVersion2()) { // begin version 2 StorageFile -> String - - if (null == pathInfo) { - return new ResponseEntity<>(logger.log(StorageMgrMessage.PATH_IS_NULL), HttpStatus.NOT_FOUND); - } - - if (pathInfo == "") { - return new ResponseEntity<>(logger.log(StorageMgrMessage.INVALID_PATH, pathInfo), HttpStatus.NOT_FOUND); - } + // Storage Manager version 2: StorageFile -> String - try { - - String relativePath = storageProvider.getRelativePath(pathInfo); - StorageFile storageFile = storageProvider.getStorageFile(relativePath); - - String response = storageProvider.getStorage().getFileContent(storageFile); + if (null == pathInfo) { + return new ResponseEntity<>(logger.log(StorageMgrMessage.PATH_IS_NULL), HttpStatus.NOT_FOUND); + } - logger.log(StorageMgrMessage.JOB_ORDER_FILE_GOT, pathInfo); + if (pathInfo == "") { + return new ResponseEntity<>(logger.log(StorageMgrMessage.INVALID_PATH, pathInfo), HttpStatus.NOT_FOUND); + } - return new ResponseEntity<>(response, HttpStatus.OK); + try { - } catch (Exception e) { + String relativePath = storageProvider.getRelativePath(pathInfo); + StorageFile storageFile = storageProvider.getStorageFile(relativePath); - String msg = logger.log(StorageMgrMessage.JOB_ORDER_FILE_CANNOT_BE_GOT, pathInfo, e.getMessage()); - return new ResponseEntity<>(msg, HttpStatus.INTERNAL_SERVER_ERROR); - } + String response = storageProvider.getStorage().getFileContent(storageFile); - } // end version 2 + logger.log(StorageMgrMessage.JOB_ORDER_FILE_GOT, pathInfo); - String response = ""; - if (pathInfo != null) { - ProseoFile proFile = ProseoFile.fromPathInfo(pathInfo, cfg); - // Find storage type - if (proFile == null || proFile.getFsType() == StorageType.ALLUXIO) { - loggerLegacy.warn("Invalid storage type for path: {}", pathInfo); - return new ResponseEntity<>(response, HttpStatus.NOT_FOUND); - } + return new ResponseEntity<>(response, HttpStatus.OK); - InputStream jofStream = proFile.getDataAsInputStream(); - if (jofStream != null) { - byte[] bytes = null; + } catch (Exception e) { - try { - bytes = java.util.Base64.getEncoder().encode(jofStream.readAllBytes()); - } catch (IOException e) { - loggerLegacy.error("Invalid job order stream"); - return new ResponseEntity<>(errorHeaders(MSG_EXCEPTION_THROWN, MSG_ID_EXCEPTION_THROWN, - e.getClass().toString() + ": " + e.getMessage()), HttpStatus.INTERNAL_SERVER_ERROR); - } - response = new String(bytes); - try { - jofStream.close(); - } catch (IOException e) { - loggerLegacy.warn("Failed to close input stream of " + pathInfo + " | " + e.getMessage()); - } - return new ResponseEntity<>(response, HttpStatus.OK); - } + String msg = logger.log(StorageMgrMessage.JOB_ORDER_FILE_CANNOT_BE_GOT, pathInfo, e.getMessage()); + return new ResponseEntity<>(msg, HttpStatus.INTERNAL_SERVER_ERROR); } - return new ResponseEntity<>(response, HttpStatus.NOT_FOUND); } - + /** * Checks if string base64 * @@ -293,29 +188,4 @@ private RestJoborder createBadResponse(String message, String stringBase64) { return response; } - /** - * Log an error and return the corresponding HTTP message header - * - * @param messageFormat the message text with parameter placeholders in - * String.format() style - * @param messageId a (unique) message id - * @param messageParameters the message parameters (optional, depending on the - * message format) - * @return an HttpHeaders object with a formatted error message - */ - private HttpHeaders errorHeaders(String messageFormat, int messageId, Object... messageParameters) { - - // Prepend message ID to parameter list - List messageParamList = new ArrayList<>(Arrays.asList(messageParameters)); - messageParamList.add(0, messageId); - - // Log the error message - String message = String.format(messageFormat, messageParamList.toArray()); - loggerLegacy.error(message); - - // Create an HTTP "Warning" header - HttpHeaders responseHeaders = new HttpHeaders(); - responseHeaders.set(HTTP_HEADER_WARNING, HTTP_MSG_PREFIX + message); - return responseHeaders; - } } diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java index c70adbc74..58507f6fa 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java @@ -12,14 +12,11 @@ import java.net.UnknownHostException; import java.text.ParseException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Date; import java.util.List; import javax.validation.Valid; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.InputStreamResource; import org.springframework.http.HttpHeaders; @@ -29,88 +26,37 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; -import com.amazonaws.services.s3.Headers; import com.nimbusds.jose.JOSEException; import com.nimbusds.jose.JWSVerifier; import com.nimbusds.jose.crypto.MACVerifier; import com.nimbusds.jwt.JWTClaimsSet; import com.nimbusds.jwt.SignedJWT; -import de.dlr.proseo.storagemgr.utils.StorageType; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; import de.dlr.proseo.logging.http.HttpPrefix; import de.dlr.proseo.logging.http.ProseoHttp; import de.dlr.proseo.logging.logger.ProseoLogger; import de.dlr.proseo.logging.messages.StorageMgrMessage; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestProductFS; -import de.dlr.proseo.storagemgr.utils.ProseoFile; -import de.dlr.proseo.storagemgr.utils.StorageLogger; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * Spring MVC controller for the prosEO Storage Manager; implements the services * required to manage any object storage, e. g. a storage based on the AWS S3 * API * - * @author Hubert Asamer * @author Dr. Thomas Bassler * @author Denys Chaykovskiy + * @author Hubert Asamer * */ @Component public class ProductControllerImpl implements ProductController { - /* Message IDs */ - private static final int MSG_ID_EXCEPTION_THROWN = 4001; - private static final int MSG_ID_FILE_NOT_FOUND = 4002; - private static final int MSG_ID_INVALID_PATH = 4003; - // private static final int MSG_ID_TOKEN_MISSING = 4004; - // private static final int MSG_ID_TOKEN_INVALID = 4005; - // private static final int MSG_ID_TOKEN_EXPIRED = 4006; - private static final int MSG_ID_TOKEN_MISMATCH = 4007; - private static final int MSG_ID_FILES_REGISTERED = 4008; - private static final int MSG_ID_FILES_LISTED = 4009; - private static final int MSG_ID_FILE_RETRIEVED = 4010; - private static final int MSG_ID_FILE_DELETED = 4011; - - /* Message strings */ - private static final String MSG_EXCEPTION_THROWN = "(E%d) Exception thrown: %s"; - private static final String MSG_FILE_NOT_FOUND = "(E%d) File %s not found"; - private static final String MSG_INVALID_PATH = "(E%d) Invalid path %s"; - // private static final String MSG_TOKEN_MISSING = "(E%d) Authentication token - // missing"; - // private static final String MSG_TOKEN_INVALID = "(E%d) Authentication token - // %s invalid (cause: %s)"; - // private static final String MSG_TOKEN_EXPIRED = "(E%d) Authentication token - // expired at %s"; - private static final String MSG_TOKEN_MISMATCH = "(E%d) Authentication token not valid for file %s"; - private static final String MSG_FILES_REGISTERED = "(I%d) Files registered: %s"; - private static final String MSG_FILES_LISTED = "(I%d) Files listed: %s"; - private static final String MSG_FILE_RETRIEVED = "(I%d) File %s retrieved from byte %d to byte %d (%d bytes transferred)"; - private static final String MSG_FILE_DELETED = "(I%d) File %s deleted"; - - /* Submessages for token evaluation */ - // private static final String MSG_TOKEN_PAYLOAD_INVALID = "The payload of the - // JWT doesn't represent a valid JSON object and a JWT claims set"; - // private static final String MSG_TOKEN_NOT_VERIFIABLE = "The JWS object - // couldn't be verified"; - // private static final String MSG_TOKEN_STATE_INVALID = "The JWS object is not - // in a signed or verified state, actual state: "; - // private static final String MSG_TOKEN_VERIFICATION_FAILED = "Verification of - // the JWT failed"; - // private static final String MSG_SECRET_TOO_SHORT = "Secret length is shorter - // than the minimum 256-bit requirement"; - // private static final String MSG_TOKEN_NOT_PARSEABLE = "Token not parseable"; - - private static final String HTTP_HEADER_WARNING = "Warning"; - private static final String HTTP_MSG_PREFIX = "199 proseo-storage-mgr "; - - /** Logger for this class */ - private static Logger loggerLegacy = LoggerFactory.getLogger(ProductControllerImpl.class); - /** A logger for this class */ private static ProseoLogger logger = new ProseoLogger(ProductControllerImpl.class); private static ProseoHttp http = new ProseoHttp(logger, HttpPrefix.STORAGE_MGR); @@ -138,86 +84,39 @@ public ResponseEntity createRestProductFS(@Valid RestProductFS re logger.trace(">>> createRestProductFs({})", (null == restProductFS ? "MISSING" : restProductFS.getProductId())); - if (storageProvider.isVersion2()) { // begin version 2 list upload source -> storage - - try { - String hostName = getLocalHostName(); - String prefix = new PathConverter(restProductFS.getProductId()).addSlashAtEnd().getPath(); - List allUploaded = new ArrayList(); - - StorageFile targetFolder = storageProvider.getStorageFile(prefix); - - for (String fileOrDir : restProductFS.getSourceFilePaths()) { - - StorageFile sourceFileOrDir = storageProvider.getAbsoluteFile(fileOrDir); - List uploaded = storageProvider.getStorage().upload(sourceFileOrDir, targetFolder); - - if (uploaded != null) - allUploaded.addAll(uploaded); - } - - allUploaded = storageProvider.getStorage().getAbsolutePath(allUploaded); + // Storage Manager version 2: list upload source -> storage - RestProductFS response = setRestProductFS(restProductFS, targetFolder.getBasePath(), true, - targetFolder.getFullPath() + "/", allUploaded, false, - "registration executed on node " + hostName); - - logger.log(StorageMgrMessage.PRODUCTS_UPLOADED_TO_STORAGE, Integer.toString(allUploaded.size()), - allUploaded.toString()); + try { + String hostName = getLocalHostName(); + String prefix = new PathConverter(restProductFS.getProductId()).addSlashAtEnd().getPath(); + List allUploaded = new ArrayList(); - return new ResponseEntity<>(response, HttpStatus.CREATED); + StorageFile targetFolder = storageProvider.getStorageFile(prefix); - } catch (Exception e) { + for (String fileOrDir : restProductFS.getSourceFilePaths()) { - String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); + StorageFile sourceFileOrDir = storageProvider.getAbsoluteFile(fileOrDir); + List uploaded = storageProvider.getStorage().upload(sourceFileOrDir, targetFolder); - return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); + if (uploaded != null) + allUploaded.addAll(uploaded); } - } // end version 2 - - // get node name info... - String hostName = null; - try { - InetAddress iAddress = InetAddress.getLocalHost(); - hostName = iAddress.getHostName(); - } catch (UnknownHostException e1) { - hostName = "(UNKNOWN)"; - } - - RestProductFS response = new RestProductFS(); + allUploaded = storageProvider.getStorage().getAbsolutePath(allUploaded); - loggerLegacy.info(restProductFS.toString()); - String pref = restProductFS.getProductId(); - if (!pref.endsWith("/")) { - pref = pref + "/"; - } + RestProductFS response = setRestProductFS(restProductFS, targetFolder.getBasePath(), true, + targetFolder.getFullPath() + "/", allUploaded, false, "registration executed on node " + hostName); - ArrayList transferSum = new ArrayList(); + logger.log(StorageMgrMessage.PRODUCTS_UPLOADED_TO_STORAGE, Integer.toString(allUploaded.size()), + allUploaded.toString()); - ProseoFile targetFile = ProseoFile.fromType(StorageType.valueOf(restProductFS.getTargetStorageType()), pref, - cfg); + return new ResponseEntity<>(response, HttpStatus.CREATED); - try { - for (String fileOrDir : restProductFS.getSourceFilePaths()) { - ProseoFile sourceFile = ProseoFile - .fromTypeFullPath(StorageType.valueOf(restProductFS.getSourceStorageType()), fileOrDir, cfg); - ArrayList transfered = sourceFile.copyTo(targetFile, true); - if (loggerLegacy.isDebugEnabled()) - loggerLegacy.debug("Files transferred: {}", transfered); - if (transfered != null) { - transferSum.addAll(transfered); - } - } - setRestProductFS(response, restProductFS, targetFile.getBasePath(), true, targetFile.getFullPath() + "/", - transferSum, false, "registration executed on node " + hostName); + } catch (Exception e) { - StorageLogger.logInfo(loggerLegacy, MSG_FILES_REGISTERED, MSG_ID_FILES_REGISTERED, transferSum.toString()); + String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); - return new ResponseEntity<>(response, HttpStatus.CREATED); - } catch (Exception e) { - return new ResponseEntity<>(errorHeaders(MSG_EXCEPTION_THROWN, MSG_ID_EXCEPTION_THROWN, - e.getClass().toString() + ": " + e.getMessage()), HttpStatus.INTERNAL_SERVER_ERROR); + return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); } } @@ -236,71 +135,38 @@ public ResponseEntity> getProductFiles(String storageType, String p if (logger.isTraceEnabled()) logger.trace(">>> getProductFiles({}, {})", storageType, prefix); - if (storageProvider.isVersion2()) { // begin version 2 - get product files + // Storage Manager version 2: get product files - if (prefix == null) - prefix = ""; + if (prefix == null) + prefix = ""; - try { - List response; - - if (storageType == null) { // add both - - response = storageProvider.getStorage(de.dlr.proseo.storagemgr.version2.model.StorageType.S3) - .getAbsoluteFiles(prefix); - response = storageProvider.getStorage(de.dlr.proseo.storagemgr.version2.model.StorageType.S3) - .addFSPrefix(response); - - response.addAll( - storageProvider.getStorage(de.dlr.proseo.storagemgr.version2.model.StorageType.POSIX) - .getAbsoluteFiles(prefix)); - response = storageProvider.getStorage(de.dlr.proseo.storagemgr.version2.model.StorageType.POSIX) - .addFSPrefix(response); + try { + List response; - } else { + if (storageType == null) { // add both - response = storageProvider - .getStorage(de.dlr.proseo.storagemgr.version2.model.StorageType.valueOf(storageType)) - .getAbsoluteFiles(prefix); - response = storageProvider - .getStorage(de.dlr.proseo.storagemgr.version2.model.StorageType.valueOf(storageType)) - .addFSPrefix(response); - } + response = storageProvider.getStorage(StorageType.S3).getAbsoluteFiles(prefix); + response = storageProvider.getStorage(StorageType.S3).addFSPrefix(response); + response.addAll(storageProvider.getStorage(StorageType.POSIX).getAbsoluteFiles(prefix)); + response = storageProvider.getStorage(StorageType.POSIX).addFSPrefix(response); - logger.log(StorageMgrMessage.PRODUCT_FILES_LISTED, response.toString()); + } else { - return new ResponseEntity<>(response, HttpStatus.OK); + response = storageProvider.getStorage(StorageType.valueOf(storageType)).getAbsoluteFiles(prefix); + response = storageProvider.getStorage(StorageType.valueOf(storageType)).addFSPrefix(response); + } - } catch (Exception e) { + logger.log(StorageMgrMessage.PRODUCT_FILES_LISTED, response.toString()); - String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); + return new ResponseEntity<>(response, HttpStatus.OK); - return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); - } + } catch (Exception e) { - } // end version 2 + String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); - List stl = new ArrayList(); - List response = new ArrayList(); - try { - if (storageType == null) { - stl.add(StorageType.S3); - stl.add(StorageType.POSIX); - } else { - stl.add(StorageType.valueOf(storageType)); - } - for (StorageType st : stl) { - listProductFiles(st, prefix, response); - } - } catch (Exception e) { - e.printStackTrace(); - return new ResponseEntity<>(errorHeaders(MSG_EXCEPTION_THROWN, MSG_ID_EXCEPTION_THROWN, - e.getClass().toString() + ": " + e.getMessage()), HttpStatus.INTERNAL_SERVER_ERROR); + return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); } - StorageLogger.logInfo(loggerLegacy, MSG_FILES_LISTED, MSG_ID_FILES_LISTED, response.toString()); - - return new ResponseEntity<>(response, HttpStatus.OK); } /** @@ -354,122 +220,58 @@ public ResponseEntity getObject(String pathInfo, String token, Long fromByte, } // token check end - if (storageProvider.isVersion2()) { // begin version 2 - storage file -> byte page - - try { - Storage storage = storageProvider.getStorage(pathInfo); - - String relativePath = storage.getRelativePath(pathInfo); - - StorageFile sourceFile = storage.getStorageFile(relativePath); - - if (sourceFile == null) { + // Storage Manager version 2: storage file -> byte page - String msg = logger.log(StorageMgrMessage.PATH_IS_NULL); - return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.BAD_REQUEST); - } - - // token check begin - if (!sourceFile.getFileName().equals(claimsSet.getSubject())) { - - String msg = logger.log(StorageMgrMessage.TOKEN_MISMATCH, sourceFile.getFileName()); - return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.UNAUTHORIZED); - } - // token check end - - InputStream stream = storage.getInputStream(sourceFile); - if (stream == null) { - - String msg = logger.log(StorageMgrMessage.FILE_NOT_FOUND, pathInfo); - return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.NOT_FOUND); - } - - HttpHeaders headers = getFilePage(sourceFile, stream, fromByte, toByte); - HttpStatus status = getOkOrPartialStatus(fromByte, toByte); - - InputStreamResource fsr = new InputStreamResource(stream); - - if (null == fromByte || null == toByte) { - logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED, pathInfo); - } else { - logger.log(StorageMgrMessage.PRODUCT_FILE_PARTIALLY_DOWNLOADED, pathInfo, Long.toString(fromByte), - Long.toString(toByte), Long.toString(toByte - fromByte)); - } - return new ResponseEntity<>(fsr, headers, status); - - } catch (Exception e) { - - if (logger.isTraceEnabled()) logger.trace("... exception thrown: ", e); - - String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); - return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); - } + try { + Storage storage = storageProvider.getStorage(pathInfo); - } // end version 2 + String relativePath = storage.getRelativePath(pathInfo); - // Download file + StorageFile sourceFile = storage.getStorageFile(relativePath); - try { - ProseoFile sourceFile = ProseoFile.fromPathInfo(pathInfo, cfg); if (sourceFile == null) { - return new ResponseEntity<>(errorHeaders(MSG_INVALID_PATH, MSG_ID_INVALID_PATH, pathInfo), - HttpStatus.BAD_REQUEST); + + String msg = logger.log(StorageMgrMessage.PATH_IS_NULL); + return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.BAD_REQUEST); } // token check begin if (!sourceFile.getFileName().equals(claimsSet.getSubject())) { - return new ResponseEntity<>( - errorHeaders(MSG_TOKEN_MISMATCH, MSG_ID_TOKEN_MISMATCH, sourceFile.getFileName()), - HttpStatus.UNAUTHORIZED); + + String msg = logger.log(StorageMgrMessage.TOKEN_MISMATCH, sourceFile.getFileName()); + return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.UNAUTHORIZED); } // token check end - InputStream stream = sourceFile.getDataAsInputStream(); + InputStream stream = storage.getInputStream(sourceFile); if (stream == null) { - return new ResponseEntity<>(errorHeaders(MSG_FILE_NOT_FOUND, MSG_ID_FILE_NOT_FOUND, pathInfo), - HttpStatus.NOT_FOUND); - } - Long fileSize = sourceFile.getLength(); - HttpHeaders headers = new HttpHeaders(); - headers.setContentDispositionFormData("attachment", sourceFile.getFileName()); - Long len = fileSize; - long from = 0; - long to = len - 1; - HttpStatus status = HttpStatus.OK; - if (fromByte != null || toByte != null) { - if (fromByte != null) { - from = fromByte; - stream.skip(from); - } - if (toByte != null) { - to = Math.min(toByte, len - 1); - } - len = to - from + 1; - headers.add(Headers.CONTENT_RANGE, String.format("bytes %d-%d/%d", from, to, fileSize)); - status = HttpStatus.PARTIAL_CONTENT; + + String msg = logger.log(StorageMgrMessage.FILE_NOT_FOUND, pathInfo); + return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.NOT_FOUND); } - // TODO The media type is not fully correct: It's OK for "application/zip", but - // e.g. for NetCDF (.nc) "application/netcdf" would be more appropriate - // headers.setContentType(new MediaType("application", - // sourceFile.getExtension())); - // PRIP always returns "application/octet-stream" in the metadata, so probably - // it's best to stay consistent here - headers.setContentType(MediaType.APPLICATION_OCTET_STREAM); - headers.setContentLength(len); + + HttpHeaders headers = getFilePage(sourceFile, stream, fromByte, toByte); + HttpStatus status = getOkOrPartialStatus(fromByte, toByte); + InputStreamResource fsr = new InputStreamResource(stream); - if (fsr != null) { - StorageLogger.logInfo(loggerLegacy, MSG_FILE_RETRIEVED, MSG_ID_FILE_RETRIEVED, pathInfo, from, to, len); - return new ResponseEntity<>(fsr, headers, status); + if (null == fromByte || null == toByte) { + logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED_FROM_STORAGE, pathInfo); + } else { + logger.log(StorageMgrMessage.PRODUCT_FILE_PARTIALLY_DOWNLOADED, pathInfo, Long.toString(fromByte), + Long.toString(toByte), Long.toString(toByte - fromByte)); } + return new ResponseEntity<>(fsr, headers, status); + } catch (Exception e) { - e.printStackTrace(); - return new ResponseEntity<>(errorHeaders(MSG_EXCEPTION_THROWN, MSG_ID_EXCEPTION_THROWN, - e.getClass().toString() + ": " + e.getMessage()), HttpStatus.INTERNAL_SERVER_ERROR); + + if (logger.isTraceEnabled()) + logger.trace("... exception thrown: ", e); + + String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); + return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); } - return new ResponseEntity<>(errorHeaders(MSG_FILE_NOT_FOUND, MSG_ID_FILE_NOT_FOUND, pathInfo), - HttpStatus.NOT_FOUND); } /** @@ -487,60 +289,34 @@ public ResponseEntity deleteProductByPathInfo(String pathInfo) { if (logger.isTraceEnabled()) logger.trace(">>> deleteProductByPathInfo({})", pathInfo); - if (storageProvider.isVersion2()) { // begin version 2 - delete files in storage - - if (null == pathInfo) { - String msg = logger.log(StorageMgrMessage.PATH_IS_NULL); - return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.NOT_FOUND); - } - - if (pathInfo == "") { - String msg = logger.log(StorageMgrMessage.INVALID_PATH, pathInfo); - return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.NOT_FOUND); - } + // Storage Manager version 2: delete files in storage - try { - String storageType = storageProvider.getStorage().getStorageType().toString(); + if (null == pathInfo) { + String msg = logger.log(StorageMgrMessage.PATH_IS_NULL); + return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.NOT_FOUND); + } - String relativePath = storageProvider.getRelativePath(pathInfo); - List deletedFilesOrDir = storageProvider.getStorage().delete(relativePath); - RestProductFS response = createRestProductFilesDeleted(deletedFilesOrDir, storageType); + if (pathInfo == "") { + String msg = logger.log(StorageMgrMessage.INVALID_PATH, pathInfo); + return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.NOT_FOUND); + } - logger.log(StorageMgrMessage.PRODUCT_FILE_DELETED, pathInfo); + try { + String storageType = storageProvider.getStorage().getStorageType().toString(); - return new ResponseEntity<>(response, HttpStatus.OK); + String relativePath = storageProvider.getRelativePath(pathInfo); + List deletedFilesOrDir = storageProvider.getStorage().delete(relativePath); + RestProductFS response = createRestProductFilesDeleted(deletedFilesOrDir, storageType); - } catch (Exception e) { + logger.log(StorageMgrMessage.PRODUCT_FILE_DELETED, pathInfo); - String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); - return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); - } + return new ResponseEntity<>(response, HttpStatus.OK); - } // end version 2 + } catch (Exception e) { - RestProductFS response = new RestProductFS(); - if (pathInfo != null) { - ProseoFile sourceFile = ProseoFile.fromPathInfo(pathInfo, cfg); - try { - ArrayList deleted = sourceFile.delete(); - if (deleted != null && !deleted.isEmpty()) { - response.setProductId(""); - response.setDeleted(true); - response.setRegistered(false); - response.setSourceFilePaths(deleted); - response.setSourceStorageType(sourceFile.getFsType().toString()); - - StorageLogger.logInfo(loggerLegacy, MSG_FILE_DELETED, MSG_ID_FILE_DELETED, pathInfo); - - return new ResponseEntity<>(response, HttpStatus.OK); - } - } catch (Exception e) { - return new ResponseEntity<>(errorHeaders(MSG_EXCEPTION_THROWN, MSG_ID_EXCEPTION_THROWN, - e.getClass().toString() + ": " + e.getMessage()), HttpStatus.INTERNAL_SERVER_ERROR); - } + String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); + return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); } - return new ResponseEntity<>(errorHeaders(MSG_FILE_NOT_FOUND, MSG_ID_FILE_NOT_FOUND, pathInfo), - HttpStatus.NOT_FOUND); } /** @@ -582,7 +358,7 @@ private HttpHeaders getFilePage(StorageFile sourceFile, InputStream stream, Long Storage storage = storageProvider.getStorage(sourceFile.getFullPath()); Long len = storage.getFileSize(sourceFile); - + HttpHeaders headers = new HttpHeaders(); headers.setContentDispositionFormData("attachment", sourceFile.getFileName()); long from = 0; @@ -615,48 +391,6 @@ private HttpHeaders getFilePage(StorageFile sourceFile, InputStream stream, Long return headers; } - /** - * Set the members of RestProductFS response. - * - * @param response the ingest info structure to update - * @param restProductFS the ingest info structure to copy product ID and - * source information from - * @param storageId the ID of the storage used - * @param registered true, if the requested files have been ingested, - * false otherwise - * @param registeredFilePath common path to the ingested files - * @param registeredFiles file names after ingestion - * @param deleted true, if the files were deleted, false otherwise - * @param msg a response message text - * @return the updated response object - */ - private RestProductFS setRestProductFS(RestProductFS response, RestProductFS restProductFS, String storageId, - Boolean registered, String registeredFilePath, List registeredFiles, Boolean deleted, String msg) { - - if (logger.isTraceEnabled()) - logger.trace(">>> setRestProductFS({}, {}, {}, {}, {}, {}, {}, {})", - (null == response ? "MISSING" : response.getProductId()), - (null == restProductFS ? "MISSING" : restProductFS.getProductId()), storageId, registered, - registeredFilePath, registeredFiles.size(), deleted, msg); - - if (response != null && restProductFS != null) { - response.setProductId(restProductFS.getProductId()); - response.setTargetStorageId(storageId); - response.setRegistered(registered); - response.setRegisteredFilePath(registeredFilePath); - response.setSourceFilePaths(restProductFS.getSourceFilePaths()); - response.setSourceStorageType(restProductFS.getSourceStorageType()); - response.setTargetStorageType(restProductFS.getTargetStorageType()); - response.setRegisteredFilesCount(Long.valueOf(registeredFiles.size())); - response.setRegisteredFilesList(registeredFiles); - response.setDeleted(deleted); - response.setMessage(msg); - } - if (logger.isDebugEnabled()) - logger.debug("Response created: {}", response); - return response; - } - /** * Check the given token for formal correctness and extract its JWT claims set * @@ -739,31 +473,6 @@ private RestProductFS createRestProductFilesDeleted(List deletedFiles, S return response; } - /** - * List file objects of repository. Collect result in response. - * - * @param st the storage type - * @param prefix relative path to list - * @param response the ingest information response to fill - */ - private void listProductFiles(StorageType st, String prefix, List response) { - - if (logger.isTraceEnabled()) - logger.trace(">>> listProductFiles({}, {})", st, prefix, response.size()); - - ProseoFile path = null; - if (prefix == null) { - path = ProseoFile.fromType(st, "", cfg); - } else { - path = ProseoFile.fromType(st, prefix + "/", cfg); - } - List files = path.list(); - for (ProseoFile f : files) { - String fs = f.getFsType().toString() + "|" + f.getFullPath(); - response.add(fs); - } - } - /** * Set the members of RestProductFS response. * @@ -823,33 +532,4 @@ private String getLocalHostName() { return "(UNKNOWN)"; } } - - /** - * Log an error and return the corresponding HTTP message header - * - * @param messageFormat the message text with parameter placeholders in - * String.format() style - * @param messageId a (unique) message id - * @param messageParameters the message parameters (optional, depending on the - * message format) - * @return an HttpHeaders object with a formatted error message - */ - private HttpHeaders errorHeaders(String messageFormat, int messageId, Object... messageParameters) { - - if (loggerLegacy.isTraceEnabled()) - loggerLegacy.trace(">>> errorHeaders({}, {}, {})", messageFormat, messageId, "messageParameters"); - - // Prepend message ID to parameter list - List messageParamList = new ArrayList<>(Arrays.asList(messageParameters)); - messageParamList.add(0, messageId); - - // Log the error message - String message = String.format(messageFormat, messageParamList.toArray()); - loggerLegacy.error(message); - - // Create an HTTP "Warning" header - HttpHeaders responseHeaders = new HttpHeaders(); - responseHeaders.set(HTTP_HEADER_WARNING, HTTP_MSG_PREFIX + message); - return responseHeaders; - } } diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index c75d0a41d..21d82268a 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -2,18 +2,10 @@ import java.io.File; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; + import java.nio.file.Paths; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.concurrent.ConcurrentSkipListSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; @@ -22,17 +14,16 @@ import de.dlr.proseo.logging.http.ProseoHttp; import de.dlr.proseo.logging.logger.ProseoLogger; import de.dlr.proseo.logging.messages.StorageMgrMessage; + import de.dlr.proseo.storagemgr.StorageManagerConfiguration; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.Exceptions.FileLockedAfterMaxCyclesException; +import de.dlr.proseo.storagemgr.cache.CacheFileStatus; import de.dlr.proseo.storagemgr.cache.FileCache; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; -import de.dlr.proseo.storagemgr.utils.StorageType; -import de.dlr.proseo.storagemgr.version2.StorageFileLocker; -import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.Exceptions.FileLockedAfterMaxCyclesException; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.utils.ProseoFile; -import de.dlr.proseo.storagemgr.utils.StorageLogger; +import de.dlr.proseo.storagemgr.utils.StorageFileLocker; /** * Spring MVC controller for the prosEO Storage Manager; implements the services @@ -46,32 +37,6 @@ @Component public class ProductfileControllerImpl implements ProductfileController { - private static final String HTTP_HEADER_WARNING = "Warning"; - private static final String HTTP_MSG_PREFIX = "199 proseo-storage-mgr "; - - private static final String MSG_EXCEPTION_THROWN = "(E%d) Exception thrown: %s"; - private static final String MSG_FILE_NOT_FOUND = "(E%d) File %s not found"; - private static final String MSG_FILE_COPIED = "(I%d) Requested object %s copied to target path %s"; - private static final String MSG_TARGET_PATH_MISSING = "(E%d) No target path given"; - private static final String MSG_FILES_UPDATED = "(I%d) Product file %s uploaded for product ID %d"; - private static final String MSG_READ_TIMEOUT = "(E%d) Read for file %s timed out after %d seconds"; - - private static final int MSG_ID_EXCEPTION_THROWN = 4051; - private static final int MSG_ID_FILE_COPIED = 4052; - private static final int MSG_ID_FILE_NOT_FOUND = 4053; - private static final int MSG_ID_FILES_UPDATED = 4054; - private static final int MSG_ID_READ_TIMEOUT = 4055; - - // Same as in ProseFileS3 - private static final int MSG_ID_TARGET_PATH_MISSING = 4100; - - private static final String MSG_FILE_NOT_FETCHED = "Requested file {} not copied"; - - // Lock table for products currently being downloaded from backend storage - private static ConcurrentSkipListSet productLockSet = new ConcurrentSkipListSet<>(); - - private static Logger loggerLegacy = LoggerFactory.getLogger(ProductfileControllerImpl.class); - /** A logger for this class */ private static ProseoLogger logger = new ProseoLogger(ProductControllerImpl.class); private static ProseoHttp http = new ProseoHttp(logger, HttpPrefix.STORAGE_MGR); @@ -105,262 +70,286 @@ public ResponseEntity getRestFileInfoByPathInfo(String pathInfo) { return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.BAD_REQUEST); } - // pathInfo is absolute path s3://bucket/.. or /storagePath/.. DOWNLOAD Storage - // -> Cache - if (storageProvider.isVersion2()) { - - StorageFileLocker fileLocker = new StorageFileLocker(pathInfo, cfg.getFileCheckWaitTime(), - cfg.getFileCheckMaxCycles()); + // 1. copy Storage -> Cache + // 2. add to cache cache.put(cache file) + // pathInfo is absolute path s3://bucket/.. or /storagePath/.. - try { - // relative path depends on path, not on actual storage - String relativePath = storageProvider.getRelativePath(pathInfo); + String absoluteStoragePath = pathInfo; - StorageFile sourceFile = storageProvider.getStorageFile(relativePath); - StorageFile targetFile = storageProvider.getCacheFile(sourceFile.getRelativePath()); + StorageFile cacheFile; + try { - FileCache cache = FileCache.getInstance(); + String relativePath = storageProvider.getRelativePath(absoluteStoragePath); + cacheFile = storageProvider.getCacheFile(relativePath); - if (!cache.containsKey(targetFile.getFullPath())) { + } catch (IOException e) { - fileLocker.lock(); - - // After lock the active thread downloaded the file and put it to the cache (see below) - // After lock the passive thread did nothing, but the file has been downloaded - // and the cache has been updated - need to check if file contains in the cache again - if (!cache.containsKey(targetFile.getFullPath())) { + String msg = logger.log(StorageMgrMessage.PRODUCT_FILE_CANNOT_BE_DOWNLOADED, e.getMessage()); + return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.BAD_REQUEST); + } - // download-thread - storageProvider.getStorage().downloadFile(sourceFile, targetFile); - logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED, targetFile.getFullPath()); - - cache.put(targetFile.getFullPath()); - } - else { - - // waiting-thread when the file downloaded and use it from cache - logger.debug("... waiting-thread when the file downloaded and use it from cache: ", targetFile.getFullPath()); - } - - } else { - logger.debug("... no download and no lock - the file is in cache: ", targetFile.getFullPath()); - } + StorageFileLocker fileLocker = new StorageFileLocker(cacheFile.getFullPath(), cfg.getFileCheckWaitTime(), + cfg.getFileCheckMaxCycles()); - RestFileInfo restFileInfo = convertToRestFileInfo(targetFile, - storageProvider.getCacheFileSize(sourceFile.getRelativePath())); + try { - return new ResponseEntity<>(restFileInfo, HttpStatus.OK); + RestFileInfo restFileInfo = copyFileStorageToCache(absoluteStoragePath, fileLocker); + return new ResponseEntity<>(restFileInfo, HttpStatus.OK); - } catch (FileLockedAfterMaxCyclesException e) { + } catch (FileLockedAfterMaxCyclesException e) { - String time = String.valueOf(cfg.getFileCheckMaxCycles() * cfg.getFileCheckWaitTime() / 1000); - String msg = logger.log(StorageMgrMessage.READ_TIME_OUT, pathInfo, time, e.getLocalizedMessage()); + String time = String.valueOf(cfg.getFileCheckMaxCycles() * cfg.getFileCheckWaitTime() / 1000); + String msg = logger.log(StorageMgrMessage.READ_TIME_OUT, absoluteStoragePath, time, + e.getLocalizedMessage()); + return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.SERVICE_UNAVAILABLE); - return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.SERVICE_UNAVAILABLE); + } catch (IOException e) { - } catch (IOException e) { + String msg = logger.log(StorageMgrMessage.PRODUCT_FILE_CANNOT_BE_DOWNLOADED, e.getMessage()); + return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.BAD_REQUEST); - String msg = logger.log(StorageMgrMessage.PRODUCT_FILE_CANNOT_BE_DOWNLOADED, e.getMessage()); - return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.BAD_REQUEST); + } catch (Exception e) { - } catch (Exception e) { + String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); + return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); - String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); - return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); - } + } finally { - finally { + fileLocker.unlock(); + logger.debug("... unlocked the file: ", pathInfo); + } + } - fileLocker.unlock(); - logger.debug("... unlocked the file: ", pathInfo); + /** + * Copy local file named pathInfo to storage manager. The target file path is: + * default mount point + productId + relative source file path + * + * @param pathInfo Source file name + * @param productId Product id + * @return Target file name + */ + @Override + public ResponseEntity updateProductfiles(String pathInfo, Long productId, Long fileSize) { - } - } // end version 2 + if (logger.isTraceEnabled()) + logger.trace(">>> updateProductfiles({}, {})", pathInfo, productId); - if (null == pathInfo || pathInfo.isBlank()) + // copies absolute external file -> cache file -> storage file + // 1. copy external -> cache + // 2. add to cache cache.put(cache file) + // 3. copy cache -> storage + // pathInfo is absolute external path - { - return new ResponseEntity<>( - errorHeaders( - StorageLogger.logError(loggerLegacy, MSG_TARGET_PATH_MISSING, MSG_ID_TARGET_PATH_MISSING)), - HttpStatus.BAD_REQUEST); + if (pathInfo == null) { + return new ResponseEntity(new RestFileInfo(), HttpStatus.BAD_REQUEST); } - ProseoFile sourceFile = ProseoFile.fromPathInfo(pathInfo, cfg); - ProseoFile targetFile = ProseoFile.fromPathInfo(cfg.getPosixCachePath() + "/" + sourceFile.getRelPathAndFile(), - cfg); + String externalPath = pathInfo; - // Acquire lock on requested product file - Instant lockRequestStartTime = Instant.now(); - Instant lockRequestTimeOut = lockRequestStartTime - .plusMillis(cfg.getFileCheckMaxCycles() * cfg.getFileCheckWaitTime()); - try { - int i = 0; - for (; i < cfg.getFileCheckMaxCycles() && Instant.now().isBefore(lockRequestTimeOut); ++i) { - synchronized (productLockSet) { - if (!productLockSet.contains(sourceFile.getFileName())) { - productLockSet.add(sourceFile.getFileName()); - break; - } - } - if (loggerLegacy.isDebugEnabled()) - loggerLegacy.debug("... waiting for concurrent access to {} to terminate", - sourceFile.getFileName()); - Thread.sleep(cfg.getFileCheckWaitTime()); - } - ; - if (i == cfg.getFileCheckMaxCycles()) { - return new ResponseEntity<>( - errorHeaders(StorageLogger.logError(loggerLegacy, MSG_READ_TIMEOUT, MSG_ID_READ_TIMEOUT, - sourceFile.getFileName(), - Duration.between(lockRequestStartTime, Instant.now()).getSeconds())), - HttpStatus.SERVICE_UNAVAILABLE); - } - } catch (InterruptedException e) { - return new ResponseEntity<>(errorHeaders(StorageLogger.logError(loggerLegacy, MSG_EXCEPTION_THROWN, - MSG_ID_EXCEPTION_THROWN, e.getClass().toString() + ": " + e.getMessage())), - HttpStatus.INTERNAL_SERVER_ERROR); - } + String relativePath = getProductFolderWithFilename(externalPath, productId); + StorageFile cacheFile = storageProvider.getCacheFile(relativePath); + + StorageFileLocker fileLocker = new StorageFileLocker(cacheFile.getFullPath(), cfg.getFileCheckWaitTime(), + cfg.getFileCheckMaxCycles()); try { - ArrayList transferredFiles = sourceFile.copyTo(targetFile, false); - if (transferredFiles != null && !transferredFiles.isEmpty()) { - RestFileInfo response = new RestFileInfo(); - response.setStorageType(targetFile.getFsType().toString()); - response.setFilePath(targetFile.getFullPath()); - response.setFileName(targetFile.getFileName()); - response.setFileSize(targetFile.getLength()); - - StorageLogger.logInfo(loggerLegacy, MSG_FILE_COPIED, MSG_ID_FILE_COPIED, sourceFile.getFullPath(), - targetFile.getFullPath()); - - return new ResponseEntity<>(response, HttpStatus.OK); - } else { - return new ResponseEntity<>(errorHeaders( - StorageLogger.logError(loggerLegacy, MSG_FILE_NOT_FOUND, MSG_ID_FILE_NOT_FOUND, pathInfo)), - HttpStatus.NOT_FOUND); - } - } catch (IllegalArgumentException e) { - return new ResponseEntity(errorHeaders(e.getMessage()), HttpStatus.BAD_REQUEST); + + RestFileInfo restFileInfo = copyFileExternalToCache(externalPath, productId, fileSize, fileLocker); + fileLocker.unlock(); + restFileInfo = copyFileCacheToStorage(relativePath, fileLocker); + + logger.log(StorageMgrMessage.PRODUCT_FILE_UPLOADED_TO_STORAGE, externalPath, productId); + return new ResponseEntity<>(restFileInfo, HttpStatus.CREATED); + } catch (Exception e) { - return new ResponseEntity<>(errorHeaders(StorageLogger.logError(loggerLegacy, MSG_EXCEPTION_THROWN, - MSG_ID_EXCEPTION_THROWN, e.getClass().toString() + ": " + e.getMessage())), - HttpStatus.INTERNAL_SERVER_ERROR); + + String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); + return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); + } finally { - productLockSet.remove(sourceFile.getFileName()); + + fileLocker.unlock(); + logger.debug("... unlocked the file: ", externalPath); } } - + /** - * Copy local file named pathInfo to storage manager. The target file path is: - * default mount point + productId + relative source file path + * Copies the file from the storage to the cache using synchronization. + * During the copying to the cache, the status of the file will be "not exists", + * after the completion the status will be set to "ready" * - * @param pathInfo Source file name - * @param productId Product id - * @return Target file name + * @param storageFilePath the file path in the storage + * @param fileLocker file locker is used for synchronization + * @return RestFileInfo + * @throws FileLockedAfterMaxCyclesException + * @throws IOException + * @throws Exception */ + private RestFileInfo copyFileStorageToCache(String storageFilePath, StorageFileLocker fileLocker) + throws FileLockedAfterMaxCyclesException, IOException, Exception { + + // x-to-cache-copy method, status "not exists" is used + + // relative path depends on path, not on actual storage + String relativePath = storageProvider.getRelativePath(storageFilePath); + + StorageFile storageFile = storageProvider.getStorageFile(relativePath); + StorageFile cacheFile = storageProvider.getCacheFile(storageFile.getRelativePath()); + + FileCache cache = FileCache.getInstance(); + + if (!cache.containsKey(cacheFile.getFullPath())) { + + fileLocker.lockOrWaitUntilUnlockedAndLock(); + + // check again, the file could be copied to cache from another thread after lock + if (!cache.containsKey(cacheFile.getFullPath())) { + + // active thread - copies the file to the cache storage and puts it to the cache + + cache.setCacheFileStatus(cacheFile.getFullPath(), CacheFileStatus.NOT_EXISTS); + + storageProvider.getStorage().downloadFile(storageFile, cacheFile); + + logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED_FROM_STORAGE, cacheFile.getFullPath()); + + cache.put(cacheFile.getFullPath()); // cache file status = READY + + } else { + + // passive thread - did nothing, waited for copied file and use it from cache + logger.debug("... waiting-thread when the file downloaded and use it from cache: ", + cacheFile.getFullPath()); + } + + } else { + + logger.debug("... no download and no lock - the file is in cache: ", cacheFile.getFullPath()); + } + + RestFileInfo restFileInfo = convertToRestFileInfo(cacheFile, + storageProvider.getCacheFileSize(storageFile.getRelativePath())); + + return restFileInfo; + } + /** - * + * Copies the file from the external source to the cache using synchronization. + * During the copying to the cache, the status of the file will be "not exists", + * after the completion the status will be set to "ready" + * + * @param externalPath external path of the file, which will be copied to the cache + * @param productId product id is used as a directory to store copied file in cache + * @param fileSize file size + * @param fileLocker file locker is used for synchronization + * @return Rest File Info + * @throws FileLockedAfterMaxCyclesException + * @throws IOException + * @throws Exception */ - @Override - public ResponseEntity updateProductfiles(String pathInfo, Long productId, Long fileSize) { + private RestFileInfo copyFileExternalToCache(String externalPath, Long productId, Long fileSize, + StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { - if (logger.isTraceEnabled()) - logger.trace(">>> updateProductfiles({}, {})", pathInfo, productId); + // x-to-cache-copy method, status "not exists" is used + + String productFolderWithFilename = getProductFolderWithFilename(externalPath, productId); + StorageFile cacheFile = storageProvider.getCacheFile(productFolderWithFilename); + + FileCache cache = FileCache.getInstance(); + + if (!cache.containsKey(cacheFile.getFullPath())) { - // pathInfo absolute path, UPLOAD absolute file -> storage - if (storageProvider.isVersion2()) { + fileLocker.lockOrWaitUntilUnlockedAndLock(); - if (pathInfo == null) { - return new ResponseEntity(new RestFileInfo(), HttpStatus.BAD_REQUEST); + // check again, the file could be copied to cache from another thread after lock + if (!cache.containsKey(cacheFile.getFullPath())) { + + // active thread - copies the file to the cache storage and puts it to the cache + + cache.setCacheFileStatus(cacheFile.getFullPath(), CacheFileStatus.NOT_EXISTS); + + storageProvider.copyAbsoluteFilesToCache(externalPath, productId); + + logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED_FROM_EXTERNAL_TO_CACHE, + cacheFile.getFullPath()); + + cache.put(cacheFile.getFullPath()); // cache file status = READY + + } else { + + // passive thread - did nothing, waited for copied file and use it from cache + logger.debug( + "... waiting-thread when the file downloaded to cache from external storage and use it from cache: ", + cacheFile.getFullPath()); } - try { - Storage storage = storageProvider.getStorage(); - String absolutePath = pathInfo; - // String relativePath = storageProvider.getRelativePath(absolutePath); - String fileName = new File(pathInfo).getName(); - String productFolderWithFilename = Paths.get(String.valueOf(productId), fileName).toString(); + } else { + + logger.debug("... no download and no lock - the file is in cache: ", cacheFile.getFullPath()); + } - StorageFile sourceFile = storageProvider.getAbsoluteFile(absolutePath); - StorageFile targetFile = storageProvider.getStorageFile(productFolderWithFilename); + RestFileInfo restFileInfo = convertToRestFileInfo(cacheFile, + storageProvider.getCacheFileSize(cacheFile.getRelativePath())); - storage.uploadFile(sourceFile, targetFile); + return restFileInfo; + } + + // TODO: WIP Special use case for cache recovery - file in cache, but not in storage + // TODO: WIP Special use case for cache state - not uploaded to storage - RestFileInfo restFileInfo = convertToRestFileInfo(targetFile, storage.getFileSize(targetFile)); + /** + * Copies the file from the cache to the storage using synchronization. + * During the copying to the cache, the status of the file will be "not exists", + * after the completion the status will be set to "ready" + * + * @param relativeCachePath relative cache path + * @param fileLocker is used for synchronization + * @return RestFileInfo + * @throws FileLockedAfterMaxCyclesException + * @throws IOException + * @throws Exception + */ + private RestFileInfo copyFileCacheToStorage(String relativeCachePath, StorageFileLocker fileLocker) + throws FileLockedAfterMaxCyclesException, IOException, Exception { - logger.log(StorageMgrMessage.PRODUCT_FILE_UPLOADED, pathInfo, productId); + Storage storage = storageProvider.getStorage(); - return new ResponseEntity<>(restFileInfo, HttpStatus.CREATED); + StorageFile cacheFile = storageProvider.getCacheFile(relativeCachePath); + StorageFile storageFile = storageProvider.getStorageFile(relativeCachePath); - } catch (Exception e) { + FileCache cache = FileCache.getInstance(); - String msg = logger.log(StorageMgrMessage.INTERNAL_ERROR, e.getMessage()); - return new ResponseEntity<>(errorHeaders(msg), HttpStatus.INTERNAL_SERVER_ERROR); - } - } // end version 2 - - RestFileInfo response = new RestFileInfo(); - if (pathInfo != null) { - ProseoFile sourceFile = ProseoFile.fromPathInfo(pathInfo, cfg); - ProseoFile targetFile = ProseoFile.fromType(StorageType.valueOf(cfg.getDefaultStorageType()), - String.valueOf(productId) + "/" + sourceFile.getFileName(), cfg); - try { - // wait until source file is really copied - if (sourceFile.getFsType() == StorageType.POSIX) { - int i = 0; - Path fp = Path.of(sourceFile.getFullPath()); - if (fp.toFile().isFile()) { - Long wait = cfg.getFileCheckWaitTime(); - Long max = cfg.getFileCheckMaxCycles(); - try { - while (Files.size(fp) < fileSize && i < max) { - if (loggerLegacy.isDebugEnabled()) { - loggerLegacy.debug("Wait for fully copied file {}", sourceFile.getFullPath()); - } - i++; - try { - Thread.sleep(wait); - } catch (InterruptedException e) { - return new ResponseEntity<>( - errorHeaders(StorageLogger.logError(loggerLegacy, MSG_READ_TIMEOUT, - MSG_ID_READ_TIMEOUT, sourceFile.getFileName(), - cfg.getFileCheckMaxCycles() * cfg.getFileCheckWaitTime() / 1000)), - HttpStatus.SERVICE_UNAVAILABLE); - } - } - } catch (IOException e) { - loggerLegacy.error("Unable to access file {}", sourceFile.getFullPath()); - return new ResponseEntity<>( - errorHeaders(StorageLogger.logError(loggerLegacy, MSG_EXCEPTION_THROWN, - MSG_ID_EXCEPTION_THROWN, e.getClass().toString() + ": " + e.getMessage())), - HttpStatus.INTERNAL_SERVER_ERROR); - } - if (i >= max) { - loggerLegacy.error(MSG_FILE_NOT_FETCHED, sourceFile.getFullPath()); - } - } - } - ArrayList transfered = sourceFile.copyTo(targetFile, false); - - if (transfered != null && !transfered.isEmpty()) { - response.setStorageType(targetFile.getFsType().toString()); - response.setFilePath(targetFile.getFullPath()); - response.setFileName(targetFile.getFileName()); - response.setFileSize(targetFile.getLength()); - - StorageLogger.logInfo(loggerLegacy, MSG_FILES_UPDATED, MSG_ID_FILES_UPDATED, pathInfo, productId); - - return new ResponseEntity<>(response, HttpStatus.CREATED); - } - } catch (Exception e) { - - return new ResponseEntity<>(errorHeaders(StorageLogger.logError(loggerLegacy, MSG_EXCEPTION_THROWN, - MSG_ID_EXCEPTION_THROWN, e.getClass().toString() + ": " + e.getMessage())), - HttpStatus.INTERNAL_SERVER_ERROR); + if (!cache.containsKey(cacheFile.getFullPath())) { + + fileLocker.lockOrWaitUntilUnlockedAndLock(); + + // check again, the file could be copied to storage from another thread after lock + if (!cache.containsKey(cacheFile.getFullPath())) { + + // active thread - copies the file to the storage and checks it as OK (WIP) + storage.uploadFile(cacheFile, storageFile); + + logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED_FROM_EXTERNAL_TO_CACHE, + storageFile.getFullPath()); + + cache.put(storageFile.getFullPath()); + + } else { + + // passive thread - did nothing, waited for copied file and use it from cache + logger.debug("... waiting-thread when the file downloaded and use it from cache: ", + storageFile.getFullPath()); } + + } else { + + logger.debug("... no download and no lock - the file is in cache: ", storageFile.getFullPath()); } - return new ResponseEntity(response, HttpStatus.NOT_FOUND); + + RestFileInfo restFileInfo = convertToRestFileInfo(storageFile, + storageProvider.getCacheFileSize(cacheFile.getRelativePath())); + + return restFileInfo; } /** @@ -368,7 +357,6 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro * * @param storageFile storage file * @param fileSize file size - * * @return rest file info */ private static RestFileInfo convertToRestFileInfo(StorageFile storageFile, long fileSize) { @@ -384,15 +372,16 @@ private static RestFileInfo convertToRestFileInfo(StorageFile storageFile, long } /** - * Create an HTTP "Warning" header with the given text message + * Gets a product folder with the file name from the given external path using + * product id * - * @param message the message text - * @return an HttpHeaders object with a warning message + * @param externalPath absolute external path + * @param productId product id + * @return product folder with the file name */ - private HttpHeaders errorHeaders(String message) { - HttpHeaders responseHeaders = new HttpHeaders(); - responseHeaders.set(HTTP_HEADER_WARNING, - HTTP_MSG_PREFIX + (null == message ? "null" : message.replaceAll("\n", " "))); - return responseHeaders; + private String getProductFolderWithFilename(String externalPath, Long productId) { + + String fileName = new File(externalPath).getName(); + return Paths.get(String.valueOf(productId), fileName).toString(); } } diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicBucketCreator.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicBucketCreator.java similarity index 96% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicBucketCreator.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicBucketCreator.java index 1285f459d..a33ea9f08 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicBucketCreator.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicBucketCreator.java @@ -3,14 +3,14 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import software.amazon.awssdk.core.waiters.WaiterResponse; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.CreateBucketRequest; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicBucketDeleter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicBucketDeleter.java similarity index 95% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicBucketDeleter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicBucketDeleter.java index eac1d3c5c..17a518936 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicBucketDeleter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicBucketDeleter.java @@ -3,14 +3,14 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicBucketListGetter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicBucketListGetter.java similarity index 96% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicBucketListGetter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicBucketListGetter.java index 009748c5d..91eb67a13 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicBucketListGetter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicBucketListGetter.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import java.util.ArrayList; @@ -12,7 +12,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.Bucket; import software.amazon.awssdk.services.s3.model.ListBucketsResponse; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileContentGetter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileContentGetter.java similarity index 96% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileContentGetter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileContentGetter.java index 0e5593a07..ed1e3a596 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileContentGetter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileContentGetter.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -11,7 +11,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.GetObjectRequest; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileDeleter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileDeleter.java similarity index 97% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileDeleter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileDeleter.java index 8b879a41a..6a18da3c5 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileDeleter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileDeleter.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import java.util.ArrayList; @@ -12,7 +12,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.Delete; import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileDownloader.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileDownloader.java similarity index 97% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileDownloader.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileDownloader.java index 4b43cd42d..9a3e72b8f 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileDownloader.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileDownloader.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.File; import java.io.IOException; @@ -20,9 +20,9 @@ import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; -import de.dlr.proseo.storagemgr.version2.FileUtils; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * S3 Atomic File Downloader diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileExistsGetter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileExistsGetter.java similarity index 96% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileExistsGetter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileExistsGetter.java index 3a6801133..fca8c343d 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileExistsGetter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileExistsGetter.java @@ -3,14 +3,14 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileListDeleter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileListDeleter.java similarity index 97% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileListDeleter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileListDeleter.java index 28f26933e..7a97d5cd2 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileListDeleter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileListDeleter.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import java.util.ArrayList; @@ -13,7 +13,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.Delete; import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileListGetter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileListGetter.java similarity index 97% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileListGetter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileListGetter.java index 4f1414caf..e226719ce 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileListGetter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileListGetter.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import java.util.ArrayList; @@ -12,7 +12,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.ListObjectsRequest; import software.amazon.awssdk.services.s3.model.ListObjectsResponse; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileSizeGetter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileSizeGetter.java similarity index 96% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileSizeGetter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileSizeGetter.java index 0a2d9d3ad..5cc057d12 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileSizeGetter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileSizeGetter.java @@ -3,14 +3,14 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileUploader.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileUploader.java similarity index 95% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileUploader.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileUploader.java index f88607812..90d69eb67 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicFileUploader.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicFileUploader.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.File; import java.io.IOException; @@ -16,8 +16,8 @@ import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * S3 Atomic Uploader diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicInputStreamGetter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicInputStreamGetter.java similarity index 95% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicInputStreamGetter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicInputStreamGetter.java index bf0c3efc3..cc927c545 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3AtomicInputStreamGetter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicInputStreamGetter.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.IOException; import java.io.InputStream; @@ -11,8 +11,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.AtomicCommand; +import de.dlr.proseo.storagemgr.utils.PathConverter; import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.GetObjectRequest; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicSynchroFileUploader.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicSynchroFileUploader.java new file mode 100644 index 000000000..d89fb87fd --- /dev/null +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3AtomicSynchroFileUploader.java @@ -0,0 +1,211 @@ +/** + * S3AtomicSynchroFileUploader.java + * + * (C) 2023 Dr. Bassler & Co. Managementberatung GmbH + */ +package de.dlr.proseo.storagemgr.s3; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; + +import de.dlr.proseo.storagemgr.Exceptions.FileLockedAfterMaxCyclesException; +import de.dlr.proseo.storagemgr.model.AtomicCommand; +import de.dlr.proseo.storagemgr.utils.PathConverter; +import de.dlr.proseo.storagemgr.utils.StorageFileLocker; + +/** + * S3 Atomic Synchro Uploader + * + * @author Denys Chaykovskiy + * + */ +public class S3AtomicSynchroFileUploader implements AtomicCommand { + + /** Info */ + private static final String INFO = "S3 ATOMIC Synchro File Uploader"; + + /** Completed Info */ + private static final String COMPLETED = "file UPLOADED"; + + /** Failed Info */ + private static final String FAILED = "file upload FAILED"; + + /** Logger for this class */ + private static Logger logger = LoggerFactory.getLogger(S3AtomicSynchroFileUploader.class); + + /** Chunk size for uploads to S3 storage (128 MB) */ + private static final Long MULTIPART_UPLOAD_PARTSIZE_BYTES = (long) (128 * 1024 * 1024); + + /** source file */ + private String sourceFile; + + /** target file or dir */ + private String targetFileOrDir; + + /** S3 Client */ + private AmazonS3 s3ClientV1; + + /** Bucket */ + private String bucket; + + /** S3 Configuration */ + private S3Configuration cfg; + + + /** + * Constructor + * + * @param s3ClientV1 s3 client + * @param bucket bucket + * @param sourceFile sourceFile + * @param targetFileOrDir target file or directory + * @param cfg s3 Configuration + + */ + public S3AtomicSynchroFileUploader(AmazonS3 s3ClientV1, String bucket, String sourceFile, String targetFileOrDir, S3Configuration cfg) { + + this.s3ClientV1 = s3ClientV1; + this.bucket = bucket; + this.sourceFile = sourceFile; + this.targetFileOrDir = targetFileOrDir; + this.cfg = cfg; + } + + /** + * Executes synchro upload of the file to s3 + * + * @return uploaded file name + */ + @Override + public String execute() throws IOException { + + if (logger.isTraceEnabled()) + logger.trace(">>> execute() - synchroUploadFile({},{})", sourceFile, targetFileOrDir); + + String targetFile = targetFileOrDir; + + if (new PathConverter(targetFileOrDir).isDirectory()) { + targetFile = Paths.get(targetFileOrDir, getFileName(sourceFile)).toString(); + targetFile = new PathConverter(targetFile).posixToS3Path().convertToSlash().getPath(); + } + + File f = new File(sourceFile); + + if (f == null || !f.isFile()) { + throw new IOException("Cannot upload to s3, source file does not exist: " + sourceFile); + } + + StorageFileLocker fileLocker = new StorageFileLocker(targetFile, cfg.getFileCheckWaitTime(), + cfg.getMaxRequestAttempts()); + + try { + + fileLocker.lockOrWaitUntilUnlockedAndLock(); + return uploadWithTranferManager(sourceFile, targetFile); + + } catch (FileLockedAfterMaxCyclesException e) { + + logger.debug("... the file is locked after max check cycles: ", targetFile, " ", e.getMessage()); + throw new IOException(e); + + } catch (InterruptedException e) { + + logger.debug("... the file is locked after max check cycles: ", targetFile, " ", e.getMessage()); + throw new IOException(e); + + } finally { + + fileLocker.unlock(); + logger.debug("... unlocked the file: ", targetFile); + } + + } + + /** + * Gets information about atomic command (mostly for logs) + * + * @return information about atomic command + */ + @Override + public String getInfo() { + return INFO + " "; + } + + /** + * Gets information about completed atomic command (mostly for logs) + * + * @return information about completed atomic command + */ + @Override + public String getCompletedInfo() { + return INFO + ": " + COMPLETED + " "; + } + + /** + * Gets information about failed atomic command (mostly for logs) + * + * @return information about failed atomic command + */ + @Override + public String getFailedInfo() { + return INFO + ": " + FAILED + " "; + } + + /** + * Gets file name + * + * @param path path + * @return file name + */ + private String getFileName(String path) { + return new File(path).getName(); + } + + + /** + * @param sourceFile source file + * @param targetFile target file + * @return uploaded file (target file) + * @throws IOException if file uploading was not successful + */ + private String uploadWithTranferManager(String sourceFile, String targetFile) throws IOException { + + TransferManager transferManager; + + try { + transferManager = TransferManagerBuilder.standard() + .withMultipartCopyPartSize(MULTIPART_UPLOAD_PARTSIZE_BYTES) + .withS3Client(s3ClientV1) + .build(); + + } catch (Exception e) { + if (logger.isTraceEnabled()) + logger.trace(getFailedInfo() + e.getMessage()); + throw new IOException(e); + } + + try { + transferManager.upload(bucket, targetFile, new File(sourceFile)).waitForCompletion(); + + if (logger.isTraceEnabled()) + logger.trace("... " + getCompletedInfo() + " - " + targetFile); + + return targetFile; + + } catch (Exception e) { + if (logger.isTraceEnabled()) + logger.trace(getFailedInfo() + e.getMessage()); + throw new IOException(e); + } finally { + transferManager.shutdownNow(false); + } + } +} \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3Configuration.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Configuration.java similarity index 98% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3Configuration.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Configuration.java index 0b9cfe4a1..4919a43fb 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3Configuration.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Configuration.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; /** * Holds the configuration settings required to connect to and interact with an diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3DAL.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3DAL.java similarity index 98% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3DAL.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3DAL.java index deb1716c2..661145677 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3DAL.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3DAL.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.File; import java.io.IOException; @@ -26,9 +26,9 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.model.AtomicCommand; -import de.dlr.proseo.storagemgr.version2.model.DefaultRetryStrategy; +import de.dlr.proseo.storagemgr.model.AtomicCommand; +import de.dlr.proseo.storagemgr.model.DefaultRetryStrategy; +import de.dlr.proseo.storagemgr.utils.PathConverter; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3Storage.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Storage.java similarity index 97% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3Storage.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Storage.java index 2fa49a06b..f37076f6a 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3Storage.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3Storage.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.File; import java.io.IOException; @@ -13,12 +13,12 @@ import java.util.List; import de.dlr.proseo.logging.logger.ProseoLogger; -import de.dlr.proseo.storagemgr.version2.FileUtils; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; -import de.dlr.proseo.storagemgr.version2.posix.PosixStorageFile; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; +import de.dlr.proseo.storagemgr.posix.PosixStorageFile; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * An implementation of the Storage interface for a S3-based file system, @@ -37,7 +37,7 @@ public class S3Storage implements Storage { private static ProseoLogger logger = new ProseoLogger(S3Storage.class); /** - * Constructor with bucket, access keys, region and end point + * Constructor with the s3 configuration * * @param cfg s3 configuration * @throws IOException if an I/O exception occurs diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageFile.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3StorageFile.java similarity index 93% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageFile.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3StorageFile.java index 6eececeb4..fb25ba8ef 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageFile.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/S3StorageFile.java @@ -3,16 +3,16 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2.s3; +package de.dlr.proseo.storagemgr.s3; import java.io.File; import java.nio.file.Paths; import org.apache.commons.io.FilenameUtils; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * Represents a file within a S3-based storage system. It implements the diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/package-info.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/package-info.java similarity index 80% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/package-info.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/package-info.java index 812dca519..c479a6000 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/s3/package-info.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/s3/package-info.java @@ -8,4 +8,4 @@ * * @author Denys Chaykovskiy */ -package de.dlr.proseo.storagemgr.version2.s3; \ No newline at end of file +package de.dlr.proseo.storagemgr.s3; \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/FileUtils.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/FileUtils.java similarity index 83% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/FileUtils.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/FileUtils.java index d76d85613..03470d1d2 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/FileUtils.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/FileUtils.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2; +package de.dlr.proseo.storagemgr.utils; import java.io.File; import java.io.FileWriter; @@ -16,6 +16,7 @@ import de.dlr.proseo.logging.logger.ProseoLogger; import de.dlr.proseo.logging.messages.StorageMgrMessage; +import de.dlr.proseo.storagemgr.Exceptions.FileLockedAfterMaxCyclesException; /** * A utility class for common file operations. It provides methods to create, @@ -90,6 +91,47 @@ public boolean createFile(String content) { } } + /** + * Creates the file with the content in a synchro modus + * + * @param content Content of the file + * @param waitTime the wait time between each cycle of checking the + * file lock status + * @param fileCheckMaxCycles file check max cycles + * @return true if file was successfully created + */ + public boolean synchroCreateFile(String content, long waitTime, long fileCheckMaxCycles) { + + if (logger.isTraceEnabled()) + logger.trace(">>> synchroCreateFile({}, {}, {})", content, waitTime, fileCheckMaxCycles); + + StorageFileLocker fileLocker = new StorageFileLocker(path, waitTime, fileCheckMaxCycles); + boolean fileCreatedStatus; + + try { + + fileLocker.lockOrWaitUntilUnlockedAndLock(); + fileCreatedStatus = createFile(content); + + } catch (FileLockedAfterMaxCyclesException e) { + + logger.debug("... the file is locked after max check cycles: ", path, " ", e.getMessage()); + fileCreatedStatus = false; + + } catch (InterruptedException e) { + + logger.debug("... the file is locked after max check cycles: ", path, " ", e.getMessage()); + fileCreatedStatus = false; + + } finally { + + fileLocker.unlock(); + logger.debug("... unlocked the file: ", path); + } + + return fileCreatedStatus; + } + /** * Gets the file size * diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/PathConverter.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/PathConverter.java similarity index 99% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/PathConverter.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/PathConverter.java index 0ec17d274..61ea2dcab 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/PathConverter.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/PathConverter.java @@ -3,7 +3,7 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2; +package de.dlr.proseo.storagemgr.utils; import java.io.File; import java.nio.file.Paths; @@ -11,7 +11,7 @@ import java.util.List; import de.dlr.proseo.logging.logger.ProseoLogger; -import de.dlr.proseo.storagemgr.version2.model.StorageType; +import de.dlr.proseo.storagemgr.model.StorageType; /** * A set of methods to convert and manipulate file paths based on different diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/ProseoFile.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/ProseoFile.java deleted file mode 100644 index d47fa1218..000000000 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/ProseoFile.java +++ /dev/null @@ -1,380 +0,0 @@ -package de.dlr.proseo.storagemgr.utils; - -import java.io.InputStream; -import java.nio.file.Paths; -import java.util.ArrayList; - -import org.apache.commons.io.FilenameUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.core.io.FileSystemResource; - -import de.dlr.proseo.storagemgr.StorageManagerConfiguration; - -/** - * Abstract definition of proseo file objects - * - * The file definition is based on: - * File system type, e.g. S3, POSIX - * Base path, e.g. S3 bucket, POSIX mount point - * Relative path and file name - * - * Example for full path: - * S3: "s3://bucket/somewhere/file.name" - * POSIX: "/mntPoint/somewhere/file.name" * - * - * Directories are represented by a "/" as last character - * - * @author melchinger - * - */ -public abstract class ProseoFile { - - /** - * Original path - */ - protected String pathInfo; - - /** - * File name - */ - protected String fileName; - - /** - * File path relative to bucket - */ - protected String relPath; - - /** - * Bucket, in which the file resides - */ - protected String basePath; - - /** - * Storage manager configuration used to get easily access to default settings. - */ - protected StorageManagerConfiguration cfg; - - /** - * Logger for this class - */ - private static Logger logger = LoggerFactory.getLogger(ProseoFile.class); - - /** - * Gets the file path information - * @return the path information - */ - public String getPathInfo() { - return pathInfo; - } - - /** - * Sets the file path information - * @param pathInfo the pathInfo to set - */ - public void setPathInfo(String pathInfo) { - this.pathInfo = pathInfo; - } - - /** - * Gets the file name - * @return the file name - */ - public String getFileName() { - return fileName; - } - - /** - * Sets the file name - * @param fileName the file name to set - */ - public void setFileName(String fileName) { - this.fileName = fileName; - } - - /** - * Gets the file path relative to the bucket - * @return the relative path - */ - public String getRelPath() { - return relPath; - } - - /** - * Sets the file path relative to the bucket - * @param relPath the relative path to set - */ - public void setRelPath(String relPath) { - this.relPath = relPath; - } - - /** - * Gets the path to the bucket - * @return the base path - */ - public String getBasePath() { - return basePath; - } - - /** - * Sets the path to the bucket - * @param basePath the base path to set - */ - public void setBasePath(String basePath) { - this.basePath = basePath; - } - - /** - * Set file name from relative path and remove it from the relative path - * (if the relative path does not end on '/') - */ - protected void buildFileName() { - - if (logger.isTraceEnabled()) logger.trace(">>> buildFileName()"); - - fileName = ""; - if (relPath != null) { - if (!relPath.endsWith("/")) { - int i = relPath.lastIndexOf('/'); - if (i > 0) { - fileName = relPath.substring(i + 1, relPath.length()); - relPath = relPath.substring(0, i); - } else { - fileName = relPath; - relPath = ""; - } - } - } - } - - /** - * Gets the complete file path relative to the bucket - * @return Relative path + file name - */ - public String getRelPathAndFile() { - - if (logger.isTraceEnabled()) logger.trace(">>> getRelPathAndFile()"); - - if (relPath.endsWith("/") || relPath.isEmpty()) { - return relPath + fileName; - } else { - return relPath + "/" + fileName; - } - } - - /** - * Checks whether the object is a directory - * @return true if object represents a directory, false otherwise - */ - public Boolean isDirectory() { - return (fileName == null) || fileName.isEmpty(); - } - - /** - * Gets the file name extension - * @return Extension of file name or an empty string, if the file name has no extension - */ - public String getExtension() { - - if (logger.isTraceEnabled()) logger.trace(">>> getExtension()"); - - if ((fileName == null) || fileName.isEmpty()) { - return ""; - } else { - return FilenameUtils.getExtension(fileName); - } - } - - /** - * Create a file object out of full path info. - * - * @param pathInfo Full path - * @param cfg a pointer to the Storage Manager configuration - * @return The new file object or null, if the operation failed - */ - public static ProseoFile fromPathInfo(String pathInfo, StorageManagerConfiguration cfg) { - - if (logger.isTraceEnabled()) logger.trace(">>> fromPathInfo({}, {})", pathInfo, cfg); - - if (pathInfo == null) { - logger.warn("pathInfo not set"); - } else { - String aPath = pathInfo.trim(); - // Find storage type - if (aPath.startsWith("s3:") || aPath.startsWith("S3:")) { - return new ProseoFileS3(aPath, true, cfg); - } else if (aPath.startsWith("alluxio:")) { - return new ProseoFileAlluxio(aPath, true, cfg); - } else { - if (!aPath.startsWith("/")) { - aPath = Paths.get(aPath).toAbsolutePath().toString(); - } - return new ProseoFilePosix(aPath, true, cfg); - } - //logger.warn("Unknown FS type in path: {}", pathInfo); - } - return null; - } - - /** - * Create file object of aType with relative path pathInfo - * - * @param aType StorageType - * @param pathInfo Relative path with bucket - * @param cfg a pointer to the Storage Manager configuration - * @return The new file object or null, if the operation failed - */ - public static ProseoFile fromType(StorageType aType, String pathInfo, StorageManagerConfiguration cfg) { - - if (logger.isTraceEnabled()) logger.trace(">>> fromType({}, {}, {})", aType, pathInfo, cfg); - - if (pathInfo != null) { - String aPath = pathInfo.trim(); - // Find storage type - switch (aType) { - case S3: - return new ProseoFileS3(aPath, false, cfg); - case ALLUXIO: - return new ProseoFileAlluxio(aPath, false, cfg); - case POSIX: - return new ProseoFilePosix(aPath, false, cfg); - case OTHER: - default: - logger.warn("Unknown FS type in path: {}", pathInfo); - return null; - } - } - logger.warn("pathInfo not set"); - return null; - } - - /** - * Create file object of aType with full path pathInfo - * - * @param aType StorageType - * @param pathInfo Full path (with type info) - * @param cfg a pointer to the Storage Manager configuration - * @return The new file object or null, if the operation failed - */ - public static ProseoFile fromTypeFullPath(StorageType aType, String pathInfo, StorageManagerConfiguration cfg) { - - if (logger.isTraceEnabled()) logger.trace(">>> fromTypeFullPath({}, {}, {})", aType, pathInfo, cfg); - - if (pathInfo != null) { - String aPath = pathInfo.trim(); - // Find storage type - switch (aType) { - case S3: - return new ProseoFileS3(aPath, true, cfg); - case ALLUXIO: - return new ProseoFileAlluxio(aPath, true, cfg); - case POSIX: - return new ProseoFilePosix(aPath, true, cfg); - case OTHER: - default: - logger.warn("Unknown FS type in path: {}", pathInfo); - return null; - } - } - logger.warn("pathInfo not set"); - return null; - } - - /** - * Create file object of aType with bucket and relative path. - * - * @param aType StorageType - * @param bucket Bucket - * @param pathInfo Relative path - * @param cfg a pointer to the Storage Manager configuration - * @return the new file object or null, if the operation failed - */ - public static ProseoFile fromTypeAndBucket(StorageType aType, String bucket, String pathInfo, StorageManagerConfiguration cfg) { - - if (logger.isTraceEnabled()) logger.trace(">>> fromTypeAndBucket({}, {}, {})", aType, bucket, pathInfo, cfg); - - if (pathInfo != null) { - String aPath = pathInfo.trim(); - // Find storage type - switch (aType) { - case S3: - return new ProseoFileS3(bucket, aPath, cfg); - case ALLUXIO: - return new ProseoFileAlluxio(bucket, aPath, cfg); - case POSIX: - return new ProseoFilePosix(bucket, aPath, cfg); - case OTHER: - default: - logger.warn("Unknown FS type in path: {}", pathInfo); - return null; - } - } - logger.warn("pathInfo not set"); - return null; - } - - // The methods below must be implemented by the derived prosEO file classes - - /** - * Gets the file system type - * @return The file system type - */ - public abstract StorageType getFsType(); - - /** - * Gets the file system resource definition - * @return the file system resource definition - */ - public abstract FileSystemResource getFileSystemResource(); - - /** - * Delete file object recursively. - * - * @return list of deleted object paths. - */ - public abstract ArrayList delete(); - - /** - * List objects recursively. - * - * @return list of file objects - */ - public abstract ArrayList list(); - - /** - * Gets the full file path - * @return The full path including type - */ - public abstract String getFullPath(); - - /** - * Gets the file content as input stream - * @return the file content (data) - */ - public abstract InputStream getDataAsInputStream(); - - /** - * Write binary byte array to file object. - * - * @param bytes Byte array - * @return true after success - * @throws Exception if an error occurs in any lower-level library - */ - public abstract Boolean writeBytes(byte[] bytes) throws Exception; - - /** - * Copy this object to target proFile - * - * @param proFile Target file object - * @param recursive Copy recursively if true - * @return List of copied target file names - * @throws Exception if an error occurs in any lower-level library - */ - public abstract ArrayList copyTo(ProseoFile proFile, Boolean recursive) throws Exception; - - /** - * Gets the object size - * @return Length of file object - */ - public abstract long getLength(); -} diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/ProseoFileAlluxio.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/ProseoFileAlluxio.java deleted file mode 100644 index e582ca516..000000000 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/ProseoFileAlluxio.java +++ /dev/null @@ -1,167 +0,0 @@ -package de.dlr.proseo.storagemgr.utils; - -import java.io.InputStream; -import java.util.ArrayList; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.core.io.FileSystemResource; - -import de.dlr.proseo.storagemgr.StorageManagerConfiguration; - -/** - * Proseo file representing alluxio. - * - * !! NOT USED AND IMPLEMENTED !! - * - * @author melchinger - * - */ -public class ProseoFileAlluxio extends ProseoFile { - - private static Logger logger = LoggerFactory.getLogger(ProseoFileAlluxio.class); - - /** - * Creates a new Alluxio file - * - * @param pathInfo Path information - * @param fullPath if it is a full path - * @param cfg The Configuration of storage manager - */ - public ProseoFileAlluxio(String pathInfo, Boolean fullPath, StorageManagerConfiguration cfg) { - - if (logger.isTraceEnabled()) logger.trace(">>> ProseoFileAlluxio({}, {}, {})", pathInfo, fullPath, cfg); - - this.cfg = cfg; - String aPath = pathInfo.trim(); - this.pathInfo = aPath; - if (fullPath) { - if (aPath.startsWith("alluxio:/") || aPath.startsWith("alluxio:/")) { - aPath = aPath.substring(9); - } - while (aPath.startsWith("/")) { - aPath = aPath.substring(1); - } - int pos = aPath.indexOf('/'); - if (pos >= 0) { - basePath = aPath.substring(0, pos); - relPath = aPath.substring(pos + 1); - } else { - basePath = ""; - relPath = aPath; - } - } else { - while (aPath.startsWith("/")) { - aPath = aPath.substring(1); - } - relPath = aPath; - //basePath = cfg.getAlluxioUnderFsS3Bucket(); - } - pathInfo = getFullPath(); - - logger.trace("ProseoFileAlluxio created: {}", this.getFullPath()); - } - - /** - * Creates a new Alluxio file - * - * @param bucket A bucket of the file - * @param pathInfo Path Information - * @param cfg The Configuration of Storage Manager - */ - public ProseoFileAlluxio(String bucket, String pathInfo, StorageManagerConfiguration cfg) { - - if (logger.isTraceEnabled()) logger.trace(">>> ProseoFileAlluxio({}, {}, {})", bucket, pathInfo, cfg); - - String aPath = pathInfo.trim(); - relPath = aPath; - basePath = bucket.trim(); - while (basePath.startsWith("/")) { - basePath = basePath.substring(1); - } - pathInfo = getFullPath(); - - logger.trace("ProseoFileAlluxio created: {}", this.getFullPath()); - } - - /* (non-Javadoc) - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#getFSType() - */ - @Override - public StorageType getFsType() { - return StorageType.ALLUXIO; - } - - /* (non-Javadoc) - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#getFullPath() - */ - @Override - public String getFullPath() { - return "alluxio://" + getRelPathAndFile(); - } - - /* (non-Javadoc) - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#getDataAsInputStream() - */ - @Override - public InputStream getDataAsInputStream() { - - throw new UnsupportedOperationException("Not implemented yet"); - } - - /* (non-Javadoc) - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#copyTo() - */ - @Override - public ArrayList copyTo(ProseoFile proFile, Boolean recursive) throws Exception { - - throw new UnsupportedOperationException("Not implemented yet"); - } - - /* (non-Javadoc) - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#delete() - */ - @Override - public ArrayList delete() { - - throw new UnsupportedOperationException("Not implemented yet"); - } - - /* (non-Javadoc) - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#list() - */ - @Override - public ArrayList list() { - - ArrayList list = new ArrayList(); - return list; - } - - /* (non-Javadoc) - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#writeBytes() - */ - @Override - public Boolean writeBytes(byte[] bytes) throws Exception { - - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Not implemented yet"); - } - - /* (non-Javadoc) - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#getFileSystemResource() - */ - @Override - public FileSystemResource getFileSystemResource() { - - throw new UnsupportedOperationException("Not implemented yet"); - } - - /* (non-Javadoc) - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#getLength() - */ - @Override - public long getLength() { - - throw new UnsupportedOperationException("Not implemented yet"); - } -} diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/ProseoFilePosix.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/ProseoFilePosix.java deleted file mode 100644 index c9922721d..000000000 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/ProseoFilePosix.java +++ /dev/null @@ -1,391 +0,0 @@ -package de.dlr.proseo.storagemgr.utils; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collection; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.filefilter.TrueFileFilter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.core.io.FileSystemResource; - -import com.amazonaws.services.s3.AmazonS3; - -import de.dlr.proseo.storagemgr.StorageManagerConfiguration; -import de.dlr.proseo.storagemgr.cache.FileCache; -import de.dlr.proseo.storagemgr.fs.s3.S3Ops; -import software.amazon.awssdk.services.s3.S3Client; - -/** - * Proseo file representing POSIX file system - * - * @author melchinger - * - */ -public class ProseoFilePosix extends ProseoFile { - - // Message IDs - private static final int MSG_ID_S3_REQUEST_FAILED_RETRYING = 4109; // same as in ProseoFileS3 - - // Message Strings - private static final String MSG_S3_REQUEST_FAILED_RETRYING = "(I%d) S3 request failed, retrying after %d ms (attempt %d of %d)"; - - /** - * Logger of this class - */ - private static Logger logger = LoggerFactory.getLogger(ProseoFilePosix.class); - - /** - * Creates a new posix file. - * - * @param pathInfo The file path and information about file - * @param fullPath Use it as full path if true, otherwise use default bucket + path info - * @param cfg the Storage Manager configuration to use - */ - public ProseoFilePosix(String pathInfo, Boolean fullPath, StorageManagerConfiguration cfg) { - - if (logger.isTraceEnabled()) logger.trace(">>> ProseoFilePosix({}, {}, {})", pathInfo, fullPath, cfg); - - this.cfg = cfg; - String aPath = pathInfo.trim(); - this.pathInfo = aPath; - while (aPath.startsWith("/")) { - aPath = aPath.substring(1); - } - if (fullPath) { - String baseBackend = Paths.get(cfg.getPosixBackendPath()).toAbsolutePath().toString(); - while (baseBackend.startsWith("/")) { - baseBackend = baseBackend.substring(1); - } - String baseCache = Paths.get(cfg.getPosixCachePath()).toAbsolutePath().toString(); - while (baseCache.startsWith("/")) { - baseCache = baseCache.substring(1); - } - if (aPath.startsWith(baseBackend)) { - basePath = baseBackend; - if (aPath.length() == baseBackend.length()) { - relPath = "/"; - } else { - relPath = aPath.substring(baseBackend.length() + 1); - } - } else if (aPath.startsWith(baseCache)) { - basePath = baseCache; - if (aPath.length() == baseCache.length()) { - relPath = "/"; - } else { - relPath = aPath.substring(baseCache.length() + 1); - } - } else { - int pos = aPath.indexOf('/'); - if (pos >= 0) { - basePath = aPath.substring(0, pos); - relPath = aPath.substring(pos + 1); - } else { - basePath = ""; - relPath = aPath; - } - } - } else { - relPath = aPath; - basePath = cfg.getPosixBackendPath().trim(); - while (basePath.startsWith("/")) { - basePath = basePath.substring(1); - } - } - buildFileName(); - pathInfo = getFullPath(); - - logger.trace("ProseoFilePosix created: {}", this.getFullPath()); - } - - /** - * Creates a new posix file. - * - * @param bucket bucket of posix file - * @param pathInfo relative path to file - * @param cfg the Storage Manager configuration to use - */ - public ProseoFilePosix(String bucket, String pathInfo, StorageManagerConfiguration cfg) { - - if (logger.isTraceEnabled()) logger.trace(">>> ProseoFilePosix({}, {}, {})", bucket, pathInfo, cfg); - - String aPath = pathInfo.trim(); - relPath = aPath; - basePath = bucket.trim(); - while (basePath.startsWith("/")) { - basePath = basePath.substring(1); - } - buildFileName(); - pathInfo = getFullPath(); - - logger.trace("ProseoFilePosix created: {}", this.getFullPath()); - } - - /* (non-Javadoc) - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#getFsType() - */ - @Override - public StorageType getFsType() { - return StorageType.POSIX; - } - - /* (non-Javadoc) - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#getFullPath() - */ - @Override - public String getFullPath() { - - if (logger.isTraceEnabled()) logger.trace(">>> getFullPath()"); - - return "/" + getBasePath() + "/" + getRelPathAndFile(); - } - - /* (non-Javadoc) - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#getDataAsInputStream() - */ - @Override - public InputStream getDataAsInputStream() { - - if (logger.isTraceEnabled()) logger.trace(">>> getDataAsInputStream()"); - - try { - return new FileInputStream(pathInfo); - } catch (FileNotFoundException e) { - logger.error("Requested POSIX file {} not found", pathInfo); - return null; - } - } - - /* (non-Javadoc) - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#writeBytes(byte[]) - */ - @Override - public Boolean writeBytes(byte[] bytes) throws IOException { - - if (logger.isTraceEnabled()) logger.trace(">>> writeBytes({})", bytes.length); - - if (bytes != null) { - // create JOF file path if not exist - File jofFile = new File(getFullPath()); - File jofFilePath = new File(jofFile.getParent()); - if (!jofFilePath.exists()) { - jofFilePath.mkdirs(); - } - FileOutputStream jofOut = new FileOutputStream(jofFile); - jofOut.write(bytes); - jofOut.close(); - logger.info("Bytes, written to {}", getFullPath()); - return true; - } - logger.warn("writeBytes, argument bytes not set"); - return false; - } - - /* (non-Javadoc) - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#copyTo(de.dlr.proseo.storagemgr.utils.ProseoFile, java.lang.Boolean) - */ - @Override - public ArrayList copyTo(ProseoFile proFile, Boolean recursive) throws Exception { - - if (logger.isTraceEnabled()) logger.trace(">>> copyTo({}, {})", - (null == proFile ? "MISSING" : proFile.getFullPath()), recursive); - - if (proFile == null) { - logger.error("Illegal call of ProseoFilePosix::copyTo(ProseoFile, Boolean) with null argument"); - return null; - } - if (logger.isDebugEnabled()) logger.debug("Copying from {} to {}", this.getFullPath(), proFile.getFullPath()); - - ArrayList result = null; - File srcFile = new File(this.getFullPath()); - switch (proFile.getFsType()) { - case S3:// create internal buckets & prefixes if not exists.. - // *** HACK FOR DDS3 - TODO Replace constants by configurable values !!! *** - long retryCount = 0, maxRetry = 3, retryInterval = 5000 /* ms */; - while (retryCount <= maxRetry) { - try { - String targetPath = null; - if (srcFile.isDirectory()) { - StorageManagerUtils.createStorageManagerInternalS3Buckets(cfg.getS3AccessKey(), cfg.getS3SecretAccessKey(), cfg.getS3EndPoint(),cfg.getS3DefaultBucket(),cfg.getS3Region()); - S3Client s3 = S3Ops.v2S3Client(cfg.getS3AccessKey(), cfg.getS3SecretAccessKey(), cfg.getS3EndPoint(), cfg.getS3Region()); - S3Ops.createFolder(s3, cfg.getS3DefaultBucket(), proFile.getRelPath()); - result = new ArrayList(); - result.add(proFile.getFullPath()); - if (recursive) { - targetPath = proFile.getRelPath(); - } - } else { - targetPath = proFile.getRelPath(); - } - if (targetPath != null) { - if (targetPath.endsWith("/")) { - targetPath = targetPath.substring(0, targetPath.length() - 1); - } - StorageManagerUtils.createStorageManagerInternalS3Buckets(cfg.getS3AccessKey(), cfg.getS3SecretAccessKey(), cfg.getS3EndPoint(),cfg.getS3DefaultBucket(),cfg.getS3Region()); - AmazonS3 s3 = S3Ops.v1S3Client(cfg.getS3AccessKey(), cfg.getS3SecretAccessKey(), cfg.getS3EndPoint(), cfg.getS3Region()); - result = S3Ops.v1Upload( - //the client - s3, - // the local POSIX source file or directory - this.getFullPath(), - // the storageId -> =BucketName - cfg.getS3DefaultBucket(), - // the final prefix of the file or directory - targetPath, - false - ); - if (null != result) { - break; // No exception and a valid result - } - } - } catch (Exception e) { - if (retryCount >= maxRetry) { - throw e; - } // else try again, see below - } - ++retryCount; - StorageLogger.logInfo(logger, MSG_S3_REQUEST_FAILED_RETRYING, MSG_ID_S3_REQUEST_FAILED_RETRYING, - retryInterval, retryCount, maxRetry); - Thread.sleep(retryInterval); - } - break; - case POSIX: - result = new ArrayList(); - ArrayList files = new ArrayList(); - if (srcFile.isDirectory()) { - if (recursive) { - File[] srcFiles = srcFile.listFiles(); - for (File f : srcFiles) { - files.add(f); - } - } else { - File targetFile = new File(proFile.getFullPath()); - if (this.isDirectory()) { - if (!targetFile.exists()) { - FileUtils.forceMkdir(targetFile); - } - result.add(proFile.getFullPath()); - } - } - } else { - if (srcFile.isFile()) { - files.add(srcFile); - } else { - logger.error("Cannot find source file {}", srcFile); - } - } - for (File f : files) { - String targetFileName = proFile.getFullPath(); - if (proFile.isDirectory()) { - targetFileName += File.separator + f.getName(); - } - File targetFile = new File(targetFileName); - if (FileCache.getInstance().containsKey(targetFile.getPath())) { // if (targetFile.exists()) - result.add(targetFile.getPath()); - } else { - FileUtils.copyFile(f, targetFile); - if (targetFile.exists()) { - targetFile.setWritable(true, false); - FileCache.getInstance().put(targetFile.getPath()); - result.add(targetFile.getPath()); - } else { - logger.error("Cannot copy from source {} to target {}", f.getCanonicalPath(), targetFile.getCanonicalPath()); - } - } - } - break; - case ALLUXIO: - break; - default: - break; - } - if (logger.isDebugEnabled()) logger.debug("Files copied: {}", result); - return result; - } - - /* (non-Javadoc) - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#delete() - */ - @Override - public ArrayList delete() { - - if (logger.isTraceEnabled()) logger.trace(">>> delete()"); - - ArrayList result = new ArrayList(); - File srcFile = new File(this.getFullPath()); - if (srcFile.isDirectory()) { - File[] files = srcFile.listFiles(); - if(files!=null) { - for(File f: files) { - if(f.isDirectory()) { - try { - FileUtils.deleteDirectory(f); - } catch (IOException e) { - e.printStackTrace(); - } - } else { - f.delete(); - } - } - } - result.add(getFullPath()); - } else if (srcFile.exists()) { - srcFile.delete(); - result.add(getFullPath()); - } - return result; - } - - /* (non-Javadoc) - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#list() - */ - @Override - public ArrayList list() { - - if (logger.isTraceEnabled()) logger.trace(">>> list()"); - - ArrayList list = new ArrayList(); - File srcFile = new File(this.getFullPath()); - if (srcFile.isDirectory()) { - Collection files = FileUtils.listFilesAndDirs(srcFile, TrueFileFilter.INSTANCE , TrueFileFilter.INSTANCE ); - for (File file : files) { - File tmpFile = new File(file.getAbsolutePath()); - if (tmpFile.isFile()) { - list.add(new ProseoFilePosix(file.getAbsolutePath(), true, cfg)); - } - } - } - return list; - } - - /* (non-Javadoc) - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#getFileSystemResource() - */ - @Override - public FileSystemResource getFileSystemResource() { - return new FileSystemResource(getFullPath()); - } - - /* (non-Javadoc) - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#getLength() - */ - @Override - public long getLength() { - - if (logger.isTraceEnabled()) logger.trace(">>> getLength()"); - - File f = new File(getFullPath()); - if (f.isFile()) { - return f.length(); - } else { - return 0; - } - } -} diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/ProseoFileS3.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/ProseoFileS3.java deleted file mode 100644 index 3116c59de..000000000 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/ProseoFileS3.java +++ /dev/null @@ -1,445 +0,0 @@ -package de.dlr.proseo.storagemgr.utils; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.core.io.FileSystemResource; - -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3URI; - -import de.dlr.proseo.storagemgr.StorageManagerConfiguration; -import de.dlr.proseo.storagemgr.cache.FileCache; -import de.dlr.proseo.storagemgr.fs.s3.S3Ops; -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; - -/** - * Proseo file representing S3 file system - * - * @author melchinger - * - */ -public class ProseoFileS3 extends ProseoFile { - - private static final int MSG_ID_TARGET_PATH_MISSING = 4100; - private static final int MSG_ID_ALLUXIO_NOT_SUPPORTED = 4101; - private static final int MSG_ID_RECURSIVE_NOT_SUPPORTED = 4102; - private static final int MSG_ID_S3_TO_S3_COPY_FAILED = 4103; - private static final int MSG_ID_S3_TO_POSIX_COPY_FAILED = 4104; - private static final int MSG_ID_S3_DELETE_FAILED = 4105; - private static final int MSG_ID_S3_LIST_FAILED = 4106; - private static final int MSG_ID_S3_GET_STREAM_FAILED = 4107; - private static final int MSG_ID_S3_WRITE_FAILED = 4108; - private static final int MSG_ID_S3_REQUEST_FAILED_RETRYING = 4109; - - private static final String MSG_TARGET_PATH_MISSING = "(E%d) No target path given"; - private static final String MSG_ALLUXIO_NOT_SUPPORTED = "(E%d) Copying S3 objects to Alluxio is not supported"; - private static final String MSG_RECURSIVE_NOT_SUPPORTED = "(E%d) Recursive copying from S3 bucket not implemented"; - private static final String MSG_S3_TO_S3_COPY_FAILED = "(E%d) Retrieval of S3 object %s/%s into S3 object %s/%s failed"; - private static final String MSG_S3_TO_POSIX_COPY_FAILED = "(E%d) Retrieval of S3 object %s into POSIX file %s failed"; - private static final String MSG_S3_DELETE_FAILED = "(E%d) Deletion of S3 object %s failed (cause: %s)"; - private static final String MSG_S3_LIST_FAILED = "(E%d) Listing of S3 object %s failed (cause: %s)"; - private static final String MSG_S3_GET_STREAM_FAILED = "(E%d) Retrieving S3 object %s as stream failed (cause: %s)"; - private static final String MSG_S3_WRITE_FAILED = "(E%d) Writing content to S3 object %s failed (cause: %s)"; - private static final String MSG_S3_REQUEST_FAILED_RETRYING = "(I%d) S3 request failed, retrying after %d ms (attempt %d of %d)"; - - - private static Logger logger = LoggerFactory.getLogger(ProseoFileS3.class); - - /** - * Create a new S3 file. - * - * @param pathInfo - * The file path - * @param fullPath - * Use it as full path if true, otherwise use default bucket + - * path info - * @param cfg - * the Storage Manager configuration to use - */ - public ProseoFileS3(String pathInfo, Boolean fullPath, StorageManagerConfiguration cfg) { - - if (logger.isTraceEnabled()) - logger.trace(">>> ProseoFileS3({}, {}, {})", pathInfo, fullPath, cfg); - - this.cfg = cfg; - String aPath = pathInfo.trim(); - this.pathInfo = aPath; - if (fullPath) { - if (aPath.startsWith("s3:/") || aPath.startsWith("S3:/")) { - aPath = aPath.substring(4); - } - while (aPath.startsWith("/")) { - aPath = aPath.substring(1); - } - int pos = aPath.indexOf('/'); - if (pos >= 0) { - basePath = aPath.substring(0, pos); - relPath = aPath.substring(pos + 1); - } else { - basePath = ""; - relPath = aPath; - } - } else { - while (aPath.startsWith("/")) { - aPath = aPath.substring(1); - } - relPath = aPath; - basePath = cfg.getS3DefaultBucket(); - } - buildFileName(); - pathInfo = getFullPath(); - - logger.trace("ProseoFileS3 created: {}", this.getFullPath()); - } - - /** - * Create a new S3 file. - * - * @param bucket - * The bucket - * @param pathInfo - * The relative path - * @param cfg - * the Storage Manager configuration to use - */ - public ProseoFileS3(String bucket, String pathInfo, StorageManagerConfiguration cfg) { - - if (logger.isTraceEnabled()) - logger.trace(">>> ProseoFileS3({}, {}, {})", bucket, pathInfo, cfg); - - String aPath = pathInfo.trim(); - relPath = aPath; - basePath = bucket.trim(); - while (basePath.startsWith("/")) { - basePath = basePath.substring(1); - } - buildFileName(); - pathInfo = getFullPath(); - - logger.trace("ProseoFileS3 created: {}", this.getFullPath()); - } - - /* - * (non-Javadoc) - * - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#getFsType() - */ - @Override - public StorageType getFsType() { - return StorageType.S3; - } - - /* - * (non-Javadoc) - * - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#getFullPath() - */ - @Override - public String getFullPath() { - - if (logger.isTraceEnabled()) - logger.trace(">>> getFullPath()"); - - return "s3://" + getBasePath() + "/" + getRelPathAndFile(); - } - - /* - * (non-Javadoc) - * - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#getDataAsInputStream() - */ - @Override - public InputStream getDataAsInputStream() { - - if (logger.isTraceEnabled()) - logger.trace(">>> getDataAsInputStream()"); - - InputStream inputStream = null; - try { - boolean success = false; - // *** HACK FOR DDS3 - TODO Replace constants by configurable values !!! *** - long retryCount = 0, maxRetry = 3, retryInterval = 5000 /* ms */; - while (retryCount <= maxRetry && !success) { - try { - S3Client s3 = S3Ops.v2S3Client(cfg.getS3AccessKey(), cfg.getS3SecretAccessKey(), cfg.getS3EndPoint(), cfg.getS3Region()); - inputStream = S3Ops.v2FetchStream(s3, getFullPath()); - if (null != inputStream) { - break; // No exception and a valid result - } - } catch (Exception e) { - if (retryCount >= maxRetry) { - throw e; - } // else try again, see below - } - ++retryCount; - StorageLogger.logInfo(logger, MSG_S3_REQUEST_FAILED_RETRYING, MSG_ID_S3_REQUEST_FAILED_RETRYING, - retryInterval, retryCount, maxRetry); - Thread.sleep(retryInterval); - } - logger.info("Successfully read from {}", getFullPath()); - } catch (Exception e) { - e.printStackTrace(); - throw new RuntimeException(StorageLogger.logError(logger, MSG_S3_GET_STREAM_FAILED, MSG_ID_S3_GET_STREAM_FAILED, - this.getFullPath(), e.getMessage())); - } - return inputStream; - } - - /* - * (non-Javadoc) - * - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#writeBytes(byte[]) - */ - @Override - public Boolean writeBytes(byte[] bytes) throws Exception { - - if (logger.isTraceEnabled()) - logger.trace(">>> writeBytes({})", bytes.length); - - if (bytes != null) { - InputStream fis = new ByteArrayInputStream(bytes); - // create internal buckets, if not existing - StorageManagerUtils.createStorageManagerInternalS3Buckets(cfg.getS3AccessKey(), cfg.getS3SecretAccessKey(), - cfg.getS3EndPoint(), getBasePath(), cfg.getS3Region()); - S3Client s3 = S3Ops.v2S3Client(cfg.getS3AccessKey(), cfg.getS3SecretAccessKey(), cfg.getS3EndPoint(), - cfg.getS3Region()); - boolean putOK = false; - try { - s3.putObject(PutObjectRequest.builder().bucket(getBasePath()).key(getRelPathAndFile()).build(), - RequestBody.fromInputStream(fis, bytes.length)); - putOK = true; - logger.info("Bytes, written to {}", getFullPath()); - } catch (Exception e) { - e.printStackTrace(); - throw new RuntimeException(StorageLogger.logError(logger, MSG_S3_WRITE_FAILED, MSG_ID_S3_WRITE_FAILED, - this.getFullPath(), e.getMessage())); - } - s3.close(); - fis.close(); - return putOK; - } - logger.warn("writeBytes, arument bytes not set"); - return false; - } - - /* - * (non-Javadoc) - * - * @see - * de.dlr.proseo.storagemgr.utils.ProseoFile#copyTo(de.dlr.proseo.storagemgr - * .utils.ProseoFile, java.lang.Boolean) - */ - @Override - public ArrayList copyTo(ProseoFile proFile, Boolean recursive) throws Exception { - - if (logger.isTraceEnabled()) - logger.trace(">>> copyTo({}, {})", (null == proFile ? "MISSING" : proFile.getFullPath()), recursive); - - if (null == proFile) { - throw new IllegalArgumentException(StorageLogger.logError(logger, MSG_TARGET_PATH_MISSING, MSG_ID_TARGET_PATH_MISSING)); - } - if (null == recursive) { - recursive = false; - } - - ArrayList result = null; - AmazonS3URI s3uri = new AmazonS3URI(this.getFullPath()); - String sourceBucket = s3uri.getBucket(); - String sourceKey = s3uri.getKey(); - switch (proFile.getFsType()) { - case S3: - // create internal buckets & prefixes if not exists.. - StorageManagerUtils.createStorageManagerInternalS3Buckets(cfg.getS3AccessKey(), cfg.getS3SecretAccessKey(), - cfg.getS3EndPoint(), cfg.getS3DefaultBucket(), cfg.getS3Region()); - AmazonS3 s3 = S3Ops.v1S3Client(cfg.getS3AccessKey(), cfg.getS3SecretAccessKey(), cfg.getS3EndPoint(), - cfg.getS3Region()); - result = S3Ops.v1Copy( - // the client - s3, - // the source S3-Bucket - sourceBucket, - // the source key - sourceKey, - // the target s3-bucket (=storageId) - cfg.getS3DefaultBucket(), - // the final prefix including productId pattern of the - // file or directory - proFile.getRelPath()); - if (null == result) { - throw new RuntimeException(StorageLogger.logError( - logger, MSG_S3_TO_S3_COPY_FAILED, MSG_ID_S3_TO_S3_COPY_FAILED, - sourceBucket, sourceKey, - cfg.getS3DefaultBucket(), proFile.getRelPath())); - } - break; - case POSIX: - result = new ArrayList(); - // TODO recursive - if (recursive) { - throw new UnsupportedOperationException(StorageLogger.logError(logger, MSG_RECURSIVE_NOT_SUPPORTED, MSG_ID_RECURSIVE_NOT_SUPPORTED)); - } - File targetFile = new File(proFile.getFullPath()); - if (this.isDirectory()) { - if (!targetFile.exists()) { - FileUtils.forceMkdir(targetFile); - } - result.add(proFile.getFullPath()); - } else { - if (FileCache.getInstance().containsKey(proFile.getFullPath())) { // if - // (targetFile.exists()) - result.add(proFile.getFullPath()); - } else { - // *** HACK FOR DDS3 - TODO Replace constants by configurable values !!! *** - long retryCount = 0, maxRetry = 3, retryInterval = 5000 /* ms */; - boolean success = false; - while (retryCount <= maxRetry ) { - if (retryCount < maxRetry) { - // Try standard download method using AWS S3 SDK V1 Transfer Manager - AmazonS3 s3v1c = S3Ops.v1S3Client(cfg.getS3AccessKey(), cfg.getS3SecretAccessKey(), cfg.getS3EndPoint(), - cfg.getS3Region()); - if (S3Ops.v1FetchFile( - // the client - s3v1c, - // the source S3-Bucket - this.getBasePath(), - // the source S3 object key - this.getRelPathAndFile(), - // the target path on POSIX file system - targetFile)) { - success = true; - break; // No exception and a valid result - } - } else { - // Last try: Use single-threaded AWS S3 SDK V2 GetObject method - S3Client s3v2c = S3Ops.v2S3Client(cfg.getS3AccessKey(), cfg.getS3SecretAccessKey(), cfg.getS3EndPoint(), - cfg.getS3Region()); - if (S3Ops.v2FetchFile( - // the client - s3v2c, - // the source S3-Bucket - this.getFullPath(), - // the final prefix including productId pattern - // of the file or directory - proFile.getFullPath())) { - success = true; - break; // No exception and a valid result - } - } - ++retryCount; - if (retryCount <= maxRetry) { - StorageLogger.logInfo(logger, MSG_S3_REQUEST_FAILED_RETRYING, MSG_ID_S3_REQUEST_FAILED_RETRYING, - retryInterval, retryCount, maxRetry); - Thread.sleep(retryInterval); - } - } - if (success) { - targetFile.setWritable(true, false); - FileCache.getInstance().put(proFile.getFullPath()); - result.add(proFile.getFullPath()); - } else { - proFile.delete(); - throw new RuntimeException(StorageLogger.logError( - logger, MSG_S3_TO_POSIX_COPY_FAILED, MSG_ID_S3_TO_POSIX_COPY_FAILED, - this.getFullPath(), proFile.getFullPath())); - } - } - } - break; - case ALLUXIO: - throw new UnsupportedOperationException(StorageLogger.logError(logger, MSG_ALLUXIO_NOT_SUPPORTED, MSG_ID_ALLUXIO_NOT_SUPPORTED)); - default: - break; - } - - return result; - } - - /* - * (non-Javadoc) - * - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#delete() - */ - @Override - public ArrayList delete() { - - if (logger.isTraceEnabled()) - logger.trace(">>> delete()"); - - ArrayList result = new ArrayList(); - AmazonS3 s3 = S3Ops.v1S3Client(cfg.getS3AccessKey(), cfg.getS3SecretAccessKey(), cfg.getS3EndPoint(), cfg.getS3Region()); - try { - S3Ops.deleteDirectory(s3, getBasePath(), getRelPathAndFile()); - result.add(getFullPath()); - } catch (Exception e) { - e.printStackTrace(); - throw new RuntimeException(StorageLogger.logError(logger, MSG_S3_DELETE_FAILED, MSG_ID_S3_DELETE_FAILED, - this.getFullPath(), e.getMessage())); - } - return result; - } - - /* - * (non-Javadoc) - * - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#list() - */ - @Override - public ArrayList list() { - - if (logger.isTraceEnabled()) - logger.trace(">>> list()"); - - ArrayList list = new ArrayList(); - try { - StorageManagerUtils.createStorageManagerInternalS3Buckets(cfg.getS3AccessKey(), cfg.getS3SecretAccessKey(), - cfg.getS3EndPoint(), cfg.getS3DefaultBucket(), cfg.getS3Region()); - AmazonS3 s3 = S3Ops.v1S3Client(cfg.getS3AccessKey(), cfg.getS3SecretAccessKey(), cfg.getS3EndPoint(), - cfg.getS3Region()); - List files = S3Ops.listObjectsInBucket(s3, getBasePath(), getRelPath()); - for (String f : files) { - list.add(new ProseoFileS3(f, true, cfg)); - } - } catch (Exception e) { - e.printStackTrace(); - throw new RuntimeException(StorageLogger.logError(logger, MSG_S3_LIST_FAILED, MSG_ID_S3_LIST_FAILED, - this.getFullPath(), e.getMessage())); - } - - return list; - } - - /* - * (non-Javadoc) - * - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#getFileSystemResource() - */ - @Override - public FileSystemResource getFileSystemResource() { - return new FileSystemResource(getFullPath()); - } - - /* - * (non-Javadoc) - * - * @see de.dlr.proseo.storagemgr.utils.ProseoFile#getLength() - */ - @Override - public long getLength() { - if (isDirectory()) { - return 0; - } else { - AmazonS3 s3 = S3Ops.v1S3Client(cfg.getS3AccessKey(), cfg.getS3SecretAccessKey(), cfg.getS3EndPoint(), - cfg.getS3Region()); - return S3Ops.getLength(s3, getBasePath(), getRelPathAndFile()); - } - } -} diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/StorageFileLocker.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java similarity index 84% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/StorageFileLocker.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java index db3e510d8..0f044fe26 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/StorageFileLocker.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java @@ -3,12 +3,12 @@ * * (C) 2022 Dr. Bassler & Co. Managementberatung GmbH */ -package de.dlr.proseo.storagemgr.version2; +package de.dlr.proseo.storagemgr.utils; import java.util.concurrent.ConcurrentSkipListSet; import de.dlr.proseo.logging.logger.ProseoLogger; -import de.dlr.proseo.storagemgr.version2.Exceptions.FileLockedAfterMaxCyclesException; +import de.dlr.proseo.storagemgr.Exceptions.FileLockedAfterMaxCyclesException; /** * A mechanism to lock and unlock files during the download process. It helps @@ -65,19 +65,23 @@ public StorageFileLocker(String path, long waitTime, long fileCheckMaxCycles) { * waiting for the concurrent access * to terminate */ - public void lock() throws FileLockedAfterMaxCyclesException, InterruptedException { + public void lockOrWaitUntilUnlockedAndLock() throws FileLockedAfterMaxCyclesException, InterruptedException { long i = 0; for (; i < fileCheckMaxCycles; ++i) { - synchronized (productLockSet) { + // lock the file if not locked (not in the set) + synchronized (productLockSet) { if (!productLockSet.contains(path)) { + productLockSet.add(path); + logger.debug("... the file {} is locked", path); break; } } + // wait, because file is locked (in the set) in another thread if (logger.isDebugEnabled()) logger.debug("... waiting for concurrent access to {} to terminate", path); Thread.sleep(waitTime); @@ -94,5 +98,6 @@ public void lock() throws FileLockedAfterMaxCyclesException, InterruptedExceptio */ public void unlock() { productLockSet.remove(path); + logger.debug("... the file {} is unlocked", path); } } \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageLogger.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageLogger.java deleted file mode 100644 index e49ba14e8..000000000 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageLogger.java +++ /dev/null @@ -1,73 +0,0 @@ -package de.dlr.proseo.storagemgr.utils; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.slf4j.Logger; - -/** - * General logger methods - * - * @author Denys Chaykovskiy - * - */ -public class StorageLogger { - - /** - * Create and log a formatted informational message - * - * @param loggger Logger object of the class - * @param messageFormat the message text with parameter placeholders in - * String.format() style - * @param messageId a (unique) message id - * @param messageParameters the message parameters (optional, depending on the - * message format) - * @return a formatted info message - */ - public static String logInfo(Logger logger, String messageFormat, int messageId, Object... messageParameters) { - - String message = createMessage(messageFormat, messageId, messageParameters); - logger.info(message); - - return message; - } - - /** - * Create and log a formatted error message - * - * @param loggger Logger object of the class - * @param messageFormat the message text with parameter placeholders in - * String.format() style - * @param messageId a (unique) message id - * @param messageParameters the message parameters (optional, depending on the - * message format) - * @return a formatted error message - */ - public static String logError(Logger logger, String messageFormat, int messageId, Object... messageParameters) { - - String message = createMessage(messageFormat, messageId, messageParameters); - logger.error(message); - - return message; - } - - /** - * Create a formatted log message - * - * @param messageFormat the message text with parameter placeholders in - * String.format() style - * @param messageId a (unique) message id - * @param messageParameters the message parameters (optional, depending on the - * message format) - * @return a formatted error message - */ - private static String createMessage(String messageFormat, int messageId, Object... messageParameters) { - - List messageParamList = new ArrayList<>(Arrays.asList(messageParameters)); - messageParamList.add(0, messageId); - - return String.format(messageFormat, messageParamList.toArray()); - } - -} diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageManagerUtils.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageManagerUtils.java deleted file mode 100644 index 2e06df2a1..000000000 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageManagerUtils.java +++ /dev/null @@ -1,169 +0,0 @@ -package de.dlr.proseo.storagemgr.utils; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.StringReader; -import java.nio.charset.Charset; -import java.util.ArrayList; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.xml.sax.ErrorHandler; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import org.xml.sax.SAXParseException; - -import de.dlr.proseo.storagemgr.fs.s3.S3Ops; -import software.amazon.awssdk.services.s3.S3Client; - -/** - * General utility methods - * - * @author melchinger - * - */ -public class StorageManagerUtils { - - private static final String DIRECTORY_ROOT = "/"; - private static final String PROTOCOL_ALLUXIO = "alluxio://"; - private static final String PROTOCOL_S3 = "s3://"; - /** - * Logger of this class - */ - private static Logger logger = LoggerFactory.getLogger(StorageManagerUtils.class); - - /** - * Creates an internal bucket for use by the Storage Manager itself; passes all exceptions on to the caller - * - * @param s3AccessKey the access key for the client - * @param s3SecretAccesKey the secret access key for the client - * @param s3Endpoint the S3 endpoint to connect to - * @param bucketName the name of the bucket to create - * @param region the region, on which the client shall operate - * @return true, if the bucket exists or was created successfully, false otherwise - */ - public static Boolean createStorageManagerInternalS3Buckets(String s3AccessKey, String s3SecretAccesKey, String s3Endpoint, - String bucketName, String region) { - - if (logger.isTraceEnabled()) logger.trace(">>> createStorageManagerInternalS3Buckets(***, ***, {}, {}, {}", - s3Endpoint, bucketName, region); - - S3Client s3 = S3Ops.v2S3Client(s3AccessKey, s3SecretAccesKey, s3Endpoint, region); - ArrayList buckets = S3Ops.listBuckets(s3); - if (!buckets.contains(bucketName)) { - String bckt = S3Ops.createBucket(s3, bucketName, region); - if (null == bckt) return false; - } - return true; - } - - /** - * Check if the provided String represents a valid XML Document - * - * @param xml the input String - * @return true, if the string is valid, false otherwise - */ - public static Boolean checkXml(String xml) { - - if (logger.isTraceEnabled()) logger.trace(">>> checkXml(String)"); - - DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); - - try { - DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); - dBuilder.setErrorHandler(new ErrorHandler() { - @Override - public void warning(SAXParseException e) throws SAXException { - } - - @Override - public void fatalError(SAXParseException e) throws SAXException { - } - - @Override - public void error(SAXParseException e) throws SAXException { - } - }); - dBuilder.parse(new InputSource(new StringReader(xml))); - return true; - } catch (Exception e) { - return false; - } - } - - /** - * Converts an InputStrem to String - * - * @param inputStream the input stream to read - * @param charset the charset of the string (defaults to UTF-8) - * @return String a string containing the data of the input stream - * @throws IOException if an I/O error occurs - */ - public static String inputStreamToString(InputStream inputStream, Charset charset) throws IOException { - - if (logger.isTraceEnabled()) logger.trace(">>> inputStreamToString(InputStream, Charset)"); - - StringBuilder stringBuilder = new StringBuilder(); - String line = null; - - try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream, charset))) { - while ((line = bufferedReader.readLine()) != null) { - stringBuilder.append(line); - } - } - inputStream.close(); - return stringBuilder.toString(); - } - - /** - * Get the storage type of a given path string - * - * @param pathInfo the path to check - * @return the storage type indicated by this path - */ - public static StorageType getFsType(String pathInfo) { - - if (logger.isTraceEnabled()) logger.trace(">>> getFsType({})", pathInfo); - - StorageType storageType = null; - if (pathInfo != null) { - // Find storage type - if (pathInfo.toLowerCase().startsWith(PROTOCOL_S3)) { // upper case and lower case protocol name allowed! - storageType = StorageType.S3; - } else if (pathInfo.startsWith(PROTOCOL_ALLUXIO)) { - storageType = StorageType.ALLUXIO; - } else if (pathInfo.startsWith(DIRECTORY_ROOT)) { - storageType = StorageType.POSIX; - } - } - return storageType; - } - /** - * Remove storage dependent path information - * - * @param pathInfo the full path string - * @return the path string without the protocol part - */ - public static String getRelativePath(String pathInfo) { - - if (logger.isTraceEnabled()) logger.trace(">>> getRelativePath({})", pathInfo); - - String relPath = null; - if (pathInfo != null) { - relPath = pathInfo.trim(); - // Remove protocol from path - if (relPath.toLowerCase().startsWith(PROTOCOL_S3)) { // upper case and lower case protocol name allowed! - relPath = DIRECTORY_ROOT + relPath.substring(PROTOCOL_S3.length()); - } else if (relPath.startsWith(PROTOCOL_ALLUXIO)) { - relPath = DIRECTORY_ROOT + relPath.substring(PROTOCOL_ALLUXIO.length()); - } - - } - return relPath; - } -} diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageType.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageType.java deleted file mode 100644 index 7826eaf5e..000000000 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageType.java +++ /dev/null @@ -1,18 +0,0 @@ -package de.dlr.proseo.storagemgr.utils; - -/** - * Storage types of prosEO - * - * @author melchinger - * - */ -public enum StorageType { - /** Type S3 */ - S3, - /** Type ALLUXIO, not used yet */ - ALLUXIO, - /** Type POSIX */ - POSIX, - /** Type OTHER, undefined type */ - OTHER -} diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/package-info.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/package-info.java similarity index 81% rename from storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/package-info.java rename to storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/package-info.java index 5cd76aeb4..0943b26bb 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/version2/package-info.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/package-info.java @@ -8,4 +8,4 @@ * * @author Denys Chaykovskiy */ -package de.dlr.proseo.storagemgr.version2; \ No newline at end of file +package de.dlr.proseo.storagemgr.utils; \ No newline at end of file diff --git a/storage-mgr/src/main/resources/application.yml b/storage-mgr/src/main/resources/application.yml index 54ef29a51..292c033a4 100644 --- a/storage-mgr/src/main/resources/application.yml +++ b/storage-mgr/src/main/resources/application.yml @@ -8,7 +8,6 @@ logging: proseo: global: storageIdPrefix: proseo - storageManagerVersion2: true # Default mount point for files to upload sourcePath: default diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java index 42575d4df..cc5ebd543 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java @@ -6,20 +6,57 @@ import java.nio.file.Paths; import java.util.List; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.posix.PosixDAL; -import de.dlr.proseo.storagemgr.version2.posix.PosixStorageFile; +import javax.annotation.PostConstruct; + +import org.junit.Rule; +import org.junit.rules.TestName; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.posix.PosixDAL; +import de.dlr.proseo.storagemgr.posix.PosixStorageFile; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * @author Denys Chaykovskiy * */ +@Component public class BaseStorageTestUtils { + + @Autowired + private StorageProvider storageProvider; + + @Autowired + private TestUtils testUtils; + + @Rule + public TestName testName = new TestName(); protected String sourcePath; protected String storagePath; protected String cachePath; + + @PostConstruct + private void init() { + + sourcePath = testUtils.getSourcePath(); + storagePath = testUtils.getStoragePath(); + cachePath = testUtils.getCachePath(); + + theTestUtils = this; + + // storageProvider = new StorageProvider(); + } + + private static BaseStorageTestUtils theTestUtils; + + public static BaseStorageTestUtils getInstance() { + + return theTestUtils; + } public StorageFile getSourceFile(String relativePath) { @@ -93,7 +130,7 @@ public String getAbsoluteSourcePath(String relativePath) { public void uploadToPosixStorage(String relativePath) { - PosixDAL posixDAL = new PosixDAL(); + PosixDAL posixDAL = new PosixDAL(storageProvider.getPosixConfigurationFromFile()); StorageFile sourceFile = getSourceFile(relativePath); StorageFile destFile = getStorageFile(relativePath); @@ -112,7 +149,7 @@ public void uploadToPosixStorage(String relativePath) { public void downloadFromPosixStorage(String relativePath) { - PosixDAL posixDAL = new PosixDAL(); + PosixDAL posixDAL = new PosixDAL(storageProvider.getPosixConfigurationFromFile()); StorageFile sourceFile = getStorageFile(relativePath); StorageFile destFile = getCacheFile(relativePath); @@ -156,4 +193,28 @@ public String getCachePath() { public String getSourcePath() { return sourcePath; } + + public static void printStorageFiles(String message, Storage storage) { + + List storageFiles; + try { + storageFiles = storage.getRelativeFiles(); + String storageType = storage.getStorageType().toString(); + TestUtils.printList(message + ". Storage " + storageType + " files || " + storage.getAbsoluteBasePath(), storageFiles); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public static void printStorageFilesWithPrefix(String message, Storage storage, String prefix) { + + List storageFiles; + try { + storageFiles = storage.getRelativeFiles(prefix); + String storageType = storage.getStorageType().toString(); + TestUtils.printList(message + ". Storage " + storageType + " files || Prefix: " + prefix + " || " + storage.getAbsoluteBasePath(), storageFiles); + } catch (IOException e) { + e.printStackTrace(); + } + } } diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StorageTestUtils.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StorageTestUtils.java deleted file mode 100644 index 16709410e..000000000 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StorageTestUtils.java +++ /dev/null @@ -1,80 +0,0 @@ -package de.dlr.proseo.storagemgr; - -import java.io.IOException; -import java.util.List; - -import javax.annotation.PostConstruct; - -import org.junit.Rule; -import org.junit.rules.TestName; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.Storage; - -/** - * @author Denys Chaykovskiy - * - */ -@Component -public class StorageTestUtils extends BaseStorageTestUtils { - - - @Autowired - private TestUtils testUtils; - - @Autowired - private StorageProvider storageProvider; - - @Rule - public TestName testName = new TestName(); - - @PostConstruct - private void init() { - - sourcePath = testUtils.getSourcePath(); - storagePath = testUtils.getStoragePath(); - cachePath = testUtils.getCachePath(); - - theTestUtils = this; - - // storageProvider = new StorageProvider(); - } - - private static StorageTestUtils theTestUtils; - - public static StorageTestUtils getInstance() { - - return theTestUtils; - } - - public void printVersion(String msg) { - - System.out.println(msg + (storageProvider.isVersion2() ? " Version-2" : " Version-1")); - } - - public static void printStorageFiles(String message, Storage storage) { - - List storageFiles; - try { - storageFiles = storage.getRelativeFiles(); - String storageType = storage.getStorageType().toString(); - TestUtils.printList(message + ". Storage " + storageType + " files || " + storage.getAbsoluteBasePath(), storageFiles); - } catch (IOException e) { - e.printStackTrace(); - } - } - - public static void printStorageFilesWithPrefix(String message, Storage storage, String prefix) { - - List storageFiles; - try { - storageFiles = storage.getRelativeFiles(prefix); - String storageType = storage.getStorageType().toString(); - TestUtils.printList(message + ". Storage " + storageType + " files || Prefix: " + prefix + " || " + storage.getAbsoluteBasePath(), storageFiles); - } catch (IOException e) { - e.printStackTrace(); - } - } -} diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java index c6ae4abd9..340027892 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java @@ -16,11 +16,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; -import de.dlr.proseo.storagemgr.version2.FileUtils; -import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; +import de.dlr.proseo.storagemgr.utils.FileUtils; /** * @author Denys Chaykovskiy @@ -321,6 +320,18 @@ public static void printList(String message, List list) { } System.out.println(); } + + /** + * @param message + * @param directoryPath + */ + public static void printDirectoryTree(String message, String directoryPath) { + + System.out.println(); + System.out.println(message); + + printDirectoryTree(directoryPath); + } /** * @param directoryPath @@ -342,7 +353,7 @@ public static void printDirectoryTree(String directoryPath) { System.out.print(" Files: " + countFilesInDirectory(directoryPath)); System.out.println(" Folders: " + countDirectoriesInDirectory(directoryPath)); - printDirectoryTree(directoryPath, ""); + printDirectoryTreeWithDepth(directoryPath, ""); System.out.println(); } @@ -350,7 +361,7 @@ public static void printDirectoryTree(String directoryPath) { * @param directoryPath * @param depth */ - private static void printDirectoryTree(String directoryPath, String depth) { + private static void printDirectoryTreeWithDepth(String directoryPath, String depth) { File directory = new File(directoryPath); @@ -374,7 +385,7 @@ private static void printDirectoryTree(String directoryPath, String depth) { for (File file : files) { if (file.isDirectory()) { System.out.println(depth + file.getName() + " "); - printDirectoryTree(file.getPath(), OUTPUT_TAB + depth); + printDirectoryTreeWithDepth(file.getPath(), OUTPUT_TAB + depth); } } } diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/UniqueStorageTestPaths.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/UniqueStorageTestPaths.java index 2ba3c7a25..a1a2da79a 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/UniqueStorageTestPaths.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/UniqueStorageTestPaths.java @@ -4,7 +4,7 @@ import org.junit.rules.TestName; -import de.dlr.proseo.storagemgr.version2.PathConverter; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * @author Denys Chaykovskiy diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java index aaab28234..71d618d84 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java @@ -20,7 +20,7 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.TestUtils; -import de.dlr.proseo.storagemgr.version2.FileUtils; +import de.dlr.proseo.storagemgr.utils.FileUtils; /** * @author Denys Chaykovskiy @@ -40,16 +40,16 @@ public class FileCacheTest { @Autowired private FileCache fileCache; - - String testCachePath; - String cachePath; - + + String testCachePath; + String cachePath; + @PostConstruct private void init() { testCachePath = testUtils.getTestCachePath(); cachePath = testUtils.getCachePath(); } - + /** * */ @@ -58,23 +58,28 @@ public void testDeleteEmptyDirectoriesToTop() { TestUtils.printMethodName(this, testName); TestUtils.createEmptyTestDirectories(); - + String emptyDirectories = testCachePath + "/d1/d2/d3"; - - TestUtils.createDirectory(emptyDirectories); - File f = new File(emptyDirectories); - - TestUtils.printDirectoryTree(testCachePath); + + // create test directories + + TestUtils.createDirectory(emptyDirectories); + + TestUtils.printDirectoryTree("Directories after creation (expectected: /d1/d2/d3 ):", testCachePath); assertTrue("Empty Directories were not created: " + emptyDirectories, f.exists()); + // delete test directories + fileCache.deleteEmptyDirectoriesToTop(emptyDirectories); - TestUtils.printDirectoryTree(cachePath); + TestUtils.printDirectoryTree("Directories after deletion (expectected: nothing)", cachePath); assertTrue("Empty Directories were not deleted: " + emptyDirectories, !f.exists()); - + + // clear + TestUtils.deleteTestDirectories(); } @@ -87,13 +92,13 @@ public void testGetLastAccessed() { TestUtils.printMethodName(this, testName); TestUtils.createEmptyTestDirectories(); fileCache.setPath(testCachePath); - + String testFile = "testLastAccessed.txt"; String path = testCachePath + "/" + testFile; FileUtils fileUtils = new FileUtils(path); fileUtils.createFile(""); - + Instant testInstant; testInstant = fileCache.getFileAccessed(path); @@ -103,7 +108,7 @@ public void testGetLastAccessed() { File f = new File(fileCache.getAccessedPath(path)); assertTrue("Last Accessed File not exists: " + f.getPath(), f.exists() && !f.isDirectory()); - + fileCache.clear(); TestUtils.deleteTestDirectories(); } @@ -127,7 +132,7 @@ public void testGetAccessedPath() { System.out.println("Accessed Path: " + accessedPath); System.out.println("Expected Accessed Path: " + expectedAccessedPath); - assertTrue("Accessed Path is wrong: " + accessedPath, accessedPath.equals(expectedAccessedPath)); + assertTrue("Accessed Path is wrong: " + accessedPath, accessedPath.equals(expectedAccessedPath)); } /** @@ -145,20 +150,19 @@ public void testGetPutContainsRemove() { String path3 = Paths.get(testCachePath + "/test1/test2/test3.txt").toString(); String path4 = Paths.get(testCachePath + "/test4.x").toString(); String path5 = Paths.get(testCachePath + "/test5.x").toString(); - - String pathNotExists = Paths.get(testCachePath + "/xxx/xxx/zzz.txt").toString(); - + + String pathNotExists = Paths.get(testCachePath + "/xxx/xxx/zzz.txt").toString(); + TestUtils.createFile(path1, ""); TestUtils.createFile(path2, ""); TestUtils.createFile(path3, ""); - + TestUtils.createFile(path4, ""); TestUtils.createFile(path5, ""); - + fileCache.putFilesToCache(testCachePath); - assertTrue("Cache does not contain 5 elements after dir init: " + fileCache.size(), - fileCache.size() == 5); + assertTrue("Cache does not contain 5 elements after dir init: " + fileCache.size(), fileCache.size() == 5); System.out.println("Before adding the element: " + path1); @@ -182,133 +186,225 @@ public void testGetPutContainsRemove() { MapCacheTest.printCache("Cache after deleting 1 element: " + path3, fileCache.getMapCache()); TestUtils.printDirectoryTree(testCachePath); - + fileCache.clear(); TestUtils.deleteTestDirectories(); } + + /** + * + */ + @Test + public void testStatus() { + + TestUtils.printMethodName(this, testName); + TestUtils.createEmptyTestDirectories(); + fileCache.setPath(testCachePath); + + String path1 = Paths.get(testCachePath + "/" + "testStatus1.txt").toString(); + String path2 = Paths.get(testCachePath + "/" + "testStatus2.txt").toString(); + String path3 = Paths.get(testCachePath + "/" + "testStatus3.txt").toString(); + + + CacheFileStatus status1; + CacheFileStatus status2; + CacheFileStatus status3; + + + // create 2 files for cache + + TestUtils.createFile(path1, ""); + TestUtils.createFile(path2, ""); + + // put 2 files to cache + + fileCache.putFilesToCache(testCachePath); + + assertTrue("Expected Cache size is 2 elements after put(). Exists: " + fileCache.size(), fileCache.size() == 2); + + // check status after put() + + status1 = fileCache.getCacheFileStatus(path1); + status2 = fileCache.getCacheFileStatus(path2); + + assertTrue("Expected cache file1 status after put() is Ready. Exists: " + status1.toString(), + status1 == CacheFileStatus.READY); + + assertTrue("Expected cache file2 status after put() is Ready. Exists: " + status2.toString(), + status2 == CacheFileStatus.READY); + + // check not exists status - file is not in cache, but has a status + + fileCache.setCacheFileStatus(path3, CacheFileStatus.NOT_EXISTS); + + status3 = fileCache.getCacheFileStatus(path3); + + assertTrue("Expected cache file3 does not exist in the cache. Exists:" + fileCache.containsKey(path3), + !fileCache.containsKey(path3)); + + assertTrue("Expected cache file3 status after setStatus(Not_exists) is Not_exists. Exists:" + status3.toString(), + status3 == CacheFileStatus.NOT_EXISTS); + + // changing status1 to copying from cache + + fileCache.setCacheFileStatus(path1, CacheFileStatus.COPYING_FROM_CACHE); + + status1 = fileCache.getCacheFileStatus(path1); + status2 = fileCache.getCacheFileStatus(path2); + + assertTrue("Expected cache file1 status after setStatus(Uploading) is Uploading. Exists:" + status1.toString(), + status1 == CacheFileStatus.COPYING_FROM_CACHE); + + assertTrue("Expected cache file2 status after is Ready. Exists:" + status2.toString(), + status2 == CacheFileStatus.READY); + + // changing status2 to copying to cache + + fileCache.setCacheFileStatus(path2, CacheFileStatus.NOT_EXISTS); + + status1 = fileCache.getCacheFileStatus(path1); + status2 = fileCache.getCacheFileStatus(path2); + + assertTrue("Expected cache file1 status after is Uploading. Exists: " + status1.toString(), + status1 == CacheFileStatus.COPYING_FROM_CACHE); + + assertTrue("Expected cache file2 status after setStatus(Downloading) is Downlading. Exists: " + status2.toString(), + status2 == CacheFileStatus.NOT_EXISTS); + + // changing status1 to Ready + + fileCache.put(path1); // status is set as ready + + status1 = fileCache.getCacheFileStatus(path1); + status2 = fileCache.getCacheFileStatus(path2); + + assertTrue("Expected cache file1 status after setStatus(Ready) is Ready. Exists: " + status1.toString(), + status1 == CacheFileStatus.READY); + + assertTrue("Expected cache file2 status after is Downloading. Exists: " + status2.toString(), + status2 == CacheFileStatus.NOT_EXISTS); + // check cache after status changes + + assertTrue("Expected 2 elements in the cache. Exists: " + fileCache.size(), fileCache.size() == 2); + + // clear + + fileCache.clear(); + TestUtils.deleteTestDirectories(); + } + /** * */ @Test public void testInterface() { - + TestUtils.printMethodName(this, testName); TestUtils.createEmptyTestDirectories(); fileCache.setPath(testCachePath); - + String path1 = Paths.get(testCachePath + "/" + "test1.txt").toString(); String path2 = Paths.get(testCachePath + "/" + "test2.txt").toString(); String path3 = Paths.get(testCachePath + "/" + "test3.txt").toString(); String pathNotExists = Paths.get(testCachePath + "/xxx/xxx/" + " zzz.txt").toString(); Instant timeNotChanged; - Instant timeNotChanged2; - Instant timeChanged; - + Instant timeNotChanged2; + Instant timeChanged; + TestUtils.createFile(path1, ""); TestUtils.createFile(path2, ""); - + fileCache.putFilesToCache(testCachePath); - - assertTrue("Cache does not contain 2 elements after dir init: " + fileCache.size(), - fileCache.size() == 2); - - // check containsKey - contains and update accessed - + + assertTrue("Cache does not contain 2 elements after dir init: " + fileCache.size(), fileCache.size() == 2); + + // check containsKey - contains and update accessed + System.out.println("Subtest: check containsKey - contains and update accessed "); - + timeChanged = fileCache.getFileAccessed(path1); - timeNotChanged = fileCache.getFileAccessed(path2); - - assertTrue("Cache does not contain an elements after dir init: " + path1, - fileCache.containsKey(path1)); - + timeNotChanged = fileCache.getFileAccessed(path2); + + assertTrue("Cache does not contain an elements after dir init: " + path1, fileCache.containsKey(path1)); + System.out.println("Path1 Time: " + timeChanged); System.out.println("Path1 time after contains(changed): " + fileCache.getFileAccessed(path1)); - + System.out.println(); - - System.out.println("Path2 Time: " + timeNotChanged); - System.out.println("Path2 time (stable): " + fileCache.getFileAccessed(path2)); - - assertTrue("Last accessed was not updated: ", - fileCache.getFileAccessed(path1).compareTo(timeChanged) > 0); - - assertTrue("Last accessed must not be updated: ", + + System.out.println("Path2 Time: " + timeNotChanged); + System.out.println("Path2 time (stable): " + fileCache.getFileAccessed(path2)); + + assertTrue("Last accessed was not updated: ", fileCache.getFileAccessed(path1).compareTo(timeChanged) > 0); + + assertTrue("Last accessed must not be updated: ", fileCache.getFileAccessed(path2).compareTo(timeNotChanged) == 0); - - assertTrue("Cache does not contain 2 elements after contains: " + fileCache.size(), - fileCache.size() == 2); - - // check containsKey - not contains - + + assertTrue("Cache does not contain 2 elements after contains: " + fileCache.size(), fileCache.size() == 2); + + // check containsKey - not contains + System.out.println("Subtest: check containsKey - not contains"); - + timeNotChanged = fileCache.getFileAccessed(path1); - timeNotChanged2 = fileCache.getFileAccessed(path2); - - assertTrue("Cache contains an element, but must not: " + pathNotExists, - !fileCache.containsKey(pathNotExists)); - + timeNotChanged2 = fileCache.getFileAccessed(path2); + + assertTrue("Cache contains an element, but must not: " + pathNotExists, !fileCache.containsKey(pathNotExists)); + System.out.println("Path1 Time: " + timeNotChanged); System.out.println("Path1 time after contains(stable): " + fileCache.getFileAccessed(path1)); - + System.out.println(); - - System.out.println("Path2 Time: " + timeNotChanged2); - System.out.println("Path2 time after contains(stable): " + fileCache.getFileAccessed(path2)); - - assertTrue("path1 Last accessed must not be updated: ", + + System.out.println("Path2 Time: " + timeNotChanged2); + System.out.println("Path2 time after contains(stable): " + fileCache.getFileAccessed(path2)); + + assertTrue("path1 Last accessed must not be updated: ", fileCache.getFileAccessed(path1).compareTo(timeNotChanged) == 0); - - assertTrue("path2 Last accessed must not be updated: ", + + assertTrue("path2 Last accessed must not be updated: ", fileCache.getFileAccessed(path2).compareTo(timeNotChanged2) == 0); - - // check put - not contains - + + // check put - not contains + System.out.println("Subtest: check containsKey - not contains"); - - assertTrue("Cache does not contain 2 elements after contains: " + fileCache.size(), - fileCache.size() == 2); - - assertTrue("Cache contains an element before put: " + path3, - !fileCache.containsKey(path3)); - + + assertTrue("Cache does not contain 2 elements after contains: " + fileCache.size(), fileCache.size() == 2); + + assertTrue("Cache contains an element before put: " + path3, !fileCache.containsKey(path3)); + TestUtils.createFile(path3, ""); fileCache.put(path3); - - assertTrue("Cache does not contain 3 elements after contains: " + fileCache.size(), - fileCache.size() == 3); - - assertTrue("Cache does not contains an element after put: " + path3, - fileCache.containsKey(path3)); - + + assertTrue("Cache does not contain 3 elements after contains: " + fileCache.size(), fileCache.size() == 3); + + assertTrue("Cache does not contains an element after put: " + path3, fileCache.containsKey(path3)); + // check put - contains, update - + System.out.println("Subtest: check put - contains, update "); - + timeChanged = fileCache.getFileAccessed(path1); - timeNotChanged = fileCache.getFileAccessed(path2); - + timeNotChanged = fileCache.getFileAccessed(path2); + fileCache.put(path1); - + System.out.println("Path1 Time: " + timeChanged); System.out.println("Path1 time after contains(changed): " + fileCache.getFileAccessed(path1)); - + System.out.println(); - - System.out.println("Path2 Time: " + timeNotChanged); - System.out.println("Path2 time (stable): " + fileCache.getFileAccessed(path2)); - - assertTrue("Last accessed was not updated: ", - fileCache.getFileAccessed(path1).compareTo(timeChanged) > 0); - - assertTrue("Last accessed must not be updated: ", + + System.out.println("Path2 Time: " + timeNotChanged); + System.out.println("Path2 time (stable): " + fileCache.getFileAccessed(path2)); + + assertTrue("Last accessed was not updated: ", fileCache.getFileAccessed(path1).compareTo(timeChanged) > 0); + + assertTrue("Last accessed must not be updated: ", fileCache.getFileAccessed(path2).compareTo(timeNotChanged) == 0); - - assertTrue("Cache does not contain 3 elements after contains: " + fileCache.size(), - fileCache.size() == 3); - + + assertTrue("Cache does not contain 3 elements after contains: " + fileCache.size(), fileCache.size() == 3); + fileCache.clear(); TestUtils.deleteTestDirectories(); } diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileSystemGeneratorTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileSystemGeneratorTest.java index 20ad2a2f9..d0cdd6c11 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileSystemGeneratorTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileSystemGeneratorTest.java @@ -1,6 +1,6 @@ package de.dlr.proseo.storagemgr.cache; -import de.dlr.proseo.storagemgr.version2.FileUtils; +import de.dlr.proseo.storagemgr.utils.FileUtils; /** * @author Denys Chaykovskiy diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileUtilsTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileUtilsTest.java index 5ac990198..4941f77e1 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileUtilsTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileUtilsTest.java @@ -18,7 +18,7 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.TestUtils; -import de.dlr.proseo.storagemgr.version2.FileUtils; +import de.dlr.proseo.storagemgr.utils.FileUtils; /** * @author Denys Chaykovskiy diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java index 6733af34c..3ee8a9092 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProductfileControllerImplTest_cache.java @@ -1,5 +1,7 @@ package de.dlr.proseo.storagemgr.cache; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; + import static org.junit.Assert.*; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -21,17 +23,16 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.MvcResult; import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import de.dlr.proseo.storagemgr.StorageManager; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; -import de.dlr.proseo.storagemgr.version2.FileUtils; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) @@ -51,7 +52,7 @@ public class ProductfileControllerImplTest_cache { public TestName testName = new TestName(); @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; String cachePath; String storagePath; @@ -60,6 +61,7 @@ public class ProductfileControllerImplTest_cache { @PostConstruct private void init() { + cachePath = testUtils.getCachePath(); storagePath = testUtils.getStoragePath(); } @@ -69,58 +71,25 @@ private void init() { @Test - public void testCache_v1Posix() throws Exception { + public void testCache_posix() throws Exception { StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion1(); storageProvider.setStorage(storageType); testCache(); - assertTrue("Expected: SM Version 1, " + " Exists: 2", !storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); } @Test - public void testCache_v2Posix() throws Exception { - - StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion2(); - storageProvider.setStorage(storageType); - - testCache(); - - assertTrue("Expected: SM Version 2, " + " Exists: 1", storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); - } - - - @Test - public void testCache_v1S3() throws Exception { - - StorageType storageType = StorageType.S3; - storageProvider.loadVersion1(); - storageProvider.setStorage(storageType); - - testCache(); - - assertTrue("Expected: SM Version 1, " + " Exists: 2", !storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); - } - - @Test - public void testCache_v2S3() throws Exception { + public void testCache_S3() throws Exception { StorageType storageType = StorageType.S3; - storageProvider.loadVersion2(); storageProvider.setStorage(storageType); testCache(); - assertTrue("Expected: SM Version 2, " + " Exists: 1", storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); } @@ -199,7 +168,6 @@ private void testCache() throws Exception { System.out.println("Content: " + mvcResult.getResponse().getContentAsString()); storageTestUtils.printCache(); - storageTestUtils.printVersion("FINISHED download-Test"); // show path of created rest job without first folder (bucket) // String expectedCachePath = new PathConverter(absolutePath).removeFirstFolder().getPath(); diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProseoFilePosixTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProseoFilePosixTest.java deleted file mode 100644 index bba52eedf..000000000 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/ProseoFilePosixTest.java +++ /dev/null @@ -1,121 +0,0 @@ -package de.dlr.proseo.storagemgr.cache; - -import static org.junit.Assert.*; - -import java.util.List; - -import javax.annotation.PostConstruct; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestName; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.orm.jpa.AutoConfigureTestEntityManager; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; - -import de.dlr.proseo.storagemgr.StorageManager; -import de.dlr.proseo.storagemgr.TestUtils; -import de.dlr.proseo.storagemgr.utils.ProseoFile; - -@RunWith(SpringJUnit4ClassRunner.class) -@SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) -@AutoConfigureTestEntityManager -public class ProseoFilePosixTest { - - @Autowired - private TestUtils testUtils; - - @Rule - public TestName testName = new TestName(); - - @Autowired - private FileCache fileCache; - - String testCachePath; - String sourceTestPath; - - @PostConstruct - private void init() { - testCachePath = testUtils.getTestCachePath(); - sourceTestPath = testUtils.getTestStoragePath(); - } - - @Test - public void testCopyTo() { - - TestUtils.printMethodName(this, testName); - TestUtils.createEmptyTestDirectories(); - fileCache.setPath(testCachePath); - - String sourcePath1 = sourceTestPath + "/" + "test1.txt"; - String sourcePath2 = sourceTestPath + "/" + "test2.txt"; - - String targetPath1 = testCachePath + "/" + "test1.txt"; - String targetPath2 = testCachePath + "/" + "test2.txt"; - - TestUtils.createFile(sourcePath1, ""); - TestUtils.createFile(sourcePath2, ""); - - ProseoFile targetFile1 = ProseoFile.fromPathInfo(targetPath1, TestUtils.getInstance().getCfg()); - ProseoFile targetFile2 = ProseoFile.fromPathInfo(targetPath2, TestUtils.getInstance().getCfg()); - - ProseoFile sourceFile; - - // add new - file not exists and not in cache, cache is empty - - sourceFile = ProseoFile.fromPathInfo(sourcePath1, TestUtils.getInstance().getCfg()); - try { - List transfered = sourceFile.copyTo(targetFile1, false); - - if (transfered != null && !transfered.isEmpty()) { - - TestUtils.printList("File copied to cache storage successfully", transfered); - } - } catch (Exception e) { - - e.printStackTrace(); - } - - assertTrue("Cache has not 1 element: " + fileCache.size(), fileCache.size() == 1); - - // add new - file not exists and not in cache, but cache has already 1 element - - sourceFile = ProseoFile.fromPathInfo(sourcePath2, TestUtils.getInstance().getCfg()); - try { - List transfered = sourceFile.copyTo(targetFile2, false); - - if (transfered != null && !transfered.isEmpty()) { - - TestUtils.printList("File copied to cache storage successfully", transfered); - } - } catch (Exception e) { - - e.printStackTrace(); - } - - assertTrue("Cache has not 2 elements: " + fileCache.size(), fileCache.size() == 2); - - // update - file exists and in cache, cache has 2 elements - - sourceFile = ProseoFile.fromPathInfo(sourcePath1, TestUtils.getInstance().getCfg()); - try { - List transfered = sourceFile.copyTo(targetFile1, false); - - if (transfered != null && !transfered.isEmpty()) { - - TestUtils.printList("File copied to cache storage successfully", transfered); - } - } catch (Exception e) { - - e.printStackTrace(); - } - - assertTrue("Cache has not 2 elements: " + fileCache.size(), fileCache.size() == 2); - - fileCache.clear(); - TestUtils.deleteTestDirectories(); - } -} diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java index 59e3a2af4..2f7212724 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_download.java @@ -5,8 +5,6 @@ import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import java.io.File; - import org.codehaus.jackson.map.ObjectMapper; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -26,11 +24,11 @@ import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestJoborder; -import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.StorageType; /** * Mock Mvc test for Product Controller @@ -68,57 +66,26 @@ public class JobOrderControllerImplTest_download { * @return String */ @Test - public void testDownload_v2Posix() throws Exception { + public void testDownload_posix() throws Exception { StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion2(); storageProvider.setStorage(storageType); downloadRestJobOrder(storageType); - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); } - @Test - public void testDownload_v1Posix() throws Exception { - - StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion1(); - storageProvider.setStorage(storageType); - - downloadRestJobOrder(storageType); - - assertTrue("Expected: SM Version1, " + " Exists: 2", !storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); - } @Test - public void testDownload_v2S3() throws Exception { + public void testDownload_S3() throws Exception { StorageType storageType = StorageType.S3; - storageProvider.loadVersion2(); storageProvider.setStorage(storageType); downloadRestJobOrder(storageType); - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); - } - - @Test - public void testDownload_v1S3() throws Exception { - - StorageType storageType = StorageType.S3; - storageProvider.loadVersion1(); - storageProvider.setStorage(storageType); - - downloadRestJobOrder(storageType); - - assertTrue("Expected: SM Version1, " + " Exists: 2", !storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); } @@ -156,7 +123,7 @@ private void downloadRestJobOrder(StorageType storageType) throws Exception { System.out.println("Created uploaded job order path: " + pathInfo); // show storage files - StorageTestUtils.printStorageFiles("After http-upload-call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("After http-upload-call", storageProvider.getStorage()); // Only 1 job order expected today, because we deleted all today-orders earlier int jobOrderCount = storageProvider.getStorage().getRelativeFiles(getJobOrderPrefixForToday()).size(); diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java index 6033b0643..e322e41ac 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/JobOrderControllerImplTest_upload.java @@ -1,19 +1,17 @@ package de.dlr.proseo.storagemgr.rest; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; - import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import java.io.File; - import org.codehaus.jackson.map.ObjectMapper; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; + import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; import org.junit.runner.RunWith; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; @@ -23,14 +21,15 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.MvcResult; import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import de.dlr.proseo.storagemgr.StorageManager; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestJoborder; -import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.StorageType; /** * Job Order Upload test for Product Controller @@ -72,26 +71,10 @@ public class JobOrderControllerImplTest_upload { public void testUpload_v2Posix() throws Exception { StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion2(); - storageProvider.setStorage(storageType); - - uploadRestJobOrder(storageType); - - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); - } - - @Test - public void testUpload_v1Posix() throws Exception { - - StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion1(); storageProvider.setStorage(storageType); uploadRestJobOrder(storageType); - assertTrue("Expected: SM Version1, " + " Exists: 2", !storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); } @@ -100,26 +83,10 @@ public void testUpload_v1Posix() throws Exception { public void testUpload_v2S3() throws Exception { StorageType storageType = StorageType.S3; - storageProvider.loadVersion2(); - storageProvider.setStorage(storageType); - - uploadRestJobOrder(storageType); - - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); - } - - @Test - public void testUpload_v1S3() throws Exception { - - StorageType storageType = StorageType.S3; - storageProvider.loadVersion1(); storageProvider.setStorage(storageType); uploadRestJobOrder(storageType); - assertTrue("Expected: SM Version1, " + " Exists: 2", !storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); } @@ -169,7 +136,7 @@ private void uploadRestJobOrder(StorageType storageType) throws Exception { System.out.println("Created job order path: " + result.getPathInfo()); // show storage files - StorageTestUtils.printStorageFiles("After http-call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("After http-call", storageProvider.getStorage()); // Only 1 job order expected today, because we deleted all today-orders earlier int jobOrderCount = storageProvider.getStorage().getRelativeFiles(getJobOrderPrefixForToday()).size(); @@ -179,7 +146,7 @@ private void uploadRestJobOrder(StorageType storageType) throws Exception { storageProvider.getStorage().delete(getJobOrderPrefixForToday()); // show storage files - StorageTestUtils.printStorageFiles("After job order cleaning", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("After job order cleaning", storageProvider.getStorage()); } // creates prefix for today only. ignores hour and file name (because it is diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java index 9a1dabb03..5f4c635b3 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_delete.java @@ -22,11 +22,11 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; import de.dlr.proseo.storagemgr.StorageManager; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.StorageType; +import de.dlr.proseo.storagemgr.model.StorageType; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * Mock Mvc test for Product Controller @@ -46,7 +46,7 @@ public class ProductControllerImplTest_delete { private MockMvc mockMvc; @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; @Rule public TestName testName = new TestName(); @@ -64,15 +64,13 @@ public class ProductControllerImplTest_delete { * @return RestProductFS */ @Test - public void testDelete_v1Posix() throws Exception { + public void testDelete_posix() throws Exception { StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion1(); storageProvider.setStorage(storageType); delete(storageProvider); - assertTrue("Expected: SM Version1, " + " Exists: 2", !storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); } @@ -85,57 +83,13 @@ public void testDelete_v1Posix() throws Exception { * @return RestProductFS */ @Test - public void testDelete_v2Posix() throws Exception { - - StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion2(); - storageProvider.setStorage(storageType); - - delete(storageProvider); - - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); - } - - /** - * Delete/remove product by product path info from prosEO storage - * - * DELETE /products pathInfo="/.." - * - * @return RestProductFS - */ - @Test - public void testDelete_v1S3() throws Exception { - - StorageType storageType = StorageType.S3; - storageProvider.loadVersion1(); - storageProvider.setStorage(storageType); - - delete(storageProvider); - - assertTrue("Expected: SM Version1, " + " Exists: 2", !storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); - } - - /** - * Delete/remove product by product path info from prosEO storage - * - * DELETE /products pathInfo="/.." - * - * @return RestProductFS - */ - @Test - public void testDelete_v2S3() throws Exception { + public void testDelete_S3() throws Exception { StorageType storageType = StorageType.S3; - storageProvider.loadVersion2(); storageProvider.setStorage(storageType); delete(storageProvider); - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); } @@ -162,10 +116,10 @@ private void delete(StorageProvider storageProvider) throws Exception { } // show storage files before http-delete-call - StorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); // show storage files with prefix before http-delete-call - StorageTestUtils.printStorageFilesWithPrefix("Before http-call", storageProvider.getStorage(), prefix); + BaseStorageTestUtils.printStorageFilesWithPrefix("Before http-call", storageProvider.getStorage(), prefix); // check count of uploaded prefix storage files int realStorageFileCount = storageProvider.getStorage().getRelativeFiles(prefix).size(); @@ -187,7 +141,7 @@ private void delete(StorageProvider storageProvider) throws Exception { TestUtils.printMvcResult(REQUEST_STRING, mvcResult); // show storage files with prefix before http-delete-call - StorageTestUtils.printStorageFilesWithPrefix("After http-call", storageProvider.getStorage(), prefix); + BaseStorageTestUtils.printStorageFilesWithPrefix("After http-call", storageProvider.getStorage(), prefix); // check files after delete (expected: 0) realStorageFileCount = storageProvider.getStorage().getRelativeFiles(prefix).size(); diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java index e821303ab..2dad0bffa 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java @@ -19,13 +19,13 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; import de.dlr.proseo.storagemgr.StorageManager; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * Mock Mvc test for Product Controller @@ -48,7 +48,7 @@ public class ProductControllerImplTest_download { private MockMvc mockMvc; @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; @Autowired private StorageProvider storageProvider; @@ -66,45 +66,20 @@ public class ProductControllerImplTest_download { * @return products string[] */ @Test - public void testDownload_v2Posix() throws Exception { + public void testDownload_posix() throws Exception { if (TESTS_ENABLED) { StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion2(); storageProvider.setStorage(storageType); downloadProductFiles(storageType); - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); } } - /** - * Downloads products with given directory prefix - * - * GET /products storageType="POSIX"&prefix="/.." - * - * @return products string[] - */ - @Test - public void testDownload_v1Posix() throws Exception { - - if (TESTS_ENABLED) { - - StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion1(); - storageProvider.setStorage(storageType); - - downloadProductFiles(storageType); - - assertTrue("Expected: SM Version1, " + " Exists: 2", !storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); - } - } /** * Downloads products with given directory prefix @@ -114,45 +89,20 @@ public void testDownload_v1Posix() throws Exception { * @return products string[] */ @Test - public void testDownload_v2S3() throws Exception { + public void testDownload_S3() throws Exception { if (TESTS_ENABLED) { StorageType storageType = StorageType.S3; - storageProvider.loadVersion2(); storageProvider.setStorage(storageType); downloadProductFiles(storageType); - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); } } - /** - * Downloads products with given directory prefix - * - * GET /products storageType="POSIX"&prefix="/.." - * - * @return products string[] - */ - @Test - public void testDownload_v1S3() throws Exception { - - if (TESTS_ENABLED) { - - StorageType storageType = StorageType.S3; - storageProvider.loadVersion1(); - storageProvider.setStorage(storageType); - - downloadProductFiles(storageType); - - assertTrue("Expected: SM Version1, " + " Exists: 2", !storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); - } - } /** * Get the data files for the product as data stream (optionally zip-compressed, @@ -201,7 +151,7 @@ private void downloadProductFiles(StorageType storageType) throws Exception { Long toByte = 7l; // show storage files - StorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); // TEST PARTIAL CONTENT // HTTP Download files (partial content) from storage @@ -254,6 +204,6 @@ private void downloadProductFiles(StorageType storageType) throws Exception { storageProvider.getStorage().delete(prefix); // show storage files after deletion - StorageTestUtils.printStorageFiles("After deletion", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("After deletion", storageProvider.getStorage()); } } \ No newline at end of file diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java index e6f1965a8..8698243a0 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_get.java @@ -23,13 +23,13 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; import de.dlr.proseo.storagemgr.StorageManager; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * Mock Mvc test for Product Controller @@ -49,7 +49,7 @@ public class ProductControllerImplTest_get { private MockMvc mockMvc; @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; @Autowired private StorageProvider storageProvider; @@ -70,33 +70,10 @@ public class ProductControllerImplTest_get { public void testGet_v2Posix() throws Exception { StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion2(); storageProvider.setStorage(storageType); getProductFiles(storageType); - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); - } - - /** - * Get products with given directory prefix - * - * GET /products storageType="POSIX"&prefix="/.." - * - * @return products string[] - */ - @Test - public void testGet_v1Posix() throws Exception { - - StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion1(); - storageProvider.setStorage(storageType); - - getProductFiles(storageType); - - assertTrue("Expected: SM Version1, " + " Exists: 2", !storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); } @@ -112,33 +89,10 @@ public void testGet_v1Posix() throws Exception { public void testGet_v2S3() throws Exception { StorageType storageType = StorageType.S3; - storageProvider.loadVersion2(); - storageProvider.setStorage(storageType); - - getProductFiles(storageType); - - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); - } - - /** - * Get products with given directory prefix - * - * GET /products storageType="POSIX"&prefix="/.." - * - * @return products string[] - */ - @Test - public void testGet_v1S3() throws Exception { - - StorageType storageType = StorageType.S3; - storageProvider.loadVersion1(); storageProvider.setStorage(storageType); getProductFiles(storageType); - assertTrue("Expected: SM Version1, " + " Exists: 2", !storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); } @@ -186,7 +140,7 @@ private void getProductFiles(StorageType storageType) throws Exception { } // show storage files - StorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); // HTTP Get files from storage MockHttpServletRequestBuilder request = MockMvcRequestBuilders.get(REQUEST_STRING) @@ -221,6 +175,6 @@ private void getProductFiles(StorageType storageType) throws Exception { storageProvider.getStorage().delete(prefix); // show storage files after deletion - StorageTestUtils.printStorageFiles("After deletion", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("After deletion", storageProvider.getStorage()); } } \ No newline at end of file diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java index 564968a88..2177c4cd8 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_getAll.java @@ -19,10 +19,10 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; import de.dlr.proseo.storagemgr.StorageManager; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; -import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.StorageType; +import de.dlr.proseo.storagemgr.model.StorageType; /** * Mock Mvc test for Product Controller @@ -57,15 +57,13 @@ public class ProductControllerImplTest_getAll { * @return products string[] */ @Test - public void testGet_v2Posix() throws Exception { + public void testGet_posix() throws Exception { StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion2(); storageProvider.setStorage(storageType); getProductFiles(storageType); - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); } @@ -78,57 +76,13 @@ public void testGet_v2Posix() throws Exception { * @return products string[] */ @Test - public void testGet_v1Posix() throws Exception { - - StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion1(); - storageProvider.setStorage(storageType); - - getProductFiles(storageType); - - assertTrue("Expected: SM Version1, " + " Exists: 2", !storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); - } - - /** - * Get products with given directory prefix - * - * GET /products storageType="POSIX"&prefix="/.." - * - * @return products string[] - */ - @Test - public void testGet_v2S3() throws Exception { - - StorageType storageType = StorageType.S3; - storageProvider.loadVersion2(); - storageProvider.setStorage(storageType); - - getProductFiles(storageType); - - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); - } - - /** - * Get products with given directory prefix - * - * GET /products storageType="POSIX"&prefix="/.." - * - * @return products string[] - */ - @Test - public void testGet_v1S3() throws Exception { + public void testGet_S3() throws Exception { StorageType storageType = StorageType.S3; - storageProvider.loadVersion1(); storageProvider.setStorage(storageType); getProductFiles(storageType); - assertTrue("Expected: SM Version1, " + " Exists: 2", !storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); } @@ -158,7 +112,7 @@ private void getProductFiles(StorageType storageType) throws Exception { TestUtils.printMethodName(this, testName); // show storage files - StorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); // HTTP Get files from storage MockHttpServletRequestBuilder request = MockMvcRequestBuilders.get(REQUEST_STRING) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java index 11bd16edc..07b09b759 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_upload.java @@ -25,13 +25,13 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; import de.dlr.proseo.storagemgr.StorageManager; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestProductFS; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageType; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * Mock Mvc test for Product Controller @@ -57,7 +57,6 @@ public class ProductControllerImplTest_upload { private StorageProvider storageProvider; private static final String REQUEST_STRING = "/proseo/storage-mgr/x/products"; - /** * Register products/files/dirs from unstructered storage in prosEO-storage @@ -67,61 +66,17 @@ public class ProductControllerImplTest_upload { * @return RestProductFS */ @Test - public void testUpload_v1Posix() throws Exception { + public void testUpload_posix() throws Exception { StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion1(); storageProvider.setStorage(storageType); uploadRestProductFS(); - assertTrue("Expected: SM Version1, " + " Exists: 2", !storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); } - - /** - * Register products/files/dirs from unstructered storage in prosEO-storage - * - * POST /products RestProductFS - * - * @return RestProductFS - */ - @Test - public void testUpload_v2Posix() throws Exception { - StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion2(); - storageProvider.setStorage(storageType); - - uploadRestProductFS(); - - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); - } - - /** - * Register products/files/dirs from unstructered storage in prosEO-storage - * - * POST /products RestProductFS - * - * @return RestProductFS - */ - @Test - public void testUpload_v1S3() throws Exception { - - StorageType storageType = StorageType.S3; - storageProvider.loadVersion1(); - storageProvider.setStorage(storageType); - - uploadRestProductFS(); - - assertTrue("Expected: SM Version1, " + " Exists: 2", !storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); - } - /** * Register products/files/dirs from unstructered storage in prosEO-storage * @@ -130,15 +85,13 @@ public void testUpload_v1S3() throws Exception { * @return RestProductFS */ @Test - public void testUpload_v2S3() throws Exception { + public void testUpload_S3() throws Exception { StorageType storageType = StorageType.S3; - storageProvider.loadVersion2(); storageProvider.setStorage(storageType); uploadRestProductFS(); - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); } @@ -190,7 +143,7 @@ private void uploadRestProductFS() throws Exception { RestProductFS restProductFS = populateRestProductFS(productId, relativePaths); // show storage files - StorageTestUtils.printStorageFiles("Before http-upload call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("Before http-upload call", storageProvider.getStorage()); // http-upload call MockHttpServletRequestBuilder request = MockMvcRequestBuilders.post(REQUEST_STRING) @@ -202,7 +155,7 @@ private void uploadRestProductFS() throws Exception { TestUtils.printMvcResult(REQUEST_STRING, mvcResult); // show storage files after http-upload - StorageTestUtils.printStorageFiles("After http-upload call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("After http-upload call", storageProvider.getStorage()); // check real with expected absolute storage paths String json = mvcResult.getResponse().getContentAsString(); diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java index b42e9690f..2bbb9921a 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_download.java @@ -22,14 +22,14 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; import de.dlr.proseo.storagemgr.StorageManager; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; -import de.dlr.proseo.storagemgr.version2.FileUtils; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * Mock Mvc test for Product Controller @@ -49,7 +49,7 @@ public class ProductfileControllerImplTest_download { private MockMvc mockMvc; @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; @Autowired private StorageProvider storageProvider; @@ -59,66 +59,26 @@ public class ProductfileControllerImplTest_download { private static final String REQUEST_STRING = "/proseo/storage-mgr/x/productfiles"; - /** - * Retrieve file from Storage Manager into locally accessible file system // DOWNLOAD TEST - * - * GET /productfiles pathInfo="/.." - * - * @return RestFileInfo - */ @Test - public void testDownload_v1Posix() throws Exception { - - StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion1(); - storageProvider.setStorage(storageType); - - download("v1Posix"); - - assertTrue("Expected: SM Version1, " + " Exists: 2", !storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); - } - - @Test - public void testDownload_v2Posix() throws Exception { + public void testDownload_posix() throws Exception { StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion2(); storageProvider.setStorage(storageType); - download("v2Posix"); + download("Posix"); - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); } - - @Test - public void testDownload_v1S3() throws Exception { - - StorageType storageType = StorageType.S3; - storageProvider.loadVersion1(); - storageProvider.setStorage(storageType); - - download("v1S3"); - - assertTrue("Expected: SM Version1, " + " Exists: 2", !storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); - } - @Test - public void testDownload_v2S3() throws Exception { + public void testDownload_S3() throws Exception { StorageType storageType = StorageType.S3; - storageProvider.loadVersion2(); storageProvider.setStorage(storageType); - download("v2S3"); + download("S3"); - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); } @@ -159,7 +119,7 @@ private void download(String testID) throws Exception { storageProvider.getStorage().upload(sourceFile, storageFile); // show storage files - StorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); // rest-download file from storage to cache String absoluteStoragePath = storageProvider.getStorage().getAbsolutePath(relativePath); @@ -173,7 +133,6 @@ private void download(String testID) throws Exception { TestUtils.printMvcResult(REQUEST_STRING, mvcResult); storageTestUtils.printCache(); - storageTestUtils.printVersion("FINISHED download-Test"); // check real with expected absolute cache path String expectedAbsoluteCachePath = new PathConverter(storageProvider.getCachePath(), relativePath).getPath(); diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java index 5272ba849..4f3b951fc 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_upload.java @@ -20,13 +20,13 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; import de.dlr.proseo.storagemgr.StorageManager; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.model.StorageType; import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; -import de.dlr.proseo.storagemgr.version2.FileUtils; -import de.dlr.proseo.storagemgr.version2.PathConverter; -import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.StorageType; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; /** * Mock Mvc test for Product Controller @@ -46,7 +46,7 @@ public class ProductfileControllerImplTest_upload { private MockMvc mockMvc; @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; @Autowired private StorageProvider storageProvider; @@ -57,57 +57,25 @@ public class ProductfileControllerImplTest_upload { private static final String REQUEST_STRING = "/proseo/storage-mgr/x/productfiles"; @Test - public void testUpload_v1Posix() throws Exception { + public void testUpload_Posix() throws Exception { StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion1(); storageProvider.setStorage(storageType); upload(); - assertTrue("Expected: SM Version1, " + " Exists: 2", !storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); } @Test - public void testUpload_v2Posix() throws Exception { - - StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion2(); - storageProvider.setStorage(storageType); - - upload(); - - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); - } - - @Test - public void testUpload_v1S3() throws Exception { - - StorageType storageType = StorageType.S3; - storageProvider.loadVersion1(); - storageProvider.setStorage(storageType); - - upload(); - - assertTrue("Expected: SM Version1, " + " Exists: 2", !storageProvider.isVersion2()); - StorageType realStorageType = storageProvider.getStorage().getStorageType(); - assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); - } - - @Test - public void testUpload_v2S3() throws Exception { + public void testUpload_S3() throws Exception { StorageType storageType = StorageType.S3; - storageProvider.loadVersion2(); storageProvider.setStorage(storageType); upload(); - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); } @@ -156,7 +124,7 @@ private void upload() throws Exception { assertTrue("Expected path: " + realRelativeStoragePath + " Exists: " + relativePath, relativePath.equals(realRelativeStoragePath)); // show storage files - StorageTestUtils.printStorageFiles("After http-call", storageProvider.getStorage()); + BaseStorageTestUtils.printStorageFiles("After http-call", storageProvider.getStorage()); // delete files with empty folders new FileUtils(absoluteSourcePath).deleteFile(); // source diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/PathConverterTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/PathConverterTest.java index 12a39c661..ecf537d4c 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/PathConverterTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/PathConverterTest.java @@ -7,6 +7,8 @@ import org.junit.Test; +import de.dlr.proseo.storagemgr.utils.PathConverter; + public class PathConverterTest { @Test diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java index 6657b6753..d1aad5592 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java @@ -18,11 +18,12 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import de.dlr.proseo.storagemgr.StorageManager; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) @@ -33,7 +34,7 @@ public class StorageProviderTest { private TestUtils testUtils; @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; @Autowired private StorageProvider storageProvider; @@ -79,7 +80,6 @@ public void testPosixPosixProvider() throws IOException { assertTrue("File for upload has not been created: " + sourceFilePath, TestUtils.fileExists(sourceFilePath)); StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion2(); storageProvider.setStorage(storageType); Storage storage = storageProvider.getStorage(); @@ -97,7 +97,7 @@ public void testPosixPosixProvider() throws IOException { assertTrue("File was not uploaded to storage: " + storageFilePath, TestUtils.fileExists(storageFilePath)); - StorageTestUtils.printStorageFileList("Storage Files (should be 1 file) ", storage.getStorageFiles()); + BaseStorageTestUtils.printStorageFileList("Storage Files (should be 1 file) ", storage.getStorageFiles()); // ----------------------- download -------------------------- @@ -115,7 +115,6 @@ public void testPosixPosixProvider() throws IOException { assertTrue("File was not downloaded from storage: " + cacheFilePath, TestUtils.fileExists(cacheFilePath)); - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); @@ -143,9 +142,4 @@ public void probaTest() throws Exception { TestUtils.deleteStorageDirectories(); } - - @Test - public void testDefaultS3PosixProvider() { - - } } diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java index 4166b8a8a..e984a337e 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java @@ -15,10 +15,11 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import de.dlr.proseo.storagemgr.StorageManager; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; -import de.dlr.proseo.storagemgr.version2.model.StorageType; -import de.dlr.proseo.storagemgr.version2.model.Storage; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageType; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) @@ -29,7 +30,7 @@ public class StorageUtilityTest { private TestUtils testUtils; @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; @Autowired private StorageProvider storageProvider; @@ -56,7 +57,6 @@ public void testUtility() throws IOException { // change type to show another storage StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion2(); storageProvider.setStorage(storageType); Storage storage = storageProvider.getStorage(); @@ -67,13 +67,13 @@ public void testUtility() throws IOException { TestUtils.printList("S3 Buckets:", s3Storage.getBuckets()); // show s3 storage files - StorageTestUtils.printStorageFiles("S3 before Action", s3Storage); + BaseStorageTestUtils.printStorageFiles("S3 before Action", s3Storage); // show posix storage files - StorageTestUtils.printStorageFiles("POSIX before Action", posixStorage); + BaseStorageTestUtils.printStorageFiles("POSIX before Action", posixStorage); // show default storage files - StorageTestUtils.printStorageFiles("Default Storage BEFORE Action", storage); + BaseStorageTestUtils.printStorageFiles("Default Storage BEFORE Action", storage); // ACTIONS // @@ -95,6 +95,6 @@ public void testUtility() throws IOException { // storage.delete(path); // show default storage files - StorageTestUtils.printStorageFiles("Default Storage AFTER Action", storage); + BaseStorageTestUtils.printStorageFiles("Default Storage AFTER Action", storage); } } diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixDALTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixDALTest.java index 13e2491fb..48a1377cb 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixDALTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixDALTest.java @@ -17,10 +17,12 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import de.dlr.proseo.storagemgr.StorageManager; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; import de.dlr.proseo.storagemgr.UniqueStorageTestPaths; -import de.dlr.proseo.storagemgr.version2.PathConverter; +import de.dlr.proseo.storagemgr.posix.PosixDAL; +import de.dlr.proseo.storagemgr.utils.PathConverter; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) @@ -31,7 +33,10 @@ public class PosixDALTest { public TestName testName = new TestName(); @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; + + @Autowired + private StorageProvider storageProvider; @Test public void test() { @@ -55,7 +60,7 @@ public void test() { String sourcePath = uniquePaths.getUniqueSourcePath(); String storagePath = uniquePaths.getUniqueStoragePath(); - PosixDAL posixDAL = new PosixDAL(); + PosixDAL posixDAL = new PosixDAL(storageProvider.getPosixConfigurationFromFile()); try { // print source files diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java index 81a097ea7..7bbc6949a 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/posix/PosixStorageTest.java @@ -17,11 +17,11 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import de.dlr.proseo.storagemgr.StorageManager; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; -import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; -import de.dlr.proseo.storagemgr.version2.model.StorageType; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) @@ -29,7 +29,7 @@ public class PosixStorageTest { @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; @Autowired private StorageProvider storageProvider; @@ -44,7 +44,6 @@ public void testPosixPosixUpload() throws IOException { TestUtils.createEmptyStorageDirectories(); StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion2(); storageProvider.setStorage(storageType); String prefix = "posix-storage-upload-test/"; @@ -68,7 +67,6 @@ public void testPosixPosixUpload() throws IOException { TestUtils.printList("Storage Files: ", uploadedPathes); assertTrue("Expected: 3, " + " Exists: " + uploadedPathes.size(), uploadedPathes.size() == 3); - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); @@ -83,7 +81,6 @@ public void testPosixPosixDownload() throws IOException { TestUtils.createEmptyStorageDirectories(); StorageType storageType = StorageType.POSIX; - storageProvider.loadVersion2(); storageProvider.setStorage(storageType); String prefix = "posix-storage-download-test/"; @@ -108,7 +105,6 @@ public void testPosixPosixDownload() throws IOException { TestUtils.printList("Source Files: ", downloadedPathes); assertTrue("Expected: 3, " + " Exists: " + downloadedPathes.size(), downloadedPathes.size() == 3); - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java index 1fac71525..bbbdba7b3 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/s3/S3StorageTest.java @@ -19,12 +19,12 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import de.dlr.proseo.storagemgr.StorageManager; -import de.dlr.proseo.storagemgr.StorageTestUtils; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; -import de.dlr.proseo.storagemgr.version2.model.StorageType; -import de.dlr.proseo.storagemgr.version2.StorageProvider; -import de.dlr.proseo.storagemgr.version2.model.Storage; -import de.dlr.proseo.storagemgr.version2.model.StorageFile; +import de.dlr.proseo.storagemgr.model.Storage; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) @@ -35,7 +35,7 @@ public class S3StorageTest { private TestUtils testUtils; @Autowired - private StorageTestUtils storageTestUtils; + private BaseStorageTestUtils storageTestUtils; @Autowired private StorageProvider storageProvider; @@ -63,7 +63,6 @@ public void testUploadFile() throws IOException { // StorageProvider storageProvider = new StorageProvider(); StorageType storageType = StorageType.S3; - storageProvider.loadVersion2(); storageProvider.setStorage(storageType); String prefix = "s3-storage-test/"; @@ -99,16 +98,13 @@ public void testUploadFile() throws IOException { TestUtils.printList("S3 Storage files after delete:", storage.getRelativeFiles()); - } catch (Exception e) { e.printStackTrace(); throw e; } - assertTrue("Expected: SM Version2, " + " Exists: 1", storageProvider.isVersion2()); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); - } } diff --git a/storage-mgr/src/test/resources/application.yml b/storage-mgr/src/test/resources/application.yml index 23eb9804e..40907a5b1 100644 --- a/storage-mgr/src/test/resources/application.yml +++ b/storage-mgr/src/test/resources/application.yml @@ -8,14 +8,13 @@ logging: proseo: global: storageIdPrefix: proseo - storageManagerVersion2: true # Default mount point for files to upload sourcePath: target/testdata/source s3: s3AccessKey: short_access_key s3SecretAccessKey: short_secret_key - s3EndPoint: http://localhost:9000 + s3EndPoint: http://127.0.0.1:9000 s3Region: eu-de s3MaxNumberOfBuckets: 50 s3DefaultBucket: proseo-data-001 diff --git a/storage-mgr/src/test/resources/docker/.gitignore b/storage-mgr/src/test/resources/docker/.gitignore new file mode 100644 index 000000000..bd5136701 --- /dev/null +++ b/storage-mgr/src/test/resources/docker/.gitignore @@ -0,0 +1,5 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore +# This directory must exist to run the minio container for storage manager tests \ No newline at end of file From cca12542642acf3b229cdfd02982c6b513cbe6bd Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 1 Feb 2024 18:28:00 +0100 Subject: [PATCH 40/79] storage-mgr: updated version in pom.xml to 0.9.5 --- storage-mgr/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage-mgr/pom.xml b/storage-mgr/pom.xml index 5deab5b92..666b354d3 100644 --- a/storage-mgr/pom.xml +++ b/storage-mgr/pom.xml @@ -6,7 +6,7 @@ de.dlr.proseo proseo - 0.9.5-SNAPSHOT + 0.9.5 @@ -20,7 +20,7 @@ - 0.9.5-SNAPSHOT + 0.9.5 ${basedir}/../.. src/main/resources/raml/storage-mgr-api.raml de.dlr.proseo.storagemgr.rest From d16eddf2469029cd6506e48b8c471de8e1b7ec53 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Fri, 2 Feb 2024 11:40:19 +0100 Subject: [PATCH 41/79] storage-mgr: removed check hasNotExistsStatus from containsKey in cache --- .../java/de/dlr/proseo/storagemgr/cache/FileCache.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java index c28def0eb..3c10f702a 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java @@ -92,7 +92,9 @@ public static FileCache getInstance() { /** * Puts the new element to map. If element exists, it will be overwritten. - * Removes the file if it is temporary + * Removes the file if it is temporary file (not a cache file). + * Calls deleteLRU before adding the file to cache to clean free space if needed. + * Set the status = "ready" and updates the last access record of the cache file. * * @param pathKey the full cache file path as a key */ @@ -134,10 +136,10 @@ public void put(String pathKey) { } /** - * Checks if the cache contains the path If not - returns false. If the physical + * Checks if the cache contains the path. If not - returns false. If the physical * file does not exits anymore - deletes the path from the cache. Returns true * if the path is available in the cache and also updates the file record of the - * last access + * last access calling put() method * * @param pathKey File path as key * @return true if pathkey is in file cache @@ -154,7 +156,7 @@ public boolean containsKey(String pathKey) { File file = new File(pathKey); - if (!file.isFile() || hasNotExistsStatus(pathKey)) { + if (!file.isFile()) { remove(pathKey); return false; From 27186b56febf4cdc6708b5229b8eeae315437d9c Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Fri, 2 Feb 2024 11:53:24 +0100 Subject: [PATCH 42/79] storage-mgr: removed uploading status from file cache status --- .../proseo/storagemgr/cache/CacheFileStatus.java | 2 -- .../proseo/storagemgr/cache/FileCacheTest.java | 16 ++++++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java index e90b4c54f..495482d08 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java @@ -13,8 +13,6 @@ public enum CacheFileStatus { /** Status READY */ READY, - /** Status UPLOADING */ - COPYING_FROM_CACHE, /** Status Cache File does not exist in the cache */ NOT_EXISTS, } \ No newline at end of file diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java index 71d618d84..2d0384f56 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java @@ -245,30 +245,30 @@ public void testStatus() { assertTrue("Expected cache file3 status after setStatus(Not_exists) is Not_exists. Exists:" + status3.toString(), status3 == CacheFileStatus.NOT_EXISTS); - // changing status1 to copying from cache + // changing status1 to NOT_EXISTS from cache - fileCache.setCacheFileStatus(path1, CacheFileStatus.COPYING_FROM_CACHE); + fileCache.setCacheFileStatus(path1, CacheFileStatus.NOT_EXISTS); status1 = fileCache.getCacheFileStatus(path1); status2 = fileCache.getCacheFileStatus(path2); - assertTrue("Expected cache file1 status after setStatus(Uploading) is Uploading. Exists:" + status1.toString(), - status1 == CacheFileStatus.COPYING_FROM_CACHE); + assertTrue("Expected cache file1 status after setStatus(NOT_EXISTS) is NOT_EXISTS. Exists:" + status1.toString(), + status1 == CacheFileStatus.NOT_EXISTS); assertTrue("Expected cache file2 status after is Ready. Exists:" + status2.toString(), status2 == CacheFileStatus.READY); - // changing status2 to copying to cache + // changing status2 to NOT_EXISTS to cache fileCache.setCacheFileStatus(path2, CacheFileStatus.NOT_EXISTS); status1 = fileCache.getCacheFileStatus(path1); status2 = fileCache.getCacheFileStatus(path2); - assertTrue("Expected cache file1 status after is Uploading. Exists: " + status1.toString(), - status1 == CacheFileStatus.COPYING_FROM_CACHE); + assertTrue("Expected cache file1 status after is NOT_EXISTS. Exists: " + status1.toString(), + status1 == CacheFileStatus.NOT_EXISTS); - assertTrue("Expected cache file2 status after setStatus(Downloading) is Downlading. Exists: " + status2.toString(), + assertTrue("Expected cache file2 status after setStatus(NOT_EXISTS) is NOT_EXISTS. Exists: " + status2.toString(), status2 == CacheFileStatus.NOT_EXISTS); // changing status1 to Ready From 9a484397d8fbd89265478d0ef6299948f50b1ff2 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Fri, 2 Feb 2024 12:13:06 +0100 Subject: [PATCH 43/79] storage-mgr: added descriptions to cache file statuses --- .../storagemgr/cache/CacheFileStatus.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java index 495482d08..55fea9e6e 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java @@ -6,13 +6,22 @@ package de.dlr.proseo.storagemgr.cache; /** - * Cache File Status shows the current status of the file in the cache + * Cache File Status shows the current status of the file in the cache storage + * and in the cache list * * @author Denys Chaykovskiy */ public enum CacheFileStatus { - /** Status READY */ + + /** + * Status READY - a file exists in the cache storage and was put to the cache + * list + */ READY, - /** Status Cache File does not exist in the cache */ - NOT_EXISTS, + + /** + * Status NOT_EXISTS - a file can exist in the cache storage, but can be + * corrupted, that's why it is not in the cache list + */ + NOT_EXISTS, } \ No newline at end of file From f8973c74f117cd3e86105e448e08cf30179f845d Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Fri, 2 Feb 2024 12:26:32 +0100 Subject: [PATCH 44/79] storage-mgr: changed cache file status to INCOMPLETE from NOT_EXISTS --- .../storagemgr/cache/CacheFileStatus.java | 6 ++-- .../proseo/storagemgr/cache/FileCache.java | 4 +-- .../rest/ProductfileControllerImpl.java | 4 +-- .../storagemgr/cache/FileCacheTest.java | 28 +++++++++---------- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java index 55fea9e6e..0719c54f0 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/CacheFileStatus.java @@ -20,8 +20,8 @@ public enum CacheFileStatus { READY, /** - * Status NOT_EXISTS - a file can exist in the cache storage, but can be - * corrupted, that's why it is not in the cache list + * Status INCOMPLETE - a file can exist in the cache storage, but can be + * corrupted (not completely uploaded and so on), that's why it is not in the cache list */ - NOT_EXISTS, + INCOMPLETE, } \ No newline at end of file diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java index 3c10f702a..7110c9d58 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/cache/FileCache.java @@ -212,7 +212,7 @@ public void setCacheFileStatus(String pathKey, CacheFileStatus status) { logger.trace(">>> setCacheFileStatus({}, {})", pathKey, status); // There is no cache file for the status == not exists - if (status != CacheFileStatus.NOT_EXISTS) { + if (status != CacheFileStatus.INCOMPLETE) { if (!mapCache.containsKey(pathKey)) { if (!new File(pathKey).exists()) { logger.log(StorageMgrMessage.CACHE_NO_FILE_FOR_PUTTING_TO_CACHE, pathKey); @@ -859,7 +859,7 @@ private boolean hasFilePrefix(String path, String prefix) { */ private boolean hasNotExistsStatus(String cacheFile) { - return getCacheFileStatus(cacheFile) == CacheFileStatus.NOT_EXISTS; + return getCacheFileStatus(cacheFile) == CacheFileStatus.INCOMPLETE; } /** diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index 21d82268a..ecd8a98de 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -207,7 +207,7 @@ private RestFileInfo copyFileStorageToCache(String storageFilePath, StorageFileL // active thread - copies the file to the cache storage and puts it to the cache - cache.setCacheFileStatus(cacheFile.getFullPath(), CacheFileStatus.NOT_EXISTS); + cache.setCacheFileStatus(cacheFile.getFullPath(), CacheFileStatus.INCOMPLETE); storageProvider.getStorage().downloadFile(storageFile, cacheFile); @@ -266,7 +266,7 @@ private RestFileInfo copyFileExternalToCache(String externalPath, Long productId // active thread - copies the file to the cache storage and puts it to the cache - cache.setCacheFileStatus(cacheFile.getFullPath(), CacheFileStatus.NOT_EXISTS); + cache.setCacheFileStatus(cacheFile.getFullPath(), CacheFileStatus.INCOMPLETE); storageProvider.copyAbsoluteFilesToCache(externalPath, productId); diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java index 2d0384f56..5a1096885 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/cache/FileCacheTest.java @@ -235,41 +235,41 @@ public void testStatus() { // check not exists status - file is not in cache, but has a status - fileCache.setCacheFileStatus(path3, CacheFileStatus.NOT_EXISTS); + fileCache.setCacheFileStatus(path3, CacheFileStatus.INCOMPLETE); status3 = fileCache.getCacheFileStatus(path3); assertTrue("Expected cache file3 does not exist in the cache. Exists:" + fileCache.containsKey(path3), !fileCache.containsKey(path3)); - assertTrue("Expected cache file3 status after setStatus(Not_exists) is Not_exists. Exists:" + status3.toString(), - status3 == CacheFileStatus.NOT_EXISTS); + assertTrue("Expected cache file3 status after setStatus(INCOMPLETE) is INCOMPLETE. Exists:" + status3.toString(), + status3 == CacheFileStatus.INCOMPLETE); - // changing status1 to NOT_EXISTS from cache + // changing status1 to INCOMPLETE - fileCache.setCacheFileStatus(path1, CacheFileStatus.NOT_EXISTS); + fileCache.setCacheFileStatus(path1, CacheFileStatus.INCOMPLETE); status1 = fileCache.getCacheFileStatus(path1); status2 = fileCache.getCacheFileStatus(path2); - assertTrue("Expected cache file1 status after setStatus(NOT_EXISTS) is NOT_EXISTS. Exists:" + status1.toString(), - status1 == CacheFileStatus.NOT_EXISTS); + assertTrue("Expected cache file1 status after setStatus(INCOMPLETE) is INCOMPLETE. Exists:" + status1.toString(), + status1 == CacheFileStatus.INCOMPLETE); assertTrue("Expected cache file2 status after is Ready. Exists:" + status2.toString(), status2 == CacheFileStatus.READY); - // changing status2 to NOT_EXISTS to cache + // changing status2 to INCOMPLETE - fileCache.setCacheFileStatus(path2, CacheFileStatus.NOT_EXISTS); + fileCache.setCacheFileStatus(path2, CacheFileStatus.INCOMPLETE); status1 = fileCache.getCacheFileStatus(path1); status2 = fileCache.getCacheFileStatus(path2); - assertTrue("Expected cache file1 status after is NOT_EXISTS. Exists: " + status1.toString(), - status1 == CacheFileStatus.NOT_EXISTS); + assertTrue("Expected cache file1 status after is INCOMPLETE. Exists: " + status1.toString(), + status1 == CacheFileStatus.INCOMPLETE); - assertTrue("Expected cache file2 status after setStatus(NOT_EXISTS) is NOT_EXISTS. Exists: " + status2.toString(), - status2 == CacheFileStatus.NOT_EXISTS); + assertTrue("Expected cache file2 status after setStatus(INCOMPLETE) is INCOMPLETE. Exists: " + status2.toString(), + status2 == CacheFileStatus.INCOMPLETE); // changing status1 to Ready @@ -282,7 +282,7 @@ public void testStatus() { status1 == CacheFileStatus.READY); assertTrue("Expected cache file2 status after is Downloading. Exists: " + status2.toString(), - status2 == CacheFileStatus.NOT_EXISTS); + status2 == CacheFileStatus.INCOMPLETE); // check cache after status changes From 85b2734a874f1d7210057ac119557e6dd314f0d4 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Mon, 5 Feb 2024 17:19:43 +0100 Subject: [PATCH 45/79] storage-mgr: set copied to cache file permissions to read-only 444 --- .../rest/ProductfileControllerImpl.java | 88 +++++++++++-------- 1 file changed, 51 insertions(+), 37 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index ecd8a98de..526341e1f 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -2,8 +2,9 @@ import java.io.File; import java.io.IOException; - +import java.nio.file.Files; import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermissions; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; @@ -37,6 +38,9 @@ @Component public class ProductfileControllerImpl implements ProductfileController { + /** The file permission with read-only permissions for all users */ + private static String READ_ONLY_FOR_ALL_USERS = "r--r--r--"; + /** A logger for this class */ private static ProseoLogger logger = new ProseoLogger(ProductControllerImpl.class); private static ProseoHttp http = new ProseoHttp(logger, HttpPrefix.STORAGE_MGR); @@ -172,14 +176,15 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro logger.debug("... unlocked the file: ", externalPath); } } - + /** - * Copies the file from the storage to the cache using synchronization. - * During the copying to the cache, the status of the file will be "not exists", - * after the completion the status will be set to "ready" + * Copies the file from the storage to the cache using synchronization. During + * the copying to the cache, the status of the file will be "not exists", after + * the completion the status will be set to "ready". Sets the file permission + * 444. * * @param storageFilePath the file path in the storage - * @param fileLocker file locker is used for synchronization + * @param fileLocker file locker is used for synchronization * @return RestFileInfo * @throws FileLockedAfterMaxCyclesException * @throws IOException @@ -187,7 +192,7 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro */ private RestFileInfo copyFileStorageToCache(String storageFilePath, StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { - + // x-to-cache-copy method, status "not exists" is used // relative path depends on path, not on actual storage @@ -206,13 +211,16 @@ private RestFileInfo copyFileStorageToCache(String storageFilePath, StorageFileL if (!cache.containsKey(cacheFile.getFullPath())) { // active thread - copies the file to the cache storage and puts it to the cache - + cache.setCacheFileStatus(cacheFile.getFullPath(), CacheFileStatus.INCOMPLETE); - + storageProvider.getStorage().downloadFile(storageFile, cacheFile); - + logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED_FROM_STORAGE, cacheFile.getFullPath()); + Files.setPosixFilePermissions(Paths.get(cacheFile.getFullPath()), + PosixFilePermissions.fromString(READ_ONLY_FOR_ALL_USERS)); + cache.put(cacheFile.getFullPath()); // cache file status = READY } else { @@ -223,7 +231,7 @@ private RestFileInfo copyFileStorageToCache(String storageFilePath, StorageFileL } } else { - + logger.debug("... no download and no lock - the file is in cache: ", cacheFile.getFullPath()); } @@ -235,23 +243,26 @@ private RestFileInfo copyFileStorageToCache(String storageFilePath, StorageFileL /** * Copies the file from the external source to the cache using synchronization. - * During the copying to the cache, the status of the file will be "not exists", - * after the completion the status will be set to "ready" + * During the copying to the cache, the status of the file will be "not exists", + * after the completion the status will be set to "ready". Sets the file + * permission 444. * - * @param externalPath external path of the file, which will be copied to the cache - * @param productId product id is used as a directory to store copied file in cache - * @param fileSize file size - * @param fileLocker file locker is used for synchronization - * @return Rest File Info + * @param externalPath external path of the file, which will be copied to the + * cache + * @param productId product id is used as a directory to store copied file in + * cache + * @param fileSize file size + * @param fileLocker file locker is used for synchronization + * @return Rest File Info * @throws FileLockedAfterMaxCyclesException - * @throws IOException + * @throws IOException * @throws Exception */ private RestFileInfo copyFileExternalToCache(String externalPath, Long productId, Long fileSize, StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { // x-to-cache-copy method, status "not exists" is used - + String productFolderWithFilename = getProductFolderWithFilename(externalPath, productId); StorageFile cacheFile = storageProvider.getCacheFile(productFolderWithFilename); @@ -265,16 +276,18 @@ private RestFileInfo copyFileExternalToCache(String externalPath, Long productId if (!cache.containsKey(cacheFile.getFullPath())) { // active thread - copies the file to the cache storage and puts it to the cache - + cache.setCacheFileStatus(cacheFile.getFullPath(), CacheFileStatus.INCOMPLETE); - + storageProvider.copyAbsoluteFilesToCache(externalPath, productId); - logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED_FROM_EXTERNAL_TO_CACHE, - cacheFile.getFullPath()); + logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED_FROM_EXTERNAL_TO_CACHE, cacheFile.getFullPath()); + + Files.setPosixFilePermissions(Paths.get(cacheFile.getFullPath()), + PosixFilePermissions.fromString(READ_ONLY_FOR_ALL_USERS)); cache.put(cacheFile.getFullPath()); // cache file status = READY - + } else { // passive thread - did nothing, waited for copied file and use it from cache @@ -284,7 +297,7 @@ private RestFileInfo copyFileExternalToCache(String externalPath, Long productId } } else { - + logger.debug("... no download and no lock - the file is in cache: ", cacheFile.getFullPath()); } @@ -294,17 +307,18 @@ private RestFileInfo copyFileExternalToCache(String externalPath, Long productId return restFileInfo; } - // TODO: WIP Special use case for cache recovery - file in cache, but not in storage + // TODO: WIP Special use case for cache recovery - file in cache, but not in + // storage // TODO: WIP Special use case for cache state - not uploaded to storage /** - * Copies the file from the cache to the storage using synchronization. - * During the copying to the cache, the status of the file will be "not exists", - * after the completion the status will be set to "ready" + * Copies the file from the cache to the storage using synchronization. During + * the copying to the cache, the status of the file will be "not exists", after + * the completion the status will be set to "ready" * * @param relativeCachePath relative cache path - * @param fileLocker is used for synchronization - * @return RestFileInfo + * @param fileLocker is used for synchronization + * @return RestFileInfo * @throws FileLockedAfterMaxCyclesException * @throws IOException * @throws Exception @@ -323,17 +337,17 @@ private RestFileInfo copyFileCacheToStorage(String relativeCachePath, StorageFil fileLocker.lockOrWaitUntilUnlockedAndLock(); - // check again, the file could be copied to storage from another thread after lock + // check again, the file could be copied to storage from another thread after + // lock if (!cache.containsKey(cacheFile.getFullPath())) { // active thread - copies the file to the storage and checks it as OK (WIP) storage.uploadFile(cacheFile, storageFile); - logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED_FROM_EXTERNAL_TO_CACHE, - storageFile.getFullPath()); + logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED_FROM_EXTERNAL_TO_CACHE, storageFile.getFullPath()); cache.put(storageFile.getFullPath()); - + } else { // passive thread - did nothing, waited for copied file and use it from cache @@ -342,7 +356,7 @@ private RestFileInfo copyFileCacheToStorage(String relativeCachePath, StorageFil } } else { - + logger.debug("... no download and no lock - the file is in cache: ", storageFile.getFullPath()); } From 062b9758d95eef72594e9386dbb7dbc3646f5ba2 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Tue, 6 Feb 2024 10:43:33 +0100 Subject: [PATCH 46/79] storage-mgr: added logs to product file controller and storage locker --- .../rest/ProductfileControllerImpl.java | 15 +++++++++++++++ .../storagemgr/utils/StorageFileLocker.java | 18 ++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index 526341e1f..ab507e37b 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -192,6 +192,9 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro */ private RestFileInfo copyFileStorageToCache(String storageFilePath, StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { + + if (logger.isTraceEnabled()) + logger.trace(">>> copyFileStorageToCache({}, {})", storageFilePath, fileLocker); // x-to-cache-copy method, status "not exists" is used @@ -260,6 +263,9 @@ private RestFileInfo copyFileStorageToCache(String storageFilePath, StorageFileL */ private RestFileInfo copyFileExternalToCache(String externalPath, Long productId, Long fileSize, StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { + + if (logger.isTraceEnabled()) + logger.trace(">>> copyFileExternalToCache({}, {}, {}, {})", externalPath, productId, fileSize, fileLocker); // x-to-cache-copy method, status "not exists" is used @@ -325,6 +331,9 @@ private RestFileInfo copyFileExternalToCache(String externalPath, Long productId */ private RestFileInfo copyFileCacheToStorage(String relativeCachePath, StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { + + if (logger.isTraceEnabled()) + logger.trace(">>> copyFileCacheToStorage({}, {})", relativeCachePath, fileLocker); Storage storage = storageProvider.getStorage(); @@ -374,6 +383,9 @@ private RestFileInfo copyFileCacheToStorage(String relativeCachePath, StorageFil * @return rest file info */ private static RestFileInfo convertToRestFileInfo(StorageFile storageFile, long fileSize) { + + if (logger.isTraceEnabled()) + logger.trace(">>> convertToRestFileInfo({}, {})", storageFile, fileSize); RestFileInfo restFileInfo = new RestFileInfo(); @@ -394,6 +406,9 @@ private static RestFileInfo convertToRestFileInfo(StorageFile storageFile, long * @return product folder with the file name */ private String getProductFolderWithFilename(String externalPath, Long productId) { + + if (logger.isTraceEnabled()) + logger.trace(">>> getProductFolderWithFilename({}, {})", externalPath, productId); String fileName = new File(externalPath).getName(); return Paths.get(String.valueOf(productId), fileName).toString(); diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java index 0f044fe26..e3b1939c3 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java @@ -46,6 +46,9 @@ public class StorageFileLocker { * @param fileCheckMaxCycles file check max cycles */ public StorageFileLocker(String path, long waitTime, long fileCheckMaxCycles) { + + if (logger.isTraceEnabled()) + logger.trace(">>> StorageFileLocker({}, {}, {})", path, waitTime, fileCheckMaxCycles); this.path = path; this.fileCheckMaxCycles = fileCheckMaxCycles; @@ -66,6 +69,9 @@ public StorageFileLocker(String path, long waitTime, long fileCheckMaxCycles) { * to terminate */ public void lockOrWaitUntilUnlockedAndLock() throws FileLockedAfterMaxCyclesException, InterruptedException { + + if (logger.isTraceEnabled()) + logger.trace(">>> lockOrWaitUntilUnlockedAndLock()"); long i = 0; @@ -97,7 +103,15 @@ public void lockOrWaitUntilUnlockedAndLock() throws FileLockedAfterMaxCyclesExce * productLockSet. */ public void unlock() { - productLockSet.remove(path); - logger.debug("... the file {} is unlocked", path); + + if (logger.isTraceEnabled()) + logger.trace(">>> unlock()"); + + if (productLockSet.remove(path)) { + logger.debug("... the file {} is unlocked", path); + } + else { + logger.debug("... the file {} was not locked (not in the concurrent list)", path); + } } } \ No newline at end of file From a304edc7c5eee4616a41e10aa0ef930738bd89a5 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Tue, 6 Feb 2024 11:07:57 +0100 Subject: [PATCH 47/79] storage-mgr: removed synchronization for copying from cache to storage --- .../rest/ProductfileControllerImpl.java | 51 ++++--------------- 1 file changed, 9 insertions(+), 42 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index ab507e37b..8029cdd21 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -160,7 +160,8 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro RestFileInfo restFileInfo = copyFileExternalToCache(externalPath, productId, fileSize, fileLocker); fileLocker.unlock(); - restFileInfo = copyFileCacheToStorage(relativePath, fileLocker); + + restFileInfo = copyFileCacheToStorage(relativePath); logger.log(StorageMgrMessage.PRODUCT_FILE_UPLOADED_TO_STORAGE, externalPath, productId); return new ResponseEntity<>(restFileInfo, HttpStatus.CREATED); @@ -173,7 +174,6 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro } finally { fileLocker.unlock(); - logger.debug("... unlocked the file: ", externalPath); } } @@ -313,62 +313,29 @@ private RestFileInfo copyFileExternalToCache(String externalPath, Long productId return restFileInfo; } - // TODO: WIP Special use case for cache recovery - file in cache, but not in - // storage - // TODO: WIP Special use case for cache state - not uploaded to storage - /** - * Copies the file from the cache to the storage using synchronization. During - * the copying to the cache, the status of the file will be "not exists", after - * the completion the status will be set to "ready" + * Copies the file from the cache to the backend storage. * * @param relativeCachePath relative cache path - * @param fileLocker is used for synchronization * @return RestFileInfo - * @throws FileLockedAfterMaxCyclesException * @throws IOException * @throws Exception */ - private RestFileInfo copyFileCacheToStorage(String relativeCachePath, StorageFileLocker fileLocker) + private RestFileInfo copyFileCacheToStorage(String relativeCachePath) throws FileLockedAfterMaxCyclesException, IOException, Exception { if (logger.isTraceEnabled()) - logger.trace(">>> copyFileCacheToStorage({}, {})", relativeCachePath, fileLocker); + logger.trace(">>> copyFileCacheToStorage({})", relativeCachePath); Storage storage = storageProvider.getStorage(); StorageFile cacheFile = storageProvider.getCacheFile(relativeCachePath); StorageFile storageFile = storageProvider.getStorageFile(relativeCachePath); + + storage.uploadFile(cacheFile, storageFile); - FileCache cache = FileCache.getInstance(); - - if (!cache.containsKey(cacheFile.getFullPath())) { - - fileLocker.lockOrWaitUntilUnlockedAndLock(); - - // check again, the file could be copied to storage from another thread after - // lock - if (!cache.containsKey(cacheFile.getFullPath())) { - - // active thread - copies the file to the storage and checks it as OK (WIP) - storage.uploadFile(cacheFile, storageFile); - - logger.log(StorageMgrMessage.PRODUCT_FILE_DOWNLOADED_FROM_EXTERNAL_TO_CACHE, storageFile.getFullPath()); - - cache.put(storageFile.getFullPath()); - - } else { - - // passive thread - did nothing, waited for copied file and use it from cache - logger.debug("... waiting-thread when the file downloaded and use it from cache: ", - storageFile.getFullPath()); - } - - } else { - - logger.debug("... no download and no lock - the file is in cache: ", storageFile.getFullPath()); - } - + logger.log(StorageMgrMessage.PRODUCT_FILE_UPLOADED_FROM_CACHE_TO_STORAGE, storageFile.getFullPath()); + RestFileInfo restFileInfo = convertToRestFileInfo(storageFile, storageProvider.getCacheFileSize(cacheFile.getRelativePath())); From 80467df078ff86820d8a0e78860f8d031ce164fc Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Fri, 9 Feb 2024 10:52:15 +0100 Subject: [PATCH 48/79] storage-mgr: added stream download from cache (if exists) for product --- .../rest/ProductControllerImpl.java | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java index 58507f6fa..9fcf52f82 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductControllerImpl.java @@ -38,6 +38,7 @@ import de.dlr.proseo.logging.messages.StorageMgrMessage; import de.dlr.proseo.storagemgr.StorageManagerConfiguration; import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.cache.FileCache; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; @@ -219,20 +220,27 @@ public ResponseEntity getObject(String pathInfo, String token, Long fromByte, } // token check end + // Storage Manager version 2: storage file -> byte page try { Storage storage = storageProvider.getStorage(pathInfo); - String relativePath = storage.getRelativePath(pathInfo); - - StorageFile sourceFile = storage.getStorageFile(relativePath); - - if (sourceFile == null) { - - String msg = logger.log(StorageMgrMessage.PATH_IS_NULL); - return new ResponseEntity<>(http.errorHeaders(msg), HttpStatus.BAD_REQUEST); + + StorageFile cacheFile = storageProvider.getCacheFile(relativePath); + FileCache cache = FileCache.getInstance(); + + StorageFile sourceFile; + if (cache.containsKey(cacheFile.getFullPath())) { + + sourceFile = cacheFile; + logger.debug("... product file found in cache - it will be downloaded from cache: {}", sourceFile.getFullPath()); + } + else { + + sourceFile = storage.getStorageFile(relativePath); + logger.debug("... no product file in cache - it will be downloaded from storage: {}", sourceFile.getFullPath()); } // token check begin From ebf9c4692476e5b27d10d49be57bc598a324f456 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Fri, 9 Feb 2024 10:56:24 +0100 Subject: [PATCH 49/79] storage-mgr: added final to read-only-for-all-users in prod file ctrl --- .../dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index 8029cdd21..9f4dbf2c3 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -39,7 +39,7 @@ public class ProductfileControllerImpl implements ProductfileController { /** The file permission with read-only permissions for all users */ - private static String READ_ONLY_FOR_ALL_USERS = "r--r--r--"; + private static final String READ_ONLY_FOR_ALL_USERS = "r--r--r--"; /** A logger for this class */ private static ProseoLogger logger = new ProseoLogger(ProductControllerImpl.class); From 93a3b0757394b35235d7472bcb1f1999bd9ddf2d Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Fri, 9 Feb 2024 11:05:21 +0100 Subject: [PATCH 50/79] storage-mgr: added test conditions for stream download from cache (WIP) --- .../rest/ProductControllerImplTest_download.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java index 2dad0bffa..4a4e85c80 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java @@ -22,6 +22,7 @@ import de.dlr.proseo.storagemgr.StorageProvider; import de.dlr.proseo.storagemgr.BaseStorageTestUtils; import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.cache.FileCache; import de.dlr.proseo.storagemgr.model.Storage; import de.dlr.proseo.storagemgr.model.StorageFile; import de.dlr.proseo.storagemgr.model.StorageType; @@ -152,6 +153,18 @@ private void downloadProductFiles(StorageType storageType) throws Exception { // show storage files BaseStorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); + + // TODO: added test conditions: 1) stream download from cache 2) stream download from storage + // the code above puts the file to the cache and covers 2), without this code 1) is covered + + /* puts to cache in order to stream download from cache + StorageFile storageFile = storageProvider.getStorageFile(relativePath); + StorageFile cacheFile = storageProvider.getCacheFile(storageFile.getRelativePath()); + storageProvider.getStorage().downloadFile(storageFile, cacheFile); + + FileCache cache = FileCache.getInstance(); + cache.put(cacheFile.getFullPath()); // cache file status = READY + */ // TEST PARTIAL CONTENT // HTTP Download files (partial content) from storage From 08627bea6ef3e8435eb18df01edd7acca0927fb9 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Mon, 12 Feb 2024 09:49:33 +0100 Subject: [PATCH 51/79] storage-mgr: integrated test check to select download from cache/storage --- .../de/dlr/proseo/storagemgr/TestUtils.java | 3 ++ .../ProductControllerImplTest_download.java | 51 ++++++++++++------- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java index 340027892..b72d3a5c9 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java @@ -190,6 +190,9 @@ public static void deleteFile(String path) { System.out.println("File was NOT deleted: " + file.getPath()); } + else { + System.out.println("File was deleted: " + file.getPath()); + } } /** diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java index 4a4e85c80..34b9f96a4 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java @@ -73,15 +73,17 @@ public void testDownload_posix() throws Exception { StorageType storageType = StorageType.POSIX; storageProvider.setStorage(storageType); + boolean downloadFileFromCache = false; - downloadProductFiles(storageType); + downloadProductFiles(storageType, downloadFileFromCache); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); + } else { + System.out.println("TESTS ARE DISABLED"); } } - /** * Downloads products with given directory prefix * @@ -96,15 +98,17 @@ public void testDownload_S3() throws Exception { StorageType storageType = StorageType.S3; storageProvider.setStorage(storageType); + boolean downloadFileFromCache = false; - downloadProductFiles(storageType); + downloadProductFiles(storageType, downloadFileFromCache); StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); + } else { + System.out.println("TESTS ARE DISABLED"); } } - /** * Get the data files for the product as data stream (optionally zip-compressed, * optionally range-restricted) @@ -124,8 +128,14 @@ public void testDownload_S3() throws Exception { * s3:/// // no storage path in s3 * // // no bucket in posix currently * + * @param storageType storage Type (S3 or POSIX) + * @param downloadFileFromCache if true, a file will be copied to cache before + * download(). If a file is in the cache, + * download() method will download it directly from + * the cache, not from the storage + * */ - private void downloadProductFiles(StorageType storageType) throws Exception { + private void downloadProductFiles(StorageType storageType, boolean downloadFileFromCache) throws Exception { TestUtils.printMethodName(this, testName); @@ -134,7 +144,7 @@ private void downloadProductFiles(StorageType storageType) throws Exception { String relativePath = new PathConverter(prefix, "productDownloadDir/productDownload3.txt").getPath(); String fileContent = "some content"; - + // create file in source storageTestUtils.createSourceFile(relativePath, fileContent); @@ -153,18 +163,22 @@ private void downloadProductFiles(StorageType storageType) throws Exception { // show storage files BaseStorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); - - // TODO: added test conditions: 1) stream download from cache 2) stream download from storage - // the code above puts the file to the cache and covers 2), without this code 1) is covered - /* puts to cache in order to stream download from cache + // puts a file to the cache in order to trigger a stream downloading from cache + // if true, a file will be copied to the cache before download() StorageFile storageFile = storageProvider.getStorageFile(relativePath); StorageFile cacheFile = storageProvider.getCacheFile(storageFile.getRelativePath()); - storageProvider.getStorage().downloadFile(storageFile, cacheFile); - FileCache cache = FileCache.getInstance(); - cache.put(cacheFile.getFullPath()); // cache file status = READY - */ + + if (downloadFileFromCache) { + + storageProvider.getStorage().downloadFile(storageFile, cacheFile); + cache.put(cacheFile.getFullPath()); // cache file status = READY + } else { + + // avoid downloading from cache - delete the file in cache if exists + TestUtils.deleteFile(cacheFile.getFullPath()); + } // TEST PARTIAL CONTENT // HTTP Download files (partial content) from storage @@ -194,8 +208,8 @@ private void downloadProductFiles(StorageType storageType) throws Exception { // HTTP Download files (FULL CONTENT) from storage System.out.println("TEST FULL CONTENT BEGIN - BEFORE HTTP CALL "); - request = MockMvcRequestBuilders.get(REQUEST_STRING) - .param("pathInfo", absoluteStoragePath).param("token", token); + request = MockMvcRequestBuilders.get(REQUEST_STRING).param("pathInfo", absoluteStoragePath).param("token", + token); mvcResult = mockMvc.perform(request).andExpect(status().isOk()).andReturn(); @@ -212,10 +226,13 @@ private void downloadProductFiles(StorageType storageType) throws Exception { assertTrue("Real path: " + realFileContent + " Expected path: " + expectedFileContent, realFileContent.equals(expectedFileContent)); System.out.println("TEST FULL CONTENT END"); - + // delete storage files with prefix storageProvider.getStorage().delete(prefix); + // delete cache file if exists + TestUtils.deleteFile(cacheFile.getFullPath()); + // show storage files after deletion BaseStorageTestUtils.printStorageFiles("After deletion", storageProvider.getStorage()); } From d5e75f8328340822e04fb1488bc421fdbe2099e2 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Mon, 12 Feb 2024 10:12:20 +0100 Subject: [PATCH 52/79] storage-mgr: added 2 product stream-downloading tests from cache --- .../ProductControllerImplTest_download.java | 60 ++++++++++++++++--- 1 file changed, 53 insertions(+), 7 deletions(-) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java index 34b9f96a4..eb7b5e7cd 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductControllerImplTest_download.java @@ -60,14 +60,13 @@ public class ProductControllerImplTest_download { private static final String REQUEST_STRING = "/proseo/storage-mgr/x/products/download"; /** - * Downloads products with given directory prefix + * Downloads products with given directory prefix from storage * * GET /products storageType="POSIX"&prefix="/.." * - * @return products string[] */ @Test - public void testDownload_posix() throws Exception { + public void testDownloadFromStorage_posix() throws Exception { if (TESTS_ENABLED) { @@ -85,14 +84,13 @@ public void testDownload_posix() throws Exception { } /** - * Downloads products with given directory prefix + * Downloads products with given directory prefix from storage * - * GET /products storageType="POSIX"&prefix="/.." + * GET /products storageType="S3"&prefix="/.." * - * @return products string[] */ @Test - public void testDownload_S3() throws Exception { + public void testDownloadfromStorage_S3() throws Exception { if (TESTS_ENABLED) { @@ -109,6 +107,54 @@ public void testDownload_S3() throws Exception { } } + /** + * Downloads products with given directory prefix from cache + * + * GET /products storageType="POSIX"&prefix="/.." + * + */ + @Test + public void testDownloadFromCache_posix() throws Exception { + + if (TESTS_ENABLED) { + + StorageType storageType = StorageType.POSIX; + storageProvider.setStorage(storageType); + boolean downloadFileFromCache = true; + + downloadProductFiles(storageType, downloadFileFromCache); + + StorageType realStorageType = storageProvider.getStorage().getStorageType(); + assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); + } else { + System.out.println("TESTS ARE DISABLED"); + } + } + + /** + * Downloads products with given directory prefix from cache + * + * GET /products storageType="S3"&prefix="/.." + * + */ + @Test + public void testDownloadfromCache_S3() throws Exception { + + if (TESTS_ENABLED) { + + StorageType storageType = StorageType.S3; + storageProvider.setStorage(storageType); + boolean downloadFileFromCache = true; + + downloadProductFiles(storageType, downloadFileFromCache); + + StorageType realStorageType = storageProvider.getStorage().getStorageType(); + assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); + } else { + System.out.println("TESTS ARE DISABLED"); + } + } + /** * Get the data files for the product as data stream (optionally zip-compressed, * optionally range-restricted) From 5a2e52c3d27e8bf0836bdb671b0f544292852c30 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Wed, 14 Feb 2024 13:14:30 +0100 Subject: [PATCH 53/79] storage-mgr: added createdLargeFile method for testing sync operations --- .../de/dlr/proseo/storagemgr/TestUtils.java | 103 ++++++++++++------ 1 file changed, 72 insertions(+), 31 deletions(-) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java index b72d3a5c9..7407bd886 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java @@ -4,6 +4,8 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.io.UnsupportedEncodingException; +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; import java.nio.file.Paths; import java.util.Arrays; import java.util.List; @@ -36,8 +38,8 @@ public class TestUtils { private static final String TARGET_DIRECTORY = "target"; @Autowired - private StorageProvider storageProvider; - + private StorageProvider storageProvider; + @Autowired private StorageManagerConfiguration cfg; @@ -157,6 +159,44 @@ public static void createFile(String path, String content) { fileUtils.createFile(content); } + /** + * Creates a large file + * + * For example: + * long fileSizeInBytes = 100L * 1024 * 1024; // 100 MB + * + * @param filePath + * @param fileSizeInBytes + */ + public static void createLargeFile(String filePath, long fileSizeInBytes) { + + File file = new File(filePath); + + if (!file.getPath().contains(TEST_DIRECTORY)) { + + System.out.println("Attempt to create file not in test dir: " + file.getPath()); + return; + } + + try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath))) { + + byte[] buffer = new byte[1024]; + long bytesWritten = 0; + + while (bytesWritten < fileSizeInBytes) { + + int bytesToWrite = (int) Math.min(buffer.length, fileSizeInBytes - bytesWritten); + bos.write(buffer, 0, bytesToWrite); + bytesWritten += bytesToWrite; + } + + System.out.println("Large file created successfully."); + + } catch (IOException e) { + e.printStackTrace(); + } + } + /** * @param path */ @@ -189,8 +229,7 @@ public static void deleteFile(String path) { if (!file.delete()) { System.out.println("File was NOT deleted: " + file.getPath()); - } - else { + } else { System.out.println("File was deleted: " + file.getPath()); } } @@ -323,17 +362,17 @@ public static void printList(String message, List list) { } System.out.println(); } - + /** * @param message * @param directoryPath */ public static void printDirectoryTree(String message, String directoryPath) { - + System.out.println(); - System.out.println(message); - - printDirectoryTree(directoryPath); + System.out.println(message); + + printDirectoryTree(directoryPath); } /** @@ -371,7 +410,8 @@ private static void printDirectoryTreeWithDepth(String directoryPath, String dep // System.out.println("FOLDER: " + directoryPath + " " + " DEPTH: " + depth); File[] files = directory.listFiles(); - if (files == null) return; + if (files == null) + return; Arrays.sort(files); /* @@ -438,7 +478,7 @@ private static int countDirectoriesInDirectory(File directory) { } return count; } - + public static String asJsonString(final Object obj) { try { return new ObjectMapper().writeValueAsString(obj); @@ -446,40 +486,41 @@ public static String asJsonString(final Object obj) { throw new RuntimeException(e); } } - + /** - * @throws IOException + * @throws IOException */ public void deleteFilesinS3Storage() throws IOException { - - deleteFilesInStorage(StorageType.S3); + + deleteFilesInStorage(StorageType.S3); } - + /** - * @throws IOException + * @throws IOException */ public void deleteFilesinPosixStorage() throws IOException { - - deleteFilesInStorage(StorageType.POSIX); + + deleteFilesInStorage(StorageType.POSIX); } - + /** - * @throws IOException + * @throws IOException */ private void deleteFilesInStorage(StorageType storageType) throws IOException { - + File file = new File(getStoragePath()); if (!file.getPath().contains(TEST_DIRECTORY)) { - System.out.println("Attempt to delete " + storageType.toString() + " storage files not from unit test: " + getStoragePath()); + System.out.println("Attempt to delete " + storageType.toString() + " storage files not from unit test: " + + getStoragePath()); return; } - + Storage storage = storageProvider.getStorage(storageType); List relativePaths = storage.getRelativeFiles(); - - for(String relativePath : relativePaths) { + + for (String relativePath : relativePaths) { StorageFile storageFile = storage.getStorageFile(relativePath); try { storage.delete(storageFile); @@ -487,24 +528,24 @@ private void deleteFilesInStorage(StorageType storageType) throws IOException { e.printStackTrace(); } } - + List storageFilesAfterDelete = storage.getRelativeFiles(); TestUtils.printList("Storage after delete all " + storageType + " files:", storageFilesAfterDelete); } public static void printMvcResult(String requestString, MvcResult mvcResult) { - + System.out.println(); System.out.println("HTTP Response"); System.out.println("Request: " + requestString); System.out.println("Status: " + mvcResult.getResponse().getStatus()); - + try { System.out.println("Content: " + mvcResult.getResponse().getContentAsString()); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } - - System.out.println(); + + System.out.println(); } } From 1402474067097c5967dfdfe17a5d5620a5ac4560 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Wed, 14 Feb 2024 13:19:30 +0100 Subject: [PATCH 54/79] storage-mgr: added createSourceFile with filesize for big files --- .../storagemgr/BaseStorageTestUtils.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java index cc5ebd543..eb35ed235 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/BaseStorageTestUtils.java @@ -122,6 +122,30 @@ public String createSourceFile(String relativePath, String fileContent) { return sourceFilePath; } + /** + * Creates a large file in source and returns absolute path of created file + * + * For example: + * long fileSizeInBytes = 100L * 1024 * 1024; // 100 MB + * + * @param relativePath + * @param fileSize + * @return absolute path of created file + */ + public String createSourceFile(String relativePath, long fileSizeInBytes) { + + String path = Paths.get(sourcePath, relativePath).toString(); + String sourceFilePath = new PathConverter(path).convertToSlash().getPath(); + + TestUtils.createLargeFile(sourceFilePath, fileSizeInBytes); + + assertTrue("File for upload in Source has not been created: " + sourceFilePath, TestUtils.fileExists(sourceFilePath)); + + System.out.println("File " + relativePath + " successfully created in Source"); + + return sourceFilePath; + } + public String getAbsoluteSourcePath(String relativePath) { String path = Paths.get(sourcePath, relativePath).toString(); From 840e542f998f07ea737fe8de36d8a47b68f3f53f Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Wed, 14 Feb 2024 13:21:45 +0100 Subject: [PATCH 55/79] storage-mgr: created syncDownload tests (WIP with threads) --- ...ctfileControllerImplTest_syncDownload.java | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java new file mode 100644 index 000000000..c933eb304 --- /dev/null +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java @@ -0,0 +1,160 @@ +package de.dlr.proseo.storagemgr.rest; + +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; + +import static org.junit.Assert.assertTrue; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.io.File; + +import org.codehaus.jackson.map.ObjectMapper; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; + +import de.dlr.proseo.storagemgr.StorageManager; +import de.dlr.proseo.storagemgr.StorageProvider; +import de.dlr.proseo.storagemgr.BaseStorageTestUtils; +import de.dlr.proseo.storagemgr.TestUtils; +import de.dlr.proseo.storagemgr.model.StorageFile; +import de.dlr.proseo.storagemgr.model.StorageType; +import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; +import de.dlr.proseo.storagemgr.utils.FileUtils; +import de.dlr.proseo.storagemgr.utils.PathConverter; + +/** + * Mock Mvc test for Product Controller + * + * @author Denys Chaykovskiy + * + */ +/** + * @throws Exception + */ +@RunWith(SpringJUnit4ClassRunner.class) +@SpringBootTest(classes = StorageManager.class, webEnvironment = WebEnvironment.RANDOM_PORT) +@AutoConfigureMockMvc +public class ProductfileControllerImplTest_syncDownload { + + @Autowired + private MockMvc mockMvc; + + @Autowired + private BaseStorageTestUtils storageTestUtils; + + @Autowired + private StorageProvider storageProvider; + + @Rule + public TestName testName = new TestName(); + + private static final String REQUEST_STRING = "/proseo/storage-mgr/x/productfiles"; + + @Test + public void testDownload_posix() throws Exception { + + StorageType storageType = StorageType.POSIX; + storageProvider.setStorage(storageType); + + syncDownload("Posix"); + + StorageType realStorageType = storageProvider.getStorage().getStorageType(); + assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); + } + + @Test + public void testDownload_S3() throws Exception { + + StorageType storageType = StorageType.S3; + storageProvider.setStorage(storageType); + + syncDownload("S3"); + + StorageType realStorageType = storageProvider.getStorage().getStorageType(); + assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); + } + + /** + * DOWNLOAD Storage -> Cache (getRestFileInfoByPathInfo) + * + * takes filename from path and productid from parameter, ignores the rest of the path + * + * INPUT + * + * absolutePath + * s3:/// // no storage path in s3 + * // // no bucket in posix currently + * + * OUTPUT + * + * Posix only (cache): // + */ + private void syncDownload(String testID) throws Exception { + + TestUtils.printMethodName(this, testName); + + // create file in source + // upload to storage + // call http-download + // check results (download in cache) + + String relativePath = "product/productFileDownload" + testID + ".txt"; + relativePath = new PathConverter(relativePath).getPath(); + + // create file in source + long fileSizeInBytes = 100L * 1024 * 1024; // 100 MB + String absoluteSourcePath = storageTestUtils.createSourceFile(relativePath, fileSizeInBytes); + + // upload file to storage from source + StorageFile sourceFile = storageProvider.getSourceFile(relativePath); + StorageFile storageFile = storageProvider.getStorageFile(relativePath); + storageProvider.getStorage().upload(sourceFile, storageFile); + + // show storage files + BaseStorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); + + // rest-download file from storage to cache + String absoluteStoragePath = storageProvider.getStorage().getAbsolutePath(relativePath); + System.out.println("Http-download call path (absolute storage path):" + absoluteStoragePath); + + MockHttpServletRequestBuilder request = MockMvcRequestBuilders.get(REQUEST_STRING) + .param("pathInfo", absoluteStoragePath); + MvcResult mvcResult = mockMvc.perform(request).andExpect(status().isOk()).andReturn(); + + // show results of http-download + TestUtils.printMvcResult(REQUEST_STRING, mvcResult); + + storageTestUtils.printCache(); + + // check real with expected absolute cache path + String expectedAbsoluteCachePath = new PathConverter(storageProvider.getCachePath(), relativePath).getPath(); + + String json = mvcResult.getResponse().getContentAsString(); + RestFileInfo result = new ObjectMapper().readValue(json, RestFileInfo.class); + String realAbsoluteCachePath = result.getFilePath(); + + System.out.println("Real cache path: " + realAbsoluteCachePath); + System.out.println("Expected cache path: " + expectedAbsoluteCachePath); + + realAbsoluteCachePath = new PathConverter(realAbsoluteCachePath).normalizeWindowsPath().getPath(); + assertTrue("Real cache path: " + realAbsoluteCachePath + " expected cache path: " + expectedAbsoluteCachePath, + realAbsoluteCachePath.equals(expectedAbsoluteCachePath)); + + assertTrue("Downloaded file from storage to cache does not exist: " + realAbsoluteCachePath, + new File(realAbsoluteCachePath).exists()); + + // delete files with empty folders + new FileUtils(absoluteSourcePath).deleteFile(); // source + new FileUtils(expectedAbsoluteCachePath).deleteFile(); // cache + storageProvider.getStorage().deleteFile(storageFile); // in storage + } +} From 35b70268bd97b1d26775722d0cf42164cb0ce1d4 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Wed, 14 Feb 2024 14:29:50 +0100 Subject: [PATCH 56/79] storage-mgr: added creation of directories in createLargeFile --- .../src/test/java/de/dlr/proseo/storagemgr/TestUtils.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java index 7407bd886..30950ea69 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/TestUtils.java @@ -178,6 +178,8 @@ public static void createLargeFile(String filePath, long fileSizeInBytes) { return; } + file.getParentFile().mkdirs(); + try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath))) { byte[] buffer = new byte[1024]; From ed9d8287a3992ba0e6a17efed705ae3976cf49b4 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Wed, 14 Feb 2024 14:34:16 +0100 Subject: [PATCH 57/79] storage-mgr: added thread processing to the sync download testing --- ...ctfileControllerImplTest_syncDownload.java | 156 +++++++++++------- 1 file changed, 98 insertions(+), 58 deletions(-) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java index c933eb304..53825c117 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java @@ -47,13 +47,13 @@ public class ProductfileControllerImplTest_syncDownload { @Autowired private MockMvc mockMvc; - + @Autowired private BaseStorageTestUtils storageTestUtils; - - @Autowired + + @Autowired private StorageProvider storageProvider; - + @Rule public TestName testName = new TestName(); @@ -61,100 +61,140 @@ public class ProductfileControllerImplTest_syncDownload { @Test public void testDownload_posix() throws Exception { - - StorageType storageType = StorageType.POSIX; + + StorageType storageType = StorageType.POSIX; storageProvider.setStorage(storageType); syncDownload("Posix"); - + StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM POSIX, " + " Exists: " + realStorageType, storageType == realStorageType); } @Test public void testDownload_S3() throws Exception { - - StorageType storageType = StorageType.S3; + + StorageType storageType = StorageType.S3; storageProvider.setStorage(storageType); syncDownload("S3"); - + StorageType realStorageType = storageProvider.getStorage().getStorageType(); assertTrue("Expected: SM S3, " + " Exists: " + realStorageType, storageType == realStorageType); } - + /** * DOWNLOAD Storage -> Cache (getRestFileInfoByPathInfo) * - * takes filename from path and productid from parameter, ignores the rest of the path + * takes filename from path and productid from parameter, ignores the rest of + * the path * - * INPUT + * INPUT * - * absolutePath - * s3:/// // no storage path in s3 - * // // no bucket in posix currently + * absolutePath s3:/// // no storage path in s3 + * // // no bucket in posix currently * - * OUTPUT + * OUTPUT * - * Posix only (cache): // + * Posix only (cache): // */ private void syncDownload(String testID) throws Exception { - + TestUtils.printMethodName(this, testName); - + // create file in source - // upload to storage - // call http-download + // upload to storage + // call http-download // check results (download in cache) - + String relativePath = "product/productFileDownload" + testID + ".txt"; relativePath = new PathConverter(relativePath).getPath(); - - // create file in source + + // create file in source long fileSizeInBytes = 100L * 1024 * 1024; // 100 MB String absoluteSourcePath = storageTestUtils.createSourceFile(relativePath, fileSizeInBytes); - + // upload file to storage from source StorageFile sourceFile = storageProvider.getSourceFile(relativePath); StorageFile storageFile = storageProvider.getStorageFile(relativePath); storageProvider.getStorage().upload(sourceFile, storageFile); - + // show storage files BaseStorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); - // rest-download file from storage to cache - String absoluteStoragePath = storageProvider.getStorage().getAbsolutePath(relativePath); - System.out.println("Http-download call path (absolute storage path):" + absoluteStoragePath); - - MockHttpServletRequestBuilder request = MockMvcRequestBuilders.get(REQUEST_STRING) - .param("pathInfo", absoluteStoragePath); - MvcResult mvcResult = mockMvc.perform(request).andExpect(status().isOk()).andReturn(); - - // show results of http-download - TestUtils.printMvcResult(REQUEST_STRING, mvcResult); + DownloadThread thread1 = new DownloadThread(relativePath); + thread1.start(); - storageTestUtils.printCache(); - - // check real with expected absolute cache path - String expectedAbsoluteCachePath = new PathConverter(storageProvider.getCachePath(), relativePath).getPath(); - - String json = mvcResult.getResponse().getContentAsString(); - RestFileInfo result = new ObjectMapper().readValue(json, RestFileInfo.class); - String realAbsoluteCachePath = result.getFilePath(); - - System.out.println("Real cache path: " + realAbsoluteCachePath); - System.out.println("Expected cache path: " + expectedAbsoluteCachePath); - - realAbsoluteCachePath = new PathConverter(realAbsoluteCachePath).normalizeWindowsPath().getPath(); - assertTrue("Real cache path: " + realAbsoluteCachePath + " expected cache path: " + expectedAbsoluteCachePath, - realAbsoluteCachePath.equals(expectedAbsoluteCachePath)); + DownloadThread thread2 = new DownloadThread(relativePath); + thread2.start(); + + // delete files with empty folders + + String cacheFile = new PathConverter(storageProvider.getCachePath(), relativePath).getPath(); + + if (new File(absoluteSourcePath).exists()) { + new FileUtils(absoluteSourcePath).deleteFile(); // source + } - assertTrue("Downloaded file from storage to cache does not exist: " + realAbsoluteCachePath, - new File(realAbsoluteCachePath).exists()); + if (new File(cacheFile).exists()) { + new FileUtils(cacheFile).deleteFile(); // cache + } - // delete files with empty folders - new FileUtils(absoluteSourcePath).deleteFile(); // source - new FileUtils(expectedAbsoluteCachePath).deleteFile(); // cache - storageProvider.getStorage().deleteFile(storageFile); // in storage + if (new File(storageFile.getFullPath()).exists()) { + storageProvider.getStorage().deleteFile(storageFile); // in storage + } + } + + private class DownloadThread extends Thread { + + private String relativePath; + + public DownloadThread(String relativePath) { + + this.relativePath = relativePath; + } + + public void run() { + + System.out.println("Thread name:" + Thread.currentThread().getName()); + + // rest-download file from storage to cache + String absoluteStoragePath = storageProvider.getStorage().getAbsolutePath(relativePath); + System.out.println("Http-download call path (absolute storage path):" + absoluteStoragePath); + + MockHttpServletRequestBuilder request = MockMvcRequestBuilders.get(REQUEST_STRING).param("pathInfo", + absoluteStoragePath); + MvcResult mvcResult; + + try { + mvcResult = mockMvc.perform(request).andExpect(status().isOk()).andReturn(); + + // show results of http-download + TestUtils.printMvcResult(REQUEST_STRING, mvcResult); + + storageTestUtils.printCache(); + + // check real with expected absolute cache path + String expectedAbsoluteCachePath = new PathConverter(storageProvider.getCachePath(), relativePath) + .getPath(); + + String json = mvcResult.getResponse().getContentAsString(); + RestFileInfo result = new ObjectMapper().readValue(json, RestFileInfo.class); + String realAbsoluteCachePath = result.getFilePath(); + + System.out.println("Real cache path: " + realAbsoluteCachePath); + System.out.println("Expected cache path: " + expectedAbsoluteCachePath); + + realAbsoluteCachePath = new PathConverter(realAbsoluteCachePath).normalizeWindowsPath().getPath(); + assertTrue("Real cache path: " + realAbsoluteCachePath + " expected cache path: " + + expectedAbsoluteCachePath, realAbsoluteCachePath.equals(expectedAbsoluteCachePath)); + + assertTrue("Downloaded file from storage to cache does not exist: " + realAbsoluteCachePath, + new File(realAbsoluteCachePath).exists()); + + } catch (Exception e) { + e.printStackTrace(); + } + } } } From 6b80d3380f2849f23858b4657f10996ba75278ae Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 15 Feb 2024 10:45:22 +0100 Subject: [PATCH 58/79] storage-mgr: created a common output for threads in sync download --- .../rest/ProductfileControllerImplTest_syncDownload.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java index 53825c117..01543055d 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java @@ -127,6 +127,9 @@ private void syncDownload(String testID) throws Exception { DownloadThread thread2 = new DownloadThread(relativePath); thread2.start(); + + thread1.join(); + thread2.join(); // delete files with empty folders From ce232bba1fad3662245f9142ef98ff6dcb162fa5 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Fri, 16 Feb 2024 13:16:04 +0100 Subject: [PATCH 59/79] storage-mgr: added third unconcurrent thread to sync download test --- .../rest/ProductfileControllerImpl.java | 25 +++++++++---------- .../proseo/storagemgr/utils/FileUtils.java | 5 ++-- .../storagemgr/utils/StorageFileLocker.java | 8 +++--- ...ctfileControllerImplTest_syncDownload.java | 18 ++++++++++--- 4 files changed, 32 insertions(+), 24 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index 9f4dbf2c3..b4daa2c89 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -120,7 +120,6 @@ public ResponseEntity getRestFileInfoByPathInfo(String pathInfo) { } finally { fileLocker.unlock(); - logger.debug("... unlocked the file: ", pathInfo); } } @@ -160,7 +159,7 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro RestFileInfo restFileInfo = copyFileExternalToCache(externalPath, productId, fileSize, fileLocker); fileLocker.unlock(); - + restFileInfo = copyFileCacheToStorage(relativePath); logger.log(StorageMgrMessage.PRODUCT_FILE_UPLOADED_TO_STORAGE, externalPath, productId); @@ -192,7 +191,7 @@ public ResponseEntity updateProductfiles(String pathInfo, Long pro */ private RestFileInfo copyFileStorageToCache(String storageFilePath, StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { - + if (logger.isTraceEnabled()) logger.trace(">>> copyFileStorageToCache({}, {})", storageFilePath, fileLocker); @@ -229,7 +228,7 @@ private RestFileInfo copyFileStorageToCache(String storageFilePath, StorageFileL } else { // passive thread - did nothing, waited for copied file and use it from cache - logger.debug("... waiting-thread when the file downloaded and use it from cache: ", + logger.debug("... waiting-thread: waited until the file was downloaded to cache from external storage and use it from cache: {}", cacheFile.getFullPath()); } @@ -263,7 +262,7 @@ private RestFileInfo copyFileStorageToCache(String storageFilePath, StorageFileL */ private RestFileInfo copyFileExternalToCache(String externalPath, Long productId, Long fileSize, StorageFileLocker fileLocker) throws FileLockedAfterMaxCyclesException, IOException, Exception { - + if (logger.isTraceEnabled()) logger.trace(">>> copyFileExternalToCache({}, {}, {}, {})", externalPath, productId, fileSize, fileLocker); @@ -298,13 +297,13 @@ private RestFileInfo copyFileExternalToCache(String externalPath, Long productId // passive thread - did nothing, waited for copied file and use it from cache logger.debug( - "... waiting-thread when the file downloaded to cache from external storage and use it from cache: ", + "... waiting-thread: waited until the file was downloaded to cache from external storage and use it from cache: {}", cacheFile.getFullPath()); } } else { - logger.debug("... no download and no lock - the file is in cache: ", cacheFile.getFullPath()); + logger.debug("... no download and no lock - the file is in cache: {}", cacheFile.getFullPath()); } RestFileInfo restFileInfo = convertToRestFileInfo(cacheFile, @@ -314,7 +313,7 @@ private RestFileInfo copyFileExternalToCache(String externalPath, Long productId } /** - * Copies the file from the cache to the backend storage. + * Copies the file from the cache to the backend storage. * * @param relativeCachePath relative cache path * @return RestFileInfo @@ -323,7 +322,7 @@ private RestFileInfo copyFileExternalToCache(String externalPath, Long productId */ private RestFileInfo copyFileCacheToStorage(String relativeCachePath) throws FileLockedAfterMaxCyclesException, IOException, Exception { - + if (logger.isTraceEnabled()) logger.trace(">>> copyFileCacheToStorage({})", relativeCachePath); @@ -331,11 +330,11 @@ private RestFileInfo copyFileCacheToStorage(String relativeCachePath) StorageFile cacheFile = storageProvider.getCacheFile(relativeCachePath); StorageFile storageFile = storageProvider.getStorageFile(relativeCachePath); - + storage.uploadFile(cacheFile, storageFile); logger.log(StorageMgrMessage.PRODUCT_FILE_UPLOADED_FROM_CACHE_TO_STORAGE, storageFile.getFullPath()); - + RestFileInfo restFileInfo = convertToRestFileInfo(storageFile, storageProvider.getCacheFileSize(cacheFile.getRelativePath())); @@ -350,7 +349,7 @@ private RestFileInfo copyFileCacheToStorage(String relativeCachePath) * @return rest file info */ private static RestFileInfo convertToRestFileInfo(StorageFile storageFile, long fileSize) { - + if (logger.isTraceEnabled()) logger.trace(">>> convertToRestFileInfo({}, {})", storageFile, fileSize); @@ -373,7 +372,7 @@ private static RestFileInfo convertToRestFileInfo(StorageFile storageFile, long * @return product folder with the file name */ private String getProductFolderWithFilename(String externalPath, Long productId) { - + if (logger.isTraceEnabled()) logger.trace(">>> getProductFolderWithFilename({}, {})", externalPath, productId); diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/FileUtils.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/FileUtils.java index 03470d1d2..cb23cf77c 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/FileUtils.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/FileUtils.java @@ -115,18 +115,17 @@ public boolean synchroCreateFile(String content, long waitTime, long fileCheckMa } catch (FileLockedAfterMaxCyclesException e) { - logger.debug("... the file is locked after max check cycles: ", path, " ", e.getMessage()); + logger.debug("... the file is locked after max check cycles: " + path + " " + e.getMessage()); fileCreatedStatus = false; } catch (InterruptedException e) { - logger.debug("... the file is locked after max check cycles: ", path, " ", e.getMessage()); + logger.debug("... the file is locked after max check cycles: " + path + " " + e.getMessage()); fileCreatedStatus = false; } finally { fileLocker.unlock(); - logger.debug("... unlocked the file: ", path); } return fileCreatedStatus; diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java index e3b1939c3..5b4874c4c 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/utils/StorageFileLocker.java @@ -82,14 +82,14 @@ public void lockOrWaitUntilUnlockedAndLock() throws FileLockedAfterMaxCyclesExce if (!productLockSet.contains(path)) { productLockSet.add(path); - logger.debug("... the file {} is locked", path); + logger.debug("... the file has been locked: {}", path); break; } } // wait, because file is locked (in the set) in another thread if (logger.isDebugEnabled()) - logger.debug("... waiting for concurrent access to {} to terminate", path); + logger.debug("... waiting for concurrent access to the file: {}", path); Thread.sleep(waitTime); } @@ -108,10 +108,10 @@ public void unlock() { logger.trace(">>> unlock()"); if (productLockSet.remove(path)) { - logger.debug("... the file {} is unlocked", path); + logger.debug("... the file has been unlocked: {}", path); } else { - logger.debug("... the file {} was not locked (not in the concurrent list)", path); + logger.debug("... the file was not locked (not in the concurrent list): {}", path); } } } \ No newline at end of file diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java index 01543055d..3de20a8e7 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java @@ -119,17 +119,26 @@ private void syncDownload(String testID) throws Exception { StorageFile storageFile = storageProvider.getStorageFile(relativePath); storageProvider.getStorage().upload(sourceFile, storageFile); - // show storage files - BaseStorageTestUtils.printStorageFiles("Before http-call", storageProvider.getStorage()); + // show storage and cache files + BaseStorageTestUtils.printStorageFiles("BEFORE http-call", storageProvider.getStorage()); + storageTestUtils.printCache(); + // concurrent threads download concurrently from storage to cache DownloadThread thread1 = new DownloadThread(relativePath); thread1.start(); DownloadThread thread2 = new DownloadThread(relativePath); thread2.start(); - + thread1.join(); thread2.join(); + + // not a concurrent thread uses downloaded file (in thread 1 or 2) from cache + DownloadThread thread3 = new DownloadThread(relativePath); + thread3.start(); + + thread3.join(); + // delete files with empty folders @@ -159,7 +168,7 @@ public DownloadThread(String relativePath) { public void run() { - System.out.println("Thread name:" + Thread.currentThread().getName()); + System.out.println("Thread name: " + Thread.currentThread().getName()); // rest-download file from storage to cache String absoluteStoragePath = storageProvider.getStorage().getAbsolutePath(relativePath); @@ -173,6 +182,7 @@ public void run() { mvcResult = mockMvc.perform(request).andExpect(status().isOk()).andReturn(); // show results of http-download + System.out.println("Thread name: " + Thread.currentThread().getName()); TestUtils.printMvcResult(REQUEST_STRING, mvcResult); storageTestUtils.printCache(); From 5df33b0f4c3bf072526e2013dd89f7e92068c8d0 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 22 Feb 2024 20:11:17 +0100 Subject: [PATCH 60/79] storage-mgr: created StreamCatcher for intercepting PrintStreams --- .../dlr/proseo/storagemgr/StreamCatcher.java | 67 +++++++++++++++++++ ...ctfileControllerImplTest_syncDownload.java | 54 ++++++++++++++- 2 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StreamCatcher.java diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StreamCatcher.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StreamCatcher.java new file mode 100644 index 000000000..0773ebb78 --- /dev/null +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StreamCatcher.java @@ -0,0 +1,67 @@ +package de.dlr.proseo.storagemgr; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; + +/** + * Stream catcher for catching an output stream + * + * @author Denys Chaykovskiy + * + */ + +public class StreamCatcher { + + /** Output header */ + private static final String OUTPUT_HEADER = "catched: "; + + /** Saved original output stream */ + final PrintStream originalSystemOut; + + /** System output content */ + ByteArrayOutputStream systemOutContent ; + + /** + * @param originalSystemOut + */ + public StreamCatcher(PrintStream originalSystemOut) { + + this.originalSystemOut = originalSystemOut; + + systemOutContent = new ByteArrayOutputStream(); + + System.setOut(new PrintStream(systemOutContent)); + } + + /** + * Gets a catched output content + * + * + * @return catched output content + */ + public List getOutput() { + + String capturedOutput = systemOutContent.toString(); + + String[] lines = capturedOutput.split(System.lineSeparator()); + + List outputLines = new ArrayList<>(); + + for (String line : lines) { + outputLines.add(OUTPUT_HEADER + line.trim()); + } + + return outputLines; + } + + /** + * Restores default system output + * + */ + public void restoreDefaultOutput() { + + System.setOut(originalSystemOut); + } +} diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java index 3de20a8e7..1dcc44e8d 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java @@ -5,7 +5,11 @@ import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.List; import org.codehaus.jackson.map.ObjectMapper; import org.junit.Rule; @@ -30,6 +34,13 @@ import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; import de.dlr.proseo.storagemgr.utils.FileUtils; import de.dlr.proseo.storagemgr.utils.PathConverter; +import de.dlr.proseo.storagemgr.StreamCatcher; + + + + +import java.util.ArrayList; + /** * Mock Mvc test for Product Controller @@ -58,6 +69,7 @@ public class ProductfileControllerImplTest_syncDownload { public TestName testName = new TestName(); private static final String REQUEST_STRING = "/proseo/storage-mgr/x/productfiles"; + @Test public void testDownload_posix() throws Exception { @@ -101,7 +113,9 @@ public void testDownload_S3() throws Exception { private void syncDownload(String testID) throws Exception { TestUtils.printMethodName(this, testName); - + + StreamCatcher streamCatcher = new StreamCatcher(System.out); + // create file in source // upload to storage // call http-download @@ -138,8 +152,11 @@ private void syncDownload(String testID) throws Exception { thread3.start(); thread3.join(); - - + + + TestUtils.printList("CATCHED LOGS AND OUTPUT", streamCatcher.getOutput()); + streamCatcher.restoreDefaultOutput(); + // delete files with empty folders String cacheFile = new PathConverter(storageProvider.getCachePath(), relativePath).getPath(); @@ -156,6 +173,19 @@ private void syncDownload(String testID) throws Exception { storageProvider.getStorage().deleteFile(storageFile); // in storage } } + + private List extractOutputLines(String capturedOutput) { + String[] lines = capturedOutput.split(System.lineSeparator()); + + List outputLines = new ArrayList<>(); + + // Add each line to the list + for (String line : lines) { + outputLines.add("YAYA" + line.trim()); + } + + return outputLines; + } private class DownloadThread extends Thread { @@ -210,4 +240,22 @@ public void run() { } } } + + public class InterceptingPrintStream extends PrintStream { + private StringBuilder capturedOutput = new StringBuilder(); + + public InterceptingPrintStream(OutputStream out) { + super(out); + } + + @Override + public void println(String x) { + capturedOutput.append(x).append(System.lineSeparator()); + super.println("YOYO" + x); + } + + public String getCapturedOutput() { + return capturedOutput.toString(); + } + } } From f3f839017540a3515cebf8594fc22fa5fd7d5a22 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 22 Feb 2024 20:20:30 +0100 Subject: [PATCH 61/79] storage-mgr: renamed to StreamIterceptor, removed duplications in test --- ...eamCatcher.java => StreamInterceptor.java} | 24 ++++++------ ...ctfileControllerImplTest_syncDownload.java | 38 ++----------------- 2 files changed, 15 insertions(+), 47 deletions(-) rename storage-mgr/src/test/java/de/dlr/proseo/storagemgr/{StreamCatcher.java => StreamInterceptor.java} (56%) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StreamCatcher.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StreamInterceptor.java similarity index 56% rename from storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StreamCatcher.java rename to storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StreamInterceptor.java index 0773ebb78..9cf0c2f58 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StreamCatcher.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/StreamInterceptor.java @@ -6,46 +6,44 @@ import java.util.List; /** - * Stream catcher for catching an output stream + * Stream interceptor for intercepting an output stream * * @author Denys Chaykovskiy * */ -public class StreamCatcher { +public class StreamInterceptor { /** Output header */ - private static final String OUTPUT_HEADER = "catched: "; + private static final String OUTPUT_HEADER = "Intercepted: "; /** Saved original output stream */ - final PrintStream originalSystemOut; + private PrintStream originalSystemOut; - /** System output content */ - ByteArrayOutputStream systemOutContent ; + /** Intercepted system output content */ + private ByteArrayOutputStream systemOutContent = new ByteArrayOutputStream();; /** * @param originalSystemOut */ - public StreamCatcher(PrintStream originalSystemOut) { + public StreamInterceptor(PrintStream originalSystemOut) { this.originalSystemOut = originalSystemOut; - systemOutContent = new ByteArrayOutputStream(); - System.setOut(new PrintStream(systemOutContent)); } /** - * Gets a catched output content + * Gets an intercepted output content * * - * @return catched output content + * @return intercepted output content */ public List getOutput() { - String capturedOutput = systemOutContent.toString(); + String interceptedOutput = systemOutContent.toString(); - String[] lines = capturedOutput.split(System.lineSeparator()); + String[] lines = interceptedOutput.split(System.lineSeparator()); List outputLines = new ArrayList<>(); diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java index 1dcc44e8d..7897e8188 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java @@ -34,7 +34,7 @@ import de.dlr.proseo.storagemgr.rest.model.RestFileInfo; import de.dlr.proseo.storagemgr.utils.FileUtils; import de.dlr.proseo.storagemgr.utils.PathConverter; -import de.dlr.proseo.storagemgr.StreamCatcher; +import de.dlr.proseo.storagemgr.StreamInterceptor; @@ -114,7 +114,7 @@ private void syncDownload(String testID) throws Exception { TestUtils.printMethodName(this, testName); - StreamCatcher streamCatcher = new StreamCatcher(System.out); + StreamInterceptor streamInterceptor = new StreamInterceptor(System.out); // create file in source // upload to storage @@ -154,8 +154,8 @@ private void syncDownload(String testID) throws Exception { thread3.join(); - TestUtils.printList("CATCHED LOGS AND OUTPUT", streamCatcher.getOutput()); - streamCatcher.restoreDefaultOutput(); + TestUtils.printList("CATCHED LOGS AND OUTPUT", streamInterceptor.getOutput()); + streamInterceptor.restoreDefaultOutput(); // delete files with empty folders @@ -173,19 +173,6 @@ private void syncDownload(String testID) throws Exception { storageProvider.getStorage().deleteFile(storageFile); // in storage } } - - private List extractOutputLines(String capturedOutput) { - String[] lines = capturedOutput.split(System.lineSeparator()); - - List outputLines = new ArrayList<>(); - - // Add each line to the list - for (String line : lines) { - outputLines.add("YAYA" + line.trim()); - } - - return outputLines; - } private class DownloadThread extends Thread { @@ -241,21 +228,4 @@ public void run() { } } - public class InterceptingPrintStream extends PrintStream { - private StringBuilder capturedOutput = new StringBuilder(); - - public InterceptingPrintStream(OutputStream out) { - super(out); - } - - @Override - public void println(String x) { - capturedOutput.append(x).append(System.lineSeparator()); - super.println("YOYO" + x); - } - - public String getCapturedOutput() { - return capturedOutput.toString(); - } - } } From 13a9d3b7a201e6c8d869511099317999eb5a62f0 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 22 Feb 2024 20:25:56 +0100 Subject: [PATCH 62/79] storage-mgr: sequence correction of the output stream restoration --- .../rest/ProductfileControllerImplTest_syncDownload.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java index 7897e8188..57768d2e7 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java @@ -153,9 +153,9 @@ private void syncDownload(String testID) throws Exception { thread3.join(); - - TestUtils.printList("CATCHED LOGS AND OUTPUT", streamInterceptor.getOutput()); + List interceptedOutput = streamInterceptor.getOutput(); streamInterceptor.restoreDefaultOutput(); + TestUtils.printList("CATCHED LOGS AND OUTPUT", interceptedOutput); // delete files with empty folders From d3d5c85cc84320a62e0f6b1ab73e40814d3e6434 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 29 Feb 2024 17:55:59 +0100 Subject: [PATCH 63/79] storage-mgr: created LogRecord to store parsed log fields --- .../de/dlr/proseo/storagemgr/LogRecord.java | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 storage-mgr/src/test/java/de/dlr/proseo/storagemgr/LogRecord.java diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/LogRecord.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/LogRecord.java new file mode 100644 index 000000000..486d9c1a4 --- /dev/null +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/LogRecord.java @@ -0,0 +1,95 @@ +package de.dlr.proseo.storagemgr; + +import java.time.Instant; + +/** + * LogRecord Class + * + * Examples: - Intercepted: 2024-02-22 20:23:08.000 TRACE 14224 --- [ Thread-6] + * d.dlr.proseo.storagemgr.utils.FileUtils : >>> + * createFile(2024-02-22T19:23:08.000469Z) - Intercepted: - + * accessed-productFileDownloadPosix.txt + * + * @author Denys Chaykovskiy + * + */ + +public class LogRecord { + + /** instant */ + private Instant timestamp; // 2024-02-22 20:23:08.000 + + /** threadName */ + private String threadName; // Thread-6 + + /** className */ + private String className; // d.dlr.proseo.storagemgr.utils.FileUtils + + /** content */ + private String content; // >>> createFile(2024-02-22T19:23:08.000469Z) + + + /** + * Constructor + * + * @param timestamp + * @param threadName + * @param className + * @param content + */ + public LogRecord(Instant timestamp, String threadName, String className, String content) { + + this.timestamp = timestamp; + this.threadName = threadName; + this.className = className; + this.content = content; + } + + /** + * Gets a timestamp + * + * @return + */ + public Instant getTimestamp() { + return timestamp; + } + + /** + * Gets a thread name + * + * @return + */ + public String getThreadName() { + return threadName; + } + + /** + * Gets a class name + * + * @return + */ + public String getClassName() { + return className; + } + + /** + * Gets a content + * + * @return + */ + public String getContent() { + return content; + } + + /** + * Prints a log record + * + */ + public void print() { + + System.out.println("Parsed Instant: " + getTimestamp()); + System.out.println("Parsed Thread : " + getThreadName()); + System.out.println("Parsed Class : " + getClassName()); + System.out.println("Parsed Content: " + getContent()); + } +} From 3576a554488f0bf58043698de366c2e778d235cd Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 29 Feb 2024 17:59:02 +0100 Subject: [PATCH 64/79] storage-mgr: created a parser for prosEO logging strings --- .../de/dlr/proseo/storagemgr/LogParser.java | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 storage-mgr/src/test/java/de/dlr/proseo/storagemgr/LogParser.java diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/LogParser.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/LogParser.java new file mode 100644 index 000000000..263144e0e --- /dev/null +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/LogParser.java @@ -0,0 +1,146 @@ +package de.dlr.proseo.storagemgr; + +import java.time.DateTimeException; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; + +/** + * LogParser Class + * + * - Intercepted: 2024-02-22 20:23:08.000 TRACE 14224 --- [ Thread-6] + * d.dlr.proseo.storagemgr.utils.FileUtils : >>> + * createFile(2024-02-22T19:23:08.000469Z) + * + * @author Denys Chaykovskiy + * + */ + +public class LogParser { + + private String logString; + + public static void main(String[] args) { + + String logString = "Intercepted: 2024-02-22 20:23:08.000 TRACE 14224 --- [ Thread-6] d.dlr.proseo.storagemgr.utils.FileUtils : >>> createFile(2024-02-22T19:23:08.000469Z)"; + + LogParser logParser = new LogParser(logString); + + logParser.parse().print(); + } + + /** + * Constructor + * + * @param logString + */ + public LogParser(String logString) { + + this.logString = logString; + } + + /** + * Checks if the string is a log string + * + * @return + */ + public boolean isLogString() { + + String[] sublines = logString.split("\\s+"); + + if (sublines.length < 11) + return false; + + // TODO: add additional conditions like thread name has "thread" + + return true; + } + + /** + * Parses a log record + * + * @return + */ + public LogRecord parse() { + + String[] sublines = logString.split("\\s+"); + +// for (String subline : sublines) { +// System.out.println(subline); +// } + + Instant timestamp = parseTimeStamp(sublines[1] + " " + sublines[2]); + + String threadName = parseThreadName(sublines[7]); + + String className = parseClassName(sublines[8]); + + String content = sublines[10]; + + for (int i = 11; i < sublines.length; i++) { + + content += " " + sublines[i]; + } + + content = parseContent(content); + + LogRecord logRecord = new LogRecord(timestamp, threadName, className, content); + + return logRecord; + } + + /** + * Parses a timestamp + * + * @param timestamp + * @return + */ + private static Instant parseTimeStamp(String timestamp) { + + // String timestampString = "2024-02-22 20:23:08.000"; + + try { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS") + .withZone(ZoneId.systemDefault()); + return Instant.from(formatter.parse(timestamp)); + + } catch (Exception e) { + System.err.println("Error parsing Instant: " + e.getMessage()); + + throw new DateTimeException(e.getMessage(), e); + } + } + + /** + * Parses a thread name + * + * @param threadName + * @return + */ + private static String parseThreadName(String threadName) { + + return threadName.replace("]", ""); + } + + /** + * Parses a class name + * + * @param className + * @return + */ + private static String parseClassName(String className) { + + return className; + } + + /** + * Parses a content + * + * @param content + * @return + */ + private static String parseContent(String content) { + + return content.replace(">>> ", ""); + } +} From 39a3d0e0f92ba3ab3f912004597ead5b02b67b29 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 29 Feb 2024 18:03:42 +0100 Subject: [PATCH 65/79] storage-mgr: created the multi-thread log analyser for prosEO logs --- .../storagemgr/MultiThreadLogAnalyser.java | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 storage-mgr/src/test/java/de/dlr/proseo/storagemgr/MultiThreadLogAnalyser.java diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/MultiThreadLogAnalyser.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/MultiThreadLogAnalyser.java new file mode 100644 index 000000000..26b8ac000 --- /dev/null +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/MultiThreadLogAnalyser.java @@ -0,0 +1,130 @@ +package de.dlr.proseo.storagemgr; + +import java.util.List; +import java.util.ArrayList; +import java.io.IOException; + +/** + * MultiThreadLogAnalyser Class + * + * + * @author Denys Chaykovskiy + * + */ + +public class MultiThreadLogAnalyser { + + /** logs */ + private List logRecords = new ArrayList<>(); + + // thread messages + + /** blocking thread message */ + private String blockingThreadMsg; + + /** waiting thread message */ + private String waitingThreadMsg; + + /** non-blocking thread message */ + private String nonBlockingThreadMsg; + + // thread names + + /** blocking thread name */ + private String blockingThreadName; + + /** waiting thread name */ + private String waitingThreadName; + + /** non-blocking thread name */ + private String nonBlockingThreadName; + + + + /** + * Constructor + * + * @param logs + * @param blockingThreadMsg + * @param waitingThreadMsg + * @param nonBlockingThreadMsg + */ + public MultiThreadLogAnalyser(List logs, String blockingThreadMsg, String waitingThreadMsg, + String nonBlockingThreadMsg) { + + this.blockingThreadMsg = blockingThreadMsg; + this.waitingThreadMsg = waitingThreadMsg; + this.nonBlockingThreadMsg = nonBlockingThreadMsg; + + for (String log : logs) { + + LogParser logParser = new LogParser(log); + + logRecords.add(logParser.parse()); + } + } + + /** + * Checks if there are logs with given messages and they are from different + * threads + * + * @return + */ + public boolean check() { + + try { + + blockingThreadName = getThreadName(blockingThreadMsg); + waitingThreadName = getThreadName(waitingThreadMsg); + nonBlockingThreadName = getThreadName(nonBlockingThreadMsg); + + } catch (Exception e) { + + System.out.println(e.getMessage()); + return false; + } + + boolean expectedDifferentThreads = areDifferent(blockingThreadName, waitingThreadName, nonBlockingThreadName); + + if (!expectedDifferentThreads) { + System.out.println("ERROR: Expected threads are not different: " + blockingThreadName + " " + waitingThreadName + + " " + nonBlockingThreadName); + } + + return expectedDifferentThreads; + + } + + /** + * Checks if 3 Strings are different + * + * @param s1 + * @param s2 + * @param s3 + * @return + */ + private boolean areDifferent(String s1, String s2, String s3) { + + return !s1.equals(s2) && !s1.equals(s3) && !s2.equals(s3); + } + + /** + * Gets a thread name from the log, which contains a given message + * + * @param message + * @return + * @throws IOException + */ + private String getThreadName(String message) throws IOException { + + for (LogRecord logRecord : logRecords) { + + if (logRecord.getContent().contains(message)) { + + return logRecord.getThreadName(); + } + } + + throw new IOException("ERROR: Cannot find a thread with this messsage: " + message); + } +} From ac2512a1b961efff0c8cb82cd3b12bad8fe1ea5b Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 29 Feb 2024 18:06:25 +0100 Subject: [PATCH 66/79] storage-mgr: added common thread roles to the multi-thread analyser --- .../proseo/storagemgr/MultiThreadLogAnalyser.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/MultiThreadLogAnalyser.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/MultiThreadLogAnalyser.java index 26b8ac000..c6faf0a52 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/MultiThreadLogAnalyser.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/MultiThreadLogAnalyser.java @@ -7,6 +7,15 @@ /** * MultiThreadLogAnalyser Class * + * There are 2 for common thread roles during synchronization for a common + * resource 1) blocking thread - blocks a free common resource, later releases + * it 2) waiting thread - waits until a common resource will be free and blocks + * it, later releases it + * + * There are 3 common thread roles during synchronization with a job to be done: + * 1) blocking thread - blocks a common resource, does a job, frees the resource + * 2) waiting thread - checks, that the resource is blocked, 3) checks, that the + * job done and just uses results. No blocking or waiting * * @author Denys Chaykovskiy * @@ -39,8 +48,6 @@ public class MultiThreadLogAnalyser { /** non-blocking thread name */ private String nonBlockingThreadName; - - /** * Constructor * @@ -87,8 +94,8 @@ public boolean check() { boolean expectedDifferentThreads = areDifferent(blockingThreadName, waitingThreadName, nonBlockingThreadName); if (!expectedDifferentThreads) { - System.out.println("ERROR: Expected threads are not different: " + blockingThreadName + " " + waitingThreadName + - " " + nonBlockingThreadName); + System.out.println("ERROR: Expected threads are not different: " + blockingThreadName + " " + + waitingThreadName + " " + nonBlockingThreadName); } return expectedDifferentThreads; From 791ee6bd6736ab38933495f4f0e4e686cc7c04ff Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 29 Feb 2024 18:09:55 +0100 Subject: [PATCH 67/79] storage-mgr: added a test scenario for the multi-thread analyser --- .../storagemgr/MultiThreadLogAnalyser.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/MultiThreadLogAnalyser.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/MultiThreadLogAnalyser.java index c6faf0a52..e3a07a569 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/MultiThreadLogAnalyser.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/MultiThreadLogAnalyser.java @@ -47,7 +47,38 @@ public class MultiThreadLogAnalyser { /** non-blocking thread name */ private String nonBlockingThreadName; + + + public static void main(String[] args) { + final String BLOCKER = "blocker"; + final String WAITER = "waiter"; + final String CONSUMER = "consumer"; + + String logString1 = "Intercepted: 2024-02-22 20:23:08.000 TRACE 14224 --- [ Thread-6] d.dlr.proseo.storagemgr.utils.FileUtils : >>> bl" + + BLOCKER + "(2024-02-22T19:23:08.000469Z)"; + String logString2 = "Intercepted: 2024-02-22 20:23:08.000 TRACE 14224 --- [ Thread-7] d.dlr.proseo.storagemgr.utils.FileUtils : >>> wa" + + WAITER + "iting(2024-02-22T19:23:08.000469Z)"; + String logString3 = "Intercepted: 2024-02-22 20:23:08.000 TRACE 14224 --- [ Thread-8] d.dlr.proseo.storagemgr.utils.FileUtils : >>> no" + + CONSUMER + "blocking(2024-02-22T19:23:08.000469Z)"; + + List logs = new ArrayList<>(); + + logs.add(logString1); + logs.add(logString2); + logs.add(logString3); + + MultiThreadLogAnalyser logAnalyser = new MultiThreadLogAnalyser(logs, BLOCKER, WAITER, CONSUMER); + + if (logAnalyser.check()) { + + System.out.println("Test is green"); + } else { + System.out.println("Test is RED"); + } + + } + /** * Constructor * From c8058d55fded6184a3b053151d0c472f586d01b6 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Thu, 29 Feb 2024 18:18:16 +0100 Subject: [PATCH 68/79] storage-mgr: moved converter, provider, utility tests to utils --- .../storagemgr/{version2 => utils}/PathConverterTest.java | 2 +- .../storagemgr/{version2 => utils}/StorageProviderTest.java | 2 +- .../storagemgr/{version2 => utils}/StorageUtilityTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename storage-mgr/src/test/java/de/dlr/proseo/storagemgr/{version2 => utils}/PathConverterTest.java (97%) rename storage-mgr/src/test/java/de/dlr/proseo/storagemgr/{version2 => utils}/StorageProviderTest.java (99%) rename storage-mgr/src/test/java/de/dlr/proseo/storagemgr/{version2 => utils}/StorageUtilityTest.java (98%) diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/PathConverterTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/utils/PathConverterTest.java similarity index 97% rename from storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/PathConverterTest.java rename to storage-mgr/src/test/java/de/dlr/proseo/storagemgr/utils/PathConverterTest.java index ecf537d4c..69dafa6a9 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/PathConverterTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/utils/PathConverterTest.java @@ -1,4 +1,4 @@ -package de.dlr.proseo.storagemgr.version2; +package de.dlr.proseo.storagemgr.utils; import static org.junit.Assert.*; diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/utils/StorageProviderTest.java similarity index 99% rename from storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java rename to storage-mgr/src/test/java/de/dlr/proseo/storagemgr/utils/StorageProviderTest.java index d1aad5592..8d88f93d4 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageProviderTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/utils/StorageProviderTest.java @@ -1,4 +1,4 @@ -package de.dlr.proseo.storagemgr.version2; +package de.dlr.proseo.storagemgr.utils; import static org.junit.Assert.*; diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/utils/StorageUtilityTest.java similarity index 98% rename from storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java rename to storage-mgr/src/test/java/de/dlr/proseo/storagemgr/utils/StorageUtilityTest.java index e984a337e..82d149490 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/version2/StorageUtilityTest.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/utils/StorageUtilityTest.java @@ -1,4 +1,4 @@ -package de.dlr.proseo.storagemgr.version2; +package de.dlr.proseo.storagemgr.utils; import java.io.IOException; From 34175782073950f56e72315c22228d0813851c88 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Fri, 1 Mar 2024 11:52:45 +0100 Subject: [PATCH 69/79] storage-mgr: integrated multi-thread analysis into prdfile download test --- .../rest/ProductfileControllerImpl.java | 5 ++- .../de/dlr/proseo/storagemgr/LogParser.java | 26 +++++++++++++--- .../storagemgr/MultiThreadLogAnalyser.java | 7 +++-- ...ctfileControllerImplTest_syncDownload.java | 31 +++++++++++++------ 4 files changed, 51 insertions(+), 18 deletions(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index b4daa2c89..eeed5e642 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -213,6 +213,7 @@ private RestFileInfo copyFileStorageToCache(String storageFilePath, StorageFileL if (!cache.containsKey(cacheFile.getFullPath())) { // active thread - copies the file to the cache storage and puts it to the cache + logger.debug("... active-thread: copies the file to the cache storage and puts it to the cache: {}", cacheFile.getFullPath()); cache.setCacheFileStatus(cacheFile.getFullPath(), CacheFileStatus.INCOMPLETE); @@ -234,7 +235,7 @@ private RestFileInfo copyFileStorageToCache(String storageFilePath, StorageFileL } else { - logger.debug("... no download and no lock - the file is in cache: ", cacheFile.getFullPath()); + logger.debug("... no download and no lock - the file is in cache: {}", cacheFile.getFullPath()); } RestFileInfo restFileInfo = convertToRestFileInfo(cacheFile, @@ -281,6 +282,8 @@ private RestFileInfo copyFileExternalToCache(String externalPath, Long productId if (!cache.containsKey(cacheFile.getFullPath())) { // active thread - copies the file to the cache storage and puts it to the cache + logger.debug("... active-thread: copies the file to the cache storage and puts it to the cache: {}", + cacheFile.getFullPath()); cache.setCacheFileStatus(cacheFile.getFullPath(), CacheFileStatus.INCOMPLETE); diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/LogParser.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/LogParser.java index 263144e0e..5623a18ae 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/LogParser.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/LogParser.java @@ -50,12 +50,28 @@ public boolean isLogString() { if (sublines.length < 11) return false; - - // TODO: add additional conditions like thread name has "thread" - + return true; } + + /** + * Checks if the string is a multi-thread log string + * + * @return + */ + public boolean isMultiThreadLogString() { + + if (!logString.contains("Thread-")) { + return false; + } + + if (!isLogString()) { + return false; + } + return true; + } + /** * Parses a log record * @@ -140,7 +156,7 @@ private static String parseClassName(String className) { * @return */ private static String parseContent(String content) { - - return content.replace(">>> ", ""); + + return content.replace(">>> ", "").replace("... ", ""); } } diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/MultiThreadLogAnalyser.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/MultiThreadLogAnalyser.java index e3a07a569..ed17ef0e8 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/MultiThreadLogAnalyser.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/MultiThreadLogAnalyser.java @@ -97,8 +97,11 @@ public MultiThreadLogAnalyser(List logs, String blockingThreadMsg, Strin for (String log : logs) { LogParser logParser = new LogParser(log); - - logRecords.add(logParser.parse()); + + if (logParser.isMultiThreadLogString()) { + + logRecords.add(logParser.parse()); + } } } diff --git a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java index 57768d2e7..d9409565a 100644 --- a/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java +++ b/storage-mgr/src/test/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImplTest_syncDownload.java @@ -5,11 +5,8 @@ import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.OutputStream; -import java.io.PrintStream; import java.util.List; +import java.io.File; import org.codehaus.jackson.map.ObjectMapper; import org.junit.Rule; @@ -35,11 +32,7 @@ import de.dlr.proseo.storagemgr.utils.FileUtils; import de.dlr.proseo.storagemgr.utils.PathConverter; import de.dlr.proseo.storagemgr.StreamInterceptor; - - - - -import java.util.ArrayList; +import de.dlr.proseo.storagemgr.MultiThreadLogAnalyser; /** @@ -155,7 +148,25 @@ private void syncDownload(String testID) throws Exception { List interceptedOutput = streamInterceptor.getOutput(); streamInterceptor.restoreDefaultOutput(); - TestUtils.printList("CATCHED LOGS AND OUTPUT", interceptedOutput); + TestUtils.printList("CATCHED LOGS AND OUTPUT", interceptedOutput); + + // Multi-thread analysis + + String cachePath = new PathConverter(storageProvider.getCachePath(), relativePath) + .getPath(); + + String blocker = "active-thread: copies the file to the cache storage and puts it to the cache: " + cachePath; + String waiter = "waiting-thread: waited until the file was downloaded to cache from external storage and use it from cache: " + cachePath; + String consumer = "no download and no lock - the file is in cache: " + cachePath; + + MultiThreadLogAnalyser logAnalyser = new MultiThreadLogAnalyser(interceptedOutput, blocker, waiter, consumer); + + if (logAnalyser.check()) { + System.out.println("Multi-thread analysis: PASSED"); + } + else { + System.out.println("Multi-thread analysis: FAILED"); + } // delete files with empty folders From 0a56b38db4d3c40230e6d93e22654a980013828a Mon Sep 17 00:00:00 2001 From: DenChaykovskiy Date: Mon, 4 Mar 2024 13:19:31 +0100 Subject: [PATCH 70/79] storage-mgr: changed log class to ProductfileControllerImpl --- .../dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java index eeed5e642..11ba6af84 100644 --- a/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java +++ b/storage-mgr/src/main/java/de/dlr/proseo/storagemgr/rest/ProductfileControllerImpl.java @@ -42,7 +42,7 @@ public class ProductfileControllerImpl implements ProductfileController { private static final String READ_ONLY_FOR_ALL_USERS = "r--r--r--"; /** A logger for this class */ - private static ProseoLogger logger = new ProseoLogger(ProductControllerImpl.class); + private static ProseoLogger logger = new ProseoLogger(ProductfileControllerImpl.class); private static ProseoHttp http = new ProseoHttp(logger, HttpPrefix.STORAGE_MGR); @Autowired From c081d287a391ad4e2b60f89042dedea282d0e1f9 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy <38767269+DenChaykovskiy@users.noreply.github.com> Date: Thu, 14 Mar 2024 15:35:17 +0100 Subject: [PATCH 71/79] Update install_all_notests.yml CI/CD: added header to install_all_notests.yml --- .github/workflows/install_all_notests.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/install_all_notests.yml b/.github/workflows/install_all_notests.yml index 6d2a0bc4d..b3eb745cb 100644 --- a/.github/workflows/install_all_notests.yml +++ b/.github/workflows/install_all_notests.yml @@ -1,10 +1,11 @@ -# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. +# install_all_notests.yml +# +# (C) 2024 Dr. Bassler & Co. Managementberatung GmbH +# +# Installs the whole prosEO project without test execution +# +# @author Denys Chaykovskiy +# name: Install all no tests From d0bca59e42d4fce340b24904532ab2fa64a3b806 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy <38767269+DenChaykovskiy@users.noreply.github.com> Date: Thu, 14 Mar 2024 15:39:33 +0100 Subject: [PATCH 72/79] CI/CD: added header to install_all.yml --- .github/workflows/install_all.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/install_all.yml b/.github/workflows/install_all.yml index 91f29e443..86fb52272 100644 --- a/.github/workflows/install_all.yml +++ b/.github/workflows/install_all.yml @@ -1,10 +1,11 @@ -# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. +# install_all.yml +# +# (C) 2024 Dr. Bassler & Co. Managementberatung GmbH +# +# Installs the whole prosEO project and runs all tests +# +# @author Denys Chaykovskiy +# name: Install all From 169ec6ff523595b1f18f4c6f098f755dcfc86b61 Mon Sep 17 00:00:00 2001 From: DenChaykovskiy <38767269+DenChaykovskiy@users.noreply.github.com> Date: Thu, 14 Mar 2024 15:40:07 +0100 Subject: [PATCH 73/79] Update install_all_notests.yml --- .github/workflows/install_all_notests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/install_all_notests.yml b/.github/workflows/install_all_notests.yml index b3eb745cb..e7c556a61 100644 --- a/.github/workflows/install_all_notests.yml +++ b/.github/workflows/install_all_notests.yml @@ -2,7 +2,7 @@ # # (C) 2024 Dr. Bassler & Co. Managementberatung GmbH # -# Installs the whole prosEO project without test execution +# Installs the whole prosEO project without test execution # # @author Denys Chaykovskiy # From a4adf30640215ccd69e8db0c3f6e7c0522dbac35 Mon Sep 17 00:00:00 2001 From: Thomas Bassler Date: Thu, 14 Mar 2024 17:39:42 +0100 Subject: [PATCH 74/79] Superfluous property "module.version" removed from all pom.xml's --- api/aip-client/pom.xml | 5 ++--- api/archive-mgr/pom.xml | 5 ++--- api/auxip-monitor/pom.xml | 5 ++--- api/base-monitor/pom.xml | 1 - api/cadip-monitor/pom.xml | 5 ++--- api/edip-monitor/pom.xml | 5 ++--- api/odip/pom.xml | 5 ++--- api/prip/pom.xml | 5 ++--- api/xbip-monitor/pom.xml | 5 ++--- base-wrapper/pom.xml | 1 - facility-mgr/pom.xml | 5 ++--- geotools/pom.xml | 5 ++--- ingestor/pom.xml | 5 ++--- interfaces/pom.xml | 1 - monitor/pom.xml | 5 ++--- notification/pom.xml | 5 ++--- order-mgr/pom.xml | 5 ++--- planner/pom.xml | 5 ++--- processor-mgr/pom.xml | 5 ++--- productclass-mgr/pom.xml | 5 ++--- samples/sample-processor/pom.xml | 8 ++------ samples/sample-wrapper/pom.xml | 5 ++--- storage-mgr/pom.xml | 5 ++--- ui/backend/pom.xml | 4 ---- ui/cli/pom.xml | 1 - ui/gui/pom.xml | 8 ++------ user-mgr/pom.xml | 5 ++--- 27 files changed, 44 insertions(+), 80 deletions(-) diff --git a/api/aip-client/pom.xml b/api/aip-client/pom.xml index 25060f9de..29c8fc076 100644 --- a/api/aip-client/pom.xml +++ b/api/aip-client/pom.xml @@ -18,7 +18,6 @@ - 1.0.0 4.7.1 4.1.48.Final ${basedir}/../.. @@ -188,14 +187,14 @@ - + - + diff --git a/api/archive-mgr/pom.xml b/api/archive-mgr/pom.xml index 7e88f3ae1..0e5f5ef74 100644 --- a/api/archive-mgr/pom.xml +++ b/api/archive-mgr/pom.xml @@ -18,7 +18,6 @@ - 1.0.0 ${basedir}/../.. src/main/resources/raml/archive-mgr-api.raml de.dlr.proseo.archivemgr.rest @@ -161,14 +160,14 @@ - + - + diff --git a/api/auxip-monitor/pom.xml b/api/auxip-monitor/pom.xml index a359f177f..e2a655975 100644 --- a/api/auxip-monitor/pom.xml +++ b/api/auxip-monitor/pom.xml @@ -17,7 +17,6 @@ - 1.0.0 4.7.1 ${basedir}/../.. src/main/resources/raml/auxip-monitor-api.raml @@ -197,14 +196,14 @@ - + - + diff --git a/api/base-monitor/pom.xml b/api/base-monitor/pom.xml index 85590ae36..9ecba04fd 100644 --- a/api/base-monitor/pom.xml +++ b/api/base-monitor/pom.xml @@ -19,7 +19,6 @@ - 1.0.0 ${basedir}/../.. diff --git a/api/cadip-monitor/pom.xml b/api/cadip-monitor/pom.xml index 11d5fffd1..dfd4601c2 100644 --- a/api/cadip-monitor/pom.xml +++ b/api/cadip-monitor/pom.xml @@ -17,7 +17,6 @@ - 1.0.0 4.7.1 ${basedir}/../.. src/main/resources/raml/cadip-monitor-api.raml @@ -203,14 +202,14 @@ - + - + diff --git a/api/edip-monitor/pom.xml b/api/edip-monitor/pom.xml index 10bcda2af..bb96b2862 100644 --- a/api/edip-monitor/pom.xml +++ b/api/edip-monitor/pom.xml @@ -17,7 +17,6 @@ - 1.0.0 ${basedir}/../.. src/main/resources/raml/edip-monitor-api.raml de.dlr.proseo.api.edipmon.rest @@ -178,14 +177,14 @@ - + - + diff --git a/api/odip/pom.xml b/api/odip/pom.xml index 6fe021f90..b20947ece 100644 --- a/api/odip/pom.xml +++ b/api/odip/pom.xml @@ -22,7 +22,6 @@ 4.7.1 - 1.0.0 ${basedir}/../.. src/main/resources/raml/odip-query.raml src/main/resources/raml/odip-oauth2.raml @@ -265,7 +264,7 @@ + value="${docker.registry}/${project.build.finalName}:${project.version}" /> @@ -273,7 +272,7 @@ + value="${docker.registry}/${project.build.finalName}:${project.version}" /> diff --git a/api/prip/pom.xml b/api/prip/pom.xml index 029aa0e91..fa4dc1c3a 100644 --- a/api/prip/pom.xml +++ b/api/prip/pom.xml @@ -19,7 +19,6 @@ 4.7.1 - 1.0.0 ${basedir}/../.. src/main/resources/raml/prip-query.raml src/main/resources/raml/prip-download.raml @@ -254,14 +253,14 @@ - + - + diff --git a/api/xbip-monitor/pom.xml b/api/xbip-monitor/pom.xml index 35af92497..36a1a9dd1 100644 --- a/api/xbip-monitor/pom.xml +++ b/api/xbip-monitor/pom.xml @@ -17,7 +17,6 @@ - 1.0.0 ${basedir}/../.. src/main/resources/raml/xbip-monitor-api.raml de.dlr.proseo.api.xbipmon.rest @@ -172,14 +171,14 @@ - + - + diff --git a/base-wrapper/pom.xml b/base-wrapper/pom.xml index 865b7a837..aa3ce3668 100644 --- a/base-wrapper/pom.xml +++ b/base-wrapper/pom.xml @@ -19,7 +19,6 @@ - 1.0.0 3.14.0.Final 1.14 diff --git a/facility-mgr/pom.xml b/facility-mgr/pom.xml index d0bb00b77..55c55d87c 100644 --- a/facility-mgr/pom.xml +++ b/facility-mgr/pom.xml @@ -18,7 +18,6 @@ - 1.0.0 ${basedir}/../.. src/main/resources/raml/facility-mgr-api.raml de.dlr.proseo.facmgr.rest @@ -161,14 +160,14 @@ - + - + diff --git a/geotools/pom.xml b/geotools/pom.xml index 0f918de1d..a1c1d5952 100644 --- a/geotools/pom.xml +++ b/geotools/pom.xml @@ -11,7 +11,6 @@ prosEO Geotools - 1.0.0 ${basedir}/../.. src/main/resources/raml/geotools-api.raml de.dlr.proseo.geotools.rest @@ -159,14 +158,14 @@ - + - + diff --git a/ingestor/pom.xml b/ingestor/pom.xml index b77378819..392b4ab33 100644 --- a/ingestor/pom.xml +++ b/ingestor/pom.xml @@ -22,7 +22,6 @@ - 1.0.0 ${basedir}/../.. src/main/resources/raml/ingestor-api.raml de.dlr.proseo.ingestor.rest @@ -183,14 +182,14 @@ - + - + diff --git a/interfaces/pom.xml b/interfaces/pom.xml index 3770787fc..41c613bb6 100644 --- a/interfaces/pom.xml +++ b/interfaces/pom.xml @@ -19,7 +19,6 @@ - 1.0.0 ${basedir}/../.. src/test/resources/raml/interface-test.raml de.dlr.proseo.interfaces diff --git a/monitor/pom.xml b/monitor/pom.xml index eae2732a1..16bc81d07 100644 --- a/monitor/pom.xml +++ b/monitor/pom.xml @@ -19,7 +19,6 @@ - 1.0.0 12.0.1 ${basedir}/../.. src/main/resources/raml/monitor-api.raml @@ -217,14 +216,14 @@ - + - + diff --git a/notification/pom.xml b/notification/pom.xml index 587303ea5..bb24f31a5 100644 --- a/notification/pom.xml +++ b/notification/pom.xml @@ -13,7 +13,6 @@ prosEO Notification Service: The component to send notifications - 1.0.0 ${basedir}/../.. src/main/resources/raml/notification-api.raml de.dlr.proseo.notification.rest @@ -163,14 +162,14 @@ - + - + diff --git a/order-mgr/pom.xml b/order-mgr/pom.xml index 31d3bffa0..3ef208d55 100644 --- a/order-mgr/pom.xml +++ b/order-mgr/pom.xml @@ -19,7 +19,6 @@ - 1.0.0 ${basedir}/../.. src/main/resources/raml/order-mgr-api.raml de.dlr.proseo.model.rest @@ -159,14 +158,14 @@ - + - + diff --git a/planner/pom.xml b/planner/pom.xml index 9e7253c86..ae82b9584 100644 --- a/planner/pom.xml +++ b/planner/pom.xml @@ -23,7 +23,6 @@ - 1.0.0 12.0.1 ${basedir}/../.. src/main/resources/raml/planner-api.raml @@ -193,14 +192,14 @@ - + - + diff --git a/processor-mgr/pom.xml b/processor-mgr/pom.xml index c7e9efe5b..3398051a9 100644 --- a/processor-mgr/pom.xml +++ b/processor-mgr/pom.xml @@ -18,7 +18,6 @@ - 1.0.0 ${basedir}/../.. src/main/resources/raml/processor-mgr-api.raml de.dlr.proseo.procmgr.rest @@ -158,14 +157,14 @@ - + - + diff --git a/productclass-mgr/pom.xml b/productclass-mgr/pom.xml index f6ddb7764..ee0b6cae2 100644 --- a/productclass-mgr/pom.xml +++ b/productclass-mgr/pom.xml @@ -18,7 +18,6 @@ - 1.0.0 ${basedir}/../.. src/main/resources/raml/productclass-mgr-api.raml de.dlr.proseo.prodclmgr.rest @@ -153,14 +152,14 @@ - + - + diff --git a/samples/sample-processor/pom.xml b/samples/sample-processor/pom.xml index c74f85603..fe9dcc183 100644 --- a/samples/sample-processor/pom.xml +++ b/samples/sample-processor/pom.xml @@ -20,10 +20,6 @@ http://www.dlr.de - - 1.0.0 - - @@ -76,14 +72,14 @@ - + - + diff --git a/samples/sample-wrapper/pom.xml b/samples/sample-wrapper/pom.xml index d62cf8b86..258ccdc41 100644 --- a/samples/sample-wrapper/pom.xml +++ b/samples/sample-wrapper/pom.xml @@ -20,7 +20,6 @@ - 1.0.0 3.2.6 @@ -77,14 +76,14 @@ - + - + diff --git a/storage-mgr/pom.xml b/storage-mgr/pom.xml index 294b8130e..922d7f451 100644 --- a/storage-mgr/pom.xml +++ b/storage-mgr/pom.xml @@ -20,7 +20,6 @@ - 1.0.0 ${basedir}/../.. src/main/resources/raml/storage-mgr-api.raml de.dlr.proseo.storagemgr.rest @@ -202,14 +201,14 @@ - + - + diff --git a/ui/backend/pom.xml b/ui/backend/pom.xml index 1e11fe213..e2ff000d7 100644 --- a/ui/backend/pom.xml +++ b/ui/backend/pom.xml @@ -17,10 +17,6 @@ http://www.dlr.de - - 1.0.0 - - org.springframework.boot diff --git a/ui/cli/pom.xml b/ui/cli/pom.xml index a58b5c42a..0c7922d3a 100644 --- a/ui/cli/pom.xml +++ b/ui/cli/pom.xml @@ -18,7 +18,6 @@ - 1.0.0 3.13.2 diff --git a/ui/gui/pom.xml b/ui/gui/pom.xml index f22034a72..1929a768a 100644 --- a/ui/gui/pom.xml +++ b/ui/gui/pom.xml @@ -17,10 +17,6 @@ http://www.dlr.de - - 1.0.0 - - org.springframework.boot @@ -131,14 +127,14 @@ - + - + diff --git a/user-mgr/pom.xml b/user-mgr/pom.xml index 36be475ab..7b14a96d3 100644 --- a/user-mgr/pom.xml +++ b/user-mgr/pom.xml @@ -18,7 +18,6 @@ - 1.0.0 ${basedir}/../.. src/main/resources/raml/user-mgr-api.raml de.dlr.proseo.usermgr.rest @@ -161,14 +160,14 @@ - + - + From e730acd066660c9c4d13447a61e1a1271c6a9754 Mon Sep 17 00:00:00 2001 From: Thomas Bassler Date: Thu, 14 Mar 2024 17:46:02 +0100 Subject: [PATCH 75/79] prosEO version updated to 1.0.1-SNAPSHOT --- api/aip-client/pom.xml | 2 +- api/archive-mgr/pom.xml | 2 +- api/auxip-monitor/pom.xml | 2 +- api/base-monitor/pom.xml | 2 +- api/cadip-monitor/pom.xml | 2 +- api/edip-monitor/pom.xml | 2 +- api/odip/pom.xml | 2 +- api/pom.xml | 2 +- api/prip/pom.xml | 2 +- api/xbip-monitor/pom.xml | 2 +- base-wrapper/pom.xml | 2 +- common/pom.xml | 2 +- .../proseo-api-aip/Dockerfile | 2 +- .../proseo-api-odip/Dockerfile | 2 +- .../proseo-api-prip/Dockerfile | 2 +- .../proseo-archive-mgr/Dockerfile | 2 +- .../proseo-facility-mgr/Dockerfile | 2 +- .../proseo-ingestor/Dockerfile | 2 +- .../proseo-notification/Dockerfile | 2 +- .../proseo-order-mgr/Dockerfile | 2 +- .../proseo-planner/Dockerfile | 2 +- .../proseo-processor-mgr/Dockerfile | 2 +- .../proseo-productclass-mgr/Dockerfile | 2 +- .../proseo-sample-wrapper/Dockerfile | 2 +- .../proseo-storage-mgr/Dockerfile | 2 +- .../proseo-ui-gui/Dockerfile | 2 +- .../proseo-user-mgr/Dockerfile | 2 +- .../files/docker-compose.yml.template | 2 +- .../proseo-api-aip/Dockerfile | 2 +- .../proseo-api-odip/Dockerfile | 2 +- .../proseo-api-prip/Dockerfile | 2 +- .../proseo-archive-mgr/Dockerfile | 2 +- .../proseo-facility-mgr/Dockerfile | 2 +- .../proseo-ingestor/Dockerfile | 2 +- .../proseo-monitor/Dockerfile | 2 +- .../proseo-notification/Dockerfile | 2 +- .../proseo-order-mgr/Dockerfile | 2 +- .../proseo-planner/Dockerfile | 2 +- .../proseo-processor-mgr/Dockerfile | 2 +- .../proseo-productclass-mgr/Dockerfile | 2 +- .../proseo-sample-wrapper/Dockerfile | 2 +- .../proseo-storage-mgr/Dockerfile | 2 +- .../proseo-ui-gui/Dockerfile | 2 +- .../proseo-user-mgr/Dockerfile | 2 +- docs/apidocs/options | 4 +- docs/index.html | 2 +- docs/modules.html | 2 +- docs/project-info.html | 2 +- docs/project-reports.html | 2 +- .../aip-client/apidocs/allclasses-index.html | 2 +- .../aip-client/apidocs/allpackages-index.html | 2 +- .../aip-client/apidocs/constant-values.html | 2 +- .../api/aipclient/AipClientApplication.html | 2 +- .../api/aipclient/AipClientConfiguration.html | 2 +- .../aipclient/AipClientSecurityConfig.html | 2 +- .../class-use/AipClientApplication.html | 2 +- .../class-use/AipClientConfiguration.html | 2 +- .../class-use/AipClientSecurityConfig.html | 2 +- .../proseo/api/aipclient/package-summary.html | 2 +- .../proseo/api/aipclient/package-tree.html | 2 +- .../dlr/proseo/api/aipclient/package-use.html | 2 +- .../aipclient/rest/AllbytimeController.html | 2 +- .../rest/AllbytimeControllerDecorator.html | 2 +- .../rest/AllbytimeControllerImpl.html | 2 +- .../api/aipclient/rest/BynameController.html | 2 +- .../rest/BynameControllerDecorator.html | 2 +- .../aipclient/rest/BynameControllerImpl.html | 2 +- .../api/aipclient/rest/BytimeController.html | 2 +- .../rest/BytimeControllerDecorator.html | 2 +- .../aipclient/rest/BytimeControllerImpl.html | 2 +- .../api/aipclient/rest/DownloadManager.html | 2 +- .../rest/IngestorproductController.html | 2 +- .../IngestorproductControllerDecorator.html | 2 +- .../rest/class-use/AllbytimeController.html | 2 +- .../AllbytimeControllerDecorator.html | 2 +- .../class-use/AllbytimeControllerImpl.html | 2 +- .../rest/class-use/BynameController.html | 2 +- .../class-use/BynameControllerDecorator.html | 2 +- .../rest/class-use/BynameControllerImpl.html | 2 +- .../rest/class-use/BytimeController.html | 2 +- .../class-use/BytimeControllerDecorator.html | 2 +- .../rest/class-use/BytimeControllerImpl.html | 2 +- .../rest/class-use/DownloadManager.html | 2 +- .../class-use/IngestorproductController.html | 2 +- .../IngestorproductControllerDecorator.html | 2 +- .../aipclient/rest/model/IngestorProduct.html | 2 +- .../api/aipclient/rest/model/Orbit.html | 2 +- .../rest/model/RestConfiguredProcessor.html | 2 +- .../rest/model/RestDownloadHistory.html | 2 +- .../aipclient/rest/model/RestParameter.html | 2 +- .../api/aipclient/rest/model/RestProduct.html | 2 +- .../aipclient/rest/model/RestProductFile.html | 2 +- .../rest/model/class-use/IngestorProduct.html | 2 +- .../aipclient/rest/model/class-use/Orbit.html | 2 +- .../class-use/RestConfiguredProcessor.html | 2 +- .../model/class-use/RestDownloadHistory.html | 2 +- .../rest/model/class-use/RestParameter.html | 2 +- .../rest/model/class-use/RestProduct.html | 2 +- .../rest/model/class-use/RestProductFile.html | 2 +- .../aipclient/rest/model/package-summary.html | 2 +- .../aipclient/rest/model/package-tree.html | 2 +- .../api/aipclient/rest/model/package-use.html | 2 +- .../api/aipclient/rest/package-summary.html | 2 +- .../api/aipclient/rest/package-tree.html | 2 +- .../api/aipclient/rest/package-use.html | 2 +- .../aip-client/apidocs/help-doc.html | 2 +- .../aip-client/apidocs/index-all.html | 2 +- docs/proseo-api/aip-client/apidocs/index.html | 4 +- .../aip-client/apidocs/overview-summary.html | 2 +- .../aip-client/apidocs/overview-tree.html | 2 +- .../proseo-api/aip-client/apidocs/search.html | 2 +- .../aip-client/apidocs/serialized-form.html | 2 +- docs/proseo-api/aip-client/index.html | 2 +- docs/proseo-api/aip-client/project-info.html | 2 +- .../aip-client/project-reports.html | 2 +- docs/proseo-api/aip-client/scm.html | 2 +- docs/proseo-api/aip-client/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../AuxipMonitor.TransferProduct.html | 2 +- .../dlr/proseo/api/auxipmon/AuxipMonitor.html | 2 +- .../api/auxipmon/AuxipMonitorApplication.html | 2 +- .../auxipmon/AuxipMonitorConfiguration.html | 2 +- .../AuxipMonitor.TransferProduct.html | 2 +- .../api/auxipmon/class-use/AuxipMonitor.html | 2 +- .../class-use/AuxipMonitorApplication.html | 2 +- .../class-use/AuxipMonitorConfiguration.html | 2 +- .../proseo/api/auxipmon/package-summary.html | 2 +- .../dlr/proseo/api/auxipmon/package-tree.html | 2 +- .../dlr/proseo/api/auxipmon/package-use.html | 2 +- .../rest/AuxipStatusControllerImpl.html | 2 +- .../api/auxipmon/rest/StatusController.html | 2 +- .../rest/StatusControllerDecorator.html | 2 +- .../class-use/AuxipStatusControllerImpl.html | 2 +- .../rest/class-use/StatusController.html | 2 +- .../class-use/StatusControllerDecorator.html | 2 +- .../rest/model/RestInterfaceStatus.html | 2 +- .../model/class-use/RestInterfaceStatus.html | 2 +- .../auxipmon/rest/model/package-summary.html | 2 +- .../api/auxipmon/rest/model/package-tree.html | 2 +- .../api/auxipmon/rest/model/package-use.html | 2 +- .../api/auxipmon/rest/package-summary.html | 2 +- .../api/auxipmon/rest/package-tree.html | 2 +- .../proseo/api/auxipmon/rest/package-use.html | 2 +- .../auxip-monitor/apidocs/help-doc.html | 2 +- .../auxip-monitor/apidocs/index-all.html | 2 +- .../auxip-monitor/apidocs/index.html | 4 +- .../apidocs/overview-summary.html | 2 +- .../auxip-monitor/apidocs/overview-tree.html | 2 +- .../auxip-monitor/apidocs/search.html | 2 +- .../apidocs/serialized-form.html | 2 +- docs/proseo-api/auxip-monitor/index.html | 2 +- .../auxip-monitor/project-info.html | 2 +- .../auxip-monitor/project-reports.html | 2 +- docs/proseo-api/auxip-monitor/scm.html | 2 +- docs/proseo-api/auxip-monitor/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../cadipmon/CadipMonitor.TransferFile.html | 2 +- .../CadipMonitor.TransferQualityInfo.html | 2 +- .../CadipMonitor.TransferSession.html | 2 +- .../dlr/proseo/api/cadipmon/CadipMonitor.html | 2 +- .../api/cadipmon/CadipMonitorApplication.html | 2 +- .../cadipmon/CadipMonitorConfiguration.html | 2 +- .../class-use/CadipMonitor.TransferFile.html | 2 +- .../CadipMonitor.TransferQualityInfo.html | 2 +- .../CadipMonitor.TransferSession.html | 2 +- .../api/cadipmon/class-use/CadipMonitor.html | 2 +- .../class-use/CadipMonitorApplication.html | 2 +- .../class-use/CadipMonitorConfiguration.html | 2 +- .../proseo/api/cadipmon/package-summary.html | 2 +- .../dlr/proseo/api/cadipmon/package-tree.html | 2 +- .../dlr/proseo/api/cadipmon/package-use.html | 2 +- .../rest/CadipStatusControllerImpl.html | 2 +- .../api/cadipmon/rest/StatusController.html | 2 +- .../rest/StatusControllerDecorator.html | 2 +- .../class-use/CadipStatusControllerImpl.html | 2 +- .../rest/class-use/StatusController.html | 2 +- .../class-use/StatusControllerDecorator.html | 2 +- .../rest/model/RestInterfaceStatus.html | 2 +- .../model/class-use/RestInterfaceStatus.html | 2 +- .../cadipmon/rest/model/package-summary.html | 2 +- .../api/cadipmon/rest/model/package-tree.html | 2 +- .../api/cadipmon/rest/model/package-use.html | 2 +- .../api/cadipmon/rest/package-summary.html | 2 +- .../api/cadipmon/rest/package-tree.html | 2 +- .../proseo/api/cadipmon/rest/package-use.html | 2 +- .../cadip-monitor/apidocs/help-doc.html | 2 +- .../cadip-monitor/apidocs/index-all.html | 2 +- .../cadip-monitor/apidocs/index.html | 4 +- .../apidocs/overview-summary.html | 2 +- .../cadip-monitor/apidocs/overview-tree.html | 2 +- .../cadip-monitor/apidocs/search.html | 2 +- .../apidocs/serialized-form.html | 2 +- docs/proseo-api/cadip-monitor/index.html | 2 +- .../cadip-monitor/project-info.html | 2 +- .../cadip-monitor/project-reports.html | 2 +- docs/proseo-api/cadip-monitor/scm.html | 2 +- docs/proseo-api/cadip-monitor/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../edip-monitor/apidocs/constant-values.html | 2 +- ...ipMonitor.DataSessionInformationBlock.html | 2 +- .../edipmon/EdipMonitor.TransferSession.html | 2 +- .../dlr/proseo/api/edipmon/EdipMonitor.html | 2 +- .../api/edipmon/EdipMonitorApplication.html | 2 +- .../api/edipmon/EdipMonitorConfiguration.html | 2 +- ...ipMonitor.DataSessionInformationBlock.html | 2 +- .../EdipMonitor.TransferSession.html | 2 +- .../api/edipmon/class-use/EdipMonitor.html | 2 +- .../class-use/EdipMonitorApplication.html | 2 +- .../class-use/EdipMonitorConfiguration.html | 2 +- .../proseo/api/edipmon/package-summary.html | 2 +- .../dlr/proseo/api/edipmon/package-tree.html | 2 +- .../dlr/proseo/api/edipmon/package-use.html | 2 +- .../rest/EdipStatusControllerImpl.html | 2 +- .../api/edipmon/rest/StatusController.html | 2 +- .../rest/StatusControllerDecorator.html | 2 +- .../class-use/EdipStatusControllerImpl.html | 2 +- .../rest/class-use/StatusController.html | 2 +- .../class-use/StatusControllerDecorator.html | 2 +- .../rest/model/RestInterfaceStatus.html | 2 +- .../model/class-use/RestInterfaceStatus.html | 2 +- .../edipmon/rest/model/package-summary.html | 2 +- .../api/edipmon/rest/model/package-tree.html | 2 +- .../api/edipmon/rest/model/package-use.html | 2 +- .../api/edipmon/rest/package-summary.html | 2 +- .../proseo/api/edipmon/rest/package-tree.html | 2 +- .../proseo/api/edipmon/rest/package-use.html | 2 +- .../edip-monitor/apidocs/help-doc.html | 2 +- .../edip-monitor/apidocs/index-all.html | 2 +- .../edip-monitor/apidocs/index.html | 4 +- .../apidocs/overview-summary.html | 2 +- .../edip-monitor/apidocs/overview-tree.html | 2 +- .../edip-monitor/apidocs/search.html | 2 +- .../edip-monitor/apidocs/serialized-form.html | 2 +- docs/proseo-api/edip-monitor/index.html | 2 +- .../proseo-api/edip-monitor/project-info.html | 2 +- .../edip-monitor/project-reports.html | 2 +- docs/proseo-api/edip-monitor/scm.html | 2 +- docs/proseo-api/edip-monitor/summary.html | 4 +- docs/proseo-api/index.html | 2 +- docs/proseo-api/modules.html | 2 +- .../odip/apidocs/allclasses-index.html | 2 +- .../odip/apidocs/allpackages-index.html | 2 +- .../odip/apidocs/constant-values.html | 2 +- .../api/odip/OAuth2TokenManager.UserInfo.html | 2 +- .../proseo/api/odip/OAuth2TokenManager.html | 2 +- .../dlr/proseo/api/odip/OdipApplication.html | 2 +- .../proseo/api/odip/OdipApplicationBase.html | 2 +- .../proseo/api/odip/OdipConfiguration.html | 2 +- .../de/dlr/proseo/api/odip/OdipHttp.html | 2 +- .../de/dlr/proseo/api/odip/OdipSecurity.html | 2 +- .../OAuth2TokenManager.UserInfo.html | 2 +- .../odip/class-use/OAuth2TokenManager.html | 2 +- .../api/odip/class-use/OdipApplication.html | 2 +- .../odip/class-use/OdipApplicationBase.html | 2 +- .../api/odip/class-use/OdipConfiguration.html | 2 +- .../proseo/api/odip/class-use/OdipHttp.html | 2 +- .../api/odip/class-use/OdipSecurity.html | 2 +- ...aExpressionVisitor.AttributeCondition.html | 2 +- .../AttributeLambdaExpressionVisitor.html | 2 +- .../api/odip/odata/CscAttributeName.html | 2 +- .../de/dlr/proseo/api/odip/odata/LogUtil.html | 2 +- .../api/odip/odata/OdipEdmProvider.html | 2 +- .../odata/OdipEntityCollectionProcessor.html | 2 +- .../api/odip/odata/OdipEntityProcessor.html | 2 +- .../dlr/proseo/api/odip/odata/OdipUtil.html | 2 +- .../odata/OdipUtilBase.OdipException.html | 2 +- .../proseo/api/odip/odata/OdipUtilBase.html | 2 +- .../OrderSqlFilterExpressionVisitor.html | 2 +- .../odata/SqlFilterExpressionVisitor.html | 2 +- .../WorkflowSqlFilterExpressionVisitor.html | 2 +- ...aExpressionVisitor.AttributeCondition.html | 2 +- .../AttributeLambdaExpressionVisitor.html | 2 +- .../odata/class-use/CscAttributeName.html | 2 +- .../api/odip/odata/class-use/LogUtil.html | 2 +- .../odip/odata/class-use/OdipEdmProvider.html | 2 +- .../OdipEntityCollectionProcessor.html | 2 +- .../odata/class-use/OdipEntityProcessor.html | 2 +- .../api/odip/odata/class-use/OdipUtil.html | 2 +- .../class-use/OdipUtilBase.OdipException.html | 2 +- .../odip/odata/class-use/OdipUtilBase.html | 2 +- .../OrderSqlFilterExpressionVisitor.html | 2 +- .../class-use/SqlFilterExpressionVisitor.html | 2 +- .../WorkflowSqlFilterExpressionVisitor.html | 2 +- .../api/odip/odata/package-summary.html | 2 +- .../proseo/api/odip/odata/package-tree.html | 2 +- .../proseo/api/odip/odata/package-use.html | 2 +- .../dlr/proseo/api/odip/package-summary.html | 2 +- .../de/dlr/proseo/api/odip/package-tree.html | 2 +- .../de/dlr/proseo/api/odip/package-use.html | 2 +- .../api/odip/rest/OAuth2TokenController.html | 2 +- .../api/odip/rest/OdipQueryController.html | 2 +- .../odip/rest/ProductionOrderController.html | 2 +- .../odip/rest/ProductionOrdersController.html | 2 +- .../rest/ProductionOrdersIdController.html | 2 +- .../proseo/api/odip/rest/TokenController.html | 2 +- .../api/odip/rest/WorkflowController.html | 2 +- .../rest/class-use/OAuth2TokenController.html | 2 +- .../rest/class-use/OdipQueryController.html | 2 +- .../class-use/ProductionOrderController.html | 2 +- .../class-use/ProductionOrdersController.html | 2 +- .../ProductionOrdersIdController.html | 2 +- .../odip/rest/class-use/TokenController.html | 2 +- .../rest/class-use/WorkflowController.html | 2 +- .../api/odip/rest/model/CscContentDate.html | 2 +- .../rest/model/CscInputProductReference.html | 2 +- .../odip/rest/model/CscProductionOrder.html | 2 +- .../api/odip/rest/model/CscWorkflow.html | 2 +- .../odip/rest/model/CscWorkflowOption.html | 2 +- .../api/odip/rest/model/OAuth2Response.html | 2 +- .../proseo/api/odip/rest/model/Status.html | 2 +- .../rest/model/class-use/CscContentDate.html | 2 +- .../class-use/CscInputProductReference.html | 2 +- .../model/class-use/CscProductionOrder.html | 2 +- .../rest/model/class-use/CscWorkflow.html | 2 +- .../model/class-use/CscWorkflowOption.html | 2 +- .../rest/model/class-use/OAuth2Response.html | 2 +- .../api/odip/rest/model/class-use/Status.html | 2 +- .../api/odip/rest/model/package-summary.html | 2 +- .../api/odip/rest/model/package-tree.html | 2 +- .../api/odip/rest/model/package-use.html | 2 +- .../proseo/api/odip/rest/package-summary.html | 2 +- .../proseo/api/odip/rest/package-tree.html | 2 +- .../dlr/proseo/api/odip/rest/package-use.html | 2 +- .../api/odip/service/ServiceConnection.html | 2 +- .../service/class-use/ServiceConnection.html | 2 +- .../api/odip/service/package-summary.html | 2 +- .../proseo/api/odip/service/package-tree.html | 2 +- .../proseo/api/odip/service/package-use.html | 2 +- .../dlr/proseo/api/odip/util/ProductUtil.html | 2 +- .../api/odip/util/class-use/ProductUtil.html | 2 +- .../proseo/api/odip/util/package-summary.html | 2 +- .../proseo/api/odip/util/package-tree.html | 2 +- .../dlr/proseo/api/odip/util/package-use.html | 2 +- docs/proseo-api/odip/apidocs/help-doc.html | 2 +- docs/proseo-api/odip/apidocs/index-all.html | 2 +- docs/proseo-api/odip/apidocs/index.html | 4 +- docs/proseo-api/odip/apidocs/options | 4 +- .../odip/apidocs/overview-summary.html | 2 +- .../odip/apidocs/overview-tree.html | 2 +- docs/proseo-api/odip/apidocs/search.html | 2 +- .../odip/apidocs/serialized-form.html | 2 +- docs/proseo-api/odip/index.html | 2 +- docs/proseo-api/odip/project-info.html | 2 +- docs/proseo-api/odip/project-reports.html | 2 +- docs/proseo-api/odip/scm.html | 2 +- docs/proseo-api/odip/summary.html | 4 +- docs/proseo-api/project-info.html | 2 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../api/prip/OAuth2TokenManager.UserInfo.html | 2 +- .../proseo/api/prip/OAuth2TokenManager.html | 2 +- .../prip/ProductionInterfaceApplication.html | 2 +- .../ProductionInterfaceConfiguration.html | 2 +- .../api/prip/ProductionInterfaceSecurity.html | 2 +- .../OAuth2TokenManager.UserInfo.html | 2 +- .../prip/class-use/OAuth2TokenManager.html | 2 +- .../ProductionInterfaceApplication.html | 2 +- .../ProductionInterfaceConfiguration.html | 2 +- .../ProductionInterfaceSecurity.html | 2 +- ...aExpressionVisitor.AttributeCondition.html | 2 +- .../AttributeLambdaExpressionVisitor.html | 2 +- .../api/prip/odata/CscAttributeName.html | 2 +- .../de/dlr/proseo/api/prip/odata/LogUtil.html | 2 +- .../api/prip/odata/ProductEdmProvider.html | 2 +- .../ProductEntityCollectionProcessor.html | 2 +- .../prip/odata/ProductEntityProcessor.html | 2 +- .../proseo/api/prip/odata/ProductUtil.html | 2 +- .../odata/SqlFilterExpressionVisitor.html | 2 +- ...aExpressionVisitor.AttributeCondition.html | 2 +- .../AttributeLambdaExpressionVisitor.html | 2 +- .../odata/class-use/CscAttributeName.html | 2 +- .../api/prip/odata/class-use/LogUtil.html | 2 +- .../odata/class-use/ProductEdmProvider.html | 2 +- .../ProductEntityCollectionProcessor.html | 2 +- .../class-use/ProductEntityProcessor.html | 2 +- .../api/prip/odata/class-use/ProductUtil.html | 2 +- .../class-use/SqlFilterExpressionVisitor.html | 2 +- .../api/prip/odata/package-summary.html | 2 +- .../proseo/api/prip/odata/package-tree.html | 2 +- .../proseo/api/prip/odata/package-use.html | 2 +- .../dlr/proseo/api/prip/package-summary.html | 2 +- .../de/dlr/proseo/api/prip/package-tree.html | 2 +- .../de/dlr/proseo/api/prip/package-use.html | 2 +- .../api/prip/rest/$valueController.html | 2 +- .../api/prip/rest/OAuth2TokenController.html | 2 +- .../api/prip/rest/ProductController.html | 2 +- .../api/prip/rest/ProductQueryController.html | 2 +- .../proseo/api/prip/rest/TokenController.html | 2 +- .../prip/rest/class-use/$valueController.html | 2 +- .../rest/class-use/OAuth2TokenController.html | 2 +- .../rest/class-use/ProductController.html | 2 +- .../class-use/ProductQueryController.html | 2 +- .../prip/rest/class-use/TokenController.html | 2 +- .../api/prip/rest/model/CscAttribute.html | 2 +- .../api/prip/rest/model/CscChecksum.html | 2 +- .../api/prip/rest/model/CscProduct.html | 2 +- .../prip/rest/model/CscProductionType.html | 2 +- .../api/prip/rest/model/CscTimeRange.html | 2 +- .../api/prip/rest/model/OAuth2Response.html | 2 +- .../rest/model/class-use/CscAttribute.html | 2 +- .../rest/model/class-use/CscChecksum.html | 2 +- .../prip/rest/model/class-use/CscProduct.html | 2 +- .../model/class-use/CscProductionType.html | 2 +- .../rest/model/class-use/CscTimeRange.html | 2 +- .../rest/model/class-use/OAuth2Response.html | 2 +- .../api/prip/rest/model/package-summary.html | 2 +- .../api/prip/rest/model/package-tree.html | 2 +- .../api/prip/rest/model/package-use.html | 2 +- .../proseo/api/prip/rest/package-summary.html | 2 +- .../proseo/api/prip/rest/package-tree.html | 2 +- .../dlr/proseo/api/prip/rest/package-use.html | 2 +- .../proseo-api-prip/apidocs/help-doc.html | 2 +- .../proseo-api-prip/apidocs/index-all.html | 2 +- .../proseo-api-prip/apidocs/index.html | 4 +- .../apidocs/overview-summary.html | 2 +- .../apidocs/overview-tree.html | 2 +- .../proseo-api-prip/apidocs/search.html | 2 +- .../apidocs/serialized-form.html | 2 +- docs/proseo-api/proseo-api-prip/index.html | 2 +- .../proseo-api-prip/project-info.html | 2 +- .../proseo-api-prip/project-reports.html | 2 +- docs/proseo-api/proseo-api-prip/scm.html | 2 +- docs/proseo-api/proseo-api-prip/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../ProductArchiveManagerApplication.html | 2 +- .../ProductArchiveManagerConfiguration.html | 2 +- .../ProductArchiveManagerSecurityConfig.html | 2 +- .../ProductArchiveManagerApplication.html | 2 +- .../ProductArchiveManagerConfiguration.html | 2 +- .../ProductArchiveManagerSecurityConfig.html | 2 +- .../proseo/archivemgr/package-summary.html | 2 +- .../dlr/proseo/archivemgr/package-tree.html | 2 +- .../de/dlr/proseo/archivemgr/package-use.html | 2 +- .../archivemgr/rest/ArchiveController.html | 2 +- .../rest/ArchiveControllerDecorator.html | 2 +- .../rest/ProductArchiveControllerImpl.html | 2 +- .../rest/ProductArchiveManager.html | 2 +- .../rest/class-use/ArchiveController.html | 2 +- .../class-use/ArchiveControllerDecorator.html | 2 +- .../ProductArchiveControllerImpl.html | 2 +- .../rest/class-use/ProductArchiveManager.html | 2 +- .../rest/model/ProductArchiveModelMapper.html | 2 +- .../rest/model/ProductArchiveRestMapper.html | 2 +- .../rest/model/RestProductArchive.html | 2 +- .../class-use/ProductArchiveModelMapper.html | 2 +- .../class-use/ProductArchiveRestMapper.html | 2 +- .../model/class-use/RestProductArchive.html | 2 +- .../rest/model/package-summary.html | 2 +- .../archivemgr/rest/model/package-tree.html | 2 +- .../archivemgr/rest/model/package-use.html | 2 +- .../archivemgr/rest/package-summary.html | 2 +- .../proseo/archivemgr/rest/package-tree.html | 2 +- .../proseo/archivemgr/rest/package-use.html | 2 +- .../proseo/archivemgr/utils/StringUtils.html | 2 +- .../utils/class-use/StringUtils.html | 2 +- .../archivemgr/utils/package-summary.html | 2 +- .../proseo/archivemgr/utils/package-tree.html | 2 +- .../proseo/archivemgr/utils/package-use.html | 2 +- .../proseo-archive-mgr/apidocs/help-doc.html | 2 +- .../proseo-archive-mgr/apidocs/index-all.html | 2 +- .../proseo-archive-mgr/apidocs/index.html | 4 +- .../proseo-archive-mgr/apidocs/options | 4 +- .../apidocs/overview-summary.html | 2 +- .../apidocs/overview-tree.html | 2 +- .../proseo-archive-mgr/apidocs/search.html | 2 +- .../apidocs/serialized-form.html | 2 +- docs/proseo-api/proseo-archive-mgr/index.html | 2 +- .../proseo-archive-mgr/project-info.html | 2 +- .../proseo-archive-mgr/project-reports.html | 2 +- docs/proseo-api/proseo-archive-mgr/scm.html | 2 +- .../proseo-archive-mgr/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../basemon/BaseMonitor.TransferControl.html | 2 +- .../dlr/proseo/api/basemon/BaseMonitor.html | 2 +- .../proseo/api/basemon/TransferObject.html | 2 +- .../BaseMonitor.TransferControl.html | 2 +- .../api/basemon/class-use/BaseMonitor.html | 2 +- .../api/basemon/class-use/TransferObject.html | 2 +- .../proseo/api/basemon/package-summary.html | 2 +- .../dlr/proseo/api/basemon/package-tree.html | 2 +- .../dlr/proseo/api/basemon/package-use.html | 2 +- .../proseo-base-monitor/apidocs/help-doc.html | 2 +- .../apidocs/index-all.html | 2 +- .../proseo-base-monitor/apidocs/index.html | 2 +- .../apidocs/overview-tree.html | 2 +- .../proseo-base-monitor/apidocs/search.html | 2 +- .../proseo-api/proseo-base-monitor/index.html | 2 +- .../proseo-base-monitor/project-info.html | 2 +- .../proseo-base-monitor/project-reports.html | 2 +- docs/proseo-api/proseo-base-monitor/scm.html | 2 +- .../proseo-base-monitor/summary.html | 4 +- docs/proseo-api/scm.html | 2 +- docs/proseo-api/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../xbip-monitor/apidocs/constant-values.html | 2 +- ...ipMonitor.DataSessionInformationBlock.html | 2 +- .../xbipmon/XbipMonitor.TransferSession.html | 2 +- .../dlr/proseo/api/xbipmon/XbipMonitor.html | 2 +- .../api/xbipmon/XbipMonitorApplication.html | 2 +- .../api/xbipmon/XbipMonitorConfiguration.html | 2 +- ...ipMonitor.DataSessionInformationBlock.html | 2 +- .../XbipMonitor.TransferSession.html | 2 +- .../api/xbipmon/class-use/XbipMonitor.html | 2 +- .../class-use/XbipMonitorApplication.html | 2 +- .../class-use/XbipMonitorConfiguration.html | 2 +- .../proseo/api/xbipmon/package-summary.html | 2 +- .../dlr/proseo/api/xbipmon/package-tree.html | 2 +- .../dlr/proseo/api/xbipmon/package-use.html | 2 +- .../api/xbipmon/rest/StatusController.html | 2 +- .../rest/StatusControllerDecorator.html | 2 +- .../rest/XbipStatusControllerImpl.html | 2 +- .../rest/class-use/StatusController.html | 2 +- .../class-use/StatusControllerDecorator.html | 2 +- .../class-use/XbipStatusControllerImpl.html | 2 +- .../rest/model/RestInterfaceStatus.html | 2 +- .../model/class-use/RestInterfaceStatus.html | 2 +- .../xbipmon/rest/model/package-summary.html | 2 +- .../api/xbipmon/rest/model/package-tree.html | 2 +- .../api/xbipmon/rest/model/package-use.html | 2 +- .../api/xbipmon/rest/package-summary.html | 2 +- .../proseo/api/xbipmon/rest/package-tree.html | 2 +- .../proseo/api/xbipmon/rest/package-use.html | 2 +- .../xbip-monitor/apidocs/help-doc.html | 2 +- .../xbip-monitor/apidocs/index-all.html | 2 +- .../xbip-monitor/apidocs/index.html | 4 +- .../apidocs/overview-summary.html | 2 +- .../xbip-monitor/apidocs/overview-tree.html | 2 +- .../xbip-monitor/apidocs/search.html | 2 +- .../xbip-monitor/apidocs/serialized-form.html | 2 +- docs/proseo-api/xbip-monitor/index.html | 2 +- .../proseo-api/xbip-monitor/project-info.html | 2 +- .../xbip-monitor/project-reports.html | 2 +- docs/proseo-api/xbip-monitor/scm.html | 2 +- docs/proseo-api/xbip-monitor/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../proseo/basewrap/BaseWrapper.ENV_VARS.html | 2 +- .../BaseWrapper.WrapperException.html | 2 +- .../de/dlr/proseo/basewrap/BaseWrapper.html | 2 +- .../de/dlr/proseo/basewrap/MD5Util.html | 2 +- .../class-use/BaseWrapper.ENV_VARS.html | 2 +- .../BaseWrapper.WrapperException.html | 2 +- .../basewrap/class-use/BaseWrapper.html | 2 +- .../proseo/basewrap/class-use/MD5Util.html | 2 +- .../dlr/proseo/basewrap/package-summary.html | 2 +- .../de/dlr/proseo/basewrap/package-tree.html | 2 +- .../de/dlr/proseo/basewrap/package-use.html | 2 +- .../basewrap/rest/HttpResponseInfo.html | 2 +- .../de/dlr/proseo/basewrap/rest/RestAuth.html | 2 +- .../basewrap/rest/RestOps.HttpMethod.html | 2 +- .../de/dlr/proseo/basewrap/rest/RestOps.html | 2 +- .../rest/class-use/HttpResponseInfo.html | 2 +- .../basewrap/rest/class-use/RestAuth.html | 2 +- .../rest/class-use/RestOps.HttpMethod.html | 2 +- .../basewrap/rest/class-use/RestOps.html | 2 +- .../proseo/basewrap/rest/package-summary.html | 2 +- .../proseo/basewrap/rest/package-tree.html | 2 +- .../dlr/proseo/basewrap/rest/package-use.html | 2 +- .../apidocs/deprecated-list.html | 2 +- .../proseo-base-wrapper/apidocs/help-doc.html | 2 +- .../apidocs/index-all.html | 2 +- docs/proseo-base-wrapper/apidocs/index.html | 4 +- .../apidocs/overview-summary.html | 2 +- .../apidocs/overview-tree.html | 2 +- docs/proseo-base-wrapper/apidocs/search.html | 2 +- .../apidocs/serialized-form.html | 2 +- docs/proseo-base-wrapper/index.html | 2 +- docs/proseo-base-wrapper/project-info.html | 2 +- docs/proseo-base-wrapper/project-reports.html | 2 +- docs/proseo-base-wrapper/scm.html | 2 +- docs/proseo-base-wrapper/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../dlr/proseo/model/enums/ArchiveType.html | 2 +- .../dlr/proseo/model/enums/FacilityState.html | 2 +- .../proseo/model/enums/JobOrderVersion.html | 2 +- .../proseo/model/enums/OrderSlicingType.html | 2 +- .../dlr/proseo/model/enums/OrderSource.html | 2 +- .../de/dlr/proseo/model/enums/OrderState.html | 2 +- .../dlr/proseo/model/enums/ParameterType.html | 2 +- .../proseo/model/enums/ProcessingLevel.html | 2 +- .../proseo/model/enums/ProductQuality.html | 2 +- .../proseo/model/enums/ProductVisibility.html | 2 +- .../proseo/model/enums/ProductionType.html | 2 +- .../dlr/proseo/model/enums/StorageType.html | 2 +- .../de/dlr/proseo/model/enums/UserRole.html | 2 +- .../model/enums/class-use/ArchiveType.html | 2 +- .../model/enums/class-use/FacilityState.html | 2 +- .../enums/class-use/JobOrderVersion.html | 2 +- .../enums/class-use/OrderSlicingType.html | 2 +- .../model/enums/class-use/OrderSource.html | 2 +- .../model/enums/class-use/OrderState.html | 2 +- .../model/enums/class-use/ParameterType.html | 2 +- .../enums/class-use/ProcessingLevel.html | 2 +- .../model/enums/class-use/ProductQuality.html | 2 +- .../enums/class-use/ProductVisibility.html | 2 +- .../model/enums/class-use/ProductionType.html | 2 +- .../model/enums/class-use/StorageType.html | 2 +- .../model/enums/class-use/UserRole.html | 2 +- .../proseo/model/enums/package-summary.html | 2 +- .../dlr/proseo/model/enums/package-tree.html | 2 +- .../dlr/proseo/model/enums/package-use.html | 2 +- .../proseo/model/joborder/BreakpointFile.html | 2 +- .../Conf.NonUniqueResultException.html | 2 +- .../de/dlr/proseo/model/joborder/Conf.html | 2 +- .../proseo/model/joborder/InputOutput.html | 2 +- .../proseo/model/joborder/IpfFileName.html | 2 +- .../dlr/proseo/model/joborder/JobOrder.html | 2 +- .../de/dlr/proseo/model/joborder/Proc.html | 2 +- .../model/joborder/ProcessingParameter.html | 2 +- .../proseo/model/joborder/SensingTime.html | 2 +- .../proseo/model/joborder/TimeInterval.html | 2 +- .../joborder/class-use/BreakpointFile.html | 2 +- .../Conf.NonUniqueResultException.html | 2 +- .../proseo/model/joborder/class-use/Conf.html | 2 +- .../model/joborder/class-use/InputOutput.html | 2 +- .../model/joborder/class-use/IpfFileName.html | 2 +- .../model/joborder/class-use/JobOrder.html | 2 +- .../proseo/model/joborder/class-use/Proc.html | 2 +- .../class-use/ProcessingParameter.html | 2 +- .../model/joborder/class-use/SensingTime.html | 2 +- .../joborder/class-use/TimeInterval.html | 2 +- .../model/joborder/package-summary.html | 2 +- .../proseo/model/joborder/package-tree.html | 2 +- .../proseo/model/joborder/package-use.html | 2 +- .../dlr/proseo/model/util/MonOrderStates.html | 2 +- .../proseo/model/util/MonServiceStates.html | 2 +- .../proseo/model/util/OrbitTimeFormatter.html | 2 +- .../de/dlr/proseo/model/util/ProseoUtil.html | 2 +- .../de/dlr/proseo/model/util/StringUtils.html | 2 +- .../model/util/class-use/MonOrderStates.html | 2 +- .../util/class-use/MonServiceStates.html | 2 +- .../util/class-use/OrbitTimeFormatter.html | 2 +- .../model/util/class-use/ProseoUtil.html | 2 +- .../model/util/class-use/StringUtils.html | 2 +- .../proseo/model/util/package-summary.html | 2 +- .../dlr/proseo/model/util/package-tree.html | 2 +- .../de/dlr/proseo/model/util/package-use.html | 2 +- .../apidocs/deprecated-list.html | 2 +- docs/proseo-common/apidocs/help-doc.html | 2 +- docs/proseo-common/apidocs/index-all.html | 2 +- docs/proseo-common/apidocs/index.html | 4 +- docs/proseo-common/apidocs/options | 4 +- .../apidocs/overview-summary.html | 2 +- docs/proseo-common/apidocs/overview-tree.html | 2 +- docs/proseo-common/apidocs/search.html | 2 +- .../apidocs/serialized-form.html | 2 +- docs/proseo-common/index.html | 2 +- docs/proseo-common/project-info.html | 2 +- docs/proseo-common/project-reports.html | 2 +- docs/proseo-common/scm.html | 2 +- docs/proseo-common/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../de/dlr/proseo/facmgr/FacilityManager.html | 2 +- .../facmgr/FacilitymgrConfiguration.html | 2 +- .../facmgr/FacilitymgrSecurityConfig.html | 2 +- .../facmgr/class-use/FacilityManager.html | 2 +- .../class-use/FacilitymgrConfiguration.html | 2 +- .../class-use/FacilitymgrSecurityConfig.html | 2 +- .../de/dlr/proseo/facmgr/package-summary.html | 2 +- .../de/dlr/proseo/facmgr/package-tree.html | 2 +- .../de/dlr/proseo/facmgr/package-use.html | 2 +- .../facmgr/rest/FacilityController.html | 2 +- .../rest/FacilityControllerDecorator.html | 2 +- .../facmgr/rest/FacmgrControllerImpl.html | 2 +- .../dlr/proseo/facmgr/rest/FacmgrManager.html | 2 +- .../rest/class-use/FacilityController.html | 2 +- .../FacilityControllerDecorator.html | 2 +- .../rest/class-use/FacmgrControllerImpl.html | 2 +- .../facmgr/rest/class-use/FacmgrManager.html | 2 +- .../proseo/facmgr/rest/model/FacmgrUtil.html | 2 +- .../rest/model/RestProcessingFacility.html | 2 +- .../rest/model/class-use/FacmgrUtil.html | 2 +- .../class-use/RestProcessingFacility.html | 2 +- .../facmgr/rest/model/package-summary.html | 2 +- .../facmgr/rest/model/package-tree.html | 2 +- .../proseo/facmgr/rest/model/package-use.html | 2 +- .../proseo/facmgr/rest/package-summary.html | 2 +- .../dlr/proseo/facmgr/rest/package-tree.html | 2 +- .../dlr/proseo/facmgr/rest/package-use.html | 2 +- .../proseo-facility-mgr/apidocs/help-doc.html | 2 +- .../apidocs/index-all.html | 2 +- docs/proseo-facility-mgr/apidocs/index.html | 4 +- .../apidocs/overview-summary.html | 2 +- .../apidocs/overview-tree.html | 2 +- docs/proseo-facility-mgr/apidocs/search.html | 2 +- .../apidocs/serialized-form.html | 2 +- docs/proseo-facility-mgr/index.html | 2 +- docs/proseo-facility-mgr/project-info.html | 2 +- docs/proseo-facility-mgr/project-reports.html | 2 +- docs/proseo-facility-mgr/scm.html | 2 +- docs/proseo-facility-mgr/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../proseo/geotools/GeotoolsApplication.html | 2 +- .../GeotoolsConfiguration.Shapefile.html | 2 +- .../geotools/GeotoolsConfiguration.html | 2 +- .../class-use/GeotoolsApplication.html | 2 +- .../GeotoolsConfiguration.Shapefile.html | 2 +- .../class-use/GeotoolsConfiguration.html | 2 +- .../dlr/proseo/geotools/package-summary.html | 2 +- .../de/dlr/proseo/geotools/package-tree.html | 2 +- .../de/dlr/proseo/geotools/package-use.html | 2 +- .../geotools/rest/ContainController.html | 2 +- .../rest/ContainControllerDecorator.html | 2 +- .../geotools/rest/ContainControllerImpl.html | 2 +- .../proseo/geotools/rest/GeotoolsUtil.html | 2 +- .../proseo/geotools/rest/InfoController.html | 2 +- .../rest/InfoControllerDecorator.html | 2 +- .../geotools/rest/InfoControllerImpl.html | 2 +- .../geotools/rest/OverlapController.html | 2 +- .../rest/OverlapControllerDecorator.html | 2 +- .../geotools/rest/OverlapControllerImpl.html | 2 +- .../geotools/rest/ShpFile.GeoFileType.html | 2 +- .../de/dlr/proseo/geotools/rest/ShpFile.html | 2 +- .../rest/class-use/ContainController.html | 2 +- .../class-use/ContainControllerDecorator.html | 2 +- .../rest/class-use/ContainControllerImpl.html | 2 +- .../geotools/rest/class-use/GeotoolsUtil.html | 2 +- .../rest/class-use/InfoController.html | 2 +- .../class-use/InfoControllerDecorator.html | 2 +- .../rest/class-use/InfoControllerImpl.html | 2 +- .../rest/class-use/OverlapController.html | 2 +- .../class-use/OverlapControllerDecorator.html | 2 +- .../rest/class-use/OverlapControllerImpl.html | 2 +- .../rest/class-use/ShpFile.GeoFileType.html | 2 +- .../geotools/rest/class-use/ShpFile.html | 2 +- .../proseo/geotools/rest/model/RestPoint.html | 2 +- .../geotools/rest/model/RestPolygon.html | 2 +- .../rest/model/class-use/RestPoint.html | 2 +- .../rest/model/class-use/RestPolygon.html | 2 +- .../geotools/rest/model/package-summary.html | 2 +- .../geotools/rest/model/package-tree.html | 2 +- .../geotools/rest/model/package-use.html | 2 +- .../proseo/geotools/rest/package-summary.html | 2 +- .../proseo/geotools/rest/package-tree.html | 2 +- .../dlr/proseo/geotools/rest/package-use.html | 2 +- docs/proseo-geotools/apidocs/help-doc.html | 2 +- docs/proseo-geotools/apidocs/index-all.html | 2 +- docs/proseo-geotools/apidocs/index.html | 4 +- docs/proseo-geotools/apidocs/options | 4 +- .../apidocs/overview-summary.html | 2 +- .../apidocs/overview-tree.html | 2 +- docs/proseo-geotools/apidocs/search.html | 2 +- .../apidocs/serialized-form.html | 2 +- docs/proseo-geotools/index.html | 2 +- docs/proseo-geotools/project-info.html | 2 +- docs/proseo-geotools/project-reports.html | 2 +- docs/proseo-geotools/scm.html | 2 +- docs/proseo-geotools/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../proseo/ingestor/IngestorApplication.html | 2 +- .../ingestor/IngestorConfiguration.html | 2 +- .../ingestor/IngestorSecurityConfig.html | 2 +- ...sageSecurityWebApplicationInitializer.html | 2 +- .../ingestor/PlannerSemaphoreClient.html | 2 +- .../class-use/IngestorApplication.html | 2 +- .../class-use/IngestorConfiguration.html | 2 +- .../class-use/IngestorSecurityConfig.html | 2 +- ...sageSecurityWebApplicationInitializer.html | 2 +- .../class-use/PlannerSemaphoreClient.html | 2 +- .../cleanup/CleanupProductThread.html | 2 +- .../class-use/CleanupProductThread.html | 2 +- .../ingestor/cleanup/package-summary.html | 2 +- .../proseo/ingestor/cleanup/package-tree.html | 2 +- .../proseo/ingestor/cleanup/package-use.html | 2 +- .../dlr/proseo/ingestor/package-summary.html | 2 +- .../de/dlr/proseo/ingestor/package-tree.html | 2 +- .../de/dlr/proseo/ingestor/package-use.html | 2 +- .../ingestor/rest/IngestController.html | 2 +- .../rest/IngestControllerDecorator.html | 2 +- .../ingestor/rest/IngestControllerImpl.html | 2 +- .../ingestor/rest/ProductController.html | 2 +- .../rest/ProductControllerDecorator.html | 2 +- .../ingestor/rest/ProductControllerImpl.html | 2 +- .../proseo/ingestor/rest/ProductIngestor.html | 2 +- .../proseo/ingestor/rest/ProductManager.html | 2 +- .../rest/class-use/IngestController.html | 2 +- .../class-use/IngestControllerDecorator.html | 2 +- .../rest/class-use/IngestControllerImpl.html | 2 +- .../rest/class-use/ProductController.html | 2 +- .../class-use/ProductControllerDecorator.html | 2 +- .../rest/class-use/ProductControllerImpl.html | 2 +- .../rest/class-use/ProductIngestor.html | 2 +- .../rest/class-use/ProductManager.html | 2 +- .../ingestor/rest/model/IngestorProduct.html | 2 +- .../dlr/proseo/ingestor/rest/model/Orbit.html | 2 +- .../ingestor/rest/model/ProductFileUtil.html | 2 +- .../ingestor/rest/model/ProductUtil.html | 2 +- .../rest/model/RestConfiguredProcessor.html | 2 +- .../rest/model/RestDownloadHistory.html | 2 +- .../ingestor/rest/model/RestParameter.html | 2 +- .../ingestor/rest/model/RestProduct.html | 2 +- .../ingestor/rest/model/RestProductFile.html | 2 +- .../rest/model/class-use/IngestorProduct.html | 2 +- .../ingestor/rest/model/class-use/Orbit.html | 2 +- .../rest/model/class-use/ProductFileUtil.html | 2 +- .../rest/model/class-use/ProductUtil.html | 2 +- .../class-use/RestConfiguredProcessor.html | 2 +- .../model/class-use/RestDownloadHistory.html | 2 +- .../rest/model/class-use/RestParameter.html | 2 +- .../rest/model/class-use/RestProduct.html | 2 +- .../rest/model/class-use/RestProductFile.html | 2 +- .../ingestor/rest/model/package-summary.html | 2 +- .../ingestor/rest/model/package-tree.html | 2 +- .../ingestor/rest/model/package-use.html | 2 +- .../proseo/ingestor/rest/package-summary.html | 2 +- .../proseo/ingestor/rest/package-tree.html | 2 +- .../dlr/proseo/ingestor/rest/package-use.html | 2 +- docs/proseo-ingestor/apidocs/help-doc.html | 2 +- docs/proseo-ingestor/apidocs/index-all.html | 2 +- docs/proseo-ingestor/apidocs/index.html | 4 +- docs/proseo-ingestor/apidocs/options | 4 +- .../apidocs/overview-summary.html | 2 +- .../apidocs/overview-tree.html | 2 +- docs/proseo-ingestor/apidocs/search.html | 2 +- .../apidocs/serialized-form.html | 2 +- docs/proseo-ingestor/index.html | 2 +- docs/proseo-ingestor/project-info.html | 2 +- docs/proseo-ingestor/project-reports.html | 2 +- docs/proseo-ingestor/scm.html | 2 +- docs/proseo-ingestor/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../interfaces/rest/DummyController.html | 2 +- .../rest/class-use/DummyController.html | 2 +- .../rest/model/IngestorProduct.html | 2 +- .../interfaces/rest/model/Joborder.html | 2 +- .../proseo/interfaces/rest/model/Orbit.html | 2 +- .../proseo/interfaces/rest/model/Posix.html | 2 +- .../rest/model/RestClassOutputParameter.html | 2 +- .../rest/model/RestConfiguredProcessor.html | 2 +- .../rest/model/RestDownloadHistory.html | 2 +- .../interfaces/rest/model/RestFileInfo.html | 2 +- .../interfaces/rest/model/RestHealth.html | 2 +- .../interfaces/rest/model/RestInfo.html | 2 +- .../rest/model/RestInputFilter.html | 2 +- .../rest/model/RestInputReference.html | 2 +- .../rest/model/RestInterfaceStatus.html | 2 +- .../interfaces/rest/model/RestJoborder.html | 2 +- .../interfaces/rest/model/RestMessage.html | 2 +- .../rest/model/RestNotificationEndpoint.html | 2 +- .../interfaces/rest/model/RestOrbitQuery.html | 2 +- .../interfaces/rest/model/RestOrder.html | 2 +- .../interfaces/rest/model/RestParameter.html | 2 +- .../interfaces/rest/model/RestProduct.html | 2 +- .../interfaces/rest/model/RestProductFS.html | 2 +- .../rest/model/RestProductFile.html | 2 +- .../interfaces/rest/model/RestStorage.html | 2 +- .../dlr/proseo/interfaces/rest/model/S3.html | 2 +- .../rest/model/SelectionRuleString.html | 2 +- .../rest/model/class-use/IngestorProduct.html | 2 +- .../rest/model/class-use/Joborder.html | 2 +- .../rest/model/class-use/Orbit.html | 2 +- .../rest/model/class-use/Posix.html | 2 +- .../class-use/RestClassOutputParameter.html | 2 +- .../class-use/RestConfiguredProcessor.html | 2 +- .../model/class-use/RestDownloadHistory.html | 2 +- .../rest/model/class-use/RestFileInfo.html | 2 +- .../rest/model/class-use/RestHealth.html | 2 +- .../rest/model/class-use/RestInfo.html | 2 +- .../rest/model/class-use/RestInputFilter.html | 2 +- .../model/class-use/RestInputReference.html | 2 +- .../model/class-use/RestInterfaceStatus.html | 2 +- .../rest/model/class-use/RestJoborder.html | 2 +- .../rest/model/class-use/RestMessage.html | 2 +- .../class-use/RestNotificationEndpoint.html | 2 +- .../rest/model/class-use/RestOrbitQuery.html | 2 +- .../rest/model/class-use/RestOrder.html | 2 +- .../rest/model/class-use/RestParameter.html | 2 +- .../rest/model/class-use/RestProduct.html | 2 +- .../rest/model/class-use/RestProductFS.html | 2 +- .../rest/model/class-use/RestProductFile.html | 2 +- .../rest/model/class-use/RestStorage.html | 2 +- .../interfaces/rest/model/class-use/S3.html | 2 +- .../model/class-use/SelectionRuleString.html | 2 +- .../rest/model/package-summary.html | 2 +- .../interfaces/rest/model/package-tree.html | 2 +- .../interfaces/rest/model/package-use.html | 2 +- .../interfaces/rest/package-summary.html | 2 +- .../proseo/interfaces/rest/package-tree.html | 2 +- .../proseo/interfaces/rest/package-use.html | 2 +- docs/proseo-interfaces/apidocs/help-doc.html | 2 +- docs/proseo-interfaces/apidocs/index-all.html | 2 +- docs/proseo-interfaces/apidocs/index.html | 4 +- .../apidocs/overview-summary.html | 2 +- .../apidocs/overview-tree.html | 2 +- docs/proseo-interfaces/apidocs/search.html | 2 +- .../apidocs/serialized-form.html | 2 +- docs/proseo-interfaces/index.html | 2 +- docs/proseo-interfaces/project-info.html | 2 +- docs/proseo-interfaces/project-reports.html | 2 +- docs/proseo-interfaces/scm.html | 2 +- docs/proseo-interfaces/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../documentation/LoggingDocumentation.html | 2 +- .../class-use/LoggingDocumentation.html | 2 +- .../documentation/package-summary.html | 2 +- .../logging/documentation/package-tree.html | 2 +- .../logging/documentation/package-use.html | 2 +- .../dlr/proseo/logging/http/HttpPrefix.html | 2 +- .../dlr/proseo/logging/http/ProseoHttp.html | 2 +- .../logging/http/class-use/HttpPrefix.html | 2 +- .../logging/http/class-use/ProseoHttp.html | 2 +- .../proseo/logging/http/package-summary.html | 2 +- .../dlr/proseo/logging/http/package-tree.html | 2 +- .../dlr/proseo/logging/http/package-use.html | 2 +- .../proseo/logging/logger/ProseoLogger.html | 2 +- .../logger/class-use/ProseoLogger.html | 2 +- .../logging/logger/package-summary.html | 2 +- .../proseo/logging/logger/package-tree.html | 2 +- .../proseo/logging/logger/package-use.html | 2 +- .../logging/messages/AipClientMessage.html | 2 +- .../logging/messages/ApiMonitorMessage.html | 2 +- .../logging/messages/FacilityMgrMessage.html | 2 +- .../logging/messages/GeneralMessage.html | 2 +- .../logging/messages/GeotoolsMessage.html | 2 +- .../logging/messages/IngestorMessage.html | 2 +- .../proseo/logging/messages/ModelMessage.html | 2 +- .../logging/messages/MonitorMessage.html | 2 +- .../logging/messages/NotificationMessage.html | 2 +- .../proseo/logging/messages/OAuthMessage.html | 2 +- .../proseo/logging/messages/OdipMessage.html | 2 +- .../logging/messages/OrderMgrMessage.html | 2 +- .../logging/messages/PlannerMessage.html | 2 +- .../proseo/logging/messages/PripMessage.html | 2 +- .../logging/messages/ProcessorMgrMessage.html | 2 +- .../messages/ProductArchiveMgrMessage.html | 2 +- .../messages/ProductClassMgrMessage.html | 2 +- .../logging/messages/ProseoMessage.html | 2 +- .../logging/messages/StorageMgrMessage.html | 2 +- .../proseo/logging/messages/UIMessage.html | 2 +- .../logging/messages/UserMgrMessage.html | 2 +- .../messages/class-use/AipClientMessage.html | 2 +- .../messages/class-use/ApiMonitorMessage.html | 2 +- .../class-use/FacilityMgrMessage.html | 2 +- .../messages/class-use/GeneralMessage.html | 2 +- .../messages/class-use/GeotoolsMessage.html | 2 +- .../messages/class-use/IngestorMessage.html | 2 +- .../messages/class-use/ModelMessage.html | 2 +- .../messages/class-use/MonitorMessage.html | 2 +- .../class-use/NotificationMessage.html | 2 +- .../messages/class-use/OAuthMessage.html | 2 +- .../messages/class-use/OdipMessage.html | 2 +- .../messages/class-use/OrderMgrMessage.html | 2 +- .../messages/class-use/PlannerMessage.html | 2 +- .../messages/class-use/PripMessage.html | 2 +- .../class-use/ProcessorMgrMessage.html | 2 +- .../class-use/ProductArchiveMgrMessage.html | 2 +- .../class-use/ProductClassMgrMessage.html | 2 +- .../messages/class-use/ProseoMessage.html | 2 +- .../messages/class-use/StorageMgrMessage.html | 2 +- .../logging/messages/class-use/UIMessage.html | 2 +- .../messages/class-use/UserMgrMessage.html | 2 +- .../logging/messages/package-summary.html | 2 +- .../proseo/logging/messages/package-tree.html | 2 +- .../proseo/logging/messages/package-use.html | 2 +- docs/proseo-logging/apidocs/help-doc.html | 2 +- docs/proseo-logging/apidocs/index-all.html | 2 +- docs/proseo-logging/apidocs/index.html | 4 +- docs/proseo-logging/apidocs/options | 4 +- .../apidocs/overview-summary.html | 2 +- .../proseo-logging/apidocs/overview-tree.html | 2 +- docs/proseo-logging/apidocs/search.html | 2 +- docs/proseo-logging/index.html | 2 +- docs/proseo-logging/project-info.html | 2 +- docs/proseo-logging/project-reports.html | 2 +- docs/proseo-logging/scm.html | 2 +- docs/proseo-logging/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../proseo-model/apidocs/constant-values.html | 2 +- .../proseo/model/ClassOutputParameter.html | 2 +- .../de/dlr/proseo/model/Configuration.html | 2 +- .../dlr/proseo/model/ConfigurationFile.html | 2 +- .../proseo/model/ConfigurationInputFile.html | 2 +- .../dlr/proseo/model/ConfiguredProcessor.html | 2 +- .../de/dlr/proseo/model/DownloadHistory.html | 2 +- .../de/dlr/proseo/model/InputFilter.html | 2 +- .../proseo/model/InputProductReference.html | 2 +- .../de/dlr/proseo/model/Job.JobState.html | 2 +- .../apidocs/de/dlr/proseo/model/Job.html | 2 +- .../proseo/model/JobStep.JobStepState.html | 2 +- .../dlr/proseo/model/JobStep.StdLogLevel.html | 2 +- .../apidocs/de/dlr/proseo/model/JobStep.html | 2 +- .../apidocs/de/dlr/proseo/model/Mission.html | 2 +- .../de/dlr/proseo/model/MonExtService.html | 2 +- .../model/MonExtServiceStateOperation.html | 2 +- .../model/MonExtServiceStateOperationDay.html | 2 +- .../MonExtServiceStateOperationMonth.html | 2 +- .../de/dlr/proseo/model/MonOrderProgress.html | 2 +- .../de/dlr/proseo/model/MonOrderState.html | 2 +- .../proseo/model/MonProductProduction.html | 2 +- .../proseo/model/MonProductProductionDay.html | 2 +- .../model/MonProductProductionHour.html | 2 +- .../model/MonProductProductionMonth.html | 2 +- .../de/dlr/proseo/model/MonService.html | 2 +- .../de/dlr/proseo/model/MonServiceState.html | 2 +- .../model/MonServiceStateOperation.html | 2 +- .../model/MonServiceStateOperationDay.html | 2 +- .../model/MonServiceStateOperationMonth.html | 2 +- .../proseo/model/NotificationEndpoint.html | 2 +- .../apidocs/de/dlr/proseo/model/Orbit.html | 2 +- .../de/dlr/proseo/model/Parameter.html | 2 +- .../apidocs/de/dlr/proseo/model/Payload.html | 2 +- .../de/dlr/proseo/model/PersistentObject.html | 2 +- .../dlr/proseo/model/ProcessingFacility.html | 2 +- .../de/dlr/proseo/model/ProcessingOrder.html | 2 +- .../de/dlr/proseo/model/Processor.html | 2 +- .../de/dlr/proseo/model/ProcessorClass.html | 2 +- .../apidocs/de/dlr/proseo/model/Product.html | 2 +- .../de/dlr/proseo/model/ProductArchive.html | 2 +- .../de/dlr/proseo/model/ProductClass.html | 2 +- .../de/dlr/proseo/model/ProductFile.html | 2 +- .../de/dlr/proseo/model/ProductQuery.html | 2 +- .../proseo/model/SimplePolicy.DeltaTime.html | 2 +- .../proseo/model/SimplePolicy.PolicyType.html | 2 +- .../de/dlr/proseo/model/SimplePolicy.html | 2 +- .../dlr/proseo/model/SimpleSelectionRule.html | 2 +- .../de/dlr/proseo/model/Spacecraft.html | 2 +- .../apidocs/de/dlr/proseo/model/Task.html | 2 +- .../apidocs/de/dlr/proseo/model/Workflow.html | 2 +- .../WorkflowOption.WorkflowOptionType.html | 2 +- .../de/dlr/proseo/model/WorkflowOption.html | 2 +- .../model/class-use/ClassOutputParameter.html | 2 +- .../proseo/model/class-use/Configuration.html | 2 +- .../model/class-use/ConfigurationFile.html | 2 +- .../class-use/ConfigurationInputFile.html | 2 +- .../model/class-use/ConfiguredProcessor.html | 2 +- .../model/class-use/DownloadHistory.html | 2 +- .../proseo/model/class-use/InputFilter.html | 2 +- .../class-use/InputProductReference.html | 2 +- .../proseo/model/class-use/Job.JobState.html | 2 +- .../de/dlr/proseo/model/class-use/Job.html | 2 +- .../model/class-use/JobStep.JobStepState.html | 2 +- .../model/class-use/JobStep.StdLogLevel.html | 2 +- .../dlr/proseo/model/class-use/JobStep.html | 2 +- .../dlr/proseo/model/class-use/Mission.html | 2 +- .../proseo/model/class-use/MonExtService.html | 2 +- .../MonExtServiceStateOperation.html | 2 +- .../MonExtServiceStateOperationDay.html | 2 +- .../MonExtServiceStateOperationMonth.html | 2 +- .../model/class-use/MonOrderProgress.html | 2 +- .../proseo/model/class-use/MonOrderState.html | 2 +- .../model/class-use/MonProductProduction.html | 2 +- .../class-use/MonProductProductionDay.html | 2 +- .../class-use/MonProductProductionHour.html | 2 +- .../class-use/MonProductProductionMonth.html | 2 +- .../proseo/model/class-use/MonService.html | 2 +- .../model/class-use/MonServiceState.html | 2 +- .../class-use/MonServiceStateOperation.html | 2 +- .../MonServiceStateOperationDay.html | 2 +- .../MonServiceStateOperationMonth.html | 2 +- .../model/class-use/NotificationEndpoint.html | 2 +- .../de/dlr/proseo/model/class-use/Orbit.html | 2 +- .../dlr/proseo/model/class-use/Parameter.html | 2 +- .../dlr/proseo/model/class-use/Payload.html | 2 +- .../model/class-use/PersistentObject.html | 2 +- .../model/class-use/ProcessingFacility.html | 2 +- .../model/class-use/ProcessingOrder.html | 2 +- .../dlr/proseo/model/class-use/Processor.html | 2 +- .../model/class-use/ProcessorClass.html | 2 +- .../dlr/proseo/model/class-use/Product.html | 2 +- .../model/class-use/ProductArchive.html | 2 +- .../proseo/model/class-use/ProductClass.html | 2 +- .../proseo/model/class-use/ProductFile.html | 2 +- .../proseo/model/class-use/ProductQuery.html | 2 +- .../class-use/SimplePolicy.DeltaTime.html | 2 +- .../class-use/SimplePolicy.PolicyType.html | 2 +- .../proseo/model/class-use/SimplePolicy.html | 2 +- .../model/class-use/SimpleSelectionRule.html | 2 +- .../proseo/model/class-use/Spacecraft.html | 2 +- .../de/dlr/proseo/model/class-use/Task.html | 2 +- .../dlr/proseo/model/class-use/Workflow.html | 2 +- .../WorkflowOption.WorkflowOptionType.html | 2 +- .../model/class-use/WorkflowOption.html | 2 +- .../dao/ClassOutputParameterRepository.html | 2 +- .../model/dao/ConfigurationRepository.html | 2 +- .../dao/ConfiguredProcessorRepository.html | 2 +- .../proseo/model/dao/FacilityRepository.html | 2 +- .../model/dao/InputFilterRepository.html | 2 +- .../dlr/proseo/model/dao/JobRepository.html | 2 +- .../proseo/model/dao/JobStepRepository.html | 2 +- .../proseo/model/dao/MissionRepository.html | 2 +- .../model/dao/MonExtServiceRepository.html | 2 +- ...ExtServiceStateOperationDayRepository.html | 2 +- ...tServiceStateOperationMonthRepository.html | 2 +- ...MonExtServiceStateOperationRepository.html | 2 +- .../model/dao/MonOrderStateRepository.html | 2 +- .../MonProductProductionDayRepository.html | 2 +- .../MonProductProductionHourRepository.html | 2 +- .../MonProductProductionMonthRepository.html | 2 +- .../model/dao/MonServiceRepository.html | 2 +- ...MonServiceStateOperationDayRepository.html | 2 +- ...nServiceStateOperationMonthRepository.html | 2 +- .../MonServiceStateOperationRepository.html | 2 +- .../model/dao/MonServiceStateRepository.html | 2 +- .../dlr/proseo/model/dao/OrbitRepository.html | 2 +- .../dlr/proseo/model/dao/OrderRepository.html | 2 +- .../model/dao/ProcessorClassRepository.html | 2 +- .../proseo/model/dao/ProcessorRepository.html | 2 +- .../model/dao/ProductArchiveRepository.html | 2 +- .../model/dao/ProductClassRepository.html | 2 +- .../model/dao/ProductFileRepository.html | 2 +- .../model/dao/ProductQueryRepository.html | 2 +- .../proseo/model/dao/ProductRepository.html | 2 +- .../model/dao/SpacecraftRepository.html | 2 +- .../dlr/proseo/model/dao/TaskRepository.html | 2 +- .../model/dao/WorkflowOptionRepository.html | 2 +- .../proseo/model/dao/WorkflowRepository.html | 2 +- .../ClassOutputParameterRepository.html | 2 +- .../class-use/ConfigurationRepository.html | 2 +- .../ConfiguredProcessorRepository.html | 2 +- .../dao/class-use/FacilityRepository.html | 2 +- .../dao/class-use/InputFilterRepository.html | 2 +- .../model/dao/class-use/JobRepository.html | 2 +- .../dao/class-use/JobStepRepository.html | 2 +- .../dao/class-use/MissionRepository.html | 2 +- .../class-use/MonExtServiceRepository.html | 2 +- ...ExtServiceStateOperationDayRepository.html | 2 +- ...tServiceStateOperationMonthRepository.html | 2 +- ...MonExtServiceStateOperationRepository.html | 2 +- .../class-use/MonOrderStateRepository.html | 2 +- .../MonProductProductionDayRepository.html | 2 +- .../MonProductProductionHourRepository.html | 2 +- .../MonProductProductionMonthRepository.html | 2 +- .../dao/class-use/MonServiceRepository.html | 2 +- ...MonServiceStateOperationDayRepository.html | 2 +- ...nServiceStateOperationMonthRepository.html | 2 +- .../MonServiceStateOperationRepository.html | 2 +- .../class-use/MonServiceStateRepository.html | 2 +- .../model/dao/class-use/OrbitRepository.html | 2 +- .../model/dao/class-use/OrderRepository.html | 2 +- .../class-use/ProcessorClassRepository.html | 2 +- .../dao/class-use/ProcessorRepository.html | 2 +- .../class-use/ProductArchiveRepository.html | 2 +- .../dao/class-use/ProductClassRepository.html | 2 +- .../dao/class-use/ProductFileRepository.html | 2 +- .../dao/class-use/ProductQueryRepository.html | 2 +- .../dao/class-use/ProductRepository.html | 2 +- .../dao/class-use/SpacecraftRepository.html | 2 +- .../model/dao/class-use/TaskRepository.html | 2 +- .../class-use/WorkflowOptionRepository.html | 2 +- .../dao/class-use/WorkflowRepository.html | 2 +- .../dlr/proseo/model/dao/package-summary.html | 2 +- .../de/dlr/proseo/model/dao/package-tree.html | 2 +- .../de/dlr/proseo/model/dao/package-use.html | 2 +- .../de/dlr/proseo/model/package-summary.html | 2 +- .../de/dlr/proseo/model/package-tree.html | 2 +- .../de/dlr/proseo/model/package-use.html | 2 +- .../model/service/ProductQueryService.html | 2 +- .../model/service/RepositoryService.html | 2 +- .../proseo/model/service/SecurityService.html | 2 +- .../class-use/ProductQueryService.html | 2 +- .../service/class-use/RepositoryService.html | 2 +- .../service/class-use/SecurityService.html | 2 +- .../proseo/model/service/package-summary.html | 2 +- .../proseo/model/service/package-tree.html | 2 +- .../dlr/proseo/model/service/package-use.html | 2 +- .../de/dlr/proseo/model/util/OrderUtil.html | 2 +- .../dlr/proseo/model/util/SelectionItem.html | 2 +- .../dlr/proseo/model/util/SelectionRule.html | 2 +- .../model/util/class-use/OrderUtil.html | 2 +- .../model/util/class-use/SelectionItem.html | 2 +- .../model/util/class-use/SelectionRule.html | 2 +- .../proseo/model/util/package-summary.html | 2 +- .../dlr/proseo/model/util/package-tree.html | 2 +- .../de/dlr/proseo/model/util/package-use.html | 2 +- .../proseo-model/apidocs/deprecated-list.html | 2 +- docs/proseo-model/apidocs/help-doc.html | 2 +- docs/proseo-model/apidocs/index-all.html | 2 +- docs/proseo-model/apidocs/index.html | 4 +- docs/proseo-model/apidocs/options | 4 +- .../apidocs/overview-summary.html | 2 +- docs/proseo-model/apidocs/overview-tree.html | 2 +- docs/proseo-model/apidocs/search.html | 2 +- docs/proseo-model/index.html | 2 +- docs/proseo-model/project-info.html | 2 +- docs/proseo-model/project-reports.html | 2 +- docs/proseo-model/scm.html | 2 +- docs/proseo-model/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../proseo/monitor/MonitorApplication.html | 2 +- .../monitor/MonitorConfiguration.Docker.html | 2 +- .../monitor/MonitorConfiguration.Service.html | 2 +- .../proseo/monitor/MonitorConfiguration.html | 2 +- .../proseo/monitor/MonitorSecurityConfig.html | 2 +- .../monitor/class-use/MonitorApplication.html | 2 +- .../MonitorConfiguration.Docker.html | 2 +- .../MonitorConfiguration.Service.html | 2 +- .../class-use/MonitorConfiguration.html | 2 +- .../class-use/MonitorSecurityConfig.html | 2 +- .../monitor/microservice/DockerService.html | 2 +- .../monitor/microservice/MicroService.html | 2 +- .../microservice/MonServiceAggregation.html | 2 +- .../monitor/microservice/MonitorServices.html | 2 +- .../microservice/class-use/DockerService.html | 2 +- .../microservice/class-use/MicroService.html | 2 +- .../class-use/MonServiceAggregation.html | 2 +- .../class-use/MonitorServices.html | 2 +- .../monitor/microservice/package-summary.html | 2 +- .../monitor/microservice/package-tree.html | 2 +- .../monitor/microservice/package-use.html | 2 +- .../proseo/monitor/order/MonitorOrders.html | 2 +- .../order/class-use/MonitorOrders.html | 2 +- .../proseo/monitor/order/package-summary.html | 2 +- .../proseo/monitor/order/package-tree.html | 2 +- .../dlr/proseo/monitor/order/package-use.html | 2 +- .../dlr/proseo/monitor/package-summary.html | 2 +- .../de/dlr/proseo/monitor/package-tree.html | 2 +- .../de/dlr/proseo/monitor/package-use.html | 2 +- .../monitor/product/MonitorProducts.html | 2 +- .../product/class-use/MonitorProducts.html | 2 +- .../monitor/product/package-summary.html | 2 +- .../proseo/monitor/product/package-tree.html | 2 +- .../proseo/monitor/product/package-use.html | 2 +- .../proseo/monitor/rest/InfoController.html | 2 +- .../monitor/rest/InfoControllerDecorator.html | 2 +- .../monitor/rest/InfoControllerImpl.html | 2 +- .../rest/class-use/InfoController.html | 2 +- .../class-use/InfoControllerDecorator.html | 2 +- .../rest/class-use/InfoControllerImpl.html | 2 +- .../proseo/monitor/rest/package-summary.html | 2 +- .../dlr/proseo/monitor/rest/package-tree.html | 2 +- .../dlr/proseo/monitor/rest/package-use.html | 2 +- docs/proseo-monitor/apidocs/help-doc.html | 2 +- docs/proseo-monitor/apidocs/index-all.html | 2 +- docs/proseo-monitor/apidocs/index.html | 4 +- docs/proseo-monitor/apidocs/options | 4 +- .../apidocs/overview-summary.html | 2 +- .../proseo-monitor/apidocs/overview-tree.html | 2 +- docs/proseo-monitor/apidocs/search.html | 2 +- docs/proseo-monitor/index.html | 2 +- docs/proseo-monitor/project-info.html | 2 +- docs/proseo-monitor/project-reports.html | 2 +- docs/proseo-monitor/scm.html | 2 +- docs/proseo-monitor/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../NotificationConfiguration.html | 2 +- .../proseo/notification/NotificationHttp.html | 2 +- .../NotificationSecurityConfig.html | 2 +- .../notification/NotificationService.html | 2 +- .../class-use/NotificationConfiguration.html | 2 +- .../class-use/NotificationHttp.html | 2 +- .../class-use/NotificationSecurityConfig.html | 2 +- .../class-use/NotificationService.html | 2 +- .../proseo/notification/package-summary.html | 2 +- .../dlr/proseo/notification/package-tree.html | 2 +- .../dlr/proseo/notification/package-use.html | 2 +- .../notification/rest/NotifyController.html | 2 +- .../rest/NotifyControllerDecorator.html | 2 +- .../rest/NotifyControllerImpl.html | 2 +- .../proseo/notification/rest/SendManager.html | 2 +- .../rest/class-use/NotifyController.html | 2 +- .../class-use/NotifyControllerDecorator.html | 2 +- .../rest/class-use/NotifyControllerImpl.html | 2 +- .../rest/class-use/SendManager.html | 2 +- .../notification/rest/model/RestMessage.html | 2 +- .../rest/model/class-use/RestMessage.html | 2 +- .../rest/model/package-summary.html | 2 +- .../notification/rest/model/package-tree.html | 2 +- .../notification/rest/model/package-use.html | 2 +- .../notification/rest/package-summary.html | 2 +- .../notification/rest/package-tree.html | 2 +- .../proseo/notification/rest/package-use.html | 2 +- .../service/ServiceConnection.html | 2 +- .../notification/service/ServiceMail.html | 2 +- .../service/class-use/ServiceConnection.html | 2 +- .../service/class-use/ServiceMail.html | 2 +- .../notification/service/package-summary.html | 2 +- .../notification/service/package-tree.html | 2 +- .../notification/service/package-use.html | 2 +- .../proseo-notification/apidocs/help-doc.html | 2 +- .../apidocs/index-all.html | 2 +- docs/proseo-notification/apidocs/index.html | 4 +- docs/proseo-notification/apidocs/options | 4 +- .../apidocs/overview-summary.html | 2 +- .../apidocs/overview-tree.html | 2 +- docs/proseo-notification/apidocs/search.html | 2 +- .../apidocs/serialized-form.html | 2 +- docs/proseo-notification/index.html | 2 +- docs/proseo-notification/project-info.html | 2 +- docs/proseo-notification/project-reports.html | 2 +- docs/proseo-notification/scm.html | 2 +- docs/proseo-notification/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../proseo/model/rest/MissionController.html | 2 +- .../rest/MissionControllerDecorator.html | 2 +- .../proseo/model/rest/OrbitController.html | 2 +- .../model/rest/OrbitControllerDecorator.html | 2 +- .../proseo/model/rest/OrderController.html | 2 +- .../model/rest/OrderControllerDecorator.html | 2 +- .../proseo/model/rest/OrderjobController.html | 2 +- .../rest/OrderjobControllerDecorator.html | 2 +- .../model/rest/OrderjobstepController.html | 2 +- .../rest/OrderjobstepControllerDecorator.html | 2 +- .../rest/class-use/MissionController.html | 2 +- .../class-use/MissionControllerDecorator.html | 2 +- .../model/rest/class-use/OrbitController.html | 2 +- .../class-use/OrbitControllerDecorator.html | 2 +- .../model/rest/class-use/OrderController.html | 2 +- .../class-use/OrderControllerDecorator.html | 2 +- .../rest/class-use/OrderjobController.html | 2 +- .../OrderjobControllerDecorator.html | 2 +- .../class-use/OrderjobstepController.html | 2 +- .../OrderjobstepControllerDecorator.html | 2 +- .../dlr/proseo/model/rest/model/JobState.html | 2 +- .../proseo/model/rest/model/JobStepState.html | 2 +- .../rest/model/RestClassOutputParameter.html | 2 +- .../model/rest/model/RestInputFilter.html | 2 +- .../model/rest/model/RestInputReference.html | 2 +- .../dlr/proseo/model/rest/model/RestJob.html | 2 +- .../proseo/model/rest/model/RestJobStep.html | 2 +- .../proseo/model/rest/model/RestMission.html | 2 +- .../rest/model/RestNotificationEndpoint.html | 2 +- .../proseo/model/rest/model/RestOrbit.html | 2 +- .../model/rest/model/RestOrbitQuery.html | 2 +- .../proseo/model/rest/model/RestOrder.html | 2 +- .../model/rest/model/RestParameter.html | 2 +- .../proseo/model/rest/model/RestPayload.html | 2 +- .../model/rest/model/RestSpacecraft.html | 2 +- .../dlr/proseo/model/rest/model/Status.html | 2 +- .../model/rest/model/StderrLogLevel.html | 2 +- .../model/rest/model/StdoutLogLevel.html | 2 +- .../model/rest/model/class-use/JobState.html | 2 +- .../rest/model/class-use/JobStepState.html | 2 +- .../class-use/RestClassOutputParameter.html | 2 +- .../rest/model/class-use/RestInputFilter.html | 2 +- .../model/class-use/RestInputReference.html | 2 +- .../model/rest/model/class-use/RestJob.html | 2 +- .../rest/model/class-use/RestJobStep.html | 2 +- .../rest/model/class-use/RestMission.html | 2 +- .../class-use/RestNotificationEndpoint.html | 2 +- .../model/rest/model/class-use/RestOrbit.html | 2 +- .../rest/model/class-use/RestOrbitQuery.html | 2 +- .../model/rest/model/class-use/RestOrder.html | 2 +- .../rest/model/class-use/RestParameter.html | 2 +- .../rest/model/class-use/RestPayload.html | 2 +- .../rest/model/class-use/RestSpacecraft.html | 2 +- .../model/rest/model/class-use/Status.html | 2 +- .../rest/model/class-use/StderrLogLevel.html | 2 +- .../rest/model/class-use/StdoutLogLevel.html | 2 +- .../model/rest/model/package-summary.html | 2 +- .../proseo/model/rest/model/package-tree.html | 2 +- .../proseo/model/rest/model/package-use.html | 2 +- .../proseo/model/rest/package-summary.html | 2 +- .../dlr/proseo/model/rest/package-tree.html | 2 +- .../de/dlr/proseo/model/rest/package-use.html | 2 +- .../de/dlr/proseo/ordermgr/OrderManager.html | 2 +- .../ordermgr/OrdermgrConfiguration.html | 2 +- .../ordermgr/OrdermgrSecurityConfig.html | 2 +- .../ordermgr/class-use/OrderManager.html | 2 +- .../class-use/OrdermgrConfiguration.html | 2 +- .../class-use/OrdermgrSecurityConfig.html | 2 +- .../ordermgr/cleanup/CleanupOrdersThread.html | 2 +- .../class-use/CleanupOrdersThread.html | 2 +- .../ordermgr/cleanup/package-summary.html | 2 +- .../proseo/ordermgr/cleanup/package-tree.html | 2 +- .../proseo/ordermgr/cleanup/package-use.html | 2 +- .../dlr/proseo/ordermgr/package-summary.html | 2 +- .../de/dlr/proseo/ordermgr/package-tree.html | 2 +- .../de/dlr/proseo/ordermgr/package-use.html | 2 +- .../ordermgr/rest/MissionControllerImpl.html | 2 +- .../ordermgr/rest/OrbitControllerImpl.html | 2 +- .../ordermgr/rest/OrderControllerImpl.html | 2 +- .../ordermgr/rest/OrderjobControllerImpl.html | 2 +- .../rest/OrderjobstepControllerImpl.html | 2 +- .../ordermgr/rest/ProcessingOrderMgr.html | 2 +- .../rest/class-use/MissionControllerImpl.html | 2 +- .../rest/class-use/OrbitControllerImpl.html | 2 +- .../rest/class-use/OrderControllerImpl.html | 2 +- .../class-use/OrderjobControllerImpl.html | 2 +- .../class-use/OrderjobstepControllerImpl.html | 2 +- .../rest/class-use/ProcessingOrderMgr.html | 2 +- .../ordermgr/rest/model/MissionUtil.html | 2 +- .../proseo/ordermgr/rest/model/OrbitUtil.html | 2 +- .../proseo/ordermgr/rest/model/RestUtil.html | 2 +- .../rest/model/class-use/MissionUtil.html | 2 +- .../rest/model/class-use/OrbitUtil.html | 2 +- .../rest/model/class-use/RestUtil.html | 2 +- .../ordermgr/rest/model/package-summary.html | 2 +- .../ordermgr/rest/model/package-tree.html | 2 +- .../ordermgr/rest/model/package-use.html | 2 +- .../proseo/ordermgr/rest/package-summary.html | 2 +- .../proseo/ordermgr/rest/package-tree.html | 2 +- .../dlr/proseo/ordermgr/rest/package-use.html | 2 +- .../apidocs/deprecated-list.html | 2 +- docs/proseo-order-mgr/apidocs/help-doc.html | 2 +- docs/proseo-order-mgr/apidocs/index-all.html | 2 +- docs/proseo-order-mgr/apidocs/index.html | 4 +- docs/proseo-order-mgr/apidocs/options | 4 +- .../apidocs/overview-summary.html | 2 +- .../apidocs/overview-tree.html | 2 +- docs/proseo-order-mgr/apidocs/search.html | 2 +- .../apidocs/serialized-form.html | 2 +- docs/proseo-order-mgr/index.html | 2 +- docs/proseo-order-mgr/project-info.html | 2 +- docs/proseo-order-mgr/project-reports.html | 2 +- docs/proseo-order-mgr/scm.html | 2 +- docs/proseo-order-mgr/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../proseo/model/rest/AcquireController.html | 2 +- .../rest/AcquireControllerDecorator.html | 2 +- .../dlr/proseo/model/rest/JobController.html | 2 +- .../model/rest/JobControllerDecorator.html | 2 +- .../proseo/model/rest/JobstepController.html | 2 +- .../rest/JobstepControllerDecorator.html | 2 +- .../proseo/model/rest/OrderController.html | 2 +- .../model/rest/OrderControllerDecorator.html | 2 +- .../rest/ProcessingfacilityController.html | 2 +- ...ProcessingfacilityControllerDecorator.html | 2 +- .../proseo/model/rest/ProductController.html | 2 +- .../rest/ProductControllerDecorator.html | 2 +- .../proseo/model/rest/ReleaseController.html | 2 +- .../rest/ReleaseControllerDecorator.html | 2 +- .../rest/class-use/AcquireController.html | 2 +- .../class-use/AcquireControllerDecorator.html | 2 +- .../model/rest/class-use/JobController.html | 2 +- .../class-use/JobControllerDecorator.html | 2 +- .../rest/class-use/JobstepController.html | 2 +- .../class-use/JobstepControllerDecorator.html | 2 +- .../model/rest/class-use/OrderController.html | 2 +- .../class-use/OrderControllerDecorator.html | 2 +- .../ProcessingfacilityController.html | 2 +- ...ProcessingfacilityControllerDecorator.html | 2 +- .../rest/class-use/ProductController.html | 2 +- .../class-use/ProductControllerDecorator.html | 2 +- .../rest/class-use/ReleaseController.html | 2 +- .../class-use/ReleaseControllerDecorator.html | 2 +- .../dlr/proseo/model/rest/model/JobState.html | 2 +- .../proseo/model/rest/model/JobStepState.html | 2 +- .../proseo/model/rest/model/PlannerPod.html | 2 +- .../rest/model/RestClassOutputParameter.html | 2 +- .../dlr/proseo/model/rest/model/RestId.html | 2 +- .../model/rest/model/RestInputFilter.html | 2 +- .../model/rest/model/RestInputReference.html | 2 +- .../dlr/proseo/model/rest/model/RestJob.html | 2 +- .../proseo/model/rest/model/RestJobGraph.html | 2 +- .../proseo/model/rest/model/RestJobStep.html | 2 +- .../model/rest/model/RestJobStepGraph.html | 2 +- .../rest/model/RestNotificationEndpoint.html | 2 +- .../proseo/model/rest/model/RestOrbit.html | 2 +- .../model/rest/model/RestOrbitQuery.html | 2 +- .../proseo/model/rest/model/RestOrder.html | 2 +- .../model/rest/model/RestParameter.html | 2 +- .../rest/model/RestProcessingFacility.html | 2 +- .../dlr/proseo/model/rest/model/Status.html | 2 +- .../model/rest/model/StderrLogLevel.html | 2 +- .../model/rest/model/StdoutLogLevel.html | 2 +- .../model/rest/model/class-use/JobState.html | 2 +- .../rest/model/class-use/JobStepState.html | 2 +- .../rest/model/class-use/PlannerPod.html | 2 +- .../class-use/RestClassOutputParameter.html | 2 +- .../model/rest/model/class-use/RestId.html | 2 +- .../rest/model/class-use/RestInputFilter.html | 2 +- .../model/class-use/RestInputReference.html | 2 +- .../model/rest/model/class-use/RestJob.html | 2 +- .../rest/model/class-use/RestJobGraph.html | 2 +- .../rest/model/class-use/RestJobStep.html | 2 +- .../model/class-use/RestJobStepGraph.html | 2 +- .../class-use/RestNotificationEndpoint.html | 2 +- .../model/rest/model/class-use/RestOrbit.html | 2 +- .../rest/model/class-use/RestOrbitQuery.html | 2 +- .../model/rest/model/class-use/RestOrder.html | 2 +- .../rest/model/class-use/RestParameter.html | 2 +- .../class-use/RestProcessingFacility.html | 2 +- .../model/rest/model/class-use/Status.html | 2 +- .../rest/model/class-use/StderrLogLevel.html | 2 +- .../rest/model/class-use/StdoutLogLevel.html | 2 +- .../model/rest/model/package-summary.html | 2 +- .../proseo/model/rest/model/package-tree.html | 2 +- .../proseo/model/rest/model/package-use.html | 2 +- .../proseo/model/rest/package-summary.html | 2 +- .../dlr/proseo/model/rest/package-tree.html | 2 +- .../de/dlr/proseo/model/rest/package-use.html | 2 +- .../de/dlr/proseo/planner/JobStepSort.html | 2 +- .../proseo/planner/PlannerResultMessage.html | 2 +- .../dlr/proseo/planner/ProductionPlanner.html | 2 +- .../ProductionPlannerConfiguration.html | 2 +- .../ProductionPlannerSecurityConfig.html | 2 +- .../proseo/planner/class-use/JobStepSort.html | 2 +- .../class-use/PlannerResultMessage.html | 2 +- .../planner/class-use/ProductionPlanner.html | 2 +- .../ProductionPlannerConfiguration.html | 2 +- .../ProductionPlannerSecurityConfig.html | 2 +- .../planner/dispatcher/JobDispatcher.html | 2 +- .../planner/dispatcher/KubeDispatcher.html | 2 +- .../planner/dispatcher/OrderDispatcher.html | 2 +- .../dispatcher/class-use/JobDispatcher.html | 2 +- .../dispatcher/class-use/KubeDispatcher.html | 2 +- .../dispatcher/class-use/OrderDispatcher.html | 2 +- .../planner/dispatcher/package-summary.html | 2 +- .../planner/dispatcher/package-tree.html | 2 +- .../planner/dispatcher/package-use.html | 2 +- .../proseo/planner/kubernetes/KubeConfig.html | 2 +- .../proseo/planner/kubernetes/KubeJob.html | 2 +- .../planner/kubernetes/KubeJobFinish.html | 2 +- .../kubernetes/class-use/KubeConfig.html | 2 +- .../planner/kubernetes/class-use/KubeJob.html | 2 +- .../kubernetes/class-use/KubeJobFinish.html | 2 +- .../planner/kubernetes/package-summary.html | 2 +- .../planner/kubernetes/package-tree.html | 2 +- .../planner/kubernetes/package-use.html | 2 +- .../dlr/proseo/planner/package-summary.html | 2 +- .../de/dlr/proseo/planner/package-tree.html | 2 +- .../de/dlr/proseo/planner/package-use.html | 2 +- .../planner/rest/AcquireControllerImpl.html | 2 +- .../planner/rest/JobControllerImpl.html | 2 +- .../planner/rest/JobstepControllerImpl.html | 2 +- .../planner/rest/OrderControllerImpl.html | 2 +- .../ProcessingfacilityControllerImpl.html | 2 +- .../planner/rest/ProductControllerImpl.html | 2 +- .../planner/rest/ReleaseControllerImpl.html | 2 +- .../rest/class-use/AcquireControllerImpl.html | 2 +- .../rest/class-use/JobControllerImpl.html | 2 +- .../rest/class-use/JobstepControllerImpl.html | 2 +- .../rest/class-use/OrderControllerImpl.html | 2 +- .../ProcessingfacilityControllerImpl.html | 2 +- .../rest/class-use/ProductControllerImpl.html | 2 +- .../rest/class-use/ReleaseControllerImpl.html | 2 +- .../proseo/planner/rest/model/PodKube.html | 2 +- .../proseo/planner/rest/model/RestUtil.html | 2 +- .../planner/rest/model/class-use/PodKube.html | 2 +- .../rest/model/class-use/RestUtil.html | 2 +- .../planner/rest/model/package-summary.html | 2 +- .../planner/rest/model/package-tree.html | 2 +- .../planner/rest/model/package-use.html | 2 +- .../proseo/planner/rest/package-summary.html | 2 +- .../dlr/proseo/planner/rest/package-tree.html | 2 +- .../dlr/proseo/planner/rest/package-use.html | 2 +- .../planner/service/ServiceConnection.html | 2 +- .../service/class-use/ServiceConnection.html | 2 +- .../planner/service/package-summary.html | 2 +- .../proseo/planner/service/package-tree.html | 2 +- .../proseo/planner/service/package-use.html | 2 +- .../dlr/proseo/planner/util/JobStepUtil.html | 2 +- .../de/dlr/proseo/planner/util/JobUtil.html | 2 +- .../proseo/planner/util/OrderPlanThread.html | 2 +- .../planner/util/OrderReleaseThread.html | 2 +- .../de/dlr/proseo/planner/util/OrderUtil.html | 2 +- .../dlr/proseo/planner/util/UtilService.html | 2 +- .../planner/util/class-use/JobStepUtil.html | 2 +- .../planner/util/class-use/JobUtil.html | 2 +- .../util/class-use/OrderPlanThread.html | 2 +- .../util/class-use/OrderReleaseThread.html | 2 +- .../planner/util/class-use/OrderUtil.html | 2 +- .../planner/util/class-use/UtilService.html | 2 +- .../proseo/planner/util/package-summary.html | 2 +- .../dlr/proseo/planner/util/package-tree.html | 2 +- .../dlr/proseo/planner/util/package-use.html | 2 +- .../apidocs/deprecated-list.html | 2 +- docs/proseo-planner/apidocs/help-doc.html | 2 +- docs/proseo-planner/apidocs/index-all.html | 2 +- docs/proseo-planner/apidocs/index.html | 4 +- docs/proseo-planner/apidocs/options | 4 +- .../apidocs/overview-summary.html | 2 +- .../proseo-planner/apidocs/overview-tree.html | 2 +- docs/proseo-planner/apidocs/search.html | 2 +- .../apidocs/serialized-form.html | 2 +- docs/proseo-planner/index.html | 2 +- docs/proseo-planner/project-info.html | 2 +- docs/proseo-planner/project-reports.html | 2 +- docs/proseo-planner/scm.html | 2 +- docs/proseo-planner/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../procmgr/ProcessorManagerApplication.html | 2 +- .../ProcessorManagerConfiguration.html | 2 +- .../ProcessorManagerSecurityConfig.html | 2 +- .../ProcessorManagerApplication.html | 2 +- .../ProcessorManagerConfiguration.html | 2 +- .../ProcessorManagerSecurityConfig.html | 2 +- .../dlr/proseo/procmgr/package-summary.html | 2 +- .../de/dlr/proseo/procmgr/package-tree.html | 2 +- .../de/dlr/proseo/procmgr/package-use.html | 2 +- .../procmgr/rest/ConfigurationController.html | 2 +- .../ConfigurationControllerDecorator.html | 2 +- .../rest/ConfigurationControllerImpl.html | 2 +- .../procmgr/rest/ConfigurationManager.html | 2 +- .../ConfiguredProcessorControllerImpl.html | 2 +- .../rest/ConfiguredProcessorManager.html | 2 +- .../rest/ConfiguredprocessorController.html | 2 +- ...onfiguredprocessorControllerDecorator.html | 2 +- .../rest/ProcessorClassControllerImpl.html | 2 +- .../procmgr/rest/ProcessorClassManager.html | 2 +- .../procmgr/rest/ProcessorController.html | 2 +- .../rest/ProcessorControllerDecorator.html | 2 +- .../procmgr/rest/ProcessorControllerImpl.html | 2 +- .../proseo/procmgr/rest/ProcessorManager.html | 2 +- .../rest/ProcessorclassController.html | 2 +- .../ProcessorclassControllerDecorator.html | 2 +- .../procmgr/rest/WorkflowController.html | 2 +- .../rest/WorkflowControllerDecorator.html | 2 +- .../procmgr/rest/WorkflowControllerImpl.html | 2 +- .../dlr/proseo/procmgr/rest/WorkflowMgr.html | 2 +- .../class-use/ConfigurationController.html | 2 +- .../ConfigurationControllerDecorator.html | 2 +- .../ConfigurationControllerImpl.html | 2 +- .../rest/class-use/ConfigurationManager.html | 2 +- .../ConfiguredProcessorControllerImpl.html | 2 +- .../class-use/ConfiguredProcessorManager.html | 2 +- .../ConfiguredprocessorController.html | 2 +- ...onfiguredprocessorControllerDecorator.html | 2 +- .../ProcessorClassControllerImpl.html | 2 +- .../rest/class-use/ProcessorClassManager.html | 2 +- .../rest/class-use/ProcessorController.html | 2 +- .../ProcessorControllerDecorator.html | 2 +- .../class-use/ProcessorControllerImpl.html | 2 +- .../rest/class-use/ProcessorManager.html | 2 +- .../class-use/ProcessorclassController.html | 2 +- .../ProcessorclassControllerDecorator.html | 2 +- .../rest/class-use/WorkflowController.html | 2 +- .../WorkflowControllerDecorator.html | 2 +- .../class-use/WorkflowControllerImpl.html | 2 +- .../procmgr/rest/class-use/WorkflowMgr.html | 2 +- .../procmgr/rest/model/ConfigurationUtil.html | 2 +- .../rest/model/ConfiguredProcessorUtil.html | 2 +- .../rest/model/ProcessorClassUtil.html | 2 +- .../procmgr/rest/model/ProcessorUtil.html | 2 +- .../rest/model/RestClassOutputParameter.html | 2 +- .../procmgr/rest/model/RestConfiguration.html | 2 +- .../rest/model/RestConfigurationFile.html | 2 +- .../model/RestConfigurationInputFile.html | 2 +- .../rest/model/RestConfiguredProcessor.html | 2 +- .../procmgr/rest/model/RestInputFilter.html | 2 +- .../procmgr/rest/model/RestParameter.html | 2 +- .../procmgr/rest/model/RestProcessor.html | 2 +- .../rest/model/RestProcessorClass.html | 2 +- .../rest/model/RestStringParameter.html | 2 +- .../proseo/procmgr/rest/model/RestTask.html | 2 +- .../procmgr/rest/model/RestWorkflow.html | 2 +- .../rest/model/RestWorkflowOption.html | 2 +- .../proseo/procmgr/rest/model/TaskUtil.html | 2 +- .../procmgr/rest/model/WorkflowUtil.html | 2 +- .../model/class-use/ConfigurationUtil.html | 2 +- .../class-use/ConfiguredProcessorUtil.html | 2 +- .../model/class-use/ProcessorClassUtil.html | 2 +- .../rest/model/class-use/ProcessorUtil.html | 2 +- .../class-use/RestClassOutputParameter.html | 2 +- .../model/class-use/RestConfiguration.html | 2 +- .../class-use/RestConfigurationFile.html | 2 +- .../class-use/RestConfigurationInputFile.html | 2 +- .../class-use/RestConfiguredProcessor.html | 2 +- .../rest/model/class-use/RestInputFilter.html | 2 +- .../rest/model/class-use/RestParameter.html | 2 +- .../rest/model/class-use/RestProcessor.html | 2 +- .../model/class-use/RestProcessorClass.html | 2 +- .../model/class-use/RestStringParameter.html | 2 +- .../rest/model/class-use/RestTask.html | 2 +- .../rest/model/class-use/RestWorkflow.html | 2 +- .../model/class-use/RestWorkflowOption.html | 2 +- .../rest/model/class-use/TaskUtil.html | 2 +- .../rest/model/class-use/WorkflowUtil.html | 2 +- .../procmgr/rest/model/package-summary.html | 2 +- .../procmgr/rest/model/package-tree.html | 2 +- .../procmgr/rest/model/package-use.html | 2 +- .../proseo/procmgr/rest/package-summary.html | 2 +- .../dlr/proseo/procmgr/rest/package-tree.html | 2 +- .../dlr/proseo/procmgr/rest/package-use.html | 2 +- .../apidocs/help-doc.html | 2 +- .../apidocs/index-all.html | 2 +- docs/proseo-processor-mgr/apidocs/index.html | 4 +- docs/proseo-processor-mgr/apidocs/options | 4 +- .../apidocs/overview-summary.html | 2 +- .../apidocs/overview-tree.html | 2 +- docs/proseo-processor-mgr/apidocs/search.html | 2 +- .../apidocs/serialized-form.html | 2 +- docs/proseo-processor-mgr/index.html | 2 +- docs/proseo-processor-mgr/project-info.html | 2 +- .../proseo-processor-mgr/project-reports.html | 2 +- docs/proseo-processor-mgr/scm.html | 2 +- docs/proseo-processor-mgr/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../prodclmgr/ProductClassConfiguration.html | 2 +- .../ProductClassManagerApplication.html | 2 +- .../prodclmgr/ProductClassSecurityConfig.html | 2 +- .../class-use/ProductClassConfiguration.html | 2 +- .../ProductClassManagerApplication.html | 2 +- .../class-use/ProductClassSecurityConfig.html | 2 +- .../dlr/proseo/prodclmgr/package-summary.html | 2 +- .../de/dlr/proseo/prodclmgr/package-tree.html | 2 +- .../de/dlr/proseo/prodclmgr/package-use.html | 2 +- .../rest/ProductClassControllerImpl.html | 2 +- .../prodclmgr/rest/ProductClassManager.html | 2 +- .../rest/ProductclassController.html | 2 +- .../rest/ProductclassControllerDecorator.html | 2 +- .../class-use/ProductClassControllerImpl.html | 2 +- .../rest/class-use/ProductClassManager.html | 2 +- .../class-use/ProductclassController.html | 2 +- .../ProductclassControllerDecorator.html | 2 +- .../prodclmgr/rest/model/DeltaTimeT0.html | 2 +- .../prodclmgr/rest/model/DeltaTimeT1.html | 2 +- .../rest/model/ProductClassUtil.html | 2 +- .../prodclmgr/rest/model/RestParameter.html | 2 +- .../rest/model/RestProductClass.html | 2 +- .../rest/model/RestSimplePolicy.html | 2 +- .../rest/model/RestSimpleSelectionRule.html | 2 +- .../rest/model/SelectionRuleString.html | 2 +- .../rest/model/class-use/DeltaTimeT0.html | 2 +- .../rest/model/class-use/DeltaTimeT1.html | 2 +- .../model/class-use/ProductClassUtil.html | 2 +- .../rest/model/class-use/RestParameter.html | 2 +- .../model/class-use/RestProductClass.html | 2 +- .../model/class-use/RestSimplePolicy.html | 2 +- .../class-use/RestSimpleSelectionRule.html | 2 +- .../model/class-use/SelectionRuleString.html | 2 +- .../prodclmgr/rest/model/package-summary.html | 2 +- .../prodclmgr/rest/model/package-tree.html | 2 +- .../prodclmgr/rest/model/package-use.html | 2 +- .../prodclmgr/rest/package-summary.html | 2 +- .../proseo/prodclmgr/rest/package-tree.html | 2 +- .../proseo/prodclmgr/rest/package-use.html | 2 +- .../apidocs/help-doc.html | 2 +- .../apidocs/index-all.html | 2 +- .../apidocs/index.html | 4 +- docs/proseo-productclass-mgr/apidocs/options | 4 +- .../apidocs/overview-summary.html | 2 +- .../apidocs/overview-tree.html | 2 +- .../apidocs/search.html | 2 +- .../apidocs/serialized-form.html | 2 +- docs/proseo-productclass-mgr/index.html | 2 +- .../proseo-productclass-mgr/project-info.html | 2 +- .../project-reports.html | 2 +- docs/proseo-productclass-mgr/scm.html | 2 +- docs/proseo-productclass-mgr/summary.html | 4 +- docs/proseo-samples/index.html | 2 +- docs/proseo-samples/modules.html | 2 +- docs/proseo-samples/project-info.html | 2 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../proseo/sampleproc/SampleProcessor.html | 2 +- .../dlr/proseo/sampleproc/SampleProduct.html | 2 +- .../sampleproc/class-use/SampleProcessor.html | 2 +- .../sampleproc/class-use/SampleProduct.html | 2 +- .../proseo/sampleproc/package-summary.html | 2 +- .../dlr/proseo/sampleproc/package-tree.html | 2 +- .../de/dlr/proseo/sampleproc/package-use.html | 2 +- .../apidocs/help-doc.html | 2 +- .../apidocs/index-all.html | 2 +- .../apidocs/index.html | 2 +- .../apidocs/overview-tree.html | 2 +- .../apidocs/search.html | 2 +- .../proseo-sample-processor/index.html | 2 +- .../proseo-sample-processor/project-info.html | 2 +- .../project-reports.html | 2 +- .../proseo-sample-processor/scm.html | 2 +- .../proseo-sample-processor/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../dlr/proseo/samplewrap/SampleWrapper.html | 2 +- .../samplewrap/class-use/SampleWrapper.html | 2 +- .../proseo/samplewrap/package-summary.html | 2 +- .../dlr/proseo/samplewrap/package-tree.html | 2 +- .../de/dlr/proseo/samplewrap/package-use.html | 2 +- .../apidocs/help-doc.html | 2 +- .../apidocs/index-all.html | 2 +- .../proseo-sample-wrapper/apidocs/index.html | 2 +- .../apidocs/overview-tree.html | 2 +- .../proseo-sample-wrapper/apidocs/search.html | 2 +- .../proseo-sample-wrapper/index.html | 2 +- .../proseo-sample-wrapper/project-info.html | 2 +- .../project-reports.html | 2 +- .../proseo-sample-wrapper/scm.html | 2 +- .../proseo-sample-wrapper/summary.html | 4 +- docs/proseo-samples/scm.html | 2 +- docs/proseo-samples/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../dlr/proseo/storagemgr/StorageManager.html | 2 +- .../StorageManagerConfiguration.html | 2 +- .../proseo/storagemgr/cache/FileCache.html | 2 +- .../dlr/proseo/storagemgr/cache/FileInfo.html | 2 +- .../cache/MapCache.SortByAccessedAsc.html | 2 +- .../cache/MapCache.SortByAccessedDesc.html | 2 +- .../cache/MapCache.SortBySizeAsc.html | 2 +- .../cache/MapCache.SortBySizeDesc.html | 2 +- .../dlr/proseo/storagemgr/cache/MapCache.html | 2 +- .../storagemgr/cache/class-use/FileCache.html | 2 +- .../storagemgr/cache/class-use/FileInfo.html | 2 +- .../class-use/MapCache.SortByAccessedAsc.html | 2 +- .../MapCache.SortByAccessedDesc.html | 2 +- .../class-use/MapCache.SortBySizeAsc.html | 2 +- .../class-use/MapCache.SortBySizeDesc.html | 2 +- .../storagemgr/cache/class-use/MapCache.html | 2 +- .../storagemgr/cache/package-summary.html | 2 +- .../proseo/storagemgr/cache/package-tree.html | 2 +- .../proseo/storagemgr/cache/package-use.html | 2 +- .../storagemgr/class-use/StorageManager.html | 2 +- .../StorageManagerConfiguration.html | 2 +- .../de/dlr/proseo/storagemgr/fs/s3/S3Ops.html | 2 +- .../storagemgr/fs/s3/class-use/S3Ops.html | 2 +- .../storagemgr/fs/s3/package-summary.html | 2 +- .../proseo/storagemgr/fs/s3/package-tree.html | 2 +- .../proseo/storagemgr/fs/s3/package-use.html | 2 +- .../proseo/storagemgr/package-summary.html | 2 +- .../dlr/proseo/storagemgr/package-tree.html | 2 +- .../de/dlr/proseo/storagemgr/package-use.html | 2 +- .../storagemgr/rest/InfoController.html | 2 +- .../rest/InfoControllerDecorator.html | 2 +- .../storagemgr/rest/InfoControllerImpl.html | 2 +- .../rest/JobOrderControllerImpl.html | 2 +- .../storagemgr/rest/JoborderController.html | 2 +- .../rest/JoborderControllerDecorator.html | 2 +- .../storagemgr/rest/ProductController.html | 2 +- .../rest/ProductControllerDecorator.html | 2 +- .../rest/ProductControllerImpl.html | 2 +- .../rest/ProductfileController.html | 2 +- .../rest/ProductfileControllerDecorator.html | 2 +- .../rest/ProductfileControllerImpl.html | 2 +- .../rest/class-use/InfoController.html | 2 +- .../class-use/InfoControllerDecorator.html | 2 +- .../rest/class-use/InfoControllerImpl.html | 2 +- .../class-use/JobOrderControllerImpl.html | 2 +- .../rest/class-use/JoborderController.html | 2 +- .../JoborderControllerDecorator.html | 2 +- .../rest/class-use/ProductController.html | 2 +- .../class-use/ProductControllerDecorator.html | 2 +- .../rest/class-use/ProductControllerImpl.html | 2 +- .../rest/class-use/ProductfileController.html | 2 +- .../ProductfileControllerDecorator.html | 2 +- .../class-use/ProductfileControllerImpl.html | 2 +- .../storagemgr/rest/model/Joborder.html | 2 +- .../proseo/storagemgr/rest/model/Posix.html | 2 +- .../storagemgr/rest/model/RestFileInfo.html | 2 +- .../storagemgr/rest/model/RestInfo.html | 2 +- .../storagemgr/rest/model/RestJoborder.html | 2 +- .../storagemgr/rest/model/RestProductFS.html | 2 +- .../dlr/proseo/storagemgr/rest/model/S3.html | 2 +- .../rest/model/class-use/Joborder.html | 2 +- .../rest/model/class-use/Posix.html | 2 +- .../rest/model/class-use/RestFileInfo.html | 2 +- .../rest/model/class-use/RestInfo.html | 2 +- .../rest/model/class-use/RestJoborder.html | 2 +- .../rest/model/class-use/RestProductFS.html | 2 +- .../storagemgr/rest/model/class-use/S3.html | 2 +- .../rest/model/package-summary.html | 2 +- .../storagemgr/rest/model/package-tree.html | 2 +- .../storagemgr/rest/model/package-use.html | 2 +- .../storagemgr/rest/package-summary.html | 2 +- .../proseo/storagemgr/rest/package-tree.html | 2 +- .../proseo/storagemgr/rest/package-use.html | 2 +- .../proseo/storagemgr/utils/ProseoFile.html | 2 +- .../storagemgr/utils/ProseoFileAlluxio.html | 2 +- .../storagemgr/utils/ProseoFilePosix.html | 2 +- .../proseo/storagemgr/utils/ProseoFileS3.html | 2 +- .../storagemgr/utils/StorageLogger.html | 2 +- .../storagemgr/utils/StorageManagerUtils.html | 2 +- .../proseo/storagemgr/utils/StorageType.html | 2 +- .../utils/class-use/ProseoFile.html | 2 +- .../utils/class-use/ProseoFileAlluxio.html | 2 +- .../utils/class-use/ProseoFilePosix.html | 2 +- .../utils/class-use/ProseoFileS3.html | 2 +- .../utils/class-use/StorageLogger.html | 2 +- .../utils/class-use/StorageManagerUtils.html | 2 +- .../utils/class-use/StorageType.html | 2 +- .../storagemgr/utils/package-summary.html | 2 +- .../proseo/storagemgr/utils/package-tree.html | 2 +- .../proseo/storagemgr/utils/package-use.html | 2 +- .../FileLockedAfterMaxCyclesException.html | 2 +- .../FileLockedAfterMaxCyclesException.html | 2 +- .../version2/Exceptions/package-summary.html | 2 +- .../version2/Exceptions/package-tree.html | 2 +- .../version2/Exceptions/package-use.html | 2 +- .../proseo/storagemgr/version2/FileUtils.html | 2 +- .../storagemgr/version2/PathConverter.html | 2 +- .../version2/StorageFileLocker.html | 2 +- .../storagemgr/version2/StorageProvider.html | 2 +- .../version2/class-use/FileUtils.html | 2 +- .../version2/class-use/PathConverter.html | 2 +- .../version2/class-use/StorageFileLocker.html | 2 +- .../version2/class-use/StorageProvider.html | 2 +- .../version2/model/AtomicCommand.html | 2 +- .../version2/model/DefaultRetryStrategy.html | 2 +- .../storagemgr/version2/model/Storage.html | 2 +- .../version2/model/StorageFile.html | 2 +- .../version2/model/StorageType.html | 2 +- .../model/class-use/AtomicCommand.html | 2 +- .../model/class-use/DefaultRetryStrategy.html | 2 +- .../version2/model/class-use/Storage.html | 2 +- .../version2/model/class-use/StorageFile.html | 2 +- .../version2/model/class-use/StorageType.html | 2 +- .../version2/model/package-summary.html | 2 +- .../version2/model/package-tree.html | 2 +- .../version2/model/package-use.html | 2 +- .../storagemgr/version2/package-summary.html | 2 +- .../storagemgr/version2/package-tree.html | 2 +- .../storagemgr/version2/package-use.html | 2 +- .../storagemgr/version2/posix/PosixDAL.html | 2 +- .../version2/posix/PosixStorage.html | 2 +- .../version2/posix/PosixStorageFile.html | 2 +- .../version2/posix/class-use/PosixDAL.html | 2 +- .../posix/class-use/PosixStorage.html | 2 +- .../posix/class-use/PosixStorageFile.html | 2 +- .../version2/posix/package-summary.html | 2 +- .../version2/posix/package-tree.html | 2 +- .../version2/posix/package-use.html | 2 +- .../version2/s3/S3AtomicBucketCreator.html | 2 +- .../version2/s3/S3AtomicBucketDeleter.html | 2 +- .../version2/s3/S3AtomicBucketListGetter.html | 2 +- .../s3/S3AtomicFileContentGetter.html | 2 +- .../version2/s3/S3AtomicFileDeleter.html | 2 +- .../version2/s3/S3AtomicFileDownloader.html | 2 +- .../version2/s3/S3AtomicFileExistsGetter.html | 2 +- .../version2/s3/S3AtomicFileListDeleter.html | 2 +- .../version2/s3/S3AtomicFileListGetter.html | 2 +- .../version2/s3/S3AtomicFileSizeGetter.html | 2 +- .../version2/s3/S3AtomicFileUploader.html | 2 +- .../s3/S3AtomicInputStreamGetter.html | 2 +- .../version2/s3/S3Configuration.html | 2 +- .../proseo/storagemgr/version2/s3/S3DAL.html | 2 +- .../storagemgr/version2/s3/S3Storage.html | 2 +- .../storagemgr/version2/s3/S3StorageFile.html | 2 +- .../s3/class-use/S3AtomicBucketCreator.html | 2 +- .../s3/class-use/S3AtomicBucketDeleter.html | 2 +- .../class-use/S3AtomicBucketListGetter.html | 2 +- .../class-use/S3AtomicFileContentGetter.html | 2 +- .../s3/class-use/S3AtomicFileDeleter.html | 2 +- .../s3/class-use/S3AtomicFileDownloader.html | 2 +- .../class-use/S3AtomicFileExistsGetter.html | 2 +- .../s3/class-use/S3AtomicFileListDeleter.html | 2 +- .../s3/class-use/S3AtomicFileListGetter.html | 2 +- .../s3/class-use/S3AtomicFileSizeGetter.html | 2 +- .../s3/class-use/S3AtomicFileUploader.html | 2 +- .../class-use/S3AtomicInputStreamGetter.html | 2 +- .../s3/class-use/S3Configuration.html | 2 +- .../version2/s3/class-use/S3DAL.html | 2 +- .../version2/s3/class-use/S3Storage.html | 2 +- .../version2/s3/class-use/S3StorageFile.html | 2 +- .../version2/s3/package-summary.html | 2 +- .../storagemgr/version2/s3/package-tree.html | 2 +- .../storagemgr/version2/s3/package-use.html | 2 +- docs/proseo-storage-mgr/apidocs/help-doc.html | 2 +- .../proseo-storage-mgr/apidocs/index-all.html | 2 +- docs/proseo-storage-mgr/apidocs/index.html | 4 +- docs/proseo-storage-mgr/apidocs/options | 4 +- .../apidocs/overview-summary.html | 2 +- .../apidocs/overview-tree.html | 2 +- docs/proseo-storage-mgr/apidocs/search.html | 2 +- .../apidocs/serialized-form.html | 2 +- docs/proseo-storage-mgr/index.html | 2 +- docs/proseo-storage-mgr/project-info.html | 2 +- docs/proseo-storage-mgr/project-reports.html | 2 +- docs/proseo-storage-mgr/scm.html | 2 +- docs/proseo-storage-mgr/summary.html | 4 +- docs/proseo-ui/index.html | 2 +- docs/proseo-ui/modules.html | 2 +- docs/proseo-ui/project-info.html | 2 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../dlr/proseo/ui/backend/LoginManager.html | 2 +- .../ui/backend/ServiceConfiguration.html | 2 +- .../proseo/ui/backend/ServiceConnection.html | 2 +- .../ui/backend/class-use/LoginManager.html | 2 +- .../class-use/ServiceConfiguration.html | 2 +- .../backend/class-use/ServiceConnection.html | 2 +- .../proseo/ui/backend/package-summary.html | 2 +- .../dlr/proseo/ui/backend/package-tree.html | 2 +- .../de/dlr/proseo/ui/backend/package-use.html | 2 +- .../proseo-ui-backend/apidocs/help-doc.html | 2 +- .../proseo-ui-backend/apidocs/index-all.html | 2 +- .../proseo-ui-backend/apidocs/index.html | 2 +- .../apidocs/overview-tree.html | 2 +- .../proseo-ui-backend/apidocs/search.html | 2 +- docs/proseo-ui/proseo-ui-backend/index.html | 2 +- .../proseo-ui-backend/project-info.html | 2 +- .../proseo-ui-backend/project-reports.html | 2 +- docs/proseo-ui/proseo-ui-backend/scm.html | 2 +- docs/proseo-ui/proseo-ui-backend/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../dlr/proseo/ui/cli/CLIConfiguration.html | 2 +- .../proseo/ui/cli/CLIUtil.Credentials.html | 2 +- .../apidocs/de/dlr/proseo/ui/cli/CLIUtil.html | 2 +- .../proseo/ui/cli/CommandLineInterface.html | 2 +- .../proseo/ui/cli/FacilityCommandRunner.html | 2 +- .../proseo/ui/cli/IngestorCommandRunner.html | 2 +- .../dlr/proseo/ui/cli/JobCommandRunner.html | 2 +- .../proseo/ui/cli/MissionCommandRunner.html | 2 +- .../dlr/proseo/ui/cli/OrderCommandRunner.html | 2 +- .../proseo/ui/cli/ProcessorCommandRunner.html | 2 +- .../ui/cli/ProductArchiveCommandRunner.html | 2 +- .../ui/cli/ProductclassCommandRunner.html | 2 +- .../dlr/proseo/ui/cli/UserCommandRunner.html | 2 +- .../proseo/ui/cli/WorkflowCommandRunner.html | 2 +- .../ui/cli/class-use/CLIConfiguration.html | 2 +- .../ui/cli/class-use/CLIUtil.Credentials.html | 2 +- .../dlr/proseo/ui/cli/class-use/CLIUtil.html | 2 +- .../cli/class-use/CommandLineInterface.html | 2 +- .../cli/class-use/FacilityCommandRunner.html | 2 +- .../cli/class-use/IngestorCommandRunner.html | 2 +- .../ui/cli/class-use/JobCommandRunner.html | 2 +- .../cli/class-use/MissionCommandRunner.html | 2 +- .../ui/cli/class-use/OrderCommandRunner.html | 2 +- .../cli/class-use/ProcessorCommandRunner.html | 2 +- .../ProductArchiveCommandRunner.html | 2 +- .../class-use/ProductclassCommandRunner.html | 2 +- .../ui/cli/class-use/UserCommandRunner.html | 2 +- .../cli/class-use/WorkflowCommandRunner.html | 2 +- .../de/dlr/proseo/ui/cli/package-summary.html | 2 +- .../de/dlr/proseo/ui/cli/package-tree.html | 2 +- .../de/dlr/proseo/ui/cli/package-use.html | 2 +- .../dlr/proseo/ui/cli/parser/CLICommand.html | 2 +- .../dlr/proseo/ui/cli/parser/CLIOption.html | 2 +- .../proseo/ui/cli/parser/CLIParameter.html | 2 +- .../dlr/proseo/ui/cli/parser/CLIParser.html | 2 +- .../dlr/proseo/ui/cli/parser/CLISyntax.html | 2 +- .../proseo/ui/cli/parser/ParsedCommand.html | 2 +- .../proseo/ui/cli/parser/ParsedOption.html | 2 +- .../proseo/ui/cli/parser/ParsedParameter.html | 2 +- .../ui/cli/parser/class-use/CLICommand.html | 2 +- .../ui/cli/parser/class-use/CLIOption.html | 2 +- .../ui/cli/parser/class-use/CLIParameter.html | 2 +- .../ui/cli/parser/class-use/CLIParser.html | 2 +- .../ui/cli/parser/class-use/CLISyntax.html | 2 +- .../cli/parser/class-use/ParsedCommand.html | 2 +- .../ui/cli/parser/class-use/ParsedOption.html | 2 +- .../cli/parser/class-use/ParsedParameter.html | 2 +- .../proseo/ui/cli/parser/package-summary.html | 2 +- .../proseo/ui/cli/parser/package-tree.html | 2 +- .../dlr/proseo/ui/cli/parser/package-use.html | 2 +- .../ui/cli/syntax2html/Syntax2HTML.html | 2 +- .../syntax2html/class-use/Syntax2HTML.html | 2 +- .../ui/cli/syntax2html/package-summary.html | 2 +- .../ui/cli/syntax2html/package-tree.html | 2 +- .../ui/cli/syntax2html/package-use.html | 2 +- .../proseo-ui-cli/apidocs/help-doc.html | 2 +- .../proseo-ui-cli/apidocs/index-all.html | 2 +- .../proseo-ui-cli/apidocs/index.html | 4 +- .../apidocs/overview-summary.html | 2 +- .../proseo-ui-cli/apidocs/overview-tree.html | 2 +- .../proseo-ui-cli/apidocs/search.html | 2 +- docs/proseo-ui/proseo-ui-cli/index.html | 2 +- .../proseo-ui/proseo-ui-cli/project-info.html | 2 +- .../proseo-ui-cli/project-reports.html | 2 +- docs/proseo-ui/proseo-ui-cli/scm.html | 2 +- docs/proseo-ui/proseo-ui-cli/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../de/dlr/proseo/ui/gui/DataCache.html | 2 +- .../ui/gui/GUIAuthenticationProvider.html | 2 +- .../proseo/ui/gui/GUIAuthenticationToken.html | 2 +- .../dlr/proseo/ui/gui/GUIBaseController.html | 2 +- .../dlr/proseo/ui/gui/GUIConfiguration.html | 2 +- .../ui/gui/GUIConfigurationController.html | 2 +- .../de/dlr/proseo/ui/gui/GUIController.html | 2 +- .../proseo/ui/gui/GUIFacilityController.html | 2 +- .../proseo/ui/gui/GUIMissionController.html | 2 +- .../dlr/proseo/ui/gui/GUIOrbitController.html | 2 +- .../dlr/proseo/ui/gui/GUIOrderController.html | 2 +- .../ui/gui/GUIProcessorClassController.html | 2 +- .../proseo/ui/gui/GUIProcessorController.html | 2 +- .../ui/gui/GUIProductClassController.html | 2 +- .../proseo/ui/gui/GUIProductController.html | 2 +- .../proseo/ui/gui/GUISettingsController.html | 2 +- .../ui/gui/GUISpaceCraftController.html | 2 +- .../ui/gui/GUIStatisticsController.html | 2 +- .../proseo/ui/gui/GUIWorkflowController.html | 2 +- .../proseo/ui/gui/GraphicalUserInterface.html | 2 +- .../de/dlr/proseo/ui/gui/OrderInfo.html | 2 +- .../ui/gui/SpringAuthenticationFilter.html | 2 +- .../proseo/ui/gui/SpringSecurityConfig.html | 2 +- .../proseo/ui/gui/class-use/DataCache.html | 2 +- .../class-use/GUIAuthenticationProvider.html | 2 +- .../gui/class-use/GUIAuthenticationToken.html | 2 +- .../ui/gui/class-use/GUIBaseController.html | 2 +- .../ui/gui/class-use/GUIConfiguration.html | 2 +- .../class-use/GUIConfigurationController.html | 2 +- .../ui/gui/class-use/GUIController.html | 2 +- .../gui/class-use/GUIFacilityController.html | 2 +- .../gui/class-use/GUIMissionController.html | 2 +- .../ui/gui/class-use/GUIOrbitController.html | 2 +- .../ui/gui/class-use/GUIOrderController.html | 2 +- .../GUIProcessorClassController.html | 2 +- .../gui/class-use/GUIProcessorController.html | 2 +- .../class-use/GUIProductClassController.html | 2 +- .../gui/class-use/GUIProductController.html | 2 +- .../gui/class-use/GUISettingsController.html | 2 +- .../class-use/GUISpaceCraftController.html | 2 +- .../class-use/GUIStatisticsController.html | 2 +- .../gui/class-use/GUIWorkflowController.html | 2 +- .../gui/class-use/GraphicalUserInterface.html | 2 +- .../proseo/ui/gui/class-use/OrderInfo.html | 2 +- .../class-use/SpringAuthenticationFilter.html | 2 +- .../gui/class-use/SpringSecurityConfig.html | 2 +- .../de/dlr/proseo/ui/gui/package-summary.html | 2 +- .../de/dlr/proseo/ui/gui/package-tree.html | 2 +- .../de/dlr/proseo/ui/gui/package-use.html | 2 +- .../proseo/ui/gui/service/MapComparator.html | 2 +- .../proseo/ui/gui/service/OrderService.html | 2 +- .../ui/gui/service/ProcessorService.html | 2 +- .../ui/gui/service/StatisticsService.html | 2 +- .../gui/service/class-use/MapComparator.html | 2 +- .../gui/service/class-use/OrderService.html | 2 +- .../service/class-use/ProcessorService.html | 2 +- .../service/class-use/StatisticsService.html | 2 +- .../ui/gui/service/package-summary.html | 2 +- .../proseo/ui/gui/service/package-tree.html | 2 +- .../proseo/ui/gui/service/package-use.html | 2 +- .../proseo-ui-gui/apidocs/help-doc.html | 2 +- .../proseo-ui-gui/apidocs/index-all.html | 2 +- .../proseo-ui-gui/apidocs/index.html | 4 +- docs/proseo-ui/proseo-ui-gui/apidocs/options | 4 +- .../apidocs/overview-summary.html | 2 +- .../proseo-ui-gui/apidocs/overview-tree.html | 2 +- .../proseo-ui-gui/apidocs/search.html | 2 +- .../apidocs/serialized-form.html | 2 +- docs/proseo-ui/proseo-ui-gui/index.html | 2 +- .../proseo-ui/proseo-ui-gui/project-info.html | 2 +- .../proseo-ui-gui/project-reports.html | 2 +- docs/proseo-ui/proseo-ui-gui/scm.html | 2 +- docs/proseo-ui/proseo-ui-gui/summary.html | 4 +- docs/proseo-ui/scm.html | 2 +- docs/proseo-ui/summary.html | 4 +- .../apidocs/allclasses-index.html | 2 +- .../apidocs/allpackages-index.html | 2 +- .../apidocs/constant-values.html | 2 +- .../usermgr/UserManagerApplication.html | 2 +- .../proseo/usermgr/UsermgrConfiguration.html | 2 +- .../proseo/usermgr/UsermgrSecurityConfig.html | 2 +- .../class-use/UserManagerApplication.html | 2 +- .../class-use/UsermgrConfiguration.html | 2 +- .../class-use/UsermgrSecurityConfig.html | 2 +- .../usermgr/dao/GroupMemberRepository.html | 2 +- .../proseo/usermgr/dao/GroupRepository.html | 2 +- .../proseo/usermgr/dao/UserRepository.html | 2 +- .../dao/class-use/GroupMemberRepository.html | 2 +- .../dao/class-use/GroupRepository.html | 2 +- .../usermgr/dao/class-use/UserRepository.html | 2 +- .../proseo/usermgr/dao/package-summary.html | 2 +- .../dlr/proseo/usermgr/dao/package-tree.html | 2 +- .../dlr/proseo/usermgr/dao/package-use.html | 2 +- .../dlr/proseo/usermgr/model/Authority.html | 2 +- .../de/dlr/proseo/usermgr/model/Group.html | 2 +- .../proseo/usermgr/model/GroupAuthority.html | 2 +- .../dlr/proseo/usermgr/model/GroupMember.html | 2 +- .../de/dlr/proseo/usermgr/model/Quota.html | 2 +- .../de/dlr/proseo/usermgr/model/User.html | 2 +- .../usermgr/model/class-use/Authority.html | 2 +- .../proseo/usermgr/model/class-use/Group.html | 2 +- .../model/class-use/GroupAuthority.html | 2 +- .../usermgr/model/class-use/GroupMember.html | 2 +- .../proseo/usermgr/model/class-use/Quota.html | 2 +- .../proseo/usermgr/model/class-use/User.html | 2 +- .../proseo/usermgr/model/package-summary.html | 2 +- .../proseo/usermgr/model/package-tree.html | 2 +- .../dlr/proseo/usermgr/model/package-use.html | 2 +- .../dlr/proseo/usermgr/package-summary.html | 2 +- .../de/dlr/proseo/usermgr/package-tree.html | 2 +- .../de/dlr/proseo/usermgr/package-use.html | 2 +- .../proseo/usermgr/rest/GroupController.html | 2 +- .../rest/GroupControllerDecorator.html | 2 +- .../usermgr/rest/GroupControllerImpl.html | 2 +- .../GroupManager.NotModifiedException.html | 2 +- .../dlr/proseo/usermgr/rest/GroupManager.html | 2 +- .../proseo/usermgr/rest/LoginController.html | 2 +- .../rest/LoginControllerDecorator.html | 2 +- .../usermgr/rest/LoginControllerImpl.html | 2 +- .../proseo/usermgr/rest/UserController.html | 2 +- .../usermgr/rest/UserControllerDecorator.html | 2 +- .../usermgr/rest/UserControllerImpl.html | 2 +- .../UserManager.NotModifiedException.html | 2 +- .../dlr/proseo/usermgr/rest/UserManager.html | 2 +- .../rest/class-use/GroupController.html | 2 +- .../class-use/GroupControllerDecorator.html | 2 +- .../rest/class-use/GroupControllerImpl.html | 2 +- .../GroupManager.NotModifiedException.html | 2 +- .../usermgr/rest/class-use/GroupManager.html | 2 +- .../rest/class-use/LoginController.html | 2 +- .../class-use/LoginControllerDecorator.html | 2 +- .../rest/class-use/LoginControllerImpl.html | 2 +- .../rest/class-use/UserController.html | 2 +- .../class-use/UserControllerDecorator.html | 2 +- .../rest/class-use/UserControllerImpl.html | 2 +- .../UserManager.NotModifiedException.html | 2 +- .../usermgr/rest/class-use/UserManager.html | 2 +- .../proseo/usermgr/rest/model/RestGroup.html | 2 +- .../proseo/usermgr/rest/model/RestQuota.html | 2 +- .../proseo/usermgr/rest/model/RestUser.html | 2 +- .../rest/model/class-use/RestGroup.html | 2 +- .../rest/model/class-use/RestQuota.html | 2 +- .../rest/model/class-use/RestUser.html | 2 +- .../usermgr/rest/model/package-summary.html | 2 +- .../usermgr/rest/model/package-tree.html | 2 +- .../usermgr/rest/model/package-use.html | 2 +- .../proseo/usermgr/rest/package-summary.html | 2 +- .../dlr/proseo/usermgr/rest/package-tree.html | 2 +- .../dlr/proseo/usermgr/rest/package-use.html | 2 +- docs/proseo-user-mgr/apidocs/help-doc.html | 2 +- docs/proseo-user-mgr/apidocs/index-all.html | 2 +- docs/proseo-user-mgr/apidocs/index.html | 4 +- docs/proseo-user-mgr/apidocs/options | 4 +- .../apidocs/overview-summary.html | 2 +- .../apidocs/overview-tree.html | 2 +- docs/proseo-user-mgr/apidocs/search.html | 2 +- .../apidocs/serialized-form.html | 2 +- docs/proseo-user-mgr/index.html | 2 +- docs/proseo-user-mgr/project-info.html | 2 +- docs/proseo-user-mgr/project-reports.html | 2 +- docs/proseo-user-mgr/scm.html | 2 +- docs/proseo-user-mgr/summary.html | 4 +- docs/scm.html | 2 +- docs/summary.html | 4 +- facility-mgr/pom.xml | 2 +- geotools/pom.xml | 2 +- ingestor/pom.xml | 2 +- interfaces/pom.xml | 2 +- logging/pom.xml | 2 +- model/pom.xml | 2 +- monitor/pom.xml | 2 +- notification/pom.xml | 4 +- order-mgr/pom.xml | 2 +- planner/pom.xml | 6 +-- pom.xml | 2 +- processor-mgr/pom.xml | 2 +- productclass-mgr/pom.xml | 2 +- samples/pom.xml | 2 +- samples/sample-processor/pom.xml | 2 +- samples/sample-wrapper/Dockerfile | 2 +- samples/sample-wrapper/pom.xml | 4 +- .../testdata/configure_proseo_test_mission.pl | 52 +++++++++---------- samples/testdata/create_data.sh.template | 4 +- samples/testdata/create_data_localhost.sh | 4 +- storage-mgr/pom.xml | 2 +- ui/backend/pom.xml | 2 +- ui/cli/pom.xml | 2 +- ui/gui/pom.xml | 2 +- ui/pom.xml | 2 +- user-mgr/pom.xml | 2 +- 2260 files changed, 2368 insertions(+), 2368 deletions(-) diff --git a/api/aip-client/pom.xml b/api/aip-client/pom.xml index 29c8fc076..6cf078504 100644 --- a/api/aip-client/pom.xml +++ b/api/aip-client/pom.xml @@ -4,7 +4,7 @@ de.dlr.proseo proseo-api - 1.0.0 + 1.0.1-SNAPSHOT aip-client diff --git a/api/archive-mgr/pom.xml b/api/archive-mgr/pom.xml index 0e5f5ef74..ed5bf987e 100644 --- a/api/archive-mgr/pom.xml +++ b/api/archive-mgr/pom.xml @@ -4,7 +4,7 @@ de.dlr.proseo proseo-api - 1.0.0 + 1.0.1-SNAPSHOT proseo-archive-mgr diff --git a/api/auxip-monitor/pom.xml b/api/auxip-monitor/pom.xml index e2a655975..4b82683d0 100644 --- a/api/auxip-monitor/pom.xml +++ b/api/auxip-monitor/pom.xml @@ -4,7 +4,7 @@ de.dlr.proseo proseo-api - 1.0.0 + 1.0.1-SNAPSHOT auxip-monitor diff --git a/api/base-monitor/pom.xml b/api/base-monitor/pom.xml index 9ecba04fd..b4c9fe690 100644 --- a/api/base-monitor/pom.xml +++ b/api/base-monitor/pom.xml @@ -5,7 +5,7 @@ de.dlr.proseo proseo-api - 1.0.0 + 1.0.1-SNAPSHOT proseo-base-monitor diff --git a/api/cadip-monitor/pom.xml b/api/cadip-monitor/pom.xml index dfd4601c2..4606ff331 100644 --- a/api/cadip-monitor/pom.xml +++ b/api/cadip-monitor/pom.xml @@ -4,7 +4,7 @@ de.dlr.proseo proseo-api - 1.0.0 + 1.0.1-SNAPSHOT cadip-monitor diff --git a/api/edip-monitor/pom.xml b/api/edip-monitor/pom.xml index bb96b2862..e438ba973 100644 --- a/api/edip-monitor/pom.xml +++ b/api/edip-monitor/pom.xml @@ -4,7 +4,7 @@ de.dlr.proseo proseo-api - 1.0.0 + 1.0.1-SNAPSHOT edip-monitor diff --git a/api/odip/pom.xml b/api/odip/pom.xml index b20947ece..e18539fe0 100644 --- a/api/odip/pom.xml +++ b/api/odip/pom.xml @@ -6,7 +6,7 @@ de.dlr.proseo proseo-api - 1.0.0 + 1.0.1-SNAPSHOT odip diff --git a/api/pom.xml b/api/pom.xml index 46f39e417..16e0ab3a7 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -5,7 +5,7 @@ de.dlr.proseo proseo - 1.0.0 + 1.0.1-SNAPSHOT proseo-api diff --git a/api/prip/pom.xml b/api/prip/pom.xml index fa4dc1c3a..62bf3bc7e 100644 --- a/api/prip/pom.xml +++ b/api/prip/pom.xml @@ -4,7 +4,7 @@ de.dlr.proseo proseo-api - 1.0.0 + 1.0.1-SNAPSHOT proseo-api-prip diff --git a/api/xbip-monitor/pom.xml b/api/xbip-monitor/pom.xml index 36a1a9dd1..f2627e26a 100644 --- a/api/xbip-monitor/pom.xml +++ b/api/xbip-monitor/pom.xml @@ -4,7 +4,7 @@ de.dlr.proseo proseo-api - 1.0.0 + 1.0.1-SNAPSHOT xbip-monitor diff --git a/base-wrapper/pom.xml b/base-wrapper/pom.xml index aa3ce3668..159710835 100644 --- a/base-wrapper/pom.xml +++ b/base-wrapper/pom.xml @@ -5,7 +5,7 @@ de.dlr.proseo proseo - 1.0.0 + 1.0.1-SNAPSHOT proseo-base-wrapper diff --git a/common/pom.xml b/common/pom.xml index 59c11d093..4ed6463a2 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -5,7 +5,7 @@ de.dlr.proseo proseo - 1.0.0 + 1.0.1-SNAPSHOT proseo-common diff --git a/deploy-single-node/proseo-images/proseo-components/proseo-api-aip/Dockerfile b/deploy-single-node/proseo-images/proseo-components/proseo-api-aip/Dockerfile index 7213f1995..39bf10c23 100644 --- a/deploy-single-node/proseo-images/proseo-components/proseo-api-aip/Dockerfile +++ b/deploy-single-node/proseo-images/proseo-components/proseo-api-aip/Dockerfile @@ -1,4 +1,4 @@ -FROM localhost:5000/proseo-aip-client:1.0.0 +FROM localhost:5000/proseo-aip-client:1.0.1-SNAPSHOT ADD application.yml /usr/share/api/aip-client/application.yml WORKDIR /usr/share/api/aip-client diff --git a/deploy-single-node/proseo-images/proseo-components/proseo-api-odip/Dockerfile b/deploy-single-node/proseo-images/proseo-components/proseo-api-odip/Dockerfile index 5030cbbbf..f4941378f 100644 --- a/deploy-single-node/proseo-images/proseo-components/proseo-api-odip/Dockerfile +++ b/deploy-single-node/proseo-images/proseo-components/proseo-api-odip/Dockerfile @@ -1,4 +1,4 @@ -FROM localhost:5000/proseo-api-odip:1.0.0 +FROM localhost:5000/proseo-api-odip:1.0.1-SNAPSHOT ADD application.yml /usr/share/api/odip/application.yml WORKDIR /usr/share/api/odip diff --git a/deploy-single-node/proseo-images/proseo-components/proseo-api-prip/Dockerfile b/deploy-single-node/proseo-images/proseo-components/proseo-api-prip/Dockerfile index 9a44e847f..93e497a21 100644 --- a/deploy-single-node/proseo-images/proseo-components/proseo-api-prip/Dockerfile +++ b/deploy-single-node/proseo-images/proseo-components/proseo-api-prip/Dockerfile @@ -1,4 +1,4 @@ -FROM localhost:5000/proseo-api-prip:1.0.0 +FROM localhost:5000/proseo-api-prip:1.0.1-SNAPSHOT ADD application.yml /usr/share/api/prip/application.yml WORKDIR /usr/share/api/prip diff --git a/deploy-single-node/proseo-images/proseo-components/proseo-archive-mgr/Dockerfile b/deploy-single-node/proseo-images/proseo-components/proseo-archive-mgr/Dockerfile index 08b6eff47..9742d7a08 100644 --- a/deploy-single-node/proseo-images/proseo-components/proseo-archive-mgr/Dockerfile +++ b/deploy-single-node/proseo-images/proseo-components/proseo-archive-mgr/Dockerfile @@ -1,3 +1,3 @@ -FROM localhost:5000/proseo-archive-mgr:1.0.0 +FROM localhost:5000/proseo-archive-mgr:1.0.1-SNAPSHOT ADD application.yml /usr/share/proseo-archive-mgr/application.yml WORKDIR /usr/share/proseo-archive-mgr/ diff --git a/deploy-single-node/proseo-images/proseo-components/proseo-facility-mgr/Dockerfile b/deploy-single-node/proseo-images/proseo-components/proseo-facility-mgr/Dockerfile index 93760ddf8..bf05c0d6e 100644 --- a/deploy-single-node/proseo-images/proseo-components/proseo-facility-mgr/Dockerfile +++ b/deploy-single-node/proseo-images/proseo-components/proseo-facility-mgr/Dockerfile @@ -1,3 +1,3 @@ -FROM localhost:5000/proseo-facility-mgr:1.0.0 +FROM localhost:5000/proseo-facility-mgr:1.0.1-SNAPSHOT ADD application.yml /usr/share/proseo-facility-mgr/application.yml WORKDIR /usr/share/proseo-facility-mgr/ diff --git a/deploy-single-node/proseo-images/proseo-components/proseo-ingestor/Dockerfile b/deploy-single-node/proseo-images/proseo-components/proseo-ingestor/Dockerfile index 6950eb0cc..20f549b59 100644 --- a/deploy-single-node/proseo-images/proseo-components/proseo-ingestor/Dockerfile +++ b/deploy-single-node/proseo-images/proseo-components/proseo-ingestor/Dockerfile @@ -1,4 +1,4 @@ -FROM localhost:5000/proseo-ingestor:1.0.0 +FROM localhost:5000/proseo-ingestor:1.0.1-SNAPSHOT ADD application.yml /usr/share/ingestor/application.yml WORKDIR /usr/share/ingestor diff --git a/deploy-single-node/proseo-images/proseo-components/proseo-notification/Dockerfile b/deploy-single-node/proseo-images/proseo-components/proseo-notification/Dockerfile index 92bf56d4c..43b940402 100644 --- a/deploy-single-node/proseo-images/proseo-components/proseo-notification/Dockerfile +++ b/deploy-single-node/proseo-images/proseo-components/proseo-notification/Dockerfile @@ -1,4 +1,4 @@ -FROM localhost:5000/proseo-notification:1.0.0 +FROM localhost:5000/proseo-notification:1.0.1-SNAPSHOT ADD application.yml /usr/share/notification/application.yml WORKDIR /usr/share/notification diff --git a/deploy-single-node/proseo-images/proseo-components/proseo-order-mgr/Dockerfile b/deploy-single-node/proseo-images/proseo-components/proseo-order-mgr/Dockerfile index d90c94e3f..f63128864 100644 --- a/deploy-single-node/proseo-images/proseo-components/proseo-order-mgr/Dockerfile +++ b/deploy-single-node/proseo-images/proseo-components/proseo-order-mgr/Dockerfile @@ -1,3 +1,3 @@ -FROM localhost:5000/proseo-order-mgr:1.0.0 +FROM localhost:5000/proseo-order-mgr:1.0.1-SNAPSHOT ADD application.yml /usr/share/proseo-order-mgr/application.yml WORKDIR /usr/share/proseo-order-mgr/ \ No newline at end of file diff --git a/deploy-single-node/proseo-images/proseo-components/proseo-planner/Dockerfile b/deploy-single-node/proseo-images/proseo-components/proseo-planner/Dockerfile index ff0a24b3f..68fe957b5 100644 --- a/deploy-single-node/proseo-images/proseo-components/proseo-planner/Dockerfile +++ b/deploy-single-node/proseo-images/proseo-components/proseo-planner/Dockerfile @@ -1,4 +1,4 @@ -FROM localhost:5000/proseo-planner:1.0.0 +FROM localhost:5000/proseo-planner:1.0.1-SNAPSHOT ADD application.yml /usr/share/planner/application.yml # Uncomment the next line, if a fixed login to a Kubernetes instance is required, and provide kube_config from the template file diff --git a/deploy-single-node/proseo-images/proseo-components/proseo-processor-mgr/Dockerfile b/deploy-single-node/proseo-images/proseo-components/proseo-processor-mgr/Dockerfile index 2f4a0f1ff..cdf553251 100644 --- a/deploy-single-node/proseo-images/proseo-components/proseo-processor-mgr/Dockerfile +++ b/deploy-single-node/proseo-images/proseo-components/proseo-processor-mgr/Dockerfile @@ -1,3 +1,3 @@ -FROM localhost:5000/proseo-processor-mgr:1.0.0 +FROM localhost:5000/proseo-processor-mgr:1.0.1-SNAPSHOT ADD application.yml /usr/share/proseo-processor-mgr/application.yml WORKDIR /usr/share/proseo-processor-mgr/ \ No newline at end of file diff --git a/deploy-single-node/proseo-images/proseo-components/proseo-productclass-mgr/Dockerfile b/deploy-single-node/proseo-images/proseo-components/proseo-productclass-mgr/Dockerfile index 1f9837fd7..ff22812c4 100644 --- a/deploy-single-node/proseo-images/proseo-components/proseo-productclass-mgr/Dockerfile +++ b/deploy-single-node/proseo-images/proseo-components/proseo-productclass-mgr/Dockerfile @@ -1,3 +1,3 @@ -FROM localhost:5000/proseo-productclass-mgr:1.0.0 +FROM localhost:5000/proseo-productclass-mgr:1.0.1-SNAPSHOT ADD application.yml /usr/share/proseo-productclass-mgr/application.yml WORKDIR /usr/share/proseo-productclass-mgr/ \ No newline at end of file diff --git a/deploy-single-node/proseo-images/proseo-components/proseo-sample-wrapper/Dockerfile b/deploy-single-node/proseo-images/proseo-components/proseo-sample-wrapper/Dockerfile index 44c31a797..afc5b60f9 100644 --- a/deploy-single-node/proseo-images/proseo-components/proseo-sample-wrapper/Dockerfile +++ b/deploy-single-node/proseo-images/proseo-components/proseo-sample-wrapper/Dockerfile @@ -1 +1 @@ -FROM localhost:5000/proseo-sample-wrapper:1.0.0 \ No newline at end of file +FROM localhost:5000/proseo-sample-wrapper:1.0.1-SNAPSHOT \ No newline at end of file diff --git a/deploy-single-node/proseo-images/proseo-components/proseo-storage-mgr/Dockerfile b/deploy-single-node/proseo-images/proseo-components/proseo-storage-mgr/Dockerfile index 516119850..3a83d45b7 100644 --- a/deploy-single-node/proseo-images/proseo-components/proseo-storage-mgr/Dockerfile +++ b/deploy-single-node/proseo-images/proseo-components/proseo-storage-mgr/Dockerfile @@ -1,3 +1,3 @@ -FROM localhost:5000/proseo-storage-mgr:1.0.0 +FROM localhost:5000/proseo-storage-mgr:1.0.1-SNAPSHOT ADD application.yml /usr/share/proseo-storage-mgr/application.yml WORKDIR /usr/share/proseo-storage-mgr/ \ No newline at end of file diff --git a/deploy-single-node/proseo-images/proseo-components/proseo-ui-gui/Dockerfile b/deploy-single-node/proseo-images/proseo-components/proseo-ui-gui/Dockerfile index fa9d0fa07..426996aba 100644 --- a/deploy-single-node/proseo-images/proseo-components/proseo-ui-gui/Dockerfile +++ b/deploy-single-node/proseo-images/proseo-components/proseo-ui-gui/Dockerfile @@ -1,3 +1,3 @@ -FROM localhost:5000/proseo-ui-gui:1.0.0 +FROM localhost:5000/proseo-ui-gui:1.0.1-SNAPSHOT ADD application.yml /usr/share/proseo-ui-gui/application.yml WORKDIR /usr/share/proseo-ui-gui/ \ No newline at end of file diff --git a/deploy-single-node/proseo-images/proseo-components/proseo-user-mgr/Dockerfile b/deploy-single-node/proseo-images/proseo-components/proseo-user-mgr/Dockerfile index efb4d865e..ea92bb175 100644 --- a/deploy-single-node/proseo-images/proseo-components/proseo-user-mgr/Dockerfile +++ b/deploy-single-node/proseo-images/proseo-components/proseo-user-mgr/Dockerfile @@ -1,3 +1,3 @@ -FROM localhost:5000/proseo-user-mgr:1.0.0 +FROM localhost:5000/proseo-user-mgr:1.0.1-SNAPSHOT ADD application.yml /usr/share/proseo-user-mgr/application.yml WORKDIR /usr/share/proseo-user-mgr/ diff --git a/deploy/loghost/prepare_monitoring/files/docker-compose.yml.template b/deploy/loghost/prepare_monitoring/files/docker-compose.yml.template index e55140d6f..daa20ccb5 100644 --- a/deploy/loghost/prepare_monitoring/files/docker-compose.yml.template +++ b/deploy/loghost/prepare_monitoring/files/docker-compose.yml.template @@ -19,7 +19,7 @@ services: monitor: hostname: monitor - image: proseo-registry.eoc.dlr.de/proseo-monitor:1.0.0-proseo + image: proseo-registry.eoc.dlr.de/proseo-monitor:1.0.1-SNAPSHOT-proseo ports: - 8080:8080 volumes: diff --git a/deploy/proseo-images/proseo-components/proseo-api-aip/Dockerfile b/deploy/proseo-images/proseo-components/proseo-api-aip/Dockerfile index 7213f1995..39bf10c23 100644 --- a/deploy/proseo-images/proseo-components/proseo-api-aip/Dockerfile +++ b/deploy/proseo-images/proseo-components/proseo-api-aip/Dockerfile @@ -1,4 +1,4 @@ -FROM localhost:5000/proseo-aip-client:1.0.0 +FROM localhost:5000/proseo-aip-client:1.0.1-SNAPSHOT ADD application.yml /usr/share/api/aip-client/application.yml WORKDIR /usr/share/api/aip-client diff --git a/deploy/proseo-images/proseo-components/proseo-api-odip/Dockerfile b/deploy/proseo-images/proseo-components/proseo-api-odip/Dockerfile index 5a6092904..fb64d8bc9 100644 --- a/deploy/proseo-images/proseo-components/proseo-api-odip/Dockerfile +++ b/deploy/proseo-images/proseo-components/proseo-api-odip/Dockerfile @@ -1,4 +1,4 @@ -FROM localhost:5000/proseo-api-odip:1.0.0 +FROM localhost:5000/proseo-api-odip:1.0.1-SNAPSHOT ADD application.yml /usr/share/api/prip/application.yml WORKDIR /usr/share/api/odip diff --git a/deploy/proseo-images/proseo-components/proseo-api-prip/Dockerfile b/deploy/proseo-images/proseo-components/proseo-api-prip/Dockerfile index 9a44e847f..93e497a21 100644 --- a/deploy/proseo-images/proseo-components/proseo-api-prip/Dockerfile +++ b/deploy/proseo-images/proseo-components/proseo-api-prip/Dockerfile @@ -1,4 +1,4 @@ -FROM localhost:5000/proseo-api-prip:1.0.0 +FROM localhost:5000/proseo-api-prip:1.0.1-SNAPSHOT ADD application.yml /usr/share/api/prip/application.yml WORKDIR /usr/share/api/prip diff --git a/deploy/proseo-images/proseo-components/proseo-archive-mgr/Dockerfile b/deploy/proseo-images/proseo-components/proseo-archive-mgr/Dockerfile index 08b6eff47..9742d7a08 100644 --- a/deploy/proseo-images/proseo-components/proseo-archive-mgr/Dockerfile +++ b/deploy/proseo-images/proseo-components/proseo-archive-mgr/Dockerfile @@ -1,3 +1,3 @@ -FROM localhost:5000/proseo-archive-mgr:1.0.0 +FROM localhost:5000/proseo-archive-mgr:1.0.1-SNAPSHOT ADD application.yml /usr/share/proseo-archive-mgr/application.yml WORKDIR /usr/share/proseo-archive-mgr/ diff --git a/deploy/proseo-images/proseo-components/proseo-facility-mgr/Dockerfile b/deploy/proseo-images/proseo-components/proseo-facility-mgr/Dockerfile index 93760ddf8..bf05c0d6e 100644 --- a/deploy/proseo-images/proseo-components/proseo-facility-mgr/Dockerfile +++ b/deploy/proseo-images/proseo-components/proseo-facility-mgr/Dockerfile @@ -1,3 +1,3 @@ -FROM localhost:5000/proseo-facility-mgr:1.0.0 +FROM localhost:5000/proseo-facility-mgr:1.0.1-SNAPSHOT ADD application.yml /usr/share/proseo-facility-mgr/application.yml WORKDIR /usr/share/proseo-facility-mgr/ diff --git a/deploy/proseo-images/proseo-components/proseo-ingestor/Dockerfile b/deploy/proseo-images/proseo-components/proseo-ingestor/Dockerfile index 2f127ddcb..8b4eb598b 100644 --- a/deploy/proseo-images/proseo-components/proseo-ingestor/Dockerfile +++ b/deploy/proseo-images/proseo-components/proseo-ingestor/Dockerfile @@ -1,3 +1,3 @@ -FROM localhost:5000/proseo-ingestor:1.0.0 +FROM localhost:5000/proseo-ingestor:1.0.1-SNAPSHOT ADD application.yml /usr/share/ingestor/application.yml WORKDIR /usr/share/ingestor diff --git a/deploy/proseo-images/proseo-components/proseo-monitor/Dockerfile b/deploy/proseo-images/proseo-components/proseo-monitor/Dockerfile index aa6204b5e..5236f559a 100644 --- a/deploy/proseo-images/proseo-components/proseo-monitor/Dockerfile +++ b/deploy/proseo-images/proseo-components/proseo-monitor/Dockerfile @@ -1,3 +1,3 @@ -FROM localhost:5000/proseo-monitor:1.0.0 +FROM localhost:5000/proseo-monitor:1.0.1-SNAPSHOT ADD application.yml /usr/share/proseo-monitor/application.yml WORKDIR /usr/share/proseo-monitor/ \ No newline at end of file diff --git a/deploy/proseo-images/proseo-components/proseo-notification/Dockerfile b/deploy/proseo-images/proseo-components/proseo-notification/Dockerfile index 92bf56d4c..43b940402 100644 --- a/deploy/proseo-images/proseo-components/proseo-notification/Dockerfile +++ b/deploy/proseo-images/proseo-components/proseo-notification/Dockerfile @@ -1,4 +1,4 @@ -FROM localhost:5000/proseo-notification:1.0.0 +FROM localhost:5000/proseo-notification:1.0.1-SNAPSHOT ADD application.yml /usr/share/notification/application.yml WORKDIR /usr/share/notification diff --git a/deploy/proseo-images/proseo-components/proseo-order-mgr/Dockerfile b/deploy/proseo-images/proseo-components/proseo-order-mgr/Dockerfile index d90c94e3f..f63128864 100644 --- a/deploy/proseo-images/proseo-components/proseo-order-mgr/Dockerfile +++ b/deploy/proseo-images/proseo-components/proseo-order-mgr/Dockerfile @@ -1,3 +1,3 @@ -FROM localhost:5000/proseo-order-mgr:1.0.0 +FROM localhost:5000/proseo-order-mgr:1.0.1-SNAPSHOT ADD application.yml /usr/share/proseo-order-mgr/application.yml WORKDIR /usr/share/proseo-order-mgr/ \ No newline at end of file diff --git a/deploy/proseo-images/proseo-components/proseo-planner/Dockerfile b/deploy/proseo-images/proseo-components/proseo-planner/Dockerfile index dd99304df..ea1b0341f 100644 --- a/deploy/proseo-images/proseo-components/proseo-planner/Dockerfile +++ b/deploy/proseo-images/proseo-components/proseo-planner/Dockerfile @@ -1,4 +1,4 @@ -FROM localhost:5000/proseo-planner:1.0.0 +FROM localhost:5000/proseo-planner:1.0.1-SNAPSHOT ADD application.yml /usr/share/planner/application.yml # Uncomment the next line, if a fixed login to a Kubernetes instance is required, and provide kube_config from the template file diff --git a/deploy/proseo-images/proseo-components/proseo-processor-mgr/Dockerfile b/deploy/proseo-images/proseo-components/proseo-processor-mgr/Dockerfile index 2f4a0f1ff..cdf553251 100644 --- a/deploy/proseo-images/proseo-components/proseo-processor-mgr/Dockerfile +++ b/deploy/proseo-images/proseo-components/proseo-processor-mgr/Dockerfile @@ -1,3 +1,3 @@ -FROM localhost:5000/proseo-processor-mgr:1.0.0 +FROM localhost:5000/proseo-processor-mgr:1.0.1-SNAPSHOT ADD application.yml /usr/share/proseo-processor-mgr/application.yml WORKDIR /usr/share/proseo-processor-mgr/ \ No newline at end of file diff --git a/deploy/proseo-images/proseo-components/proseo-productclass-mgr/Dockerfile b/deploy/proseo-images/proseo-components/proseo-productclass-mgr/Dockerfile index 1f9837fd7..ff22812c4 100644 --- a/deploy/proseo-images/proseo-components/proseo-productclass-mgr/Dockerfile +++ b/deploy/proseo-images/proseo-components/proseo-productclass-mgr/Dockerfile @@ -1,3 +1,3 @@ -FROM localhost:5000/proseo-productclass-mgr:1.0.0 +FROM localhost:5000/proseo-productclass-mgr:1.0.1-SNAPSHOT ADD application.yml /usr/share/proseo-productclass-mgr/application.yml WORKDIR /usr/share/proseo-productclass-mgr/ \ No newline at end of file diff --git a/deploy/proseo-images/proseo-components/proseo-sample-wrapper/Dockerfile b/deploy/proseo-images/proseo-components/proseo-sample-wrapper/Dockerfile index 44c31a797..afc5b60f9 100644 --- a/deploy/proseo-images/proseo-components/proseo-sample-wrapper/Dockerfile +++ b/deploy/proseo-images/proseo-components/proseo-sample-wrapper/Dockerfile @@ -1 +1 @@ -FROM localhost:5000/proseo-sample-wrapper:1.0.0 \ No newline at end of file +FROM localhost:5000/proseo-sample-wrapper:1.0.1-SNAPSHOT \ No newline at end of file diff --git a/deploy/proseo-images/proseo-components/proseo-storage-mgr/Dockerfile b/deploy/proseo-images/proseo-components/proseo-storage-mgr/Dockerfile index 516119850..3a83d45b7 100644 --- a/deploy/proseo-images/proseo-components/proseo-storage-mgr/Dockerfile +++ b/deploy/proseo-images/proseo-components/proseo-storage-mgr/Dockerfile @@ -1,3 +1,3 @@ -FROM localhost:5000/proseo-storage-mgr:1.0.0 +FROM localhost:5000/proseo-storage-mgr:1.0.1-SNAPSHOT ADD application.yml /usr/share/proseo-storage-mgr/application.yml WORKDIR /usr/share/proseo-storage-mgr/ \ No newline at end of file diff --git a/deploy/proseo-images/proseo-components/proseo-ui-gui/Dockerfile b/deploy/proseo-images/proseo-components/proseo-ui-gui/Dockerfile index fa9d0fa07..426996aba 100644 --- a/deploy/proseo-images/proseo-components/proseo-ui-gui/Dockerfile +++ b/deploy/proseo-images/proseo-components/proseo-ui-gui/Dockerfile @@ -1,3 +1,3 @@ -FROM localhost:5000/proseo-ui-gui:1.0.0 +FROM localhost:5000/proseo-ui-gui:1.0.1-SNAPSHOT ADD application.yml /usr/share/proseo-ui-gui/application.yml WORKDIR /usr/share/proseo-ui-gui/ \ No newline at end of file diff --git a/deploy/proseo-images/proseo-components/proseo-user-mgr/Dockerfile b/deploy/proseo-images/proseo-components/proseo-user-mgr/Dockerfile index efb4d865e..ea92bb175 100644 --- a/deploy/proseo-images/proseo-components/proseo-user-mgr/Dockerfile +++ b/deploy/proseo-images/proseo-components/proseo-user-mgr/Dockerfile @@ -1,3 +1,3 @@ -FROM localhost:5000/proseo-user-mgr:1.0.0 +FROM localhost:5000/proseo-user-mgr:1.0.1-SNAPSHOT ADD application.yml /usr/share/proseo-user-mgr/application.yml WORKDIR /usr/share/proseo-user-mgr/ diff --git a/docs/apidocs/options b/docs/apidocs/options index 201eb812d..037f69158 100644 --- a/docs/apidocs/options +++ b/docs/apidocs/options @@ -16,10 +16,10 @@ -docencoding 'UTF-8' -doctitle -'prosEO 1.0.0 API' +'prosEO 1.0.1-SNAPSHOT API' -linkoffline 'https://docs.oracle.com/en/java/javase/11/docs/api' '/Users/thomas/Documents/Thomas Beruf/Managementberatung/30 Kunden/DLR-EOC/Copernicus/S5P/S5P Processing Subsystem/prosEO/target/javadoc-bundle-options' -use -version -windowtitle -'prosEO 1.0.0 API' \ No newline at end of file +'prosEO 1.0.1-SNAPSHOT API' \ No newline at end of file diff --git a/docs/index.html b/docs/index.html index 6bc7a8ea2..ec0781877 100644 --- a/docs/index.html +++ b/docs/index.html @@ -23,7 +23,7 @@