From 7a1c6ac6d7b9c334208bb22fd9c114a7de18d1c0 Mon Sep 17 00:00:00 2001 From: "igor.petrenko" Date: Tue, 23 Apr 2024 19:45:44 +0300 Subject: [PATCH 1/5] OAP-125 IoStreams#out: add parameter throwError if file exists --- .../src/test/java/oap/io/IoStreamsTest.java | 16 ++++ .../src/main/java/oap/io/IoStreams.java | 94 ++++++++++++++++--- .../java/oap/io/SafeFileOutputStream.java | 6 +- pom.xml | 2 +- 4 files changed, 104 insertions(+), 14 deletions(-) diff --git a/oap-stdlib-test/src/test/java/oap/io/IoStreamsTest.java b/oap-stdlib-test/src/test/java/oap/io/IoStreamsTest.java index e3f582bb17..9d2b9f97f7 100644 --- a/oap-stdlib-test/src/test/java/oap/io/IoStreamsTest.java +++ b/oap-stdlib-test/src/test/java/oap/io/IoStreamsTest.java @@ -31,6 +31,7 @@ import oap.testng.TestDirectoryFixture; import oap.util.Arrays; import oap.util.Lists; +import org.apache.commons.io.FileExistsException; import org.apache.commons.lang3.RandomStringUtils; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -38,6 +39,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.UncheckedIOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -53,6 +55,7 @@ import static oap.testng.Asserts.assertFile; import static oap.testng.Asserts.pathOfTestResource; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class IoStreamsTest extends Fixtures { private final TestDirectoryFixture testDirectoryFixture; @@ -151,4 +154,17 @@ public void compressionLevel() { System.out.println( encoding + ":\t" + content.length() + " -> " + path.toFile().length() ); } } + + @Test + public void testThrowIfFileExists() throws IOException { + Path testFile = testDirectoryFixture.testPath( "testFile" ); + testFile.toFile().createNewFile(); + + assertThatThrownBy( () -> { + try( var _ = IoStreams.out( testFile, new IoStreams.OutOptions().withThrowIfFileExists( true ) ) ) { + System.out.println( "!" ); + } + } ).isInstanceOf( UncheckedIOException.class ) + .hasCauseInstanceOf( FileExistsException.class ); + } } diff --git a/oap-stdlib/src/main/java/oap/io/IoStreams.java b/oap-stdlib/src/main/java/oap/io/IoStreams.java index 4345c0e33c..19e25b872c 100644 --- a/oap-stdlib/src/main/java/oap/io/IoStreams.java +++ b/oap-stdlib/src/main/java/oap/io/IoStreams.java @@ -23,17 +23,20 @@ */ package oap.io; +import com.google.common.base.Preconditions; import com.google.common.io.ByteStreams; import io.airlift.compress.bzip2.BZip2HadoopStreams; import io.airlift.compress.gzip.JdkGzipHadoopStreams; import io.airlift.compress.lz4.Lz4HadoopStreams; import io.airlift.compress.zstd.ZstdHadoopStreams; import lombok.SneakyThrows; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; import oap.io.ProgressInputStream.Progress; import oap.util.Stream; import oap.util.Strings; import oap.util.function.Try; +import org.apache.commons.io.FileExistsException; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; @@ -57,7 +60,6 @@ import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; -import static com.google.common.base.Preconditions.checkArgument; import static java.nio.charset.StandardCharsets.UTF_8; import static oap.io.ProgressInputStream.progress; import static oap.util.function.Functions.empty.consume; @@ -92,7 +94,7 @@ public static Stream lines( Path path ) { } public static Stream lines( Path path, Encoding encoding ) { - return lines( path, encoding, p -> {} ); + return lines( path, encoding, _ -> {} ); } public static Stream lines( Path path, Encoding encoding, Consumer progress ) { @@ -199,22 +201,45 @@ public static OutputStream out( Path path, Encoding encoding, int bufferSize, bo return out( path, encoding, bufferSize, append, false ); } - @SneakyThrows public static OutputStream out( Path path, Encoding encoding, int bufferSize, boolean append, boolean safe ) { - checkArgument( !append || encoding.appendable, encoding + " is not appendable" ); + return out( path, new OutOptions() + .withBufferSize( bufferSize ).withEncoding( encoding ).withAppend( append ) + .withSafe( safe ) ); + } + + @SneakyThrows + public static OutputStream out( Path path, OutOptions options ) { + if( options.encoding == null ) { + options.withEncodingFrom( path ); + } + + Preconditions.checkArgument( !options.throwIfFileExists || !options.append ); + Preconditions.checkArgument( !options.append || options.encoding.appendable, options.encoding + " is not appendable" ); + Files.ensureFile( path ); - if( append ) Files.ensureFileEncodingValid( path ); - OutputStream outputStream = safe - ? new SafeFileOutputStream( path, append, encoding ) - : new FileOutputStream( path.toFile(), append ); + if( options.append ) Files.ensureFileEncodingValid( path ); + + if( options.throwIfFileExists ) { + Path filePath = options.safe ? SafeFileOutputStream.getUnsafePath( path ) : path; + + if( !filePath.toFile().createNewFile() ) { + throw new UncheckedIOException( new FileExistsException( path.toFile() ) ); + } + } + + OutputStream outputStream = options.safe + ? new SafeFileOutputStream( path, options.append, options.encoding ) + : new FileOutputStream( path.toFile(), options.append ); + OutputStream fos = - bufferSize > 0 && encoding != Encoding.GZIP ? new BufferedOutputStream( outputStream, bufferSize ) + options.bufferSize > 0 && options.encoding != Encoding.GZIP + ? new BufferedOutputStream( outputStream, options.bufferSize ) : outputStream; - return switch( encoding ) { + return switch( options.encoding ) { case GZIP -> GZIP_HADOOP_STREAMS.createOutputStream( fos ); case BZIP2 -> { OutputStream os = BZIP2_HADOOP_STREAMS.createOutputStream( fos ); - yield bufferSize > 0 ? new BufferedOutputStream( os, bufferSize ) : os; + yield options.bufferSize > 0 ? new BufferedOutputStream( os, options.bufferSize ) : os; } case ZIP -> { var zip = new ZipOutputStream( fos ); @@ -224,7 +249,7 @@ public static OutputStream out( Path path, Encoding encoding, int bufferSize, bo case LZ4 -> LZ4_HADOOP_STREAMS.createOutputStream( fos ); case ZSTD -> { OutputStream os = ZSTD_HADOOP_STREAMS.createOutputStream( fos ); - yield bufferSize > 0 ? new BufferedOutputStream( os, bufferSize ) : os; + yield options.bufferSize > 0 ? new BufferedOutputStream( os, options.bufferSize ) : os; } case PLAIN, ORC, PARQUET, AVRO -> fos; }; @@ -370,4 +395,49 @@ public Path resolve( Path path ) { return Paths.get( s.substring( 0, s.length() - from( path ).extension.length() ) + extension ); } } + + @ToString + public static class OutOptions { + protected int bufferSize = DEFAULT_BUFFER; + protected Encoding encoding; + protected boolean append = false; + protected boolean safe = false; + protected boolean throwIfFileExists = false; + + public OutOptions withBufferSize( int bufferSize ) { + this.bufferSize = bufferSize; + + return this; + } + + public OutOptions withEncoding( Encoding encoding ) { + this.encoding = encoding; + + return this; + } + + public OutOptions withEncodingFrom( Path path ) { + this.encoding = Encoding.from( path ); + + return this; + } + + public OutOptions withAppend( boolean append ) { + this.append = append; + + return this; + } + + public OutOptions withSafe( boolean safe ) { + this.safe = safe; + + return this; + } + + public OutOptions withThrowIfFileExists( boolean throwIfFileExists ) { + this.throwIfFileExists = throwIfFileExists; + + return this; + } + } } diff --git a/oap-stdlib/src/main/java/oap/io/SafeFileOutputStream.java b/oap-stdlib/src/main/java/oap/io/SafeFileOutputStream.java index b5e073357d..57016b4fae 100644 --- a/oap-stdlib/src/main/java/oap/io/SafeFileOutputStream.java +++ b/oap-stdlib/src/main/java/oap/io/SafeFileOutputStream.java @@ -33,7 +33,7 @@ public class SafeFileOutputStream extends FileOutputStream { private final Path path; public SafeFileOutputStream( Path path, boolean append, IoStreams.Encoding encoding ) throws IOException { - super( path + ".unsafe", append ); + super( getUnsafePath( path ).toFile(), append ); this.path = path; if( append && java.nio.file.Files.exists( path ) ) { @@ -46,6 +46,10 @@ public SafeFileOutputStream( Path path, boolean append, IoStreams.Encoding encod } } + public static Path getUnsafePath( Path path ) { + return Paths.get( path.toString() + ".unsafe" ); + } + @Override public void close() throws IOException { // https://stackoverflow.com/questions/25910173/fileoutputstream-close-is-not-always-writing-bytes-to-file-system diff --git a/pom.xml b/pom.xml index e7f8e92716..bd813b92fd 100644 --- a/pom.xml +++ b/pom.xml @@ -70,7 +70,7 @@ - 22.1.1 + 22.1.2 21.0.0 21.0.1 From ecfbb834a11caacfc8022c872f755802e37d7656 Mon Sep 17 00:00:00 2001 From: "igor.petrenko" Date: Tue, 23 Apr 2024 20:00:13 +0300 Subject: [PATCH 2/5] OAP-125 IoStreams#out: add parameter throwError if file exists --- .../test/java/oap/ws/sso/JwtTokenGeneratorExtractorTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/oap-ws/oap-ws-sso-api/src/test/java/oap/ws/sso/JwtTokenGeneratorExtractorTest.java b/oap-ws/oap-ws-sso-api/src/test/java/oap/ws/sso/JwtTokenGeneratorExtractorTest.java index 818f2f380d..23c4919378 100644 --- a/oap-ws/oap-ws-sso-api/src/test/java/oap/ws/sso/JwtTokenGeneratorExtractorTest.java +++ b/oap-ws/oap-ws-sso-api/src/test/java/oap/ws/sso/JwtTokenGeneratorExtractorTest.java @@ -26,11 +26,11 @@ import oap.testng.Fixtures; import oap.testng.SystemTimerFixture; -import oap.util.Dates; import oap.util.Pair; import oap.ws.sso.AbstractUserTest.TestSecurityRolesProvider; import oap.ws.sso.AbstractUserTest.TestUser; import org.joda.time.DateTime; +import org.joda.time.DateTimeUtils; import org.testng.annotations.Test; import java.util.Date; @@ -52,8 +52,7 @@ public JwtTokenGeneratorExtractorTest() { @Test public void generateAndExtractToken() { - Dates.setTimeFixed( 2024, 4, 23, 14, 45, 56, 12 ); - + DateTimeUtils.setCurrentMillisFixed( DateTimeUtils.currentTimeMillis() ); Pair token = jwtTokenGenerator.generateAccessToken( new TestUser( "email@email.com", "password", Pair.of( "org1", "ADMIN" ) ) ); assertNotNull( token._1 ); From 4165e554fa0caef4be933540eaff7a47c758d31d Mon Sep 17 00:00:00 2001 From: "igor.petrenko" Date: Wed, 24 Apr 2024 11:11:49 +0300 Subject: [PATCH 3/5] OAP-125 IoStreams#out: add parameter throwError if file exists --- .../test/java/oap/logstream/LoggerTest.java | 2 +- .../main/java/oap/io/FileExistsException.java | 13 + .../java/oap/io/FileNotFoundException.java | 11 + oap-stdlib/src/main/java/oap/io/Files.java | 23 +- .../src/main/java/oap/io/IOException.java | 18 ++ .../src/main/java/oap/io/IoStreams.java | 290 +++++++++--------- .../java/oap/io/MalformedURLException.java | 7 + oap-stdlib/src/main/java/oap/io/Paths.java | 12 +- .../src/main/java/oap/io/Resources.java | 8 +- .../src/main/java/oap/util/Throwables.java | 16 +- 10 files changed, 234 insertions(+), 166 deletions(-) create mode 100644 oap-stdlib/src/main/java/oap/io/FileExistsException.java create mode 100644 oap-stdlib/src/main/java/oap/io/FileNotFoundException.java create mode 100644 oap-stdlib/src/main/java/oap/io/IOException.java create mode 100644 oap-stdlib/src/main/java/oap/io/MalformedURLException.java diff --git a/oap-formats/oap-logstream/oap-logstream-test/src/test/java/oap/logstream/LoggerTest.java b/oap-formats/oap-logstream/oap-logstream-test/src/test/java/oap/logstream/LoggerTest.java index 777f6366bd..fc3fe8b4b3 100644 --- a/oap-formats/oap-logstream/oap-logstream-test/src/test/java/oap/logstream/LoggerTest.java +++ b/oap-formats/oap-logstream/oap-logstream-test/src/test/java/oap/logstream/LoggerTest.java @@ -29,8 +29,8 @@ import oap.logstream.disk.DiskLoggerBackend; import oap.logstream.net.client.SocketLoggerBackend; import oap.logstream.net.server.SocketLoggerServer; -import oap.message.server.MessageHttpHandler; import oap.message.client.MessageSender; +import oap.message.server.MessageHttpHandler; import oap.template.BinaryUtils; import oap.template.Types; import oap.testng.Fixtures; diff --git a/oap-stdlib/src/main/java/oap/io/FileExistsException.java b/oap-stdlib/src/main/java/oap/io/FileExistsException.java new file mode 100644 index 0000000000..672ce45594 --- /dev/null +++ b/oap-stdlib/src/main/java/oap/io/FileExistsException.java @@ -0,0 +1,13 @@ +package oap.io; + +import java.io.File; + +public class FileExistsException extends IOException { + public FileExistsException( File file ) { + super( "File " + file + " exists" ); + } + + public FileExistsException( IOException cause ) { + super( cause ); + } +} diff --git a/oap-stdlib/src/main/java/oap/io/FileNotFoundException.java b/oap-stdlib/src/main/java/oap/io/FileNotFoundException.java new file mode 100644 index 0000000000..7e291eb57c --- /dev/null +++ b/oap-stdlib/src/main/java/oap/io/FileNotFoundException.java @@ -0,0 +1,11 @@ +package oap.io; + +public class FileNotFoundException extends IOException { + public FileNotFoundException( String message ) { + super( message ); + } + + public FileNotFoundException( java.io.FileNotFoundException cause ) { + super( cause ); + } +} diff --git a/oap-stdlib/src/main/java/oap/io/Files.java b/oap-stdlib/src/main/java/oap/io/Files.java index 89b72a732a..19b5092a7c 100644 --- a/oap-stdlib/src/main/java/oap/io/Files.java +++ b/oap-stdlib/src/main/java/oap/io/Files.java @@ -57,7 +57,6 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; -import java.io.UncheckedIOException; import java.net.MalformedURLException; import java.net.URL; import java.nio.file.CopyOption; @@ -509,7 +508,7 @@ public static void rename( Path sourcePath, Path destPath ) { ensureFile( destPath ); java.nio.file.Files.move( sourcePath, destPath, ATOMIC_MOVE, REPLACE_EXISTING ); } catch( IOException e ) { - throw new UncheckedIOException( "cannot rename " + sourcePath + " to " + destPath, e ); + throw new oap.io.IOException( "cannot rename " + sourcePath + " to " + destPath, e ); } } @@ -526,7 +525,7 @@ public static Path ensureDirectory( Path path ) { } } - public static void move( Path source, Path target, CopyOption... options ) { + public static void move( Path source, Path target, CopyOption... options ) throws oap.io.IOException { try { ensureFile( target ); java.nio.file.Files.move( source, target, options ); @@ -535,7 +534,7 @@ public static void move( Path source, Path target, CopyOption... options ) { } } - public static void setPosixPermissions( Path path, Set permissions ) { + public static void setPosixPermissions( Path path, Set permissions ) throws oap.io.IOException { try { java.nio.file.Files.setPosixFilePermissions( path, permissions ); } catch( IOException e ) { @@ -543,11 +542,11 @@ public static void setPosixPermissions( Path path, Set perm } } - public static void setPosixPermissions( Path path, PosixFilePermission... permissions ) { + public static void setPosixPermissions( Path path, PosixFilePermission... permissions ) throws oap.io.IOException { setPosixPermissions( path, Sets.of( permissions ) ); } - public static Set getPosixPermissions( Path path ) { + public static Set getPosixPermissions( Path path ) throws oap.io.IOException { try { return java.nio.file.Files.getPosixFilePermissions( path, LinkOption.NOFOLLOW_LINKS ); } catch( IOException e ) { @@ -555,7 +554,7 @@ public static Set getPosixPermissions( Path path ) { } } - public static boolean isDirectoryEmpty( Path directory ) { + public static boolean isDirectoryEmpty( Path directory ) throws oap.io.IOException { try( DirectoryStream dirStream = java.nio.file.Files.newDirectoryStream( directory ) ) { return !dirStream.iterator().hasNext(); } catch( IOException e ) { @@ -563,11 +562,11 @@ public static boolean isDirectoryEmpty( Path directory ) { } } - public static void setLastModifiedTime( Path path, DateTime dateTime ) { + public static void setLastModifiedTime( Path path, DateTime dateTime ) throws oap.io.IOException { setLastModifiedTime( path, dateTime.getMillis() ); } - public static void setLastModifiedTime( Path path, long ms ) { + public static void setLastModifiedTime( Path path, long ms ) throws oap.io.IOException { try { java.nio.file.Files.setLastModifiedTime( path, FileTime.fromMillis( ms ) ); } catch( IOException e ) { @@ -654,7 +653,7 @@ public static boolean fileNotEmpty( Path path ) { return isFileNotEmpty( path ); } - public static boolean isFileNotEmpty( final Path path ) { + public static boolean isFileNotEmpty( final Path path ) throws oap.io.IOException { try( InputStream is = IoStreams.in( path ); InputStreamReader isr = new InputStreamReader( is ); BufferedReader reader = new BufferedReader( isr ) ) { @@ -670,7 +669,7 @@ public static boolean exists( Path path ) { return java.nio.file.Files.exists( path ); } - public static long getLastModifiedTime( Path path ) { + public static long getLastModifiedTime( Path path ) throws oap.io.IOException { try { return java.nio.file.Files.getLastModifiedTime( path ).toMillis(); } catch( IOException e ) { @@ -678,7 +677,7 @@ public static long getLastModifiedTime( Path path ) { } } - public static boolean createFile( Path file ) { + public static boolean createFile( Path file ) throws oap.io.IOException { try { java.nio.file.Files.createFile( file ); return true; diff --git a/oap-stdlib/src/main/java/oap/io/IOException.java b/oap-stdlib/src/main/java/oap/io/IOException.java new file mode 100644 index 0000000000..1a1dd5f4cf --- /dev/null +++ b/oap-stdlib/src/main/java/oap/io/IOException.java @@ -0,0 +1,18 @@ +package oap.io; + +public class IOException extends RuntimeException { + public IOException() { + } + + public IOException( String message ) { + super( message ); + } + + public IOException( String message, Throwable cause ) { + super( message, cause ); + } + + public IOException( Throwable cause ) { + super( cause ); + } +} diff --git a/oap-stdlib/src/main/java/oap/io/IoStreams.java b/oap-stdlib/src/main/java/oap/io/IoStreams.java index 19e25b872c..adb1bf29db 100644 --- a/oap-stdlib/src/main/java/oap/io/IoStreams.java +++ b/oap-stdlib/src/main/java/oap/io/IoStreams.java @@ -29,14 +29,13 @@ import io.airlift.compress.gzip.JdkGzipHadoopStreams; import io.airlift.compress.lz4.Lz4HadoopStreams; import io.airlift.compress.zstd.ZstdHadoopStreams; -import lombok.SneakyThrows; import lombok.ToString; import lombok.extern.slf4j.Slf4j; import oap.io.ProgressInputStream.Progress; import oap.util.Stream; import oap.util.Strings; +import oap.util.Throwables; import oap.util.function.Try; -import org.apache.commons.io.FileExistsException; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; @@ -50,7 +49,6 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; -import java.io.UncheckedIOException; import java.net.URL; import java.net.URLConnection; import java.nio.file.Path; @@ -72,42 +70,44 @@ public class IoStreams { public static final ZstdHadoopStreams ZSTD_HADOOP_STREAMS = new ZstdHadoopStreams(); public static final JdkGzipHadoopStreams GZIP_HADOOP_STREAMS = new JdkGzipHadoopStreams(); - public static Stream lines( URL url ) { + public static Stream lines( URL url ) throws oap.io.IOException { return lines( url, Encoding.from( url ), consume() ); } - public static Stream lines( URL url, Consumer progress ) { + public static Stream lines( URL url, Consumer progress ) throws oap.io.IOException { return lines( url, Encoding.from( url ), progress ); } - @SneakyThrows - public static Stream lines( URL url, Encoding encoding, Consumer progress ) { - log.trace( "loading {}...", url ); - URLConnection connection = url.openConnection(); - InputStream stream = connection.getInputStream(); - return lines( stream, encoding, progress( connection.getContentLengthLong(), progress ) ) - .onClose( Try.run( stream::close ) ); + public static Stream lines( URL url, Encoding encoding, Consumer progress ) throws oap.io.IOException { + try { + log.trace( "loading {}...", url ); + URLConnection connection = url.openConnection(); + InputStream stream = connection.getInputStream(); + return lines( stream, encoding, progress( connection.getContentLengthLong(), progress ) ).onClose( Try.run( stream::close ) ); + } catch( IOException e ) { + throw Throwables.propagate( e ); + } } - public static Stream lines( Path path ) { + public static Stream lines( Path path ) throws oap.io.IOException { return lines( path, Encoding.from( path ), consume() ); } - public static Stream lines( Path path, Encoding encoding ) { + public static Stream lines( Path path, Encoding encoding ) throws oap.io.IOException { return lines( path, encoding, _ -> {} ); } - public static Stream lines( Path path, Encoding encoding, Consumer progress ) { + public static Stream lines( Path path, Encoding encoding, Consumer progress ) throws oap.io.IOException { InputStream stream = in( path, Encoding.PLAIN ); return lines( stream, encoding, progress( path.toFile().length(), progress ) ) .onClose( Try.run( stream::close ) ); } - private static Stream lines( InputStream stream, Encoding encoding, Progress progress ) { + private static Stream lines( InputStream stream, Encoding encoding, Progress progress ) throws oap.io.IOException { return lines( in( new ProgressInputStream( stream, progress ), encoding ) ); } - public static Stream lines( InputStream stream ) { + public static Stream lines( InputStream stream ) throws oap.io.IOException { return lines( stream, false ); } @@ -121,7 +121,7 @@ public static Stream lines( InputStream stream, boolean autoClose ) { return Stream.of( ustream ); } - public static void write( Path path, Encoding encoding, java.util.stream.Stream lines ) throws UncheckedIOException { + public static void write( Path path, Encoding encoding, java.util.stream.Stream lines ) throws oap.io.IOException { Files.ensureFile( path ); try( OutputStream out = out( path, encoding, DEFAULT_BUFFER, false, false ) ) { @@ -130,227 +130,233 @@ public static void write( Path path, Encoding encoding, java.util.stream.Stream< out.write( line.getBytes() ); out.write( '\n' ); } catch( IOException e ) { - throw new UncheckedIOException( e ); + throw Throwables.propagate( e ); } } ); } catch( IOException e ) { - throw new UncheckedIOException( e ); + throw Throwables.propagate( e ); } } - public static void write( Path path, Encoding encoding, String value ) throws UncheckedIOException { + public static void write( Path path, Encoding encoding, String value ) throws oap.io.IOException { write( path, encoding, value, false ); } - public static void write( Path path, Encoding encoding, InputStream in ) throws UncheckedIOException { + public static void write( Path path, Encoding encoding, InputStream in ) throws oap.io.IOException { write( path, encoding, in, Progress.EMPTY ); } - public static void write( Path path, Encoding encoding, InputStream in, Progress progress ) throws UncheckedIOException { + public static void write( Path path, Encoding encoding, InputStream in, Progress progress ) throws oap.io.IOException { write( path, encoding, in, false, false, progress ); } - public static void write( Path path, Encoding encoding, InputStream in, Progress progress, boolean append ) throws UncheckedIOException { + public static void write( Path path, Encoding encoding, InputStream in, Progress progress, boolean append ) throws oap.io.IOException { write( path, encoding, in, append, false, progress ); } - public static void write( Path path, Encoding encoding, String value, boolean append ) throws UncheckedIOException { + public static void write( Path path, Encoding encoding, String value, boolean append ) throws oap.io.IOException { write( path, encoding, new ByteArrayInputStream( Strings.toByteArray( value ) ), append, false, Progress.EMPTY ); - } - public static void write( Path path, Encoding encoding, InputStream in, boolean append, boolean safe, Progress progress ) throws UncheckedIOException { + public static void write( Path path, Encoding encoding, InputStream in, boolean append, boolean safe, Progress progress ) throws oap.io.IOException { Files.ensureFile( path ); try( OutputStream out = out( path, encoding, DEFAULT_BUFFER, append, safe ) ) { ByteStreams.copy( new ProgressInputStream( in, progress ), out ); } catch( IOException e ) { - throw new UncheckedIOException( e ); + throw Throwables.propagate( e ); } } - public static void write( Path path, Encoding encoding, InputStream in, boolean append, boolean safe ) { + public static void write( Path path, Encoding encoding, InputStream in, boolean append, boolean safe ) throws oap.io.IOException { Files.ensureFile( path ); try( OutputStream out = out( path, encoding, DEFAULT_BUFFER, append, safe ) ) { ByteStreams.copy( in, out ); } catch( IOException e ) { - throw new UncheckedIOException( e ); + throw Throwables.propagate( e ); } } - public static FixedLengthArrayOutputStream out( byte[] bytes ) { + public static FixedLengthArrayOutputStream out( byte[] bytes ) throws oap.io.IOException { return new FixedLengthArrayOutputStream( bytes ); } - public static OutputStream out( Path path ) { + public static OutputStream out( Path path ) throws oap.io.IOException { return out( path, Encoding.from( path ), DEFAULT_BUFFER ); } - public static OutputStream out( Path path, Encoding encoding ) { + public static OutputStream out( Path path, Encoding encoding ) throws oap.io.IOException { return out( path, encoding, DEFAULT_BUFFER ); } - public static OutputStream out( Path path, Encoding encoding, int bufferSize ) { + public static OutputStream out( Path path, Encoding encoding, int bufferSize ) throws oap.io.IOException { return out( path, encoding, bufferSize, false ); } - public static OutputStream out( Path path, Encoding encoding, boolean append ) { + public static OutputStream out( Path path, Encoding encoding, boolean append ) throws oap.io.IOException { return out( path, encoding, DEFAULT_BUFFER, append ); } - public static OutputStream out( Path path, Encoding encoding, int bufferSize, boolean append ) { + public static OutputStream out( Path path, Encoding encoding, int bufferSize, boolean append ) throws oap.io.IOException { return out( path, encoding, bufferSize, append, false ); } - public static OutputStream out( Path path, Encoding encoding, int bufferSize, boolean append, boolean safe ) { + public static OutputStream out( Path path, Encoding encoding, int bufferSize, boolean append, boolean safe ) throws oap.io.IOException { return out( path, new OutOptions() .withBufferSize( bufferSize ).withEncoding( encoding ).withAppend( append ) .withSafe( safe ) ); } - @SneakyThrows - public static OutputStream out( Path path, OutOptions options ) { - if( options.encoding == null ) { - options.withEncodingFrom( path ); - } + public static OutputStream out( Path path, OutOptions options ) throws oap.io.IOException { + try { + if( options.encoding == null ) { + options.withEncodingFrom( path ); + } - Preconditions.checkArgument( !options.throwIfFileExists || !options.append ); - Preconditions.checkArgument( !options.append || options.encoding.appendable, options.encoding + " is not appendable" ); + Preconditions.checkArgument( !options.throwIfFileExists || !options.append ); + Preconditions.checkArgument( !options.append || options.encoding.appendable, options.encoding + " is not appendable" ); - Files.ensureFile( path ); - if( options.append ) Files.ensureFileEncodingValid( path ); + Files.ensureFile( path ); + if( options.append ) Files.ensureFileEncodingValid( path ); - if( options.throwIfFileExists ) { - Path filePath = options.safe ? SafeFileOutputStream.getUnsafePath( path ) : path; + if( options.throwIfFileExists ) { + Path filePath = options.safe ? SafeFileOutputStream.getUnsafePath( path ) : path; - if( !filePath.toFile().createNewFile() ) { - throw new UncheckedIOException( new FileExistsException( path.toFile() ) ); + if( !filePath.toFile().createNewFile() ) { + throw new FileExistsException( path.toFile() ); + } } - } - OutputStream outputStream = options.safe - ? new SafeFileOutputStream( path, options.append, options.encoding ) - : new FileOutputStream( path.toFile(), options.append ); - - OutputStream fos = - options.bufferSize > 0 && options.encoding != Encoding.GZIP - ? new BufferedOutputStream( outputStream, options.bufferSize ) - : outputStream; - return switch( options.encoding ) { - case GZIP -> GZIP_HADOOP_STREAMS.createOutputStream( fos ); - case BZIP2 -> { - OutputStream os = BZIP2_HADOOP_STREAMS.createOutputStream( fos ); - yield options.bufferSize > 0 ? new BufferedOutputStream( os, options.bufferSize ) : os; - } - case ZIP -> { - var zip = new ZipOutputStream( fos ); - zip.putNextEntry( new ZipEntry( path.getFileName().toString() ) ); - yield zip; - } - case LZ4 -> LZ4_HADOOP_STREAMS.createOutputStream( fos ); - case ZSTD -> { - OutputStream os = ZSTD_HADOOP_STREAMS.createOutputStream( fos ); - yield options.bufferSize > 0 ? new BufferedOutputStream( os, options.bufferSize ) : os; - } - case PLAIN, ORC, PARQUET, AVRO -> fos; - }; + OutputStream outputStream = options.safe + ? new SafeFileOutputStream( path, options.append, options.encoding ) + : new FileOutputStream( path.toFile(), options.append ); + + OutputStream fos = + options.bufferSize > 0 && options.encoding != Encoding.GZIP + ? new BufferedOutputStream( outputStream, options.bufferSize ) + : outputStream; + return switch( options.encoding ) { + case GZIP -> GZIP_HADOOP_STREAMS.createOutputStream( fos ); + case BZIP2 -> { + OutputStream os = BZIP2_HADOOP_STREAMS.createOutputStream( fos ); + yield options.bufferSize > 0 ? new BufferedOutputStream( os, options.bufferSize ) : os; + } + case ZIP -> { + ZipOutputStream zip = new ZipOutputStream( fos ); + zip.putNextEntry( new ZipEntry( path.getFileName().toString() ) ); + yield zip; + } + case LZ4 -> LZ4_HADOOP_STREAMS.createOutputStream( fos ); + case ZSTD -> { + OutputStream os = ZSTD_HADOOP_STREAMS.createOutputStream( fos ); + yield options.bufferSize > 0 ? new BufferedOutputStream( os, options.bufferSize ) : os; + } + case PLAIN, ORC, PARQUET, AVRO -> fos; + }; + } catch( IOException e ) { + throw Throwables.propagate( e ); + } } - public static InputStream in( Path path, Encoding encoding ) { + public static InputStream in( Path path, Encoding encoding ) throws oap.io.IOException { return in( path, encoding, DEFAULT_BUFFER ); } - public static InputStream in( URL url, Encoding encoding ) { + public static InputStream in( URL url, Encoding encoding ) throws oap.io.IOException { return in( url, encoding, DEFAULT_BUFFER ); } - public static InputStream in( Path path ) { + public static InputStream in( Path path ) throws oap.io.IOException { return in( path, DEFAULT_BUFFER ); } - public static InputStream in( URL url ) { + public static InputStream in( URL url ) throws oap.io.IOException { return in( url, DEFAULT_BUFFER ); } - public static InputStream in( Path path, int bufferSIze ) { + public static InputStream in( Path path, int bufferSIze ) throws oap.io.IOException { return in( path, Encoding.from( path ), bufferSIze ); } - public static InputStream in( URL url, int bufferSIze ) { + public static InputStream in( URL url, int bufferSIze ) throws oap.io.IOException { return in( url, Encoding.from( url ), bufferSIze ); } - public static InputStream in( Path path, Encoding encoding, int bufferSize ) { + public static InputStream in( Path path, Encoding encoding, int bufferSize ) throws oap.io.IOException { try { FileInputStream fileInputStream = new FileInputStream( path.toFile() ); - return decoded( - bufferSize > 0 ? new BufferedInputStream( fileInputStream, bufferSize ) : fileInputStream, encoding ); + return decoded( bufferSize > 0 ? new BufferedInputStream( fileInputStream, bufferSize ) : fileInputStream, encoding ); } catch( IOException e ) { - throw new UncheckedIOException( "couldn't open file " + path, e ); + throw new oap.io.IOException( "couldn't open file " + path, e ); } } - public static InputStream in( URL url, Encoding encoding, int bufferSize ) { + public static InputStream in( URL url, Encoding encoding, int bufferSize ) throws oap.io.IOException { try { - var inputStream = url.openStream(); - return decoded( - bufferSize > 0 ? new BufferedInputStream( inputStream, bufferSize ) : inputStream, encoding ); + InputStream inputStream = url.openStream(); + return decoded( bufferSize > 0 ? new BufferedInputStream( inputStream, bufferSize ) : inputStream, encoding ); } catch( IOException e ) { - throw new UncheckedIOException( "couldn't open file " + url, e ); + throw new oap.io.IOException( "couldn't open file " + url, e ); } } - @SneakyThrows - public static InputStream in( InputStream stream, Encoding encoding ) { + public static InputStream in( InputStream stream, Encoding encoding ) throws oap.io.IOException { return decoded( stream, encoding ); } - @SneakyThrows - public static String asString( InputStream stream, Encoding encoding ) { - return IOUtils.toString( decoded( stream, encoding ), UTF_8 ); + public static String asString( InputStream stream, Encoding encoding ) throws oap.io.IOException { + try { + return IOUtils.toString( decoded( stream, encoding ), UTF_8 ); + } catch( IOException e ) { + throw Throwables.propagate( e ); + } } - @SneakyThrows - private static InputStream decoded( InputStream stream, Encoding encoding ) { - switch( encoding ) { - case GZIP: - try { - return GZIP_HADOOP_STREAMS.createInputStream( stream ); - } catch( Exception e ) { - stream.close(); - } - case BZIP2: - try { - return BZIP2_HADOOP_STREAMS.createInputStream( stream ); - } catch( Exception e ) { - stream.close(); - } - case ZIP: - try { - ZipInputStream zip = new ZipInputStream( stream ); - if( zip.getNextEntry() == null ) - throw new IllegalArgumentException( "zip stream contains no entries" ); - return zip; - } catch( Exception e ) { - stream.close(); - } - case PLAIN, ORC, PARQUET, AVRO: - return stream; - case LZ4: - try { - return LZ4_HADOOP_STREAMS.createInputStream( stream ); - } catch( Exception e ) { - stream.close(); - throw e; - } - case ZSTD: - try { - return ZSTD_HADOOP_STREAMS.createInputStream( stream ); - } catch( Exception e ) { - stream.close(); - throw e; - } - default: - throw new IllegalArgumentException( "Unknown encoding " + encoding ); + private static InputStream decoded( InputStream stream, Encoding encoding ) throws oap.io.IOException { + try { + switch( encoding ) { + case GZIP: + try { + return GZIP_HADOOP_STREAMS.createInputStream( stream ); + } catch( Exception e ) { + stream.close(); + } + case BZIP2: + try { + return BZIP2_HADOOP_STREAMS.createInputStream( stream ); + } catch( Exception e ) { + stream.close(); + } + case ZIP: + try { + ZipInputStream zip = new ZipInputStream( stream ); + if( zip.getNextEntry() == null ) { + throw new IllegalArgumentException( "zip stream contains no entries" ); + } + return zip; + } catch( Exception e ) { + stream.close(); + } + case PLAIN, ORC, PARQUET, AVRO: + return stream; + case LZ4: + try { + return LZ4_HADOOP_STREAMS.createInputStream( stream ); + } catch( Exception e ) { + stream.close(); + throw e; + } + case ZSTD: + try { + return ZSTD_HADOOP_STREAMS.createInputStream( stream ); + } catch( Exception e ) { + stream.close(); + throw e; + } + default: + throw new IllegalArgumentException( "Unknown encoding " + encoding ); + } + } catch( IOException e ) { + throw Throwables.propagate( e ); } } @@ -386,7 +392,11 @@ public static Encoding from( URL url ) { } public static Encoding from( String name ) { - for( var e : values() ) if( e.compressed && StringUtils.endsWithIgnoreCase( name, e.extension ) ) return e; + for( Encoding e : values() ) { + if( e.compressed && StringUtils.endsWithIgnoreCase( name, e.extension ) ) { + return e; + } + } return PLAIN; } diff --git a/oap-stdlib/src/main/java/oap/io/MalformedURLException.java b/oap-stdlib/src/main/java/oap/io/MalformedURLException.java new file mode 100644 index 0000000000..c79269e89b --- /dev/null +++ b/oap-stdlib/src/main/java/oap/io/MalformedURLException.java @@ -0,0 +1,7 @@ +package oap.io; + +public class MalformedURLException extends IOException { + public MalformedURLException( java.net.MalformedURLException cause ) { + super( cause ); + } +} diff --git a/oap-stdlib/src/main/java/oap/io/Paths.java b/oap-stdlib/src/main/java/oap/io/Paths.java index ae155ae87f..be4a71d1ff 100644 --- a/oap-stdlib/src/main/java/oap/io/Paths.java +++ b/oap-stdlib/src/main/java/oap/io/Paths.java @@ -24,14 +24,18 @@ package oap.io; -import lombok.SneakyThrows; +import oap.util.Throwables; +import java.net.MalformedURLException; import java.net.URL; import java.nio.file.Path; public class Paths { - @SneakyThrows - public static URL toUrl( Path path ) { - return path.toUri().toURL(); + public static URL toUrl( Path path ) throws oap.io.MalformedURLException { + try { + return path.toUri().toURL(); + } catch( MalformedURLException e ) { + throw Throwables.propagate( e ); + } } } diff --git a/oap-stdlib/src/main/java/oap/io/Resources.java b/oap-stdlib/src/main/java/oap/io/Resources.java index 350d79462d..fc0192becb 100644 --- a/oap-stdlib/src/main/java/oap/io/Resources.java +++ b/oap-stdlib/src/main/java/oap/io/Resources.java @@ -28,11 +28,11 @@ import oap.io.content.ContentReader; import oap.util.Lists; import oap.util.Stream; +import oap.util.Throwables; import oap.util.function.Try; import org.apache.commons.collections4.EnumerationUtils; import java.io.IOException; -import java.io.UncheckedIOException; import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; @@ -110,7 +110,7 @@ public static List readResourcesAsString( Class contextClass, String ret.add( ContentReader.read( is, ContentReader.ofString() ) ); } } catch( IOException e ) { - throw new UncheckedIOException( e ); + throw Throwables.propagate( e ); } return ret; } @@ -195,8 +195,8 @@ public static List urls( String atPackage, String... ext ) { @SneakyThrows public static List urls( Predicate filter ) { return Stream.of( ClassPath.from( Thread.currentThread() - .getContextClassLoader() ) - .getResources() ) + .getContextClassLoader() ) + .getResources() ) .filter( ri -> filter.test( ri.getResourceName() ) ) .map( ClassPath.ResourceInfo::url ) .toList(); diff --git a/oap-stdlib/src/main/java/oap/util/Throwables.java b/oap-stdlib/src/main/java/oap/util/Throwables.java index 8897891c22..928ccfa923 100644 --- a/oap-stdlib/src/main/java/oap/util/Throwables.java +++ b/oap-stdlib/src/main/java/oap/util/Throwables.java @@ -26,18 +26,24 @@ import lombok.Lombok; +import java.io.FileNotFoundException; import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.concurrent.ExecutionException; +import java.net.MalformedURLException; public final class Throwables { private Throwables() { } public static RuntimeException propagate( Throwable throwable ) { - if( throwable instanceof IOException ) throw new UncheckedIOException( ( IOException ) throwable ); - else if( throwable instanceof java.util.concurrent.ExecutionException ) throw new oap.concurrent.ExecutionException( ( ExecutionException ) throwable ); - else if( throwable instanceof java.util.concurrent.TimeoutException ) throw new oap.concurrent.TimeoutException( throwable ); + switch( throwable ) { + case FileNotFoundException fnf -> throw new oap.io.FileNotFoundException( fnf ); + case MalformedURLException murl -> throw new oap.io.MalformedURLException( murl ); + case IOException _ -> throw new oap.io.IOException( throwable ); + case java.util.concurrent.ExecutionException ee -> throw new oap.concurrent.ExecutionException( ee ); + case java.util.concurrent.TimeoutException _ -> throw new oap.concurrent.TimeoutException( throwable ); + case null, default -> { + } + } throw Lombok.sneakyThrow( throwable ); } From 8ee6a68cbbb3e8172d26f43ae1fb91059a42984b Mon Sep 17 00:00:00 2001 From: "igor.petrenko" Date: Wed, 24 Apr 2024 11:26:24 +0300 Subject: [PATCH 4/5] OAP-125 IoStreams#out: add parameter throwError if file exists --- .../main/java/oap/io/SafeFileOutputStream.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/oap-stdlib/src/main/java/oap/io/SafeFileOutputStream.java b/oap-stdlib/src/main/java/oap/io/SafeFileOutputStream.java index 57016b4fae..cdd22c8c96 100644 --- a/oap-stdlib/src/main/java/oap/io/SafeFileOutputStream.java +++ b/oap-stdlib/src/main/java/oap/io/SafeFileOutputStream.java @@ -23,12 +23,15 @@ */ package oap.io; +import lombok.extern.slf4j.Slf4j; + import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; import java.nio.file.Paths; +@Slf4j public class SafeFileOutputStream extends FileOutputStream { private final Path path; @@ -52,15 +55,17 @@ public static Path getUnsafePath( Path path ) { @Override public void close() throws IOException { - // https://stackoverflow.com/questions/25910173/fileoutputstream-close-is-not-always-writing-bytes-to-file-system - getFD().sync(); - super.close(); + try { + // https://stackoverflow.com/questions/25910173/fileoutputstream-close-is-not-always-writing-bytes-to-file-system + getFD().sync(); + super.close(); + } catch( IOException e ) { + log.error( e.getMessage(), e ); + } final Path unsafePath = Paths.get( this.path + ".unsafe" ); if( !java.nio.file.Files.exists( unsafePath ) || java.nio.file.Files.size( unsafePath ) == 0 ) Files.delete( unsafePath ); else Files.rename( unsafePath, this.path ); } - - } From c42fd964b45c60c6b6aedc1b4c986dfda9e65843 Mon Sep 17 00:00:00 2001 From: "igor.petrenko" Date: Wed, 24 Apr 2024 11:41:49 +0300 Subject: [PATCH 5/5] OAP-125 IoStreams#out: add parameter throwError if file exists --- oap-stdlib-test/src/test/java/oap/io/IoStreamsTest.java | 5 +---- oap-stdlib-test/src/test/java/oap/util/ThrowablesTest.java | 3 +-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/oap-stdlib-test/src/test/java/oap/io/IoStreamsTest.java b/oap-stdlib-test/src/test/java/oap/io/IoStreamsTest.java index 9d2b9f97f7..0cd48bd3b3 100644 --- a/oap-stdlib-test/src/test/java/oap/io/IoStreamsTest.java +++ b/oap-stdlib-test/src/test/java/oap/io/IoStreamsTest.java @@ -31,7 +31,6 @@ import oap.testng.TestDirectoryFixture; import oap.util.Arrays; import oap.util.Lists; -import org.apache.commons.io.FileExistsException; import org.apache.commons.lang3.RandomStringUtils; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -39,7 +38,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.UncheckedIOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -164,7 +162,6 @@ public void testThrowIfFileExists() throws IOException { try( var _ = IoStreams.out( testFile, new IoStreams.OutOptions().withThrowIfFileExists( true ) ) ) { System.out.println( "!" ); } - } ).isInstanceOf( UncheckedIOException.class ) - .hasCauseInstanceOf( FileExistsException.class ); + } ).isInstanceOf( oap.io.FileExistsException.class ); } } diff --git a/oap-stdlib-test/src/test/java/oap/util/ThrowablesTest.java b/oap-stdlib-test/src/test/java/oap/util/ThrowablesTest.java index 49ea74d9a8..8fe249031e 100644 --- a/oap-stdlib-test/src/test/java/oap/util/ThrowablesTest.java +++ b/oap-stdlib-test/src/test/java/oap/util/ThrowablesTest.java @@ -27,10 +27,9 @@ import org.testng.annotations.Test; import java.io.IOException; -import java.io.UncheckedIOException; public class ThrowablesTest { - @Test( expectedExceptions = UncheckedIOException.class, expectedExceptionsMessageRegExp = "java.io.IOException: test" ) + @Test( expectedExceptions = oap.io.IOException.class, expectedExceptionsMessageRegExp = "java.io.IOException: test" ) public void propagateIOException() { throw Throwables.propagate( new IOException( "test" ) ); }