From 5ad2f4902e6bda91d4b7c9eb03057a863172d83d Mon Sep 17 00:00:00 2001 From: "igor.petrenko" Date: Mon, 16 Oct 2023 15:00:22 +0300 Subject: [PATCH 1/2] oap-template: disk cache --- .../java/oap/tools/TemplateClassCompiler.java | 105 ++++++++++++---- .../main/java/oap/template/JavaTemplate.java | 9 +- .../java/oap/template/TemplateEngine.java | 20 +-- .../main/resources/META-INF/oap-module.conf | 4 +- .../oap/tools/TemplateClassCompilerTest.java | 116 +++++++++++------- .../oap/tools/TemplateClassSupplierTest.java | 95 +++++++------- 6 files changed, 223 insertions(+), 126 deletions(-) diff --git a/oap-stdlib/src/main/java/oap/tools/TemplateClassCompiler.java b/oap-stdlib/src/main/java/oap/tools/TemplateClassCompiler.java index e47c9830a9..3077900edb 100644 --- a/oap-stdlib/src/main/java/oap/tools/TemplateClassCompiler.java +++ b/oap-stdlib/src/main/java/oap/tools/TemplateClassCompiler.java @@ -5,9 +5,12 @@ import io.micrometer.core.instrument.Timer; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import oap.io.content.ContentReader; import oap.util.Result; import org.jetbrains.annotations.NotNull; +import org.joda.time.DateTimeUtils; +import javax.annotation.Nullable; import javax.tools.DiagnosticCollector; import javax.tools.FileObject; import javax.tools.ForwardingJavaFileManager; @@ -21,6 +24,8 @@ import java.io.IOException; import java.io.StringWriter; import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -29,6 +34,8 @@ import java.util.concurrent.TimeUnit; import static java.util.Collections.singleton; +import static oap.io.content.ContentWriter.ofBytes; +import static oap.io.content.ContentWriter.ofString; /** * Compiles provided java sources using in-memory strategy @@ -36,25 +43,31 @@ * JAR file's descriptors while compiling many files, * closing it after operation. Result of compilation may be * fetched with 'compiledJavaFiles' field. - * + *

* USAGE: * List javaFiles = ...; * var compilationResult = new TemplateClassCompiler().compile( javaFiles ); * if ( compilationResult.isSuccess() ) { - * var compiledClasses = compilationResult.getSuccessValue(); - * ... // here 'TemplateClassSupplier' might be using to load & initialize the compiled classes + * var compiledClasses = compilationResult.getSuccessValue(); + * ... // here 'TemplateClassSupplier' might be using to load & initialize the compiled classes * } */ @Slf4j public class TemplateClassCompiler { private static final Counter METRICS_COMPILE = Metrics.counter( "oap_template", "type", "compile" ); private static final Timer METRICS_COMPILE_TIME = Metrics.timer( "oap_template", "type", "compile_time_in_millis" ); + private static final Counter METRICS_DISK = Metrics.counter( "oap_template", "type", "disk" ); private static final Counter METRICS_ERROR = Metrics.counter( "oap_template", "type", "error" ); private final JavaCompiler javaCompiler = ToolProvider.getSystemJavaCompiler(); private final List options = getCompilationOptions(); + private final Path diskCache; DiagnosticCollector diagnostics = null; Map compiledJavaFiles = new HashMap<>(); + public TemplateClassCompiler( @Nullable Path diskCache ) { + this.diskCache = diskCache; + } + public static class SourceJavaFile extends SimpleJavaFileObject { final String javaName; private final String content; @@ -131,7 +144,7 @@ public JavaFileObject getJavaFileForOutput( Location location, String name, Java public void close() { try { super.close(); - } catch ( IOException e ) { + } catch( IOException e ) { throw new RuntimeException( e ); } } @@ -139,49 +152,97 @@ public void close() { /** * Main entry-point for compilation + * * @param javaFiles list of files to be compiled * @return map of compilation result or failures */ public Result, String> compile( List javaFiles ) { compiledJavaFiles = new HashMap<>(); diagnostics = new DiagnosticCollector<>(); - try ( var fileManager = new MemoryFileManager( compiledJavaFiles, ToolProvider.getSystemJavaCompiler() ) ) { + try( var fileManager = new MemoryFileManager( compiledJavaFiles, ToolProvider.getSystemJavaCompiler() ) ) { boolean compilationOk = javaFiles - .stream() - .map( javaFile -> compileSingleFile( fileManager, javaFile ) ) - .peek( compileResult -> { - if ( compileResult.isSuccess() ) METRICS_COMPILE.increment(); - else METRICS_ERROR.increment(); - } ) - .allMatch( Result::isSuccess ); - if ( compilationOk ) { + .stream() + .map( javaFile -> compileSingleFile( fileManager, javaFile, diskCache ) ) + .allMatch( Result::isSuccess ); + if( compilationOk ) { return Result.success( compiledJavaFiles ); } return Result.failure( diagnostics.getDiagnostics().toString() ); } } - private Result, String> compileSingleFile( JavaFileManager fileManager, SourceJavaFile javaFile ) { + private Result, String> compileSingleFile( + JavaFileManager fileManager, SourceJavaFile javaFile, + Path diskCache ) { + if( diskCache != null ) { + var sourceFile = diskCache.resolve( javaFile.javaName + ".java" ); + var classFile = diskCache.resolve( javaFile.javaName + ".class" ); + if( Files.exists( classFile ) && Files.exists( sourceFile ) && sourceContentEquals( sourceFile, javaFile.content ) ) { + var bytes = oap.io.Files.read( classFile, ContentReader.ofBytes() ); + + compiledJavaFiles.put( javaFile.javaName, new CompiledJavaFile( javaFile.javaName, bytes ) ); + + var currentTimeMillis = DateTimeUtils.currentTimeMillis(); + oap.io.Files.setLastModifiedTime( sourceFile, currentTimeMillis ); + oap.io.Files.setLastModifiedTime( classFile, currentTimeMillis ); + + METRICS_DISK.increment(); + + return Result.success( compiledJavaFiles ); + } + } + // Now compile! StringWriter outer = new StringWriter(); JavaCompiler.CompilationTask compilationTask = javaCompiler.getTask( - outer, - fileManager, - diagnostics, - options, - null, - singleton( javaFile ) ); + outer, + fileManager, + diagnostics, + options, + null, + singleton( javaFile ) ); long time = System.currentTimeMillis(); boolean compilationOk = compilationTask.call(); long tookForCompilation = System.currentTimeMillis() - time; METRICS_COMPILE_TIME.record( tookForCompilation, TimeUnit.MILLISECONDS ); - log.trace( "Compiling class '{}' ({}) took {} ms", javaFile.javaName, compilationOk ? "SUCCESS" : "FAILURE", tookForCompilation ); - if ( compilationOk ) return Result.success( compiledJavaFiles ); + log.trace( "Compiling class '{}' ({}) took {} ms", javaFile.javaName, + compilationOk ? "SUCCESS" : "FAILURE", tookForCompilation ); + if( compilationOk ) { + METRICS_COMPILE.increment(); + if( diskCache != null ) { + for( var source : compiledJavaFiles.values() ) { + var javaPath = diskCache.resolve( javaFile.javaName + ".java" ); + var classPath = diskCache.resolve( javaFile.javaName + ".class" ); + + try { + oap.io.Files.write( javaPath, javaFile.content, ofString() ); + var bytes = source.toByteArray(); + oap.io.Files.write( classPath, bytes, ofBytes() ); + } catch( Exception e ) { + oap.io.Files.delete( javaPath ); + oap.io.Files.delete( classPath ); + } + } + } + + return Result.success( compiledJavaFiles ); + } + METRICS_ERROR.increment(); log.warn( "Compilation '{}' failed with error: {}", javaFile.javaName, outer ); return Result.failure( diagnostics.getDiagnostics().toString() ); } + private boolean sourceContentEquals( Path sourceFile, String filecontent ) { + var source = oap.io.Files.read( sourceFile, ContentReader.ofString() ); + boolean res = filecontent.equals( source ); + if( !res ) { + log.warn( "{}: file content != template content", sourceFile ); + } + + return res; + } + /** * @return result of compilation in case of failure */ diff --git a/oap-template/src/main/java/oap/template/JavaTemplate.java b/oap-template/src/main/java/oap/template/JavaTemplate.java index 9029938d03..d0e629b34f 100644 --- a/oap-template/src/main/java/oap/template/JavaTemplate.java +++ b/oap-template/src/main/java/oap/template/JavaTemplate.java @@ -32,6 +32,7 @@ import oap.util.function.TriConsumer; import org.jetbrains.annotations.NotNull; +import java.nio.file.Path; import java.util.Map; import java.util.function.Supplier; @@ -42,19 +43,19 @@ public class JavaTemplate type, TA acc, AstRoot ast ) { + public JavaTemplate( String name, String template, TypeRef type, TA acc, AstRoot ast, Path diskCache ) { this.acc = acc; try { // generate jav file for template Render render = generateSourceFileForTemplate( name, template, type, acc, ast ); var fullTemplateName = getClass().getPackage().getName() + "." + render.nameEscaped(); // compile source of template into class - var compileResult = new TemplateClassCompiler().compile( Lists.of( new TemplateClassCompiler.SourceJavaFile( fullTemplateName, render.out() ) ) ); + var compileResult = new TemplateClassCompiler( diskCache ).compile( Lists.of( new TemplateClassCompiler.SourceJavaFile( fullTemplateName, render.out() ) ) ); // instantiate class representing template Class clazz = new TemplateClassSupplier( compileResult.getSuccessValue() ).loadClasses( Lists.of( fullTemplateName ) ).get( fullTemplateName ).getSuccessValue(); cons = ( TriConsumer>, TA> ) clazz - .getDeclaredConstructor() - .newInstance(); + .getDeclaredConstructor() + .newInstance(); } catch( Exception e ) { throw new TemplateException( e ); } diff --git a/oap-template/src/main/java/oap/template/TemplateEngine.java b/oap-template/src/main/java/oap/template/TemplateEngine.java index c8fc03167c..dc1a1ac4af 100644 --- a/oap-template/src/main/java/oap/template/TemplateEngine.java +++ b/oap-template/src/main/java/oap/template/TemplateEngine.java @@ -60,18 +60,22 @@ @Slf4j public class TemplateEngine implements Runnable, AutoCloseable { - public final Path tmpPath; + public final Path diskCache; public final long ttl; private final Map> builtInFunction = new HashMap<>(); private final Cache templates; public long maxSize = 1_000_000L; - public TemplateEngine( Path tmpPath ) { - this( tmpPath, Dates.d( 30 ) ); + public TemplateEngine() { + this( null ); } - public TemplateEngine( Path tmpPath, long ttl ) { - this.tmpPath = tmpPath; + public TemplateEngine( Path diskCache ) { + this( diskCache, Dates.d( 30 ) ); + } + + public TemplateEngine( Path diskCache, long ttl ) { + this.diskCache = diskCache; this.ttl = ttl; templates = CacheBuilder.newBuilder() @@ -161,7 +165,7 @@ Template getTemplate( String name, TypeRef type if( postProcess != null ) postProcess.accept( ast ); - var tf = new JavaTemplate<>( name + '_' + id, template, type, acc, ast ); + var tf = new JavaTemplate<>( name + '_' + id, template, type, acc, ast, diskCache ); return new TemplateFunction( tf ); } ); @@ -196,7 +200,7 @@ public long getCacheSize() { public void run() { templates.cleanUp(); var now = System.currentTimeMillis(); - try( Stream stream = Files.walk( tmpPath ) ) { + try( Stream stream = Files.walk( diskCache ) ) { stream .forEach( path -> { try { @@ -210,7 +214,7 @@ public void run() { } ); log.info( "TemplateEngine has done its work in {} ms", System.currentTimeMillis() - now ); } catch( IOException e ) { - log.error( "Could not walk through '{}'", tmpPath, e ); + log.error( "Could not walk through '{}'", diskCache, e ); } } diff --git a/oap-template/src/main/resources/META-INF/oap-module.conf b/oap-template/src/main/resources/META-INF/oap-module.conf index e4cd537014..181dfb410a 100644 --- a/oap-template/src/main/resources/META-INF/oap-module.conf +++ b/oap-template/src/main/resources/META-INF/oap-module.conf @@ -3,8 +3,8 @@ services { oap-template-engine { implementation = oap.template.TemplateEngine parameters { - ttl = 30d - tmpPath = /tmp/template +// ttl = 30d +// diskCache = /tmp/template } supervision { schedule = true diff --git a/oap-template/src/test/java/oap/tools/TemplateClassCompilerTest.java b/oap-template/src/test/java/oap/tools/TemplateClassCompilerTest.java index bf80d77a84..c469282cc9 100644 --- a/oap-template/src/test/java/oap/tools/TemplateClassCompilerTest.java +++ b/oap-template/src/test/java/oap/tools/TemplateClassCompilerTest.java @@ -1,5 +1,10 @@ package oap.tools; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.prometheus.PrometheusConfig; +import io.micrometer.prometheus.PrometheusMeterRegistry; +import oap.testng.Fixtures; +import oap.testng.TestDirectoryFixture; import oap.util.Lists; import oap.util.Result; import org.testng.annotations.Test; @@ -8,61 +13,86 @@ import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -public class TemplateClassCompilerTest { +public class TemplateClassCompilerTest extends Fixtures { + public TemplateClassCompilerTest() { + fixture( TestDirectoryFixture.FIXTURE ); + } + private static String sourceNoImport = """ - public class Bad { - public int getRandomNumber() { - return RandomGenerator.StreamableGenerator.of("L128X256MixRandom").nextInt(); - } + public class Bad { + public int getRandomNumber() { + return RandomGenerator.StreamableGenerator.of("L128X256MixRandom").nextInt(); } - """; + } + """; private static String sourceA = """ - import java.util.random.RandomGenerator; - public class A { - public int getRandomNumber() { - return RandomGenerator.StreamableGenerator.of("L128X256MixRandom").nextInt(); - } + import java.util.random.RandomGenerator; + public class A { + public int getRandomNumber() { + return RandomGenerator.StreamableGenerator.of("L128X256MixRandom").nextInt(); } - """; + } + """; private static String sourceB = """ - import java.util.random.RandomGenerator; - public class B { - public static class InnerB extends B { - @Override - public double getRandomNumber() { - return 0.0; - } - } + import java.util.random.RandomGenerator; + public class B { + public static class InnerB extends B { + @Override public double getRandomNumber() { - return RandomGenerator.StreamableGenerator.of("Xoroshiro128PlusPlus").nextDouble(); - } - public static void main(String[] args) { - System.err.println( new B().getRandomNumber() ); + return 0.0; } } - """; + public double getRandomNumber() { + return RandomGenerator.StreamableGenerator.of("Xoroshiro128PlusPlus").nextDouble(); + } + public static void main(String[] args) { + System.err.println( new B().getRandomNumber() ); + } + } + """; @Test public void testCompileSingleFileOk() { - var compiler = new TemplateClassCompiler(); - Result, String> result = compiler.compile( + var prometheusRegistry = new PrometheusMeterRegistry( PrometheusConfig.DEFAULT ); + Metrics.addRegistry( prometheusRegistry ); + try { + + var compiler = new TemplateClassCompiler( TestDirectoryFixture.testDirectory() ); + Result, String> result = compiler.compile( Lists.of( - new TemplateClassCompiler.SourceJavaFile( "A", sourceA ) + new TemplateClassCompiler.SourceJavaFile( "A", sourceA ) ) - ); + ); - assertThat( result.isSuccess() ).isTrue(); + assertThat( result.isSuccess() ).isTrue(); + + var compiler2 = new TemplateClassCompiler( TestDirectoryFixture.testDirectory() ); + Result, String> result2 = compiler2.compile( + Lists.of( + new TemplateClassCompiler.SourceJavaFile( "A", sourceA ) + ) + ); + + assertThat( result2.isSuccess() ).isTrue(); + + String scrape = prometheusRegistry.scrape(); + assertThat( scrape ).contains( "oap_template_total{type=\"disk\",} 1.0" ); + assertThat( scrape ).contains( "oap_template_total{type=\"compile\",} 1.0" ); + + } finally { + Metrics.removeRegistry( prometheusRegistry ); + } } @Test public void testCompileBatchFilesOk() { - var compiler = new TemplateClassCompiler(); + var compiler = new TemplateClassCompiler( null ); Result, String> result = compiler.compile( - Lists.of( - new TemplateClassCompiler.SourceJavaFile( "A", sourceA ), - new TemplateClassCompiler.SourceJavaFile( "B", sourceB ) - ) + Lists.of( + new TemplateClassCompiler.SourceJavaFile( "A", sourceA ), + new TemplateClassCompiler.SourceJavaFile( "B", sourceB ) + ) ); assertThat( result.isSuccess() ).isTrue(); @@ -70,11 +100,11 @@ public void testCompileBatchFilesOk() { @Test public void testCompileSingleFileSyntaxError() { - var compiler = new TemplateClassCompiler(); + var compiler = new TemplateClassCompiler( null ); Result, String> result = compiler.compile( - Lists.of( - new TemplateClassCompiler.SourceJavaFile( "Bad", sourceNoImport ) - ) + Lists.of( + new TemplateClassCompiler.SourceJavaFile( "Bad", sourceNoImport ) + ) ); assertThat( result.isSuccess() ).isFalse(); @@ -83,11 +113,11 @@ public void testCompileSingleFileSyntaxError() { @Test public void testCompileInvalidSourceError() { - var compiler = new TemplateClassCompiler(); + var compiler = new TemplateClassCompiler( null ); Result, String> result = compiler.compile( - Lists.of( - new TemplateClassCompiler.SourceJavaFile( "InvalidJavaFileName", sourceA ) - ) + Lists.of( + new TemplateClassCompiler.SourceJavaFile( "InvalidJavaFileName", sourceA ) + ) ); assertThat( result.isSuccess() ).isFalse(); diff --git a/oap-template/src/test/java/oap/tools/TemplateClassSupplierTest.java b/oap-template/src/test/java/oap/tools/TemplateClassSupplierTest.java index 079a623cf9..bef94248c8 100644 --- a/oap-template/src/test/java/oap/tools/TemplateClassSupplierTest.java +++ b/oap-template/src/test/java/oap/tools/TemplateClassSupplierTest.java @@ -12,46 +12,47 @@ public class TemplateClassSupplierTest { private static String sourceA = """ - import java.util.random.RandomGenerator; - public class A { - public int getRandomNumber() { - RandomGenerator.StreamableGenerator.of("L128X256MixRandom").nextInt(); - return 5; - } + import java.util.random.RandomGenerator; + public class A { + public int getRandomNumber() { + RandomGenerator.StreamableGenerator.of("L128X256MixRandom").nextInt(); + return 5; } - """; + } + """; private static String sourceB = """ - import java.util.random.RandomGenerator; - public class B { - public static class InnerB extends B { - @Override - public double getRandomNumber() { - return 0.0; - } - } + import java.util.random.RandomGenerator; + public class B { + public static class InnerB extends B { + @Override public double getRandomNumber() { - return RandomGenerator.StreamableGenerator.of("Xoroshiro128PlusPlus").nextDouble(); - } - public static void main(String[] args) { - System.err.println( new B().getRandomNumber() ); + return 0.0; } } - """; + public double getRandomNumber() { + return RandomGenerator.StreamableGenerator.of("Xoroshiro128PlusPlus").nextDouble(); + } + public static void main(String[] args) { + System.err.println( new B().getRandomNumber() ); + } + } + """; private static String sourceC = """ - public class C { - public String get() { - return "OK"; - } + public class C { + public String get() { + return "OK"; } - """; + } + """; + @Test public void compileAndRunClassA() throws Exception { - var compiler = new TemplateClassCompiler(); + var compiler = new TemplateClassCompiler( null ); compiler.compile( - Lists.of( - new TemplateClassCompiler.SourceJavaFile( "A", sourceA ) - ) + Lists.of( + new TemplateClassCompiler.SourceJavaFile( "A", sourceA ) + ) ); var supplier = new TemplateClassSupplier( compiler.compiledJavaFiles ); @@ -65,11 +66,11 @@ public void compileAndRunClassA() throws Exception { @Test public void compileAndRunClassInnerB() throws Exception { - var compiler = new TemplateClassCompiler(); + var compiler = new TemplateClassCompiler( null ); compiler.compile( - Lists.of( - new TemplateClassCompiler.SourceJavaFile( "B", sourceB ) - ) + Lists.of( + new TemplateClassCompiler.SourceJavaFile( "B", sourceB ) + ) ); var supplier = new TemplateClassSupplier( compiler.compiledJavaFiles ); @@ -83,13 +84,13 @@ public void compileAndRunClassInnerB() throws Exception { @Test public void compileAndRunClassBAndInnerBWithinSingleClassLoader() throws Exception { - var compiler = new TemplateClassCompiler(); + var compiler = new TemplateClassCompiler( null ); compiler.compile( - Lists.of( - new TemplateClassCompiler.SourceJavaFile( "A", sourceA ), - new TemplateClassCompiler.SourceJavaFile( "B", sourceB ), - new TemplateClassCompiler.SourceJavaFile( "C", sourceC ) - ) + Lists.of( + new TemplateClassCompiler.SourceJavaFile( "A", sourceA ), + new TemplateClassCompiler.SourceJavaFile( "B", sourceB ), + new TemplateClassCompiler.SourceJavaFile( "C", sourceC ) + ) ); var classLoader = new TemplateClassSupplier.TemplateClassLoader( compiler.compiledJavaFiles ); @@ -115,17 +116,17 @@ public void compileAndRunClassBAndInnerBWithinSingleClassLoader() throws Excepti @Test public void compile2TimesAndLoadIntoSingleClassLoader() { - var compiler = new TemplateClassCompiler(); + var compiler = new TemplateClassCompiler( null ); var result1 = compiler.compile( - Lists.of( - new TemplateClassCompiler.SourceJavaFile( "A", sourceA ) - ) + Lists.of( + new TemplateClassCompiler.SourceJavaFile( "A", sourceA ) + ) ).getSuccessValue(); var result2 = compiler.compile( - Lists.of( - new TemplateClassCompiler.SourceJavaFile( "B", sourceB ), - new TemplateClassCompiler.SourceJavaFile( "C", sourceC ) - ) + Lists.of( + new TemplateClassCompiler.SourceJavaFile( "B", sourceB ), + new TemplateClassCompiler.SourceJavaFile( "C", sourceC ) + ) ).getSuccessValue(); Map allCompiledClasses = new HashMap<>( result1 ); From 4a962a6fafbc8d86165a9ff4b8a1f2769ae69426 Mon Sep 17 00:00:00 2001 From: "igor.petrenko" Date: Mon, 16 Oct 2023 15:21:53 +0300 Subject: [PATCH 2/2] oap-template: disk cache --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1ee4968c86..db28792354 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ - 18.11.10 + 18.11.11 7.8.0 3.24.2