Skip to content

Commit

Permalink
oap-template: disk cache
Browse files Browse the repository at this point in the history
  • Loading branch information
@galaxina
galaxina committed Oct 16, 2023
1 parent 8bbdf0f commit 5ad2f49
Show file tree
Hide file tree
Showing 6 changed files with 223 additions and 126 deletions.
105 changes: 83 additions & 22 deletions oap-stdlib/src/main/java/oap/tools/TemplateClassCompiler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,12 @@
import io.micrometer.core.instrument.Timer;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import oap.io.content.ContentReader;
import oap.util.Result;
import org.jetbrains.annotations.NotNull;
import org.joda.time.DateTimeUtils;

import javax.annotation.Nullable;
import javax.tools.DiagnosticCollector;
import javax.tools.FileObject;
import javax.tools.ForwardingJavaFileManager;
Expand All @@ -21,6 +24,8 @@
import java.io.IOException;
import java.io.StringWriter;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
Expand All @@ -29,32 +34,40 @@
import java.util.concurrent.TimeUnit;

import static java.util.Collections.singleton;
import static oap.io.content.ContentWriter.ofBytes;
import static oap.io.content.ContentWriter.ofString;

/**
* Compiles provided java sources using in-memory strategy
* (no I/O at all), using memory file system. It saves
* JAR file's descriptors while compiling many files,
* closing it after operation. Result of compilation may be
* fetched with 'compiledJavaFiles' field.
*
* <p>
* USAGE:
* List<TemplateClassCompiler.SourceJavaFile> javaFiles = ...;
* var compilationResult = new TemplateClassCompiler().compile( javaFiles );
* if ( compilationResult.isSuccess() ) {
* var compiledClasses = compilationResult.getSuccessValue();
* ... // here 'TemplateClassSupplier' might be using to load & initialize the compiled classes
* var compiledClasses = compilationResult.getSuccessValue();
* ... // here 'TemplateClassSupplier' might be using to load & initialize the compiled classes
* }
*/
@Slf4j
public class TemplateClassCompiler {
private static final Counter METRICS_COMPILE = Metrics.counter( "oap_template", "type", "compile" );
private static final Timer METRICS_COMPILE_TIME = Metrics.timer( "oap_template", "type", "compile_time_in_millis" );
private static final Counter METRICS_DISK = Metrics.counter( "oap_template", "type", "disk" );
private static final Counter METRICS_ERROR = Metrics.counter( "oap_template", "type", "error" );
private final JavaCompiler javaCompiler = ToolProvider.getSystemJavaCompiler();
private final List<String> options = getCompilationOptions();
private final Path diskCache;
DiagnosticCollector<JavaFileObject> diagnostics = null;
Map<String, CompiledJavaFile> compiledJavaFiles = new HashMap<>();

public TemplateClassCompiler( @Nullable Path diskCache ) {
this.diskCache = diskCache;
}

public static class SourceJavaFile extends SimpleJavaFileObject {
final String javaName;
private final String content;
Expand Down Expand Up @@ -131,57 +144,105 @@ public JavaFileObject getJavaFileForOutput( Location location, String name, Java
public void close() {
try {
super.close();
} catch ( IOException e ) {
} catch( IOException e ) {
throw new RuntimeException( e );
}
}
}

/**
* Main entry-point for compilation
*
* @param javaFiles list of files to be compiled
* @return map of compilation result or failures
*/
public Result<Map<String, CompiledJavaFile>, String> compile( List<SourceJavaFile> javaFiles ) {
compiledJavaFiles = new HashMap<>();
diagnostics = new DiagnosticCollector<>();
try ( var fileManager = new MemoryFileManager( compiledJavaFiles, ToolProvider.getSystemJavaCompiler() ) ) {
try( var fileManager = new MemoryFileManager( compiledJavaFiles, ToolProvider.getSystemJavaCompiler() ) ) {
boolean compilationOk = javaFiles
.stream()
.map( javaFile -> compileSingleFile( fileManager, javaFile ) )
.peek( compileResult -> {
if ( compileResult.isSuccess() ) METRICS_COMPILE.increment();
else METRICS_ERROR.increment();
} )
.allMatch( Result::isSuccess );
if ( compilationOk ) {
.stream()
.map( javaFile -> compileSingleFile( fileManager, javaFile, diskCache ) )
.allMatch( Result::isSuccess );
if( compilationOk ) {
return Result.success( compiledJavaFiles );
}
return Result.failure( diagnostics.getDiagnostics().toString() );
}
}

private Result<Map<String, CompiledJavaFile>, String> compileSingleFile( JavaFileManager fileManager, SourceJavaFile javaFile ) {
private Result<Map<String, CompiledJavaFile>, String> compileSingleFile(
JavaFileManager fileManager, SourceJavaFile javaFile,
Path diskCache ) {
if( diskCache != null ) {
var sourceFile = diskCache.resolve( javaFile.javaName + ".java" );
var classFile = diskCache.resolve( javaFile.javaName + ".class" );
if( Files.exists( classFile ) && Files.exists( sourceFile ) && sourceContentEquals( sourceFile, javaFile.content ) ) {
var bytes = oap.io.Files.read( classFile, ContentReader.ofBytes() );

compiledJavaFiles.put( javaFile.javaName, new CompiledJavaFile( javaFile.javaName, bytes ) );

var currentTimeMillis = DateTimeUtils.currentTimeMillis();
oap.io.Files.setLastModifiedTime( sourceFile, currentTimeMillis );
oap.io.Files.setLastModifiedTime( classFile, currentTimeMillis );

METRICS_DISK.increment();

return Result.success( compiledJavaFiles );
}
}

// Now compile!
StringWriter outer = new StringWriter();
JavaCompiler.CompilationTask compilationTask =
javaCompiler.getTask(
outer,
fileManager,
diagnostics,
options,
null,
singleton( javaFile ) );
outer,
fileManager,
diagnostics,
options,
null,
singleton( javaFile ) );
long time = System.currentTimeMillis();
boolean compilationOk = compilationTask.call();
long tookForCompilation = System.currentTimeMillis() - time;
METRICS_COMPILE_TIME.record( tookForCompilation, TimeUnit.MILLISECONDS );
log.trace( "Compiling class '{}' ({}) took {} ms", javaFile.javaName, compilationOk ? "SUCCESS" : "FAILURE", tookForCompilation );
if ( compilationOk ) return Result.success( compiledJavaFiles );
log.trace( "Compiling class '{}' ({}) took {} ms", javaFile.javaName,
compilationOk ? "SUCCESS" : "FAILURE", tookForCompilation );
if( compilationOk ) {
METRICS_COMPILE.increment();
if( diskCache != null ) {
for( var source : compiledJavaFiles.values() ) {
var javaPath = diskCache.resolve( javaFile.javaName + ".java" );
var classPath = diskCache.resolve( javaFile.javaName + ".class" );

try {
oap.io.Files.write( javaPath, javaFile.content, ofString() );
var bytes = source.toByteArray();
oap.io.Files.write( classPath, bytes, ofBytes() );
} catch( Exception e ) {
oap.io.Files.delete( javaPath );
oap.io.Files.delete( classPath );
}
}
}

return Result.success( compiledJavaFiles );
}
METRICS_ERROR.increment();
log.warn( "Compilation '{}' failed with error: {}", javaFile.javaName, outer );
return Result.failure( diagnostics.getDiagnostics().toString() );
}

private boolean sourceContentEquals( Path sourceFile, String filecontent ) {
var source = oap.io.Files.read( sourceFile, ContentReader.ofString() );
boolean res = filecontent.equals( source );
if( !res ) {
log.warn( "{}: file content != template content", sourceFile );
}

return res;
}

/**
* @return result of compilation in case of failure
*/
Expand Down
9 changes: 5 additions & 4 deletions oap-template/src/main/java/oap/template/JavaTemplate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import oap.util.function.TriConsumer;
import org.jetbrains.annotations.NotNull;

import java.nio.file.Path;
import java.util.Map;
import java.util.function.Supplier;

Expand All @@ -42,19 +43,19 @@ public class JavaTemplate<TIn, TOut, TOutMutable, TA extends TemplateAccumulator
private final TA acc;

@SuppressWarnings( "unchecked" )
public JavaTemplate( String name, String template, TypeRef<TIn> type, TA acc, AstRoot ast ) {
public JavaTemplate( String name, String template, TypeRef<TIn> type, TA acc, AstRoot ast, Path diskCache ) {
this.acc = acc;
try {
// generate jav file for template
Render render = generateSourceFileForTemplate( name, template, type, acc, ast );
var fullTemplateName = getClass().getPackage().getName() + "." + render.nameEscaped();
// compile source of template into class
var compileResult = new TemplateClassCompiler().compile( Lists.of( new TemplateClassCompiler.SourceJavaFile( fullTemplateName, render.out() ) ) );
var compileResult = new TemplateClassCompiler( diskCache ).compile( Lists.of( new TemplateClassCompiler.SourceJavaFile( fullTemplateName, render.out() ) ) );
// instantiate class representing template
Class<?> clazz = new TemplateClassSupplier( compileResult.getSuccessValue() ).loadClasses( Lists.of( fullTemplateName ) ).get( fullTemplateName ).getSuccessValue();
cons = ( TriConsumer<TIn, Map<String, Supplier<String>>, TA> ) clazz
.getDeclaredConstructor()
.newInstance();
.getDeclaredConstructor()
.newInstance();
} catch( Exception e ) {
throw new TemplateException( e );
}
Expand Down
20 changes: 12 additions & 8 deletions oap-template/src/main/java/oap/template/TemplateEngine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,18 +60,22 @@

@Slf4j
public class TemplateEngine implements Runnable, AutoCloseable {
public final Path tmpPath;
public final Path diskCache;
public final long ttl;
private final Map<String, List<Method>> builtInFunction = new HashMap<>();
private final Cache<String, TemplateFunction> templates;
public long maxSize = 1_000_000L;

public TemplateEngine( Path tmpPath ) {
this( tmpPath, Dates.d( 30 ) );
public TemplateEngine() {
this( null );
}

public TemplateEngine( Path tmpPath, long ttl ) {
this.tmpPath = tmpPath;
public TemplateEngine( Path diskCache ) {
this( diskCache, Dates.d( 30 ) );
}

public TemplateEngine( Path diskCache, long ttl ) {
this.diskCache = diskCache;
this.ttl = ttl;

templates = CacheBuilder.newBuilder()
Expand Down Expand Up @@ -161,7 +165,7 @@ Template<TIn, TOut, TOutMutable, TA> getTemplate( String name, TypeRef<TIn> type
if( postProcess != null )
postProcess.accept( ast );

var tf = new JavaTemplate<>( name + '_' + id, template, type, acc, ast );
var tf = new JavaTemplate<>( name + '_' + id, template, type, acc, ast, diskCache );
return new TemplateFunction( tf );
} );

Expand Down Expand Up @@ -196,7 +200,7 @@ public long getCacheSize() {
public void run() {
templates.cleanUp();
var now = System.currentTimeMillis();
try( Stream<Path> stream = Files.walk( tmpPath ) ) {
try( Stream<Path> stream = Files.walk( diskCache ) ) {
stream
.forEach( path -> {
try {
Expand All @@ -210,7 +214,7 @@ public void run() {
} );
log.info( "TemplateEngine has done its work in {} ms", System.currentTimeMillis() - now );
} catch( IOException e ) {
log.error( "Could not walk through '{}'", tmpPath, e );
log.error( "Could not walk through '{}'", diskCache, e );
}
}

Expand Down
4 changes: 2 additions & 2 deletions oap-template/src/main/resources/META-INF/oap-module.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ services {
oap-template-engine {
implementation = oap.template.TemplateEngine
parameters {
ttl = 30d
tmpPath = /tmp/template
// ttl = 30d
// diskCache = /tmp/template
}
supervision {
schedule = true
Expand Down
Loading

0 comments on commit 5ad2f49

Please sign in to comment.