Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

oap-template: disk cache #209

Merged
merged 2 commits into from
Oct 16, 2023
Merged

oap-template: disk cache #209

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5ad2f49
oap-template: disk cache
galaxina Oct 16, 2023
4a962a6
oap-template: disk cache
galaxina Oct 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 83 additions & 22 deletions oap-stdlib/src/main/java/oap/tools/TemplateClassCompiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,12 @@
import io.micrometer.core.instrument.Timer;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import oap.io.content.ContentReader;
import oap.util.Result;
import org.jetbrains.annotations.NotNull;
import org.joda.time.DateTimeUtils;

import javax.annotation.Nullable;
import javax.tools.DiagnosticCollector;
import javax.tools.FileObject;
import javax.tools.ForwardingJavaFileManager;
Expand All @@ -21,6 +24,8 @@
import java.io.IOException;
import java.io.StringWriter;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
Expand All @@ -29,32 +34,40 @@
import java.util.concurrent.TimeUnit;

import static java.util.Collections.singleton;
import static oap.io.content.ContentWriter.ofBytes;
import static oap.io.content.ContentWriter.ofString;

/**
* Compiles provided java sources using in-memory strategy
* (no I/O at all), using memory file system. It saves
* JAR file's descriptors while compiling many files,
* closing it after operation. Result of compilation may be
* fetched with 'compiledJavaFiles' field.
*
* <p>
* USAGE:
* List<TemplateClassCompiler.SourceJavaFile> javaFiles = ...;
* var compilationResult = new TemplateClassCompiler().compile( javaFiles );
* if ( compilationResult.isSuccess() ) {
* var compiledClasses = compilationResult.getSuccessValue();
* ... // here 'TemplateClassSupplier' might be using to load & initialize the compiled classes
* var compiledClasses = compilationResult.getSuccessValue();
* ... // here 'TemplateClassSupplier' might be using to load & initialize the compiled classes
* }
*/
@Slf4j
public class TemplateClassCompiler {
private static final Counter METRICS_COMPILE = Metrics.counter( "oap_template", "type", "compile" );
private static final Timer METRICS_COMPILE_TIME = Metrics.timer( "oap_template", "type", "compile_time_in_millis" );
private static final Counter METRICS_DISK = Metrics.counter( "oap_template", "type", "disk" );
private static final Counter METRICS_ERROR = Metrics.counter( "oap_template", "type", "error" );
private final JavaCompiler javaCompiler = ToolProvider.getSystemJavaCompiler();
private final List<String> options = getCompilationOptions();
private final Path diskCache;
DiagnosticCollector<JavaFileObject> diagnostics = null;
Map<String, CompiledJavaFile> compiledJavaFiles = new HashMap<>();

public TemplateClassCompiler( @Nullable Path diskCache ) {
this.diskCache = diskCache;
}

public static class SourceJavaFile extends SimpleJavaFileObject {
final String javaName;
private final String content;
Expand Down Expand Up @@ -131,57 +144,105 @@ public JavaFileObject getJavaFileForOutput( Location location, String name, Java
public void close() {
try {
super.close();
} catch ( IOException e ) {
} catch( IOException e ) {
throw new RuntimeException( e );
}
}
}

/**
* Main entry-point for compilation
*
* @param javaFiles list of files to be compiled
* @return map of compilation result or failures
*/
public Result<Map<String, CompiledJavaFile>, String> compile( List<SourceJavaFile> javaFiles ) {
compiledJavaFiles = new HashMap<>();
diagnostics = new DiagnosticCollector<>();
try ( var fileManager = new MemoryFileManager( compiledJavaFiles, ToolProvider.getSystemJavaCompiler() ) ) {
try( var fileManager = new MemoryFileManager( compiledJavaFiles, ToolProvider.getSystemJavaCompiler() ) ) {
boolean compilationOk = javaFiles
.stream()
.map( javaFile -> compileSingleFile( fileManager, javaFile ) )
.peek( compileResult -> {
if ( compileResult.isSuccess() ) METRICS_COMPILE.increment();
else METRICS_ERROR.increment();
} )
.allMatch( Result::isSuccess );
if ( compilationOk ) {
.stream()
.map( javaFile -> compileSingleFile( fileManager, javaFile, diskCache ) )
.allMatch( Result::isSuccess );
if( compilationOk ) {
return Result.success( compiledJavaFiles );
}
return Result.failure( diagnostics.getDiagnostics().toString() );
}
}

private Result<Map<String, CompiledJavaFile>, String> compileSingleFile( JavaFileManager fileManager, SourceJavaFile javaFile ) {
private Result<Map<String, CompiledJavaFile>, String> compileSingleFile(
JavaFileManager fileManager, SourceJavaFile javaFile,
Path diskCache ) {
if( diskCache != null ) {
var sourceFile = diskCache.resolve( javaFile.javaName + ".java" );
var classFile = diskCache.resolve( javaFile.javaName + ".class" );
if( Files.exists( classFile ) && Files.exists( sourceFile ) && sourceContentEquals( sourceFile, javaFile.content ) ) {
var bytes = oap.io.Files.read( classFile, ContentReader.ofBytes() );

compiledJavaFiles.put( javaFile.javaName, new CompiledJavaFile( javaFile.javaName, bytes ) );

var currentTimeMillis = DateTimeUtils.currentTimeMillis();
oap.io.Files.setLastModifiedTime( sourceFile, currentTimeMillis );
oap.io.Files.setLastModifiedTime( classFile, currentTimeMillis );

METRICS_DISK.increment();

return Result.success( compiledJavaFiles );
}
}

// Now compile!
StringWriter outer = new StringWriter();
JavaCompiler.CompilationTask compilationTask =
javaCompiler.getTask(
outer,
fileManager,
diagnostics,
options,
null,
singleton( javaFile ) );
outer,
fileManager,
diagnostics,
options,
null,
singleton( javaFile ) );
long time = System.currentTimeMillis();
boolean compilationOk = compilationTask.call();
long tookForCompilation = System.currentTimeMillis() - time;
METRICS_COMPILE_TIME.record( tookForCompilation, TimeUnit.MILLISECONDS );
log.trace( "Compiling class '{}' ({}) took {} ms", javaFile.javaName, compilationOk ? "SUCCESS" : "FAILURE", tookForCompilation );
if ( compilationOk ) return Result.success( compiledJavaFiles );
log.trace( "Compiling class '{}' ({}) took {} ms", javaFile.javaName,
compilationOk ? "SUCCESS" : "FAILURE", tookForCompilation );
if( compilationOk ) {
METRICS_COMPILE.increment();
if( diskCache != null ) {
for( var source : compiledJavaFiles.values() ) {
var javaPath = diskCache.resolve( javaFile.javaName + ".java" );
var classPath = diskCache.resolve( javaFile.javaName + ".class" );

try {
oap.io.Files.write( javaPath, javaFile.content, ofString() );
var bytes = source.toByteArray();
oap.io.Files.write( classPath, bytes, ofBytes() );
} catch( Exception e ) {
oap.io.Files.delete( javaPath );
oap.io.Files.delete( classPath );
}
}
}

return Result.success( compiledJavaFiles );
}
METRICS_ERROR.increment();
log.warn( "Compilation '{}' failed with error: {}", javaFile.javaName, outer );
return Result.failure( diagnostics.getDiagnostics().toString() );
}

private boolean sourceContentEquals( Path sourceFile, String filecontent ) {
var source = oap.io.Files.read( sourceFile, ContentReader.ofString() );
boolean res = filecontent.equals( source );
if( !res ) {
log.warn( "{}: file content != template content", sourceFile );
}

return res;
}

/**
* @return result of compilation in case of failure
*/
Expand Down
9 changes: 5 additions & 4 deletions oap-template/src/main/java/oap/template/JavaTemplate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import oap.util.function.TriConsumer;
import org.jetbrains.annotations.NotNull;

import java.nio.file.Path;
import java.util.Map;
import java.util.function.Supplier;

Expand All @@ -42,19 +43,19 @@ public class JavaTemplate<TIn, TOut, TOutMutable, TA extends TemplateAccumulator
private final TA acc;

@SuppressWarnings( "unchecked" )
public JavaTemplate( String name, String template, TypeRef<TIn> type, TA acc, AstRoot ast ) {
public JavaTemplate( String name, String template, TypeRef<TIn> type, TA acc, AstRoot ast, Path diskCache ) {
this.acc = acc;
try {
// generate jav file for template
Render render = generateSourceFileForTemplate( name, template, type, acc, ast );
var fullTemplateName = getClass().getPackage().getName() + "." + render.nameEscaped();
// compile source of template into class
var compileResult = new TemplateClassCompiler().compile( Lists.of( new TemplateClassCompiler.SourceJavaFile( fullTemplateName, render.out() ) ) );
var compileResult = new TemplateClassCompiler( diskCache ).compile( Lists.of( new TemplateClassCompiler.SourceJavaFile( fullTemplateName, render.out() ) ) );
// instantiate class representing template
Class<?> clazz = new TemplateClassSupplier( compileResult.getSuccessValue() ).loadClasses( Lists.of( fullTemplateName ) ).get( fullTemplateName ).getSuccessValue();
cons = ( TriConsumer<TIn, Map<String, Supplier<String>>, TA> ) clazz
.getDeclaredConstructor()
.newInstance();
.getDeclaredConstructor()
.newInstance();
} catch( Exception e ) {
throw new TemplateException( e );
}
Expand Down
20 changes: 12 additions & 8 deletions oap-template/src/main/java/oap/template/TemplateEngine.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,18 +60,22 @@

@Slf4j
public class TemplateEngine implements Runnable, AutoCloseable {
public final Path tmpPath;
public final Path diskCache;
public final long ttl;
private final Map<String, List<Method>> builtInFunction = new HashMap<>();
private final Cache<String, TemplateFunction> templates;
public long maxSize = 1_000_000L;

public TemplateEngine( Path tmpPath ) {
this( tmpPath, Dates.d( 30 ) );
public TemplateEngine() {
this( null );
}

public TemplateEngine( Path tmpPath, long ttl ) {
this.tmpPath = tmpPath;
public TemplateEngine( Path diskCache ) {
this( diskCache, Dates.d( 30 ) );
}

public TemplateEngine( Path diskCache, long ttl ) {
this.diskCache = diskCache;
this.ttl = ttl;

templates = CacheBuilder.newBuilder()
Expand Down Expand Up @@ -161,7 +165,7 @@ Template<TIn, TOut, TOutMutable, TA> getTemplate( String name, TypeRef<TIn> type
if( postProcess != null )
postProcess.accept( ast );

var tf = new JavaTemplate<>( name + '_' + id, template, type, acc, ast );
var tf = new JavaTemplate<>( name + '_' + id, template, type, acc, ast, diskCache );
return new TemplateFunction( tf );
} );

Expand Down Expand Up @@ -196,7 +200,7 @@ public long getCacheSize() {
public void run() {
templates.cleanUp();
var now = System.currentTimeMillis();
try( Stream<Path> stream = Files.walk( tmpPath ) ) {
try( Stream<Path> stream = Files.walk( diskCache ) ) {
stream
.forEach( path -> {
try {
Expand All @@ -210,7 +214,7 @@ public void run() {
} );
log.info( "TemplateEngine has done its work in {} ms", System.currentTimeMillis() - now );
} catch( IOException e ) {
log.error( "Could not walk through '{}'", tmpPath, e );
log.error( "Could not walk through '{}'", diskCache, e );
}
}

Expand Down
4 changes: 2 additions & 2 deletions oap-template/src/main/resources/META-INF/oap-module.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ services {
oap-template-engine {
implementation = oap.template.TemplateEngine
parameters {
ttl = 30d
tmpPath = /tmp/template
// ttl = 30d
// diskCache = /tmp/template
}
supervision {
schedule = true
Expand Down
Loading
Loading