Skip to content

Commit

Permalink
Merge pull request #1400 from virtualcell/faster-biosim-logging
Browse files Browse the repository at this point in the history 
Fast in-memory biosim logging
  • Loading branch information
@CodeByDrescher
CodeByDrescher authored Dec 13, 2024
2 parents bc9506d + 0426620 commit 67ddfba
Show file tree
Hide file tree
Showing 10 changed files with 194 additions and 224 deletions.
35 changes: 0 additions & 35 deletions .github/workflows/objective-c-xcode-debug.yml
View file

This file was deleted.

48 changes: 27 additions & 21 deletions vcell-cli/src/main/java/org/vcell/cli/run/ExecuteImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,45 +40,47 @@ public static void batchMode(File dirOfArchivesToProcess, File outputDir, CLIRec
for (File inputFile : inputFiles){
String bioModelBaseName = FileUtils.getBaseName(inputFile.getName());
String outputBaseDir = outputDir.getAbsolutePath(); // bioModelBaseName = input file without the path
String targetOutputDir = Paths.get(outputBaseDir, bioModelBaseName).toString();
File adjustedOutputDir = new File(targetOutputDir);

File targetOutputDir = Paths.get(outputBaseDir, bioModelBaseName).toFile();
logger.info("Preparing output directory...");
// we don't want to accidentally delete the input...
// if the output is a subset of the input file's housing directory, we shouldn't delete!!
if (!inputFile.getParentFile().getCanonicalPath().contains(adjustedOutputDir.getCanonicalPath()))
RunUtils.removeAndMakeDirs(adjustedOutputDir);
try {
BiosimulationLog.generateStatusYaml(inputFile.getAbsolutePath(), targetOutputDir); // generate Status YAML
} catch (XMLException e){
throw new RuntimeException("Python call did not process correctly:", e);
if (!inputFile.getParentFile().getCanonicalPath().contains(targetOutputDir.getCanonicalPath())) {
RunUtils.removeAndMakeDirs(targetOutputDir);
}
}
try {
for (File inputFile : inputFiles) {
String inputFileName = inputFile.getName();
System.out.println("\n\n");
logger.info("Processing " + inputFileName + "(" + inputFile + ")");
String bioModelBaseName = FileUtils.getBaseName(inputFile.getName());
String outputBaseDir = outputDir.getAbsolutePath(); // bioModelBaseName = input file without the path
String targetOutputDir = Paths.get(outputBaseDir, bioModelBaseName).toString();

Span span = null;
try {
span = Tracer.startSpan(Span.ContextType.OMEX_EXECUTE, inputFileName, Map.of("filename", inputFileName));
if (inputFileName.endsWith("vcml"))
BiosimulationLog.initialize(inputFile.getAbsolutePath(), targetOutputDir); // generate Status YAML

System.out.println("\n\n");
logger.info("Processing " + inputFileName + "(" + inputFile + ")");

if (inputFileName.endsWith("vcml")) {
singleExecVcml(inputFile, outputDir, cliLogger);
if (inputFileName.endsWith("omex"))
runSingleExecOmex(inputFile, outputDir, cliLogger,
bKeepTempFiles, bExactMatchOnly, bSmallMeshOverride);
} catch (ExecutionException e){
} else if (inputFileName.endsWith("omex")) {
runSingleExecOmex(inputFile, outputDir, cliLogger, bKeepTempFiles, bExactMatchOnly, bSmallMeshOverride);
}
} catch (ExecutionException e) {
logger.error("Error caught executing batch mode", e);
Tracer.failure(e, "Error caught executing batch mode");
failedFiles.add(inputFileName);
} catch (Exception e){
} catch (Exception e) {
Tracer.failure(e, "Error caught executing batch mode");
failedFiles.add(inputFileName);
throw e;
} finally {
if (span != null) {
span.close();
}
BiosimulationLog.instance().close();
}
}
if (failedFiles.isEmpty()){
Expand Down Expand Up @@ -138,10 +140,14 @@ public static void singleMode(File inputFile, File rootOutputDir, CLIRecordable
if (!inputFile.getParentFile().getCanonicalPath().contains(adjustedOutputDir.getCanonicalPath()))
RunUtils.removeAndMakeDirs(adjustedOutputDir);

BiosimulationLog.generateStatusYaml(inputFile.getAbsolutePath(), targetOutputDir); // generate Status YAML
try {
BiosimulationLog.initialize(inputFile.getAbsolutePath(), targetOutputDir); // generate Status YAML

ExecuteImpl.singleExecOmex(inputFile, rootOutputDir, cliLogger, bKeepTempFiles, bExactMatchOnly,
bEncapsulateOutput, bSmallMeshOverride, bBioSimMode);
ExecuteImpl.singleExecOmex(inputFile, rootOutputDir, cliLogger, bKeepTempFiles, bExactMatchOnly,
bEncapsulateOutput, bSmallMeshOverride, bBioSimMode);
} finally {
BiosimulationLog.instance().close();
}
}

public static void singleMode(File inputFile, File outputDir, CLIRecordable cliLogger) throws Exception {
Expand Down Expand Up @@ -232,7 +238,7 @@ public static void singleExecVcml(File vcmlFile, File outputDir, CLIRecordable c

private static void singleExecOmex(File inputFile, File rootOutputDir, CLIRecordable cliRecorder,
boolean bKeepTempFiles, boolean bExactMatchOnly, boolean bEncapsulateOutput, boolean bSmallMeshOverride, boolean bBioSimMode)
throws ExecutionException, PythonStreamException, IOException, InterruptedException, BiosimulationsHdfWriterException {
throws ExecutionException, PythonStreamException, IOException, BiosimulationsHdfWriterException {

ExecutionJob requestedExecution = new ExecutionJob(inputFile, rootOutputDir, cliRecorder,
bKeepTempFiles, bExactMatchOnly, bEncapsulateOutput, bSmallMeshOverride);
Expand Down
33 changes: 18 additions & 15 deletions vcell-cli/src/main/java/org/vcell/cli/run/ExecutionJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
Expand All @@ -25,10 +26,12 @@ public class ExecutionJob {

private final static Logger logger = LogManager.getLogger(ExecutionJob.class);

private long startTime, endTime;
private long startTime_ms, endTime_ms;
private boolean bExactMatchOnly, bSmallMeshOverride, bKeepTempFiles;
private StringBuilder logOmexMessage;
private String inputFilePath, bioModelBaseName, outputBaseDir, outputDir;
private String inputFilePath;
private String bioModelBaseName;
private String outputDir;
private boolean anySedmlDocumentHasSucceeded = false; // set to true if at least one sedml document run is successful
private boolean anySedmlDocumentHasFailed = false; // set to true if at least one sedml document run fails

Expand Down Expand Up @@ -58,7 +61,7 @@ public ExecutionJob(File inputFile, File rootOutputDir, CLIRecordable cliRecorde

this.inputFilePath = inputFile.getAbsolutePath();
this.bioModelBaseName = FileUtils.getBaseName(inputFile.getName()); // input file without the path
this.outputBaseDir = rootOutputDir.getAbsolutePath();
String outputBaseDir = rootOutputDir.getAbsolutePath();
this.outputDir = bEncapsulateOutput ? Paths.get(outputBaseDir, bioModelBaseName).toString() : outputBaseDir;
this.bKeepTempFiles = bKeepTempFiles;
this.bExactMatchOnly = bExactMatchOnly;
Expand All @@ -78,7 +81,7 @@ private ExecutionJob(){
*/
public void preprocessArchive() throws PythonStreamException, IOException {
// Start the clock
this.startTime = System.currentTimeMillis();
this.startTime_ms = System.currentTimeMillis();

// Beginning of Execution
logger.info("Executing OMEX archive `{}`", this.inputFile.getName());
Expand Down Expand Up @@ -108,7 +111,7 @@ public void preprocessArchive() throws PythonStreamException, IOException {
}

// Update Status
BiosimulationLog.updateOmexStatusYml(BiosimulationLog.Status.RUNNING, outputDir, "0");
BiosimulationLog.instance().updateOmexStatusYml(BiosimulationLog.Status.RUNNING, 0.0);
}

/**
Expand All @@ -128,7 +131,7 @@ public void executeArchive(boolean isBioSimSedml) throws BiosimulationsHdfWriter
this.queueAllSedml();

for (String sedmlLocation : this.sedmlLocations){
SedmlJob job = new SedmlJob(sedmlLocation, this.omexHandler, this.inputFile, new File(this.outputBaseDir),
SedmlJob job = new SedmlJob(sedmlLocation, this.omexHandler, this.inputFile,
this.outputDir, this.sedmlPath2d3d.toString(), this.cliRecorder,
this.bKeepTempFiles, this.bExactMatchOnly, this.bSmallMeshOverride, this.logOmexMessage);
if (!job.preProcessDoc()){
Expand Down Expand Up @@ -166,13 +169,13 @@ public void executeArchive(boolean isBioSimSedml) throws BiosimulationsHdfWriter
* @throws PythonStreamException if calls to the python-shell instance are not working correctly
* @throws IOException if there are system I/O issues
*/
public void postProcessessArchive() throws InterruptedException, PythonStreamException, IOException {
public void postProcessessArchive() throws IOException {
omexHandler.deleteExtractedOmex();

this.endTime = System.currentTimeMillis();
long elapsedTime = this.endTime - this.startTime;
int duration = (int) Math.ceil(elapsedTime / 1000.0);
logger.info("Omex " + inputFile.getName() + " processing completed (" + Integer.toString(duration) + "s)");
this.endTime_ms = System.currentTimeMillis();
long elapsedTime_ms = this.endTime_ms - this.startTime_ms;
double duration_s = elapsedTime_ms / 1000.0;
logger.info("Omex " + inputFile.getName() + " processing completed (" + duration_s + "s)");
//
// failure if at least one of the documents in the omex archive fails
//
Expand All @@ -181,24 +184,24 @@ public void postProcessessArchive() throws InterruptedException, PythonStreamExc
if (anySedmlDocumentHasSucceeded) { // some succeeded, some failed
error = " At least one document in this archive failed to execute";
}
BiosimulationLog.updateOmexStatusYml(BiosimulationLog.Status.FAILED, outputDir, duration + "");
BiosimulationLog.instance().updateOmexStatusYml(BiosimulationLog.Status.FAILED, duration_s);
logger.error(error);
logOmexMessage.append(error);
cliRecorder.writeErrorList(new Exception("exception not recorded"), bioModelBaseName);
} else {
BiosimulationLog.updateOmexStatusYml(BiosimulationLog.Status.SUCCEEDED, outputDir, duration + "");
BiosimulationLog.instance().updateOmexStatusYml(BiosimulationLog.Status.SUCCEEDED, duration_s);
cliRecorder.writeFullSuccessList(bioModelBaseName);
logOmexMessage.append(" Done");

}
BiosimulationLog.setOutputMessage("null", "null", outputDir, "omex", logOmexMessage.toString());
BiosimulationLog.instance().setOutputMessage("null", "null", "omex", logOmexMessage.toString());

logger.debug("Finished Execution of Archive: " + bioModelBaseName);
}

private void queueAllSedml() throws PythonStreamException, InterruptedException, IOException {
for (String sedmlLocation: sedmlLocations){
BiosimulationLog.updateSedmlDocStatusYml(sedmlLocation, BiosimulationLog.Status.QUEUED, outputDir);
BiosimulationLog.instance().updateSedmlDocStatusYml(sedmlLocation, BiosimulationLog.Status.QUEUED);
}
}

Expand Down
Loading

0 comments on commit 67ddfba

Please sign in to comment.