diff --git a/JShellAPI/build.gradle b/JShellAPI/build.gradle index 1ea77da..efe627c 100644 --- a/JShellAPI/build.gradle +++ b/JShellAPI/build.gradle @@ -24,6 +24,9 @@ repositories { dependencies { implementation project(':JShellWrapper') implementation 'org.springframework.boot:spring-boot-starter-web' + implementation 'com.github.docker-java:docker-java-transport-httpclient5:3.3.4' + implementation 'com.github.docker-java:docker-java-core:3.3.4' + testImplementation 'org.springframework.boot:spring-boot-starter-test' annotationProcessor "org.springframework.boot:spring-boot-configuration-processor" } diff --git a/JShellAPI/src/main/java/org/togetherjava/jshellapi/Config.java b/JShellAPI/src/main/java/org/togetherjava/jshellapi/Config.java index 5079ca3..a207103 100644 --- a/JShellAPI/src/main/java/org/togetherjava/jshellapi/Config.java +++ b/JShellAPI/src/main/java/org/togetherjava/jshellapi/Config.java @@ -11,7 +11,9 @@ public record Config( long maxAliveSessions, int dockerMaxRamMegaBytes, double dockerCPUsUsage, - long schedulerSessionKillScanRateSeconds) { + long schedulerSessionKillScanRateSeconds, + long dockerResponseTimeout, + long dockerConnectionTimeout) { public Config { if(regularSessionTimeoutSeconds <= 0) throw new RuntimeException("Invalid value " + regularSessionTimeoutSeconds); if(oneTimeSessionTimeoutSeconds <= 0) throw new RuntimeException("Invalid value " + oneTimeSessionTimeoutSeconds); @@ -21,5 +23,7 @@ public record Config( if(dockerMaxRamMegaBytes <= 0) throw new RuntimeException("Invalid value " + dockerMaxRamMegaBytes); if(dockerCPUsUsage <= 0) throw new RuntimeException("Invalid value " + dockerCPUsUsage); if(schedulerSessionKillScanRateSeconds <= 0) throw new RuntimeException("Invalid value " + schedulerSessionKillScanRateSeconds); + if(dockerResponseTimeout <= 0) throw new RuntimeException("Invalid value " + dockerResponseTimeout); + if(dockerConnectionTimeout <= 0) throw new RuntimeException("Invalid value " + dockerConnectionTimeout); } } diff --git a/JShellAPI/src/main/java/org/togetherjava/jshellapi/service/DockerService.java b/JShellAPI/src/main/java/org/togetherjava/jshellapi/service/DockerService.java new file mode 100644 index 0000000..b19ad14 --- /dev/null +++ b/JShellAPI/src/main/java/org/togetherjava/jshellapi/service/DockerService.java @@ -0,0 +1,149 @@ +package org.togetherjava.jshellapi.service; + +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.async.ResultCallback; +import com.github.dockerjava.api.command.PullImageResultCallback; +import com.github.dockerjava.api.model.*; +import com.github.dockerjava.core.DefaultDockerClientConfig; +import com.github.dockerjava.core.DockerClientImpl; +import com.github.dockerjava.httpclient5.ApacheDockerHttpClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.togetherjava.jshellapi.Config; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.TimeUnit; + +@Service +public class DockerService implements DisposableBean { + private static final Logger LOGGER = LoggerFactory.getLogger(DockerService.class); + private static final String WORKER_LABEL = "jshell-api-worker"; + private static final UUID WORKER_UNIQUE_ID = UUID.randomUUID(); + + private final DockerClient client; + + public DockerService(Config config) { + DefaultDockerClientConfig clientConfig = DefaultDockerClientConfig.createDefaultConfigBuilder().build(); + ApacheDockerHttpClient httpClient = new ApacheDockerHttpClient.Builder() + .dockerHost(clientConfig.getDockerHost()) + .sslConfig(clientConfig.getSSLConfig()) + .responseTimeout(Duration.ofSeconds(config.dockerResponseTimeout())) + .connectionTimeout(Duration.ofSeconds(config.dockerConnectionTimeout())) + .build(); + this.client = DockerClientImpl.getInstance(clientConfig, httpClient); + + cleanupLeftovers(WORKER_UNIQUE_ID); + } + + private void cleanupLeftovers(UUID currentId) { + for (Container container : client.listContainersCmd().withLabelFilter(Set.of(WORKER_LABEL)).exec()) { + String containerHumanName = container.getId() + " " + Arrays.toString(container.getNames()); + LOGGER.info("Found worker container '{}'", containerHumanName); + if (!container.getLabels().get(WORKER_LABEL).equals(currentId.toString())) { + LOGGER.info("Killing container '{}'", containerHumanName); + client.killContainerCmd(container.getId()).exec(); + } + } + } + + public String spawnContainer( + long maxMemoryMegs, long cpus, String name, Duration evalTimeout, long sysoutLimit + ) throws InterruptedException { + String imageName = "togetherjava.org:5001/togetherjava/jshellwrapper"; + boolean presentLocally = client.listImagesCmd() + .withFilter("reference", List.of(imageName)) + .exec() + .stream() + .flatMap(it -> Arrays.stream(it.getRepoTags())) + .anyMatch(it -> it.endsWith(":master")); + + if (!presentLocally) { + client.pullImageCmd(imageName) + .withTag("master") + .exec(new PullImageResultCallback()) + .awaitCompletion(5, TimeUnit.MINUTES); + } + + return client.createContainerCmd( + imageName + ":master" + ) + .withHostConfig( + HostConfig.newHostConfig() + .withAutoRemove(true) + .withInit(true) + .withCapDrop(Capability.ALL) + .withNetworkMode("none") + .withPidsLimit(2000L) + .withReadonlyRootfs(true) + .withMemory(maxMemoryMegs * 1024 * 1024) + .withCpuCount(cpus) + ) + .withStdinOpen(true) + .withAttachStdin(true) + .withAttachStderr(true) + .withAttachStdout(true) + .withEnv("evalTimeoutSeconds=" + evalTimeout.toSeconds(), "sysOutCharLimit=" + sysoutLimit) + .withLabels(Map.of(WORKER_LABEL, WORKER_UNIQUE_ID.toString())) + .withName(name) + .exec() + .getId(); + } + + public InputStream startAndAttachToContainer(String containerId, InputStream stdin) throws IOException { + PipedInputStream pipeIn = new PipedInputStream(); + PipedOutputStream pipeOut = new PipedOutputStream(pipeIn); + + client.attachContainerCmd(containerId) + .withLogs(true) + .withFollowStream(true) + .withStdOut(true) + .withStdErr(true) + .withStdIn(stdin) + .exec(new ResultCallback.Adapter<>() { + @Override + public void onNext(Frame object) { + try { + String payloadString = new String(object.getPayload(), StandardCharsets.UTF_8); + if (object.getStreamType() == StreamType.STDOUT) { + pipeOut.write(object.getPayload()); + } else { + LOGGER.warn( + "Received STDERR from container {}: {}", + containerId, + payloadString + ); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + }); + + client.startContainerCmd(containerId).exec(); + return pipeIn; + } + + public void killContainerByName(String name) { + for (Container container : client.listContainersCmd().withNameFilter(Set.of(name)).exec()) { + client.killContainerCmd(container.getId()).exec(); + } + } + + public boolean isDead(String containerName) { + return client.listContainersCmd().withNameFilter(Set.of(containerName)).exec().isEmpty(); + } + + @Override + public void destroy() throws Exception { + LOGGER.info("destroy() called. Destroying all containers..."); + cleanupLeftovers(UUID.randomUUID()); + client.close(); + } + +} diff --git a/JShellAPI/src/main/java/org/togetherjava/jshellapi/service/JShellService.java b/JShellAPI/src/main/java/org/togetherjava/jshellapi/service/JShellService.java index 5e556b7..27c2bc4 100644 --- a/JShellAPI/src/main/java/org/togetherjava/jshellapi/service/JShellService.java +++ b/JShellAPI/src/main/java/org/togetherjava/jshellapi/service/JShellService.java @@ -4,12 +4,10 @@ import org.togetherjava.jshellapi.dto.*; import org.togetherjava.jshellapi.exceptions.DockerException; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.IOException; +import java.io.*; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -18,7 +16,6 @@ public class JShellService implements Closeable { private final JShellSessionService sessionService; private final String id; - private Process process; private final BufferedWriter writer; private final BufferedReader reader; @@ -26,8 +23,10 @@ public class JShellService implements Closeable { private final long timeout; private final boolean renewable; private boolean doingOperation; + private final DockerService dockerService; - public JShellService(JShellSessionService sessionService, String id, long timeout, boolean renewable, long evalTimeout, int sysOutCharLimit, int maxMemory, double cpus, String startupScript) throws DockerException { + public JShellService(DockerService dockerService, JShellSessionService sessionService, String id, long timeout, boolean renewable, long evalTimeout, int sysOutCharLimit, int maxMemory, double cpus, String startupScript) throws DockerException { + this.dockerService = dockerService; this.sessionService = sessionService; this.id = id; this.timeout = timeout; @@ -39,30 +38,23 @@ public JShellService(JShellSessionService sessionService, String id, long timeou Files.createDirectories(errorLogs.getParent()); Files.createFile(errorLogs); } - process = new ProcessBuilder( - "docker", - "run", - "--rm", - "-i", - "--init", - "--cap-drop=ALL", - "--network=none", - "--pids-limit=2000", - "--read-only", - "--memory=" + maxMemory + "m", - "--cpus=" + cpus, - "--name", containerName(), - "-e", "\"evalTimeoutSeconds=%d\"".formatted(evalTimeout), - "-e", "\"sysOutCharLimit=%d\"".formatted(sysOutCharLimit), - "togetherjava.org:5001/togetherjava/jshellwrapper:master") - .directory(new File("..")) - .redirectError(errorLogs.toFile()) - .start(); - writer = process.outputWriter(); - reader = process.inputReader(); + String containerId = dockerService.spawnContainer( + maxMemory, + (long) Math.ceil(cpus), + containerName(), + Duration.ofSeconds(evalTimeout), + sysOutCharLimit + ); + PipedInputStream containerInput = new PipedInputStream(); + this.writer = new BufferedWriter(new OutputStreamWriter(new PipedOutputStream(containerInput))); + InputStream containerOutput = dockerService.startAndAttachToContainer( + containerId, + containerInput + ); + reader = new BufferedReader(new InputStreamReader(containerOutput)); writer.write(sanitize(startupScript)); writer.newLine(); - } catch (IOException e) { + } catch (IOException | InterruptedException e) { throw new DockerException(e); } this.doingOperation = false; @@ -73,6 +65,10 @@ public Optional eval(String code) throws DockerException { return Optional.empty(); } } + if (isClosed()) { + close(); + return Optional.empty(); + } updateLastTimeout(); if(!code.endsWith("\n")) code += '\n'; try { @@ -86,7 +82,7 @@ public Optional eval(String code) throws DockerException { checkContainerOK(); return Optional.of(readResult()); - } catch (IOException | NumberFormatException ex) { + } catch (DockerException | IOException | NumberFormatException ex) { close(); throw new DockerException(ex); } finally { @@ -185,27 +181,22 @@ public String id() { @Override public void close() { - process.destroyForcibly(); try { try { writer.close(); } finally { reader.close(); } - new ProcessBuilder("docker", "kill", containerName()) - .directory(new File("..")) - .start() - .waitFor(); - } catch(IOException | InterruptedException ex) { + dockerService.killContainerByName(containerName()); + } catch(IOException ex) { throw new RuntimeException(ex); } - process = null; sessionService.notifyDeath(id); } @Override public boolean isClosed() { - return process == null; + return dockerService.isDead(containerName()); } private void updateLastTimeout() { diff --git a/JShellAPI/src/main/java/org/togetherjava/jshellapi/service/JShellSessionService.java b/JShellAPI/src/main/java/org/togetherjava/jshellapi/service/JShellSessionService.java index 0498a90..dfc8612 100644 --- a/JShellAPI/src/main/java/org/togetherjava/jshellapi/service/JShellSessionService.java +++ b/JShellAPI/src/main/java/org/togetherjava/jshellapi/service/JShellSessionService.java @@ -18,7 +18,9 @@ public class JShellSessionService { private Config config; private StartupScriptsService startupScriptsService; private ScheduledExecutorService scheduler; + private DockerService dockerService; private final Map jshellSessions = new HashMap<>(); + private void initScheduler() { scheduler = Executors.newSingleThreadScheduledExecutor(); scheduler.scheduleAtFixedRate(() -> { @@ -74,6 +76,7 @@ private synchronized JShellService createSession(String id, long sessionTimeout, throw new ResponseStatusException(HttpStatus.TOO_MANY_REQUESTS, "Too many sessions, try again later :(."); } JShellService service = new JShellService( + dockerService, this, id, sessionTimeout, @@ -97,4 +100,9 @@ public void setConfig(Config config) { public void setStartupScriptsService(StartupScriptsService startupScriptsService) { this.startupScriptsService = startupScriptsService; } + + @Autowired + public void setDockerService(DockerService dockerService) { + this.dockerService = dockerService; + } } diff --git a/JShellAPI/src/main/resources/application.properties b/JShellAPI/src/main/resources/application.properties index c1d4a3d..35a1925 100644 --- a/JShellAPI/src/main/resources/application.properties +++ b/JShellAPI/src/main/resources/application.properties @@ -11,4 +11,8 @@ jshellapi.dockerMaxRamMegaBytes=100 jshellapi.dockerCPUsUsage=0.5 # Internal config -jshellapi.schedulerSessionKillScanRateSeconds=60 \ No newline at end of file +jshellapi.schedulerSessionKillScanRateSeconds=60 + +# Docker service config +jshellapi.dockerResponseTimeout=60 +jshellapi.dockerConnectionTimeout=60 \ No newline at end of file