allow scripted changes to logged data

pom.xml

@@ -131,7 +131,6 @@
     	<groupId>org.jenkins-ci.plugins</groupId>
     	<artifactId>script-security</artifactId>
     	<version>1.39</version>
-    	<scope>test</scope>
     </dependency>
     <dependency>
     	<groupId>org.powermock</groupId>
@@ -157,7 +156,8 @@
        <version>2.1.5</version>
        <scope>test</scope>
     </dependency>
-    <dependency>
+
+      <dependency>
     	<groupId>org.hamcrest</groupId>
     	<artifactId>java-hamcrest</artifactId>
     	<version>2.0.0.0</version>

src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java

@@ -25,6 +25,10 @@
 package jenkins.plugins.logstash;
 
 import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+
+import java.nio.charset.Charset;
 
 import org.kohsuke.stapler.DataBoundConstructor;
 
@@ -35,8 +39,14 @@
 import hudson.model.BuildListener;
 import hudson.tasks.BuildWrapper;
 import hudson.tasks.BuildWrapperDescriptor;
+import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SecureGroovyScript;
+
+import javax.annotation.CheckForNull;
+
+import org.kohsuke.stapler.DataBoundSetter;
 
 /**
+ * Logstash note on each output line.
  *
  * This BuildWrapper is not used anymore.
  * We just keep it to be able to convert projects that have the BuildWrapper configured at startup or when posting the xml via the rest api
@@ -48,13 +58,21 @@
 public class LogstashBuildWrapper extends BuildWrapper
 {
 
+  @CheckForNull
+  private SecureGroovyScript secureGroovyScript;
+
   /**
    * Create a new {@link LogstashBuildWrapper}.
    */
   @DataBoundConstructor
   public LogstashBuildWrapper()
   {}
 
+  @DataBoundSetter
+  public void setSecureGroovyScript(@CheckForNull SecureGroovyScript script) {
+    this.secureGroovyScript = script != null ? script.configuringWithNonKeyItem() : null;
+  }
+
   /**
    * {@inheritDoc}
    */
@@ -73,6 +91,22 @@ public DescriptorImpl getDescriptor()
     return (DescriptorImpl)super.getDescriptor();
   }
 
+  @CheckForNull
+  public SecureGroovyScript getSecureGroovyScript() {
+    return secureGroovyScript;
+  }
+
+  // Method to encapsulate calls for unit-testing
+  LogstashWriter getLogStashWriter(AbstractBuild<?, ?> build, OutputStream errorStream) {
+    Charset charset = build.getCharset();
+    LogstashScriptProcessor processor = null;
+    if (secureGroovyScript != null) {
+      processor = new LogstashScriptProcessor(secureGroovyScript, new OutputStreamWriter(errorStream, charset));
+    }
+
+    return new LogstashWriter(build, errorStream, null, charset, processor);
+  }
+
   /**
    * Registers {@link LogstashBuildWrapper} as a {@link BuildWrapper}.
    */

src/main/java/jenkins/plugins/logstash/LogstashConsoleLogFilter.java

@@ -2,28 +2,41 @@
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.OutputStreamWriter;
 import java.io.Serializable;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import javax.annotation.CheckForNull;
+
 import hudson.Extension;
 import hudson.console.ConsoleLogFilter;
 import hudson.model.AbstractBuild;
 import hudson.model.AbstractProject;
 import hudson.model.Run;
 
+import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SecureGroovyScript;
+
 @Extension(ordinal = 1000)
 public class LogstashConsoleLogFilter extends ConsoleLogFilter implements Serializable
 {
 
   private static Logger LOGGER = Logger.getLogger(LogstashConsoleLogFilter.class.getName());
 
+  @CheckForNull
+  private SecureGroovyScript secureGroovyScript = null;
   private transient Run<?, ?> run;
   public LogstashConsoleLogFilter() {};
 
   public LogstashConsoleLogFilter(Run<?, ?> run)
+  {
+    this(run, null);
+  }
+
+  public LogstashConsoleLogFilter(Run<?, ?> run, SecureGroovyScript script)
   {
     this.run = run;
+    this.secureGroovyScript = script;
   }
   private static final long serialVersionUID = 1L;
 
@@ -41,7 +54,7 @@ public OutputStream decorateLogger(Run build, OutputStream logger) throws IOExce
     {
       if (isLogstashEnabled(build))
       {
-        LogstashWriter logstash = getLogStashWriter(build, logger);
+        LogstashWriter logstash = getLogStashWriter(build, logger, secureGroovyScript);
         return new LogstashOutputStream(logger, logstash);
       }
       else
@@ -51,7 +64,7 @@ public OutputStream decorateLogger(Run build, OutputStream logger) throws IOExce
     }
     if (run != null)
     {
-      LogstashWriter logstash = getLogStashWriter(run, logger);
+      LogstashWriter logstash = getLogStashWriter(run, logger, secureGroovyScript);
       return new LogstashOutputStream(logger, logstash);
     }
     else
@@ -60,9 +73,13 @@ public OutputStream decorateLogger(Run build, OutputStream logger) throws IOExce
     }
   }
 
-  LogstashWriter getLogStashWriter(Run<?, ?> build, OutputStream errorStream)
+  LogstashWriter getLogStashWriter(Run<?, ?> build, OutputStream errorStream, SecureGroovyScript script)
   {
-    return new LogstashWriter(build, errorStream, null, build.getCharset());
+    LogstashScriptProcessor processor = null;
+    if (secureGroovyScript != null) {
+      processor = new LogstashScriptProcessor(secureGroovyScript, new OutputStreamWriter(errorStream, build.getCharset()));
+    }
+    return new LogstashWriter(build, errorStream, null, build.getCharset(), processor);
   }
 
   private boolean isLogstashEnabled(Run<?, ?> build)
@@ -76,8 +93,10 @@ private boolean isLogstashEnabled(Run<?, ?> build)
     if (build.getParent() instanceof AbstractProject)
     {
       AbstractProject<?, ?> project = (AbstractProject<?, ?>)build.getParent();
-      if (project.getProperty(LogstashJobProperty.class) != null)
+      LogstashJobProperty jobProperty = project.getProperty(LogstashJobProperty.class);
+      if (jobProperty != null)
       {
+        this.secureGroovyScript = jobProperty.getSecureGroovyScript();
         return true;
       }
     }

src/main/java/jenkins/plugins/logstash/LogstashJobProperty.java

@@ -1,6 +1,9 @@
 package jenkins.plugins.logstash;
 
+import javax.annotation.CheckForNull;
+
 import org.kohsuke.stapler.DataBoundConstructor;
+import org.kohsuke.stapler.DataBoundSetter;
 import org.kohsuke.stapler.StaplerRequest;
 
 import hudson.Extension;
@@ -9,18 +12,32 @@
 import hudson.model.JobProperty;
 import hudson.model.JobPropertyDescriptor;
 import net.sf.json.JSONObject;
+import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SecureGroovyScript;
 
 /**
  * This JobProperty is a marker to decide if logs should be sent to an indexer.
  *
  */
 public class LogstashJobProperty extends JobProperty<Job<?, ?>>
 {
+  @CheckForNull
+  private SecureGroovyScript secureGroovyScript = null;
 
   @DataBoundConstructor
   public LogstashJobProperty()
   {}
 
+  @DataBoundSetter
+  public void setSecureGroovyScript(@CheckForNull SecureGroovyScript script)
+  {
+    this.secureGroovyScript = script != null ? script.configuringWithNonKeyItem() : null;
+  }
+
+  public SecureGroovyScript getSecureGroovyScript()
+  {
+    return this.secureGroovyScript;
+  }
+
   @Extension
   public static class DescriptorImpl extends JobPropertyDescriptor
   {

src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java

@@ -79,6 +79,7 @@ public void flush() throws IOException {
    */
   @Override
   public void close() throws IOException {
+    logstash.close();
     delegate.close();
     super.close();
   }

src/main/java/jenkins/plugins/logstash/LogstashPayloadProcessor.java

@@ -0,0 +1,50 @@
+/*
+ * The MIT License
+ *
+ * Copyright 2017 Red Hat inc, and individual contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package jenkins.plugins.logstash;
+
+import net.sf.json.JSONObject;
+
+/**
+ * Interface describing processors of persisted payload.
+ *
+ * @author Aleksandar Kostadinov
+ * @since 1.4.0
+ */
+public interface LogstashPayloadProcessor {
+  /**
+   * Modifies a JSON payload compatible with the Logstash schema.
+   *
+   * @param payload the JSON payload that has been constructed so far.
+   * @return The formatted JSON object, can be null to ignore this payload.
+   */
+  JSONObject process(JSONObject payload) throws Exception;
+
+  /**
+   * Finalizes any operations, for example returns cashed lines at end of build.
+   *
+   * @return A formatted JSON object, can be null when it has nothing.
+   */
+  JSONObject finish() throws Exception;
+}

src/main/java/jenkins/plugins/logstash/LogstashScriptProcessor.java

@@ -0,0 +1,117 @@
+/*
+ * The MIT License
+ *
+ * Copyright 2017 Red Hat inc. and individual contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package jenkins.plugins.logstash;
+
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.util.LinkedHashMap;
+
+import javax.annotation.Nonnull;
+
+import groovy.lang.Binding;
+
+import net.sf.json.JSONObject;
+
+import jenkins.model.Jenkins;
+import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SecureGroovyScript;
+import org.jenkinsci.plugins.scriptsecurity.sandbox.whitelists.Whitelisted;
+
+/**
+ * This class is handling custom groovy script processing of JSON payload.
+ * Each call to process executes the script provided in job configuration.
+ * Script is executed under the same binding each time so that it has ability
+ * to persist data during build execution if desired by script author.
+ * When build is finished, script will receive null as the payload and can
+ * return any cached but non-sent data back for persisting.
+ * The return value of script is the payload to be persisted unless null.
+ *
+ * @author Aleksandar Kostadinov
+ * @since 1.4.0
+ */
+public class LogstashScriptProcessor implements LogstashPayloadProcessor{
+  @Nonnull
+  private final SecureGroovyScript script;
+
+  @Nonnull
+  private final OutputStreamWriter consoleWriter;
+
+  /** Groovy binding for script execution */
+  @Nonnull
+  private final Binding binding;
+
+  /** Classloader for script execution */
+  @Nonnull
+  private final ClassLoader classLoader;
+
+  public LogstashScriptProcessor(SecureGroovyScript script, OutputStreamWriter consoleWriter) {
+    this.script = script;
+    this.consoleWriter = consoleWriter;
+
+    // TODO: should we put variables in the binding like manager, job, etc.?
+    binding = new Binding();
+    binding.setVariable("console", new BuildConsoleWrapper());
+
+    // if users need access to types defined by any plugin we may decide to
+    // switch to Jenkins.getInstance().getPluginManager().uberClassLoader
+    classLoader = LogstashScriptProcessor.class.getClassLoader();
+  }
+
+  /**
+   * Helper method to allow logging to build console.
+   */
+  private void buildLogPrintln(Object o) throws IOException {
+    consoleWriter.write(o.toString() + "\n");
+    consoleWriter.flush();
+  }
+
+  /*
+   * good examples in:
+   *  https://github.com/jenkinsci/envinject-plugin/blob/master/src/main/java/org/jenkinsci/plugins/envinject/service/EnvInjectEnvVars.java
+   *  https://github.com/jenkinsci/groovy-postbuild-plugin/pull/11/files
+   */
+  @Override
+  public JSONObject process(JSONObject payload) throws Exception {
+    binding.setVariable("payload", payload);
+    script.evaluate(classLoader, binding);
+    return (JSONObject) binding.getVariable("payload");
+  }
+
+  @Override
+  public JSONObject finish() throws Exception {
+    buildLogPrintln("Tearing down Script Log Processor..");
+    return process(new JSONObject());
+  }
+
+  /**
+   * Helper to allow access from sandboxed script to output messages to console.
+   */
+  private class BuildConsoleWrapper {
+    @Whitelisted
+    public void println(Object o) throws IOException {
+      buildLogPrintln(o);
+    }
+  }
+}