Merge branch 'release/v0.15.0'

README.rst

@@ -53,7 +53,7 @@ to build from the source just clone source and package with maven
     cd logisland
     mvn clean install
 
-the final package is available at `logisland-assembly/target/logisland-0.14.0-bin-hdp2.5.tar.gz`
+the final package is available at `logisland-assembly/target/logisland-0.15.0-bin-hdp2.5.tar.gz`
 
 You can also download the `latest release build <https://github.com/Hurence/logisland/releases>`_
 
@@ -76,9 +76,9 @@ Alternatively you can deploy **logisland** on any linux server from which Kafka
     curl -s http://d3kbcqa49mib13.cloudfront.net/spark-2.1.0-bin-hadoop2.7.tgz | tar -xz -C /usr/local/
     export SPARK_HOME=/usr/local/spark-2.1.0-bin-hadoop2.7
 
-    # install Logisland 0.14.0
-    curl -s https://github.com/Hurence/logisland/releases/download/v0.10.0/logisland-0.14.0-bin-hdp2.5.tar.gz  | tar -xz -C /usr/local/
-    cd /usr/local/logisland-0.14.0
+    # install Logisland 0.15.0
+    curl -s https://github.com/Hurence/logisland/releases/download/v0.10.0/logisland-0.15.0-bin-hdp2.5.tar.gz  | tar -xz -C /usr/local/
+    cd /usr/local/logisland-0.15.0
 
     # launch a logisland job
     bin/logisland.sh --conf conf/index-apache-logs.yml
@@ -107,9 +107,9 @@ Launching logisland streaming apps is just easy as unarchiving logisland distrib
 
 .. code-block:: sh
 
-    # install Logisland 0.14.0
-    curl -s https://github.com/Hurence/logisland/releases/download/v0.10.0/logisland-0.14.0-bin-hdp2.5.tar.gz  | tar -xz -C /usr/local/
-    cd /usr/local/logisland-0.14.0
+    # install Logisland 0.15.0
+    curl -s https://github.com/Hurence/logisland/releases/download/v0.10.0/logisland-0.15.0-bin-hdp2.5.tar.gz  | tar -xz -C /usr/local/
+    cd /usr/local/logisland-0.15.0
     bin/logisland.sh --conf conf/index-apache-logs.yml
 
 
@@ -130,7 +130,7 @@ The first part is the `ProcessingEngine` configuration (here a Spark streaming e
 
 .. code-block:: yaml
 
-    version: 0.14.0
+    version: 0.15.0
     documentation: LogIsland job config file
     engine:
       component: com.hurence.logisland.engine.spark.KafkaStreamProcessingEngine

apitester/index.js

@@ -0,0 +1,32 @@
+const express = require('express');
+const app = express();
+const port = process.env.PORT || 3000;
+const fs = require('fs');
+const promisify = require('util').promisify;
+const filename = './logisland.json';
+const bodyParser = require('body-parser');
+
+app.use(bodyParser.json());
+app.listen(port);
+
+const routes = {
+
+  getDataflows: async function(req, res) {
+    console.log("Updating dataflow for logisland job", req.params.jobId);
+    let content = JSON.parse(await promisify(fs.readFile)(filename));
+    let fileDate = (await promisify(fs.stat)(filename)).mtime;
+    res.append('Last-Modified', fileDate.toUTCString()).json(content);
+  },
+
+  notifyActiveDataflows: function(req, res) {
+    console.log("Logisland active dataflows for job", req.params.jobId);
+    console.log(JSON.stringify(req.body));
+    res.sendStatus(200);
+  }
+};
+
+app.route('/dataflows/:jobId')
+  .get(routes.getDataflows)
+  .post(routes.notifyActiveDataflows);
+
+console.log('Logisland RESTful API server started on: ' + port);

apitester/logisland.json

@@ -0,0 +1,87 @@
+{
+	"lastModified": "2018-06-30T10:00:24Z",
+
+	"streams": [{
+			"name": "kafka_in",
+			"component": "com.hurence.logisland.stream.spark.KafkaRecordStreamParallelProcessing",
+			"config": [{
+					"key": "kafka.input.topics",
+					"value": "logisland_raw"
+				},
+
+				{
+					"key": "kafka.output.topics",
+					"value": "logisland_events"
+				},
+				{
+					"key": "kafka.error.topics",
+					"value": "logisland_errors"
+				},
+				{
+					"key": "kafka.input.topics.serializer",
+					"value": "none"
+				},
+				{
+					"key": "kafka.output.topics.serializer",
+					"value": "com.hurence.logisland.serializer.KryoSerializer"
+				},
+				{
+					"key": "kafka.error.topics.serializer",
+					"value": "com.hurence.logisland.serializer.JsonSerializer"
+				},
+				{
+					"key": "kafka.metadata.broker.list",
+					"value": "sandbox:9092"
+				},
+				{
+					"key": "kafka.zookeeper.quorum",
+					"value": "sandbox:2181"
+				},
+				{
+					"key": "kafka.topic.autoCreate",
+					"value": "true"
+				},
+				{
+					"key": "kafka.topic.default.partitions",
+					"value": "4"
+				},
+				{
+					"key": "kafka.topic.default.replicationFactor",
+					"value": "1"
+				}
+			],
+			"pipeline": {
+				"lastModified": "2018-05-30T12:02:35Z",
+				"processors": [{
+						"component": "com.hurence.logisland.processor.SplitText",
+						"name": "apache_parser",
+						"documentation": "parse apache logs with a regexp",
+						"config": [
+              {
+                                 "key": "record.type",
+                                 "value": "apache_logs"
+                               },
+							{
+								"key": "value.regex",
+								"value": "(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+\\[([\\w:\\/]+\\s[+\\-]\\d{4})\\]\\s+\"(\\S+)\\s+(\\S+)\\s*(\\S*)\"\\s+(\\S+)\\s+(\\S+)"
+							},
+							{
+								"key": "value.fields",
+								"value": "src_ip,identd,user,record_time,http_method,http_query,http_version,http_status,bytes_out"
+							}
+						]
+					},
+					{
+						"name": "outLog",
+						"component": "com.hurence.logisland.processor.DebugStream",
+						"config": [{
+							"key": "event.serializer",
+							"value": "json"
+						}]
+
+					}
+	]
+}
+}
+]
+}