Split apart Extract creation into an ExtractAdapter, to support Hyper…

… extracts in the future. Add Makefile for managing releases
jlmorton · May 29, 2018 · 95c253e · 95c253e
1 parent b92aaf8
commit 95c253e
Show file tree

Hide file tree

Showing 15 changed files with 312 additions and 172 deletions.
diff --git a/Makefile b/Makefile
@@ -0,0 +1,30 @@
+MKDIR_P = mkdir -p
+NAME = tableau-sdk-wrapper
+VERSION = 1.1
+BUILD_DIR = ${NAME}-${VERSION}
+ARTIFACT = tableau-${VERSION}.jar
+ARCHIVE = ${NAME}-${VERSION}.zip
+TARGET = target
+SAMPLES = samples
+BIN_DIR = bin
+
+all: mkdir copy_files archive
+
+mkdir:
+	${MKDIR_P} ${BUILD_DIR}/bin
+	${MKDIR_P} ${BUILD_DIR}/lib
+	${MKDIR_P} ${BUILD_DIR}/samples
+	${MKDIR_P} ${BUILD_DIR}/tmp
+	${MKDIR_P} ${BUILD_DIR}/logs
+
+copy_files:
+	cp ${TARGET}/${ARTIFACT} ${BUILD_DIR}/lib
+	cp -a ${SAMPLES}/* ${BUILD_DIR}/samples/
+	cp -a ${BIN_DIR}/* ${BUILD_DIR}/bin/
+
+archive:
+	zip -r ${ARCHIVE} ${BUILD_DIR}
+
+clean:
+	rm -rf ${BUILD_DIR}
+	rm -f ${ARCHIVE}
diff --git a/bin/extract.sh b/bin/extract.sh
@@ -2,7 +2,7 @@
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."
 LIB="$DIR/lib"
 MAX_MEMORY="2g"
-JARFILE="$LIB/tableau-1.0.jar"
+JARFILE="$LIB/tableau-1.1.jar"
 SDK_DIR="$LIB/tableausdk-linux64-10300.18.0510.1135"
 JAVA_SDK_DIR="$SDK_DIR/lib64/tableausdk/Java/"
 

diff --git a/bin/publish.sh b/bin/publish.sh
@@ -2,7 +2,7 @@
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 BASE_PATH="$DIR/..";
 LIB="$BASE_PATH/lib"
-JARFILE="$LIB/tableau-1.0.jar"
+JARFILE="$LIB/tableau-1.1.jar"
 SDK_DIR="$LIB/tableausdk-linux64-10300.18.0510.1135"
 JAVA_SDK_DIR="$SDK_DIR/lib64/tableausdk/Java/"
 

diff --git a/pom.xml b/pom.xml
@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>net.jlmorton</groupId>
     <artifactId>tableau</artifactId>
-    <version>1.0</version>
+    <version>1.1</version>
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <java.language.version>1.8</java.language.version>

diff --git a/src/main/java/net/jlmorton/tableau/CommandLinePropertySource.java b/src/main/java/net/jlmorton/tableau/CommandLinePropertySource.java
@@ -1,10 +1,15 @@
 package net.jlmorton.tableau;
 
 import org.apache.commons.cli.*;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 import java.io.File;
+import java.io.IOException;
 
 class CommandLinePropertySource {
+    private static final Logger LOGGER = LogManager.getLogger(CommandLinePropertySource.class);
+
     private final String[] args;
 
     CommandLinePropertySource(String... args) {
@@ -16,8 +21,9 @@ Properties getProperties() {
 
         return new Properties() {
             @Override
-            public String getSchemaPath() {
-                return commandLine.getOptionValue("schema");
+            public Schema getSchema() {
+                String schemaPath = commandLine.getOptionValue("schema");
+                return createSchemaFromJson(schemaPath);
             }
 
             @Override
@@ -79,13 +85,18 @@ public boolean isExtract() {
                 return commandLine.hasOption("extract");
             }
 
-            @Override
-            public boolean isAppend() {
-                return commandLine.hasOption("append");
-            }
         };
     }
 
+    private Schema createSchemaFromJson(String schemaPath) {
+        try {
+            return Schema.fromJson(schemaPath);
+        } catch (IOException e) {
+            LOGGER.error("Error creating schema with path {}", schemaPath, e);
+            throw new RuntimeException(e);
+        }
+    }
+
     private CommandLine parseCommandLineOptions() {
         CommandLineParser parser = new DefaultParser();
 
@@ -97,6 +108,7 @@ private CommandLine parseCommandLineOptions() {
 
             return commandLine;
         } catch (ParseException e) {
+            LOGGER.error("Could not parse command line options", e);
             throw new RuntimeException("Could not parse command line options", e);
         }
     }
@@ -105,13 +117,11 @@ private static Options getOptions() {
         Options options = new Options();
         options.addOption("s", "schema", true, "Schema file for extract");
         options.addOption("f", "file", true, "CSV file to import");
-        options.addOption("a", "append", false, "Append to existing extract");
-        options.addOption("o", "output", true, "Output file name, or name of existing extract in append mode");
         options.addOption("t", "threads", true, "Number of threads (default: 1)");
         options.addOption("p", "publish", false, "Publish an extract to Tableau (requires --extract, --site, --project, --datasource, --username --password, and --url,");
         options.addOption("s", "site", true, "Tableau site name to publish");
         options.addOption("c", "project", true, "Project name to publish to");
-        options.addOption("e", "extract", true, "Filename of extract to publish");
+        options.addOption("e", "extract", true, "Filename of extract");
         options.addOption("d", "datasource", true, "Name of datasource to publish");
         options.addOption("u", "url", true, "Tableau Server URL for publishing");
         options.addOption("n", "username", true, "Tableau Server username for publishing");

diff --git a/src/main/java/net/jlmorton/tableau/ExtractAdapter.java b/src/main/java/net/jlmorton/tableau/ExtractAdapter.java
@@ -0,0 +1,19 @@
+package net.jlmorton.tableau;
+
+import com.tableausoftware.TableauException;
+import com.tableausoftware.extract.Row;
+import com.tableausoftware.extract.TableDefinition;
+
+public interface ExtractAdapter {
+    void openExtract() throws TableauException;
+
+    void closeExtract();
+
+    void insertRow(Row row) throws TableauException;
+
+    TableDefinition getTableDefinition();
+
+    default String getTableName() {
+        return "Extract";
+    }
+}
diff --git a/src/main/java/net/jlmorton/tableau/ExtractWriter.java b/src/main/java/net/jlmorton/tableau/ExtractWriter.java
@@ -1,7 +1,9 @@
 package net.jlmorton.tableau;
 
-import com.tableausoftware.extract.Extract;
+import com.tableausoftware.TableauException;
 
 public interface ExtractWriter {
-    Extract createExtract();
+    void writeExtract() throws TableauException;
+
+    void closeExtract();
 }
diff --git a/src/main/java/net/jlmorton/tableau/ExtractWriterImpl.java b/src/main/java/net/jlmorton/tableau/ExtractWriterImpl.java
diff --git a/src/main/java/net/jlmorton/tableau/Main.java b/src/main/java/net/jlmorton/tableau/Main.java
@@ -1,13 +1,11 @@
 package net.jlmorton.tableau;
 
 import com.tableausoftware.TableauException;
-import com.tableausoftware.extract.Extract;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import java.io.File;
-import java.io.IOException;
 import java.util.Objects;
 
 public class Main {
@@ -44,21 +42,20 @@ private static void publish(Properties properties) throws TableauException {
         Publisher.publish(properties);
     }
 
-    private static void createExtract(Properties properties) throws IOException {
+    private static void createExtract(Properties properties) throws TableauException {
         validatePropertiesForExtract(properties);
 
-        Schema schema = Schema.fromJson(properties.getSchemaPath());
-
-        LOGGER.info("Creating Extract {}", schema.getName());
+        LOGGER.info("Creating Extract {}", properties.getSchema().getName());
         LOGGER.info("CSV File Path: {}", properties.getCsvFile());
         LOGGER.info("Extract Path: {}", properties.getExtractFilePath());
         LOGGER.info("Number of Threads: {}", properties.getNumberOfThreads());
 
         RowInputSource rowInputSource = new CsvInputSource(properties.getCsvFile());
+        ExtractAdapter extractAdapter = new TdeExtractAdapter(properties);
 
-        ExtractWriter extractWriter = new ExtractWriterImpl(schema, rowInputSource, properties);
-        Extract extract = extractWriter.createExtract();
-        extract.close();
+        ExtractWriter extractWriter = new MultiThreadedExtractWriter(properties, rowInputSource, extractAdapter);
+        extractWriter.writeExtract();
+        extractWriter.closeExtract();
     }
 
     private static void validatePropertiesForPublishing(Properties properties) {
@@ -84,7 +81,7 @@ private static void validatePropertiesForPublishing(Properties properties) {
     private static void validatePropertiesForExtract(Properties properties) {
         boolean hasExtractPath = !Objects.isNull(properties.getExtractFilePath());
         boolean hasCsvFilePath = !Objects.isNull(properties.getCsvFile());
-        boolean hasSchemaPath = StringUtils.isNotBlank(properties.getSchemaPath());
+        boolean hasSchemaPath = !Objects.isNull(properties.getSchema());
 
         if (!(hasExtractPath && hasCsvFilePath && hasSchemaPath)) {
             LOGGER.error("Must provide extract path, CSV file path, and Schema path when creating an extract");