Merge pull request #57 from stavrospapadopoulos/dev

Support for new compression methods

.travis.yml

@@ -3,22 +3,39 @@ sudo: required
 dist: trusty
 
 install:
-    #Install lcov and MPICH
-    - sudo apt-get -y install lcov mpich zlib1g-dev libssl-dev libgtest-dev
+    # Install lcov, MPICH, zlib, OpenSSL, Gtest and LZ4
+    - sudo apt-get -y install lcov mpich zlib1g-dev libssl-dev libgtest-dev liblz4-dev
+    # Install Zstandard
+    - wget https://github.com/facebook/zstd/archive/v1.0.0.tar.gz
+    - tar xf v1.0.0.tar.gz
+    - cd zstd-1.0.0
+    - sudo make install PREFIX='/usr'
+    - cd $TRAVIS_BUILD_DIR
+    # Install Blosc
+    - git clone https://github.com/Blosc/c-blosc 
+    - cd c-blosc
+    - mkdir build
+    - cd build
+    - cmake -DCMAKE_INSTALL_PREFIX='/usr' ..
+    - cmake --build .
+    - sudo cmake --build . --target install
+    - cd $TRAVIS_BUILD_DIR
+    # Install Google Test
     - cd /usr/src/gtest
     - sudo cmake .
     - sudo make
     - sudo mv libgtest* /usr/lib/
     - cd $TRAVIS_BUILD_DIR
-    # install lcov to coveralls conversion + upload tool
+    # Install lcov to coveralls conversion + upload tool
     - gem install coveralls-lcov
 
 before_script:
     - lcov --directory . --zerocounters
 
 script:
-    - make CXX=mpic++ INCLUDE_PATHS=-I/usr/include/mpich MPILIB=-lmpich -j 4
-    - make test CXX=mpic++ INCLUDE_PATHS=-I/usr/include/mpich MPILIB=-lmpich 
+    - make clean
+    - make -j 4
+    - make test -j 4 
     - lcov --directory . --capture --output-file coverage.info
     - lcov --list coverage.info # debug before upload
 

Makefile

@@ -48,6 +48,28 @@ ifeq ($(VERBOSE),1)
   CPPFLAGS += -DVERBOSE
 endif
 
+# --- MAC address interface --- #
+MAC_ADDRESS_INTERFACE =
+ifneq ($(MAC_ADDRESS_INTERFACE),)
+  CPPFLAGS += -DTILEDB_MAC_ADDRESS_INTERFACE=$(MAC_ADDRESS_INTERFACE)
+endif
+
+# --- Compression levels --- #
+COMPRESSION_LEVEL_GZIP =
+ifneq ($(COMPRESSION_LEVEL_GZIP),)
+  CPPFLAGS += -DTILEDB_COMPRESSION_LEVEL_GZIP=$(COMPRESSION_LEVEL_GZIP)
+endif
+
+COMPRESSION_LEVEL_ZSTD =
+ifneq ($(COMPRESSION_LEVEL_ZSTD),)
+  CPPFLAGS += -DTILEDB_COMPRESSION_LEVEL_ZSTD=$(COMPRESSION_LEVEL_ZSTD)
+endif
+
+COMPRESSION_LEVEL_BLOSC =
+ifneq ($(COMPRESSION_LEVEL_BLOSC),)
+  CPPFLAGS += -DTILEDB_COMPRESSION_LEVEL_BLOSC=$(COMPRESSION_LEVEL_BLOSC)
+endif
+
 # --- Use parallel sort --- #
 USE_PARALLEL_SORT =
 ifeq ($(USE_PARALLEL_SORT),1)
@@ -137,7 +159,10 @@ ifdef TRAVIS
 endif
 
 # --- Libraries --- #
-ZLIB = -lz
+ZLIB = -lz 
+ZSTD = -lzstd
+LZ4 = -llz4
+BLOSC = -lblosc
 OPENSSLLIB = -lcrypto
 GTESTLIB = -lgtest -lgtest_main
 MPILIB =
@@ -237,7 +262,8 @@ $(CORE_LIB_DIR)/libtiledb.$(SHLIB_EXT): $(CORE_OBJ)
 	@mkdir -p $(CORE_LIB_DIR)
 	@echo "Creating dynamic library libtiledb.$(SHLIB_EXT)"
 	@$(CXX) $(SHLIB_FLAGS) $(SONAME) -o $@ $^ $(LIBRARY_PATHS) $(MPILIB) \
-		$(PTHREADLIB) $(ZLIB) $(OPENSSLLIB) $(OPENMP_FLAG)
+		$(PTHREADLIB) $(ZLIB) $(ZSTD) $(LZ4) $(BLOSC) \
+                $(OPENSSLLIB) $(OPENMP_FLAG)
 
 $(CORE_LIB_DIR)/libtiledb.a: $(CORE_OBJ)
 	@mkdir -p $(CORE_LIB_DIR)
@@ -275,8 +301,9 @@ $(EXAMPLES_OBJ_DIR)/%.o: $(EXAMPLES_SRC_DIR)/%.cc
 $(EXAMPLES_BIN_DIR)/%: $(EXAMPLES_OBJ_DIR)/%.o $(CORE_LIB_DIR)/libtiledb.a
 	@mkdir -p $(EXAMPLES_BIN_DIR)
 	@echo "Creating $@"
-	@$(CXX) -std=gnu++11 -o $@ $^ $(LIBRARY_PATHS) $(MPILIB) $(ZLIB) \
-		 $(PTHREADLIB) $(OPENSSLLIB) $(OPENMP_FLAG) 
+	@$(CXX) -std=gnu++11 -o $@ $^ $(LIBRARY_PATHS) $(MPILIB) \
+                 $(ZLIB) $(LZ4) $(ZSTD) $(BLOSC) \
+                 $(PTHREADLIB) $(OPENSSLLIB) $(OPENMP_FLAG) 
 
 # --- Cleaning --- #
 
@@ -308,8 +335,9 @@ $(TEST_OBJ_DIR)/%.o: $(TEST_SRC_DIR)/%.cc
 
 $(TEST_BIN_DIR)/tiledb_test: $(TEST_OBJ) $(CORE_LIB_DIR)/libtiledb.a
 	@mkdir -p $(TEST_BIN_DIR)
-	@echo "Creating test_cmd"
-	@$(CXX) -o $@ $^ $(LIBRARY_PATHS) $(MPILIB) $(ZLIB) \
+	@echo "Creating tiledb_test"
+	@$(CXX) -o $@ $^ $(LIBRARY_PATHS) $(MPILIB) \
+                $(ZLIB) $(ZSTD) $(LZ4) $(BLOSC) \
 		$(PTHREADLIB) $(OPENSSLLIB) $(GTESTLIB) $(OPENMP_FLAG) 
 
 # --- Cleaning --- #

core/include/array/array.h

@@ -39,7 +39,7 @@
 #include "array_sorted_write_state.h"
 #include "array_schema.h"
 #include "book_keeping.h"
-#include "config.h"
+#include "storage_manager_config.h"
 #include "constants.h"
 #include "fragment.h"
 #include <pthread.h>
@@ -138,7 +138,7 @@ class Array {
   const std::vector<int>& attribute_ids() const;
 
   /** Returns the configuration parameters. */
-  const Config* config() const;
+  const StorageManagerConfig* config() const;
 
   /** Returns the number of fragments in this array. */
   int fragment_num() const;
@@ -307,7 +307,7 @@ class Array {
       const char** attributes,
       int attribute_num,
       const void* subarray,
-      const Config* config,
+      const StorageManagerConfig* config,
       Array* array_clone = NULL);
 
   /**
@@ -471,7 +471,7 @@ class Array {
    */
   std::vector<int> attribute_ids_;
   /** Configuration parameters. */
-  const Config* config_;
+  const StorageManagerConfig* config_;
   /** The array fragments. */
   std::vector<Fragment*> fragments_;
   /** 

core/include/array/array_schema.h

@@ -294,6 +294,9 @@ class ArraySchema {
   /** Returns the type of the i-th attribute, or NULL if 'i' is invalid. */
   int type(int i) const;
 
+  /** Returns the type size of the i-th attribute. */
+  size_t type_size(int i) const;
+
   /** Returns the number of attributes with variable-sized values. */
   int var_attribute_num() const;
 
@@ -707,7 +710,16 @@ class ArraySchema {
    * The compression type for each attribute (plus one extra at the end for the
    * coordinates. It can be one of the following: 
    *    - TILEDB_NO_COMPRESSION
-   *    - TILEDB_GZIP. 
+   *    - TILEDB_GZIP
+   *    - TILEDB_ZSTD 
+   *    - TILEDB_LZ4 
+   *    - TILEDB_BLOSC 
+   *    - TILEDB_BLOSC_LZ4 
+   *    - TILEDB_BLOSC_LZ4HC 
+   *    - TILEDB_BLOSC_SNAPPY 
+   *    - TILEDB_BLOSC_ZLIB 
+   *    - TILEDB_BLOSC_ZSTD 
+   *    - TILEDB_RLE 
    */
   std::vector<int> compression_;
   /** Auxiliary variable used when calculating Hilbert ids. */

core/include/array/array_schema_c.h

@@ -69,7 +69,16 @@ typedef struct ArraySchemaC {
    * The compression type for each attribute (plus one extra at the end for the
    * coordinates. It can be one of the following: 
    *    - TILEDB_NO_COMPRESSION
-   *    - TILEDB_GZIP. 
+   *    - TILEDB_GZIP 
+   *    - TILEDB_ZSTD 
+   *    - TILEDB_LZ4 
+   *    - TILEDB_BLOSC 
+   *    - TILEDB_BLOSC_LZ4 
+   *    - TILEDB_BLOSC_LZ4HC 
+   *    - TILEDB_BLOSC_SNAPPY 
+   *    - TILEDB_BLOSC_ZLIB 
+   *    - TILEDB_BLOSC_ZSTD 
+   *    - TILEDB_RLE 
    */
   int* compression_;
   /** 

core/include/c_api/c_api.h

@@ -100,7 +100,7 @@ typedef struct TileDB_Config {
   /** 
    * The method for reading data from a file. 
    * It can be one of the following: 
-   *    - TILEDB_IO_MMAP
+   *    - TILEDB_IO_MMAP (default)
    *      TileDB will use mmap.
    *    - TILEDB_IO_READ
    *      TileDB will use standard OS read.
@@ -111,7 +111,7 @@ typedef struct TileDB_Config {
   /** 
    * The method for writing data to a file. 
    * It can be one of the following: 
-   *    - TILEDB_IO_WRITE
+   *    - TILEDB_IO_WRITE (default)
    *      TileDB will use standard OS write.
    *    - TILEDB_IO_MPI
    *      TileDB will use MPI-IO write. 
@@ -233,6 +233,15 @@ typedef struct TileDB_ArraySchema {
    * coordinates). It can be one of the following: 
    *    - TILEDB_NO_COMPRESSION
    *    - TILEDB_GZIP 
+   *    - TILEDB_ZSTD 
+   *    - TILEDB_LZ4 
+   *    - TILEDB_BLOSC 
+   *    - TILEDB_BLOSC_LZ4 
+   *    - TILEDB_BLOSC_LZ4HC 
+   *    - TILEDB_BLOSC_SNAPPY 
+   *    - TILEDB_BLOSC_ZLIB 
+   *    - TILEDB_BLOSC_ZSTD 
+   *    - TILEDB_RLE
    *
    * If it is *NULL*, then the default TILEDB_NO_COMPRESSION is used for all
    * attributes.
@@ -737,6 +746,15 @@ typedef struct TileDB_MetadataSchema {
    * key). It can be one of the following: 
    *    - TILEDB_NO_COMPRESSION
    *    - TILEDB_GZIP 
+   *    - TILEDB_ZSTD 
+   *    - TILEDB_LZ4 
+   *    - TILEDB_BLOSC 
+   *    - TILEDB_BLOSC_LZ4 
+   *    - TILEDB_BLOSC_LZ4HC 
+   *    - TILEDB_BLOSC_SNAPPY 
+   *    - TILEDB_BLOSC_ZLIB 
+   *    - TILEDB_BLOSC_ZSTD 
+   *    - TILEDB_RLE
    *
    * If it is *NULL*, then the default TILEDB_NO_COMPRESSION is used for all
    * attributes.

core/include/c_api/constants.h

@@ -37,7 +37,7 @@
 #include <limits.h>
 
 /** Version. */
-#define TILEDB_VERSION                          "0.4.0"
+#define TILEDB_VERSION                          "0.5.0"
 
 /**@{*/
 /** Return code. */  
@@ -134,6 +134,15 @@
 /** Compression type. */
 #define TILEDB_NO_COMPRESSION                        0
 #define TILEDB_GZIP                                  1
+#define TILEDB_ZSTD                                  2
+#define TILEDB_LZ4                                   3
+#define TILEDB_BLOSC                                 4
+#define TILEDB_BLOSC_LZ4                             5
+#define TILEDB_BLOSC_LZ4HC                           6
+#define TILEDB_BLOSC_SNAPPY                          7
+#define TILEDB_BLOSC_ZLIB                            8
+#define TILEDB_BLOSC_ZSTD                            9
+#define TILEDB_RLE                                  10
 /**@}*/
 
 /**@{*/
@@ -167,7 +176,30 @@
 #define TILEDB_SORTED_BUFFER_VAR_SIZE         10000000  // ~10MB
 /**@}*/
 
-/** The alignment to assist vectorization. */
-#define ALIGNMENT 64
+/**@{*/
+/** Compression levels. */
+#ifndef TILEDB_COMPRESSION_LEVEL_GZIP
+#  define TILEDB_COMPRESSION_LEVEL_GZIP      Z_DEFAULT_COMPRESSION
+#endif
+#ifndef TILEDB_COMPRESSION_LEVEL_ZSTD
+#  define TILEDB_COMPRESSION_LEVEL_ZSTD                          1
+#endif
+#ifndef TILEDB_COMPRESSION_LEVEL_BLOSC
+#  define TILEDB_COMPRESSION_LEVEL_BLOSC                         5
+#endif
+/**@}*/
+
+/**@{*/
+/** MAC address interface. */
+#if defined(__APPLE__) && defined(__MACH__)
+  #ifndef TILEDB_MAC_ADDRESS_INTERFACE
+    #define TILEDB_MAC_ADDRESS_INTERFACE                       en0
+  #endif
+#else
+  #ifndef TILEDB_MAC_ADDRESS_INTERFACE
+    #define TILEDB_MAC_ADDRESS_INTERFACE                      eth0
+  #endif
+#endif
+/**@}*/
 
 #endif