Skip to content
This repository has been archived by the owner on Jun 12, 2023. It is now read-only.

Commit

Permalink
Merge pull request #58 from Intel-HLS/dev
Browse files Browse the repository at this point in the history
Version 0.5.0
  • Loading branch information
stavrospapadopoulos authored Jan 13, 2017
2 parents 82ef689 + a400510 commit a54a7a7
Show file tree
Hide file tree
Showing 26 changed files with 2,528 additions and 231 deletions.
27 changes: 22 additions & 5 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,39 @@ sudo: required
dist: trusty

install:
#Install lcov and MPICH
- sudo apt-get -y install lcov mpich zlib1g-dev libssl-dev libgtest-dev
# Install lcov, MPICH, zlib, OpenSSL, Gtest and LZ4
- sudo apt-get -y install lcov mpich zlib1g-dev libssl-dev libgtest-dev liblz4-dev
# Install Zstandard
- wget https://github.com/facebook/zstd/archive/v1.0.0.tar.gz
- tar xf v1.0.0.tar.gz
- cd zstd-1.0.0
- sudo make install PREFIX='/usr'
- cd $TRAVIS_BUILD_DIR
# Install Blosc
- git clone https://github.com/Blosc/c-blosc
- cd c-blosc
- mkdir build
- cd build
- cmake -DCMAKE_INSTALL_PREFIX='/usr' ..
- cmake --build .
- sudo cmake --build . --target install
- cd $TRAVIS_BUILD_DIR
# Install Google Test
- cd /usr/src/gtest
- sudo cmake .
- sudo make
- sudo mv libgtest* /usr/lib/
- cd $TRAVIS_BUILD_DIR
# install lcov to coveralls conversion + upload tool
# Install lcov to coveralls conversion + upload tool
- gem install coveralls-lcov

before_script:
- lcov --directory . --zerocounters

script:
- make CXX=mpic++ INCLUDE_PATHS=-I/usr/include/mpich MPILIB=-lmpich -j 4
- make test CXX=mpic++ INCLUDE_PATHS=-I/usr/include/mpich MPILIB=-lmpich
- make clean
- make -j 4
- make test -j 4
- lcov --directory . --capture --output-file coverage.info
- lcov --list coverage.info # debug before upload

Expand Down
40 changes: 34 additions & 6 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,28 @@ ifeq ($(VERBOSE),1)
CPPFLAGS += -DVERBOSE
endif

# --- MAC address interface --- #
MAC_ADDRESS_INTERFACE =
ifneq ($(MAC_ADDRESS_INTERFACE),)
CPPFLAGS += -DTILEDB_MAC_ADDRESS_INTERFACE=$(MAC_ADDRESS_INTERFACE)
endif

# --- Compression levels --- #
COMPRESSION_LEVEL_GZIP =
ifneq ($(COMPRESSION_LEVEL_GZIP),)
CPPFLAGS += -DTILEDB_COMPRESSION_LEVEL_GZIP=$(COMPRESSION_LEVEL_GZIP)
endif

COMPRESSION_LEVEL_ZSTD =
ifneq ($(COMPRESSION_LEVEL_ZSTD),)
CPPFLAGS += -DTILEDB_COMPRESSION_LEVEL_ZSTD=$(COMPRESSION_LEVEL_ZSTD)
endif

COMPRESSION_LEVEL_BLOSC =
ifneq ($(COMPRESSION_LEVEL_BLOSC),)
CPPFLAGS += -DTILEDB_COMPRESSION_LEVEL_BLOSC=$(COMPRESSION_LEVEL_BLOSC)
endif

# --- Use parallel sort --- #
USE_PARALLEL_SORT =
ifeq ($(USE_PARALLEL_SORT),1)
Expand Down Expand Up @@ -137,7 +159,10 @@ ifdef TRAVIS
endif

# --- Libraries --- #
ZLIB = -lz
ZLIB = -lz
ZSTD = -lzstd
LZ4 = -llz4
BLOSC = -lblosc
OPENSSLLIB = -lcrypto
GTESTLIB = -lgtest -lgtest_main
MPILIB =
Expand Down Expand Up @@ -237,7 +262,8 @@ $(CORE_LIB_DIR)/libtiledb.$(SHLIB_EXT): $(CORE_OBJ)
@mkdir -p $(CORE_LIB_DIR)
@echo "Creating dynamic library libtiledb.$(SHLIB_EXT)"
@$(CXX) $(SHLIB_FLAGS) $(SONAME) -o $@ $^ $(LIBRARY_PATHS) $(MPILIB) \
$(PTHREADLIB) $(ZLIB) $(OPENSSLLIB) $(OPENMP_FLAG)
$(PTHREADLIB) $(ZLIB) $(ZSTD) $(LZ4) $(BLOSC) \
$(OPENSSLLIB) $(OPENMP_FLAG)

$(CORE_LIB_DIR)/libtiledb.a: $(CORE_OBJ)
@mkdir -p $(CORE_LIB_DIR)
Expand Down Expand Up @@ -275,8 +301,9 @@ $(EXAMPLES_OBJ_DIR)/%.o: $(EXAMPLES_SRC_DIR)/%.cc
$(EXAMPLES_BIN_DIR)/%: $(EXAMPLES_OBJ_DIR)/%.o $(CORE_LIB_DIR)/libtiledb.a
@mkdir -p $(EXAMPLES_BIN_DIR)
@echo "Creating $@"
@$(CXX) -std=gnu++11 -o $@ $^ $(LIBRARY_PATHS) $(MPILIB) $(ZLIB) \
$(PTHREADLIB) $(OPENSSLLIB) $(OPENMP_FLAG)
@$(CXX) -std=gnu++11 -o $@ $^ $(LIBRARY_PATHS) $(MPILIB) \
$(ZLIB) $(LZ4) $(ZSTD) $(BLOSC) \
$(PTHREADLIB) $(OPENSSLLIB) $(OPENMP_FLAG)

# --- Cleaning --- #

Expand Down Expand Up @@ -308,8 +335,9 @@ $(TEST_OBJ_DIR)/%.o: $(TEST_SRC_DIR)/%.cc

$(TEST_BIN_DIR)/tiledb_test: $(TEST_OBJ) $(CORE_LIB_DIR)/libtiledb.a
@mkdir -p $(TEST_BIN_DIR)
@echo "Creating test_cmd"
@$(CXX) -o $@ $^ $(LIBRARY_PATHS) $(MPILIB) $(ZLIB) \
@echo "Creating tiledb_test"
@$(CXX) -o $@ $^ $(LIBRARY_PATHS) $(MPILIB) \
$(ZLIB) $(ZSTD) $(LZ4) $(BLOSC) \
$(PTHREADLIB) $(OPENSSLLIB) $(GTESTLIB) $(OPENMP_FLAG)

# --- Cleaning --- #
Expand Down
8 changes: 4 additions & 4 deletions core/include/array/array.h
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
#include "array_sorted_write_state.h"
#include "array_schema.h"
#include "book_keeping.h"
#include "config.h"
#include "storage_manager_config.h"
#include "constants.h"
#include "fragment.h"
#include <pthread.h>
Expand Down Expand Up @@ -138,7 +138,7 @@ class Array {
const std::vector<int>& attribute_ids() const;

/** Returns the configuration parameters. */
const Config* config() const;
const StorageManagerConfig* config() const;

/** Returns the number of fragments in this array. */
int fragment_num() const;
Expand Down Expand Up @@ -307,7 +307,7 @@ class Array {
const char** attributes,
int attribute_num,
const void* subarray,
const Config* config,
const StorageManagerConfig* config,
Array* array_clone = NULL);

/**
Expand Down Expand Up @@ -471,7 +471,7 @@ class Array {
*/
std::vector<int> attribute_ids_;
/** Configuration parameters. */
const Config* config_;
const StorageManagerConfig* config_;
/** The array fragments. */
std::vector<Fragment*> fragments_;
/**
Expand Down
14 changes: 13 additions & 1 deletion core/include/array/array_schema.h
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,9 @@ class ArraySchema {
/** Returns the type of the i-th attribute, or NULL if 'i' is invalid. */
int type(int i) const;

/** Returns the type size of the i-th attribute. */
size_t type_size(int i) const;

/** Returns the number of attributes with variable-sized values. */
int var_attribute_num() const;

Expand Down Expand Up @@ -707,7 +710,16 @@ class ArraySchema {
* The compression type for each attribute (plus one extra at the end for the
* coordinates. It can be one of the following:
* - TILEDB_NO_COMPRESSION
* - TILEDB_GZIP.
* - TILEDB_GZIP
* - TILEDB_ZSTD
* - TILEDB_LZ4
* - TILEDB_BLOSC
* - TILEDB_BLOSC_LZ4
* - TILEDB_BLOSC_LZ4HC
* - TILEDB_BLOSC_SNAPPY
* - TILEDB_BLOSC_ZLIB
* - TILEDB_BLOSC_ZSTD
* - TILEDB_RLE
*/
std::vector<int> compression_;
/** Auxiliary variable used when calculating Hilbert ids. */
Expand Down
11 changes: 10 additions & 1 deletion core/include/array/array_schema_c.h
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,16 @@ typedef struct ArraySchemaC {
* The compression type for each attribute (plus one extra at the end for the
* coordinates. It can be one of the following:
* - TILEDB_NO_COMPRESSION
* - TILEDB_GZIP.
* - TILEDB_GZIP
* - TILEDB_ZSTD
* - TILEDB_LZ4
* - TILEDB_BLOSC
* - TILEDB_BLOSC_LZ4
* - TILEDB_BLOSC_LZ4HC
* - TILEDB_BLOSC_SNAPPY
* - TILEDB_BLOSC_ZLIB
* - TILEDB_BLOSC_ZSTD
* - TILEDB_RLE
*/
int* compression_;
/**
Expand Down
22 changes: 20 additions & 2 deletions core/include/c_api/c_api.h
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ typedef struct TileDB_Config {
/**
* The method for reading data from a file.
* It can be one of the following:
* - TILEDB_IO_MMAP
* - TILEDB_IO_MMAP (default)
* TileDB will use mmap.
* - TILEDB_IO_READ
* TileDB will use standard OS read.
Expand All @@ -111,7 +111,7 @@ typedef struct TileDB_Config {
/**
* The method for writing data to a file.
* It can be one of the following:
* - TILEDB_IO_WRITE
* - TILEDB_IO_WRITE (default)
* TileDB will use standard OS write.
* - TILEDB_IO_MPI
* TileDB will use MPI-IO write.
Expand Down Expand Up @@ -233,6 +233,15 @@ typedef struct TileDB_ArraySchema {
* coordinates). It can be one of the following:
* - TILEDB_NO_COMPRESSION
* - TILEDB_GZIP
* - TILEDB_ZSTD
* - TILEDB_LZ4
* - TILEDB_BLOSC
* - TILEDB_BLOSC_LZ4
* - TILEDB_BLOSC_LZ4HC
* - TILEDB_BLOSC_SNAPPY
* - TILEDB_BLOSC_ZLIB
* - TILEDB_BLOSC_ZSTD
* - TILEDB_RLE
*
* If it is *NULL*, then the default TILEDB_NO_COMPRESSION is used for all
* attributes.
Expand Down Expand Up @@ -737,6 +746,15 @@ typedef struct TileDB_MetadataSchema {
* key). It can be one of the following:
* - TILEDB_NO_COMPRESSION
* - TILEDB_GZIP
* - TILEDB_ZSTD
* - TILEDB_LZ4
* - TILEDB_BLOSC
* - TILEDB_BLOSC_LZ4
* - TILEDB_BLOSC_LZ4HC
* - TILEDB_BLOSC_SNAPPY
* - TILEDB_BLOSC_ZLIB
* - TILEDB_BLOSC_ZSTD
* - TILEDB_RLE
*
* If it is *NULL*, then the default TILEDB_NO_COMPRESSION is used for all
* attributes.
Expand Down
38 changes: 35 additions & 3 deletions core/include/c_api/constants.h
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
#include <limits.h>

/** Version. */
#define TILEDB_VERSION "0.4.0"
#define TILEDB_VERSION "0.5.0"

/**@{*/
/** Return code. */
Expand Down Expand Up @@ -134,6 +134,15 @@
/** Compression type. */
#define TILEDB_NO_COMPRESSION 0
#define TILEDB_GZIP 1
#define TILEDB_ZSTD 2
#define TILEDB_LZ4 3
#define TILEDB_BLOSC 4
#define TILEDB_BLOSC_LZ4 5
#define TILEDB_BLOSC_LZ4HC 6
#define TILEDB_BLOSC_SNAPPY 7
#define TILEDB_BLOSC_ZLIB 8
#define TILEDB_BLOSC_ZSTD 9
#define TILEDB_RLE 10
/**@}*/

/**@{*/
Expand Down Expand Up @@ -167,7 +176,30 @@
#define TILEDB_SORTED_BUFFER_VAR_SIZE 10000000 // ~10MB
/**@}*/

/** The alignment to assist vectorization. */
#define ALIGNMENT 64
/**@{*/
/** Compression levels. */
#ifndef TILEDB_COMPRESSION_LEVEL_GZIP
# define TILEDB_COMPRESSION_LEVEL_GZIP Z_DEFAULT_COMPRESSION
#endif
#ifndef TILEDB_COMPRESSION_LEVEL_ZSTD
# define TILEDB_COMPRESSION_LEVEL_ZSTD 1
#endif
#ifndef TILEDB_COMPRESSION_LEVEL_BLOSC
# define TILEDB_COMPRESSION_LEVEL_BLOSC 5
#endif
/**@}*/

/**@{*/
/** MAC address interface. */
#if defined(__APPLE__) && defined(__MACH__)
#ifndef TILEDB_MAC_ADDRESS_INTERFACE
#define TILEDB_MAC_ADDRESS_INTERFACE en0
#endif
#else
#ifndef TILEDB_MAC_ADDRESS_INTERFACE
#define TILEDB_MAC_ADDRESS_INTERFACE eth0
#endif
#endif
/**@}*/

#endif
Loading

0 comments on commit a54a7a7

Please sign in to comment.