ensure bulk load data to LMAX using sstPartitioner

Summary:
- register custom SstPartitionerFactory to rocksdb
- on bulk load, register index number used to the SstPartitionerFactory
- on compaction, SstPartitionerFactory create SstParitioner that will split sst files with keys overlapping with bulk load indexes
- on bulk load completion, trigger a compaction when sst file ingestion fails and retry

Reviewed By: hermanlee

Differential Revision: D41586846

fbshipit-source-id: fc33994a58636a1d3baf00607c3e7d2d517545a9

mysql-test/suite/rocksdb/r/add_index_inplace.result

@@ -298,6 +298,7 @@ rocksdb_bulk_load_allow_unsorted	OFF
 rocksdb_bulk_load_fail_if_not_bottommost_level	OFF
 rocksdb_bulk_load_partial_index	ON
 rocksdb_bulk_load_size	1000
+rocksdb_bulk_load_use_sst_partitioner	OFF
 show session variables like 'rocksdb_bulk_load%';
 Variable_name	Value
 rocksdb_bulk_load	ON
@@ -306,6 +307,7 @@ rocksdb_bulk_load_allow_unsorted	OFF
 rocksdb_bulk_load_fail_if_not_bottommost_level	OFF
 rocksdb_bulk_load_partial_index	ON
 rocksdb_bulk_load_size	1000
+rocksdb_bulk_load_use_sst_partitioner	OFF
 CREATE TABLE t1 (i INT, j INT, PRIMARY KEY (i)) ENGINE = ROCKSDB;
 INSERT INTO t1 VALUES (1,1);
 # Disconnecting on con1
@@ -352,6 +354,7 @@ rocksdb_bulk_load_allow_unsorted	OFF
 rocksdb_bulk_load_fail_if_not_bottommost_level	OFF
 rocksdb_bulk_load_partial_index	ON
 rocksdb_bulk_load_size	1000
+rocksdb_bulk_load_use_sst_partitioner	OFF
 CREATE TABLE t1 (a VARCHAR(30)) ENGINE=RocksDB;
 INSERT INTO t1 (a) VALUES (REPEAT("a", 30));
 INSERT INTO t1 (a) VALUES (REPEAT("a", 30));

mysql-test/suite/rocksdb/r/bulk_load_use_sst_partitioner.result

@@ -0,0 +1,41 @@
+SET rocksdb_bulk_load_use_sst_partitioner=0;
+SET rocksdb_bulk_load_fail_if_not_bottommost_level=1;
+SET rocksdb_bulk_load_allow_sk=1;
+CREATE TABLE t1(a INT, b INT, c INT, PRIMARY KEY(a), KEY k1(b, c), KEY k2(c) COMMENT 'cfname=rev:cf1') ENGINE=ROCKSDB;
+CREATE TABLE t2(a INT, b INT, c INT, PRIMARY KEY(a), KEY k1(b, c), KEY k2(c) COMMENT 'cfname=rev:cf1') ENGINE=ROCKSDB;
+CREATE TABLE t3(a INT, b INT, c INT, PRIMARY KEY(a), KEY k1(b, c), KEY k2(c) COMMENT 'cfname=rev:cf1') ENGINE=ROCKSDB;
+SET rocksdb_bulk_load=1;
+INSERT INTO t1 VALUES (1, 1, 1), (2, 1, 1), (3, 2, 1);
+INSERT INTO t3 VALUES (1, 1, 3), (2, 1, 3), (3, 2, 3);
+SET rocksdb_bulk_load=0;
+set global rocksdb_compact_cf="";
+set global rocksdb_compact_cf="rev:cf1";
+SET rocksdb_bulk_load=1;
+INSERT INTO t2 VALUES(1, 1, 2);
+INSERT INTO t2 VALUES(2, 1, 2);
+INSERT INTO t2 VALUES(3, 2, 2);
+SET rocksdb_bulk_load=0;
+ERROR HY000: [(null)] bulk load error: Operation failed. Try again.: Files cannot be ingested to Lmax. Please make sure key range of Lmax and ongoing compaction's output to Lmaxdoes not overlap with files to ingest.
+select * from t2;
+a	b	c
+SET rocksdb_bulk_load_use_sst_partitioner=1;
+CREATE TABLE t4(a INT, b INT, c INT, PRIMARY KEY(a), KEY k1(b, c), KEY k2(c) COMMENT 'cfname=rev:cf1') ENGINE=ROCKSDB;
+CREATE TABLE t5(a INT, b INT, c INT, PRIMARY KEY(a), KEY k1(b, c), KEY k2(c) COMMENT 'cfname=rev:cf1') ENGINE=ROCKSDB;
+CREATE TABLE t6(a INT, b INT, c INT, PRIMARY KEY(a), KEY k1(b, c), KEY k2(c) COMMENT 'cfname=rev:cf1') ENGINE=ROCKSDB;
+SET rocksdb_bulk_load=1;
+INSERT INTO t4 VALUES (1, 1, 4), (2, 1, 4), (3, 2, 4);
+INSERT INTO t6 VALUES (1, 1, 6), (2, 1, 6), (3, 2, 6);
+SET rocksdb_bulk_load=0;
+set global rocksdb_compact_cf="";
+set global rocksdb_compact_cf="rev:cf1";
+SET rocksdb_bulk_load=1;
+INSERT INTO t5 VALUES(1, 1, 5);
+INSERT INTO t5 VALUES(2, 1, 5);
+INSERT INTO t5 VALUES(3, 2, 5);
+SET rocksdb_bulk_load=0;
+select * from t5;
+a	b	c
+1	1	5
+2	1	5
+3	2	5
+drop table t1, t2, t3, t4, t5, t6;

mysql-test/suite/rocksdb/r/rocksdb.result

@@ -912,6 +912,7 @@ rocksdb_bulk_load_allow_unsorted	OFF
 rocksdb_bulk_load_fail_if_not_bottommost_level	OFF
 rocksdb_bulk_load_partial_index	ON
 rocksdb_bulk_load_size	1000
+rocksdb_bulk_load_use_sst_partitioner	OFF
 rocksdb_bytes_per_sync	0
 rocksdb_cache_dump	ON
 rocksdb_cache_high_pri_pool_ratio	0.000000

mysql-test/suite/rocksdb/t/bulk_load_use_sst_partitioner.test

@@ -0,0 +1,80 @@
+# test cases verify the features that ensure the bulk load sst files
+# will be placed in rocksdb bottommost level
+--source include/have_rocksdb.inc
+--source include/count_sessions.inc
+
+--disable_query_log
+call mtr.add_suppression("failed to bulk load. status code = 13, status = Operation failed. Try again.");
+call mtr.add_suppression("Error 504 finalizing last SST file while setting bulk loading variable");
+--enable_query_log
+
+# test case:
+# verify bulk load fail when bottommost lvl has sst file with overlap key range
+SET rocksdb_bulk_load_use_sst_partitioner=0;
+SET rocksdb_bulk_load_fail_if_not_bottommost_level=1;
+SET rocksdb_bulk_load_allow_sk=1;
+
+# index numbers in t2 will be greater than index number in t1 and less than
+# index numbers in t3
+CREATE TABLE t1(a INT, b INT, c INT, PRIMARY KEY(a), KEY k1(b, c), KEY k2(c) COMMENT 'cfname=rev:cf1') ENGINE=ROCKSDB;
+CREATE TABLE t2(a INT, b INT, c INT, PRIMARY KEY(a), KEY k1(b, c), KEY k2(c) COMMENT 'cfname=rev:cf1') ENGINE=ROCKSDB;
+CREATE TABLE t3(a INT, b INT, c INT, PRIMARY KEY(a), KEY k1(b, c), KEY k2(c) COMMENT 'cfname=rev:cf1') ENGINE=ROCKSDB;
+
+SET rocksdb_bulk_load=1;
+INSERT INTO t1 VALUES (1, 1, 1), (2, 1, 1), (3, 2, 1);
+INSERT INTO t3 VALUES (1, 1, 3), (2, 1, 3), (3, 2, 3);
+SET rocksdb_bulk_load=0;
+
+# compact the LMAX sst files created in the bulk load before
+set global rocksdb_compact_cf="";
+set global rocksdb_compact_cf="rev:cf1";
+
+SET rocksdb_bulk_load=1;
+INSERT INTO t2 VALUES(1, 1, 2);
+INSERT INTO t2 VALUES(2, 1, 2);
+INSERT INTO t2 VALUES(3, 2, 2);
+
+# verify we get an error
+--error ER_UNKNOWN_ERROR
+SET rocksdb_bulk_load=0;
+
+# verify data not loaded to db
+select * from t2;
+
+# test case:
+# same as previous test case, except rocksdb_bulk_load_use_sst_partitioner
+# is set
+SET rocksdb_bulk_load_use_sst_partitioner=1;
+
+CREATE TABLE t4(a INT, b INT, c INT, PRIMARY KEY(a), KEY k1(b, c), KEY k2(c) COMMENT 'cfname=rev:cf1') ENGINE=ROCKSDB;
+CREATE TABLE t5(a INT, b INT, c INT, PRIMARY KEY(a), KEY k1(b, c), KEY k2(c) COMMENT 'cfname=rev:cf1') ENGINE=ROCKSDB;
+CREATE TABLE t6(a INT, b INT, c INT, PRIMARY KEY(a), KEY k1(b, c), KEY k2(c) COMMENT 'cfname=rev:cf1') ENGINE=ROCKSDB;
+
+SET rocksdb_bulk_load=1;
+INSERT INTO t4 VALUES (1, 1, 4), (2, 1, 4), (3, 2, 4);
+INSERT INTO t6 VALUES (1, 1, 6), (2, 1, 6), (3, 2, 6);
+SET rocksdb_bulk_load=0;
+
+# compact the LMAX sst files created in the bulk load before
+set global rocksdb_compact_cf="";
+set global rocksdb_compact_cf="rev:cf1";
+
+SET rocksdb_bulk_load=1;
+INSERT INTO t5 VALUES(1, 1, 5);
+INSERT INTO t5 VALUES(2, 1, 5);
+INSERT INTO t5 VALUES(3, 2, 5);
+
+# now there is no error
+SET rocksdb_bulk_load=0;
+
+# verify data loaded to db
+select * from t5;
+
+# clean up
+drop table t1, t2, t3, t4, t5, t6;
+
+--let SEARCH_FILE=$MYSQLTEST_VARDIR/log/mysqld.1.err
+--let SEARCH_PATTERN=MyRocks: failed to bulk load, retry with compaction.
+--source include/search_pattern_in_file.inc
+
+--source include/wait_until_count_sessions.inc

mysql-test/suite/rocksdb_sys_vars/r/rocksdb_bulk_load_use_sst_partitioner_basic.result

@@ -0,0 +1,121 @@
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(1);
+INSERT INTO valid_values VALUES(0);
+INSERT INTO valid_values VALUES('on');
+INSERT INTO valid_values VALUES('off');
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO invalid_values VALUES('\'aaa\'');
+INSERT INTO invalid_values VALUES('\'bbb\'');
+SET @start_global_value = @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+SELECT @start_global_value;
+@start_global_value
+0
+SET @start_session_value = @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+SELECT @start_session_value;
+@start_session_value
+0
+'# Setting to valid values in global scope#'
+"Trying to set variable @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER to 1"
+SET @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER   = 1;
+SELECT @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+1
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER = DEFAULT;
+SELECT @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+0
+"Trying to set variable @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER to 0"
+SET @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER   = 0;
+SELECT @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+0
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER = DEFAULT;
+SELECT @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+0
+"Trying to set variable @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER to on"
+SET @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER   = on;
+SELECT @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+1
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER = DEFAULT;
+SELECT @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+0
+"Trying to set variable @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER to off"
+SET @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER   = off;
+SELECT @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+0
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER = DEFAULT;
+SELECT @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+0
+'# Setting to valid values in session scope#'
+"Trying to set variable @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER to 1"
+SET @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER   = 1;
+SELECT @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+1
+"Setting the session scope variable back to default"
+SET @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER = DEFAULT;
+SELECT @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+0
+"Trying to set variable @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER to 0"
+SET @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER   = 0;
+SELECT @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+0
+"Setting the session scope variable back to default"
+SET @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER = DEFAULT;
+SELECT @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+0
+"Trying to set variable @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER to on"
+SET @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER   = on;
+SELECT @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+1
+"Setting the session scope variable back to default"
+SET @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER = DEFAULT;
+SELECT @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+0
+"Trying to set variable @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER to off"
+SET @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER   = off;
+SELECT @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+0
+"Setting the session scope variable back to default"
+SET @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER = DEFAULT;
+SELECT @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+0
+'# Testing with invalid values in global scope #'
+"Trying to set variable @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER to 'aaa'"
+SET @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER   = 'aaa';
+Got one of the listed errors
+SELECT @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+0
+"Trying to set variable @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER to 'bbb'"
+SET @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER   = 'bbb';
+Got one of the listed errors
+SELECT @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+0
+SET @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER = @start_global_value;
+SELECT @@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@global.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+0
+SET @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER = @start_session_value;
+SELECT @@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER;
+@@session.ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+0
+DROP TABLE valid_values;
+DROP TABLE invalid_values;

mysql-test/suite/rocksdb_sys_vars/t/rocksdb_bulk_load_use_sst_partitioner_basic.test

@@ -0,0 +1,19 @@
+--source include/have_rocksdb.inc
+
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(1);
+INSERT INTO valid_values VALUES(0);
+INSERT INTO valid_values VALUES('on');
+INSERT INTO valid_values VALUES('off');
+
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO invalid_values VALUES('\'aaa\'');
+INSERT INTO invalid_values VALUES('\'bbb\'');
+
+--let $sys_var=ROCKSDB_BULK_LOAD_USE_SST_PARTITIONER
+--let $read_only=0
+--let $session=1
+--source ../include/rocksdb_sys_var.inc
+
+DROP TABLE valid_values;
+DROP TABLE invalid_values;

storage/rocksdb/CMakeLists.txt

@@ -290,6 +290,7 @@ SET(ROCKSDB_SOURCES
   logger.h
   rdb_datadic.cc rdb_datadic.h
   rdb_iterator.cc rdb_iterator.h
+  rdb_sst_partitioner_factory.h
   rdb_cf_options.cc rdb_cf_options.h
   rdb_cf_manager.cc rdb_cf_manager.h
   rdb_converter.cc rdb_converter.h