CBL-6400: Port the changes to QueryParser in 3.2 branch (to support U…

C/tests/c4ArrayIndexTest.cc

@@ -11,8 +11,6 @@
 #include "c4Query.h"
 #include "c4Query.hh"
 
-// Disabled pending CBL-6400
-#if 0
 class ArrayIndexTest : public C4Test {
   public:
     explicit ArrayIndexTest(int opt) : C4Test(opt) {}
@@ -132,20 +130,26 @@ constexpr std::string_view p0004 =
 N_WAY_TEST_CASE_METHOD(ArrayIndexTest, "Create Array Index with Empty Path", "[C][ArrayIndex]") {
     const auto defaultColl = REQUIRED(c4db_getDefaultCollection(db, ERROR_INFO()));
     C4Error    err{};
-    createArrayIndex(defaultColl, "arridx"_sl, nullslice, "", &err);
+    {
+        ExpectingExceptions x;
+        createArrayIndex(defaultColl, "arridx"_sl, nullslice, "", &err);
+    }
     CHECK(err.code == kC4ErrorInvalidQuery);
 }
 
 // 2. TestCreateArrayIndexWithInvalidExpressions
 N_WAY_TEST_CASE_METHOD(ArrayIndexTest, "Create Array Index with Invalid Expressions", "[C][ArrayIndex]") {
     const auto defaultColl = REQUIRED(c4db_getDefaultCollection(db, ERROR_INFO()));
     C4Error    err{};
+    {
+        ExpectingExceptions x;
 
-    createArrayIndex(defaultColl, "arridx"_sl, R"([".address.state", "", ".address.city"])", "contacts", &err);
-    CHECK(err.code == kC4ErrorInvalidQuery);
+        createArrayIndex(defaultColl, "arridx"_sl, R"([".address.state", "", ".address.city"])", "contacts", &err);
+        CHECK(err.code == kC4ErrorInvalidQuery);
 
-    createArrayIndex(defaultColl, "arridx"_sl, R"([".address.state", , ".address.city"])", "contacts", &err);
-    CHECK(err.code == kC4ErrorInvalidQuery);
+        createArrayIndex(defaultColl, "arridx"_sl, R"([".address.state", , ".address.city"])", "contacts", &err);
+        CHECK(err.code == kC4ErrorInvalidQuery);
+    }
 }
 
 // 3. TestCreateUpdateDeleteArrayIndexSingleLevel
@@ -605,7 +609,7 @@ N_WAY_TEST_CASE_METHOD(ArrayIndexTest, "Unnest Nested Non-Scalar Array", "[C][Un
 // 5. TestUnnestSingleLevelArrayWithGroupBy
 // Disabled until group-by is fixed
 // See https://jira.issues.couchbase.com/browse/CBL-6327
-#    if 0
+#if 0
 TEST_CASE_METHOD(ArrayIndexTest, "Unnest Single Level Array With Group By", "[C][Unnest]") {
     C4Collection* coll = createCollection(db, {"profiles"_sl, "_default"_sl});
     importTestData(coll);
@@ -616,7 +620,7 @@ TEST_CASE_METHOD(ArrayIndexTest, "Unnest Single Level Array With Group By", "[C]
     c4::ref queryenum = REQUIRED(c4query_run(query, nullslice, nullptr));
     validateQuery(queryenum, {});
 }
-#    endif
+#endif
 
 // 6. TestUnnestWithoutAlias
 N_WAY_TEST_CASE_METHOD(ArrayIndexTest, "Unnest Without Alias", "[C][Unnest]") {
@@ -671,5 +675,3 @@ N_WAY_TEST_CASE_METHOD(ArrayIndexTest, "Unnest Array Literal Not Supported", "[C
     REQUIRE(!query);
     CHECK(err.code == kC4ErrorInvalidQuery);
 }
-
-#endif

C/tests/c4QueryTest.cc

@@ -776,8 +776,6 @@ N_WAY_TEST_CASE_METHOD(C4QueryTest, "C4Query Join", "[Query][C]") {
     c4queryenum_release(e);
 }
 
-// Disabled pending CBL-6400
-#if 0
 N_WAY_TEST_CASE_METHOD(C4QueryTest, "C4Query UNNEST", "[Query][C][Unnest]") {
     for ( int withIndex = 0; withIndex <= 1; ++withIndex ) {
         if ( withIndex ) {
@@ -1072,7 +1070,6 @@ N_WAY_TEST_CASE_METHOD(NestedQueryTest, "C4Query Nested UNNEST - Missing Array",
         CHECK(run2(nullptr, 2) == results);
     }
 }
-#endif
 
 N_WAY_TEST_CASE_METHOD(C4QueryTest, "C4Query Seek", "[Query][C]") {
     compile(json5("['=', ['.', 'contact', 'address', 'state'], 'CA']"));

LiteCore/Query/IndexSpec.cc

@@ -31,7 +31,8 @@ namespace litecore {
         , queryLanguage(queryLanguage_)
         , options(std::move(opt)) {
         auto whichOpts = options.index();
-        if ( (type == kFullText && whichOpts != 1 && whichOpts != 0) || (type == kVector && whichOpts != 2) )
+        if ( (type == kFullText && whichOpts != 1 && whichOpts != 0) || (type == kVector && whichOpts != 2)
+             || (type == kArray && whichOpts != 3) )
             error::_throw(error::LiteCoreError::InvalidParameter, "Invalid options type for index");
     }
 
@@ -57,17 +58,24 @@ namespace litecore {
             switch ( queryLanguage ) {
                 case QueryLanguage::kJSON:
                     {
-                        _doc = Doc::fromJSON(expression).detach();
-                        if ( !_doc ) error::_throw(error::InvalidQuery, "Invalid JSON in index expression");
+                        if ( auto doc = Doc::fromJSON(expression); doc ) _doc = doc.detach();
+                        else
+                            error::_throw(error::InvalidQuery, "Invalid JSON in index expression");
                         break;
                     }
                 case QueryLanguage::kN1QL:
                     try {
-                        int           errPos;
-                        FLMutableDict result = n1ql::parse(string(expression), &errPos);
-                        if ( !result ) { throw Query::parseError("N1QL syntax error in index expression", errPos); }
-                        alloc_slice json(FLValue_ToJSON(FLValue(result)));
-                        FLMutableDict_Release(result);
+                        alloc_slice json;
+                        if ( !expression.empty() ) {
+                            int           errPos;
+                            FLMutableDict result = n1ql::parse(string(expression), &errPos);
+                            if ( !result ) { throw Query::parseError("N1QL syntax error in index expression", errPos); }
+                            json = FLValue_ToJSON(FLValue(result));
+                            FLMutableDict_Release(result);
+                        } else {
+                            // n1ql parser won't compile empty string to empty array. Do it manually.
+                            json = "[]";
+                        }
                         _doc = Doc::fromJSON(json).detach();
                     } catch ( const std::runtime_error& ) {
                         error::_throw(error::InvalidQuery, "Invalid N1QL in index expression");
@@ -88,7 +96,8 @@ namespace litecore {
             // of expressions.
             what = qt::requiredArray(doc.root(), "Index JSON");
         }
-        if ( what.empty() ) error::_throw(error::InvalidQuery, "Index WHAT list cannot be empty");
+        // Array Inddex can have empty what.
+        if ( type != kArray && what.empty() ) error::_throw(error::InvalidQuery, "Index WHAT list cannot be empty");
         return what;
     }
 
@@ -101,5 +110,61 @@ namespace litecore {
         return nullptr;
     }
 
+    // Turning unnestPath in C4IndexOptions to an array in JSON expresion.
+    // Ex. students[].interests -> [[".students"],[".interests"]]
+    FLArray IndexSpec::unnestPaths() const {
+        const ArrayOptions* arrayOpts = arrayOptions();
+        if ( !arrayOpts || !arrayOpts->unnestPath )
+            error::_throw(error::InvalidParameter, "IndexOptions for ArrayIndex must include unnestPath.");
 
+        Doc doc(unnestDoc());
+        if ( auto dict = doc.asDict(); dict ) {
+            if ( auto whatVal = qt::getCaseInsensitive(dict, "WHAT"); whatVal )
+                return qt::requiredArray(whatVal, "Index WHAT term");
+        }
+        return nullptr;
+    }
+
+    FLDoc IndexSpec::unnestDoc() const {
+        // Precondition: arrayOptions() && arrayOptions()->unnestPath
+        if ( !_unnestDoc ) {
+            try {
+                string_view unnestPath = arrayOptions()->unnestPath;
+                // Split unnestPath from the options by "[]."
+                std::vector<std::string_view> splitPaths;
+                for ( size_t pos = 0; pos != string::npos && pos < unnestPath.length(); ) {
+                    size_t next = unnestPath.find(KeyStore::kUnnestLevelSeparator, pos);
+                    if ( next == string::npos ) {
+                        splitPaths.push_back(unnestPath.substr(pos));
+                        pos = string::npos;
+                    } else {
+                        splitPaths.push_back(unnestPath.substr(pos, next - pos));
+                        pos = next + KeyStore::kUnnestLevelSeparator.size;
+                    }
+                }
+                if ( splitPaths.empty() )
+                    error::_throw(error::InvalidParameter,
+                                  "IndexOptions for ArrayIndex must have non-empty unnestPath.");
+
+                string n1qlUnnestPaths = string(splitPaths[0]);
+                for ( unsigned i = 1; i < splitPaths.size(); ++i ) {
+                    n1qlUnnestPaths += ", ";
+                    n1qlUnnestPaths += splitPaths[i];
+                }
+                int           errPos;
+                FLMutableDict result = n1ql::parse(n1qlUnnestPaths, &errPos);
+                if ( !result ) {
+                    string msg = "N1QL syntax error in unnestPath \"" + n1qlUnnestPaths + "\"";
+                    throw Query::parseError(msg.c_str(), errPos);
+                }
+
+                alloc_slice json{FLValue_ToJSON(FLValue(result))};
+                FLMutableDict_Release(result);
+                _unnestDoc = Doc::fromJSON(json).detach();
+            } catch ( const std::runtime_error& exc ) {
+                error::_throw(error::InvalidQuery, "Invalid N1QL in unnestPath (%s)", exc.what());
+            }
+        }
+        return _unnestDoc;
+    }
 }  // namespace litecore

LiteCore/Query/IndexSpec.hh

@@ -91,6 +91,9 @@ namespace litecore {
         /** The optional WHERE clause: the condition for a partial index */
         FLArray where() const;
 
+        /** The nested unnestPath from arrayOptions, as separated by "[]." is turned to an array. */
+        FLArray unnestPaths() const;
+
         std::string const name;           ///< Name of index
         Type const        type;           ///< Type of index
         alloc_slice const expression;     ///< The query expression
@@ -99,8 +102,10 @@ namespace litecore {
 
       private:
         FLDoc doc() const;
+        FLDoc unnestDoc() const;
 
-        mutable FLDoc _doc = nullptr;
+        mutable FLDoc _doc       = nullptr;
+        mutable FLDoc _unnestDoc = nullptr;
     };
 
 }  // namespace litecore

LiteCore/Query/SQLiteKeyStore+ArrayIndexes.cc

@@ -14,74 +14,115 @@
 #include "SQLiteDataFile.hh"
 #include "QueryTranslator.hh"
 #include "SQLUtil.hh"
+#include "SecureDigest.hh"
 #include "StringUtil.hh"
 #include "Array.hh"
 
 using namespace std;
-using namespace fleece;
 using namespace fleece::impl;
 
 namespace litecore {
 
     bool SQLiteKeyStore::createArrayIndex(const IndexSpec& spec) {
+        auto currSpec = db().getIndex(spec.name);
+        if ( currSpec ) {
+            // If there is already index with the index name,
+            // eiher delete the current one, or use it (return false)
+            while ( true ) {
+                if ( currSpec->type != IndexSpec::kArray ) break;
+                if ( !currSpec->arrayOptions() ) break;
+                if ( currSpec->arrayOptions()->unnestPath != spec.arrayOptions()->unnestPath ) break;
+                if ( !currSpec->what() || !spec.what() ) break;
+                alloc_slice currWhat = FLValue_ToJSON(FLValue(currSpec->what()));
+                alloc_slice specWhat = FLValue_ToJSON(FLValue(spec.what()));
+                if ( currWhat != specWhat ) break;
+                // Same index spec and unnestPath
+                return false;
+            }
+            db().deleteIndex(*currSpec);
+        }
+
+        // the following will throw if !spec.arrayOptions() || !spec.arrayOpeionts()->unnestPath
+        Array::iterator itPath((const Array*)spec.unnestPaths());
+        string          plainTableName, unnestTableName;
+        for ( ; itPath; ++itPath ) {
+            std::tie(plainTableName, unnestTableName) =
+                    createUnnestedTable(itPath.value(), plainTableName, unnestTableName);
+        }
         Array::iterator iExprs((const Array*)spec.what());
-        string          arrayTableName = createUnnestedTable(iExprs.value());
-        return createIndex(spec, arrayTableName, ++iExprs);
+        return createIndex(spec, plainTableName, iExprs);
     }
 
-    string SQLiteKeyStore::createUnnestedTable(const Value* expression) {
+    std::pair<string, string> SQLiteKeyStore::createUnnestedTable(const Value* expression, string plainParentTable,
+                                                                  string parentTable) {
         // Derive the table name from the expression it unnests:
-        string          kvTableName = tableName();
-        QueryTranslator qp(db(), "", kvTableName);
-        string          unnestTableName = qp.unnestedTableName(FLValue(expression));
+        if ( plainParentTable.empty() ) plainParentTable = parentTable = tableName();
+        QueryTranslator qp(db(), "", plainParentTable);
+        string          plainTableName    = qp.unnestedTableName(FLValue(expression));
+        string          unnestTableName   = hexName(plainTableName);
+        string          quotedParentTable = CONCAT(sqlIdentifier(parentTable));
 
         // Create the index table, unless an identical one already exists:
         string sql = CONCAT("CREATE TABLE " << sqlIdentifier(unnestTableName)
                                             << " "
                                                "(docid INTEGER NOT NULL REFERENCES "
-                                            << sqlIdentifier(kvTableName)
+                                            << sqlIdentifier(parentTable)
                                             << "(rowid), "
                                                " i INTEGER NOT NULL,"
                                                " body BLOB NOT NULL, "
-                                               " CONSTRAINT pk PRIMARY KEY (docid, i)) "
-                                               "WITHOUT ROWID");
+                                               " CONSTRAINT pk PRIMARY KEY (docid, i))");
         if ( !db().schemaExistsWithSQL(unnestTableName, "table", unnestTableName, sql) ) {
             LogTo(QueryLog, "Creating UNNEST table '%s' on %s", unnestTableName.c_str(),
                   expression->toJSON(true).asString().c_str());
             db().exec(sql);
 
             qp.setBodyColumnName("new.body");
             string eachExpr = qp.eachExpressionSQL(FLValue(expression));
+            bool   nested   = plainParentTable.find(KeyStore::kUnnestSeparator) != string::npos;
 
             // Populate the index-table with data from existing documents:
-            db().exec(CONCAT("INSERT INTO " << sqlIdentifier(unnestTableName)
-                                            << " (docid, i, body) "
-                                               "SELECT new.rowid, _each.rowid, _each.value "
-                                            << "FROM " << sqlIdentifier(kvTableName) << " as new, " << eachExpr
-                                            << " AS _each "
-                                               "WHERE (new.flags & 1) = 0"));
+            if ( !nested ) {
+                db().exec(CONCAT("INSERT INTO " << sqlIdentifier(unnestTableName)
+                                                << " (docid, i, body) "
+                                                   "SELECT new.rowid, _each.rowid, _each.value "
+                                                << "FROM " << sqlIdentifier(parentTable) << " as new, " << eachExpr
+                                                << " AS _each "
+                                                   "WHERE (new.flags & 1) = 0"));
+            } else {
+                db().exec(CONCAT("INSERT INTO " << sqlIdentifier(unnestTableName)
+                                                << " (docid, i, body) "
+                                                   "SELECT new.rowid, _each.rowid, _each.value "
+                                                << "FROM " << sqlIdentifier(parentTable) << " as new, " << eachExpr
+                                                << " AS _each"));
+            }
 
             // Set up triggers to keep the index-table up to date
             // ...on insertion:
             string insertTriggerExpr = CONCAT("INSERT INTO " << sqlIdentifier(unnestTableName)
                                                              << " (docid, i, body) "
                                                                 "SELECT new.rowid, _each.rowid, _each.value "
                                                              << "FROM " << eachExpr << " AS _each ");
-            createTrigger(unnestTableName, "ins", "AFTER INSERT", "WHEN (new.flags & 1) = 0", insertTriggerExpr);
-
             // ...on delete:
             string deleteTriggerExpr = CONCAT("DELETE FROM " << sqlIdentifier(unnestTableName)
                                                              << " "
                                                                 "WHERE docid = old.rowid");
-            createTrigger(unnestTableName, "del", "BEFORE DELETE", "WHEN (old.flags & 1) = 0", deleteTriggerExpr);
+            if ( !nested ) {
+                createTrigger(unnestTableName, "ins", "AFTER INSERT", "WHEN (new.flags & 1) = 0", insertTriggerExpr,
+                              quotedParentTable);
+                createTrigger(unnestTableName, "del", "BEFORE DELETE", "WHEN (old.flags & 1) = 0", deleteTriggerExpr,
+                              quotedParentTable);
 
-            // ...on update:
-            createTrigger(unnestTableName, "preupdate", "BEFORE UPDATE OF body, flags", "WHEN (old.flags & 1) = 0",
-                          deleteTriggerExpr);
-            createTrigger(unnestTableName, "postupdate", "AFTER UPDATE OF body, flags", "WHEN (new.flags & 1 = 0)",
-                          insertTriggerExpr);
+                // ...on update:
+                createTrigger(unnestTableName, "preupdate", "BEFORE UPDATE OF body, flags", "WHEN (old.flags & 1) = 0",
+                              deleteTriggerExpr, quotedParentTable);
+                createTrigger(unnestTableName, "postupdate", "AFTER UPDATE OF body, flags", "WHEN (new.flags & 1 = 0)",
+                              insertTriggerExpr, quotedParentTable);
+            } else {
+                createTrigger(unnestTableName, "ins", "AFTER INSERT", "", insertTriggerExpr, quotedParentTable);
+                createTrigger(unnestTableName, "del", "BEFORE DELETE", "", deleteTriggerExpr, quotedParentTable);
+            }
         }
-        return unnestTableName;
+        return {plainTableName, unnestTableName};
     }
 
 }  // namespace litecore

LiteCore/Query/SQLiteKeyStore+Indexes.cc

@@ -89,9 +89,10 @@ namespace litecore {
     bool SQLiteKeyStore::createIndex(const IndexSpec& spec, const string& sourceTableName,
                                      Array::iterator& expressions) {
         Assert(spec.type != IndexSpec::kFullText && spec.type != IndexSpec::kVector);
-        QueryTranslator qp(db(), "", sourceTableName);
-        qp.writeCreateIndex(spec.name, sourceTableName, (FLArrayIterator&)expressions, spec.where(),
-                            (spec.type != IndexSpec::kValue));
+        string          hexName{spec.type == IndexSpec::kArray ? litecore::hexName(sourceTableName) : ""};
+        QueryTranslator qp(db(), "", hexName.empty() ? sourceTableName : hexName);
+        qp.writeCreateIndex(spec.name, hexName.empty() ? sourceTableName : hexName, (FLArrayIterator&)expressions,
+                            spec.where(), (spec.type != IndexSpec::kValue));
         string sql = qp.SQL();
         return db().createIndex(spec, this, sourceTableName, sql);
     }