CBL-6400: Port the changes to QueryParser in 3.2 branch (to support U…

C/tests/c4ArrayIndexTest.cc

@@ -11,8 +11,6 @@
 #include "c4Query.h"
 #include "c4Query.hh"
 
-// Disabled pending CBL-6400
-#if 0
 class ArrayIndexTest : public C4Test {
   public:
     explicit ArrayIndexTest(int opt) : C4Test(opt) {}
@@ -132,20 +130,26 @@ constexpr std::string_view p0004 =
 N_WAY_TEST_CASE_METHOD(ArrayIndexTest, "Create Array Index with Empty Path", "[C][ArrayIndex]") {
     const auto defaultColl = REQUIRED(c4db_getDefaultCollection(db, ERROR_INFO()));
     C4Error    err{};
-    createArrayIndex(defaultColl, "arridx"_sl, nullslice, "", &err);
+    {
+        ExpectingExceptions x;
+        createArrayIndex(defaultColl, "arridx"_sl, nullslice, "", &err);
+    }
     CHECK(err.code == kC4ErrorInvalidQuery);
 }
 
 // 2. TestCreateArrayIndexWithInvalidExpressions
 N_WAY_TEST_CASE_METHOD(ArrayIndexTest, "Create Array Index with Invalid Expressions", "[C][ArrayIndex]") {
     const auto defaultColl = REQUIRED(c4db_getDefaultCollection(db, ERROR_INFO()));
     C4Error    err{};
+    {
+        ExpectingExceptions x;
 
-    createArrayIndex(defaultColl, "arridx"_sl, R"([".address.state", "", ".address.city"])", "contacts", &err);
-    CHECK(err.code == kC4ErrorInvalidQuery);
+        createArrayIndex(defaultColl, "arridx"_sl, R"([".address.state", "", ".address.city"])", "contacts", &err);
+        CHECK(err.code == kC4ErrorInvalidQuery);
 
-    createArrayIndex(defaultColl, "arridx"_sl, R"([".address.state", , ".address.city"])", "contacts", &err);
-    CHECK(err.code == kC4ErrorInvalidQuery);
+        createArrayIndex(defaultColl, "arridx"_sl, R"([".address.state", , ".address.city"])", "contacts", &err);
+        CHECK(err.code == kC4ErrorInvalidQuery);
+    }
 }
 
 // 3. TestCreateUpdateDeleteArrayIndexSingleLevel
@@ -605,7 +609,7 @@ N_WAY_TEST_CASE_METHOD(ArrayIndexTest, "Unnest Nested Non-Scalar Array", "[C][Un
 // 5. TestUnnestSingleLevelArrayWithGroupBy
 // Disabled until group-by is fixed
 // See https://jira.issues.couchbase.com/browse/CBL-6327
-#    if 0
+#if 0
 TEST_CASE_METHOD(ArrayIndexTest, "Unnest Single Level Array With Group By", "[C][Unnest]") {
     C4Collection* coll = createCollection(db, {"profiles"_sl, "_default"_sl});
     importTestData(coll);
@@ -616,7 +620,7 @@ TEST_CASE_METHOD(ArrayIndexTest, "Unnest Single Level Array With Group By", "[C]
     c4::ref queryenum = REQUIRED(c4query_run(query, nullslice, nullptr));
     validateQuery(queryenum, {});
 }
-#    endif
+#endif
 
 // 6. TestUnnestWithoutAlias
 N_WAY_TEST_CASE_METHOD(ArrayIndexTest, "Unnest Without Alias", "[C][Unnest]") {
@@ -671,5 +675,3 @@ N_WAY_TEST_CASE_METHOD(ArrayIndexTest, "Unnest Array Literal Not Supported", "[C
     REQUIRE(!query);
     CHECK(err.code == kC4ErrorInvalidQuery);
 }
-
-#endif

C/tests/c4QueryTest.cc

@@ -776,8 +776,6 @@ N_WAY_TEST_CASE_METHOD(C4QueryTest, "C4Query Join", "[Query][C]") {
     c4queryenum_release(e);
 }
 
-// Disabled pending CBL-6400
-#if 0
 N_WAY_TEST_CASE_METHOD(C4QueryTest, "C4Query UNNEST", "[Query][C][Unnest]") {
     for ( int withIndex = 0; withIndex <= 1; ++withIndex ) {
         if ( withIndex ) {
@@ -1072,7 +1070,6 @@ N_WAY_TEST_CASE_METHOD(NestedQueryTest, "C4Query Nested UNNEST - Missing Array",
         CHECK(run2(nullptr, 2) == results);
     }
 }
-#endif
 
 N_WAY_TEST_CASE_METHOD(C4QueryTest, "C4Query Seek", "[Query][C]") {
     compile(json5("['=', ['.', 'contact', 'address', 'state'], 'CA']"));

LiteCore/Query/IndexSpec.cc

@@ -19,6 +19,7 @@
 #include "n1ql_parser.hh"
 #include "Query.hh"
 #include "MutableDict.hh"
+#include "StringUtil.hh"
 
 namespace litecore {
     using namespace fleece;
@@ -31,7 +32,8 @@ namespace litecore {
         , queryLanguage(queryLanguage_)
         , options(std::move(opt)) {
         auto whichOpts = options.index();
-        if ( (type == kFullText && whichOpts != 1 && whichOpts != 0) || (type == kVector && whichOpts != 2) )
+        if ( (type == kFullText && whichOpts != 1 && whichOpts != 0) || (type == kVector && whichOpts != 2)
+             || (type == kArray && whichOpts != 3) )
             error::_throw(error::LiteCoreError::InvalidParameter, "Invalid options type for index");
     }
 
@@ -42,7 +44,10 @@ namespace litecore {
         spec._doc = nullptr;
     }
 
-    IndexSpec::~IndexSpec() { FLDoc_Release(_doc); }
+    IndexSpec::~IndexSpec() {
+        FLDoc_Release(_doc);
+        FLDoc_Release(_unnestDoc);
+    }
 
     void IndexSpec::validateName() const {
         if ( name.empty() ) { error::_throw(error::LiteCoreError::InvalidParameter, "Index name must not be empty"); }
@@ -57,17 +62,24 @@ namespace litecore {
             switch ( queryLanguage ) {
                 case QueryLanguage::kJSON:
                     {
-                        _doc = Doc::fromJSON(expression).detach();
-                        if ( !_doc ) error::_throw(error::InvalidQuery, "Invalid JSON in index expression");
+                        if ( auto doc = Doc::fromJSON(expression); doc ) _doc = doc.detach();
+                        else
+                            error::_throw(error::InvalidQuery, "Invalid JSON in index expression");
                         break;
                     }
                 case QueryLanguage::kN1QL:
                     try {
-                        int           errPos;
-                        FLMutableDict result = n1ql::parse(string(expression), &errPos);
-                        if ( !result ) { throw Query::parseError("N1QL syntax error in index expression", errPos); }
-                        alloc_slice json(FLValue_ToJSON(FLValue(result)));
-                        FLMutableDict_Release(result);
+                        alloc_slice json;
+                        if ( !expression.empty() ) {
+                            int           errPos;
+                            FLMutableDict result = n1ql::parse(string(expression), &errPos);
+                            if ( !result ) { throw Query::parseError("N1QL syntax error in index expression", errPos); }
+                            json = FLValue_ToJSON(FLValue(result));
+                            FLMutableDict_Release(result);
+                        } else {
+                            // n1ql parser won't compile empty string to empty array. Do it manually.
+                            json = "[]";
+                        }
                         _doc = Doc::fromJSON(json).detach();
                     } catch ( const std::runtime_error& ) {
                         error::_throw(error::InvalidQuery, "Invalid N1QL in index expression");
@@ -88,7 +100,8 @@ namespace litecore {
             // of expressions.
             what = qt::requiredArray(doc.root(), "Index JSON");
         }
-        if ( what.empty() ) error::_throw(error::InvalidQuery, "Index WHAT list cannot be empty");
+        // Array Inddex can have empty what.
+        if ( type != kArray && what.empty() ) error::_throw(error::InvalidQuery, "Index WHAT list cannot be empty");
         return what;
     }
 
@@ -101,5 +114,46 @@ namespace litecore {
         return nullptr;
     }
 
+    // Turning unnestPath in C4IndexOptions to an array in JSON expresion.
+    // Ex. students[].interests -> [[".students"],[".interests"]]
+    FLArray IndexSpec::unnestPaths() const {
+        const ArrayOptions* arrayOpts = arrayOptions();
+        if ( !arrayOpts || !arrayOpts->unnestPath )
+            error::_throw(error::InvalidParameter, "IndexOptions for ArrayIndex must include unnestPath.");
+
+        Doc doc(unnestDoc());
+        if ( auto dict = doc.asDict(); dict ) {
+            if ( auto whatVal = qt::getCaseInsensitive(dict, "WHAT"); whatVal )
+                return qt::requiredArray(whatVal, "Index WHAT term");
+        }
+        return nullptr;
+    }
+
+    FLDoc IndexSpec::unnestDoc() const {
+        // Precondition: arrayOptions() && arrayOptions()->unnestPath
+        if ( !_unnestDoc ) {
+            try {
+                string n1qlUnnestPaths{arrayOptions()->unnestPath};
+                if ( n1qlUnnestPaths.empty() )
+                    error::_throw(error::InvalidParameter,
+                                  "IndexOptions for ArrayIndex must have non-empty unnestPath.");
+
+                // Turning "students[].interests" to "students, interests"
+                litecore::replace(n1qlUnnestPaths, KeyStore::kUnnestLevelSeparator, ", ");
+                int           errPos;
+                FLMutableDict result = n1ql::parse(n1qlUnnestPaths, &errPos);
+                if ( !result ) {
+                    string msg = "N1QL syntax error in unnestPath \"" + n1qlUnnestPaths + "\"";
+                    throw Query::parseError(msg.c_str(), errPos);
+                }
 
+                alloc_slice json{FLValue_ToJSON(FLValue(result))};
+                FLMutableDict_Release(result);
+                _unnestDoc = Doc::fromJSON(json).detach();
+            } catch ( const std::runtime_error& exc ) {
+                error::_throw(error::InvalidQuery, "Invalid N1QL in unnestPath (%s)", exc.what());
+            }
+        }
+        return _unnestDoc;
+    }
 }  // namespace litecore

LiteCore/Query/IndexSpec.hh

@@ -91,6 +91,9 @@ namespace litecore {
         /** The optional WHERE clause: the condition for a partial index */
         FLArray where() const;
 
+        /** The nested unnestPath from arrayOptions, as separated by "[]." is turned to an array. */
+        FLArray unnestPaths() const;
+
         std::string const name;           ///< Name of index
         Type const        type;           ///< Type of index
         alloc_slice const expression;     ///< The query expression
@@ -99,8 +102,10 @@ namespace litecore {
 
       private:
         FLDoc doc() const;
+        FLDoc unnestDoc() const;
 
-        mutable FLDoc _doc = nullptr;
+        mutable FLDoc _doc       = nullptr;
+        mutable FLDoc _unnestDoc = nullptr;
     };
 
 }  // namespace litecore

LiteCore/Query/SQLiteKeyStore+ArrayIndexes.cc

@@ -14,74 +14,116 @@
 #include "SQLiteDataFile.hh"
 #include "QueryTranslator.hh"
 #include "SQLUtil.hh"
+#include "SecureDigest.hh"
 #include "StringUtil.hh"
 #include "Array.hh"
 
 using namespace std;
-using namespace fleece;
 using namespace fleece::impl;
 
 namespace litecore {
 
     bool SQLiteKeyStore::createArrayIndex(const IndexSpec& spec) {
+        auto currSpec = db().getIndex(spec.name);
+        if ( currSpec ) {
+            // If there is already index with the index name,
+            // eiher delete the current one, or use it (return false)
+            bool same = true;
+            if ( currSpec->type != IndexSpec::kArray || !currSpec->arrayOptions()
+                 || currSpec->arrayOptions()->unnestPath != spec.arrayOptions()->unnestPath || !currSpec->what()
+                 || !spec.what() )
+                same = false;
+            else {
+                alloc_slice currWhat = FLValue_ToJSON(FLValue(currSpec->what()));
+                alloc_slice specWhat = FLValue_ToJSON(FLValue(spec.what()));
+                if ( currWhat != specWhat ) same = false;
+            }
+
+            if ( same ) return false;
+            else
+                db().deleteIndex(*currSpec);
+        }
+
+        string plainTableName, unnestTableName;
+        // the following will throw if !spec.arrayOptions() || !spec.arrayOptions()->unnestPath
+        for ( Array::iterator itPath((const Array*)spec.unnestPaths()); itPath; ++itPath ) {
+            std::tie(plainTableName, unnestTableName) =
+                    createUnnestedTable(itPath.value(), plainTableName, unnestTableName);
+        }
         Array::iterator iExprs((const Array*)spec.what());
-        string          arrayTableName = createUnnestedTable(iExprs.value());
-        return createIndex(spec, arrayTableName, ++iExprs);
+        return createIndex(spec, plainTableName, iExprs);
     }
 
-    string SQLiteKeyStore::createUnnestedTable(const Value* expression) {
+    std::pair<string, string> SQLiteKeyStore::createUnnestedTable(const Value* expression, string plainParentTable,
+                                                                  string parentTable) {
         // Derive the table name from the expression it unnests:
-        string          kvTableName = tableName();
-        QueryTranslator qp(db(), "", kvTableName);
-        string          unnestTableName = qp.unnestedTableName(FLValue(expression));
+        if ( plainParentTable.empty() ) plainParentTable = parentTable = tableName();
+        QueryTranslator qp(db(), "", plainParentTable);
+        string          plainTableName    = qp.unnestedTableName(FLValue(expression));
+        string          unnestTableName   = hexName(plainTableName);
+        string          quotedParentTable = CONCAT(sqlIdentifier(parentTable));
 
         // Create the index table, unless an identical one already exists:
         string sql = CONCAT("CREATE TABLE " << sqlIdentifier(unnestTableName)
                                             << " "
                                                "(docid INTEGER NOT NULL REFERENCES "
-                                            << sqlIdentifier(kvTableName)
+                                            << sqlIdentifier(parentTable)
                                             << "(rowid), "
                                                " i INTEGER NOT NULL,"
                                                " body BLOB NOT NULL, "
-                                               " CONSTRAINT pk PRIMARY KEY (docid, i)) "
-                                               "WITHOUT ROWID");
+                                               " CONSTRAINT pk PRIMARY KEY (docid, i))");
         if ( !db().schemaExistsWithSQL(unnestTableName, "table", unnestTableName, sql) ) {
             LogTo(QueryLog, "Creating UNNEST table '%s' on %s", unnestTableName.c_str(),
                   expression->toJSON(true).asString().c_str());
             db().exec(sql);
 
             qp.setBodyColumnName("new.body");
             string eachExpr = qp.eachExpressionSQL(FLValue(expression));
+            bool   nested   = plainParentTable.find(KeyStore::kUnnestSeparator) != string::npos;
 
             // Populate the index-table with data from existing documents:
-            db().exec(CONCAT("INSERT INTO " << sqlIdentifier(unnestTableName)
-                                            << " (docid, i, body) "
-                                               "SELECT new.rowid, _each.rowid, _each.value "
-                                            << "FROM " << sqlIdentifier(kvTableName) << " as new, " << eachExpr
-                                            << " AS _each "
-                                               "WHERE (new.flags & 1) = 0"));
+            if ( !nested ) {
+                db().exec(CONCAT("INSERT INTO " << sqlIdentifier(unnestTableName)
+                                                << " (docid, i, body) "
+                                                   "SELECT new.rowid, _each.rowid, _each.value "
+                                                << "FROM " << sqlIdentifier(parentTable) << " as new, " << eachExpr
+                                                << " AS _each "
+                                                   "WHERE (new.flags & 1) = 0"));
+            } else {
+                db().exec(CONCAT("INSERT INTO " << sqlIdentifier(unnestTableName)
+                                                << " (docid, i, body) "
+                                                   "SELECT new.rowid, _each.rowid, _each.value "
+                                                << "FROM " << sqlIdentifier(parentTable) << " as new, " << eachExpr
+                                                << " AS _each"));
+            }
 
             // Set up triggers to keep the index-table up to date
             // ...on insertion:
             string insertTriggerExpr = CONCAT("INSERT INTO " << sqlIdentifier(unnestTableName)
                                                              << " (docid, i, body) "
                                                                 "SELECT new.rowid, _each.rowid, _each.value "
                                                              << "FROM " << eachExpr << " AS _each ");
-            createTrigger(unnestTableName, "ins", "AFTER INSERT", "WHEN (new.flags & 1) = 0", insertTriggerExpr);
-
             // ...on delete:
             string deleteTriggerExpr = CONCAT("DELETE FROM " << sqlIdentifier(unnestTableName)
                                                              << " "
                                                                 "WHERE docid = old.rowid");
-            createTrigger(unnestTableName, "del", "BEFORE DELETE", "WHEN (old.flags & 1) = 0", deleteTriggerExpr);
+            if ( !nested ) {
+                createTrigger(unnestTableName, "ins", "AFTER INSERT", "WHEN (new.flags & 1) = 0", insertTriggerExpr,
+                              quotedParentTable);
+                createTrigger(unnestTableName, "del", "BEFORE DELETE", "WHEN (old.flags & 1) = 0", deleteTriggerExpr,
+                              quotedParentTable);
 
-            // ...on update:
-            createTrigger(unnestTableName, "preupdate", "BEFORE UPDATE OF body, flags", "WHEN (old.flags & 1) = 0",
-                          deleteTriggerExpr);
-            createTrigger(unnestTableName, "postupdate", "AFTER UPDATE OF body, flags", "WHEN (new.flags & 1 = 0)",
-                          insertTriggerExpr);
+                // ...on update:
+                createTrigger(unnestTableName, "preupdate", "BEFORE UPDATE OF body, flags", "WHEN (old.flags & 1) = 0",
+                              deleteTriggerExpr, quotedParentTable);
+                createTrigger(unnestTableName, "postupdate", "AFTER UPDATE OF body, flags", "WHEN (new.flags & 1 = 0)",
+                              insertTriggerExpr, quotedParentTable);
+            } else {
+                createTrigger(unnestTableName, "ins", "AFTER INSERT", "", insertTriggerExpr, quotedParentTable);
+                createTrigger(unnestTableName, "del", "BEFORE DELETE", "", deleteTriggerExpr, quotedParentTable);
+            }
         }
-        return unnestTableName;
+        return {plainTableName, unnestTableName};
     }
 
 }  // namespace litecore

LiteCore/Query/SQLiteKeyStore+Indexes.cc

@@ -89,8 +89,9 @@ namespace litecore {
     bool SQLiteKeyStore::createIndex(const IndexSpec& spec, const string& sourceTableName,
                                      Array::iterator& expressions) {
         Assert(spec.type != IndexSpec::kFullText && spec.type != IndexSpec::kVector);
-        QueryTranslator qp(db(), "", sourceTableName);
-        qp.writeCreateIndex(spec.name, sourceTableName, (FLArrayIterator&)expressions, spec.where(),
+        string          name{spec.type == IndexSpec::kArray ? litecore::hexName(sourceTableName) : sourceTableName};
+        QueryTranslator qp(db(), "", name);
+        qp.writeCreateIndex(spec.name, name, (FLArrayIterator&)expressions, spec.where(),
                             (spec.type != IndexSpec::kValue));
         string sql = qp.SQL();
         return db().createIndex(spec, this, sourceTableName, sql);

LiteCore/Query/Translator/ExprNodes.cc

@@ -326,12 +326,18 @@ namespace litecore::qt {
 
         string_view lastComponent;
         if ( path.count() > 0 ) lastComponent = ctx.newString(path.get(path.count() - 1).first);
-        return new (ctx) PropertyNode(source, result, ctx.newString(string(path)), lastComponent, sqliteFn);
+        return new (ctx) PropertyNode(source, result, ctx.newString(string(path)), lastComponent, sqliteFn,
+                                      ctx.select ? ctx.select->hasGroupBy() : false);
     }
 
     PropertyNode::PropertyNode(SourceNode* C4NULLABLE src, WhatNode* C4NULLABLE result, string_view path,
-                               string_view lastComponent, string_view fn)
-        : _source(src), _result(result), _path(path), _lastComponent(lastComponent), _sqliteFn(fn) {}
+                               string_view lastComponent, string_view fn, bool hasGroupBy)
+        : _source(src)
+        , _result(result)
+        , _path(path)
+        , _lastComponent(lastComponent)
+        , _sqliteFn(fn)
+        , _hasGroupBy(hasGroupBy) {}
 
     string_view PropertyNode::asColumnName() const {
         if ( !_path.empty() ) return _lastComponent;