Skip to content

Commit c3ecb46

Browse files
authored
CBL-6400: Port the changes to QueryParser in 3.2 branch (to support U… (#2186)
* CBL-6400: Port the changes to QueryParser in 3.2 branch (to support UNNEST) to 4.0 (master branch)
1 parent 842c8c9 commit c3ecb46

20 files changed

+281
-117
lines changed

C/tests/c4ArrayIndexTest.cc

+13-11
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@
1111
#include "c4Query.h"
1212
#include "c4Query.hh"
1313

14-
// Disabled pending CBL-6400
15-
#if 0
1614
class ArrayIndexTest : public C4Test {
1715
public:
1816
explicit ArrayIndexTest(int opt) : C4Test(opt) {}
@@ -132,20 +130,26 @@ constexpr std::string_view p0004 =
132130
N_WAY_TEST_CASE_METHOD(ArrayIndexTest, "Create Array Index with Empty Path", "[C][ArrayIndex]") {
133131
const auto defaultColl = REQUIRED(c4db_getDefaultCollection(db, ERROR_INFO()));
134132
C4Error err{};
135-
createArrayIndex(defaultColl, "arridx"_sl, nullslice, "", &err);
133+
{
134+
ExpectingExceptions x;
135+
createArrayIndex(defaultColl, "arridx"_sl, nullslice, "", &err);
136+
}
136137
CHECK(err.code == kC4ErrorInvalidQuery);
137138
}
138139

139140
// 2. TestCreateArrayIndexWithInvalidExpressions
140141
N_WAY_TEST_CASE_METHOD(ArrayIndexTest, "Create Array Index with Invalid Expressions", "[C][ArrayIndex]") {
141142
const auto defaultColl = REQUIRED(c4db_getDefaultCollection(db, ERROR_INFO()));
142143
C4Error err{};
144+
{
145+
ExpectingExceptions x;
143146

144-
createArrayIndex(defaultColl, "arridx"_sl, R"([".address.state", "", ".address.city"])", "contacts", &err);
145-
CHECK(err.code == kC4ErrorInvalidQuery);
147+
createArrayIndex(defaultColl, "arridx"_sl, R"([".address.state", "", ".address.city"])", "contacts", &err);
148+
CHECK(err.code == kC4ErrorInvalidQuery);
146149

147-
createArrayIndex(defaultColl, "arridx"_sl, R"([".address.state", , ".address.city"])", "contacts", &err);
148-
CHECK(err.code == kC4ErrorInvalidQuery);
150+
createArrayIndex(defaultColl, "arridx"_sl, R"([".address.state", , ".address.city"])", "contacts", &err);
151+
CHECK(err.code == kC4ErrorInvalidQuery);
152+
}
149153
}
150154

151155
// 3. TestCreateUpdateDeleteArrayIndexSingleLevel
@@ -605,7 +609,7 @@ N_WAY_TEST_CASE_METHOD(ArrayIndexTest, "Unnest Nested Non-Scalar Array", "[C][Un
605609
// 5. TestUnnestSingleLevelArrayWithGroupBy
606610
// Disabled until group-by is fixed
607611
// See https://jira.issues.couchbase.com/browse/CBL-6327
608-
# if 0
612+
#if 0
609613
TEST_CASE_METHOD(ArrayIndexTest, "Unnest Single Level Array With Group By", "[C][Unnest]") {
610614
C4Collection* coll = createCollection(db, {"profiles"_sl, "_default"_sl});
611615
importTestData(coll);
@@ -616,7 +620,7 @@ TEST_CASE_METHOD(ArrayIndexTest, "Unnest Single Level Array With Group By", "[C]
616620
c4::ref queryenum = REQUIRED(c4query_run(query, nullslice, nullptr));
617621
validateQuery(queryenum, {});
618622
}
619-
# endif
623+
#endif
620624

621625
// 6. TestUnnestWithoutAlias
622626
N_WAY_TEST_CASE_METHOD(ArrayIndexTest, "Unnest Without Alias", "[C][Unnest]") {
@@ -671,5 +675,3 @@ N_WAY_TEST_CASE_METHOD(ArrayIndexTest, "Unnest Array Literal Not Supported", "[C
671675
REQUIRE(!query);
672676
CHECK(err.code == kC4ErrorInvalidQuery);
673677
}
674-
675-
#endif

C/tests/c4QueryTest.cc

-3
Original file line numberDiff line numberDiff line change
@@ -776,8 +776,6 @@ N_WAY_TEST_CASE_METHOD(C4QueryTest, "C4Query Join", "[Query][C]") {
776776
c4queryenum_release(e);
777777
}
778778

779-
// Disabled pending CBL-6400
780-
#if 0
781779
N_WAY_TEST_CASE_METHOD(C4QueryTest, "C4Query UNNEST", "[Query][C][Unnest]") {
782780
for ( int withIndex = 0; withIndex <= 1; ++withIndex ) {
783781
if ( withIndex ) {
@@ -1072,7 +1070,6 @@ N_WAY_TEST_CASE_METHOD(NestedQueryTest, "C4Query Nested UNNEST - Missing Array",
10721070
CHECK(run2(nullptr, 2) == results);
10731071
}
10741072
}
1075-
#endif
10761073

10771074
N_WAY_TEST_CASE_METHOD(C4QueryTest, "C4Query Seek", "[Query][C]") {
10781075
compile(json5("['=', ['.', 'contact', 'address', 'state'], 'CA']"));

LiteCore/Query/IndexSpec.cc

+64-10
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include "n1ql_parser.hh"
2020
#include "Query.hh"
2121
#include "MutableDict.hh"
22+
#include "StringUtil.hh"
2223

2324
namespace litecore {
2425
using namespace fleece;
@@ -31,7 +32,8 @@ namespace litecore {
3132
, queryLanguage(queryLanguage_)
3233
, options(std::move(opt)) {
3334
auto whichOpts = options.index();
34-
if ( (type == kFullText && whichOpts != 1 && whichOpts != 0) || (type == kVector && whichOpts != 2) )
35+
if ( (type == kFullText && whichOpts != 1 && whichOpts != 0) || (type == kVector && whichOpts != 2)
36+
|| (type == kArray && whichOpts != 3) )
3537
error::_throw(error::LiteCoreError::InvalidParameter, "Invalid options type for index");
3638
}
3739

@@ -42,7 +44,10 @@ namespace litecore {
4244
spec._doc = nullptr;
4345
}
4446

45-
IndexSpec::~IndexSpec() { FLDoc_Release(_doc); }
47+
IndexSpec::~IndexSpec() {
48+
FLDoc_Release(_doc);
49+
FLDoc_Release(_unnestDoc);
50+
}
4651

4752
void IndexSpec::validateName() const {
4853
if ( name.empty() ) { error::_throw(error::LiteCoreError::InvalidParameter, "Index name must not be empty"); }
@@ -57,17 +62,24 @@ namespace litecore {
5762
switch ( queryLanguage ) {
5863
case QueryLanguage::kJSON:
5964
{
60-
_doc = Doc::fromJSON(expression).detach();
61-
if ( !_doc ) error::_throw(error::InvalidQuery, "Invalid JSON in index expression");
65+
if ( auto doc = Doc::fromJSON(expression); doc ) _doc = doc.detach();
66+
else
67+
error::_throw(error::InvalidQuery, "Invalid JSON in index expression");
6268
break;
6369
}
6470
case QueryLanguage::kN1QL:
6571
try {
66-
int errPos;
67-
FLMutableDict result = n1ql::parse(string(expression), &errPos);
68-
if ( !result ) { throw Query::parseError("N1QL syntax error in index expression", errPos); }
69-
alloc_slice json(FLValue_ToJSON(FLValue(result)));
70-
FLMutableDict_Release(result);
72+
alloc_slice json;
73+
if ( !expression.empty() ) {
74+
int errPos;
75+
FLMutableDict result = n1ql::parse(string(expression), &errPos);
76+
if ( !result ) { throw Query::parseError("N1QL syntax error in index expression", errPos); }
77+
json = FLValue_ToJSON(FLValue(result));
78+
FLMutableDict_Release(result);
79+
} else {
80+
// n1ql parser won't compile empty string to empty array. Do it manually.
81+
json = "[]";
82+
}
7183
_doc = Doc::fromJSON(json).detach();
7284
} catch ( const std::runtime_error& ) {
7385
error::_throw(error::InvalidQuery, "Invalid N1QL in index expression");
@@ -88,7 +100,8 @@ namespace litecore {
88100
// of expressions.
89101
what = qt::requiredArray(doc.root(), "Index JSON");
90102
}
91-
if ( what.empty() ) error::_throw(error::InvalidQuery, "Index WHAT list cannot be empty");
103+
// Array Inddex can have empty what.
104+
if ( type != kArray && what.empty() ) error::_throw(error::InvalidQuery, "Index WHAT list cannot be empty");
92105
return what;
93106
}
94107

@@ -101,5 +114,46 @@ namespace litecore {
101114
return nullptr;
102115
}
103116

117+
// Turning unnestPath in C4IndexOptions to an array in JSON expresion.
118+
// Ex. students[].interests -> [[".students"],[".interests"]]
119+
FLArray IndexSpec::unnestPaths() const {
120+
const ArrayOptions* arrayOpts = arrayOptions();
121+
if ( !arrayOpts || !arrayOpts->unnestPath )
122+
error::_throw(error::InvalidParameter, "IndexOptions for ArrayIndex must include unnestPath.");
123+
124+
Doc doc(unnestDoc());
125+
if ( auto dict = doc.asDict(); dict ) {
126+
if ( auto whatVal = qt::getCaseInsensitive(dict, "WHAT"); whatVal )
127+
return qt::requiredArray(whatVal, "Index WHAT term");
128+
}
129+
return nullptr;
130+
}
131+
132+
FLDoc IndexSpec::unnestDoc() const {
133+
// Precondition: arrayOptions() && arrayOptions()->unnestPath
134+
if ( !_unnestDoc ) {
135+
try {
136+
string n1qlUnnestPaths{arrayOptions()->unnestPath};
137+
if ( n1qlUnnestPaths.empty() )
138+
error::_throw(error::InvalidParameter,
139+
"IndexOptions for ArrayIndex must have non-empty unnestPath.");
140+
141+
// Turning "students[].interests" to "students, interests"
142+
litecore::replace(n1qlUnnestPaths, KeyStore::kUnnestLevelSeparator, ", ");
143+
int errPos;
144+
FLMutableDict result = n1ql::parse(n1qlUnnestPaths, &errPos);
145+
if ( !result ) {
146+
string msg = "N1QL syntax error in unnestPath \"" + n1qlUnnestPaths + "\"";
147+
throw Query::parseError(msg.c_str(), errPos);
148+
}
104149

150+
alloc_slice json{FLValue_ToJSON(FLValue(result))};
151+
FLMutableDict_Release(result);
152+
_unnestDoc = Doc::fromJSON(json).detach();
153+
} catch ( const std::runtime_error& exc ) {
154+
error::_throw(error::InvalidQuery, "Invalid N1QL in unnestPath (%s)", exc.what());
155+
}
156+
}
157+
return _unnestDoc;
158+
}
105159
} // namespace litecore

LiteCore/Query/IndexSpec.hh

+6-1
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,9 @@ namespace litecore {
9191
/** The optional WHERE clause: the condition for a partial index */
9292
FLArray where() const;
9393

94+
/** The nested unnestPath from arrayOptions, as separated by "[]." is turned to an array. */
95+
FLArray unnestPaths() const;
96+
9497
std::string const name; ///< Name of index
9598
Type const type; ///< Type of index
9699
alloc_slice const expression; ///< The query expression
@@ -99,8 +102,10 @@ namespace litecore {
99102

100103
private:
101104
FLDoc doc() const;
105+
FLDoc unnestDoc() const;
102106

103-
mutable FLDoc _doc = nullptr;
107+
mutable FLDoc _doc = nullptr;
108+
mutable FLDoc _unnestDoc = nullptr;
104109
};
105110

106111
} // namespace litecore

LiteCore/Query/SQLiteKeyStore+ArrayIndexes.cc

+67-25
Original file line numberDiff line numberDiff line change
@@ -14,74 +14,116 @@
1414
#include "SQLiteDataFile.hh"
1515
#include "QueryTranslator.hh"
1616
#include "SQLUtil.hh"
17+
#include "SecureDigest.hh"
1718
#include "StringUtil.hh"
1819
#include "Array.hh"
1920

2021
using namespace std;
21-
using namespace fleece;
2222
using namespace fleece::impl;
2323

2424
namespace litecore {
2525

2626
bool SQLiteKeyStore::createArrayIndex(const IndexSpec& spec) {
27+
auto currSpec = db().getIndex(spec.name);
28+
if ( currSpec ) {
29+
// If there is already index with the index name,
30+
// eiher delete the current one, or use it (return false)
31+
bool same = true;
32+
if ( currSpec->type != IndexSpec::kArray || !currSpec->arrayOptions()
33+
|| currSpec->arrayOptions()->unnestPath != spec.arrayOptions()->unnestPath || !currSpec->what()
34+
|| !spec.what() )
35+
same = false;
36+
else {
37+
alloc_slice currWhat = FLValue_ToJSON(FLValue(currSpec->what()));
38+
alloc_slice specWhat = FLValue_ToJSON(FLValue(spec.what()));
39+
if ( currWhat != specWhat ) same = false;
40+
}
41+
42+
if ( same ) return false;
43+
else
44+
db().deleteIndex(*currSpec);
45+
}
46+
47+
string plainTableName, unnestTableName;
48+
// the following will throw if !spec.arrayOptions() || !spec.arrayOptions()->unnestPath
49+
for ( Array::iterator itPath((const Array*)spec.unnestPaths()); itPath; ++itPath ) {
50+
std::tie(plainTableName, unnestTableName) =
51+
createUnnestedTable(itPath.value(), plainTableName, unnestTableName);
52+
}
2753
Array::iterator iExprs((const Array*)spec.what());
28-
string arrayTableName = createUnnestedTable(iExprs.value());
29-
return createIndex(spec, arrayTableName, ++iExprs);
54+
return createIndex(spec, plainTableName, iExprs);
3055
}
3156

32-
string SQLiteKeyStore::createUnnestedTable(const Value* expression) {
57+
std::pair<string, string> SQLiteKeyStore::createUnnestedTable(const Value* expression, string plainParentTable,
58+
string parentTable) {
3359
// Derive the table name from the expression it unnests:
34-
string kvTableName = tableName();
35-
QueryTranslator qp(db(), "", kvTableName);
36-
string unnestTableName = qp.unnestedTableName(FLValue(expression));
60+
if ( plainParentTable.empty() ) plainParentTable = parentTable = tableName();
61+
QueryTranslator qp(db(), "", plainParentTable);
62+
string plainTableName = qp.unnestedTableName(FLValue(expression));
63+
string unnestTableName = hexName(plainTableName);
64+
string quotedParentTable = CONCAT(sqlIdentifier(parentTable));
3765

3866
// Create the index table, unless an identical one already exists:
3967
string sql = CONCAT("CREATE TABLE " << sqlIdentifier(unnestTableName)
4068
<< " "
4169
"(docid INTEGER NOT NULL REFERENCES "
42-
<< sqlIdentifier(kvTableName)
70+
<< sqlIdentifier(parentTable)
4371
<< "(rowid), "
4472
" i INTEGER NOT NULL,"
4573
" body BLOB NOT NULL, "
46-
" CONSTRAINT pk PRIMARY KEY (docid, i)) "
47-
"WITHOUT ROWID");
74+
" CONSTRAINT pk PRIMARY KEY (docid, i))");
4875
if ( !db().schemaExistsWithSQL(unnestTableName, "table", unnestTableName, sql) ) {
4976
LogTo(QueryLog, "Creating UNNEST table '%s' on %s", unnestTableName.c_str(),
5077
expression->toJSON(true).asString().c_str());
5178
db().exec(sql);
5279

5380
qp.setBodyColumnName("new.body");
5481
string eachExpr = qp.eachExpressionSQL(FLValue(expression));
82+
bool nested = plainParentTable.find(KeyStore::kUnnestSeparator) != string::npos;
5583

5684
// Populate the index-table with data from existing documents:
57-
db().exec(CONCAT("INSERT INTO " << sqlIdentifier(unnestTableName)
58-
<< " (docid, i, body) "
59-
"SELECT new.rowid, _each.rowid, _each.value "
60-
<< "FROM " << sqlIdentifier(kvTableName) << " as new, " << eachExpr
61-
<< " AS _each "
62-
"WHERE (new.flags & 1) = 0"));
85+
if ( !nested ) {
86+
db().exec(CONCAT("INSERT INTO " << sqlIdentifier(unnestTableName)
87+
<< " (docid, i, body) "
88+
"SELECT new.rowid, _each.rowid, _each.value "
89+
<< "FROM " << sqlIdentifier(parentTable) << " as new, " << eachExpr
90+
<< " AS _each "
91+
"WHERE (new.flags & 1) = 0"));
92+
} else {
93+
db().exec(CONCAT("INSERT INTO " << sqlIdentifier(unnestTableName)
94+
<< " (docid, i, body) "
95+
"SELECT new.rowid, _each.rowid, _each.value "
96+
<< "FROM " << sqlIdentifier(parentTable) << " as new, " << eachExpr
97+
<< " AS _each"));
98+
}
6399

64100
// Set up triggers to keep the index-table up to date
65101
// ...on insertion:
66102
string insertTriggerExpr = CONCAT("INSERT INTO " << sqlIdentifier(unnestTableName)
67103
<< " (docid, i, body) "
68104
"SELECT new.rowid, _each.rowid, _each.value "
69105
<< "FROM " << eachExpr << " AS _each ");
70-
createTrigger(unnestTableName, "ins", "AFTER INSERT", "WHEN (new.flags & 1) = 0", insertTriggerExpr);
71-
72106
// ...on delete:
73107
string deleteTriggerExpr = CONCAT("DELETE FROM " << sqlIdentifier(unnestTableName)
74108
<< " "
75109
"WHERE docid = old.rowid");
76-
createTrigger(unnestTableName, "del", "BEFORE DELETE", "WHEN (old.flags & 1) = 0", deleteTriggerExpr);
110+
if ( !nested ) {
111+
createTrigger(unnestTableName, "ins", "AFTER INSERT", "WHEN (new.flags & 1) = 0", insertTriggerExpr,
112+
quotedParentTable);
113+
createTrigger(unnestTableName, "del", "BEFORE DELETE", "WHEN (old.flags & 1) = 0", deleteTriggerExpr,
114+
quotedParentTable);
77115

78-
// ...on update:
79-
createTrigger(unnestTableName, "preupdate", "BEFORE UPDATE OF body, flags", "WHEN (old.flags & 1) = 0",
80-
deleteTriggerExpr);
81-
createTrigger(unnestTableName, "postupdate", "AFTER UPDATE OF body, flags", "WHEN (new.flags & 1 = 0)",
82-
insertTriggerExpr);
116+
// ...on update:
117+
createTrigger(unnestTableName, "preupdate", "BEFORE UPDATE OF body, flags", "WHEN (old.flags & 1) = 0",
118+
deleteTriggerExpr, quotedParentTable);
119+
createTrigger(unnestTableName, "postupdate", "AFTER UPDATE OF body, flags", "WHEN (new.flags & 1 = 0)",
120+
insertTriggerExpr, quotedParentTable);
121+
} else {
122+
createTrigger(unnestTableName, "ins", "AFTER INSERT", "", insertTriggerExpr, quotedParentTable);
123+
createTrigger(unnestTableName, "del", "BEFORE DELETE", "", deleteTriggerExpr, quotedParentTable);
124+
}
83125
}
84-
return unnestTableName;
126+
return {plainTableName, unnestTableName};
85127
}
86128

87129
} // namespace litecore

LiteCore/Query/SQLiteKeyStore+Indexes.cc

+3-2
Original file line numberDiff line numberDiff line change
@@ -89,8 +89,9 @@ namespace litecore {
8989
bool SQLiteKeyStore::createIndex(const IndexSpec& spec, const string& sourceTableName,
9090
Array::iterator& expressions) {
9191
Assert(spec.type != IndexSpec::kFullText && spec.type != IndexSpec::kVector);
92-
QueryTranslator qp(db(), "", sourceTableName);
93-
qp.writeCreateIndex(spec.name, sourceTableName, (FLArrayIterator&)expressions, spec.where(),
92+
string name{spec.type == IndexSpec::kArray ? litecore::hexName(sourceTableName) : sourceTableName};
93+
QueryTranslator qp(db(), "", name);
94+
qp.writeCreateIndex(spec.name, name, (FLArrayIterator&)expressions, spec.where(),
9495
(spec.type != IndexSpec::kValue));
9596
string sql = qp.SQL();
9697
return db().createIndex(spec, this, sourceTableName, sql);

LiteCore/Query/Translator/ExprNodes.cc

+9-3
Original file line numberDiff line numberDiff line change
@@ -326,12 +326,18 @@ namespace litecore::qt {
326326

327327
string_view lastComponent;
328328
if ( path.count() > 0 ) lastComponent = ctx.newString(path.get(path.count() - 1).first);
329-
return new (ctx) PropertyNode(source, result, ctx.newString(string(path)), lastComponent, sqliteFn);
329+
return new (ctx) PropertyNode(source, result, ctx.newString(string(path)), lastComponent, sqliteFn,
330+
ctx.select ? ctx.select->hasGroupBy() : false);
330331
}
331332

332333
PropertyNode::PropertyNode(SourceNode* C4NULLABLE src, WhatNode* C4NULLABLE result, string_view path,
333-
string_view lastComponent, string_view fn)
334-
: _source(src), _result(result), _path(path), _lastComponent(lastComponent), _sqliteFn(fn) {}
334+
string_view lastComponent, string_view fn, bool hasGroupBy)
335+
: _source(src)
336+
, _result(result)
337+
, _path(path)
338+
, _lastComponent(lastComponent)
339+
, _sqliteFn(fn)
340+
, _hasGroupBy(hasGroupBy) {}
335341

336342
string_view PropertyNode::asColumnName() const {
337343
if ( !_path.empty() ) return _lastComponent;

0 commit comments

Comments
 (0)