Skip to content

Commit 54ee740

Browse files
waliscwalisc
committed
Merge branch 'dev' of github.com:jembi/JeMPI into dev_delete_indexs
2 parents 17d2f82 + 4ebf98a commit 54ee740

27 files changed

+1247
-61
lines changed

‎JeMPI_Apps/JeMPI_Configuration/src/main/scala/configuration/Ast.scala

+2
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,6 @@ object Ast {
1616

1717
case class Eq(name: Variable) extends Expression
1818

19+
case class Null(name: Variable) extends Expression
20+
1921
}

‎JeMPI_Apps/JeMPI_Configuration/src/main/scala/configuration/CustomDgraphQueries.scala

+91-30
Original file line numberDiff line numberDiff line change
@@ -175,14 +175,14 @@ object CustomDgraphQueries {
175175
expr match {
176176
case Ast.Or(x) =>
177177
"("
178-
+ (for (k <- x.zipWithIndex)
178+
+ (for (k <- x.filter(v => !v.isInstanceOf[Ast.Null]).zipWithIndex)
179179
yield
180180
if (k._2 == 0) expression(k._1)
181181
else " && " + expression(k._1)).mkString
182182
+ ")"
183183
case Ast.And(x) =>
184184
"("
185-
+ (for (k <- x.zipWithIndex)
185+
+ (for (k <- x.filter(v => !v.isInstanceOf[Ast.Null]).zipWithIndex)
186186
yield
187187
if (k._2 == 0) expression(k._1)
188188
else " || " + expression(k._1)).mkString
@@ -193,6 +193,8 @@ object CustomDgraphQueries {
193193
Utils.snakeCaseToCamelCase(variable.name) + "IsBlank"
194194
case Ast.Eq(variable) =>
195195
Utils.snakeCaseToCamelCase(variable.name) + "IsBlank"
196+
case Ast.Null (variable) =>
197+
"false"
196198
case _ =>
197199
"ERROR"
198200
}
@@ -256,13 +258,58 @@ object CustomDgraphQueries {
256258

257259
def emitRuleTemplate(name: String, rule: Rule): Unit = {
258260

261+
case class VarMeta(astName: String,
262+
func:String,
263+
funcParam: Option[Function1[String, String]],
264+
funcName: String,
265+
distance:Option[Integer])
266+
259267
val vars = for (v <- rule.vars) yield v
268+
var varsMeta = Map[String, List[VarMeta]]()
269+
270+
vars.foreach(v => {
271+
if (!varsMeta.contains(v)){
272+
varsMeta += (v -> List[VarMeta]())
273+
}
274+
})
275+
276+
277+
var currentFuncIndex:Int = -1
260278
val text = rule.text
261279
val expression: Ast.Expression = ParseRule.parse(text)
262-
val varsMap = vars.zipWithIndex
263-
.toMap[String, Int]
264-
.map((k, i) => (k, ("A".head + i).toChar.toString))
265-
var meta = Map[String, (String, Option[Integer])]()
280+
281+
282+
def addMeta(v: String,
283+
astName: String,
284+
func:String,
285+
funcParam: Option[Function1[String, String]],
286+
distance:Option[Integer]): String = {
287+
288+
if (!varsMeta.contains(v)){
289+
varsMeta += (v -> List[VarMeta]())
290+
}
291+
292+
val existingMeta = varsMeta(v).find(m => {
293+
m.astName == astName &&
294+
m.func == func &&
295+
m.funcParam.toString == funcParam.toString &&
296+
m.distance.toString == distance.toString
297+
})
298+
299+
if (existingMeta.isDefined){
300+
return existingMeta.get.funcName
301+
}
302+
else{
303+
val newMeta = VarMeta(astName, func, funcParam, getFuncIndex(), distance)
304+
varsMeta = varsMeta + (v -> (varsMeta(v) :+ newMeta ))
305+
return newMeta.funcName
306+
}
307+
}
308+
309+
def getFuncIndex(): String ={
310+
currentFuncIndex = currentFuncIndex + 1
311+
return ("A".head + currentFuncIndex).toChar.toString
312+
}
266313

267314
def main_func(expression: Ast.Expression): String = {
268315
expression match {
@@ -283,25 +330,41 @@ object CustomDgraphQueries {
283330
case Ast.Not(x) =>
284331
"NOT (" + main_func(x) + ")"
285332
case Ast.Match(variable, distance) =>
286-
meta += (variable.name -> ("match", Option(distance)))
287-
"uid(" + variable.name + ")"
333+
val uidName = addMeta(variable.name, "match", "match", None, Option(distance))
334+
"uid(" + uidName + ")"
288335
case Ast.Eq(variable) =>
289-
meta += (variable.name -> ("eq", None))
290-
"uid(" + variable.name + ")"
336+
val uidName = addMeta(variable.name, "eq", "eq", None, None)
337+
"uid(" + uidName + ")"
338+
case Ast.Null(variable) =>
339+
val uidName = addMeta(variable.name, "null", "eq", Option(new Function1[String, String] {
340+
def apply(x: String): String = "\"\""
341+
}), None)
342+
"uid(" + uidName + ")"
291343
case _ =>
292344
"ERROR"
293345
}
294346
}
295347

296-
def createScalerFunc(): Unit = {
297-
vars.foreach(v => {
298-
val fn = meta(v)._1
299-
writer.println(
300-
s"""${" " * 12}all(func:type(GoldenRecord)) @filter($fn(GoldenRecord.$v, $$$v${
301-
if (meta(v)._2.isDefined) ", " + meta(v)._2.get
348+
def getFilterParams(v: String, metaInfo: VarMeta):String = {
349+
if (metaInfo.funcParam.isDefined) {
350+
return metaInfo.funcParam.get.apply(v)
351+
}
352+
353+
return s"""
354+
$$$v${
355+
if (metaInfo.distance.isDefined) ", " + metaInfo.distance.get
302356
else
303357
""
304-
})) {
358+
}
359+
"""
360+
}
361+
362+
def createScalerFunc(): Unit = {
363+
varsMeta.foreach((v, mL) => {
364+
val m = mL(0)
365+
val fn = m.func
366+
writer.println(
367+
s"""${" " * 12}all(func:type(GoldenRecord)) @filter($fn(GoldenRecord.$v, ${getFilterParams(v, m)})) {
305368
|${" " * 15}uid
306369
|${" " * 15}GoldenRecord.source_id {
307370
|${" " * 18}uid
@@ -320,16 +383,17 @@ object CustomDgraphQueries {
320383
}
321384

322385
def createFilterFunc(all_func_str: String): Unit = {
323-
vars.foreach(v => {
324-
val fn = meta(v)._1
325-
writer.println(
326-
s"""${" " * 12}var(func:type(GoldenRecord)) @filter($fn(GoldenRecord.$v, $$$v${
327-
if (meta(v)._2.isDefined) ", " + meta(v)._2.get else ""
328-
})) {
329-
|${" " * 15}${varsMap(v)} as uid
330-
|${" " * 12}}""".stripMargin
331-
)
386+
varsMeta.foreach((v, mL) => {
387+
mL.foreach(m => {
388+
val fn = m.func
389+
writer.println(
390+
s"""${" " * 12}var(func:type(GoldenRecord)) @filter($fn(GoldenRecord.$v, ${getFilterParams(v, m)})) {
391+
|${" " * 15}${m.funcName} as uid
392+
|${" " * 12}}""".stripMargin
393+
)
394+
})
332395
})
396+
333397
writer.println(s"""${" " * 12}all(func:type(GoldenRecord)) @filter${
334398
if (all_func_str.startsWith("(")) "" else "("
335399
}$all_func_str${
@@ -351,9 +415,6 @@ object CustomDgraphQueries {
351415
}
352416

353417
var all_func_str = main_func(expression)
354-
varsMap.foreach((k, v) => {
355-
all_func_str = all_func_str.replace("uid(" + k + ")", "uid(" + v + ")")
356-
})
357418

358419
writer.print(
359420
s"""${" " * 3}private static final String ${name.toUpperCase} =
@@ -369,7 +430,7 @@ object CustomDgraphQueries {
369430
})
370431
writer.println(") {")
371432

372-
if (vars.length == 1)
433+
if (varsMeta.size == 1 && varsMeta.values.headOption.get.size < 2)
373434
createScalerFunc()
374435
else
375436
createFilterFunc(all_func_str)

‎JeMPI_Apps/JeMPI_Configuration/src/main/scala/configuration/CustomLinkerDeterministic.scala

+12-4
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,10 @@ object CustomLinkerDeterministic {
7474
val left = field + "L"
7575
val right = field + "R"
7676
s"isMatch($left, $right)"
77+
case Ast.Null(variable) =>
78+
val field = Utils.snakeCaseToCamelCase(variable.name)
79+
val left = field + "L"
80+
s"StringUtils.isBlank($left)"
7781
case _ =>
7882
"ERROR"
7983
}
@@ -89,15 +93,19 @@ object CustomLinkerDeterministic {
8993
|""".stripMargin)
9094
} else {
9195
val z = map.zipWithIndex
96+
var definedProperties: List[String] = List()
9297
z.foreach((map, index) => {
9398
val expression: Ast.Expression = ParseRule.parse(map._2.text)
9499
val expr_1 = checkNullExpression(expression)
95100
map._2.vars.foreach(v => {
96101
val field = Utils.snakeCaseToCamelCase(v)
97-
val left = field + "L"
98-
val right = field + "R"
99-
writer.println(" " * 6 + s"final var $left = goldenRecord.$field;")
100-
writer.println(" " * 6 + s"final var $right = interaction.$field;")
102+
if (!definedProperties.contains(field)){
103+
val left = field + "L"
104+
val right = field + "R"
105+
writer.println(" " * 6 + s"final var $left = goldenRecord.$field;")
106+
writer.println(" " * 6 + s"final var $right = interaction.$field;")
107+
definedProperties = definedProperties :+ field
108+
}
101109
})
102110
if (index < z.size - 1) {
103111
writer.println(s""" if ($expr_1) {

‎JeMPI_Apps/JeMPI_Configuration/src/main/scala/configuration/ParseRule.scala

+4-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ object ParseRule extends JavaTokenParsers {
2424
* @return
2525
*/
2626
private def leftExpression: Parser[Expression] =
27-
comment.? ~> not | brackets | matchField | eqField <~ comment.?
27+
comment.? ~> not | brackets | matchField | eqField | nullField <~ comment.?
2828

2929
private def brackets: Parser[Expression] =
3030
"(" ~> expression <~ ")"
@@ -54,6 +54,9 @@ object ParseRule extends JavaTokenParsers {
5454
case (left: Expression) ~ (right: Seq[Expression]) => And(left +: right)
5555
}
5656

57+
private def nullField: Parser[Null] =
58+
"null" ~ "(" ~>! variable <~! ")" ^^ (parameter => Null.apply(parameter))
59+
5760
private def or: Parser[Or] =
5861
(and | leftExpression) ~ ("or" ~> (and | leftExpression)).+ ^^ {
5962
case left ~ right => Or(left +: right)

‎JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/Routes.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ public static Route patchIidNewGidLink(
130130
return entity(Jackson.unmarshaller(NotificationResolution.class),
131131
obj -> onComplete(Ask.patchIidNewGidLink(actorSystem,
132132
backEnd,
133-
obj.newGoldenId(),
133+
obj.currentGoldenId(),
134134
obj.interactionId()),
135135
result -> result.isSuccess()
136136
? result.get()

‎JeMPI_Apps/JeMPI_LibMPI/src/main/java/org/jembi/jempi/libmpi/dgraph/CustomDgraphQueries.java

+31-13
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@ final class CustomDgraphQueries {
2222
private static final String QUERY_LINK_DETERMINISTIC_A =
2323
"""
2424
query query_link_deterministic_a($national_id: string) {
25-
all(func:type(GoldenRecord)) @filter(eq(GoldenRecord.national_id, $national_id)) {
25+
all(func:type(GoldenRecord)) @filter(eq(GoldenRecord.national_id,
26+
$national_id
27+
)) {
2628
uid
2729
GoldenRecord.source_id {
2830
uid
@@ -44,13 +46,19 @@ query query_link_deterministic_a($national_id: string) {
4446
private static final String QUERY_LINK_DETERMINISTIC_B =
4547
"""
4648
query query_link_deterministic_b($given_name: string, $family_name: string, $phone_number: string) {
47-
var(func:type(GoldenRecord)) @filter(eq(GoldenRecord.given_name, $given_name)) {
49+
var(func:type(GoldenRecord)) @filter(eq(GoldenRecord.given_name,
50+
$given_name
51+
)) {
4852
A as uid
4953
}
50-
var(func:type(GoldenRecord)) @filter(eq(GoldenRecord.family_name, $family_name)) {
54+
var(func:type(GoldenRecord)) @filter(eq(GoldenRecord.family_name,
55+
$family_name
56+
)) {
5157
B as uid
5258
}
53-
var(func:type(GoldenRecord)) @filter(eq(GoldenRecord.phone_number, $phone_number)) {
59+
var(func:type(GoldenRecord)) @filter(eq(GoldenRecord.phone_number,
60+
$phone_number
61+
)) {
5462
C as uid
5563
}
5664
all(func:type(GoldenRecord)) @filter(uid(A) AND uid(B) AND uid(C)) {
@@ -75,21 +83,31 @@ query query_link_deterministic_b($given_name: string, $family_name: string, $pho
7583
private static final String QUERY_LINK_PROBABILISTIC =
7684
"""
7785
query query_link_probabilistic($given_name: string, $family_name: string, $city: string, $phone_number: string, $national_id: string) {
78-
var(func:type(GoldenRecord)) @filter(match(GoldenRecord.given_name, $given_name, 3)) {
79-
A as uid
80-
}
81-
var(func:type(GoldenRecord)) @filter(match(GoldenRecord.family_name, $family_name, 3)) {
82-
B as uid
83-
}
84-
var(func:type(GoldenRecord)) @filter(match(GoldenRecord.city, $city, 3)) {
86+
var(func:type(GoldenRecord)) @filter(match(GoldenRecord.city,
87+
$city, 3
88+
)) {
8589
C as uid
8690
}
87-
var(func:type(GoldenRecord)) @filter(match(GoldenRecord.phone_number, $phone_number, 2)) {
91+
var(func:type(GoldenRecord)) @filter(match(GoldenRecord.phone_number,
92+
$phone_number, 2
93+
)) {
8894
D as uid
8995
}
90-
var(func:type(GoldenRecord)) @filter(match(GoldenRecord.national_id, $national_id, 3)) {
96+
var(func:type(GoldenRecord)) @filter(match(GoldenRecord.family_name,
97+
$family_name, 3
98+
)) {
99+
B as uid
100+
}
101+
var(func:type(GoldenRecord)) @filter(match(GoldenRecord.national_id,
102+
$national_id, 3
103+
)) {
91104
E as uid
92105
}
106+
var(func:type(GoldenRecord)) @filter(match(GoldenRecord.given_name,
107+
$given_name, 3
108+
)) {
109+
A as uid
110+
}
93111
all(func:type(GoldenRecord)) @filter(((uid(A) AND uid(B)) OR (uid(A) AND uid(C)) OR (uid(B) AND uid(C))) OR uid(D) OR uid(E)) {
94112
uid
95113
GoldenRecord.source_id {

‎JeMPI_Apps/JeMPI_UI/src/components/reviewLink/ReviewLink.tsx

+4
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ const ReviewLink = () => {
7575
} = useLinkReview(payload, refineSearchQuery, candidateThreshold)
7676
const { linkRecords, createNewGoldenRecord } = useRelink()
7777

78+
const shouldDisableCreateGoldenRecordButton = (): boolean =>
79+
!(goldenRecord && goldenRecord?.linkRecords.length > 1);
80+
7881
const mutateNotification = useMutation({
7982
mutationFn: (request: NotificationRequest) =>
8083
apiClient.updateNotification(request),
@@ -320,6 +323,7 @@ const ReviewLink = () => {
320323
<Button
321324
variant="outlined"
322325
onClick={() => setIsNewGoldenRecordDialogOpen(true)}
326+
disabled={shouldDisableCreateGoldenRecordButton()}
323327
>
324328
Create new golden record
325329
</Button>

‎devops/windows/base-docker-linux/z-stack-2-reboot-hub-images-db-init.sh

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash
2+
3+
set -e
4+
set -u
5+
6+
echo
7+
echo "Down stack"
8+
source ./helper/scripts/d-stack-08-rm.sh
9+
source ./helper/scripts/d-stack-09-wait-removed.sh
10+
11+
echo
12+
echo "Up app containers"
13+
source ./helper/scripts/d-stack-01-create-dirs.sh
14+
sleep 2
15+
source ./helper/scripts/d-stack-02-deploy-0.sh
16+
sleep 2
17+
source ./helper/scripts/d-stack-03-up-hub-containers.sh
18+

‎devops/windows/base-docker-wsl/conf/env/conf-env-high-1-pc.template

+3
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ export DATA_DGRAPH_ALPHA_01_DIR=${PROJECT_DATA_DIR}/dgraph/alpha-01
3030
export DATA_DGRAPH_ALPHA_02_DIR=${PROJECT_DATA_DIR}/dgraph/alpha-02
3131
export DATA_DGRAPH_ALPHA_03_DIR=${PROJECT_DATA_DIR}/dgraph/alpha-03
3232
export DATA_POSTGRESQL_DIR=${PROJECT_DATA_DIR}/postgres
33+
export DATA_POSTGRESQL_DB_DIR=${PROJECT_DATA_DIR}/postgres_db
34+
export POSTGRES_BACKUP_DIRECTORY=${PROJECT_DATA_DIR}/backups/postgres
35+
export DGRAPH_BACKUP_DIRECTORY=${PROJECT_DATA_DIR}/backups/dgraph
3336

3437
export SCALE_KEYCLOAK_TEST_SERVER=${SCALE_KEYCLOAK_TEST_SERVER}
3538
export SCALE_NGINX=${SCALE_NGINX}

‎devops/windows/base-docker-wsl/conf/env/conf-env-low-1-pc.template

+3
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ export DATA_KAFKA_01_DIR=${PROJECT_DATA_DIR}/kafka-01
2626
export DATA_DGRAPH_ZERO_01_DIR=${PROJECT_DATA_DIR}/dgraph/zero-01
2727
export DATA_DGRAPH_ALPHA_01_DIR=${PROJECT_DATA_DIR}/dgraph/alpha-01
2828
export DATA_POSTGRESQL_DIR=${PROJECT_DATA_DIR}/postgres
29+
export DATA_POSTGRESQL_DB_DIR=${PROJECT_DATA_DIR}/postgres_db
30+
export POSTGRES_BACKUP_DIRECTORY=${PROJECT_DATA_DIR}/backups/postgres
31+
export DGRAPH_BACKUP_DIRECTORY=${PROJECT_DATA_DIR}/backups/dgraph
2932

3033
export SCALE_KEYCLOAK_TEST_SERVER=${SCALE_KEYCLOAK_TEST_SERVER}
3134
export SCALE_NGINX=${SCALE_NGINX}

0 commit comments

Comments
 (0)