Skip to content

Commit 66b03bd

Browse files
guillermauguillermau
committed
feat(api): improve merge for externalData
refs: #299
1 parent 0a146d7 commit 66b03bd

File tree

8 files changed

+185
-33
lines changed

8 files changed

+185
-33
lines changed

‎api/src/core/adapters/dbApi/kysely/createPgSoftwareExternalDataRepository.ts‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,5 +154,42 @@ export const createPgSoftwareExternalDataRepository = (db: Kysely<Database>): So
154154
.where("sourceSlug", "=", sourceSlug)
155155
.execute()
156156
.then(rows => rows.length > 0);
157+
},
158+
getOtherIdentifierIdsBySourceURL: async ({ sourceURL }) => {
159+
const request = db
160+
.selectFrom("software_external_datas")
161+
.leftJoin("sources", "sources.slug", "software_external_datas.sourceSlug")
162+
.select(["softwareId", "identifiers"])
163+
.where("sources.url", "!=", sourceURL);
164+
165+
const externalData = await request.execute();
166+
167+
if (!externalData) return undefined;
168+
169+
return externalData.reduce(
170+
(acc, externalDataItem) => {
171+
if (
172+
!externalDataItem.identifiers ||
173+
externalDataItem.identifiers.length === 0 ||
174+
!externalDataItem.softwareId
175+
)
176+
return acc;
177+
178+
const formatedUrl = new URL(sourceURL).toString();
179+
180+
const foundIdentiers = externalDataItem.identifiers.filter(
181+
identifer => identifer.subjectOf?.url.toString() === formatedUrl
182+
);
183+
184+
if (foundIdentiers.length === 0) return acc;
185+
186+
if (foundIdentiers.length > 2)
187+
throw Error("Database corrupted, shouldn't have same source on this object");
188+
189+
acc[foundIdentiers[0].value] = externalDataItem.softwareId;
190+
return acc;
191+
},
192+
{} as Record<string, number>
193+
);
157194
}
158195
});

‎api/src/core/adapters/dbApi/kysely/createPgSourceRepository.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ export const createPgSourceRepository = (db: Kysely<Database>): SourceRepository
1616
.selectAll()
1717
.where("slug", "=", params.name)
1818
.orderBy("priority", "asc")
19-
.executeTakeFirstOrThrow()
20-
.then(row => stripNullOrUndefinedValues(row)),
19+
.executeTakeFirst()
20+
.then(row => row ? stripNullOrUndefinedValues(row): row),
2121
getMainSource: async () =>
2222
db
2323
.selectFrom("sources")

‎api/src/core/ports/DbApiV2.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ export interface SoftwareExternalDataRepository {
119119
getAll: () => Promise<DatabaseDataType.SoftwareExternalDataRow[] | undefined>;
120120
delete: (params: { sourceSlug: string; externalId: string }) => Promise<boolean>;
121121
getSimilarSoftwarePk: (params: { externalId: string; sourceSlug: string }) => Promise<{ softwareId: number }[]>;
122+
getOtherIdentifierIdsBySourceURL: (params: { sourceURL: string }) => Promise<Record<string, number> | undefined>;
122123
}
123124

124125
type CnllPrestataire = {

‎api/src/core/usecases/createSoftware.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,4 +294,6 @@ describe("fetches software extra data (from different providers)", () => {
294294
});
295295
expectToEqual(similardExternalDataUpdated?.length, 3);
296296
});
297+
298+
// TODO Another case : register
297299
});

‎api/src/core/usecases/createSoftware.ts‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@ export const makeCreateSofware: (dbApi: DbApiV2) => CreateSoftware =
4141

4242
let softwareId: number | undefined = undefined;
4343

44+
const source = await dbApi.source.getByName({ name: sourceSlug });
45+
if (!source) throw new Error("Source slug is unknown");
46+
4447
const named = await dbApi.software.getByName(softwareName);
4548

4649
if (named) {
@@ -59,7 +62,25 @@ export const makeCreateSofware: (dbApi: DbApiV2) => CreateSoftware =
5962
}
6063
}
6164

62-
// TODO Resolve with other identifiers
65+
// Check if identifiers is saved in external data
66+
const savedIdentifers = await dbApi.softwareExternalData.getOtherIdentifierIdsBySourceURL({
67+
sourceURL: source.url
68+
});
69+
if (savedIdentifers && externalIdForSource && Object.hasOwn(savedIdentifers, externalIdForSource)) {
70+
// There is no externalId for this source, but it's already save and we know where !
71+
console.info(
72+
`[UC:Import] Importing ${softwareName}(${externalIdForSource}) from ${source.slug}: Adding externalData to software #${savedIdentifers[externalIdForSource]}`
73+
);
74+
await dbApi.softwareExternalData.saveIds([
75+
{
76+
softwareId: savedIdentifers[externalIdForSource],
77+
sourceSlug: source.slug,
78+
externalId: externalIdForSource
79+
}
80+
]);
81+
82+
softwareId = savedIdentifers[externalIdForSource];
83+
}
6384

6485
if (!softwareId) {
6586
console.log(logTitle, `The software package isn't save yet, let's create it`);

‎api/src/core/usecases/getPopulatedSoftware.ts‎

Lines changed: 8 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { DatabaseDataType, DbApiV2, PopulatedExternalData } from "../ports/DbApiV2";
2+
import { mergeObjects } from "../utils";
23
import { Software } from "./readWriteSillData";
34

45
export type MakeGetPopulatedSoftware = (dbApi: DbApiV2) => GetPopulatedSoftware;
@@ -30,8 +31,11 @@ export const makeGetPopulatedSoftware: MakeGetPopulatedSoftware = (dbApi: DbApiV
3031
sofware.map(async softwareItem => {
3132
const formatedSoftwareUI = formatSoftwareRowToUISoftware(softwareItem);
3233

33-
const similarSoftwareIds = await dbApi.similarSoftware.getById({ softwareId: softwareItem.id });
34+
const similarSoftwareIds = await dbApi.software.getSimilarSoftwareExternalDataPks({
35+
softwareId: softwareItem.id
36+
});
3437
console.log(similarSoftwareIds);
38+
// WIP : Either we point to an actual software or we compute a softwareUI from ExternalData
3539

3640
const externalData = await dbApi.softwareExternalData.getPopulatedBySoftwareId({
3741
softwareId: softwareItem.id
@@ -134,6 +138,7 @@ const formatExternalDataRowToUISoftware = (
134138
"@type": "Person",
135139
name: dev.name,
136140
url: dev.url,
141+
identifiers: dev.identifiers,
137142
affiliations: dev["@type"] === "Person" ? dev.affiliations : []
138143
})),
139144
officialWebsiteUrl: externalDataRow.websiteUrl,
@@ -157,13 +162,9 @@ const mergeExternalData = (externalData: PopulatedExternalData[]) => {
157162
if (externalData.length === 0) throw Error("Nothing to merge, the array should be filled");
158163
if (externalData.length === 1) return stripExternalDataFromSource(externalData[0]);
159164

160-
const sortedExternalData = externalData.sort((firstItem, secondItem) => firstItem.priority - secondItem.priority);
165+
const [first, ...nexts] = externalData.sort((firstItem, secondItem) => secondItem.priority - firstItem.priority);
161166

162-
// TODO Merge
163-
const mergedItem = sortedExternalData.reduce((savedExternalDataItem, currentExternalDataItem) => {
164-
console.error(savedExternalDataItem);
165-
return currentExternalDataItem;
166-
}, sortedExternalData[0]);
167+
const mergedItem = mergeObjects(first, nexts);
167168

168169
return stripExternalDataFromSource(mergedItem);
169170
};
@@ -175,24 +176,3 @@ const stripExternalDataFromSource = (
175176

176177
return externalDataItem;
177178
};
178-
179-
/* const mergeObjects = (obj1: PopulatedExternalData, obj2: PopulatedExternalData): PopulatedExternalData => {
180-
const result: PopulatedExternalData = { ...obj1 };
181-
182-
for (const key in obj2) {
183-
if (obj2.hasOwnProperty(key)) {
184-
const value1 = obj1[key as keyof PopulatedExternalData];
185-
const value2 = obj2[key as keyof PopulatedExternalData];
186-
187-
if (value1 === undefined || value1 === null || value1 === '') {
188-
result[key as keyof PopulatedExternalData] = value2;
189-
} else if (Array.isArray(value1) && Array.isArray(value2)) {
190-
result[key] = Array.from(new Set([...value1, ...value2]));
191-
} else {
192-
result[key] = value1;
193-
}
194-
}
195-
}
196-
197-
return result;
198-
} */

‎api/src/core/utils.ts‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,3 +170,114 @@ export const identifersUtils = {
170170
};
171171
}
172172
};
173+
174+
function isEmpty(value: any): boolean {
175+
return (
176+
value === null ||
177+
value === undefined ||
178+
(Array.isArray(value) && value.length === 0) ||
179+
(typeof value === "string" && value.trim() === "")
180+
);
181+
}
182+
183+
const isEqual = (var1: any, var2: any): boolean => {
184+
// Check if both values are strictly equal
185+
if (var1 === var2) {
186+
return true;
187+
}
188+
189+
// Check if both values are of the same type
190+
if (typeof var1 !== typeof var2) {
191+
return false;
192+
}
193+
194+
// Handle null and undefined cases
195+
if (var1 === null || var2 === null) {
196+
return var1 === var2;
197+
}
198+
199+
// Handle arrays
200+
if (Array.isArray(var1) && Array.isArray(var2)) {
201+
if (var1.length !== var2.length) {
202+
return false;
203+
}
204+
for (let i = 0; i < var1.length; i++) {
205+
if (!isDeepIncludedInArray(var1[i], var2)) {
206+
return false;
207+
}
208+
}
209+
return true;
210+
}
211+
212+
// Handle objects
213+
if (typeof var1 === "object" && typeof var2 === "object") {
214+
const keysA = Object.keys(var1);
215+
const keysB = Object.keys(var2);
216+
217+
if (keysA.length !== keysB.length) {
218+
return false;
219+
}
220+
221+
for (let key of keysA) {
222+
if (!keysB.includes(key) || !isEqual(var1[key], var2[key])) {
223+
console.log(var1[key], " !==", var2[key]);
224+
return false;
225+
}
226+
}
227+
228+
return true;
229+
}
230+
231+
// If none of the above conditions are met, the values are not equal
232+
return false;
233+
};
234+
235+
const isDeepIncludedInArray = (var1: any, arrayToCheck: any[]): boolean => {
236+
return arrayToCheck.some(element => isEqual(var1, element));
237+
};
238+
239+
function mergeArrays(arr1: any[], arr2: any[]): any[] {
240+
const merged = [...arr1, ...arr2];
241+
return merged.reduce((acc, item) => {
242+
if (isDeepIncludedInArray(item, acc)) return acc;
243+
return [item, ...acc];
244+
}, []);
245+
}
246+
247+
export const mergeObjects = <T extends Object>(obj1: T, obj2: T | T[]): T => {
248+
if (Array.isArray(obj2)) {
249+
if (obj2.length === 0) return obj1;
250+
251+
const [first, ...rest] = obj2;
252+
return mergeObjects(obj1, mergeObjects(first, rest));
253+
}
254+
255+
// Case both objects
256+
if (Object.keys(obj1).length === 0) return obj2;
257+
if (Object.keys(obj2).length === 0) return obj2;
258+
259+
const result: T = obj1;
260+
261+
for (const key in obj2) {
262+
if (obj2.hasOwnProperty(key)) {
263+
const value1 = obj1[key as keyof T];
264+
const value2 = obj2[key as keyof T];
265+
266+
if (isEmpty(value1)) {
267+
result[key as keyof T] = value2;
268+
} else if (Array.isArray(value1) && Array.isArray(value2)) {
269+
if (value1.length === 0) {
270+
result[key as keyof T] = value2;
271+
} else {
272+
(result[key as keyof T] as any[]) = mergeArrays(value1, value2);
273+
}
274+
} else if (typeof value1 === "object" && typeof value2 === "object") {
275+
(result[key as keyof T] as Object) = mergeObjects(value1 as Object, value2 as Object);
276+
} else {
277+
result[key as keyof T] = value2;
278+
}
279+
}
280+
}
281+
282+
return result;
283+
};

‎api/src/tools/test.helpers.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { Db } from "../core/ports/DbApi";
33
import { DeclarationFormData, InstanceFormData, SoftwareFormData, Source } from "../core/usecases/readWriteSillData";
44
import { Kysely } from "kysely";
55
import { Database } from "../core/adapters/dbApi/kysely/kysely.database";
6-
import { ExternalDataOrigin } from "../lib/ApiTypes";
6+
import { ExternalDataOriginKind } from "../lib/ApiTypes";
77

88
export const testPgUrl = "postgresql://sill:pg_password@localhost:5432/sill";
99

@@ -149,7 +149,7 @@ export const resetDB = async (db: Kysely<Database>) => {
149149
.insertInto("sources")
150150
.values({
151151
...testSource,
152-
kind: testSource.kind as ExternalDataOrigin
152+
kind: testSource.kind as ExternalDataOriginKind
153153
})
154154
.execute();
155155
};

0 commit comments

Comments
 (0)