Skip to content

Commit 9d9d039

Browse files
adrien2padrien2p
authored
fix(): Index integration tests flackyness (#13953)
* fix(): Index integration tests flackyness * fix * Create twenty-eels-remain.md * fix * fix * fix * fix * finalize * finalize * finalize * finalize * finalize * finalize * chore: empty commit * finalize * finalize * chore: empty commit * finalize * finalize
1 parent c6556d1 commit 9d9d039

File tree

14 files changed

+110
-87
lines changed

14 files changed

+110
-87
lines changed

‎.changeset/twenty-eels-remain.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
"@medusajs/utils": patch
3+
"@medusajs/medusa-utils": patch
4+
---
5+
6+
fix(): Index integration tests flackyness

‎integration-tests/helpers/wait-for-index.ts‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
export interface WaitForIndexOptions {
77
timeout?: number
88
pollInterval?: number
9+
isLink?: boolean
910
}
1011

1112
/**
@@ -18,19 +19,22 @@ export async function waitForIndexedEntities(
1819
entityIds: string[],
1920
options: WaitForIndexOptions = {}
2021
): Promise<void> {
21-
const { timeout = 120000, pollInterval = 100 } = options
22+
const { timeout = 30000, pollInterval = 250 } = options
2223
const startTime = Date.now()
2324

2425
// Normalize the entity name to match partition table naming convention
25-
const normalizedName = entityName.toLowerCase().replace(/[^a-z0-9_]/g, "_")
26+
const normalizedName = !options.isLink
27+
? entityName.toLowerCase().replace(/[^a-z0-9_]/g, "_")
28+
: `link${entityName.toLowerCase()}`
2629
const partitionTableName = `cat_${normalizedName}`
30+
const normalizedEntityName = options.isLink ? `Link${entityName}` : entityName
2731

2832
while (Date.now() - startTime < timeout) {
2933
try {
3034
// Query the index_data table to check if all entities are indexed
3135
const result = await dbConnection.raw(
32-
`SELECT id FROM index_data WHERE name = ? AND id = ANY(?) AND staled_at IS NULL`,
33-
[entityName, entityIds]
36+
`SELECT id FROM index_data WHERE id = ANY(?) AND staled_at IS NULL`,
37+
[entityIds]
3438
)
3539

3640
const indexedIds = result.rows
@@ -62,15 +66,15 @@ export async function waitForIndexedEntities(
6266
return
6367
}
6468
} catch (error) {
65-
// Continue polling on database errors
69+
console.error(error)
6670
}
6771

6872
await new Promise((resolve) => setTimeout(resolve, pollInterval))
6973
}
7074

71-
throw new Error(
75+
console.error(
7276
`Entities [${entityIds.join(
7377
", "
74-
)}] of type '${entityName}' were not fully replicated to partition table within ${timeout}ms`
78+
)}] of type '${normalizedEntityName}' were not fully replicated to partition table within ${timeout}ms`
7579
)
7680
}

‎integration-tests/modules/__tests__/cart/store/cart.completion.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -872,6 +872,8 @@ medusaIntegrationTestRunner({
872872
}),
873873
])
874874

875+
await new Promise((resolve) => setTimeout(resolve, 100))
876+
875877
const { result: fullOrder } = await getOrderDetailWorkflow(
876878
appContainer
877879
).run({

‎integration-tests/modules/__tests__/index/query-index.spec.ts‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -112,15 +112,15 @@ medusaIntegrationTestRunner({
112112
testSuite: ({ getContainer, dbConnection, api, dbConfig }) => {
113113
let appContainer
114114

115-
beforeAll(() => {
116-
appContainer = getContainer()
117-
})
115+
describe("Index engine - Query.index", () => {
116+
beforeAll(() => {
117+
appContainer = getContainer()
118+
})
118119

119-
afterAll(() => {
120-
process.env.ENABLE_INDEX_MODULE = "false"
121-
})
120+
afterAll(() => {
121+
process.env.ENABLE_INDEX_MODULE = "false"
122+
})
122123

123-
describe("Index engine - Query.index", () => {
124124
beforeEach(async () => {
125125
await createAdminUser(dbConnection, adminHeaders, appContainer)
126126
})
@@ -134,7 +134,7 @@ medusaIntegrationTestRunner({
134134
name: "Medusa Brand",
135135
})
136136

137-
await link.create({
137+
const [createdLink] = await link.create({
138138
[Modules.PRODUCT]: {
139139
product_id: products.find((p) => p.title === "Extra product").id,
140140
},
@@ -166,6 +166,14 @@ medusaIntegrationTestRunner({
166166
)
167167
),
168168
waitForIndexedEntities(dbConnection, "Brand", [brand.id]),
169+
waitForIndexedEntities(
170+
dbConnection,
171+
"ProductProductBrandBrand",
172+
[createdLink.id],
173+
{
174+
isLink: true,
175+
}
176+
),
169177
])
170178

171179
const resultset = await fetchAndRetry(
@@ -584,7 +592,7 @@ medusaIntegrationTestRunner({
584592
name: "Medusa Brand",
585593
})
586594

587-
await link.create({
595+
const [createdLink] = await link.create({
588596
[Modules.PRODUCT]: {
589597
product_id: products.find((p) => p.title === "Extra product").id,
590598
},
@@ -616,6 +624,14 @@ medusaIntegrationTestRunner({
616624
)
617625
),
618626
waitForIndexedEntities(dbConnection, "Brand", [brand.id]),
627+
waitForIndexedEntities(
628+
dbConnection,
629+
"ProductProductBrandBrand",
630+
[createdLink.id],
631+
{
632+
isLink: true,
633+
}
634+
),
619635
])
620636

621637
const resultset = await fetchAndRetry(
@@ -636,6 +652,7 @@ medusaIntegrationTestRunner({
636652
waitSeconds: 1.5,
637653
}
638654
)
655+
639656
expect(resultset.data.length).toEqual(1)
640657
})
641658
})

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@
150150
"test:chunk": "./scripts/run-workspace-unit-tests-in-chunks.sh",
151151
"test:integration:packages:fast": "turbo run test:integration --concurrency=1 --no-daemon --no-cache --force --filter='./packages/core/*' --filter='./packages/medusa' --filter='./packages/modules/*' --filter='./packages/modules/providers/*' --filter='!./packages/modules/{workflow-engine-redis,index,product,order,cart}'",
152152
"test:integration:packages:slow": "turbo run test:integration --concurrency=1 --no-daemon --no-cache --force --filter='./packages/modules/{workflow-engine-redis,index,product,order,cart}'",
153-
"test:integration:packages": "turbo run test:integration --concurrency=2 --no-daemon --no-cache --force --filter='./packages/core/*' --filter='./packages/medusa' --filter='./packages/modules/*' --filter='./packages/modules/providers/*'",
153+
"test:integration:packages": "turbo run test:integration --concurrency=1 --no-daemon --no-cache --force --filter='./packages/core/*' --filter='./packages/medusa' --filter='./packages/modules/*' --filter='./packages/modules/providers/*'",
154154
"test:integration:api": "turbo run test:integration:chunk --no-daemon --no-cache --force --filter=integration-tests-api",
155155
"test:integration:http": "turbo run test:integration:chunk --no-daemon --no-cache --force --filter=integration-tests-http",
156156
"test:integration:modules": "turbo run test:integration:chunk --no-daemon --no-cache --force --filter=integration-tests-modules",

‎packages/core/utils/src/common/execute-with-concurrency.ts‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import { promiseAll } from "./promise-all"
2+
13
/**
24
* Execute functions with a concurrency limit
35
* @param functions Array of functions to execute in parallel
@@ -7,25 +9,24 @@ export async function executeWithConcurrency<T>(
79
functions: (() => Promise<T>)[],
810
concurrency: number
911
): Promise<PromiseSettledResult<Awaited<T>>[]> {
10-
const results: PromiseSettledResult<Awaited<T>>[] = new Array(
11-
functions.length
12-
)
12+
const functionsLength = functions.length
13+
const results: PromiseSettledResult<Awaited<T>>[] = new Array(functionsLength)
1314
let currentIndex = 0
1415

1516
const executeNext = async (): Promise<void> => {
16-
while (currentIndex < functions.length) {
17+
while (currentIndex < functionsLength) {
1718
const index = currentIndex++
1819
const result = await Promise.allSettled([functions[index]()])
1920
results[index] = result[0]
2021
}
2122
}
2223

2324
const workers: Promise<void>[] = []
24-
for (let i = 0; i < concurrency; i++) {
25+
for (let i = 0; i < Math.min(concurrency, functionsLength); i++) {
2526
workers.push(executeNext())
2627
}
2728

28-
await Promise.all(workers)
29+
await promiseAll(workers)
2930

3031
return results
3132
}

‎packages/medusa-test-utils/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
"axios": "^1.13.1",
3333
"express": "^4.21.0",
3434
"get-port": "^5.1.1",
35-
"randomatic": "^3.1.1"
35+
"ulid": "^2.3.0"
3636
},
3737
"peerDependencies": {
3838
"@medusajs/framework": "2.11.2",

‎packages/medusa-test-utils/src/database.ts‎

Lines changed: 49 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,8 @@ export const dbTestUtilFactory = (): any => ({
237237
let hasIndexTables = false
238238

239239
const tablesToTruncate: string[] = []
240+
const allTablesToVerify: string[] = []
241+
240242
for (const { table_name } of tableNames) {
241243
if (mainPartitionTables.includes(table_name)) {
242244
hasIndexTables = true
@@ -246,20 +248,61 @@ export const dbTestUtilFactory = (): any => ({
246248
table_name.startsWith(skipIndexPartitionPrefix) ||
247249
mainPartitionTables.includes(table_name)
248250
) {
251+
allTablesToVerify.push(table_name)
249252
continue
250253
}
251254

252255
tablesToTruncate.push(`${schema}."${table_name}"`)
256+
allTablesToVerify.push(table_name)
253257
}
254-
if (tablesToTruncate.length > 0) {
255-
await runRawQuery(`TRUNCATE ${tablesToTruncate.join(", ")};`)
258+
259+
const allTablesToTruncase = [
260+
...tablesToTruncate,
261+
...(hasIndexTables ? mainPartitionTables : []),
262+
].join(", ")
263+
264+
if (allTablesToTruncase) {
265+
await runRawQuery(`TRUNCATE ${allTablesToTruncase};`)
256266
}
257267

258-
if (hasIndexTables) {
259-
await runRawQuery(
260-
`TRUNCATE ${schema}.index_data, ${schema}.index_relation;`
261-
)
268+
const verifyEmpty = async (maxRetries = 5) => {
269+
for (let retry = 0; retry < maxRetries; retry++) {
270+
const countQueries = allTablesToVerify.map(
271+
(tableName) =>
272+
`SELECT '${tableName}' as table_name, COUNT(*) as count FROM ${schema}."${tableName}"`
273+
)
274+
275+
const { rows: counts } = await runRawQuery(
276+
countQueries.join(" UNION ALL ")
277+
)
278+
279+
const nonEmptyTables = counts.filter(
280+
(row: { table_name: string; count: string }) =>
281+
parseInt(row.count) > 0
282+
)
283+
284+
if (nonEmptyTables.length === 0) {
285+
return true
286+
}
287+
288+
if (retry < maxRetries - 1) {
289+
await new Promise((resolve) => setTimeout(resolve, 100))
290+
} else {
291+
const tableList = nonEmptyTables
292+
.map(
293+
(t: { table_name: string; count: string }) =>
294+
`${t.table_name}(${t.count})`
295+
)
296+
.join(", ")
297+
logger.warn(
298+
`Some tables still contain data after truncate: ${tableList}`
299+
)
300+
}
301+
}
302+
return false
262303
}
304+
305+
await verifyEmpty()
263306
} catch (error) {
264307
logger.error("Error during database teardown:", error)
265308
throw error

‎packages/medusa-test-utils/src/id-map.ts‎

Lines changed: 0 additions & 19 deletions
This file was deleted.

‎packages/medusa-test-utils/src/index.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
export * as TestDatabaseUtils from "./database"
22
export * as TestEventUtils from "./events"
3-
export { default as IdMap } from "./id-map"
43
export * from "./init-modules"
54
export * as JestUtils from "./jest"
65
export * from "./medusa-test-runner"

0 commit comments

Comments
 (0)