feat(medusa): Rollout index engine behind feature flag (medusajs#11431)

**What**
- Add index engine feature flag
- apply it to the `store/products` end point as well as `admin/products`
- Query builder various fixes
- search capabilities on full data of every entities. The `q` search will be applied to all involved joined table for selection/where clauses

Co-authored-by: Carlos R. L. Rodrigues <[email protected]>

Author: adrien2p

.changeset/silver-baboons-drop.md

@@ -0,0 +1,8 @@
+---
+"@medusajs/index": patch
+"@medusajs/medusa": patch
+"@medusajs/utils": patch
+"@medusajs/modules-sdk": patch
+---
+
+chore(medusa): index engine feature flag

integration-tests/modules/__tests__/index/query.index.ts

@@ -9,6 +9,8 @@ import {
 
 jest.setTimeout(120000)
 
+// NOTE: In this tests, both API are used to query, we use object pattern and string pattern
+
 process.env.ENABLE_INDEX_MODULE = "true"
 
 medusaIntegrationTestRunner({
@@ -23,12 +25,9 @@ medusaIntegrationTestRunner({
       process.env.ENABLE_INDEX_MODULE = "false"
     })
 
-    beforeEach(async () => {
-      await createAdminUser(dbConnection, adminHeaders, appContainer)
-    })
-
     describe("Index engine - Query.index", () => {
-      it("should use query.index to query the index module and hydrate the data", async () => {
+      beforeEach(async () => {
+        await createAdminUser(dbConnection, adminHeaders, appContainer)
         const shippingProfile = (
           await api.post(
             `/admin/shipping-profiles`,
@@ -40,6 +39,7 @@ medusaIntegrationTestRunner({
         const payload = [
           {
             title: "Test Product",
+            status: "published",
             description: "test-product-description",
             shipping_profile_id: shippingProfile.id,
             options: [{ title: "Denominations", values: ["100"] }],
@@ -66,6 +66,7 @@ medusaIntegrationTestRunner({
           {
             title: "Extra product",
             description: "extra description",
+            status: "published",
             shipping_profile_id: shippingProfile.id,
             options: [{ title: "Colors", values: ["Red"] }],
             variants: new Array(2).fill(0).map((_, i) => ({
@@ -88,13 +89,16 @@ medusaIntegrationTestRunner({
           },
         ]
 
-        for (const data of payload) {
-          await api.post("/admin/products", data, adminHeaders).catch((err) => {
+        await api
+          .post("/admin/products/batch", { create: payload }, adminHeaders)
+          .catch((err) => {
             console.log(err)
           })
-        }
-        await setTimeout(5000)
 
+        await setTimeout(2000)
+      })
+
+      it("should use query.index to query the index module and hydrate the data", async () => {
         const query = appContainer.resolve(
           ContainerRegistrationKeys.QUERY
         ) as RemoteQueryFunction
@@ -105,7 +109,7 @@ medusaIntegrationTestRunner({
             "id",
             "description",
             "status",
-
+            "title",
             "variants.sku",
             "variants.barcode",
             "variants.material",
@@ -120,17 +124,25 @@ medusaIntegrationTestRunner({
             "variants.prices.amount": { $gt: 30 },
           },
           pagination: {
+            take: 10,
+            skip: 0,
             order: {
               "variants.prices.amount": "DESC",
             },
           },
         })
 
+        expect(resultset.metadata).toEqual({
+          count: 2,
+          skip: 0,
+          take: 10,
+        })
         expect(resultset.data).toEqual([
           {
             id: expect.any(String),
             description: "extra description",
-            status: "draft",
+            title: "Extra product",
+            status: "published",
             variants: [
               {
                 sku: "extra-variant-0",
@@ -194,7 +206,8 @@ medusaIntegrationTestRunner({
           {
             id: expect.any(String),
             description: "test-product-description",
-            status: "draft",
+            title: "Test Product",
+            status: "published",
             variants: [
               {
                 sku: "test-variant-1",
@@ -234,6 +247,104 @@ medusaIntegrationTestRunner({
           },
         ])
       })
+
+      it("should use query.index to query the index module sorting by price desc", async () => {
+        const query = appContainer.resolve(
+          ContainerRegistrationKeys.QUERY
+        ) as RemoteQueryFunction
+
+        const resultset = await query.index({
+          entity: "product",
+          fields: [
+            "id",
+            "variants.prices.amount",
+            "variants.prices.currency_code",
+          ],
+          filters: {
+            "variants.prices.currency_code": "USD",
+          },
+          pagination: {
+            take: 1,
+            skip: 0,
+            order: {
+              "variants.prices.amount": "DESC",
+            },
+          },
+        })
+
+        // Limiting to 1 on purpose to keep it simple and check the correct order is maintained
+        expect(resultset.data).toEqual([
+          {
+            id: expect.any(String),
+            variants: expect.arrayContaining([
+              expect.objectContaining({
+                prices: expect.arrayContaining([
+                  {
+                    amount: 20,
+                    currency_code: "CAD",
+                    id: expect.any(String),
+                  },
+                  {
+                    amount: 80,
+                    currency_code: "USD",
+                    id: expect.any(String),
+                  },
+                ]),
+              }),
+            ]),
+          },
+        ])
+
+        const resultset2 = await query.index({
+          entity: "product",
+          fields: [
+            "id",
+            "variants.prices.amount",
+            "variants.prices.currency_code",
+          ],
+          filters: {
+            variants: {
+              prices: {
+                currency_code: "USD",
+              },
+            },
+          },
+          pagination: {
+            take: 1,
+            skip: 0,
+            order: {
+              variants: {
+                prices: {
+                  amount: "ASC",
+                },
+              },
+            },
+          },
+        })
+
+        // Limiting to 1 on purpose to keep it simple and check the correct order is maintained
+        expect(resultset2.data).toEqual([
+          {
+            id: expect.any(String),
+            variants: [
+              expect.objectContaining({
+                prices: expect.arrayContaining([
+                  {
+                    amount: 30,
+                    currency_code: "USD",
+                    id: expect.any(String),
+                  },
+                  {
+                    amount: 50,
+                    currency_code: "EUR",
+                    id: expect.any(String),
+                  },
+                ]),
+              }),
+            ],
+          },
+        ])
+      })
     })
   },
 })

packages/core/modules-sdk/src/remote-query/query.ts

@@ -14,6 +14,7 @@ import {
   MedusaError,
   isObject,
   remoteQueryObjectFromString,
+  unflattenObjectKeys,
 } from "@medusajs/utils"
 import { RemoteQuery } from "./remote-query"
 import { toRemoteQuery } from "./to-remote-query"
@@ -211,7 +212,9 @@ export class Query {
       : ({} as any)
     const pagination = queryOptions.pagination as any
     if (pagination?.order) {
-      pagination.order = { [mainEntity]: pagination.order }
+      pagination.order = {
+        [mainEntity]: unflattenObjectKeys(pagination?.order),
+      }
     }
 
     const indexResponse = (await this.#indexModule.query({

packages/core/modules-sdk/src/remote-query/remote-query.ts

@@ -18,6 +18,8 @@ import { isPresent, isString, toPascalCase } from "@medusajs/utils"
 import { MedusaModule } from "../medusa-module"
 
 const BASE_PREFIX = ""
+const MAX_BATCH_SIZE = 4000
+const MAX_CONCURRENT_REQUESTS = 10
 export class RemoteQuery {
   private remoteJoiner: RemoteJoiner
   private modulesMap: Map<string, LoadedModule> = new Map()
@@ -182,6 +184,102 @@ export class RemoteQuery {
     }
   }
 
+  private async fetchRemoteDataBatched(args: {
+    serviceName: string
+    keyField: string
+    service: any
+    methodName: string
+    filters: any
+    options: any
+    ids: (unknown | unknown[])[]
+  }): Promise<any[]> {
+    const {
+      serviceName,
+      keyField,
+      service,
+      methodName,
+      filters,
+      options,
+      ids,
+    } = args
+
+    const getBatch = function* (
+      idArray: (unknown | unknown[])[],
+      batchSize: number
+    ) {
+      for (let i = 0; i < idArray.length; i += batchSize) {
+        yield idArray.slice(i, i + batchSize)
+      }
+    }
+
+    const idsToFetch = getBatch(ids, MAX_BATCH_SIZE)
+    const results: any[] = []
+    let running = 0
+    const fetchPromises: Promise<void>[] = []
+
+    const processBatch = async (batch: (unknown | unknown[])[]) => {
+      running++
+      const batchFilters = { ...filters, [keyField]: batch }
+      let result
+
+      try {
+        if (RemoteQuery.traceFetchRemoteData) {
+          result = await RemoteQuery.traceFetchRemoteData(
+            async () => service[methodName](batchFilters, options),
+            serviceName,
+            methodName,
+            options
+          )
+        } else {
+          result = await service[methodName](batchFilters, options)
+        }
+        results.push(result)
+      } finally {
+        running--
+        processAllBatches()
+      }
+    }
+
+    let batchesDone: (value: void) => void = () => {}
+    const awaitBatches = new Promise((ok) => {
+      batchesDone = ok
+    })
+    const processAllBatches = async () => {
+      let isDone = false
+      while (running < MAX_CONCURRENT_REQUESTS) {
+        const nextBatch = idsToFetch.next()
+        if (nextBatch.done) {
+          isDone = true
+          break
+        }
+
+        const batch = nextBatch.value
+        fetchPromises.push(processBatch(batch))
+      }
+
+      if (isDone) {
+        await Promise.all(fetchPromises)
+        batchesDone()
+      }
+    }
+
+    processAllBatches()
+    await awaitBatches
+
+    const flattenedResults = results.reduce((acc, result) => {
+      if (
+        Array.isArray(result) &&
+        result.length === 2 &&
+        Array.isArray(result[0])
+      ) {
+        return acc.concat(result[0])
+      }
+      return acc.concat(result)
+    }, [])
+
+    return flattenedResults
+  }
+
   public async remoteFetchData(
     expand: RemoteExpandProperty,
     keyField: string,
@@ -267,6 +365,19 @@ export class RemoteQuery {
       options.take = null
     }
 
+    if (ids && ids.length >= MAX_BATCH_SIZE && !hasPagination) {
+      const data = await this.fetchRemoteDataBatched({
+        serviceName: serviceConfig.serviceName,
+        keyField,
+        service,
+        methodName,
+        filters,
+        options,
+        ids,
+      })
+      return { data }
+    }
+
     let result: any
     if (RemoteQuery.traceFetchRemoteData) {
       result = await RemoteQuery.traceFetchRemoteData(