feat: query.index (medusajs#11348)

What:
 - `query.index` helper. It queries the index module, and aggregate the rest of requested fields/relations if needed like `query.graph`.
 
Not covered in this PR:
 - Hydrate only sub entities returned by the query. Example: 1 out of 5 variants have returned, it should only hydrate the data of the single entity, currently it will merge all the variants of the product.
 - Generate types of indexed data
 
 example:
 ```ts
 const query = container.resolve(ContainerRegistrationKeys.QUERY)
        
 await query.index({
  entity: "product",
  fields: [
    "id",
    "description",
    "status",
    "variants.sku",
    "variants.barcode",
    "variants.material",
    "variants.options.value",
    "variants.prices.amount",
    "variants.prices.currency_code",
    "variants.inventory_items.inventory.sku",
    "variants.inventory_items.inventory.description",
  ],
  filters: {
    "variants.sku": { $like: "%-1" },
    "variants.prices.amount": { $gt: 30 },
  },
  pagination: {
    order: {
      "variants.prices.amount": "DESC",
    },
  },
})
```
This query return all products where at least one variant has the title ending in `-1` and at least one price bigger than `30`.
 
The Index Module only hold the data used to paginate and filter, and the returned object is:
```json
{
  "id": "prod_01JKEAM2GJZ14K64R0DHK0JE72",
  "title": null,
  "variants": [
    {
      "id": "variant_01JKEAM2HC89GWS95F6GF9C6YA",
      "sku": "extra-variant-1",
      "prices": [
        {
          "id": "price_01JKEAM2JADEWWX72F8QDP6QXT",
          "amount": 80,
          "currency_code": "USD"
        }
      ]
    }
  ]
}
```

All the rest of the fields will be hydrated from their respective modules, and the final result will be:

```json
{
  "id": "prod_01JKEAY2RJTF8TW9A23KTGY1GD",
  "description": "extra description",
  "status": "draft",
  "variants": [
    {
      "sku": "extra-variant-1",
      "barcode": null,
      "material": null,
      "id": "variant_01JKEAY2S945CRZ6X4QZJ7GVBJ",
      "options": [
        {
          "value": "Red"
        }
      ],
      "prices": [
        {
          "amount": 20,
          "currency_code": "CAD",
          "id": "price_01JKEAY2T2EEYSWZHPGG11B7W7"
        },
        {
          "amount": 80,
          "currency_code": "USD",
          "id": "price_01JKEAY2T2NJK2E5468RK84CAR"
        }
      ],
      "inventory_items": [
        {
          "variant_id": "variant_01JKEAY2S945CRZ6X4QZJ7GVBJ",
          "inventory_item_id": "iitem_01JKEAY2SNY2AWEHPZN0DDXVW6",
          "inventory": {
            "sku": "extra-variant-1",
            "description": "extra variant 1",
            "id": "iitem_01JKEAY2SNY2AWEHPZN0DDXVW6"
          }
        }
      ]
    }
  ]
}
```

Co-authored-by: Adrien de Peretti <[email protected]>

Author: carlos-r-l-rodrigues

.changeset/fresh-beers-visit.md

@@ -0,0 +1,8 @@
+---
+"@medusajs/orchestration": patch
+"@medusajs/modules-sdk": patch
+"@medusajs/types": patch
+"@medusajs/index": patch
+---
+
+feat: query.index

integration-tests/modules/__tests__/index/query.index.ts

@@ -0,0 +1,239 @@
+import { medusaIntegrationTestRunner } from "@medusajs/test-utils"
+import { RemoteQueryFunction } from "@medusajs/types"
+import { ContainerRegistrationKeys, defaultCurrencies } from "@medusajs/utils"
+import { setTimeout } from "timers/promises"
+import {
+  adminHeaders,
+  createAdminUser,
+} from "../../../helpers/create-admin-user"
+
+jest.setTimeout(120000)
+
+process.env.ENABLE_INDEX_MODULE = "true"
+
+medusaIntegrationTestRunner({
+  testSuite: ({ getContainer, dbConnection, api, dbConfig }) => {
+    let appContainer
+
+    beforeAll(() => {
+      appContainer = getContainer()
+    })
+
+    afterAll(() => {
+      process.env.ENABLE_INDEX_MODULE = "false"
+    })
+
+    beforeEach(async () => {
+      await createAdminUser(dbConnection, adminHeaders, appContainer)
+    })
+
+    describe("Index engine - Query.index", () => {
+      it("should use query.index to query the index module and hydrate the data", async () => {
+        const shippingProfile = (
+          await api.post(
+            `/admin/shipping-profiles`,
+            { name: "Test", type: "default" },
+            adminHeaders
+          )
+        ).data.shipping_profile
+
+        const payload = [
+          {
+            title: "Test Product",
+            description: "test-product-description",
+            shipping_profile_id: shippingProfile.id,
+            options: [{ title: "Denominations", values: ["100"] }],
+            variants: [
+              {
+                title: `Test variant 1`,
+                sku: `test-variant-1`,
+                prices: [
+                  {
+                    currency_code: Object.values(defaultCurrencies)[0].code,
+                    amount: 30,
+                  },
+                  {
+                    currency_code: Object.values(defaultCurrencies)[2].code,
+                    amount: 50,
+                  },
+                ],
+                options: {
+                  Denominations: "100",
+                },
+              },
+            ],
+          },
+          {
+            title: "Extra product",
+            description: "extra description",
+            shipping_profile_id: shippingProfile.id,
+            options: [{ title: "Colors", values: ["Red"] }],
+            variants: new Array(2).fill(0).map((_, i) => ({
+              title: `extra variant ${i}`,
+              sku: `extra-variant-${i}`,
+              prices: [
+                {
+                  currency_code: Object.values(defaultCurrencies)[1].code,
+                  amount: 20,
+                },
+                {
+                  currency_code: Object.values(defaultCurrencies)[0].code,
+                  amount: 80,
+                },
+              ],
+              options: {
+                Colors: "Red",
+              },
+            })),
+          },
+        ]
+
+        for (const data of payload) {
+          await api.post("/admin/products", data, adminHeaders).catch((err) => {
+            console.log(err)
+          })
+        }
+        await setTimeout(5000)
+
+        const query = appContainer.resolve(
+          ContainerRegistrationKeys.QUERY
+        ) as RemoteQueryFunction
+
+        const resultset = await query.index({
+          entity: "product",
+          fields: [
+            "id",
+            "description",
+            "status",
+
+            "variants.sku",
+            "variants.barcode",
+            "variants.material",
+            "variants.options.value",
+            "variants.prices.amount",
+            "variants.prices.currency_code",
+            "variants.inventory_items.inventory.sku",
+            "variants.inventory_items.inventory.description",
+          ],
+          filters: {
+            "variants.sku": { $like: "%-1" },
+            "variants.prices.amount": { $gt: 30 },
+          },
+          pagination: {
+            order: {
+              "variants.prices.amount": "DESC",
+            },
+          },
+        })
+
+        expect(resultset.data).toEqual([
+          {
+            id: expect.any(String),
+            description: "extra description",
+            status: "draft",
+            variants: [
+              {
+                sku: "extra-variant-0",
+                barcode: null,
+                material: null,
+                id: expect.any(String),
+                options: [
+                  {
+                    value: "Red",
+                  },
+                ],
+                inventory_items: [
+                  {
+                    variant_id: expect.any(String),
+                    inventory_item_id: expect.any(String),
+                    inventory: {
+                      sku: "extra-variant-0",
+                      description: "extra variant 0",
+                      id: expect.any(String),
+                    },
+                  },
+                ],
+                prices: expect.arrayContaining([]),
+              },
+              {
+                sku: "extra-variant-1",
+                barcode: null,
+                material: null,
+                id: expect.any(String),
+                options: [
+                  {
+                    value: "Red",
+                  },
+                ],
+                prices: expect.arrayContaining([
+                  {
+                    amount: 20,
+                    currency_code: "CAD",
+                    id: expect.any(String),
+                  },
+                  {
+                    amount: 80,
+                    currency_code: "USD",
+                    id: expect.any(String),
+                  },
+                ]),
+                inventory_items: [
+                  {
+                    variant_id: expect.any(String),
+                    inventory_item_id: expect.any(String),
+                    inventory: {
+                      sku: "extra-variant-1",
+                      description: "extra variant 1",
+                      id: expect.any(String),
+                    },
+                  },
+                ],
+              },
+            ],
+          },
+          {
+            id: expect.any(String),
+            description: "test-product-description",
+            status: "draft",
+            variants: [
+              {
+                sku: "test-variant-1",
+                barcode: null,
+                material: null,
+                id: expect.any(String),
+                options: [
+                  {
+                    value: "100",
+                  },
+                ],
+                prices: expect.arrayContaining([
+                  {
+                    amount: 30,
+                    currency_code: "USD",
+                    id: expect.any(String),
+                  },
+                  {
+                    amount: 50,
+                    currency_code: "EUR",
+                    id: expect.any(String),
+                  },
+                ]),
+                inventory_items: [
+                  {
+                    variant_id: expect.any(String),
+                    inventory_item_id: expect.any(String),
+                    inventory: {
+                      sku: "test-variant-1",
+                      description: "Test variant 1",
+                      id: expect.any(String),
+                    },
+                  },
+                ],
+              },
+            ],
+          },
+        ])
+      })
+    })
+  },
+})

integration-tests/modules/__tests__/index/sync.spec.ts

@@ -198,7 +198,6 @@ medusaIntegrationTestRunner({
       expect(updatedResults.length).toBe(1)
       expect(updatedResults[0].variants.length).toBe(1)
 
-      /*
       let staledRaws = await dbConnection.raw(
         'SELECT * FROM "index_data" WHERE "staled_at" IS NOT NULL'
       )
@@ -209,7 +208,6 @@ medusaIntegrationTestRunner({
         'SELECT * FROM "index_relation" WHERE "staled_at" IS NOT NULL'
       )
       expect(staledRaws.rows.length).toBe(0)
-      */
     })
   },
 })

packages/core/modules-sdk/src/medusa-app.ts

@@ -1,6 +1,7 @@
 import { RemoteFetchDataCallback } from "@medusajs/orchestration"
 import {
   ExternalModuleDeclaration,
+  IIndexService,
   ILinkMigrationsPlanner,
   InternalModuleDeclaration,
   LoadedModule,
@@ -27,13 +28,13 @@ import {
   promiseAll,
 } from "@medusajs/utils"
 import { asValue } from "awilix"
+import { Link } from "./link"
 import {
   MedusaModule,
   MigrationOptions,
   ModuleBootstrapOptions,
   RegisterModuleJoinerConfig,
 } from "./medusa-module"
-import { Link } from "./link"
 import { createQuery, RemoteQuery } from "./remote-query"
 import { MODULE_SCOPE } from "./types"
 
@@ -562,13 +563,20 @@ async function MedusaApp_({
     return getMigrationPlanner(options, linkModules)
   }
 
+  const indexModule = sharedContainer_.resolve(Modules.INDEX, {
+    allowUnregistered: true,
+  }) as IIndexService
+
   return {
     onApplicationShutdown,
     onApplicationPrepareShutdown,
     onApplicationStart,
     modules: allModules,
     link: remoteLink,
-    query: createQuery(remoteQuery) as any, // TODO: rm any once we remove the old RemoteQueryFunction and rely on the Query object instead,
+    query: createQuery({
+      remoteQuery,
+      indexModule,
+    }) as any, // TODO: rm any once we remove the old RemoteQueryFunction and rely on the Query object instead,
     entitiesMap,
     gqlSchema: schema,
     notFound,

packages/core/modules-sdk/src/remote-query/__fixtures__/parse-filters.ts

@@ -1,6 +1,6 @@
+import { ModuleJoinerConfig } from "@medusajs/types"
 import { defineJoinerConfig } from "@medusajs/utils"
 import { MedusaModule } from "../../medusa-module"
-import { ModuleJoinerConfig } from "@medusajs/types"
 
 const customModuleJoinerConfig = defineJoinerConfig("custom_user", {
   schema: `
@@ -62,12 +62,12 @@ const pricingJoinerConfig = defineJoinerConfig("pricing", {
     }
 
     type Price {
-      amount: Int
+      amount: Float
       deep_nested_price: DeepNestedPrice
     }
     
     type DeepNestedPrice {
-      amount: Int
+      amount: Float
     }
   `,
   alias: [