break: redesign (#125)

* wip * wip * fix * wip * wip * wip * fix * fix * names * test: add tests for GET /users/:address/wearables * add tests for definitions * add tests for error cases on GET /users/:address/wearables * add cache tests for wearables endpoint * chore: emotes (#122) * chore: add emotes just duplicated all logic from wearables * chore: abstraction in definitions-fetcher * chore: abstract wearablesFetcher and emotesFetcher in itemsFetcher * chore: fix tests * fix: tests * chore: item-fetcher queries record * test: add emotes integration tests (#124) * chore: create test components with local fetch and the graph mock to avoid external calls * test: add emote integration tests * chore: add types to test function * chore: add integration tests for names * chore: change ropsten to goerli * feat: add LANDs endpoint * test: add lands integration tests * chore: remove old definitions logic * chore: delete nfts.ts unused code * chore: remove commented code * chore: readd old endpoints (#127) * chore: re add old endpoints * chore: small stuff * chore: remove non used functions and dont export functions * chore: stuff * chore: stuff --------- Co-authored-by: Hugo Arregui <[email protected]> Co-authored-by: Alejo Thomas Ortega <[email protected]>
decentraland · Mar 16, 2023 · 96d5e27 · 96d5e27
1 parent 393a60a
commit 96d5e27
Show file tree

Hide file tree

Showing 44 changed files with 4,228 additions and 1,677 deletions.
diff --git a/package.json b/package.json
@@ -14,6 +14,7 @@
     "@types/lru-cache": "^7.10.10",
     "@types/node": "^18.11.18",
     "@well-known-components/test-helpers": "^1.4.0",
+    "ethereumjs-wallet": "^1.0.2",
     "typescript": "^4.9.5"
   },
   "prettier": {
@@ -33,7 +34,7 @@
     "@well-known-components/logger": "^3.0.0",
     "@well-known-components/metrics": "^2.0.1",
     "@well-known-components/thegraph-component": "^1.4.1",
-    "dcl-catalyst-client": "^12.0.1",
+    "dcl-catalyst-client": "^14.0.9",
     "dcl-catalyst-commons": "9.0.16",
     "lodash": "^4.17.21",
     "lru-cache": "^7.14.1"

diff --git a/src/adapters/definitions-fetcher.ts b/src/adapters/definitions-fetcher.ts
@@ -0,0 +1,52 @@
+import LRU from 'lru-cache'
+import { IBaseComponent } from '@well-known-components/interfaces'
+import { AppComponents, Definition } from '../types'
+import { extractEmoteDefinitionFromEntity, extractWearableDefinitionFromEntity } from './definitions'
+import { Entity } from '@dcl/schemas'
+
+export type DefinitionsFetcher = IBaseComponent & {
+  fetchWearablesDefinitions(urns: string[]): Promise<(Definition | undefined)[]>
+  fetchEmotesDefinitions(urns: string[]): Promise<(Definition | undefined)[]>
+}
+
+export async function createDefinitionsFetcherComponent({
+  config,
+  content
+}: Pick<AppComponents, 'logs' | 'config' | 'content'>): Promise<DefinitionsFetcher> {
+  const itemsSize = (await config.getNumber('ITEMS_CACHE_MAX_SIZE')) ?? 1000
+  const itemsAge = (await config.getNumber('ITEMS_CACHE_MAX_AGE')) ?? 600000 // 10 minutes by default
+
+  // TODO create lower case cache, get/set wrapped to set the key to lowercase
+  const itemDefinitionsCache = new LRU<string, Definition>({
+    max: itemsSize,
+    ttl: itemsAge
+  })
+
+  async function fetchItemsDefinitions(
+    urns: string[],
+    mapEntityToDefinition: (components: Pick<AppComponents, 'content'>, entity: Entity) => Definition
+  ): Promise<(Definition | undefined)[]> {
+    const nonCachedURNs: string[] = []
+    for (const urn of urns) {
+      const definition = itemDefinitionsCache.get(urn.toLowerCase())
+      if (!definition) {
+        nonCachedURNs.push(urn)
+      }
+    }
+
+    if (nonCachedURNs.length !== 0) {
+      const entities = await content.fetchEntitiesByPointers(nonCachedURNs)
+      for (const entity of entities) {
+        const definition = mapEntityToDefinition({ content }, entity)
+        itemDefinitionsCache.set(definition.id.toLowerCase(), definition)
+      }
+    }
+
+    return urns.map((urn) => itemDefinitionsCache.get(urn.toLowerCase()))
+  }
+
+  return {
+    fetchWearablesDefinitions: (urns) => fetchItemsDefinitions(urns, extractWearableDefinitionFromEntity),
+    fetchEmotesDefinitions: (urns) => fetchItemsDefinitions(urns, extractEmoteDefinitionFromEntity)
+  }
+}
diff --git a/src/adapters/definitions.ts b/src/adapters/definitions.ts
@@ -2,6 +2,7 @@ import { EmoteCategory, Entity } from '@dcl/schemas'
 import { AppComponents } from '../types'
 
 export function extractWearableDefinitionFromEntity(components: Pick<AppComponents, 'content'>, entity: Entity) {
+  // TODO: metadata can be null according to the schema, should we add a check before access?
   const metadata = entity.metadata
   const representations = metadata.data.representations.map((representation: any) =>
     mapRepresentation(components, representation, entity)

diff --git a/src/adapters/items-fetcher.ts b/src/adapters/items-fetcher.ts
@@ -0,0 +1,160 @@
+import LRU from 'lru-cache'
+import { AppComponents, Limits, ItemsResult, Item, ItemFetcher, ItemFetcherError, ItemFetcherErrorCode } from '../types'
+import { ISubgraphComponent } from '@well-known-components/thegraph-component'
+import { compareByRarity } from '../logic/utils'
+
+const THE_GRAPH_PAGE_SIZE = 1000
+
+// TODO cache metrics
+type ItemsQueryResponse = {
+  nfts: ItemFromQuery[]
+}
+
+export type ItemFromQuery = {
+  urn: string
+  id: string
+  tokenId: string
+  transferredAt: number
+  item: {
+    rarity: string
+    price: number
+  }
+}
+
+const QUERIES: Record<ItemCategory, string> = {
+  emote: createQueryForCategory('emote'),
+  wearable: createQueryForCategory('wearable')
+}
+
+function groupItemsByURN(items: ItemFromQuery[]): Item[] {
+  const itemsByURN = new Map<string, Item>()
+
+  items.forEach((item) => {
+    const individualData = {
+      id: item.id,
+      tokenId: item.tokenId,
+      transferredAt: item.transferredAt,
+      price: item.item.price
+    }
+    if (itemsByURN.has(item.urn)) {
+      const itemFromMap = itemsByURN.get(item.urn)!
+      itemFromMap.individualData.push(individualData)
+      itemFromMap.amount = itemFromMap.amount + 1
+    } else {
+      itemsByURN.set(item.urn, {
+        urn: item.urn,
+        individualData: [individualData],
+        rarity: item.item.rarity,
+        amount: 1
+      })
+    }
+  })
+
+  return Array.from(itemsByURN.values())
+}
+
+type ItemCategory = 'wearable' | 'emote'
+
+function createQueryForCategory(category: ItemCategory) {
+  return `query fetchItemsByOwner($owner: String, $idFrom: String) {
+    nfts(
+      where: { id_gt: $idFrom, owner: $owner, category: "${category}"},
+      orderBy: id,
+      orderDirection: asc,
+      first: ${THE_GRAPH_PAGE_SIZE}
+    ) {
+      urn,
+      id,
+      tokenId,
+      category
+      transferredAt,
+      item {
+        rarity,
+        price
+      }
+    }
+  }`
+}
+
+async function runItemsQuery(
+  subgraph: ISubgraphComponent,
+  address: string,
+  category: ItemCategory
+): Promise<ItemFromQuery[]> {
+  const items = []
+  const owner = address.toLowerCase()
+  let idFrom = ''
+  let result: ItemsQueryResponse
+  const query = QUERIES[category]
+  do {
+    result = await subgraph.query<ItemsQueryResponse>(query, {
+      owner,
+      idFrom
+    })
+
+    if (result.nfts.length === 0) {
+      break
+    }
+
+    for (const nft of result.nfts) {
+      items.push(nft)
+    }
+
+    idFrom = items[items.length - 1].id
+  } while (result.nfts.length === THE_GRAPH_PAGE_SIZE)
+  return items
+}
+
+export async function createWearableFetcherComponent(components: Pick<AppComponents, 'logs' | 'config' | 'theGraph'>) {
+  return createItemFetcherComponent(components, 'wearable', true)
+}
+
+export async function createEmoteFetcherComponent(components: Pick<AppComponents, 'logs' | 'config' | 'theGraph'>) {
+  return createItemFetcherComponent(components, 'emote', false)
+}
+
+async function createItemFetcherComponent(
+  { config, theGraph, logs }: Pick<AppComponents, 'logs' | 'config' | 'theGraph'>,
+  category: ItemCategory,
+  includeEthereum: boolean
+): Promise<ItemFetcher> {
+  const itemsSize = (await config.getNumber('ITEMS_CACHE_MAX_SIZE')) ?? 1000
+  const itemsAge = (await config.getNumber('ITEMS_CACHE_MAX_AGE')) ?? 600000 // 10 minutes by default
+  const logger = logs.getLogger(`${category}-fetcher`)
+
+  const cache = new LRU<string, Item[]>({
+    max: itemsSize,
+    ttl: itemsAge,
+    fetchMethod: async function (address: string, staleValue: Item[]) {
+      try {
+        const [ethereumItems, maticItems] = await Promise.all([
+          includeEthereum
+            ? runItemsQuery(theGraph.ethereumCollectionsSubgraph, address, category)
+            : ([] as ItemFromQuery[]),
+          runItemsQuery(theGraph.maticCollectionsSubgraph, address, category)
+        ])
+
+        return groupItemsByURN(ethereumItems.concat(maticItems)).sort(compareByRarity)
+      } catch (err: any) {
+        logger.error(err)
+        return staleValue
+      }
+    }
+  })
+
+  async function fetchByOwner(address: string, { offset, limit }: Limits): Promise<ItemsResult> {
+    const results = await cache.fetch(address)
+    if (results === undefined) {
+      throw new ItemFetcherError(ItemFetcherErrorCode.CANNOT_FETCH_ITEMS, `Cannot fetch ${category}s for ${address}`)
+    }
+    const totalAmount = results.length
+    return {
+      items: results.slice(offset, offset + limit),
+      totalAmount
+    }
+  }
+
+  return {
+    fetchByOwner
+  }
+}