Use new trade bulk api (#610)

Co-authored-by: Alexander Drozdov <snosme@gmail.com>

renderer/src/web/price-check/trade/TradeBulk.vue

@@ -9,12 +9,12 @@
           <button class="btn flex items-center mr-1" :style="{ background: selectedCurr !== 'chaos' ? 'transparent' : undefined }"
             @click="selectedCurr = 'chaos'">
             <img src="/images/chaos.png" class="trade-bulk-currency-icon">
-            <span>{{ result.chaos.total }}</span>
+            <span>{{ result.chaos.listed.value?.total ?? '?' }}</span>
           </button>
           <button class="btn flex items-center mr-1" :style="{ background: selectedCurr !== 'exa' ? 'transparent' : undefined }"
             @click="selectedCurr = 'exa'">
             <img src="/images/exa.png" class="trade-bulk-currency-icon">
-            <span>{{ result.exa.total }}</span>
+            <span>{{ result.exa.listed.value?.total ?? '?' }}</span>
           </button>
           <span class="ml-1"><online-filter :filters="filters" /></span>
         </div>
@@ -87,9 +87,9 @@
 </template>
 
 <script lang="ts">
-import { defineComponent, PropType, inject, ref, computed, watch, ComputedRef, shallowRef } from 'vue'
+import { defineComponent, PropType, inject, computed, watch, ComputedRef, Ref, shallowRef, shallowReactive } from 'vue'
 import { useI18n } from 'vue-i18n'
-import { BulkSearch, execBulkSearch, PricingResult, requestResults } from './pathofexile-bulk'
+import { BulkSearch, execBulkSearch, PricingResult } from './pathofexile-bulk'
 import { getTradeEndpoint } from './common'
 import { selected as league } from '../../background/Leagues'
 import { AppConfig } from '@/web/Config'
@@ -102,13 +102,13 @@ import OnlineFilter from './OnlineFilter.vue'
 const slowdown = artificialSlowdown(900)
 
 function useBulkApi () {
-  type BulkSearchExtended = BulkSearch & {
-    exa: { listed: ComputedRef<PricingResult[]> }
-    chaos: { listed: ComputedRef<PricingResult[]> }
-  }
+  type BulkSearchExtended = Record<'exa' | 'chaos', {
+    listed: Ref<BulkSearch | null>
+    listedLazy: ComputedRef<PricingResult[]>
+  }>
 
   let searchId = 0
-  const error = ref<string | null>(null)
+  const error = shallowRef<string | null>(null)
   const result = shallowRef<BulkSearchExtended | null>(null)
 
   async function search (item: ParsedItem, filters: ItemFilters) {
@@ -118,17 +118,20 @@ function useBulkApi () {
       result.value = null
 
       const _searchId = searchId
-      const _result = await execBulkSearch(item, filters)
+
+      // override, because at league start many players set wrong price, and this breaks optimistic search
+      const have = (item.info.refName === 'Chaos Orb')
+        ? ['exalted']
+        : (item.info.refName === 'Exalted Orb')
+            ? ['chaos']
+            : ['exalted', 'chaos']
+
+      const optimisticSearch = await execBulkSearch(
+        item, filters, have, { accountName: AppConfig().accountName })
       if (_searchId === searchId) {
         result.value = {
-          exa: {
-            ..._result.exa,
-            listed: getResultsByQuery(_result.exa)
-          },
-          chaos: {
-            ..._result.chaos,
-            listed: getResultsByQuery(_result.chaos)
-          }
+          exa: getResultsByHave(item, filters, optimisticSearch, 'exalted'),
+          chaos: getResultsByHave(item, filters, optimisticSearch, 'chaos')
         }
       }
     } catch (err) {
@@ -136,16 +139,33 @@ function useBulkApi () {
     }
   }
 
-  function getResultsByQuery (query: BulkSearch['exa' | 'chaos']) {
-    const items = shallowRef<PricingResult[]>([])
-    let requested = false
+  function getResultsByHave (
+    item: ParsedItem,
+    filters: ItemFilters,
+    preloaded: Array<BulkSearch | null>,
+    have: 'exalted' | 'chaos'
+  ) {
+    const _result = shallowRef(
+      preloaded.some(res => res?.haveTag === have)
+        ? shallowReactive(preloaded.find(res => res?.haveTag === have)!)
+        : null)
+    const items = shallowRef<PricingResult[]>(_result.value?.listed ?? [])
+    let requested: boolean = (_result.value != null)
 
-    return computed(() => {
-      if (query.total && !requested) {
+    const listedLazy = computed(() => {
+      if (!requested) {
         ;(async function () {
           try {
             requested = true
-            items.value = await requestResults(query.queryId, query.listedIds.slice(0, 20), { accountName: AppConfig().accountName })
+            _result.value = shallowReactive((await execBulkSearch(
+              item, filters, [have], { accountName: AppConfig().accountName }))[0]!
+            )
+            items.value = _result.value.listed
+            const otherHave = (have === 'exalted')
+              ? result.value?.chaos?.listed.value!
+              : result.value?.exa?.listed.value!
+            // fix best guess we did while making optimistic search
+            otherHave.total -= _result.value.total
           } catch (err) {
             error.value = (err as Error).message
           }
@@ -154,6 +174,8 @@ function useBulkApi () {
 
       return items.value
     })
+
+    return { listed: _result, listedLazy }
   }
 
   return { error, result, search }
@@ -175,7 +197,7 @@ export default defineComponent({
     const widget = computed(() => AppConfig<PriceCheckWidget>('price-check')!)
     const { error, result, search } = useBulkApi()
 
-    const selectedCurr = ref<'chaos' | 'exa'>('chaos')
+    const selectedCurr = shallowRef<'chaos' | 'exa'>('chaos')
 
     watch(() => props.item, (item) => {
       slowdown.reset(item)
@@ -185,21 +207,20 @@ export default defineComponent({
       const arr = Array(20)
       if (!slowdown.isReady.value || !result.value) return arr
 
-      const listed = result.value[selectedCurr.value].listed.value
+      const listed = result.value[selectedCurr.value].listedLazy.value
       arr.splice(0, listed.length, ...listed)
       return arr
     })
 
     watch(result, () => {
-      if (result.value) {
-        const { exa, chaos } = result.value
-        selectedCurr.value = (exa.total > chaos.total) ? 'exa' : 'chaos'
-        // override, because at league start many players set wrong price, and this breaks auto-detection
-        if (props.item.info.refName === 'Chaos Orb') {
-          selectedCurr.value = 'exa'
-        } else if (props.item.info.refName === 'Exalted Orb') {
-          selectedCurr.value = 'chaos'
-        }
+      const exaTotal = result.value?.exa.listed.value?.total
+      const chaosTotal = result.value?.chaos.listed.value?.total
+      if (exaTotal == null) {
+        selectedCurr.value = 'chaos'
+      } else if (chaosTotal == null) {
+        selectedCurr.value = 'exa'
+      } else {
+        selectedCurr.value = (exaTotal > chaosTotal) ? 'exa' : 'chaos'
       }
     })
 
@@ -216,7 +237,7 @@ export default defineComponent({
       execSearch: () => { search(props.item, props.filters) },
       showSeller: computed(() => widget.value.showSeller),
       openTradeLink (isExternal: boolean) {
-        const link = `https://${getTradeEndpoint()}/trade/exchange/${league.value}/${result.value![selectedCurr.value].queryId}`
+        const link = `https://${getTradeEndpoint()}/trade/exchange/${league.value}/${result.value![selectedCurr.value].listed.value!.queryId}`
         if (isExternal) {
           window.open(link)
         } else {

renderer/src/web/price-check/trade/TradeListing.vue

@@ -83,8 +83,8 @@
 <script lang="ts">
 import { defineComponent, computed, watch, PropType, inject, shallowReactive, shallowRef } from 'vue'
 import { useI18n } from 'vue-i18n'
-import { requestTradeResultList, requestResults, createTradeRequest, PricingResult } from './pathofexile-trade'
-import { getTradeEndpoint, SearchResult } from './common'
+import { requestTradeResultList, requestResults, createTradeRequest, PricingResult, SearchResult } from './pathofexile-trade'
+import { getTradeEndpoint } from './common'
 import { AppConfig } from '@/web/Config'
 import { PriceCheckWidget } from '@/web/overlay/interfaces'
 import { ItemFilters, StatFilter } from '../filters/interfaces'

renderer/src/web/price-check/trade/common.ts

@@ -15,12 +15,8 @@ export interface Account {
   }
 }
 
-export interface SearchResult {
-  id: string
-  result: string[]
-  total: number
-  inexact?: boolean
-  error?: {
+export type TradeResponse<T> = (T & { error?: null }) | {
+  error: {
     code: number
     message: string
   }

renderer/src/web/price-check/trade/pathofexile-bulk.ts

@@ -1,13 +1,13 @@
 import { DateTime } from 'luxon'
 import { MainProcess } from '@/web/background/IPC'
-import { SearchResult, Account, getTradeEndpoint, RATE_LIMIT_RULES, adjustRateLimits, tradeTag, preventQueueCreation } from './common'
+import { TradeResponse, Account, getTradeEndpoint, RATE_LIMIT_RULES, adjustRateLimits, tradeTag, preventQueueCreation } from './common'
 import { RateLimiter } from './RateLimiter'
 import { ItemFilters } from '../filters/interfaces'
 import { ParsedItem } from '@/parser'
 import { Cache } from './Cache'
 
 interface TradeRequest { /* eslint-disable camelcase */
-  engine: 'legacy' // TODO: blocked by https://www.pathofexile.com/forum/view-thread/3265663
+  engine: 'new'
   query: {
     status: { option: 'online' | 'onlineleague' | 'any' }
     have: string[]
@@ -15,21 +15,29 @@ interface TradeRequest { /* eslint-disable camelcase */
     minimum?: number
     fulfillable?: null
   }
+  sort: { have: 'asc' }
+}
+
+interface SearchResult {
+  id: string
+  result: Record<string, FetchResult>
+  total: number
 }
 
 interface FetchResult {
   id: string
   listing: {
-    // indexed: string // not available in legacy engine now
-    price: {
+    indexed: string
+    offers: Array<{
       exchange: {
+        currency: string
         amount: number
       }
       item: {
         amount: number
         stock: number
       }
-    }
+    }>
     account: Account
   }
 }
@@ -53,8 +61,7 @@ async function requestTradeResultList (body: TradeRequest, leagueId: string): Pr
 
   if (!data) {
     preventQueueCreation([
-      { count: 2, limiters: RATE_LIMIT_RULES.EXCHANGE },
-      { count: 1, limiters: RATE_LIMIT_RULES.FETCH }
+      { count: 1, limiters: RATE_LIMIT_RULES.EXCHANGE }
     ])
 
     await RateLimiter.waitMulti(RATE_LIMIT_RULES.EXCHANGE)
@@ -69,99 +76,115 @@ async function requestTradeResultList (body: TradeRequest, leagueId: string): Pr
     })
     adjustRateLimits(RATE_LIMIT_RULES.EXCHANGE, response.headers)
 
-    data = await response.json() as SearchResult
-    if (data.error) {
-      throw new Error(data.error.message)
+    const _data = await response.json() as TradeResponse<SearchResult>
+    if (_data.error) {
+      throw new Error(_data.error.message)
+    } else {
+      data = _data
     }
 
-    cache.set<SearchResult>([body, leagueId], data, Cache.deriveTtl(...RATE_LIMIT_RULES.EXCHANGE, ...RATE_LIMIT_RULES.FETCH))
+    cache.set<SearchResult>([body, leagueId], data, Cache.deriveTtl(...RATE_LIMIT_RULES.EXCHANGE))
   }
 
   return data
 }
 
-export async function requestResults (
-  queryId: string,
-  resultIds: string[],
-  opts: { accountName: string }
-): Promise<PricingResult[]> {
-  interface ResponseT { result: FetchResult[], error: SearchResult['error'] }
-  let data = cache.get<ResponseT>(resultIds)
-
-  if (!data) {
-    await RateLimiter.waitMulti(RATE_LIMIT_RULES.FETCH)
-
-    const response = await fetch(`${MainProcess.CORS}https://${getTradeEndpoint()}/api/trade/fetch/${resultIds.join(',')}?query=${queryId}&exchange`)
-    adjustRateLimits(RATE_LIMIT_RULES.FETCH, response.headers)
-
-    data = await response.json() as ResponseT
-    if (data.error) {
-      throw new Error(data.error.message)
-    }
-
-    cache.set<ResponseT>(resultIds, data, Cache.deriveTtl(...RATE_LIMIT_RULES.EXCHANGE, ...RATE_LIMIT_RULES.FETCH))
+function toPricingResult (
+  result: FetchResult,
+  opts: { accountName: string },
+  offer: number
+): PricingResult {
+  return {
+    id: result.id,
+    relativeDate: DateTime.fromISO(result.listing.indexed).toRelative({ style: 'short' }) ?? '',
+    exchangeAmount: result.listing.offers[offer].exchange.amount,
+    itemAmount: result.listing.offers[offer].item.amount,
+    stock: result.listing.offers[offer].item.stock,
+    isMine: (result.listing.account.name === opts.accountName),
+    ign: result.listing.account.lastCharacterName,
+    accountName: result.listing.account.name,
+    accountStatus: result.listing.account.online
+      ? (result.listing.account.online.status === 'afk' ? 'afk' : 'online')
+      : 'offline'
   }
-
-  return data.result
-    .filter(result => result != null) // { gone: true }
-    .map<PricingResult>(result => {
-    return {
-      id: result.id,
-      relativeDate: DateTime.fromISO('').toRelative({ style: 'short' }) ?? '',
-      exchangeAmount: result.listing.price.exchange.amount,
-      itemAmount: result.listing.price.item.amount,
-      stock: result.listing.price.item.stock,
-      isMine: (result.listing.account.name === opts.accountName),
-      ign: result.listing.account.lastCharacterName,
-      accountName: result.listing.account.name,
-      accountStatus: result.listing.account.online
-        ? (result.listing.account.online.status === 'afk' ? 'afk' : 'online')
-        : 'offline'
-    }
-  })
 }
 
 export interface BulkSearch {
-  exa: {
-    queryId: string
-    total: number
-    listedIds: string[]
-  }
-  chaos: {
-    queryId: string
-    total: number
-    listedIds: string[]
+  queryId: string
+  haveTag: string
+  total: number
+  listed: PricingResult[]
+}
+
+function createTradeRequest (filters: ItemFilters, item: ParsedItem, have: string[]): TradeRequest {
+  return {
+    engine: 'new',
+    query: {
+      have: have,
+      want: [tradeTag(item)!],
+      status: {
+        option: filters.trade.offline
+          ? 'any'
+          : (filters.trade.onlineInLeague ? 'onlineleague' : 'online')
+      },
+      minimum: (filters.stackSize && !filters.stackSize.disabled) ? filters.stackSize.value : undefined
+      // fulfillable: null
+    },
+    sort: { have: 'asc' }
   }
 }
 
-const HAVE_CURRENCY = ['exalted', 'chaos']
+const SHOW_RESULTS = 20
+const API_FETCH_LIMIT = 100
 
-export async function execBulkSearch (item: ParsedItem, filters: ItemFilters): Promise<BulkSearch> {
-  const resultByHave = await Promise.all(HAVE_CURRENCY.map(async (have) => {
-    const query = await requestTradeResultList({
-      engine: 'legacy',
-      query: {
-        have: [have],
-        want: [tradeTag(item)!],
-        status: {
-          option: filters.trade.offline
-            ? 'any'
-            : (filters.trade.onlineInLeague ? 'onlineleague' : 'online')
-        },
-        minimum: (filters.stackSize && !filters.stackSize.disabled) ? filters.stackSize.value : undefined
-        // fulfillable: null
-      }
-    }, filters.trade.league)
+export async function execBulkSearch (
+  item: ParsedItem,
+  filters: ItemFilters,
+  have: string[],
+  opts: { accountName: string }
+): Promise<Array<BulkSearch | null>> {
+  const query = await requestTradeResultList(
+    createTradeRequest(filters, item, have),
+    filters.trade.league
+  )
+
+  const offer = 0
+  const results = Object.values(query.result)
+    .filter(result => result.listing.offers.length === 1)
+
+  const resultByHave = have.map(tradeTag => {
+    const resultsTag = results.filter(result => result.listing.offers[offer].exchange.currency === tradeTag)
+
+    const loadedOnDemand = (
+      tradeTag === 'chaos' &&
+      resultsTag.length < SHOW_RESULTS &&
+      query.total > API_FETCH_LIMIT
+    )
+    if (loadedOnDemand) return null
+
+    const listed = resultsTag
+      .sort((a, b) =>
+        (a.listing.offers[offer].exchange.amount / a.listing.offers[offer].item.amount) -
+        (b.listing.offers[offer].exchange.amount / b.listing.offers[offer].item.amount))
+      .slice(0, SHOW_RESULTS)
+      .map(result => toPricingResult(result, opts, offer))
+
+    const chaosIsLoaded = (
+      tradeTag === 'exalted' &&
+      resultsTag.length < results.length &&
+      ((results.length - resultsTag.length) >= SHOW_RESULTS || query.total <= API_FETCH_LIMIT)
+    )
 
     return {
       queryId: query.id,
-      total: query.total,
-      listedIds: query.result
+      haveTag: tradeTag,
+      // this is a best guess when making request with multiple `have` currencies
+      total: (chaosIsLoaded)
+        ? resultsTag.length
+        : (query.total - (results.length - resultsTag.length)),
+      listed: listed
     }
-  }))
+  })
 
-  return {
-    exa: resultByHave[0],
-    chaos: resultByHave[1]
-  }
+  return resultByHave
 }

renderer/src/web/price-check/trade/pathofexile-trade.ts

@@ -3,7 +3,7 @@ import { ItemFilters, StatFilter, INTERNAL_TRADE_IDS, InternalTradeId } from '..
 import { setProperty as propSet } from 'dot-prop'
 import { DateTime } from 'luxon'
 import { MainProcess } from '@/web/background/IPC'
-import { SearchResult, Account, getTradeEndpoint, adjustRateLimits, RATE_LIMIT_RULES, preventQueueCreation } from './common'
+import { TradeResponse, Account, getTradeEndpoint, adjustRateLimits, RATE_LIMIT_RULES, preventQueueCreation } from './common'
 import { STAT_BY_REF } from '@/assets/data'
 import { RateLimiter } from './RateLimiter'
 import { ModifierType } from '@/parser/modifiers'
@@ -182,6 +182,13 @@ interface TradeRequest { /* eslint-disable camelcase */
   }
 }
 
+export interface SearchResult {
+  id: string
+  result: string[]
+  total: number
+  inexact?: boolean
+}
+
 interface FetchResult {
   id: string
   item: {
@@ -512,9 +519,11 @@ export async function requestTradeResultList (body: TradeRequest, leagueId: stri
     })
     adjustRateLimits(RATE_LIMIT_RULES.SEARCH, response.headers)
 
-    data = await response.json() as SearchResult
-    if (data.error) {
-      throw new Error(data.error.message)
+    const _data = await response.json() as TradeResponse<SearchResult>
+    if (_data.error) {
+      throw new Error(_data.error.message)
+    } else {
+      data = _data
     }
 
     cache.set<SearchResult>([body, leagueId], data, Cache.deriveTtl(...RATE_LIMIT_RULES.SEARCH, ...RATE_LIMIT_RULES.FETCH))
@@ -528,8 +537,7 @@ export async function requestResults (
   resultIds: string[],
   opts: { accountName: string }
 ): Promise<PricingResult[]> {
-  interface ResponseT { result: FetchResult[], error: SearchResult['error'] }
-  let data = cache.get<ResponseT>(resultIds)
+  let data = cache.get<FetchResult[]>(resultIds)
 
   if (!data) {
     await RateLimiter.waitMulti(RATE_LIMIT_RULES.FETCH)
@@ -537,15 +545,17 @@ export async function requestResults (
     const response = await fetch(`${MainProcess.CORS}https://${getTradeEndpoint()}/api/trade/fetch/${resultIds.join(',')}?query=${queryId}`)
     adjustRateLimits(RATE_LIMIT_RULES.FETCH, response.headers)
 
-    data = await response.json() as ResponseT
-    if (data.error) {
-      throw new Error(data.error.message)
+    const _data = await response.json() as TradeResponse<{ result: FetchResult[] }>
+    if (_data.error) {
+      throw new Error(_data.error.message)
+    } else {
+      data = _data.result
     }
 
-    cache.set<ResponseT>(resultIds, data, Cache.deriveTtl(...RATE_LIMIT_RULES.SEARCH, ...RATE_LIMIT_RULES.FETCH))
+    cache.set<FetchResult[]>(resultIds, data, Cache.deriveTtl(...RATE_LIMIT_RULES.SEARCH, ...RATE_LIMIT_RULES.FETCH))
   }
 
-  return data.result.map<PricingResult>(result => {
+  return data.map<PricingResult>(result => {
     return {
       id: result.id,
       itemLevel: result.item.ilvl,