From 6b4473b8d9fc692ccdcfdb2e92494b671c4bb3ef Mon Sep 17 00:00:00 2001
From: Alexander Drozdov <snosme@gmail.com>
Date: Thu, 26 Jan 2023 14:26:22 +0200
Subject: [PATCH] feat: OCR

---
 ipc/types.ts                                  |  13 ++
 main/build/script.cjs                         |   9 +
 main/electron-builder.yml                     |   2 +-
 main/package.json                             |   4 +-
 main/src/main.ts                              |   4 +-
 main/src/shortcuts/Shortcuts.ts               |  42 +++-
 main/src/vision/HeistGemFinder.ts             | 125 ++++++++++++
 main/src/vision/link-main.ts                  |  42 ++++
 main/src/vision/link-worker.ts                |  28 +++
 main/src/vision/utils.ts                      |  75 +++++++
 main/src/vision/wasm-bindings.ts              |  56 ++++++
 main/src/windowing/GameWindow.ts              |   4 +
 main/yarn.lock                                |  25 ++-
 renderer/package.json                         |   1 +
 renderer/src/assets/data/index.ts             |  21 ++
 renderer/src/web/Config.ts                    |  16 +-
 renderer/src/web/overlay/WidgetItemSearch.vue | 188 +++++++++++++-----
 renderer/src/web/overlay/widgets.ts           |   1 +
 renderer/src/web/settings/SettingsWindow.vue  |   4 +
 renderer/src/web/settings/item-search.vue     |  38 ++++
 renderer/yarn.lock                            |   5 +
 21 files changed, 641 insertions(+), 62 deletions(-)
 create mode 100644 main/src/vision/HeistGemFinder.ts
 create mode 100644 main/src/vision/link-main.ts
 create mode 100644 main/src/vision/link-worker.ts
 create mode 100644 main/src/vision/utils.ts
 create mode 100644 main/src/vision/wasm-bindings.ts
 create mode 100644 renderer/src/web/settings/item-search.vue

diff --git a/ipc/types.ts b/ipc/types.ts
index 8aa980b3..d5a41aca 100644
--- a/ipc/types.ts
+++ b/ipc/types.ts
@@ -8,6 +8,7 @@ export interface HostConfig {
   disableUpdateDownload: boolean
   logLevel: string
   windowTitle: string
+  language: string
 }
 
 export interface ShortcutAction {
@@ -17,6 +18,9 @@ export interface ShortcutAction {
     type: 'copy-item'
     focusOverlay?: boolean
     target: string
+  } | {
+    type: 'ocr-text'
+    target: 'heist-gems'
   } | {
     type: 'trigger-event'
     target: string
@@ -69,6 +73,7 @@ export type IpcEvent =
   IpcHostConfig |
   IpcWidgetAction |
   IpcItemText |
+  IpcOcrText |
   IpcConfigChanged |
   IpcUserAction
 
@@ -142,6 +147,14 @@ type IpcItemText =
     focusOverlay: boolean
   }>
 
+type IpcOcrText =
+  Event<'MAIN->CLIENT::ocr-text', {
+    target: string
+    pressTime: number
+    ocrTime: number
+    paragraphs: string[]
+  }>
+
 type IpcGameLog =
   Event<'MAIN->CLIENT::game-log', {
     lines: string[]
diff --git a/main/build/script.cjs b/main/build/script.cjs
index d4733f20..78e8135b 100644
--- a/main/build/script.cjs
+++ b/main/build/script.cjs
@@ -19,6 +19,14 @@ const electronRunner = (() => {
   }
 })()
 
+const visionBuild = esbuild.build({
+  entryPoints: ['src/vision/link-worker.ts'],
+  bundle: true,
+  platform: 'node',
+  outfile: 'dist/vision.js',
+  watch: isDev
+})
+
 const mainBuild = esbuild.build({
   entryPoints: ['src/main.ts'],
   bundle: true,
@@ -36,6 +44,7 @@ const mainBuild = esbuild.build({
 })
 
 Promise.all([
+  visionBuild,
   mainBuild
 ])
 .then(() => { if (isDev) electronRunner.restart() })
diff --git a/main/electron-builder.yml b/main/electron-builder.yml
index 28bf4c12..ad8b12ac 100644
--- a/main/electron-builder.yml
+++ b/main/electron-builder.yml
@@ -6,7 +6,7 @@ files:
   - "package.json"
   - from: "dist"
     to: "."
-    filter: ["main.js"]
+    filter: ["main.js", "vision.js"]
   - from: "../renderer/dist"
     to: "."
 extraMetadata:
diff --git a/main/package.json b/main/package.json
index 9d54e8ea..d369e7c9 100644
--- a/main/package.json
+++ b/main/package.json
@@ -16,7 +16,7 @@
   },
   "main": "dist/main.js",
   "dependencies": {
-    "electron-overlay-window": "3.0.0",
+    "electron-overlay-window": "3.2.0",
     "uiohook-napi": "1.5.x"
   },
   "devDependencies": {
@@ -27,6 +27,8 @@
     "@types/ini": "^1.3.30",
     "@types/node": "16.x.x",
     "@types/ws": "^8.5.3",
+    "@wokwi/bmp-ts": "^3.0.0",
+    "comlink": "^4.3.1",
     "electron": "19.1.9",
     "electron-builder": "23.3.3",
     "electron-updater": "^5.0.1",
diff --git a/main/src/main.ts b/main/src/main.ts
index b08b631a..86dcd08e 100644
--- a/main/src/main.ts
+++ b/main/src/main.ts
@@ -36,10 +36,10 @@ app.on('ready', async () => {
     async () => {
       const overlay = new OverlayWindow(eventPipe, logger, poeWindow, httpProxy)
       new OverlayVisibility(eventPipe, overlay, gameConfig)
-      const shortcuts = new Shortcuts(logger, overlay, poeWindow, gameConfig, eventPipe)
+      const shortcuts = await Shortcuts.create(logger, overlay, poeWindow, gameConfig, eventPipe)
       eventPipe.onEventAnyClient('CLIENT->MAIN::update-host-config', (cfg) => {
         overlay.updateOpts(cfg.overlayKey, cfg.windowTitle)
-        shortcuts.updateActions(cfg.shortcuts, cfg.stashScroll, cfg.restoreClipboard)
+        shortcuts.updateActions(cfg.shortcuts, cfg.stashScroll, cfg.restoreClipboard, cfg.language)
         gameLogWatcher.restart(cfg.clientLog)
         gameConfig.readConfig(cfg.gameConfig)
         appUpdater.updateOpts(!cfg.disableUpdateDownload)
diff --git a/main/src/shortcuts/Shortcuts.ts b/main/src/shortcuts/Shortcuts.ts
index afd85ed1..44ac5480 100644
--- a/main/src/shortcuts/Shortcuts.ts
+++ b/main/src/shortcuts/Shortcuts.ts
@@ -4,6 +4,7 @@ import { isModKey, KeyToElectron, mergeTwoHotkeys } from '../../../ipc/KeyToCode
 import { typeInChat, stashSearch } from './text-box'
 import { WidgetAreaTracker } from '../windowing/WidgetAreaTracker'
 import { HostClipboard } from './HostClipboard'
+import { OcrWorker } from '../vision/link-main'
 import type { ShortcutAction } from '../../../ipc/types'
 import type { Logger } from '../RemoteLogger'
 import type { OverlayWindow } from '../windowing/OverlayWindow'
@@ -20,12 +21,25 @@ export class Shortcuts {
   private areaTracker: WidgetAreaTracker
   private clipboard: HostClipboard
 
-  constructor (
+  static async create (
+    logger: Logger,
+    overlay: OverlayWindow,
+    poeWindow: GameWindow,
+    gameConfig: GameConfig,
+    server: ServerEvents
+  ) {
+    const ocrWorker = await OcrWorker.create()
+    const shortcuts = new Shortcuts(logger, overlay, poeWindow, gameConfig, server, ocrWorker)
+    return shortcuts
+  }
+
+  private constructor (
     private logger: Logger,
     private overlay: OverlayWindow,
     private poeWindow: GameWindow,
     private gameConfig: GameConfig,
-    private server: ServerEvents
+    private server: ServerEvents,
+    private ocrWorker: OcrWorker
   ) {
     this.areaTracker = new WidgetAreaTracker(server, overlay)
     this.clipboard = new HostClipboard(logger)
@@ -69,9 +83,10 @@ export class Shortcuts {
     })
   }
 
-  updateActions (actions: ShortcutAction[], stashScroll: boolean, restoreClipboard: boolean) {
+  updateActions (actions: ShortcutAction[], stashScroll: boolean, restoreClipboard: boolean, language: string) {
     this.stashScroll = stashScroll
     this.clipboard.updateOptions(restoreClipboard)
+    this.ocrWorker.updateOptions(language)
 
     const copyItemShortcut = mergeTwoHotkeys('Ctrl + C', this.gameConfig.showModsKey)
     if (copyItemShortcut !== 'Ctrl + C') {
@@ -154,6 +169,27 @@ export class Shortcuts {
             (entry.keepModKeys) ? entry.shortcut.split(' + ').filter(key => isModKey(key)) : undefined,
             this.gameConfig.showModsKey
           )
+        } else if (entry.action.type === 'ocr-text' && entry.action.target === 'heist-gems') {
+          if (process.platform !== 'win32') return
+
+          const { action } = entry
+          const pressTime = Date.now()
+          const imageData = this.poeWindow.screenshot()
+          this.ocrWorker.findHeistGems({
+            width: this.poeWindow.bounds.width,
+            height: this.poeWindow.bounds.height,
+            data: imageData
+          }).then(result => {
+            this.server.sendEventTo('last-active', {
+              name: 'MAIN->CLIENT::ocr-text',
+              payload: {
+                target: action.target,
+                pressTime,
+                ocrTime: result.elapsed,
+                paragraphs: result.recognized.map(p => p.text)
+              }
+            })
+          }).catch(() => {})
         }
       })
 
diff --git a/main/src/vision/HeistGemFinder.ts b/main/src/vision/HeistGemFinder.ts
new file mode 100644
index 00000000..40f9a723
--- /dev/null
+++ b/main/src/vision/HeistGemFinder.ts
@@ -0,0 +1,125 @@
+import fs from 'fs/promises'
+import Bmp from '@wokwi/bmp-ts'
+import * as Bindings from './wasm-bindings'
+import { cv, tessApi } from './wasm-bindings'
+import {
+  findNonZeroWeights,
+  groupWeightedPoints,
+  findLines,
+  hsvToU8,
+  timeIt,
+  ImageData
+} from './utils'
+
+const REFERENCE_HEIGHT = 600
+const TEMPLATE_THRESHOLD = 0.75
+const LINE_Y_DIST_TOLERANCE = 4
+const TEXT_HSV_MIN = hsvToU8(173, 31, 31)
+const TEXT_HSV_MAX = hsvToU8(180, 100, 100)
+
+interface OcrResult {
+  elapsed: number
+  matches: number
+  clusters: number
+  linesMin: number
+  linesMax: number
+  recognized: Array<{ text: string, confidence: number }>
+}
+
+export class HeistGemFinder {
+  private constructor (
+    private readonly needleMat: any,
+    private readonly hsvMin: any,
+    private readonly hsvMax: any
+  ) {}
+
+  static async create (binDir: string): Promise<HeistGemFinder> {
+    const needleImg = Bmp.decode(await fs.readFile(binDir + '/heist-lock.bmp'), { toRGBA: true })
+    const needleMat = Bindings.cvMatFromImage(needleImg)
+    cv.cvtColor(needleMat, needleMat, cv.COLOR_RGBA2GRAY)
+
+    const hsvMin = new cv.Mat(3, 1, cv.CV_8U)
+    hsvMin.data.set(TEXT_HSV_MIN)
+    const hsvMax = new cv.Mat(3, 1, cv.CV_8U)
+    hsvMax.data.set(TEXT_HSV_MAX)
+
+    return new HeistGemFinder(needleMat, hsvMin, hsvMax)
+  }
+
+  ocrScreenshot (screenshot: ImageData): OcrResult {
+    let elapsed = 0
+    const colorMat = Bindings.cvMatFromImage(screenshot)
+
+    const scale = screenshot.height / REFERENCE_HEIGHT
+    const graySize = new cv.Size(Math.floor(screenshot.width / scale), REFERENCE_HEIGHT)
+    const grayMat = new cv.Mat()
+    elapsed += timeIt(() => {
+      if (scale > 2.1) {
+        cv.resize(colorMat, grayMat, new cv.Size(graySize.width * 2, graySize.height * 2), 0, 0, cv.INTER_LINEAR)
+        cv.resize(grayMat, grayMat, new cv.Size(graySize.width, graySize.height), 0, 0, cv.INTER_LINEAR)
+      } else {
+        cv.resize(colorMat, grayMat, new cv.Size(graySize.width, graySize.height), 0, 0, cv.INTER_LINEAR)
+      }
+      cv.cvtColor(grayMat, grayMat, cv.COLOR_BGR2GRAY)
+    })
+
+    const { needleMat } = this
+    const matchesMat = grayMat
+    let matches: ReturnType<typeof findNonZeroWeights>
+    elapsed += timeIt(() => {
+      cv.matchTemplate(grayMat, needleMat, matchesMat, cv.TM_CCOEFF_NORMED)
+      cv.threshold(matchesMat, matchesMat, TEMPLATE_THRESHOLD, 1, cv.THRESH_TOZERO)
+      matches = findNonZeroWeights(matchesMat)
+    })
+    matchesMat.delete()
+    const clusteredMatches = groupWeightedPoints(matches!, Math.hypot(needleMat.cols, needleMat.rows))
+    const lines = findLines(clusteredMatches, LINE_Y_DIST_TOLERANCE)
+
+    const recognizedLines: OcrResult['recognized'] = []
+    for (const line of lines) {
+      const topLeft = new cv.Point(
+        (Math.min(line[0].x, line[1].x) + needleMat.cols) * scale,
+        (Math.min(line[0].y, line[1].y) - 1) * scale)
+      const bottomRight = new cv.Point(
+        Math.max(line[0].x, line[1].x) * scale,
+        (Math.max(line[0].y, line[1].y) + needleMat.rows) * scale)
+
+      const roiSize = new cv.Size(bottomRight.x - topLeft.x, bottomRight.y - topLeft.y)
+      const roiRect = new cv.Rect(topLeft, roiSize)
+      const roiColor = colorMat.roi(roiRect)
+      const roiHsv = new cv.Mat()
+
+      elapsed += timeIt(() => {
+        cv.cvtColor(roiColor, roiHsv, cv.COLOR_BGR2HSV_FULL)
+        cv.inRange(roiHsv, this.hsvMin, this.hsvMax, roiHsv)
+        cv.bitwise_not(roiHsv, roiHsv)
+      })
+      roiColor.delete()
+
+      Bindings.ocrSetImage(roiHsv.data, roiHsv.cols, roiHsv.rows, roiHsv.channels())
+      roiHsv.delete()
+      tessApi.SetVariable('tessedit_pageseg_mode', '7') // single line mode
+      elapsed += timeIt(() => {
+        tessApi.Recognize()
+      })
+      const text = tessApi.GetUTF8Text().trim()
+      const confidence = tessApi.MeanTextConf()
+      if (text.length > 0 && confidence > 30) {
+        recognizedLines.push({ text, confidence })
+      }
+    }
+    colorMat.delete()
+
+    const linesWeight = lines.flatMap(([p0, p1]) => ([p0.weight, p1.weight]))
+    const results = {
+      elapsed,
+      matches: matches!.length,
+      clusters: clusteredMatches.length,
+      linesMin: Math.min(...linesWeight),
+      linesMax: Math.max(...linesWeight),
+      recognized: recognizedLines
+    }
+    // console.log(results)
+    return results
+  }
+}
diff --git a/main/src/vision/link-main.ts b/main/src/vision/link-main.ts
new file mode 100644
index 00000000..101e364f
--- /dev/null
+++ b/main/src/vision/link-main.ts
@@ -0,0 +1,42 @@
+import { Worker } from 'worker_threads'
+import * as Comlink from 'comlink'
+import nodeEndpoint from 'comlink/dist/umd/node-adapter'
+import type { WorkerAPI } from './link-worker'
+import type { ImageData } from './utils'
+import { app } from 'electron'
+import path from 'path'
+
+export class OcrWorker {
+  private binDir = path.join(app.getPath('userData'), 'apt-data/cv-ocr')
+  private api: Comlink.Remote<WorkerAPI>
+  private lang = ''
+
+  private constructor () {
+    const worker = new Worker(__dirname + '/vision.js')
+    this.api = Comlink.wrap<WorkerAPI>(nodeEndpoint(worker))
+  }
+
+  static async create () {
+    const worker = new OcrWorker()
+    try {
+      await worker.api.init(worker.binDir)
+    } catch {}
+    return worker
+  }
+
+  async updateOptions (lang: string) {
+    try {
+      if (lang !== this.lang) {
+        await this.api.changeLanguage(lang, this.binDir)
+      }
+    } catch {} finally {
+      this.lang = lang
+    }
+  }
+
+  async findHeistGems (image: ImageData) {
+    const result = await this.api.findHeistGems(
+      Comlink.transfer(image, [image.data.buffer]))
+    return result
+  }
+}
diff --git a/main/src/vision/link-worker.ts b/main/src/vision/link-worker.ts
new file mode 100644
index 00000000..2cf24369
--- /dev/null
+++ b/main/src/vision/link-worker.ts
@@ -0,0 +1,28 @@
+import { parentPort } from 'worker_threads'
+import * as Comlink from 'comlink'
+import nodeEndpoint from 'comlink/dist/umd/node-adapter'
+import * as Bindings from './wasm-bindings'
+import { HeistGemFinder } from './HeistGemFinder'
+import { ImageData } from './utils'
+
+let _heistGems: HeistGemFinder
+let _changeLangPromise = Promise.resolve()
+
+const WorkerBody = {
+  async init (binDir: string) {
+    await Bindings.init(binDir)
+    _heistGems = await HeistGemFinder.create(binDir)
+  },
+  async changeLanguage (lang: string, binDir: string) {
+    await _changeLangPromise
+    _changeLangPromise = Bindings.changeLanguage(lang, binDir)
+    await _changeLangPromise
+  },
+  async findHeistGems (screenshot: ImageData) {
+    await _changeLangPromise
+    return _heistGems.ocrScreenshot(screenshot)
+  }
+}
+Comlink.expose(WorkerBody, nodeEndpoint(parentPort!))
+
+export type WorkerAPI = Comlink.Remote<typeof WorkerBody>
diff --git a/main/src/vision/utils.ts b/main/src/vision/utils.ts
new file mode 100644
index 00000000..df91ce53
--- /dev/null
+++ b/main/src/vision/utils.ts
@@ -0,0 +1,75 @@
+import { cv } from './wasm-bindings'
+
+export interface ImageData {
+  width: number
+  height: number
+  data: Uint8Array
+}
+
+export interface WeightedPoint {
+  x: number
+  y: number
+  weight: number
+}
+
+export type LinePoints = [WeightedPoint, WeightedPoint]
+
+export function findNonZeroWeights (matchResult: any): WeightedPoint[] {
+  const locations = new cv.Mat()
+  cv.findNonZero(matchResult, locations)
+  const weights = Array<WeightedPoint>(locations.rows)
+  for (let i = 0; i < locations.rows; ++i) {
+    const x = locations.intAt(i, 0)
+    const y = locations.intAt(i, 1)
+    const weight = matchResult.floatAt(y, x)
+    weights[i] = { x, y, weight }
+  }
+  locations.delete()
+  return weights
+}
+
+export function groupWeightedPoints (weights: WeightedPoint[], radius: number): WeightedPoint[] {
+  // similar to non-maximum suppression
+  const maxWeighted: WeightedPoint[] = []
+  for (const point of weights) {
+    const closeIdx = maxWeighted.findIndex(maxPoint => {
+      const dist = Math.hypot(point.x - maxPoint.x, point.y - maxPoint.y)
+      return dist < radius
+    })
+    if (closeIdx === -1) {
+      maxWeighted.push(point)
+    } else if (point.weight > maxWeighted[closeIdx].weight) {
+      maxWeighted[closeIdx] = point
+    }
+  }
+  return maxWeighted
+}
+
+export function findLines (points: WeightedPoint[], yTolerance: number): LinePoints[] {
+  points.sort((a, b) => a.x - b.x)
+  const lines: LinePoints[] = []
+  for (let idxA = 0; idxA < points.length; ++idxA) {
+    for (let idxB = idxA + 1; idxB < points.length; ++idxB) {
+      const pointA = points[idxA]
+      const pointB = points[idxB]
+      if (Math.abs(pointA.y - pointB.y) > yTolerance) continue
+      lines.push([pointA, pointB])
+      break
+    }
+  }
+  return lines
+}
+
+export function hsvToU8 (h: number, s: number, v: number) {
+  return [
+    Math.round(h * 255 / 360),
+    Math.round(s * 255 / 100),
+    Math.round(v * 255 / 100),
+  ]
+}
+
+export function timeIt (syncFn: () => void): number {
+  const startTime = performance.now()
+  syncFn()
+  return performance.now() - startTime
+}
diff --git a/main/src/vision/wasm-bindings.ts b/main/src/vision/wasm-bindings.ts
new file mode 100644
index 00000000..df70a52b
--- /dev/null
+++ b/main/src/vision/wasm-bindings.ts
@@ -0,0 +1,56 @@
+import fs from 'fs/promises'
+import type { ImageData } from './utils'
+
+let tessModule: any
+export let tessApi: any
+export let cv: any
+
+const langMap = new Map([
+  ['en', 'eng'],
+  ['ru', 'rus'],
+  // ['cmn-Hant', 'chi_tra'],
+])
+
+export async function init (binDir: string) {
+  if (process.platform !== 'win32') {
+    // so far only tested on Windows with BGRA images
+    throw new Error('Unsupported platform')
+  }
+
+  const tessInstantiate = (await import('file://' + binDir + '/tesseract-core-simd.js')).default
+  tessModule = await tessInstantiate()
+  tessApi = new tessModule.TessBaseAPI()
+
+  const cvPromise = (await import('file://' + binDir + '/opencv.js')).default
+  cv = await cvPromise
+}
+
+export async function changeLanguage (lang: string, binDir: string) {
+  if (!langMap.has(lang)) {
+    throw new Error('Unsupported language')
+  }
+  lang = langMap.get(lang)!
+  const langData = await fs.readFile(binDir + `/${lang}.traineddata`)
+  tessModule.FS.writeFile(`${lang}.traineddata`, langData)
+  if (tessApi.Init(null, lang, tessModule.OEM_DEFAULT)) {
+    throw new Error('Could not initialize tesseract.')
+  }
+  tessModule.FS.unlink(`${lang}.traineddata`)
+}
+
+export function ocrSetImage (data: Uint8Array, width: number, height: number, bpp: number) {
+  const imgPtr = tessModule._malloc(data.byteLength)
+  tessModule.HEAPU8.set(data, imgPtr)
+  if (bpp === 0) {
+    tessApi.SetImage(imgPtr, width, height, 0, Math.ceil(width / 8))
+  } else {
+    tessApi.SetImage(imgPtr, width, height, bpp, width * bpp)
+  }
+  tessModule._free(imgPtr)
+}
+
+export function cvMatFromImage (img: ImageData) {
+  const mat = new cv.Mat(img.height, img.width, cv.CV_8UC4)
+  mat.data.set(img.data)
+  return mat
+}
diff --git a/main/src/windowing/GameWindow.ts b/main/src/windowing/GameWindow.ts
index 27615fa7..7e81bd95 100644
--- a/main/src/windowing/GameWindow.ts
+++ b/main/src/windowing/GameWindow.ts
@@ -44,4 +44,8 @@ export class GameWindow extends EventEmitter {
       cb(e.hasAccess)
     })
   }
+
+  screenshot () {
+    return OverlayController.screenshot()
+  }
 }
diff --git a/main/yarn.lock b/main/yarn.lock
index 0831f590..410047f2 100644
--- a/main/yarn.lock
+++ b/main/yarn.lock
@@ -216,6 +216,11 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-16.18.6.tgz#87846192fd51b693368fad3e99123169225621d4"
   integrity sha512-vmYJF0REqDyyU0gviezF/KHq/fYaUbFhkcNbQCuPGFQj6VTbXuHZoxs/Y7mutWe73C8AC6l9fFu8mSYiBAqkGA==
 
+"@types/node@^10.11.7":
+  version "10.17.60"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-10.17.60.tgz#35f3d6213daed95da7f0f73e75bcc6980e90597b"
+  integrity sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==
+
 "@types/plist@^3.0.1":
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/@types/plist/-/plist-3.0.2.tgz#61b3727bba0f5c462fe333542534a0c3e19ccb01"
@@ -253,6 +258,13 @@
   dependencies:
     "@types/yargs-parser" "*"
 
+"@wokwi/bmp-ts@^3.0.0":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/@wokwi/bmp-ts/-/bmp-ts-3.0.0.tgz#826b635268f32c69784089a1b56843ffc1dfb455"
+  integrity sha512-fBoRZQEd8RHdFq4gaS6eU69fYZjqGIz8myYPLm0gJSosxqHWpge8/3+6yCyc8b8afcb/ptjSlnEVP93t00fWDQ==
+  dependencies:
+    "@types/node" "^10.11.7"
+
 abort-controller@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392"
@@ -676,6 +688,11 @@ combined-stream@^1.0.8:
   dependencies:
     delayed-stream "~1.0.0"
 
+comlink@^4.3.1:
+  version "4.3.1"
+  resolved "https://registry.yarnpkg.com/comlink/-/comlink-4.3.1.tgz#0c6b9d69bcd293715c907c33fe8fc45aecad13c5"
+  integrity sha512-+YbhUdNrpBZggBAHWcgQMLPLH1KDF3wJpeqrCKieWQ8RL7atmgsgTQko1XEBK6PsecfopWNntopJ+ByYG1lRaA==
+
 commander@2.9.0:
   version "2.9.0"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4"
@@ -937,10 +954,10 @@ electron-osx-sign@^0.6.0:
     minimist "^1.2.0"
     plist "^3.0.1"
 
-electron-overlay-window@3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/electron-overlay-window/-/electron-overlay-window-3.0.0.tgz#d34c4d97190f26a99d46271a61031faefdf10fa3"
-  integrity sha512-7mexgNUEgAJaxorG3aa1k9YIr1V9LkMjt7ZhPfG4Eq1AfQEgWyfHUrndQDw75/JohnyNFPzG/fVaCimDi4mcVA==
+electron-overlay-window@3.2.0:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/electron-overlay-window/-/electron-overlay-window-3.2.0.tgz#caedec2b9418e8260c9b9debdda4ab00e242f76d"
+  integrity sha512-EqUucgLJw3QnnMn0S2yEDC3MVcAEE7Aa87VSdhnepeCvKKKc7nwgo/Trfl6ritLXrbH3lSIdOGdMcS9g3M4jGA==
   dependencies:
     node-gyp-build "4.x.x"
     throttle-debounce "5.x.x"
diff --git a/renderer/package.json b/renderer/package.json
index fed9be08..e3dd018e 100644
--- a/renderer/package.json
+++ b/renderer/package.json
@@ -16,6 +16,7 @@
     "apexcharts": "^3.23.1",
     "dot-prop": "7.x.x",
     "fast-deep-equal": "^3.1.3",
+    "fastest-levenshtein": "^1.0.16",
     "luxon": "3.x.x",
     "object-hash": "^3.0.0",
     "sockette": "^2.0.6",
diff --git a/renderer/src/assets/data/index.ts b/renderer/src/assets/data/index.ts
index dfc58fd8..52b78def 100644
--- a/renderer/src/assets/data/index.ts
+++ b/renderer/src/assets/data/index.ts
@@ -12,6 +12,8 @@ export let ITEM_BY_TRANSLATED = (ns: BaseType['namespace'], name: string): BaseT
 export let ITEM_BY_REF = (ns: BaseType['namespace'], name: string): BaseType[] | undefined => undefined
 export let ITEMS_ITERATOR = function * (includes: string, andIncludes?: string[]): Generator<BaseType> {}
 
+export let ALTQ_GEM_NAMES = function * (): Generator<string> {}
+
 export let STAT_BY_MATCH_STR = (name: string): { matcher: StatMatcher, stat: Stat } | undefined => undefined
 export let STAT_BY_REF = (name: string): Stat | undefined => undefined
 export let STATS_ITERATOR = function * (includes: string, andIncludes?: string[]): Generator<Stat> {}
@@ -52,6 +54,24 @@ function ndjsonFindLines<T> (ndjson: string) {
   }
 }
 
+function itemNamesFromLines (items: Generator<BaseType>) {
+  let cached = ''
+  return function * (): Generator<string> {
+    if (!cached.length) {
+      for (const item of items) {
+        cached += (item.name + '\n')
+      }
+    }
+
+    let start = 0
+    while (start !== cached.length) {
+      const end = cached.indexOf('\n', start)
+      yield cached.slice(start, end)
+      start = end + 1
+    }
+  }
+}
+
 async function loadItems (language: string) {
   const ndjson = await (await fetch(`${import.meta.env.BASE_URL}data/${language}/items.ndjson`)).text()
   const INDEX_WIDTH = 2
@@ -80,6 +100,7 @@ async function loadItems (language: string) {
   ITEM_BY_TRANSLATED = commonFind(indexNames, 'name')
   ITEM_BY_REF = commonFind(indexRefNames, 'refName')
   ITEMS_ITERATOR = ndjsonFindLines<BaseType>(ndjson)
+  ALTQ_GEM_NAMES = itemNamesFromLines(ITEMS_ITERATOR('altQuality":["Anomalous'))
 }
 
 async function loadStats (language: string) {
diff --git a/renderer/src/web/Config.ts b/renderer/src/web/Config.ts
index 45b26060..420ca024 100644
--- a/renderer/src/web/Config.ts
+++ b/renderer/src/web/Config.ts
@@ -499,9 +499,11 @@ function upgradeConfig (_config: Config): Config {
 
   if (config.configVersion < 15) {
     const priceCheck = config.widgets.find(w => w.wmType === 'price-check') as widget.PriceCheckWidget
-
     priceCheck.builtinBrowser = false
 
+    const itemSearch = config.widgets.find(w => w.wmType === 'item-search') as widget.ItemSearchWidget
+    itemSearch.ocrGemsKey = null
+
     config.configVersion = 15
   }
 
@@ -610,6 +612,15 @@ function getConfigForHost (): HostConfig {
           }
         })
       }
+    } else if (widget.wmType === 'item-search') {
+      const itemSearch = widget as widget.ItemSearchWidget
+      if (itemSearch.ocrGemsKey) {
+        actions.push({
+          shortcut: itemSearch.ocrGemsKey,
+          keepModKeys: true,
+          action: { type: 'ocr-text', target: 'heist-gems' }
+        })
+      }
     }
   }
 
@@ -622,6 +633,7 @@ function getConfigForHost (): HostConfig {
     overlayKey: config.overlayKey,
     disableUpdateDownload: config.disableUpdateDownload,
     logLevel: config.logLevel,
-    windowTitle: config.windowTitle
+    windowTitle: config.windowTitle,
+    language: config.language
   }
 }
diff --git a/renderer/src/web/overlay/WidgetItemSearch.vue b/renderer/src/web/overlay/WidgetItemSearch.vue
index d107a5ac..5a337d40 100644
--- a/renderer/src/web/overlay/WidgetItemSearch.vue
+++ b/renderer/src/web/overlay/WidgetItemSearch.vue
@@ -1,10 +1,11 @@
 <template>
-  <widget :config="config" move-handles="tl" readonly :removable="false">
-    <div class="widget-default-style p-1" style="min-width: 24rem;">
-      <div v-if="starred.length"
-        class="mb-1 flex gap-x-2 py-1 pl-1 pr-2 bg-gray-800 rounded">
-        <div v-for="item in starred" :key="item.name"
-          class="flex flex-col">
+  <widget :config="config" :move-handles="['tl', 'bl']" :removable="false" :inline-edit="false">
+    <div class="widget-default-style flex flex-col p-1 gap-1" style="min-width: 24rem;">
+      <transition-group v-if="starred.length" tag="div"
+        :enter-active-class="$style.starredItemEnter"
+        class="flex gap-x-1 py-1 pr-1 bg-gray-800 rounded">
+        <div v-for="item in starred" :key="item.name + item.discr"
+          :class="$style.starredItem">
           <item-quick-price
             :item-img="item.icon"
             :price="item.price"
@@ -14,59 +15,67 @@
           <div v-if="item.discr"
             class="ml-1 truncate" style="max-width: 7rem;">{{ t(item.discr) }}</div>
         </div>
-      </div>
-      <div class="flex gap-x-1 bg-gray-800 p-1 rounded-t">
-        <input type="text" :placeholder="t('Search by name…')" class="rounded bg-gray-900 px-1 flex-1"
-          v-model="searchValue">
-        <button @click="clearItems" class="btn"><i class="fas fa-times" /> {{ t('Reset items') }}</button>
-      </div>
-      <div class="flex bg-gray-800 gap-x-2 px-2 mb-px1 py-1">
-        <span>{{ t('Heist target:') }}</span>
-        <div class="flex gap-x-1">
-          <button :class="{ 'border': (typeFilter === 'gem') }" class="rounded px-2 bg-gray-900"
-            @click="typeFilter = 'gem'">{{ t('Skill Gem') }}</button>
-          <button :class="{ 'border': (typeFilter === 'replica') }" class="rounded px-2 bg-gray-900"
-            @click="typeFilter = 'replica'">{{ t('Replicas') }}, <span class="line-through text-gray-600">Base items</span></button>
+      </transition-group>
+      <ui-timeout v-if="!showSearch"
+        ref="showTimeout"
+        @timeout="makeInvisible"
+        class="self-center" :ms="4000" />
+      <div v-else class="bg-gray-800 rounded">
+        <div class="flex gap-x-1 p-1">
+          <input type="text" :placeholder="t('Search by name…')" class="rounded bg-gray-900 px-1 flex-1"
+            v-model="searchValue">
+          <button @click="clearItems" class="btn"><i class="fas fa-times" /> {{ t('Reset items') }}</button>
         </div>
-      </div>
-      <div class="flex flex-col bg-gray-800 rounded-b">
-        <div v-for="item in (results || [])" :key="item.name">
-          <div class="flex" :class="$style.itemWrapper">
-            <div class="w-8 h-8 flex items-center justify-center">
-              <img :src="item.icon" class="max-w-full max-h-full overflow-hidden">
-            </div>
-            <div>
-              <div class="h-8 flex items-center px-1">{{ item.name }}</div>
-              <div v-if="item.gem" class="flex gap-x-1">
-                <button v-for="altQuality in item.gem.altQuality" :key="altQuality"
-                  @click="selectItem(item, { altQuality })"
-                  >{{ t(altQuality) }}</button>
+        <div class="flex gap-x-2 px-2 mb-px1 py-1">
+          <span>{{ t('Heist target:') }}</span>
+          <div class="flex gap-x-1">
+            <button :class="{ 'border': (typeFilter === 'gem') }" class="rounded px-2 bg-gray-900"
+              @click="typeFilter = 'gem'">{{ t('Skill Gem') }}</button>
+            <button :class="{ 'border': (typeFilter === 'replica') }" class="rounded px-2 bg-gray-900"
+              @click="typeFilter = 'replica'">{{ t('Replicas') }}, <span class="line-through text-gray-600">Base items</span></button>
+          </div>
+        </div>
+        <div class="flex flex-col">
+          <div v-for="item in (results || [])" :key="item.name">
+            <div class="flex" :class="$style.itemWrapper">
+              <div class="w-8 h-8 flex items-center justify-center">
+                <img :src="item.icon" class="max-w-full max-h-full overflow-hidden">
               </div>
-              <div v-else-if="item.unique" class="flex gap-x-1">
-                <button  @click="selectItem(item, { unique: true })"
-                  >{{ t('Select') }}</button>
+              <div>
+                <div class="h-8 flex items-center px-1">{{ item.name }}</div>
+                <div v-if="item.gem" class="flex gap-x-1">
+                  <button v-for="altQuality in item.gem.altQuality" :key="altQuality"
+                    @click="selectItem(item, { altQuality })"
+                    >{{ t(altQuality) }}</button>
+                </div>
+                <div v-else-if="item.unique" class="flex gap-x-1">
+                  <button  @click="selectItem(item, { unique: true })"
+                    >{{ t('Select') }}</button>
+                </div>
               </div>
             </div>
           </div>
+          <div v-if="results === false"
+            class="text-center p-8 max-w-xs"><i class="fas fa-search" /> {{ t('too_many') }}</div>
+          <div v-else-if="!results.length"
+            class="text-center p-8 max-w-xs"><i class="fas fa-exclamation-triangle" /> {{ t('not_found') }}</div>
         </div>
-        <div v-if="results === false"
-          class="text-center p-8 max-w-xs"><i class="fas fa-search" /> {{ t('too_many') }}</div>
-        <div v-else-if="!results.length"
-          class="text-center p-8 max-w-xs"><i class="fas fa-exclamation-triangle" /> {{ t('not_found') }}</div>
       </div>
     </div>
   </widget>
 </template>
 
 <script lang="ts">
-import { defineComponent, PropType, shallowRef, ref, computed } from 'vue'
+import { defineComponent, PropType, shallowRef, ref, computed, nextTick, inject } from 'vue'
 import { useI18n } from 'vue-i18n'
-import { ItemSearchWidget } from './interfaces'
+import { distance } from 'fastest-levenshtein'
+import { ItemSearchWidget, WidgetManager } from './interfaces'
 import ItemQuickPrice from '@/web/ui/ItemQuickPrice.vue'
 import Widget from './Widget.vue'
-import { BaseType, ITEMS_ITERATOR } from '@/assets/data'
+import { BaseType, ITEMS_ITERATOR, CLIENT_STRINGS as _$, ALTQ_GEM_NAMES, ITEM_BY_TRANSLATED } from '@/assets/data'
 import { AppConfig } from '@/web/Config'
 import { findPriceByQuery, autoCurrency } from '@/web/background/Prices'
+import { Host } from '@/web/background/IPC'
 
 interface SelectedItem {
   name: string
@@ -80,6 +89,11 @@ function useSelectedItems () {
   const items = ref<SelectedItem[]>([])
 
   function addItem (newItem: SelectedItem) {
+    if (items.value.some(item =>
+      item.name === newItem.name &&
+      item.discr === newItem.discr
+    )) return false
+
     if (items.value.length < 5) {
       items.value.push(newItem)
       items.value.sort((a, b) => {
@@ -88,6 +102,7 @@ function useSelectedItems () {
     } else {
       items.value = [newItem]
     }
+    return true
   }
 
   function clearItems () {
@@ -133,6 +148,35 @@ function findItems (opts: {
   return out.slice(0, MAX_RESULTS_VISIBLE)
 }
 
+function fuzzyFindHeistGem (badStr: string) {
+  badStr = badStr.toLowerCase()
+
+  const qualities = [
+    ['Anomalous', _$.QUALITY_ANOMALOUS.toString().slice(2, -2)],
+    ['Divergent', _$.QUALITY_DIVERGENT.toString().slice(2, -2)],
+    ['Phantasmal', _$.QUALITY_PHANTASMAL.toString().slice(2, -2)]
+  ]
+
+  let bestMatch: { name: string, altQuality: string }
+  let minDist = Infinity
+  for (const name of ALTQ_GEM_NAMES()) {
+    for (const [altQuality, reStr] of qualities) {
+      const exactStr = reStr.replace('(.*)', name).toLowerCase()
+      if (Math.abs(exactStr.length - badStr.length) > 5) {
+        continue
+      }
+
+      const dist = distance(badStr, exactStr)
+      if (dist < minDist) {
+        bestMatch = { name, altQuality }
+        if (dist === 0) return bestMatch
+        minDist = dist
+      }
+    }
+  }
+  return bestMatch!
+}
+
 export default defineComponent({
   components: { Widget, ItemQuickPrice },
   props: {
@@ -141,13 +185,34 @@ export default defineComponent({
       required: true
     }
   },
-  setup () {
+  setup (props) {
+    const wm = inject<WidgetManager>('wm')!
+    const { t } = useI18n()
+
+    const showTimeout = shallowRef<{ reset:() => void } | null>(null)
+
+    nextTick(() => {
+      props.config.wmFlags = ['invisible-on-blur']
+    })
+
     const searchValue = shallowRef('')
     const { items: starred, addItem, clearItems } = useSelectedItems()
 
     const typeFilter = shallowRef<'gem' | 'replica'>('gem')
 
-    function selectItem (item: BaseType, opts: { altQuality?: string, unique?: true }) {
+    Host.onEvent('MAIN->CLIENT::ocr-text', (e) => {
+      if (e.target !== 'heist-gems') return
+
+      for (const para of e.paragraphs) {
+        const res = fuzzyFindHeistGem(para)
+        selectItem(
+          ITEM_BY_TRANSLATED('GEM', res.name)![0],
+          { altQuality: res.altQuality, withTimeout: true }
+        )
+      }
+    })
+
+    function selectItem (item: BaseType, opts: { altQuality?: string, unique?: true, withTimeout?: true }) {
       let price: ReturnType<typeof findPriceByQuery>
       if (opts.altQuality) {
         price = findPriceByQuery({
@@ -162,18 +227,20 @@ export default defineComponent({
           variant: item.unique!.base
         })
       }
-      addItem({
+      const isAdded = addItem({
         name: item.name,
         icon: item.icon,
         discr: opts.altQuality,
         chaos: price?.chaos,
         price: (price != null) ? autoCurrency(price.chaos) : undefined
       })
+      if (isAdded && opts.withTimeout) {
+        showTimeout.value?.reset()
+        props.config.wmFlags = []
+      }
       searchValue.value = ''
     }
 
-    const { t } = useI18n()
-
     return {
       t,
       searchValue,
@@ -198,8 +265,16 @@ export default defineComponent({
         }
       }),
       selectItem,
-      clearItems,
-      starred
+      clearItems () {
+        clearItems()
+        props.config.wmFlags = ['invisible-on-blur']
+      },
+      starred,
+      showSearch: wm.active,
+      showTimeout,
+      makeInvisible () {
+        props.config.wmFlags = ['invisible-on-blur']
+      }
     }
   }
 })
@@ -225,6 +300,21 @@ export default defineComponent({
     @apply bg-gray-700;
   }
 }
+
+.starredItem {
+  display: flex;
+  flex-direction: column;
+  @apply rounded px-1;
+}
+
+@keyframes starredItemEnter {
+  0% { @apply bg-transparent; }
+  50% { @apply bg-gray-700; }
+  100% { @apply bg-transparent; }
+}
+.starredItemEnter {
+  animation: starredItemEnter 0.8s linear;
+}
 </style>
 
 <i18n>
diff --git a/renderer/src/web/overlay/widgets.ts b/renderer/src/web/overlay/widgets.ts
index fa60c350..a2e0613a 100644
--- a/renderer/src/web/overlay/widgets.ts
+++ b/renderer/src/web/overlay/widgets.ts
@@ -85,4 +85,5 @@ export interface ImageStripWidget extends Widget {
 
 export interface ItemSearchWidget extends Widget {
   anchor: Anchor
+  ocrGemsKey: string | null
 }
diff --git a/renderer/src/web/settings/SettingsWindow.vue b/renderer/src/web/settings/SettingsWindow.vue
index 7ce5f79e..3ddcdf35 100644
--- a/renderer/src/web/settings/SettingsWindow.vue
+++ b/renderer/src/web/settings/SettingsWindow.vue
@@ -65,6 +65,7 @@ import SettingsDebug from './debug.vue'
 import SettingsMaps from './maps/maps.vue'
 import SettingsStashSearch from './stash-search.vue'
 import SettingsStopwatch from './stopwatch.vue'
+import SettingsItemSearch from './item-search.vue'
 
 function shuffle<T> (array: T[]): T[] {
   let currentIndex = array.length
@@ -190,6 +191,8 @@ function menuByType (type?: string) {
       return [[SettingsItemcheck, SettingsMaps]]
     case 'price-check':
       return [[SettingsPricecheck]]
+    case 'item-search':
+      return [[SettingsItemSearch]]
     default:
       return [
         [SettingsHotkeys, SettingsChat],
@@ -351,6 +354,7 @@ function flatJoin<T, J> (arr: T[][], joinEl: () => J) {
     "Price check": "Прайс-чек",
     "Maps": "Карты",
     "Item info": "Проверка предмета",
+    "Item search": "Поиск предметов",
     "Debug": "Debug",
     "Quit": "Выход",
     "Chat": "Чат",
diff --git a/renderer/src/web/settings/item-search.vue b/renderer/src/web/settings/item-search.vue
new file mode 100644
index 00000000..453802d9
--- /dev/null
+++ b/renderer/src/web/settings/item-search.vue
@@ -0,0 +1,38 @@
+<template>
+  <div class="max-w-md p-2">
+    <div class="mb-4 flex">
+      <label class="flex-1">{{ t('Perform an OCR for a Skill Gem') }}</label>
+      <hotkey-input v-model="ocrGemsKey" class="w-48" />
+    </div>
+  </div>
+</template>
+
+<script lang="ts">
+import { defineComponent } from 'vue'
+import { useI18n } from 'vue-i18n'
+import { configProp, configModelValue, findWidget } from './utils'
+import { ItemSearchWidget } from '@/web/overlay/interfaces'
+import HotkeyInput from './HotkeyInput.vue'
+
+export default defineComponent({
+  name: 'Item search',
+  components: { HotkeyInput },
+  props: configProp(),
+  setup (props) {
+    const { t } = useI18n()
+
+    return {
+      t,
+      ocrGemsKey: configModelValue(() => findWidget<ItemSearchWidget>('item-search', props.config)!, 'ocrGemsKey')
+    }
+  }
+})
+</script>
+
+<i18n>
+{
+  "ru": {
+    "Perform an OCR for a Skill Gem": "Выполнить OCR для гема"
+  }
+}
+</i18n>
diff --git a/renderer/yarn.lock b/renderer/yarn.lock
index 142fd8f6..c5cebcc9 100644
--- a/renderer/yarn.lock
+++ b/renderer/yarn.lock
@@ -1218,6 +1218,11 @@ fast-levenshtein@^2.0.6:
   resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917"
   integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==
 
+fastest-levenshtein@^1.0.16:
+  version "1.0.16"
+  resolved "https://registry.yarnpkg.com/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz#210e61b6ff181de91ea9b3d1b84fdedd47e034e5"
+  integrity sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==
+
 fastq@^1.6.0:
   version "1.13.0"
   resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.13.0.tgz#616760f88a7526bdfc596b7cab8c18938c36b98c"