update parser #67

* uhhhhhhhhhhhh, maybe pseudo will get support before GGG

* psesudo only ele

* remove git ignored files

* fix ignore

renderer/package.json

@@ -2,6 +2,7 @@
   "name": "exiled-exchange-2",
   "version": "0.0.0",
   "private": true,
+  "type": "module",
   "scripts": {
     "dev": "vite",
     "lint": "eslint --ext .ts,.vue src",

renderer/public/.DS_Store/.gitignore

@@ -131,6 +131,8 @@ dist
 
 __pycache__
 
+__pycache__
+
 vendor/json-api/*.json
 vendor/json-api/en/*.json
 vendor/json-api/ru/*.json
@@ -142,16 +144,18 @@ vendor/client/descriptions/*
 
 vendor/client/tables/*
 !vendor/client/tables/index.ts
+
 !vendor/client/tables/ArmourTypesOverride.json
+!vendor/client/tables/en/ArmourTypesOverride.json
+!vendor/client/tables/ru/ArmourTypesOverride.json
+!vendor/client/tables/ko/ArmourTypesOverride.json
+!vendor/client/tables/cmn-Hant/ArmourTypesOverride.json
+
 !vendor/client/tables/en/.gitkeep
 !vendor/client/tables/ru/.gitkeep
 !vendor/client/tables/ko/.gitkeep
 !vendor/client/tables/cmn-Hant/.gitkeep
 
-!vendor/client/tables/en/ArmourTypesOverride.json
-!vendor/client/tables/ru/ArmourTypesOverride.json
-!vendor/client/tables/ko/ArmourTypesOverride.json
-!vendor/client/tables/cmn-Hant/ArmourTypesOverride.json
 
 
 
@@ -162,8 +166,11 @@ vendor/client/tables/*
 !vendor/json-api/cmn-Hant/.gitkeep
 
 
-EXPORT/tables/English/*.json
+EXPORT/tables/**/*.json
-EXPORT/tables/Korean/*.json
+vendor/client/tables/*.json
-EXPORT/tables/Russian/*.json
+!vendor/client/tables/ArmourTypesOverride.json
-EXPORT/tables/Traditional Chinese/*.json
+vendor/json-api/*.json
-
+vendor/json-api/en/*.json
+vendor/json-api/ru/*.json
+vendor/json-api/ko/*.json
+vendor/json-api/cmn-Hant/*.json

renderer/public/.DS_Store/EXPORT/config.json

@@ -2,7 +2,9 @@
   "steam": "C:\\Program Files (x86)\\Steam\\steamapps\\common\\Path of Exile 2",
   "translations": [
     "English",
-    "Russian"
+    "Russian",
+    "Korean",
+    "Traditional Chinese"
   ],
   "tables": [
     {

renderer/public/.DS_Store/copy-tables.sh

@@ -0,0 +1,19 @@
+
+echo "Copying files from EXPORT/tables/English to data/en"
+cp -R EXPORT/tables/English/* vendor/client/tables/en/
+
+echo "Copying files from EXPORT/tables/Russian to data/ru"
+cp -R EXPORT/tables/Russian/* vendor/client/tables/ru/
+
+echo "Copying files from EXPORT/tables/Korean to data/ko"
+cp -R EXPORT/tables/Korean/* vendor/client/tables/ko/
+
+echo "Copying files from EXPORT/tables/Traditional Chinese to data/cmn-Hant"
+cp -R EXPORT/tables/Traditional\ Chinese/* vendor/client/tables/cmn-Hant/
+
+echo "Copying not generated files from english to other languages"
+
+# Copy without overwriting English -> data/ru
+cp -n vendor/client/tables/en/* vendor/client/tables/ru/
+cp -n vendor/client/tables/en/* vendor/client/tables/ko/
+cp -n vendor/client/tables/en/* vendor/client/tables/cmn-Hant/

renderer/public/.DS_Store/package.json

@@ -25,7 +25,7 @@
     "start": "node dist/index.js",
     "items": "ts-node items.ts",
     "stats": "ts-node stats.ts",
-    "copy-files": "cp -R EXPORT/tables/English/* vendor/client/tables/",
+    "copy-files": "sh ./copy-tables.sh",
     "push-ndjson": "bash ./push-ndjson.sh",
     "anon-vodka-python": "python ./vendor/client/parser.py",
     "langs-anon-vodka-python": "python ./vendor/client/parserRunner.py",

renderer/public/.DS_Store/vendor/client/parser.py

@@ -20,6 +20,12 @@ import urllib.parse
 
 import requests
 
+LANG_CODES_TO_NAMES = {
+    "ru": "Russian",
+    "ko": "Korean",
+    "cmn-Hant": "Traditional Chinese",
+}
+
 
 class Parser:
     def get_script_dir(self):
@@ -27,7 +33,9 @@ class Parser:
         return os.path.dirname(os.path.realpath(__file__))
 
     def load_file(self, file):
-        return json.loads(open(f"{self.base_dir}/{self.lang}/{file}.json").read())
+        return json.loads(
+            open(f"{self.base_dir}/{self.lang}/{file}.json", encoding="utf-8").read()
+        )
 
     def __init__(self, lang="en"):
         self.lang = lang
@@ -47,13 +55,19 @@ class Parser:
         self.mods_file = self.load_file("Mods")
         # NOTE: could need to add local here?
         self.trade_stats = json.loads(
-            open(f"{self.cwd}/../json-api/en/stats.json").read()
+            open(
+                f"{self.cwd}/../json-api/{self.lang}/stats.json", encoding="utf-8"
+            ).read()
         )  # content of https://www.pathofexile.com/api/trade2/data/stats
         self.trade_items = json.loads(
-            open(f"{self.cwd}/../json-api/en/items.json").read()
+            open(
+                f"{self.cwd}/../json-api/{self.lang}/items.json", encoding="utf-8"
+            ).read()
         )  # content of https://www.pathofexile.com/api/trade2/data/items
         self.trade_exchange_items = json.loads(
-            open(f"{self.cwd}/../json-api/en/static.json").read()
+            open(
+                f"{self.cwd}/../json-api/{self.lang}/static.json", encoding="utf-8"
+            ).read()
         )  # content of https://www.pathofexile.com/api/trade2/data/static
 
         self.items = {}
@@ -115,9 +129,15 @@ class Parser:
 
                 self.stats_trade_ids[text][type].append(id)
 
-    def parse_mod(self, id, english):
+    def parse_mod(self, id, english, log=False):
+        if log:
+            print(
+                "===================================================================="
+            )
+            print(f"[id:{id}] {english}")
         matchers = []
         ref = None
+
         for lang in english:
             lang = self.convert_stat_name(lang)
 
@@ -151,11 +171,38 @@ class Parser:
                 "matchers": matchers,
             }
 
+    def parse_translation_line(self, stats_translations, i, id, log=False):
+        if log:
+            print(
+                "===================================================================="
+            )
+            print(f"[i:{i}, id:{id}] {stats_translations[i]}")
+            print(f"[i:{i+1}, id:{id}] {stats_translations[i+1]}")
+            print(f"[i:{i+2}, id:{id}] {stats_translations[i+2]}")
+            print(f"[i:{i+3}, id:{id}] {stats_translations[i+3]}")
+        line = stats_translations[i + 3].strip()  # skip first 2 characters
+        start = line.find('"')
+        end = line.rfind('"')
+        line = line[start + 1 : end]
+
+        # convert to array so we can add the negated option later on, if one exists
+        line = [line]
+
+        negate_line = stats_translations[i + 4].strip()
+        if "lang" not in negate_line and "negate" in negate_line:
+            # mod has a negated version
+            end = negate_line.find("negate")
+            negate_line = negate_line[negate_line.find('"') + 1 : end + len("negate")]
+            line.append(negate_line)
+
+        self.parse_mod(id, line, log=log)
+
     def parse_translation_file(self, file):
         dir = f"{self.cwd}/descriptions/{file}"
         print("Parsing", dir)
         stats_translations = open(dir, encoding="utf-16").read().split("\n")
-        for i in range(0, len(stats_translations)):
+        should_log = True
+        for i in range(0, 100):
             line = stats_translations[i]
 
             if line == "description":
@@ -163,24 +210,29 @@ class Parser:
                 id = (
                     stats_translations[i + 1].strip()[2:].replace('"', "")
                 )  # skip first 2 characters
-                english = stats_translations[i + 3].strip()  # skip first 2 characters
+                if self.lang == "en":
-                start = english.find('"')
+                    self.parse_translation_line(
-                end = english.rfind('"')
+                        stats_translations, i, id, log=should_log
-                english = english[start + 1 : end]
+                    )
-
+                    should_log = False
-                # convert to array so we can add the negated option later on, if one exists
+                else:
-                english = [english]
+                    j = i
-
+                    while (
-                negate_english = stats_translations[i + 4].strip()
+                        j + 5 < 100
-                if "lang" not in negate_english and "negate" in negate_english:
+                        and stats_translations[j + 1] != "description"
-                    # mod has a negated version
+                        and LANG_CODES_TO_NAMES[self.lang]
-                    end = negate_english.find("negate")
+                        not in stats_translations[j + 1]
-                    negate_english = negate_english[
+                    ):
-                        negate_english.find('"') + 1 : end + len("negate")
+                        j += 1
-                    ]
+                    if should_log:
-                    english.append(negate_english)
+                        print(
-
+                            f"Parsing [j:{j + 1}, id:{id}] {stats_translations[j + 1]}"
-                self.parse_mod(id, english)
+                        )
+                    if stats_translations[j + 1] != "description":
+                        self.parse_translation_line(
+                            stats_translations, j, id, log=should_log
+                        )
+                        should_log = False
 
     def parse_mods(self):
         for stat in self.stats_file:
@@ -206,6 +258,7 @@ class Parser:
                 translation = self.mod_translations.get(stats_id)
                 if translation:
                     ref = translation.get("ref")
+                    print(translation)
                     matchers = translation.get("matchers")
                     ids = self.stats_trade_ids.get(matchers[0].get("string"))
                     # if ref.lower() == "bow attacks fire an additional arrow":
@@ -440,37 +493,41 @@ class Parser:
             if gem:
                 out.update({"gem": gem})
 
-            f.write(json.dumps(out) + "\n")
+            f.write(json.dumps(out, ensure_ascii=False) + "\n")
 
         for item in self.unique_items:
-            f.write(json.dumps(item) + "\n")
+            f.write(json.dumps(item, ensure_ascii=False) + "\n")
 
         f.close()
 
         # somehow not a thing? - possibly missing some data
-        self.mods["physical_local_damage_+%"] = {
+        # self.mods["physical_local_damage_+%"] = {
-            "ref": "#% increased Physical Damage",
+        #     "ref": "#% increased Physical Damage",
-            "better": 1,
+        #     "better": 1,
-            "id": "physical_local_damage_+%",
+        #     "id": "physical_local_damage_+%",
-            "matchers": [{"string": "#% increased Physical Damage"}],
+        #     "matchers": [{"string": "#% increased Physical Damage"}],
-            "trade": {
+        #     "trade": {
-                "ids": {
+        #         "ids": {
-                    "explicit": ["explicit.stat_419810844"],
+        #             "explicit": ["explicit.stat_419810844"],
-                    "fractured": ["fractured.stat_419810844"],
+        #             "fractured": ["fractured.stat_419810844"],
-                    "rune": ["rune.stat_419810844"],
+        #             "rune": ["rune.stat_419810844"],
-                }
+        #         }
-            },
+        #     },
-        }
+        # }
 
         seen = set()
-        m = open(f"{self.out_dir}/stats.ndjson", "w", encoding="utf-8")
+        m = open(
+            f"{self.out_dir}/stats.ndjson",
+            "w",
+            encoding="utf-8",
+        )
         for mod in self.mods.values():
             id = mod.get("id")
 
             if id in seen:
                 continue
 
-            m.write(json.dumps(mod) + "\n")
+            m.write(json.dumps(mod, ensure_ascii=False) + "\n")
             seen.add(id)
         m.close()
 
@@ -479,21 +536,23 @@ class Parser:
             "w",
             encoding="utf-8",
         ) as f:
-            f.write(json.dumps(self.items, indent=4))
+            f.write(json.dumps(self.items, indent=4, ensure_ascii=False))
 
         with open(
             f"{self.get_script_dir()}/pyDumps/{self.lang+'-out'}/mods_dump.json",
             "w",
             encoding="utf-8",
         ) as f:
-            f.write(json.dumps(self.mods, indent=4))
+            f.write(json.dumps(self.mods, indent=4, ensure_ascii=False))
 
         with open(
             f"{self.get_script_dir()}/pyDumps/{self.lang+'-out'}/matchers_no_trade_ids.json",
             "w",
             encoding="utf-8",
         ) as f:
-            f.write(json.dumps(self.matchers_no_trade_ids, indent=4))
+            f.write(
+                json.dumps(self.matchers_no_trade_ids, indent=4, ensure_ascii=False)
+            )
 
     def run(self):
         self.parse_trade_ids()

renderer/public/.DS_Store/vendor/client/pyDumps/ru-out/matchers_no_trade_ids.json

@@ -0,0 +1 @@
+[]

renderer/public/.DS_Store/vendor/client/pyDumps/ru-out/mods_dump.json

@@ -0,0 +1,25 @@
+{
+    "physical_local_damage_+%": {
+        "ref": "#% increased Physical Damage",
+        "better": 1,
+        "id": "physical_local_damage_+%",
+        "matchers": [
+            {
+                "string": "#% increased Physical Damage"
+            }
+        ],
+        "trade": {
+            "ids": {
+                "explicit": [
+                    "explicit.stat_419810844"
+                ],
+                "fractured": [
+                    "fractured.stat_419810844"
+                ],
+                "rune": [
+                    "rune.stat_419810844"
+                ]
+            }
+        }
+    }
+}

renderer/public/.DS_Store/vendor/client/pyDumps/ru/stats.ndjson

@@ -0,0 +1 @@
+{"ref": "#% increased Physical Damage", "better": 1, "id": "physical_local_damage_+%", "matchers": [{"string": "#% increased Physical Damage"}], "trade": {"ids": {"explicit": ["explicit.stat_419810844"], "fractured": ["fractured.stat_419810844"], "rune": ["rune.stat_419810844"]}}}

renderer/src/web/price-check/trade/pathofexile-trade.ts

@@ -704,7 +704,7 @@ export async function requestTradeResultList(
       {
         method: "POST",
         headers: {
-          "Accept": "application/json",
+          Accept: "application/json",
           "Content-Type": "application/json",
         },
         body: JSON.stringify(body),