Merge branch 'master' into restock-visualselector-refactor

changedetectionio/content_fetchers/res/puppeteer_fetch.js

@@ -18,6 +18,7 @@ module.exports = async ({page, context}) => {
 
     await page.setBypassCSP(true)
     await page.setExtraHTTPHeaders(req_headers);
+    var total_size = 0;
 
     if (user_agent) {
         await page.setUserAgent(user_agent);
@@ -42,102 +43,89 @@ module.exports = async ({page, context}) => {
         height: 768,
         deviceScaleFactor: 1,
     });
-
     await page.setRequestInterception(true);
-    if (disk_cache_dir) {
+    await page.setCacheEnabled(false);
-        console.log(">>>>>>>>>>>>>>> LOCAL DISK CACHE ENABLED <<<<<<<<<<<<<<<<<<<<<");
+
+
+    await page.evaluateOnNewDocument('navigator.serviceWorker.register = () => { console.warn("Service Worker registration blocked by Playwright")}');
+
+    await page.evaluateOnNewDocument(`
+   
+  const toBlob = HTMLCanvasElement.prototype.toBlob;
+  const toDataURL = HTMLCanvasElement.prototype.toDataURL;
+
+    HTMLCanvasElement.prototype.manipulate = function() {
+    console.warn("ma");
+    const {width, height} = this;
+    const context = this.getContext('2d');
+    var dt = new Date();
+    
+    const shift = {
+      'r': dt.getDay()-3,
+      'g': dt.getDay()-3,
+      'b': dt.getDay()-3
+    };
+    console.log(shift);
+    const matt = context.getImageData(0, 0, width, height);
+    for (let i = 0; i < height; i += Math.max(1, parseInt(height / 10))) {
+      for (let j = 0; j < width; j += Math.max(1, parseInt(width / 10))) {
+        const n = ((i * (width * 4)) + (j * 4));
+        matt.data[n + 0] = matt.data[n + 0] + shift.r;
+        matt.data[n + 1] = matt.data[n + 1] + shift.g;
+        matt.data[n + 2] = matt.data[n + 2] + shift.b;
+      }
     }
-    const fs = require('fs');
+    context.putImageData(matt, 0, 0);
-    const crypto = require('crypto');
+  };
 
-    function file_is_expired(file_path) {
+  Object.defineProperty(HTMLCanvasElement.prototype, 'toBlob', {
-        if (!fs.existsSync(file_path)) {
+    value: function() {
-            return true;
+    console.warn("toblob");
+      if (true) {
+        try {
+          this.manipulate();
         }
-        var stats = fs.statSync(file_path);
+        catch(e) {
-        const now_date = new Date();
+          console.warn('manipulation failed', e);
-        const expire_seconds = 300;
-        if ((now_date / 1000) - (stats.mtime.getTime() / 1000) > expire_seconds) {
-            console.log("CACHE EXPIRED: " + file_path);
-            return true;
         }
-        return false;
+      }
-
+      return toBlob.apply(this, arguments);
     }
-
+  });
-    page.on('request', async (request) => {
+  Object.defineProperty(HTMLCanvasElement.prototype, 'toDataURL', {
-        // General blocking of requests that waste traffic
+    value: function() {
-        if (block_url_list.some(substring => request.url().toLowerCase().includes(substring))) return request.abort();
+        console.warn("todata");
-
+      if (true) {
-        if (disk_cache_dir) {
+        try {
-            const url = request.url();
+          this.manipulate();
-            const key = crypto.createHash('md5').update(url).digest("hex");
-            const dir_path = disk_cache_dir + key.slice(0, 1) + '/' + key.slice(1, 2) + '/' + key.slice(2, 3) + '/';
-
-            // https://stackoverflow.com/questions/4482686/check-synchronously-if-file-directory-exists-in-node-js
-
-            if (fs.existsSync(dir_path + key)) {
-                console.log("* CACHE HIT , using - " + dir_path + key + " - " + url);
-                const cached_data = fs.readFileSync(dir_path + key);
-                // @todo headers can come from dir_path+key+".meta" json file
-                request.respond({
-                    status: 200,
-                    //contentType: 'text/html', //@todo
-                    body: cached_data
-                });
-                return;
-            }
         }
-        request.continue();
+        catch(e) {
+          console.warn('manipulation failed', e);
+        }
+      }
+      return toDataURL.apply(this, arguments);
+    }
+  });
+
+
+  Object.defineProperty(navigator, 'webdriver', {get: () => false});
+`)
+
+    await page.emulateTimezone('America/Chicago');
+
+    var r = await page.goto(url, {
+        waitUntil: 'load', timeout: 0
     });
 
-
+// https://github.com/puppeteer/puppeteer/issues/2479#issuecomment-408263504
-    if (disk_cache_dir) {
+    if (r === null) {
-        page.on('response', async (response) => {
+        r = await page.waitForResponse(() => true);
-            const url = response.url();
-            // Basic filtering for sane responses
-            if (response.request().method() != 'GET' || response.request().resourceType() == 'xhr' || response.request().resourceType() == 'document' || response.status() != 200) {
-                console.log("Skipping (not useful) - Status:" + response.status() + " Method:" + response.request().method() + " ResourceType:" + response.request().resourceType() + " " + url);
-                return;
-            }
-            if (no_cache_list.some(substring => url.toLowerCase().includes(substring))) {
-                console.log("Skipping (no_cache_list) - " + url);
-                return;
-            }
-            if (url.toLowerCase().includes('data:')) {
-                console.log("Skipping (embedded-data) - " + url);
-                return;
-            }
-            response.buffer().then(buffer => {
-                if (buffer.length > 100) {
-                    console.log("Cache - Saving " + response.request().method() + " - " + url + " - " + response.request().resourceType());
-
-                    const key = crypto.createHash('md5').update(url).digest("hex");
-                    const dir_path = disk_cache_dir + key.slice(0, 1) + '/' + key.slice(1, 2) + '/' + key.slice(2, 3) + '/';
-
-                    if (!fs.existsSync(dir_path)) {
-                        fs.mkdirSync(dir_path, {recursive: true})
-                    }
-
-                    if (fs.existsSync(dir_path + key)) {
-                        if (file_is_expired(dir_path + key)) {
-                            fs.writeFileSync(dir_path + key, buffer);
-                        }
-                    } else {
-                        fs.writeFileSync(dir_path + key, buffer);
-                    }
-                }
-            });
-        });
     }
 
-    const r = await page.goto(url, {
+    await page.waitForTimeout(4000);
-        waitUntil: 'load'
-    });
-
-    await page.waitForTimeout(1000);
     await page.waitForTimeout(extra_wait_ms);
 
+
     if (execute_js) {
         await page.evaluate(execute_js);
         await page.waitForTimeout(200);
@@ -176,6 +164,8 @@ module.exports = async ({page, context}) => {
     }
 
     var html = await page.content();
+    page.close();
+
     return {
         data: {
             'content': html,
@@ -183,8 +173,9 @@ module.exports = async ({page, context}) => {
             'instock_data': instock_data,
             'screenshot': b64s,
             'status_code': r.status(),
-            'xpath_data': xpath_data
+            'xpath_data': xpath_data,
+            'total_size': total_size
         },
         type: 'application/json',
     };
-};
+};

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -156,6 +156,8 @@ function isItemInStock() {
         const element = elementsToScan[i];
         // outside the 'fold' or some weird text in the heading area
         // .getBoundingClientRect() was causing a crash in chrome 119, can only be run on contentVisibility != hidden
+
+         // Should be in the "above the fold" plus about 150px
         if (element.getBoundingClientRect().top + window.scrollY >= vh + 150 || element.getBoundingClientRect().top + window.scrollY <= 100) {
             continue
         }