Add jsonpath support for checkver and hash extraction

Unravel if-blocks in checkver

Use new jsonpath for openssl, nuget, dart and bfg

bin/checkver.ps1

@@ -52,28 +52,37 @@ $queue | % {
     $name, $json = $_
 
     $githubRegex = "\/releases\/tag\/(?:v)?([\d.]+)"
-    if ($json.checkver -is [String]) {
+
+    $url = $json.homepage
+    if($json.checkver.url) {
+        $url = $json.checkver.url
+    }
+    $regex = ""
+    $jsonpath = ""
+
     if ($json.checkver -eq "github") {
         if (!$json.homepage.StartsWith("https://github.com/")) {
             write-host "ERROR: $name checkver expects the homepage to be a github repository" -f DarkYellow
         }
-
         $url = $json.homepage + "/releases/latest"
         $regex = $githubRegex
-        } else {
-            $url = $json.homepage
-            $regex = $json.checkver
     }
-    } else {
+
     if ($json.checkver.github) {
         $url = $json.checkver.github + "/releases/latest"
         $regex = $githubRegex
-        } else {
+    }
-            $url = $json.checkver.url
-            if(!$url) { $url = $json.homepage }
 
+    if($json.checkver.re) {
         $regex = $json.checkver.re
     }
+
+    if($json.checkver.jp) {
+        $jsonpath = $json.checkver.jp
+    }
+
+    if(!$jsonpath -and !$regex) {
+        $regex = $json.checkver
     }
 
     $state = new-object psobject @{
@@ -81,6 +90,7 @@ $queue | % {
         url = $url;
         regex = $regex;
         json = $json;
+        jsonpath = $jsonpath;
     }
 
     $wc.downloadstringasync($url, $state)
@@ -99,6 +109,8 @@ while($in_progress -gt 0) {
     $url = $state.url
     $expected_ver = $json.version
     $regexp = $state.regex
+    $jsonpath = $state.jsonpath
+    $ver = ""
 
     $err = $ev.sourceeventargs.error
     $page = $ev.sourceeventargs.result
@@ -108,12 +120,39 @@ while($in_progress -gt 0) {
     if($err) {
         write-host -f darkred $err.message
         write-host -f darkred "URL $url is not valid"
-    } else {
+        continue
+    }
+
+    if($jsonpath -and $regexp) {
+        write-host -f darkred "'jp' and 're' shouldn't be used together"
+        continue
+    }
+
+    if($jsonpath) {
+        $ver = json_path ($page | ConvertFrom-Json -ea stop) $jsonpath
+        if(!$ver) {
+            write-host -f darkred "couldn't find '$jsonpath' in $url"
+            continue
+        }
+    }
+
+    if($regexp) {
         if($page -match $regexp) {
             $ver = $matches[1]
             if(!$ver) {
                 $ver = $matches['version']
             }
+        } else {
+            write-host -f darkred "couldn't match '$regexp' in $url"
+            continue
+        }
+    }
+
+    if(!$ver) {
+        write-host -f darkred "couldn't find new version in $url"
+        continue
+    }
+
     if($ver -eq $expected_ver) {
         write-host "$ver" -f darkgreen
 
@@ -135,11 +174,6 @@ while($in_progress -gt 0) {
             autoupdate $app $dir $json $ver $matches
         }
     }
-
-        } else {
-            write-host "couldn't match '$regexp' in $url" -f darkred
-        }
-    }
 }
 
 set_https_protocols $original

bucket/bfg.json

@@ -1,7 +1,7 @@
 {
     "homepage": "https://rtyley.github.io/bfg-repo-cleaner/",
     "license": "GPL",
-    "version": "1.12.15",
+    "version": "1.12.14",
     "url": "https://repo1.maven.org/maven2/com/madgag/bfg/1.12.15/bfg-1.12.15.jar",
     "hash": "330af214a0fed320c591afc1046b0f31e8a438f290da09672973aeaa6411b09d",
     "suggest": {
@@ -17,7 +17,7 @@
     ",
     "checkver": {
         "url": "https://search.maven.org/solrsearch/select/?q=g:com.madgag+AND+a:bfg",
-        "re": "\"latestVersion\":\"([\\d.]+)\""
+        "jp": "$.response.docs[0].latestVersion"
     },
     "autoupdate": {
         "url": "https://repo1.maven.org/maven2/com/madgag/bfg/$version/bfg-$version.jar"

bucket/dart.json

@@ -18,7 +18,7 @@
     },
     "checkver": {
         "url": "https://storage.googleapis.com/dart-archive/channels/stable/release/latest/VERSION",
-        "re": "\"version\":\\s*\"(.*)\","
+        "jp": "$.version"
     },
     "autoupdate": {
         "architecture": {

bucket/nuget.json

@@ -1,13 +1,13 @@
 {
     "homepage": "https://www.nuget.org/",
-    "version": "3.5.0",
+    "version": "3.4.1",
     "license": "Apache 2.0",
     "url": "https://dist.nuget.org/win-x86-commandline/v3.5.0/NuGet.exe",
     "hash": "399ec24c26ed54d6887cde61994bb3d1cada7956c1b19ff880f06f060c039918",
     "bin": "NuGet.exe",
     "checkver": {
         "url": "https://dist.nuget.org/index.json",
-        "re": "latest\", \"version\": \"([\\d.]+)\""
+        "jp": "$.artifacts[0].versions[0].version"
     },
     "autoupdate": {
         "url": "https://dist.nuget.org/win-x86-commandline/v$version/NuGet.exe"

bucket/openssl.json

@@ -29,8 +29,8 @@
             }
         },
         "hash": {
-            "mode": "extract",
+            "mode": "json",
-            "find": "$basename.*\\s.*\\s.*\\s.*\\s.*\"([\\da-f]{64})\"",
+            "jp": "$.files.$basename.sha256",
             "url": "https://slproweb.com/download/win32_openssl_hashes.json"
         }
     }

lib/autoupdate.ps1

@@ -4,6 +4,7 @@ TODO
  - tests (single arch, without hashes etc.)
  - clean up
 #>
+. "$psscriptroot\..\lib\json.ps1"
 
 function substitute([String] $str, [Hashtable] $params) {
     $params.GetEnumerator() | % {
@@ -55,8 +56,48 @@ function find_hash_in_rdf([String] $url, [String] $filename)
     return $digest.sha256
 }
 
-function get_hash_for_app([String] $app, $config, [String] $version, [String] $url, [Hashtable] $substitutions)
+function find_hash_in_textfile([String] $url, [String] $basename, [String] $type, [String] $regex) {
-{
+    $hashfile = $null
+
+    try {
+        $hashfile = (new-object net.webclient).downloadstring($url)
+    } catch [system.net.webexception] {
+        write-host -f darkred $_
+        write-host -f darkred "URL $url is not valid"
+        return
+    }
+
+    if ($regex -eq $null) {
+        $regex = "([a-z0-9]+)"
+    }
+    $regex = substitute $regex @{'$basename' = [regex]::Escape($basename)}
+
+    if ($hashfile -match $regex) {
+        $hash = $matches[1]
+
+        if ($type -and !($type -eq "sha256")) {
+            $hash = $type + ":$hash"
+        }
+
+        return $hash
+    }
+}
+
+function find_hash_in_json([String] $url, [String] $basename, [String] $jsonpath) {
+    $json = $null
+
+    try {
+        $json = (new-object net.webclient).downloadstring($url) | convertfrom-json -ea stop
+    } catch [system.net.webexception] {
+        write-host -f darkred $_
+        write-host -f darkred "URL $url is not valid"
+        return
+    }
+
+    return json_path $json $jsonpath $basename
+}
+
+function get_hash_for_app([String] $app, $config, [String] $version, [String] $url, [Hashtable] $substitutions) {
     $hash = $null
 
     <#
@@ -67,37 +108,22 @@ function get_hash_for_app([String] $app, $config, [String] $version, [String] $u
     #>
     $hashmode = $config.mode
     $basename = fname($url)
-    if ($hashmode -eq "extract") {
+
     $hashfile_url = substitute $config.url @{'$url' = $url}
     $hashfile_url = substitute $hashfile_url $substitutions
-        $hashfile = $null
 
-        try {
+    if ($hashmode -eq "extract") {
-            $hashfile = (new-object net.webclient).downloadstring($hashfile_url)
+        return find_hash_in_textfile $hashfile_url $basename $config.type $config.find
-        } catch [system.net.webexception] {
-            write-host -f darkred $_
-            write-host -f darkred "URL $hashfile_url is not valid"
-            return $null
     }
 
-        $regex = $config.find
+    if ($hashmode -eq "json") {
-        if ($regex -eq $null) {
+        return find_hash_in_json $hashfile_url $basename $config.jp
-            $regex = "([a-z0-9]+)"
-        }
-        $regex = substitute $regex @{'$basename' = [regex]::Escape($basename)}
-
-        if ($hashfile -match $regex) {
-            $hash = $matches[1]
-
-            if ($config.type -and !($config.type -eq "sha256")) {
-                $hash = $config.type + ":$hash"
     }
 
-            return $hash
+    if ($hashmode -eq "rdf") {
+        return find_hash_in_rdf $hashfile_url $basename
     }
-    } elseif ($hashmode -eq "rdf") {
+
-        return find_hash_in_rdf $config.url $basename
-    } else {
     Write-Host "Download files to compute hashes!" -f DarkYellow
     try {
         dl_with_cache $app $version $url $null $null $true
@@ -109,7 +135,6 @@ function get_hash_for_app([String] $app, $config, [String] $version, [String] $u
     $file = fullpath (cache_path $app $version $url)
     return compute_hash $file "sha256"
 }
-}
 
 function update_manifest_with_new_version($json, [String] $version, [String] $url, [String] $hash, $architecture = $null)
 {
@@ -168,10 +193,12 @@ function get_version_substitutions([String] $version, [Hashtable] $matches)
         '$buildVersion' = $firstPart.Split('.') | Select-Object -skip 3 -first 1;
         '$preReleaseVersion' = $lastPart;
     }
+    if($matches) {
         $matches.Remove(0)
         $matches.GetEnumerator() | % {
             $versionVariables.Add('$match' + (Get-Culture).TextInfo.ToTitleCase($_.Name), $_.Value)
         }
+    }
     return $versionVariables
 }
 

lib/json.ps1

@@ -86,3 +86,30 @@ Function ConvertToPrettyJson {
         $output
     }
 }
+
+function json_path([Object] $json, [String] $jsonpath, [String] $basename) {
+    $result = $json
+    $isJsonPath = $jsonpath.StartsWith("`$")
+    $jsonpath.split(".") | ForEach-Object {
+        $el = $_
+
+        # substitute the base filename into the jsonpath
+        if($el.StartsWith("`$basename")) {
+            $el = $el.Replace("`$basename", $basename)
+        }
+
+        # skip $ if it's jsonpath format
+        if($el -eq "`$" -and $isJsonPath) {
+            return
+        }
+
+        if($el -match "^(?<property>\w+)\[(?<index>\d+)\]$") {
+            $property = $matches['property']
+            $result = $result.$property[$matches['index']]
+            return
+        }
+
+        $result = $result.$el
+    }
+    return $result
+}

schema.json

@@ -96,10 +96,15 @@
                             "format": "regex",
                             "type": "string"
                         },
+                        "jp": {
+                            "pattern": "^\\$\\..*$",
+                            "type": "string"
+                        },
                         "mode": {
                             "enum": [
                                 "download",
                                 "extract",
+                                "json",
                                 "rdf"
                             ]
                         },
@@ -156,6 +161,10 @@
                         "url": {
                             "format": "uri",
                             "type": "string"
+                        },
+                        "jp": {
+                            "pattern": "^\\$\\..*$",
+                            "type": "string"
                         }
                     },
                     "type": "object"