Add jsonpath support for checkver and hash extraction

Unravel if-blocks in checkver Use new jsonpath for openssl, nuget, dart and bfg
2025-12-12 02:56:03 +00:00 · 2017-03-03 21:15:37 +01:00
parent 314a27e598
commit 825f19aa52
8 changed files with 192 additions and 95 deletions
--- a/bin/checkver.ps1
+++ b/bin/checkver.ps1
@@ -52,28 +52,37 @@ $queue | % {
    $name, $json = $_

    $githubRegex = "\/releases\/tag\/(?:v)?([\d.]+)"
-    if ($json.checkver -is [String]) {
+
+    $url = $json.homepage
+    if($json.checkver.url) {
+        $url = $json.checkver.url
+    }
+    $regex = ""
+    $jsonpath = ""
+
    if ($json.checkver -eq "github") {
        if (!$json.homepage.StartsWith("https://github.com/")) {
            write-host "ERROR: $name checkver expects the homepage to be a github repository" -f DarkYellow
        }
-
        $url = $json.homepage + "/releases/latest"
        $regex = $githubRegex
-        } else {
-            $url = $json.homepage
-            $regex = $json.checkver
    }
-    } else {
+
    if ($json.checkver.github) {
        $url = $json.checkver.github + "/releases/latest"
        $regex = $githubRegex
-        } else {
-            $url = $json.checkver.url
-            if(!$url) { $url = $json.homepage }
+    }

+    if($json.checkver.re) {
        $regex = $json.checkver.re
    }
+
+    if($json.checkver.jp) {
+        $jsonpath = $json.checkver.jp
+    }
+
+    if(!$jsonpath -and !$regex) {
+        $regex = $json.checkver
    }

    $state = new-object psobject @{
@@ -81,6 +90,7 @@ $queue | % {
        url = $url;
        regex = $regex;
        json = $json;
+        jsonpath = $jsonpath;
    }

    $wc.downloadstringasync($url, $state)
@@ -99,6 +109,8 @@ while($in_progress -gt 0) {
    $url = $state.url
    $expected_ver = $json.version
    $regexp = $state.regex
+    $jsonpath = $state.jsonpath
+    $ver = ""

    $err = $ev.sourceeventargs.error
    $page = $ev.sourceeventargs.result
@@ -108,12 +120,39 @@ while($in_progress -gt 0) {
    if($err) {
        write-host -f darkred $err.message
        write-host -f darkred "URL $url is not valid"
-    } else {
+        continue
+    }
+
+    if($jsonpath -and $regexp) {
+        write-host -f darkred "'jp' and 're' shouldn't be used together"
+        continue
+    }
+
+    if($jsonpath) {
+        $ver = json_path ($page | ConvertFrom-Json -ea stop) $jsonpath
+        if(!$ver) {
+            write-host -f darkred "couldn't find '$jsonpath' in $url"
+            continue
+        }
+    }
+
+    if($regexp) {
        if($page -match $regexp) {
            $ver = $matches[1]
            if(!$ver) {
                $ver = $matches['version']
            }
+        } else {
+            write-host -f darkred "couldn't match '$regexp' in $url"
+            continue
+        }
+    }
+
+    if(!$ver) {
+        write-host -f darkred "couldn't find new version in $url"
+        continue
+    }
+
    if($ver -eq $expected_ver) {
        write-host "$ver" -f darkgreen

@@ -135,11 +174,6 @@ while($in_progress -gt 0) {
            autoupdate $app $dir $json $ver $matches
        }
    }
-
-        } else {
-            write-host "couldn't match '$regexp' in $url" -f darkred
-        }
-    }
 }

 set_https_protocols $original
--- a/bucket/bfg.json
+++ b/bucket/bfg.json
@@ -1,7 +1,7 @@
 {
    "homepage": "https://rtyley.github.io/bfg-repo-cleaner/",
    "license": "GPL",
-    "version": "1.12.15",
+    "version": "1.12.14",
    "url": "https://repo1.maven.org/maven2/com/madgag/bfg/1.12.15/bfg-1.12.15.jar",
    "hash": "330af214a0fed320c591afc1046b0f31e8a438f290da09672973aeaa6411b09d",
    "suggest": {
@@ -17,7 +17,7 @@
    ",
    "checkver": {
        "url": "https://search.maven.org/solrsearch/select/?q=g:com.madgag+AND+a:bfg",
-        "re": "\"latestVersion\":\"([\\d.]+)\""
+        "jp": "$.response.docs[0].latestVersion"
    },
    "autoupdate": {
        "url": "https://repo1.maven.org/maven2/com/madgag/bfg/$version/bfg-$version.jar"
--- a/bucket/dart.json
+++ b/bucket/dart.json
@@ -18,7 +18,7 @@
    },
    "checkver": {
        "url": "https://storage.googleapis.com/dart-archive/channels/stable/release/latest/VERSION",
-        "re": "\"version\":\\s*\"(.*)\","
+        "jp": "$.version"
    },
    "autoupdate": {
        "architecture": {
--- a/bucket/nuget.json
+++ b/bucket/nuget.json
@@ -1,13 +1,13 @@
 {
    "homepage": "https://www.nuget.org/",
-    "version": "3.5.0",
+    "version": "3.4.1",
    "license": "Apache 2.0",
    "url": "https://dist.nuget.org/win-x86-commandline/v3.5.0/NuGet.exe",
    "hash": "399ec24c26ed54d6887cde61994bb3d1cada7956c1b19ff880f06f060c039918",
    "bin": "NuGet.exe",
    "checkver": {
        "url": "https://dist.nuget.org/index.json",
-        "re": "latest\", \"version\": \"([\\d.]+)\""
+        "jp": "$.artifacts[0].versions[0].version"
    },
    "autoupdate": {
        "url": "https://dist.nuget.org/win-x86-commandline/v$version/NuGet.exe"
--- a/bucket/openssl.json
+++ b/bucket/openssl.json
@@ -29,8 +29,8 @@
            }
        },
        "hash": {
-            "mode": "extract",
-            "find": "$basename.*\\s.*\\s.*\\s.*\\s.*\"([\\da-f]{64})\"",
+            "mode": "json",
+            "jp": "$.files.$basename.sha256",
            "url": "https://slproweb.com/download/win32_openssl_hashes.json"
        }
    }
--- a/lib/autoupdate.ps1
+++ b/lib/autoupdate.ps1
@@ -4,6 +4,7 @@ TODO
 - tests (single arch, without hashes etc.)
 - clean up
 #>
+. "$psscriptroot\..\lib\json.ps1"

 function substitute([String] $str, [Hashtable] $params) {
    $params.GetEnumerator() | % {
@@ -55,8 +56,48 @@ function find_hash_in_rdf([String] $url, [String] $filename)
    return $digest.sha256
 }

-function get_hash_for_app([String] $app, $config, [String] $version, [String] $url, [Hashtable] $substitutions)
-{
+function find_hash_in_textfile([String] $url, [String] $basename, [String] $type, [String] $regex) {
+    $hashfile = $null
+
+    try {
+        $hashfile = (new-object net.webclient).downloadstring($url)
+    } catch [system.net.webexception] {
+        write-host -f darkred $_
+        write-host -f darkred "URL $url is not valid"
+        return
+    }
+
+    if ($regex -eq $null) {
+        $regex = "([a-z0-9]+)"
+    }
+    $regex = substitute $regex @{'$basename' = [regex]::Escape($basename)}
+
+    if ($hashfile -match $regex) {
+        $hash = $matches[1]
+
+        if ($type -and !($type -eq "sha256")) {
+            $hash = $type + ":$hash"
+        }
+
+        return $hash
+    }
+}
+
+function find_hash_in_json([String] $url, [String] $basename, [String] $jsonpath) {
+    $json = $null
+
+    try {
+        $json = (new-object net.webclient).downloadstring($url) | convertfrom-json -ea stop
+    } catch [system.net.webexception] {
+        write-host -f darkred $_
+        write-host -f darkred "URL $url is not valid"
+        return
+    }
+
+    return json_path $json $jsonpath $basename
+}
+
+function get_hash_for_app([String] $app, $config, [String] $version, [String] $url, [Hashtable] $substitutions) {
    $hash = $null

    <#
@@ -67,37 +108,22 @@ function get_hash_for_app([String] $app, $config, [String] $version, [String] $u
    #>
    $hashmode = $config.mode
    $basename = fname($url)
-    if ($hashmode -eq "extract") {
+
    $hashfile_url = substitute $config.url @{'$url' = $url}
    $hashfile_url = substitute $hashfile_url $substitutions
-        $hashfile = $null

-        try {
-            $hashfile = (new-object net.webclient).downloadstring($hashfile_url)
-        } catch [system.net.webexception] {
-            write-host -f darkred $_
-            write-host -f darkred "URL $hashfile_url is not valid"
-            return $null
+    if ($hashmode -eq "extract") {
+        return find_hash_in_textfile $hashfile_url $basename $config.type $config.find
    }

-        $regex = $config.find
-        if ($regex -eq $null) {
-            $regex = "([a-z0-9]+)"
-        }
-        $regex = substitute $regex @{'$basename' = [regex]::Escape($basename)}
-
-        if ($hashfile -match $regex) {
-            $hash = $matches[1]
-
-            if ($config.type -and !($config.type -eq "sha256")) {
-                $hash = $config.type + ":$hash"
+    if ($hashmode -eq "json") {
+        return find_hash_in_json $hashfile_url $basename $config.jp
    }

-            return $hash
+    if ($hashmode -eq "rdf") {
+        return find_hash_in_rdf $hashfile_url $basename
    }
-    } elseif ($hashmode -eq "rdf") {
-        return find_hash_in_rdf $config.url $basename
-    } else {
+
    Write-Host "Download files to compute hashes!" -f DarkYellow
    try {
        dl_with_cache $app $version $url $null $null $true
@@ -108,7 +134,6 @@ function get_hash_for_app([String] $app, $config, [String] $version, [String] $u
    }
    $file = fullpath (cache_path $app $version $url)
    return compute_hash $file "sha256"
-    }
 }

 function update_manifest_with_new_version($json, [String] $version, [String] $url, [String] $hash, $architecture = $null)
@@ -168,10 +193,12 @@ function get_version_substitutions([String] $version, [Hashtable] $matches)
        '$buildVersion' = $firstPart.Split('.') | Select-Object -skip 3 -first 1;
        '$preReleaseVersion' = $lastPart;
    }
+    if($matches) {
        $matches.Remove(0)
        $matches.GetEnumerator() | % {
            $versionVariables.Add('$match' + (Get-Culture).TextInfo.ToTitleCase($_.Name), $_.Value)
        }
+    }
    return $versionVariables
 }

--- a/lib/json.ps1
+++ b/lib/json.ps1
@@ -86,3 +86,30 @@ Function ConvertToPrettyJson {
        $output
    }
 }
+
+function json_path([Object] $json, [String] $jsonpath, [String] $basename) {
+    $result = $json
+    $isJsonPath = $jsonpath.StartsWith("`$")
+    $jsonpath.split(".") | ForEach-Object {
+        $el = $_
+
+        # substitute the base filename into the jsonpath
+        if($el.StartsWith("`$basename")) {
+            $el = $el.Replace("`$basename", $basename)
+        }
+
+        # skip $ if it's jsonpath format
+        if($el -eq "`$" -and $isJsonPath) {
+            return
+        }
+
+        if($el -match "^(?<property>\w+)\[(?<index>\d+)\]$") {
+            $property = $matches['property']
+            $result = $result.$property[$matches['index']]
+            return
+        }
+
+        $result = $result.$el
+    }
+    return $result
+}
--- a/schema.json
+++ b/schema.json
@@ -96,10 +96,15 @@
                            "format": "regex",
                            "type": "string"
                        },
+                        "jp": {
+                            "pattern": "^\\$\\..*$",
+                            "type": "string"
+                        },
                        "mode": {
                            "enum": [
                                "download",
                                "extract",
+                                "json",
                                "rdf"
                            ]
                        },
@@ -156,6 +161,10 @@
                        "url": {
                            "format": "uri",
                            "type": "string"
+                        },
+                        "jp": {
+                            "pattern": "^\\$\\..*$",
+                            "type": "string"
                        }
                    },
                    "type": "object"