# Must included with 'json.ps1' function format_hash([String] $hash) { $hash = $hash.toLower() # Workaround for GitHub API: # `"digest": "sha256:"` if ($hash -like 'sha256:*') { $hash = $hash.Substring(7) # Remove prefix 'sha256:' } switch ($hash.Length) { 32 { $hash = "md5:$hash" } # md5 40 { $hash = "sha1:$hash" } # sha1 64 { $hash = $hash } # sha256 128 { $hash = "sha512:$hash" } # sha512 default { $hash = $null } } return $hash } function find_hash_in_rdf([String] $url, [String] $basename) { $xml = $null try { # Download and parse RDF XML file $wc = New-Object Net.Webclient $wc.Headers.Add('Referer', (strip_filename $url)) $wc.Headers.Add('User-Agent', (Get-UserAgent)) $data = $wc.DownloadData($url) [xml]$xml = (Get-Encoding($wc)).GetString($data) } catch [System.Net.WebException] { Write-Host $_ -ForegroundColor DarkRed Write-Host "URL $url is not valid" -ForegroundColor DarkRed return $null } # Find file content $digest = $xml.RDF.Content | Where-Object { [String]$_.about -eq $basename } return format_hash $digest.sha256 } function find_hash_in_textfile([String] $url, [Hashtable] $substitutions, [String] $regex) { $hashfile = $null $templates = @{ '$md5' = '([a-fA-F0-9]{32})' '$sha1' = '([a-fA-F0-9]{40})' '$sha256' = '([a-fA-F0-9]{64})' '$sha512' = '([a-fA-F0-9]{128})' '$checksum' = '([a-fA-F0-9]{32,128})' '$base64' = '([a-zA-Z0-9+\/=]{24,88})' } try { $wc = New-Object Net.Webclient $wc.Headers.Add('Referer', (strip_filename $url)) $wc.Headers.Add('User-Agent', (Get-UserAgent)) $data = $wc.DownloadData($url) $ms = New-Object System.IO.MemoryStream $ms.Write($data, 0, $data.Length) $ms.Seek(0, 0) | Out-Null if ($data[0] -eq 0x1F -and $data[1] -eq 0x8B) { $ms = New-Object System.IO.Compression.GZipStream($ms, [System.IO.Compression.CompressionMode]::Decompress) } $hashfile = (New-Object System.IO.StreamReader($ms, (Get-Encoding $wc))).ReadToEnd() } catch [system.net.webexception] { Write-Host $_ -ForegroundColor DarkRed Write-Host "URL $url is not valid" -ForegroundColor DarkRed return } if ($regex.Length -eq 0) { $regex = '^\s*([a-fA-F0-9]+)\s*$' } $regex = substitute $regex $templates $false $regex = substitute $regex $substitutions $true if ($hashfile -match $regex) { debug $regex $hash = $matches[1] -replace '\s', '' } # convert base64 encoded hash values if ($hash -match '^(?:[A-Za-z0-9+\/]{4})*(?:[A-Za-z0-9+\/]{2}==|[A-Za-z0-9+\/]{3}=|[A-Za-z0-9+\/]{4})$') { $base64 = $matches[0] if (!($hash -match '^[a-fA-F0-9]+$') -and $hash.Length -notin @(32, 40, 64, 128)) { try { $hash = ([System.Convert]::FromBase64String($base64) | ForEach-Object { $_.ToString('x2') }) -join '' } catch { $hash = $hash } } } # find hash with filename in $hashfile if ($hash.Length -eq 0) { $filenameRegex = "([a-fA-F0-9]{32,128})[\x20\t]+.*`$basename(?:\s|$)|`$basename[\x20\t]+.*?([a-fA-F0-9]{32,128})" $filenameRegex = substitute $filenameRegex $substitutions $true if ($hashfile -match $filenameRegex) { debug $filenameRegex $hash = $matches[1] } $metalinkRegex = ']+>([a-fA-F0-9]{64})' if ($hashfile -match $metalinkRegex) { debug $metalinkRegex $hash = $matches[1] } } return format_hash $hash } function find_hash_in_json([String] $url, [Hashtable] $substitutions, [String] $jsonpath) { $json = $null try { $wc = New-Object Net.Webclient $wc.Headers.Add('Referer', (strip_filename $url)) $wc.Headers.Add('User-Agent', (Get-UserAgent)) if (($url -match '^https?://api\.github\.com/.*') -and (Get-GitHubToken)) { $wc.Headers.Add('Authorization', "Bearer $(Get-GitHubToken)") $wc.Headers.Add('X-GitHub-Api-Version', '2022-11-28') } $data = $wc.DownloadData($url) $ms = New-Object System.IO.MemoryStream $ms.Write($data, 0, $data.Length) $ms.Seek(0, 0) | Out-Null if ($data[0] -eq 0x1F -and $data[1] -eq 0x8B) { $ms = New-Object System.IO.Compression.GZipStream($ms, [System.IO.Compression.CompressionMode]::Decompress) } $json = (New-Object System.IO.StreamReader($ms, (Get-Encoding $wc))).ReadToEnd() } catch [System.Net.WebException] { Write-Host $_ -ForegroundColor DarkRed Write-Host "URL $url is not valid" -ForegroundColor DarkRed return } debug $jsonpath $hash = json_path $json $jsonpath $substitutions if (!$hash) { $hash = json_path_legacy $json $jsonpath $substitutions } return format_hash $hash } function find_hash_in_xml([String] $url, [Hashtable] $substitutions, [String] $xpath) { $xml = $null try { $wc = New-Object Net.Webclient $wc.Headers.Add('Referer', (strip_filename $url)) $wc.Headers.Add('User-Agent', (Get-UserAgent)) $data = $wc.DownloadData($url) $ms = New-Object System.IO.MemoryStream $ms.Write($data, 0, $data.Length) $ms.Seek(0, 0) | Out-Null if ($data[0] -eq 0x1F -and $data[1] -eq 0x8B) { $ms = New-Object System.IO.Compression.GZipStream($ms, [System.IO.Compression.CompressionMode]::Decompress) } $xml = [xml]((New-Object System.IO.StreamReader($ms, (Get-Encoding $wc))).ReadToEnd()) } catch [system.net.webexception] { Write-Host $_ -ForegroundColor DarkRed Write-Host "URL $url is not valid" -ForegroundColor DarkRed return } # Replace placeholders if ($substitutions) { $xpath = substitute $xpath $substitutions } # Find all `significant namespace declarations` from the XML file $nsList = $xml.SelectNodes('//namespace::*[not(. = ../../namespace::*)]') # Then add them into the NamespaceManager $nsmgr = New-Object System.Xml.XmlNamespaceManager($xml.NameTable) $nsList | ForEach-Object { $nsmgr.AddNamespace($_.LocalName, $_.Value) } debug $xpath debug $nsmgr # Getting hash from XML, using XPath $hash = $xml.SelectSingleNode($xpath, $nsmgr).'#text' return format_hash $hash } function find_hash_in_headers([String] $url) { $hash = $null try { $req = [System.Net.WebRequest]::Create($url) $req.Referer = (strip_filename $url) $req.AllowAutoRedirect = $false $req.UserAgent = (Get-UserAgent) $req.Timeout = 2000 $req.Method = 'HEAD' $res = $req.GetResponse() if (([int]$res.StatusCode -ge 300) -and ([int]$res.StatusCode -lt 400)) { if ($res.Headers['Digest'] -match 'SHA-256=([^,]+)' -or $res.Headers['Digest'] -match 'SHA=([^,]+)' -or $res.Headers['Digest'] -match 'MD5=([^,]+)') { $hash = ([System.Convert]::FromBase64String($matches[1]) | ForEach-Object { $_.ToString('x2') }) -join '' debug $hash } } $res.Close() } catch [System.Net.WebException] { Write-Host $_ -ForegroundColor DarkRed Write-Host "URL $url is not valid" -ForegroundColor DarkRed return } return format_hash $hash } function get_hash_for_app([String] $app, $config, [String] $version, [String] $url, [Hashtable] $substitutions) { $hash = $null $hashmode = $config.mode $originurl = strip_fragment $url $basename = [System.Web.HttpUtility]::UrlDecode((url_remote_filename($url))) $substitutions = $substitutions.Clone() $substitutions.Add('$url', $originurl) $substitutions.Add('$baseurl', (strip_filename $originurl).TrimEnd('/')) $substitutions.Add('$basename', $basename) $substitutions.Add('$urlNoExt', (strip_ext $originurl)) $substitutions.Add('$basenameNoExt', (strip_ext $basename)) debug $substitutions $hashfile_url = substitute $config.url $substitutions debug $hashfile_url if ($hashfile_url) { Write-Host 'Searching hash for ' -ForegroundColor DarkYellow -NoNewline Write-Host $basename -ForegroundColor Green -NoNewline Write-Host ' in ' -ForegroundColor DarkYellow -NoNewline Write-Host $hashfile_url -ForegroundColor Green } if ($hashmode.Length -eq 0 -and $config.url.Length -ne 0) { $hashmode = 'extract' } $jsonpath = '' if ($config.jp) { $jsonpath = $config.jp $hashmode = 'json' } if ($config.jsonpath) { $jsonpath = $config.jsonpath $hashmode = 'json' } $regex = '' if ($config.find) { $regex = $config.find } if ($config.regex) { $regex = $config.regex } $xpath = '' if ($config.xpath) { $xpath = $config.xpath $hashmode = 'xpath' } if (!$hashfile_url -and $url -match '^(?:.*fosshub.com\/).*(?:\/|\?dwl=)(?.*)$') { $hashmode = 'fosshub' } if (!$hashfile_url -and $url -match '(?:downloads\.)?sourceforge.net\/projects?\/(?[^\/]+)\/(?:files\/)?(?.*)') { $hashmode = 'sourceforge' } if (!$hashfile_url -and $url -match 'https:\/\/github\.com\/(?[^\/]+)\/(?[^\/]+)\/releases\/download\/[^\/]+\/[^\/]+') { $hashmode = 'github' } switch ($hashmode) { 'extract' { $hash = find_hash_in_textfile $hashfile_url $substitutions $regex } 'json' { $hash = find_hash_in_json $hashfile_url $substitutions $jsonpath } 'xpath' { $hash = find_hash_in_xml $hashfile_url $substitutions $xpath } 'rdf' { $hash = find_hash_in_rdf $hashfile_url $basename } 'metalink' { $hash = find_hash_in_headers $url if (!$hash) { $hash = find_hash_in_textfile "$url.meta4" $substitutions } } 'fosshub' { $hash = find_hash_in_textfile $url $substitutions ($matches.filename + '.*?"sha256":"([a-fA-F0-9]{64})"') } 'sourceforge' { # change the URL because downloads.sourceforge.net doesn't have checksums $hashfile_url = (strip_filename (strip_fragment "https://sourceforge.net/projects/$($matches['project'])/files/$($matches['file'])")).TrimEnd('/') $hash = find_hash_in_textfile $hashfile_url $substitutions '"$basename":.*?"sha1":\s*"([a-fA-F0-9]{40})"' } 'github' { $hashfile_url = "https://api.github.com/repos/$($matches['owner'])/$($matches['repo'])/releases" $hash = find_hash_in_json $hashfile_url $substitutions ("$..assets[?(@.browser_download_url == '" + $originurl + "')].digest") } } if ($hash) { # got one! Write-Host 'Found: ' -ForegroundColor DarkYellow -NoNewline Write-Host $hash -ForegroundColor Green -NoNewline Write-Host ' using ' -ForegroundColor DarkYellow -NoNewline Write-Host "$((Get-Culture).TextInfo.ToTitleCase($hashmode)) Mode" -ForegroundColor Green return $hash } elseif ($hashfile_url) { Write-Host -f DarkYellow "Could not find hash in $hashfile_url" } Write-Host 'Downloading ' -ForegroundColor DarkYellow -NoNewline Write-Host $basename -ForegroundColor Green -NoNewline Write-Host ' to compute hashes!' -ForegroundColor DarkYellow try { Invoke-CachedDownload $app $version $url $null $null $true } catch [system.net.webexception] { Write-Host $_ -ForegroundColor DarkRed Write-Host "URL $url is not valid" -ForegroundColor DarkRed return $null } $file = cache_path $app $version $url $hash = (Get-FileHash -Path $file -Algorithm SHA256).Hash.ToLower() Write-Host 'Computed hash: ' -ForegroundColor DarkYellow -NoNewline Write-Host $hash -ForegroundColor Green return $hash } function Update-ManifestProperty { <# .SYNOPSIS Update propert(y|ies) in manifest .DESCRIPTION Update selected propert(y|ies) to given version in manifest. .PARAMETER Manifest Manifest to be updated .PARAMETER Property Selected propert(y|ies) to be updated .PARAMETER AppName Software name .PARAMETER Version Given software version .PARAMETER Substitutions Hashtable of internal substitutable variables .OUTPUTS System.Boolean Flag that indicate if there are any changed properties #> [CmdletBinding(SupportsShouldProcess = $true)] [OutputType([Boolean])] param ( [Parameter(Mandatory = $true, Position = 1)] [PSCustomObject] $Manifest, [Parameter(ValueFromPipeline = $true, Position = 2)] [String[]] $Property, [String] $AppName, [String] $Version, [Alias('Matches')] [HashTable] $Substitutions ) begin { $hasManifestChanged = $false } process { foreach ($currentProperty in $Property) { if ($currentProperty -eq 'hash') { # Update hash if ($Manifest.hash) { # Global $newURL = substitute $Manifest.autoupdate.url $Substitutions $newHash = HashHelper -AppName $AppName -Version $Version -HashExtraction $Manifest.autoupdate.hash -URL $newURL -Substitutions $Substitutions $Manifest.hash, $hasPropertyChanged = PropertyHelper -Property $Manifest.hash -Value $newHash $hasManifestChanged = $hasManifestChanged -or $hasPropertyChanged } else { # Arch-spec $Manifest.architecture | Get-Member -MemberType NoteProperty | ForEach-Object { $arch = $_.Name $newURL = substitute (arch_specific 'url' $Manifest.autoupdate $arch) $Substitutions $newHash = HashHelper -AppName $AppName -Version $Version -HashExtraction (arch_specific 'hash' $Manifest.autoupdate $arch) -URL $newURL -Substitutions $Substitutions $Manifest.architecture.$arch.hash, $hasPropertyChanged = PropertyHelper -Property $Manifest.architecture.$arch.hash -Value $newHash $hasManifestChanged = $hasManifestChanged -or $hasPropertyChanged } } } elseif ($Manifest.$currentProperty -and $Manifest.autoupdate.$currentProperty) { # Update other property (global) $autoupdateProperty = $Manifest.autoupdate.$currentProperty $newValue = substitute $autoupdateProperty $Substitutions if (($autoupdateProperty.GetType().Name -eq 'Object[]') -and ($autoupdateProperty.Length -eq 1)) { # Make sure it's an array $newValue = , $newValue } $Manifest.$currentProperty, $hasPropertyChanged = PropertyHelper -Property $Manifest.$currentProperty -Value $newValue $hasManifestChanged = $hasManifestChanged -or $hasPropertyChanged } elseif ($Manifest.architecture) { # Update other property (arch-spec) $Manifest.architecture | Get-Member -MemberType NoteProperty | ForEach-Object { $arch = $_.Name if ($Manifest.architecture.$arch.$currentProperty -and ($Manifest.autoupdate.architecture.$arch.$currentProperty -or $Manifest.autoupdate.$currentProperty)) { $autoupdateProperty = @(arch_specific $currentProperty $Manifest.autoupdate $arch) $newValue = substitute $autoupdateProperty $Substitutions if (($autoupdateProperty.GetType().Name -eq 'Object[]') -and ($autoupdateProperty.Length -eq 1)) { # Make sure it's an array $newValue = , $newValue } $Manifest.architecture.$arch.$currentProperty, $hasPropertyChanged = PropertyHelper -Property $Manifest.architecture.$arch.$currentProperty -Value $newValue $hasManifestChanged = $hasManifestChanged -or $hasPropertyChanged } } } } } end { if ($Version -ne '' -and $Manifest.version -ne $Version) { $Manifest.version = $Version $hasManifestChanged = $true } return $hasManifestChanged } } function Get-VersionSubstitution { param ( [String] $Version, [Hashtable] $CustomMatches ) $firstPart = $Version.Split('-') | Select-Object -First 1 $lastPart = $Version.Split('-') | Select-Object -Last 1 $versionVariables = @{ '$version' = $Version '$dotVersion' = ($Version -replace '[._-]', '.') '$underscoreVersion' = ($Version -replace '[._-]', '_') '$dashVersion' = ($Version -replace '[._-]', '-') '$cleanVersion' = ($Version -replace '[._-]', '') '$majorVersion' = $firstPart.Split('.') | Select-Object -First 1 '$minorVersion' = $firstPart.Split('.') | Select-Object -Skip 1 -First 1 '$patchVersion' = $firstPart.Split('.') | Select-Object -Skip 2 -First 1 '$buildVersion' = $firstPart.Split('.') | Select-Object -Skip 3 -First 1 '$preReleaseVersion' = $lastPart } if ($Version -match '(?\d+\.\d+(?:\.\d+)?)(?.*)') { $versionVariables.Add('$matchHead', $Matches['head']) $versionVariables.Add('$matchTail', $Matches['tail']) } if ($CustomMatches) { $CustomMatches.GetEnumerator() | ForEach-Object { if ($_.Name -ne '0') { $versionVariables.Add('$match' + (Get-Culture).TextInfo.ToTitleCase($_.Name), $_.Value) } } } return $versionVariables } function Invoke-AutoUpdate { param ( [String] $AppName, [String] $Path, [PSObject] $Manifest, [String] $Version, [Hashtable] $CustomMatches ) Write-Host "Autoupdating $AppName" -ForegroundColor DarkCyan $substitutions = Get-VersionSubstitution $Version $CustomMatches # update properties $updatedProperties = @(@($Manifest.autoupdate.PSObject.Properties.Name) -ne 'architecture') if ($Manifest.autoupdate.architecture) { $updatedProperties += $Manifest.autoupdate.architecture.PSObject.Properties | ForEach-Object { $_.Value.PSObject.Properties.Name } } if ($updatedProperties -contains 'url') { $updatedProperties += 'hash' } $updatedProperties = $updatedProperties | Select-Object -Unique debug [$updatedProperties] $hasChanged = Update-ManifestProperty -Manifest $Manifest -Property $updatedProperties -AppName $AppName -Version $Version -Substitutions $substitutions if ($hasChanged) { # write file Write-Host "Writing updated $AppName manifest" -ForegroundColor DarkGreen # Accept unusual Unicode characters # 'Set-Content -Encoding ASCII' don't works in PowerShell 5 # Wait for 'UTF8NoBOM' Encoding in PowerShell 7 # $Manifest | ConvertToPrettyJson | Set-Content -Path (Join-Path $Path "$AppName.json") -Encoding UTF8NoBOM [System.IO.File]::WriteAllLines($Path, (ConvertToPrettyJson $Manifest)) # notes $note = "`nUpdating note:" if ($Manifest.autoupdate.note) { $note += "`nno-arch: $($Manifest.autoupdate.note)" $hasNote = $true } if ($Manifest.autoupdate.architecture) { '64bit', '32bit', 'arm64' | ForEach-Object { if ($Manifest.autoupdate.architecture.$_.note) { $note += "`n$_-arch: $($Manifest.autoupdate.architecture.$_.note)" $hasNote = $true } } } if ($hasNote) { Write-Host $note -ForegroundColor DarkYellow } } else { # This if-else branch may not be in use. Write-Host "No updates for $AppName" -ForegroundColor DarkGray } } ## Helper Functions function PropertyHelper { <# .SYNOPSIS Helper of updating property .DESCRIPTION Update manifest property (String, Array or PSCustomObject). .PARAMETER Property Property to be updated .PARAMETER Value New property values Update line by line .OUTPUTS System.Object[] The first element is new property, the second element is change flag #> param ( [Object]$Property, [Object]$Value ) $hasChanged = $false if (@($Property).Length -lt @($Value).Length) { $Property = $Value $hasChanged = $true } else { switch ($Property.GetType().Name) { 'String' { $Value = $Value -as [String] if ($null -ne $Value) { $Property = $Value $hasChanged = $true } } 'Object[]' { $Value = @($Value) for ($i = 0; $i -lt $Value.Length; $i++) { $Property[$i], $hasItemChanged = PropertyHelper -Property $Property[$i] -Value $Value[$i] $hasChanged = $hasChanged -or $hasItemChanged } } 'PSCustomObject' { if ($Value -is [PSObject]) { foreach ($name in $Property.PSObject.Properties.Name) { if ($Value.$name) { $Property.$name, $hasItemChanged = PropertyHelper -Property $Property.$name -Value $Value.$name $hasChanged = $hasChanged -or $hasItemChanged } } } } } } return $Property, $hasChanged } function HashHelper { <# .SYNOPSIS Helper of getting file hash(es) .DESCRIPTION Extract or calculate file hash(es). If hash extraction templates are less then URLs, the last template will be reused for the rest URLs. .PARAMETER AppName Software name .PARAMETER Version Given software version .PARAMETER HashExtraction Hash extraction template(s) .PARAMETER URL New download URL(s), used to calculate hash locally (fallback) .PARAMETER Substitutions Hashtable of internal substitutable variables .OUTPUTS System.String Hash value (single URL) System.String[] Hash values (multi URLs) #> param ( [String] $AppName, [String] $Version, [PSObject[]] $HashExtraction, [String[]] $URL, [HashTable] $Substitutions ) $hash = @() for ($i = 0; $i -lt $URL.Length; $i++) { if ($null -eq $HashExtraction) { $currentHashExtraction = $null } else { $currentHashExtraction = $HashExtraction[$i], $HashExtraction[-1] | Select-Object -First 1 } $hash += get_hash_for_app $AppName $currentHashExtraction $Version $URL[$i] $Substitutions if ($null -eq $hash[$i]) { throw "Could not update $AppName, hash for $(url_remote_filename $URL[$i]) failed!" } } return $hash }