From 5b64e41999b49588f1588ddbeefa058d554ba3c7 Mon Sep 17 00:00:00 2001 From: Kasra Bigdeli Date: Thu, 16 Jul 2026 22:19:30 -0700 Subject: [PATCH 1/9] Safely clean up orphaned certificates --- src/user/system/CertbotManager.ts | 149 ++++++++++++++++++++++++++---- 1 file changed, 132 insertions(+), 17 deletions(-) diff --git a/src/user/system/CertbotManager.ts b/src/user/system/CertbotManager.ts index ec26246..1a5982d 100644 --- a/src/user/system/CertbotManager.ts +++ b/src/user/system/CertbotManager.ts @@ -1,3 +1,4 @@ +import { X509Certificate } from 'crypto' import ApiStatusCodes from '../../api/ApiStatusCodes' import DockerApi from '../../docker/DockerApi' import CaptainConstants, { @@ -14,6 +15,25 @@ const WEBROOT_PATH_IN_CAPTAIN = CaptainConstants.nginxDomainSpecificHtmlDir const shouldUseStaging = false // CaptainConstants.isDebug; +const ORPHAN_CERTIFICATE_EXPIRY_THRESHOLD_MS = 48 * 60 * 60 * 1000 +const CERTBOT_RENEWAL_CONFIG_DIRECTORY = + CaptainConstants.letsEncryptEtcPath + '/renewal' + +export function shouldDeleteOrphanedCertificate( + certificateName: string, + activeDomains: string[], + expiryDate: number, + currentTime = Date.now() +): boolean { + const activeDomainSet = new Set( + activeDomains.map((domain) => domain.toLowerCase()) + ) + + return ( + !activeDomainSet.has(certificateName.toLowerCase()) && + expiryDate <= currentTime + ORPHAN_CERTIFICATE_EXPIRY_THRESHOLD_MS + ) +} function isCertCommandSuccess(output: string) { // https://github.com/certbot/certbot/blob/099c6c8b240400b928d6b349e023e5e8414611e6/certbot/certbot/_internal/main.py#L516 @@ -268,7 +288,79 @@ class CertbotManager { }) } - renewAllCerts() { + cleanupExpiringOrphanedCertificates(activeDomains: string[]) { + const self = this + + return fs + .readdir(CERTBOT_RENEWAL_CONFIG_DIRECTORY) + .then(function (renewalConfigFiles) { + let cleanupPromise = Promise.resolve() + + renewalConfigFiles + .filter((fileName) => fileName.endsWith('.conf')) + .map((fileName) => fileName.slice(0, -'.conf'.length)) + .forEach((certificateName) => { + cleanupPromise = cleanupPromise.then(function () { + try { + self.domainValidOrThrow(certificateName) + } catch (error) { + Logger.e( + `Skipping invalid Certbot certificate name ${certificateName}: ${error}` + ) + return + } + + const certificatePath = + CaptainConstants.letsEncryptEtcPath + + `/live/${certificateName}/cert.pem` + + return fs + .readFile(certificatePath) + .then(function (certificatePem) { + const certificate = new X509Certificate( + certificatePem + ) + const expiryDate = Date.parse( + certificate.validTo + ) + + if ( + Number.isNaN(expiryDate) || + !shouldDeleteOrphanedCertificate( + certificateName, + activeDomains, + expiryDate + ) + ) { + return + } + + return self + .runCommand([ + 'certbot', + 'delete', + '--cert-name', + certificateName, + ]) + .then(function () { + Logger.d( + `Deleted orphaned certificate nearing expiration: ${certificateName}` + ) + }) + }) + .catch(function (error) { + Logger.e( + `Skipping cleanup for certificate ${certificateName}: ${error}` + ) + }) + }) + }) + + return cleanupPromise + }) + } + + renewAllCerts(activeDomains: string[]) { const self = this /* @@ -282,24 +374,47 @@ class CertbotManager { it can be run as frequently as you want - since it will usually take no action. */ - const cmd = ['certbot', 'renew'] - - if (shouldUseStaging) { - cmd.push('--staging') - } - - return Promise.resolve() // - .then(function () { - return self.ensureAllCurrentlyRegisteredDomainsHaveDirs() + return self + .cleanupExpiringOrphanedCertificates(activeDomains) + .catch(function (error) { + // Cleanup must never prevent active certificates from renewing. + Logger.e( + `Orphaned certificate cleanup failed; continuing with active certificate renewal: ${error}` + ) }) .then(function () { - return self.runCommand(cmd) - }) - .then(function (output) { - // Ignore output :) - }) - .catch(function (err) { - Logger.e(err) + let renewalPromise = Promise.resolve() + + activeDomains.forEach((domainName) => { + renewalPromise = renewalPromise.then(function () { + const cmd = [ + 'certbot', + 'renew', + '--cert-name', + domainName, + ] + + if (shouldUseStaging) { + cmd.push('--staging') + } + + return self + .ensureDomainHasDirectory(domainName) + .then(function () { + return self.runCommand(cmd) + }) + .then(function () { + // Ignore output :) + }) + .catch(function (error) { + Logger.e( + `Failed to renew certificate ${domainName}: ${error}` + ) + }) + }) + }) + + return renewalPromise }) } From 002318921c829faa92a54c6a41784a3c05c2f901 Mon Sep 17 00:00:00 2001 From: Kasra Bigdeli Date: Thu, 16 Jul 2026 22:19:37 -0700 Subject: [PATCH 2/9] Collect active SSL domains --- src/user/system/LoadBalancerManager.ts | 36 ++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/src/user/system/LoadBalancerManager.ts b/src/user/system/LoadBalancerManager.ts index 1c7b628..d343b4d 100644 --- a/src/user/system/LoadBalancerManager.ts +++ b/src/user/system/LoadBalancerManager.ts @@ -919,6 +919,35 @@ class LoadBalancerManager { }) } + getActiveSslDomains() { + const self = this + + return Promise.all([ + self.getServerList(), + self.dataStore.getHasRootSsl(), + self.dataStore.getHasRegistrySsl(), + ]).then(function ([servers, hasRootSsl, hasRegistrySsl]) { + const activeDomains: string[] = servers + .filter((server) => server.hasSsl) + .map((server) => server.publicDomain) + const rootDomain = self.dataStore.getRootDomain() + + if (hasRootSsl) { + activeDomains.push( + `${CaptainConstants.configs.captainSubDomain}.${rootDomain}` + ) + } + + if (hasRegistrySsl) { + activeDomains.push( + `${CaptainConstants.registrySubDomain}.${rootDomain}` + ) + } + + return Array.from(new Set(activeDomains)) + }) + } + renewAllCertsAndReload() { const self = this @@ -936,8 +965,11 @@ class LoadBalancerManager { 1000 * 3600 * 20.3 ) - return self.certbotManager - .renewAllCerts() // + return self + .getActiveSslDomains() + .then(function (activeDomains) { + return self.certbotManager.renewAllCerts(activeDomains) + }) .then(function () { Logger.d('Updating Load Balancer - renewAllCerts') return self.rePopulateNginxConfigFile() From 869aace8f37192b04c6b16b7d564511f06814326 Mon Sep 17 00:00:00 2001 From: Kasra Bigdeli Date: Thu, 16 Jul 2026 22:19:42 -0700 Subject: [PATCH 3/9] Test orphaned certificate lifecycle --- tests/CertCommandGenerator.test.ts | 50 +++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/tests/CertCommandGenerator.test.ts b/tests/CertCommandGenerator.test.ts index 02d5b3e..53a5c7f 100644 --- a/tests/CertCommandGenerator.test.ts +++ b/tests/CertCommandGenerator.test.ts @@ -1,4 +1,7 @@ -import { CertCommandGenerator } from '../src/user/system/CertbotManager' +import { + CertCommandGenerator, + shouldDeleteOrphanedCertificate, +} from '../src/user/system/CertbotManager' const defaultCommand = 'certbot certonly --domain ${domainName}' const exampleRule = { @@ -59,3 +62,48 @@ test('falls back to default command when rule command is null', () => { expect(generator.getCertbotCertCommand('nullcommand.com', fakeWebroot)) .toEqual([ 'certbot', 'certonly', '--domain', 'nullcommand.com' ]) }) + +describe('orphaned certificate cleanup', () => { + const currentTime = Date.parse('2026-07-16T12:00:00Z') + + test('deletes orphaned certificates expiring within 48 hours', () => { + expect( + shouldDeleteOrphanedCertificate( + 'expired.example.com', + [], + currentTime - 1, + currentTime + ) + ).toBe(true) + expect( + shouldDeleteOrphanedCertificate( + '48-hours.example.com', + [], + currentTime + 48 * 60 * 60 * 1000, + currentTime + ) + ).toBe(true) + }) + + test('keeps orphaned certificates with more than 48 hours remaining', () => { + expect( + shouldDeleteOrphanedCertificate( + '49-hours.example.com', + [], + currentTime + 49 * 60 * 60 * 1000, + currentTime + ) + ).toBe(false) + }) + + test('keeps active certificates even when they are expired', () => { + expect( + shouldDeleteOrphanedCertificate( + 'active.example.com', + ['ACTIVE.EXAMPLE.COM'], + currentTime - 1, + currentTime + ) + ).toBe(false) + }) +}) From 2453ff8f7d8337aad474a014ce3ead3a7aae3f5c Mon Sep 17 00:00:00 2001 From: Kasra Bigdeli Date: Thu, 16 Jul 2026 22:19:47 -0700 Subject: [PATCH 4/9] Document orphaned certificate cleanup --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e0441b..e3b361f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Improved: Descriptive error on installations on incompatible systems (e.g. Proxmox LXC) [issues-2326](https://github.com/caprover/caprover/issues/2326) - Improved: Moved one click app creation process to backend for more stability [PR-2334](https://github.com/caprover/caprover/pull/2334) - Improved: Reduced the Backup size by excluding the GoAccess logs [PR-2336](https://github.com/caprover/caprover/pull/2336) +- Fixed: Removed orphaned SSL certificates shortly before expiration to prevent failed renewal attempts [Issue-2397](https://github.com/caprover/caprover/issues/2397) - Fixed: Prevented malformed `config-captain.json` from being overwritten [Issue-858](https://github.com/caprover/caprover/issues/858) - Fixed: Git webhook deployments failing when webhook payloads exceed 100 KB [Issue-1393](https://github.com/caprover/caprover/issues/1393) From 36eed6141d2136c04f695584017c05e1edac8998 Mon Sep 17 00:00:00 2001 From: Kasra Bigdeli Date: Fri, 17 Jul 2026 17:29:34 -0700 Subject: [PATCH 5/9] Observe orphaned certificates without deleting them --- src/user/system/CertbotManager.ts | 80 ++++++++++--------------------- 1 file changed, 24 insertions(+), 56 deletions(-) diff --git a/src/user/system/CertbotManager.ts b/src/user/system/CertbotManager.ts index 1a5982d..f9ead9c 100644 --- a/src/user/system/CertbotManager.ts +++ b/src/user/system/CertbotManager.ts @@ -19,7 +19,7 @@ const ORPHAN_CERTIFICATE_EXPIRY_THRESHOLD_MS = 48 * 60 * 60 * 1000 const CERTBOT_RENEWAL_CONFIG_DIRECTORY = CaptainConstants.letsEncryptEtcPath + '/renewal' -export function shouldDeleteOrphanedCertificate( +export function isExpiringOrphanedCertificate( certificateName: string, activeDomains: string[], expiryDate: number, @@ -288,7 +288,7 @@ class CertbotManager { }) } - cleanupExpiringOrphanedCertificates(activeDomains: string[]) { + logExpiringOrphanedCertificates(activeDomains: string[]) { const self = this return fs @@ -326,7 +326,7 @@ class CertbotManager { if ( Number.isNaN(expiryDate) || - !shouldDeleteOrphanedCertificate( + !isExpiringOrphanedCertificate( certificateName, activeDomains, expiryDate @@ -335,22 +335,13 @@ class CertbotManager { return } - return self - .runCommand([ - 'certbot', - 'delete', - '--cert-name', - certificateName, - ]) - .then(function () { - Logger.d( - `Deleted orphaned certificate nearing expiration: ${certificateName}` - ) - }) + Logger.d( + `Orphaned certificate eligible for deletion (no action taken): ${certificateName}` + ) }) .catch(function (error) { Logger.e( - `Skipping cleanup for certificate ${certificateName}: ${error}` + `Skipping orphan candidate check for certificate ${certificateName}: ${error}` ) }) }) @@ -360,7 +351,7 @@ class CertbotManager { }) } - renewAllCerts(activeDomains: string[]) { + renewAllCerts() { const self = this /* @@ -374,47 +365,24 @@ class CertbotManager { it can be run as frequently as you want - since it will usually take no action. */ - return self - .cleanupExpiringOrphanedCertificates(activeDomains) - .catch(function (error) { - // Cleanup must never prevent active certificates from renewing. - Logger.e( - `Orphaned certificate cleanup failed; continuing with active certificate renewal: ${error}` - ) + const cmd = ['certbot', 'renew'] + + if (shouldUseStaging) { + cmd.push('--staging') + } + + return Promise.resolve() // + .then(function () { + return self.ensureAllCurrentlyRegisteredDomainsHaveDirs() }) .then(function () { - let renewalPromise = Promise.resolve() - - activeDomains.forEach((domainName) => { - renewalPromise = renewalPromise.then(function () { - const cmd = [ - 'certbot', - 'renew', - '--cert-name', - domainName, - ] - - if (shouldUseStaging) { - cmd.push('--staging') - } - - return self - .ensureDomainHasDirectory(domainName) - .then(function () { - return self.runCommand(cmd) - }) - .then(function () { - // Ignore output :) - }) - .catch(function (error) { - Logger.e( - `Failed to renew certificate ${domainName}: ${error}` - ) - }) - }) - }) - - return renewalPromise + return self.runCommand(cmd) + }) + .then(function (output) { + // Ignore output :) + }) + .catch(function (err) { + Logger.e(err) }) } From e4a9584d3453c70655e758c0f6beb9c865c08727 Mon Sep 17 00:00:00 2001 From: Kasra Bigdeli Date: Fri, 17 Jul 2026 17:29:47 -0700 Subject: [PATCH 6/9] Run orphan observation after legacy renewal --- src/user/system/LoadBalancerManager.ts | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/user/system/LoadBalancerManager.ts b/src/user/system/LoadBalancerManager.ts index d343b4d..dee5f68 100644 --- a/src/user/system/LoadBalancerManager.ts +++ b/src/user/system/LoadBalancerManager.ts @@ -965,15 +965,27 @@ class LoadBalancerManager { 1000 * 3600 * 20.3 ) - return self - .getActiveSslDomains() - .then(function (activeDomains) { - return self.certbotManager.renewAllCerts(activeDomains) - }) + return self.certbotManager + .renewAllCerts() // .then(function () { Logger.d('Updating Load Balancer - renewAllCerts') return self.rePopulateNginxConfigFile() }) + .then(function () { + return self + .getActiveSslDomains() + .then(function (activeDomains) { + return self.certbotManager.logExpiringOrphanedCertificates( + activeDomains + ) + }) + .catch(function (error) { + // Observation must never affect certificate renewal or NGINX reload. + Logger.e( + `Orphaned certificate observation failed (no action taken): ${error}` + ) + }) + }) } } From 3699872941ef98f3a13f8c7b33353f20b8135753 Mon Sep 17 00:00:00 2001 From: Kasra Bigdeli Date: Fri, 17 Jul 2026 17:30:02 -0700 Subject: [PATCH 7/9] Test orphan certificate observation --- tests/CertCommandGenerator.test.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/CertCommandGenerator.test.ts b/tests/CertCommandGenerator.test.ts index 53a5c7f..6dd5e96 100644 --- a/tests/CertCommandGenerator.test.ts +++ b/tests/CertCommandGenerator.test.ts @@ -1,6 +1,6 @@ import { CertCommandGenerator, - shouldDeleteOrphanedCertificate, + isExpiringOrphanedCertificate, } from '../src/user/system/CertbotManager' const defaultCommand = 'certbot certonly --domain ${domainName}' @@ -63,12 +63,12 @@ test('falls back to default command when rule command is null', () => { .toEqual([ 'certbot', 'certonly', '--domain', 'nullcommand.com' ]) }) -describe('orphaned certificate cleanup', () => { +describe('orphaned certificate observation', () => { const currentTime = Date.parse('2026-07-16T12:00:00Z') - test('deletes orphaned certificates expiring within 48 hours', () => { + test('flags orphaned certificates expiring within 48 hours', () => { expect( - shouldDeleteOrphanedCertificate( + isExpiringOrphanedCertificate( 'expired.example.com', [], currentTime - 1, @@ -76,7 +76,7 @@ describe('orphaned certificate cleanup', () => { ) ).toBe(true) expect( - shouldDeleteOrphanedCertificate( + isExpiringOrphanedCertificate( '48-hours.example.com', [], currentTime + 48 * 60 * 60 * 1000, @@ -85,9 +85,9 @@ describe('orphaned certificate cleanup', () => { ).toBe(true) }) - test('keeps orphaned certificates with more than 48 hours remaining', () => { + test('does not flag orphaned certificates with more than 48 hours remaining', () => { expect( - shouldDeleteOrphanedCertificate( + isExpiringOrphanedCertificate( '49-hours.example.com', [], currentTime + 49 * 60 * 60 * 1000, @@ -96,9 +96,9 @@ describe('orphaned certificate cleanup', () => { ).toBe(false) }) - test('keeps active certificates even when they are expired', () => { + test('does not flag active certificates even when they are expired', () => { expect( - shouldDeleteOrphanedCertificate( + isExpiringOrphanedCertificate( 'active.example.com', ['ACTIVE.EXAMPLE.COM'], currentTime - 1, From 5f0711dda1e4b123008afb1d506e28e62af4e77f Mon Sep 17 00:00:00 2001 From: Kasra Bigdeli Date: Fri, 17 Jul 2026 17:30:06 -0700 Subject: [PATCH 8/9] Describe orphan certificate observation --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3b361f..503eeb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ - Improved: Descriptive error on installations on incompatible systems (e.g. Proxmox LXC) [issues-2326](https://github.com/caprover/caprover/issues/2326) - Improved: Moved one click app creation process to backend for more stability [PR-2334](https://github.com/caprover/caprover/pull/2334) - Improved: Reduced the Backup size by excluding the GoAccess logs [PR-2336](https://github.com/caprover/caprover/pull/2336) -- Fixed: Removed orphaned SSL certificates shortly before expiration to prevent failed renewal attempts [Issue-2397](https://github.com/caprover/caprover/issues/2397) +- Improved: Logged orphaned SSL certificates shortly before expiration for a safe observation period [Issue-2397](https://github.com/caprover/caprover/issues/2397) - Fixed: Prevented malformed `config-captain.json` from being overwritten [Issue-858](https://github.com/caprover/caprover/issues/858) - Fixed: Git webhook deployments failing when webhook payloads exceed 100 KB [Issue-1393](https://github.com/caprover/caprover/issues/1393) From 5192066f940c6bc6d0f95859f635c6481d4aa4ea Mon Sep 17 00:00:00 2001 From: Kasra Bigdeli Date: Fri, 17 Jul 2026 17:47:42 -0700 Subject: [PATCH 9/9] Update CHANGELOG.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 503eeb1..f0bf782 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ - Improved: Descriptive error on installations on incompatible systems (e.g. Proxmox LXC) [issues-2326](https://github.com/caprover/caprover/issues/2326) - Improved: Moved one click app creation process to backend for more stability [PR-2334](https://github.com/caprover/caprover/pull/2334) - Improved: Reduced the Backup size by excluding the GoAccess logs [PR-2336](https://github.com/caprover/caprover/pull/2336) -- Improved: Logged orphaned SSL certificates shortly before expiration for a safe observation period [Issue-2397](https://github.com/caprover/caprover/issues/2397) +- Improved: Logged expired or soon-to-expire orphaned SSL certificates for a safe observation period [Issue-2397](https://github.com/caprover/caprover/issues/2397) - Fixed: Prevented malformed `config-captain.json` from being overwritten [Issue-858](https://github.com/caprover/caprover/issues/858) - Fixed: Git webhook deployments failing when webhook payloads exceed 100 KB [Issue-1393](https://github.com/caprover/caprover/issues/1393)