lee/caprover
1
0
Fork 0
mirror of https://github.com/caprover/caprover synced 2025-10-30 10:07:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #2182 from dshook/goaccess

Browse Source 
Add Goaccess log reporting!
This commit is contained in:
Kasra Bigdeli
2024-12-21 22:33:22 -08:00
committed by GitHub
parent 45cf63427c 119ef3e6c0
commit 3f7e6093b0
18 changed files with 650 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
src/datastore/DataStore.ts
View File

@@ -8,6 +8,8 @@ import {
IAutomatedCleanupConfigs,
} from '../models/AutomatedCleanupConfigs'
import CapRoverTheme from '../models/CapRoverTheme'
import { GoAccessInfo } from '../models/GoAccessInfo'
import { NetDataInfo } from '../models/NetDataInfo'
import CaptainConstants from '../utils/CaptainConstants'
import CaptainEncryptor from '../utils/Encryptor'
import Utils from '../utils/Utils'
@@ -15,7 +17,6 @@ import AppsDataStore from './AppsDataStore'
import ProDataStore from './ProDataStore'
import ProjectsDataStore from './ProjectsDataStore'
import RegistriesDataStore from './RegistriesDataStore'
import { NetDataInfo } from '../models/NetDataInfo'
// keys:
const NAMESPACE = 'namespace'
@@ -26,6 +27,7 @@ const FORCE_ROOT_SSL = 'forceRootSsl'
const HAS_REGISTRY_SSL = 'hasRegistrySsl'
const EMAIL_ADDRESS = 'emailAddress'
const NET_DATA_INFO = 'netDataInfo'
const GOACCESS_INFO = 'goAccessInfo'
const NGINX_BASE_CONFIG = 'nginxBaseConfig'
const NGINX_CAPTAIN_CONFIG = 'nginxCaptainConfig'
const CUSTOM_ONE_CLICK_APP_URLS = 'oneClickAppUrls'
@@ -242,6 +244,23 @@ class DataStore {
})
}
getGoAccessInfo() {
const self = this
const goAccessInfo = (self.data.get(GOACCESS_INFO) ||
{}) as GoAccessInfo
goAccessInfo.isEnabled = goAccessInfo.isEnabled || false
goAccessInfo.data.rotationFrequencyCron =
goAccessInfo.data.rotationFrequencyCron ?? '0 0 1 * *' // monthly
return Promise.resolve(goAccessInfo)
}
setGoAccessInfo(goAccessInfo: GoAccessInfo) {
const self = this
return Promise.resolve().then(function () {
return self.data.set(GOACCESS_INFO, goAccessInfo)
})
}
getRootDomain() {
return this.data.get(CUSTOM_DOMAIN) || DEFAULT_CAPTAIN_ROOT_DOMAIN
}

89
src/docker/DockerApi.ts
View File

@@ -6,6 +6,7 @@ import {
IAppEnvVar,
IAppPort,
IAppVolume,
VolumesTypes,
} from '../models/AppDefinition'
import { DockerAuthObj, DockerRegistryConfig } from '../models/DockerAuthObj'
import { DockerSecret } from '../models/DockerSecret'
@@ -15,7 +16,6 @@ import {
IDockerApiPort,
IDockerContainerResource,
PreDeployFunction,
VolumesTypes,
} from '../models/OtherTypes'
import { ServerDockerInfo } from '../models/ServerDockerInfo'
import BuildLog from '../user/BuildLog'
@@ -65,6 +65,37 @@ export abstract class IDockerUpdateOrders {
}
export type IDockerUpdateOrder = 'auto' | 'stopFirst' | 'startFirst'
export interface CreateContainerParams {
containerName?: string
imageName: string
/** Override command to run the container with */
command?: string[]
/** an array, hostPath & containerPath, mode */
volumes: IAppVolume[]
network: string
/**
*
* [
* {
* key: 'somekey'
* value: 'some value'
* }
* ]
*/
arrayOfEnvKeyAndValue: IAppEnvVar[]
addedCapabilities?: string[]
addedSecOptions?: string[]
authObj?: DockerAuthObj
/** If true, have the container always restart,
* If false act as a one off job runner that cleans up after itself */
sticky: boolean
/** If set true, waits for the container to exit before returning */
wait?: boolean
}
class DockerApi {
private dockerode: Docker
@@ -503,7 +534,7 @@ class DockerApi {
}
/**
* Creates a volume thar restarts unless stopped
* Creates a container that restarts unless stopped
* @param containerName
* @param imageName
* @param volumes an array, hostPath & containerPath, mode
@@ -528,9 +559,40 @@ class DockerApi {
addedSecOptions: string[],
authObj: DockerAuthObj | undefined
) {
return this.createContainer({
containerName,
imageName,
volumes,
network,
arrayOfEnvKeyAndValue,
addedCapabilities,
addedSecOptions,
authObj,
sticky: true,
})
}
/**
* Creates a docker container
*/
createContainer({
containerName,
imageName,
command,
volumes,
network,
arrayOfEnvKeyAndValue,
addedCapabilities,
addedSecOptions,
authObj,
sticky,
wait = false,
}: CreateContainerParams) {
const self = this
Logger.d(`Creating Sticky Container: ${imageName}`)
Logger.d(
`Creating ${sticky ? 'Sticky' : 'Ephemeral'} Container: ${imageName}`
)
const volumesMapped: string[] = []
volumes = volumes || []
@@ -548,6 +610,8 @@ class DockerApi {
envs.push(`${e.key}=${e.value}`)
}
let container: Docker.Container | undefined = undefined
return Promise.resolve()
.then(function () {
return self.pullImage(imageName, authObj)
@@ -557,6 +621,7 @@ class DockerApi {
name: containerName,
Image: imageName,
Env: envs,
Cmd: command,
HostConfig: {
Binds: volumesMapped,
CapAdd: addedCapabilities,
@@ -569,15 +634,27 @@ class DockerApi {
CaptainConstants.configs.defaultMaxLogSize,
},
},
RestartPolicy: {
Name: 'always',
},
...(sticky
? {
RestartPolicy: {
Name: 'always',
},
}
: {}),
AutoRemove: !CaptainConstants.isDebug && !sticky,
},
})
})
.then(function (data) {
container = data
return data.start()
})
.then(function (startInfo) {
if (wait && container) {
return container.wait()
}
return startInfo
})
}
retag(currentName: string, targetName: string) {

5
src/models/AppDefinition.ts
View File

@@ -7,6 +7,11 @@ export interface IAppEnvVar {
value: string
}
export const enum VolumesTypes {
BIND = 'bind',
VOLUME = 'volume',
}
export interface IAppVolume {
containerPath: string
volumeName?: string

7
src/models/GoAccessInfo.ts Normal file
View File

@@ -0,0 +1,7 @@
export class GoAccessInfo {
public isEnabled: boolean
public data: {
rotationFrequencyCron: string
logRetentionDays?: number
}
}

1
src/models/IServerBlockDetails.ts
View File

@@ -14,4 +14,5 @@ export interface IServerBlockDetails {
customErrorPagesDirectory: string
staticWebRoot: string
redirectToPath?: string
logAccessPath?: string
}

5
src/models/OtherTypes.ts
View File

@@ -5,11 +5,6 @@ export type CaptainError = {
apiMessage: string
}
export abstract class VolumesTypes {
public static readonly BIND = 'bind'
public static readonly VOLUME = 'volume'
}
export type AnyError = any
export type PreDeployFunction = (

199
src/routes/user/system/SystemRouter.ts
View File

@@ -1,18 +1,21 @@
import express = require('express')
import fs from 'fs/promises'
import path from 'path'
import validator from 'validator'
import ApiStatusCodes from '../../../api/ApiStatusCodes'
import BaseApi from '../../../api/BaseApi'
import DockerApi from '../../../docker/DockerApi'
import DockerUtils from '../../../docker/DockerUtils'
import InjectionExtractor from '../../../injection/InjectionExtractor'
import { IAppDef } from '../../../models/AppDefinition'
import { AutomatedCleanupConfigsCleaner } from '../../../models/AutomatedCleanupConfigs'
import CaptainManager from '../../../user/system/CaptainManager'
import VersionManager from '../../../user/system/VersionManager'
import CaptainConstants from '../../../utils/CaptainConstants'
import Logger from '../../../utils/Logger'
import Utils from '../../../utils/Utils'
import SystemRouteSelfHostRegistry from './selfhostregistry/SystemRouteSelfHostRegistry'
import ThemesRouter from './ThemesRouter'
import SystemRouteSelfHostRegistry from './selfhostregistry/SystemRouteSelfHostRegistry'
const router = express.Router()
@@ -287,6 +290,200 @@ router.post('/netdata/', function (req, res, next) {
.catch(ApiStatusCodes.createCatcher(res))
})
router.get('/goaccess/', function (req, res, next) {
const dataStore =
InjectionExtractor.extractUserFromInjected(res).user.dataStore
return Promise.resolve()
.then(function () {
return dataStore.getGoAccessInfo()
})
.then(function (goAccessInfo) {
const baseApi = new BaseApi(
ApiStatusCodes.STATUS_OK,
'GoAccess info retrieved'
)
baseApi.data = goAccessInfo
res.send(baseApi)
})
.catch(ApiStatusCodes.createCatcher(res))
})
router.post('/goaccess/', function (req, res, next) {
const goAccessInfo = req.body.goAccessInfo
return Promise.resolve()
.then(function () {
return CaptainManager.get().updateGoAccessInfo(goAccessInfo)
})
.then(function () {
const baseApi = new BaseApi(
ApiStatusCodes.STATUS_OK,
'GoAccess info is updated'
)
res.send(baseApi)
})
.catch(ApiStatusCodes.createCatcher(res))
})
router.get('/goaccess/:appName/files', async function (req, res, next) {
const dataStore =
InjectionExtractor.extractUserFromInjected(res).user.dataStore
const goAccessInfo = await dataStore.getGoAccessInfo()
const loadBalanceManager = CaptainManager.get().getLoadBalanceManager()
const appName = req.params.appName
if (!appName) {
const baseApi = new BaseApi(
ApiStatusCodes.STATUS_ERROR_GENERIC,
'Invalid appName'
)
baseApi.data = []
res.send(baseApi)
return
}
if (!goAccessInfo.isEnabled) {
const baseApi = new BaseApi(
ApiStatusCodes.STATUS_ERROR_GENERIC,
'GoAccess not enabled'
)
baseApi.data = []
res.send(baseApi)
return
}
const directoryPath = path.join(
CaptainConstants.nginxSharedLogsPathOnHost,
appName
)
let appDefinition: IAppDef | undefined = undefined
return Promise.resolve()
.then(function () {
// Ensure a valid appName parameter
return dataStore.getAppsDataStore().getAppDefinition(appName)
})
.then(function (data) {
appDefinition = data
return fs.readdir(directoryPath)
})
.then(function (files) {
return Promise.all(
files
// Make sure to only return the generated reports and not folders or the live report
// That will be added back later
.filter(
(f) => f.endsWith('.html') && !f.endsWith('Live.html')
)
.map((file) => {
return fs
.stat(path.join(directoryPath, file))
.then(function (fileStats) {
return {
name: file,
time: fileStats.mtime,
}
})
})
)
})
.then(function (linkData) {
const baseUrl = `/user/system/goaccess/`
const baseApi = new BaseApi(
ApiStatusCodes.STATUS_OK,
'GoAccess info retrieved'
)
const linkList = linkData.map((d) => {
const { domainName, fileName } =
loadBalanceManager.parseLogPath(d.name)
return {
domainName,
name: fileName,
lastModifiedTime: d.time,
url: baseUrl + `${appName}/files/${d.name}`,
}
})
// Add in the live report for all sites even if it might not exist yet since they're dynamic
const allDomains = [
`${appName}.${dataStore.getRootDomain()}`,
...appDefinition!.customDomain.map((d) => d.publicDomain),
]
for (const domain of allDomains) {
const name =
loadBalanceManager.getLogName(appName, domain) +
'--Live.html'
linkList.push({
domainName: domain,
name,
lastModifiedTime: new Date(),
url: baseUrl + `${appName}/files/${name}`,
})
}
linkList.sort(
(a, b) =>
b.lastModifiedTime.getTime() - a.lastModifiedTime.getTime()
)
baseApi.data = linkList
res.send(baseApi)
})
.catch(ApiStatusCodes.createCatcher(res))
})
router.get('/goaccess/:appName/files/:file', async function (req, res, next) {
const { appName, file } = req.params
const { domainName, fileName } = CaptainManager.get()
.getLoadBalanceManager()
.parseLogPath(file)
if (fileName.includes('Live')) {
// Dynamically update the live reports by running the catchup script for the particular domain
await DockerApi.get().createContainer({
imageName: CaptainConstants.configs.goAccessImageName,
command: ['./catchupLog.sh'],
volumes: [
{
hostPath: CaptainConstants.nginxSharedLogsPathOnHost,
containerPath: CaptainConstants.nginxSharedLogsPath,
mode: 'rw',
},
],
network: CaptainConstants.captainNetworkName,
arrayOfEnvKeyAndValue: [
{
key: 'FILE_PREFIX',
value: `${appName}--${domainName}`,
},
],
sticky: false,
wait: true,
})
}
const path = `${appName}/${file}`
res.sendFile(
path,
{ root: CaptainConstants.nginxSharedLogsPathOnHost },
function (error) {
if (error !== undefined) {
Logger.e(error, 'Error getting GoAccess report ' + path)
const baseApi = new BaseApi(
ApiStatusCodes.NOT_FOUND,
'Report not found'
)
res.send(baseApi)
}
}
)
})
router.get('/nginxconfig/', function (req, res, next) {
return Promise.resolve()
.then(function () {

85
src/user/system/CaptainManager.ts
View File

@@ -3,6 +3,7 @@ import ApiStatusCodes from '../../api/ApiStatusCodes'
import DataStore from '../../datastore/DataStore'
import DataStoreProvider from '../../datastore/DataStoreProvider'
import DockerApi from '../../docker/DockerApi'
import { GoAccessInfo } from '../../models/GoAccessInfo'
import { IRegistryInfo, IRegistryTypes } from '../../models/IRegistryInfo'
import { NetDataInfo } from '../../models/NetDataInfo'
import CaptainConstants from '../../utils/CaptainConstants'
@@ -248,6 +249,13 @@ class CaptainManager {
.then(function () {
return self.diskCleanupManager.init()
})
.then(function () {
return self.dataStore.getGoAccessInfo()
})
.then(function (goAccessInfo) {
// Ensure GoAccess container restart
return self.updateGoAccessInfo(goAccessInfo)
})
.then(function () {
self.inited = true
@@ -671,6 +679,83 @@ class CaptainManager {
})
}
updateGoAccessInfo(goAccessInfo: GoAccessInfo) {
const self = this
const dockerApi = this.dockerApi
const enabled = goAccessInfo.isEnabled
// Validate cron schedules
if (!Utils.validateCron(goAccessInfo.data.rotationFrequencyCron)) {
throw ApiStatusCodes.createError(
ApiStatusCodes.ILLEGAL_PARAMETER,
'Invalid cron schedule'
)
}
const crontabFilePath = `${
CaptainConstants.goaccessConfigPathBase
}/crontab.txt`
return Promise.resolve()
.then(function () {
return self.dataStore.setGoAccessInfo(goAccessInfo)
})
.then(function () {
const cronFile = [
`${goAccessInfo.data.rotationFrequencyCron} /processLogs.sh`,
].join('\n')
return fs.outputFile(crontabFilePath, cronFile)
})
.then(function () {
return dockerApi.ensureContainerStoppedAndRemoved(
CaptainConstants.goAccessContainerName,
CaptainConstants.captainNetworkName
)
})
.then(function () {
if (enabled) {
return dockerApi.createStickyContainer(
CaptainConstants.goAccessContainerName,
CaptainConstants.configs.goAccessImageName,
[
{
hostPath:
CaptainConstants.nginxSharedLogsPathOnHost,
containerPath:
CaptainConstants.nginxSharedLogsPath,
mode: 'rw',
},
{
hostPath: crontabFilePath,
containerPath:
CaptainConstants.goAccessCrontabPath,
mode: 'ro',
},
],
CaptainConstants.captainNetworkName,
[
{
key: 'LOG_RETENTION_DAYS',
value: (
goAccessInfo.data.logRetentionDays ?? -1
).toString(),
},
],
[],
['apparmor:unconfined'],
undefined
)
}
})
.then(function () {
Logger.d(
'Updating Load Balancer - CaptainManager.updateGoAccess'
)
return self.loadBalancerManager.rePopulateNginxConfigFile()
})
}
getNodesInfo() {
const dockerApi = this.dockerApi

14
src/user/system/DiskCleanupManager.ts
View File

@@ -5,6 +5,7 @@ import DataStore from '../../datastore/DataStore'
import DockerApi from '../../docker/DockerApi'
import { IAutomatedCleanupConfigs } from '../../models/AutomatedCleanupConfigs'
import Logger from '../../utils/Logger'
import Utils from '../../utils/Utils'
export default class DiskCleanupManager {
private job: CronJob | undefined
@@ -171,18 +172,7 @@ export default class DiskCleanupManager {
return // no need to validate cron schedule
}
try {
const testJob = new CronJob(
configs.cronSchedule, // cronTime
function () {
// nothing
}, // onTick
null, // onComplete
false, // start
configs.timezone // timezone
)
testJob.stop()
} catch (e) {
if (!Utils.validateCron(configs.cronSchedule)) {
throw ApiStatusCodes.createError(
ApiStatusCodes.ILLEGAL_PARAMETER,
'Invalid cron schedule'

60
src/user/system/LoadBalancerManager.ts
View File

@@ -6,6 +6,7 @@ import { v4 as uuid } from 'uuid'
import ApiStatusCodes from '../../api/ApiStatusCodes'
import DataStore from '../../datastore/DataStore'
import DockerApi from '../../docker/DockerApi'
import { IAllAppDefinitions } from '../../models/AppDefinition'
import { IServerBlockDetails } from '../../models/IServerBlockDetails'
import LoadBalancerInfo from '../../models/LoadBalancerInfo'
import { AnyError } from '../../models/OtherTypes'
@@ -279,11 +280,21 @@ class LoadBalancerManager {
rootDomain: string
) {
const servers: IServerBlockDetails[] = []
const self = this
let apps: IAllAppDefinitions
return dataStore
.getAppsDataStore()
.getAppDefinitions()
.then(function (apps) {
.then(function (loadedApps) {
apps = loadedApps
})
.then(function () {
return dataStore.getGoAccessInfo()
})
.then(function (goAccessInfo) {
const logAccess = goAccessInfo.isEnabled
Object.keys(apps).forEach(function (appName) {
const webApp = apps[appName]
const httpBasicAuth =
@@ -315,6 +326,12 @@ class LoadBalancerManager {
serverWithSubDomain.nginxConfigTemplate =
nginxConfigTemplate
serverWithSubDomain.httpBasicAuth = httpBasicAuth
serverWithSubDomain.logAccessPath = logAccess
? self.getLogPath(
appName,
serverWithSubDomain.publicDomain
)
: undefined
if (
webApp.redirectDomain &&
@@ -346,6 +363,9 @@ class LoadBalancerManager {
staticWebRoot: '',
customErrorPagesDirectory: '',
httpBasicAuth: httpBasicAuth,
logAccessPath: logAccess
? self.getLogPath(appName, d.publicDomain)
: undefined,
}
if (
webApp.redirectDomain &&
@@ -383,6 +403,31 @@ class LoadBalancerManager {
)
}
getLogPath(appName: string, domainName: string) {
return `${CaptainConstants.nginxSharedLogsPath}/${this.getLogName(appName, domainName)}`
}
getLogName(appName: string, domainName: string) {
return `${appName}--${domainName}--access.log`
}
// Parses out the app and domain name from the log path original constructed in getLogPath
// then updated when processing the logs into file names that have timestamps that look like
// appname--some-alias.localhost--access.log--2024-10-30T01:50.html
// or appname--speed4.captain.localhost--access.log--Current.html
parseLogPath(logPath: string): { domainName: string; fileName: string } {
const splitName = logPath.split('--')
const fileName =
splitName.length > 3
? `${splitName[3].replace('.html', '')}`
: logPath
return {
domainName: splitName[1],
fileName,
}
}
getInfo() {
return new Promise<LoadBalancerInfo>(function (resolve, reject) {
const url = `http://${CaptainConstants.nginxServiceName}/nginx_status`
@@ -439,6 +484,7 @@ class LoadBalancerManager {
const registryDomain = `${
CaptainConstants.registrySubDomain
}.${dataStore.getRootDomain()}`
let logAccess = false
let hasRootSsl = false
@@ -450,6 +496,10 @@ class LoadBalancerManager {
return Promise.resolve()
.then(function () {
return dataStore.getGoAccessInfo()
})
.then(function (goAccessInfo) {
logAccess = goAccessInfo.isEnabled
return dataStore.getNginxConfig()
})
.then(function (nginxConfig) {
@@ -490,6 +540,9 @@ class LoadBalancerManager {
CaptainConstants.nginxStaticRootDir +
CaptainConstants.nginxDomainSpecificHtmlDir
}/${captainDomain}`,
logAccessPath: logAccess
? CaptainConstants.nginxSharedLogsPath
: undefined,
},
registry: {
crtPath: self.getSslCertPath(registryDomain),
@@ -804,6 +857,11 @@ class LoadBalancerManager {
CaptainConstants.nginxSharedPathOnNginx,
hostPath: CaptainConstants.nginxSharedPathOnHost,
},
{
hostPath:
CaptainConstants.nginxSharedLogsPathOnHost,
containerPath: CaptainConstants.nginxSharedLogsPath,
},
],
[CaptainConstants.captainNetworkName],
undefined,

12
src/utils/CaptainConstants.ts
View File

@@ -37,6 +37,8 @@ const configs = {
netDataImageName: 'caprover/netdata:v1.34.1',
goAccessImageName: 'dshook/goaccess',
registryImageName: 'registry:2',
appPlaceholderImageName: 'caprover/caprover-placeholder-app:latest',
@@ -109,6 +111,10 @@ const data = {
nginxDefaultHtmlDir: '/default',
nginxSharedLogsPath: '/var/log/nginx-shared',
goAccessCrontabPath: '/var/spool/cron/crontabs/root',
letsEncryptEtcPathOnNginx: '/letencrypt/etc',
nginxDomainSpecificHtmlDir: '/domains',
@@ -141,6 +147,8 @@ const data = {
perAppNginxConfigPathBase:
CAPTAIN_ROOT_DIRECTORY_GENERATED + '/nginx/conf.d',
goaccessConfigPathBase: CAPTAIN_ROOT_DIRECTORY_GENERATED + '/goaccess',
captainDataDirectory: CAPTAIN_DATA_DIRECTORY,
letsEncryptLibPath: CAPTAIN_DATA_DIRECTORY + '/letencrypt/lib',
@@ -151,6 +159,8 @@ const data = {
nginxSharedPathOnHost: CAPTAIN_DATA_DIRECTORY + '/nginx-shared',
nginxSharedLogsPathOnHost: CAPTAIN_DATA_DIRECTORY + '/shared-logs',
debugSourceDirectory: '', // Only used in debug mode
// ********************* Local Docker Constants ************************
@@ -163,6 +173,8 @@ const data = {
certbotServiceName: 'captain-certbot',
goAccessContainerName: 'captain-goaccess-container',
netDataContainerName: 'captain-netdata-container',
registryServiceName: 'captain-registry',

18
src/utils/Utils.ts
View File

@@ -1,3 +1,4 @@
import { CronJob } from 'cron'
import * as crypto from 'crypto'
import { remove } from 'fs-extra'
import * as yaml from 'yaml'
@@ -162,4 +163,21 @@ export default class Utils {
throw 'Domain name is not accepted. Custom domain cannot be subdomain of root domain.'
}
}
static validateCron(schedule: string) {
try {
const testJob = new CronJob(
schedule, // cronTime
function () {}, // onTick
null, // onComplete
false, // start
'UTC' // timezone
)
testJob.stop()
} catch (e) {
return false
}
return true
}
}