From 2d53f27e3231d0cf8c49f9fd19a8ff697bca2a11 Mon Sep 17 00:00:00 2001 From: Dillon Shook Date: Mon, 4 Nov 2024 20:45:21 -0500 Subject: [PATCH] PR feedback --- dockerfiles/goaccess-files/catchupLog.sh | 9 +++++++++ dockerfiles/goaccess-files/processLogs.sh | 18 +++++++++++++++++ src/datastore/DataStore.ts | 2 +- src/docker/DockerApi.ts | 11 ----------- src/routes/user/system/SystemRouter.ts | 9 +++++---- src/user/system/CaptainManager.ts | 24 ++++++++++++----------- src/user/system/LoadBalancerManager.ts | 18 ++++++++++++++--- 7 files changed, 61 insertions(+), 30 deletions(-) diff --git a/dockerfiles/goaccess-files/catchupLog.sh b/dockerfiles/goaccess-files/catchupLog.sh index 815afd7..5d4f37a 100755 --- a/dockerfiles/goaccess-files/catchupLog.sh +++ b/dockerfiles/goaccess-files/catchupLog.sh @@ -1,3 +1,12 @@ +#!/bin/sh + +# This script is a much simpler version of the processLogs script +# It simply loops over the up to date nginx log files and creates a +# GoAccess report for it with a consistent name so it will be overwritten +# next time this runs. + +# The --restore and --persist flags let GoAccess process the logs incrementally +# so it doesn't have to process the whole log file from scratch each time for logFile in /var/log/nginx-shared/*.log; do diff --git a/dockerfiles/goaccess-files/processLogs.sh b/dockerfiles/goaccess-files/processLogs.sh index 0b9fb43..5b7c28c 100755 --- a/dockerfiles/goaccess-files/processLogs.sh +++ b/dockerfiles/goaccess-files/processLogs.sh @@ -1,5 +1,23 @@ +#!/bin/sh + currentDateTime=$(date +"%Y-%m-%dT%H:%M") +# This script does a few things to process the nginx logs into GoAccess reports +# First it loops through all the non empty .log files in the shared volume +# (there should only be one per app/domain combo) +# For each one it: +# - Creates a directory for all the app reports if it doesn't exist +# - Copies the live log file into a temporary $rotateLog +# - Empties the original log file +# - Creates a new GoAccess report with the temp log file +# - Then gzip's the temp log to archive it and so it won't be picked up by the loop +# The reason for doing the copy then truncate is to avoid needing to send the +# "kill -USR1 `cat /var/run/nginx.pid`" signal to NGINX to have it pick up a new log file +# because this container doesn't have access to the NGINX of course. + +# After creating the reports, this also checks to see if there are any logs and reports older +# than the days specified by the $LOG_RETENTION_DAYS environment variable and deletes them if so + echo "Checking logs to process" for logFile in /var/log/nginx-shared/*.log; do diff --git a/src/datastore/DataStore.ts b/src/datastore/DataStore.ts index ed4142e..9261187 100644 --- a/src/datastore/DataStore.ts +++ b/src/datastore/DataStore.ts @@ -253,7 +253,7 @@ class DataStore { goAccessInfo.data.rotationFrequencyCron ?? '0 0 1 * *' // monthly goAccessInfo.data.catchupFrequencyCron = goAccessInfo.data.catchupFrequencyCron ?? '*/10 * * * *' // every 10 minutes - return goAccessInfo + return Promise.resolve(goAccessInfo) } setGoAccessInfo(goAccessInfo: GoAccessInfo) { diff --git a/src/docker/DockerApi.ts b/src/docker/DockerApi.ts index bf3f228..3a99ef9 100644 --- a/src/docker/DockerApi.ts +++ b/src/docker/DockerApi.ts @@ -815,17 +815,6 @@ class DockerApi { return self.dockerode.getService(serviceName).remove() } - getSharedVol(name: string) { - return this.dockerode.getVolume(name) - } - - createSharedVol(name: string) { - return this.dockerode.createVolume({ Name: name }).catch((err) => { - Logger.d(err) - throw err - }) - } - deleteVols(vols: string[]) { const self = this diff --git a/src/routes/user/system/SystemRouter.ts b/src/routes/user/system/SystemRouter.ts index 5c99695..9955b65 100644 --- a/src/routes/user/system/SystemRouter.ts +++ b/src/routes/user/system/SystemRouter.ts @@ -292,10 +292,11 @@ router.get('/goaccess/', function (req, res, next) { const dataStore = InjectionExtractor.extractUserFromInjected(res).user.dataStore - const goAccessInfo = dataStore.getGoAccessInfo() - return Promise.resolve() .then(function () { + return dataStore.getGoAccessInfo() + }) + .then(function (goAccessInfo) { const baseApi = new BaseApi( ApiStatusCodes.STATUS_OK, 'GoAccess info retrieved' @@ -323,11 +324,11 @@ router.post('/goaccess/', function (req, res, next) { .catch(ApiStatusCodes.createCatcher(res)) }) -router.get('/goaccess/:appName/files', function (req, res, next) { +router.get('/goaccess/:appName/files', async function (req, res, next) { const dataStore = InjectionExtractor.extractUserFromInjected(res).user.dataStore - const goAccessInfo = dataStore.getGoAccessInfo() + const goAccessInfo = await dataStore.getGoAccessInfo() const appName = req.params.appName diff --git a/src/user/system/CaptainManager.ts b/src/user/system/CaptainManager.ts index 56e8976..8aa0c29 100644 --- a/src/user/system/CaptainManager.ts +++ b/src/user/system/CaptainManager.ts @@ -249,6 +249,13 @@ class CaptainManager { .then(function () { return self.diskCleanupManager.init() }) + .then(function () { + return self.dataStore.getGoAccessInfo() + }) + .then(function (goAccessInfo) { + // Ensure GoAccess container restart + return self.updateGoAccessInfo(goAccessInfo) + }) .then(function () { self.inited = true @@ -697,10 +704,12 @@ class CaptainManager { return self.dataStore.setGoAccessInfo(goAccessInfo) }) .then(function () { - return fs.outputFile( - crontabFilePath, - self.generateGoAccessCrontab(goAccessInfo) - ) + const cronFile = [ + `${goAccessInfo.data.catchupFrequencyCron} /catchupLog.sh`, + `${goAccessInfo.data.rotationFrequencyCron} /processLogs.sh`, + ].join('\n') + + return fs.outputFile(crontabFilePath, cronFile) }) .then(function () { return dockerApi.ensureContainerStoppedAndRemoved( @@ -752,13 +761,6 @@ class CaptainManager { }) } - generateGoAccessCrontab(goAccessInfo: GoAccessInfo): string { - return [ - `${goAccessInfo.data.catchupFrequencyCron} /catchupLog.sh`, - `${goAccessInfo.data.rotationFrequencyCron} /processLogs.sh`, - ].join('\n') - } - getNodesInfo() { const dockerApi = this.dockerApi diff --git a/src/user/system/LoadBalancerManager.ts b/src/user/system/LoadBalancerManager.ts index 33dcd82..db6ecc5 100644 --- a/src/user/system/LoadBalancerManager.ts +++ b/src/user/system/LoadBalancerManager.ts @@ -6,6 +6,7 @@ import { v4 as uuid } from 'uuid' import ApiStatusCodes from '../../api/ApiStatusCodes' import DataStore from '../../datastore/DataStore' import DockerApi from '../../docker/DockerApi' +import { IAllAppDefinitions } from '../../models/AppDefinition' import { IServerBlockDetails } from '../../models/IServerBlockDetails' import LoadBalancerInfo from '../../models/LoadBalancerInfo' import { AnyError } from '../../models/OtherTypes' @@ -280,12 +281,19 @@ class LoadBalancerManager { ) { const servers: IServerBlockDetails[] = [] const self = this + let apps: IAllAppDefinitions return dataStore .getAppsDataStore() .getAppDefinitions() - .then(function (apps) { - const logAccess = dataStore.getGoAccessInfo().isEnabled + .then(function (loadedApps) { + apps = loadedApps + }) + .then(function () { + return dataStore.getGoAccessInfo() + }) + .then(function (goAccessInfo) { + const logAccess = goAccessInfo.isEnabled Object.keys(apps).forEach(function (appName) { const webApp = apps[appName] @@ -455,7 +463,7 @@ class LoadBalancerManager { const registryDomain = `${ CaptainConstants.registrySubDomain }.${dataStore.getRootDomain()}` - const logAccess = dataStore.getGoAccessInfo().isEnabled + let logAccess = false let hasRootSsl = false @@ -467,6 +475,10 @@ class LoadBalancerManager { return Promise.resolve() .then(function () { + return dataStore.getGoAccessInfo() + }) + .then(function (goAccessInfo) { + logAccess = goAccessInfo.isEnabled return dataStore.getNginxConfig() }) .then(function (nginxConfig) {