PR feedback

dockerfiles/goaccess-files/catchupLog.sh

@@ -1,3 +1,12 @@
+#!/bin/sh
+
+# This script is a much simpler version of the processLogs script
+# It simply loops over the up to date nginx log files and creates a
+# GoAccess report for it with a consistent name so it will be overwritten
+# next time this runs.
+
+# The --restore and --persist flags let GoAccess process the logs incrementally
+# so it doesn't have to process the whole log file from scratch each time
 
 for logFile in /var/log/nginx-shared/*.log; do
 

dockerfiles/goaccess-files/processLogs.sh

@@ -1,5 +1,23 @@
+#!/bin/sh
+
 currentDateTime=$(date +"%Y-%m-%dT%H:%M")
 
+# This script does a few things to process the nginx logs into GoAccess reports
+# First it loops through all the non empty .log files in the shared volume
+# (there should only be one per app/domain combo)
+# For each one it:
+# - Creates a directory for all the app reports if it doesn't exist
+# - Copies the live log file into a temporary $rotateLog
+# - Empties the original log file
+# - Creates a new GoAccess report with the temp log file
+# - Then gzip's the temp log to archive it and so it won't be picked up by the loop
+# The reason for doing the copy then truncate is to avoid needing to send the
+# "kill -USR1 `cat /var/run/nginx.pid`" signal to NGINX to have it pick up a new log file
+# because this container doesn't have access to the NGINX of course.
+
+# After creating the reports, this also checks to see if there are any logs and reports older
+# than the days specified by the $LOG_RETENTION_DAYS environment variable and deletes them if so
+
 echo "Checking logs to process"
 for logFile in /var/log/nginx-shared/*.log; do
 

src/datastore/DataStore.ts

@@ -253,7 +253,7 @@ class DataStore {
             goAccessInfo.data.rotationFrequencyCron ?? '0 0 1 * *' // monthly
         goAccessInfo.data.catchupFrequencyCron =
             goAccessInfo.data.catchupFrequencyCron ?? '*/10 * * * *' // every 10 minutes
-        return goAccessInfo
+        return Promise.resolve(goAccessInfo)
     }
 
     setGoAccessInfo(goAccessInfo: GoAccessInfo) {

src/docker/DockerApi.ts

@@ -815,17 +815,6 @@ class DockerApi {
         return self.dockerode.getService(serviceName).remove()
     }
 
-    getSharedVol(name: string) {
-        return this.dockerode.getVolume(name)
-    }
-
-    createSharedVol(name: string) {
-        return this.dockerode.createVolume({ Name: name }).catch((err) => {
-            Logger.d(err)
-            throw err
-        })
-    }
-
     deleteVols(vols: string[]) {
         const self = this
 

src/routes/user/system/SystemRouter.ts

@@ -292,10 +292,11 @@ router.get('/goaccess/', function (req, res, next) {
     const dataStore =
         InjectionExtractor.extractUserFromInjected(res).user.dataStore
 
-    const goAccessInfo = dataStore.getGoAccessInfo()
-
     return Promise.resolve()
         .then(function () {
+            return dataStore.getGoAccessInfo()
+        })
+        .then(function (goAccessInfo) {
             const baseApi = new BaseApi(
                 ApiStatusCodes.STATUS_OK,
                 'GoAccess info retrieved'
@@ -323,11 +324,11 @@ router.post('/goaccess/', function (req, res, next) {
         .catch(ApiStatusCodes.createCatcher(res))
 })
 
-router.get('/goaccess/:appName/files', function (req, res, next) {
+router.get('/goaccess/:appName/files', async function (req, res, next) {
     const dataStore =
         InjectionExtractor.extractUserFromInjected(res).user.dataStore
 
-    const goAccessInfo = dataStore.getGoAccessInfo()
+    const goAccessInfo = await dataStore.getGoAccessInfo()
 
     const appName = req.params.appName
 

src/user/system/CaptainManager.ts

@@ -249,6 +249,13 @@ class CaptainManager {
             .then(function () {
                 return self.diskCleanupManager.init()
             })
+            .then(function () {
+                return self.dataStore.getGoAccessInfo()
+            })
+            .then(function (goAccessInfo) {
+                // Ensure GoAccess container restart
+                return self.updateGoAccessInfo(goAccessInfo)
+            })
             .then(function () {
                 self.inited = true
 
@@ -697,10 +704,12 @@ class CaptainManager {
                 return self.dataStore.setGoAccessInfo(goAccessInfo)
             })
             .then(function () {
-                return fs.outputFile(
-                    crontabFilePath,
-                    self.generateGoAccessCrontab(goAccessInfo)
-                )
+                const cronFile = [
+                    `${goAccessInfo.data.catchupFrequencyCron} /catchupLog.sh`,
+                    `${goAccessInfo.data.rotationFrequencyCron} /processLogs.sh`,
+                ].join('\n')
+
+                return fs.outputFile(crontabFilePath, cronFile)
             })
             .then(function () {
                 return dockerApi.ensureContainerStoppedAndRemoved(
@@ -752,13 +761,6 @@ class CaptainManager {
             })
     }
 
-    generateGoAccessCrontab(goAccessInfo: GoAccessInfo): string {
-        return [
-            `${goAccessInfo.data.catchupFrequencyCron} /catchupLog.sh`,
-            `${goAccessInfo.data.rotationFrequencyCron} /processLogs.sh`,
-        ].join('\n')
-    }
-
     getNodesInfo() {
         const dockerApi = this.dockerApi
 

src/user/system/LoadBalancerManager.ts

@@ -6,6 +6,7 @@ import { v4 as uuid } from 'uuid'
 import ApiStatusCodes from '../../api/ApiStatusCodes'
 import DataStore from '../../datastore/DataStore'
 import DockerApi from '../../docker/DockerApi'
+import { IAllAppDefinitions } from '../../models/AppDefinition'
 import { IServerBlockDetails } from '../../models/IServerBlockDetails'
 import LoadBalancerInfo from '../../models/LoadBalancerInfo'
 import { AnyError } from '../../models/OtherTypes'
@@ -280,12 +281,19 @@ class LoadBalancerManager {
     ) {
         const servers: IServerBlockDetails[] = []
         const self = this
+        let apps: IAllAppDefinitions
 
         return dataStore
             .getAppsDataStore()
             .getAppDefinitions()
-            .then(function (apps) {
-                const logAccess = dataStore.getGoAccessInfo().isEnabled
+            .then(function (loadedApps) {
+                apps = loadedApps
+            })
+            .then(function () {
+                return dataStore.getGoAccessInfo()
+            })
+            .then(function (goAccessInfo) {
+                const logAccess = goAccessInfo.isEnabled
 
                 Object.keys(apps).forEach(function (appName) {
                     const webApp = apps[appName]
@@ -455,7 +463,7 @@ class LoadBalancerManager {
         const registryDomain = `${
             CaptainConstants.registrySubDomain
         }.${dataStore.getRootDomain()}`
-        const logAccess = dataStore.getGoAccessInfo().isEnabled
+        let logAccess = false
 
         let hasRootSsl = false
 
@@ -467,6 +475,10 @@ class LoadBalancerManager {
 
         return Promise.resolve()
             .then(function () {
+                return dataStore.getGoAccessInfo()
+            })
+            .then(function (goAccessInfo) {
+                logAccess = goAccessInfo.isEnabled
                 return dataStore.getNginxConfig()
             })
             .then(function (nginxConfig) {