lee/pangolin
1
0
Fork 0
mirror of https://github.com/fosrl/pangolin.git synced 2025-12-17 21:45:51 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge branch 'dev' into audit-logs

Browse Source
This commit is contained in:
Owen
2025-10-24 11:15:39 -07:00
parent 2a644c3f88 b542d82553
commit 10a5af67aa
9 changed files with 35 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
server/lib/billing/usageService.ts
View File

@@ -1,5 +1,4 @@
import { eq, sql, and } from "drizzle-orm"; import { eq, sql, and } from "drizzle-orm";
import NodeCache from "node-cache";
import { v4 as uuidv4 } from "uuid"; import { v4 as uuidv4 } from "uuid";
import { PutObjectCommand } from "@aws-sdk/client-s3"; import { PutObjectCommand } from "@aws-sdk/client-s3";
import * as fs from "fs/promises"; import * as fs from "fs/promises";
@@ -20,6 +19,7 @@ import logger from "@server/logger";
import { sendToClient } from "#dynamic/routers/ws"; import { sendToClient } from "#dynamic/routers/ws";
import { build } from "@server/build"; import { build } from "@server/build";
import { s3Client } from "@server/lib/s3"; import { s3Client } from "@server/lib/s3";
import cache from "@server/lib/cache";
interface StripeEvent { interface StripeEvent {
identifier?: string; identifier?: string;
@@ -43,7 +43,6 @@ export function noop() {
} }
export class UsageService { export class UsageService {
private cache: NodeCache;
private bucketName: string | undefined; private bucketName: string | undefined;
private currentEventFile: string | null = null; private currentEventFile: string | null = null;
private currentFileStartTime: number = 0; private currentFileStartTime: number = 0;
@@ -51,7 +50,6 @@ export class UsageService {
private uploadingFiles: Set<string> = new Set(); private uploadingFiles: Set<string> = new Set();
constructor() { constructor() {
this.cache = new NodeCache({ stdTTL: 300 }); // 5 minute TTL
if (noop()) { if (noop()) {
return; return;
} }
@@ -399,7 +397,7 @@ export class UsageService {
featureId: FeatureId featureId: FeatureId
): Promise<string | null> { ): Promise<string | null> {
const cacheKey = `customer_${orgId}_${featureId}`; const cacheKey = `customer_${orgId}_${featureId}`;
const cached = this.cache.get<string>(cacheKey); const cached = cache.get<string>(cacheKey);
if (cached) { if (cached) {
return cached; return cached;
@@ -422,7 +420,7 @@ export class UsageService {
const customerId = customer.customerId; const customerId = customer.customerId;
// Cache the result // Cache the result
this.cache.set(cacheKey, customerId); cache.set(cacheKey, customerId, 300); // 5 minute TTL
return customerId; return customerId;
} catch (error) { } catch (error) {
@@ -700,10 +698,6 @@ export class UsageService {
await this.uploadFileToS3(); await this.uploadFileToS3();
} }
public clearCache(): void {
this.cache.flushAll();
}
/** /**
* Scan the events directory for files older than 1 minute and upload them if not empty. * Scan the events directory for files older than 1 minute and upload them if not empty.
*/ */

5
server/lib/cache.ts Normal file
View File

@@ -0,0 +1,5 @@
import NodeCache from "node-cache";
export const cache = new NodeCache({ stdTTL: 3600, checkperiod: 120 });
export default cache;

11
server/private/lib/certificates.ts
View File

@@ -16,8 +16,8 @@ import { certificates, db } from "@server/db";
import { and, eq, isNotNull, or, inArray, sql } from "drizzle-orm"; import { and, eq, isNotNull, or, inArray, sql } from "drizzle-orm";
import { decryptData } from "@server/lib/encryption"; import { decryptData } from "@server/lib/encryption";
import * as fs from "fs"; import * as fs from "fs";
import NodeCache from "node-cache";
import logger from "@server/logger"; import logger from "@server/logger";
import cache from "@server/lib/cache";
let encryptionKeyPath = ""; let encryptionKeyPath = "";
let encryptionKeyHex = ""; let encryptionKeyHex = "";
@@ -51,9 +51,6 @@ export type CertificateResult = {
updatedAt?: number | null; updatedAt?: number | null;
}; };
// --- In-Memory Cache Implementation ---
const certificateCache = new NodeCache({ stdTTL: 180 }); // Cache for 3 minutes (180 seconds)
export async function getValidCertificatesForDomains( export async function getValidCertificatesForDomains(
domains: Set<string>, domains: Set<string>,
useCache: boolean = true useCache: boolean = true
@@ -67,7 +64,8 @@ export async function getValidCertificatesForDomains(
// 1. Check cache first if enabled // 1. Check cache first if enabled
if (useCache) { if (useCache) {
for (const domain of domains) { for (const domain of domains) {
const cachedCert = certificateCache.get<CertificateResult>(domain); const cacheKey = `cert:${domain}`;
const cachedCert = cache.get<CertificateResult>(cacheKey);
if (cachedCert) { if (cachedCert) {
finalResults.push(cachedCert); // Valid cache hit finalResults.push(cachedCert); // Valid cache hit
} else { } else {
@@ -180,7 +178,8 @@ export async function getValidCertificatesForDomains(
// Add to cache for future requests, using the *requested domain* as the key // Add to cache for future requests, using the *requested domain* as the key
if (useCache) { if (useCache) {
certificateCache.set(domain, resultCert); const cacheKey = `cert:${domain}`;
cache.set(cacheKey, resultCert, 180);
} }
} }
} }

17
server/routers/badger/verifySession.ts
View File

@@ -24,7 +24,6 @@ import logger from "@server/logger";
import HttpCode from "@server/types/HttpCode"; import HttpCode from "@server/types/HttpCode";
import { NextFunction, Request, Response } from "express"; import { NextFunction, Request, Response } from "express";
import createHttpError from "http-errors"; import createHttpError from "http-errors";
import NodeCache from "node-cache";
import { z } from "zod"; import { z } from "zod";
import { fromError } from "zod-validation-error"; import { fromError } from "zod-validation-error";
import { getCountryCodeForIp } from "@server/lib/geoip"; import { getCountryCodeForIp } from "@server/lib/geoip";
@@ -32,11 +31,7 @@ import { getOrgTierData } from "#dynamic/lib/billing";
import { TierId } from "@server/lib/billing/tiers"; import { TierId } from "@server/lib/billing/tiers";
import { verifyPassword } from "@server/auth/password"; import { verifyPassword } from "@server/auth/password";
import { logRequestAudit } from "./logRequestAudit"; import { logRequestAudit } from "./logRequestAudit";
import cache from "@server/lib/cache";
// We'll see if this speeds anything up
const cache = new NodeCache({
stdTTL: 5 // seconds
});
const verifyResourceSessionSchema = z.object({ const verifyResourceSessionSchema = z.object({
sessions: z.record(z.string()).optional(), sessions: z.record(z.string()).optional(),
@@ -165,7 +160,7 @@ export async function verifyResourceSession(
} }
resourceData = result; resourceData = result;
cache.set(resourceCacheKey, resourceData); cache.set(resourceCacheKey, resourceData, 5);
} }
const { resource, pincode, password, headerAuth } = resourceData; const { resource, pincode, password, headerAuth } = resourceData;
@@ -425,7 +420,7 @@ export async function verifyResourceSession(
headerAuth.headerAuthHash headerAuth.headerAuthHash
) )
) { ) {
cache.set(clientHeaderAuthKey, clientHeaderAuth); cache.set(clientHeaderAuthKey, clientHeaderAuth, 5);
logger.debug("Resource allowed because header auth is valid"); logger.debug("Resource allowed because header auth is valid");
logRequestAudit( logRequestAudit(
@@ -524,7 +519,7 @@ export async function verifyResourceSession(
); );
resourceSession = result?.resourceSession; resourceSession = result?.resourceSession;
cache.set(sessionCacheKey, resourceSession); cache.set(sessionCacheKey, resourceSession, 5);
} }
if (resourceSession?.isRequestToken) { if (resourceSession?.isRequestToken) {
@@ -651,7 +646,7 @@ export async function verifyResourceSession(
resource resource
); );
cache.set(userAccessCacheKey, allowedUserData); cache.set(userAccessCacheKey, allowedUserData, 5);
} }
if ( if (
@@ -885,7 +880,7 @@ async function checkRules(
if (!rules) { if (!rules) {
rules = await getResourceRules(resourceId); rules = await getResourceRules(resourceId);
cache.set(ruleCacheKey, rules); cache.set(ruleCacheKey, rules, 5);
} }
if (rules.length === 0) { if (rules.length === 0) {

5
server/routers/newt/dockerSocket.ts
View File

@@ -1,10 +1,5 @@
import NodeCache from "node-cache";
import { sendToClient } from "#dynamic/routers/ws"; import { sendToClient } from "#dynamic/routers/ws";
export const dockerSocketCache = new NodeCache({
stdTTL: 3600 // seconds
});
export function fetchContainers(newtId: string) { export function fetchContainers(newtId: string) {
const payload = { const payload = {
type: `newt/socket/fetch`, type: `newt/socket/fetch`,

8
server/routers/newt/handleSocketMessages.ts
View File

@@ -1,8 +1,8 @@
import { MessageHandler } from "@server/routers/ws"; import { MessageHandler } from "@server/routers/ws";
import logger from "@server/logger"; import logger from "@server/logger";
import { dockerSocketCache } from "./dockerSocket";
import { Newt } from "@server/db"; import { Newt } from "@server/db";
import { applyNewtDockerBlueprint } from "@server/lib/blueprints/applyNewtDockerBlueprint"; import { applyNewtDockerBlueprint } from "@server/lib/blueprints/applyNewtDockerBlueprint";
import cache from "@server/lib/cache";
export const handleDockerStatusMessage: MessageHandler = async (context) => { export const handleDockerStatusMessage: MessageHandler = async (context) => {
const { message, client, sendToClient } = context; const { message, client, sendToClient } = context;
@@ -24,8 +24,8 @@ export const handleDockerStatusMessage: MessageHandler = async (context) => {
if (available) { if (available) {
logger.info(`Newt ${newt.newtId} has Docker socket access`); logger.info(`Newt ${newt.newtId} has Docker socket access`);
dockerSocketCache.set(`${newt.newtId}:socketPath`, socketPath, 0); cache.set(`${newt.newtId}:socketPath`, socketPath, 0);
dockerSocketCache.set(`${newt.newtId}:isAvailable`, available, 0); cache.set(`${newt.newtId}:isAvailable`, available, 0);
} else { } else {
logger.warn(`Newt ${newt.newtId} does not have Docker socket access`); logger.warn(`Newt ${newt.newtId} does not have Docker socket access`);
} }
@@ -54,7 +54,7 @@ export const handleDockerContainersMessage: MessageHandler = async (
); );
if (containers && containers.length > 0) { if (containers && containers.length > 0) {
dockerSocketCache.set(`${newt.newtId}:dockerContainers`, containers, 0); cache.set(`${newt.newtId}:dockerContainers`, containers, 0);
} else { } else {
logger.warn(`Newt ${newt.newtId} does not have Docker containers`); logger.warn(`Newt ${newt.newtId} does not have Docker containers`);
} }

8
server/routers/site/listSites.ts
View File

@@ -9,14 +9,12 @@ import createHttpError from "http-errors";
import { z } from "zod"; import { z } from "zod";
import { fromError } from "zod-validation-error"; import { fromError } from "zod-validation-error";
import { OpenAPITags, registry } from "@server/openApi"; import { OpenAPITags, registry } from "@server/openApi";
import NodeCache from "node-cache";
import semver from "semver"; import semver from "semver";
import cache from "@server/lib/cache";
const newtVersionCache = new NodeCache({ stdTTL: 3600 }); // 1 hours in seconds
async function getLatestNewtVersion(): Promise<string | null> { async function getLatestNewtVersion(): Promise<string | null> {
try { try {
const cachedVersion = newtVersionCache.get<string>("latestNewtVersion"); const cachedVersion = cache.get<string>("latestNewtVersion");
if (cachedVersion) { if (cachedVersion) {
return cachedVersion; return cachedVersion;
} }
@@ -48,7 +46,7 @@ async function getLatestNewtVersion(): Promise<string | null> {
const latestVersion = tags[0].name; const latestVersion = tags[0].name;
newtVersionCache.set("latestNewtVersion", latestVersion); cache.set("latestNewtVersion", latestVersion);
return latestVersion; return latestVersion;
} catch (error: any) { } catch (error: any) {

12
server/routers/site/socketIntegration.ts
View File

@@ -12,9 +12,9 @@ import stoi from "@server/lib/stoi";
import { sendToClient } from "#dynamic/routers/ws"; import { sendToClient } from "#dynamic/routers/ws";
import { import {
fetchContainers, fetchContainers,
dockerSocketCache,
dockerSocket dockerSocket
} from "../newt/dockerSocket"; } from "../newt/dockerSocket";
import cache from "@server/lib/cache";
export interface ContainerNetwork { export interface ContainerNetwork {
networkId: string; networkId: string;
@@ -157,7 +157,7 @@ async function triggerFetch(siteId: number) {
// clear the cache for this Newt ID so that the site has to keep asking for the containers // clear the cache for this Newt ID so that the site has to keep asking for the containers
// this is to ensure that the site always gets the latest data // this is to ensure that the site always gets the latest data
dockerSocketCache.del(`${newt.newtId}:dockerContainers`); cache.del(`${newt.newtId}:dockerContainers`);
return { siteId, newtId: newt.newtId }; return { siteId, newtId: newt.newtId };
} }
@@ -165,7 +165,7 @@ async function triggerFetch(siteId: number) {
async function queryContainers(siteId: number) { async function queryContainers(siteId: number) {
const { newt } = await getSiteAndNewt(siteId); const { newt } = await getSiteAndNewt(siteId);
const result = dockerSocketCache.get( const result = cache.get(
`${newt.newtId}:dockerContainers` `${newt.newtId}:dockerContainers`
) as Container[]; ) as Container[];
if (!result) { if (!result) {
@@ -182,7 +182,7 @@ async function isDockerAvailable(siteId: number): Promise<boolean> {
const { newt } = await getSiteAndNewt(siteId); const { newt } = await getSiteAndNewt(siteId);
const key = `${newt.newtId}:isAvailable`; const key = `${newt.newtId}:isAvailable`;
const isAvailable = dockerSocketCache.get(key); const isAvailable = cache.get(key);
return !!isAvailable; return !!isAvailable;
} }
@@ -196,8 +196,8 @@ async function getDockerStatus(
const mappedKeys = keys.map((x) => `${newt.newtId}:${x}`); const mappedKeys = keys.map((x) => `${newt.newtId}:${x}`);
const result = { const result = {
isAvailable: dockerSocketCache.get(mappedKeys[0]) as boolean, isAvailable: cache.get(mappedKeys[0]) as boolean,
socketPath: dockerSocketCache.get(mappedKeys[1]) as string | undefined socketPath: cache.get(mappedKeys[1]) as string | undefined
}; };
return result; return result;

8
server/routers/user/inviteUser.ts
View File

@@ -1,4 +1,3 @@
import NodeCache from "node-cache";
import { Request, Response, NextFunction } from "express"; import { Request, Response, NextFunction } from "express";
import { z } from "zod"; import { z } from "zod";
import { db } from "@server/db"; import { db } from "@server/db";
@@ -20,8 +19,7 @@ import { UserType } from "@server/types/UserTypes";
import { usageService } from "@server/lib/billing/usageService"; import { usageService } from "@server/lib/billing/usageService";
import { FeatureId } from "@server/lib/billing"; import { FeatureId } from "@server/lib/billing";
import { build } from "@server/build"; import { build } from "@server/build";
import cache from "@server/lib/cache";
const regenerateTracker = new NodeCache({ stdTTL: 3600, checkperiod: 600 });
const inviteUserParamsSchema = z const inviteUserParamsSchema = z
.object({ .object({
@@ -182,7 +180,7 @@ export async function inviteUser(
} }
if (existingInvite.length) { if (existingInvite.length) {
const attempts = regenerateTracker.get<number>(email) || 0; const attempts = cache.get<number>(email) || 0;
if (attempts >= 3) { if (attempts >= 3) {
return next( return next(
createHttpError( createHttpError(
@@ -192,7 +190,7 @@ export async function inviteUser(
); );
} }
regenerateTracker.set(email, attempts + 1); cache.set(email, attempts + 1);
const inviteId = existingInvite[0].inviteId; // Retrieve the original inviteId const inviteId = existingInvite[0].inviteId; // Retrieve the original inviteId
const token = generateRandomString( const token = generateRandomString(