diff --git a/server/lib/blueprints/applyBlueprint.ts b/server/lib/blueprints/applyBlueprint.ts new file mode 100644 index 00000000..6f5fc4ee --- /dev/null +++ b/server/lib/blueprints/applyBlueprint.ts @@ -0,0 +1,118 @@ +import { db, newts, Target } from "@server/db"; +import { Config, ConfigSchema } from "./types"; +import { ResourcesResults, updateResources } from "./resources"; +import { fromError } from "zod-validation-error"; +import logger from "@server/logger"; +import { resources, targets, sites } from "@server/db"; +import { eq, and, asc, or, ne, count, isNotNull } from "drizzle-orm"; +import { addTargets } from "@server/routers/newt/targets"; + +export async function applyBlueprint( + orgId: string, + configData: unknown, + siteId?: number +): Promise { + // Validate the input data + const validationResult = ConfigSchema.safeParse(configData); + if (!validationResult.success) { + throw new Error(fromError(validationResult.error).toString()); + } + + const config: Config = validationResult.data; + + try { + let resourcesResults: ResourcesResults = []; + await db.transaction(async (trx) => { + resourcesResults = await updateResources(orgId, config, trx, siteId); + }); + + logger.debug( + `Successfully updated resources for org ${orgId}: ${JSON.stringify(resourcesResults)}` + ); + + // We need to update the targets on the newts from the successfully updated information + for (const result of resourcesResults) { + for (const target of result.targetsToUpdate) { + const [site] = await db + .select() + .from(sites) + .innerJoin(newts, eq(sites.siteId, newts.siteId)) + .where( + and( + eq(sites.siteId, target.siteId), + eq(sites.orgId, orgId), + eq(sites.type, "newt"), + isNotNull(sites.pubKey) + ) + ) + .limit(1); + + if (site) { + logger.debug( + `Updating target ${target.targetId} on site ${site.sites.siteId}` + ); + + await addTargets( + site.newt.newtId, + [target], + result.resource.protocol, + result.resource.proxyPort + ); + } + } + } + } catch (error) { + logger.error(`Failed to update database from config: ${error}`); + throw error; + } +} + +// await updateDatabaseFromConfig("org_i21aifypnlyxur2", { +// resources: { +// "resource-nice-id": { +// name: "this is my resource", +// protocol: "http", +// "full-domain": "level1.test.example.com", +// "host-header": "example.com", +// "tls-server-name": "example.com", +// auth: { +// pincode: 123456, +// password: "sadfasdfadsf", +// "sso-enabled": true, +// "sso-roles": ["Member"], +// "sso-users": ["owen@fossorial.io"], +// "whitelist-users": ["owen@fossorial.io"] +// }, +// targets: [ +// { +// site: "glossy-plains-viscacha-rat", +// hostname: "localhost", +// method: "http", +// port: 8000, +// healthcheck: { +// port: 8000, +// hostname: "localhost" +// } +// }, +// { +// site: "glossy-plains-viscacha-rat", +// hostname: "localhost", +// method: "http", +// port: 8001 +// } +// ] +// }, + // "resource-nice-id2": { + // name: "http server", + // protocol: "tcp", + // "proxy-port": 3000, + // targets: [ + // { + // site: "glossy-plains-viscacha-rat", + // hostname: "localhost", + // port: 3000, + // } + // ] + // } +// } +// }); diff --git a/server/lib/blueprints/applyNewtDockerBlueprint.ts b/server/lib/blueprints/applyNewtDockerBlueprint.ts new file mode 100644 index 00000000..f69e4854 --- /dev/null +++ b/server/lib/blueprints/applyNewtDockerBlueprint.ts @@ -0,0 +1,53 @@ +import { sendToClient } from "@server/routers/ws"; +import { processContainerLabels } from "./parseDockerContainers"; +import { applyBlueprint } from "./applyBlueprint"; +import { db, sites } from "@server/db"; +import { eq } from "drizzle-orm"; +import logger from "@server/logger"; + +export async function applyNewtDockerBlueprint( + siteId: number, + newtId: string, + containers: any +) { + const [site] = await db + .select() + .from(sites) + .where(eq(sites.siteId, siteId)) + .limit(1); + + if (!site) { + logger.warn("Site not found in applyNewtDockerBlueprint"); + return; + } + + // logger.debug(`Applying Docker blueprint to site: ${siteId}`); + // logger.debug(`Containers: ${JSON.stringify(containers, null, 2)}`); + + try { + const blueprint = processContainerLabels(containers); + + logger.debug(`Received Docker blueprint: ${JSON.stringify(blueprint)}`); + + // Update the blueprint in the database + await applyBlueprint(site.orgId, blueprint, site.siteId); + } catch (error) { + logger.error(`Failed to update database from config: ${error}`); + await sendToClient(newtId, { + type: "newt/blueprint/results", + data: { + success: false, + message: `Failed to update database from config: ${error}` + } + }); + return; + } + + await sendToClient(newtId, { + type: "newt/blueprint/results", + data: { + success: true, + message: "Config updated successfully" + } + }); +} diff --git a/server/lib/blueprints/parseDockerContainers.ts b/server/lib/blueprints/parseDockerContainers.ts new file mode 100644 index 00000000..03ab609f --- /dev/null +++ b/server/lib/blueprints/parseDockerContainers.ts @@ -0,0 +1,284 @@ +import logger from "@server/logger"; +import { setNestedProperty } from "./parseDotNotation"; + +export type DockerLabels = { + [key: string]: string; +}; + +export type ParsedObject = { + [key: string]: any; +}; + +type ContainerPort = { + privatePort: number; + publicPort: number; + type: string; + ip: string; +}; + +type Container = { + id: string; + name: string; + image: string; + state: string; + status: string; + ports: ContainerPort[] | null; + labels: DockerLabels; + created: number; + networks: { [key: string]: any }; + hostname: string; +}; + +type Target = { + hostname?: string; + port?: number; + method?: string; + enabled?: boolean; + [key: string]: any; +}; + +type ResourceConfig = { + [key: string]: any; + targets?: (Target | null)[]; +}; + +function getContainerPort(container: Container): number | null { + if (!container.ports || container.ports.length === 0) { + return null; + } + // Return the first port's privatePort + return container.ports[0].privatePort; + // return container.ports[0].publicPort; +} + +export function processContainerLabels(containers: Container[]): { + resources: { [key: string]: ResourceConfig }; +} { + const result: { resources: { [key: string]: ResourceConfig } } = { + resources: {} + }; + + // Process each container + containers.forEach((container) => { + if (container.state !== "running") { + return; + } + + const resourceLabels: DockerLabels = {}; + + // Filter labels that start with "pangolin.resources." + Object.entries(container.labels).forEach(([key, value]) => { + if (key.startsWith("pangolin.resources.")) { + // remove the pangolin. prefix + const strippedKey = key.replace("pangolin.", ""); + resourceLabels[strippedKey] = value; + } + }); + + // Skip containers with no resource labels + if (Object.keys(resourceLabels).length === 0) { + return; + } + + // Parse the labels using the existing parseDockerLabels logic + const tempResult: ParsedObject = {}; + Object.entries(resourceLabels).forEach(([key, value]) => { + setNestedProperty(tempResult, key, value); + }); + + // Merge into main result + if (tempResult.resources) { + Object.entries(tempResult.resources).forEach( + ([resourceKey, resourceConfig]: [string, any]) => { + // Initialize resource if it doesn't exist + if (!result.resources[resourceKey]) { + result.resources[resourceKey] = {}; + } + + // Merge all properties except targets + Object.entries(resourceConfig).forEach( + ([propKey, propValue]) => { + if (propKey !== "targets") { + result.resources[resourceKey][propKey] = + propValue; + } + } + ); + + // Handle targets specially + if ( + resourceConfig.targets && + Array.isArray(resourceConfig.targets) + ) { + const resource = result.resources[resourceKey]; + if (resource) { + if (!resource.targets) { + resource.targets = []; + } + + resourceConfig.targets.forEach( + (target: any, targetIndex: number) => { + // check if the target is an empty object + if ( + typeof target === "object" && + Object.keys(target).length === 0 + ) { + logger.debug( + `Skipping null target at index ${targetIndex} for resource ${resourceKey}` + ); + resource.targets!.push(null); + return; + } + + // Ensure targets array is long enough + while ( + resource.targets!.length <= targetIndex + ) { + resource.targets!.push({}); + } + + // Set default hostname and port if not provided + const finalTarget = { ...target }; + if (!finalTarget.hostname) { + finalTarget.hostname = + container.name || + container.hostname; + } + if (!finalTarget.port) { + const containerPort = + getContainerPort(container); + if (containerPort !== null) { + finalTarget.port = containerPort; + } + } + + // Merge with existing target data + resource.targets![targetIndex] = { + ...resource.targets![targetIndex], + ...finalTarget + }; + } + ); + } + } + } + ); + } + }); + + return result; +} + +// // Test example +// const testContainers: Container[] = [ +// { +// id: "57e056cb0e3a", +// name: "nginx1", +// image: "nginxdemos/hello", +// state: "running", +// status: "Up 4 days", +// ports: [ +// { +// privatePort: 80, +// publicPort: 8000, +// type: "tcp", +// ip: "0.0.0.0" +// } +// ], +// labels: { +// "resources.nginx.name": "nginx", +// "resources.nginx.full-domain": "nginx.example.com", +// "resources.nginx.protocol": "http", +// "resources.nginx.targets[0].enabled": "true" +// }, +// created: 1756942725, +// networks: { +// owen_default: { +// networkId: +// "cb131c0f1d5d8ef7158660e77fc370508f5a563e1f9829b53a1945ae3725b58c" +// } +// }, +// hostname: "57e056cb0e3a" +// }, +// { +// id: "58e056cb0e3b", +// name: "nginx2", +// image: "nginxdemos/hello", +// state: "running", +// status: "Up 4 days", +// ports: [ +// { +// privatePort: 80, +// publicPort: 8001, +// type: "tcp", +// ip: "0.0.0.0" +// } +// ], +// labels: { +// "resources.nginx.name": "nginx", +// "resources.nginx.full-domain": "nginx.example.com", +// "resources.nginx.protocol": "http", +// "resources.nginx.targets[1].enabled": "true" +// }, +// created: 1756942726, +// networks: { +// owen_default: { +// networkId: +// "cb131c0f1d5d8ef7158660e77fc370508f5a563e1f9829b53a1945ae3725b58c" +// } +// }, +// hostname: "58e056cb0e3b" +// }, +// { +// id: "59e056cb0e3c", +// name: "api-server", +// image: "my-api:latest", +// state: "running", +// status: "Up 2 days", +// ports: [ +// { +// privatePort: 3000, +// publicPort: 3000, +// type: "tcp", +// ip: "0.0.0.0" +// } +// ], +// labels: { +// "resources.api.name": "API Server", +// "resources.api.protocol": "http", +// "resources.api.targets[0].enabled": "true", +// "resources.api.targets[0].hostname": "custom-host", +// "resources.api.targets[0].port": "3001" +// }, +// created: 1756942727, +// networks: { +// owen_default: { +// networkId: +// "cb131c0f1d5d8ef7158660e77fc370508f5a563e1f9829b53a1945ae3725b58c" +// } +// }, +// hostname: "59e056cb0e3c" +// }, +// { +// id: "d0e29b08361c", +// name: "beautiful_wilson", +// image: "bolkedebruin/rdpgw:latest", +// state: "exited", +// status: "Exited (0) 4 hours ago", +// ports: null, +// labels: {}, +// created: 1757359039, +// networks: { +// bridge: { +// networkId: +// "ea7f56dfc9cc476b8a3560b5b570d0fe8a6a2bc5e8343ab1ed37822086e89687" +// } +// }, +// hostname: "d0e29b08361c" +// } +// ]; + +// // Test the function +// const result = processContainerLabels(testContainers); +// console.log("Processed result:"); +// console.log(JSON.stringify(result, null, 2)); diff --git a/server/lib/blueprints/parseDotNotation.ts b/server/lib/blueprints/parseDotNotation.ts new file mode 100644 index 00000000..87509d39 --- /dev/null +++ b/server/lib/blueprints/parseDotNotation.ts @@ -0,0 +1,109 @@ +export function setNestedProperty(obj: any, path: string, value: string): void { + const keys = path.split("."); + let current = obj; + + for (let i = 0; i < keys.length - 1; i++) { + const key = keys[i]; + + // Handle array notation like "targets[0]" + const arrayMatch = key.match(/^(.+)\[(\d+)\]$/); + + if (arrayMatch) { + const [, arrayKey, indexStr] = arrayMatch; + const index = parseInt(indexStr, 10); + + // Initialize array if it doesn't exist + if (!current[arrayKey]) { + current[arrayKey] = []; + } + + // Ensure array is long enough + while (current[arrayKey].length <= index) { + current[arrayKey].push({}); + } + + current = current[arrayKey][index]; + } else { + // Regular object property + if (!current[key]) { + current[key] = {}; + } + current = current[key]; + } + } + + // Set the final value + const finalKey = keys[keys.length - 1]; + const arrayMatch = finalKey.match(/^(.+)\[(\d+)\]$/); + + if (arrayMatch) { + const [, arrayKey, indexStr] = arrayMatch; + const index = parseInt(indexStr, 10); + + if (!current[arrayKey]) { + current[arrayKey] = []; + } + + // Ensure array is long enough + while (current[arrayKey].length <= index) { + current[arrayKey].push(null); + } + + current[arrayKey][index] = convertValue(value); + } else { + current[finalKey] = convertValue(value); + } +} + +// Helper function to convert string values to appropriate types +export function convertValue(value: string): any { + // Convert boolean strings + if (value === "true") return true; + if (value === "false") return false; + + // Convert numeric strings + if (/^\d+$/.test(value)) { + const num = parseInt(value, 10); + return num; + } + + if (/^\d*\.\d+$/.test(value)) { + const num = parseFloat(value); + return num; + } + + // Return as string + return value; +} + +// // Example usage: +// const dockerLabels: DockerLabels = { +// "resources.resource-nice-id.name": "this is my resource", +// "resources.resource-nice-id.protocol": "http", +// "resources.resource-nice-id.full-domain": "level1.test3.example.com", +// "resources.resource-nice-id.host-header": "example.com", +// "resources.resource-nice-id.tls-server-name": "example.com", +// "resources.resource-nice-id.auth.pincode": "123456", +// "resources.resource-nice-id.auth.password": "sadfasdfadsf", +// "resources.resource-nice-id.auth.sso-enabled": "true", +// "resources.resource-nice-id.auth.sso-roles[0]": "Member", +// "resources.resource-nice-id.auth.sso-users[0]": "owen@fossorial.io", +// "resources.resource-nice-id.auth.whitelist-users[0]": "owen@fossorial.io", +// "resources.resource-nice-id.targets[0].hostname": "localhost", +// "resources.resource-nice-id.targets[0].method": "http", +// "resources.resource-nice-id.targets[0].port": "8000", +// "resources.resource-nice-id.targets[0].healthcheck.port": "8000", +// "resources.resource-nice-id.targets[0].healthcheck.hostname": "localhost", +// "resources.resource-nice-id.targets[1].hostname": "localhost", +// "resources.resource-nice-id.targets[1].method": "http", +// "resources.resource-nice-id.targets[1].port": "8001", +// "resources.resource-nice-id2.name": "this is other resource", +// "resources.resource-nice-id2.protocol": "tcp", +// "resources.resource-nice-id2.proxy-port": "3000", +// "resources.resource-nice-id2.targets[0].hostname": "localhost", +// "resources.resource-nice-id2.targets[0].port": "3000" +// }; + +// // Parse the labels +// const parsed = parseDockerLabels(dockerLabels); +// console.log(JSON.stringify(parsed, null, 2)); diff --git a/server/lib/blueprints/resources.ts b/server/lib/blueprints/resources.ts new file mode 100644 index 00000000..e1c1209f --- /dev/null +++ b/server/lib/blueprints/resources.ts @@ -0,0 +1,745 @@ +import { + domains, + orgDomains, + Resource, + resourcePincode, + resourceWhitelist, + roleResources, + roles, + Target, + Transaction, + userOrgs, + userResources, + users +} from "@server/db"; +import { resources, targets, sites } from "@server/db"; +import { eq, and, asc, or, ne, count, isNotNull } from "drizzle-orm"; +import { Config, ConfigSchema, isTargetsOnlyResource, TargetData } from "./types"; +import logger from "@server/logger"; +import { pickPort } from "@server/routers/target/helpers"; +import { resourcePassword } from "@server/db"; +import { hashPassword } from "@server/auth/password"; + +export type ResourcesResults = { + resource: Resource; + targetsToUpdate: Target[]; +}[]; + +export async function updateResources( + orgId: string, + config: Config, + trx: Transaction, + siteId?: number +): Promise { + let results: ResourcesResults = []; + + for (const [resourceNiceId, resourceData] of Object.entries( + config.resources + )) { + let targetsToUpdate: Target[] = []; + let resource: Resource; + + async function createTarget( // reusable function to create a target + resourceId: number, + targetData: TargetData + ) { + let targetSiteId = targetData.site; + let site; + + if (targetSiteId) { + // Look up site by niceId + [site] = await trx + .select({ siteId: sites.siteId }) + .from(sites) + .where( + and( + eq(sites.niceId, targetSiteId), + eq(sites.orgId, orgId) + ) + ) + .limit(1); + } else if (siteId) { + // Use the provided siteId directly, but verify it belongs to the org + [site] = await trx + .select({ siteId: sites.siteId }) + .from(sites) + .where( + and(eq(sites.siteId, siteId), eq(sites.orgId, orgId)) + ) + .limit(1); + } else { + throw new Error(`Target site ID is required`); + } + + if (!site) { + throw new Error( + `Site not found: ${targetSiteId} in org ${orgId}` + ); + } + + let internalPortToCreate; + if (!targetData["internal-port"]) { + const { internalPort, targetIps } = await pickPort( + site.siteId!, + trx + ); + internalPortToCreate = internalPort; + } else { + internalPortToCreate = targetData["internal-port"]; + } + + // Create target + const [newTarget] = await trx + .insert(targets) + .values({ + resourceId: resourceId, + siteId: site.siteId, + ip: targetData.hostname, + method: targetData.method, + port: targetData.port, + enabled: targetData.enabled, + internalPort: internalPortToCreate + }) + .returning(); + + targetsToUpdate.push(newTarget); + } + + // Find existing resource by niceId and orgId + const [existingResource] = await trx + .select() + .from(resources) + .where( + and( + eq(resources.niceId, resourceNiceId), + eq(resources.orgId, orgId) + ) + ) + .limit(1); + + const http = resourceData.protocol == "http"; + const protocol = + resourceData.protocol == "http" ? "tcp" : resourceData.protocol; + + if (existingResource) { + let domain; + if (http) { + domain = await getDomain( + existingResource.resourceId, + resourceData["full-domain"]!, + orgId, + trx + ); + } + + // check if the only key in the resource is targets, if so, skip the update + if ( + isTargetsOnlyResource(resourceData) + ) { + logger.debug( + `Skipping update for resource ${existingResource.resourceId} as only targets are provided` + ); + resource = existingResource; + } else { + // Update existing resource + [resource] = await trx + .update(resources) + .set({ + name: resourceData.name || "Unnamed Resource", + protocol: protocol || "http", + http: http, + proxyPort: http ? null : resourceData["proxy-port"], + fullDomain: http ? resourceData["full-domain"] : null, + subdomain: domain ? domain.subdomain : null, + domainId: domain ? domain.domainId : null, + enabled: resourceData.enabled ? true : false, + sso: resourceData.auth?.["sso-enabled"] || false, + ssl: resourceData.ssl ? true : false, + setHostHeader: resourceData["host-header"] || null, + tlsServerName: resourceData["tls-server-name"] || null, + emailWhitelistEnabled: resourceData.auth?.[ + "whitelist-users" + ] + ? resourceData.auth["whitelist-users"].length > 0 + : false + }) + .where( + eq(resources.resourceId, existingResource.resourceId) + ) + .returning(); + + await trx + .delete(resourcePassword) + .where( + eq( + resourcePassword.resourceId, + existingResource.resourceId + ) + ); + if (resourceData.auth?.password) { + const passwordHash = await hashPassword( + resourceData.auth.password + ); + + await trx.insert(resourcePassword).values({ + resourceId: existingResource.resourceId, + passwordHash + }); + } + + await trx + .delete(resourcePincode) + .where( + eq( + resourcePincode.resourceId, + existingResource.resourceId + ) + ); + if (resourceData.auth?.pincode) { + const pincodeHash = await hashPassword( + resourceData.auth.pincode.toString() + ); + + await trx.insert(resourcePincode).values({ + resourceId: existingResource.resourceId, + pincodeHash, + digitLength: 6 + }); + } + + if (resourceData.auth?.["sso-roles"]) { + const ssoRoles = resourceData.auth?.["sso-roles"]; + await syncRoleResources( + existingResource.resourceId, + ssoRoles, + orgId, + trx + ); + } + + if (resourceData.auth?.["sso-users"]) { + const ssoUsers = resourceData.auth?.["sso-users"]; + await syncUserResources( + existingResource.resourceId, + ssoUsers, + orgId, + trx + ); + } + + if (resourceData.auth?.["whitelist-users"]) { + const whitelistUsers = + resourceData.auth?.["whitelist-users"]; + await syncWhitelistUsers( + existingResource.resourceId, + whitelistUsers, + orgId, + trx + ); + } + } + + const existingResourceTargets = await trx + .select() + .from(targets) + .where(eq(targets.resourceId, existingResource.resourceId)) + .orderBy(asc(targets.targetId)); + + // Create new targets + for (const [index, targetData] of resourceData.targets.entries()) { + if (!targetData) { + // If targetData is null or an empty object, we can skip it + continue; + } + const existingTarget = existingResourceTargets[index]; + if (existingTarget) { + let targetSiteId = targetData.site; + let site; + + if (targetSiteId) { + // Look up site by niceId + [site] = await trx + .select({ siteId: sites.siteId }) + .from(sites) + .where( + and( + eq(sites.niceId, targetSiteId), + eq(sites.orgId, orgId) + ) + ) + .limit(1); + } else if (siteId) { + // Use the provided siteId directly, but verify it belongs to the org + [site] = await trx + .select({ siteId: sites.siteId }) + .from(sites) + .where( + and( + eq(sites.siteId, siteId), + eq(sites.orgId, orgId) + ) + ) + .limit(1); + } else { + throw new Error(`Target site ID is required`); + } + + if (!site) { + throw new Error( + `Site not found: ${targetSiteId} in org ${orgId}` + ); + } + + // update this target + const [updatedTarget] = await trx + .update(targets) + .set({ + siteId: site.siteId, + ip: targetData.hostname, + method: http ? targetData.method : null, + port: targetData.port, + enabled: targetData.enabled + }) + .where(eq(targets.targetId, existingTarget.targetId)) + .returning(); + + if (checkIfTargetChanged(existingTarget, updatedTarget)) { + let internalPortToUpdate; + if (!targetData["internal-port"]) { + const { internalPort, targetIps } = await pickPort( + site.siteId!, + trx + ); + internalPortToUpdate = internalPort; + } else { + internalPortToUpdate = targetData["internal-port"]; + } + + const [finalUpdatedTarget] = await trx // this double is so we can check the whole target before and after + .update(targets) + .set({ + internalPort: internalPortToUpdate + }) + .where( + eq(targets.targetId, existingTarget.targetId) + ) + .returning(); + + targetsToUpdate.push(finalUpdatedTarget); + } + } else { + await createTarget(existingResource.resourceId, targetData); + } + } + + if (existingResourceTargets.length > resourceData.targets.length) { + const targetsToDelete = existingResourceTargets.slice( + resourceData.targets.length + ); + for (const target of targetsToDelete) { + await trx + .delete(targets) + .where(eq(targets.targetId, target.targetId)); + } + } + + logger.debug(`Updated resource ${existingResource.resourceId}`); + } else { + // create a brand new resource + let domain; + if (http) { + domain = await getDomain( + undefined, + resourceData["full-domain"]!, + orgId, + trx + ); + } + + // Create new resource + const [newResource] = await trx + .insert(resources) + .values({ + orgId, + niceId: resourceNiceId, + name: resourceData.name || "Unnamed Resource", + protocol: resourceData.protocol || "http", + http: http, + proxyPort: http ? null : resourceData["proxy-port"], + fullDomain: http ? resourceData["full-domain"] : null, + subdomain: domain ? domain.subdomain : null, + domainId: domain ? domain.domainId : null, + enabled: resourceData.enabled ? true : false, + sso: resourceData.auth?.["sso-enabled"] || false, + setHostHeader: resourceData["host-header"] || null, + tlsServerName: resourceData["tls-server-name"] || null, + ssl: resourceData.ssl ? true : false + }) + .returning(); + + if (resourceData.auth?.password) { + const passwordHash = await hashPassword( + resourceData.auth.password + ); + + await trx.insert(resourcePassword).values({ + resourceId: newResource.resourceId, + passwordHash + }); + } + + if (resourceData.auth?.pincode) { + const pincodeHash = await hashPassword( + resourceData.auth.pincode.toString() + ); + + await trx.insert(resourcePincode).values({ + resourceId: newResource.resourceId, + pincodeHash, + digitLength: 6 + }); + } + + resource = newResource; + + const [adminRole] = await trx + .select() + .from(roles) + .where(and(eq(roles.isAdmin, true), eq(roles.orgId, orgId))) + .limit(1); + + if (!adminRole) { + throw new Error(`Admin role not found`); + } + + await trx.insert(roleResources).values({ + roleId: adminRole.roleId, + resourceId: newResource.resourceId + }); + + if (resourceData.auth?.["sso-roles"]) { + const ssoRoles = resourceData.auth?.["sso-roles"]; + await syncRoleResources( + newResource.resourceId, + ssoRoles, + orgId, + trx + ); + } + + if (resourceData.auth?.["sso-users"]) { + const ssoUsers = resourceData.auth?.["sso-users"]; + await syncUserResources( + newResource.resourceId, + ssoUsers, + orgId, + trx + ); + } + + if (resourceData.auth?.["whitelist-users"]) { + const whitelistUsers = resourceData.auth?.["whitelist-users"]; + await syncWhitelistUsers( + newResource.resourceId, + whitelistUsers, + orgId, + trx + ); + } + + // Create new targets + for (const targetData of resourceData.targets) { + if (!targetData) { + // If targetData is null or an empty object, we can skip it + continue; + } + await createTarget(newResource.resourceId, targetData); + } + + logger.debug(`Created resource ${newResource.resourceId}`); + } + + results.push({ + resource: resource, + targetsToUpdate, + }); + } + + return results; +} + +async function syncRoleResources( + resourceId: number, + ssoRoles: string[], + orgId: string, + trx: Transaction +) { + const existingRoleResources = await trx + .select() + .from(roleResources) + .where(eq(roleResources.resourceId, resourceId)); + + for (const roleName of ssoRoles) { + if (roleName === "Admin") { + continue; // never add admin access + } + + const [role] = await trx + .select() + .from(roles) + .where(and(eq(roles.name, roleName), eq(roles.orgId, orgId))) + .limit(1); + + if (!role) { + throw new Error(`Role not found: ${roleName} in org ${orgId}`); + } + + const existingRoleResource = existingRoleResources.find( + (rr) => rr.roleId === role.roleId + ); + + if (!existingRoleResource) { + await trx.insert(roleResources).values({ + roleId: role.roleId, + resourceId: resourceId + }); + } + } + + for (const existingRoleResource of existingRoleResources) { + const [role] = await trx + .select() + .from(roles) + .where(eq(roles.roleId, existingRoleResource.roleId)) + .limit(1); + + if (role.isAdmin) { + continue; // never remove admin access + } + + if (role && !ssoRoles.includes(role.name)) { + await trx + .delete(roleResources) + .where( + and( + eq(roleResources.roleId, existingRoleResource.roleId), + eq(roleResources.resourceId, resourceId) + ) + ); + } + } +} + +async function syncUserResources( + resourceId: number, + ssoUsers: string[], + orgId: string, + trx: Transaction +) { + const existingUserResources = await trx + .select() + .from(userResources) + .where(eq(userResources.resourceId, resourceId)); + + for (const email of ssoUsers) { + const [user] = await trx + .select() + .from(users) + .innerJoin(userOrgs, eq(users.userId, userOrgs.userId)) + .where(and(eq(users.email, email), eq(userOrgs.orgId, orgId))) + .limit(1); + + if (!user) { + throw new Error(`User not found: ${email} in org ${orgId}`); + } + + const existingUserResource = existingUserResources.find( + (rr) => rr.userId === user.user.userId + ); + + if (!existingUserResource) { + await trx.insert(userResources).values({ + userId: user.user.userId, + resourceId: resourceId + }); + } + } + + for (const existingUserResource of existingUserResources) { + const [user] = await trx + .select() + .from(users) + .innerJoin(userOrgs, eq(users.userId, userOrgs.userId)) + .where( + and( + eq(users.userId, existingUserResource.userId), + eq(userOrgs.orgId, orgId) + ) + ) + .limit(1); + + if (user && user.user.email && !ssoUsers.includes(user.user.email)) { + await trx + .delete(userResources) + .where( + and( + eq(userResources.userId, existingUserResource.userId), + eq(userResources.resourceId, resourceId) + ) + ); + } + } +} + +async function syncWhitelistUsers( + resourceId: number, + whitelistUsers: string[], + orgId: string, + trx: Transaction +) { + const existingWhitelist = await trx + .select() + .from(resourceWhitelist) + .where(eq(resourceWhitelist.resourceId, resourceId)); + + for (const email of whitelistUsers) { + const [user] = await trx + .select() + .from(users) + .innerJoin(userOrgs, eq(users.userId, userOrgs.userId)) + .where(and(eq(users.email, email), eq(userOrgs.orgId, orgId))) + .limit(1); + + if (!user) { + throw new Error(`User not found: ${email} in org ${orgId}`); + } + + const existingWhitelistEntry = existingWhitelist.find( + (w) => w.email === email + ); + + if (!existingWhitelistEntry) { + await trx.insert(resourceWhitelist).values({ + email, + resourceId: resourceId + }); + } + } + + for (const existingWhitelistEntry of existingWhitelist) { + if (!whitelistUsers.includes(existingWhitelistEntry.email)) { + await trx + .delete(resourceWhitelist) + .where( + and( + eq(resourceWhitelist.resourceId, resourceId), + eq( + resourceWhitelist.email, + existingWhitelistEntry.email + ) + ) + ); + } + } +} + +function checkIfTargetChanged( + existing: Target | undefined, + incoming: Target | undefined +): boolean { + if (!existing && incoming) return true; + if (existing && !incoming) return true; + if (!existing || !incoming) return false; + + if (existing.ip !== incoming.ip) return true; + if (existing.port !== incoming.port) return true; + if (existing.siteId !== incoming.siteId) return true; + + return false; +} + +async function getDomain( + resourceId: number | undefined, + fullDomain: string, + orgId: string, + trx: Transaction +) { + const [fullDomainExists] = await trx + .select({ resourceId: resources.resourceId }) + .from(resources) + .where( + and( + eq(resources.fullDomain, fullDomain), + eq(resources.orgId, orgId), + resourceId + ? ne(resources.resourceId, resourceId) + : isNotNull(resources.resourceId) + ) + ) + .limit(1); + + if (fullDomainExists) { + throw new Error( + `Resource already exists: ${fullDomain} in org ${orgId}` + ); + } + + const domain = await getDomainId(orgId, fullDomain, trx); + + if (!domain) { + throw new Error( + `Domain not found for full-domain: ${fullDomain} in org ${orgId}` + ); + } + + return domain; +} + +async function getDomainId( + orgId: string, + fullDomain: string, + trx: Transaction +): Promise<{ subdomain: string | null; domainId: string } | null> { + const possibleDomains = await trx + .select() + .from(domains) + .innerJoin(orgDomains, eq(domains.domainId, orgDomains.domainId)) + .where(and(eq(orgDomains.orgId, orgId), eq(domains.verified, true))) + .execute(); + + if (possibleDomains.length === 0) { + return null; + } + + const validDomains = possibleDomains.filter((domain) => { + if (domain.domains.type == "ns") { + return ( + fullDomain === domain.domains.baseDomain || + fullDomain.endsWith(`.${domain.domains.baseDomain}`) + ); + } else if (domain.domains.type == "cname") { + return fullDomain === domain.domains.baseDomain; + } + }); + + if (validDomains.length === 0) { + return null; + } + + const domainSelection = validDomains[0].domains; + const baseDomain = domainSelection.baseDomain; + + // remove the base domain of the domain + let subdomain = null; + if (domainSelection.type == "ns") { + if (fullDomain != baseDomain) { + subdomain = fullDomain.replace(`.${baseDomain}`, ""); + } + } + + // Return the first valid domain + return { + subdomain: subdomain, + domainId: domainSelection.domainId + }; +} diff --git a/server/lib/blueprints/types.ts b/server/lib/blueprints/types.ts new file mode 100644 index 00000000..62e390a8 --- /dev/null +++ b/server/lib/blueprints/types.ts @@ -0,0 +1,232 @@ +import { z } from "zod"; + +export const SiteSchema = z.object({ + name: z.string().min(1).max(100), + "docker-socket-enabled": z.boolean().optional().default(true) +}); + +// Schema for individual target within a resource +export const TargetSchema = z.object({ + site: z.string().optional(), + method: z.enum(["http", "https", "h2c"]).optional(), + hostname: z.string(), + port: z.number().int().min(1).max(65535), + enabled: z.boolean().optional().default(true), + "internal-port": z.number().int().min(1).max(65535).optional(), +}); +export type TargetData = z.infer; + +export const AuthSchema = z.object({ + // pincode has to have 6 digits + pincode: z.number().min(100000).max(999999).optional(), + password: z.string().min(1).optional(), + "sso-enabled": z.boolean().optional().default(false), + "sso-roles": z + .array(z.string()) + .optional() + .default([]) + .refine((roles) => !roles.includes("Admin"), { + message: "Admin role cannot be included in sso-roles" + }), + "sso-users": z.array(z.string().email()).optional().default([]), + "whitelist-users": z.array(z.string().email()).optional().default([]) +}); + +// Schema for individual resource +export const ResourceSchema = z + .object({ + name: z.string().optional(), + protocol: z.enum(["http", "tcp", "udp"]).optional(), + ssl: z.boolean().optional(), + "full-domain": z.string().optional(), + "proxy-port": z.number().int().min(1).max(65535).optional(), + enabled: z.boolean().optional(), + targets: z.array(TargetSchema.nullable()).optional().default([]), + auth: AuthSchema.optional(), + "host-header": z.string().optional(), + "tls-server-name": z.string().optional() + }) + .refine( + (resource) => { + if (isTargetsOnlyResource(resource)) { + return true; + } + + // Otherwise, require name and protocol for full resource definition + return ( + resource.name !== undefined && resource.protocol !== undefined + ); + }, + { + message: + "Resource must either be targets-only (only 'targets' field) or have both 'name' and 'protocol' fields at a minimum", + path: ["name", "protocol"] + } + ) + .refine( + (resource) => { + if (isTargetsOnlyResource(resource)) { + return true; + } + + // If protocol is http, all targets must have method field + if (resource.protocol === "http") { + return resource.targets.every( + (target) => target == null || target.method !== undefined + ); + } + // If protocol is tcp or udp, no target should have method field + if (resource.protocol === "tcp" || resource.protocol === "udp") { + return resource.targets.every( + (target) => target == null || target.method === undefined + ); + } + return true; + }, + (resource) => { + if (resource.protocol === "http") { + return { + message: + "When protocol is 'http', all targets must have a 'method' field", + path: ["targets"] + }; + } + return { + message: + "When protocol is 'tcp' or 'udp', targets must not have a 'method' field", + path: ["targets"] + }; + } + ) + .refine( + (resource) => { + if (isTargetsOnlyResource(resource)) { + return true; + } + + // If protocol is http, it must have a full-domain + if (resource.protocol === "http") { + return ( + resource["full-domain"] !== undefined && + resource["full-domain"].length > 0 + ); + } + return true; + }, + { + message: + "When protocol is 'http', a 'full-domain' must be provided", + path: ["full-domain"] + } + ) + .refine( + (resource) => { + if (isTargetsOnlyResource(resource)) { + return true; + } + + // If protocol is tcp or udp, it must have both proxy-port + if (resource.protocol === "tcp" || resource.protocol === "udp") { + return resource["proxy-port"] !== undefined; + } + return true; + }, + { + message: + "When protocol is 'tcp' or 'udp', 'proxy-port' must be provided", + path: ["proxy-port", "exit-node"] + } + ) + .refine( + (resource) => { + // Skip validation for targets-only resources + if (isTargetsOnlyResource(resource)) { + return true; + } + + // If protocol is tcp or udp, it must not have auth + if (resource.protocol === "tcp" || resource.protocol === "udp") { + return resource.auth === undefined; + } + return true; + }, + { + message: + "When protocol is 'tcp' or 'udp', 'auth' must not be provided", + path: ["auth"] + } + ); + +export function isTargetsOnlyResource(resource: any): boolean { + return Object.keys(resource).length === 1 && resource.targets; +} + +// Schema for the entire configuration object +export const ConfigSchema = z + .object({ + resources: z.record(z.string(), ResourceSchema).optional().default({}), + sites: z.record(z.string(), SiteSchema).optional().default({}) + }) + .refine( + // Enforce the full-domain uniqueness across resources in the same stack + (config) => { + // Extract all full-domain values with their resource keys + const fullDomainMap = new Map(); + + Object.entries(config.resources).forEach( + ([resourceKey, resource]) => { + const fullDomain = resource["full-domain"]; + if (fullDomain) { + // Only process if full-domain is defined + if (!fullDomainMap.has(fullDomain)) { + fullDomainMap.set(fullDomain, []); + } + fullDomainMap.get(fullDomain)!.push(resourceKey); + } + } + ); + + // Find duplicates + const duplicates = Array.from(fullDomainMap.entries()).filter( + ([_, resourceKeys]) => resourceKeys.length > 1 + ); + + return duplicates.length === 0; + }, + (config) => { + // Extract duplicates for error message + const fullDomainMap = new Map(); + + Object.entries(config.resources).forEach( + ([resourceKey, resource]) => { + const fullDomain = resource["full-domain"]; + if (fullDomain) { + // Only process if full-domain is defined + if (!fullDomainMap.has(fullDomain)) { + fullDomainMap.set(fullDomain, []); + } + fullDomainMap.get(fullDomain)!.push(resourceKey); + } + } + ); + + const duplicates = Array.from(fullDomainMap.entries()) + .filter(([_, resourceKeys]) => resourceKeys.length > 1) + .map( + ([fullDomain, resourceKeys]) => + `'${fullDomain}' used by resources: ${resourceKeys.join(", ")}` + ) + .join("; "); + + return { + message: `Duplicate 'full-domain' values found: ${duplicates}`, + path: ["resources"] + }; + } + ); + +// Type inference from the schema +export type Site = z.infer; +export type Target = z.infer; +export type Resource = z.infer; +export type Config = z.infer; diff --git a/server/routers/newt/handleApplyBlueprintMessage.ts b/server/routers/newt/handleApplyBlueprintMessage.ts new file mode 100644 index 00000000..68158799 --- /dev/null +++ b/server/routers/newt/handleApplyBlueprintMessage.ts @@ -0,0 +1,73 @@ +import { db, newts } from "@server/db"; +import { MessageHandler } from "../ws"; +import { exitNodes, Newt, resources, sites, Target, targets } from "@server/db"; +import { eq, and, sql, inArray } from "drizzle-orm"; +import logger from "@server/logger"; +import { applyBlueprint } from "@server/lib/blueprints/applyBlueprint"; + +export const handleApplyBlueprintMessage: MessageHandler = async (context) => { + const { message, client, sendToClient } = context; + const newt = client as Newt; + + logger.debug("Handling apply blueprint message!"); + + if (!newt) { + logger.warn("Newt not found"); + return; + } + + if (!newt.siteId) { + logger.warn("Newt has no site!"); // TODO: Maybe we create the site here? + return; + } + + // get the site + const [site] = await db + .select() + .from(sites) + .where(eq(sites.siteId, newt.siteId)); + + if (!site) { + logger.warn("Site not found for newt"); + return; + } + + const { blueprint } = message.data; + if (!blueprint) { + logger.warn("No blueprint provided"); + return; + } + + logger.debug(`Received blueprint: ${blueprint}`); + + try { + const blueprintParsed = JSON.parse(blueprint); + // Update the blueprint in the database + await applyBlueprint(site.orgId, blueprintParsed, site.siteId); + } catch (error) { + logger.error(`Failed to update database from config: ${error}`); + return { + message: { + type: "newt/blueprint/results", + data: { + success: false, + message: `Failed to update database from config: ${error}` + } + }, + broadcast: false, // Send to all clients + excludeSender: false // Include sender in broadcast + }; + } + + return { + message: { + type: "newt/blueprint/results", + data: { + success: true, + message: "Config updated successfully" + } + }, + broadcast: false, // Send to all clients + excludeSender: false // Include sender in broadcast + }; +}; diff --git a/server/routers/newt/handleNewtRegisterMessage.ts b/server/routers/newt/handleNewtRegisterMessage.ts index 3c7ecaff..eef78765 100644 --- a/server/routers/newt/handleNewtRegisterMessage.ts +++ b/server/routers/newt/handleNewtRegisterMessage.ts @@ -10,6 +10,7 @@ import { getNextAvailableClientSubnet } from "@server/lib/ip"; import { selectBestExitNode, verifyExitNodeOrgAccess } from "@server/lib/exitNodes"; +import { fetchContainers } from "./dockerSocket"; export type ExitNodePingResult = { exitNodeId: number; @@ -76,6 +77,15 @@ export const handleNewtRegisterMessage: MessageHandler = async (context) => { return; } + logger.debug(`Docker socket enabled: ${oldSite.dockerSocketEnabled}`); + + if (oldSite.dockerSocketEnabled) { + logger.debug( + "Site has docker socket enabled - requesting docker containers" + ); + fetchContainers(newt.newtId); + } + let siteSubnet = oldSite.subnet; let exitNodeIdToQuery = oldSite.exitNodeId; if (exitNodeId && (oldSite.exitNodeId !== exitNodeId || !oldSite.subnet)) { diff --git a/server/routers/newt/handleSocketMessages.ts b/server/routers/newt/handleSocketMessages.ts index 01b7be60..aceca37d 100644 --- a/server/routers/newt/handleSocketMessages.ts +++ b/server/routers/newt/handleSocketMessages.ts @@ -2,6 +2,7 @@ import { MessageHandler } from "../ws"; import logger from "@server/logger"; import { dockerSocketCache } from "./dockerSocket"; import { Newt } from "@server/db"; +import { applyNewtDockerBlueprint } from "@server/lib/blueprints/applyNewtDockerBlueprint"; export const handleDockerStatusMessage: MessageHandler = async (context) => { const { message, client, sendToClient } = context; @@ -57,4 +58,15 @@ export const handleDockerContainersMessage: MessageHandler = async ( } else { logger.warn(`Newt ${newt.newtId} does not have Docker containers`); } + + if (!newt.siteId) { + logger.warn("Newt has no site!"); + return; + } + + await applyNewtDockerBlueprint( + newt.siteId, + newt.newtId, + containers + ); }; diff --git a/server/routers/newt/index.ts b/server/routers/newt/index.ts index 08f047e3..9642a637 100644 --- a/server/routers/newt/index.ts +++ b/server/routers/newt/index.ts @@ -4,4 +4,5 @@ export * from "./handleNewtRegisterMessage"; export * from "./handleReceiveBandwidthMessage"; export * from "./handleGetConfigMessage"; export * from "./handleSocketMessages"; -export * from "./handleNewtPingRequestMessage"; \ No newline at end of file +export * from "./handleNewtPingRequestMessage"; +export * from "./handleApplyBlueprintMessage"; \ No newline at end of file diff --git a/server/routers/ws/messageHandlers.ts b/server/routers/ws/messageHandlers.ts index a30daf43..8ca33b8a 100644 --- a/server/routers/ws/messageHandlers.ts +++ b/server/routers/ws/messageHandlers.ts @@ -4,7 +4,8 @@ import { handleGetConfigMessage, handleDockerStatusMessage, handleDockerContainersMessage, - handleNewtPingRequestMessage + handleNewtPingRequestMessage, + handleApplyBlueprintMessage } from "../newt"; import { handleOlmRegisterMessage, @@ -23,7 +24,8 @@ export const messageHandlers: Record = { "olm/ping": handleOlmPingMessage, "newt/socket/status": handleDockerStatusMessage, "newt/socket/containers": handleDockerContainersMessage, - "newt/ping/request": handleNewtPingRequestMessage + "newt/ping/request": handleNewtPingRequestMessage, + "newt/blueprint/apply": handleApplyBlueprintMessage, }; startOlmOfflineChecker(); // this is to handle the offline check for olms