diff --git a/.gitignore b/.gitignore index cd66e7e28..9506fd0cf 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,11 @@ dist/ .env !.env.example +# Per-developer / deployment config override. `config.default.json` ships in +# the repo and is the authoritative source; `config.json` (if present) wins +# at runtime for the local machine only. +config.json + # this is for jetbrain IDEs .idea/ /puter @@ -63,3 +68,5 @@ AGENTS.md coverage/ *.log undefined +servers.json +config.*.json diff --git a/.husky/pre-commit b/.husky/pre-commit old mode 100644 new mode 100755 index 86db09d5a..20df1a7ed --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,13 +1,15 @@ #!/usr/bin/env sh -tmpfile="$(mktemp)" -git diff --cached --name-only -z --diff-filter=ACMR -- \ - '*.js' '*.mjs' '*.cjs' '*.jsx' '*.ts' '*.tsx' '*.vue' \ - > "$tmpfile" +files=$(git diff --cached --name-only --diff-filter=ACMR | \ + grep -E '^(src/backend|extensions)/.*\.(js|mjs|cjs|ts)$' || true) -if [ -s "$tmpfile" ]; then - xargs -0 eslint --fix < "$tmpfile" || true - xargs -0 git add < "$tmpfile" +if [ -z "$files" ]; then + exit 0 fi -rm -f "$tmpfile" +echo "$files" | xargs npx eslint --fix --no-warn-ignored +status=$? + +echo "$files" | xargs git add + +exit $status diff --git a/.is_puter_repository b/.is_puter_repository deleted file mode 100644 index e69de29bb..000000000 diff --git a/.prettierignore b/.prettierignore index 0e796c3f1..2c6cc8099 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,4 @@ node_modules dist -build \ No newline at end of file +build +**/*.dbmig.js \ No newline at end of file diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 000000000..9b0bb437a --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,4 @@ +{ + "tabWidth": 4, + "singleQuote": true +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 916be7a53..000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,78 +0,0 @@ -# Contributing to Puter - -Welcome to Puter, the open-source distributed internet operating system. We're excited to have you contribute to our project, whether you're reporting bugs, suggesting new features, or contributing code. This guide will help you get started with contributing to Puter in different ways. - -
- -# Report bugs - -Before reporting a bug, please check [the issues on our GitHub repository](https://github.com/HeyPuter/puter/issues) to see if the bug has already been reported. If it has, you can add a comment to the existing issue with any additional information you have. - -If you find a new bug in Puter, please [open an issue on our GitHub repository](https://github.com/HeyPuter/puter/issues/new). We'll do our best to address the issue as soon as possible. When reporting a bug, please include as much information as possible, including: - -- A clear and descriptive title -- A description of the issue -- Steps to reproduce the bug -- Expected behavior -- Actual behavior -- Screenshots, if applicable -- Your host operating system and browser -- Your Puter version, location, ... - -Please open a separate issue for each bug you find. - -Maintainers will apply the appropriate labels to your issue. - -
- -# Suggest new features - -If you have an idea for a new feature in Puter, please open a new discussion thread on our [GitHub repository](https://github.com/HeyPuter/puter/discussions) to discuss your idea with the community. We'll do our best to respond to your suggestion as soon as possible. - -When suggesting a new feature, please include as much information as possible, including: - -- A clear and descriptive title -- A description of the feature -- The problem the feature will solve -- Any relevant screenshots or mockups -- Any relevant links or resources - -
- -# Contribute code - -If you'd like to contribute code to Puter, you need to fork the project and submit a pull request. If this is your first time contributing to an open-source project, we recommend reading this short guide by GitHub on [how to contribute to a project](https://docs.github.com/en/get-started/exploring-projects-on-github/contributing-to-a-project). - -We'll review your pull request and work with you to get your changes merged into the project. - -
- -## PR Standards - -We expect the following from pull requests (it makes things easier): -- If you're closing an issue, please reference that issue in the PR description -- Avoid whitespace changes -- No regressions for "appspace" (Puter apps) - -
- -## Code Review - -Once you've submitted your pull request, the project maintainers will review your changes. We may suggest some changes or improvements. This is a normal part of the process, and your contributions are greatly appreciated! - -
- -## Contribution License Agreement (CLA) - -Like many open source projects, we require contributors to sign a Contribution License Agreement (CLA) before we can accept your code. When you open a pull request for the first time, a bot will automatically add a comment with a link to the CLA. You can sign the CLA electronically by following the link and filling out the form. - -
- -# Getting Help - -If you have any questions about Puter, please feel free to reach out to us through the following channels: - -- [Discord](https://discord.com/invite/PQcx7Teh8u) -- [Reddit](https://www.reddit.com/r/Puter/) -- [Twitter](https://twitter.com/HeyPuter) -- [Email](mailto:support@puter.com) diff --git a/Dockerfile b/Dockerfile index 5c5dc0e4c..b9abf58ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -82,6 +82,7 @@ HEALTHCHECK --interval=30s --timeout=3s \ CMD wget --no-verbose --tries=1 --spider http://puter.localhost:4100/test || exit 1 ENV NO_VAR_RUNTUME=1 +ENV NODE_OPTIONS=--enable-source-maps # Attempt to fix `lru-cache@11.0.2` missing after build stage # by doing a redundant `npm install` at this stage diff --git a/config.default.json b/config.default.json new file mode 100644 index 000000000..7f5f21341 --- /dev/null +++ b/config.default.json @@ -0,0 +1,42 @@ +{ + "config_name": "oss-default", + "env": "dev", + "port": 4100, + "protocol": "http", + "domain": "puter.localhost", + "cookie_name": "puter_auth_token", + "jwt_secret": "dev-jwt-secret-change-me", + "url_signature_secret": "dev-url-signature-secret-change-me", + "allow_all_host_values": true, + "allow_no_host_header": true, + "enable_public_folders": true, + "is_storage_limited": false, + "min_pass_length": 6, + "static_hosting_domain": "site.puter.localhost", + "static_hosting_domain_alt": "host.puter.localhost", + "private_app_hosting_domain": "app.puter.localhost", + "private_app_hosting_domain_alt": "dev.puter.localhost", + "captcha": { "enabled": false }, + "default_user_group": "78b1b1dd-c959-44d2-b02c-8735671f9997", + "default_temp_group": "b7220104-7905-4985-b996-649fdcdb3c8f", + "storage_capacity": 104857600, + "strict_email_verification_required": false, + "gui_assets_root": "./src/gui", + "puterjs_root": "./src/puter-js/dist", + "builtin_apps": { + "dev-center": "./src/dev-center" + }, + "extensions": [ + "./extensions" + ], + "database": { + "engine": "sqlite", + "path": "volatile/runtime/puter-database.sqlite" + }, + "s3": { + "localConfig": { + "dataDir": "volatile/runtime/fauxqs-data", + "s3StorageDir": "volatile/runtime/fauxqs-s3-data" + } + } +} diff --git a/config.template.jsonc b/config.template.jsonc new file mode 100644 index 000000000..771d00753 --- /dev/null +++ b/config.template.jsonc @@ -0,0 +1,313 @@ +{ + // Comprehensive template — every key the backend or shipped extensions read. + // Copy to `config.json` and trim what you don't need; unset keys fall back to + // documented defaults (see src/backend/types.ts for per-field comments). + // Each setting lives at exactly one canonical key — there are no fallback + // aliases. Values shown are illustrative, not production secrets. + + "config_name": "template", + "env": "dev", + "version": "0.0.0", + "serverId": "node-1", + "port": 4100, + "pub_port": 4100, + "protocol": "http", + "domain": "puter.localhost", + "origin": "http://puter.localhost:4100", + "api_base_url": "http://api.puter.localhost:4100", + "static_hosting_domain": "site.puter.localhost", + "static_hosting_domain_alt": "host.puter.localhost", + "private_app_hosting_domain": "app.puter.localhost", + "private_app_hosting_domain_alt": "dev.puter.localhost", + "allow_all_host_values": true, + "allow_no_host_header": true, + "allow_nipio_domains": false, + "custom_domains_enabled": false, + "enable_ip_validation": false, + "no_browser_launch": false, + "jwt_secret": "change-me", + "url_signature_secret": "change-me", + "cookie_name": "puter_auth_token", + "min_pass_length": 6, + "allow_system_login": false, + "strict_email_verification_required": false, + "captcha": { + "enabled": false, + "difficulty": "medium" + }, + "oidc": { + "providers": { + "google": { + "client_id": "", + "client_secret": "", + "scopes": "openid email profile" + }, + "custom-oidc": { + "client_id": "", + "client_secret": "", + "authorization_endpoint": "", + "token_endpoint": "", + "userinfo_endpoint": "", + "scopes": "openid email profile" + } + } + }, + "default_user_group": "78b1b1dd-c959-44d2-b02c-8735671f9997", + "default_temp_group": "b7220104-7905-4985-b996-649fdcdb3c8f", + "enable_public_folders": true, + "s3": { + "localConfig": { + "inMemory": false, + "host": "127.0.0.1", + "port": 4566, + "dataDir": "volatile/runtime/fauxqs-data", + "s3StorageDir": "volatile/runtime/fauxqs-s3-data" + }, + "_remote_example": { + "s3Config": { + "useCredentialChain": false, + "endpoint": "https://s3.example.com", + "accessKeyId": "", + "secretAccessKey": "", + "region": "us-west-2" + } + } + }, + "s3_bucket": "puter-local", + "s3_region": "us-west-2", + "region": "us-west-2", + "storage_capacity": 104857600, + "is_storage_limited": false, + "available_device_storage": 0, + "thumbnailStore": { + "name": "puter-local", + "endpoint": "", + "credentials": { + "accessKeyId": "", + "secretAccessKey": "" + } + }, + "database": { + "engine": "sqlite", + "path": "volatile/runtime/puter-database.sqlite", + "targetVersion": 0, + "host": "", + "port": 3306, + "user": "", + "password": "", + "database": "", + "replica": { + "host": "", + "port": 3306, + "user": "", + "password": "", + "database": "" + } + }, + "dynamo": { + "endpoint": "http://localhost:8000", + "path": "", + "aws": { + "access_key": "", + "secret_key": "", + "region": "us-west-2" + } + }, + "redis": { + "useMock": true, + "startupNodes": [ + { + "host": "127.0.0.1", + "port": 7000 + } + ] + }, + "pager": { + "pagerduty": { + "enabled": false, + "routingKey": "" + } + }, + "email": { + "from": "\"Puter\" ", + "host": "smtp.example.com", + "port": 587, + "secure": false, + "service": "", + "auth": { + "user": "", + "pass": "" + } + }, + "clickhouse": { + "url": "http://127.0.0.1:8123", + "username": "", + "password": "", + "request_timeout": 15000, + "max_buffer_size": 100000, + "batch_size": 500, + "flush_interval_ms": 5000 + }, + "cf_file_cache": { + "endpoint": "https://example.com/invalidate", + "throttle_ms": 500 + }, + "rate_limit": { + "backend": "redis" + }, + "providers": { + "_": "All AI / integration drivers read from here. Provider id = driver-side identifier.", + "claude": { + "apiKey": "" + }, + "openai-completion": { + "apiKey": "" + }, + "gemini": { + "apiKey": "" + }, + "groq": { + "apiKey": "" + }, + "deepseek": { + "apiKey": "" + }, + "mistral": { + "apiKey": "" + }, + "xai": { + "apiKey": "" + }, + "openrouter": { + "apiKey": "", + "apiBaseUrl": "https://openrouter.ai/api/v1" + }, + "together-ai": { + "apiKey": "" + }, + "ollama": { + "enabled": false, + "apiBaseUrl": "http://localhost:11434" + }, + "openai-image-generation": { + "apiKey": "" + }, + "gemini-image-generation": { + "apiKey": "" + }, + "together-image-generation": { + "apiKey": "" + }, + "cloudflare-image-generation": { + "apiToken": "", + "accountId": "", + "apiBaseUrl": "https://api.cloudflare.com/client/v4" + }, + "xai-image-generation": { + "apiKey": "" + }, + "openai-video-generation": { + "apiKey": "" + }, + "together-video-generation": { + "apiKey": "" + }, + "gemini-video-generation": { + "apiKey": "" + }, + "openai": { + "apiKey": "" + }, + "elevenlabs": { + "apiKey": "", + "apiBaseUrl": "https://api.elevenlabs.io", + "defaultVoiceId": "", + "speechToSpeechModelId": "" + }, + "aws-polly": { + "access_key": "", + "secret_key": "", + "region": "us-west-2" + }, + "aws-textract": { + "access_key": "", + "secret_key": "", + "region": "us-west-2" + }, + "mistral-ocr": { + "apiKey": "" + } + }, + "broadcast": { + "peers": [ + { + "peerId": "peer-a", + "webhook": true, + "webhook_url": "https://peer-a.example.com/broadcast/webhook", + "webhook_secret": "shared-secret" + } + ], + "webhook": { + "peerId": "this-node", + "secret": "shared-secret" + }, + "webhook_replay_window_seconds": 300, + "outbound_flush_ms": 2000 + }, + "peers": { + "signaller_url": "wss://signaller.example.com", + "fallback_ice": [], + "turn": { + "cloudflare_turn_service_id": "", + "cloudflare_turn_api_token": "", + "ttl": 86400 + }, + "internal_auth_secret": "" + }, + "wisp": { + "server": "wss://wisp.example.com" + }, + "workers": { + "XAUTHKEY": "", + "ACCOUNTID": "", + "namespace": "", + "internetExposedUrl": "https://api.puter.com", + "loggingUrl": "" + }, + "entri": { + "applicationId": "", + "secret": "" + }, + + "gui_assets_root": "./src/gui", + "gui_profile": "development", + "builtin_apps": { + "dev-center": "./src/dev-center" + }, + "use_bundled_gui": false, + "gui_bundle": "/dist/bundle.min.js", + "gui_css": "/dist/bundle.min.css", + "gui_puterjs_bundle": "https://js.puter.com/v2/", + "gui_params": { + "title": "Puter", + "short_description": "Your personal cloud computer", + "social_media_image": "" + }, + "native_apps_root": "", + "client_libs_root": "", + "puterjs_root": "./src/puter-js/dist", + "feature_flags": { + "example_flag": false + }, + "blockedEmailDomains": [], + "support_email": "support@puter.com", + "reserved_words": [], + "max_subdomains_per_user": 10, + "server_health": { + "db_liveness_latency_fail_ms": 1500, + "stale_health_loop_fail_ms": 0 + }, + "extensions": [ + "./extensions" + ] +} diff --git a/eslint.config.js b/eslint.config.js index abfaeb3ce..b07eab4e9 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -1,204 +1,131 @@ import js from '@eslint/js'; -import stylistic from '@stylistic/eslint-plugin'; import tseslintPlugin from '@typescript-eslint/eslint-plugin'; import tseslintParser from '@typescript-eslint/parser'; +import prettierPlugin from 'eslint-plugin-prettier'; +import prettierConfig from 'eslint-config-prettier'; import { defineConfig } from 'eslint/config'; import globals from 'globals'; -import bangSpaceIf from './eslint/bang-space-if.js'; -import controlStructureSpacing from './eslint/control-structure-spacing.js'; -import spaceUnaryOpsWithException from './eslint/space-unary-ops-with-exception.js'; -export const rules = { - 'no-invalid-this': 'error', - 'no-unused-vars': ['error', { - vars: 'all', - args: 'after-used', - caughtErrors: 'none', - ignoreRestSiblings: false, - ignoreUsingDeclarations: false, - reportUsedIgnorePattern: false, - argsIgnorePattern: '^_', - destructuredArrayIgnorePattern: '^_', +// typescript-eslint's flat/recommended preset is an array of configs (base + +// eslint-recommended overrides + recommended rules). Flatten its rules so we +// can apply them via our own `files`-scoped blocks. +const tsRecommendedRules = tseslintPlugin.configs['flat/recommended'].reduce( + (acc, cfg) => ({ ...acc, ...(cfg.rules ?? {}) }), + {}, +); - }], - curly: ['error', 'multi-line'], - '@stylistic/curly-newline': ['error', 'always'], - '@stylistic/object-curly-spacing': ['error', 'always'], - '@stylistic/indent': ['error', 4, { - SwitchCase: 1, - CallExpression: { - arguments: 1, - }, - }], - '@stylistic/indent-binary-ops': ['error', 4], - '@stylistic/array-bracket-newline': ['error', 'consistent'], - '@stylistic/semi': ['error', 'always'], - '@stylistic/quotes': ['error', 'single', { 'avoidEscape': true }], - '@stylistic/function-call-argument-newline': ['error', 'consistent'], - '@stylistic/function-paren-newline': ['error', 'multiline-arguments'], - '@stylistic/arrow-spacing': ['error', { before: true, after: true }], - '@stylistic/space-before-function-paren': 'error', - '@stylistic/key-spacing': ['error', { 'beforeColon': false, 'afterColon': true }], - '@stylistic/keyword-spacing': ['error', { 'before': true, 'after': true }], - '@stylistic/no-multiple-empty-lines': ['error', { max: 1, maxEOF: 0 }], - '@stylistic/comma-spacing': ['error', { 'before': false, 'after': true }], - '@stylistic/comma-dangle': ['error', 'always-multiline'], - '@stylistic/object-property-newline': ['error', { allowAllPropertiesOnSameLine: true }], - '@stylistic/dot-location': ['error', 'property'], - '@stylistic/space-infix-ops': ['error'], - 'no-undef': 'error', - 'custom/control-structure-spacing': 'error', - 'custom/bang-space-if': 'error', - '@stylistic/no-trailing-spaces': 'error', - '@stylistic/space-before-blocks': ['error', 'always'], - 'prefer-template': 'error', - '@stylistic/no-mixed-spaces-and-tabs': ['error', 'smart-tabs'], - 'custom/space-unary-ops-with-exception': ['error', { words: true, nonwords: false }], - '@stylistic/no-multi-spaces': ['error', { exceptions: { 'VariableDeclarator': true } }], - '@stylistic/type-annotation-spacing': 'error', - '@stylistic/type-generic-spacing': 'error', - '@stylistic/type-named-tuple-spacing': ['error'], - 'no-use-before-define': ['error', { - 'functions': false, - }], - '@stylistic/array-bracket-spacing': ['error', 'never'], - '@stylistic/linebreak-style': ['error', 'unix'], - 'no-useless-computed-key': 'error', - 'no-sequences': [ - 'error', { - allowInParentheses: false, - }, - ], +const prettierRules = { + ...prettierConfig.rules, + 'prettier/prettier': 'error', }; -const tsRules = { - '@typescript-eslint/no-explicit-any': 'warn', - '@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_', caughtErrors: 'none' }], - '@typescript-eslint/ban-ts-comment': 'warn', - '@typescript-eslint/consistent-type-definitions': ['error', 'interface'], +const unusedVarsOptions = { + args: 'after-used', + argsIgnorePattern: '^_', + varsIgnorePattern: '^_', + caughtErrorsIgnorePattern: '^_', + destructuredArrayIgnorePattern: '^_', + ignoreRestSiblings: true, }; -const sharedPlugins = { - js, - '@stylistic': stylistic, - custom: { - rules: { - 'control-structure-spacing': controlStructureSpacing, - 'bang-space-if': bangSpaceIf, - 'space-unary-ops-with-exception': spaceUnaryOpsWithException, - }, - }, +const preferConstOptions = { + destructuring: 'all', + ignoreReadBeforeAssign: false, }; -const sharedJsConfig = { - rules, - plugins: sharedPlugins, +const lintedGlobals = { + ...globals.node, + extension: 'readonly', + config: 'readonly', + global_config: 'readonly', }; -const recommendedJsConfig = { - ...sharedJsConfig, - extends: ['js/recommended'], -}; +const jsFiles = [ + 'src/backend/**/*.{js,mjs,cjs}', + 'extensions/**/*.{js,mjs,cjs}', +]; -const createTsConfig = ({ files, project, ignores = [], globals: tsGlobals }) => ({ +const tsIgnores = [ + '**/*.test.ts', + '**/*.test.mts', + '**/*.spec.ts', + '**/*.spec.mts', + 'src/backend/test/**', + 'src/backend/tools/**', + 'src/backend/vitest.config.ts', + 'src/backend/vitest.bench.config.ts', +]; + +const createTsConfig = ({ files, project }) => ({ files, - ignores, + ignores: tsIgnores, languageOptions: { parser: tseslintParser, - ...(tsGlobals ? { globals: tsGlobals } : {}), + globals: lintedGlobals, parserOptions: { ecmaVersion: 'latest', sourceType: 'module', - project, + projectService: { defaultProject: project }, + tsconfigRootDir: import.meta.dirname, }, }, plugins: { '@typescript-eslint': tseslintPlugin, + prettier: prettierPlugin, + }, + rules: { + ...tsRecommendedRules, + ...prettierRules, + '@typescript-eslint/no-unused-vars': ['error', unusedVarsOptions], + '@typescript-eslint/no-explicit-any': 'warn', + 'prefer-const': ['error', preferConstOptions], }, - rules: tsRules, }); -const backendConfig = { - ...recommendedJsConfig, - files: [ - 'src/backend/**/*.{js,mjs,cjs,ts}', - 'src/putility/**/*.{js,mjs,cjs,ts}', - ], - ignores: [ - '**/*.test.js', - '**/*.test.ts', - '**/*.test.mts', - ], - languageOptions: { globals: globals.node }, -}; - -const testConfig = { - ...sharedJsConfig, - files: [ - '**/*.test.js', - '**/*.test.ts', - '**/*.test.mts', - ], - languageOptions: { globals: { ...globals.node, ...globals.vitest } }, -}; - -const extensionConfig = { - ...recommendedJsConfig, - files: ['extensions/**/*.{js,mjs,cjs,ts}'], - languageOptions: { - globals: { - extension: 'readonly', - config: 'readonly', - global_config: 'readonly', - ...globals.node, - }, - }, -}; - -const frontendConfig = { - ...recommendedJsConfig, - files: ['**/*.{js,mjs,cjs,ts}', 'src/gui/src/**/*.js'], - ignores: [ - 'src/backend/**/*.{js,mjs,cjs,ts}', - 'extensions/**/*.{js,mjs,cjs,ts}', - 'submodules/**', - '**/*.test.{js,ts,mts,mjs}', - '**/*.min.js', - '**/*.min.cjs', - '**/*.min.mjs', - '**/socket.io.js', - '**/dist/*.js', - 'src/gui/src/lib/**', - 'src/gui/dist/**', - ], - languageOptions: { - globals: { - ...globals.browser, - ...globals.jquery, - i18n: 'readonly', - puter: 'readonly', - }, - }, -}; - export default defineConfig([ + { + ignores: [ + '**/*.dbmig.js', + 'dist/**', + 'build/**', + 'volatile/**', + 'node_modules/**', + 'puter.js/**', + 'apps/**', + 'experiment/**', + 'workers/**', + 'src/public/**', + 'src/gui/**', + 'src/docs/**', + 'src/puter-js/**', + 'src/useapi/**', + 'src/worker/**', + 'submodules/**', + 'tests/**', + 'tools/**', + ], + }, + { + files: jsFiles, + ignores: ['**/*.test.js'], + plugins: { + js, + prettier: prettierPlugin, + }, + extends: ['js/recommended'], + languageOptions: { + ecmaVersion: 'latest', + sourceType: 'module', + globals: lintedGlobals, + }, + rules: { + ...prettierRules, + 'no-unused-vars': ['error', unusedVarsOptions], + 'prefer-const': ['error', preferConstOptions], + }, + }, createTsConfig({ - files: ['**/*.test.ts', '**/*.test.mts', '**/*.test.setup.ts'], - ignores: ['tests/playwright/tests/**/*.ts'], - project: './tests/tsconfig.json', - globals: { ...globals.node, ...globals.vitest }, - }), - createTsConfig({ - files: ['**/*.ts'], - ignores: ['**/*.test.ts', '**/*.test.mts', 'extensions/**/*.ts'], + files: ['src/backend/**/*.ts', 'extensions/**/*.ts'], project: './tsconfig.json', }), - createTsConfig({ - files: ['extensions/**/*.ts'], - project: './extensions/tsconfig.json', - }), - backendConfig, - testConfig, - extensionConfig, - frontendConfig, ]); diff --git a/eslint/bang-space-if.js b/eslint/bang-space-if.js deleted file mode 100644 index 7b6c0b8dd..000000000 --- a/eslint/bang-space-if.js +++ /dev/null @@ -1,69 +0,0 @@ -// eslint-plugin-bang-space-if/index.js -'use strict'; - -/** @type {import('eslint').ESLint.Plugin} */ -export default { - meta: { - type: 'layout', - docs: { - description: - "Require a space after a top-level '!' in an if(...) condition (e.g., `if ( ! entry )`).", - recommended: false, - }, - fixable: 'whitespace', - schema: [], // no options - }, - create (context) { - const source = context.getSourceCode(); - - // Unwrap ParenthesizedExpression layers, if any - function unwrapParens (node) { - let n = node; - // ESLint/ESTree: ParenthesizedExpression is supported by espree - while ( n && n.type === 'ParenthesizedExpression' ) { - n = n.expression; - } - return n; - } - - return { - IfStatement (ifNode) { - const testRaw = ifNode.test; - if ( ! testRaw ) return; - - const test = unwrapParens(testRaw); - if ( !test || test.type !== 'UnaryExpression' || test.operator !== '!' ) { - return; // only top-level `!` expressions - } - - // Ignore boolean-cast `!!x` cases to avoid producing `! !x` - if ( test.argument && test.argument.type === 'UnaryExpression' && test.argument.operator === '!' ) { - return; - } - - // Grab operator and argument tokens - const opToken = source.getFirstToken(test); // should be '!' - const argToken = source.getTokenAfter(opToken, { includeComments: false }); - if ( !opToken || !argToken ) return; - - // Compute current whitespace between '!' and the argument - const between = source.text.slice(opToken.range[1], argToken.range[0]); - - // We want exactly one space - if ( between === ' ' ) return; - - context.report({ - node: test, - loc: { - start: opToken.loc.end, - end: argToken.loc.start, - }, - message: "Expected a single space after top-level '!' in if(...) condition.", - fix (fixer) { - return fixer.replaceTextRange([opToken.range[1], argToken.range[0]], ' '); - }, - }); - }, - }; - }, -};;;; diff --git a/eslint/control-structure-spacing.js b/eslint/control-structure-spacing.js deleted file mode 100644 index bd2bf2ed9..000000000 --- a/eslint/control-structure-spacing.js +++ /dev/null @@ -1,206 +0,0 @@ -export default { - meta: { - type: 'layout', - docs: { - description: 'enforce spacing inside parentheses for control structures only', - category: 'Stylistic Issues', - }, - fixable: 'whitespace', - schema: [], - messages: { - missingSpaceAfterOpen: 'Missing space after opening parenthesis in control structure.', - missingSpaceBeforeClose: 'Missing space before closing parenthesis in control structure.', - unexpectedSpaceAfterOpen: 'Unexpected space after opening parenthesis in function call.', - unexpectedSpaceBeforeClose: 'Unexpected space before closing parenthesis in function call.', - }, - }, - - create (context) { - const sourceCode = context.getSourceCode(); - - function checkControlStructureSpacing (node) { - // For control structures, we need to find the parentheses around the condition/test - let conditionNode; - - if ( node.type === 'IfStatement' || node.type === 'WhileStatement' || node.type === 'DoWhileStatement' ) { - conditionNode = node.test; - } else if ( node.type === 'ForStatement' || node.type === 'ForInStatement' || node.type === 'ForOfStatement' ) { - // For loops, we want the parentheses around the entire for clause - conditionNode = node; - } else if ( node.type === 'SwitchStatement' ) { - conditionNode = node.discriminant; - } else if ( node.type === 'CatchClause' ) { - conditionNode = node.param; - } - - if ( ! conditionNode ) return; - - // Find the opening paren - it should be right before the condition starts - const openParen = sourceCode.getTokenBefore(conditionNode, token => token.value === '('); - if ( !openParen || openParen.value !== '(' ) return; - - // Find the closing paren - it should be right after the condition ends - const closeParen = sourceCode.getTokenAfter(conditionNode, token => token.value === ')'); - if ( !closeParen || closeParen.value !== ')' ) return; - - const afterOpen = sourceCode.getTokenAfter(openParen); - const beforeClose = sourceCode.getTokenBefore(closeParen); - - { - const contentBetweenParens = sourceCode.getText().slice(openParen.range[1], closeParen.range[0]); - const isSingleCharVariable = /^\s*[a-zA-Z_$]\s*$/.test(contentBetweenParens); - - // Skip spacing requirements for single character variables - if ( isSingleCharVariable ) { - return; - } - } - - // Control structures should have spacing - if ( afterOpen && openParen.range[1] === afterOpen.range[0] ) { - context.report({ - node, - loc: openParen.loc, - messageId: 'missingSpaceAfterOpen', - fix (fixer) { - return fixer.insertTextAfter(openParen, ' '); - }, - }); - } - - if ( beforeClose && beforeClose.range[1] === closeParen.range[0] ) { - context.report({ - node, - loc: closeParen.loc, - messageId: 'missingSpaceBeforeClose', - fix (fixer) { - return fixer.insertTextBefore(closeParen, ' '); - }, - }); - } - } - - function checkForLoopSpacing (node) { - // For loops are special - we need to find the opening paren after the 'for' keyword - // and the closing paren before the body - const forKeyword = sourceCode.getFirstToken(node); - if ( !forKeyword || forKeyword.value !== 'for' ) return; - - const openParen = sourceCode.getTokenAfter(forKeyword, token => token.value === '('); - if ( ! openParen ) return; - - // The closing paren should be right before the body - const closeParen = sourceCode.getTokenBefore(node.body, token => token.value === ')'); - if ( ! closeParen ) return; - - const afterOpen = sourceCode.getTokenAfter(openParen); - const beforeClose = sourceCode.getTokenBefore(closeParen); - - if ( afterOpen && openParen.range[1] === afterOpen.range[0] ) { - context.report({ - node, - loc: openParen.loc, - messageId: 'missingSpaceAfterOpen', - fix (fixer) { - return fixer.insertTextAfter(openParen, ' '); - }, - }); - } - - if ( beforeClose && beforeClose.range[1] === closeParen.range[0] ) { - context.report({ - node, - loc: closeParen.loc, - messageId: 'missingSpaceBeforeClose', - fix (fixer) { - return fixer.insertTextBefore(closeParen, ' '); - }, - }); - } - } - - function checkFunctionCallSpacing (node) { - // Find the opening parenthesis for this function call - const openParen = sourceCode.getFirstToken(node, token => token.value === '('); - const closeParen = sourceCode.getLastToken(node, token => token.value === ')'); - - if ( !openParen || !closeParen ) return; - // Defer multi-line call/new formatting to stylistic paren/argument rules. - if ( openParen.loc.start.line !== closeParen.loc.end.line ) return; - - const afterOpen = sourceCode.getTokenAfter(openParen); - const beforeClose = sourceCode.getTokenBefore(closeParen); - - // Function calls should NOT have spacing on the same line (multi-line calls are allowed) - if ( afterOpen && openParen.range[1] !== afterOpen.range[0] ) { - const spaceAfter = sourceCode.getText().slice(openParen.range[1], afterOpen.range[0]); - if ( /^\s+$/.test(spaceAfter) && !spaceAfter.includes('\n') ) { - context.report({ - node, - loc: openParen.loc, - messageId: 'unexpectedSpaceAfterOpen', - fix (fixer) { - return fixer.removeRange([openParen.range[1], afterOpen.range[0]]); - }, - }); - } - } - - if ( beforeClose && beforeClose.range[1] !== closeParen.range[0] ) { - const spaceBefore = sourceCode.getText().slice(beforeClose.range[1], closeParen.range[0]); - if ( /^\s+$/.test(spaceBefore) && !spaceBefore.includes('\n') ) { - context.report({ - node, - loc: closeParen.loc, - messageId: 'unexpectedSpaceBeforeClose', - fix (fixer) { - return fixer.removeRange([beforeClose.range[1], closeParen.range[0]]); - }, - }); - } - } - } - - return { - // Control structures that should have spacing - IfStatement (node) { - checkControlStructureSpacing(node); - }, - WhileStatement (node) { - checkControlStructureSpacing(node); - }, - DoWhileStatement (node) { - checkControlStructureSpacing(node); - }, - SwitchStatement (node) { - checkControlStructureSpacing(node); - }, - CatchClause (node) { - if ( node.param ) { - checkControlStructureSpacing(node); - } - }, - - // For loops need special handling - ForStatement (node) { - checkForLoopSpacing(node); - }, - ForInStatement (node) { - checkForLoopSpacing(node); - }, - ForOfStatement (node) { - checkForLoopSpacing(node); - }, - - // Function calls that should NOT have spacing - CallExpression (node) { - checkFunctionCallSpacing(node); - }, - NewExpression (node) { - if ( node.arguments.length > 0 || sourceCode.getLastToken(node).value === ')' ) { - checkFunctionCallSpacing(node); - } - }, - }; - }, -}; diff --git a/eslint/mandatory.eslint.config.js b/eslint/mandatory.eslint.config.js deleted file mode 100644 index 5f0e4962d..000000000 --- a/eslint/mandatory.eslint.config.js +++ /dev/null @@ -1,86 +0,0 @@ -import tseslintPlugin from '@typescript-eslint/eslint-plugin'; -import { defineConfig } from 'eslint/config'; -import globals from 'globals'; - -const backendLanguageOptions = { - globals: { - // Current, intentionally supported globals - extension: 'readonly', - config: 'readonly', - global_config: 'readonly', - - // Older not entirely ideal globals - use: 'readonly', // <-- older import mechanism - def: 'readonly', // <-- older import mechanism - kv: 'readonly', // <-- should be passed/imported - ll: 'readonly', // <-- questionable - - // Language/environment globals - ...globals.node, - }, -}; - -const mandatoryRules = { - 'no-undef': 'error', - 'no-use-before-define': ['error', { - 'functions': false, - }], - 'no-invalid-this': 'warn', -}; - -export default defineConfig([ - { - ignores: [ - 'src/backend/src/modules/apps/AppInformationService.js', // TEMPORARY - SHOULD BE FIXED! - 'src/backend/src/services/worker/WorkerService.js', // TEMPORARY - SHOULD BE FIXED! - 'src/backend/src/public/**/*', // We may be able to delete this! I don't think it's used - - // These files run in the worker environment, so these rules don't apply - 'src/backend/src/services/worker/dist/**/*.{js,cjs,mjs}', - 'src/backend/src/services/worker/src/**/*.{js,cjs,mjs}', - 'src/backend/src/services/worker/template/puter-portable.js', - ], - }, - { - plugins: { - '@typescript-eslint': tseslintPlugin, - }, - }, - { - files: [ - 'src/backend/**/*.{js,mjc,cjs}', - 'extensions/**/*.{js,mjc,cjs}', - ], - ignores: [ - 'src/backend/src/services/database/sqlite_setup/**/*.js', - ], - rules: mandatoryRules, - languageOptions: { - ...backendLanguageOptions, - }, - }, - { - files: [ - 'src/backend/src/services/database/sqlite_setup/**/*.js', - ], - rules: mandatoryRules, - languageOptions: { - globals: { - read: 'readonly', - write: 'readonly', - log: 'readonly', - ...globals.node, - }, - }, - }, - { - files: [ - 'src/backend/**/*.{ts}', - 'extensions/**/*.{ts}', - ], - rules: mandatoryRules, - languageOptions: { - ...backendLanguageOptions, - }, - }, -]); diff --git a/eslint/space-unary-ops-with-exception.js b/eslint/space-unary-ops-with-exception.js deleted file mode 100644 index 83a48d366..000000000 --- a/eslint/space-unary-ops-with-exception.js +++ /dev/null @@ -1,37 +0,0 @@ -import ruleComposer from 'eslint-rule-composer'; - -// Adjust this require to match the package you use for the rule. -// For eslint-stylistic v2+ the package is "@stylistic/eslint-plugin" -import stylistic from '@stylistic/eslint-plugin'; -const baseRule = stylistic.rules['space-unary-ops']; - -// unwrap nested parentheses -function unwrapParens (node) { - let n = node; - while ( n && n.type === 'ParenthesizedExpression' ) n = n.expression; - return n; -} - -function isTopLevelBangInIfTest (node) { - if ( !node || node.type !== 'UnaryExpression' || node.operator !== '!' ) return false; - - // Walk up through ancestors manually using .parent (safe in ESLint) - let current = node; - let parent = current.parent; - - // Skip ParenthesizedExpression layers - while ( parent && parent.type === 'ParenthesizedExpression' ) { - current = parent; - parent = parent.parent; - } - - return parent && parent.type === 'IfStatement' && unwrapParens(parent.test) === node; -} - -// Filter out ONLY the reports for top-level ! inside if(...) condition -export default ruleComposer.filterReports(baseRule, (problem, context) => { - const { node } = problem; - // If this particular report is about a top-level ! in an if(...) test, - // suppress it. Otherwise, keep the original report. - return !isTopLevelBangInIfTest(node, context); -}); diff --git a/extensions/.gitkeep b/extensions/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/extensions/README.md b/extensions/README.md deleted file mode 100644 index 6186a41dd..000000000 --- a/extensions/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Extension System Development Guide - -## Where to find documentation - -### Here -Documentation for extensions is [here](src/backend/doc/extensions/README.md). - -### Not Here - -Outdated documentation for extensions is [here](../doc/contributors/extensions/README.md). -This documentation may include some topics that are missing from the current documentation. Eventually those topics should be updated and transferred to the current documentation so that this documentation may be removed. diff --git a/extensions/api.d.ts b/extensions/api.d.ts deleted file mode 100644 index 3dd3b3e13..000000000 --- a/extensions/api.d.ts +++ /dev/null @@ -1,207 +0,0 @@ - -import type APIError from '@heyputer/backend/src/api/APIError.js'; -import type query from '@heyputer/backend/src/om/query/query'; -import type { Actor } from '@heyputer/backend/src/services/auth/Actor.js'; -import type { ServicesMap } from '@heyputer/backend/src/services/BaseService.d.ts'; -import type { BaseDatabaseAccessService } from '@heyputer/backend/src/services/database/BaseDatabaseAccessService.d.ts'; -import type { DynamoKVStore } from '@heyputer/backend/src/services/repositories/DynamoKVStore/DynamoKVStore.ts'; -import type { IUser } from '@heyputer/backend/src/services/User.js'; -import type { Context } from '@heyputer/backend/src/util/context.js'; -import type kvjs from '@heyputer/kv.js'; -import type { RequestHandler } from 'express'; -import type { Cluster } from 'ioredis'; -import type FSNodeContext from '../src/backend/src/filesystem/FSNodeContext.js'; -import type helpers from '../src/backend/src/helpers.js'; -import type { ICompleteArguments } from '../src/backend/src/services/ai/chat/providers/types.ts'; -import type * as ExtensionControllerExports from './ExtensionController/src/ExtensionController.ts'; -import type { s3ClientProvider } from '../src/backend/src/clients/s3/s3ClientProvider.js'; - -declare global { - namespace Express { - interface Request { - services: { - get: ( - string: T, - ) => T extends keyof ServicesMap ? ServicesMap[T] : unknown; - }; - actor?: Actor; - rawBody: Buffer; - /** @deprecated use actor instead */ - user: IUser; - } - } -} - -export type { Cluster } from 'ioredis'; - -export interface EndpointOptions { - allowedMethods?: string[]; - subdomain?: string; - noauth?: boolean; - mw?: RequestHandler[]; - otherOpts?: Record & { - json?: boolean; - noReallyItsJson?: boolean; - }; -} - -// Driver interface types -interface ParameterDefinition { - type: 'string' | 'number' | 'boolean' | 'object' | 'array'; - optional: boolean; -} -interface MethodDefinition { - description: string; - parameters: Record; -} -interface DriverInterface { - description: string; - methods: Record; -} - -export type HttpMethod = 'get' | 'post' | 'put' | 'delete' | 'patch'; - -export type AddRouteFunction = ( - path: string, - options: EndpointOptions, - handler: RequestHandler, -) => void; - -export type RouterMethods = { - [K in HttpMethod]: { - (path: string, options: EndpointOptions, handler: RequestHandler): void; - (path: string, handler: RequestHandler, options?: EndpointOptions): void; - }; -}; - -interface CoreRuntimeModule { - util: { - helpers: typeof helpers; - }; - redisClient: Cluster; - kvjs: kvjs - s3ClientProvider: typeof s3ClientProvider; - Context: typeof Context; - APIError: typeof APIError; -} - -interface FilesystemModule { - FSNodeContext: FSNodeContext; - selectors: unknown; -} - -export interface ExtensionEventTypeMap { - 'metering:registerAvailablePolicies': { - availablePolicies: unknown[] - }, - 'create.drivers': { - createDriver: (interface: string, service: string, executors: any) => any; - }; - 'create.permissions': { - grant_to_everyone: (permission: string) => void; - grant_to_users: (permission: string) => void; - }; - 'create.interfaces': { - createInterface: (interface: string, interfaces: DriverInterface) => void; - }; - 'puter.gui.addons': { - bodyContent: string; - headContent: string; - guiParams: { - env: string; - app_origin: string; - api_origin: string; - gui_origin: string; - asset_dir: string; - launch_options: unknown; - app_name_regex: RegExp; - app_name_max_length: number; - app_title_max_length: number; - hosting_domain: string; - subdomain_regex: RegExp; - subdomain_max_length: number; - domain: string; - protocol: string; - api_base_url: string; - app?: { name: string, uid: string } & Record; - [key: string]: unknown; - }; - }; - 'app.changed': { - app_uid: string; - action: 'updated' | 'deleted'; - }; - 'app.privateAccess.check': { - appUid: string; - userUid?: string | null; - requestHost?: string; - requestPath?: string; - result: { - allowed: boolean; - redirectUrl?: string; - reason?: string; - checkedBy?: string; - }; - }; - 'app.privateAccess.resolveLaunch': { - appUid: string; - appName?: string; - userUid?: string | null; - source?: string; - args?: Record; - result: { - hasAccess: boolean; - fallbackAppName?: string; - fallbackArgs?: Record; - reason?: string; - checkedBy?: string; - }; - }; - 'ai.prompt.validate': { - actor: Actor; - actor, - completionId: string, - allow: boolean, - intended_service: string, - parameters: ICompleteArguments - } - 'outer.cacheUpdate': { cacheKey: string | string[], ttlSeconds?: number, data?: unknown } -} - -interface Extension extends RouterMethods { - exports: Record; - span: ((label: string, fn: () => T) => () => T) & { - run(label: string, fn: () => T): T; - run(fn: () => T): T; - }; - config: Record; - - on( - name: E, - listener: (event: ExtensionEventTypeMap[E], metadata?: { from_outside?: boolean }) => void | Promise - ): void; - on(name: string, listener: (event: T, metadata?: { from_outside?: boolean }) => void | Promise): void - - import(module: 'data'): { - db: BaseDatabaseAccessService; - kv: DynamoKVStore; - cache: Cluster; - s3ClientProvider: typeof s3ClientProvider; - }; - import(module: 'core'): CoreRuntimeModule; - import(module: 'fs'): FilesystemModule; - import(module: 'query'): typeof query; - import(module: 'extensionController'): typeof ExtensionControllerExports; - import( - module: T - ): T extends `service:${infer R extends keyof ServicesMap}` - ? ServicesMap[R] - : unknown; -} - -declare global { - // Declare the extension variable - const extension: Extension; - const config: Record; - const global_config: Record; -} diff --git a/extensions/app-telemetry/app-user-count.ts b/extensions/app-telemetry/app-user-count.ts deleted file mode 100644 index 0f7f3e44d..000000000 --- a/extensions/app-telemetry/app-user-count.ts +++ /dev/null @@ -1,138 +0,0 @@ -const { Eq } = extension.import('query'); -const { db } = extension.import('data'); -const { APIError, Context } = extension.import('core'); -const app_es = extension.import('service:es:app') as any; -const svc_permission = extension.import('service:permission') as any; - -const DEFAULT_LIMIT = 100; -const MAX_LIMIT = 1000; -const MAX_OFFSET = 100_000; - -const parseIntegerParam = ( - value: unknown, - { - key, - min, - max, - fallback, - }: { key: string, min: number, max: number, fallback: number }, -) => { - if ( value === undefined || value === null ) return fallback; - - const parsed = typeof value === 'number' - ? value - : (typeof value === 'string' && value.trim() !== '' - ? Number(value) - : Number.NaN); - - if ( !Number.isFinite(parsed) || !Number.isInteger(parsed) ) { - throw APIError.create('field_invalid', undefined, { - key, - expected: `an integer between ${min} and ${max}`, - got: value, - }); - } - - if ( parsed < min || parsed > max ) { - throw APIError.create('field_invalid', undefined, { - key, - expected: `an integer between ${min} and ${max}`, - got: parsed, - }); - } - - return parsed; -}; - -extension.on('create.interfaces', (event) => { - event.createInterface('app-telemetry', { - description: 'Provides methods for getting app telemetry', - methods: { - get_users: { - description: 'Returns users who have used your app', - parameters: { - app_uuid: { - type: 'string', - optional: false, - }, - limit: { - type: 'number', - optional: true, - }, - offset: { - type: 'number', - optional: true, - }, - }, - }, - user_count: { - description: 'Returns number of users who have used your app', - parameters: { - app_uuid: { - type: 'string', - optional: false, - }, - }, - }, - }, - }); -}); - -extension.on('create.drivers', event => { - event.createDriver('app-telemetry', 'app-telemetry', { - async get_users ({ app_uuid, limit, offset }: { app_uuid: string, limit?: number, offset?: number }) { - const safeLimit = parseIntegerParam(limit, { - key: 'limit', - min: 1, - max: MAX_LIMIT, - fallback: DEFAULT_LIMIT, - }); - const safeOffset = parseIntegerParam(offset, { - key: 'offset', - min: 0, - max: MAX_OFFSET, - fallback: 0, - }); - - // first lets make sure executor owns this app - const [result] = (await app_es.select({ predicate: new Eq({ key: 'uid', value: app_uuid }) })); - if ( ! result ) { - throw APIError.create('permission_denied'); - } - if ( ! (await svc_permission.check(Context.get('actor'), `apps-of-user:${result.values_.owner.uuid}:write`, { no_cache: true })) ) { - throw APIError.create('permission_denied'); - } - - // Fetch and return users - const users: Array<{ username: string, uuid: string }> = await db.read( - `SELECT user.username, user.uuid FROM user_to_app_permissions - INNER JOIN user ON user_to_app_permissions.user_id = user.id - WHERE permission = 'flag:app-is-authenticated' AND app_id=? ORDER BY (dt IS NOT NULL), dt, user_id LIMIT ? OFFSET ?`, - [result.private_meta.mysql_id, safeLimit, safeOffset], - ); - return users.map(e => { - return { user: e.username, user_uuid: e.uuid }; - }); - }, - async user_count ({ app_uuid }: { app_uuid: string }) { - // first lets make sure executor owns this app - const [result] = (await app_es.select({ predicate: new Eq({ key: 'uid', value: app_uuid }) })); - if ( ! result ) { - throw APIError.create('permission_denied'); - } - - // Fetch and return authenticated user count - const [data] = await db.read( - `SELECT count(*) FROM user_to_app_permissions - WHERE permission = 'flag:app-is-authenticated' AND app_id=?;`, - [result.private_meta.mysql_id], - ); - const count = data['count(*)']; - return count; - }, - }); -}); - -extension.on('create.permissions', (event) => { - event.grant_to_everyone('service:app-telemetry:ii:app-telemetry'); -}); diff --git a/extensions/app-telemetry/index.d.ts b/extensions/app-telemetry/index.d.ts deleted file mode 100644 index 2fc1c14d9..000000000 --- a/extensions/app-telemetry/index.d.ts +++ /dev/null @@ -1 +0,0 @@ -import '../api.js'; \ No newline at end of file diff --git a/extensions/app-telemetry/package.json b/extensions/app-telemetry/package.json deleted file mode 100644 index fcc665103..000000000 --- a/extensions/app-telemetry/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "@heyputer/app-telemetry", - "main": "app-user-count.js", - "type": "module", - "scripts": { - "postinstall": "tsc --noCheck", - "test": "echo \"Error: no test specified\" && exit 1" - }, - "devDependencies": { - "typescript": "^5.9.3" - } -} \ No newline at end of file diff --git a/extensions/app-telemetry/tsconfig.json b/extensions/app-telemetry/tsconfig.json deleted file mode 100644 index 6b590838b..000000000 --- a/extensions/app-telemetry/tsconfig.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2024", - "module": "nodenext", - "moduleResolution": "nodenext", - "rootDir": "./", - "strict": true, - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "allowSyntheticDefaultImports": true, - "skipLibCheck": true, - "sourceMap": true, - }, - "include": [ - "./**/*.ts", - "./**/*.d.ts" - ], - "exclude": [ - "**/*.test.ts", - "**/*.spec.ts", - "**/test/**", - "**/tests/**", - "node_modules", - "dist", - "*.js" - ] -} \ No newline at end of file diff --git a/extensions/appTelemetry.ts b/extensions/appTelemetry.ts new file mode 100644 index 000000000..e363d5769 --- /dev/null +++ b/extensions/appTelemetry.ts @@ -0,0 +1,120 @@ +import { Context } from '@heyputer/backend/src/core'; +import { HttpError } from '@heyputer/backend/src/core/http'; +import { extension } from '@heyputer/backend/src/extensions'; + +const clients = extension.import('client'); +const stores = extension.import('store'); +const services = extension.import('service'); + +const DEFAULT_LIMIT = 100; +const MAX_LIMIT = 1000; +const MAX_OFFSET = 100_000; + +const parseIntParam = ( + value: unknown, + { + key, + min, + max, + fallback, + }: { key: string; min: number; max: number; fallback: number }, +): number => { + if (value === undefined || value === null) return fallback; + const parsed = + typeof value === 'number' + ? value + : typeof value === 'string' && value.trim() !== '' + ? Number(value) + : NaN; + if ( + !Number.isFinite(parsed) || + !Number.isInteger(parsed) || + parsed < min || + parsed > max + ) { + throw new HttpError( + 400, + `${key} must be an integer between ${min} and ${max}`, + ); + } + return parsed; +}; + +extension.get( + '/app-telemetry/users', + { subdomain: 'api', requireAuth: true }, + async (req, res) => { + const { app_uuid } = req.query as Record; + if (!app_uuid) throw new HttpError(400, 'Missing `app_uuid`'); + + const safeLimit = parseIntParam(req.query.limit, { + key: 'limit', + min: 1, + max: MAX_LIMIT, + fallback: DEFAULT_LIMIT, + }); + const safeOffset = parseIntParam(req.query.offset, { + key: 'offset', + min: 0, + max: MAX_OFFSET, + fallback: 0, + }); + + const app = await stores.app.getByUid(app_uuid); + if (!app) throw new HttpError(404, 'App not found'); + + // `apps-of-user::write` — the implicator keys on the owner's + // UUID, not the numeric id. Look up the owner explicitly. v1 got + // this for free because its entity-storage layer eager-joined the + // owner row; v2's AppStore.getByUid returns the raw row with only + // `owner_user_id` populated. + const ownerId = (app as { owner_user_id?: number }).owner_user_id; + if (!ownerId) throw new HttpError(404, 'App owner not found'); + const owner = (await stores.user.getById(ownerId)) as { + uuid?: string; + } | null; + if (!owner?.uuid) throw new HttpError(404, 'App owner not found'); + + const actor = Context.get('actor'); + const ownsApp = await services.permission + .check(actor!, `apps-of-user:${owner.uuid}:write`) + .catch(() => false); + if (!ownsApp) throw new HttpError(403, 'Permission denied'); + + const users = await clients.db.read( + `SELECT u.username, u.uuid FROM user_to_app_permissions p + INNER JOIN user u ON p.user_id = u.id + WHERE p.permission = 'flag:app-is-authenticated' AND p.app_id = ? + ORDER BY (p.dt IS NOT NULL), p.dt, p.user_id + LIMIT ? OFFSET ?`, + [(app as Record).id, safeLimit, safeOffset], + ); + + res.json( + (users as Array<{ username: string; uuid: string }>).map((e) => ({ + user: e.username, + user_uuid: e.uuid, + })), + ); + }, +); + +extension.get( + '/app-telemetry/user-count', + { subdomain: 'api', requireAuth: true }, + async (req, res) => { + const { app_uuid } = req.query as Record; + if (!app_uuid) throw new HttpError(400, 'Missing `app_uuid`'); + + const app = await stores.app.getByUid(app_uuid); + if (!app) throw new HttpError(404, 'App not found'); + + const [row] = (await clients.db.read( + `SELECT COUNT(*) AS n FROM user_to_app_permissions + WHERE permission = 'flag:app-is-authenticated' AND app_id = ?`, + [(app as Record).id], + )) as Array<{ n: number }>; + + res.json({ count: row?.n ?? 0 }); + }, +); diff --git a/extensions/data.js b/extensions/data.js deleted file mode 100644 index e72dbba49..000000000 --- a/extensions/data.js +++ /dev/null @@ -1,34 +0,0 @@ -//@extension priority -10000 - -const { redisClient, kvjs, s3ClientProvider } = extension.import('core'); -const svc_database = extension.import('service:database'); -const svc_kvstore = extension.import('service:puter-kvstore'); - -// Methods on the object from `.as()` come from TraitsFeature.js, -// and they are already bound to their respective instance. -const simplified_kv = { ...svc_kvstore.as('puter-kvstore') }; - -const original_get = simplified_kv.get; -const original_set = simplified_kv.set; - -simplified_kv.get = (...a) => { - if ( typeof a[0] === 'string' ) { - return original_get({ key: a[0] }); - } - return original_get(...a); -}; - -simplified_kv.set = (...a) => { - if ( typeof a[0] === 'string' ) { - return original_set({ key: a[0], value: a[1] }); - } - return original_set(...a); -}; - -extension.exports = { - db: svc_database.get(), - kv: simplified_kv, - cache: redisClient, - kvjs: kvjs, - s3ClientProvider, -}; diff --git a/extensions/extensionController/package.json b/extensions/extensionController/package.json deleted file mode 100644 index 7035aa372..000000000 --- a/extensions/extensionController/package.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "@puter/extension-controller", - "version": "1.0.0", - "description": "", - "main": "src/index.js", - "type": "module", - "scripts": { - "postinstall": "tsc --noCheck" - }, - "keywords": [], - "author": "", - "license": "ISC", - "devDependencies": { - "@types/node": "^24.9.1", - "ts-node": "^10.9.2", - "typescript": "^5.9.3" - }, - "dependencies": { - "http-status-codes": "^2.3.0", - "stripe": "^19.1.0" - } -} \ No newline at end of file diff --git a/extensions/extensionController/puter.json b/extensions/extensionController/puter.json deleted file mode 100644 index 5fce7a8c0..000000000 --- a/extensions/extensionController/puter.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "priority": -10 -} \ No newline at end of file diff --git a/extensions/extensionController/src/ExtensionController.ts b/extensions/extensionController/src/ExtensionController.ts deleted file mode 100644 index 5529a9ab6..000000000 --- a/extensions/extensionController/src/ExtensionController.ts +++ /dev/null @@ -1,238 +0,0 @@ -import type { NextFunction, Request, Response } from 'express'; -import { StatusCodes } from 'http-status-codes'; -import type { - EndpointOptions, - HttpMethod, - RouterMethods, -} from '../../api.d.ts'; -declare const extension: Partial>; -/** - * Class decorator to set prefix on prototype and register routes on instantiation - * @argument prefix - prefix for all routes under the class - * @argument [adminUsernames] - gate all routes behind admin username check - */ -export const Controller = ( - prefix: string, - adminUsernames?: string[], - allowedAppIds?: string[], -): ClassDecorator => { - return (target: Function) => { - target.prototype.__controllerPrefix = prefix; - target.prototype.__allowedAppIds = allowedAppIds; - target.prototype.__adminUsernames = adminUsernames - ? [...adminUsernames, 'admin', 'system'] - : undefined; - }; -}; - -/** - * Method decorator factory that collects route metadata - */ -interface RouteMeta { - method: HttpMethod; - path: string; - options?: EndpointOptions | undefined; - handler: (req: Request, res: Response, next: NextFunction) => void | Promise; - adminUsernames?: string[]; - allowedAppIds?: string[]; -} - -const createMethodDecorator = (method: HttpMethod) => { - return ( - path: string, - routeOptions?: EndpointOptions & { allowedAppIds?: string[] }, - adminUsernames?: string[], - ) => { - const { allowedAppIds, ...options } = routeOptions ?? {}; - return ( - target: (req: Request, res: Response, next: NextFunction) => void | Promise, - _context: ClassMethodDecoratorContext< - This, - ( - this: This, - ...args: [req: Request, res: Response, next: NextFunction] - ) => void | Promise - >, - ) => { - _context.addInitializer(function () { - // eslint-disable-next-line no-invalid-this - const proto = Object.getPrototypeOf(this); // will be bound to class - if ( ! proto.__routes ) { - proto.__routes = []; - } - proto.__routes.push({ - method, - path, - options: options as EndpointOptions | undefined, - adminUsernames: adminUsernames - ? [...adminUsernames, 'admin', 'system'] - : undefined, - allowedAppIds, - handler: target, - }); - }); - }; - }; -}; - -// HTTP method decorators -export const Get = createMethodDecorator('get'); -export const Post = createMethodDecorator('post'); -export const Put = createMethodDecorator('put'); -export const Delete = createMethodDecorator('delete'); -// TODO DS: add others as needed (patch, etc) - -interface HttpErrorOptions { - cause?: unknown; - legacyCode?: string; - code?: string; - fields?: Record; -} - -const isHttpErrorOptions = (value: unknown): value is HttpErrorOptions => { - if ( !value || typeof value !== 'object' || Array.isArray(value) ) { - return false; - } - - return ( - Object.prototype.hasOwnProperty.call(value, 'cause') - || Object.prototype.hasOwnProperty.call(value, 'legacyCode') - || Object.prototype.hasOwnProperty.call(value, 'code') - || Object.prototype.hasOwnProperty.call(value, 'fields') - ); -}; - -export class HttpError extends Error { - statusCode: number; - legacyCode?: string; - code?: string; - fields?: Record; - constructor ( - statusCode: StatusCodes, - message: string, - causeOrOptions?: unknown, - legacyCode?: string, - ) { - const options = isHttpErrorOptions(causeOrOptions) - ? causeOrOptions - : undefined; - const cause = options - ? options.cause - : causeOrOptions; - const resolvedLegacyCode = legacyCode ?? options?.legacyCode; - const code = options?.code; - super( - `${statusCode} - ${message}`, - cause !== undefined ? { cause } : undefined, - ); - this.statusCode = statusCode; - this.legacyCode = resolvedLegacyCode; - this.code = code; - this.fields = options?.fields; - } -} - -// Registers all routes from a decorated controller instance to an Express router -export class ExtensionController { - logger?: Console; - // TODO DS: make this work with other express-like routers - registerRoutes () { - const logger = this.logger || console; - const prefix = Object.getPrototypeOf(this).__controllerPrefix || ''; - const adminsForController = Object.getPrototypeOf(this).__adminUsernames as - | string[] - | undefined; - const allowedAppIdsForController = Object.getPrototypeOf(this).__allowedAppIds as - | string[] - | undefined; - const routes: RouteMeta[] = Object.getPrototypeOf(this).__routes || []; - for ( const route of routes ) { - const fullPath = `${prefix}/${route.path}`.replace(/\/+/g, '/'); - const adminsForRoute = route.adminUsernames - ? adminsForController - ? adminsForController.concat(route.adminUsernames) - : route.adminUsernames - : adminsForController - ? adminsForController - : undefined; - const allowedAppIds = route.allowedAppIds - ? allowedAppIdsForController - ? allowedAppIdsForController.concat(route.allowedAppIds) - : route.allowedAppIds - : allowedAppIdsForController - ? allowedAppIdsForController - : undefined; - - if ( ! extension[route.method] ) { - throw new Error(`Unsupported HTTP method: ${route.method}`); - } else { - logger.log(`Registering route: [${route.method.toUpperCase()}] ${fullPath}`); - - (extension[route.method] as RouterMethods[HttpMethod])( - fullPath, - route.options || {}, - async (req, res, next) => { - try { - if ( adminsForRoute || allowedAppIds ) { - if ( ! req.actor ) { - throw new HttpError(StatusCodes.UNAUTHORIZED, 'Unauthenticated'); - } - } - if ( adminsForRoute ) { - if ( ! adminsForRoute.includes(req.actor!.type.user.username) ) { - throw new HttpError( - StatusCodes.FORBIDDEN, - 'Only admins may request this resource.', - ); - } - } - if ( allowedAppIds ) { - if ( ( req.actor!.type?.app?.uid && !allowedAppIds.includes(req.actor!.type.app.uid) ) ) { - throw new HttpError( - StatusCodes.FORBIDDEN, - 'This app may not request this resource.', - ); - } - } - return await route.handler.bind(this)(req, res, next); - } catch ( error ) { - if ( error instanceof HttpError ) { - const payload: Record = { - error: error.message, - }; - if ( error.legacyCode ) { - payload.code = error.legacyCode; - } - if ( error.code ) { - if ( payload.code === undefined ) { - payload.code = error.code; - } else { - payload.errorCode = error.code; - } - } - if ( error.fields ) { - for ( const [key, value] of Object.entries(error.fields) ) { - if ( payload[key] !== undefined ) { - continue; - } - payload[key] = value; - } - } - res.status(error.statusCode).send(payload); - logger.warn('httpError:', error); - return; - } - if ( error instanceof Error ) { - res.status(StatusCodes.INTERNAL_SERVER_ERROR).send({ error: error.message }); - logger.error('Non-http error:', error); - return; - } - res.status(StatusCodes.INTERNAL_SERVER_ERROR).send({ error: 'An unknown error occurred' }); - logger.error('An unknown error occurred:', error); - } - }, - ); - } - } - } -} diff --git a/extensions/extensionController/src/index.ts b/extensions/extensionController/src/index.ts deleted file mode 100644 index d4d276998..000000000 --- a/extensions/extensionController/src/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { Controller, Delete, ExtensionController, Get, HttpError, Post, Put } from './ExtensionController.js'; - -extension.exports = { - ExtensionController, - Controller, - Get, - Put, - Post, - Delete, - HttpError, -}; - -export { - Controller, Delete, ExtensionController, Get, HttpError, Post, Put, -}; diff --git a/extensions/extensionController/tsconfig.json b/extensions/extensionController/tsconfig.json deleted file mode 100644 index c9cbd48a9..000000000 --- a/extensions/extensionController/tsconfig.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2024", - "module": "nodenext", - "moduleResolution": "nodenext", - "strict": true, - "forceConsistentCasingInFileNames": true, - "skipLibCheck": true, - "sourceMap": true, - "noEmitOnError": true, - "noImplicitAny": false, - "allowJs": true, - "checkJs": false, - }, - "include": [ - "./**/*.ts", - "./**/*.d.ts" - ], - "exclude": [ - "**/*.test.ts", - "**/*.spec.ts", - "**/test/**", - "**/tests/**", - "node_modules", - "dist", - "*.js" - ] -} \ No newline at end of file diff --git a/extensions/fsv2/package-lock.json b/extensions/fsv2/package-lock.json deleted file mode 100644 index dbf6c2349..000000000 --- a/extensions/fsv2/package-lock.json +++ /dev/null @@ -1,2653 +0,0 @@ -{ - "name": "@heyputer/prodfsv2", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@heyputer/prodfsv2", - "version": "1.0.0", - "hasInstallScript": true, - "license": "ISC", - "dependencies": { - "@aws-sdk/s3-request-presigner": "^3.1021.0", - "body-parser": "^2.2.0", - "busboy": "^1.6.0", - "http-status-codes": "^2.3.0", - "stripe": "^20.1.2", - "uuid": "^13.0.0" - }, - "devDependencies": { - "@aws-sdk/client-s3": "^3.1021.0", - "@types/express": "^4.17.21", - "@types/node": "^24.9.1", - "ts-node": "^10.9.2", - "typescript": "^5.9.3" - } - }, - "node_modules/@aws-crypto/crc32": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-5.2.0.tgz", - "integrity": "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/util": "^5.2.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-crypto/crc32c": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/crc32c/-/crc32c-5.2.0.tgz", - "integrity": "sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/util": "^5.2.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-crypto/sha1-browser": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha1-browser/-/sha1-browser-5.2.0.tgz", - "integrity": "sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/supports-web-crypto": "^5.2.0", - "@aws-crypto/util": "^5.2.0", - "@aws-sdk/types": "^3.222.0", - "@aws-sdk/util-locate-window": "^3.0.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-crypto/sha1-browser/node_modules/@smithy/is-array-buffer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", - "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/sha1-browser/node_modules/@smithy/util-buffer-from": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", - "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/is-array-buffer": "^2.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/sha1-browser/node_modules/@smithy/util-utf8": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", - "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/util-buffer-from": "^2.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/sha256-browser": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", - "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/sha256-js": "^5.2.0", - "@aws-crypto/supports-web-crypto": "^5.2.0", - "@aws-crypto/util": "^5.2.0", - "@aws-sdk/types": "^3.222.0", - "@aws-sdk/util-locate-window": "^3.0.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/is-array-buffer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", - "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-buffer-from": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", - "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/is-array-buffer": "^2.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-utf8": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", - "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/util-buffer-from": "^2.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/sha256-js": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", - "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/util": "^5.2.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-crypto/supports-web-crypto": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", - "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-crypto/util": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", - "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.222.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-crypto/util/node_modules/@smithy/is-array-buffer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", - "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/util/node_modules/@smithy/util-buffer-from": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", - "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/is-array-buffer": "^2.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/util/node_modules/@smithy/util-utf8": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", - "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/util-buffer-from": "^2.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/client-s3": { - "version": "3.1021.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1021.0.tgz", - "integrity": "sha512-BCfggq8gYSjlKOZlMSVApix3cgKAQIWGeoJFX/AU5HMvqz1BZBEw83jJFL9LYrqTPCocH8NGl++1Xr70ro+jcg==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/sha1-browser": "5.2.0", - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.26", - "@aws-sdk/credential-provider-node": "^3.972.29", - "@aws-sdk/middleware-bucket-endpoint": "^3.972.8", - "@aws-sdk/middleware-expect-continue": "^3.972.8", - "@aws-sdk/middleware-flexible-checksums": "^3.974.6", - "@aws-sdk/middleware-host-header": "^3.972.8", - "@aws-sdk/middleware-location-constraint": "^3.972.8", - "@aws-sdk/middleware-logger": "^3.972.8", - "@aws-sdk/middleware-recursion-detection": "^3.972.9", - "@aws-sdk/middleware-sdk-s3": "^3.972.27", - "@aws-sdk/middleware-ssec": "^3.972.8", - "@aws-sdk/middleware-user-agent": "^3.972.28", - "@aws-sdk/region-config-resolver": "^3.972.10", - "@aws-sdk/signature-v4-multi-region": "^3.996.15", - "@aws-sdk/types": "^3.973.6", - "@aws-sdk/util-endpoints": "^3.996.5", - "@aws-sdk/util-user-agent-browser": "^3.972.8", - "@aws-sdk/util-user-agent-node": "^3.973.14", - "@smithy/config-resolver": "^4.4.13", - "@smithy/core": "^3.23.13", - "@smithy/eventstream-serde-browser": "^4.2.12", - "@smithy/eventstream-serde-config-resolver": "^4.3.12", - "@smithy/eventstream-serde-node": "^4.2.12", - "@smithy/fetch-http-handler": "^5.3.15", - "@smithy/hash-blob-browser": "^4.2.13", - "@smithy/hash-node": "^4.2.12", - "@smithy/hash-stream-node": "^4.2.12", - "@smithy/invalid-dependency": "^4.2.12", - "@smithy/md5-js": "^4.2.12", - "@smithy/middleware-content-length": "^4.2.12", - "@smithy/middleware-endpoint": "^4.4.28", - "@smithy/middleware-retry": "^4.4.46", - "@smithy/middleware-serde": "^4.2.16", - "@smithy/middleware-stack": "^4.2.12", - "@smithy/node-config-provider": "^4.3.12", - "@smithy/node-http-handler": "^4.5.1", - "@smithy/protocol-http": "^5.3.12", - "@smithy/smithy-client": "^4.12.8", - "@smithy/types": "^4.13.1", - "@smithy/url-parser": "^4.2.12", - "@smithy/util-base64": "^4.3.2", - "@smithy/util-body-length-browser": "^4.2.2", - "@smithy/util-body-length-node": "^4.2.3", - "@smithy/util-defaults-mode-browser": "^4.3.44", - "@smithy/util-defaults-mode-node": "^4.2.48", - "@smithy/util-endpoints": "^3.3.3", - "@smithy/util-middleware": "^4.2.12", - "@smithy/util-retry": "^4.2.13", - "@smithy/util-stream": "^4.5.21", - "@smithy/util-utf8": "^4.2.2", - "@smithy/util-waiter": "^4.2.14", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/core": { - "version": "3.973.26", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.973.26.tgz", - "integrity": "sha512-A/E6n2W42ruU+sfWk+mMUOyVXbsSgGrY3MJ9/0Az5qUdG67y8I6HYzzoAa+e/lzxxl1uCYmEL6BTMi9ZiZnplQ==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.6", - "@aws-sdk/xml-builder": "^3.972.16", - "@smithy/core": "^3.23.13", - "@smithy/node-config-provider": "^4.3.12", - "@smithy/property-provider": "^4.2.12", - "@smithy/protocol-http": "^5.3.12", - "@smithy/signature-v4": "^5.3.12", - "@smithy/smithy-client": "^4.12.8", - "@smithy/types": "^4.13.1", - "@smithy/util-base64": "^4.3.2", - "@smithy/util-middleware": "^4.2.12", - "@smithy/util-utf8": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/crc64-nvme": { - "version": "3.972.5", - "resolved": "https://registry.npmjs.org/@aws-sdk/crc64-nvme/-/crc64-nvme-3.972.5.tgz", - "integrity": "sha512-2VbTstbjKdT+yKi8m7b3a9CiVac+pL/IY2PHJwsaGkkHmuuqkJZIErPck1h6P3T9ghQMLSdMPyW6Qp7Di5swFg==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.972.24", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.24.tgz", - "integrity": "sha512-FWg8uFmT6vQM7VuzELzwVo5bzExGaKHdubn0StjgrcU5FvuLExUe+k06kn/40uKv59rYzhez8eFNM4yYE/Yb/w==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.26", - "@aws-sdk/types": "^3.973.6", - "@smithy/property-provider": "^4.2.12", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.972.26", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.26.tgz", - "integrity": "sha512-CY4ppZ+qHYqcXqBVi//sdHST1QK3KzOEiLtpLsc9W2k2vfZPKExGaQIsOwcyvjpjUEolotitmd3mUNY56IwDEA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.26", - "@aws-sdk/types": "^3.973.6", - "@smithy/fetch-http-handler": "^5.3.15", - "@smithy/node-http-handler": "^4.5.1", - "@smithy/property-provider": "^4.2.12", - "@smithy/protocol-http": "^5.3.12", - "@smithy/smithy-client": "^4.12.8", - "@smithy/types": "^4.13.1", - "@smithy/util-stream": "^4.5.21", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.972.28", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.28.tgz", - "integrity": "sha512-wXYvq3+uQcZV7k+bE4yDXCTBdzWTU9x/nMiKBfzInmv6yYK1veMK0AKvRfRBd72nGWYKcL6AxwiPg9z/pYlgpw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.26", - "@aws-sdk/credential-provider-env": "^3.972.24", - "@aws-sdk/credential-provider-http": "^3.972.26", - "@aws-sdk/credential-provider-login": "^3.972.28", - "@aws-sdk/credential-provider-process": "^3.972.24", - "@aws-sdk/credential-provider-sso": "^3.972.28", - "@aws-sdk/credential-provider-web-identity": "^3.972.28", - "@aws-sdk/nested-clients": "^3.996.18", - "@aws-sdk/types": "^3.973.6", - "@smithy/credential-provider-imds": "^4.2.12", - "@smithy/property-provider": "^4.2.12", - "@smithy/shared-ini-file-loader": "^4.4.7", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-login": { - "version": "3.972.28", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.28.tgz", - "integrity": "sha512-ZSTfO6jqUTCysbdBPtEX5OUR//3rbD0lN7jO3sQeS2Gjr/Y+DT6SbIJ0oT2cemNw3UzKu97sNONd1CwNMthuZQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.26", - "@aws-sdk/nested-clients": "^3.996.18", - "@aws-sdk/types": "^3.973.6", - "@smithy/property-provider": "^4.2.12", - "@smithy/protocol-http": "^5.3.12", - "@smithy/shared-ini-file-loader": "^4.4.7", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.972.29", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.29.tgz", - "integrity": "sha512-clSzDcvndpFJAggLDnDb36sPdlZYyEs5Zm6zgZjjUhwsJgSWiWKwFIXUVBcbruidNyBdbpOv2tNDL9sX8y3/0g==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/credential-provider-env": "^3.972.24", - "@aws-sdk/credential-provider-http": "^3.972.26", - "@aws-sdk/credential-provider-ini": "^3.972.28", - "@aws-sdk/credential-provider-process": "^3.972.24", - "@aws-sdk/credential-provider-sso": "^3.972.28", - "@aws-sdk/credential-provider-web-identity": "^3.972.28", - "@aws-sdk/types": "^3.973.6", - "@smithy/credential-provider-imds": "^4.2.12", - "@smithy/property-provider": "^4.2.12", - "@smithy/shared-ini-file-loader": "^4.4.7", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-process": { - "version": "3.972.24", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.24.tgz", - "integrity": "sha512-Q2k/XLrFXhEztPHqj4SLCNID3hEPdlhh1CDLBpNnM+1L8fq7P+yON9/9M1IGN/dA5W45v44ylERfXtDAlmMNmw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.26", - "@aws-sdk/types": "^3.973.6", - "@smithy/property-provider": "^4.2.12", - "@smithy/shared-ini-file-loader": "^4.4.7", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.972.28", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.28.tgz", - "integrity": "sha512-IoUlmKMLEITFn1SiCTjPfR6KrE799FBo5baWyk/5Ppar2yXZoUdaRqZzJzK6TcJxx450M8m8DbpddRVYlp5R/A==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.26", - "@aws-sdk/nested-clients": "^3.996.18", - "@aws-sdk/token-providers": "3.1021.0", - "@aws-sdk/types": "^3.973.6", - "@smithy/property-provider": "^4.2.12", - "@smithy/shared-ini-file-loader": "^4.4.7", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.972.28", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.28.tgz", - "integrity": "sha512-d+6h0SD8GGERzKe27v5rOzNGKOl0D+l0bWJdqrxH8WSQzHzjsQFIAPgIeOTUwBHVsKKwtSxc91K/SWax6XgswQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.26", - "@aws-sdk/nested-clients": "^3.996.18", - "@aws-sdk/types": "^3.973.6", - "@smithy/property-provider": "^4.2.12", - "@smithy/shared-ini-file-loader": "^4.4.7", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-bucket-endpoint": { - "version": "3.972.8", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.972.8.tgz", - "integrity": "sha512-WR525Rr2QJSETa9a050isktyWi/4yIGcmY3BQ1kpHqb0LqUglQHCS8R27dTJxxWNZvQ0RVGtEZjTCbZJpyF3Aw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.6", - "@aws-sdk/util-arn-parser": "^3.972.3", - "@smithy/node-config-provider": "^4.3.12", - "@smithy/protocol-http": "^5.3.12", - "@smithy/types": "^4.13.1", - "@smithy/util-config-provider": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-expect-continue": { - "version": "3.972.8", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.972.8.tgz", - "integrity": "sha512-5DTBTiotEES1e2jOHAq//zyzCjeMB78lEHd35u15qnrid4Nxm7diqIf9fQQ3Ov0ChH1V3Vvt13thOnrACmfGVQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.6", - "@smithy/protocol-http": "^5.3.12", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-flexible-checksums": { - "version": "3.974.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.974.6.tgz", - "integrity": "sha512-YckB8k1ejbyCg/g36gUMFLNzE4W5cERIa4MtsdO+wpTmJEP0+TB7okWIt7d8TDOvnb7SwvxJ21E4TGOBxFpSWQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/crc32": "5.2.0", - "@aws-crypto/crc32c": "5.2.0", - "@aws-crypto/util": "5.2.0", - "@aws-sdk/core": "^3.973.26", - "@aws-sdk/crc64-nvme": "^3.972.5", - "@aws-sdk/types": "^3.973.6", - "@smithy/is-array-buffer": "^4.2.2", - "@smithy/node-config-provider": "^4.3.12", - "@smithy/protocol-http": "^5.3.12", - "@smithy/types": "^4.13.1", - "@smithy/util-middleware": "^4.2.12", - "@smithy/util-stream": "^4.5.21", - "@smithy/util-utf8": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-host-header": { - "version": "3.972.8", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.972.8.tgz", - "integrity": "sha512-wAr2REfKsqoKQ+OkNqvOShnBoh+nkPurDKW7uAeVSu6kUECnWlSJiPvnoqxGlfousEY/v9LfS9sNc46hjSYDIQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.6", - "@smithy/protocol-http": "^5.3.12", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-location-constraint": { - "version": "3.972.8", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.972.8.tgz", - "integrity": "sha512-KaUoFuoFPziIa98DSQsTPeke1gvGXlc5ZGMhy+b+nLxZ4A7jmJgLzjEF95l8aOQN2T/qlPP3MrAyELm8ExXucw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.6", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-logger": { - "version": "3.972.8", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.972.8.tgz", - "integrity": "sha512-CWl5UCM57WUFaFi5kB7IBY1UmOeLvNZAZ2/OZ5l20ldiJ3TiIz1pC65gYj8X0BCPWkeR1E32mpsCk1L1I4n+lA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.6", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-recursion-detection": { - "version": "3.972.9", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.972.9.tgz", - "integrity": "sha512-/Wt5+CT8dpTFQxEJ9iGy/UGrXr7p2wlIOEHvIr/YcHYByzoLjrqkYqXdJjd9UIgWjv7eqV2HnFJen93UTuwfTQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.6", - "@aws/lambda-invoke-store": "^0.2.2", - "@smithy/protocol-http": "^5.3.12", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3": { - "version": "3.972.27", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.27.tgz", - "integrity": "sha512-gomO6DZwx+1D/9mbCpcqO5tPBqYBK7DtdgjTIjZ4yvfh/S7ETwAPS0XbJgP2JD8Ycr5CwVrEkV1sFtu3ShXeOw==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.26", - "@aws-sdk/types": "^3.973.6", - "@aws-sdk/util-arn-parser": "^3.972.3", - "@smithy/core": "^3.23.13", - "@smithy/node-config-provider": "^4.3.12", - "@smithy/protocol-http": "^5.3.12", - "@smithy/signature-v4": "^5.3.12", - "@smithy/smithy-client": "^4.12.8", - "@smithy/types": "^4.13.1", - "@smithy/util-config-provider": "^4.2.2", - "@smithy/util-middleware": "^4.2.12", - "@smithy/util-stream": "^4.5.21", - "@smithy/util-utf8": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-ssec": { - "version": "3.972.8", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.972.8.tgz", - "integrity": "sha512-wqlK0yO/TxEC2UsY9wIlqeeutF6jjLe0f96Pbm40XscTo57nImUk9lBcw0dPgsm0sppFtAkSlDrfpK+pC30Wqw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.6", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.972.28", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.28.tgz", - "integrity": "sha512-cfWZFlVh7Va9lRay4PN2A9ARFzaBYcA097InT5M2CdRS05ECF5yaz86jET8Wsl2WcyKYEvVr/QNmKtYtafUHtQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.26", - "@aws-sdk/types": "^3.973.6", - "@aws-sdk/util-endpoints": "^3.996.5", - "@smithy/core": "^3.23.13", - "@smithy/protocol-http": "^5.3.12", - "@smithy/types": "^4.13.1", - "@smithy/util-retry": "^4.2.13", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/nested-clients": { - "version": "3.996.18", - "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.996.18.tgz", - "integrity": "sha512-c7ZSIXrESxHKx2Mcopgd8AlzZgoXMr20fkx5ViPWPOLBvmyhw9VwJx/Govg8Ef/IhEon5R9l53Z8fdYSEmp6VA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.26", - "@aws-sdk/middleware-host-header": "^3.972.8", - "@aws-sdk/middleware-logger": "^3.972.8", - "@aws-sdk/middleware-recursion-detection": "^3.972.9", - "@aws-sdk/middleware-user-agent": "^3.972.28", - "@aws-sdk/region-config-resolver": "^3.972.10", - "@aws-sdk/types": "^3.973.6", - "@aws-sdk/util-endpoints": "^3.996.5", - "@aws-sdk/util-user-agent-browser": "^3.972.8", - "@aws-sdk/util-user-agent-node": "^3.973.14", - "@smithy/config-resolver": "^4.4.13", - "@smithy/core": "^3.23.13", - "@smithy/fetch-http-handler": "^5.3.15", - "@smithy/hash-node": "^4.2.12", - "@smithy/invalid-dependency": "^4.2.12", - "@smithy/middleware-content-length": "^4.2.12", - "@smithy/middleware-endpoint": "^4.4.28", - "@smithy/middleware-retry": "^4.4.46", - "@smithy/middleware-serde": "^4.2.16", - "@smithy/middleware-stack": "^4.2.12", - "@smithy/node-config-provider": "^4.3.12", - "@smithy/node-http-handler": "^4.5.1", - "@smithy/protocol-http": "^5.3.12", - "@smithy/smithy-client": "^4.12.8", - "@smithy/types": "^4.13.1", - "@smithy/url-parser": "^4.2.12", - "@smithy/util-base64": "^4.3.2", - "@smithy/util-body-length-browser": "^4.2.2", - "@smithy/util-body-length-node": "^4.2.3", - "@smithy/util-defaults-mode-browser": "^4.3.44", - "@smithy/util-defaults-mode-node": "^4.2.48", - "@smithy/util-endpoints": "^3.3.3", - "@smithy/util-middleware": "^4.2.12", - "@smithy/util-retry": "^4.2.13", - "@smithy/util-utf8": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/region-config-resolver": { - "version": "3.972.10", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.972.10.tgz", - "integrity": "sha512-1dq9ToC6e070QvnVhhbAs3bb5r6cQ10gTVc6cyRV5uvQe7P138TV2uG2i6+Yok4bAkVAcx5AqkTEBUvWEtBlsQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.6", - "@smithy/config-resolver": "^4.4.13", - "@smithy/node-config-provider": "^4.3.12", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner": { - "version": "3.1021.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/s3-request-presigner/-/s3-request-presigner-3.1021.0.tgz", - "integrity": "sha512-kkIzsIAc7wnG7vVRkZFIwJ3noOyF3S6ozOQ9t2KxzPde1LsmpmPwYbmiB91DzdfuGySdk4Hpb0JmHh4KhGECXQ==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/signature-v4-multi-region": "^3.996.15", - "@aws-sdk/types": "^3.973.6", - "@aws-sdk/util-format-url": "^3.972.8", - "@smithy/middleware-endpoint": "^4.4.28", - "@smithy/protocol-http": "^5.3.12", - "@smithy/smithy-client": "^4.12.8", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region": { - "version": "3.996.15", - "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.15.tgz", - "integrity": "sha512-Ukw2RpqvaL96CjfH/FgfBmy/ZosHBqoHBCFsN61qGg99F33vpntIVii8aNeh65XuOja73arSduskoa4OJea9RQ==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/middleware-sdk-s3": "^3.972.27", - "@aws-sdk/types": "^3.973.6", - "@smithy/protocol-http": "^5.3.12", - "@smithy/signature-v4": "^5.3.12", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/token-providers": { - "version": "3.1021.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1021.0.tgz", - "integrity": "sha512-TKY6h9spUk3OLs5v1oAgW9mAeBE3LAGNBwJokLy96wwmd4W2v/tYlXseProyed9ValDj2u1jK/4Rg1T+1NXyJA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.26", - "@aws-sdk/nested-clients": "^3.996.18", - "@aws-sdk/types": "^3.973.6", - "@smithy/property-provider": "^4.2.12", - "@smithy/shared-ini-file-loader": "^4.4.7", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/types": { - "version": "3.973.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.973.6.tgz", - "integrity": "sha512-Atfcy4E++beKtwJHiDln2Nby8W/mam64opFPTiHEqgsthqeydFS1pY+OUlN1ouNOmf8ArPU/6cDS65anOP3KQw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/util-arn-parser": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-arn-parser/-/util-arn-parser-3.972.3.tgz", - "integrity": "sha512-HzSD8PMFrvgi2Kserxuff5VitNq2sgf3w9qxmskKDiDTThWfVteJxuCS9JXiPIPtmCrp+7N9asfIaVhBFORllA==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/util-endpoints": { - "version": "3.996.5", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.996.5.tgz", - "integrity": "sha512-Uh93L5sXFNbyR5sEPMzUU8tJ++Ku97EY4udmC01nB8Zu+xfBPwpIwJ6F7snqQeq8h2pf+8SGN5/NoytfKgYPIw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.6", - "@smithy/types": "^4.13.1", - "@smithy/url-parser": "^4.2.12", - "@smithy/util-endpoints": "^3.3.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/util-format-url": { - "version": "3.972.8", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-format-url/-/util-format-url-3.972.8.tgz", - "integrity": "sha512-J6DS9oocrgxM8xlUTTmQOuwRF6rnAGEujAN9SAzllcrQmwn5iJ58ogxy3SEhD0Q7JZvlA5jvIXBkpQRqEqlE9A==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.6", - "@smithy/querystring-builder": "^4.2.12", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/util-locate-window": { - "version": "3.965.5", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.965.5.tgz", - "integrity": "sha512-WhlJNNINQB+9qtLtZJcpQdgZw3SCDCpXdUJP7cToGwHbCWCnRckGlc6Bx/OhWwIYFNAn+FIydY8SZ0QmVu3xTQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/util-user-agent-browser": { - "version": "3.972.8", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.972.8.tgz", - "integrity": "sha512-B3KGXJviV2u6Cdw2SDY2aDhoJkVfY/Q/Trwk2CMSkikE1Oi6gRzxhvhIfiRpHfmIsAhV4EA54TVEX8K6CbHbkA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.6", - "@smithy/types": "^4.13.1", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.973.14", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.973.14.tgz", - "integrity": "sha512-vNSB/DYaPOyujVZBg/zUznH9QC142MaTHVmaFlF7uzzfg3CgT9f/l4C0Yi+vU/tbBhxVcXVB90Oohk5+o+ZbWw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/middleware-user-agent": "^3.972.28", - "@aws-sdk/types": "^3.973.6", - "@smithy/node-config-provider": "^4.3.12", - "@smithy/types": "^4.13.1", - "@smithy/util-config-provider": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "aws-crt": ">=1.0.0" - }, - "peerDependenciesMeta": { - "aws-crt": { - "optional": true - } - } - }, - "node_modules/@aws-sdk/xml-builder": { - "version": "3.972.16", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.16.tgz", - "integrity": "sha512-iu2pyvaqmeatIJLURLqx9D+4jKAdTH20ntzB6BFwjyN7V960r4jK32mx0Zf7YbtOYAbmbtQfDNuL60ONinyw7A==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.13.1", - "fast-xml-parser": "5.5.8", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws/lambda-invoke-store": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.2.4.tgz", - "integrity": "sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@cspotcode/source-map-support": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", - "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "0.3.9" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", - "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.0.3", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - }, - "node_modules/@smithy/chunked-blob-reader": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader/-/chunked-blob-reader-5.2.2.tgz", - "integrity": "sha512-St+kVicSyayWQca+I1rGitaOEH6uKgE8IUWoYnnEX26SWdWQcL6LvMSD19Lg+vYHKdT9B2Zuu7rd3i6Wnyb/iw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/chunked-blob-reader-native": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader-native/-/chunked-blob-reader-native-4.2.3.tgz", - "integrity": "sha512-jA5k5Udn7Y5717L86h4EIv06wIr3xn8GM1qHRi/Nf31annXcXHJjBKvgztnbn2TxH3xWrPBfgwHsOwZf0UmQWw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/util-base64": "^4.3.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/config-resolver": { - "version": "4.4.13", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.4.13.tgz", - "integrity": "sha512-iIzMC5NmOUP6WL6o8iPBjFhUhBZ9pPjpUpQYWMUFQqKyXXzOftbfK8zcQCz/jFV1Psmf05BK5ypx4K2r4Tnwdg==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/node-config-provider": "^4.3.12", - "@smithy/types": "^4.13.1", - "@smithy/util-config-provider": "^4.2.2", - "@smithy/util-endpoints": "^3.3.3", - "@smithy/util-middleware": "^4.2.12", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/core": { - "version": "3.23.13", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.23.13.tgz", - "integrity": "sha512-J+2TT9D6oGsUVXVEMvz8h2EmdVnkBiy2auCie4aSJMvKlzUtO5hqjEzXhoCUkIMo7gAYjbQcN0g/MMSXEhDs1Q==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.12", - "@smithy/types": "^4.13.1", - "@smithy/url-parser": "^4.2.12", - "@smithy/util-base64": "^4.3.2", - "@smithy/util-body-length-browser": "^4.2.2", - "@smithy/util-middleware": "^4.2.12", - "@smithy/util-stream": "^4.5.21", - "@smithy/util-utf8": "^4.2.2", - "@smithy/uuid": "^1.1.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/credential-provider-imds": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.2.12.tgz", - "integrity": "sha512-cr2lR792vNZcYMriSIj+Um3x9KWrjcu98kn234xA6reOAFMmbRpQMOv8KPgEmLLtx3eldU6c5wALKFqNOhugmg==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/node-config-provider": "^4.3.12", - "@smithy/property-provider": "^4.2.12", - "@smithy/types": "^4.13.1", - "@smithy/url-parser": "^4.2.12", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/eventstream-codec": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-4.2.12.tgz", - "integrity": "sha512-FE3bZdEl62ojmy8x4FHqxq2+BuOHlcxiH5vaZ6aqHJr3AIZzwF5jfx8dEiU/X0a8RboyNDjmXjlbr8AdEyLgiA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/crc32": "5.2.0", - "@smithy/types": "^4.13.1", - "@smithy/util-hex-encoding": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-browser": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-4.2.12.tgz", - "integrity": "sha512-XUSuMxlTxV5pp4VpqZf6Sa3vT/Q75FVkLSpSSE3KkWBvAQWeuWt1msTv8fJfgA4/jcJhrbrbMzN1AC/hvPmm5A==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/eventstream-serde-universal": "^4.2.12", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-config-resolver": { - "version": "4.3.12", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-4.3.12.tgz", - "integrity": "sha512-7epsAZ3QvfHkngz6RXQYseyZYHlmWXSTPOfPmXkiS+zA6TBNo1awUaMFL9vxyXlGdoELmCZyZe1nQE+imbmV+Q==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-node": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-4.2.12.tgz", - "integrity": "sha512-D1pFuExo31854eAvg89KMn9Oab/wEeJR6Buy32B49A9Ogdtx5fwZPqBHUlDzaCDpycTFk2+fSQgX689Qsk7UGA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/eventstream-serde-universal": "^4.2.12", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-universal": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-4.2.12.tgz", - "integrity": "sha512-+yNuTiyBACxOJUTvbsNsSOfH9G9oKbaJE1lNL3YHpGcuucl6rPZMi3nrpehpVOVR2E07YqFFmtwpImtpzlouHQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/eventstream-codec": "^4.2.12", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/fetch-http-handler": { - "version": "5.3.15", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.3.15.tgz", - "integrity": "sha512-T4jFU5N/yiIfrtrsb9uOQn7RdELdM/7HbyLNr6uO/mpkj1ctiVs7CihVr51w4LyQlXWDpXFn4BElf1WmQvZu/A==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.12", - "@smithy/querystring-builder": "^4.2.12", - "@smithy/types": "^4.13.1", - "@smithy/util-base64": "^4.3.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/hash-blob-browser": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-4.2.13.tgz", - "integrity": "sha512-YrF4zWKh+ghLuquldj6e/RzE3xZYL8wIPfkt0MqCRphVICjyyjH8OwKD7LLlKpVEbk4FLizFfC1+gwK6XQdR3g==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/chunked-blob-reader": "^5.2.2", - "@smithy/chunked-blob-reader-native": "^4.2.3", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/hash-node": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.2.12.tgz", - "integrity": "sha512-QhBYbGrbxTkZ43QoTPrK72DoYviDeg6YKDrHTMJbbC+A0sml3kSjzFtXP7BtbyJnXojLfTQldGdUR0RGD8dA3w==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.13.1", - "@smithy/util-buffer-from": "^4.2.2", - "@smithy/util-utf8": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/hash-stream-node": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-4.2.12.tgz", - "integrity": "sha512-O3YbmGExeafuM/kP7Y8r6+1y0hIh3/zn6GROx0uNlB54K9oihAL75Qtc+jFfLNliTi6pxOAYZrRKD9A7iA6UFw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.13.1", - "@smithy/util-utf8": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/invalid-dependency": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.2.12.tgz", - "integrity": "sha512-/4F1zb7Z8LOu1PalTdESFHR0RbPwHd3FcaG1sI3UEIriQTWakysgJr65lc1jj6QY5ye7aFsisajotH6UhWfm/g==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/is-array-buffer": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-4.2.2.tgz", - "integrity": "sha512-n6rQ4N8Jj4YTQO3YFrlgZuwKodf4zUFs7EJIWH86pSCWBaAtAGBFfCM7Wx6D2bBJ2xqFNxGBSrUWswT3M0VJow==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/md5-js": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-4.2.12.tgz", - "integrity": "sha512-W/oIpHCpWU2+iAkfZYyGWE+qkpuf3vEXHLxQQDx9FPNZTTdnul0dZ2d/gUFrtQ5je1G2kp4cjG0/24YueG2LbQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.13.1", - "@smithy/util-utf8": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/middleware-content-length": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.2.12.tgz", - "integrity": "sha512-YE58Yz+cvFInWI/wOTrB+DbvUVz/pLn5mC5MvOV4fdRUc6qGwygyngcucRQjAhiCEbmfLOXX0gntSIcgMvAjmA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.12", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/middleware-endpoint": { - "version": "4.4.28", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.4.28.tgz", - "integrity": "sha512-p1gfYpi91CHcs5cBq982UlGlDrxoYUX6XdHSo91cQ2KFuz6QloHosO7Jc60pJiVmkWrKOV8kFYlGFFbQ2WUKKQ==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/core": "^3.23.13", - "@smithy/middleware-serde": "^4.2.16", - "@smithy/node-config-provider": "^4.3.12", - "@smithy/shared-ini-file-loader": "^4.4.7", - "@smithy/types": "^4.13.1", - "@smithy/url-parser": "^4.2.12", - "@smithy/util-middleware": "^4.2.12", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/middleware-retry": { - "version": "4.4.46", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.4.46.tgz", - "integrity": "sha512-SpvWNNOPOrKQGUqZbEPO+es+FRXMWvIyzUKUOYdDgdlA6BdZj/R58p4umoQ76c2oJC44PiM7mKizyyex1IJzow==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/node-config-provider": "^4.3.12", - "@smithy/protocol-http": "^5.3.12", - "@smithy/service-error-classification": "^4.2.12", - "@smithy/smithy-client": "^4.12.8", - "@smithy/types": "^4.13.1", - "@smithy/util-middleware": "^4.2.12", - "@smithy/util-retry": "^4.2.13", - "@smithy/uuid": "^1.1.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/middleware-serde": { - "version": "4.2.16", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.2.16.tgz", - "integrity": "sha512-beqfV+RZ9RSv+sQqor3xroUUYgRFCGRw6niGstPG8zO9LgTl0B0MCucxjmrH/2WwksQN7UUgI7KNANoZv+KALA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/core": "^3.23.13", - "@smithy/protocol-http": "^5.3.12", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/middleware-stack": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-4.2.12.tgz", - "integrity": "sha512-kruC5gRHwsCOuyCd4ouQxYjgRAym2uDlCvQ5acuMtRrcdfg7mFBg6blaxcJ09STpt3ziEkis6bhg1uwrWU7txw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/node-config-provider": { - "version": "4.3.12", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-4.3.12.tgz", - "integrity": "sha512-tr2oKX2xMcO+rBOjobSwVAkV05SIfUKz8iI53rzxEmgW3GOOPOv0UioSDk+J8OpRQnpnhsO3Af6IEBabQBVmiw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/property-provider": "^4.2.12", - "@smithy/shared-ini-file-loader": "^4.4.7", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/node-http-handler": { - "version": "4.5.1", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.5.1.tgz", - "integrity": "sha512-ejjxdAXjkPIs9lyYyVutOGNOraqUE9v/NjGMKwwFrfOM354wfSD8lmlj8hVwUzQmlLLF4+udhfCX9Exnbmvfzw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.12", - "@smithy/querystring-builder": "^4.2.12", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/property-provider": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-4.2.12.tgz", - "integrity": "sha512-jqve46eYU1v7pZ5BM+fmkbq3DerkSluPr5EhvOcHxygxzD05ByDRppRwRPPpFrsFo5yDtCYLKu+kreHKVrvc7A==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/protocol-http": { - "version": "5.3.12", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.3.12.tgz", - "integrity": "sha512-fit0GZK9I1xoRlR4jXmbLhoN0OdEpa96ul8M65XdmXnxXkuMxM0Y8HDT0Fh0Xb4I85MBvBClOzgSrV1X2s1Hxw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/querystring-builder": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.2.12.tgz", - "integrity": "sha512-6wTZjGABQufekycfDGMEB84BgtdOE/rCVTov+EDXQ8NHKTUNIp/j27IliwP7tjIU9LR+sSzyGBOXjeEtVgzCHg==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.13.1", - "@smithy/util-uri-escape": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/querystring-parser": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-4.2.12.tgz", - "integrity": "sha512-P2OdvrgiAKpkPNKlKUtWbNZKB1XjPxM086NeVhK+W+wI46pIKdWBe5QyXvhUm3MEcyS/rkLvY8rZzyUdmyDZBw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/service-error-classification": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-4.2.12.tgz", - "integrity": "sha512-LlP29oSQN0Tw0b6D0Xo6BIikBswuIiGYbRACy5ujw/JgWSzTdYj46U83ssf6Ux0GyNJVivs2uReU8pt7Eu9okQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.13.1" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/shared-ini-file-loader": { - "version": "4.4.7", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.4.7.tgz", - "integrity": "sha512-HrOKWsUb+otTeo1HxVWeEb99t5ER1XrBi/xka2Wv6NVmTbuCUC1dvlrksdvxFtODLBjsC+PHK+fuy2x/7Ynyiw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/signature-v4": { - "version": "5.3.12", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.3.12.tgz", - "integrity": "sha512-B/FBwO3MVOL00DaRSXfXfa/TRXRheagt/q5A2NM13u7q+sHS59EOVGQNfG7DkmVtdQm5m3vOosoKAXSqn/OEgw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/is-array-buffer": "^4.2.2", - "@smithy/protocol-http": "^5.3.12", - "@smithy/types": "^4.13.1", - "@smithy/util-hex-encoding": "^4.2.2", - "@smithy/util-middleware": "^4.2.12", - "@smithy/util-uri-escape": "^4.2.2", - "@smithy/util-utf8": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/smithy-client": { - "version": "4.12.8", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.12.8.tgz", - "integrity": "sha512-aJaAX7vHe5i66smoSSID7t4rKY08PbD8EBU7DOloixvhOozfYWdcSYE4l6/tjkZ0vBZhGjheWzB2mh31sLgCMA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/core": "^3.23.13", - "@smithy/middleware-endpoint": "^4.4.28", - "@smithy/middleware-stack": "^4.2.12", - "@smithy/protocol-http": "^5.3.12", - "@smithy/types": "^4.13.1", - "@smithy/util-stream": "^4.5.21", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/types": { - "version": "4.13.1", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.13.1.tgz", - "integrity": "sha512-787F3yzE2UiJIQ+wYW1CVg2odHjmaWLGksnKQHUrK/lYZSEcy1msuLVvxaR/sI2/aDe9U+TBuLsXnr3vod1g0g==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/url-parser": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-4.2.12.tgz", - "integrity": "sha512-wOPKPEpso+doCZGIlr+e1lVI6+9VAKfL4kZWFgzVgGWY2hZxshNKod4l2LXS3PRC9otH/JRSjtEHqQ/7eLciRA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/querystring-parser": "^4.2.12", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-base64": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-4.3.2.tgz", - "integrity": "sha512-XRH6b0H/5A3SgblmMa5ErXQ2XKhfbQB+Fm/oyLZ2O2kCUrwgg55bU0RekmzAhuwOjA9qdN5VU2BprOvGGUkOOQ==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/util-buffer-from": "^4.2.2", - "@smithy/util-utf8": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-body-length-browser": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-4.2.2.tgz", - "integrity": "sha512-JKCrLNOup3OOgmzeaKQwi4ZCTWlYR5H4Gm1r2uTMVBXoemo1UEghk5vtMi1xSu2ymgKVGW631e2fp9/R610ZjQ==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-body-length-node": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-4.2.3.tgz", - "integrity": "sha512-ZkJGvqBzMHVHE7r/hcuCxlTY8pQr1kMtdsVPs7ex4mMU+EAbcXppfo5NmyxMYi2XU49eqaz56j2gsk4dHHPG/g==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-buffer-from": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-4.2.2.tgz", - "integrity": "sha512-FDXD7cvUoFWwN6vtQfEta540Y/YBe5JneK3SoZg9bThSoOAC/eGeYEua6RkBgKjGa/sz6Y+DuBZj3+YEY21y4Q==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/is-array-buffer": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-config-provider": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-4.2.2.tgz", - "integrity": "sha512-dWU03V3XUprJwaUIFVv4iOnS1FC9HnMHDfUrlNDSh4315v0cWyaIErP8KiqGVbf5z+JupoVpNM7ZB3jFiTejvQ==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-defaults-mode-browser": { - "version": "4.3.44", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.44.tgz", - "integrity": "sha512-eZg6XzaCbVr2S5cAErU5eGBDaOVTuTo1I65i4tQcHENRcZ8rMWhQy1DaIYUSLyZjsfXvmCqZrstSMYyGFocvHA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/property-provider": "^4.2.12", - "@smithy/smithy-client": "^4.12.8", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-defaults-mode-node": { - "version": "4.2.48", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.48.tgz", - "integrity": "sha512-FqOKTlqSaoV3nzO55pMs5NBnZX8EhoI0DGmn9kbYeXWppgHD6dchyuj2HLqp4INJDJbSrj6OFYJkAh/WhSzZPg==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/config-resolver": "^4.4.13", - "@smithy/credential-provider-imds": "^4.2.12", - "@smithy/node-config-provider": "^4.3.12", - "@smithy/property-provider": "^4.2.12", - "@smithy/smithy-client": "^4.12.8", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-endpoints": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-3.3.3.tgz", - "integrity": "sha512-VACQVe50j0HZPjpwWcjyT51KUQ4AnsvEaQ2lKHOSL4mNLD0G9BjEniQ+yCt1qqfKfiAHRAts26ud7hBjamrwig==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/node-config-provider": "^4.3.12", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-hex-encoding": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-4.2.2.tgz", - "integrity": "sha512-Qcz3W5vuHK4sLQdyT93k/rfrUwdJ8/HZ+nMUOyGdpeGA1Wxt65zYwi3oEl9kOM+RswvYq90fzkNDahPS8K0OIg==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-middleware": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.2.12.tgz", - "integrity": "sha512-Er805uFUOvgc0l8nv0e0su0VFISoxhJ/AwOn3gL2NWNY2LUEldP5WtVcRYSQBcjg0y9NfG8JYrCJaYDpupBHJQ==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-retry": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-4.2.13.tgz", - "integrity": "sha512-qQQsIvL0MGIbUjeSrg0/VlQ3jGNKyM3/2iU3FPNgy01z+Sp4OvcaxbgIoFOTvB61ZoohtutuOvOcgmhbD0katQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/service-error-classification": "^4.2.12", - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-stream": { - "version": "4.5.21", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.5.21.tgz", - "integrity": "sha512-KzSg+7KKywLnkoKejRtIBXDmwBfjGvg1U1i/etkC7XSWUyFCoLno1IohV2c74IzQqdhX5y3uE44r/8/wuK+A7Q==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/fetch-http-handler": "^5.3.15", - "@smithy/node-http-handler": "^4.5.1", - "@smithy/types": "^4.13.1", - "@smithy/util-base64": "^4.3.2", - "@smithy/util-buffer-from": "^4.2.2", - "@smithy/util-hex-encoding": "^4.2.2", - "@smithy/util-utf8": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-uri-escape": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-4.2.2.tgz", - "integrity": "sha512-2kAStBlvq+lTXHyAZYfJRb/DfS3rsinLiwb+69SstC9Vb0s9vNWkRwpnj918Pfi85mzi42sOqdV72OLxWAISnw==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-utf8": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-4.2.2.tgz", - "integrity": "sha512-75MeYpjdWRe8M5E3AW0O4Cx3UadweS+cwdXjwYGBW5h/gxxnbeZ877sLPX/ZJA9GVTlL/qG0dXP29JWFCD1Ayw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/util-buffer-from": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-waiter": { - "version": "4.2.14", - "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-4.2.14.tgz", - "integrity": "sha512-2zqq5o/oizvMaFUlNiTyZ7dbgYv1a893aGut2uaxtbzTx/VYYnRxWzDHuD/ftgcw94ffenua+ZNLrbqwUYE+Bg==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.13.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/uuid": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@smithy/uuid/-/uuid-1.1.2.tgz", - "integrity": "sha512-O/IEdcCUKkubz60tFbGA7ceITTAJsty+lBjNoorP4Z6XRqaFb/OjQjZODophEcuq68nKm6/0r+6/lLQ+XVpk8g==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@tsconfig/node10": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz", - "integrity": "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node12": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", - "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node14": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", - "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node16": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", - "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/body-parser": { - "version": "1.19.6", - "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", - "integrity": "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/connect": "*", - "@types/node": "*" - } - }, - "node_modules/@types/connect": { - "version": "3.4.38", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", - "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/express": { - "version": "4.17.25", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", - "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.33", - "@types/qs": "*", - "@types/serve-static": "^1" - } - }, - "node_modules/@types/express-serve-static-core": { - "version": "4.19.8", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.8.tgz", - "integrity": "sha512-02S5fmqeoKzVZCHPZid4b8JH2eM5HzQLZWN2FohQEy/0eXTq8VXZfSN6Pcr3F6N9R/vNrj7cpgbhjie6m/1tCA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*", - "@types/send": "*" - } - }, - "node_modules/@types/http-errors": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz", - "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/mime": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", - "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/node": { - "version": "24.12.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.0.tgz", - "integrity": "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ==", - "devOptional": true, - "license": "MIT", - "dependencies": { - "undici-types": "~7.16.0" - } - }, - "node_modules/@types/qs": { - "version": "6.15.0", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.0.tgz", - "integrity": "sha512-JawvT8iBVWpzTrz3EGw9BTQFg3BQNmwERdKE22vlTxawwtbyUSlMppvZYKLZzB5zgACXdXxbD3m1bXaMqP/9ow==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/range-parser": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", - "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/send": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz", - "integrity": "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/serve-static": { - "version": "1.15.10", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.10.tgz", - "integrity": "sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/http-errors": "*", - "@types/node": "*", - "@types/send": "<1" - } - }, - "node_modules/@types/serve-static/node_modules/@types/send": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.6.tgz", - "integrity": "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/mime": "^1", - "@types/node": "*" - } - }, - "node_modules/acorn": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", - "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", - "dev": true, - "license": "MIT", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-walk": { - "version": "8.3.5", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.5.tgz", - "integrity": "sha512-HEHNfbars9v4pgpW6SO1KSPkfoS0xVOM/9UzkJltjlsHZmJasxg8aXkuZa7SMf8vKGIBhpUsPluQSqhJFCqebw==", - "dev": true, - "license": "MIT", - "dependencies": { - "acorn": "^8.11.0" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/arg": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", - "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", - "dev": true, - "license": "MIT" - }, - "node_modules/body-parser": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz", - "integrity": "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==", - "license": "MIT", - "dependencies": { - "bytes": "^3.1.2", - "content-type": "^1.0.5", - "debug": "^4.4.3", - "http-errors": "^2.0.0", - "iconv-lite": "^0.7.0", - "on-finished": "^2.4.1", - "qs": "^6.14.1", - "raw-body": "^3.0.1", - "type-is": "^2.0.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/bowser": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", - "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", - "dev": true, - "license": "MIT" - }, - "node_modules/busboy": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", - "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", - "dependencies": { - "streamsearch": "^1.1.0" - }, - "engines": { - "node": ">=10.16.0" - } - }, - "node_modules/bytes": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", - "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/call-bound": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", - "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "get-intrinsic": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/content-type": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", - "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/create-require": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", - "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/diff": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.4.tgz", - "integrity": "sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.3.1" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/ee-first": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", - "license": "MIT" - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/fast-xml-builder": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.4.tgz", - "integrity": "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "dependencies": { - "path-expression-matcher": "^1.1.3" - } - }, - "node_modules/fast-xml-parser": { - "version": "5.5.8", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.5.8.tgz", - "integrity": "sha512-Z7Fh2nVQSb2d+poDViM063ix2ZGt9jmY1nWhPfHBOK2Hgnb/OW3P4Et3P/81SEej0J7QbWtJqxO05h8QYfK7LQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "dependencies": { - "fast-xml-builder": "^1.1.4", - "path-expression-matcher": "^1.2.0", - "strnum": "^2.2.0" - }, - "bin": { - "fxparser": "src/cli/cli.js" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", - "license": "MIT", - "dependencies": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" - }, - "engines": { - "node": ">= 0.8" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/http-status-codes": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/http-status-codes/-/http-status-codes-2.3.0.tgz", - "integrity": "sha512-RJ8XvFvpPM/Dmc5SV+dC4y5PCeOhT3x1Hq0NU3rjGeg5a/CqlhZ7uudknPwZFz4aeAXDcbAyaeP7GAo9lvngtA==", - "license": "MIT" - }, - "node_modules/iconv-lite": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", - "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "license": "ISC" - }, - "node_modules/make-error": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", - "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", - "dev": true, - "license": "ISC" - }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/media-typer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", - "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", - "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", - "license": "MIT", - "dependencies": { - "mime-db": "^1.54.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/object-inspect": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", - "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/on-finished": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", - "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", - "license": "MIT", - "dependencies": { - "ee-first": "1.1.1" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/path-expression-matcher": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.2.0.tgz", - "integrity": "sha512-DwmPWeFn+tq7TiyJ2CxezCAirXjFxvaiD03npak3cRjlP9+OjTmSy1EpIrEbh+l6JgUundniloMLDQ/6VTdhLQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/qs": { - "version": "6.15.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.0.tgz", - "integrity": "sha512-mAZTtNCeetKMH+pSjrb76NAM8V9a05I9aBZOHztWy/UqcJdQYNsf59vrRKWnojAT9Y+GbIvoTBC++CPHqpDBhQ==", - "license": "BSD-3-Clause", - "dependencies": { - "side-channel": "^1.1.0" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/raw-body": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", - "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", - "license": "MIT", - "dependencies": { - "bytes": "~3.1.2", - "http-errors": "~2.0.1", - "iconv-lite": "~0.7.0", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" - }, - "node_modules/setprototypeof": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", - "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", - "license": "ISC" - }, - "node_modules/side-channel": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", - "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.3", - "side-channel-list": "^1.0.0", - "side-channel-map": "^1.0.1", - "side-channel-weakmap": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-list": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", - "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-map": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", - "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-weakmap": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", - "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3", - "side-channel-map": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/streamsearch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", - "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/stripe": { - "version": "20.4.1", - "resolved": "https://registry.npmjs.org/stripe/-/stripe-20.4.1.tgz", - "integrity": "sha512-axCguHItc8Sxt0HC6aSkdVRPffjYPV7EQqZRb2GkIa8FzWDycE7nHJM19C6xAIynH1Qp1/BHiopSi96jGBxT0w==", - "license": "MIT", - "engines": { - "node": ">=16" - }, - "peerDependencies": { - "@types/node": ">=16" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/strnum": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.2.tgz", - "integrity": "sha512-DnR90I+jtXNSTXWdwrEy9FakW7UX+qUZg28gj5fk2vxxl7uS/3bpI4fjFYVmdK9etptYBPNkpahuQnEwhwECqA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT" - }, - "node_modules/toidentifier": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", - "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", - "license": "MIT", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/ts-node": { - "version": "10.9.2", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", - "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@cspotcode/source-map-support": "^0.8.0", - "@tsconfig/node10": "^1.0.7", - "@tsconfig/node12": "^1.0.7", - "@tsconfig/node14": "^1.0.0", - "@tsconfig/node16": "^1.0.2", - "acorn": "^8.4.1", - "acorn-walk": "^8.1.1", - "arg": "^4.1.0", - "create-require": "^1.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "v8-compile-cache-lib": "^3.0.1", - "yn": "3.1.1" - }, - "bin": { - "ts-node": "dist/bin.js", - "ts-node-cwd": "dist/bin-cwd.js", - "ts-node-esm": "dist/bin-esm.js", - "ts-node-script": "dist/bin-script.js", - "ts-node-transpile-only": "dist/bin-transpile.js", - "ts-script": "dist/bin-script-deprecated.js" - }, - "peerDependencies": { - "@swc/core": ">=1.2.50", - "@swc/wasm": ">=1.2.50", - "@types/node": "*", - "typescript": ">=2.7" - }, - "peerDependenciesMeta": { - "@swc/core": { - "optional": true - }, - "@swc/wasm": { - "optional": true - } - } - }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD" - }, - "node_modules/type-is": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", - "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", - "license": "MIT", - "dependencies": { - "content-type": "^1.0.5", - "media-typer": "^1.1.0", - "mime-types": "^3.0.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/undici-types": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", - "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", - "devOptional": true, - "license": "MIT" - }, - "node_modules/unpipe": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/uuid": { - "version": "13.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.0.tgz", - "integrity": "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist-node/bin/uuid" - } - }, - "node_modules/v8-compile-cache-lib": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", - "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", - "dev": true, - "license": "MIT" - }, - "node_modules/yn": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", - "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - } - } -} diff --git a/extensions/fsv2/package.json b/extensions/fsv2/package.json deleted file mode 100644 index 88707a94a..000000000 --- a/extensions/fsv2/package.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "name": "@heyputer/fs", - "version": "1.0.0", - "description": "", - "main": "src/index.js", - "type": "module", - "scripts": { - "postinstall": "tsc --noCheck", - "test": "echo \"Error: no test specified\" && exit 1" - }, - "keywords": [], - "author": "", - "license": "ISC", - "devDependencies": { - "@types/express": "^4.17.21", - "@types/node": "^24.9.1", - "ts-node": "^10.9.2", - "typescript": "^5.9.3", - "@aws-sdk/client-s3": "^3.1021.0" - }, - "dependencies": { - "@aws-sdk/s3-request-presigner": "^3.1021.0", - "body-parser": "^2.2.0", - "busboy": "^1.6.0", - "http-status-codes": "^2.3.0", - "stripe": "^20.1.2", - "uuid": "^13.0.0" - } -} diff --git a/extensions/fsv2/src/.gitignore b/extensions/fsv2/src/.gitignore deleted file mode 100644 index 4e57eef88..000000000 --- a/extensions/fsv2/src/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.js -*.js.map diff --git a/extensions/fsv2/src/eventHandlers/FSEntryCacheInvalidationEventHandler.ts b/extensions/fsv2/src/eventHandlers/FSEntryCacheInvalidationEventHandler.ts deleted file mode 100644 index 21be28d42..000000000 --- a/extensions/fsv2/src/eventHandlers/FSEntryCacheInvalidationEventHandler.ts +++ /dev/null @@ -1,153 +0,0 @@ -import type { FSEntryRepository } from '../repositories/FSEntryRepository.js'; -import type { - FsRemoveNodeEventPayload, - FsRemoveNodeTarget, - OuterGuiItemEventPayload, -} from './types.js'; - -export class FSEntryCacheInvalidationEventHandler { - #fsEntryRepository: FSEntryRepository; - - constructor (fsEntryRepository: FSEntryRepository) { - this.#fsEntryRepository = fsEntryRepository; - this.#registerHandlers(); - } - - #registerHandlers (): void { - extension.on('outer.gui.item.added', async (event: OuterGuiItemEventPayload) => { - await this.#runSafely(() => this.#handleOuterGuiItemEvent(event), 'outer.gui.item.added'); - }); - extension.on('outer.gui.item.updated', async (event: OuterGuiItemEventPayload) => { - await this.#runSafely(() => this.#handleOuterGuiItemEvent(event), 'outer.gui.item.updated'); - }); - extension.on('outer.gui.item.moved', async (event: OuterGuiItemEventPayload) => { - await this.#runSafely(() => this.#handleOuterGuiItemEvent(event), 'outer.gui.item.moved'); - }); - extension.on('fs.remove.node', async (event: FsRemoveNodeEventPayload) => { - await this.#runSafely(() => this.#handleRemoveNodeEvent(event), 'fs.remove.node'); - }); - } - - async #runSafely (handler: () => Promise, eventName: string): Promise { - try { - await handler(); - } catch ( error ) { - console.error(`prodfsv2 cache invalidation failed for ${eventName}`, error); - } - } - - #toUserIds (value: unknown): number[] { - if ( ! Array.isArray(value) ) { - return []; - } - - const userIds: number[] = []; - for ( const item of value ) { - const numeric = Number(item); - if ( Number.isInteger(numeric) && numeric > 0 ) { - userIds.push(numeric); - } - } - return userIds; - } - - #toNonEmptyString (value: unknown): string | null { - if ( typeof value !== 'string' ) { - return null; - } - const trimmed = value.trim(); - return trimmed.length > 0 ? trimmed : null; - } - - #isUnrecognizedTargetKeyError (error: unknown): boolean { - if ( ! (error instanceof Error) ) { - return false; - } - return error.message.includes('unrecognize key for FSNodeContext.get:'); - } - - async #readTargetValue (target: FsRemoveNodeTarget, keys: string[]): Promise { - if ( typeof target.get !== 'function' ) { - return undefined; - } - - for ( const key of keys ) { - try { - return await target.get(key); - } catch ( error ) { - if ( this.#isUnrecognizedTargetKeyError(error) ) { - continue; - } - throw error; - } - } - - return undefined; - } - - #extractUidFromEntry (value: unknown): string | null { - if ( !value || typeof value !== 'object' ) { - return null; - } - const entry = value as { uid?: unknown; uuid?: unknown }; - return this.#toNonEmptyString(entry.uid) ?? this.#toNonEmptyString(entry.uuid); - } - - async #handleOuterGuiItemEvent (event: OuterGuiItemEventPayload): Promise { - const userIds = this.#toUserIds(event?.user_id_list); - const response = event?.response ?? {}; - - const path = this.#toNonEmptyString(response.path); - const oldPath = this.#toNonEmptyString(response.old_path); - const uid = - this.#toNonEmptyString(response.uid) - ?? this.#toNonEmptyString(response.uuid) - ?? this.#toNonEmptyString(response.id); - - const tasks: Promise[] = []; - for ( const userId of userIds ) { - if ( path ) { - tasks.push(this.#fsEntryRepository.invalidateEntryCacheByPathForUser(userId, path)); - } - if ( oldPath && oldPath !== path ) { - tasks.push(this.#fsEntryRepository.invalidateEntryCacheByPathForUser(userId, oldPath)); - } - } - if ( uid ) { - tasks.push(this.#fsEntryRepository.invalidateEntryCacheByUuid(uid)); - } - - if ( tasks.length > 0 ) { - await Promise.all(tasks); - } - } - - async #handleRemoveNodeEvent (event: FsRemoveNodeEventPayload): Promise { - const target = event?.target; - if ( !target || typeof target.get !== 'function' ) { - return; - } - - const userIdValue = await this.#readTargetValue(target, ['user_id']); - const pathValue = await this.#readTargetValue(target, ['path']); - const uidValue = - await this.#readTargetValue(target, ['uid', 'uuid']) - ?? this.#extractUidFromEntry(await this.#readTargetValue(target, ['entry'])); - - const userId = Number(userIdValue); - const path = this.#toNonEmptyString(pathValue); - const uuid = this.#toNonEmptyString(uidValue); - - const tasks: Promise[] = []; - if ( Number.isInteger(userId) && userId > 0 && path ) { - tasks.push(this.#fsEntryRepository.invalidateEntryCacheByPathForUser(userId, path)); - } - if ( uuid ) { - tasks.push(this.#fsEntryRepository.invalidateEntryCacheByUuid(uuid)); - } - - if ( tasks.length > 0 ) { - await Promise.all(tasks); - } - } -} diff --git a/extensions/fsv2/src/globals.d.ts b/extensions/fsv2/src/globals.d.ts deleted file mode 100644 index 7f0006112..000000000 --- a/extensions/fsv2/src/globals.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -/// - -export {}; diff --git a/extensions/fsv2/src/index.d.ts b/extensions/fsv2/src/index.d.ts deleted file mode 100644 index 55986b5e6..000000000 --- a/extensions/fsv2/src/index.d.ts +++ /dev/null @@ -1,6 +0,0 @@ -import '@heyputer/backend/src/services/User.js'; -declare module '../../packages/puter/src/backend/src/services/User.d.ts' { - export interface IUser { - stripe_customer_id?: string; - } -} \ No newline at end of file diff --git a/extensions/fsv2/src/index.ts b/extensions/fsv2/src/index.ts deleted file mode 100644 index f8784d3da..000000000 --- a/extensions/fsv2/src/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -import type { DynamoKVStore } from '@heyputer/backend/src/services/DynamoKVStore/DynamoKVStore.js'; -import { FSController } from './controllers/FSController.js'; -import { FSEntryCacheInvalidationEventHandler } from './eventHandlers/FSEntryCacheInvalidationEventHandler.js'; -import { FSEntryRepository } from './repositories/FSEntryRepository.js'; -import { S3StorageProvider } from './repositories/S3FileStorageRepository.js'; -import { FSEntryService } from './services/FSEntryService.js'; - -const databaseService = extension.import('service:database'); -const { cache, s3ClientProvider } = extension.import('data'); -const eventService = extension.import('service:event'); -const kvStore = extension.import('service:puter-kvstore') as DynamoKVStore; -const filesystemDb = databaseService; - -const fsEntryRepository = new FSEntryRepository(filesystemDb, cache, kvStore); -const s3StorageProvider = new S3StorageProvider(s3ClientProvider); -const fsEntryService = new FSEntryService(fsEntryRepository, s3StorageProvider); - -const fsController = new FSController(fsEntryService, eventService); -(fsController as unknown as { registerRoutes: () => void }).registerRoutes(); -new FSEntryCacheInvalidationEventHandler(fsEntryRepository); diff --git a/extensions/fsv2/src/repositories/FSEntryRepository.ts b/extensions/fsv2/src/repositories/FSEntryRepository.ts deleted file mode 100644 index 5d7984801..000000000 --- a/extensions/fsv2/src/repositories/FSEntryRepository.ts +++ /dev/null @@ -1,1362 +0,0 @@ -import type { BaseDatabaseAccessService } from '@heyputer/backend/src/services/database/BaseDatabaseAccessService.js'; -import type { DynamoKVStore } from '@heyputer/backend/src/services/DynamoKVStore/DynamoKVStore.js'; -import type { Cluster } from 'ioredis'; -import { posix as pathPosix } from 'node:path'; -import { v4 as uuidv4 } from 'uuid'; -import { - FSEntry, - FSEntryCreateInput, - PendingUploadCreateInput, - PendingUploadSession, -} from '../types/FSEntry.js'; -import { runWithConcurrencyLimit } from '../utils/concurrency.js'; -import { - normalizePendingUploadSession, - PendingUploadSessionStatus, - toPendingUploadSession, - toPendingUploadSessionExpiresAtSeconds, - toPendingUploadSessionKey, - withPendingUploadSessionStatus, -} from './pendingUploadSessionHelpers.js'; -import type { - FSEntryRow, - NormalizedEntryWrite, - ReadEntriesByPathsOptions, -} from './types.js'; - -const { HttpError } = extension.import('extensionController'); - -const ENTRY_CACHE_TTL_SECONDS = 60; -const BULK_QUERY_CHUNK_SIZE = 200; -const DEFAULT_DB_CHUNK_CONCURRENCY = 4; - -export class FSEntryRepository { - #db: BaseDatabaseAccessService; - #cache: Cluster; - #kvStore: DynamoKVStore; - - constructor (db: BaseDatabaseAccessService, cache: Cluster, kvStore: DynamoKVStore) { - this.#db = db; - this.#cache = cache; - this.#kvStore = kvStore; - } - - #insertIgnoreIntoFsentriesSql (): string { - return this.#db.case({ - sqlite: 'INSERT OR IGNORE INTO fsentries', - otherwise: 'INSERT IGNORE INTO fsentries', - }); - } - - #normalizePath (path: string): string { - const trimmed = path.trim(); - if ( trimmed.length === 0 ) { - throw new HttpError(400, 'Path cannot be empty'); - } - - let normalized = pathPosix.normalize(trimmed); - if ( ! normalized.startsWith('/') ) { - normalized = `/${normalized}`; - } - if ( normalized.length > 1 && normalized.endsWith('/') ) { - normalized = normalized.slice(0, -1); - } - - return normalized; - } - - #toBoolean (value: number | boolean | null | undefined): boolean { - if ( typeof value === 'boolean' ) { - return value; - } - return Number(value ?? 0) === 1; - } - - #toNullableBoolean (value: number | boolean | null | undefined): boolean | null { - if ( value === null || value === undefined ) { - return null; - } - return this.#toBoolean(value); - } - - #mapFSEntryRow (row: FSEntryRow): FSEntry { - return { - id: Number(row.id), - uuid: row.uuid, - uid: row.uuid, - userId: Number(row.user_id), - parentId: row.parent_id === null ? null : Number(row.parent_id), - parentUid: row.parent_uid, - path: row.path, - name: row.name, - isDir: this.#toBoolean(row.is_dir), - bucket: row.bucket, - bucketRegion: row.bucket_region, - publicToken: row.public_token, - fileRequestToken: row.file_request_token, - isShortcut: this.#toBoolean(row.is_shortcut), - shortcutTo: row.shortcut_to, - associatedAppId: row.associated_app_id, - layout: row.layout, - sortBy: row.sort_by, - sortOrder: row.sort_order, - isPublic: this.#toNullableBoolean(row.is_public), - thumbnail: row.thumbnail, - immutable: this.#toBoolean(row.immutable), - metadata: row.metadata, - modified: Number(row.modified), - created: row.created === null ? null : Number(row.created), - accessed: row.accessed === null ? null : Number(row.accessed), - size: row.size === null ? null : Number(row.size), - symlinkPath: row.symlink_path, - isSymlink: this.#toBoolean(row.is_symlink), - }; - } - - #entryCacheKeys (entry: FSEntry): string[] { - return [ - `prodfsv2:fsentry:id:${entry.id}`, - `prodfsv2:fsentry:uuid:${entry.uuid}`, - `prodfsv2:fsentry:path:${entry.userId}:${entry.path}`, - `prodfsv2:fsentry:path:any:${entry.path}`, - ]; - } - - async #readEntryFromCache (cacheKey: string): Promise { - try { - const cached = await this.#cache.get(cacheKey); - if ( ! cached ) { - return null; - } - return JSON.parse(cached) as FSEntry; - } catch { - return null; - } - } - - async #writeEntryToCache (entry: FSEntry): Promise { - try { - const serialized = JSON.stringify(entry); - await Promise.all(this.#entryCacheKeys(entry).map((cacheKey) => { - return this.#cache.setex(cacheKey, ENTRY_CACHE_TTL_SECONDS, serialized); - })); - } catch { - // Best effort cache write. - } - } - - async #invalidateEntryCache (entry: FSEntry): Promise { - try { - const keys = this.#entryCacheKeys(entry); - if ( keys.length > 0 ) { - await this.#cache.del(...keys); - } - } catch { - // Best effort cache invalidation. - } - } - - async invalidateEntryCacheByPathForUser (userId: number, path: string): Promise { - const normalizedPath = this.#normalizePath(path); - const cacheKeys: string[] = [ - `prodfsv2:fsentry:path:${userId}:${normalizedPath}`, - `prodfsv2:fsentry:path:any:${normalizedPath}`, - ]; - - const rows = await this.#db.read( - 'SELECT * FROM fsentries WHERE user_id = ? AND path = ? LIMIT 1', - [userId, normalizedPath], - ) as FSEntryRow[]; - const row = rows[0]; - - if ( row ) { - const entry = this.#mapFSEntryRow(row); - await this.#invalidateEntryCache(entry); - return; - } - - try { - await this.#cache.del(...cacheKeys); - } catch { - // Best effort cache invalidation. - } - } - - async invalidateEntryCacheByUuid (uuid: string): Promise { - if ( typeof uuid !== 'string' || uuid.length === 0 ) { - return; - } - - const rows = await this.#db.read( - 'SELECT * FROM fsentries WHERE uuid = ? LIMIT 1', - [uuid], - ) as FSEntryRow[]; - const row = rows[0]; - - if ( row ) { - const entry = this.#mapFSEntryRow(row); - await this.#invalidateEntryCache(entry); - return; - } - - const cached = await this.#readEntryFromCache(`prodfsv2:fsentry:uuid:${uuid}`); - if ( cached ) { - await this.#invalidateEntryCache(cached); - return; - } - - try { - await this.#cache.del(`prodfsv2:fsentry:uuid:${uuid}`); - } catch { - // Best effort cache invalidation. - } - } - - #chunk (values: T[], size: number): T[][] { - if ( values.length === 0 ) { - return []; - } - const chunks: T[][] = []; - for ( let index = 0; index < values.length; index += size ) { - chunks.push(values.slice(index, index + size)); - } - return chunks; - } - - async #writePendingUploadSessions (sessions: PendingUploadSession[], operationName: string): Promise { - if ( sessions.length === 0 ) { - return; - } - - try { - await this.#kvStore.batchPut({ - items: sessions.map((session) => ({ - key: toPendingUploadSessionKey(session.sessionId), - value: session, - expireAt: toPendingUploadSessionExpiresAtSeconds(session.expiresAt), - })), - }); - } catch ( error ) { - if ( error instanceof Error ) { - throw error; - } - throw new Error(`Failed to ${operationName}`); - } - } - - async #getPendingUploadSessionsBySessionIds ( - sessionIds: string[], - ): Promise> { - const uniqueSessionIds = Array.from(new Set(sessionIds)); - const sessionsById = new Map(); - if ( uniqueSessionIds.length === 0 ) { - return sessionsById; - } - - const rawValues = await this.#kvStore.get({ - key: uniqueSessionIds.map((sessionId) => toPendingUploadSessionKey(sessionId)), - }); - if ( ! Array.isArray(rawValues) ) { - return sessionsById; - } - - for ( let index = 0; index < uniqueSessionIds.length; index++ ) { - const sessionId = uniqueSessionIds[index]; - const rawValue = rawValues[index]; - if ( ! sessionId ) { - continue; - } - - const normalizedSession = normalizePendingUploadSession(rawValue, sessionId); - if ( normalizedSession ) { - sessionsById.set(sessionId, normalizedSession); - } - } - - return sessionsById; - } - - async #markPendingSessionsWithStatus ( - sessionIds: string[], - status: PendingUploadSessionStatus, - reason: string | null, - ): Promise { - if ( sessionIds.length === 0 ) { - return; - } - - const sessionsById = await this.#getPendingUploadSessionsBySessionIds(sessionIds); - const now = Date.now(); - const updatedSessions = Array.from(new Set(sessionIds)) - .map((sessionId) => { - const session = sessionsById.get(sessionId); - if ( ! session ) { - return null; - } - - return withPendingUploadSessionStatus(session, status, reason, now); - }) - .filter((session): session is PendingUploadSession => Boolean(session)); - - await this.#writePendingUploadSessions( - updatedSessions, - `mark pending upload sessions as ${status}`, - ); - } - - async #readEntriesByPathsForUser ( - userId: number, - paths: string[], - options: ReadEntriesByPathsOptions = {}, - ): Promise> { - const useTryHardRead = Boolean(options.useTryHardRead); - const skipCache = Boolean(options.skipCache); - const normalizedPaths = Array.from(new Set(paths - .map((path) => this.#normalizePath(path)) - .filter((path) => path.length > 0))); - const entriesByPath = new Map(); - if ( normalizedPaths.length === 0 ) { - return entriesByPath; - } - - const missingPaths: string[] = []; - if ( skipCache ) { - missingPaths.push(...normalizedPaths); - } else { - const cacheReads = await Promise.all(normalizedPaths.map(async (path) => { - const cacheKey = `prodfsv2:fsentry:path:${userId}:${path}`; - const cachedEntry = await this.#readEntryFromCache(cacheKey); - return { path, cachedEntry }; - })); - - for ( const cacheRead of cacheReads ) { - if ( cacheRead.cachedEntry ) { - entriesByPath.set(cacheRead.path, cacheRead.cachedEntry); - } else { - missingPaths.push(cacheRead.path); - } - } - } - - const chunks = this.#chunk(missingPaths, BULK_QUERY_CHUNK_SIZE); - const chunkResults = await runWithConcurrencyLimit( - chunks, - DEFAULT_DB_CHUNK_CONCURRENCY, - async (chunk) => { - if ( chunk.length === 0 ) { - return []; - } - - const placeholders = chunk.map(() => '?').join(', '); - const rows = (useTryHardRead ? await this.#db.tryHardRead( - `SELECT * FROM fsentries WHERE user_id = ? AND path IN (${placeholders})`, - [userId, ...chunk], - ) : await this.#db.read( - `SELECT * FROM fsentries WHERE user_id = ? AND path IN (${placeholders})`, - [userId, ...chunk], - )) as FSEntryRow[]; - - const entries = rows.map((row) => this.#mapFSEntryRow(row)); - if ( entries.length > 0 ) { - await Promise.all(entries.map((entry) => this.#writeEntryToCache(entry))); - } - return entries; - }, - ); - for ( const chunkEntries of chunkResults ) { - for ( const entry of chunkEntries ) { - entriesByPath.set(entry.path, entry); - } - } - - return entriesByPath; - } - - #pathDepth (path: string): number { - return path.split('/').filter(Boolean).length; - } - - async #ensureDirectoryPathsForUser ( - userId: number, - requiredPaths: string[], - ): Promise<{ - requiredEntryMap: Map; - createdEntryMap: Map; - }> { - const normalizedRequiredPaths = Array.from(new Set(requiredPaths - .map((path) => this.#normalizePath(path)) - .filter((path) => path !== '/'))); - const requiredEntryMap = new Map(); - const createdEntryMap = new Map(); - if ( normalizedRequiredPaths.length === 0 ) { - return { - requiredEntryMap, - createdEntryMap, - }; - } - - const candidateDirSet = new Set(); - for ( const requiredPath of normalizedRequiredPaths ) { - let cursor = requiredPath; - while ( cursor !== '/' ) { - candidateDirSet.add(cursor); - cursor = pathPosix.dirname(cursor); - } - } - - const candidatePaths = Array.from(candidateDirSet); - const allEntries = await this.#readEntriesByPathsForUser(userId, candidatePaths); - for ( const path of candidatePaths ) { - const entry = allEntries.get(path); - if ( entry && !entry.isDir ) { - throw new HttpError(409, `Path is not a directory: ${path}`); - } - } - - const missingPaths = candidatePaths - .filter((path) => !allEntries.has(path)) - .sort((pathA, pathB) => this.#pathDepth(pathA) - this.#pathDepth(pathB)); - if ( missingPaths.length > 0 ) { - const uniqueDepths = Array.from(new Set(missingPaths.map((path) => this.#pathDepth(path)))) - .sort((depthA, depthB) => depthA - depthB); - - for ( const depth of uniqueDepths ) { - const pathsAtDepth = missingPaths.filter((path) => this.#pathDepth(path) === depth); - if ( pathsAtDepth.length === 0 ) { - continue; - } - - const now = Math.floor(Date.now() / 1000); - const insertRows: unknown[] = []; - const valuePlaceholders: string[] = []; - const expectedUuidByPath = new Map(); - for ( const dirPath of pathsAtDepth ) { - const parentPath = pathPosix.dirname(dirPath); - const parentEntry = parentPath === '/' - ? null - : allEntries.get(parentPath); - if ( parentPath !== '/' && !parentEntry ) { - throw new Error(`Parent directory not resolved while creating ${dirPath}`); - } - if ( parentEntry && !parentEntry.isDir ) { - throw new HttpError(409, `Path is not a directory: ${parentPath}`); - } - - const expectedUuid = uuidv4(); - expectedUuidByPath.set(dirPath, expectedUuid); - valuePlaceholders.push('(?, ?, ?, ?, ?, ?, 1, ?, ?, ?, 0, 0)'); - insertRows.push( - expectedUuid, - userId, - parentEntry ? parentEntry.id : null, - parentEntry ? parentEntry.uuid : null, - pathPosix.basename(dirPath), - dirPath, - now, - now, - now, - ); - } - - try { - await this.#db.write( - `${this.#insertIgnoreIntoFsentriesSql()} ( - uuid, - user_id, - parent_id, - parent_uid, - name, - path, - is_dir, - created, - modified, - accessed, - immutable, - size - ) VALUES ${valuePlaceholders.join(', ')}`, - insertRows, - ); - } catch { - // Concurrent create may have already inserted some/all rows. - } - - const insertedEntries = await this.#readEntriesByPathsForUser( - userId, - pathsAtDepth, - { useTryHardRead: true }, - ); - for ( const path of pathsAtDepth ) { - let insertedEntry = insertedEntries.get(path); - if ( ! insertedEntry ) { - insertedEntry = await this.#ensureDirectoryPath(path, userId, true); - } - if ( ! insertedEntry.isDir ) { - throw new HttpError(409, `Path is not a directory: ${path}`); - } - if ( expectedUuidByPath.get(path) === insertedEntry.uuid ) { - createdEntryMap.set(path, insertedEntry); - } - allEntries.set(path, insertedEntry); - } - } - } - - for ( const requiredPath of normalizedRequiredPaths ) { - const entry = allEntries.get(requiredPath); - if ( ! entry ) { - throw new Error(`Failed to resolve directory path: ${requiredPath}`); - } - if ( ! entry.isDir ) { - throw new HttpError(409, `Path is not a directory: ${requiredPath}`); - } - requiredEntryMap.set(requiredPath, entry); - } - - return { - requiredEntryMap, - createdEntryMap, - }; - } - - async #getEntryByPathAndUser (path: string, userId: number): Promise { - const normalizedPath = this.#normalizePath(path); - const cacheKey = `prodfsv2:fsentry:path:${userId}:${normalizedPath}`; - const cached = await this.#readEntryFromCache(cacheKey); - if ( cached ) { - return cached; - } - - const rows = await this.#db.read( - 'SELECT * FROM fsentries WHERE path = ? AND user_id = ? LIMIT 1', - [normalizedPath, userId], - ) as FSEntryRow[]; - const row = rows[0]; - if ( ! row ) { - return null; - } - const entry = this.#mapFSEntryRow(row); - await this.#writeEntryToCache(entry); - return entry; - } - - async #ensureDirectoryPath (path: string, userId: number, createPaths: boolean): Promise { - const normalizedPath = this.#normalizePath(path); - - const existingEntry = await this.#getEntryByPathAndUser(normalizedPath, userId); - if ( existingEntry ) { - if ( ! existingEntry.isDir ) { - throw new HttpError(409, `Path is not a directory: ${normalizedPath}`); - } - return existingEntry; - } - - if ( ! createPaths ) { - throw new HttpError(404, `Parent path does not exist: ${normalizedPath}`); - } - - if ( normalizedPath === '/' ) { - throw new HttpError(400, 'Cannot create root directory'); - } - - const parentPath = pathPosix.dirname(normalizedPath); - const parentEntry = parentPath === '/' - ? null - : await this.#ensureDirectoryPath(parentPath, userId, true); - const dirName = pathPosix.basename(normalizedPath); - const now = Math.floor(Date.now() / 1000); - - try { - await this.#db.write( - `${this.#insertIgnoreIntoFsentriesSql()} ( - uuid, - user_id, - parent_id, - parent_uid, - name, - path, - is_dir, - created, - modified, - accessed, - immutable, - size - ) VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, 0, 0)`, - [ - uuidv4(), - userId, - parentEntry ? parentEntry.id : null, - parentEntry ? parentEntry.uuid : null, - dirName, - normalizedPath, - now, - now, - now, - ], - ); - } catch { - // If another request created it first, we'll fetch it below. - } - - const resolvedEntries = await this.#readEntriesByPathsForUser( - userId, - [normalizedPath], - { useTryHardRead: true }, - ); - const resolvedEntry = resolvedEntries.get(normalizedPath) ?? null; - if ( ! resolvedEntry ) { - throw new Error(`Failed to resolve directory path: ${normalizedPath}`); - } - if ( ! resolvedEntry.isDir ) { - throw new HttpError(409, `Path is not a directory: ${normalizedPath}`); - } - - return resolvedEntry; - } - - #serializeMetadata (input: FSEntryCreateInput): string | null { - if ( typeof input.metadata === 'string' ) { - return input.metadata; - } - - const metadataObject: Record = - input.metadata && typeof input.metadata === 'object' - ? { ...input.metadata } - : {}; - - if ( input.contentType ) { - metadataObject.contentType = input.contentType; - } - if ( input.checksumSha256 ) { - metadataObject.checksumSha256 = input.checksumSha256; - } - - if ( Object.keys(metadataObject).length === 0 ) { - return null; - } - - return JSON.stringify(metadataObject); - } - - async getEntryByPath (path: string): Promise { - const normalizedPath = this.#normalizePath(path); - const cacheKey = `prodfsv2:fsentry:path:any:${normalizedPath}`; - const cached = await this.#readEntryFromCache(cacheKey); - if ( cached ) { - return cached; - } - - const rows = await this.#db.read( - 'SELECT * FROM fsentries WHERE path = ? LIMIT 1', - [normalizedPath], - ) as FSEntryRow[]; - const row = rows[0]; - if ( ! row ) { - return null; - } - const entry = this.#mapFSEntryRow(row); - await this.#writeEntryToCache(entry); - return entry; - } - - async getEntriesByPaths (paths: string[]): Promise> { - const normalizedPaths = Array.from(new Set( - paths.map((path) => this.#normalizePath(path)).filter((path) => path.length > 0), - )); - const entriesByPath = new Map(); - if ( normalizedPaths.length === 0 ) { - return entriesByPath; - } - - const missingPaths: string[] = []; - const cacheReads = await Promise.all(normalizedPaths.map(async (path) => { - const cacheKey = `prodfsv2:fsentry:path:any:${path}`; - const cachedEntry = await this.#readEntryFromCache(cacheKey); - return { path, cachedEntry }; - })); - for ( const { path, cachedEntry } of cacheReads ) { - if ( cachedEntry ) { - entriesByPath.set(path, cachedEntry); - } else { - missingPaths.push(path); - } - } - - if ( missingPaths.length > 0 ) { - const chunks = this.#chunk(missingPaths, BULK_QUERY_CHUNK_SIZE); - const chunkResults = await runWithConcurrencyLimit( - chunks, - DEFAULT_DB_CHUNK_CONCURRENCY, - async (chunk) => { - if ( chunk.length === 0 ) { - return []; - } - const placeholders = chunk.map(() => '?').join(', '); - const rows = await this.#db.read( - `SELECT * FROM fsentries WHERE path IN (${placeholders})`, - chunk, - ) as FSEntryRow[]; - const entries = rows.map((row) => this.#mapFSEntryRow(row)); - await Promise.all(entries.map((entry) => this.#writeEntryToCache(entry))); - return entries; - }, - ); - for ( const chunkEntries of chunkResults ) { - for ( const entry of chunkEntries ) { - entriesByPath.set(entry.path, entry); - } - } - } - - return entriesByPath; - } - - async getEntryByUuid (id: string): Promise { - const cacheKey = `prodfsv2:fsentry:uuid:${id}`; - const cached = await this.#readEntryFromCache(cacheKey); - if ( cached ) { - return cached; - } - - const rows = await this.#db.read( - 'SELECT * FROM fsentries WHERE uuid = ? LIMIT 1', - [id], - ) as FSEntryRow[]; - const row = rows[0]; - if ( ! row ) { - return null; - } - const entry = this.#mapFSEntryRow(row); - await this.#writeEntryToCache(entry); - return entry; - } - - async getEntryById (id: number): Promise { - const cacheKey = `prodfsv2:fsentry:id:${id}`; - const cached = await this.#readEntryFromCache(cacheKey); - if ( cached ) { - return cached; - } - - const rows = await this.#db.read( - 'SELECT * FROM fsentries WHERE id = ? LIMIT 1', - [id], - ) as FSEntryRow[]; - const row = rows[0]; - if ( ! row ) { - return null; - } - const entry = this.#mapFSEntryRow(row); - await this.#writeEntryToCache(entry); - return entry; - } - - async updateEntryThumbnailByUuidForUser (userId: number, uuid: string, thumbnail: string | null): Promise { - const now = Math.floor(Date.now() / 1000); - const writeResult = await this.#db.write( - `UPDATE fsentries - SET thumbnail = ?, - modified = ?, - accessed = ? - WHERE uuid = ? AND user_id = ?`, - [thumbnail, now, now, uuid, userId], - ); - if ( typeof writeResult === 'object' && writeResult !== null ) { - const writeResultRecord = writeResult as Record; - const anyRowsAffected = writeResultRecord.anyRowsAffected; - if ( typeof anyRowsAffected === 'boolean' && !anyRowsAffected ) { - throw new HttpError(404, 'File entry was not found for thumbnail update'); - } - - const affectedRowsRaw = writeResultRecord.affectedRows; - const affectedRows = Number(affectedRowsRaw); - if ( - affectedRowsRaw !== undefined - && Number.isFinite(affectedRows) - && affectedRows <= 0 - ) { - throw new HttpError(404, 'File entry was not found for thumbnail update'); - } - } - - const refreshedRows = await this.#db.tryHardRead( - 'SELECT * FROM fsentries WHERE uuid = ? AND user_id = ? LIMIT 1', - [uuid, userId], - ) as FSEntryRow[]; - const refreshedRow = refreshedRows[0]; - if ( ! refreshedRow ) { - throw new HttpError(404, 'File entry was not found for thumbnail update'); - } - - const updatedEntry = this.#mapFSEntryRow(refreshedRow); - await this.#invalidateEntryCache(updatedEntry); - await this.#writeEntryToCache(updatedEntry); - return updatedEntry; - } - - async resolveParentDirectory (userId: number, parentPath: string, createPaths: boolean): Promise { - return this.#ensureDirectoryPath(parentPath, userId, createPaths); - } - - async getEntryByPathForUser ( - path: string, - userId: number, - options: ReadEntriesByPathsOptions = {}, - ): Promise { - if ( !options.useTryHardRead && !options.skipCache ) { - return this.#getEntryByPathAndUser(path, userId); - } - - const normalizedPath = this.#normalizePath(path); - const entriesByPath = await this.#readEntriesByPathsForUser( - userId, - [normalizedPath], - options, - ); - return entriesByPath.get(normalizedPath) ?? null; - } - - async getEntriesByPathsForUser ( - userId: number, - paths: string[], - options: ReadEntriesByPathsOptions = {}, - ): Promise<(FSEntry | null)[]> { - const entriesByPath = await this.#readEntriesByPathsForUser(userId, paths, options); - return paths.map((path) => { - const normalizedPath = this.#normalizePath(path); - return entriesByPath.get(normalizedPath) ?? null; - }); - } - - async resolveParentDirectoriesBatch ( - userId: number, - requests: { parentPath: string; createPaths: boolean }[], - ): Promise { - const { parentEntries } = await this.resolveParentDirectoriesBatchWithCreated(userId, requests); - return parentEntries; - } - - async resolveParentDirectoriesBatchWithCreated ( - userId: number, - requests: { parentPath: string; createPaths: boolean }[], - ): Promise<{ - parentEntries: FSEntry[]; - createdDirectoryEntries: FSEntry[]; - }> { - if ( requests.length === 0 ) { - return { - parentEntries: [], - createdDirectoryEntries: [], - }; - } - - const parentPathsToEnsure = requests - .filter((request) => request.createPaths) - .map((request) => request.parentPath); - const { createdEntryMap } = await this.#ensureDirectoryPathsForUser(userId, parentPathsToEnsure); - - const allParentPaths = requests.map((request) => request.parentPath); - const parentEntriesByPath = await this.#readEntriesByPathsForUser(userId, allParentPaths); - const parentEntries = allParentPaths.map((path) => { - const normalizedPath = this.#normalizePath(path); - const parentEntry = parentEntriesByPath.get(normalizedPath); - if ( ! parentEntry ) { - throw new HttpError(404, `Parent path does not exist: ${normalizedPath}`); - } - if ( ! parentEntry.isDir ) { - throw new HttpError(409, `Path is not a directory: ${normalizedPath}`); - } - return parentEntry; - }); - - return { - parentEntries, - createdDirectoryEntries: Array.from(createdEntryMap.values()), - }; - } - - async ensureDirectoriesForUser ( - userId: number, - requests: { path: string; createPaths: boolean }[], - ): Promise { - const { entries } = await this.ensureDirectoriesForUserWithCreated(userId, requests); - return entries; - } - - async ensureDirectoriesForUserWithCreated ( - userId: number, - requests: { path: string; createPaths: boolean }[], - ): Promise<{ - entries: FSEntry[]; - createdDirectoryEntries: FSEntry[]; - }> { - if ( requests.length === 0 ) { - return { - entries: [], - createdDirectoryEntries: [], - }; - } - - const normalizedRequests = requests.map((request) => { - const normalizedPath = this.#normalizePath(request.path); - if ( normalizedPath === '/' ) { - throw new HttpError(400, 'Cannot create root directory'); - } - return { - path: normalizedPath, - createPaths: request.createPaths, - }; - }); - - const pathsToEnsure = normalizedRequests - .filter((request) => request.createPaths) - .map((request) => request.path); - const { createdEntryMap } = await this.#ensureDirectoryPathsForUser(userId, pathsToEnsure); - - const allPaths = normalizedRequests.map((request) => request.path); - const entriesByPath = await this.#readEntriesByPathsForUser(userId, allPaths); - - const entries = normalizedRequests.map((request) => { - const entry = entriesByPath.get(request.path); - if ( ! entry ) { - throw new HttpError(404, `Directory path does not exist: ${request.path}`); - } - if ( ! entry.isDir ) { - throw new HttpError(409, `Path is not a directory: ${request.path}`); - } - return entry; - }); - - return { - entries, - createdDirectoryEntries: Array.from(createdEntryMap.values()), - }; - } - - async createEntry (fsEntry: FSEntryCreateInput, createPaths = true): Promise { - const [entry] = await this.batchCreateEntries([fsEntry], createPaths); - if ( ! entry ) { - throw new Error('Failed to create entry'); - } - return entry; - } - - async batchCreateEntries (entries: FSEntryCreateInput[], createPaths = true): Promise { - if ( entries.length === 0 ) { - return []; - } - - const normalizedEntries: NormalizedEntryWrite[] = entries.map((entryInput, index) => { - const targetPath = this.#normalizePath(entryInput.path); - if ( targetPath === '/' ) { - throw new HttpError(400, 'Cannot write to root path'); - } - - const parentPath = this.#normalizePath(pathPosix.dirname(targetPath)); - if ( parentPath === '/' ) { - throw new HttpError(400, 'Cannot write directly under root path'); - } - - const size = Number(entryInput.size); - if ( Number.isNaN(size) || size < 0 ) { - throw new HttpError(400, `Invalid size for path ${targetPath}`); - } - - return { - index, - input: entryInput, - userId: entryInput.userId, - targetPath, - parentPath, - fileName: pathPosix.basename(targetPath), - metadataJson: this.#serializeMetadata(entryInput), - bucket: entryInput.bucket ?? null, - bucketRegion: entryInput.bucketRegion ?? null, - size, - createPaths: entryInput.createMissingParents ?? createPaths, - }; - }); - - const duplicatePathSet = new Set(); - for ( const normalizedEntry of normalizedEntries ) { - const dedupeKey = `${normalizedEntry.userId}:${normalizedEntry.targetPath}`; - if ( duplicatePathSet.has(dedupeKey) ) { - throw new HttpError(409, `Batch contains duplicate target path: ${normalizedEntry.targetPath}`); - } - duplicatePathSet.add(dedupeKey); - } - - const entriesByUser = new Map(); - for ( const normalizedEntry of normalizedEntries ) { - const userEntries = entriesByUser.get(normalizedEntry.userId) ?? []; - userEntries.push(normalizedEntry); - entriesByUser.set(normalizedEntry.userId, userEntries); - } - - const resultsByIndex = new Map(); - for ( const [userId, userEntries] of entriesByUser ) { - const parentEntries = await this.resolveParentDirectoriesBatch( - userId, - userEntries.map((entry) => ({ - parentPath: entry.parentPath, - createPaths: entry.createPaths, - })), - ); - const parentByPath = new Map(); - for ( const parentEntry of parentEntries ) { - parentByPath.set(parentEntry.path, parentEntry); - } - - const existingEntriesByPath = await this.#readEntriesByPathsForUser( - userId, - userEntries.map((entry) => entry.targetPath), - { - useTryHardRead: true, - skipCache: true, - }, - ); - - const now = Math.floor(Date.now() / 1000); - const updateOperations: Array<{ - existingEntry: FSEntry; - updatedEntry: FSEntry; - promise: Promise; - }> = []; - const updatedResultsByIndex = new Map(); - const insertCandidates: NormalizedEntryWrite[] = []; - - for ( const entry of userEntries ) { - const parentEntry = parentByPath.get(entry.parentPath); - if ( ! parentEntry ) { - throw new Error(`Failed to resolve parent directory for ${entry.targetPath}`); - } - - const existingEntry = existingEntriesByPath.get(entry.targetPath); - if ( existingEntry ) { - if ( ! entry.input.overwrite ) { - throw new HttpError(409, `Entry already exists at ${entry.targetPath}`); - } - if ( existingEntry.isDir ) { - throw new HttpError(409, `Cannot overwrite a directory at ${entry.targetPath}`); - } - - const updatedEntry = { - ...existingEntry, - bucket: entry.bucket, - bucketRegion: entry.bucketRegion, - parentId: parentEntry.id, - parentUid: parentEntry.uuid, - associatedAppId: entry.input.associatedAppId ?? null, - isPublic: entry.input.isPublic === undefined ? null : Boolean(entry.input.isPublic), - thumbnail: entry.input.thumbnail ?? null, - immutable: Boolean(entry.input.immutable), - name: entry.fileName, - path: entry.targetPath, - metadata: entry.metadataJson, - modified: now, - accessed: now, - size: entry.size, - }; - updateOperations.push({ - existingEntry, - updatedEntry, - promise: this.#db.write( - `UPDATE fsentries - SET bucket = ?, - bucket_region = ?, - parent_id = ?, - parent_uid = ?, - associated_app_id = ?, - is_public = ?, - thumbnail = ?, - immutable = ?, - name = ?, - path = ?, - metadata = ?, - modified = ?, - accessed = ?, - size = ? - WHERE id = ?`, - [ - entry.bucket, - entry.bucketRegion, - parentEntry.id, - parentEntry.uuid, - entry.input.associatedAppId ?? null, - entry.input.isPublic === undefined ? null : (entry.input.isPublic ? 1 : 0), - entry.input.thumbnail ?? null, - entry.input.immutable ? 1 : 0, - entry.fileName, - entry.targetPath, - entry.metadataJson, - now, - now, - entry.size, - existingEntry.id, - ], - ), - }); - updatedResultsByIndex.set(entry.index, updatedEntry); - continue; - } - - insertCandidates.push(entry); - } - - if ( updateOperations.length > 0 ) { - const updateResults = await Promise.allSettled(updateOperations.map((operation) => operation.promise)); - const successfulUpdateOperations = updateResults.flatMap((result, index) => { - if ( result.status !== 'fulfilled' ) { - return []; - } - const operation = updateOperations[index]; - return operation ? [operation] : []; - }); - if ( successfulUpdateOperations.length > 0 ) { - await Promise.all(successfulUpdateOperations.map((operation) => { - return this.#invalidateEntryCache(operation.existingEntry); - })); - await Promise.all(successfulUpdateOperations.map((operation) => { - return this.#writeEntryToCache(operation.updatedEntry); - })); - } - - const failedUpdate = updateResults.find((result) => result.status === 'rejected'); - if ( failedUpdate?.status === 'rejected' ) { - throw (failedUpdate.reason instanceof Error - ? failedUpdate.reason - : new Error('Failed to update fsentries batch')); - } - } - - const insertChunks = this.#chunk(insertCandidates, BULK_QUERY_CHUNK_SIZE); - await runWithConcurrencyLimit( - insertChunks, - DEFAULT_DB_CHUNK_CONCURRENCY, - async (insertChunk) => { - if ( insertChunk.length === 0 ) { - return; - } - - const valuePlaceholders: string[] = []; - const values: unknown[] = []; - for ( const entry of insertChunk ) { - const parentEntry = parentByPath.get(entry.parentPath); - if ( ! parentEntry ) { - throw new Error(`Failed to resolve parent directory for ${entry.targetPath}`); - } - - valuePlaceholders.push('(?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'); - values.push( - entry.input.uuid, - entry.bucket, - entry.bucketRegion, - userId, - parentEntry.id, - parentEntry.uuid, - entry.input.associatedAppId ?? null, - entry.input.isPublic === undefined ? null : (entry.input.isPublic ? 1 : 0), - entry.input.thumbnail ?? null, - entry.input.immutable ? 1 : 0, - entry.fileName, - entry.targetPath, - entry.metadataJson, - now, - now, - now, - entry.size, - ); - } - - await this.#db.write( - `INSERT INTO fsentries ( - uuid, - bucket, - bucket_region, - user_id, - parent_id, - parent_uid, - associated_app_id, - is_dir, - is_public, - thumbnail, - immutable, - name, - path, - metadata, - modified, - created, - accessed, - size - ) VALUES ${valuePlaceholders.join(', ')}`, - values, - ); - }, - ); - - const insertedEntriesByUuid = new Map(); - if ( insertCandidates.length > 0 ) { - const insertUuidChunks = this.#chunk( - insertCandidates.map((entry) => entry.input.uuid), - BULK_QUERY_CHUNK_SIZE, - ); - - const insertedChunkResults = await runWithConcurrencyLimit( - insertUuidChunks, - DEFAULT_DB_CHUNK_CONCURRENCY, - async (insertUuidChunk) => { - if ( insertUuidChunk.length === 0 ) { - return []; - } - - const placeholders = insertUuidChunk.map(() => '?').join(', '); - const rows = await this.#db.tryHardRead( - `SELECT * FROM fsentries WHERE user_id = ? AND uuid IN (${placeholders})`, - [userId, ...insertUuidChunk], - ) as FSEntryRow[]; - - const insertedEntries = rows.map((row) => this.#mapFSEntryRow(row)); - if ( insertedEntries.length > 0 ) { - await Promise.all(insertedEntries.map((entry) => this.#writeEntryToCache(entry))); - } - return insertedEntries; - }, - ); - for ( const insertedEntries of insertedChunkResults ) { - for ( const insertedEntry of insertedEntries ) { - insertedEntriesByUuid.set(insertedEntry.uuid, insertedEntry); - } - } - } - - for ( const entry of userEntries ) { - const updatedResult = updatedResultsByIndex.get(entry.index); - if ( updatedResult ) { - resultsByIndex.set(entry.index, updatedResult); - continue; - } - - const insertedResult = insertedEntriesByUuid.get(entry.input.uuid); - if ( insertedResult ) { - resultsByIndex.set(entry.index, insertedResult); - continue; - } - - throw new Error(`Failed to load final entry for ${entry.targetPath}`); - } - } - - const createdEntries: FSEntry[] = []; - for ( let index = 0; index < entries.length; index++ ) { - const entry = resultsByIndex.get(index); - if ( ! entry ) { - throw new Error(`Failed to resolve entry result at index ${index}`); - } - createdEntries.push(entry); - } - return createdEntries; - } - - async createPendingEntry (entry: PendingUploadCreateInput): Promise { - const [createdEntry] = await this.batchCreatePendingEntries([entry]); - if ( ! createdEntry ) { - throw new Error('Failed to create pending upload entry'); - } - return createdEntry; - } - - async batchCreatePendingEntries (entries: PendingUploadCreateInput[]): Promise { - if ( entries.length === 0 ) { - return []; - } - const now = Date.now(); - const pendingSessions = entries.map((entry) => toPendingUploadSession(entry, now)); - await this.#writePendingUploadSessions(pendingSessions, 'create pending upload sessions'); - return pendingSessions; - } - - async getPendingEntryBySessionId (sessionId: string): Promise { - const value = await this.#kvStore.get({ - key: toPendingUploadSessionKey(sessionId), - }); - return normalizePendingUploadSession(value, sessionId); - } - - async getPendingEntriesBySessionIds (sessionIds: string[]): Promise<(PendingUploadSession | null)[]> { - if ( sessionIds.length === 0 ) { - return []; - } - - const entriesBySessionId = await this.#getPendingUploadSessionsBySessionIds(sessionIds); - return sessionIds.map((sessionId) => entriesBySessionId.get(sessionId) ?? null); - } - - async markPendingEntryCompleted (sessionId: string): Promise { - await this.#markPendingSessionsWithStatus([sessionId], 'completed', null); - } - - async markPendingEntryFailed (sessionId: string, reason: string): Promise { - await this.#markPendingSessionsWithStatus([sessionId], 'failed', reason); - } - - async markPendingEntriesFailed (sessionIds: string[], reason: string): Promise { - await this.#markPendingSessionsWithStatus(sessionIds, 'failed', reason); - } - - async abortPendingEntry (sessionId: string, reason: string): Promise { - await this.#markPendingSessionsWithStatus([sessionId], 'aborted', reason); - } - - async completePendingEntry (sessionId: string, finalData: FSEntryCreateInput): Promise { - const [completedEntry] = await this.batchCompletePendingEntries([{ sessionId, finalData }]); - if ( ! completedEntry ) { - throw new Error('Failed to complete pending entry'); - } - return completedEntry; - } - - async batchCompletePendingEntries ( - entries: { sessionId: string; finalData: FSEntryCreateInput }[], - ): Promise { - if ( entries.length === 0 ) { - return []; - } - - const completedEntries = await this.batchCreateEntries( - entries.map((entry) => entry.finalData), - true, - ); - - await this.#markPendingSessionsWithStatus( - entries.map((entry) => entry.sessionId), - 'completed', - null, - ); - - return completedEntries; - } - - async getUserStorageAllowance (userId: number): Promise<{ curr: number; max: number }> { - const [usageRows, userRows] = await Promise.all([ - this.#db.read( - 'SELECT COALESCE(SUM(size), 0) AS totalUsage FROM fsentries WHERE user_id = ?', - [userId], - ) as Promise<{ totalUsage: number }[]>, - this.#db.read( - 'SELECT free_storage AS freeStorage FROM user WHERE id = ? LIMIT 1', - [userId], - ) as Promise<{ freeStorage: number | null }[]>, - ]); - const usageRow = usageRows[0]; - const userRow = userRows[0]; - - const curr = Number(usageRow?.totalUsage ?? 0); - let max = Number(userRow?.freeStorage ?? global_config.storage_capacity ?? 0); - - if ( ! global_config.is_storage_limited ) { - const availableDeviceStorage = Number(global_config.available_device_storage ?? 0); - max = availableDeviceStorage > 0 ? availableDeviceStorage : Number.MAX_SAFE_INTEGER; - } - - return { curr, max }; - } -} diff --git a/extensions/fsv2/src/repositories/S3FileStorageRepository.ts b/extensions/fsv2/src/repositories/S3FileStorageRepository.ts deleted file mode 100644 index 453874aa3..000000000 --- a/extensions/fsv2/src/repositories/S3FileStorageRepository.ts +++ /dev/null @@ -1,383 +0,0 @@ -import { - AbortMultipartUploadCommand, - CompleteMultipartUploadCommand, - CreateMultipartUploadCommand, - DeleteObjectCommand, - PutObjectCommand, - type S3Client, - UploadPartCommand, -} from '@aws-sdk/client-s3'; -import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; -import type { s3ClientProvider } from '@heyputer/backend/src/clients/s3/s3ClientProvider.js'; -import { Readable } from 'node:stream'; -import type { - MultipartCompleteInput, - ServerUploadInput, - SignedMultipartPartUrlsInput, - SignedUploadInput, - SignedUploadPart, - SignedUploadResult, -} from './s3Types.js'; - -export class S3StorageProvider { - - #s3ClientProvider: typeof s3ClientProvider; - - constructor (s3Provider: typeof s3ClientProvider) { - this.#s3ClientProvider = s3Provider; - } - - #getClientForRegion (region: string): S3Client { - return this.#s3ClientProvider.get(region); - } - - getMaxSingleUploadSize (): number { - return this.#s3ClientProvider.maxSingleUploadSize; - } - - getMultipartPartSize (): number { - return this.#s3ClientProvider.partSize; - } - - #resolveMultipartPartSize (requestedPartSize?: number): number { - return Math.max( - this.getMaxSingleUploadSize(), - requestedPartSize ?? this.getMultipartPartSize(), - ); - } - - async createSignedUploadUrl (fileMetadata: SignedUploadInput, region: string): Promise { - const [result] = await this.batchCreateSignedUploadUrls([fileMetadata], region); - if ( ! result ) { - throw new Error('Failed to create signed upload url'); - } - return result; - } - - async batchCreateSignedUploadUrls (filesMetadata: SignedUploadInput[], region: string): Promise { - const client = this.#getClientForRegion(region); - const now = Date.now(); - const settledResults = await Promise.allSettled(filesMetadata.map(async (fileMetadata) => { - const expiresInSeconds = Math.max(60, Math.min(60 * 60, fileMetadata.expiresInSeconds)); - const expiresAt = now + expiresInSeconds * 1000; - const maxSingleUploadSize = this.getMaxSingleUploadSize(); - const shouldUseSingleUpload = fileMetadata.uploadMode === 'single' - && fileMetadata.size <= maxSingleUploadSize; - - if ( shouldUseSingleUpload ) { - const command = new PutObjectCommand({ - Bucket: fileMetadata.bucket, - Key: fileMetadata.objectKey, - ContentType: fileMetadata.contentType, - }); - const url = await getSignedUrl(client, command, { expiresIn: expiresInSeconds }); - return { - uploadMode: 'single' as const, - expiresAt, - url, - }; - } - - const multipartPartSize = this.#resolveMultipartPartSize(fileMetadata.multipartPartSize); - const multipartPartCount = Math.max(1, Math.ceil(fileMetadata.size / multipartPartSize)); - let multipartUploadId: string | undefined; - - try { - const multipartResult = await client.send(new CreateMultipartUploadCommand({ - Bucket: fileMetadata.bucket, - Key: fileMetadata.objectKey, - ContentType: fileMetadata.contentType, - })); - - if ( ! multipartResult.UploadId ) { - throw new Error('Failed to initialize multipart upload'); - } - - multipartUploadId = multipartResult.UploadId; - const partUrls = await this.createSignedMultipartPartUrls({ - bucket: fileMetadata.bucket, - objectKey: fileMetadata.objectKey, - multipartUploadId, - partNumbers: Array.from({ length: multipartPartCount }, (_, index) => index + 1), - expiresInSeconds, - }, region); - - return { - uploadMode: 'multipart' as const, - expiresAt, - multipartUploadId, - multipartPartSize, - multipartPartCount, - multipartPartUrls: partUrls, - }; - } catch ( error ) { - if ( multipartUploadId ) { - try { - await this.abortMutipartUpload( - multipartUploadId, - region, - fileMetadata.bucket, - fileMetadata.objectKey, - ); - } catch { - // Best effort cleanup for partially initialized multipart uploads. - } - } - throw error; - } - })); - - const failedResults = settledResults.filter((result) => result.status === 'rejected'); - if ( failedResults.length > 0 ) { - await Promise.allSettled(settledResults.map((result, index) => { - if ( result.status !== 'fulfilled' ) { - return Promise.resolve(); - } - if ( result.value.uploadMode !== 'multipart' || !result.value.multipartUploadId ) { - return Promise.resolve(); - } - const fileMetadata = filesMetadata[index]; - if ( ! fileMetadata ) { - return Promise.resolve(); - } - return this.abortMutipartUpload( - result.value.multipartUploadId, - region, - fileMetadata.bucket, - fileMetadata.objectKey, - ); - })); - - const firstFailure = failedResults[0]?.reason; - if ( firstFailure instanceof Error ) { - throw firstFailure; - } - throw new Error('Failed to create signed upload urls'); - } - - return settledResults.map((result) => { - if ( result.status !== 'fulfilled' ) { - throw new Error('Failed to create signed upload urls'); - } - return result.value; - }); - } - - async createSignedMultipartPartUrls ( - input: SignedMultipartPartUrlsInput, - region: string, - ): Promise { - const client = this.#getClientForRegion(region); - const expiresInSeconds = Math.max(60, Math.min(60 * 60, input.expiresInSeconds)); - - return Promise.all(input.partNumbers.map(async (partNumber) => { - const command = new UploadPartCommand({ - Bucket: input.bucket, - Key: input.objectKey, - UploadId: input.multipartUploadId, - PartNumber: partNumber, - }); - const url = await getSignedUrl(client, command, { - expiresIn: expiresInSeconds, - }); - return { - partNumber, - url, - }; - })); - } - - async completeMultipartUpload (input: MultipartCompleteInput, region: string): Promise { - const client = this.#getClientForRegion(region); - await client.send(new CompleteMultipartUploadCommand({ - Bucket: input.bucket, - Key: input.objectKey, - UploadId: input.multipartUploadId, - MultipartUpload: { - Parts: [...input.parts] - .sort((partA, partB) => partA.partNumber - partB.partNumber) - .map((part) => ({ - PartNumber: part.partNumber, - ETag: part.etag, - })), - }, - })); - } - - async abortMutipartUpload (uploadId: string, region: string, bucket: string, objectKey: string): Promise { - const client = this.#getClientForRegion(region); - await client.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: objectKey, - UploadId: uploadId, - })); - } - - async uploadFromServer (input: ServerUploadInput, region: string): Promise { - const client = this.#getClientForRegion(region); - const maxSingleUploadSize = this.getMaxSingleUploadSize(); - const resolvedContentLength = this.#resolveContentLength(input); - const shouldUseMultipart = input.body instanceof Readable - ? resolvedContentLength === undefined || resolvedContentLength > maxSingleUploadSize - : resolvedContentLength !== undefined && resolvedContentLength > maxSingleUploadSize; - - if ( ! shouldUseMultipart ) { - await client.send(new PutObjectCommand({ - Bucket: input.bucket, - Key: input.objectKey, - ContentType: input.contentType, - Body: input.body, - ...(input.contentLength !== undefined ? { ContentLength: input.contentLength } : {}), - })); - return; - } - - await this.#uploadFromServerMultipart(input, region, this.#resolveMultipartPartSize()); - } - - async deleteObject (bucket: string, objectKey: string, region: string): Promise { - const client = this.#getClientForRegion(region); - await client.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: objectKey, - })); - } - - #resolveContentLength (input: ServerUploadInput): number | undefined { - if ( Number.isFinite(input.contentLength) && Number(input.contentLength) >= 0 ) { - return Number(input.contentLength); - } - if ( Number.isFinite(input.sizeHint) && Number(input.sizeHint) >= 0 ) { - return Number(input.sizeHint); - } - if ( Buffer.isBuffer(input.body) || input.body instanceof Uint8Array ) { - return input.body.byteLength; - } - if ( typeof input.body === 'string' ) { - return Buffer.byteLength(input.body); - } - return undefined; - } - - #toBuffer (chunk: unknown): Buffer { - if ( Buffer.isBuffer(chunk) ) { - return chunk; - } - if ( chunk instanceof Uint8Array ) { - return Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength); - } - if ( typeof chunk === 'string' ) { - return Buffer.from(chunk); - } - throw new Error('Unsupported chunk type for multipart upload'); - } - - async #uploadFromServerMultipart ( - input: ServerUploadInput, - region: string, - partSize: number, - ): Promise { - const client = this.#getClientForRegion(region); - const createResult = await client.send(new CreateMultipartUploadCommand({ - Bucket: input.bucket, - Key: input.objectKey, - ContentType: input.contentType, - })); - - const uploadId = createResult.UploadId; - if ( ! uploadId ) { - throw new Error('Failed to initialize multipart upload'); - } - - const completedParts: Array<{ ETag: string; PartNumber: number; }> = []; - let partNumber = 1; - - const uploadPart = async (partBody: Buffer) => { - const uploadPartResult = await client.send(new UploadPartCommand({ - Bucket: input.bucket, - Key: input.objectKey, - UploadId: uploadId, - PartNumber: partNumber, - Body: partBody, - ContentLength: partBody.byteLength, - })); - if ( ! uploadPartResult.ETag ) { - throw new Error(`Multipart upload returned no ETag for part ${partNumber}`); - } - completedParts.push({ - ETag: uploadPartResult.ETag, - PartNumber: partNumber, - }); - partNumber++; - }; - - try { - if ( Buffer.isBuffer(input.body) || input.body instanceof Uint8Array ) { - const bufferBody = this.#toBuffer(input.body); - for ( let offset = 0; offset < bufferBody.byteLength; offset += partSize ) { - const partBody = bufferBody.subarray(offset, offset + partSize); - await uploadPart(partBody); - } - } else if ( typeof input.body === 'string' ) { - const bufferBody = Buffer.from(input.body); - for ( let offset = 0; offset < bufferBody.byteLength; offset += partSize ) { - const partBody = bufferBody.subarray(offset, offset + partSize); - await uploadPart(partBody); - } - } else if ( input.body instanceof Readable ) { - let pendingChunk: Buffer = Buffer.alloc(0); - for await ( const chunk of input.body ) { - const chunkBuffer = this.#toBuffer(chunk); - if ( chunkBuffer.byteLength === 0 ) { - continue; - } - pendingChunk = pendingChunk.byteLength === 0 - ? chunkBuffer - : Buffer.concat([pendingChunk, chunkBuffer]); - while ( pendingChunk.byteLength >= partSize ) { - const partBody = pendingChunk.subarray(0, partSize); - await uploadPart(partBody); - pendingChunk = pendingChunk.subarray(partSize); - } - } - if ( pendingChunk.byteLength > 0 ) { - await uploadPart(pendingChunk); - } - } else { - throw new Error('Unsupported body type for multipart upload'); - } - - if ( completedParts.length === 0 ) { - await client.send(new AbortMultipartUploadCommand({ - Bucket: input.bucket, - Key: input.objectKey, - UploadId: uploadId, - })); - await client.send(new PutObjectCommand({ - Bucket: input.bucket, - Key: input.objectKey, - ContentType: input.contentType, - Body: Buffer.alloc(0), - ContentLength: 0, - })); - return; - } - - await client.send(new CompleteMultipartUploadCommand({ - Bucket: input.bucket, - Key: input.objectKey, - UploadId: uploadId, - MultipartUpload: { - Parts: completedParts, - }, - })); - } catch ( error ) { - await client.send(new AbortMultipartUploadCommand({ - Bucket: input.bucket, - Key: input.objectKey, - UploadId: uploadId, - })).catch(() => undefined); - throw error; - } - } -} diff --git a/extensions/fsv2/src/services/FSEntryService.ts b/extensions/fsv2/src/services/FSEntryService.ts deleted file mode 100644 index 447e736fe..000000000 --- a/extensions/fsv2/src/services/FSEntryService.ts +++ /dev/null @@ -1,1779 +0,0 @@ -import { posix as pathPosix } from 'node:path'; -import { createHash } from 'node:crypto'; -import { Readable, Transform } from 'node:stream'; -import type { TransformCallback } from 'node:stream'; -import { v4 as uuidv4 } from 'uuid'; -import { FSEntryRepository } from '../repositories/FSEntryRepository.js'; -import { S3StorageProvider } from '../repositories/S3FileStorageRepository.js'; -import type { - MultipartCompletePart, - SignedUploadResult, -} from '../repositories/s3Types.js'; -import { - FSEntry, - FSEntryCreateInput, - FSEntryWriteInput, - PendingUploadCreateInput, - PendingUploadSession, -} from '../types/FSEntry.js'; -import { - BinaryPayload, - CompleteWriteRequest, - CompleteWriteResponse, - SignMultipartPartsRequest, - SignMultipartPartsResponse, - SignedWriteRequest, - SignedWriteResponse, - UploadMode, - WriteRequest, - WriteResponse, -} from '../types/requests.js'; -import type { - BatchWritePrepareRequest, - NormalizedWriteInput, - PreparedBatchWrite, - UploadedBatchWriteItem, - UploadPayload, - UploadPreparedBatchItemInput, - UploadProgressTrackerLike, -} from './types.js'; -import { runWithConcurrencyLimitSettled } from '../utils/concurrency.js'; - -const { HttpError } = extension.import('extensionController'); - -const DEFAULT_CONTENT_TYPE = 'application/octet-stream'; -const DEFAULT_SIGNED_UPLOAD_EXPIRY_SECONDS = 60 * 15; - -interface WriteTargetResolutionInput { - index: number; - normalizedInput: NormalizedWriteInput; -} - -interface WriteTargetResolutionResult { - index: number; - normalizedInput: NormalizedWriteInput; - existingEntry: FSEntry | null; - wasOverwrite: boolean; -} - -interface SignedMultipartCleanupTarget { - bucket: string; - bucketRegion: string; - objectKey: string; - signedUploadResult: SignedUploadResult; -} - -interface StartSignedWriteResult { - response: SignedWriteResponse; - createdDirectoryEntries: FSEntry[]; -} - -interface BatchStartSignedWriteResult { - responses: SignedWriteResponse[]; - createdDirectoryEntries: FSEntry[]; -} - -export class FSEntryService { - #fsEntryRepository: FSEntryRepository; - #s3StorageProvider: S3StorageProvider; - - constructor ( - fsEntryRepository: FSEntryRepository, - s3StorageProvider: S3StorageProvider, - ) { - this.#fsEntryRepository = fsEntryRepository; - this.#s3StorageProvider = s3StorageProvider; - } - - #normalizePath (path: string): string { - const trimmedPath = path.trim(); - if ( trimmedPath.length === 0 ) { - throw new HttpError(400, 'Path cannot be empty'); - } - if ( trimmedPath === '~' || trimmedPath.startsWith('~/') ) { - throw new HttpError(400, 'Home path must be resolved before write'); - } - - let normalizedPath = pathPosix.normalize(trimmedPath); - if ( ! normalizedPath.startsWith('/') ) { - normalizedPath = `/${normalizedPath}`; - } - if ( normalizedPath.length > 1 && normalizedPath.endsWith('/') ) { - normalizedPath = normalizedPath.slice(0, -1); - } - return normalizedPath; - } - - #resolveBucket (metadata: FSEntryWriteInput): string { - const bucket = metadata.bucket ?? global_config.s3_bucket ?? 'puter-local'; - if ( typeof bucket !== 'string' || bucket.length === 0 ) { - throw new HttpError(500, 'Missing S3 bucket configuration'); - } - return bucket; - } - - #resolveBucketRegion (metadata: FSEntryWriteInput): string { - const bucketRegion = metadata.bucketRegion - ?? global_config.s3_region - ?? global_config.region - ?? 'us-west-2'; - - if ( typeof bucketRegion !== 'string' || bucketRegion.length === 0 ) { - throw new HttpError(500, 'Missing S3 region configuration'); - } - - return bucketRegion; - } - - #normalizeWriteInput (userId: number, metadata: FSEntryWriteInput): NormalizedWriteInput { - const normalizedPath = this.#normalizePath(metadata.path); - if ( normalizedPath === '/' ) { - throw new HttpError(400, 'Cannot write to root path'); - } - - const size = Number(metadata.size); - if ( Number.isNaN(size) || size < 0 ) { - throw new HttpError(400, 'Invalid file size'); - } - - const metadataRecord = metadata as unknown as Record; - const dedupeName = Boolean( - metadata.dedupeName - ?? metadataRecord.dedupe_name, - ); - - return { - userId, - path: normalizedPath, - size, - contentType: metadata.contentType ?? DEFAULT_CONTENT_TYPE, - checksumSha256: metadata.checksumSha256, - metadata: metadata.metadata, - thumbnail: metadata.thumbnail, - associatedAppId: metadata.associatedAppId, - overwrite: Boolean(metadata.overwrite), - dedupeName, - createMissingParents: Boolean(metadata.createMissingParents), - immutable: Boolean(metadata.immutable), - isPublic: metadata.isPublic, - multipartPartSize: metadata.multipartPartSize, - bucket: this.#resolveBucket(metadata), - bucketRegion: this.#resolveBucketRegion(metadata), - }; - } - - async #findDedupedPath ( - targetPath: string, - reservedPaths: Set, - loadExistingEntry: (path: string) => Promise, - ): Promise { - const parentPath = pathPosix.dirname(targetPath); - const extension = pathPosix.extname(targetPath); - const fileName = pathPosix.basename(targetPath, extension); - - for ( let suffix = 1; suffix < 100_000; suffix++ ) { - const dedupedPath = pathPosix.join(parentPath, `${fileName} (${suffix})${extension}`); - if ( reservedPaths.has(dedupedPath) ) { - continue; - } - const existingEntry = await loadExistingEntry(dedupedPath); - if ( ! existingEntry ) { - return dedupedPath; - } - } - - throw new HttpError(409, 'Unable to resolve deduped file path'); - } - - async #resolveWriteTargets ( - userId: number, - inputs: WriteTargetResolutionInput[], - ): Promise { - const reservedPaths = new Set(); - const existingEntryCache = new Map>(); - const initialPaths = Array.from(new Set(inputs.map((input) => input.normalizedInput.path))); - const initialEntries = await this.#fsEntryRepository.getEntriesByPathsForUser( - userId, - initialPaths, - { - useTryHardRead: true, - skipCache: true, - }, - ); - for ( let index = 0; index < initialPaths.length; index++ ) { - const path = initialPaths[index]; - if ( ! path ) { - continue; - } - existingEntryCache.set(path, Promise.resolve(initialEntries[index] ?? null)); - } - - const loadExistingEntry = async (path: string): Promise => { - const cachedPromise = existingEntryCache.get(path); - if ( cachedPromise ) { - return await cachedPromise; - } - - const readPromise = this.#fsEntryRepository.getEntryByPathForUser(path, userId, { - useTryHardRead: true, - skipCache: true, - }); - existingEntryCache.set(path, readPromise); - return await readPromise; - }; - - const results: WriteTargetResolutionResult[] = []; - for ( const input of inputs ) { - let normalizedInput = input.normalizedInput; - let existingEntry = await loadExistingEntry(normalizedInput.path); - const pathReservedInBatch = reservedPaths.has(normalizedInput.path); - - if ( pathReservedInBatch || existingEntry ) { - if ( normalizedInput.dedupeName ) { - const dedupedPath = await this.#findDedupedPath( - normalizedInput.path, - reservedPaths, - loadExistingEntry, - ); - normalizedInput = { - ...normalizedInput, - path: dedupedPath, - }; - existingEntry = await loadExistingEntry(dedupedPath); - } else if ( pathReservedInBatch ) { - throw new HttpError(409, `Batch contains duplicate target path: ${normalizedInput.path}`); - } - } - - if ( existingEntry && existingEntry.isDir ) { - throw new HttpError(409, 'Cannot overwrite an existing directory'); - } - if ( existingEntry && !normalizedInput.overwrite ) { - throw new HttpError(409, 'A file already exists at this path and overwrite was not requested'); - } - - reservedPaths.add(normalizedInput.path); - results.push({ - index: input.index, - normalizedInput, - existingEntry, - wasOverwrite: Boolean(existingEntry), - }); - } - - return results; - } - - #toCreateInput (normalizedInput: NormalizedWriteInput, objectKey: string): FSEntryCreateInput { - return { - userId: normalizedInput.userId, - uuid: objectKey, - path: normalizedInput.path, - size: normalizedInput.size, - contentType: normalizedInput.contentType, - checksumSha256: normalizedInput.checksumSha256, - metadata: normalizedInput.metadata, - thumbnail: normalizedInput.thumbnail, - associatedAppId: normalizedInput.associatedAppId, - overwrite: normalizedInput.overwrite, - createMissingParents: normalizedInput.createMissingParents, - immutable: normalizedInput.immutable, - isPublic: normalizedInput.isPublic, - multipartPartSize: normalizedInput.multipartPartSize, - bucket: normalizedInput.bucket, - bucketRegion: normalizedInput.bucketRegion, - }; - } - - #determineUploadMode (requestUploadMode: UploadMode | 'auto' | undefined, size: number): UploadMode { - const maxSingleUploadSize = this.#s3StorageProvider.getMaxSingleUploadSize(); - if ( requestUploadMode === 'multipart' ) { - return 'multipart'; - } - if ( requestUploadMode === 'single' ) { - return size > maxSingleUploadSize ? 'multipart' : 'single'; - } - return size > maxSingleUploadSize ? 'multipart' : 'single'; - } - - #resolveStorageMax ( - allowanceMax: number, - storageAllowanceMaxOverride?: number, - ): number { - if ( allowanceMax === Number.MAX_SAFE_INTEGER ) { - return allowanceMax; - } - if ( storageAllowanceMaxOverride === undefined ) { - return allowanceMax; - } - if ( !Number.isFinite(storageAllowanceMaxOverride) || storageAllowanceMaxOverride < 0 ) { - return allowanceMax; - } - return Math.max(allowanceMax, storageAllowanceMaxOverride); - } - - async #assertStorageAllowance ( - userId: number, - incomingSize: number, - existingSize = 0, - storageAllowanceMaxOverride?: number, - ): Promise { - const allowance = await this.#fsEntryRepository.getUserStorageAllowance(userId); - const maxStorage = this.#resolveStorageMax(allowance.max, storageAllowanceMaxOverride); - if ( maxStorage === Number.MAX_SAFE_INTEGER ) { - return; - } - - const projectedUsage = allowance.curr - existingSize + incomingSize; - if ( projectedUsage > maxStorage ) { - throw new HttpError(413, 'Storage limit reached'); - } - } - - async #assertStorageAllowanceForBatch ( - userId: number, - sizeChanges: Array<{ incomingSize: number; existingSize: number }>, - storageAllowanceMaxOverride?: number, - ): Promise { - if ( sizeChanges.length === 0 ) { - return; - } - - const allowance = await this.#fsEntryRepository.getUserStorageAllowance(userId); - const maxStorage = this.#resolveStorageMax(allowance.max, storageAllowanceMaxOverride); - if ( maxStorage === Number.MAX_SAFE_INTEGER ) { - return; - } - - let projectedUsage = allowance.curr; - for ( const sizeChange of sizeChanges ) { - projectedUsage = projectedUsage - sizeChange.existingSize + sizeChange.incomingSize; - } - - if ( projectedUsage > maxStorage ) { - throw new HttpError(413, 'Storage limit reached'); - } - } - - #toErrorMessage (error: unknown): string { - if ( error instanceof Error ) { - return error.message; - } - return 'Unknown error'; - } - - #toError (error: unknown, fallbackMessage: string): Error { - if ( error instanceof Error ) { - return error; - } - return new Error(fallbackMessage); - } - - #toMultipartParts (parts: CompleteWriteRequest['parts']): MultipartCompletePart[] { - if ( !parts || parts.length === 0 ) { - return []; - } - return parts.map((part) => ({ - partNumber: Number(part.partNumber), - etag: part.etag, - })); - } - - #parseSessionMetadata (session: PendingUploadSession): FSEntryCreateInput { - if ( ! session.metadataJson ) { - throw new HttpError(500, 'Upload session metadata is missing'); - } - - const parsedMetadata = JSON.parse(session.metadataJson) as FSEntryCreateInput; - return { - ...parsedMetadata, - userId: session.userId, - uuid: session.objectKey, - path: session.targetPath, - size: session.size, - contentType: session.contentType, - checksumSha256: session.checksumSha256 ?? undefined, - bucket: session.bucket ?? undefined, - bucketRegion: session.bucketRegion ?? undefined, - overwrite: Boolean(session.overwriteTargetUid), - }; - } - - #isBinaryPayload (value: unknown): value is BinaryPayload { - return Boolean( - value - && typeof value === 'object' - && 'base64' in value - && typeof (value as BinaryPayload).base64 === 'string', - ); - } - - #isNodeStream (value: unknown): value is Readable { - return Boolean(value && typeof value === 'object' && typeof (value as Readable).pipe === 'function'); - } - - #isWebReadableStream (value: unknown): value is ReadableStream { - return Boolean(value && typeof value === 'object' && typeof (value as ReadableStream).getReader === 'function'); - } - - #createCountingStream ( - source: Readable, - uploadTracker?: UploadProgressTrackerLike, - ): { stream: Readable; uploadedSize: () => number; contentHashSha256: () => string } { - let uploadedBytes = 0; - const hash = createHash('sha256'); - const countingStream = new Transform({ - transform (chunk: unknown, _encoding: string, callback: TransformCallback) { - let chunkLength = 0; - if ( Buffer.isBuffer(chunk) || chunk instanceof Uint8Array ) { - chunkLength = chunk.byteLength; - hash.update(chunk); - } else if ( typeof chunk === 'string' ) { - chunkLength = Buffer.byteLength(chunk); - hash.update(chunk); - } - uploadedBytes += chunkLength; - if ( chunkLength > 0 && uploadTracker ) { - uploadTracker.add(chunkLength); - } - callback(null, chunk as Buffer | Uint8Array | string); - }, - }); - - source.on('error', (error) => { - countingStream.destroy(error); - }); - source.pipe(countingStream); - - return { - stream: countingStream, - uploadedSize: () => uploadedBytes, - contentHashSha256: () => hash.digest('hex'), - }; - } - - async #toUploadBody ( - content: WriteRequest['fileContent'], - encoding: WriteRequest['encoding'], - uploadTracker?: UploadProgressTrackerLike, - ): Promise { - if ( Buffer.isBuffer(content) ) { - const hash = createHash('sha256'); - hash.update(content); - return { - body: content, - contentLength: content.byteLength, - uploadedSize: () => content.byteLength, - contentHashSha256: hash.digest('hex'), - }; - } - if ( this.#isBinaryPayload(content) ) { - const buffer = Buffer.from(content.base64, 'base64'); - const hash = createHash('sha256'); - hash.update(buffer); - return { - body: buffer, - contentLength: buffer.byteLength, - uploadedSize: () => buffer.byteLength, - contentHashSha256: hash.digest('hex'), - }; - } - if ( typeof content === 'string' ) { - if ( encoding === 'base64' ) { - const buffer = Buffer.from(content, 'base64'); - const hash = createHash('sha256'); - hash.update(buffer); - return { - body: buffer, - contentLength: buffer.byteLength, - uploadedSize: () => buffer.byteLength, - contentHashSha256: hash.digest('hex'), - }; - } - const buffer = Buffer.from(content, encoding ?? 'utf8'); - const hash = createHash('sha256'); - hash.update(buffer); - return { - body: buffer, - contentLength: buffer.byteLength, - uploadedSize: () => buffer.byteLength, - contentHashSha256: hash.digest('hex'), - }; - } - if ( content instanceof Uint8Array ) { - const hash = createHash('sha256'); - hash.update(content); - return { - body: content, - contentLength: content.byteLength, - uploadedSize: () => content.byteLength, - contentHashSha256: hash.digest('hex'), - }; - } - if ( content instanceof ArrayBuffer ) { - const buffer = Buffer.from(content); - const hash = createHash('sha256'); - hash.update(buffer); - return { - body: buffer, - contentLength: buffer.byteLength, - uploadedSize: () => buffer.byteLength, - contentHashSha256: hash.digest('hex'), - }; - } - if ( this.#isNodeStream(content) ) { - const streamPayload = this.#createCountingStream(content, uploadTracker); - return { - body: streamPayload.stream, - uploadedSize: streamPayload.uploadedSize, - contentHashSha256: null, - finalizeContentHashSha256: () => streamPayload.contentHashSha256(), - }; - } - if ( this.#isWebReadableStream(content) ) { - const reader = content.getReader(); - const asyncIterable = { - async *[Symbol.asyncIterator] (): AsyncGenerator { - while ( true ) { - const readResult = await reader.read(); - if ( readResult.done ) { - return; - } - if ( readResult.value ) { - yield readResult.value; - } - } - }, - }; - const streamPayload = this.#createCountingStream( - Readable.from(asyncIterable), - uploadTracker, - ); - return { - body: streamPayload.stream, - uploadedSize: streamPayload.uploadedSize, - contentHashSha256: null, - finalizeContentHashSha256: () => streamPayload.contentHashSha256(), - }; - } - if ( content instanceof Blob ) { - const reader = content.stream().getReader(); - const asyncIterable = { - async *[Symbol.asyncIterator] (): AsyncGenerator { - while ( true ) { - const readResult = await reader.read(); - if ( readResult.done ) { - return; - } - if ( readResult.value ) { - yield readResult.value; - } - } - }, - }; - const streamPayload = this.#createCountingStream( - Readable.from(asyncIterable), - uploadTracker, - ); - return { - body: streamPayload.stream, - contentLength: Number.isFinite(content.size) ? content.size : undefined, - uploadedSize: streamPayload.uploadedSize, - contentHashSha256: null, - finalizeContentHashSha256: () => streamPayload.contentHashSha256(), - }; - } - - throw new HttpError(400, 'Unsupported file content payload'); - } - - async #cleanupPreparedBatchUploads ( - preparedBatch: PreparedBatchWrite, - uploadedItems: UploadedBatchWriteItem[], - ): Promise { - const cleanupTargets = uploadedItems.map((uploadedItem) => { - const preparedItem = preparedBatch.itemsByIndex.get(uploadedItem.index); - if ( !preparedItem || preparedItem.wasOverwrite ) { - return null; - } - - return { - bucket: preparedItem.normalizedInput.bucket, - bucketRegion: preparedItem.normalizedInput.bucketRegion, - objectKey: uploadedItem.objectKey, - }; - }).filter((target): target is { - bucket: string; - bucketRegion: string; - objectKey: string; - } => Boolean(target)); - - if ( cleanupTargets.length === 0 ) { - return; - } - - const cleanupResults = await Promise.allSettled(cleanupTargets.map((target) => { - return this.#s3StorageProvider.deleteObject( - target.bucket, - target.objectKey, - target.bucketRegion, - ); - })); - - const cleanupFailures = cleanupResults.filter((result) => result.status === 'rejected'); - if ( cleanupFailures.length > 0 ) { - console.error('prodfsv2 failed to clean up batch upload objects', cleanupFailures); - } - } - - getMaxSingleUploadSize (): number { - return this.#s3StorageProvider.getMaxSingleUploadSize(); - } - - async #cleanupSignedMultipartUploads (uploads: SignedMultipartCleanupTarget[]): Promise { - if ( uploads.length === 0 ) { - return; - } - - const cleanupResults = await Promise.allSettled(uploads.map((upload) => { - if ( upload.signedUploadResult.uploadMode !== 'multipart' || !upload.signedUploadResult.multipartUploadId ) { - return Promise.resolve(); - } - - return this.#s3StorageProvider.abortMutipartUpload( - upload.signedUploadResult.multipartUploadId, - upload.bucketRegion, - upload.bucket, - upload.objectKey, - ); - })); - - const cleanupFailures = cleanupResults.filter((result) => result.status === 'rejected'); - if ( cleanupFailures.length > 0 ) { - console.error('prodfsv2 failed to abort signed multipart uploads', cleanupFailures); - } - } - - #toSignedMultipartCleanupTargets ( - items: Array<{ - index: number; - normalizedInput: NormalizedWriteInput; - }>, - objectKeys: string[], - signedResultsByIndex: Map, - ): SignedMultipartCleanupTarget[] { - return items.map((item, index) => { - const signedUploadResult = signedResultsByIndex.get(item.index); - const objectKey = objectKeys[index]; - if ( !signedUploadResult || !objectKey ) { - return null; - } - - return { - bucket: item.normalizedInput.bucket, - bucketRegion: item.normalizedInput.bucketRegion, - objectKey, - signedUploadResult, - }; - }).filter((upload): upload is SignedMultipartCleanupTarget => Boolean(upload)); - } - - #toSignedWriteResponse ( - sessionId: string, - normalizedInput: NormalizedWriteInput, - objectKey: string, - signedUploadResult: SignedUploadResult, - ): SignedWriteResponse { - return { - sessionId, - uploadMode: signedUploadResult.uploadMode, - objectKey, - bucket: normalizedInput.bucket, - bucketRegion: normalizedInput.bucketRegion, - contentType: normalizedInput.contentType, - expiresAt: signedUploadResult.expiresAt, - ...(signedUploadResult.url ? { url: signedUploadResult.url } : {}), - ...(signedUploadResult.multipartUploadId ? { multipartUploadId: signedUploadResult.multipartUploadId } : {}), - ...(signedUploadResult.multipartPartSize ? { multipartPartSize: signedUploadResult.multipartPartSize } : {}), - ...(signedUploadResult.multipartPartCount ? { multipartPartCount: signedUploadResult.multipartPartCount } : {}), - ...(signedUploadResult.multipartPartUrls ? { multipartPartUrls: signedUploadResult.multipartPartUrls } : {}), - }; - } - - #toDirectorySignedWriteResponse ( - fsEntry: FSEntry, - directoryCreated: boolean, - ): SignedWriteResponse { - return { - sessionId: '', - uploadMode: 'single', - objectKey: fsEntry.uuid, - bucket: fsEntry.bucket ?? '', - bucketRegion: fsEntry.bucketRegion ?? '', - contentType: 'inode/directory', - expiresAt: Date.now(), - directoryCreated, - fsEntry, - }; - } - - async entryExistsByPath (path: string): Promise { - const entry = await this.#fsEntryRepository.getEntryByPath(path); - return entry !== null; - } - - async getAncestorChain (path: string): Promise> { - const paths: string[] = []; - let cursor = this.#normalizePath(path); - while ( cursor !== '/' ) { - paths.push(cursor); - cursor = pathPosix.dirname(cursor); - } - - const entriesByPath = await this.#fsEntryRepository.getEntriesByPaths(paths); - - const ancestors: Array<{ uid: string; path: string }> = []; - for ( const p of paths ) { - const entry = entriesByPath.get(p); - if ( entry ) { - ancestors.push({ uid: entry.uid, path: entry.path }); - } - } - return ancestors; - } - - async prepareBatchWrites ( - userId: number, - writeRequests: BatchWritePrepareRequest[], - storageAllowanceMax?: number, - ): Promise { - if ( writeRequests.length === 0 ) { - return { - userId, - items: [], - itemsByIndex: new Map(), - ...(storageAllowanceMax !== undefined ? { storageAllowanceMax } : {}), - }; - } - - const normalizedRequests = writeRequests.map((writeRequest, index) => { - const normalizedInput = this.#normalizeWriteInput(userId, writeRequest.fileMetadata); - const requestedThumbnail = writeRequest.thumbnailData ?? normalizedInput.thumbnail ?? null; - normalizedInput.thumbnail = null; - return { - index, - normalizedInput, - requestedThumbnail, - guiMetadata: writeRequest.guiMetadata, - }; - }); - - const resolvedTargets = await this.#resolveWriteTargets( - userId, - normalizedRequests.map((request) => ({ - index: request.index, - normalizedInput: request.normalizedInput, - })), - ); - const resolvedTargetMap = new Map( - resolvedTargets.map((resolvedTarget) => [resolvedTarget.index, resolvedTarget]), - ); - const resolvedRequests = normalizedRequests.map((request) => { - const resolvedTarget = resolvedTargetMap.get(request.index); - if ( ! resolvedTarget ) { - throw new Error(`Failed to resolve write target for index ${request.index}`); - } - return { - ...request, - normalizedInput: resolvedTarget.normalizedInput, - existingEntry: resolvedTarget.existingEntry, - wasOverwrite: resolvedTarget.wasOverwrite, - }; - }); - - await this.#fsEntryRepository.resolveParentDirectoriesBatch( - userId, - resolvedRequests.map((item) => ({ - parentPath: pathPosix.dirname(item.normalizedInput.path), - createPaths: item.normalizedInput.createMissingParents, - })), - ); - - const items = resolvedRequests.map((item) => ({ - index: item.index, - normalizedInput: item.normalizedInput, - existingEntry: item.existingEntry, - objectKey: item.existingEntry?.uuid ?? uuidv4(), - wasOverwrite: item.wasOverwrite, - requestedThumbnail: item.requestedThumbnail, - guiMetadata: item.guiMetadata, - })); - const itemsByIndex = new Map(); - for ( const item of items ) { - itemsByIndex.set(item.index, item); - } - - return { - userId, - items, - itemsByIndex, - ...(storageAllowanceMax !== undefined ? { storageAllowanceMax } : {}), - }; - } - - async assertStorageAllowanceForPreparedBatch ( - preparedBatch: PreparedBatchWrite, - uploadedItems?: UploadedBatchWriteItem[], - storageAllowanceMaxOverride?: number, - ): Promise { - if ( preparedBatch.items.length === 0 ) { - return; - } - - const uploadedItemMap = new Map(); - if ( uploadedItems ) { - for ( const uploadedItem of uploadedItems ) { - uploadedItemMap.set(uploadedItem.index, uploadedItem); - } - } - - const sizeChanges = preparedBatch.items.map((item) => { - const uploadedItem = uploadedItemMap.get(item.index); - return { - incomingSize: uploadedItem ? uploadedItem.uploadedSize : item.normalizedInput.size, - existingSize: item.existingEntry?.size ?? 0, - }; - }); - - const storageAllowanceMax = storageAllowanceMaxOverride ?? preparedBatch.storageAllowanceMax; - await this.#assertStorageAllowanceForBatch(preparedBatch.userId, sizeChanges, storageAllowanceMax); - } - - async uploadPreparedBatchItem ( - input: UploadPreparedBatchItemInput, - ): Promise { - const preparedItem = input.preparedBatch.itemsByIndex.get(input.itemIndex); - if ( ! preparedItem ) { - throw new HttpError(400, `Batch metadata was not found for index ${input.itemIndex}`); - } - - const uploadBody = await this.#toUploadBody( - input.fileContent, - input.encoding, - input.uploadTracker, - ); - - await this.#s3StorageProvider.uploadFromServer({ - bucket: preparedItem.normalizedInput.bucket, - objectKey: preparedItem.objectKey, - contentType: preparedItem.normalizedInput.contentType, - body: uploadBody.body, - ...(uploadBody.contentLength !== undefined ? { contentLength: uploadBody.contentLength } : {}), - ...(Number.isFinite(preparedItem.normalizedInput.size) - ? { sizeHint: preparedItem.normalizedInput.size } - : {}), - }, preparedItem.normalizedInput.bucketRegion); - - const uploadedSize = uploadBody.uploadedSize(); - if ( input.uploadTracker ) { - const currentTrackedSize = Number(input.uploadTracker.progress ?? 0); - if ( uploadedSize > currentTrackedSize ) { - input.uploadTracker.add(uploadedSize - currentTrackedSize); - } - } - - return { - index: preparedItem.index, - objectKey: preparedItem.objectKey, - uploadedSize, - contentHashSha256: uploadBody.finalizeContentHashSha256 - ? uploadBody.finalizeContentHashSha256() - : uploadBody.contentHashSha256, - }; - } - - async finalizePreparedBatchWrites ( - preparedBatch: PreparedBatchWrite, - uploadedItems: UploadedBatchWriteItem[], - ): Promise { - try { - if ( preparedBatch.items.length !== uploadedItems.length ) { - throw new HttpError(400, 'Some batch files were missing upload content'); - } - - await this.assertStorageAllowanceForPreparedBatch(preparedBatch, uploadedItems); - - const uploadedItemMap = new Map(); - for ( const uploadedItem of uploadedItems ) { - uploadedItemMap.set(uploadedItem.index, uploadedItem); - } - - const createInputs = preparedBatch.items.map((item) => { - const uploadedItem = uploadedItemMap.get(item.index); - if ( ! uploadedItem ) { - throw new HttpError(400, `Missing uploaded file content for index ${item.index}`); - } - item.normalizedInput.size = uploadedItem.uploadedSize; - return this.#toCreateInput(item.normalizedInput, uploadedItem.objectKey); - }); - - const fsEntries = await this.#fsEntryRepository.batchCreateEntries(createInputs, true); - return preparedBatch.items.map((item, index) => { - const fsEntry = fsEntries[index]; - if ( ! fsEntry ) { - throw new Error(`Failed to resolve batch write result at index ${index}`); - } - const uploadedItem = uploadedItemMap.get(item.index); - return { - fsEntry, - wasOverwrite: item.wasOverwrite, - requestedThumbnail: item.requestedThumbnail, - contentHashSha256: uploadedItem?.contentHashSha256 ?? null, - }; - }); - } catch ( error ) { - await this.#cleanupPreparedBatchUploads(preparedBatch, uploadedItems); - throw error; - } - } - - async startUrlWrite ( - userId: number, - signedWriteRequest: SignedWriteRequest, - storageAllowanceMax?: number, - ): Promise { - const result = await this.startUrlWriteWithCreatedDirectories( - userId, - signedWriteRequest, - storageAllowanceMax, - ); - return result.response; - } - - async startUrlWriteWithCreatedDirectories ( - userId: number, - signedWriteRequest: SignedWriteRequest, - storageAllowanceMax?: number, - ): Promise { - let normalizedInput = this.#normalizeWriteInput(userId, signedWriteRequest.fileMetadata); - if ( signedWriteRequest.directory ) { - const { - entries, - createdDirectoryEntries, - } = await this.#fsEntryRepository.ensureDirectoriesForUserWithCreated( - userId, - [{ - path: normalizedInput.path, - createPaths: normalizedInput.createMissingParents, - }], - ); - const [directoryEntry] = entries; - if ( ! directoryEntry ) { - throw new Error('Failed to resolve directory entry after start write'); - } - const createdDirectoryPathSet = new Set(createdDirectoryEntries.map((entry) => entry.path)); - return { - response: this.#toDirectorySignedWriteResponse( - directoryEntry, - createdDirectoryPathSet.has(normalizedInput.path), - ), - createdDirectoryEntries, - }; - } - - const [resolvedTarget] = await this.#resolveWriteTargets(userId, [{ - index: 0, - normalizedInput, - }]); - if ( ! resolvedTarget ) { - throw new Error('Failed to resolve write target'); - } - normalizedInput = resolvedTarget.normalizedInput; - const existingEntry = resolvedTarget.existingEntry; - - const existingSize = existingEntry?.size ?? 0; - const parentPath = pathPosix.dirname(normalizedInput.path); - const [, { - parentEntries, - createdDirectoryEntries, - }] = await Promise.all([ - this.#assertStorageAllowance(userId, normalizedInput.size, existingSize, storageAllowanceMax), - this.#fsEntryRepository.resolveParentDirectoriesBatchWithCreated( - userId, - [{ - parentPath, - createPaths: normalizedInput.createMissingParents, - }], - ), - ]); - const [parentEntry] = parentEntries; - if ( ! parentEntry ) { - throw new Error('Failed to resolve parent directory for signed write'); - } - - const objectKey = existingEntry?.uuid ?? uuidv4(); - const uploadMode = this.#determineUploadMode(signedWriteRequest.uploadMode, normalizedInput.size); - const expiresInSeconds = signedWriteRequest.expiresInSeconds ?? DEFAULT_SIGNED_UPLOAD_EXPIRY_SECONDS; - const createInput = this.#toCreateInput(normalizedInput, objectKey); - - const signedUploadResult = await this.#s3StorageProvider.createSignedUploadUrl({ - bucket: normalizedInput.bucket, - objectKey, - size: normalizedInput.size, - contentType: normalizedInput.contentType, - uploadMode, - expiresInSeconds, - multipartPartSize: normalizedInput.multipartPartSize, - }, normalizedInput.bucketRegion); - - const sessionId = uuidv4(); - const pendingUploadInput: PendingUploadCreateInput = { - sessionId, - userId, - appId: normalizedInput.associatedAppId ?? null, - parentUid: parentEntry.uuid, - parentPath: parentEntry.path, - targetName: pathPosix.basename(normalizedInput.path), - targetPath: normalizedInput.path, - overwriteTargetUid: existingEntry?.uuid ?? null, - contentType: normalizedInput.contentType, - size: normalizedInput.size, - checksumSha256: normalizedInput.checksumSha256 ?? null, - uploadMode, - multipartUploadId: signedUploadResult.multipartUploadId ?? null, - multipartPartSize: signedUploadResult.multipartPartSize ?? null, - multipartPartCount: signedUploadResult.multipartPartCount ?? null, - storageProvider: 's3', - bucket: normalizedInput.bucket, - bucketRegion: normalizedInput.bucketRegion, - objectKey, - metadataJson: JSON.stringify(createInput), - expiresAt: signedUploadResult.expiresAt, - }; - - try { - await this.#fsEntryRepository.createPendingEntry(pendingUploadInput); - } catch ( error ) { - await this.#cleanupSignedMultipartUploads([{ - bucket: normalizedInput.bucket, - bucketRegion: normalizedInput.bucketRegion, - objectKey, - signedUploadResult, - }]); - throw error; - } - - return { - response: this.#toSignedWriteResponse(sessionId, normalizedInput, objectKey, signedUploadResult), - createdDirectoryEntries, - }; - } - - async batchStartUrlWrites ( - userId: number, - signedWriteRequests: SignedWriteRequest[], - storageAllowanceMax?: number, - ): Promise { - const result = await this.batchStartUrlWritesWithCreatedDirectories( - userId, - signedWriteRequests, - storageAllowanceMax, - ); - return result.responses; - } - - async batchStartUrlWritesWithCreatedDirectories ( - userId: number, - signedWriteRequests: SignedWriteRequest[], - storageAllowanceMax?: number, - ): Promise { - if ( signedWriteRequests.length === 0 ) { - return { - responses: [], - createdDirectoryEntries: [], - }; - } - - const normalizedRequests = signedWriteRequests.map((signedWriteRequest, index) => ({ - index, - request: signedWriteRequest, - isDirectory: Boolean(signedWriteRequest.directory), - normalizedInput: this.#normalizeWriteInput(userId, signedWriteRequest.fileMetadata), - })); - const responsesByIndex = new Map(); - const createdDirectoryEntriesByPath = new Map(); - - const directoryItems = normalizedRequests.filter((item) => item.isDirectory); - const directoryPathSet = new Set(); - for ( const directoryItem of directoryItems ) { - const targetPath = directoryItem.normalizedInput.path; - if ( directoryPathSet.has(targetPath) ) { - throw new HttpError(409, `Batch contains duplicate target path: ${targetPath}`); - } - directoryPathSet.add(targetPath); - } - if ( directoryItems.length > 0 ) { - const { - entries: ensuredDirectoryEntries, - createdDirectoryEntries, - } = await this.#fsEntryRepository.ensureDirectoriesForUserWithCreated( - userId, - directoryItems.map((item) => ({ - path: item.normalizedInput.path, - createPaths: item.normalizedInput.createMissingParents, - })), - ); - for ( const createdDirectoryEntry of createdDirectoryEntries ) { - createdDirectoryEntriesByPath.set(createdDirectoryEntry.path, createdDirectoryEntry); - } - - for ( let index = 0; index < directoryItems.length; index++ ) { - const item = directoryItems[index]; - const directoryEntry = ensuredDirectoryEntries[index]; - if ( !item || !directoryEntry ) { - throw new Error('Failed to build directory response from batch start data'); - } - responsesByIndex.set( - item.index, - this.#toDirectorySignedWriteResponse( - directoryEntry, - createdDirectoryEntriesByPath.has(item.normalizedInput.path), - ), - ); - } - } - - const fileItems = normalizedRequests.filter((item) => !item.isDirectory); - if ( fileItems.length > 0 ) { - const resolvedTargets = await this.#resolveWriteTargets( - userId, - fileItems.map((item) => ({ - index: item.index, - normalizedInput: item.normalizedInput, - })), - ); - const resolvedTargetMap = new Map( - resolvedTargets.map((resolvedTarget) => [resolvedTarget.index, resolvedTarget]), - ); - const resolvedFileItems = fileItems.map((item) => { - const resolvedTarget = resolvedTargetMap.get(item.index); - if ( ! resolvedTarget ) { - throw new Error(`Failed to resolve write target for batch index ${item.index}`); - } - - return { - ...item, - normalizedInput: resolvedTarget.normalizedInput, - existingEntry: resolvedTarget.existingEntry, - }; - }); - - const allowanceChecks: Array<{ incomingSize: number; existingSize: number }> = []; - for ( const item of resolvedFileItems ) { - allowanceChecks.push({ - incomingSize: item.normalizedInput.size, - existingSize: item.existingEntry?.size ?? 0, - }); - } - const [, { - parentEntries, - createdDirectoryEntries: createdParentDirectoryEntries, - }] = await Promise.all([ - this.#assertStorageAllowanceForBatch(userId, allowanceChecks, storageAllowanceMax), - this.#fsEntryRepository.resolveParentDirectoriesBatchWithCreated( - userId, - resolvedFileItems.map((item) => ({ - parentPath: pathPosix.dirname(item.normalizedInput.path), - createPaths: item.normalizedInput.createMissingParents, - })), - ), - ]); - for ( const createdParentDirectoryEntry of createdParentDirectoryEntries ) { - createdDirectoryEntriesByPath.set(createdParentDirectoryEntry.path, createdParentDirectoryEntry); - } - - const objectKeys = resolvedFileItems.map((item) => { - return item.existingEntry?.uuid ?? uuidv4(); - }); - const uploadModes = resolvedFileItems.map((item) => { - return this.#determineUploadMode(item.request.uploadMode, item.normalizedInput.size); - }); - const sessionIds = resolvedFileItems.map(() => uuidv4()); - - const signedResultsByIndex = new Map(); - const writesByRegion = new Map>(); - for ( let index = 0; index < resolvedFileItems.length; index++ ) { - const item = resolvedFileItems[index]; - const objectKey = objectKeys[index]; - const uploadMode = uploadModes[index]; - if ( !item || !objectKey || !uploadMode ) { - throw new Error('Failed to build batch signed upload request'); - } - const regionEntries = writesByRegion.get(item.normalizedInput.bucketRegion) ?? []; - regionEntries.push({ - requestIndex: item.index, - input: { - bucket: item.normalizedInput.bucket, - objectKey, - size: item.normalizedInput.size, - contentType: item.normalizedInput.contentType, - uploadMode, - expiresInSeconds: item.request.expiresInSeconds ?? DEFAULT_SIGNED_UPLOAD_EXPIRY_SECONDS, - multipartPartSize: item.normalizedInput.multipartPartSize, - }, - }); - writesByRegion.set(item.normalizedInput.bucketRegion, regionEntries); - } - - const regionResults = await Promise.allSettled(Array.from(writesByRegion.entries()).map(async ([region, regionWrites]) => { - const signedResults = await this.#s3StorageProvider.batchCreateSignedUploadUrls( - regionWrites.map((item) => item.input), - region, - ); - for ( let index = 0; index < regionWrites.length; index++ ) { - const regionWrite = regionWrites[index]; - const signedResult = signedResults[index]; - if ( !regionWrite || !signedResult ) { - throw new Error('Failed to map signed upload result to request'); - } - signedResultsByIndex.set(regionWrite.requestIndex, signedResult); - } - })); - const signedMultipartCleanupTargets = this.#toSignedMultipartCleanupTargets( - resolvedFileItems, - objectKeys, - signedResultsByIndex, - ); - - const failedRegionResult = regionResults.find((result) => result.status === 'rejected'); - if ( failedRegionResult?.status === 'rejected' ) { - await this.#cleanupSignedMultipartUploads(signedMultipartCleanupTargets); - - throw this.#toError(failedRegionResult.reason, 'Failed to create batch signed upload urls'); - } - - try { - const pendingInputs: PendingUploadCreateInput[] = []; - for ( let index = 0; index < resolvedFileItems.length; index++ ) { - const item = resolvedFileItems[index]; - const parentEntry = parentEntries[index]; - const objectKey = objectKeys[index]; - const sessionId = sessionIds[index]; - const uploadMode = uploadModes[index]; - const existingEntry = item?.existingEntry; - if ( !item || !parentEntry || !objectKey || !sessionId || !uploadMode ) { - throw new Error('Failed to build pending upload input from batch start data'); - } - const signedUploadResult = signedResultsByIndex.get(item.index); - if ( ! signedUploadResult ) { - throw new Error('Failed to resolve signed upload result for batch start data'); - } - - const createInput = this.#toCreateInput(item.normalizedInput, objectKey); - pendingInputs.push({ - sessionId, - userId, - appId: item.normalizedInput.associatedAppId ?? null, - parentUid: parentEntry.uuid, - parentPath: parentEntry.path, - targetName: pathPosix.basename(item.normalizedInput.path), - targetPath: item.normalizedInput.path, - overwriteTargetUid: existingEntry?.uuid ?? null, - contentType: item.normalizedInput.contentType, - size: item.normalizedInput.size, - checksumSha256: item.normalizedInput.checksumSha256 ?? null, - uploadMode, - multipartUploadId: signedUploadResult.multipartUploadId ?? null, - multipartPartSize: signedUploadResult.multipartPartSize ?? null, - multipartPartCount: signedUploadResult.multipartPartCount ?? null, - storageProvider: 's3', - bucket: item.normalizedInput.bucket, - bucketRegion: item.normalizedInput.bucketRegion, - objectKey, - metadataJson: JSON.stringify(createInput), - expiresAt: signedUploadResult.expiresAt, - }); - } - - await this.#fsEntryRepository.batchCreatePendingEntries(pendingInputs); - - for ( let index = 0; index < resolvedFileItems.length; index++ ) { - const item = resolvedFileItems[index]; - const sessionId = sessionIds[index]; - const objectKey = objectKeys[index]; - if ( !item || !sessionId || !objectKey ) { - throw new Error('Failed to build signed write response from batch start data'); - } - const signedUploadResult = signedResultsByIndex.get(item.index); - if ( ! signedUploadResult ) { - throw new Error('Failed to resolve signed upload result for batch response data'); - } - responsesByIndex.set(item.index, this.#toSignedWriteResponse( - sessionId, - item.normalizedInput, - objectKey, - signedUploadResult, - )); - } - } catch ( error ) { - await this.#cleanupSignedMultipartUploads(signedMultipartCleanupTargets); - throw error; - } - } - - const responses = normalizedRequests.map((request) => { - const response = responsesByIndex.get(request.index); - if ( ! response ) { - throw new Error(`Failed to resolve signed batch response for index ${request.index}`); - } - return response; - }); - return { - responses, - createdDirectoryEntries: Array.from(createdDirectoryEntriesByPath.values()), - }; - } - - async signMultipartParts ( - userId: number, - request: SignMultipartPartsRequest, - ): Promise { - if ( ! request?.uploadId ) { - throw new HttpError(400, 'Missing uploadId'); - } - if ( !Array.isArray(request.partNumbers) || request.partNumbers.length === 0 ) { - throw new HttpError(400, 'Missing partNumbers'); - } - - const uniquePartNumbers = Array.from(new Set(request.partNumbers.map((value) => Number(value)))); - if ( uniquePartNumbers.some((partNumber) => !Number.isInteger(partNumber) || partNumber <= 0) ) { - throw new HttpError(400, 'Invalid partNumbers'); - } - - const session = await this.#fsEntryRepository.getPendingEntryBySessionId(request.uploadId); - if ( ! session ) { - throw new HttpError(404, 'Upload session was not found'); - } - if ( session.userId !== userId ) { - throw new HttpError(403, 'Upload session access denied'); - } - if ( session.status !== 'pending' ) { - throw new HttpError(409, `Upload session is not pending (status=${session.status})`); - } - if ( session.expiresAt < Date.now() ) { - await this.#fsEntryRepository.markPendingEntryFailed(session.sessionId, 'Upload session expired'); - throw new HttpError(400, 'Upload session expired'); - } - if ( session.uploadMode !== 'multipart' ) { - throw new HttpError(400, 'Upload session is not multipart'); - } - if ( ! session.multipartUploadId ) { - throw new HttpError(400, 'Multipart upload id missing from session'); - } - const multipartPartCount = session.multipartPartCount; - if ( - multipartPartCount !== null - && uniquePartNumbers.some((partNumber) => partNumber > multipartPartCount) - ) { - throw new HttpError(400, 'Part number exceeds multipart part count'); - } - if ( !session.bucket || !session.bucketRegion ) { - throw new HttpError(500, 'Upload session storage metadata is missing'); - } - - const expiresInSeconds = request.expiresInSeconds ?? DEFAULT_SIGNED_UPLOAD_EXPIRY_SECONDS; - const multipartPartUrls = await this.#s3StorageProvider.createSignedMultipartPartUrls({ - bucket: session.bucket, - objectKey: session.objectKey, - multipartUploadId: session.multipartUploadId, - partNumbers: uniquePartNumbers, - expiresInSeconds, - }, session.bucketRegion); - - const expiresAt = Date.now() + Math.max(60, Math.min(60 * 60, expiresInSeconds)) * 1000; - - return { - uploadId: session.sessionId, - multipartUploadId: session.multipartUploadId, - objectKey: session.objectKey, - bucket: session.bucket, - bucketRegion: session.bucketRegion, - expiresAt, - multipartPartUrls, - }; - } - - async completeUrlWrite (userId: number, completeWriteRequest: CompleteWriteRequest): Promise { - const session = await this.#fsEntryRepository.getPendingEntryBySessionId(completeWriteRequest.uploadId); - if ( ! session ) { - throw new HttpError(404, 'Upload session was not found'); - } - if ( session.userId !== userId ) { - throw new HttpError(403, 'Upload session access denied'); - } - if ( session.status !== 'pending' ) { - throw new HttpError(409, `Upload session is not pending (status=${session.status})`); - } - if ( session.expiresAt < Date.now() ) { - await this.#fsEntryRepository.markPendingEntryFailed(session.sessionId, 'Upload session expired'); - throw new HttpError(400, 'Upload session expired'); - } - - const createInput = this.#parseSessionMetadata(session); - const requestedThumbnail = completeWriteRequest.thumbnailData ?? createInput.thumbnail ?? null; - createInput.thumbnail = null; - - try { - if ( session.uploadMode === 'multipart' ) { - if ( ! session.multipartUploadId ) { - throw new HttpError(400, 'Multipart upload id missing from session'); - } - - const completeParts = this.#toMultipartParts(completeWriteRequest.parts); - if ( completeParts.length === 0 ) { - throw new HttpError(400, 'Multipart upload completion requires parts'); - } - - await this.#s3StorageProvider.completeMultipartUpload({ - bucket: session.bucket ?? createInput.bucket ?? this.#resolveBucket(createInput), - objectKey: session.objectKey, - multipartUploadId: session.multipartUploadId, - parts: completeParts, - }, session.bucketRegion ?? createInput.bucketRegion ?? this.#resolveBucketRegion(createInput)); - } - - const fsEntry = await this.#fsEntryRepository.completePendingEntry(session.sessionId, createInput); - return { - sessionId: session.sessionId, - fsEntry, - wasOverwrite: Boolean(session.overwriteTargetUid), - requestedThumbnail, - }; - } catch ( error ) { - await this.#fsEntryRepository.markPendingEntryFailed( - session.sessionId, - error instanceof Error ? error.message : 'Unknown error while completing upload', - ); - throw error; - } - } - - async batchCompleteUrlWrite ( - userId: number, - completeWriteRequests: CompleteWriteRequest[], - ): Promise { - if ( completeWriteRequests.length === 0 ) { - return []; - } - - const uploadIds = completeWriteRequests.map((request) => request.uploadId); - const uniqueUploadIds = new Set(uploadIds); - if ( uniqueUploadIds.size !== uploadIds.length ) { - throw new HttpError(409, 'Batch contains duplicate upload session ids'); - } - - const sessions = await this.#fsEntryRepository.getPendingEntriesBySessionIds(uploadIds); - const completionItems: Array<{ - index: number; - request: CompleteWriteRequest; - session: PendingUploadSession; - finalData: FSEntryCreateInput; - requestedThumbnail: string | null | undefined; - }> = []; - const expiredSessionIds: string[] = []; - - for ( let index = 0; index < completeWriteRequests.length; index++ ) { - const request = completeWriteRequests[index]; - const session = sessions[index]; - if ( !request || !session ) { - throw new HttpError(404, 'Upload session was not found'); - } - if ( session.userId !== userId ) { - throw new HttpError(403, 'Upload session access denied'); - } - if ( session.status !== 'pending' ) { - throw new HttpError(409, `Upload session is not pending (status=${session.status})`); - } - if ( session.expiresAt < Date.now() ) { - expiredSessionIds.push(session.sessionId); - continue; - } - - const finalData = this.#parseSessionMetadata(session); - const requestedThumbnail = request.thumbnailData ?? finalData.thumbnail ?? null; - finalData.thumbnail = null; - completionItems.push({ - index, - request, - session, - finalData, - requestedThumbnail, - }); - } - - if ( expiredSessionIds.length > 0 ) { - await this.#fsEntryRepository.markPendingEntriesFailed(expiredSessionIds, 'Upload session expired'); - throw new HttpError(400, 'Upload session expired'); - } - - const multipartItems = completionItems.filter((item) => item.session.uploadMode === 'multipart'); - const multipartCompletions = await Promise.allSettled(multipartItems.map(async (item) => { - if ( ! item.session.multipartUploadId ) { - throw new HttpError(400, 'Multipart upload id missing from session'); - } - - const completeParts = this.#toMultipartParts(item.request.parts); - if ( completeParts.length === 0 ) { - throw new HttpError(400, 'Multipart upload completion requires parts'); - } - - await this.#s3StorageProvider.completeMultipartUpload({ - bucket: item.session.bucket ?? item.finalData.bucket ?? this.#resolveBucket(item.finalData), - objectKey: item.session.objectKey, - multipartUploadId: item.session.multipartUploadId, - parts: completeParts, - }, item.session.bucketRegion ?? item.finalData.bucketRegion ?? this.#resolveBucketRegion(item.finalData)); - })); - - const failedMultipartItems: Array<{ sessionId: string; reason: unknown }> = []; - for ( let index = 0; index < multipartCompletions.length; index++ ) { - const completion = multipartCompletions[index]; - const multipartItem = multipartItems[index]; - if ( completion?.status === 'rejected' && multipartItem ) { - failedMultipartItems.push({ - sessionId: multipartItem.session.sessionId, - reason: completion.reason, - }); - } - } - - if ( failedMultipartItems.length > 0 ) { - await Promise.all(failedMultipartItems.map((item) => { - return this.#fsEntryRepository.markPendingEntryFailed(item.sessionId, this.#toErrorMessage(item.reason)); - })); - - const firstReason = failedMultipartItems[0]?.reason; - if ( firstReason instanceof HttpError ) { - throw firstReason; - } - if ( firstReason instanceof Error ) { - throw firstReason; - } - throw new Error('Failed to complete multipart upload'); - } - - const completedEntries = await this.#fsEntryRepository.batchCompletePendingEntries( - completionItems.map((item) => ({ - sessionId: item.session.sessionId, - finalData: item.finalData, - })), - ); - - const responseByIndex = new Map(); - for ( let index = 0; index < completionItems.length; index++ ) { - const completionItem = completionItems[index]; - const completedEntry = completedEntries[index]; - if ( !completionItem || !completedEntry ) { - throw new Error('Failed to build completed batch write response'); - } - - responseByIndex.set(completionItem.index, { - sessionId: completionItem.session.sessionId, - fsEntry: completedEntry, - wasOverwrite: Boolean(completionItem.session.overwriteTargetUid), - requestedThumbnail: completionItem.requestedThumbnail, - }); - } - - const response: CompleteWriteResponse[] = []; - for ( let index = 0; index < completeWriteRequests.length; index++ ) { - const result = responseByIndex.get(index); - if ( ! result ) { - throw new Error(`Failed to resolve completed batch response for index ${index}`); - } - response.push(result); - } - return response; - } - - async abortUrlWrite (userId: number, uploadId: string): Promise { - const session = await this.#fsEntryRepository.getPendingEntryBySessionId(uploadId); - if ( ! session ) { - return; - } - if ( session.userId !== userId ) { - throw new HttpError(403, 'Upload session access denied'); - } - - try { - const bucket = session.bucket; - const bucketRegion = session.bucketRegion; - if ( bucket && bucketRegion ) { - if ( session.uploadMode === 'multipart' && session.multipartUploadId ) { - await this.#s3StorageProvider.abortMutipartUpload( - session.multipartUploadId, - bucketRegion, - bucket, - session.objectKey, - ); - } else { - await this.#s3StorageProvider.deleteObject(bucket, session.objectKey, bucketRegion); - } - } - } finally { - await this.#fsEntryRepository.abortPendingEntry(session.sessionId, 'Upload aborted by caller'); - } - } - - async write ( - userId: number, - writeRequest: WriteRequest, - uploadTracker?: UploadProgressTrackerLike, - storageAllowanceMax?: number, - ): Promise { - - let normalizedInput = this.#normalizeWriteInput(userId, writeRequest.fileMetadata); - const [resolvedTarget] = await this.#resolveWriteTargets(userId, [{ - index: 0, - normalizedInput, - }]); - if ( ! resolvedTarget ) { - throw new Error('Failed to resolve write target'); - } - normalizedInput = resolvedTarget.normalizedInput; - const existingEntry = resolvedTarget.existingEntry; - const requestedThumbnail = writeRequest.thumbnailData ?? normalizedInput.thumbnail ?? null; - normalizedInput.thumbnail = null; - - const existingSize = existingEntry?.size ?? 0; - await this.#assertStorageAllowance(userId, normalizedInput.size, existingSize, storageAllowanceMax); - - const uploadBody = await this.#toUploadBody( - writeRequest.fileContent, - writeRequest.encoding, - uploadTracker, - ); - const objectKey = existingEntry?.uuid ?? uuidv4(); - await this.#s3StorageProvider.uploadFromServer({ - bucket: normalizedInput.bucket, - objectKey, - contentType: normalizedInput.contentType, - body: uploadBody.body, - ...(uploadBody.contentLength !== undefined ? { contentLength: uploadBody.contentLength } : {}), - ...(Number.isFinite(normalizedInput.size) - ? { sizeHint: normalizedInput.size } - : {}), - }, normalizedInput.bucketRegion); - - const uploadedSize = uploadBody.uploadedSize(); - if ( uploadTracker ) { - const currentTrackedSize = Number(uploadTracker.progress ?? 0); - if ( uploadedSize > currentTrackedSize ) { - uploadTracker.add(uploadedSize - currentTrackedSize); - } - } - if ( uploadedSize > normalizedInput.size ) { - await this.#assertStorageAllowance(userId, uploadedSize, existingSize, storageAllowanceMax); - } - normalizedInput.size = uploadedSize; - const contentHashSha256 = uploadBody.finalizeContentHashSha256 - ? uploadBody.finalizeContentHashSha256() - : uploadBody.contentHashSha256; - - const createInput = this.#toCreateInput(normalizedInput, objectKey); - const fsEntry = await this.#fsEntryRepository.createEntry( - createInput, - normalizedInput.createMissingParents, - ); - - return { - fsEntry, - wasOverwrite: Boolean(existingEntry), - requestedThumbnail, - contentHashSha256, - }; - } - - async batchWrites ( - userId: number, - writeRequests: WriteRequest[], - storageAllowanceMax?: number, - ): Promise { - if ( writeRequests.length === 0 ) { - return []; - } - const preparedBatch = await this.prepareBatchWrites( - userId, - writeRequests.map((writeRequest) => ({ - fileMetadata: writeRequest.fileMetadata, - thumbnailData: writeRequest.thumbnailData, - guiMetadata: writeRequest.guiMetadata, - })), - storageAllowanceMax, - ); - await this.assertStorageAllowanceForPreparedBatch(preparedBatch, undefined, storageAllowanceMax); - - const uploadResults = await runWithConcurrencyLimitSettled( - writeRequests, - 8, - async (writeRequest, index) => { - return this.uploadPreparedBatchItem({ - preparedBatch, - itemIndex: index, - fileContent: writeRequest.fileContent, - encoding: writeRequest.encoding, - }); - }, - ); - const uploadedItems = uploadResults - .filter((result): result is PromiseFulfilledResult => result.status === 'fulfilled') - .map((result) => result.value); - const failedUpload = uploadResults.find((result) => result.status === 'rejected'); - if ( failedUpload?.status === 'rejected' ) { - await this.#cleanupPreparedBatchUploads(preparedBatch, uploadedItems); - throw this.#toError(failedUpload.reason, 'Failed to upload batch write item'); - } - - return this.finalizePreparedBatchWrites(preparedBatch, uploadedItems); - } - - async cleanupPreparedBatchUploads ( - preparedBatch: PreparedBatchWrite, - uploadedItems: UploadedBatchWriteItem[], - ): Promise { - await this.#cleanupPreparedBatchUploads(preparedBatch, uploadedItems); - } - - async updateEntryThumbnail ( - userId: number, - entryUuid: string, - thumbnail: string | null, - ): Promise { - if ( typeof entryUuid !== 'string' || entryUuid.length === 0 ) { - throw new HttpError(400, 'Invalid file entry identifier for thumbnail update'); - } - - return this.#fsEntryRepository.updateEntryThumbnailByUuidForUser( - userId, - entryUuid, - thumbnail, - ); - } - - async getUsersStorageAllowance (userId: string | number): Promise<{ curr: number; max: number }> { - const numericUserId = typeof userId === 'string' ? Number(userId) : userId; - if ( Number.isNaN(numericUserId) ) { - throw new HttpError(400, 'Invalid user id'); - } - return this.#fsEntryRepository.getUserStorageAllowance(numericUserId); - } -} diff --git a/extensions/fsv2/tsconfig.json b/extensions/fsv2/tsconfig.json deleted file mode 100644 index 358d469bd..000000000 --- a/extensions/fsv2/tsconfig.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2024", - "module": "nodenext", - "moduleResolution": "nodenext", - "strict": true, - "forceConsistentCasingInFileNames": true, - "skipLibCheck": true, - "sourceMap": true, - "removeComments": true, - "noEmitOnError": true, - "noImplicitAny": false, - "allowJs": true, - "checkJs": false, - }, - "include": [ - "./**/*.ts", - "./**/*.d.ts" - ], - "exclude": [ - "**/*.test.ts", - "**/*.spec.ts", - "**/test/**", - "**/tests/**", - "node_modules", - "dist", - "*.js" - ] -} \ No newline at end of file diff --git a/extensions/installedApps.ts b/extensions/installedApps.ts new file mode 100644 index 000000000..78ef40985 --- /dev/null +++ b/extensions/installedApps.ts @@ -0,0 +1,74 @@ +import { Context } from '@heyputer/backend/src/core'; +import { HttpError } from '@heyputer/backend/src/core/http'; +import { extension } from '@heyputer/backend/src/extensions'; +import { getAppIconUrl } from '@heyputer/backend/src/util/appIcon.js'; + +const clients = extension.import('client'); + +const ALLOWED_ORDER_BY = [ + 'id', + 'name', + 'uid', + 'title', + 'installed_at', +] as const; +const ORDER_BY_FIELD_MAP: Record = { + id: 'apps.id', + name: 'apps.name', + uid: 'apps.uid', + title: 'apps.title', + installed_at: 'installed_at', +}; + +extension.get( + '/installedApps', + { subdomain: 'api', requireUserActor: true }, + async (req, res) => { + const actor = Context.get('actor'); + if (!actor?.user?.id) + throw new HttpError(401, 'Authentication required'); + + const orderBy = String(req.query.orderBy ?? 'installed_at'); + if (!(ALLOWED_ORDER_BY as readonly string[]).includes(orderBy)) { + throw new HttpError( + 400, + `Invalid orderBy. Allowed: ${ALLOWED_ORDER_BY.join(', ')}`, + ); + } + + const page = Math.max(Number(req.query.page) || 1, 1); + const limit = Math.min( + Math.max(Number(req.query.limit) || 100, 1), + 100, + ); + const offset = (page - 1) * limit; + const orderByField = ORDER_BY_FIELD_MAP[orderBy]; + const sortDirection = req.query.desc ? 'DESC' : 'ASC'; + + const installedApps = (await clients.db.read( + `SELECT + apps.name, + apps.uid, + apps.title, + apps.description, + apps.icon, + MIN(perm.dt) AS installed_at + FROM apps + LEFT JOIN user_to_app_permissions AS perm ON apps.id = perm.app_id + WHERE perm.user_id = ? + GROUP BY apps.id, apps.name, apps.uid, apps.title, apps.description + ORDER BY ${orderByField} ${sortDirection} + LIMIT ? + OFFSET ?`, + [actor.user.id, limit, offset], + )) as Array>; + + const apiBaseUrl = extension.config.api_base_url as string | undefined; + res.json( + installedApps.map((app) => ({ + ...app, + iconUrl: getAppIconUrl(app, { apiBaseUrl }), + })), + ); + }, +); diff --git a/extensions/installedApps/package.json b/extensions/installedApps/package.json deleted file mode 100644 index 0f5a2f7cf..000000000 --- a/extensions/installedApps/package.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "name": "@puter/extension-controller", - "version": "1.0.0", - "description": "", - "main": "src/index.js", - "type": "module", - "scripts": { - "postinstall": "tsc --noCheck" - }, - "keywords": [], - "author": "", - "license": "ISC", - "devDependencies": { - "@types/express": "^4.17.21", - "@types/node": "^24.9.1", - "ts-node": "^10.9.2", - "typescript": "^5.9.3" - }, - "dependencies": { - "http-status-codes": "^2.3.0", - "stripe": "^19.1.0" - } -} \ No newline at end of file diff --git a/extensions/installedApps/src/controllers/InstalledAppsController.ts b/extensions/installedApps/src/controllers/InstalledAppsController.ts deleted file mode 100644 index c24062639..000000000 --- a/extensions/installedApps/src/controllers/InstalledAppsController.ts +++ /dev/null @@ -1,73 +0,0 @@ -import type { BaseDatabaseAccessService } from '@heyputer/backend/src/services/database/BaseDatabaseAccessService.js'; -import { Request, Response } from 'express'; -import type { } from '../../../api.js'; - -const { Controller, Get, ExtensionController, HttpError } = extension.import('extensionController'); - -const getAppIconUrl = extension.import('core').util.helpers.get_app_icon_url; - -@Controller('/installedApps') -export class InstalledAppsController extends ExtensionController { - - static ALLOWED_ORDER_BY = ['id', 'name', 'uid', 'title', 'installed_at']; - static ORDER_BY_FIELD_MAP: Record = { - id: 'apps.id', - name: 'apps.name', - uid: 'apps.uid', - title: 'apps.title', - installed_at: 'installed_at', - }; - #db: BaseDatabaseAccessService; - constructor (db: BaseDatabaseAccessService) { - super(); - this.#db = db; - } - - @Get('/', { subdomain: 'api' }) - async getInstalledApps (req: Request, res: Response): Promise { - const actor = req.actor; - if ( ! actor ) { - throw new HttpError(401, 'actor not found in context'); - } - if ( actor.type.app ) { - throw new HttpError(403, 'Apps are not allowed to access this resource'); - } - req.query.orderBy ??= 'installed_at'; - if ( ! InstalledAppsController.ALLOWED_ORDER_BY.includes(req.query.orderBy) ) { - throw new HttpError(400, `Invalid orderBy field. Allowed fields are: ${InstalledAppsController.ALLOWED_ORDER_BY.join(', ')}`); - } - - const page = Math.max(req.query.page || 1, 1); - const limit = Math.min(Math.max(req.query.limit || 100, 1), 100); - const offset = (page - 1) * limit; - const orderByField = InstalledAppsController.ORDER_BY_FIELD_MAP[req.query.orderBy]; - const sortDirection = req.query.desc ? 'DESC' : 'ASC'; - - const installedApps = await this.#db.read( - `SELECT - apps.name, - apps.uid, - apps.title, - apps.description, - apps.icon, - MIN(perm.dt) AS installed_at - FROM apps - LEFT JOIN user_to_app_permissions AS perm ON apps.id = perm.app_id - WHERE perm.user_id = ? - GROUP BY apps.id, apps.name, apps.uid, apps.title, apps.description - ORDER BY ${orderByField} ${sortDirection} - LIMIT ? - OFFSET ?`, - [actor.type.user.id, limit, offset], - ) as { - name: string; - uid: string; - title: string; - description: string; - installed_at: Date; - last_opened: Date | null; - }[]; - - res.send(installedApps.map((app) => ({ ...app, iconUrl: getAppIconUrl(app) }))); - } -} diff --git a/extensions/installedApps/src/index.ts b/extensions/installedApps/src/index.ts deleted file mode 100644 index 859f22a10..000000000 --- a/extensions/installedApps/src/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { InstalledAppsController } from './controllers/InstalledAppsController.js'; - -const installedAppsController = new InstalledAppsController(extension.import('data').db); - -installedAppsController.registerRoutes(); \ No newline at end of file diff --git a/extensions/installedApps/tsconfig.json b/extensions/installedApps/tsconfig.json deleted file mode 100644 index c9cbd48a9..000000000 --- a/extensions/installedApps/tsconfig.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2024", - "module": "nodenext", - "moduleResolution": "nodenext", - "strict": true, - "forceConsistentCasingInFileNames": true, - "skipLibCheck": true, - "sourceMap": true, - "noEmitOnError": true, - "noImplicitAny": false, - "allowJs": true, - "checkJs": false, - }, - "include": [ - "./**/*.ts", - "./**/*.d.ts" - ], - "exclude": [ - "**/*.test.ts", - "**/*.spec.ts", - "**/test/**", - "**/tests/**", - "node_modules", - "dist", - "*.js" - ] -} \ No newline at end of file diff --git a/extensions/jsconfig.json b/extensions/jsconfig.json deleted file mode 100644 index c04279b51..000000000 --- a/extensions/jsconfig.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2024", - "module": "node16", - "moduleResolution": "node16", - "rootDir": ".", - "paths": { - "../src/*": [ - "../src/*" - ] - }, - "allowJs": true, - "checkJs": true - }, - "include": [ - "./**/*.js", - "./**/*.d.ts" -, "../src/backend/src/deprecated/filesystem/PuterS3StorageStrategy.js" ] -} \ No newline at end of file diff --git a/extensions/legacyFileSystem/PuterFSProvider.js b/extensions/legacyFileSystem/PuterFSProvider.js deleted file mode 100644 index 8fe9d65ed..000000000 --- a/extensions/legacyFileSystem/PuterFSProvider.js +++ /dev/null @@ -1,1098 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const STUCK_STATUS_TIMEOUT = 10 * 1000; -const STUCK_ALARM_TIMEOUT = 20 * 1000; - -// Temporary limit -const MAX_DIRECTORY_DEPTH = 35; - -import crypto from 'node:crypto'; -import path_ from 'node:path'; -import { v4 as uuidv4 } from 'uuid'; - -const { db } = extension.import('data'); - -const svc_metering = extension.import('service:meteringService'); -const svc_fs = extension.import('service:filesystem'); -const { stuck_detector_stream, hashing_stream } = extension.import('core').util.streamutil; - -// TODO: filesystem providers should not need to call EventService -const svc_event = extension.import('service:event'); - -// TODO: filesystem providers REALLY SHOULD NOT implement ACL logic! -const svc_acl = extension.import('service:acl'); - -// TODO: these services ought to be part of this extension -const svc_size = extension.import('service:sizeService'); -const svc_resource = extension.import('service:resourceService'); - -// TODO: depending on mountpoint service will not be necessary -// once the storage provider is moved to this extension -const svc_mountpoint = extension.import('service:mountpoint'); - -const { - APIError, - Actor, - Context, - UserActorType, - TDetachable, - MultiDetachable, -} = extension.import('core'); - -const { - get_user, -} = extension.import('core').util.helpers; - -const { - ParallelTasks, - getTracer, -} = extension.import('core').util.otelutil; - -const { - TYPE_DIRECTORY, -} = extension.import('core').fs; - -const { - NodeChildSelector, - NodeUIDSelector, - NodeInternalIDSelector, - NodeRawEntrySelector, -} = extension.import('core').fs.selectors; - -const { - FSNodeContext, - capabilities, -} = extension.import('fs'); - -const { - // MODE_READ, - MODE_WRITE, -} = extension.import('fs').lock; - -// ^ Yep I know, import('fs') and import('core').fs is confusing and -// redundant... this will be cleaned up as the new API is developed - -const { - // MODE_READ, - RESOURCE_STATUS_PENDING_CREATE, -} = extension.import('fs').resource; - -const { - UploadProgressTracker, -} = extension.import('fs').util; - -export default class PuterFSProvider { - constructor ({ fsEntryController, storageController }) { - this.fsEntryController = fsEntryController; - this.storageController = storageController; - this.name = 'puterfs'; - } - - // #region depth limit helpers - /** - * Number of path segments (directory depth). Root or empty path = 0. - * @param {string} path - * @returns {number} - */ - #pathDepth (path) { - if ( !path || typeof path !== 'string' ) return 0; - return path_.normalize(path).split(path_.sep).filter(Boolean).length; - } - - /** - * Max relative depth of the source tree (0 for a file, 1+ for directory tree). - * Used to enforce MAX_DIRECTORY_DEPTH when moving or copying. - * @param {FSNode} node - * @returns {Promise} - */ - async #getSourceTreeMaxRelativeDepth (node) { - await node.fetchEntry(); - if ( ! node.entry.is_dir ) return 0; - const child_uuids = await this.fsEntryController.fast_get_direct_descendants(await node.get('uid')); - let max = 0; - for ( const child_uuid of child_uuids ) { - const child_node = await svc_fs.node(new NodeUIDSelector(child_uuid)); - const child_relative = 1 + await this.#getSourceTreeMaxRelativeDepth(child_node); - max = Math.max(max, child_relative); - } - return max; - } - - /** - * Throws if destination depth plus source tree depth would exceed MAX_DIRECTORY_DEPTH. - * @param {number} destinationPathDepth - * @param {FSNode} sourceNode - */ - async #assertDepthLimitForTreeOp (destinationPathDepth, sourceNode) { - const source_relative = await this.#getSourceTreeMaxRelativeDepth(sourceNode); - const max_depth = destinationPathDepth + source_relative; - if ( max_depth > MAX_DIRECTORY_DEPTH ) { - throw APIError.create('directory_depth_limit_exceeded', null, { - limit: MAX_DIRECTORY_DEPTH, - would_be: max_depth, - }); - } - } - // #endregion - - // TODO: should this be a static member instead? - get_capabilities () { - return new Set([ - capabilities.THUMBNAIL, - capabilities.UPDATE_THUMBNAIL, - capabilities.UUID, - capabilities.OPERATION_TRACE, - capabilities.READDIR_UUID_MODE, - capabilities.READDIRSTAT_UUID, - capabilities.PUTER_SHORTCUT, - - capabilities.COPY_TREE, - capabilities.GET_RECURSIVE_SIZE, - - capabilities.READ, - capabilities.WRITE, - capabilities.CASE_SENSITIVE, - capabilities.SYMLINK, - capabilities.TRASH, - ]); - } - - // #region PuterOnly - async update_thumbnail ({ context, node, thumbnail }) { - const { - actor: inputActor, - } = context.values; - const actor = inputActor ?? Context.get('actor'); - - context = context ?? Context.get(); - const services = context.get('services'); - - // TODO: this ACL check should not be here, but there's no LL method yet - // and it's possible we will never implement the thumbnail - // capability for any other filesystem type - - const svc_acl = services.get('acl'); - if ( ! await svc_acl.check(actor, node, 'write') ) { - throw await svc_acl.get_safe_acl_error(actor, node, 'write'); - } - - const uid = await node.get('uid'); - - const entryOp = await this.fsEntryController.update(uid, { - thumbnail, - }); - - (async () => { - await entryOp.awaitDone(); - svc_event.emit('fs.write.file', { - node, - context, - }); - })(); - - return node; - } - - async puter_shortcut ({ parent, name, user, target }) { - const user_id = user?.id ?? Context.get('actor')?.type?.user?.id; - await target.fetchEntry({ thumbnail: true }); - - const ts = Math.round(Date.now() / 1000); - const uid = uuidv4(); - - svc_resource.register({ - uid, - status: RESOURCE_STATUS_PENDING_CREATE, - }); - - const raw_fsentry = { - is_shortcut: 1, - shortcut_to: target.mysql_id, - is_dir: target.entry.is_dir, - thumbnail: target.entry.thumbnail, - uuid: uid, - parent_uid: await parent.get('uid'), - path: path_.join(await parent.get('path'), name), - user_id: user_id, - name, - created: ts, - updated: ts, - modified: ts, - immutable: false, - }; - - const entryOp = await this.fsEntryController.insert(raw_fsentry); - - (async () => { - await entryOp.awaitDone(); - svc_resource.free(uid); - })(); - - const node = await svc_fs.node(new NodeUIDSelector(uid)); - - svc_event.emit('fs.create.shortcut', { - node, - context: Context.get(), - }); - - return node; - } - // #endregion - - // #region Optimization - /** - * The readdirstat_uuid operation is only available for filesystem - * immplementations with READDIR_UUID_MODE enabled. This implements - * an optimized readdir operation when the UUID is already known. - * @param {*} param0 - */ - async readdirstat_uuid ({ - uuid, - options = {}, - }) { - const entries = await this.fsEntryController.get_descendants_full(uuid, options); - const nodes = Promise.all(Array.prototype.map.call(entries, raw_entry => { - const node = svc_fs.node(new NodeRawEntrySelector(raw_entry, { - found_thumbnail: options.thumbnail, - })); - node.found = true; // TODO: how is it possible for this to be false? - return node; - })); - return nodes; - }; - // #endregion - - // #region Standard FS - - /** - * Check if a given node exists. - * - * @param {Object} param - * @param {NodeSelector} param.selector - The selector used for checking. - * @returns {Promise} - True if the node exists, false otherwise. - */ - async quick_check ({ - selector, - }) { - // shortcut: has full path - if ( selector?.path ) { - const entry = await this.fsEntryController.findByPath(selector.path); - return Boolean(entry); - } - - // shortcut: has uid - if ( selector?.uid ) { - const entry = await this.fsEntryController.findByUID(selector.uid); - return Boolean(entry); - } - - // shortcut: parent uid + child name - if ( selector instanceof NodeChildSelector && selector.parent instanceof NodeUIDSelector ) { - return await this.fsEntryController.nameExistsUnderParent( - selector.parent.uid, - selector.name, - ); - } - - // shortcut: parent id + child name - if ( selector instanceof NodeChildSelector && selector.parent instanceof NodeInternalIDSelector ) { - return await this.fsEntryController.nameExistsUnderParentID( - selector.parent.id, - selector.name, - ); - } - - return false; - } - - async unlink ({ context, node, options = {} }) { - if ( await node.get('type') === TYPE_DIRECTORY ) { - throw new APIError(409, 'Cannot unlink a directory.'); - } - - await this.#rmnode({ context, node, options }); - } - - async rmdir ({ context, node, options = {} }) { - if ( await node.get('type') !== TYPE_DIRECTORY ) { - throw new APIError(409, 'Cannot rmdir a file.'); - } - - if ( await node.get('immutable') ) { - throw APIError.create('immutable'); - } - - const children = await this.fsEntryController.fast_get_direct_descendants(await node.get('uid')); - - if ( children.length > 0 && !options.ignore_not_empty ) { - throw APIError.create('not_empty'); - } - - await this.#rmnode({ context, node, options }); - } - - /** - * Create a new directory. - * - * @param {Object} param - * @param {Context} param.context - * @param {FSNode} param.parent - * @param {string} param.name - * @param {boolean} param.immutable - * @returns {Promise} - */ - async mkdir ({ context, parent, name, immutable }) { - const { actor, thumbnail } = context.values; - - const ts = Math.round(Date.now() / 1000); - const uid = uuidv4(); - - const existing = await svc_fs.node(new NodeChildSelector(parent.selector, name)); - - if ( await existing.exists() ) { - throw APIError.create('item_with_same_name_exists', null, { - entry_name: name, - }); - } - - if ( ! await parent.exists() ) { - throw APIError.create('subject_does_not_exist'); - } - - const new_path = path_.join(await parent.get('path'), name); - if ( this.#pathDepth(new_path) > MAX_DIRECTORY_DEPTH ) { - throw APIError.create('directory_depth_limit_exceeded', null, { - limit: MAX_DIRECTORY_DEPTH, - would_be: this.#pathDepth(new_path), - }); - } - - svc_resource.register({ - uid, - status: RESOURCE_STATUS_PENDING_CREATE, - }); - - const raw_fsentry = { - is_dir: 1, - uuid: uid, - parent_uid: await parent.get('uid'), - path: path_.join(await parent.get('path'), name), - user_id: actor.type.user.id, - name, - created: ts, - accessed: ts, - modified: ts, - immutable: immutable ?? false, - ...(thumbnail ? { - thumbnail: thumbnail, - } : {}), - }; - - const entryOp = await this.fsEntryController.insert(raw_fsentry); - - await entryOp.awaitDone(); - svc_resource.free(uid); - - const node = await svc_fs.node(new NodeUIDSelector(uid)); - - svc_event.emit('fs.create.directory', { - node, - context: Context.get(), - }); - - return node; - } - - async read ({ context, node, version_id, range }) { - const svc_mountpoint = context.get('services').get('mountpoint'); - const storage = svc_mountpoint.get_storage(this.constructor.name); - const location = await node.get('s3:location') ?? {}; - const stream = (await storage.create_read_stream(await node.get('uid'), { - // TODO: fs:decouple-s3 - bucket: location.bucket, - bucket_region: location.bucket_region, - version_id, - key: location.key, - memory_file: node.entry, - ...(range ? { range } : {}), - })); - return stream; - } - - async stat ({ - selector, - options, - controls, - node, - }) { - // For Puter FS nodes, we assume we will obtain all properties from - // fsEntryController, except for 'thumbnail' unless it's - // explicitly requested. - - if ( options.tracer == null ) { - options.tracer = getTracer(); - } - - if ( options.op ) { - options.trace_options = { - parent: options.op.span, - }; - } - - let entry; - - // stat doesn't work with RawEntrySelector - if ( selector instanceof NodeRawEntrySelector ) { - selector = new NodeUIDSelector(node.uid); - } - - await new Promise (rslv => { - const detachables = new MultiDetachable(); - - const callback = (_resolver) => { - detachables.as(TDetachable).detach(); - rslv(); - }; - - // either the resource is free - { - // no detachale because waitForResource returns a - // Promise that will be resolved when the resource - // is free no matter what, and then it will be - // garbage collected. - svc_resource.waitForResource(selector).then(callback.bind(null, 'resourceService')); - } - - // or pending information about the resource - // becomes available - { - // detachable is needed here because waitForEntry keeps - // a map of listeners in memory, and this event may - // never occur. If this never occurs, waitForResource - // is guaranteed to resolve eventually, and then this - // detachable will be detached by `callback` so the - // listener can be garbage collected. - const det = this.fsEntryController.waitForEntry(node, callback.bind(null, 'fsEntryService')); - if ( det ) detachables.add(det); - } - }); - - const maybe_uid = node.uid; - if ( svc_resource.getResourceInfo(maybe_uid) ) { - entry = await this.fsEntryController.get(maybe_uid, options); - controls.log.debug('got an entry from the future'); - } else { - entry = await this.fsEntryController.find(selector, options); - } - - if ( ! entry ) { - if ( this.log_fsentriesNotFound ) { - controls.log.warn(`entry not found: ${selector.describe(true)}`); - } - } - - if ( entry === null || typeof entry !== 'object' ) { - return null; - } - - if ( entry.id ) { - controls.provide_selector(new NodeInternalIDSelector('mysql', entry.id, { - source: 'FSNodeContext optimization', - })); - } - - return entry; - } - - async copy_tree ({ context, source, parent, target_name }) { - // Context - const actor = (context ?? Context).get('actor'); - const user = actor.type.user; - - const tracer = getTracer(); - const uuid = uuidv4(); - const timestamp = Math.round(Date.now() / 1000); - await parent.fetchEntry(); - await source.fetchEntry({ thumbnail: true }); - - const destination_path = path_.join(await parent.get('path'), target_name); - await this.#assertDepthLimitForTreeOp(this.#pathDepth(destination_path), source); - - // New filesystem entry - const raw_fsentry = { - uuid, - is_dir: source.entry.is_dir, - ...(source.entry.is_shortcut ? { - is_shortcut: source.entry.is_shortcut, - shortcut_to: source.entry.shortcut_to, - } : {}), - parent_uid: parent.uid, - name: target_name, - created: timestamp, - modified: timestamp, - - path: path_.join(await parent.get('path'), target_name), - - // if property exists but the value is undefined, - // it will still be included in the INSERT, causing - // an error - ...(source.entry.thumbnail ? - { thumbnail: source.entry.thumbnail } : {}), - - user_id: user.id, - }; - - svc_event.emit('fs.pending.file', { - fsentry: FSNodeContext.sanitize_pending_entry_info(raw_fsentry), - context: context, - }); - - if ( await source.get('has-s3') ) { - Object.assign(raw_fsentry, { - size: source.entry.size, - associated_app_id: source.entry.associated_app_id, - bucket: source.entry.bucket, - bucket_region: source.entry.bucket_region, - }); - - await tracer.startActiveSpan('fs:cp:storage-copy', async span => { - let progress_tracker = new UploadProgressTracker(); - - svc_event.emit('fs.storage.progress.copy', { - upload_tracker: progress_tracker, - context, - meta: { - item_uid: uuid, - item_path: raw_fsentry.path, - }, - }); - - const storage = context.get('storage'); - const state_copy = storage.create_copy(); - await state_copy.run({ - src_node: source, - dst_storage: { - key: uuid, - bucket: raw_fsentry.bucket, - bucket_region: raw_fsentry.bucket_region, - }, - storage_api: { progress_tracker }, - }); - - span.end(); - }); - } - - { - await svc_size.add_node_size(undefined, source, user); - } - - svc_resource.register({ - uid: uuid, - status: RESOURCE_STATUS_PENDING_CREATE, - }); - - const entryOp = await this.fsEntryController.insert(raw_fsentry); - - let node; - - const tasks = new ParallelTasks({ tracer, max: 4 }); - await context.arun('fs:cp:parallel-portion', async () => { - // Add child copy tasks if this is a directory - if ( source.entry.is_dir ) { - const children = await this.fsEntryController.fast_get_direct_descendants(source.uid); - for ( const child_uuid of children ) { - tasks.add('fs:cp:copy-child', async () => { - const child_node = await svc_fs.node(new NodeUIDSelector(child_uuid)); - const child_name = await child_node.get('name'); - - await this.copy_tree({ - context, - source: await svc_fs.node(new NodeUIDSelector(child_uuid)), - parent: await svc_fs.node(new NodeUIDSelector(uuid)), - target_name: child_name, - }); - }); - } - } - - // Add task to await entry - tasks.add('fs:cp:entry-op', async () => { - await entryOp.awaitDone(); - svc_resource.free(uuid); - const copy_fsNode = await svc_fs.node(new NodeUIDSelector(uuid)); - copy_fsNode.entry = raw_fsentry; - copy_fsNode.found = true; - copy_fsNode.path = raw_fsentry.path; - - node = copy_fsNode; - - svc_event.emit('fs.create.file', { - node, - context, - }); - }, { force: true }); - - await tasks.awaitAll(); - }); - - node = node || await svc_fs.node(new NodeUIDSelector(uuid)); - - // TODO: What event do we emit? How do we know if we're overwriting? - return node; - } - - async move ({ context, node, new_parent, new_name, metadata }) { - const old_path = await node.get('path'); - const new_path = path_.join(await new_parent.get('path'), new_name); - - await this.#assertDepthLimitForTreeOp(this.#pathDepth(new_path), node); - - const op_update = await this.fsEntryController.update(node.uid, { - ...( - await node.get('parent_uid') !== await new_parent.get('uid') - ? { parent_uid: await new_parent.get('uid') } - : {} - ), - path: new_path, - name: new_name, - ...(metadata ? { metadata } : {}), - }); - - node.entry.name = new_name; - node.entry.path = new_path; - - // NOTE: this is a safeguard passed to update_child_paths to isolate - // changes to the owner's directory tree, ut this may need to be - // removed in the future. - const user_id = await node.get('user_id'); - - await op_update.awaitDone(); - - await svc_fs.update_child_paths(old_path, node.entry.path, user_id); - - const promises = []; - promises.push(svc_event.emit('fs.move.file', { - context, - moved: node, - old_path, - })); - promises.push(svc_event.emit('fs.rename', { - uid: await node.get('uid'), - new_name, - })); - - return node; - } - - async readdir ({ node }) { - const uuid = await node.get('uid'); - const child_uuids = await this.fsEntryController.fast_get_direct_descendants(uuid); - return child_uuids; - } - - async directory_has_name ({ parent, name }) { - const uid = await parent.get('uid'); - - let check_dupe = await db.read( - 'SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1', - [uid, name], - ); - - return !!check_dupe[0]; - } - - /** - * Write a new file to the filesystem. Throws an error if the destination - * already exists. - * - * @param {Object} param - * @param {Context} param.context - * @param {FSNode} param.parent: The parent directory of the file. - * @param {string} param.name: The name of the file. - * @param {File} param.file: The file to write. - * @returns {Promise} - */ - async write_new ({ context, parent, name, file }) { - const { - tmp, fsentry_tmp, message, actor: inputActor, app_id, - } = context.values; - const actor = inputActor ?? Context.get('actor'); - - const uid = uuidv4(); - - // determine bucket region - let bucket_region = global_config.s3_region ?? global_config.region ?? 'us-west-2'; - let bucket = global_config.s3_bucket ?? 'puter-local'; - - if ( ! await svc_acl.check(actor, parent, 'write') ) { - throw await svc_acl.get_safe_acl_error(actor, parent, 'write'); - } - - const storage_resp = await this.#storage_upload({ - uuid: uid, - bucket, - bucket_region, - file, - tmp: { - ...tmp, - path: path_.join(await parent.get('path'), name), - }, - }); - - fsentry_tmp.thumbnail = await fsentry_tmp.thumbnail_promise; - delete fsentry_tmp.thumbnail_promise; - - const timestamp = Math.round(Date.now() / 1000); - const raw_fsentry = { - uuid: uid, - is_dir: 0, - user_id: actor.type.user.id, - created: timestamp, - accessed: timestamp, - modified: timestamp, - parent_uid: await parent.get('uid'), - name, - size: file.size, - path: path_.join(await parent.get('path'), name), - ...fsentry_tmp, - bucket_region, - bucket, - associated_app_id: app_id ?? null, - }; - - svc_event.emit('fs.pending.file', { - fsentry: FSNodeContext.sanitize_pending_entry_info(raw_fsentry), - context, - }); - - svc_resource.register({ - uid, - status: RESOURCE_STATUS_PENDING_CREATE, - }); - - const filesize = file.size; - svc_size.change_usage(actor.type.user.id, filesize); - - // Meter ingress - const ownerId = await parent.get('user_id'); - const ownerActor = new Actor({ - type: new UserActorType({ - user: await get_user({ id: ownerId }), - }), - }); - - svc_metering.incrementUsage(ownerActor, 'filesystem:ingress:bytes', filesize); - - const entryOp = await this.fsEntryController.insert(raw_fsentry); - - (async () => { - await entryOp.awaitDone(); - svc_resource.free(uid); - - const new_item_node = await svc_fs.node(new NodeUIDSelector(uid)); - const new_item = await new_item_node.get('entry'); - const store_version_id = storage_resp.VersionId; - if ( store_version_id ) { - // insert version into db - db.write( - 'INSERT INTO `fsentry_versions` (`user_id`, `fsentry_id`, `fsentry_uuid`, `version_id`, `message`, `ts_epoch`) VALUES (?, ?, ?, ?, ?, ?)', - [ - actor.type.user.id, - new_item.id, - new_item.uuid, - store_version_id, - message ?? null, - timestamp, - ], - ); - } - })(); - - const node = await svc_fs.node(new NodeUIDSelector(uid)); - - svc_event.emit('fs.create.file', { - node, - context, - }); - - return node; - } - - /** - * Overwrite an existing file. Throws an error if the destination does not - * exist. - * - * @param {Object} param - * @param {Context} param.context - * @param {FSNodeContext} param.node: The node to write to. - * @param {File} param.file: The file to write. - * @returns {Promise} - */ - async write_overwrite ({ context, node, file }) { - const { - tmp, fsentry_tmp, message, actor: inputActor, - } = context.values; - const actor = inputActor ?? Context.get('actor'); - - if ( ! await svc_acl.check(actor, node, 'write') ) { - throw await svc_acl.get_safe_acl_error(actor, node, 'write'); - } - - const uid = await node.get('uid'); - - const bucket_region = node.entry.bucket_region; - const bucket = node.entry.bucket; - - const state_upload = await this.#storage_upload({ - uuid: node.entry.uuid, - bucket, - bucket_region, - file, - tmp: { - ...tmp, - path: await node.get('path'), - }, - }); - - if ( fsentry_tmp?.thumbnail_promise ) { - fsentry_tmp.thumbnail = await fsentry_tmp.thumbnail_promise; - delete fsentry_tmp.thumbnail_promise; - } - - const ts = Math.round(Date.now() / 1000); - const raw_fsentry_delta = { - modified: ts, - accessed: ts, - size: file.size, - ...fsentry_tmp, - }; - - svc_resource.register({ - uid, - status: RESOURCE_STATUS_PENDING_CREATE, - }); - - const filesize = file.size; - svc_size.change_usage(actor.type.user.id, filesize); - - // Meter ingress - const ownerId = await node.get('user_id'); - const ownerActor = new Actor({ - type: new UserActorType({ - user: await get_user({ id: ownerId }), - }), - }); - svc_metering.incrementUsage(ownerActor, 'filesystem:ingress:bytes', filesize); - - const entryOp = await this.fsEntryController.update(uid, raw_fsentry_delta); - - // depends on fsentry, does not depend on S3 - const entryOpPromise = (async () => { - await entryOp.awaitDone(); - svc_resource.free(uid); - })(); - - (async () => { - await entryOpPromise; - svc_event.emit('fs.write.file', { - node, - context, - }); - })(); - - // TODO (xiaochen): determine if this can be removed, post_insert handler need - // to skip events from other servers (why? 1. current write logic is inside - // the local server 2. broadcast system conduct "fire-and-forget" behavior) - state_upload.post_insert({ - db, user: actor.type.user, node, uid, message, ts, - }); - - return node; - } - - async get_recursive_size ({ node }) { - const uuid = await node.get('uid'); - const cte_query = ` - WITH RECURSIVE descendant_cte AS ( - SELECT uuid, parent_uid, size - FROM fsentries - WHERE parent_uid = ? - - UNION ALL - - SELECT f.uuid, f.parent_uid, f.size - FROM fsentries f - INNER JOIN descendant_cte d - ON f.parent_uid = d.uuid - ) - SELECT SUM(size) AS total_size FROM descendant_cte - `; - const rows = await db.read(cte_query, [uuid]); - return rows[0].total_size; - } - - // #endregion - - // #region internal - - /** - * @param {Object} param - * @param {File} param.file: The file to write. - * @returns - */ - async #storage_upload ({ - uuid, - bucket, - bucket_region, - file, - tmp, - }) { - const storage = svc_mountpoint.get_storage(this.constructor.name); - - bucket ??= global_config.s3_bucket; - bucket_region ??= global_config.s3_region ?? global_config.region; - - let upload_tracker = new UploadProgressTracker(); - - svc_event.emit('fs.storage.upload-progress', { - upload_tracker, - context: Context.get(), - meta: { - item_uid: uuid, - item_path: tmp.path, - }, - }); - - if ( ! file.buffer ) { - let stream = file.stream; - let alarm_timeout = null; - stream = stuck_detector_stream(stream, { - timeout: STUCK_STATUS_TIMEOUT, - on_stuck: () => { - console.warn('Upload stream stuck might be stuck', { - bucket_region, - bucket, - uuid, - }); - alarm_timeout = setTimeout(() => { - extension.errors.report('fs.write.s3-upload', { - message: 'Upload stream stuck for too long', - alarm: true, - extra: { - bucket_region, - bucket, - uuid, - }, - }); - }, STUCK_ALARM_TIMEOUT); - }, - on_unstuck: () => { - clearTimeout(alarm_timeout); - }, - }); - file = { ...file, stream }; - } - - let hashPromise; - if ( file.buffer ) { - const hash = crypto.createHash('sha256'); - hash.update(file.buffer); - hashPromise = Promise.resolve(hash.digest('hex')); - } else { - const hs = hashing_stream(file.stream); - file.stream = hs.stream; - hashPromise = hs.hashPromise; - } - - hashPromise.then(hash => { - svc_event.emit('outer.fs.write-hash', { - hash, uuid, - }); - }); - - const state_upload = storage.create_upload(); - - try { - await this.storageController.upload({ - uid: uuid, - file, - storage_meta: { bucket, bucket_region }, - storage_api: { progress_tracker: upload_tracker }, - }); - } catch (e) { - extension.errors.report('fs.write.storage-upload', { - source: e || new Error('unknown'), - trace: true, - alarm: true, - extra: { - bucket_region, - bucket, - uuid, - }, - }); - throw APIError.create('upload_failed'); - } - - return state_upload; - } - - async #rmnode ({ node, options }) { - // Services - if ( !options.override_immutable && await node.get('immutable') ) { - throw new APIError(403, 'File is immutable.'); - } - - const userId = await node.get('user_id'); - const fileSize = await node.get('size'); - svc_size.change_usage( - userId, - -1 * fileSize, - ); - - const ownerActor = new Actor({ - type: new UserActorType({ - user: await get_user({ id: userId }), - }), - }); - - svc_metering.incrementUsage(ownerActor, 'filesystem:delete:bytes', fileSize); - - const tracer = getTracer(); - const tasks = new ParallelTasks({ tracer, max: 4 }); - - tasks.add('remove-fsentry', async () => { - await this.fsEntryController.delete(await node.get('uid')); - }); - - if ( await node.get('has-s3') ) { - tasks.add('remove-from-s3', async () => { - const storage = Context.get('storage'); - const state_delete = storage.create_delete(); - await state_delete.run({ - node: node, - }); - }); - } - - await tasks.awaitAll(); - } - // #endregion -} diff --git a/extensions/legacyFileSystem/fsentries/BaseOperation.js b/extensions/legacyFileSystem/fsentries/BaseOperation.js deleted file mode 100644 index a59c2df82..000000000 --- a/extensions/legacyFileSystem/fsentries/BaseOperation.js +++ /dev/null @@ -1,31 +0,0 @@ -import { TeePromise } from 'teepromise'; - -export default class BaseOperation { - static STATUS_PENDING = {}; - static STATUS_RUNNING = {}; - static STATUS_DONE = {}; - - /** @type {PromiseLike & { resolve: () => void }} */ - #donePromise; - - constructor () { - this.status_ = this.constructor.STATUS_PENDING; - this.#donePromise = new TeePromise(); - } - get status () { - return this.status_; - } - set status (status) { - this.status_ = status; - if ( status === this.constructor.STATUS_DONE ) { - this.#donePromise.resolve(); - } - } - async awaitDone () { - await this.#donePromise; - } - async onComplete (fn) { - await this.#donePromise; - fn(); - } -} diff --git a/extensions/legacyFileSystem/fsentries/Delete.js b/extensions/legacyFileSystem/fsentries/Delete.js deleted file mode 100644 index a8dcdad91..000000000 --- a/extensions/legacyFileSystem/fsentries/Delete.js +++ /dev/null @@ -1,18 +0,0 @@ -import BaseOperation from './BaseOperation.js'; - -export default class extends BaseOperation { - constructor (uuid) { - super(); - this.uuid = uuid; - } - - getStatement () { - const statement = 'DELETE FROM fsentries WHERE uuid = ? LIMIT 1'; - const values = [this.uuid]; - return { statement, values }; - } - - apply (answer) { - answer.entry = null; - } -} diff --git a/extensions/legacyFileSystem/fsentries/FSEntryController.js b/extensions/legacyFileSystem/fsentries/FSEntryController.js deleted file mode 100644 index 0ce5b3802..000000000 --- a/extensions/legacyFileSystem/fsentries/FSEntryController.js +++ /dev/null @@ -1,610 +0,0 @@ -import { TeePromise } from 'teepromise'; -import BaseOperation from './BaseOperation.js'; -import Delete from './Delete.js'; -import Insert from './Insert.js'; -import Update from './Update.js'; - -const { db } = extension.import('data'); -const svc_params = extension.import('service:params'); - -const { PuterPath } = extension.import('fs'); - -const { - RootNodeSelector, - NodeChildSelector, - NodeUIDSelector, - NodePathSelector, - NodeInternalIDSelector, -} = extension.import('core').fs.selectors; - -export default class FSEntryController { - static CONCERN = 'filesystem'; - - static STATUS_READY = {}; - static STATUS_RUNNING_JOB = {}; - - constructor () { - this.status = FSEntryController.STATUS_READY; - - this.currentState = { - queue: [], - updating_uuids: {}, - }; - this.deferredState = { - queue: [], - updating_uuids: {}, - }; - - this.entryListeners_ = {}; - - this.mkPromiseForQueueSize_(); - - // this list of properties is for read operations - // (originally in FSEntryFetcher) - this.defaultProperties = [ - 'id', - 'associated_app_id', - 'uuid', - 'public_token', - 'bucket', - 'bucket_region', - 'file_request_token', - 'user_id', - 'parent_uid', - 'is_dir', - 'is_public', - 'is_shortcut', - 'is_symlink', - 'symlink_path', - 'shortcut_to', - 'sort_by', - 'sort_order', - 'immutable', - 'name', - 'metadata', - 'modified', - 'created', - 'accessed', - 'size', - 'layout', - 'path', - ]; - - this.subdomainProperties = [ - 'uuid', - 'subdomain', - ]; - } - - init () { - svc_params.createParameters('fsentry-service', [ - { - id: 'max_queue', - description: 'Maximum queue size', - default: 50, - }, - ], this); - - } - - mkPromiseForQueueSize_ () { - this.queueSizePromise = new Promise((resolve, reject) => { - this.queueSizeResolve = resolve; - }); - } - - // #region write operations - async insert (entry) { - const op = new Insert(entry); - await this.enqueue_(op); - return op; - } - - async update (uuid, entry) { - const op = new Update(uuid, entry); - await this.enqueue_(op); - return op; - } - - async delete (uuid) { - const op = new Delete(uuid); - await this.enqueue_(op); - return op; - } - // #endregion - - // #region read operations - async fast_get_descendants (uuid) { - return (await db.read(` - WITH RECURSIVE descendant_cte AS ( - SELECT uuid, parent_uid - FROM fsentries - WHERE parent_uid = ? - - UNION ALL - - SELECT f.uuid, f.parent_uid - FROM fsentries f - INNER JOIN descendant_cte d ON f.parent_uid = d.uuid - ) - SELECT uuid FROM descendant_cte - `, [uuid])).map(x => x.uuid); - } - - async fast_get_direct_descendants (uuid) { - return (uuid === PuterPath.NULL_UUID - ? await db.read('SELECT uuid FROM fsentries WHERE parent_uid IS NULL') - : await db.read( - 'SELECT uuid FROM fsentries WHERE parent_uid = ?', - [uuid], - )).map(x => x.uuid); - } - - waitForEntry (node, callback) { - // *** uncomment to debug slow waits *** - // console.log('ATTEMPT TO WAIT FOR', selector.describe()) - let selector = node.get_selector_of_type(NodeUIDSelector); - if ( selector === null ) { - // console.log(new Error('========')); - return; - } - - const entry_already_enqueued = - Object.prototype.hasOwnProperty.call(this.currentState.updating_uuids, selector.value) || - Object.prototype.hasOwnProperty.call(this.deferredState.updating_uuids, selector.value) ; - - if ( entry_already_enqueued ) { - callback(); - return; - } - - const k = `uid:${selector.value}`; - if ( ! Object.prototype.hasOwnProperty.call(this.entryListeners_, k) ) { - this.entryListeners_[k] = []; - } - - const det = { - detach: () => { - const i = this.entryListeners_[k].indexOf(callback); - if ( i === -1 ) return; - this.entryListeners_[k].splice(i, 1); - if ( this.entryListeners_[k].length === 0 ) { - delete this.entryListeners_[k]; - } - }, - }; - - this.entryListeners_[k].push(callback); - - return det; - } - - async get (uuid, fetch_entry_options) { - const answer = {}; - for ( const op of this.currentState.queue ) { - if ( op.uuid != uuid ) continue; - op.apply(answer); - } - for ( const op of this.deferredState.queue ) { - if ( op.uuid != uuid ) continue; - op.apply(answer); - op.apply(answer); - } - if ( answer.is_diff ) { - const base_entry = await this.find( - new NodeUIDSelector(uuid), - fetch_entry_options, - ); - answer.entry = { ...base_entry, ...answer.entry }; - } - return answer.entry; - } - - /** - * Returns UUIDs of child fsentries under the specified - * parent fsentry - * @param {string} uuid - UUID of parent fsentry - * @returns fsentry[] - */ - async get_descendants (uuid) { - return uuid === PuterPath.NULL_UUID - ? await db.read( - 'SELECT uuid FROM fsentries WHERE parent_uid IS NULL', - [uuid], - ) - : await db.read( - 'SELECT uuid FROM fsentries WHERE parent_uid = ?', - [uuid], - ) - ; - } - - /** - * Returns full fsentry nodes for entries under the specified - * parent fsentry - * @param {string} uuid - UUID of parent fsentry - * @returns fsentry[] - */ - async get_descendants_full (uuid, fetch_entry_options) { - const { thumbnail } = fetch_entry_options; - const columns = `${ - [ - ...this.defaultProperties.map(v => `f.${v}`), - ...this.subdomainProperties - .map(v => `s.${v} AS subdomain_${v}`), - ].join(', ') - }${thumbnail ? ', thumbnail' : ''}`; - const results_with_dupes = uuid === PuterPath.NULL_UUID - ? await db.read( - `SELECT ${columns} FROM fsentries WHERE parent_uid IS NULL`, - [uuid], - ) - : await db.read( - `SELECT ${columns} FROM fsentries AS f ` + - 'LEFT JOIN subdomains AS s ON f.id=s.root_dir_id ' + - 'WHERE parent_uid = ? ORDER BY f.id', - [uuid], - ) - ; - - const byId = new Map(); - for ( const row of results_with_dupes ) { - const id = row.id; - let entry = byId.get(id); - if ( ! entry ) { - entry = { ...row }; - if ( thumbnail ) entry.thumbnail = row.thumbnail; - entry.subdomains = []; - byId.set(id, entry); - } - if ( row.subdomain_uuid != null ) { - entry.subdomains.push({ - uuid: row.subdomain_uuid, - subdomain: row.subdomain_subdomain, - }); - } - } - return Array.from(byId.values()); - } - - async get_recursive_size (uuid) { - const cte_query = ` - WITH RECURSIVE descendant_cte AS ( - SELECT uuid, parent_uid, size - FROM fsentries - WHERE parent_uid = ? - - UNION ALL - - SELECT f.uuid, f.parent_uid, f.size - FROM fsentries f - INNER JOIN descendant_cte d - ON f.parent_uid = d.uuid - ) - SELECT SUM(size) AS total_size FROM descendant_cte - `; - const rows = await db.read(cte_query, [uuid]); - return rows[0].total_size; - } - - /** - * Finds a filesystem entry using the provided selector. - * @param {Object} selector - The selector object specifying how to find the entry - * @param {Object} fetch_entry_options - Options for fetching the entry - * @returns {Promise} The filesystem entry or null if not found - */ - async find (selector, fetch_entry_options) { - if ( selector instanceof RootNodeSelector ) { - return selector.entry; - } - if ( selector instanceof NodePathSelector ) { - return await this.findByPath(selector.value, fetch_entry_options); - } - if ( selector instanceof NodeUIDSelector ) { - return await this.findByUID(selector.value, fetch_entry_options); - } - if ( selector instanceof NodeInternalIDSelector ) { - return await this.findByID(selector.id, fetch_entry_options); - } - if ( selector instanceof NodeChildSelector ) { - let id; - - if ( selector.parent instanceof RootNodeSelector ) { - id = await this.findNameInRoot(selector.name); - } else { - const parentEntry = await this.find(selector.parent); - if ( ! parentEntry ) return null; - id = await this.findNameInParent(parentEntry.uuid, selector.name); - } - - if ( id === undefined ) return null; - if ( typeof id !== 'number' ) { - throw new Error( - 'unexpected type for id value', - typeof id, - id, - ); - } - return this.find(new NodeInternalIDSelector('mysql', id)); - } - } - - /** - * Finds a filesystem entry by its UUID. - * @param {string} uuid - The UUID of the entry to find - * @param {Object} fetch_entry_options - Options including thumbnail flag - * @returns {Promise} The filesystem entry or undefined if not found - */ - async findByUID (uuid, fetch_entry_options = {}) { - const { thumbnail } = fetch_entry_options; - - let fsentry = await db.tryHardRead( - `SELECT ${ - this.defaultProperties.join(', ') - }${thumbnail ? ', thumbnail' : '' - } FROM fsentries WHERE uuid = ? LIMIT 1`, - [uuid], - ); - - return fsentry[0]; - } - - /** - * Finds a filesystem entry by its internal database ID. - * @param {number} id - The internal ID of the entry to find - * @param {Object} fetch_entry_options - Options including thumbnail flag - * @returns {Promise} The filesystem entry or undefined if not found - */ - async findByID (id, fetch_entry_options = {}) { - const { thumbnail } = fetch_entry_options; - - let fsentry = await db.tryHardRead( - `SELECT ${ - this.defaultProperties.join(', ') - }${thumbnail ? ', thumbnail' : '' - } FROM fsentries WHERE id = ? LIMIT 1`, - [id], - ); - - return fsentry[0]; - } - - /** - * Finds a filesystem entry by its full path. - * @param {string} path - The full path of the entry to find - * @param {Object} fetch_entry_options - Options including thumbnail flag and tracer - * @returns {Promise} The filesystem entry or false if not found - */ - async findByPath (path, fetch_entry_options = {}) { - const { thumbnail } = fetch_entry_options; - - if ( path === '/' ) { - return this.find(new RootNodeSelector()); - } - - const parts = path.split('/').filter(path => path !== ''); - if ( parts.length === 0 ) { - // TODO: invalid path; this should be an error - return false; - } - - // TODO: use a closure table for more efficient path resolving - let parent_uid = null; - let result; - - const resultColsSql = this.defaultProperties.join(', ') + - (thumbnail ? ', thumbnail' : ''); - - result = await db.read( - `SELECT ${ resultColsSql - } FROM fsentries WHERE path=? LIMIT 1`, - [path], - ); - - // using knex instead - - if ( result[0] ) return result[0]; - - const loop = async () => { - for ( let i = 0 ; i < parts.length ; i++ ) { - const part = parts[i]; - const isLast = i == parts.length - 1; - const colsSql = isLast ? resultColsSql : 'uuid'; - if ( parent_uid === null ) { - result = await db.read( - `SELECT ${ colsSql - } FROM fsentries WHERE parent_uid IS NULL AND name=? LIMIT 1`, - [part], - ); - } else { - result = await db.read( - `SELECT ${ colsSql - } FROM fsentries WHERE parent_uid=? AND name=? LIMIT 1`, - [parent_uid, part], - ); - } - - if ( ! result[0] ) return false; - parent_uid = result[0].uuid; - } - }; - - if ( fetch_entry_options.tracer ) { - const tracer = fetch_entry_options.tracer; - const options = fetch_entry_options.trace_options; - await tracer.startActiveSpan( - 'fs:sql:findByPath', - ...(options ? [options] : []), - async span => { - await loop(); - span.end(); - }, - ); - } else { - await loop(); - } - - return result[0]; - } - - /** - * Finds the ID of a child entry with the given name in the root directory. - * @param {string} name - The name of the child entry to find - * @returns {Promise} The ID of the child entry or undefined if not found - */ - async findNameInRoot (name) { - let child_id = await db.read( - 'SELECT `id` FROM `fsentries` WHERE `parent_uid` IS NULL AND name = ? LIMIT 1', - [name], - ); - return child_id[0]?.id; - } - - /** - * Finds the ID of a child entry with the given name under a specific parent. - * @param {string} parent_uid - The UUID of the parent directory - * @param {string} name - The name of the child entry to find - * @returns {Promise} The ID of the child entry or undefined if not found - */ - async findNameInParent (parent_uid, name) { - let child_id = await db.read( - 'SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1', - [parent_uid, name], - ); - return child_id[0]?.id; - } - - /** - * Checks if an entry with the given name exists under a specific parent. - * @param {string} parent_uid - The UUID of the parent directory - * @param {string} name - The name to check for - * @returns {Promise} True if the name exists under the parent, false otherwise - */ - async nameExistsUnderParent (parent_uid, name) { - let check_dupe = await db.read( - 'SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1', - [parent_uid, name], - ); - return !!check_dupe[0]; - } - - /** - * Checks if an entry with the given name exists under a parent specified by ID. - * @param {number} parent_id - The internal ID of the parent directory - * @param {string} name - The name to check for - * @returns {Promise} True if the name exists under the parent, false otherwise - */ - async nameExistsUnderParentID (parent_id, name) { - const parent = await this.findByID(parent_id); - if ( ! parent ) { - return false; - } - return this.nameExistsUnderParent(parent.uuid, name); - } - // #endregion - - // #region queue logic - async enqueue_ (op) { - const tp = new TeePromise(); - while ( - this.currentState.queue.length > this.max_queue || - this.deferredState.queue.length > this.max_queue - ) { - await this.queueSizePromise; - } - - if ( ! (op instanceof BaseOperation) ) { - throw new Error('Invalid operation'); - } - - const state = this.status === FSEntryController.STATUS_READY ? - this.currentState : this.deferredState; - - if ( ! Object.prototype.hasOwnProperty.call(state.updating_uuids, op.uuid) ) { - state.updating_uuids[op.uuid] = []; - } - state.updating_uuids[op.uuid].push(state.queue.length); - - state.queue.push(op); - - // DRY: same pattern as FSOperationContext:provideValue - // DRY: same pattern as FSOperationContext:rejectValue - if ( Object.prototype.hasOwnProperty.call(this.entryListeners_, op.uuid) ) { - const listeners = this.entryListeners_[op.uuid]; - - delete this.entryListeners_[op.uuid]; - - for ( const lis of listeners ) lis(); - } - - this.checkShouldExec_(); - - await op.awaitDone(); - } - - checkShouldExec_ () { - if ( this.status !== FSEntryController.STATUS_READY ) return; - if ( this.currentState.queue.length === 0 ) return; - this.exec_(); - } - - async exec_ () { - if ( this.status !== FSEntryController.STATUS_READY ) { - throw new Error('Duplicate exec_ call'); - } - - const queue = this.currentState.queue; - - this.status = FSEntryController.STATUS_RUNNING_JOB; - - // const conn = await db_primary.promise().getConnection(); - // await conn.beginTransaction(); - - for ( const op of queue ) { - op.status = op.constructor.STATUS_RUNNING; - // await conn.execute(stmt, values); - } - - // await conn.commit(); - // conn.release(); - - // const stmtAndVals = queue.map(op => op.getStatementAndValues()); - // const stmts = stmtAndVals.map(x => x.stmt).join('; '); - // const vals = stmtAndVals.reduce((acc, x) => acc.concat(x.values), []); - - // *** uncomment to debug batch queries *** - // this.log.debug({ stmts, vals }); - // console.log('<<========================'); - // console.log({ stmts, vals }); - // console.log('>>========================'); - - // this.log.debug('array?', Array.isArray(vals)) - - await db.batch_write(queue.map(op => op.getStatement())); - - for ( const op of queue ) { - op.status = op.constructor.STATUS_DONE; - } - - this.flipState_(); - this.status = FSEntryController.STATUS_READY; - - for ( const op of queue ) { - op.status = op.constructor.STATUS_DONE; - } - - this.checkShouldExec_(); - } - - flipState_ () { - this.currentState = this.deferredState; - this.deferredState = { - queue: [], - updating_uuids: {}, - }; - const queueSizeResolve = this.queueSizeResolve; - this.mkPromiseForQueueSize_(); - queueSizeResolve(); - } - // #endregion -} diff --git a/extensions/legacyFileSystem/fsentries/Insert.js b/extensions/legacyFileSystem/fsentries/Insert.js deleted file mode 100644 index f072b2d78..000000000 --- a/extensions/legacyFileSystem/fsentries/Insert.js +++ /dev/null @@ -1,72 +0,0 @@ -import { safeHasOwnProperty } from '../lib/objectfn.js'; -import BaseOperation from './BaseOperation.js'; - -export default class extends BaseOperation { - static requiredForCreate = [ - 'uuid', - 'parent_uid', - ]; - - static allowedForCreate = [ - ...this.requiredForCreate, - 'name', - 'user_id', - 'is_dir', - 'created', - 'modified', - 'immutable', - 'shortcut_to', - 'is_shortcut', - 'metadata', - 'bucket', - 'bucket_region', - 'thumbnail', - 'accessed', - 'size', - 'symlink_path', - 'is_symlink', - 'associated_app_id', - 'path', - ]; - - constructor (entry) { - super(); - const requiredForCreate = this.constructor.requiredForCreate; - const allowedForCreate = this.constructor.allowedForCreate; - - { - const sanitized_entry = {}; - for ( const k of allowedForCreate ) { - if ( safeHasOwnProperty(entry, k) ) { - sanitized_entry[k] = entry[k]; - } - } - entry = sanitized_entry; - } - - for ( const k of requiredForCreate ) { - if ( ! safeHasOwnProperty(entry, k) ) { - throw new Error(`Missing required property: ${k}`); - } - } - - this.entry = entry; - } - - getStatement () { - const fields = Object.keys(this.entry); - const statement = 'INSERT INTO fsentries ' + - `(${fields.join(', ')}) ` + - `VALUES (${fields.map(() => '?').join(', ')})`; - const values = fields.map(k => this.entry[k]); - return { statement, values }; - } - - apply (answer) { - answer.entry = { ...this.entry }; - } - - get uuid () { - return this.entry.uuid; - } -}; diff --git a/extensions/legacyFileSystem/fsentries/Update.js b/extensions/legacyFileSystem/fsentries/Update.js deleted file mode 100644 index 6b778705b..000000000 --- a/extensions/legacyFileSystem/fsentries/Update.js +++ /dev/null @@ -1,52 +0,0 @@ -import { safeHasOwnProperty } from '../lib/objectfn.js'; -import BaseOperation from './BaseOperation.js'; - -export default class extends BaseOperation { - static allowedForUpdate = [ - 'name', - 'parent_uid', - 'user_id', - 'modified', - 'shortcut_to', - 'metadata', - 'thumbnail', - 'size', - 'path', - ]; - - constructor (uuid, entry) { - super(); - const allowedForUpdate = this.constructor.allowedForUpdate; - - { - const sanitized_entry = {}; - for ( const k of allowedForUpdate ) { - if ( safeHasOwnProperty(entry, k) ) { - sanitized_entry[k] = entry[k]; - } - } - entry = sanitized_entry; - } - - this.uuid = uuid; - this.entry = entry; - } - - getStatement () { - const fields = Object.keys(this.entry); - const statement = 'UPDATE fsentries SET ' + - `${fields.map(k => `${k} = ?`).join(', ')} ` + - 'WHERE uuid = ? LIMIT 1'; - const values = fields.map(k => this.entry[k]); - values.push(this.uuid); - return { statement, values }; - } - - apply (answer) { - if ( ! answer.entry ) { - answer.is_diff = true; - answer.entry = {}; - } - Object.assign(answer.entry, this.entry); - } -}; diff --git a/extensions/legacyFileSystem/lib/objectfn.js b/extensions/legacyFileSystem/lib/objectfn.js deleted file mode 100644 index f32f7dcab..000000000 --- a/extensions/legacyFileSystem/lib/objectfn.js +++ /dev/null @@ -1,16 +0,0 @@ -/** - * Instead of `myObject.hasOwnProperty(k)`, always write: - * `safeHasOwnProperty(myObject, k)`. - * - * This is a less verbose way to call `Object.prototype.hasOwnProperty.call`. - * This prevents unexpected behavior when `hasOwnProperty` is overridden, - * which is especially possible for objects parsed from user-sent JSON. - * - * explanation: https://eslint.org/docs/latest/rules/no-prototype-builtins - * @param {*} o - * @param {...any} a - * @returns - */ -export const safeHasOwnProperty = (o, ...a) => { - return Object.prototype.hasOwnProperty.call(o, ...a); -}; diff --git a/extensions/legacyFileSystem/main.js b/extensions/legacyFileSystem/main.js deleted file mode 100644 index f35f7bd44..000000000 --- a/extensions/legacyFileSystem/main.js +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import FSEntryController from './fsentries/FSEntryController.js'; -import PuterFSProvider from './PuterFSProvider.js'; -import ProxyStorageController from './storage/ProxyStorageController.js'; -import S3StorageController from './storage/S3StorageController.js'; - -const svc_event = extension.import('service:event'); - -const fsEntryController = new FSEntryController(); -const storageController = new ProxyStorageController(); - -extension.on('init', async () => { - fsEntryController.init(); - - // Keep track of possible storage strategies for puterfs here - let defaultStorage = 'S3'; - const storageStrategies = { - 'S3': new S3StorageController(), - }; - - // Emit the "create storage strategies" event - const event = { - createStorageStrategy (name, implementation) { - storageStrategies[name] = implementation; - if ( implementation === undefined ) { - throw new Error('createStorageStrategy was called wrong'); - } - if ( implementation.forceDefault ) { - defaultStorage = name; - } - }, - }; - // Awaiting the event ensures all the storage strategies are registered - await svc_event.emit('puterfs.storage.create', event); - - let configuredStorage = defaultStorage; - if ( config.storage ) configuredStorage = config.storage; - - // Not we can select the configured strategy - const storageToUse = storageStrategies[configuredStorage]; - storageController.setDelegate(storageToUse); - - // The StorageController may need to await some asynchronous operations - // before it's ready to be used. - await storageController.init(); - -}); - -extension.on('create.filesystem-types', event => { - const fsProvider = new PuterFSProvider({ - fsEntryController, - storageController, - }); - event.createFilesystemType('puterfs', { - mount ({ path }) { - return fsProvider; - }, - }); -}); diff --git a/extensions/legacyFileSystem/package-lock.json b/extensions/legacyFileSystem/package-lock.json deleted file mode 100644 index c59189723..000000000 --- a/extensions/legacyFileSystem/package-lock.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "puterfs", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "dependencies": { - "teepromise": "^0.1.1", - "uuid": "^13.0.0" - } - }, - "node_modules/teepromise": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/teepromise/-/teepromise-0.1.1.tgz", - "integrity": "sha512-BS++sfQYjtOdPvBCb3sd0mNYfPcZKFjSx1yA85Yz/BAAQ3jyZAINd5iB7p70Z8D0Q4XElRwKaa4/lPEP4EHyiw==", - "license": "MIT" - }, - "node_modules/uuid": { - "version": "13.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.0.tgz", - "integrity": "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist-node/bin/uuid" - } - } - } -} diff --git a/extensions/legacyFileSystem/package.json b/extensions/legacyFileSystem/package.json deleted file mode 100644 index 6f920af14..000000000 --- a/extensions/legacyFileSystem/package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "main": "main.js", - "type": "module", - "dependencies": { - "teepromise": "^0.1.1", - "uuid": "^13.0.0", - "@aws-sdk/client-s3": "^3.1021.0" - } -} diff --git a/extensions/legacyFileSystem/storage/ProxyStorageController.js b/extensions/legacyFileSystem/storage/ProxyStorageController.js deleted file mode 100644 index 1627c1838..000000000 --- a/extensions/legacyFileSystem/storage/ProxyStorageController.js +++ /dev/null @@ -1,24 +0,0 @@ -export default class { - constructor (delegate) { - this.delegate = delegate ?? null; - } - setDelegate (delegate) { - this.delegate = delegate; - } - - init (...a) { - return this.delegate.init(...a); - } - upload (...a) { - return this.delegate.upload(...a); - } - copy (...a) { - return this.delegate.copy(...a); - } - delete (...a) { - return this.delegate.delete(...a); - } - read (...a) { - return this.delegate.read(...a); - } -} diff --git a/extensions/legacyFileSystem/storage/S3StorageController.js b/extensions/legacyFileSystem/storage/S3StorageController.js deleted file mode 100644 index 0f34be6c3..000000000 --- a/extensions/legacyFileSystem/storage/S3StorageController.js +++ /dev/null @@ -1,397 +0,0 @@ -import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CopyObjectCommand, CreateMultipartUploadCommand, DeleteObjectCommand, GetObjectCommand, PutObjectCommand, UploadPartCommand, UploadPartCopyCommand } from '@aws-sdk/client-s3'; -import { Readable } from 'stream'; -import { TeePromise } from 'teepromise'; - -const { s3ClientProvider } = extension.import('data'); - -const { Context } = extension.import('core'); - -const { - chunk_stream, -} = extension.import('core').util.streamutil; - -const { - simple_retry, -} = extension.import('core').util.retryutil; - -const { - EWMA, -} = extension.import('core').util.opmath; - -export default class S3StorageController { - forceDefault = true; - async init () { - this.clients_ = {}; - this.config = global_config; - - this.global_average_S3_part_time = new EWMA({ - initial: 4000, // average from local testing - alpha: 0.1, - }); - } - - #get_client (region) { - - return s3ClientProvider.get(region); - } - - async upload ({ uid, file, storage_meta, storage_api }) { - const { progress_tracker } = storage_api; - - const { - bucket_region, - bucket, - } = storage_meta; - - const client = this.#get_client(bucket_region); - - if ( file.buffer ) { - const [s3_error, s3_eventual_success, _s3_resp] = await simple_retry(async () => { - const ret = await client.send(new PutObjectCommand({ - Bucket: bucket, - Key: uid, - Body: file.buffer, - })); - progress_tracker.set_total(file.size); - progress_tracker.set(file.size); - return ret; - }, 3, 200); - - if ( ! s3_eventual_success ) { - throw s3_error; - } - - return; // AKA "} else {{" - } - - const [s3_error, s3_eventual_success, _s3_resp] = await simple_retry(async () => { - return await this.#upload_stream({ - bucket_region, - bucket, - key: uid, - stream: file.stream, - on_progress: evt => { - progress_tracker.set_total(file.size); - progress_tracker.set(evt.uploaded); - }, - }); - }, 3, 200); - - if ( ! s3_eventual_success ) { - throw s3_error; - } - } - - async copy ({ src_node, dst_storage, storage_api }) { - const { - progress_tracker, - } = storage_api; - - const src_storage = await src_node.get('s3:location'); - - const size = await src_node.get('size'); - if ( size < 4 * 1000 ** 3 - 100 ) { - const ret = await this.#copy_simple({ - src_key: src_storage.key, - src_bucket: src_storage.bucket, - - dst_key: dst_storage.key, - dst_bucket_region: dst_storage.bucket_region, - dst_bucket: dst_storage.bucket, - }); - progress_tracker.set_total(size); - progress_tracker.set(size); - return ret; - } - - return await this.#copy_multipart({ - src_key: src_storage.key, - src_bucket: src_storage.bucket, - - dst_key: dst_storage.key, - dst_bucket_region: dst_storage.bucket_region, - dst_bucket: dst_storage.bucket, - - size, - - on_progress: evt => { - const x = Context.get(); - progress_tracker.set_total(size); - progress_tracker.set(evt.uploaded); - }, - }); - } - async delete ({ node }) { - const node_storage = await node.get('s3:location'); - - const client = this.#get_client(node_storage.bucket_region); - - return await client.send(new DeleteObjectCommand({ - Bucket: node_storage.bucket, - Key: node_storage.key, - })); - } - async read ({ location, range, version_id }) { - const { bucket_region, bucket, key } = location; - const client = this.#get_client(bucket_region); - - const response = await client.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - ...(range ? { Range: range } : {}), - ...(version_id ? { VersionId: version_id } : {}), - })); - - const stream = Readable.from(response.Body); - - return stream; - } - - async #upload_stream ({ bucket_region, bucket, key, stream, on_progress }) { - const client = this.#get_client(bucket_region); - - const multipart_upload = await client.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - })); - - let ret; // return value - - try { - const part_size = 1024 * 1024 * 5; // 5MB - // - - // get each part while streaming - const chunk_iterator = chunk_stream( - stream, - part_size, - this.global_average_S3_part_time, - ); - let i = 0; - let uploaded_bytes = 0; - let upload_promises = []; - const upload_results = []; - - let tp; - let count_parts_being_uploaded = 0; - - let check_queue; - - let queue_empty_promise = null; - - const upload_part = async part => { - - if ( count_parts_being_uploaded >= 4 ) { - console.log('too many concurrent part uploads; halting'); - tp = new TeePromise(); - await tp; - } - - const part_number = ++i; - - count_parts_being_uploaded++; - - const upload_promise = (async () => { - - const ts_start = Date.now(); - - const [err, success, result] = await simple_retry(async () => { - return await client.send(new UploadPartCommand({ - Bucket: bucket, - Key: key, - PartNumber: part_number, - UploadId: multipart_upload.UploadId, - Body: part, - })); - }, 3, 50); - - if ( err || !success ) { - throw err; - } - - const ts_end = Date.now(); - const elapsed = ts_end - ts_start; - const elapsed_per_part_size = elapsed * (part.length / part_size); - // this.global_average_S3_part_time.put(elapsed); - this.global_average_S3_part_time.put(elapsed_per_part_size); - - uploaded_bytes += part.length; - on_progress({ uploaded: uploaded_bytes }); - - count_parts_being_uploaded--; - if ( tp ) { - const p = tp; - tp = null; - p.resolve(); - } - - check_queue(); - - return result; - })(); - - upload_promises.push(upload_promise); - }; - - const part_queue = []; - - check_queue = () => { - if ( part_queue.length > 0 ) { - const part = part_queue.shift(); - upload_part(part); - if ( part_queue.length == 0 ) { - if ( queue_empty_promise ) { - const p = queue_empty_promise; - queue_empty_promise = null; - p.resolve(); - } - } - } - }; - - for await ( const chunk of chunk_iterator ) { - await upload_part(chunk); - } - - // If the file is empty we still need to upload a part - if ( i === 0 ) { - const upload_promise = (async () => { - const [err, success, result] = await simple_retry(async () => { - return await client.send(new UploadPartCommand({ - Bucket: bucket, - Key: key, - PartNumber: 1, - UploadId: multipart_upload.UploadId, - Body: Buffer.alloc(0), - })); - }, 3, 50); - - if ( err || !success ) { - throw err; - } - - on_progress({ uploaded: uploaded_bytes }); - return result; - })(); - - upload_promises.push(upload_promise); - } - - if ( part_queue.length > 0 ) { - queue_empty_promise = new TeePromise(); - await queue_empty_promise; - } - - const some_results = await Promise.all(upload_promises); - upload_results.push(...some_results); - - try { - // complete the upload - ret = await client.send(new CompleteMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: multipart_upload.UploadId, - MultipartUpload: { - Parts: upload_results.map((_, i) => ({ - PartNumber: i + 1, - ETag: _.ETag, - })), - }, - })); - } catch ( e ) { - console.warn(`catch block: ${e.message}`); - } - } catch ( e ) { - console.error(`error: ${e.message}`); - // abort the upload - await client.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: multipart_upload.UploadId, - })); - - throw e; - } - - return ret; - } - async #copy_simple ({ - dst_bucket_region, - dst_bucket, - src_bucket, - src_key, - dst_key, - }) { - const client = this.#get_client(dst_bucket_region); - - const ret = await client.send(new CopyObjectCommand({ - Bucket: dst_bucket, - Key: dst_key, - CopySource: `${src_bucket}/${src_key}`, - })); - - return ret; - } - - async #copy_multipart ({ - dst_bucket_region, - dst_bucket, - src_bucket, - src_key, - dst_key, - on_progress, - size, - }) { - const client = this.#get_client(dst_bucket_region); - - const multipart_upload = await client.send(new CreateMultipartUploadCommand({ - Bucket: dst_bucket, - Key: dst_key, - })); - - const part_size = 4 * 1024 * 1024 * 1024; // 4GiB - - const results = []; - - let part_number_i = 0; - for ( let byte_start = 0 ; byte_start < size ; byte_start += part_size ) { - const part_number = ++part_number_i; - // byte range is inclusive... WTF? - const byte_end = Math.min(byte_start + part_size, size) - 1; - - const [err, success, result] = await simple_retry(async () => { - const params = { - Bucket: dst_bucket, - Key: dst_key, - PartNumber: part_number, - UploadId: multipart_upload.UploadId, - CopySource: `${src_bucket}/${src_key}`, - CopySourceRange: `bytes=${byte_start}-${byte_end}`, - }; - return await client.send(new UploadPartCopyCommand(params)); - }, 3, 50); - - if ( err || !success ) { - throw err; - } - - results.push(result); - - on_progress({ uploaded: byte_end + 1 }); - } - - const ret = await client.send(new CompleteMultipartUploadCommand({ - Bucket: dst_bucket, - Key: dst_key, - UploadId: multipart_upload.UploadId, - MultipartUpload: { - Parts: results.map((_, i) => ({ - PartNumber: i + 1, - ETag: _.CopyPartResult.ETag, - })), - }, - })); - - return ret; - } -} diff --git a/extensions/metering.ts b/extensions/metering.ts new file mode 100644 index 000000000..9333ea919 --- /dev/null +++ b/extensions/metering.ts @@ -0,0 +1,114 @@ +import { Context } from '@heyputer/backend/src/core'; +import { HttpError } from '@heyputer/backend/src/core/http'; +import { + controllersContainers, + driversContainers, +} from '@heyputer/backend/src/exports'; +import { extension } from '@heyputer/backend/src/extensions'; + +const services = extension.import('service'); +const clients = extension.import('client'); + +// Cached on first request — the underlying cost catalogues are baked into +// driver/controller source so they only change on deploy. +let cachedAllCosts: Record[] | null = null; + +function collectAllCosts(): Record[] { + const all: Record[] = []; + const collect = ( + source: Record, + kind: 'driver' | 'controller', + ) => { + for (const [name, instance] of Object.entries(source)) { + const fn = ( + instance as { + getReportedCosts?: () => Record[]; + } + )?.getReportedCosts; + if (typeof fn !== 'function') continue; + try { + const entries = fn.call(instance); + if (!Array.isArray(entries)) continue; + for (const entry of entries) { + all.push({ ...entry, registry: kind, registryKey: name }); + } + } catch (e) { + console.warn( + `[metering] getReportedCosts failed for ${kind}:${name}:`, + (e as Error).message, + ); + } + } + }; + collect(driversContainers as Record, 'driver'); + collect(controllersContainers as Record, 'controller'); + return all; +} + +extension.get( + '/metering/usage', + { subdomain: 'api', requireAuth: true }, + async (req, res) => { + const actor = Context.get('actor'); + if (!actor?.user) throw new HttpError(401, 'Authentication required'); + + const [actorUsage, allowanceInfo] = await Promise.all([ + services.metering.getActorCurrentMonthUsageDetails(actor), + services.metering.getAllowedUsage(actor), + ]); + res.json({ ...actorUsage, allowanceInfo }); + }, +); + +extension.get( + '/metering/usage/:appIdOrName', + { subdomain: 'api', requireAuth: true }, + async (req, res) => { + const actor = Context.get('actor'); + if (!actor?.user) throw new HttpError(401, 'Authentication required'); + + let appId = String(req.params.appIdOrName ?? ''); + if (!appId) throw new HttpError(400, 'appId parameter is required'); + + // If not a UUID-shaped app UID, look up by name + if (!appId.startsWith('app-')) { + const appRows = (await clients.db.read( + 'SELECT `uid` FROM `apps` WHERE `name` = ? LIMIT 1', + [appId], + )) as Array<{ uid: string }>; + if (appRows.length > 0) { + appId = appRows[0].uid; + } else { + throw new HttpError(404, 'App not found'); + } + } + + const appUsage = + await services.metering.getActorCurrentMonthAppUsageDetails( + actor, + appId, + ); + res.json(appUsage); + }, +); + +extension.get( + '/metering/globalUsage', + { subdomain: 'api', adminOnly: true }, + async (_req, res) => { + const globalUsage = await services.metering.getGlobalUsage(); + res.json(globalUsage); + }, +); + +// First hit walks the registries; subsequent hits serve the in-memory cache. +extension.get( + '/metering/allCosts', + { subdomain: 'api', requireAuth: true }, + async (_req, res) => { + if (!cachedAllCosts) { + cachedAllCosts = collectAllCosts(); + } + res.json({ costs: cachedAllCosts }); + }, +); diff --git a/extensions/metering/config.json b/extensions/metering/config.json deleted file mode 100644 index 1f5505941..000000000 --- a/extensions/metering/config.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "unlimitedUsage": false, - "unlimitedAllowList": [ - "admin" - ], - "allowedGlobalUsageUsers": [ - "nj", - "salazareos" - ], - "priority": 10 -} \ No newline at end of file diff --git a/extensions/metering/controllers/UsageController.ts b/extensions/metering/controllers/UsageController.ts deleted file mode 100644 index 06d3aa6a8..000000000 --- a/extensions/metering/controllers/UsageController.ts +++ /dev/null @@ -1,96 +0,0 @@ -/* global extension */ -import type { BaseDatabaseAccessService } from '@heyputer/backend/src/services/database/BaseDatabaseAccessService.js'; -import type { MeteringService } from '@heyputer/backend/src/services/MeteringService/MeteringService.js'; -import type { - Request, - Response, -} from 'express'; - -const { Controller, Get, ExtensionController } = extension.import('extensionController'); - -@Controller('/metering') -export class UsageController extends ExtensionController { - #meteringService: MeteringService; - #sqlClient: BaseDatabaseAccessService; - - constructor ( - meteringService: MeteringService, - sqlClient: BaseDatabaseAccessService, - ) { - super(); - this.#meteringService = meteringService; - this.#sqlClient = sqlClient; - } - - @Get('usage', { subdomain: 'api' }) - async getUsage (req: Request, res: Response) { - const actor = req.actor; - if ( ! actor ) { - throw Error('actor not found in context'); - } - const actorUsagePromise = this.#meteringService.getActorCurrentMonthUsageDetails(actor); - const actorAllowanceInfoPromise = this.#meteringService.getAllowedUsage(actor); - - const [actorUsage, allowanceInfo] = await Promise.all([ - actorUsagePromise, - actorAllowanceInfoPromise, - ]); - res.status(200).json({ ...actorUsage, allowanceInfo }); - return; - } - - @Get('usage/:appIdOrName', { subdomain: 'api' }) - async getUsageByApp (req: Request, res: Response) { - const actor = req.actor; - if ( ! actor ) { - throw Error('actor not found in context'); - } - const appIdOrName = req.params.appIdOrName; - if ( ! appIdOrName ) { - res.status(400).json({ error: 'appId parameter is required' }); - return; - } - if ( typeof appIdOrName !== 'string' ) { - res.status(400).json({ error: 'appId parameter must be a string' }); - return; - } - - let appId = appIdOrName; - if ( !appIdOrName.startsWith('app-') || !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(appIdOrName.split('app-')[1]) ) { - // Check if the part after 'app-' is a valid UUID (v4) - const appRows = await this.#sqlClient.read( - 'SELECT `uid` FROM `apps` WHERE `name` = ? LIMIT 1', - [appIdOrName], - ); - if ( appRows.length > 0 ) { - appId = appRows[0].uid; - } else { - res.status(404).json({ error: 'App not found' }); - return; - } - } else { - appId = appIdOrName; - } - - const appUsage = - await this.#meteringService.getActorCurrentMonthAppUsageDetails( - actor, - appId, - ); - - res.status(200).json(appUsage); - return; - } - - @Get('globalUsage', { subdomain: 'api' }, extension.config.allowedGlobalUsageUsers || []) - async getGlobalUsage (req: Request, res: Response) { - const actor = req.actor; - if ( ! actor ) { - throw Error('actor not found in context'); - } - - const globalUsage = await this.#meteringService.getGlobalUsage(); - res.status(200).json(globalUsage); - return; - } -} diff --git a/extensions/metering/eventListeners/subscriptionEvents.ts b/extensions/metering/eventListeners/subscriptionEvents.ts deleted file mode 100644 index e15f8f717..000000000 --- a/extensions/metering/eventListeners/subscriptionEvents.ts +++ /dev/null @@ -1,29 +0,0 @@ -extension.on('metering:overrideDefaultSubscription', async (event) => { - // bit of a stub implementation for OSS, technically can be always free if you set this config true - if ( config.unlimitedUsage ) { - console.warn('WARNING!!! unlimitedUsage is enabled, this is not recommended for production use'); - event.defaultSubscriptionId = 'unlimited'; - } -}); - -extension.on('metering:registerAvailablePolicies', async (event) => { - // bit of a stub implementation for OSS, technically can be always free if you set this config true - if ( config.unlimitedUsage || config.unlimitedAllowList?.length ) { - event.availablePolicies.push({ - id: 'unlimited', - monthUsageAllowance: 5_000_000 * 1_000_000 * 100, // unless you're like, jeff's, mark's, and elon's illegitamate son, you probably won't hit $5m a month - monthlyStorageAllowance: 100_000 * 1024 * 1024, // 100MiB but ignored in local dev - }); - } -}); - -extension.on('metering:getUserSubscription', async (event) => { - const userName = event?.actor?.type?.user?.username; - if ( config.unlimitedAllowList?.includes(userName) ) { - event.userSubscriptionId; - } - else { - event.userSubscriptionId = event?.actor?.type?.user?.subscription?.active ? event.actor.type.user.subscription?.tier : undefined; - } - // default location for user sub, but can techinically be anywhere else or fetched on request -}); diff --git a/extensions/metering/main.ts b/extensions/metering/main.ts deleted file mode 100644 index 79102f3a2..000000000 --- a/extensions/metering/main.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { UsageController } from './controllers/UsageController.js'; -import './eventListeners/subscriptionEvents.js'; - -const meteringService = extension.import('service:meteringService'); -const sqlClient = extension.import('service:database'); - -const controller = new UsageController(meteringService, sqlClient); -controller.registerRoutes(); diff --git a/extensions/metering/package.json b/extensions/metering/package.json deleted file mode 100644 index 47c7f2e17..000000000 --- a/extensions/metering/package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "@heyputer/extension-metering-service", - "main": "main.js", - "type": "module", - "scripts": { - "postinstall": "tsc --noCheck" - }, - "devDependencies": { - "typescript": "^5.9.3" - } -} diff --git a/extensions/metering/tsconfig.json b/extensions/metering/tsconfig.json deleted file mode 100644 index 3e9daf662..000000000 --- a/extensions/metering/tsconfig.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2024", - "module": "nodenext", - "moduleResolution": "nodenext", - "strict": true, - "forceConsistentCasingInFileNames": true, - "skipLibCheck": true, - "sourceMap": true, - "noEmitOnError": true, - "noImplicitAny": false, - "allowJs": true, - "checkJs": false, - }, - "include": [ - "./**/*.ts", - "./**/*.d.ts" - ], - "exclude": [ - "**/*.test.ts", - "**/*.spec.ts", - "**/test/**", - "**/tests/**", - "node_modules", - "dist", - "*.js" - ] -} diff --git a/extensions/metering/types.ts b/extensions/metering/types.ts deleted file mode 100644 index 69a5a7cfd..000000000 --- a/extensions/metering/types.ts +++ /dev/null @@ -1 +0,0 @@ -import '../api.js'; diff --git a/extensions/serverInfo/index.ts b/extensions/serverInfo.ts similarity index 64% rename from extensions/serverInfo/index.ts rename to extensions/serverInfo.ts index b829011f9..196723598 100644 --- a/extensions/serverInfo/index.ts +++ b/extensions/serverInfo.ts @@ -1,16 +1,11 @@ -/* global config, extension */ -import type { - Request, - Response, -} from 'express'; +import { extension } from '@heyputer/backend/src/extensions'; import fs from 'fs/promises'; import os from 'os'; -const { Controller, Get, ExtensionController } = extension.import('extensionController'); -@Controller('/serverInfo', [...config.allowedUsernames]) -class ServerInfoController extends ExtensionController { - @Get('', { subdomain: 'api' }) - async getServerInfo (_req: Request, res: Response) { +extension.get( + '/serverInfo', + { subdomain: 'api', adminOnly: true }, + async (_req, res) => { const osData = { platform: os.platform(), type: os.type(), @@ -40,18 +35,26 @@ class ServerInfoController extends ExtensionController { pretty: `${Math.floor(uptimeSeconds / 86400)}d ${Math.floor((uptimeSeconds % 86400) / 3600)}h ${Math.floor((uptimeSeconds % 3600) / 60)}m`, }; - let diskData = { total: 'N/A', free: 'N/A', used: 'N/A' }; + let diskData: Record = { + total: 'N/A', + free: 'N/A', + used: 'N/A', + }; try { const stats = await fs.statfs('/'); - const totalGB = (stats.blocks * stats.bsize / 1073741824); - const freeGB = (stats.bfree * stats.bsize / 1073741824); + const totalGB = (stats.blocks * stats.bsize) / 1073741824; + const freeGB = (stats.bfree * stats.bsize) / 1073741824; const usedGB = (totalGB - freeGB).toFixed(2); - diskData = { total: totalGB.toFixed(2), free: freeGB.toFixed(2), used: usedGB }; - } catch ( err ) { + diskData = { + total: totalGB.toFixed(2), + free: freeGB.toFixed(2), + used: usedGB, + }; + } catch (err) { console.error('Disk stats error:', err); } - const response = { + res.json({ os: osData, cpu: cpuData, ram: ramData, @@ -59,10 +62,6 @@ class ServerInfoController extends ExtensionController { disk: diskData, loadavg: os.loadavg(), hostname: os.hostname(), - }; - - res.json(response); - } -} - -(new ServerInfoController()).registerRoutes(); + }); + }, +); diff --git a/extensions/serverInfo/config.json b/extensions/serverInfo/config.json deleted file mode 100644 index 77cf15d65..000000000 --- a/extensions/serverInfo/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "allowedUsernames": [ - "puter" -] -} \ No newline at end of file diff --git a/extensions/serverInfo/package.json b/extensions/serverInfo/package.json deleted file mode 100644 index 56157804f..000000000 --- a/extensions/serverInfo/package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "@heyputer/server-info-extension", - "main": "index.js", - "type": "module", - "scripts": { - "postinstall": "tsc --noCheck" - }, - "devDependencies": { - "typescript": "^5.9.3" - } -} diff --git a/extensions/serverInfo/tsconfig.json b/extensions/serverInfo/tsconfig.json deleted file mode 100644 index 3e9daf662..000000000 --- a/extensions/serverInfo/tsconfig.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2024", - "module": "nodenext", - "moduleResolution": "nodenext", - "strict": true, - "forceConsistentCasingInFileNames": true, - "skipLibCheck": true, - "sourceMap": true, - "noEmitOnError": true, - "noImplicitAny": false, - "allowJs": true, - "checkJs": false, - }, - "include": [ - "./**/*.ts", - "./**/*.d.ts" - ], - "exclude": [ - "**/*.test.ts", - "**/*.spec.ts", - "**/test/**", - "**/tests/**", - "node_modules", - "dist", - "*.js" - ] -} diff --git a/extensions/serverInfo/types.ts b/extensions/serverInfo/types.ts deleted file mode 100644 index 69a5a7cfd..000000000 --- a/extensions/serverInfo/types.ts +++ /dev/null @@ -1 +0,0 @@ -import '../api.js'; diff --git a/extensions/thumbnails.ts b/extensions/thumbnails.ts new file mode 100644 index 000000000..a2c6cc929 --- /dev/null +++ b/extensions/thumbnails.ts @@ -0,0 +1,227 @@ +import { + DeleteObjectCommand, + GetObjectCommand, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { extension } from '@heyputer/backend/src/extensions'; +import crypto from 'node:crypto'; +import sharp from 'sharp'; +const clients = extension.import('client'); + +const MAX_THUMBNAIL_BYTES = 2 * 1024 * 1024; +const MAX_THUMBNAIL_PIXELS = 64e6; + +// S3 client + bucket config — lazily resolved after boot from config. +let s3Client: S3Client | null = null; +let thumbnailBucketName = 'puter-local'; +let extensionBucketEndpoint = 'http://127.0.0.1:4566/puter-local/'; + +function getClient(): S3Client { + if (s3Client) return s3Client; + + // Top-level `thumbnailStore` config when the extension should use a + // dedicated S3 bucket instead of the main one. + const thumbStore = extension.config.thumbnailStore; + + if (thumbStore?.endpoint && thumbStore.credentials) { + s3Client = new S3Client({ + region: 'auto', + endpoint: thumbStore.endpoint, + credentials: thumbStore.credentials, + }); + thumbnailBucketName = thumbStore.name ?? 'puter-local'; + extensionBucketEndpoint = thumbStore.endpoint; + } else { + // Fall back to the project's S3 wrapper. `clients.s3` is the Puter + // `S3Client` wrapper (region-cache + lifecycle), not an AWS + // `S3Client`. Call `.get()` to obtain the underlying AWS client that + // `getSignedUrl` / `.send(command)` both expect. + const wrapper = clients.s3; + s3Client = wrapper.get(); + } + return s3Client; +} + +function base64ParseDataUrl(dataURL: string) { + dataURL = dataURL.slice(5); + const mimeType = dataURL.split(';')[0]; + const data = Buffer.from(dataURL.split(',')[1], 'base64'); + return { mimeType, data }; +} + +// Strictly decode a data: URL and validate the decoded image. Encoded-string +// length lies about decoded byte count (whitespace, padding) and says nothing +// about pixel count — a 2MB PNG can decompress to hundreds of MB of raster. +async function decodeAndValidateThumbnail( + dataURL: string, +): Promise<{ mimeType: string; data: Buffer } | null> { + const commaIdx = dataURL.indexOf(','); + if (commaIdx === -1) return null; + const mimeType = dataURL.slice(5, commaIdx).split(';')[0]; + + const data = Buffer.from(dataURL.slice(commaIdx + 1), 'base64'); + if (data.length === 0 || data.length > MAX_THUMBNAIL_BYTES) return null; + + try { + await sharp(data, { + limitInputPixels: MAX_THUMBNAIL_PIXELS, + density: 72, + failOn: 'error', + }).metadata(); + } catch { + return null; + } + + return { mimeType, data }; +} + +// ── thumbnail.created ─────────────────────────────────────────────── +// Intercept data-URL thumbnails before they hit the DB: upload to S3 +// and replace the URL with an s3:// pointer. + +extension.on('thumbnail.created', async (event: Record) => { + const url = event.url; + if (typeof url !== 'string' || !url.startsWith('data:')) return; + + const decoded = await decodeAndValidateThumbnail(url); + if (!decoded) { + event.url = null; + return; + } + + const key = crypto.randomUUID(); + event.url = `s3://${thumbnailBucketName}/${key}`; + + await getClient().send( + new PutObjectCommand({ + Bucket: thumbnailBucketName, + Key: key, + Body: decoded.data, + ContentType: decoded.mimeType, + }), + ); +}); + +// ── thumbnail.upload.prepare ──────────────────────────────────────── +// Generate pre-signed upload URLs so the client can PUT directly to S3. + +extension.on( + 'thumbnail.upload.prepare', + async (event: Record) => { + if (!event || !Array.isArray(event.items)) return; + const client = getClient(); + + for (const item of event.items as Array>) { + if (!item || typeof item !== 'object') { + throw new Error('thumbnail.upload.prepare item is invalid'); + } + + const contentType = + typeof item.contentType === 'string' + ? item.contentType.trim() + : ''; + if (!contentType) continue; + + if (item.size !== undefined) { + const size = Number(item.size); + if ( + !Number.isFinite(size) || + size < 0 || + size > MAX_THUMBNAIL_BYTES + ) + continue; + } + + const key = crypto.randomUUID(); + const command = new PutObjectCommand({ + Bucket: thumbnailBucketName, + Key: key, + ContentType: contentType, + }); + item.uploadUrl = await getSignedUrl(client, command, { + expiresIn: 900, + }); + item.thumbnailUrl = `s3://${thumbnailBucketName}/${key}`; + } + }, +); + +// ── thumbnail.read ────────────────────────────────────────────────── +// Convert s3:// or legacy https:// thumbnails to signed URLs. + +extension.on('thumbnail.read', async (entry: Record) => { + const thumb = entry.thumbnail; + if (typeof thumb !== 'string' || !thumb) return; + const client = getClient(); + + if (thumb.startsWith('s3://')) { + const [bucket, key] = thumb.slice(5).split('/'); + entry.thumbnail = await getSignedUrl( + client, + new GetObjectCommand({ Bucket: bucket, Key: key }), + { expiresIn: 604800 }, + ); + } else if ( + thumb.startsWith('https') && + thumb.includes(new URL(extensionBucketEndpoint).hostname) + ) { + // Legacy format — remove after full migration + const [bucket, key] = new URL(thumb).pathname.slice(1).split('/'); + entry.thumbnail = await getSignedUrl( + client, + new GetObjectCommand({ Bucket: bucket, Key: key }), + { expiresIn: 604800 }, + ); + } else if (thumb.startsWith('data')) { + // Inline data-URL migration: upload to S3 and update the DB entry. + const key = crypto.randomUUID(); + const { mimeType, data } = base64ParseDataUrl(thumb); + const newUrl = `s3://${thumbnailBucketName}/${key}`; + + await client.send( + new PutObjectCommand({ + Bucket: thumbnailBucketName, + Key: key, + Body: data, + ContentType: mimeType, + }), + ); + + // Best-effort async DB update + const uuid = entry.uuid ?? entry.uid; + if (uuid) { + clients.db + .write( + 'UPDATE `fsentries` SET `thumbnail` = ? WHERE `uuid` = ?', + [newUrl, uuid], + ) + .catch((err: unknown) => + console.warn('[thumbnails] inline migration failed', err), + ); + } + + entry.thumbnail = await getSignedUrl( + client, + new GetObjectCommand({ Bucket: thumbnailBucketName, Key: key }), + { expiresIn: 604800 }, + ); + } +}); + +// ── fs.remove.node ────────────────────────────────────────────────── +// Delete S3 thumbnail when the file is removed. + +extension.on( + 'fs.remove.node', + async ({ target }: { target: Record }) => { + const thumbnailUrl = target.thumbnail as string | undefined; + if (!thumbnailUrl || !thumbnailUrl.startsWith('s3://')) return; + + const [bucket, key] = thumbnailUrl.slice(5).split('/'); + await getClient().send( + new DeleteObjectCommand({ Bucket: bucket, Key: key }), + ); + }, +); diff --git a/extensions/thumbnails/package.json b/extensions/thumbnails/package.json deleted file mode 100644 index 60266b7fc..000000000 --- a/extensions/thumbnails/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "thumbnail", - "version": "1.0.0", - "main": "thumbnailBucketStore.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "keywords": [], - "author": "", - "license": "UNLICENSED", - "description": "", - "dependencies": { - "@aws-sdk/s3-request-presigner": "^3.1020.0", - "@aws-sdk/client-s3": "^3.1020.0" - } -} diff --git a/extensions/thumbnails/thumbnailBucketStore.js b/extensions/thumbnails/thumbnailBucketStore.js deleted file mode 100644 index 6b2473110..000000000 --- a/extensions/thumbnails/thumbnailBucketStore.js +++ /dev/null @@ -1,194 +0,0 @@ -const { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand } = require('@aws-sdk/client-s3'); -const { getSignedUrl } = require('@aws-sdk/s3-request-presigner'); -const { Context } = extension.import('core'); -const /**@type {any}*/ svc_fs = extension.import('service:filesystem'); -const { - NodeUIDSelector, -} = extension.import('core').fs.selectors; - -const extensionBucketInfo = global_config.services?.thumbnails?.bucket; -const client = extensionBucketInfo?.endpoint && extensionBucketInfo?.credentials ? new S3Client({ - region: 'auto', - endpoint: extensionBucketInfo.endpoint, - credentials: extensionBucketInfo.credentials, -}) : extension.import('data').s3ClientProvider.get(); -const MAX_THUMBNAIL_BYTES = 2 * 1024 * 1024; - -const thumbnailBucketName = extensionBucketInfo?.name || 'puter-local'; -const extensionBucketEndpoint = extensionBucketInfo?.endpoint || 'http://127.0.0.1:4566/puter-local/'; - -// A not-user-input-safe base64 data url parser. -function base64ParseDataUrl (dataURL) { - dataURL = dataURL.slice(5); - const mimeType = dataURL.split(';')[0]; - const data = Buffer.from(dataURL.split(',')[1], 'base64'); - return { mimeType, data }; -} - -function estimateDataUrlSize (dataURL) { - const commaIndex = dataURL.indexOf(','); - const base64 = commaIndex === -1 ? dataURL : dataURL.slice(commaIndex + 1); - return Math.ceil(base64.length * 3 / 4); -} - -extension.on('thumbnail.created', async (event) => { - const url = event.url; - if ( typeof url !== 'string' || !url.startsWith('data:') ) { - return; - } - if ( estimateDataUrlSize(url) > MAX_THUMBNAIL_BYTES ) { - event.url = null; - return; - } - - const key = crypto.randomUUID(); - - // Inject in the s3 internal URL in place of the data URL before the operation goes to DB - event.url = `s3://${thumbnailBucketName}/${key}`; - - // Parse base64 URL created from thumbnail service - const { mimeType, data } = base64ParseDataUrl(url); - - // Upload thumbnail - const params = { - Bucket: thumbnailBucketName, - Key: key, - Body: data, - ContentType: mimeType, - }; - await client.send(new PutObjectCommand(params)); -}); - -extension.on('thumbnail.upload.prepare', async (event) => { - if ( !event || !Array.isArray(event.items) ) { - return; - } - - for ( const item of event.items ) { - if ( !item || typeof item !== 'object' ) { - throw new Error('thumbnail.upload.prepare item is invalid'); - } - - const contentType = typeof item.contentType === 'string' - ? item.contentType.trim() - : ''; - if ( ! contentType ) { - continue; - } - - if ( item.size !== undefined ) { - const size = Number(item.size); - if ( !Number.isFinite(size) || size < 0 ) { - continue; - } - if ( size > MAX_THUMBNAIL_BYTES ) { - continue; - } - } - - const key = crypto.randomUUID(); - const bucket = thumbnailBucketName; - const command = new PutObjectCommand({ - Bucket: bucket, - Key: key, - ContentType: contentType, - }); - const uploadUrl = await getSignedUrl(client, command, { expiresIn: 900 }); - - item.uploadUrl = uploadUrl; - item.thumbnailUrl = `s3://${bucket}/${key}`; - } -}); - -let in_progress_thumbs = {}; - -extension.on('thumbnail.read', async (/**@type {any}*/entry) => { - if ( entry.thumbnail && entry.thumbnail.startsWith('s3://') ) { - // Parse s3 URL - const [bucket, key] = entry.thumbnail.slice(5).split('/'); - - // Get signed url and inject it into the thumbnail read event - entry.thumbnail = await getSignedUrl( - client, - new GetObjectCommand({ Bucket: bucket, Key: key }), - { expiresIn: 604800 }, - ); - } else if ( entry.thumbnail.startsWith('https') && entry.thumbnail.includes(new URL(extensionBucketEndpoint).hostname) ) { - // Remove after migration - let [bucket, key] = new URL(entry.thumbnail).pathname.slice(1).split('/'); - - // Get signed url and inject it into the thumbnail read event - entry.thumbnail = await getSignedUrl( - client, - new GetObjectCommand({ Bucket: bucket, Key: key }), - { expiresIn: 604800 }, - ); - } else if ( entry.thumbnail.startsWith('data') && Context.get('req') && !in_progress_thumbs[entry.uuid] ) { - in_progress_thumbs[entry.uuid] = true; - const newNode = await svc_fs.node(new NodeUIDSelector(entry.uuid)); - const key = crypto.randomUUID(); - const { mimeType, data } = base64ParseDataUrl(entry.thumbnail); - const newUrl = `s3://${thumbnailBucketName}/${key}`; - // Upload thumbnail - const params = { - Bucket: thumbnailBucketName, - Key: key, - Body: data, - ContentType: mimeType, - }; - (async () => { - await client.send(new PutObjectCommand(params)); - await newNode.provider.update_thumbnail({ - context: Context.get(), - node: newNode, - thumbnail: newUrl, - }); - delete in_progress_thumbs[entry.uuid]; - })(); - } -}); - -extension.on('fs.remove.node', async ({ target }) => { - let thumbnailUrl; - if ( ! target.thumbnail ) { - // Stat the entry since we weren't given a thumbnail - const controls = { - log: target.log, - provide_selector: selector => { - target.selector = selector; - }, - }; - const newTarget = await target.provider.stat({ - selector: target.selector, - options: { thumbnail: true }, - node: target, - controls, - }); - - // There is REALLY just no thumbnail - if ( ! newTarget.thumbnail ) - { - return; - } - - thumbnailUrl = newTarget.thumbnail; - } else { - // We were immediately given a thumbnail - thumbnailUrl = target.thumbnail; - } - - // Not an S3 thumbnail, likely older format like data URL - if ( !thumbnailUrl || !thumbnailUrl.startsWith('s3://') ) - { - return; - } - - const [bucket, key] = thumbnailUrl.slice(5).split('/'); - - // Delete thumbnail from S3 - const params = { - Bucket: bucket, - Key: key, - }; - await client.send(new DeleteObjectCommand(params)); -}); diff --git a/extensions/tsconfig.json b/extensions/tsconfig.json deleted file mode 100644 index faaa2cc0e..000000000 --- a/extensions/tsconfig.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2024", - "module": "node16", - "moduleResolution": "node16", - "allowJs": true, - "rootDir": "./", - "strict": true, - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "allowSyntheticDefaultImports": true, - "skipLibCheck": true, - "sourceMap": true, - }, - "include": [ - "./**/*.ts", - "./**/*.d.ts", - "./**/*.d.mts", - "./**/*.d.cts" - ], - "exclude": [ - "**/*.test.ts", - "**/*.spec.ts", - "**/test/**", - "**/tests/**", - "node_modules", - "dist" - ] -} \ No newline at end of file diff --git a/extensions/utilities.js b/extensions/utilities.js deleted file mode 100644 index f7818b64e..000000000 --- a/extensions/utilities.js +++ /dev/null @@ -1,9 +0,0 @@ -//@extension priority -10000 - -extension.exports = {}; - -extension.exports.sleep = async (seconds) => { - await new Promise(resolve => { - setTimeout(resolve, seconds); - }); -}; diff --git a/extensions/whoami.ts b/extensions/whoami.ts new file mode 100644 index 000000000..5bc792f67 --- /dev/null +++ b/extensions/whoami.ts @@ -0,0 +1,178 @@ +import { Context } from '@heyputer/backend/src/core'; +import { extension } from '@heyputer/backend/src/extensions'; +import { getTaskbarItems } from '@heyputer/backend/src/util/taskbarItems.js'; +import TimeAgo from 'javascript-time-ago'; +import localeEn from 'javascript-time-ago/locale/en'; + +const stores = extension.import('store'); +const services = extension.import('service'); +const clients = extension.import('client'); + +const timeago = (() => { + TimeAgo.addDefaultLocale(localeEn); + return new TimeAgo('en-US'); +})(); + +// Allowlist of `config.feature_flags` keys safe to surface via /whoami. +// Anything not listed here stays server-side, so internal flags +// (payment_bypass, staff_only_*, etc.) cannot leak by accident. Add a +// flag here when, and only when, the client actually needs to read it. +const CLIENT_VISIBLE_FEATURE_FLAGS: ReadonlySet = new Set([ + 'create_shortcut', + 'download_directory', + 'prompt_user_when_navigation_away_from_puter', +]); + +extension.get( + '/whoami', + { subdomain: 'api', requireAuth: true }, + async (req, res) => { + const actor = Context.get('actor'); + if (!actor?.user?.id) { + res.status(401).json({ error: 'Authentication required' }); + return; + } + + const isUser = !actor.app; + const user = await stores.user.getById(actor.user.id); + if (!user) { + res.status(404).json({ error: 'User not found' }); + return; + } + + const oidcOnly = user.password === null; + const ALLOWED_ICON_SIZES = new Set([16, 32, 64, 128, 256, 512]); + const rawIconSize = + typeof req.query?.icon_size === 'string' + ? Number(req.query.icon_size) + : undefined; + const iconSize = + rawIconSize !== undefined && ALLOWED_ICON_SIZES.has(rawIconSize) + ? rawIconSize + : undefined; + const noIcons = !iconSize; + + // Feature flags come from `config.feature_flags`. We only forward keys + // listed in CLIENT_VISIBLE_FEATURE_FLAGS so internal flags can't leak. + // Non-boolean values (e.g. `"true"` as a string) are coerced so the + // client never has to guess. + const rawFlags = extension.config.feature_flags ?? {}; + const feature_flags: Record = {}; + for (const [k, v] of Object.entries(rawFlags)) { + if (CLIENT_VISIBLE_FEATURE_FLAGS.has(k)) { + feature_flags[k] = Boolean(v); + } + } + + const details: Record = { + username: user.username, + uuid: user.uuid, + email: user.email, + unconfirmed_email: user.email, + email_confirmed: user.email_confirmed || user.username === 'admin', + requires_email_confirmation: user.requires_email_confirmation, + desktop_bg_url: user.desktop_bg_url, + desktop_bg_color: user.desktop_bg_color, + desktop_bg_fit: user.desktop_bg_fit, + is_temp: user.password === null && user.email === null, + oidc_only: oidcOnly, + taskbar_items: isUser + ? await getTaskbarItems( + user, + { + clients, + stores, + services, + apiBaseUrl: String( + extension.config.api_base_url ?? '', + ), + }, + { iconSize, noIcons }, + ) + : undefined, + otp: !!user.otp_enabled, + feature_flags, + human_readable_age: user.timestamp + ? timeago.format(new Date(user.timestamp as string)) + : null, + }; + + // OIDC revalidate URL for password-less accounts + if (oidcOnly) { + try { + const providers = await services.oidc.getEnabledProviderIds(); + const provider = providers?.[0]; + if (provider) { + const callbackUrl = + services.oidc.getCallbackUrl?.('login') ?? ''; + const origin = callbackUrl.replace( + /\/auth\/oidc\/callback\/login$/, + '', + ); + details.oidc_revalidate_url = `${origin}/auth/oidc/${provider}/start?flow=revalidate&user_uuid=${encodeURIComponent(user.uuid)}`; + } + } catch { + // OIDC not configured + } + } + + // Directories — only sent to user actors + if (isUser) { + const directories: Record = {}; + const nameToProp: Record = { + desktop_uuid: `/${user.username}/Desktop`, + appdata_uuid: `/${user.username}/AppData`, + documents_uuid: `/${user.username}/Documents`, + pictures_uuid: `/${user.username}/Pictures`, + videos_uuid: `/${user.username}/Videos`, + trash_uuid: `/${user.username}/Trash`, + }; + for (const k in nameToProp) { + directories[nameToProp[k]] = user[k]; + } + details.directories = directories; + } + + // Last activity + if (user.last_activity_ts) { + try { + details.last_activity_ts = Math.round( + new Date(user.last_activity_ts as string).getTime() / 1000, + ); + } catch { + /* ignore parse error */ + } + } + + // Strip sensitive fields for app actors + if (!isUser) { + const canReadEmail = await services.permission + .check(actor, `user:${user.uuid}:email:read`) + .catch(() => false); + if (!canReadEmail) { + delete details.email; + delete details.unconfirmed_email; + } + delete details.desktop_bg_url; + delete details.desktop_bg_color; + delete details.desktop_bg_fit; + delete details.human_readable_age; + } + + if (actor.app) { + details.app_name = actor.app.uid; + } + + try { + await clients.event.emitAndWait( + 'whoami.details', + { user, details, isUser }, + {}, + ); + } catch { + /* best-effort */ + } + + res.json(details); + }, +); diff --git a/extensions/whoami/main.js b/extensions/whoami/main.js deleted file mode 100644 index b0b0f479c..000000000 --- a/extensions/whoami/main.js +++ /dev/null @@ -1 +0,0 @@ -import './routes.js'; diff --git a/extensions/whoami/package.json b/extensions/whoami/package.json deleted file mode 100644 index 0fb6d98a3..000000000 --- a/extensions/whoami/package.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "@heyputer/extension-whoami", - "main": "main.js", - "type": "module", - "dependencies": { - "javascript-time-ago": "^2.5.12" - } -} diff --git a/extensions/whoami/routes.js b/extensions/whoami/routes.js deleted file mode 100644 index 391904595..000000000 --- a/extensions/whoami/routes.js +++ /dev/null @@ -1,256 +0,0 @@ -// static imports -import _path from 'fs'; -import TimeAgo from 'javascript-time-ago'; -import localeEn from 'javascript-time-ago/locale/en'; - -// runtime imports -const { UserActorType, AppUnderUserActorType } = extension.import('core'); -const { - id2uuid, - get_descendants, - suggest_app_for_fsentry, - is_shared_with_anyone, - get_app, - get_taskbar_items, -} = extension.import('core').util.helpers; - -const timeago = (() => { - TimeAgo.addDefaultLocale(localeEn); - return new TimeAgo('en-US'); -})(); - -const whoami_common = ({ is_user, user }) => { - const details = {}; - - // User's immutable default (often called "system") directories' - // alternative (to path) identifiers are sent to the user's client - // (but not to apps; they don't need this information) - if ( is_user ) { - const directories = details.directories = {}; - const name_to_path = { - 'desktop_uuid': `/${user.username}/Desktop`, - 'appdata_uuid': `/${user.username}/AppData`, - 'documents_uuid': `/${user.username}/Documents`, - 'pictures_uuid': `/${user.username}/Pictures`, - 'videos_uuid': `/${user.username}/Videos`, - 'trash_uuid': `/${user.username}/Trash`, - }; - for ( const k in name_to_path ) { - directories[name_to_path[k]] = user[k]; - } - } - - if ( user.last_activity_ts ) { - - // Create a Date object and get the epoch timestamp - let epoch; - try { - epoch = new Date(user.last_activity_ts).getTime(); - // round to 1 decimal place - epoch = Math.round(epoch / 1000); - } catch ( e ) { - console.error('Error parsing last_activity_ts', e); - } - - // add last_activity_ts - details.last_activity_ts = epoch; - } - - return details; -}; - -extension.get('/whoami', { subdomain: 'api' }, async (req, res, next) => { - const actor = req.actor; - - if ( ! actor ) { - throw Error('actor not found in context'); - } - - const is_user = actor.type instanceof UserActorType; - - if ( req.query.icon_size ) { - const ALLOWED_SIZES = ['16', '32', '64', '128', '256', '512']; - - if ( ! ALLOWED_SIZES.includes(req.query.icon_size) ) { - res.status(400).send({ error: 'Invalid icon_size' }); - } - } - - const oidc_only = req.user.password === null; - const details = { - username: req.user.username, - uuid: req.user.uuid, - email: req.user.email, - unconfirmed_email: req.user.email, - email_confirmed: req.user.email_confirmed - || req.user.username === 'admin', - requires_email_confirmation: req.user.requires_email_confirmation, - desktop_bg_url: req.user.desktop_bg_url, - desktop_bg_color: req.user.desktop_bg_color, - desktop_bg_fit: req.user.desktop_bg_fit, - is_temp: (req.user.password === null && req.user.email === null), - oidc_only, - ...(oidc_only ? await (async () => { - try { - const svc_oidc = req.services.get('oidc'); - const providers = await svc_oidc.getEnabledProviderIds(); - const origin = (svc_oidc.global_config?.origin || '').replace(/\/$/, ''); - const provider = providers && providers[0]; - if ( provider ) { - return { - oidc_revalidate_url: `${origin}/auth/oidc/${provider}/start?flow=revalidate&user_id=${req.user.id}`, - }; - } - return {}; - } catch ( _e ) { - return {}; - } - })() : {}), - taskbar_items: await get_taskbar_items(req.user, { - ...(req.query.icon_size - ? { icon_size: req.query.icon_size } - : { no_icons: true }), - }), - referral_code: req.user.referral_code, - otp: !!req.user.otp_enabled, - human_readable_age: timeago.format(new Date(req.user.timestamp)), - hasDevAccountAccess: !!req.actor.type.user.metadata?.hasDevAccountAccess, - ...(req.new_token ? { token: req.token } : {}), - is_user_token: true, // gets deleted if not a user token - }; - - // TODO: redundant? GetUserService already puts these values on 'user' - // Get whoami values from other services - const /** @type {any} */ svc_whoami = req.services.get('whoami'); - - const /** @type {any} */ svc_permission = req.services.get('permission'); - - const provider_details = await svc_whoami.get_details({ - user: req.user, - actor: actor, - }); - Object.assign(details, provider_details); - - if ( ! is_user ) { - // When apps call /whoami they should not see these attributes - // delete details.username; - // delete details.uuid; - - if ( ! (await svc_permission.check(actor, `user:${details.uuid}:email:read`, { no_cache: true })) ) { - delete details.email; - delete details.unconfirmed_email; - } - - delete details.desktop_bg_url; - delete details.desktop_bg_color; - delete details.desktop_bg_fit; - delete details.taskbar_items; - delete details.token; - delete details.human_readable_age; - delete details.is_user_token; - } - - if ( actor.type instanceof AppUnderUserActorType ) { - details.app_name = actor.type.app.name; - - // IDEA: maybe we do this in the future - // details.app = { - // name: actor.type.app.name, - // }; - } - - Object.assign(details, whoami_common({ is_user, user: req.user })); - - res.send(details); -}); - -extension.post('/whoami', { subdomain: 'api' }, async (req, res) => { - const actor = req.actor; - if ( ! actor ) { - throw Error('actor not found in context'); - } - - const is_user = actor.type instanceof UserActorType; - if ( ! is_user ) { - throw Error('actor is not a user'); - } - - let desktop_items = []; - - // check if user asked for desktop items - if ( req.query.return_desktop_items === 1 || req.query.return_desktop_items === '1' || req.query.return_desktop_items === 'true' ) { - // by cached desktop id - if ( req.user.desktop_id ) { - // TODO: Check if used anywhere, maybe remove - // eslint-disable-next-line no-undef - desktop_items = await db.read(`SELECT * FROM fsentries - WHERE user_id = ? AND parent_uid = ?`, - [req.user.id, await id2uuid(req.user.desktop_id)]); - } - // by desktop path - else { - desktop_items = await get_descendants(`${req.user.username }/Desktop`, req.user, 1, true); - } - - // clean up desktop items and add some extra information - if ( desktop_items.length > 0 ) { - if ( desktop_items.length > 0 ) { - for ( let i = 0; i < desktop_items.length; i++ ) { - if ( desktop_items[i].id !== null ) { - // suggested_apps for files - if ( ! desktop_items[i].is_dir ) { - desktop_items[i].suggested_apps = await suggest_app_for_fsentry(desktop_items[i], { user: req.user }); - } - // is_shared - desktop_items[i].is_shared = await is_shared_with_anyone(desktop_items[i].id); - - // associated_app - if ( desktop_items[i].associated_app_id ) { - const app = await get_app({ id: desktop_items[i].associated_app_id }); - - // remove some privileged information - delete app.id; - delete app.approved_for_listing; - delete app.approved_for_opening_items; - delete app.godmode; - delete app.owner_user_id; - // add to array - desktop_items[i].associated_app = app; - - } else { - desktop_items[i].associated_app = {}; - } - - // remove associated_app_id since it's sensitive info - // delete desktop_items[i].associated_app_id; - } - // id is sesitive info - delete desktop_items[i].id; - delete desktop_items[i].user_id; - delete desktop_items[i].bucket; - desktop_items[i].path = _path.join('/', req.user.username, desktop_items[i].name); - } - } - } - } - - const oidc_only = req.user.password === null; - // send user object - res.send(Object.assign({ - username: req.user.username, - uuid: req.user.uuid, - email: req.user.email, - email_confirmed: req.user.email_confirmed - || req.user.username === 'admin', - requires_email_confirmation: req.user.requires_email_confirmation, - desktop_bg_url: req.user.desktop_bg_url, - desktop_bg_color: req.user.desktop_bg_color, - desktop_bg_fit: req.user.desktop_bg_fit, - is_temp: (req.user.password === null && req.user.email === null), - oidc_only, - taskbar_items: await get_taskbar_items(req.user), - desktop_items: desktop_items, - referral_code: req.user.referral_code, - hasDevAccountAccess: !!req.actor.user.metadata?.hasDevAccountAccess, - }, whoami_common({ is_user, user: req.user }))); -}); diff --git a/extensions/worker-sandbox.js b/extensions/worker-sandbox.js deleted file mode 100644 index b0926cf7d..000000000 --- a/extensions/worker-sandbox.js +++ /dev/null @@ -1,201 +0,0 @@ -const page = ` - - - - - - Puter Worker Sandbox Playground - - - -
-

Puter Worker Sandbox Playground

-

Use this page to interact with the puter APIs in the same sandbox as your worker.

-
- - -
-
-
-

Code

- -
-
-

Logs

-

-            
-
-
- - - - -`; - -extension.get('/', { noauth: true, subdomain: 'worker-sandbox' }, (req, res) => { - res.type('html').send(page); -}); diff --git a/extensions/workerSandbox.ts b/extensions/workerSandbox.ts new file mode 100644 index 000000000..a5cb6cbe2 --- /dev/null +++ b/extensions/workerSandbox.ts @@ -0,0 +1,98 @@ +import { extension } from '@heyputer/backend/src/extensions'; + +const page = ` + + + + + + Puter Worker Sandbox Playground + + + +
+

Puter Worker Sandbox Playground

+

Use this page to interact with the puter APIs in the same sandbox as your worker.

+
+ + +
+
+
+

Code

+ +
+
+

Logs

+

+            
+
+
+ + + + +`; + +extension.get( + '/', + { requireAuth: false, subdomain: 'worker-sandbox' }, + (_req, res) => { + res.type('html').send(page); + }, +); diff --git a/mod_packages/testex/package.json b/mod_packages/testex/package.json deleted file mode 100644 index f11778892..000000000 --- a/mod_packages/testex/package.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/mods/README.md b/mods/README.md deleted file mode 100644 index 61796e509..000000000 --- a/mods/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Puter Mods - -A list of Puter mods which may be expanded in the future. - -**Contributions of new mods are welcome.** - -## kdmod - -- **location:** [./kdmod](./kdmod) -- **description:** - > "kernel dev mod"; specifically for the devex needs of - > GitHub user KernelDeimos and provided in case anyone else - > finds it of any use. diff --git a/mods/mods_available/example-singlefile.js b/mods/mods_available/example-singlefile.js deleted file mode 100644 index 3cac2740a..000000000 --- a/mods/mods_available/example-singlefile.js +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -extension.get('/example-onefile-get', (req, res) => { - res.send('Hello World!'); -}); - -extension.on('install', ({ services }) => { - // console.log('install was called'); -}); diff --git a/mods/mods_available/example/main.js b/mods/mods_available/example/main.js deleted file mode 100644 index 6ca4b6940..000000000 --- a/mods/mods_available/example/main.js +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -extension.get('/example-mod-get', (req, res) => { - res.send('Hello World!'); -}); - -extension.on('install', ({ services }) => { - // console.log('install was called'); -}); diff --git a/mods/mods_available/example/package.json b/mods/mods_available/example/package.json deleted file mode 100644 index 175f513aa..000000000 --- a/mods/mods_available/example/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "example-puter-extension", - "version": "1.0.0", - "description": "", - "main": "main.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "keywords": [], - "author": "", - "license": "AGPL-3.0-only" -} diff --git a/mods/mods_available/kdmod/CustomPuterService.js b/mods/mods_available/kdmod/CustomPuterService.js deleted file mode 100644 index 45266315e..000000000 --- a/mods/mods_available/kdmod/CustomPuterService.js +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const path = require('path'); - -class CustomPuterService extends use.Service { - async _init () { - const svc_puterHomepage = this.services.get('puter-homepage'); - svc_puterHomepage.register_script('/custom-gui/main.js'); - } - '__on_install.routes' (_, { app }) { - const require = this.require; - const express = require('express'); - const path_ = require('path'); - - app.use( - '/custom-gui', - express.static(path.join(__dirname, 'gui')), - ); - } -} - -module.exports = { CustomPuterService }; diff --git a/mods/mods_available/kdmod/README.md b/mods/mods_available/kdmod/README.md deleted file mode 100644 index d7fa3de3a..000000000 --- a/mods/mods_available/kdmod/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Kernel Dev Mod - -This mod makes testing and debugging easier. - -## Current Features: -- A service-script adds `reqex` to the `window` object in the client, - which contains a bunch of example requests to internal API endpoints. diff --git a/mods/mods_available/kdmod/ShareTestService.js b/mods/mods_available/kdmod/ShareTestService.js deleted file mode 100644 index c4854e8f5..000000000 --- a/mods/mods_available/kdmod/ShareTestService.js +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -// TODO: accessing these imports directly from a mod is not really -// the way mods are intended to work; this is temporary until -// we have these things registered in "useapi". -const { - get_user, - invalidate_cached_user, - deleteUser, -} = require('../../../src/backend/src/helpers.js'); -const { HLWrite } = require('../../../src/backend/src/filesystem/hl_operations/hl_write.js'); -const { LLRead } = require('../../../src/backend/src/filesystem/ll_operations/ll_read.js'); -const { Actor, UserActorType } - = require('../../../src/backend/src/services/auth/Actor.js'); -const { DB_WRITE } = require('../../../src/backend/src/services/database/consts.js'); -const { - RootNodeSelector, - NodeChildSelector, - NodePathSelector, -} = require('../../../src/backend/src/filesystem/node/selectors.js'); -const { Context } = require('../../../src/backend/src/util/context.js'); - -class ShareTestService extends use.Service { - static MODULES = { - uuidv4: require('uuid').v4, - }; - - async _init () { - - this.scenarios = require('./data/sharetest_scenarios'); - - const svc_db = this.services.get('database'); - this.db = svc_db.get(svc_db.DB_WRITE, 'share-test'); - } - - async runit () { - await this.teardown_(); - await this.setup_(); - - const results = []; - - for ( const scenario of this.scenarios ) { - if ( ! scenario.title ) { - scenario.title = scenario.sequence.map(step => step.title).join('; '); - } - results.push({ - title: scenario.title, - report: await this.run_scenario_(scenario), - }); - } - - await this.teardown_(); - return results; - } - - async setup_ () { - await this.create_test_user_('testuser_eric'); - await this.create_test_user_('testuser_stan'); - await this.create_test_user_('testuser_kyle'); - await this.create_test_user_('testuser_kenny'); - } - async run_scenario_ (scenario) { - let error; - // Run sequence - for ( const step of scenario.sequence ) { - const method = this[`__scenario:${step.call}`]; - const user = await get_user({ username: step.as }); - const actor = await Actor.create(UserActorType, { user }); - const generated = { user, actor }; - const report = await Context.get().sub({ user, actor }) - .arun(async () => { - return await method.call(this, generated, step.with); - }); - if ( report ) { - error = { step: step.title, report }; - break; - } - } - return error; - } - async teardown_ () { - await this.delete_test_user_('testuser_eric'); - await this.delete_test_user_('testuser_stan'); - await this.delete_test_user_('testuser_kyle'); - await this.delete_test_user_('testuser_kenny'); - } - - async create_test_user_ (username) { - await this.db.write( - ` - INSERT INTO user (uuid, username, email, free_storage, password) - VALUES (?, ?, ?, ?, ?) - `, - [ - this.modules.uuidv4(), - username, - `${username}@example.com`, - 1024 * 1024 * 500, // 500 MiB - this.modules.uuidv4(), - ], - ); - const user = await get_user({ username }); - const svc_user = this.services.get('user'); - await svc_user.generate_default_fsentries({ user }); - invalidate_cached_user(user); - return user; - } - - async delete_test_user_ (username) { - const user = await get_user({ username }); - if ( ! user ) return; - await deleteUser(user.id); - } - - // API for scenarios - async '__scenario:create-example-file' ( - { actor, user }, - { name, contents }, - ) { - const svc_fs = this.services.get('filesystem'); - const parent = await svc_fs.node(new NodePathSelector(`/${user.username}/Desktop`)); - console.log( - 'test -> create-example-file', - user, - name, - contents, - ); - const buffer = Buffer.from(contents); - const file = { - size: buffer.length, - name: name, - type: 'application/octet-stream', - buffer, - }; - const hl_write = new HLWrite(); - await hl_write.run({ - actor, - user, - destination_or_parent: parent, - specified_name: name, - file, - }); - } - async '__scenario:assert-no-access' ( - { actor, user }, - { path }, - ) { - const svc_fs = this.services.get('filesystem'); - const node = await svc_fs.node(new NodePathSelector(path)); - const ll_read = new LLRead(); - let expected_e; try { - const stream = await ll_read.run({ - fsNode: node, - actor, - }); - } catch (e) { - expected_e = e; - } - if ( ! expected_e ) { - return { message: 'expected error, got none' }; - } - } - async '__scenario:grant' ( - { actor, user }, - { to, permission }, - ) { - const svc_permission = this.services.get('permission'); - await svc_permission.grant_user_user_permission(actor, to, permission, {}, {}); - } - async '__scenario:assert-access' ( - { actor, user }, - { path, level }, - ) { - const svc_fs = this.services.get('filesystem'); - const svc_acl = this.services.get('acl'); - const node = await svc_fs.node(new NodePathSelector(path)); - const has_read = await svc_acl.check(actor, node, 'read'); - const has_write = await svc_acl.check(actor, node, 'write'); - - if ( level !== 'write' && level !== 'read' ) { - return { - message: 'unexpected value for "level" parameter', - }; - } - - if ( level === 'read' && has_write ) { - return { - message: 'expected read-only but actor can write', - }; - } - if ( level === 'read' && !has_read ) { - return { - message: 'expected read access but no read access', - }; - } - if ( level === 'write' && (!has_write || !has_read) ) { - return { - message: 'expected write access but no write access', - }; - } - if ( level === 'manage' && (!has_write || !has_read) ) { - return { - message: 'expected write access but no write access', - }; - } - } -} - -module.exports = { - ShareTestService, -}; diff --git a/mods/mods_available/kdmod/data/sharetest_scenarios.js b/mods/mods_available/kdmod/data/sharetest_scenarios.js deleted file mode 100644 index 02c3d61ac..000000000 --- a/mods/mods_available/kdmod/data/sharetest_scenarios.js +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -module.exports = [ - { - sequence: [ - { - title: 'Kyle creates a file', - call: 'create-example-file', - as: 'testuser_kyle', - with: { - name: 'example.txt', - contents: 'secret file', - }, - }, - { - title: 'Eric tries to access it', - call: 'assert-no-access', - as: 'testuser_eric', - with: { - path: '/testuser_kyle/Desktop/example.txt', - }, - }, - ], - }, - { - sequence: [ - { - title: 'Stan creates a file', - call: 'create-example-file', - as: 'testuser_stan', - with: { - name: 'example.txt', - contents: 'secret file', - }, - }, - { - title: 'Stan grants permission to Eric', - call: 'grant', - as: 'testuser_stan', - with: { - to: 'testuser_eric', - permission: 'fs:/testuser_stan/Desktop/example.txt:read', - }, - }, - { - title: 'Eric tries to access it', - call: 'assert-access', - as: 'testuser_eric', - with: { - path: '/testuser_stan/Desktop/example.txt', - level: 'read', - }, - }, - ], - }, - { - sequence: [ - { - title: 'Stan grants Kyle\'s file to Eric', - call: 'grant', - as: 'testuser_stan', - with: { - to: 'testuser_eric', - permission: 'fs:/testuser_kyle/Desktop/example.txt:read', - }, - }, - { - title: 'Eric tries to access it', - call: 'assert-no-access', - as: 'testuser_eric', - with: { - path: '/testuser_kyle/Desktop/example.txt', - }, - }, - ], - }, -]; diff --git a/mods/mods_available/kdmod/gui/main.js b/mods/mods_available/kdmod/gui/main.js deleted file mode 100644 index cdcb7f723..000000000 --- a/mods/mods_available/kdmod/gui/main.js +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const request_examples = [ - { - name: 'entity storage app read', - fetch: async (args) => { - return await fetch(`${window.api_origin}/drivers/call`, { - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${puter.authToken}`, - }, - body: JSON.stringify({ - interface: 'puter-apps', - method: 'read', - args, - }), - method: 'POST', - }); - }, - out: async (resp) => { - const data = await resp.json(); - if ( ! data.success ) return data; - return data.result; - }, - exec: async function exec (...a) { - const resp = await this.fetch(...a); - return await this.out(resp); - }, - }, - { - name: 'entity storage app select all', - fetch: async () => { - return await fetch(`${window.api_origin}/drivers/call`, { - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${puter.authToken}`, - }, - body: JSON.stringify({ - interface: 'puter-apps', - method: 'select', - args: { predicate: [] }, - }), - method: 'POST', - }); - }, - out: async (resp) => { - const data = await resp.json(); - if ( ! data.success ) return data; - return data.result; - }, - exec: async function exec (...a) { - const resp = await this.fetch(...a); - return await this.out(resp); - }, - }, - { - name: 'grant permission from a user to a user', - fetch: async (user, perm) => { - return await fetch(`${window.api_origin}/auth/grant-user-user`, { - 'headers': { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${puter.authToken}`, - }, - 'body': JSON.stringify({ - target_username: user, - permission: perm, - }), - 'method': 'POST', - }); - }, - out: async (resp) => { - const data = await resp.json(); - return data; - }, - exec: async function exec (...a) { - const resp = await this.fetch(...a); - return await this.out(resp); - }, - }, - { - name: 'write file', - fetch: async (path, str) => { - const endpoint = `${window.api_origin}/write`; - const token = puter.authToken; - - const blob = new Blob([str], { type: 'text/plain' }); - const formData = new FormData(); - formData.append('create_missing_ancestors', true); - formData.append('path', path); - formData.append('size', 8); - formData.append('overwrite', true); - formData.append('file', blob, 'something.txt'); - - const response = await fetch(endpoint, { - method: 'POST', - headers: { 'Authorization': `Bearer ${token}` }, - body: formData, - }); - return await response.json(); - }, - }, -]; - -globalThis.reqex = request_examples; - -globalThis.service_script(api => { - api.on_ready(() => { - }); -}); diff --git a/mods/mods_available/kdmod/package.json b/mods/mods_available/kdmod/package.json deleted file mode 100644 index 26da3846e..000000000 --- a/mods/mods_available/kdmod/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "custom-puter-mod", - "version": "1.0.0", - "description": "", - "main": "module.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "keywords": [], - "author": "", - "license": "AGPL-3.0-only" -} diff --git a/mods/mods_available/test-actions/main.js b/mods/mods_available/test-actions/main.js deleted file mode 100644 index 3d234f151..000000000 --- a/mods/mods_available/test-actions/main.js +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Test-actions extension: declarative actions page for testing user suspension - * and other admin actions. All changes in this single file. - */ - -const { db } = extension.import('data'); -const { invalidate_cached_user } = use('core.util.helpers'); - -// Declarative actions: id, label, and inputs drive the generated GUI. -const ACTIONS = [ - { - id: 'suspend-user', - label: 'Suspend user', - inputs: [ - { name: 'username', label: 'Username', type: 'text' }, - ], - }, - // Add more actions here; each needs a handler in INVOKE_HANDLERS. -]; - -// Handlers for each action id. Receives (req, res, body). -const INVOKE_HANDLERS = { - 'suspend-user': async (req, res, body) => { - const username = body?.username?.trim(); - if ( ! username ) { - return res.status(400).json({ ok: false, error: 'username is required' }); - } - const svc_get_user = req.services.get('get-user'); - const user = await svc_get_user.get_user({ username }); - if ( ! user ) { - return res.status(404).json({ ok: false, error: 'User not found' }); - } - await db.write('UPDATE `user` SET suspended = 1 WHERE id = ? LIMIT 1', [user.id]); - invalidate_cached_user(user); - // Cache invalidation would require backend helpers (ESM); skipped here. - return res.json({ ok: true, message: `User "${username}" suspended.` }); - }, -}; - -const PAGE_HTML = (actionsJson) => ` - - - - Test actions - - - -

Test actions

-
- - - -`; - -extension.get('/test-actions', (req, res) => { - res.setHeader('Content-Type', 'text/html; charset=utf-8'); - res.send(PAGE_HTML(JSON.stringify(ACTIONS))); -}); - -extension.post('/test-actions/invoke/:actionId', async (req, res) => { - const actionId = req.params.actionId; - const handler = INVOKE_HANDLERS[actionId]; - if ( ! handler ) { - return res.status(404).json({ ok: false, error: 'Unknown action' }); - } - return handler(req, res, req.body || {}); -}); - -extension.on('ai.prompt.validate', async event => { - console.log('ai.prompt.validate'); - const messages = event.parameters?.messages ?? []; - console.log(`ai prompt validate: ${messages.length} messages`); - - console.log('is user suspended?', event.actor.type.user.suspended); -}); diff --git a/mods/mods_available/test-actions/package.json b/mods/mods_available/test-actions/package.json deleted file mode 100644 index d75646375..000000000 --- a/mods/mods_available/test-actions/package.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "@heyputer/test-actions", - "version": "1.0.0", - "description": "Actions for test purposes", - "main": "main.js", - "type": "module", - "private": true -} \ No newline at end of file diff --git a/mods/mods_available/testex.js b/mods/mods_available/testex.js deleted file mode 100644 index 07911b801..000000000 --- a/mods/mods_available/testex.js +++ /dev/null @@ -1,145 +0,0 @@ -// Test extension for event listeners - -extension.on('ai.prompt.complete', event => { - console.log('GOT AI.PROMPT.COMPLETE EVENT', event); -}); - -extension.on('ai.prompt.validate', event => { - console.log('GOT AI.PROMPT.VALIDATE EVENT', event); -}); - -extension.on('app.new-icon', event => { - console.log('GOT APP.NEW-ICON EVENT', event); -}); - -extension.on('app.rename', event => { - console.log('GOT APP.RENAME EVENT', event); -}); - -extension.on('apps.invalidate', event => { - console.log('GOT APPS.INVALIDATE EVENT', event); -}); - -extension.on('email.validate', event => { - console.log('GOT EMAIL.VALIDATE EVENT', event); -}); - -extension.on('fs.create.directory', event => { - console.log('GOT FS.CREATE.DIRECTORY EVENT', event); -}); - -extension.on('fs.create.file', event => { - console.log('GOT FS.CREATE.FILE EVENT', event); -}); - -extension.on('fs.create.shortcut', event => { - console.log('GOT FS.CREATE.SHORTCUT EVENT', event); -}); - -extension.on('fs.create.symlink', event => { - console.log('GOT FS.CREATE.SYMLINK EVENT', event); -}); - -extension.on('fs.move.file', event => { - console.log('GOT FS.MOVE.FILE EVENT', event); -}); - -extension.on('fs.pending.file', event => { - console.log('GOT FS.PENDING.FILE EVENT', event); -}); - -extension.on('fs.storage.progress.copy', event => { - console.log('GOT FS.STORAGE.PROGRESS.COPY EVENT', event); -}); - -extension.on('fs.storage.upload-progress', event => { - console.log('GOT FS.STORAGE.UPLOAD-PROGRESS EVENT', event); -}); - -extension.on('fs.write.file', event => { - console.log('GOT FS.WRITE.FILE EVENT', event); -}); - -extension.on('ip.validate', event => { - console.log('GOT IP.VALIDATE EVENT', event); -}); - -extension.on('outer.fs.write-hash', event => { - console.log('GOT OUTER.FS.WRITE-HASH EVENT', event); -}); - -extension.on('outer.gui.item.added', event => { - console.log('GOT OUTER.GUI.ITEM.ADDED EVENT', event); -}); - -extension.on('outer.gui.item.moved', event => { - console.log('GOT OUTER.GUI.ITEM.MOVED EVENT', event); -}); - -extension.on('outer.gui.item.pending', event => { - console.log('GOT OUTER.GUI.ITEM.PENDING EVENT', event); -}); - -extension.on('outer.gui.item.updated', event => { - console.log('GOT OUTER.GUI.ITEM.UPDATED EVENT', event); -}); - -extension.on('outer.gui.notif.ack', event => { - console.log('GOT OUTER.GUI.NOTIF.ACK EVENT', event); -}); - -extension.on('outer.gui.notif.message', event => { - console.log('GOT OUTER.GUI.NOTIF.MESSAGE EVENT', event); -}); - -extension.on('outer.gui.notif.persisted', event => { - console.log('GOT OUTER.GUI.NOTIF.PERSISTED EVENT', event); -}); - -extension.on('outer.gui.notif.unreads', event => { - console.log('GOT OUTER.GUI.NOTIF.UNREADS EVENT', event); -}); - -extension.on('outer.gui.submission.done', event => { - console.log('GOT OUTER.GUI.SUBMISSION.DONE EVENT', event); -}); - -extension.on('puter-exec.submission.done', event => { - console.log('GOT PUTER-EXEC.SUBMISSION.DONE EVENT', event); -}); - -extension.on('request.measured', event => { - console.log('GOT REQUEST.MEASURED EVENT', event); -}); - -extension.on('template-service.hello', event => { - console.log('GOT TEMPLATE-SERVICE.HELLO EVENT', event); -}); - -extension.on('usages.query', event => { - console.log('GOT USAGES.QUERY EVENT', event); -}); - -extension.on('user.email-changed', event => { - console.log('GOT USER.EMAIL-CHANGED EVENT', event); -}); - -extension.on('user.email-confirmed', event => { - console.log('GOT USER.EMAIL-CONFIRMED EVENT', event); -}); - -extension.on('user.save_account', event => { - console.log('GOT USER.SAVE_ACCOUNT EVENT', event); -}); - -extension.on('web.socket.connected', event => { - console.log('GOT WEB.SOCKET.CONNECTED EVENT', event); -}); - -extension.on('web.socket.user-connected', event => { - console.log('GOT WEB.SOCKET.USER-CONNECTED EVENT', event); -}); - -extension.on('wisp.get-policy', event => { - console.log('GOT WISP.GET-POLICY EVENT', event); -}); diff --git a/mods/mods_enabled/.gitignore b/mods/mods_enabled/.gitignore deleted file mode 100644 index d6b7ef32c..000000000 --- a/mods/mods_enabled/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/package-lock.json b/package-lock.json index 92f4df159..5f6f004a6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,7 @@ "": { "name": "puter.com", "version": "2.5.1", + "hasInstallScript": true, "license": "AGPL-3.0-only", "workspaces": [ "src/*", @@ -15,37 +16,19 @@ ], "dependencies": { "@ai-sdk/openai": "^3.0.25", - "@anthropic-ai/sdk": "^0.68.0", - "@aws-sdk/client-dynamodb": "^3.490.0", "@aws-sdk/client-s3": "^3.1020.0", - "@aws-sdk/client-secrets-manager": "^3.879.0", - "@aws-sdk/client-sns": "^3.907.0", - "@aws-sdk/credential-providers": "^3.1021.0", - "@aws-sdk/lib-dynamodb": "^3.490.0", - "@google/genai": "^1.19.0", + "@aws-sdk/s3-request-presigner": "^3.1028.0", "@heyputer/putility": "^1.0.2", - "@paralleldrive/cuid2": "^2.2.2", - "@stylistic/eslint-plugin-js": "^4.4.1", "ai": "^6.0.73", "dedent": "^1.5.3", - "dynalite": "^4.0.0", - "express": "^4.18.2", - "express-xml-bodyparser": "^0.4.1", - "fauxqs": "^2.5.0", - "file-type": "21.3.3", "javascript-time-ago": "^2.5.11", - "json-colorizer": "^3.0.1", - "music-metadata": "11.12.3", - "open": "^10.1.0", - "parse-domain": "^8.2.2", - "string-template": "^1.0.0", - "uuid": "^9.0.1" + "open": "^10.1.0" }, "devDependencies": { "@eslint/js": "^9.35.0", "@playwright/test": "^1.56.1", "@stylistic/eslint-plugin": "^5.3.1", - "@types/express": "^4.17.21", + "@types/express": "^5.0.0", "@types/mime-types": "^3.0.1", "@types/uuid": "^10.0.0", "@typescript-eslint/eslint-plugin": "^8.46.1", @@ -56,19 +39,20 @@ "clean-css": "^5.3.2", "dotenv": "^16.4.5", "eslint": "^9.35.0", + "eslint-config-prettier": "^10.1.8", + "eslint-plugin-prettier": "^5.5.5", "eslint-rule-composer": "^0.3.0", "globals": "^15.15.0", - "html-entities": "^2.3.3", "html-webpack-plugin": "^5.6.0", "husky": "^9.1.7", "license-check-and-add": "^4.0.5", "mocha": "^7.2.0", "nodemon": "^3.1.0", + "prettier": "^3.8.3", "simple-git": "^3.32.3", "typescript": "^5.4.5", "uglify-js": "^3.17.4", "vite-plugin-static-copy": "^3.3.0", - "vitest": "^4.0.14", "webpack": "^5.88.2", "webpack-cli": "^5.1.1", "yaml": "^2.8.1" @@ -77,22 +61,18 @@ "node": ">=24.0.0" }, "optionalDependencies": { - "@emnapi/core": "^1.9.2", - "@emnapi/runtime": "^1.9.2", - "sharp": "^0.34.4", - "sharp-bmp": "^0.1.5", - "sharp-ico": "^0.1.5" + "sharp": "^0.34.4" } }, "node_modules/@ai-sdk/gateway": { - "version": "3.0.95", - "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.95.tgz", - "integrity": "sha512-ZmUNNbZl3V42xwQzPaNUi+s8eqR2lnrxf0bvB6YbLXpLjHYv0k2Y78t12cNOfY0bxGeuVVTLyk856uLuQIuXEQ==", + "version": "3.0.104", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.104.tgz", + "integrity": "sha512-ZKX5n74io8VIRlhIMSLWVlvT3sXC8Z7cZ9GHuWBWZDVi96+62AIsWuLGvMfcBA1STYuSoDrp6rIziZmvrTq0TA==", "license": "Apache-2.0", "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.23", - "@vercel/oidc": "3.1.0" + "@vercel/oidc": "3.2.0" }, "engines": { "node": ">=18" @@ -102,9 +82,9 @@ } }, "node_modules/@ai-sdk/openai": { - "version": "3.0.52", - "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-3.0.52.tgz", - "integrity": "sha512-4Rr8NCGmfWTz6DCUvixn9UmyZcMatiHn0zWoMzI3JCUe9R1P/vsPOpCBALKoSzVYOjyJnhtnVIbfUKujcS39uw==", + "version": "3.0.53", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-3.0.53.tgz", + "integrity": "sha512-Wld+Rbc05KaUn08uBt06eEuwcgalcIFtIl32Yp+GxuZXUQwOb6YeAuq+C6da4ch6BurFoqEaLemJVwjBb7x+PQ==", "license": "Apache-2.0", "dependencies": { "@ai-sdk/provider": "3.0.8", @@ -167,14 +147,15 @@ } }, "node_modules/@asamuzakjp/css-color": { - "version": "5.1.9", - "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-5.1.9.tgz", - "integrity": "sha512-zd9c/Wdso6v1U7v6w3i/hbAr4K7NaSHImdpvmLt+Y9ea5BhilnIGNkfhOJ7FEIuPipAnE9tZeDOll05WDT0kgg==", + "version": "5.1.11", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-5.1.11.tgz", + "integrity": "sha512-KVw6qIiCTUQhByfTd78h2yD1/00waTmm9uy/R7Ck/ctUyAPj+AEDLkQIdJW0T8+qGgj3j5bpNKK7Q3G+LedJWg==", "dev": true, "license": "MIT", "dependencies": { - "@csstools/css-calc": "^3.1.1", - "@csstools/css-color-parser": "^4.0.2", + "@asamuzakjp/generational-cache": "^1.0.1", + "@csstools/css-calc": "^3.2.0", + "@csstools/css-color-parser": "^4.1.0", "@csstools/css-parser-algorithms": "^4.0.0", "@csstools/css-tokenizer": "^4.0.0" }, @@ -183,12 +164,13 @@ } }, "node_modules/@asamuzakjp/dom-selector": { - "version": "7.0.9", - "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-7.0.9.tgz", - "integrity": "sha512-r3ElRr7y8ucyN2KdICwGsmj19RoN13CLCa/pvGydghWK6ZzeKQ+TcDjVdtEZz2ElpndM5jXw//B9CEee0mWnVg==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-7.1.1.tgz", + "integrity": "sha512-67RZDnYRc8H/8MLDgQCDE//zoqVFwajkepHZgmXrbwybzXOEwOWGPYGmALYl9J2DOLfFPPs6kKCqmbzV895hTQ==", "dev": true, "license": "MIT", "dependencies": { + "@asamuzakjp/generational-cache": "^1.0.1", "@asamuzakjp/nwsapi": "^2.3.9", "bidi-js": "^1.0.3", "css-tree": "^3.2.1", @@ -198,6 +180,16 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" } }, + "node_modules/@asamuzakjp/generational-cache": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@asamuzakjp/generational-cache/-/generational-cache-1.0.1.tgz", + "integrity": "sha512-wajfB8KqzMCN2KGNFdLkReeHncd0AslUSrvHVvvYWuU8ghncRJoA50kT3zP9MVL0+9g4/67H+cdvBskj9THPzg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, "node_modules/@asamuzakjp/nwsapi": { "version": "2.3.9", "resolved": "https://registry.npmjs.org/@asamuzakjp/nwsapi/-/nwsapi-2.3.9.tgz", @@ -407,116 +399,49 @@ "node": ">=14.0.0" } }, - "node_modules/@aws-sdk/client-cloudwatch": { - "version": "3.1028.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudwatch/-/client-cloudwatch-3.1028.0.tgz", - "integrity": "sha512-VKo8xQZzCO8CpwfytSP1pr75+YGUPqg2m8Ki3WKhkDOr7qKU6GssDoob3cCR9QThmgTLc7D+pM1SOUmvRccnhw==", - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/credential-provider-node": "^3.972.30", - "@aws-sdk/middleware-host-header": "^3.972.9", - "@aws-sdk/middleware-logger": "^3.972.9", - "@aws-sdk/middleware-recursion-detection": "^3.972.10", - "@aws-sdk/middleware-user-agent": "^3.972.29", - "@aws-sdk/region-config-resolver": "^3.972.11", - "@aws-sdk/types": "^3.973.7", - "@aws-sdk/util-endpoints": "^3.996.6", - "@aws-sdk/util-user-agent-browser": "^3.972.9", - "@aws-sdk/util-user-agent-node": "^3.973.15", - "@smithy/config-resolver": "^4.4.14", - "@smithy/core": "^3.23.14", - "@smithy/fetch-http-handler": "^5.3.16", - "@smithy/hash-node": "^4.2.13", - "@smithy/invalid-dependency": "^4.2.13", - "@smithy/middleware-compression": "^4.3.43", - "@smithy/middleware-content-length": "^4.2.13", - "@smithy/middleware-endpoint": "^4.4.29", - "@smithy/middleware-retry": "^4.5.0", - "@smithy/middleware-serde": "^4.2.17", - "@smithy/middleware-stack": "^4.2.13", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/node-http-handler": "^4.5.2", - "@smithy/protocol-http": "^5.3.13", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", - "@smithy/url-parser": "^4.2.13", - "@smithy/util-base64": "^4.3.2", - "@smithy/util-body-length-browser": "^4.2.2", - "@smithy/util-body-length-node": "^4.2.3", - "@smithy/util-defaults-mode-browser": "^4.3.45", - "@smithy/util-defaults-mode-node": "^4.2.49", - "@smithy/util-endpoints": "^3.3.4", - "@smithy/util-middleware": "^4.2.13", - "@smithy/util-retry": "^4.3.0", - "@smithy/util-utf8": "^4.2.2", - "@smithy/util-waiter": "^4.2.15", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/client-cloudwatch/node_modules/@smithy/node-http-handler": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.5.2.tgz", - "integrity": "sha512-/oD7u8M0oj2ZTFw7GkuuHWpIxtWdLlnyNkbrWcyVYhd5RJNDuczdkb0wfnQICyNFrVPlr8YHOhamjNy3zidhmA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.13", - "@smithy/querystring-builder": "^4.2.13", - "@smithy/types": "^4.14.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@aws-sdk/client-cognito-identity": { - "version": "3.1028.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.1028.0.tgz", - "integrity": "sha512-T94NnSifr6PPd66exnzK+QS7tQfo/tTw7ZvkHh6FmcepK4mFNm12s5OEIu/XYiVWSX4RaBCwYIAZvcjqXnWHGQ==", + "version": "3.1032.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.1032.0.tgz", + "integrity": "sha512-TVgbjyb1fJoHZDoBAmW85hNcx00zxi5qXFG3wvS/2C213Q2PusCQIih7Zlub9mKE3iRtES5epxazFmp8jVeLyQ==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/credential-provider-node": "^3.972.30", - "@aws-sdk/middleware-host-header": "^3.972.9", - "@aws-sdk/middleware-logger": "^3.972.9", - "@aws-sdk/middleware-recursion-detection": "^3.972.10", - "@aws-sdk/middleware-user-agent": "^3.972.29", - "@aws-sdk/region-config-resolver": "^3.972.11", - "@aws-sdk/types": "^3.973.7", - "@aws-sdk/util-endpoints": "^3.996.6", - "@aws-sdk/util-user-agent-browser": "^3.972.9", - "@aws-sdk/util-user-agent-node": "^3.973.15", - "@smithy/config-resolver": "^4.4.14", - "@smithy/core": "^3.23.14", - "@smithy/fetch-http-handler": "^5.3.16", - "@smithy/hash-node": "^4.2.13", - "@smithy/invalid-dependency": "^4.2.13", - "@smithy/middleware-content-length": "^4.2.13", - "@smithy/middleware-endpoint": "^4.4.29", - "@smithy/middleware-retry": "^4.5.0", - "@smithy/middleware-serde": "^4.2.17", - "@smithy/middleware-stack": "^4.2.13", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/node-http-handler": "^4.5.2", - "@smithy/protocol-http": "^5.3.13", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", - "@smithy/url-parser": "^4.2.13", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/credential-provider-node": "^3.972.32", + "@aws-sdk/middleware-host-header": "^3.972.10", + "@aws-sdk/middleware-logger": "^3.972.10", + "@aws-sdk/middleware-recursion-detection": "^3.972.11", + "@aws-sdk/middleware-user-agent": "^3.972.31", + "@aws-sdk/region-config-resolver": "^3.972.12", + "@aws-sdk/types": "^3.973.8", + "@aws-sdk/util-endpoints": "^3.996.7", + "@aws-sdk/util-user-agent-browser": "^3.972.10", + "@aws-sdk/util-user-agent-node": "^3.973.17", + "@smithy/config-resolver": "^4.4.16", + "@smithy/core": "^3.23.15", + "@smithy/fetch-http-handler": "^5.3.17", + "@smithy/hash-node": "^4.2.14", + "@smithy/invalid-dependency": "^4.2.14", + "@smithy/middleware-content-length": "^4.2.14", + "@smithy/middleware-endpoint": "^4.4.30", + "@smithy/middleware-retry": "^4.5.3", + "@smithy/middleware-serde": "^4.2.18", + "@smithy/middleware-stack": "^4.2.14", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/node-http-handler": "^4.5.3", + "@smithy/protocol-http": "^5.3.14", + "@smithy/smithy-client": "^4.12.11", + "@smithy/types": "^4.14.1", + "@smithy/url-parser": "^4.2.14", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-body-length-node": "^4.2.3", - "@smithy/util-defaults-mode-browser": "^4.3.45", - "@smithy/util-defaults-mode-node": "^4.2.49", - "@smithy/util-endpoints": "^3.3.4", - "@smithy/util-middleware": "^4.2.13", - "@smithy/util-retry": "^4.3.0", + "@smithy/util-defaults-mode-browser": "^4.3.47", + "@smithy/util-defaults-mode-node": "^4.2.52", + "@smithy/util-endpoints": "^3.4.1", + "@smithy/util-middleware": "^4.2.14", + "@smithy/util-retry": "^4.3.2", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, @@ -524,138 +449,108 @@ "node": ">=20.0.0" } }, - "node_modules/@aws-sdk/client-cognito-identity/node_modules/@smithy/node-http-handler": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.5.2.tgz", - "integrity": "sha512-/oD7u8M0oj2ZTFw7GkuuHWpIxtWdLlnyNkbrWcyVYhd5RJNDuczdkb0wfnQICyNFrVPlr8YHOhamjNy3zidhmA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.13", - "@smithy/querystring-builder": "^4.2.13", - "@smithy/types": "^4.14.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@aws-sdk/client-dynamodb": { - "version": "3.1028.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.1028.0.tgz", - "integrity": "sha512-OkO2p9Wm+6CccOfQcdYjvCAJdUfBSWbmrIKXGh/qbBjp0B8d1MsYl1Exps5OzRSzqLVuTUVjPJCkgSMJF/mPqg==", + "version": "3.1032.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.1032.0.tgz", + "integrity": "sha512-kkXiZBNdWCQAg/8opqAu10TxzdpqMkcGrNAT2ScdfWhCpzYZ2pmSpP8W7BOlA32jYIWnYrEdb808UZsNWYBPAA==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/credential-provider-node": "^3.972.30", - "@aws-sdk/dynamodb-codec": "^3.972.28", - "@aws-sdk/middleware-endpoint-discovery": "^3.972.10", - "@aws-sdk/middleware-host-header": "^3.972.9", - "@aws-sdk/middleware-logger": "^3.972.9", - "@aws-sdk/middleware-recursion-detection": "^3.972.10", - "@aws-sdk/middleware-user-agent": "^3.972.29", - "@aws-sdk/region-config-resolver": "^3.972.11", - "@aws-sdk/types": "^3.973.7", - "@aws-sdk/util-endpoints": "^3.996.6", - "@aws-sdk/util-user-agent-browser": "^3.972.9", - "@aws-sdk/util-user-agent-node": "^3.973.15", - "@smithy/config-resolver": "^4.4.14", - "@smithy/core": "^3.23.14", - "@smithy/fetch-http-handler": "^5.3.16", - "@smithy/hash-node": "^4.2.13", - "@smithy/invalid-dependency": "^4.2.13", - "@smithy/middleware-content-length": "^4.2.13", - "@smithy/middleware-endpoint": "^4.4.29", - "@smithy/middleware-retry": "^4.5.0", - "@smithy/middleware-serde": "^4.2.17", - "@smithy/middleware-stack": "^4.2.13", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/node-http-handler": "^4.5.2", - "@smithy/protocol-http": "^5.3.13", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", - "@smithy/url-parser": "^4.2.13", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/credential-provider-node": "^3.972.32", + "@aws-sdk/dynamodb-codec": "^3.973.1", + "@aws-sdk/middleware-endpoint-discovery": "^3.972.11", + "@aws-sdk/middleware-host-header": "^3.972.10", + "@aws-sdk/middleware-logger": "^3.972.10", + "@aws-sdk/middleware-recursion-detection": "^3.972.11", + "@aws-sdk/middleware-user-agent": "^3.972.31", + "@aws-sdk/region-config-resolver": "^3.972.12", + "@aws-sdk/types": "^3.973.8", + "@aws-sdk/util-endpoints": "^3.996.7", + "@aws-sdk/util-user-agent-browser": "^3.972.10", + "@aws-sdk/util-user-agent-node": "^3.973.17", + "@smithy/config-resolver": "^4.4.16", + "@smithy/core": "^3.23.15", + "@smithy/fetch-http-handler": "^5.3.17", + "@smithy/hash-node": "^4.2.14", + "@smithy/invalid-dependency": "^4.2.14", + "@smithy/middleware-content-length": "^4.2.14", + "@smithy/middleware-endpoint": "^4.4.30", + "@smithy/middleware-retry": "^4.5.3", + "@smithy/middleware-serde": "^4.2.18", + "@smithy/middleware-stack": "^4.2.14", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/node-http-handler": "^4.5.3", + "@smithy/protocol-http": "^5.3.14", + "@smithy/smithy-client": "^4.12.11", + "@smithy/types": "^4.14.1", + "@smithy/url-parser": "^4.2.14", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-body-length-node": "^4.2.3", - "@smithy/util-defaults-mode-browser": "^4.3.45", - "@smithy/util-defaults-mode-node": "^4.2.49", - "@smithy/util-endpoints": "^3.3.4", - "@smithy/util-middleware": "^4.2.13", - "@smithy/util-retry": "^4.3.0", + "@smithy/util-defaults-mode-browser": "^4.3.47", + "@smithy/util-defaults-mode-node": "^4.2.52", + "@smithy/util-endpoints": "^3.4.1", + "@smithy/util-middleware": "^4.2.14", + "@smithy/util-retry": "^4.3.2", "@smithy/util-utf8": "^4.2.2", - "@smithy/util-waiter": "^4.2.15", + "@smithy/util-waiter": "^4.2.16", "tslib": "^2.6.2" }, "engines": { "node": ">=20.0.0" } }, - "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/node-http-handler": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.5.2.tgz", - "integrity": "sha512-/oD7u8M0oj2ZTFw7GkuuHWpIxtWdLlnyNkbrWcyVYhd5RJNDuczdkb0wfnQICyNFrVPlr8YHOhamjNy3zidhmA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.13", - "@smithy/querystring-builder": "^4.2.13", - "@smithy/types": "^4.14.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@aws-sdk/client-polly": { - "version": "3.1028.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-polly/-/client-polly-3.1028.0.tgz", - "integrity": "sha512-/5BvY3U1rR7yqKHQOxOUJkit7rTbOeExUSRCpSNMx1+TzdTWj6u9OAxcNPUwmVCuRWoDTvCsX8DFHbOcOesNAw==", + "version": "3.1032.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-polly/-/client-polly-3.1032.0.tgz", + "integrity": "sha512-sueBgoP/nnsUS1h5karBuX70Odu6H4xs9qm1NJCmAWN5ROWAqC7Xwg10MY2uDfn3vtlJORq3xggcXnm+UEslSA==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/credential-provider-node": "^3.972.30", - "@aws-sdk/eventstream-handler-node": "^3.972.13", - "@aws-sdk/middleware-eventstream": "^3.972.9", - "@aws-sdk/middleware-host-header": "^3.972.9", - "@aws-sdk/middleware-logger": "^3.972.9", - "@aws-sdk/middleware-recursion-detection": "^3.972.10", - "@aws-sdk/middleware-user-agent": "^3.972.29", - "@aws-sdk/region-config-resolver": "^3.972.11", - "@aws-sdk/types": "^3.973.7", - "@aws-sdk/util-endpoints": "^3.996.6", - "@aws-sdk/util-user-agent-browser": "^3.972.9", - "@aws-sdk/util-user-agent-node": "^3.973.15", - "@smithy/config-resolver": "^4.4.14", - "@smithy/core": "^3.23.14", - "@smithy/eventstream-serde-browser": "^4.2.13", - "@smithy/eventstream-serde-config-resolver": "^4.3.13", - "@smithy/eventstream-serde-node": "^4.2.13", - "@smithy/fetch-http-handler": "^5.3.16", - "@smithy/hash-node": "^4.2.13", - "@smithy/invalid-dependency": "^4.2.13", - "@smithy/middleware-content-length": "^4.2.13", - "@smithy/middleware-endpoint": "^4.4.29", - "@smithy/middleware-retry": "^4.5.0", - "@smithy/middleware-serde": "^4.2.17", - "@smithy/middleware-stack": "^4.2.13", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/node-http-handler": "^4.5.2", - "@smithy/protocol-http": "^5.3.13", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", - "@smithy/url-parser": "^4.2.13", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/credential-provider-node": "^3.972.32", + "@aws-sdk/eventstream-handler-node": "^3.972.14", + "@aws-sdk/middleware-eventstream": "^3.972.10", + "@aws-sdk/middleware-host-header": "^3.972.10", + "@aws-sdk/middleware-logger": "^3.972.10", + "@aws-sdk/middleware-recursion-detection": "^3.972.11", + "@aws-sdk/middleware-user-agent": "^3.972.31", + "@aws-sdk/region-config-resolver": "^3.972.12", + "@aws-sdk/types": "^3.973.8", + "@aws-sdk/util-endpoints": "^3.996.7", + "@aws-sdk/util-user-agent-browser": "^3.972.10", + "@aws-sdk/util-user-agent-node": "^3.973.17", + "@smithy/config-resolver": "^4.4.16", + "@smithy/core": "^3.23.15", + "@smithy/eventstream-serde-browser": "^4.2.14", + "@smithy/eventstream-serde-config-resolver": "^4.3.14", + "@smithy/eventstream-serde-node": "^4.2.14", + "@smithy/fetch-http-handler": "^5.3.17", + "@smithy/hash-node": "^4.2.14", + "@smithy/invalid-dependency": "^4.2.14", + "@smithy/middleware-content-length": "^4.2.14", + "@smithy/middleware-endpoint": "^4.4.30", + "@smithy/middleware-retry": "^4.5.3", + "@smithy/middleware-serde": "^4.2.18", + "@smithy/middleware-stack": "^4.2.14", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/node-http-handler": "^4.5.3", + "@smithy/protocol-http": "^5.3.14", + "@smithy/smithy-client": "^4.12.11", + "@smithy/types": "^4.14.1", + "@smithy/url-parser": "^4.2.14", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-body-length-node": "^4.2.3", - "@smithy/util-defaults-mode-browser": "^4.3.45", - "@smithy/util-defaults-mode-node": "^4.2.49", - "@smithy/util-endpoints": "^3.3.4", - "@smithy/util-middleware": "^4.2.13", - "@smithy/util-retry": "^4.3.0", - "@smithy/util-stream": "^4.5.22", + "@smithy/util-defaults-mode-browser": "^4.3.47", + "@smithy/util-defaults-mode-node": "^4.2.52", + "@smithy/util-endpoints": "^3.4.1", + "@smithy/util-middleware": "^4.2.14", + "@smithy/util-retry": "^4.3.2", + "@smithy/util-stream": "^4.5.23", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, @@ -663,210 +558,115 @@ "node": ">=20.0.0" } }, - "node_modules/@aws-sdk/client-polly/node_modules/@smithy/node-http-handler": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.5.2.tgz", - "integrity": "sha512-/oD7u8M0oj2ZTFw7GkuuHWpIxtWdLlnyNkbrWcyVYhd5RJNDuczdkb0wfnQICyNFrVPlr8YHOhamjNy3zidhmA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.13", - "@smithy/querystring-builder": "^4.2.13", - "@smithy/types": "^4.14.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@aws-sdk/client-s3": { - "version": "3.1028.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1028.0.tgz", - "integrity": "sha512-KL8PREFJxyWXUjMQR6Krq/OjZ5qbcV1QFjtA7Q7oMW5XaFO9YoSBtBxQeeXO4um6vYSmRVYVDTvEKZDcNbyeXw==", + "version": "3.1032.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1032.0.tgz", + "integrity": "sha512-A1wjVhV3IgsZ5td2l4AWgK03EjZ+ldwbiorxuO1hPf7RHJtSdr6oq/gKzyUwP7Tm7ma/M2xS/tplg5C8XB8RWg==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha1-browser": "5.2.0", "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/credential-provider-node": "^3.972.30", - "@aws-sdk/middleware-bucket-endpoint": "^3.972.9", - "@aws-sdk/middleware-expect-continue": "^3.972.9", - "@aws-sdk/middleware-flexible-checksums": "^3.974.7", - "@aws-sdk/middleware-host-header": "^3.972.9", - "@aws-sdk/middleware-location-constraint": "^3.972.9", - "@aws-sdk/middleware-logger": "^3.972.9", - "@aws-sdk/middleware-recursion-detection": "^3.972.10", - "@aws-sdk/middleware-sdk-s3": "^3.972.28", - "@aws-sdk/middleware-ssec": "^3.972.9", - "@aws-sdk/middleware-user-agent": "^3.972.29", - "@aws-sdk/region-config-resolver": "^3.972.11", - "@aws-sdk/signature-v4-multi-region": "^3.996.16", - "@aws-sdk/types": "^3.973.7", - "@aws-sdk/util-endpoints": "^3.996.6", - "@aws-sdk/util-user-agent-browser": "^3.972.9", - "@aws-sdk/util-user-agent-node": "^3.973.15", - "@smithy/config-resolver": "^4.4.14", - "@smithy/core": "^3.23.14", - "@smithy/eventstream-serde-browser": "^4.2.13", - "@smithy/eventstream-serde-config-resolver": "^4.3.13", - "@smithy/eventstream-serde-node": "^4.2.13", - "@smithy/fetch-http-handler": "^5.3.16", - "@smithy/hash-blob-browser": "^4.2.14", - "@smithy/hash-node": "^4.2.13", - "@smithy/hash-stream-node": "^4.2.13", - "@smithy/invalid-dependency": "^4.2.13", - "@smithy/md5-js": "^4.2.13", - "@smithy/middleware-content-length": "^4.2.13", - "@smithy/middleware-endpoint": "^4.4.29", - "@smithy/middleware-retry": "^4.5.0", - "@smithy/middleware-serde": "^4.2.17", - "@smithy/middleware-stack": "^4.2.13", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/node-http-handler": "^4.5.2", - "@smithy/protocol-http": "^5.3.13", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", - "@smithy/url-parser": "^4.2.13", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/credential-provider-node": "^3.972.32", + "@aws-sdk/middleware-bucket-endpoint": "^3.972.10", + "@aws-sdk/middleware-expect-continue": "^3.972.10", + "@aws-sdk/middleware-flexible-checksums": "^3.974.9", + "@aws-sdk/middleware-host-header": "^3.972.10", + "@aws-sdk/middleware-location-constraint": "^3.972.10", + "@aws-sdk/middleware-logger": "^3.972.10", + "@aws-sdk/middleware-recursion-detection": "^3.972.11", + "@aws-sdk/middleware-sdk-s3": "^3.972.30", + "@aws-sdk/middleware-ssec": "^3.972.10", + "@aws-sdk/middleware-user-agent": "^3.972.31", + "@aws-sdk/region-config-resolver": "^3.972.12", + "@aws-sdk/signature-v4-multi-region": "^3.996.18", + "@aws-sdk/types": "^3.973.8", + "@aws-sdk/util-endpoints": "^3.996.7", + "@aws-sdk/util-user-agent-browser": "^3.972.10", + "@aws-sdk/util-user-agent-node": "^3.973.17", + "@smithy/config-resolver": "^4.4.16", + "@smithy/core": "^3.23.15", + "@smithy/eventstream-serde-browser": "^4.2.14", + "@smithy/eventstream-serde-config-resolver": "^4.3.14", + "@smithy/eventstream-serde-node": "^4.2.14", + "@smithy/fetch-http-handler": "^5.3.17", + "@smithy/hash-blob-browser": "^4.2.15", + "@smithy/hash-node": "^4.2.14", + "@smithy/hash-stream-node": "^4.2.14", + "@smithy/invalid-dependency": "^4.2.14", + "@smithy/md5-js": "^4.2.14", + "@smithy/middleware-content-length": "^4.2.14", + "@smithy/middleware-endpoint": "^4.4.30", + "@smithy/middleware-retry": "^4.5.3", + "@smithy/middleware-serde": "^4.2.18", + "@smithy/middleware-stack": "^4.2.14", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/node-http-handler": "^4.5.3", + "@smithy/protocol-http": "^5.3.14", + "@smithy/smithy-client": "^4.12.11", + "@smithy/types": "^4.14.1", + "@smithy/url-parser": "^4.2.14", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-body-length-node": "^4.2.3", - "@smithy/util-defaults-mode-browser": "^4.3.45", - "@smithy/util-defaults-mode-node": "^4.2.49", - "@smithy/util-endpoints": "^3.3.4", - "@smithy/util-middleware": "^4.2.13", - "@smithy/util-retry": "^4.3.0", - "@smithy/util-stream": "^4.5.22", + "@smithy/util-defaults-mode-browser": "^4.3.47", + "@smithy/util-defaults-mode-node": "^4.2.52", + "@smithy/util-endpoints": "^3.4.1", + "@smithy/util-middleware": "^4.2.14", + "@smithy/util-retry": "^4.3.2", + "@smithy/util-stream": "^4.5.23", "@smithy/util-utf8": "^4.2.2", - "@smithy/util-waiter": "^4.2.15", + "@smithy/util-waiter": "^4.2.16", "tslib": "^2.6.2" }, "engines": { "node": ">=20.0.0" } }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/node-http-handler": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.5.2.tgz", - "integrity": "sha512-/oD7u8M0oj2ZTFw7GkuuHWpIxtWdLlnyNkbrWcyVYhd5RJNDuczdkb0wfnQICyNFrVPlr8YHOhamjNy3zidhmA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.13", - "@smithy/querystring-builder": "^4.2.13", - "@smithy/types": "^4.14.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@aws-sdk/client-secrets-manager": { - "version": "3.1028.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-secrets-manager/-/client-secrets-manager-3.1028.0.tgz", - "integrity": "sha512-Vj+pgAb8raFIxUh0WCFI3fhYY68lN1tGFPj+EFauB8EUbgz9BA3TuI8plFRXvq+h9m1gXvW/VbQTzCxkoeChdA==", - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/credential-provider-node": "^3.972.30", - "@aws-sdk/middleware-host-header": "^3.972.9", - "@aws-sdk/middleware-logger": "^3.972.9", - "@aws-sdk/middleware-recursion-detection": "^3.972.10", - "@aws-sdk/middleware-user-agent": "^3.972.29", - "@aws-sdk/region-config-resolver": "^3.972.11", - "@aws-sdk/types": "^3.973.7", - "@aws-sdk/util-endpoints": "^3.996.6", - "@aws-sdk/util-user-agent-browser": "^3.972.9", - "@aws-sdk/util-user-agent-node": "^3.973.15", - "@smithy/config-resolver": "^4.4.14", - "@smithy/core": "^3.23.14", - "@smithy/fetch-http-handler": "^5.3.16", - "@smithy/hash-node": "^4.2.13", - "@smithy/invalid-dependency": "^4.2.13", - "@smithy/middleware-content-length": "^4.2.13", - "@smithy/middleware-endpoint": "^4.4.29", - "@smithy/middleware-retry": "^4.5.0", - "@smithy/middleware-serde": "^4.2.17", - "@smithy/middleware-stack": "^4.2.13", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/node-http-handler": "^4.5.2", - "@smithy/protocol-http": "^5.3.13", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", - "@smithy/url-parser": "^4.2.13", - "@smithy/util-base64": "^4.3.2", - "@smithy/util-body-length-browser": "^4.2.2", - "@smithy/util-body-length-node": "^4.2.3", - "@smithy/util-defaults-mode-browser": "^4.3.45", - "@smithy/util-defaults-mode-node": "^4.2.49", - "@smithy/util-endpoints": "^3.3.4", - "@smithy/util-middleware": "^4.2.13", - "@smithy/util-retry": "^4.3.0", - "@smithy/util-utf8": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/client-secrets-manager/node_modules/@smithy/node-http-handler": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.5.2.tgz", - "integrity": "sha512-/oD7u8M0oj2ZTFw7GkuuHWpIxtWdLlnyNkbrWcyVYhd5RJNDuczdkb0wfnQICyNFrVPlr8YHOhamjNy3zidhmA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.13", - "@smithy/querystring-builder": "^4.2.13", - "@smithy/types": "^4.14.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@aws-sdk/client-sns": { - "version": "3.1028.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sns/-/client-sns-3.1028.0.tgz", - "integrity": "sha512-izYwsbiPwCQxmGAl2KAT2ZWcZd6Gg2+ylw51zOblwxRZIJ0xXHiltDr5gXadXaa3B5h02oW3TNe8Qq/PNK7iBg==", + "version": "3.1032.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sns/-/client-sns-3.1032.0.tgz", + "integrity": "sha512-WebJZGkQArdZ4YTvZZKmHdqbkcG4hyf6fzba1Z2yG+fIyzNB/MTODRHPByzaq9tKL8bK6wWCU9/9dgLCPIs4cQ==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/credential-provider-node": "^3.972.30", - "@aws-sdk/middleware-host-header": "^3.972.9", - "@aws-sdk/middleware-logger": "^3.972.9", - "@aws-sdk/middleware-recursion-detection": "^3.972.10", - "@aws-sdk/middleware-user-agent": "^3.972.29", - "@aws-sdk/region-config-resolver": "^3.972.11", - "@aws-sdk/types": "^3.973.7", - "@aws-sdk/util-endpoints": "^3.996.6", - "@aws-sdk/util-user-agent-browser": "^3.972.9", - "@aws-sdk/util-user-agent-node": "^3.973.15", - "@smithy/config-resolver": "^4.4.14", - "@smithy/core": "^3.23.14", - "@smithy/fetch-http-handler": "^5.3.16", - "@smithy/hash-node": "^4.2.13", - "@smithy/invalid-dependency": "^4.2.13", - "@smithy/middleware-content-length": "^4.2.13", - "@smithy/middleware-endpoint": "^4.4.29", - "@smithy/middleware-retry": "^4.5.0", - "@smithy/middleware-serde": "^4.2.17", - "@smithy/middleware-stack": "^4.2.13", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/node-http-handler": "^4.5.2", - "@smithy/protocol-http": "^5.3.13", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", - "@smithy/url-parser": "^4.2.13", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/credential-provider-node": "^3.972.32", + "@aws-sdk/middleware-host-header": "^3.972.10", + "@aws-sdk/middleware-logger": "^3.972.10", + "@aws-sdk/middleware-recursion-detection": "^3.972.11", + "@aws-sdk/middleware-user-agent": "^3.972.31", + "@aws-sdk/region-config-resolver": "^3.972.12", + "@aws-sdk/types": "^3.973.8", + "@aws-sdk/util-endpoints": "^3.996.7", + "@aws-sdk/util-user-agent-browser": "^3.972.10", + "@aws-sdk/util-user-agent-node": "^3.973.17", + "@smithy/config-resolver": "^4.4.16", + "@smithy/core": "^3.23.15", + "@smithy/fetch-http-handler": "^5.3.17", + "@smithy/hash-node": "^4.2.14", + "@smithy/invalid-dependency": "^4.2.14", + "@smithy/middleware-content-length": "^4.2.14", + "@smithy/middleware-endpoint": "^4.4.30", + "@smithy/middleware-retry": "^4.5.3", + "@smithy/middleware-serde": "^4.2.18", + "@smithy/middleware-stack": "^4.2.14", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/node-http-handler": "^4.5.3", + "@smithy/protocol-http": "^5.3.14", + "@smithy/smithy-client": "^4.12.11", + "@smithy/types": "^4.14.1", + "@smithy/url-parser": "^4.2.14", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-body-length-node": "^4.2.3", - "@smithy/util-defaults-mode-browser": "^4.3.45", - "@smithy/util-defaults-mode-node": "^4.2.49", - "@smithy/util-endpoints": "^3.3.4", - "@smithy/util-middleware": "^4.2.13", - "@smithy/util-retry": "^4.3.0", + "@smithy/util-defaults-mode-browser": "^4.3.47", + "@smithy/util-defaults-mode-node": "^4.2.52", + "@smithy/util-endpoints": "^3.4.1", + "@smithy/util-middleware": "^4.2.14", + "@smithy/util-retry": "^4.3.2", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, @@ -874,66 +674,51 @@ "node": ">=20.0.0" } }, - "node_modules/@aws-sdk/client-sns/node_modules/@smithy/node-http-handler": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.5.2.tgz", - "integrity": "sha512-/oD7u8M0oj2ZTFw7GkuuHWpIxtWdLlnyNkbrWcyVYhd5RJNDuczdkb0wfnQICyNFrVPlr8YHOhamjNy3zidhmA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.13", - "@smithy/querystring-builder": "^4.2.13", - "@smithy/types": "^4.14.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@aws-sdk/client-sqs": { - "version": "3.1028.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sqs/-/client-sqs-3.1028.0.tgz", - "integrity": "sha512-lysXICi1hSF0J2zUasEhNU8JeQQduJBfdfB0EvqpJ/8Dw904Ls7ZOOKR40Pkyfy4EPdm+VuDBDRVRXZM9hPIcw==", + "version": "3.1032.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sqs/-/client-sqs-3.1032.0.tgz", + "integrity": "sha512-n102sARTLi53Da0JT/2Kvg/bQ4bv+JqA+YQ8OlaM4CgsPn61sMv0x9PxdF6s/KbgZ2HMwYBszNzuvUttN+Beqg==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/credential-provider-node": "^3.972.30", - "@aws-sdk/middleware-host-header": "^3.972.9", - "@aws-sdk/middleware-logger": "^3.972.9", - "@aws-sdk/middleware-recursion-detection": "^3.972.10", - "@aws-sdk/middleware-sdk-sqs": "^3.972.19", - "@aws-sdk/middleware-user-agent": "^3.972.29", - "@aws-sdk/region-config-resolver": "^3.972.11", - "@aws-sdk/types": "^3.973.7", - "@aws-sdk/util-endpoints": "^3.996.6", - "@aws-sdk/util-user-agent-browser": "^3.972.9", - "@aws-sdk/util-user-agent-node": "^3.973.15", - "@smithy/config-resolver": "^4.4.14", - "@smithy/core": "^3.23.14", - "@smithy/fetch-http-handler": "^5.3.16", - "@smithy/hash-node": "^4.2.13", - "@smithy/invalid-dependency": "^4.2.13", - "@smithy/md5-js": "^4.2.13", - "@smithy/middleware-content-length": "^4.2.13", - "@smithy/middleware-endpoint": "^4.4.29", - "@smithy/middleware-retry": "^4.5.0", - "@smithy/middleware-serde": "^4.2.17", - "@smithy/middleware-stack": "^4.2.13", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/node-http-handler": "^4.5.2", - "@smithy/protocol-http": "^5.3.13", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", - "@smithy/url-parser": "^4.2.13", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/credential-provider-node": "^3.972.32", + "@aws-sdk/middleware-host-header": "^3.972.10", + "@aws-sdk/middleware-logger": "^3.972.10", + "@aws-sdk/middleware-recursion-detection": "^3.972.11", + "@aws-sdk/middleware-sdk-sqs": "^3.972.20", + "@aws-sdk/middleware-user-agent": "^3.972.31", + "@aws-sdk/region-config-resolver": "^3.972.12", + "@aws-sdk/types": "^3.973.8", + "@aws-sdk/util-endpoints": "^3.996.7", + "@aws-sdk/util-user-agent-browser": "^3.972.10", + "@aws-sdk/util-user-agent-node": "^3.973.17", + "@smithy/config-resolver": "^4.4.16", + "@smithy/core": "^3.23.15", + "@smithy/fetch-http-handler": "^5.3.17", + "@smithy/hash-node": "^4.2.14", + "@smithy/invalid-dependency": "^4.2.14", + "@smithy/md5-js": "^4.2.14", + "@smithy/middleware-content-length": "^4.2.14", + "@smithy/middleware-endpoint": "^4.4.30", + "@smithy/middleware-retry": "^4.5.3", + "@smithy/middleware-serde": "^4.2.18", + "@smithy/middleware-stack": "^4.2.14", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/node-http-handler": "^4.5.3", + "@smithy/protocol-http": "^5.3.14", + "@smithy/smithy-client": "^4.12.11", + "@smithy/types": "^4.14.1", + "@smithy/url-parser": "^4.2.14", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-body-length-node": "^4.2.3", - "@smithy/util-defaults-mode-browser": "^4.3.45", - "@smithy/util-defaults-mode-node": "^4.2.49", - "@smithy/util-endpoints": "^3.3.4", - "@smithy/util-middleware": "^4.2.13", - "@smithy/util-retry": "^4.3.0", + "@smithy/util-defaults-mode-browser": "^4.3.47", + "@smithy/util-defaults-mode-node": "^4.2.52", + "@smithy/util-endpoints": "^3.4.1", + "@smithy/util-middleware": "^4.2.14", + "@smithy/util-retry": "^4.3.2", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, @@ -941,64 +726,49 @@ "node": ">=20.0.0" } }, - "node_modules/@aws-sdk/client-sqs/node_modules/@smithy/node-http-handler": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.5.2.tgz", - "integrity": "sha512-/oD7u8M0oj2ZTFw7GkuuHWpIxtWdLlnyNkbrWcyVYhd5RJNDuczdkb0wfnQICyNFrVPlr8YHOhamjNy3zidhmA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.13", - "@smithy/querystring-builder": "^4.2.13", - "@smithy/types": "^4.14.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@aws-sdk/client-textract": { - "version": "3.1028.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-textract/-/client-textract-3.1028.0.tgz", - "integrity": "sha512-zZ84Kf92/x/HoT9jowB5ulhOJuRtaoXm8XAQE+AGVt6NK1JxECENDnMVHueJqedhIypRjmTLqQjhcGFJE+uNxw==", + "version": "3.1032.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-textract/-/client-textract-3.1032.0.tgz", + "integrity": "sha512-aqOnp0aEiqCQQ/ceLTMAjtCsmIdWAeuOyjz9dIzwgcNZKqUPwf+rproTSl8XIHmorqaBb8donzPaLTcAkMr+Yw==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/credential-provider-node": "^3.972.30", - "@aws-sdk/middleware-host-header": "^3.972.9", - "@aws-sdk/middleware-logger": "^3.972.9", - "@aws-sdk/middleware-recursion-detection": "^3.972.10", - "@aws-sdk/middleware-user-agent": "^3.972.29", - "@aws-sdk/region-config-resolver": "^3.972.11", - "@aws-sdk/types": "^3.973.7", - "@aws-sdk/util-endpoints": "^3.996.6", - "@aws-sdk/util-user-agent-browser": "^3.972.9", - "@aws-sdk/util-user-agent-node": "^3.973.15", - "@smithy/config-resolver": "^4.4.14", - "@smithy/core": "^3.23.14", - "@smithy/fetch-http-handler": "^5.3.16", - "@smithy/hash-node": "^4.2.13", - "@smithy/invalid-dependency": "^4.2.13", - "@smithy/middleware-content-length": "^4.2.13", - "@smithy/middleware-endpoint": "^4.4.29", - "@smithy/middleware-retry": "^4.5.0", - "@smithy/middleware-serde": "^4.2.17", - "@smithy/middleware-stack": "^4.2.13", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/node-http-handler": "^4.5.2", - "@smithy/protocol-http": "^5.3.13", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", - "@smithy/url-parser": "^4.2.13", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/credential-provider-node": "^3.972.32", + "@aws-sdk/middleware-host-header": "^3.972.10", + "@aws-sdk/middleware-logger": "^3.972.10", + "@aws-sdk/middleware-recursion-detection": "^3.972.11", + "@aws-sdk/middleware-user-agent": "^3.972.31", + "@aws-sdk/region-config-resolver": "^3.972.12", + "@aws-sdk/types": "^3.973.8", + "@aws-sdk/util-endpoints": "^3.996.7", + "@aws-sdk/util-user-agent-browser": "^3.972.10", + "@aws-sdk/util-user-agent-node": "^3.973.17", + "@smithy/config-resolver": "^4.4.16", + "@smithy/core": "^3.23.15", + "@smithy/fetch-http-handler": "^5.3.17", + "@smithy/hash-node": "^4.2.14", + "@smithy/invalid-dependency": "^4.2.14", + "@smithy/middleware-content-length": "^4.2.14", + "@smithy/middleware-endpoint": "^4.4.30", + "@smithy/middleware-retry": "^4.5.3", + "@smithy/middleware-serde": "^4.2.18", + "@smithy/middleware-stack": "^4.2.14", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/node-http-handler": "^4.5.3", + "@smithy/protocol-http": "^5.3.14", + "@smithy/smithy-client": "^4.12.11", + "@smithy/types": "^4.14.1", + "@smithy/url-parser": "^4.2.14", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-body-length-node": "^4.2.3", - "@smithy/util-defaults-mode-browser": "^4.3.45", - "@smithy/util-defaults-mode-node": "^4.2.49", - "@smithy/util-endpoints": "^3.3.4", - "@smithy/util-middleware": "^4.2.13", - "@smithy/util-retry": "^4.3.0", + "@smithy/util-defaults-mode-browser": "^4.3.47", + "@smithy/util-defaults-mode-node": "^4.2.52", + "@smithy/util-endpoints": "^3.4.1", + "@smithy/util-middleware": "^4.2.14", + "@smithy/util-retry": "^4.3.2", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, @@ -1006,38 +776,23 @@ "node": ">=20.0.0" } }, - "node_modules/@aws-sdk/client-textract/node_modules/@smithy/node-http-handler": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.5.2.tgz", - "integrity": "sha512-/oD7u8M0oj2ZTFw7GkuuHWpIxtWdLlnyNkbrWcyVYhd5RJNDuczdkb0wfnQICyNFrVPlr8YHOhamjNy3zidhmA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.13", - "@smithy/querystring-builder": "^4.2.13", - "@smithy/types": "^4.14.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@aws-sdk/core": { - "version": "3.973.27", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.973.27.tgz", - "integrity": "sha512-CUZ5m8hwMCH6OYI4Li/WgMfIEx10Q2PLI9Y3XOUTPGZJ53aZ0007jCv+X/ywsaERyKPdw5MRZWk877roQksQ4A==", + "version": "3.974.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.974.1.tgz", + "integrity": "sha512-gy/gffKz0zaHDaqRiLCdIvgHmaAL/HXuAtMcBP7euYSFx4BsbsdlfmUBJag+Gqe62z6/XuloKyQyaiH+kS3Vrg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.7", - "@aws-sdk/xml-builder": "^3.972.17", - "@smithy/core": "^3.23.14", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/property-provider": "^4.2.13", - "@smithy/protocol-http": "^5.3.13", - "@smithy/signature-v4": "^5.3.13", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", + "@aws-sdk/types": "^3.973.8", + "@aws-sdk/xml-builder": "^3.972.18", + "@smithy/core": "^3.23.15", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/property-provider": "^4.2.14", + "@smithy/protocol-http": "^5.3.14", + "@smithy/signature-v4": "^5.3.14", + "@smithy/smithy-client": "^4.12.11", + "@smithy/types": "^4.14.1", "@smithy/util-base64": "^4.3.2", - "@smithy/util-middleware": "^4.2.13", + "@smithy/util-middleware": "^4.2.14", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, @@ -1046,12 +801,12 @@ } }, "node_modules/@aws-sdk/crc64-nvme": { - "version": "3.972.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/crc64-nvme/-/crc64-nvme-3.972.6.tgz", - "integrity": "sha512-NMbiqKdruhwwgI6nzBVe2jWMkXjaoQz2YOs3rFX+2F3gGyrJDkDPwMpV/RsTFeq2vAQ055wZNtOXFK4NYSkM8g==", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/crc64-nvme/-/crc64-nvme-3.972.7.tgz", + "integrity": "sha512-QUagVVBbC8gODCF6e1aV0mE2TXWB9Opz4k8EJFdNrujUVQm5R4AjJa1mpOqzwOuROBzqJU9zawzig7M96L8Ejg==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.0", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1059,15 +814,15 @@ } }, "node_modules/@aws-sdk/credential-provider-cognito-identity": { - "version": "3.972.22", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.972.22.tgz", - "integrity": "sha512-ih6ORpme4i2qJqGckOQ9Lt2iiZ+5tm3bnfsT5TwoPyFnuDURXv3OdhYa3Nr/m0iJr38biqKYKdGKb5GR1KB2hw==", + "version": "3.972.24", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.972.24.tgz", + "integrity": "sha512-i6eMWlKfgQkNY3S/kg1ZnBZm2lhd6r8B3yobCalvrfCCGjvthREwsyDViuWl7gOWSvuUjQEFFGcJhGbCstcqJg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/nested-clients": "^3.996.19", - "@aws-sdk/types": "^3.973.7", - "@smithy/property-provider": "^4.2.13", - "@smithy/types": "^4.14.0", + "@aws-sdk/nested-clients": "^3.996.21", + "@aws-sdk/types": "^3.973.8", + "@smithy/property-provider": "^4.2.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1075,15 +830,15 @@ } }, "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.972.25", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.25.tgz", - "integrity": "sha512-6QfI0wv4jpG5CrdO/AO0JfZ2ux+tKwJPrUwmvxXF50vI5KIypKVGNF6b4vlkYEnKumDTI1NX2zUBi8JoU5QU3A==", + "version": "3.972.27", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.27.tgz", + "integrity": "sha512-xfUt2CUZDC+Tf16A6roD1b4pk/nrXdkoLY3TEhv198AXDtBo5xUJP1zd0e8SmuKLN4PpIBX96OizZbmMlcI6oQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/types": "^3.973.7", - "@smithy/property-provider": "^4.2.13", - "@smithy/types": "^4.14.0", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/types": "^3.973.8", + "@smithy/property-provider": "^4.2.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1091,60 +846,45 @@ } }, "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.972.27", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.27.tgz", - "integrity": "sha512-3V3Usj9Gs93h865DqN4M2NWJhC5kXU9BvZskfN3+69omuYlE3TZxOEcVQtBGLOloJB7BVfJKXVLqeNhOzHqSlQ==", + "version": "3.972.29", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.29.tgz", + "integrity": "sha512-hjNeYb6oLyHgMihra83ie0J/T2y9om3cy1qC90h9DRgvYXEoN4BCFf8bHguZjKhXunnv7YkmZRuYL5Mkk77eCA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/types": "^3.973.7", - "@smithy/fetch-http-handler": "^5.3.16", - "@smithy/node-http-handler": "^4.5.2", - "@smithy/property-provider": "^4.2.13", - "@smithy/protocol-http": "^5.3.13", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", - "@smithy/util-stream": "^4.5.22", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/types": "^3.973.8", + "@smithy/fetch-http-handler": "^5.3.17", + "@smithy/node-http-handler": "^4.5.3", + "@smithy/property-provider": "^4.2.14", + "@smithy/protocol-http": "^5.3.14", + "@smithy/smithy-client": "^4.12.11", + "@smithy/types": "^4.14.1", + "@smithy/util-stream": "^4.5.23", "tslib": "^2.6.2" }, "engines": { "node": ">=20.0.0" } }, - "node_modules/@aws-sdk/credential-provider-http/node_modules/@smithy/node-http-handler": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.5.2.tgz", - "integrity": "sha512-/oD7u8M0oj2ZTFw7GkuuHWpIxtWdLlnyNkbrWcyVYhd5RJNDuczdkb0wfnQICyNFrVPlr8YHOhamjNy3zidhmA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.13", - "@smithy/querystring-builder": "^4.2.13", - "@smithy/types": "^4.14.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.972.29", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.29.tgz", - "integrity": "sha512-SiBuAnXecCbT/OpAf3vqyI/AVE3mTaYr9ShXLybxZiPLBiPCCOIWSGAtYYGQWMRvobBTiqOewaB+wcgMMZI2Aw==", + "version": "3.972.31", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.31.tgz", + "integrity": "sha512-PuQ7e8WYzAPpzvFcajxf8c0LqSzakVHVlKw8M0oubk8Kf347YOCCqT1seQrHs5AdZuIh2RD9LX4O+Xa5ImEBfQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/credential-provider-env": "^3.972.25", - "@aws-sdk/credential-provider-http": "^3.972.27", - "@aws-sdk/credential-provider-login": "^3.972.29", - "@aws-sdk/credential-provider-process": "^3.972.25", - "@aws-sdk/credential-provider-sso": "^3.972.29", - "@aws-sdk/credential-provider-web-identity": "^3.972.29", - "@aws-sdk/nested-clients": "^3.996.19", - "@aws-sdk/types": "^3.973.7", - "@smithy/credential-provider-imds": "^4.2.13", - "@smithy/property-provider": "^4.2.13", - "@smithy/shared-ini-file-loader": "^4.4.8", - "@smithy/types": "^4.14.0", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/credential-provider-env": "^3.972.27", + "@aws-sdk/credential-provider-http": "^3.972.29", + "@aws-sdk/credential-provider-login": "^3.972.31", + "@aws-sdk/credential-provider-process": "^3.972.27", + "@aws-sdk/credential-provider-sso": "^3.972.31", + "@aws-sdk/credential-provider-web-identity": "^3.972.31", + "@aws-sdk/nested-clients": "^3.996.21", + "@aws-sdk/types": "^3.973.8", + "@smithy/credential-provider-imds": "^4.2.14", + "@smithy/property-provider": "^4.2.14", + "@smithy/shared-ini-file-loader": "^4.4.9", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1152,18 +892,18 @@ } }, "node_modules/@aws-sdk/credential-provider-login": { - "version": "3.972.29", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.29.tgz", - "integrity": "sha512-OGOslTbOlxXexKMqhxCEbBQbUIfuhGxU5UXw3Fm56ypXHvrXH4aTt/xb5Y884LOoteP1QST1lVZzHfcTnWhiPQ==", + "version": "3.972.31", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.31.tgz", + "integrity": "sha512-bBmWDmtSpmLOZR6a0kmowBcVL1hiL8Vlap/RXeMpFd7JbWl87YcwqL6T9LH/0oBVEZXu1dUZAtojgSuZgMO5xw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/nested-clients": "^3.996.19", - "@aws-sdk/types": "^3.973.7", - "@smithy/property-provider": "^4.2.13", - "@smithy/protocol-http": "^5.3.13", - "@smithy/shared-ini-file-loader": "^4.4.8", - "@smithy/types": "^4.14.0", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/nested-clients": "^3.996.21", + "@aws-sdk/types": "^3.973.8", + "@smithy/property-provider": "^4.2.14", + "@smithy/protocol-http": "^5.3.14", + "@smithy/shared-ini-file-loader": "^4.4.9", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1171,22 +911,22 @@ } }, "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.972.30", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.30.tgz", - "integrity": "sha512-FMnAnWxc8PG+ZrZ2OBKzY4luCUJhe9CG0B9YwYr4pzrYGLXBS2rl+UoUvjGbAwiptxRL6hyA3lFn03Bv1TLqTw==", + "version": "3.972.32", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.32.tgz", + "integrity": "sha512-9aj0x9hGYUondBZSD0XkksAdHhOKttFw4BWpLCeggeg40qSJxGrAP++g0GCm0VqWc1WtC/NRFiAVzPCy56vmog==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/credential-provider-env": "^3.972.25", - "@aws-sdk/credential-provider-http": "^3.972.27", - "@aws-sdk/credential-provider-ini": "^3.972.29", - "@aws-sdk/credential-provider-process": "^3.972.25", - "@aws-sdk/credential-provider-sso": "^3.972.29", - "@aws-sdk/credential-provider-web-identity": "^3.972.29", - "@aws-sdk/types": "^3.973.7", - "@smithy/credential-provider-imds": "^4.2.13", - "@smithy/property-provider": "^4.2.13", - "@smithy/shared-ini-file-loader": "^4.4.8", - "@smithy/types": "^4.14.0", + "@aws-sdk/credential-provider-env": "^3.972.27", + "@aws-sdk/credential-provider-http": "^3.972.29", + "@aws-sdk/credential-provider-ini": "^3.972.31", + "@aws-sdk/credential-provider-process": "^3.972.27", + "@aws-sdk/credential-provider-sso": "^3.972.31", + "@aws-sdk/credential-provider-web-identity": "^3.972.31", + "@aws-sdk/types": "^3.973.8", + "@smithy/credential-provider-imds": "^4.2.14", + "@smithy/property-provider": "^4.2.14", + "@smithy/shared-ini-file-loader": "^4.4.9", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1194,16 +934,16 @@ } }, "node_modules/@aws-sdk/credential-provider-process": { - "version": "3.972.25", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.25.tgz", - "integrity": "sha512-HR7ynNRdNhNsdVCOCegy1HsfsRzozCOPtD3RzzT1JouuaHobWyRfJzCBue/3jP7gECHt+kQyZUvwg/cYLWurNQ==", + "version": "3.972.27", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.27.tgz", + "integrity": "sha512-1CZvfb1WzudWWIFAVQkd1OI/T1RxPcSvNWzNsb2BMBVsBJzBtB8dV5f2nymHVU4UqwxipdVt/DAbgdDRf33JDg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/types": "^3.973.7", - "@smithy/property-provider": "^4.2.13", - "@smithy/shared-ini-file-loader": "^4.4.8", - "@smithy/types": "^4.14.0", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/types": "^3.973.8", + "@smithy/property-provider": "^4.2.14", + "@smithy/shared-ini-file-loader": "^4.4.9", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1211,18 +951,18 @@ } }, "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.972.29", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.29.tgz", - "integrity": "sha512-HWv4SEq3jZDYPlwryZVef97+U8CxxRos5mK8sgGO1dQaFZpV5giZLzqGE5hkDmh2csYcBO2uf5XHjPTpZcJlig==", + "version": "3.972.31", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.31.tgz", + "integrity": "sha512-x8Mx18S48XMl9bEEpYwmXDTvjWGPIfDadReN37Lc099/DUrlL4Zs9T9rwwggo6DkKS1aev6v+MTUx7JTa87TZQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/nested-clients": "^3.996.19", - "@aws-sdk/token-providers": "3.1026.0", - "@aws-sdk/types": "^3.973.7", - "@smithy/property-provider": "^4.2.13", - "@smithy/shared-ini-file-loader": "^4.4.8", - "@smithy/types": "^4.14.0", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/nested-clients": "^3.996.21", + "@aws-sdk/token-providers": "3.1032.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/property-provider": "^4.2.14", + "@smithy/shared-ini-file-loader": "^4.4.9", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1230,17 +970,17 @@ } }, "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.972.29", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.29.tgz", - "integrity": "sha512-PdMBza1WEKEUPFEmMGCfnU2RYCz9MskU2e8JxjyUOsMKku7j9YaDKvbDi2dzC0ihFoM6ods2SbhfAAro+Gwlew==", + "version": "3.972.31", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.31.tgz", + "integrity": "sha512-zfuNMIkGfjYsHis9qytYf74Bcmq6Ji9Xwf4w53baRCI/b2otTwZv3SW1uRiJ5Di7999QzRGhHZ96+eUeo3gSOA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/nested-clients": "^3.996.19", - "@aws-sdk/types": "^3.973.7", - "@smithy/property-provider": "^4.2.13", - "@smithy/shared-ini-file-loader": "^4.4.8", - "@smithy/types": "^4.14.0", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/nested-clients": "^3.996.21", + "@aws-sdk/types": "^3.973.8", + "@smithy/property-provider": "^4.2.14", + "@smithy/shared-ini-file-loader": "^4.4.9", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1248,30 +988,30 @@ } }, "node_modules/@aws-sdk/credential-providers": { - "version": "3.1028.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.1028.0.tgz", - "integrity": "sha512-ceaO4TnRycUoJl/1hNCdwWJLKHVF4R82YtBY6QE2SF1JVkrnR/WZu/yF1n1fPLuvYQlXW7OIOT9df7t2bfXM2Q==", + "version": "3.1032.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.1032.0.tgz", + "integrity": "sha512-OT+4/Kf62PKslLoJJGEzpQricgaMOuXFNf65FR56i3QugNLlNHfkhDWkk1CDvN1NYCHZbYZMJusdGjxVxQKgjQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-cognito-identity": "3.1028.0", - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/credential-provider-cognito-identity": "^3.972.22", - "@aws-sdk/credential-provider-env": "^3.972.25", - "@aws-sdk/credential-provider-http": "^3.972.27", - "@aws-sdk/credential-provider-ini": "^3.972.29", - "@aws-sdk/credential-provider-login": "^3.972.29", - "@aws-sdk/credential-provider-node": "^3.972.30", - "@aws-sdk/credential-provider-process": "^3.972.25", - "@aws-sdk/credential-provider-sso": "^3.972.29", - "@aws-sdk/credential-provider-web-identity": "^3.972.29", - "@aws-sdk/nested-clients": "^3.996.19", - "@aws-sdk/types": "^3.973.7", - "@smithy/config-resolver": "^4.4.14", - "@smithy/core": "^3.23.14", - "@smithy/credential-provider-imds": "^4.2.13", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/property-provider": "^4.2.13", - "@smithy/types": "^4.14.0", + "@aws-sdk/client-cognito-identity": "3.1032.0", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/credential-provider-cognito-identity": "^3.972.24", + "@aws-sdk/credential-provider-env": "^3.972.27", + "@aws-sdk/credential-provider-http": "^3.972.29", + "@aws-sdk/credential-provider-ini": "^3.972.31", + "@aws-sdk/credential-provider-login": "^3.972.31", + "@aws-sdk/credential-provider-node": "^3.972.32", + "@aws-sdk/credential-provider-process": "^3.972.27", + "@aws-sdk/credential-provider-sso": "^3.972.31", + "@aws-sdk/credential-provider-web-identity": "^3.972.31", + "@aws-sdk/nested-clients": "^3.996.21", + "@aws-sdk/types": "^3.973.8", + "@smithy/config-resolver": "^4.4.16", + "@smithy/core": "^3.23.15", + "@smithy/credential-provider-imds": "^4.2.14", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/property-provider": "^4.2.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1279,15 +1019,14 @@ } }, "node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.972.28", - "resolved": "https://registry.npmjs.org/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.972.28.tgz", - "integrity": "sha512-wx5jKLKPVJRsr/dwK9Xp26+SDb95xHlZU9Bgm2AglnMxQ0DlRlq3PyKlGi9y0OCuWZ7hLNcQJ7uDSN+PgsiuGg==", + "version": "3.973.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.973.1.tgz", + "integrity": "sha512-BuxJyHW+fnuGLFZ84z5txzlfKXLVbf3hmWH4wQ9q5a/P6O5slNg6j2eUE2kQMYWt3A3PheUR4tgRBUC7j9i/nQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.27", - "@smithy/core": "^3.23.14", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", + "@aws-sdk/core": "^3.974.1", + "@smithy/core": "^3.23.15", + "@smithy/types": "^4.14.1", "@smithy/util-base64": "^4.3.2", "tslib": "^2.6.2" }, @@ -1309,14 +1048,14 @@ } }, "node_modules/@aws-sdk/eventstream-handler-node": { - "version": "3.972.13", - "resolved": "https://registry.npmjs.org/@aws-sdk/eventstream-handler-node/-/eventstream-handler-node-3.972.13.tgz", - "integrity": "sha512-2Pi1kD0MDkMAxDHqvpi/hKMs9hXUYbj2GLEjCwy+0jzfLChAsF50SUYnOeTI+RztA+Ic4pnLAdB03f1e8nggxQ==", + "version": "3.972.14", + "resolved": "https://registry.npmjs.org/@aws-sdk/eventstream-handler-node/-/eventstream-handler-node-3.972.14.tgz", + "integrity": "sha512-m4X56gxG76/CKfxNVbOFuYwnAZcHgS6HOH8lgp15HoGHIAVTcZfZrXvcYzJFOMLEJgVn+JHBu6EiNV+xSNXXFg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.7", - "@smithy/eventstream-codec": "^4.2.13", - "@smithy/types": "^4.14.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/eventstream-codec": "^4.2.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1324,36 +1063,36 @@ } }, "node_modules/@aws-sdk/lib-dynamodb": { - "version": "3.1028.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/lib-dynamodb/-/lib-dynamodb-3.1028.0.tgz", - "integrity": "sha512-/DRWE5DPlM74xrSf6AqT8mxNo+ZSUiegwTVdZAVxa9iaQqZn4ZVgv8gP8fPrhG7LA8Twr9uuUlVz2B8tb0pSAA==", + "version": "3.1032.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/lib-dynamodb/-/lib-dynamodb-3.1032.0.tgz", + "integrity": "sha512-rYGhqP1H0Fy4r1yvWTmEAx0qqy1Zd9OzI8pPkXo6KSEDjZ4EwU+6QN1V+KLX3XTU6FQouF5LTvqLtl/CW4gxyQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.27", + "@aws-sdk/core": "^3.974.1", "@aws-sdk/util-dynamodb": "^3.996.2", - "@smithy/core": "^3.23.14", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", + "@smithy/core": "^3.23.15", + "@smithy/smithy-client": "^4.12.11", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { "node": ">=20.0.0" }, "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1028.0" + "@aws-sdk/client-dynamodb": "^3.1032.0" } }, "node_modules/@aws-sdk/middleware-bucket-endpoint": { - "version": "3.972.9", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.972.9.tgz", - "integrity": "sha512-COToYKgquDyligbcAep7ygs48RK+mwe/IYprq4+TSrVFzNOYmzWvHf6werpnKV5VYpRiwdn+Wa5ZXkPqLVwcTg==", + "version": "3.972.10", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.972.10.tgz", + "integrity": "sha512-Vbc2frZH7wXlMNd+ZZSXUEs/l1Sv8Jj4zUnIfwrYF5lwaLdXHZ9xx4U3rjUcaye3HRhFVc+E5DbBxpRAbB16BA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.7", + "@aws-sdk/types": "^3.973.8", "@aws-sdk/util-arn-parser": "^3.972.3", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/protocol-http": "^5.3.13", - "@smithy/types": "^4.14.0", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/protocol-http": "^5.3.14", + "@smithy/types": "^4.14.1", "@smithy/util-config-provider": "^4.2.2", "tslib": "^2.6.2" }, @@ -1362,16 +1101,16 @@ } }, "node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.10", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.972.10.tgz", - "integrity": "sha512-b3hf8dPxWonxFKgxBijMehVblgbY0gPprTvyuHYMxnOPfiCIY467kZltPoeOCQYLr9v0v0HuL9fIGtT6utd15w==", + "version": "3.972.11", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.972.11.tgz", + "integrity": "sha512-vXARCZVFQHdsd6qPPZyC/hh+5x2XsCYKqUQDCqnUlpGpChMpDojOOacQWdLJ+FFXKN8X3cmLOGrtgx/zysCKqQ==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/endpoint-cache": "^3.972.5", - "@aws-sdk/types": "^3.973.7", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/protocol-http": "^5.3.13", - "@smithy/types": "^4.14.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/protocol-http": "^5.3.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1379,14 +1118,14 @@ } }, "node_modules/@aws-sdk/middleware-eventstream": { - "version": "3.972.9", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-eventstream/-/middleware-eventstream-3.972.9.tgz", - "integrity": "sha512-ypgOvpWxQTCnQyDHGxnTviqqANE7FIIzII7VczJnTPCJcJlu17hMQXnvE47aKSKsawVJAaaRsyOEbHQuLJF9ng==", + "version": "3.972.10", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-eventstream/-/middleware-eventstream-3.972.10.tgz", + "integrity": "sha512-QUqLs7Af1II9X4fCRAu+EGHG3KHyOp4RkuLhRKoA3NuFlh6TL8i+zXBl8w2LUxqm44B/Kom45hgSlwA1SpTsXQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.7", - "@smithy/protocol-http": "^5.3.13", - "@smithy/types": "^4.14.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/protocol-http": "^5.3.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1394,14 +1133,14 @@ } }, "node_modules/@aws-sdk/middleware-expect-continue": { - "version": "3.972.9", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.972.9.tgz", - "integrity": "sha512-V/FNCjFxnh4VGu+HdSiW4Yg5GELihA1MIDSAdsEPvuayXBVmr0Jaa6jdLAZLH38KYXl/vVjri9DQJWnTAujHEA==", + "version": "3.972.10", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.972.10.tgz", + "integrity": "sha512-2Yn0f1Qiq/DjxYR3wfI3LokXnjOhFM7Ssn4LTdFDIxRMCE6I32MAsVnhPX1cUZsuVA9tiZtwwhlSLAtFGxAZlQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.7", - "@smithy/protocol-http": "^5.3.13", - "@smithy/types": "^4.14.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/protocol-http": "^5.3.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1409,23 +1148,23 @@ } }, "node_modules/@aws-sdk/middleware-flexible-checksums": { - "version": "3.974.7", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.974.7.tgz", - "integrity": "sha512-uU4/ch2CLHB8Phu1oTKnnQ4e8Ujqi49zEnQYBhWYT53zfFvtJCdGsaOoypBr8Fm/pmCBssRmGoIQ4sixgdLP9w==", + "version": "3.974.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.974.9.tgz", + "integrity": "sha512-ye6xVuMEQ5NCT+yQOryGYsuCXnOwu7iGFGzV+qpXZOWtqXIAAaFostapxj6RCubw36rekVwmdB2lcspFuyNfYQ==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/crc32": "5.2.0", "@aws-crypto/crc32c": "5.2.0", "@aws-crypto/util": "5.2.0", - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/crc64-nvme": "^3.972.6", - "@aws-sdk/types": "^3.973.7", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/crc64-nvme": "^3.972.7", + "@aws-sdk/types": "^3.973.8", "@smithy/is-array-buffer": "^4.2.2", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/protocol-http": "^5.3.13", - "@smithy/types": "^4.14.0", - "@smithy/util-middleware": "^4.2.13", - "@smithy/util-stream": "^4.5.22", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/protocol-http": "^5.3.14", + "@smithy/types": "^4.14.1", + "@smithy/util-middleware": "^4.2.14", + "@smithy/util-stream": "^4.5.23", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, @@ -1434,14 +1173,14 @@ } }, "node_modules/@aws-sdk/middleware-host-header": { - "version": "3.972.9", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.972.9.tgz", - "integrity": "sha512-je5vRdNw4SkuTnmRbFZLdye4sQ0faLt8kwka5wnnSU30q1mHO4X+idGEJOOE+Tn1ME7Oryn05xxkDvIb3UaLaQ==", + "version": "3.972.10", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.972.10.tgz", + "integrity": "sha512-IJSsIMeVQ8MMCPbuh1AbltkFhLBLXn7aejzfX5YKT/VLDHn++Dcz8886tXckE+wQssyPUhaXrJhdakO2VilRhg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.7", - "@smithy/protocol-http": "^5.3.13", - "@smithy/types": "^4.14.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/protocol-http": "^5.3.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1449,13 +1188,13 @@ } }, "node_modules/@aws-sdk/middleware-location-constraint": { - "version": "3.972.9", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.972.9.tgz", - "integrity": "sha512-TyfOi2XNdOZpNKeTJwRUsVAGa+14nkyMb2VVGG+eDgcWG/ed6+NUo72N3hT6QJioxym80NSinErD+LBRF0Ir1w==", + "version": "3.972.10", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.972.10.tgz", + "integrity": "sha512-rI3NZvJcEvjoD0+0PI0iUAwlPw2IlSlhyvgBK/3WkKJQE/YiKFedd9dMN2lVacdNxPNhxL/jzQaKQdrGtQagjQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.7", - "@smithy/types": "^4.14.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1463,13 +1202,13 @@ } }, "node_modules/@aws-sdk/middleware-logger": { - "version": "3.972.9", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.972.9.tgz", - "integrity": "sha512-HsVgDrruhqI28RkaXALm8grJ7Agc1wF6Et0xh6pom8NdO2VdO/SD9U/tPwUjewwK/pVoka+EShBxyCvgsPCtog==", + "version": "3.972.10", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.972.10.tgz", + "integrity": "sha512-OOuGvvz1Dm20SjZo5oEBePFqxt5nf8AwkNDSyUHvD9/bfNASmstcYxFAHUowy4n6Io7mWUZ04JURZwSBvyQanQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.7", - "@smithy/types": "^4.14.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1477,15 +1216,15 @@ } }, "node_modules/@aws-sdk/middleware-recursion-detection": { - "version": "3.972.10", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.972.10.tgz", - "integrity": "sha512-RVQQbq5orQ/GHUnXvqEOj2HHPBJm+mM+ySwZKS5UaLBwra5ugRtiH09PLUoOZRl7a1YzaOzXSuGbn9iD5j60WQ==", + "version": "3.972.11", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.972.11.tgz", + "integrity": "sha512-+zz6f79Kj9V5qFK2P+D8Ehjnw4AhphAlCAsPjUqEcInA9umtSSKMrHbSagEeOIsDNuvVrH98bjRHcyQukTrhaQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.7", + "@aws-sdk/types": "^3.973.8", "@aws/lambda-invoke-store": "^0.2.2", - "@smithy/protocol-http": "^5.3.13", - "@smithy/types": "^4.14.0", + "@smithy/protocol-http": "^5.3.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1493,23 +1232,23 @@ } }, "node_modules/@aws-sdk/middleware-sdk-s3": { - "version": "3.972.28", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.28.tgz", - "integrity": "sha512-qJHcJQH9UNPUrnPlRtCozKjtqAaypQ5IgQxTNoPsVYIQeuwNIA8Rwt3NvGij1vCDYDfCmZaPLpnJEHlZXeFqmg==", + "version": "3.972.30", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.30.tgz", + "integrity": "sha512-hoQRxjJu4tt3gEOQin21rJKotClJC+x7AmCh9ylRct1DJeaNI/BRlFxMbuhJe54bG6xANPagSs0my8K30QyV9g==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/types": "^3.973.7", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/types": "^3.973.8", "@aws-sdk/util-arn-parser": "^3.972.3", - "@smithy/core": "^3.23.14", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/protocol-http": "^5.3.13", - "@smithy/signature-v4": "^5.3.13", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", + "@smithy/core": "^3.23.15", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/protocol-http": "^5.3.14", + "@smithy/signature-v4": "^5.3.14", + "@smithy/smithy-client": "^4.12.11", + "@smithy/types": "^4.14.1", "@smithy/util-config-provider": "^4.2.2", - "@smithy/util-middleware": "^4.2.13", - "@smithy/util-stream": "^4.5.22", + "@smithy/util-middleware": "^4.2.14", + "@smithy/util-stream": "^4.5.23", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, @@ -1518,14 +1257,14 @@ } }, "node_modules/@aws-sdk/middleware-sdk-sqs": { - "version": "3.972.19", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sqs/-/middleware-sdk-sqs-3.972.19.tgz", - "integrity": "sha512-S7AWsrOTcs52AdS4uWPtP6n7tloOscfeNfJWK4wvNPJBI01lrfHb6g+tYRckwDzruhhdaPpn/CARZ+YPw6oMGw==", + "version": "3.972.20", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sqs/-/middleware-sdk-sqs-3.972.20.tgz", + "integrity": "sha512-yt0w5FKyH8Or7OT/Bp3fDRAtI4/f6uaaRKnW9TmU9qv8c1HFh43C9nQYZ26IcyRm+tYFdrB65yNTav/YThu36A==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.7", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/smithy-client": "^4.12.11", + "@smithy/types": "^4.14.1", "@smithy/util-hex-encoding": "^4.2.2", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" @@ -1535,13 +1274,13 @@ } }, "node_modules/@aws-sdk/middleware-ssec": { - "version": "3.972.9", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.972.9.tgz", - "integrity": "sha512-wSA2BR7L0CyBNDJeSrleIIzC+DzL93YNTdfU0KPGLiocK6YsRv1nPAzPF+BFSdcs0Qa5ku5Kcf4KvQcWwKGenQ==", + "version": "3.972.10", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.972.10.tgz", + "integrity": "sha512-Gli9A0u8EVVb+5bFDGS/QbSVg28w/wpEidg1ggVcSj65BDTdGR6punsOcVjqdiu1i42WHWo51MCvARPIIz9juw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.7", - "@smithy/types": "^4.14.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1549,18 +1288,18 @@ } }, "node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.972.29", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.29.tgz", - "integrity": "sha512-f/sIRzuTfEjg6NsbMYvye2VsmnQoNgntntleQyx5uGacUYzszbfIlO3GcI6G6daWUmTm0IDZc11qMHWwF0o0mQ==", + "version": "3.972.31", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.31.tgz", + "integrity": "sha512-L+hXN2HDomlIsWSHW5DVD7ppccCeRnlHXZ5uHG34ePTjF5bm0I1fmrJLbUGiW97xRXWryit5cjdP4Sx2FwiGog==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/types": "^3.973.7", - "@aws-sdk/util-endpoints": "^3.996.6", - "@smithy/core": "^3.23.14", - "@smithy/protocol-http": "^5.3.13", - "@smithy/types": "^4.14.0", - "@smithy/util-retry": "^4.3.0", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/types": "^3.973.8", + "@aws-sdk/util-endpoints": "^3.996.7", + "@smithy/core": "^3.23.15", + "@smithy/protocol-http": "^5.3.14", + "@smithy/types": "^4.14.1", + "@smithy/util-retry": "^4.3.2", "tslib": "^2.6.2" }, "engines": { @@ -1568,47 +1307,47 @@ } }, "node_modules/@aws-sdk/nested-clients": { - "version": "3.996.19", - "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.996.19.tgz", - "integrity": "sha512-uFkmCDXvmQYLanlYdOFS0+MQWkrj9wPMt/ZCc/0J0fjPim6F5jBVBmEomvGY/j77ILW6GTPwN22Jc174Mhkw6Q==", + "version": "3.996.21", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.996.21.tgz", + "integrity": "sha512-Me3d/ua2lb2G0bQfFmvCeQQp3+nN6GSPqMxDmi/IQlQ8CrlpQ5C0JJHpz2AnOUkEFI0lBNrAL3Vnt29l44ndkA==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/middleware-host-header": "^3.972.9", - "@aws-sdk/middleware-logger": "^3.972.9", - "@aws-sdk/middleware-recursion-detection": "^3.972.10", - "@aws-sdk/middleware-user-agent": "^3.972.29", - "@aws-sdk/region-config-resolver": "^3.972.11", - "@aws-sdk/types": "^3.973.7", - "@aws-sdk/util-endpoints": "^3.996.6", - "@aws-sdk/util-user-agent-browser": "^3.972.9", - "@aws-sdk/util-user-agent-node": "^3.973.15", - "@smithy/config-resolver": "^4.4.14", - "@smithy/core": "^3.23.14", - "@smithy/fetch-http-handler": "^5.3.16", - "@smithy/hash-node": "^4.2.13", - "@smithy/invalid-dependency": "^4.2.13", - "@smithy/middleware-content-length": "^4.2.13", - "@smithy/middleware-endpoint": "^4.4.29", - "@smithy/middleware-retry": "^4.5.0", - "@smithy/middleware-serde": "^4.2.17", - "@smithy/middleware-stack": "^4.2.13", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/node-http-handler": "^4.5.2", - "@smithy/protocol-http": "^5.3.13", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", - "@smithy/url-parser": "^4.2.13", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/middleware-host-header": "^3.972.10", + "@aws-sdk/middleware-logger": "^3.972.10", + "@aws-sdk/middleware-recursion-detection": "^3.972.11", + "@aws-sdk/middleware-user-agent": "^3.972.31", + "@aws-sdk/region-config-resolver": "^3.972.12", + "@aws-sdk/types": "^3.973.8", + "@aws-sdk/util-endpoints": "^3.996.7", + "@aws-sdk/util-user-agent-browser": "^3.972.10", + "@aws-sdk/util-user-agent-node": "^3.973.17", + "@smithy/config-resolver": "^4.4.16", + "@smithy/core": "^3.23.15", + "@smithy/fetch-http-handler": "^5.3.17", + "@smithy/hash-node": "^4.2.14", + "@smithy/invalid-dependency": "^4.2.14", + "@smithy/middleware-content-length": "^4.2.14", + "@smithy/middleware-endpoint": "^4.4.30", + "@smithy/middleware-retry": "^4.5.3", + "@smithy/middleware-serde": "^4.2.18", + "@smithy/middleware-stack": "^4.2.14", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/node-http-handler": "^4.5.3", + "@smithy/protocol-http": "^5.3.14", + "@smithy/smithy-client": "^4.12.11", + "@smithy/types": "^4.14.1", + "@smithy/url-parser": "^4.2.14", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-body-length-node": "^4.2.3", - "@smithy/util-defaults-mode-browser": "^4.3.45", - "@smithy/util-defaults-mode-node": "^4.2.49", - "@smithy/util-endpoints": "^3.3.4", - "@smithy/util-middleware": "^4.2.13", - "@smithy/util-retry": "^4.3.0", + "@smithy/util-defaults-mode-browser": "^4.3.47", + "@smithy/util-defaults-mode-node": "^4.2.52", + "@smithy/util-endpoints": "^3.4.1", + "@smithy/util-middleware": "^4.2.14", + "@smithy/util-retry": "^4.3.2", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, @@ -1616,31 +1355,35 @@ "node": ">=20.0.0" } }, - "node_modules/@aws-sdk/nested-clients/node_modules/@smithy/node-http-handler": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.5.2.tgz", - "integrity": "sha512-/oD7u8M0oj2ZTFw7GkuuHWpIxtWdLlnyNkbrWcyVYhd5RJNDuczdkb0wfnQICyNFrVPlr8YHOhamjNy3zidhmA==", + "node_modules/@aws-sdk/region-config-resolver": { + "version": "3.972.12", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.972.12.tgz", + "integrity": "sha512-QQI43Mxd53nBij0pm8HXC+t4IOC6gnhhZfzxE0OATQyO6QfPV4e+aTIRRuAJKA6Nig/cR8eLwPryqYTX9ZrjAQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/protocol-http": "^5.3.13", - "@smithy/querystring-builder": "^4.2.13", - "@smithy/types": "^4.14.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/config-resolver": "^4.4.16", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { - "node": ">=18.0.0" + "node": ">=20.0.0" } }, - "node_modules/@aws-sdk/region-config-resolver": { - "version": "3.972.11", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.972.11.tgz", - "integrity": "sha512-6Q8B1dcx6BBqUTY1Mc/eROKA0FImEEY5VPSd6AGPEUf0ErjExz4snVqa9kNJSoVDV1rKaNf3qrWojgcKW+SdDg==", + "node_modules/@aws-sdk/s3-request-presigner": { + "version": "3.1032.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/s3-request-presigner/-/s3-request-presigner-3.1032.0.tgz", + "integrity": "sha512-LFaI5JQhiOmJDjKK02ir9oERU9AmxdyEvzv332oPDzAzWeNH06sZ1WsF3xRBBE5tbEH2jIc79N8EqDCY0s5kKQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.7", - "@smithy/config-resolver": "^4.4.14", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/types": "^4.14.0", + "@aws-sdk/signature-v4-multi-region": "^3.996.18", + "@aws-sdk/types": "^3.973.8", + "@aws-sdk/util-format-url": "^3.972.10", + "@smithy/middleware-endpoint": "^4.4.30", + "@smithy/protocol-http": "^5.3.14", + "@smithy/smithy-client": "^4.12.11", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1648,16 +1391,16 @@ } }, "node_modules/@aws-sdk/signature-v4-multi-region": { - "version": "3.996.16", - "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.16.tgz", - "integrity": "sha512-EMdXYB4r/k5RWq86fugjRhid5JA+Z6MpS7n4sij4u5/C+STrkvuf9aFu41rJA9MjUzxCLzv8U2XL8cH2GSRYpQ==", + "version": "3.996.18", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.18.tgz", + "integrity": "sha512-4KT8UXRmvNAP5zKq9UI1MIwbnmSChZncBt89RKu/skMqZSSWGkBZTAJsZ+no+txfmF3kVaUFv31CTBZkQ5BJpQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/middleware-sdk-s3": "^3.972.28", - "@aws-sdk/types": "^3.973.7", - "@smithy/protocol-http": "^5.3.13", - "@smithy/signature-v4": "^5.3.13", - "@smithy/types": "^4.14.0", + "@aws-sdk/middleware-sdk-s3": "^3.972.30", + "@aws-sdk/types": "^3.973.8", + "@smithy/protocol-http": "^5.3.14", + "@smithy/signature-v4": "^5.3.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1665,17 +1408,17 @@ } }, "node_modules/@aws-sdk/token-providers": { - "version": "3.1026.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1026.0.tgz", - "integrity": "sha512-Ieq/HiRrbEtrYP387Nes0XlR7H1pJiJOZKv+QyQzMYpvTiDs0VKy2ZB3E2Zf+aFovWmeE7lRE4lXyF7dYM6GgA==", + "version": "3.1032.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1032.0.tgz", + "integrity": "sha512-n+PU8Z+gll7p3wDrH+Wo6fkt8sPrVnq30YYM6Ryga95oJlEneNMEbDHj0iqjMX3V7gaGdJo/hJWyPo4lscP+mA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.27", - "@aws-sdk/nested-clients": "^3.996.19", - "@aws-sdk/types": "^3.973.7", - "@smithy/property-provider": "^4.2.13", - "@smithy/shared-ini-file-loader": "^4.4.8", - "@smithy/types": "^4.14.0", + "@aws-sdk/core": "^3.974.1", + "@aws-sdk/nested-clients": "^3.996.21", + "@aws-sdk/types": "^3.973.8", + "@smithy/property-provider": "^4.2.14", + "@smithy/shared-ini-file-loader": "^4.4.9", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1683,12 +1426,12 @@ } }, "node_modules/@aws-sdk/types": { - "version": "3.973.7", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.973.7.tgz", - "integrity": "sha512-reXRwoJ6CfChoqAsBszUYajAF8Z2LRE+CRcKocvFSMpIiLOtYU3aJ9trmn6VVPAzbbY5LXF+FfmUslbXk1SYFg==", + "version": "3.973.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.973.8.tgz", + "integrity": "sha512-gjlAdtHMbtR9X5iIhVUvbVcy55KnznpC6bkDUWW9z915bi0ckdUr5cjf16Kp6xq0bP5HBD2xzgbL9F9Quv5vUw==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.0", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1723,15 +1466,30 @@ } }, "node_modules/@aws-sdk/util-endpoints": { - "version": "3.996.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.996.6.tgz", - "integrity": "sha512-2nUQ+2ih7CShuKHpGSIYvvAIOHy52dOZguYG36zptBukhw6iFwcvGfG0tes0oZFWQqEWvgZe9HLWaNlvXGdOrg==", + "version": "3.996.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.996.7.tgz", + "integrity": "sha512-ty4LQxN1QC+YhUP28NfEgZDEGXkyqOQy+BDriBozqHsrYO4JMgiPhfizqOGF7P+euBTZ5Ez6SKlLAMCLo8tzmw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.7", - "@smithy/types": "^4.14.0", - "@smithy/url-parser": "^4.2.13", - "@smithy/util-endpoints": "^3.3.4", + "@aws-sdk/types": "^3.973.8", + "@smithy/types": "^4.14.1", + "@smithy/url-parser": "^4.2.14", + "@smithy/util-endpoints": "^3.4.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/util-format-url": { + "version": "3.972.10", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-format-url/-/util-format-url-3.972.10.tgz", + "integrity": "sha512-DEKiHNJVtNxdyTeQspzY+15Po/kHm6sF0Cs4HV9Q2+lplB63+DrvdeiSoOSdWEWAoO2RcY1veoXVDz2tWxWCgQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.8", + "@smithy/querystring-builder": "^4.2.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -1751,27 +1509,27 @@ } }, "node_modules/@aws-sdk/util-user-agent-browser": { - "version": "3.972.9", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.972.9.tgz", - "integrity": "sha512-sn/LMzTbGjYqCCF24390WxPd6hkpoSptiUn5DzVp4cD71yqw+yGEGm1YCxyEoPXyc8qciM8UzLJcZBFslxo5Uw==", + "version": "3.972.10", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.972.10.tgz", + "integrity": "sha512-FAzqXvfEssGdSIz8ejatan0bOdx1qefBWKF/gWmVBXIP1HkS7v/wjjaqrAGGKvyihrXTXW00/2/1nTJtxpXz7g==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.7", - "@smithy/types": "^4.14.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/types": "^4.14.1", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.973.15", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.973.15.tgz", - "integrity": "sha512-fYn3s9PtKdgQkczGZCFMgkNEe8aq1JCVbnRqjqN9RSVW43xn2RV9xdcZ3z01a48Jpkuh/xCmBKJxdLOo4Ozg7w==", + "version": "3.973.17", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.973.17.tgz", + "integrity": "sha512-utF5qjjbuJQuU9VdCkWl7L87sr93cApsrD+uxGfUnlafX8iyEzJrb7EZnufjThURZVTOtelRMXrblWxpefElUg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/middleware-user-agent": "^3.972.29", - "@aws-sdk/types": "^3.973.7", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/types": "^4.14.0", + "@aws-sdk/middleware-user-agent": "^3.972.31", + "@aws-sdk/types": "^3.973.8", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/types": "^4.14.1", "@smithy/util-config-provider": "^4.2.2", "tslib": "^2.6.2" }, @@ -1788,13 +1546,14 @@ } }, "node_modules/@aws-sdk/xml-builder": { - "version": "3.972.17", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.17.tgz", - "integrity": "sha512-Ra7hjqAZf1OXRRMueB13qex7mFJRDK/pgCvdSFemXBT8KCGnQDPoKzHY1SjN+TjJVmnpSF14W5tJ1vDamFu+Gg==", + "version": "3.972.20", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.20.tgz", + "integrity": "sha512-MDcUfroaMAnDAHn29vN781t0wudR8zjfgg+r3s5otx8TJXFWg01NZB7HvHkBbOf7UUmKEwIZf5kHxiaVUgwjlQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.0", - "fast-xml-parser": "5.5.8", + "@nodable/entities": "2.1.0", + "@smithy/types": "^4.14.1", + "fast-xml-parser": "5.7.2", "tslib": "^2.6.2" }, "engines": { @@ -1810,183 +1569,11 @@ "node": ">=18.0.0" } }, - "node_modules/@babel/code-frame": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", - "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", - "license": "MIT", - "dependencies": { - "@babel/helper-validator-identifier": "^7.28.5", - "js-tokens": "^4.0.0", - "picocolors": "^1.1.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/code-frame/node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "license": "MIT" - }, - "node_modules/@babel/compat-data": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.0.tgz", - "integrity": "sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/core": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.0.tgz", - "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", - "dev": true, - "license": "MIT", - "peer": true, - "dependencies": { - "@babel/code-frame": "^7.29.0", - "@babel/generator": "^7.29.0", - "@babel/helper-compilation-targets": "^7.28.6", - "@babel/helper-module-transforms": "^7.28.6", - "@babel/helpers": "^7.28.6", - "@babel/parser": "^7.29.0", - "@babel/template": "^7.28.6", - "@babel/traverse": "^7.29.0", - "@babel/types": "^7.29.0", - "@jridgewell/remapping": "^2.3.5", - "convert-source-map": "^2.0.0", - "debug": "^4.1.0", - "gensync": "^1.0.0-beta.2", - "json5": "^2.2.3", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/babel" - } - }, - "node_modules/@babel/core/node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/@babel/generator": { - "version": "7.29.1", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.1.tgz", - "integrity": "sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw==", - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.29.0", - "@babel/types": "^7.29.0", - "@jridgewell/gen-mapping": "^0.3.12", - "@jridgewell/trace-mapping": "^0.3.28", - "jsesc": "^3.0.2" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-compilation-targets": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.28.6.tgz", - "integrity": "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/compat-data": "^7.28.6", - "@babel/helper-validator-option": "^7.27.1", - "browserslist": "^4.24.0", - "lru-cache": "^5.1.1", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-compilation-targets/node_modules/lru-cache": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^3.0.2" - } - }, - "node_modules/@babel/helper-compilation-targets/node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/@babel/helper-compilation-targets/node_modules/yallist": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", - "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", - "dev": true, - "license": "ISC" - }, - "node_modules/@babel/helper-globals": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", - "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-imports": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.28.6.tgz", - "integrity": "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/traverse": "^7.28.6", - "@babel/types": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-transforms": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.6.tgz", - "integrity": "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-module-imports": "^7.28.6", - "@babel/helper-validator-identifier": "^7.28.5", - "@babel/traverse": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, "node_modules/@babel/helper-string-parser": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -1996,39 +1583,17 @@ "version": "7.28.5", "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-option": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", - "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" } }, - "node_modules/@babel/helpers": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.2.tgz", - "integrity": "sha512-HoGuUs4sCZNezVEKdVcwqmZN8GoHirLUcLaYVNBK2J0DadGtdcqgr3BCbvH8+XUo4NGjNl3VOtSjEKNzqfFgKw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/template": "^7.28.6", - "@babel/types": "^7.29.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/@babel/parser": { "version": "7.29.2", "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.2.tgz", "integrity": "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA==", + "dev": true, "license": "MIT", "dependencies": { "@babel/types": "^7.29.0" @@ -2049,42 +1614,11 @@ "node": ">=6.9.0" } }, - "node_modules/@babel/template": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", - "integrity": "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==", - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.28.6", - "@babel/parser": "^7.28.6", - "@babel/types": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.0.tgz", - "integrity": "sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA==", - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.29.0", - "@babel/generator": "^7.29.0", - "@babel/helper-globals": "^7.28.0", - "@babel/parser": "^7.29.0", - "@babel/template": "^7.28.6", - "@babel/types": "^7.29.0", - "debug": "^4.3.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/@babel/types": { "version": "7.29.0", "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", + "dev": true, "license": "MIT", "dependencies": { "@babel/helper-string-parser": "^7.27.1", @@ -2127,21 +1661,6 @@ "specificity": "bin/cli.js" } }, - "node_modules/@canvas/image-data": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@canvas/image-data/-/image-data-1.1.0.tgz", - "integrity": "sha512-QdObRRjRbcXGmM1tmJ+MrHcaz1MftF2+W7YI+MsphnsCrmtyfS0d5qJbk0MeSbUeyM/jCb0hmnkXPsy026L7dA==", - "license": "MIT" - }, - "node_modules/@colors/colors": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", - "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", - "license": "MIT", - "engines": { - "node": ">=0.1.90" - } - }, "node_modules/@csstools/color-helpers": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.0.2.tgz", @@ -2163,9 +1682,9 @@ } }, "node_modules/@csstools/css-calc": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.1.1.tgz", - "integrity": "sha512-HJ26Z/vmsZQqs/o3a6bgKslXGFAungXGbinULZO3eMsOyNJHeBBZfup5FiZInOghgoM4Hwnmw+OgbJCNg1wwUQ==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.2.0.tgz", + "integrity": "sha512-bR9e6o2BDB12jzN/gIbjHa5wLJ4UjD1CB9pM7ehlc0ddk6EBz+yYS1EV2MF55/HUxrHcB/hehAyt5vhsA3hx7w==", "dev": true, "funding": [ { @@ -2187,9 +1706,9 @@ } }, "node_modules/@csstools/css-color-parser": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.0.2.tgz", - "integrity": "sha512-0GEfbBLmTFf0dJlpsNU7zwxRIH0/BGEMuXLTCvFYxuL1tNhqzTbtnFICyJLTNK4a+RechKP75e7w42ClXSnJQw==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.1.0.tgz", + "integrity": "sha512-U0KhLYmy2GVj6q4T3WaAe6NPuFYCPQoE3b0dRGxejWDgcPp8TP7S5rVdM5ZrFaqu4N67X8YaPBw14dQSYx3IyQ==", "dev": true, "funding": [ { @@ -2204,7 +1723,7 @@ "license": "MIT", "dependencies": { "@csstools/color-helpers": "^6.0.2", - "@csstools/css-calc": "^3.1.1" + "@csstools/css-calc": "^3.2.0" }, "engines": { "node": ">=20.19.0" @@ -2230,7 +1749,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=20.19.0" }, @@ -2239,9 +1757,9 @@ } }, "node_modules/@csstools/css-syntax-patches-for-csstree": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.2.tgz", - "integrity": "sha512-5GkLzz4prTIpoyeUiIu3iV6CSG3Plo7xRVOFPKI7FVEJ3mZ0A8SwK0XU3Gl7xAkiQ+mDyam+NNp875/C5y+jSA==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.3.tgz", + "integrity": "sha512-SH60bMfrRCJF3morcdk57WklujF4Jr/EsQUzqkarfHXEFcAR1gg7fS/chAE922Sehgzc1/+Tz5H3Ypa1HiEKrg==", "dev": true, "funding": [ { @@ -2279,22 +1797,10 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=20.19.0" } }, - "node_modules/@dabh/diagnostics": { - "version": "2.0.8", - "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.8.tgz", - "integrity": "sha512-R4MSXTVnuMzGD7bzHdW2ZhhdPC/igELENcq5IjEverBvq5hn1SXCWcsi6eSsdWP0/Ur+SItRRjAktmdoX/8R/Q==", - "license": "MIT", - "dependencies": { - "@so-ric/colorspace": "^1.1.6", - "enabled": "2.0.x", - "kuler": "^2.0.0" - } - }, "node_modules/@discoveryjs/json-ext": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz", @@ -2305,28 +1811,37 @@ "node": ">=10.0.0" } }, - "node_modules/@emnapi/wasi-threads": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", - "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "node_modules/@emnapi/core": { + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.2.tgz", + "integrity": "sha512-UC+ZhH3XtczQYfOlu3lNEkdW/p4dsJ1r/bP7H8+rhao3TTTMO1ATq/4DdIi23XuGoFY+Cz0JmCbdVl0hz9jZcA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", "license": "MIT", "optional": true, "dependencies": { "tslib": "^2.4.0" } }, - "node_modules/@ericcornelissen/lregexp": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@ericcornelissen/lregexp/-/lregexp-1.0.8.tgz", - "integrity": "sha512-asaQOkMr8CzYVpmCnBvEe8zejcv8Q1+f3Q1qssOF85njH1VTJfSQUYwwClE41SRdvZmN26uL08GuQk04VN2myA==", + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "dev": true, "license": "MIT", + "optional": true, "dependencies": { - "is-supported-regexp-flag": "^2.0.0" - }, - "engines": { - "bun": "^1.2.0", - "deno": "^2.0.0", - "node": "^12.20.0 || ^14.13.0 || ^15 || ^16 || ^17 || ^18 || ^19 || ^20 || ^21 || ^22 || ^23 || ^24 || ^25" + "tslib": "^2.4.0" } }, "node_modules/@esbuild/linux-x64": { @@ -2349,6 +1864,7 @@ "version": "4.9.1", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", + "dev": true, "license": "MIT", "dependencies": { "eslint-visitor-keys": "^3.4.3" @@ -2367,6 +1883,7 @@ "version": "3.4.3", "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, "license": "Apache-2.0", "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -2379,6 +1896,7 @@ "version": "4.12.2", "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, "license": "MIT", "engines": { "node": "^12.0.0 || ^14.0.0 || >=16.0.0" @@ -2388,6 +1906,7 @@ "version": "0.21.2", "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.2.tgz", "integrity": "sha512-nJl2KGTlrf9GjLimgIru+V/mzgSK0ABCDQRvxw5BjURL7WfH5uoWmizbH7QB6MmnMBd8cIC9uceWnezL1VZWWw==", + "dev": true, "license": "Apache-2.0", "dependencies": { "@eslint/object-schema": "^2.1.7", @@ -2402,12 +1921,14 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, "license": "MIT" }, "node_modules/@eslint/config-array/node_modules/brace-expansion": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.13.tgz", - "integrity": "sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w==", + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", + "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", + "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -2418,6 +1939,7 @@ "version": "3.1.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -2430,6 +1952,7 @@ "version": "0.4.2", "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", + "dev": true, "license": "Apache-2.0", "dependencies": { "@eslint/core": "^0.17.0" @@ -2442,6 +1965,7 @@ "version": "0.17.0", "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", + "dev": true, "license": "Apache-2.0", "dependencies": { "@types/json-schema": "^7.0.15" @@ -2454,6 +1978,7 @@ "version": "3.3.5", "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.5.tgz", "integrity": "sha512-4IlJx0X0qftVsN5E+/vGujTRIFtwuLbNsVUe7TO6zYPDR1O6nFwvwhIKEKSrl6dZchmYBITazxKoUYOjdtjlRg==", + "dev": true, "license": "MIT", "dependencies": { "ajv": "^6.14.0", @@ -2477,12 +2002,14 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, "license": "MIT" }, "node_modules/@eslint/eslintrc/node_modules/brace-expansion": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.13.tgz", - "integrity": "sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w==", + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", + "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", + "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -2493,6 +2020,7 @@ "version": "14.0.0", "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=18" @@ -2505,6 +2033,7 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, "license": "MIT", "engines": { "node": ">= 4" @@ -2514,6 +2043,7 @@ "version": "3.1.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -2526,6 +2056,7 @@ "version": "9.39.4", "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.4.tgz", "integrity": "sha512-nE7DEIchvtiFTwBw4Lfbu59PG+kCofhjsKaCWzxTpt4lfRjRMqG6uMBzKXuEcyXhOHoUp9riAm7/aWYGhXZ9cw==", + "dev": true, "license": "MIT", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2538,6 +2069,7 @@ "version": "2.1.7", "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", + "dev": true, "license": "Apache-2.0", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2547,6 +2079,7 @@ "version": "0.4.1", "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", + "dev": true, "license": "Apache-2.0", "dependencies": { "@eslint/core": "^0.17.0", @@ -2617,18 +2150,6 @@ "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "license": "MIT" }, - "node_modules/@fastify/busboy": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-1.2.1.tgz", - "integrity": "sha512-7PQA7EH43S0CxcOa9OeAnaeA0oQ+e/DHNPZwSQM9CQHW76jle5+OvLdibRp/Aafs9KXbLhxyjOTkRjWUbQEd3Q==", - "license": "MIT", - "dependencies": { - "text-decoding": "^1.0.0" - }, - "engines": { - "node": ">=14" - } - }, "node_modules/@fastify/cors": { "version": "11.2.0", "resolved": "https://registry.npmjs.org/@fastify/cors/-/cors-11.2.0.tgz", @@ -2739,110 +2260,6 @@ "ipaddr.js": "^2.1.0" } }, - "node_modules/@firebase/app-types": { - "version": "0.9.4", - "resolved": "https://registry.npmjs.org/@firebase/app-types/-/app-types-0.9.4.tgz", - "integrity": "sha512-crX9TA5SVYZwLPG7/R16IsH8FLlgkPXjJUVhsVpHVDSqJiq3D/NuFTM5ctxGTExXAOeIn//69tQw47CPerM8MQ==", - "license": "Apache-2.0", - "dependencies": { - "@firebase/logger": "0.5.0" - } - }, - "node_modules/@firebase/app-types/node_modules/@firebase/logger": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@firebase/logger/-/logger-0.5.0.tgz", - "integrity": "sha512-cGskaAvkrnh42b3BA3doDWeBmuHFO/Mx5A83rbRDYakPjO9bJtRL3dX7javzc2Rr/JHZf4HlterTW2lUkfeN4g==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.1.0" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@firebase/auth-interop-types": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/@firebase/auth-interop-types/-/auth-interop-types-0.1.7.tgz", - "integrity": "sha512-yA/dTveGGPcc85JP8ZE/KZqfGQyQTBCV10THdI8HTlP1GDvNrhr//J5jAt58MlsCOaO3XmC4DqScPBbtIsR/EA==", - "license": "Apache-2.0", - "peerDependencies": { - "@firebase/app-types": "0.x", - "@firebase/util": "1.x" - } - }, - "node_modules/@firebase/component": { - "version": "0.5.21", - "resolved": "https://registry.npmjs.org/@firebase/component/-/component-0.5.21.tgz", - "integrity": "sha512-12MMQ/ulfygKpEJpseYMR0HunJdlsLrwx2XcEs40M18jocy2+spyzHHEwegN3x/2/BLFBjR5247Etmz0G97Qpg==", - "license": "Apache-2.0", - "dependencies": { - "@firebase/util": "1.7.3", - "tslib": "^2.1.0" - } - }, - "node_modules/@firebase/database": { - "version": "0.13.10", - "resolved": "https://registry.npmjs.org/@firebase/database/-/database-0.13.10.tgz", - "integrity": "sha512-KRucuzZ7ZHQsRdGEmhxId5jyM2yKsjsQWF9yv0dIhlxYg0D8rCVDZc/waoPKA5oV3/SEIoptF8F7R1Vfe7BCQA==", - "license": "Apache-2.0", - "dependencies": { - "@firebase/auth-interop-types": "0.1.7", - "@firebase/component": "0.5.21", - "@firebase/logger": "0.3.4", - "@firebase/util": "1.7.3", - "faye-websocket": "0.11.4", - "tslib": "^2.1.0" - } - }, - "node_modules/@firebase/database-compat": { - "version": "0.2.10", - "resolved": "https://registry.npmjs.org/@firebase/database-compat/-/database-compat-0.2.10.tgz", - "integrity": "sha512-fK+IgUUqVKcWK/gltzDU+B1xauCOfY6vulO8lxoNTkcCGlSxuTtwsdqjGkFmgFRMYjXFWWJ6iFcJ/vXahzwCtA==", - "license": "Apache-2.0", - "dependencies": { - "@firebase/component": "0.5.21", - "@firebase/database": "0.13.10", - "@firebase/database-types": "0.9.17", - "@firebase/logger": "0.3.4", - "@firebase/util": "1.7.3", - "tslib": "^2.1.0" - } - }, - "node_modules/@firebase/database-types": { - "version": "0.9.17", - "resolved": "https://registry.npmjs.org/@firebase/database-types/-/database-types-0.9.17.tgz", - "integrity": "sha512-YQm2tCZyxNtEnlS5qo5gd2PAYgKCy69tUKwioGhApCFThW+mIgZs7IeYeJo2M51i4LCixYUl+CvnOyAnb/c3XA==", - "license": "Apache-2.0", - "dependencies": { - "@firebase/app-types": "0.8.1", - "@firebase/util": "1.7.3" - } - }, - "node_modules/@firebase/database-types/node_modules/@firebase/app-types": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@firebase/app-types/-/app-types-0.8.1.tgz", - "integrity": "sha512-p75Ow3QhB82kpMzmOntv866wH9eZ3b4+QbUY+8/DA5Zzdf1c8Nsk8B7kbFpzJt4wwHMdy5LTF5YUnoTc1JiWkw==", - "license": "Apache-2.0" - }, - "node_modules/@firebase/logger": { - "version": "0.3.4", - "resolved": "https://registry.npmjs.org/@firebase/logger/-/logger-0.3.4.tgz", - "integrity": "sha512-hlFglGRgZEwoyClZcGLx/Wd+zoLfGmbDkFx56mQt/jJ0XMbfPqwId1kiPl0zgdWZX+D8iH+gT6GuLPFsJWgiGw==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.1.0" - } - }, - "node_modules/@firebase/util": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@firebase/util/-/util-1.7.3.tgz", - "integrity": "sha512-wxNqWbqokF551WrJ9BIFouU/V5SL1oYCGx1oudcirdhadnQRFH5v1sjgGL7cUV/UsekSycygphdrF2lxBxOYKg==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "tslib": "^2.1.0" - } - }, "node_modules/@fontsource/inter": { "version": "5.2.8", "resolved": "https://registry.npmjs.org/@fontsource/inter/-/inter-5.2.8.tgz", @@ -2852,187 +2269,10 @@ "url": "https://github.com/sponsors/ayuhito" } }, - "node_modules/@google-cloud/firestore": { - "version": "4.15.1", - "resolved": "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-4.15.1.tgz", - "integrity": "sha512-2PWsCkEF1W02QbghSeRsNdYKN1qavrHBP3m72gPDMHQSYrGULOaTi7fSJquQmAtc4iPVB2/x6h80rdLHTATQtA==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "fast-deep-equal": "^3.1.1", - "functional-red-black-tree": "^1.0.1", - "google-gax": "^2.24.1", - "protobufjs": "^6.8.6" - }, - "engines": { - "node": ">=10.10.0" - } - }, - "node_modules/@google-cloud/firestore/node_modules/long": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", - "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", - "license": "Apache-2.0", - "optional": true - }, - "node_modules/@google-cloud/firestore/node_modules/protobufjs": { - "version": "6.11.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.4.tgz", - "integrity": "sha512-5kQWPaJHi1WoCpjTGszzQ32PG2F4+wRY6BmAT4Vfw56Q2FZ4YZzK20xUYQH4YkfehY1e6QSICrJquM6xXZNcrw==", - "hasInstallScript": true, - "license": "BSD-3-Clause", - "optional": true, - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/long": "^4.0.1", - "@types/node": ">=13.7.0", - "long": "^4.0.0" - }, - "bin": { - "pbjs": "bin/pbjs", - "pbts": "bin/pbts" - } - }, - "node_modules/@google-cloud/paginator": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-3.0.7.tgz", - "integrity": "sha512-jJNutk0arIQhmpUUQJPJErsojqo834KcyB6X7a1mxuic8i1tKXxde8E69IZxNZawRIlZdIK2QY4WALvlK5MzYQ==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "arrify": "^2.0.0", - "extend": "^3.0.2" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@google-cloud/projectify": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-2.1.1.tgz", - "integrity": "sha512-+rssMZHnlh0twl122gXY4/aCrk0G1acBqkHFfYddtsqpYXGxA29nj9V5V9SfC+GyOG00l650f6lG9KL+EpFEWQ==", - "license": "Apache-2.0", - "optional": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/@google-cloud/promisify": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-2.0.4.tgz", - "integrity": "sha512-j8yRSSqswWi1QqUGKVEKOG03Q7qOoZP6/h2zN2YO+F5h2+DHU0bSrHCK9Y7lo2DI9fBd8qGAw795sf+3Jva4yA==", - "license": "Apache-2.0", - "optional": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/@google-cloud/storage": { - "version": "5.20.5", - "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-5.20.5.tgz", - "integrity": "sha512-lOs/dCyveVF8TkVFnFSF7IGd0CJrTm91qiK6JLu+Z8qiT+7Ag0RyVhxZIWkhiACqwABo7kSHDm8FdH8p2wxSSw==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "@google-cloud/paginator": "^3.0.7", - "@google-cloud/projectify": "^2.0.0", - "@google-cloud/promisify": "^2.0.0", - "abort-controller": "^3.0.0", - "arrify": "^2.0.0", - "async-retry": "^1.3.3", - "compressible": "^2.0.12", - "configstore": "^5.0.0", - "duplexify": "^4.0.0", - "ent": "^2.2.0", - "extend": "^3.0.2", - "gaxios": "^4.0.0", - "google-auth-library": "^7.14.1", - "hash-stream-validation": "^0.2.2", - "mime": "^3.0.0", - "mime-types": "^2.0.8", - "p-limit": "^3.0.1", - "pumpify": "^2.0.0", - "retry-request": "^4.2.2", - "stream-events": "^1.0.4", - "teeny-request": "^7.1.3", - "uuid": "^8.0.0", - "xdg-basedir": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@google-cloud/storage/node_modules/gcp-metadata": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.3.1.tgz", - "integrity": "sha512-x850LS5N7V1F3UcV7PoupzGsyD6iVwTVvsh3tbXfkctZnBnjW5yu5z1/3k3SehF7TyoTIe78rJs02GMMy+LF+A==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "gaxios": "^4.0.0", - "json-bigint": "^1.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@google-cloud/storage/node_modules/google-auth-library": { - "version": "7.14.1", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-7.14.1.tgz", - "integrity": "sha512-5Rk7iLNDFhFeBYc3s8l1CqzbEBcdhwR193RlD4vSNFajIcINKI8W8P0JLmBpwymHqqWbX34pJDQu39cSy/6RsA==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "arrify": "^2.0.0", - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "fast-text-encoding": "^1.0.0", - "gaxios": "^4.0.0", - "gcp-metadata": "^4.2.0", - "gtoken": "^5.0.4", - "jws": "^4.0.0", - "lru-cache": "^6.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@google-cloud/storage/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "license": "ISC", - "optional": true, - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@google-cloud/storage/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "license": "MIT", - "optional": true, - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/@google/genai": { - "version": "1.49.0", - "resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.49.0.tgz", - "integrity": "sha512-hO69Zl0H3x+L0KL4stl1pLYgnqnwHoLqtKy6MRlNnW8TAxjqMdOUVafomKd4z1BePkzoxJWbYILny9a2Zk43VQ==", + "version": "1.50.1", + "resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.50.1.tgz", + "integrity": "sha512-YbkX7H9+1Pt8wOt7DDREy8XSoiL6fRDzZQRyaVBarFf8MR3zHGqVdvM4cLbDXqPhxqvegZShgfxb8kw9C7YhAQ==", "license": "Apache-2.0", "dependencies": { "google-auth-library": "^10.3.0", @@ -3052,15 +2292,6 @@ } } }, - "node_modules/@google/generative-ai": { - "version": "0.21.0", - "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.21.0.tgz", - "integrity": "sha512-7XhUbtnlkSEZK15kN3t+tzIMxsbKm/dSkKBFalj+20NvPKe1kBY7mR2P7vuijEn+f06z5+A8bVGKO0v39cr6Wg==", - "license": "Apache-2.0", - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@grpc/grpc-js": { "version": "1.14.3", "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.3.tgz", @@ -3092,100 +2323,6 @@ "node": ">=6" } }, - "node_modules/@hapi/b64": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@hapi/b64/-/b64-5.0.0.tgz", - "integrity": "sha512-ngu0tSEmrezoiIaNGG6rRvKOUkUuDdf4XTPnONHGYfSGRmDqPZX5oJL6HAdKTo1UQHECbdB4OzhWrfgVppjHUw==", - "license": "BSD-3-Clause", - "dependencies": { - "@hapi/hoek": "9.x.x" - } - }, - "node_modules/@hapi/boom": { - "version": "9.1.4", - "resolved": "https://registry.npmjs.org/@hapi/boom/-/boom-9.1.4.tgz", - "integrity": "sha512-Ls1oH8jaN1vNsqcaHVYJrKmgMcKsC1wcp8bujvXrHaAqD2iDYq3HoOwsxwo09Cuda5R5nC0o0IxlrlTuvPuzSw==", - "license": "BSD-3-Clause", - "dependencies": { - "@hapi/hoek": "9.x.x" - } - }, - "node_modules/@hapi/bourne": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@hapi/bourne/-/bourne-2.1.0.tgz", - "integrity": "sha512-i1BpaNDVLJdRBEKeJWkVO6tYX6DMFBuwMhSuWqLsY4ufeTKGVuV5rBsUhxPayXqnnWHgXUAmWK16H/ykO5Wj4Q==", - "license": "BSD-3-Clause" - }, - "node_modules/@hapi/cryptiles": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/@hapi/cryptiles/-/cryptiles-5.1.0.tgz", - "integrity": "sha512-fo9+d1Ba5/FIoMySfMqPBR/7Pa29J2RsiPrl7bkwo5W5o+AN1dAYQRi4SPrPwwVxVGKjgLOEWrsvt1BonJSfLA==", - "license": "BSD-3-Clause", - "dependencies": { - "@hapi/boom": "9.x.x" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/@hapi/hoek": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz", - "integrity": "sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ==", - "license": "BSD-3-Clause" - }, - "node_modules/@hapi/iron": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/@hapi/iron/-/iron-6.0.0.tgz", - "integrity": "sha512-zvGvWDufiTGpTJPG1Y/McN8UqWBu0k/xs/7l++HVU535NLHXsHhy54cfEMdW7EjwKfbBfM9Xy25FmTiobb7Hvw==", - "license": "BSD-3-Clause", - "dependencies": { - "@hapi/b64": "5.x.x", - "@hapi/boom": "9.x.x", - "@hapi/bourne": "2.x.x", - "@hapi/cryptiles": "5.x.x", - "@hapi/hoek": "9.x.x" - } - }, - "node_modules/@hapi/podium": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/@hapi/podium/-/podium-4.1.3.tgz", - "integrity": "sha512-ljsKGQzLkFqnQxE7qeanvgGj4dejnciErYd30dbrYzUOF/FyS/DOF97qcrT3bhoVwCYmxa6PEMhxfCPlnUcD2g==", - "license": "BSD-3-Clause", - "dependencies": { - "@hapi/hoek": "9.x.x", - "@hapi/teamwork": "5.x.x", - "@hapi/validate": "1.x.x" - } - }, - "node_modules/@hapi/teamwork": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/@hapi/teamwork/-/teamwork-5.1.1.tgz", - "integrity": "sha512-1oPx9AE5TIv+V6Ih54RP9lTZBso3rP8j4Xhb6iSVwPXtAM+sDopl5TFMv5Paw73UnpZJ9gjcrTE1BXrWt9eQrg==", - "license": "BSD-3-Clause", - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/@hapi/topo": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/@hapi/topo/-/topo-5.1.0.tgz", - "integrity": "sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg==", - "license": "BSD-3-Clause", - "dependencies": { - "@hapi/hoek": "^9.0.0" - } - }, - "node_modules/@hapi/validate": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@hapi/validate/-/validate-1.1.3.tgz", - "integrity": "sha512-/XMR0N0wjw0Twzq2pQOzPBZlDzkekGcoCtzO314BpIEsbXdYGthQUbxgkGDf4nhk1+IPDAsXqWjMohRQYO06UA==", - "license": "BSD-3-Clause", - "dependencies": { - "@hapi/hoek": "^9.0.0", - "@hapi/topo": "^5.0.0" - } - }, "node_modules/@heyputer/backend": { "resolved": "src/backend", "link": true @@ -3200,17 +2337,6 @@ "integrity": "sha512-YhVtzz7ZA/HmuaDvzZZhhUyQWBvp3/TXeY4jULssTdLJwT+tEM4BTYHXttORX+V5auvrYinjj8dNFQnby5T82w==", "license": "MIT" }, - "node_modules/@heyputer/multest": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/@heyputer/multest/-/multest-0.0.2.tgz", - "integrity": "sha512-Hr4U9Z2/oMIyCKv+cgO3pNzncA4PAlkS9RotZwoicfzR2BLLu2Rjjp3/HpcLmtE7UEt0hlRMYP+ImMZPYGNwkg==", - "license": "UNLICENSED", - "dependencies": { - "append-field": "^1.0.0", - "busboy": "^1.6.0", - "form-data": "^4.0.0" - } - }, "node_modules/@heyputer/puter-wisp": { "resolved": "src/puter-wisp", "link": true @@ -3223,32 +2349,53 @@ "resolved": "src/putility", "link": true }, + "node_modules/@heyputer/worker": { + "resolved": "src/worker", + "link": true + }, "node_modules/@humanfs/core": { - "version": "0.19.1", - "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", - "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==", + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", + "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==", + "dev": true, "license": "Apache-2.0", + "dependencies": { + "@humanfs/types": "^0.15.0" + }, "engines": { "node": ">=18.18.0" } }, "node_modules/@humanfs/node": { - "version": "0.16.7", - "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz", - "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==", + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz", + "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==", + "dev": true, "license": "Apache-2.0", "dependencies": { - "@humanfs/core": "^0.19.1", + "@humanfs/core": "^0.19.2", + "@humanfs/types": "^0.15.0", "@humanwhocodes/retry": "^0.4.0" }, "engines": { "node": ">=18.18.0" } }, + "node_modules/@humanfs/types": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz", + "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, "node_modules/@humanwhocodes/module-importer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, "license": "Apache-2.0", "engines": { "node": ">=12.22" @@ -3262,6 +2409,7 @@ "version": "0.4.3", "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, "license": "Apache-2.0", "engines": { "node": ">=18.18" @@ -3363,6 +2511,9 @@ "cpu": [ "arm" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3379,6 +2530,9 @@ "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3395,6 +2549,9 @@ "cpu": [ "ppc64" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3411,6 +2568,9 @@ "cpu": [ "riscv64" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3427,6 +2587,9 @@ "cpu": [ "s390x" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3443,6 +2606,9 @@ "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3459,6 +2625,9 @@ "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3475,6 +2644,9 @@ "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3491,6 +2663,9 @@ "cpu": [ "arm" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3513,6 +2688,9 @@ "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3535,6 +2713,9 @@ "cpu": [ "ppc64" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3557,6 +2738,9 @@ "cpu": [ "riscv64" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3579,6 +2763,9 @@ "cpu": [ "s390x" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3601,6 +2788,9 @@ "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3623,6 +2813,9 @@ "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3645,6 +2838,9 @@ "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3748,714 +2944,14 @@ "integrity": "sha512-JH8ZL/ywcJyR9MmJ5BNqZllXNZQqQbnVZOqpPQqE1vHiFgAw4NHbvE0FOduNU8IX9babitBT46571OnPTT0Zcw==", "license": "MIT" }, - "node_modules/@istanbuljs/load-nyc-config": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", - "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "camelcase": "^5.3.1", - "find-up": "^4.1.0", - "get-package-type": "^0.1.0", - "js-yaml": "^3.13.1", - "resolve-from": "^5.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "license": "MIT", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/js-yaml": { - "version": "3.14.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", - "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/resolve-from": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", - "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/@istanbuljs/schema": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", - "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@jimp/core": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/core/-/core-1.6.1.tgz", - "integrity": "sha512-+BoKC5G6hkrSy501zcJ2EpfnllP+avPevcBfRcZe/CW+EwEfY6X1EZ8QWyT7NpDIvEEJb1fdJnMMfUnFkxmw9A==", - "license": "MIT", - "dependencies": { - "@jimp/file-ops": "1.6.1", - "@jimp/types": "1.6.1", - "@jimp/utils": "1.6.1", - "await-to-js": "^3.0.0", - "exif-parser": "^0.1.12", - "file-type": "^21.3.3", - "mime": "3" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/diff": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/diff/-/diff-1.6.1.tgz", - "integrity": "sha512-YkKDPdHjLgo1Api3+Bhc0GLAygldlpt97NfOKoNg1U6IUNXA6X2MgosCjPfSBiSvJvrrz1fsIR+/4cfYXBI/HQ==", - "license": "MIT", - "dependencies": { - "@jimp/plugin-resize": "1.6.1", - "@jimp/types": "1.6.1", - "@jimp/utils": "1.6.1", - "pixelmatch": "^5.3.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/file-ops": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/file-ops/-/file-ops-1.6.1.tgz", - "integrity": "sha512-T+gX6osHjprbDRad0/B71Evyre7ZdVY1z/gFGEG9Z8KOtZPKboWvPeP2UjbZYWQLy9UKCPQX1FNAnDiOPkJL7w==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/js-bmp": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/js-bmp/-/js-bmp-1.6.1.tgz", - "integrity": "sha512-xzWzNT4/u5zGrTT3Tme9sGU7YzIKxi13+BCQwLqACbt5DXf9SAfdzRkopZQnmDko+6In5nqaT89Gjs43/WdnYQ==", - "license": "MIT", - "dependencies": { - "@jimp/core": "1.6.1", - "@jimp/types": "1.6.1", - "@jimp/utils": "1.6.1", - "bmp-ts": "^1.0.9" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/js-gif": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/js-gif/-/js-gif-1.6.1.tgz", - "integrity": "sha512-YjY2W26rQa05XhanYhRZ7dingCiNN+T2Ymb1JiigIbABY0B28wHE3v3Cf1/HZPWGu0hOg36ylaKgV5KxF2M58w==", - "license": "MIT", - "dependencies": { - "@jimp/core": "1.6.1", - "@jimp/types": "1.6.1", - "gifwrap": "^0.10.1", - "omggif": "^1.0.10" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/js-jpeg": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/js-jpeg/-/js-jpeg-1.6.1.tgz", - "integrity": "sha512-HT9H3yOmlOFzYmdI15IYdfy6ggQhSRIaHeA+OTJSEORXBqEo97sUZu/DsgHIcX5NJ7TkJBTgZ9BZXsV6UbsyMg==", - "license": "MIT", - "dependencies": { - "@jimp/core": "1.6.1", - "@jimp/types": "1.6.1", - "jpeg-js": "^0.4.4" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/js-png": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/js-png/-/js-png-1.6.1.tgz", - "integrity": "sha512-SZ/KVhI5UjcSzzlXsXdIi/LhJ7UShf2NkMOtVrbZQcGzsqNtynAelrOXeoTxcanfVqmNhAoVHg8yR2cYoqrYjA==", - "license": "MIT", - "dependencies": { - "@jimp/core": "1.6.1", - "@jimp/types": "1.6.1", - "pngjs": "^7.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/js-tiff": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/js-tiff/-/js-tiff-1.6.1.tgz", - "integrity": "sha512-jDG/eJquID1M4MBlKMmDRBmz2TpXMv7TUyu2nIRUxhlUc2ogC82T+VQUkca9GJH1BBJ9dx5sSE5dGkWNjIbZxw==", - "license": "MIT", - "dependencies": { - "@jimp/core": "1.6.1", - "@jimp/types": "1.6.1", - "utif2": "^4.1.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-blit": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/plugin-blit/-/plugin-blit-1.6.1.tgz", - "integrity": "sha512-MwnI7C7K81uWddY9FLw1fCOIy6SsPIUftUz36Spt7jisCn8/40DhQMlSxpxTNelnZb/2SnloFimQfRZAmHLOqQ==", - "license": "MIT", - "dependencies": { - "@jimp/types": "1.6.1", - "@jimp/utils": "1.6.1", - "zod": "^3.23.8" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-blit/node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/@jimp/plugin-blur": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/plugin-blur/-/plugin-blur-1.6.1.tgz", - "integrity": "sha512-lIo7Tzp5jQu30EFFSK/phXANK3citKVEjepDjQ6ljHoIFtuMRrnybnmI2Md24ulvWlDaz+hh3n6qrMb8ydwhZQ==", - "license": "MIT", - "dependencies": { - "@jimp/core": "1.6.1", - "@jimp/utils": "1.6.1" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-circle": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/plugin-circle/-/plugin-circle-1.6.1.tgz", - "integrity": "sha512-kK1PavY6cKHNNKce37vdV4Tmpc1/zDKngGoeOV3j+EMatoHFZUinV3s6F9aWryPs3A0xhCLZgdJ6Zeea1d5LCQ==", - "license": "MIT", - "dependencies": { - "@jimp/types": "1.6.1", - "zod": "^3.23.8" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-circle/node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/@jimp/plugin-color": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/plugin-color/-/plugin-color-1.6.1.tgz", - "integrity": "sha512-LtUN1vAP+LRlZAtTNVhDRSiXx+26Kbz3zJaG6a5k59gQ95jgT5mknnF8lxkHcqJthM4MEk3/tPxkdJpEybyF/A==", - "license": "MIT", - "dependencies": { - "@jimp/core": "1.6.1", - "@jimp/types": "1.6.1", - "@jimp/utils": "1.6.1", - "tinycolor2": "^1.6.0", - "zod": "^3.23.8" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-color/node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/@jimp/plugin-contain": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/plugin-contain/-/plugin-contain-1.6.1.tgz", - "integrity": "sha512-m0qhrfA8jkTqretGv4w+T/ADFR4GwBpE0sCOC2uJ0dzr44/ddOMsIdrpi89kabqYiPYIrxkgdCVCLm3zn1Vkkg==", - "license": "MIT", - "dependencies": { - "@jimp/core": "1.6.1", - "@jimp/plugin-blit": "1.6.1", - "@jimp/plugin-resize": "1.6.1", - "@jimp/types": "1.6.1", - "@jimp/utils": "1.6.1", - "zod": "^3.23.8" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-contain/node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/@jimp/plugin-cover": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/plugin-cover/-/plugin-cover-1.6.1.tgz", - "integrity": "sha512-hZytnsth0zoll6cPf434BrT+p/v569Wr5tyO6Dp0dH1IDPhzhB5F38sZGMLDo7bzQiN9JFVB3fxkcJ/WYCJ3Mg==", - "license": "MIT", - "dependencies": { - "@jimp/core": "1.6.1", - "@jimp/plugin-crop": "1.6.1", - "@jimp/plugin-resize": "1.6.1", - "@jimp/types": "1.6.1", - "zod": "^3.23.8" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-cover/node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/@jimp/plugin-crop": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/plugin-crop/-/plugin-crop-1.6.1.tgz", - "integrity": "sha512-EerRSLlclXyKDnYc/H9w/1amZW7b7v3OGi/VlerPd2M/pAu5X8TkyYWtfqYCXnNp1Ixtd8oCo9zGfY9zoXT4rg==", - "license": "MIT", - "dependencies": { - "@jimp/core": "1.6.1", - "@jimp/types": "1.6.1", - "@jimp/utils": "1.6.1", - "zod": "^3.23.8" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-crop/node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/@jimp/plugin-displace": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/plugin-displace/-/plugin-displace-1.6.1.tgz", - "integrity": "sha512-K07QVl7xQwIfD6KfxRV/c3E9e7ZBXxUXdWuvoTWcKHL2qV48MOF5Nqbz/aJW4ThnQARIsxvYlZjPFiqkCjlU+g==", - "license": "MIT", - "dependencies": { - "@jimp/types": "1.6.1", - "@jimp/utils": "1.6.1", - "zod": "^3.23.8" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-displace/node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/@jimp/plugin-dither": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/plugin-dither/-/plugin-dither-1.6.1.tgz", - "integrity": "sha512-+2V+GCV2WycMoX1/z977TkZ8Zq/4MVSKElHYatgUqtwXMi2fDK2gKYU2g9V39IqFvTJsTIsK0+58VFz/ROBVew==", - "license": "MIT", - "dependencies": { - "@jimp/types": "1.6.1" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-fisheye": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/plugin-fisheye/-/plugin-fisheye-1.6.1.tgz", - "integrity": "sha512-XtS5ZyoZ0vxZxJ6gkqI63SivhtI58vX95foMPM+cyzYkRsJXMOYCr8DScxF5bp4Xr003NjYm/P+7+08tibwzHA==", - "license": "MIT", - "dependencies": { - "@jimp/types": "1.6.1", - "@jimp/utils": "1.6.1", - "zod": "^3.23.8" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-fisheye/node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/@jimp/plugin-flip": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/plugin-flip/-/plugin-flip-1.6.1.tgz", - "integrity": "sha512-ws38W/sGj7LobNRayQ83garxiktOyWxM5vO/y4a/2cy9v65SLEUzVkrj+oeAaUSSObdz4HcCEla7XtGlnAGAaA==", - "license": "MIT", - "dependencies": { - "@jimp/types": "1.6.1", - "zod": "^3.23.8" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-flip/node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/@jimp/plugin-hash": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/plugin-hash/-/plugin-hash-1.6.1.tgz", - "integrity": "sha512-sZt6ZcMX6i8vFWb4GYnw0pR/o9++ef0dTVcboTB5B/g7nrxCODIB4wfEkJ/YqZM5wUvol77K1qeS0/rVO6z21A==", - "license": "MIT", - "dependencies": { - "@jimp/core": "1.6.1", - "@jimp/js-bmp": "1.6.1", - "@jimp/js-jpeg": "1.6.1", - "@jimp/js-png": "1.6.1", - "@jimp/js-tiff": "1.6.1", - "@jimp/plugin-color": "1.6.1", - "@jimp/plugin-resize": "1.6.1", - "@jimp/types": "1.6.1", - "@jimp/utils": "1.6.1", - "any-base": "^1.1.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-mask": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/plugin-mask/-/plugin-mask-1.6.1.tgz", - "integrity": "sha512-SIG0/FcmEj3tkwFxc7fAGLO8o4uNzMpSOdQOhbCgxefQKq5wOVMk9BQx/sdMPBwtMLr9WLq0GzLA/rk6t2v20A==", - "license": "MIT", - "dependencies": { - "@jimp/types": "1.6.1", - "zod": "^3.23.8" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-mask/node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/@jimp/plugin-print": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/plugin-print/-/plugin-print-1.6.1.tgz", - "integrity": "sha512-BYVz/X3Xzv8XYilVeDy11NOp0h7BTDjlOtu0BekIFHP1yHVd24AXNzbOy52XlzYZWQ0Dl36HOHEpl/nSNrzc6w==", - "license": "MIT", - "dependencies": { - "@jimp/core": "1.6.1", - "@jimp/js-jpeg": "1.6.1", - "@jimp/js-png": "1.6.1", - "@jimp/plugin-blit": "1.6.1", - "@jimp/types": "1.6.1", - "parse-bmfont-ascii": "^1.0.6", - "parse-bmfont-binary": "^1.0.6", - "parse-bmfont-xml": "^1.1.6", - "simple-xml-to-json": "^1.2.2", - "zod": "^3.23.8" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-print/node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/@jimp/plugin-quantize": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/plugin-quantize/-/plugin-quantize-1.6.1.tgz", - "integrity": "sha512-J2En9PLURfP+vwYDtuZ9T8yBW6BWYZBScydAjRiPBmJfEhTcNQqiiQODrZf7EqbbX/Sy5H6dAeRiqkgoV9N6Ww==", - "license": "MIT", - "dependencies": { - "image-q": "^4.0.0", - "zod": "^3.23.8" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-quantize/node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/@jimp/plugin-resize": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/plugin-resize/-/plugin-resize-1.6.1.tgz", - "integrity": "sha512-CLkrtJoIz2HdWnpYiN6p8KYcPc00rCH/SUu6o+lfZL05Q4uhecJlnvXuj9x+U6mDn3ldPmJj6aZqMHuUJzdVqg==", - "license": "MIT", - "dependencies": { - "@jimp/core": "1.6.1", - "@jimp/types": "1.6.1", - "zod": "^3.23.8" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-resize/node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/@jimp/plugin-rotate": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/plugin-rotate/-/plugin-rotate-1.6.1.tgz", - "integrity": "sha512-nOjVjbbj705B02ksysKnh0POAwEBXZtJ9zQ5qC+X7Tavl3JNn+P3BzQovbBxLPSbUSld6XID9z5ijin4PtOAUg==", - "license": "MIT", - "dependencies": { - "@jimp/core": "1.6.1", - "@jimp/plugin-crop": "1.6.1", - "@jimp/plugin-resize": "1.6.1", - "@jimp/types": "1.6.1", - "@jimp/utils": "1.6.1", - "zod": "^3.23.8" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-rotate/node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/@jimp/plugin-threshold": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/plugin-threshold/-/plugin-threshold-1.6.1.tgz", - "integrity": "sha512-JOKv9F8s6tnVLf4sB/2fF0F339EFnHvgEdFYugO6VhowKLsap0pEZmLyE/DlRnYtIj2RddHZVxVMp/eKJ04l2Q==", - "license": "MIT", - "dependencies": { - "@jimp/core": "1.6.1", - "@jimp/plugin-color": "1.6.1", - "@jimp/plugin-hash": "1.6.1", - "@jimp/types": "1.6.1", - "@jimp/utils": "1.6.1", - "zod": "^3.23.8" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/plugin-threshold/node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/@jimp/types": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/types/-/types-1.6.1.tgz", - "integrity": "sha512-leI7YbveTNi565m910XgIOwXyuu074H5qazAD1357HImJSv2hqxnWXpwxQbadGWZ7goZRYBDZy5lpqud0p7q5w==", - "license": "MIT", - "dependencies": { - "zod": "^3.23.8" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@jimp/types/node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/@jimp/utils": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@jimp/utils/-/utils-1.6.1.tgz", - "integrity": "sha512-veFPRd93FCnS7AgmCkPgARVGoDRrJ9cm1ujuNyA+UfQ5VKbED2002sm5XfFLFwTsKC8j04heTrwe+tU1dluXOw==", - "license": "MIT", - "dependencies": { - "@jimp/types": "1.6.1", - "tinycolor2": "^1.6.0" - }, - "engines": { - "node": ">=18" - } - }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.0", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@jridgewell/remapping": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", - "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", "dev": true, "license": "MIT", "dependencies": { - "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, @@ -4463,6 +2959,7 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, "license": "MIT", "engines": { "node": ">=6.0.0" @@ -4483,12 +2980,14 @@ "version": "1.5.5", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, "license": "MIT" }, "node_modules/@jridgewell/trace-mapping": { "version": "0.3.31", "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, "license": "MIT", "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", @@ -4562,9 +3061,9 @@ } }, "node_modules/@napi-rs/wasm-runtime": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.3.tgz", - "integrity": "sha512-xK9sGVbJWYb08+mTJt3/YV24WxvxpXcXtP6B172paPZ+Ts69Re9dAr7lKwJoeIx8OoeuimEiRZ7umkiUVClmmQ==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", + "integrity": "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==", "dev": true, "license": "MIT", "optional": true, @@ -4581,17 +3080,29 @@ } }, "node_modules/@noble/hashes": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", - "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-2.0.1.tgz", + "integrity": "sha512-XlOlEbQcE9fmuXxrVTXCTlG2nlRXa9Rj3rr5Ue/+tX+nmkgbX720YHh0VR3hBF9xDvwnb8D2shVGOwNx+ulArw==", "license": "MIT", "engines": { - "node": "^14.21.3 || >=16" + "node": ">= 20.19.0" }, "funding": { "url": "https://paulmillr.com/funding/" } }, + "node_modules/@nodable/entities": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-2.1.0.tgz", + "integrity": "sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -4635,74 +3146,75 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", "license": "Apache-2.0", - "peer": true, "engines": { "node": ">=8.0.0" } }, "node_modules/@opentelemetry/api-logs": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.49.1.tgz", - "integrity": "sha512-kaNl/T7WzyMUQHQlVq7q0oV4Kev6+0xFwqzofryC66jgGMacd0QH5TwfpbUwSTby+SdAdprAe5UKMvBw4tKS5Q==", + "version": "0.54.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.54.2.tgz", + "integrity": "sha512-4MTVwwmLgUh5QrJnZpYo6YRO5IBLAggf2h8gWDblwRagDStY13aEvt7gGk3jewrMaPlHiF83fENhIx0HO97/cQ==", "license": "Apache-2.0", - "peer": true, "dependencies": { - "@opentelemetry/api": "^1.0.0" + "@opentelemetry/api": "^1.3.0" }, "engines": { "node": ">=14" } }, "node_modules/@opentelemetry/auto-instrumentations-node": { - "version": "0.43.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/auto-instrumentations-node/-/auto-instrumentations-node-0.43.0.tgz", - "integrity": "sha512-2WvHUSi/QVeVG8ObPD0Ls6WevfIbQjspxIQRuHaQFWXhmEwy/MsEcoQUjbNKXwO5516aS04GTydKEoRKsMwhdA==", + "version": "0.52.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/auto-instrumentations-node/-/auto-instrumentations-node-0.52.1.tgz", + "integrity": "sha512-4QaRTZifSoYnh27B3JA7z7YwE0Nwkd824pDeonAQVijeLLsenhZB1japualZ6mF9lY8VdQId9KkNsgmCGdJVNQ==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/instrumentation-amqplib": "^0.35.0", - "@opentelemetry/instrumentation-aws-lambda": "^0.39.0", - "@opentelemetry/instrumentation-aws-sdk": "^0.39.1", - "@opentelemetry/instrumentation-bunyan": "^0.36.0", - "@opentelemetry/instrumentation-cassandra-driver": "^0.36.0", - "@opentelemetry/instrumentation-connect": "^0.34.0", - "@opentelemetry/instrumentation-cucumber": "^0.4.0", - "@opentelemetry/instrumentation-dataloader": "^0.7.0", - "@opentelemetry/instrumentation-dns": "^0.34.0", - "@opentelemetry/instrumentation-express": "^0.36.1", - "@opentelemetry/instrumentation-fastify": "^0.34.0", - "@opentelemetry/instrumentation-fs": "^0.10.0", - "@opentelemetry/instrumentation-generic-pool": "^0.34.0", - "@opentelemetry/instrumentation-graphql": "^0.38.1", - "@opentelemetry/instrumentation-grpc": "^0.49.1", - "@opentelemetry/instrumentation-hapi": "^0.35.0", - "@opentelemetry/instrumentation-http": "^0.49.1", - "@opentelemetry/instrumentation-ioredis": "^0.38.0", - "@opentelemetry/instrumentation-knex": "^0.34.0", - "@opentelemetry/instrumentation-koa": "^0.38.0", - "@opentelemetry/instrumentation-lru-memoizer": "^0.35.0", - "@opentelemetry/instrumentation-memcached": "^0.34.0", - "@opentelemetry/instrumentation-mongodb": "^0.41.0", - "@opentelemetry/instrumentation-mongoose": "^0.36.0", - "@opentelemetry/instrumentation-mysql": "^0.36.0", - "@opentelemetry/instrumentation-mysql2": "^0.36.0", - "@opentelemetry/instrumentation-nestjs-core": "^0.35.0", - "@opentelemetry/instrumentation-net": "^0.34.0", - "@opentelemetry/instrumentation-pg": "^0.39.1", - "@opentelemetry/instrumentation-pino": "^0.36.0", - "@opentelemetry/instrumentation-redis": "^0.37.0", - "@opentelemetry/instrumentation-redis-4": "^0.37.0", - "@opentelemetry/instrumentation-restify": "^0.36.0", - "@opentelemetry/instrumentation-router": "^0.35.0", - "@opentelemetry/instrumentation-socket.io": "^0.37.0", - "@opentelemetry/instrumentation-tedious": "^0.8.0", - "@opentelemetry/instrumentation-winston": "^0.35.0", - "@opentelemetry/resource-detector-alibaba-cloud": "^0.28.7", - "@opentelemetry/resource-detector-aws": "^1.4.0", - "@opentelemetry/resource-detector-container": "^0.3.7", - "@opentelemetry/resource-detector-gcp": "^0.29.7", - "@opentelemetry/resources": "^1.12.0", - "@opentelemetry/sdk-node": "^0.49.1" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/instrumentation-amqplib": "^0.43.0", + "@opentelemetry/instrumentation-aws-lambda": "^0.47.0", + "@opentelemetry/instrumentation-aws-sdk": "^0.46.0", + "@opentelemetry/instrumentation-bunyan": "^0.42.0", + "@opentelemetry/instrumentation-cassandra-driver": "^0.42.0", + "@opentelemetry/instrumentation-connect": "^0.40.0", + "@opentelemetry/instrumentation-cucumber": "^0.10.0", + "@opentelemetry/instrumentation-dataloader": "^0.13.0", + "@opentelemetry/instrumentation-dns": "^0.40.0", + "@opentelemetry/instrumentation-express": "^0.44.0", + "@opentelemetry/instrumentation-fastify": "^0.41.0", + "@opentelemetry/instrumentation-fs": "^0.16.0", + "@opentelemetry/instrumentation-generic-pool": "^0.40.0", + "@opentelemetry/instrumentation-graphql": "^0.44.0", + "@opentelemetry/instrumentation-grpc": "^0.54.0", + "@opentelemetry/instrumentation-hapi": "^0.42.0", + "@opentelemetry/instrumentation-http": "^0.54.0", + "@opentelemetry/instrumentation-ioredis": "^0.44.0", + "@opentelemetry/instrumentation-kafkajs": "^0.4.0", + "@opentelemetry/instrumentation-knex": "^0.41.0", + "@opentelemetry/instrumentation-koa": "^0.44.0", + "@opentelemetry/instrumentation-lru-memoizer": "^0.41.0", + "@opentelemetry/instrumentation-memcached": "^0.40.0", + "@opentelemetry/instrumentation-mongodb": "^0.48.0", + "@opentelemetry/instrumentation-mongoose": "^0.43.0", + "@opentelemetry/instrumentation-mysql": "^0.42.0", + "@opentelemetry/instrumentation-mysql2": "^0.42.1", + "@opentelemetry/instrumentation-nestjs-core": "^0.41.0", + "@opentelemetry/instrumentation-net": "^0.40.0", + "@opentelemetry/instrumentation-pg": "^0.47.1", + "@opentelemetry/instrumentation-pino": "^0.43.0", + "@opentelemetry/instrumentation-redis": "^0.43.0", + "@opentelemetry/instrumentation-redis-4": "^0.43.0", + "@opentelemetry/instrumentation-restify": "^0.42.0", + "@opentelemetry/instrumentation-router": "^0.41.0", + "@opentelemetry/instrumentation-socket.io": "^0.43.0", + "@opentelemetry/instrumentation-tedious": "^0.15.0", + "@opentelemetry/instrumentation-undici": "^0.7.1", + "@opentelemetry/instrumentation-winston": "^0.41.0", + "@opentelemetry/resource-detector-alibaba-cloud": "^0.29.4", + "@opentelemetry/resource-detector-aws": "^1.7.0", + "@opentelemetry/resource-detector-azure": "^0.2.12", + "@opentelemetry/resource-detector-container": "^0.5.0", + "@opentelemetry/resource-detector-gcp": "^0.29.13", + "@opentelemetry/resources": "^1.24.0", + "@opentelemetry/sdk-node": "^0.54.0" }, "engines": { "node": ">=14" @@ -4711,18 +3223,160 @@ "@opentelemetry/api": "^1.4.1" } }, - "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/exporter-trace-otlp-grpc": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.49.1.tgz", - "integrity": "sha512-Zbd7f3zF7fI2587MVhBizaW21cO/SordyrZGtMtvhoxU6n4Qb02Gx71X4+PzXH620e0+JX+Pcr9bYb1HTeVyJA==", + "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/context-async-hooks": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-1.27.0.tgz", + "integrity": "sha512-CdZ3qmHCwNhFAzjTgHqrDQ44Qxcpz43cVxZRhOs+Ns/79ug+Mr84Bkb626bkJLkA3+BLimA5YAEVRlJC6pFb7g==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/core": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.27.0.tgz", + "integrity": "sha512-yQPKnK5e+76XuiqUH/gKyS8wv/7qITd5ln56QkBTf3uggr0VkXOXfcaAuG330UfdYu83wsyoBwqwxigpIG+Jkg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/exporter-logs-otlp-grpc": { + "version": "0.54.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-grpc/-/exporter-logs-otlp-grpc-0.54.2.tgz", + "integrity": "sha512-MQNmV5r96+5n3axLFgNYtVy62x8Ru7VERZH3zgC50KDcIKWCiQT3vHOtzakhzd1Wq0HqOgu6bzKdwzneSoDrEQ==", "license": "Apache-2.0", "dependencies": { "@grpc/grpc-js": "^1.7.1", - "@opentelemetry/core": "1.22.0", - "@opentelemetry/otlp-grpc-exporter-base": "0.49.1", - "@opentelemetry/otlp-transformer": "0.49.1", - "@opentelemetry/resources": "1.22.0", - "@opentelemetry/sdk-trace-base": "1.22.0" + "@opentelemetry/core": "1.27.0", + "@opentelemetry/otlp-grpc-exporter-base": "0.54.2", + "@opentelemetry/otlp-transformer": "0.54.2", + "@opentelemetry/sdk-logs": "0.54.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/exporter-logs-otlp-http": { + "version": "0.54.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-http/-/exporter-logs-otlp-http-0.54.2.tgz", + "integrity": "sha512-wYeCSbX2XWX2wFslnfQ/YFUolO0fj2nUiGI7oEQWpLKSg40Lc4xOOW14X/EXOkCCijhP7bigo6nvyEQlxEVLjA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.54.2", + "@opentelemetry/core": "1.27.0", + "@opentelemetry/otlp-exporter-base": "0.54.2", + "@opentelemetry/otlp-transformer": "0.54.2", + "@opentelemetry/sdk-logs": "0.54.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/exporter-logs-otlp-proto": { + "version": "0.54.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-proto/-/exporter-logs-otlp-proto-0.54.2.tgz", + "integrity": "sha512-agrzFbSNmIy6dhkyg41ERlEDUDqkaUJj2n/tVRFp9Tl+6wyNVPsqmwU5RWJOXpyK+lYH/znv6A47VpTeJF0lrw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.54.2", + "@opentelemetry/core": "1.27.0", + "@opentelemetry/otlp-exporter-base": "0.54.2", + "@opentelemetry/otlp-transformer": "0.54.2", + "@opentelemetry/resources": "1.27.0", + "@opentelemetry/sdk-logs": "0.54.2", + "@opentelemetry/sdk-trace-base": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/exporter-trace-otlp-grpc": { + "version": "0.54.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.54.2.tgz", + "integrity": "sha512-tmxiCYhQdPrzwlM6O7VQeNP9PBjKhaiOo54wFxQFZQcoVaDiOOES4+6PwHU1eW+43mDsgdQHN5AHSRHVLe9jDA==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.7.1", + "@opentelemetry/core": "1.27.0", + "@opentelemetry/otlp-grpc-exporter-base": "0.54.2", + "@opentelemetry/otlp-transformer": "0.54.2", + "@opentelemetry/resources": "1.27.0", + "@opentelemetry/sdk-trace-base": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/exporter-trace-otlp-http": { + "version": "0.54.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.54.2.tgz", + "integrity": "sha512-BgWKKyD/h2zpISdmYHN/sapwTjvt1P4p5yx4xeBV8XAEqh4OQUhOtSGFG80+nPQ1F8of3mKOT1DDoDbJp1u25w==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.27.0", + "@opentelemetry/otlp-exporter-base": "0.54.2", + "@opentelemetry/otlp-transformer": "0.54.2", + "@opentelemetry/resources": "1.27.0", + "@opentelemetry/sdk-trace-base": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/exporter-trace-otlp-proto": { + "version": "0.54.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-proto/-/exporter-trace-otlp-proto-0.54.2.tgz", + "integrity": "sha512-XSmm1N2wAhoWDXP1q/N6kpLebWaxl6VIADv4WA5QWKHLRpF3gLz5NAWNJBR8ygsvv8jQcrwnXgwfnJ18H3v1fg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.27.0", + "@opentelemetry/otlp-exporter-base": "0.54.2", + "@opentelemetry/otlp-transformer": "0.54.2", + "@opentelemetry/resources": "1.27.0", + "@opentelemetry/sdk-trace-base": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/exporter-zipkin": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-zipkin/-/exporter-zipkin-1.27.0.tgz", + "integrity": "sha512-eGMY3s4QprspFZojqsuQyQpWNFpo+oNVE/aosTbtvAlrJBAlvXcwwsOROOHOd8Y9lkU4i0FpQW482rcXkgwCSw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.27.0", + "@opentelemetry/resources": "1.27.0", + "@opentelemetry/sdk-trace-base": "1.27.0", + "@opentelemetry/semantic-conventions": "1.27.0" }, "engines": { "node": ">=14" @@ -4731,358 +3385,235 @@ "@opentelemetry/api": "^1.0.0" } }, - "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/core": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.22.0.tgz", - "integrity": "sha512-0VoAlT6x+Xzik1v9goJ3pZ2ppi6+xd3aUfg4brfrLkDBHRIVjMP0eBHrKrhB+NKcDyMAg8fAbGL3Npg/F6AwWA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" - } - }, - "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/otlp-transformer": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.49.1.tgz", - "integrity": "sha512-Z+koA4wp9L9e3jkFacyXTGphSWTbOKjwwXMpb0CxNb0kjTHGUxhYRN8GnkLFsFo5NbZPjP07hwAqeEG/uCratQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api-logs": "0.49.1", - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0", - "@opentelemetry/sdk-logs": "0.49.1", - "@opentelemetry/sdk-metrics": "1.22.0", - "@opentelemetry/sdk-trace-base": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.9.0" - } - }, - "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-logs": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.49.1.tgz", - "integrity": "sha512-gCzYWsJE0h+3cuh3/cK+9UwlVFyHvj3PReIOCDOmdeXOp90ZjKRoDOJBc3mvk1LL6wyl1RWIivR8Rg9OToyesw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.4.0 <1.9.0", - "@opentelemetry/api-logs": ">=0.39.1" - } - }, - "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-metrics": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.22.0.tgz", - "integrity": "sha512-k6iIx6H3TZ+BVMr2z8M16ri2OxWaljg5h8ihGJxi/KQWcjign6FEaEzuigXt5bK9wVEhqAcWLCfarSftaNWkkg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0", - "lodash.merge": "^4.6.2" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.9.0" - } - }, - "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/resources": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.22.0.tgz", - "integrity": "sha512-+vNeIFPH2hfcNL0AJk/ykJXoUCtR1YaDUZM+p3wZNU4Hq98gzq+7b43xbkXjadD9VhWIUQqEwXyY64q6msPj6A==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/semantic-conventions": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" - } - }, - "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/sdk-trace-base": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.22.0.tgz", - "integrity": "sha512-pfTuSIpCKONC6vkTpv6VmACxD+P1woZf4q0K46nSUvXFvOFqjBYKFaAMkKD3M1mlKUUh0Oajwj35qNjMl80m1Q==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0", - "@opentelemetry/semantic-conventions": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" - } - }, "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/otlp-exporter-base": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.49.1.tgz", - "integrity": "sha512-z6sHliPqDgJU45kQatAettY9/eVF58qVPaTuejw9YWfSRqid9pXPYeegDCSdyS47KAUgAtm+nC28K3pfF27HWg==", + "version": "0.54.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.54.2.tgz", + "integrity": "sha512-NrNyxu6R/bGAwanhz1HI0aJWKR6xUED4TjCH4iWMlAfyRukGbI9Kt/Akd2sYLwRKNhfS+sKetKGCUQPMDyYYMA==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.22.0" + "@opentelemetry/core": "1.27.0", + "@opentelemetry/otlp-transformer": "0.54.2" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": "^1.0.0" - } - }, - "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/otlp-exporter-base/node_modules/@opentelemetry/core": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.22.0.tgz", - "integrity": "sha512-0VoAlT6x+Xzik1v9goJ3pZ2ppi6+xd3aUfg4brfrLkDBHRIVjMP0eBHrKrhB+NKcDyMAg8fAbGL3Npg/F6AwWA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" + "@opentelemetry/api": "^1.3.0" } }, "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/otlp-grpc-exporter-base": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-grpc-exporter-base/-/otlp-grpc-exporter-base-0.49.1.tgz", - "integrity": "sha512-DNDNUWmOqtKTFJAyOyHHKotVox0NQ/09ETX8fUOeEtyNVHoGekAVtBbvIA3AtK+JflP7LC0PTjlLfruPM3Wy6w==", + "version": "0.54.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-grpc-exporter-base/-/otlp-grpc-exporter-base-0.54.2.tgz", + "integrity": "sha512-HZtACQuLhgDcgNa9arGnVVGV28sSGQ+iwRgICWikFKiVxUsoWffqBvTxPa6G3DUTg5R+up97j/zxubEyxSAOHg==", "license": "Apache-2.0", "dependencies": { "@grpc/grpc-js": "^1.7.1", - "@opentelemetry/core": "1.22.0", - "@opentelemetry/otlp-exporter-base": "0.49.1", - "protobufjs": "^7.2.3" + "@opentelemetry/core": "1.27.0", + "@opentelemetry/otlp-exporter-base": "0.54.2", + "@opentelemetry/otlp-transformer": "0.54.2" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": "^1.0.0" + "@opentelemetry/api": "^1.3.0" } }, - "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/otlp-grpc-exporter-base/node_modules/@opentelemetry/core": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.22.0.tgz", - "integrity": "sha512-0VoAlT6x+Xzik1v9goJ3pZ2ppi6+xd3aUfg4brfrLkDBHRIVjMP0eBHrKrhB+NKcDyMAg8fAbGL3Npg/F6AwWA==", + "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/otlp-transformer": { + "version": "0.54.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.54.2.tgz", + "integrity": "sha512-2tIjahJlMRRUz0A2SeE+qBkeBXBFkSjR0wqJ08kuOqaL8HNGan5iZf+A8cfrfmZzPUuMKCyY9I+okzFuFs6gKQ==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/semantic-conventions": "1.22.0" + "@opentelemetry/api-logs": "0.54.2", + "@opentelemetry/core": "1.27.0", + "@opentelemetry/resources": "1.27.0", + "@opentelemetry/sdk-logs": "0.54.2", + "@opentelemetry/sdk-metrics": "1.27.0", + "@opentelemetry/sdk-trace-base": "1.27.0", + "protobufjs": "^7.3.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/propagator-b3": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-b3/-/propagator-b3-1.27.0.tgz", + "integrity": "sha512-pTsko3gnMioe3FeWcwTQR3omo5C35tYsKKwjgTCTVCgd3EOWL9BZrMfgLBmszrwXABDfUrlAEFN/0W0FfQGynQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/propagator-jaeger": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-jaeger/-/propagator-jaeger-1.27.0.tgz", + "integrity": "sha512-EI1bbK0wn0yIuKlc2Qv2LKBRw6LiUWevrjCF80fn/rlaB+7StAi8Y5s8DBqAYNpY7v1q86+NjU18v7hj2ejU3A==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/resources": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.27.0.tgz", + "integrity": "sha512-jOwt2VJ/lUD5BLc+PMNymDrUCpm5PKi1E9oSVYAvz01U/VdndGmrtV3DU1pG4AwlYhJRHbHfOUIlpBeXCPw6QQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.27.0", + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/sdk-logs": { + "version": "0.54.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.54.2.tgz", + "integrity": "sha512-yIbYqDLS/AtBbPjCjh6eSToGNRMqW2VR8RrKEy+G+J7dFG7pKoptTH5T+XlKPleP9NY8JZYIpgJBlI+Osi0rFw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.54.2", + "@opentelemetry/core": "1.27.0", + "@opentelemetry/resources": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/sdk-metrics": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.27.0.tgz", + "integrity": "sha512-JzWgzlutoXCydhHWIbLg+r76m+m3ncqvkCcsswXAQ4gqKS+LOHKhq+t6fx1zNytvLuaOUBur7EvWxECc4jPQKg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.27.0", + "@opentelemetry/resources": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/sdk-node": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-node/-/sdk-node-0.49.1.tgz", - "integrity": "sha512-feBIT85ndiSHXsQ2gfGpXC/sNeX4GCHLksC4A9s/bfpUbbgbCSl0RvzZlmEpCHarNrkZMwFRi4H0xFfgvJEjrg==", + "version": "0.54.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-node/-/sdk-node-0.54.2.tgz", + "integrity": "sha512-afn8GBpA7Gb55aU0LUxIQ+oe6QxLhsf+Te9iw12Non3ZAspzdoCcfz5+hqecwpuVpEDdnj5iSalF7VVaL2pDeg==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/api-logs": "0.49.1", - "@opentelemetry/core": "1.22.0", - "@opentelemetry/exporter-trace-otlp-grpc": "0.49.1", - "@opentelemetry/exporter-trace-otlp-http": "0.49.1", - "@opentelemetry/exporter-trace-otlp-proto": "0.49.1", - "@opentelemetry/exporter-zipkin": "1.22.0", - "@opentelemetry/instrumentation": "0.49.1", - "@opentelemetry/resources": "1.22.0", - "@opentelemetry/sdk-logs": "0.49.1", - "@opentelemetry/sdk-metrics": "1.22.0", - "@opentelemetry/sdk-trace-base": "1.22.0", - "@opentelemetry/sdk-trace-node": "1.22.0", - "@opentelemetry/semantic-conventions": "1.22.0" + "@opentelemetry/api-logs": "0.54.2", + "@opentelemetry/core": "1.27.0", + "@opentelemetry/exporter-logs-otlp-grpc": "0.54.2", + "@opentelemetry/exporter-logs-otlp-http": "0.54.2", + "@opentelemetry/exporter-logs-otlp-proto": "0.54.2", + "@opentelemetry/exporter-trace-otlp-grpc": "0.54.2", + "@opentelemetry/exporter-trace-otlp-http": "0.54.2", + "@opentelemetry/exporter-trace-otlp-proto": "0.54.2", + "@opentelemetry/exporter-zipkin": "1.27.0", + "@opentelemetry/instrumentation": "0.54.2", + "@opentelemetry/resources": "1.27.0", + "@opentelemetry/sdk-logs": "0.54.2", + "@opentelemetry/sdk-metrics": "1.27.0", + "@opentelemetry/sdk-trace-base": "1.27.0", + "@opentelemetry/sdk-trace-node": "1.27.0", + "@opentelemetry/semantic-conventions": "1.27.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.9.0" + "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, - "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/core": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.22.0.tgz", - "integrity": "sha512-0VoAlT6x+Xzik1v9goJ3pZ2ppi6+xd3aUfg4brfrLkDBHRIVjMP0eBHrKrhB+NKcDyMAg8fAbGL3Npg/F6AwWA==", + "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/sdk-trace-base": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.27.0.tgz", + "integrity": "sha512-btz6XTQzwsyJjombpeqCX6LhiMQYpzt2pIYNPnw0IPO/3AhT6yjnf8Mnv3ZC2A4eRYOjqrg+bfaXg9XHDRJDWQ==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/semantic-conventions": "1.22.0" + "@opentelemetry/core": "1.27.0", + "@opentelemetry/resources": "1.27.0", + "@opentelemetry/semantic-conventions": "1.27.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" + "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, - "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/resources": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.22.0.tgz", - "integrity": "sha512-+vNeIFPH2hfcNL0AJk/ykJXoUCtR1YaDUZM+p3wZNU4Hq98gzq+7b43xbkXjadD9VhWIUQqEwXyY64q6msPj6A==", + "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/sdk-trace-node": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-1.27.0.tgz", + "integrity": "sha512-dWZp/dVGdUEfRBjBq2BgNuBlFqHCxyyMc8FsN0NX15X07mxSUO0SZRLyK/fdAVrde8nqFI/FEdMH4rgU9fqJfQ==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/semantic-conventions": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" - } - }, - "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/sdk-logs": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.49.1.tgz", - "integrity": "sha512-gCzYWsJE0h+3cuh3/cK+9UwlVFyHvj3PReIOCDOmdeXOp90ZjKRoDOJBc3mvk1LL6wyl1RWIivR8Rg9OToyesw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.4.0 <1.9.0", - "@opentelemetry/api-logs": ">=0.39.1" - } - }, - "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/sdk-metrics": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.22.0.tgz", - "integrity": "sha512-k6iIx6H3TZ+BVMr2z8M16ri2OxWaljg5h8ihGJxi/KQWcjign6FEaEzuigXt5bK9wVEhqAcWLCfarSftaNWkkg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0", - "lodash.merge": "^4.6.2" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.9.0" - } - }, - "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/sdk-trace-base": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.22.0.tgz", - "integrity": "sha512-pfTuSIpCKONC6vkTpv6VmACxD+P1woZf4q0K46nSUvXFvOFqjBYKFaAMkKD3M1mlKUUh0Oajwj35qNjMl80m1Q==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0", - "@opentelemetry/semantic-conventions": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" - } - }, - "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/sdk-trace-node": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-1.22.0.tgz", - "integrity": "sha512-gTGquNz7ue8uMeiWPwp3CU321OstQ84r7PCDtOaCicjbJxzvO8RZMlEC4geOipTeiF88kss5n6w+//A0MhP1lQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/context-async-hooks": "1.22.0", - "@opentelemetry/core": "1.22.0", - "@opentelemetry/propagator-b3": "1.22.0", - "@opentelemetry/propagator-jaeger": "1.22.0", - "@opentelemetry/sdk-trace-base": "1.22.0", + "@opentelemetry/context-async-hooks": "1.27.0", + "@opentelemetry/core": "1.27.0", + "@opentelemetry/propagator-b3": "1.27.0", + "@opentelemetry/propagator-jaeger": "1.27.0", + "@opentelemetry/sdk-trace-base": "1.27.0", "semver": "^7.5.2" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" - } - }, - "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/sdk-trace-node/node_modules/@opentelemetry/context-async-hooks": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-1.22.0.tgz", - "integrity": "sha512-Nfdxyg8YtWqVWkyrCukkundAjPhUXi93JtVQmqDT1mZRVKqA7e2r7eJCrI+F651XUBMp0hsOJSGiFk3QSpaIJw==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" - } - }, - "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/sdk-trace-node/node_modules/@opentelemetry/propagator-b3": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-b3/-/propagator-b3-1.22.0.tgz", - "integrity": "sha512-qBItJm9ygg/jCB5rmivyGz1qmKZPsL/sX715JqPMFgq++Idm0x+N9sLQvWFHFt2+ZINnCSojw7FVBgFW6izcXA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" - } - }, - "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/sdk-trace-node/node_modules/@opentelemetry/propagator-jaeger": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-jaeger/-/propagator-jaeger-1.22.0.tgz", - "integrity": "sha512-pMLgst3QIwrUfepraH5WG7xfpJ8J3CrPKrtINK0t7kBkuu96rn+HDYQ8kt3+0FXvrZI8YJE77MCQwnJWXIrgpA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" + "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "node_modules/@opentelemetry/auto-instrumentations-node/node_modules/@opentelemetry/semantic-conventions": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.22.0.tgz", - "integrity": "sha512-CAOgFOKLybd02uj/GhCdEeeBjOS0yeoDeo/CA7ASBSmenpZHAKGB3iDm/rv3BQLcabb/OprDEsSQ1y0P8A7Siw==", + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz", + "integrity": "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==", "license": "Apache-2.0", "engines": { "node": ">=14" } }, + "node_modules/@opentelemetry/context-async-hooks": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-1.28.0.tgz", + "integrity": "sha512-igcl4Ve+F1N2063PJUkesk/GkYyuGIWinYkSyAFTnIj3gzrOgvOA4k747XNdL47HRRL1w/qh7UW8NDuxOLvKFA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, "node_modules/@opentelemetry/core": { - "version": "1.30.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.30.1.tgz", - "integrity": "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==", + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.28.0.tgz", + "integrity": "sha512-ZLwRMV+fNDpVmF2WYUdBHlq0eOWtEaUJSusrzjGnBt7iSRvfjFE3RXYUZJrqou/wIDWV0DwQ5KIfYe9WXg9Xqw==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/semantic-conventions": "1.28.0" + "@opentelemetry/semantic-conventions": "1.27.0" }, "engines": { "node": ">=14" @@ -5092,27 +3623,153 @@ } }, "node_modules/@opentelemetry/core/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz", + "integrity": "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-grpc": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-grpc/-/exporter-logs-otlp-grpc-0.55.0.tgz", + "integrity": "sha512-ykqawCL0ILJWyCJlxCPSAlqQXZ6x2bQsxAVUu8S3z22XNqY5SMx0rl2d93XnvnrOwtcfm+sM9ZhbGh/i5AZ9xw==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.7.1", + "@opentelemetry/core": "1.28.0", + "@opentelemetry/otlp-grpc-exporter-base": "0.55.0", + "@opentelemetry/otlp-transformer": "0.55.0", + "@opentelemetry/sdk-logs": "0.55.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-http": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-http/-/exporter-logs-otlp-http-0.55.0.tgz", + "integrity": "sha512-fpFObWWq+DoLVrBU2dyMEaVkibByEkmKQZIUIjW/4j7lwIsTgW7aJCoD9RYFVB/tButcqov5Es2C0J2wTjM2tg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.55.0", + "@opentelemetry/core": "1.28.0", + "@opentelemetry/otlp-exporter-base": "0.55.0", + "@opentelemetry/otlp-transformer": "0.55.0", + "@opentelemetry/sdk-logs": "0.55.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-http/node_modules/@opentelemetry/api-logs": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.55.0.tgz", + "integrity": "sha512-3cpa+qI45VHYcA5c0bHM6VHo9gicv3p5mlLHNG3rLyjQU8b7e0st1rWtrUn3JbZ3DwwCfhKop4eQ9UuYlC6Pkg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-proto": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-proto/-/exporter-logs-otlp-proto-0.55.0.tgz", + "integrity": "sha512-vjE+DxUr+cUpxikdKCPiLZM5Wx7g1bywjCG76TQocvsA7Tmbb9p0t1+8gPlu9AGH7VEzPwDxxpN4p1ajpOurzQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.55.0", + "@opentelemetry/core": "1.28.0", + "@opentelemetry/otlp-exporter-base": "0.55.0", + "@opentelemetry/otlp-transformer": "0.55.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/sdk-logs": "0.55.0", + "@opentelemetry/sdk-trace-base": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-proto/node_modules/@opentelemetry/api-logs": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.55.0.tgz", + "integrity": "sha512-3cpa+qI45VHYcA5c0bHM6VHo9gicv3p5mlLHNG3rLyjQU8b7e0st1rWtrUn3JbZ3DwwCfhKop4eQ9UuYlC6Pkg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-proto/node_modules/@opentelemetry/resources": { "version": "1.28.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz", - "integrity": "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.28.0.tgz", + "integrity": "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-proto/node_modules/@opentelemetry/sdk-trace-base": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.28.0.tgz", + "integrity": "sha512-ceUVWuCpIao7Y5xE02Xs3nQi0tOGmMea17ecBdwtCvdo9ekmO+ijc9RFDgfifMl7XCBf41zne/1POM3LqSTZDA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-proto/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz", + "integrity": "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==", "license": "Apache-2.0", "engines": { "node": ">=14" } }, "node_modules/@opentelemetry/exporter-metrics-otlp-grpc": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-grpc/-/exporter-metrics-otlp-grpc-0.40.0.tgz", - "integrity": "sha512-1kIEi2G4uVrxqZV+9M09Il2XeylAUOpNXg1vpS50R2CgP99u6ICzu+xhENXWucaVya+tRduwrhkVzSagyP7Mtw==", + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-grpc/-/exporter-metrics-otlp-grpc-0.55.0.tgz", + "integrity": "sha512-9V6t3Tz1yFmXKXDjphgPSAsfQuG0bz9PlAVRY28D+wuG7Ut/Fv43XuccktdAcdrXx1MvCHdCcVmINLyYXLqTIg==", "license": "Apache-2.0", "dependencies": { "@grpc/grpc-js": "^1.7.1", - "@opentelemetry/core": "1.14.0", - "@opentelemetry/exporter-metrics-otlp-http": "0.40.0", - "@opentelemetry/otlp-grpc-exporter-base": "0.40.0", - "@opentelemetry/otlp-transformer": "0.40.0", - "@opentelemetry/resources": "1.14.0", - "@opentelemetry/sdk-metrics": "1.14.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/exporter-metrics-otlp-http": "0.55.0", + "@opentelemetry/otlp-exporter-base": "0.55.0", + "@opentelemetry/otlp-grpc-exporter-base": "0.55.0", + "@opentelemetry/otlp-transformer": "0.55.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/sdk-metrics": "1.28.0" }, "engines": { "node": ">=14" @@ -5121,141 +3778,58 @@ "@opentelemetry/api": "^1.3.0" } }, - "node_modules/@opentelemetry/exporter-metrics-otlp-grpc/node_modules/@opentelemetry/api-logs": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.40.0.tgz", - "integrity": "sha512-8WRuvGnfnbeR9ifGjLN8kklk2fkd0gBT6aN7NHO9zeYF/6qacAViD3bwAKqGXKnJgl39l1EU41I9diqUjamEEQ==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@opentelemetry/api": "^1.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/@opentelemetry/exporter-metrics-otlp-grpc/node_modules/@opentelemetry/core": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.14.0.tgz", - "integrity": "sha512-MnMZ+sxsnlzloeuXL2nm5QcNczt/iO82UOeQQDHhV83F2fP3sgntW2evvtoxJki0MBLxEsh5ADD7PR/Hn5uzjw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "1.14.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" - } - }, - "node_modules/@opentelemetry/exporter-metrics-otlp-grpc/node_modules/@opentelemetry/otlp-transformer": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.40.0.tgz", - "integrity": "sha512-YrJgVVAsJHibENSbYmC1x+5jAmkAGZ9yrgmHxc6IyqM3D1mryhqBvMRDD31JoavPYelkS7dmrXWM8g7swX0B+g==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api-logs": "0.40.0", - "@opentelemetry/core": "1.14.0", - "@opentelemetry/resources": "1.14.0", - "@opentelemetry/sdk-logs": "0.40.0", - "@opentelemetry/sdk-metrics": "1.14.0", - "@opentelemetry/sdk-trace-base": "1.14.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.5.0" - } - }, - "node_modules/@opentelemetry/exporter-metrics-otlp-grpc/node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-logs": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.40.0.tgz", - "integrity": "sha512-/JG7DOLo/Y3VR9azPXlXNRGQff3gp7nQbWl5cFD2SmlYqUrzMq1OjbksZLVztDu1+ynbFunseUG11SxhoxvSRg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.14.0", - "@opentelemetry/resources": "1.14.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.4.0 <1.5.0", - "@opentelemetry/api-logs": ">=0.39.1" - } - }, - "node_modules/@opentelemetry/exporter-metrics-otlp-grpc/node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-trace-base": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.14.0.tgz", - "integrity": "sha512-NzRGt3PS+HPKfQYMb6Iy8YYc5OKA73qDwci/6ujOIvyW9vcqBJSWbjZ8FeLEAmuatUB5WrRhEKu9b0sIiIYTrQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.14.0", - "@opentelemetry/resources": "1.14.0", - "@opentelemetry/semantic-conventions": "1.14.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" - } - }, "node_modules/@opentelemetry/exporter-metrics-otlp-grpc/node_modules/@opentelemetry/resources": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.14.0.tgz", - "integrity": "sha512-qRfWIgBxxl3z47E036Aey0Lj2ZjlFb27Q7Xnj1y1z/P293RXJZGLtcfn/w8JF7v1Q2hs3SDGxz7Wb9Dko1YUQA==", + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.28.0.tgz", + "integrity": "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.14.0", - "@opentelemetry/semantic-conventions": "1.14.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" + "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "node_modules/@opentelemetry/exporter-metrics-otlp-grpc/node_modules/@opentelemetry/sdk-metrics": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.14.0.tgz", - "integrity": "sha512-F0JXmLqT4LmsaiaE28fl0qMtc5w0YuMWTHt1hnANTNX8hxW4IKSv9+wrYG7BZd61HEbPm032Re7fXyzzNA6nIw==", + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.28.0.tgz", + "integrity": "sha512-43tqMK/0BcKTyOvm15/WQ3HLr0Vu/ucAl/D84NO7iSlv6O4eOprxSHa3sUtmYkaZWHqdDJV0AHVz/R6u4JALVQ==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.14.0", - "@opentelemetry/resources": "1.14.0", - "lodash.merge": "4.6.2" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.5.0" + "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "node_modules/@opentelemetry/exporter-metrics-otlp-grpc/node_modules/@opentelemetry/semantic-conventions": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.14.0.tgz", - "integrity": "sha512-rJfCY8rCWz3cb4KI6pEofnytvMPuj3YLQwoscCCYZ5DkdiPjo15IQ0US7+mjcWy9H3fcZIzf2pbJZ7ck/h4tug==", + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz", + "integrity": "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==", "license": "Apache-2.0", "engines": { "node": ">=14" } }, "node_modules/@opentelemetry/exporter-metrics-otlp-http": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-http/-/exporter-metrics-otlp-http-0.40.0.tgz", - "integrity": "sha512-4ferfcHOyYAhy+7Xk/vMWGBI6yafeOxpLWKrRjzNFAGKzD78teOnPTvyaCecPF0nviTF4VuwT2ECgon6Q/bFBQ==", + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-http/-/exporter-metrics-otlp-http-0.55.0.tgz", + "integrity": "sha512-3MqDNZzgXmLaiVo9gs9kCw/zPEaZYKIT0+jeMWscWHL/jrA9BNArTOYWUHEPabAQmWQ2BbvgNC7yzlqjoynQwA==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.14.0", - "@opentelemetry/otlp-exporter-base": "0.40.0", - "@opentelemetry/otlp-transformer": "0.40.0", - "@opentelemetry/resources": "1.14.0", - "@opentelemetry/sdk-metrics": "1.14.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/otlp-exporter-base": "0.55.0", + "@opentelemetry/otlp-transformer": "0.55.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/sdk-metrics": "1.28.0" }, "engines": { "node": ">=14" @@ -5264,575 +3838,241 @@ "@opentelemetry/api": "^1.3.0" } }, - "node_modules/@opentelemetry/exporter-metrics-otlp-http/node_modules/@opentelemetry/api-logs": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.40.0.tgz", - "integrity": "sha512-8WRuvGnfnbeR9ifGjLN8kklk2fkd0gBT6aN7NHO9zeYF/6qacAViD3bwAKqGXKnJgl39l1EU41I9diqUjamEEQ==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@opentelemetry/api": "^1.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/@opentelemetry/exporter-metrics-otlp-http/node_modules/@opentelemetry/core": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.14.0.tgz", - "integrity": "sha512-MnMZ+sxsnlzloeuXL2nm5QcNczt/iO82UOeQQDHhV83F2fP3sgntW2evvtoxJki0MBLxEsh5ADD7PR/Hn5uzjw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "1.14.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" - } - }, - "node_modules/@opentelemetry/exporter-metrics-otlp-http/node_modules/@opentelemetry/otlp-transformer": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.40.0.tgz", - "integrity": "sha512-YrJgVVAsJHibENSbYmC1x+5jAmkAGZ9yrgmHxc6IyqM3D1mryhqBvMRDD31JoavPYelkS7dmrXWM8g7swX0B+g==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api-logs": "0.40.0", - "@opentelemetry/core": "1.14.0", - "@opentelemetry/resources": "1.14.0", - "@opentelemetry/sdk-logs": "0.40.0", - "@opentelemetry/sdk-metrics": "1.14.0", - "@opentelemetry/sdk-trace-base": "1.14.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.5.0" - } - }, - "node_modules/@opentelemetry/exporter-metrics-otlp-http/node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-logs": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.40.0.tgz", - "integrity": "sha512-/JG7DOLo/Y3VR9azPXlXNRGQff3gp7nQbWl5cFD2SmlYqUrzMq1OjbksZLVztDu1+ynbFunseUG11SxhoxvSRg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.14.0", - "@opentelemetry/resources": "1.14.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.4.0 <1.5.0", - "@opentelemetry/api-logs": ">=0.39.1" - } - }, - "node_modules/@opentelemetry/exporter-metrics-otlp-http/node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-trace-base": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.14.0.tgz", - "integrity": "sha512-NzRGt3PS+HPKfQYMb6Iy8YYc5OKA73qDwci/6ujOIvyW9vcqBJSWbjZ8FeLEAmuatUB5WrRhEKu9b0sIiIYTrQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.14.0", - "@opentelemetry/resources": "1.14.0", - "@opentelemetry/semantic-conventions": "1.14.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" - } - }, "node_modules/@opentelemetry/exporter-metrics-otlp-http/node_modules/@opentelemetry/resources": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.14.0.tgz", - "integrity": "sha512-qRfWIgBxxl3z47E036Aey0Lj2ZjlFb27Q7Xnj1y1z/P293RXJZGLtcfn/w8JF7v1Q2hs3SDGxz7Wb9Dko1YUQA==", + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.28.0.tgz", + "integrity": "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.14.0", - "@opentelemetry/semantic-conventions": "1.14.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" + "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "node_modules/@opentelemetry/exporter-metrics-otlp-http/node_modules/@opentelemetry/sdk-metrics": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.14.0.tgz", - "integrity": "sha512-F0JXmLqT4LmsaiaE28fl0qMtc5w0YuMWTHt1hnANTNX8hxW4IKSv9+wrYG7BZd61HEbPm032Re7fXyzzNA6nIw==", + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.28.0.tgz", + "integrity": "sha512-43tqMK/0BcKTyOvm15/WQ3HLr0Vu/ucAl/D84NO7iSlv6O4eOprxSHa3sUtmYkaZWHqdDJV0AHVz/R6u4JALVQ==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.14.0", - "@opentelemetry/resources": "1.14.0", - "lodash.merge": "4.6.2" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.5.0" + "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "node_modules/@opentelemetry/exporter-metrics-otlp-http/node_modules/@opentelemetry/semantic-conventions": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.14.0.tgz", - "integrity": "sha512-rJfCY8rCWz3cb4KI6pEofnytvMPuj3YLQwoscCCYZ5DkdiPjo15IQ0US7+mjcWy9H3fcZIzf2pbJZ7ck/h4tug==", + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz", + "integrity": "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==", "license": "Apache-2.0", "engines": { "node": ">=14" } }, "node_modules/@opentelemetry/exporter-trace-otlp-grpc": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.40.0.tgz", - "integrity": "sha512-/UW/6s1WBHkFgdwizouUCEGZPt7NE0Y5xpuFuHqQF/KyjcHzTWibXzB/XWOSS81X55FUxrI3Icoeptk7vtxJFQ==", + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.55.0.tgz", + "integrity": "sha512-ohIkCLn2Wc3vhhFuf1bH8kOXHMEdcWiD847x7f3Qfygc+CGiatGLzQYscTcEYsWGMV22gVwB/kVcNcx5a3o8gA==", "license": "Apache-2.0", "dependencies": { "@grpc/grpc-js": "^1.7.1", - "@opentelemetry/core": "1.14.0", - "@opentelemetry/otlp-grpc-exporter-base": "0.40.0", - "@opentelemetry/otlp-transformer": "0.40.0", - "@opentelemetry/resources": "1.14.0", - "@opentelemetry/sdk-trace-base": "1.14.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/otlp-grpc-exporter-base": "0.55.0", + "@opentelemetry/otlp-transformer": "0.55.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/sdk-trace-base": "1.28.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": "^1.0.0" - } - }, - "node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/api-logs": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.40.0.tgz", - "integrity": "sha512-8WRuvGnfnbeR9ifGjLN8kklk2fkd0gBT6aN7NHO9zeYF/6qacAViD3bwAKqGXKnJgl39l1EU41I9diqUjamEEQ==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@opentelemetry/api": "^1.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/core": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.14.0.tgz", - "integrity": "sha512-MnMZ+sxsnlzloeuXL2nm5QcNczt/iO82UOeQQDHhV83F2fP3sgntW2evvtoxJki0MBLxEsh5ADD7PR/Hn5uzjw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "1.14.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" - } - }, - "node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/otlp-transformer": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.40.0.tgz", - "integrity": "sha512-YrJgVVAsJHibENSbYmC1x+5jAmkAGZ9yrgmHxc6IyqM3D1mryhqBvMRDD31JoavPYelkS7dmrXWM8g7swX0B+g==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api-logs": "0.40.0", - "@opentelemetry/core": "1.14.0", - "@opentelemetry/resources": "1.14.0", - "@opentelemetry/sdk-logs": "0.40.0", - "@opentelemetry/sdk-metrics": "1.14.0", - "@opentelemetry/sdk-trace-base": "1.14.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.5.0" - } - }, - "node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-logs": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.40.0.tgz", - "integrity": "sha512-/JG7DOLo/Y3VR9azPXlXNRGQff3gp7nQbWl5cFD2SmlYqUrzMq1OjbksZLVztDu1+ynbFunseUG11SxhoxvSRg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.14.0", - "@opentelemetry/resources": "1.14.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.4.0 <1.5.0", - "@opentelemetry/api-logs": ">=0.39.1" - } - }, - "node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-metrics": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.14.0.tgz", - "integrity": "sha512-F0JXmLqT4LmsaiaE28fl0qMtc5w0YuMWTHt1hnANTNX8hxW4IKSv9+wrYG7BZd61HEbPm032Re7fXyzzNA6nIw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.14.0", - "@opentelemetry/resources": "1.14.0", - "lodash.merge": "4.6.2" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.5.0" + "@opentelemetry/api": "^1.3.0" } }, "node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/resources": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.14.0.tgz", - "integrity": "sha512-qRfWIgBxxl3z47E036Aey0Lj2ZjlFb27Q7Xnj1y1z/P293RXJZGLtcfn/w8JF7v1Q2hs3SDGxz7Wb9Dko1YUQA==", + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.28.0.tgz", + "integrity": "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.14.0", - "@opentelemetry/semantic-conventions": "1.14.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" + "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/sdk-trace-base": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.14.0.tgz", - "integrity": "sha512-NzRGt3PS+HPKfQYMb6Iy8YYc5OKA73qDwci/6ujOIvyW9vcqBJSWbjZ8FeLEAmuatUB5WrRhEKu9b0sIiIYTrQ==", + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.28.0.tgz", + "integrity": "sha512-ceUVWuCpIao7Y5xE02Xs3nQi0tOGmMea17ecBdwtCvdo9ekmO+ijc9RFDgfifMl7XCBf41zne/1POM3LqSTZDA==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.14.0", - "@opentelemetry/resources": "1.14.0", - "@opentelemetry/semantic-conventions": "1.14.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" + "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/semantic-conventions": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.14.0.tgz", - "integrity": "sha512-rJfCY8rCWz3cb4KI6pEofnytvMPuj3YLQwoscCCYZ5DkdiPjo15IQ0US7+mjcWy9H3fcZIzf2pbJZ7ck/h4tug==", + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz", + "integrity": "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==", "license": "Apache-2.0", "engines": { "node": ">=14" } }, "node_modules/@opentelemetry/exporter-trace-otlp-http": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.49.1.tgz", - "integrity": "sha512-KOLtZfZvIrpGZLVvblKsiVQT7gQUZNKcUUH24Zz6Xbi7LJb9Vt6xtUZFYdR5IIjvt47PIqBKDWUQlU0o1wAsRw==", + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.55.0.tgz", + "integrity": "sha512-lMiNic63EVHpW+eChmLD2CieDmwQBFi72+LFbh8+5hY0ShrDGrsGP/zuT5MRh7M/vM/UZYO/2A/FYd7CMQGR7A==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/otlp-exporter-base": "0.49.1", - "@opentelemetry/otlp-transformer": "0.49.1", - "@opentelemetry/resources": "1.22.0", - "@opentelemetry/sdk-trace-base": "1.22.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/otlp-exporter-base": "0.55.0", + "@opentelemetry/otlp-transformer": "0.55.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/sdk-trace-base": "1.28.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": "^1.0.0" - } - }, - "node_modules/@opentelemetry/exporter-trace-otlp-http/node_modules/@opentelemetry/core": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.22.0.tgz", - "integrity": "sha512-0VoAlT6x+Xzik1v9goJ3pZ2ppi6+xd3aUfg4brfrLkDBHRIVjMP0eBHrKrhB+NKcDyMAg8fAbGL3Npg/F6AwWA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" - } - }, - "node_modules/@opentelemetry/exporter-trace-otlp-http/node_modules/@opentelemetry/otlp-exporter-base": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.49.1.tgz", - "integrity": "sha512-z6sHliPqDgJU45kQatAettY9/eVF58qVPaTuejw9YWfSRqid9pXPYeegDCSdyS47KAUgAtm+nC28K3pfF27HWg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.0.0" - } - }, - "node_modules/@opentelemetry/exporter-trace-otlp-http/node_modules/@opentelemetry/otlp-transformer": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.49.1.tgz", - "integrity": "sha512-Z+koA4wp9L9e3jkFacyXTGphSWTbOKjwwXMpb0CxNb0kjTHGUxhYRN8GnkLFsFo5NbZPjP07hwAqeEG/uCratQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api-logs": "0.49.1", - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0", - "@opentelemetry/sdk-logs": "0.49.1", - "@opentelemetry/sdk-metrics": "1.22.0", - "@opentelemetry/sdk-trace-base": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.9.0" - } - }, - "node_modules/@opentelemetry/exporter-trace-otlp-http/node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-logs": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.49.1.tgz", - "integrity": "sha512-gCzYWsJE0h+3cuh3/cK+9UwlVFyHvj3PReIOCDOmdeXOp90ZjKRoDOJBc3mvk1LL6wyl1RWIivR8Rg9OToyesw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.4.0 <1.9.0", - "@opentelemetry/api-logs": ">=0.39.1" - } - }, - "node_modules/@opentelemetry/exporter-trace-otlp-http/node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-metrics": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.22.0.tgz", - "integrity": "sha512-k6iIx6H3TZ+BVMr2z8M16ri2OxWaljg5h8ihGJxi/KQWcjign6FEaEzuigXt5bK9wVEhqAcWLCfarSftaNWkkg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0", - "lodash.merge": "^4.6.2" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.9.0" + "@opentelemetry/api": "^1.3.0" } }, "node_modules/@opentelemetry/exporter-trace-otlp-http/node_modules/@opentelemetry/resources": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.22.0.tgz", - "integrity": "sha512-+vNeIFPH2hfcNL0AJk/ykJXoUCtR1YaDUZM+p3wZNU4Hq98gzq+7b43xbkXjadD9VhWIUQqEwXyY64q6msPj6A==", + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.28.0.tgz", + "integrity": "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/semantic-conventions": "1.22.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" + "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "node_modules/@opentelemetry/exporter-trace-otlp-http/node_modules/@opentelemetry/sdk-trace-base": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.22.0.tgz", - "integrity": "sha512-pfTuSIpCKONC6vkTpv6VmACxD+P1woZf4q0K46nSUvXFvOFqjBYKFaAMkKD3M1mlKUUh0Oajwj35qNjMl80m1Q==", + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.28.0.tgz", + "integrity": "sha512-ceUVWuCpIao7Y5xE02Xs3nQi0tOGmMea17ecBdwtCvdo9ekmO+ijc9RFDgfifMl7XCBf41zne/1POM3LqSTZDA==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0", - "@opentelemetry/semantic-conventions": "1.22.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" + "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "node_modules/@opentelemetry/exporter-trace-otlp-http/node_modules/@opentelemetry/semantic-conventions": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.22.0.tgz", - "integrity": "sha512-CAOgFOKLybd02uj/GhCdEeeBjOS0yeoDeo/CA7ASBSmenpZHAKGB3iDm/rv3BQLcabb/OprDEsSQ1y0P8A7Siw==", + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz", + "integrity": "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==", "license": "Apache-2.0", "engines": { "node": ">=14" } }, "node_modules/@opentelemetry/exporter-trace-otlp-proto": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-proto/-/exporter-trace-otlp-proto-0.49.1.tgz", - "integrity": "sha512-n8ON/c9pdMyYAfSFWKkgsPwjYoxnki+6Olzo+klKfW7KqLWoyEkryNkbcMIYnGGNXwdkMIrjoaP0VxXB26Oxcg==", + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-proto/-/exporter-trace-otlp-proto-0.55.0.tgz", + "integrity": "sha512-qxiJFP+bBZW3+goHCGkE1ZdW9gJU0fR7eQ6OP+Rz5oGtEBbq4nkGodhb7C9FJlEFlE2siPtCxoeupV0gtYynag==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/otlp-exporter-base": "0.49.1", - "@opentelemetry/otlp-proto-exporter-base": "0.49.1", - "@opentelemetry/otlp-transformer": "0.49.1", - "@opentelemetry/resources": "1.22.0", - "@opentelemetry/sdk-trace-base": "1.22.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/otlp-exporter-base": "0.55.0", + "@opentelemetry/otlp-transformer": "0.55.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/sdk-trace-base": "1.28.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": "^1.0.0" - } - }, - "node_modules/@opentelemetry/exporter-trace-otlp-proto/node_modules/@opentelemetry/core": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.22.0.tgz", - "integrity": "sha512-0VoAlT6x+Xzik1v9goJ3pZ2ppi6+xd3aUfg4brfrLkDBHRIVjMP0eBHrKrhB+NKcDyMAg8fAbGL3Npg/F6AwWA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" - } - }, - "node_modules/@opentelemetry/exporter-trace-otlp-proto/node_modules/@opentelemetry/otlp-exporter-base": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.49.1.tgz", - "integrity": "sha512-z6sHliPqDgJU45kQatAettY9/eVF58qVPaTuejw9YWfSRqid9pXPYeegDCSdyS47KAUgAtm+nC28K3pfF27HWg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.0.0" - } - }, - "node_modules/@opentelemetry/exporter-trace-otlp-proto/node_modules/@opentelemetry/otlp-transformer": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.49.1.tgz", - "integrity": "sha512-Z+koA4wp9L9e3jkFacyXTGphSWTbOKjwwXMpb0CxNb0kjTHGUxhYRN8GnkLFsFo5NbZPjP07hwAqeEG/uCratQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api-logs": "0.49.1", - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0", - "@opentelemetry/sdk-logs": "0.49.1", - "@opentelemetry/sdk-metrics": "1.22.0", - "@opentelemetry/sdk-trace-base": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.9.0" - } - }, - "node_modules/@opentelemetry/exporter-trace-otlp-proto/node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-logs": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.49.1.tgz", - "integrity": "sha512-gCzYWsJE0h+3cuh3/cK+9UwlVFyHvj3PReIOCDOmdeXOp90ZjKRoDOJBc3mvk1LL6wyl1RWIivR8Rg9OToyesw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.4.0 <1.9.0", - "@opentelemetry/api-logs": ">=0.39.1" - } - }, - "node_modules/@opentelemetry/exporter-trace-otlp-proto/node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-metrics": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.22.0.tgz", - "integrity": "sha512-k6iIx6H3TZ+BVMr2z8M16ri2OxWaljg5h8ihGJxi/KQWcjign6FEaEzuigXt5bK9wVEhqAcWLCfarSftaNWkkg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0", - "lodash.merge": "^4.6.2" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.9.0" + "@opentelemetry/api": "^1.3.0" } }, "node_modules/@opentelemetry/exporter-trace-otlp-proto/node_modules/@opentelemetry/resources": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.22.0.tgz", - "integrity": "sha512-+vNeIFPH2hfcNL0AJk/ykJXoUCtR1YaDUZM+p3wZNU4Hq98gzq+7b43xbkXjadD9VhWIUQqEwXyY64q6msPj6A==", + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.28.0.tgz", + "integrity": "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/semantic-conventions": "1.22.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" + "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "node_modules/@opentelemetry/exporter-trace-otlp-proto/node_modules/@opentelemetry/sdk-trace-base": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.22.0.tgz", - "integrity": "sha512-pfTuSIpCKONC6vkTpv6VmACxD+P1woZf4q0K46nSUvXFvOFqjBYKFaAMkKD3M1mlKUUh0Oajwj35qNjMl80m1Q==", + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.28.0.tgz", + "integrity": "sha512-ceUVWuCpIao7Y5xE02Xs3nQi0tOGmMea17ecBdwtCvdo9ekmO+ijc9RFDgfifMl7XCBf41zne/1POM3LqSTZDA==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0", - "@opentelemetry/semantic-conventions": "1.22.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" + "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "node_modules/@opentelemetry/exporter-trace-otlp-proto/node_modules/@opentelemetry/semantic-conventions": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.22.0.tgz", - "integrity": "sha512-CAOgFOKLybd02uj/GhCdEeeBjOS0yeoDeo/CA7ASBSmenpZHAKGB3iDm/rv3BQLcabb/OprDEsSQ1y0P8A7Siw==", + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz", + "integrity": "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==", "license": "Apache-2.0", "engines": { "node": ">=14" } }, "node_modules/@opentelemetry/exporter-zipkin": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-zipkin/-/exporter-zipkin-1.22.0.tgz", - "integrity": "sha512-XcFs6rGvcTz0qW5uY7JZDYD0yNEXdekXAb6sFtnZgY/cHY6BQ09HMzOjv9SX+iaXplRDcHr1Gta7VQKM1XXM6g==", + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-zipkin/-/exporter-zipkin-1.28.0.tgz", + "integrity": "sha512-AMwr3eGXaPEH7gk8yhcUcen31VXy1yU5VJETu0pCfGpggGCYmhm0FKgYBpL5/vlIgQJWU/sW2vIjCL7aSilpKg==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0", - "@opentelemetry/sdk-trace-base": "1.22.0", - "@opentelemetry/semantic-conventions": "1.22.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/sdk-trace-base": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" }, "engines": { "node": ">=14" @@ -5841,72 +4081,57 @@ "@opentelemetry/api": "^1.0.0" } }, - "node_modules/@opentelemetry/exporter-zipkin/node_modules/@opentelemetry/core": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.22.0.tgz", - "integrity": "sha512-0VoAlT6x+Xzik1v9goJ3pZ2ppi6+xd3aUfg4brfrLkDBHRIVjMP0eBHrKrhB+NKcDyMAg8fAbGL3Npg/F6AwWA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" - } - }, "node_modules/@opentelemetry/exporter-zipkin/node_modules/@opentelemetry/resources": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.22.0.tgz", - "integrity": "sha512-+vNeIFPH2hfcNL0AJk/ykJXoUCtR1YaDUZM+p3wZNU4Hq98gzq+7b43xbkXjadD9VhWIUQqEwXyY64q6msPj6A==", + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.28.0.tgz", + "integrity": "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/semantic-conventions": "1.22.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" + "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "node_modules/@opentelemetry/exporter-zipkin/node_modules/@opentelemetry/sdk-trace-base": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.22.0.tgz", - "integrity": "sha512-pfTuSIpCKONC6vkTpv6VmACxD+P1woZf4q0K46nSUvXFvOFqjBYKFaAMkKD3M1mlKUUh0Oajwj35qNjMl80m1Q==", + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.28.0.tgz", + "integrity": "sha512-ceUVWuCpIao7Y5xE02Xs3nQi0tOGmMea17ecBdwtCvdo9ekmO+ijc9RFDgfifMl7XCBf41zne/1POM3LqSTZDA==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0", - "@opentelemetry/semantic-conventions": "1.22.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" + "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "node_modules/@opentelemetry/exporter-zipkin/node_modules/@opentelemetry/semantic-conventions": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.22.0.tgz", - "integrity": "sha512-CAOgFOKLybd02uj/GhCdEeeBjOS0yeoDeo/CA7ASBSmenpZHAKGB3iDm/rv3BQLcabb/OprDEsSQ1y0P8A7Siw==", + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz", + "integrity": "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==", "license": "Apache-2.0", "engines": { "node": ">=14" } }, "node_modules/@opentelemetry/instrumentation": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.49.1.tgz", - "integrity": "sha512-0DLtWtaIppuNNRRllSD4bjU8ZIiLp1cDXvJEbp752/Zf+y3gaLNaoGRGIlX4UHhcsrmtL+P2qxi3Hodi8VuKiQ==", + "version": "0.54.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.54.2.tgz", + "integrity": "sha512-go6zpOVoZVztT9r1aPd79Fr3OWiD4N24bCPJsIKkBses8oyFo12F/Ew3UBTdIu6hsW4HC4MVEJygG6TEyJI/lg==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/api-logs": "0.49.1", - "@types/shimmer": "^1.0.2", - "import-in-the-middle": "1.7.1", + "@opentelemetry/api-logs": "0.54.2", + "@types/shimmer": "^1.2.0", + "import-in-the-middle": "^1.8.1", "require-in-the-middle": "^7.1.1", "semver": "^7.5.2", "shimmer": "^1.2.1" @@ -5919,14 +4144,14 @@ } }, "node_modules/@opentelemetry/instrumentation-amqplib": { - "version": "0.35.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-amqplib/-/instrumentation-amqplib-0.35.0.tgz", - "integrity": "sha512-rb3hIWA7f0HXpXpfElnGC6CukRxy58/OJ6XYlTzpZJtNJPao7BuobZjkQEscaRYhUzgi7X7R1aKkIUOTV5JFrg==", + "version": "0.43.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-amqplib/-/instrumentation-amqplib-0.43.0.tgz", + "integrity": "sha512-ALjfQC+0dnIEcvNYsbZl/VLh7D2P1HhFF4vicRKHhHFIUV3Shpg4kXgiek5PLhmeKSIPiUB25IYH5RIneclL4A==", "license": "Apache-2.0", "dependencies": { "@opentelemetry/core": "^1.8.0", - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -5936,16 +4161,14 @@ } }, "node_modules/@opentelemetry/instrumentation-aws-lambda": { - "version": "0.39.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-aws-lambda/-/instrumentation-aws-lambda-0.39.0.tgz", - "integrity": "sha512-D+oG/hIBDdwCNq7Y6BEuddjcwDVD0C8NhBE7A85mRZ9RLG0bKoWrhIdVvbpqEoa0U5AWe9Y98RX4itNg7WTy4w==", + "version": "0.47.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-aws-lambda/-/instrumentation-aws-lambda-0.47.0.tgz", + "integrity": "sha512-0BidKDPziHWGl5mnpLuh7ob1X3KpR0UN3QcJkcxIsOMylBbMMp9EoB55dHsTMoNO7bx2uyeY0iirEuTchjF1gQ==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/propagator-aws-xray": "^1.3.1", - "@opentelemetry/resources": "^1.8.0", - "@opentelemetry/semantic-conventions": "^1.0.0", - "@types/aws-lambda": "8.10.122" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0", + "@types/aws-lambda": "8.10.143" }, "engines": { "node": ">=14" @@ -5955,15 +4178,15 @@ } }, "node_modules/@opentelemetry/instrumentation-aws-sdk": { - "version": "0.39.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-aws-sdk/-/instrumentation-aws-sdk-0.39.1.tgz", - "integrity": "sha512-QnvIMVpzRYqQHSXydGUksbhBjPbMyHSUBwi6ocN7gEXoI711+tIY3R1cfRutl0u3M67A/fAvPI3IgACfJaFORg==", + "version": "0.46.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-aws-sdk/-/instrumentation-aws-sdk-0.46.0.tgz", + "integrity": "sha512-EyxGQVYhgY8OI4/CKzqamUswiEVlua6DJcsmkeNSykZrDGs78jPfssbqoMQGetywHWPZBRVJN4Ba/7aB5iLHBA==", "license": "Apache-2.0", "dependencies": { "@opentelemetry/core": "^1.8.0", - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/propagation-utils": "^0.30.7", - "@opentelemetry/semantic-conventions": "^1.0.0" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/propagation-utils": "^0.30.12", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -5973,13 +4196,13 @@ } }, "node_modules/@opentelemetry/instrumentation-bunyan": { - "version": "0.36.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-bunyan/-/instrumentation-bunyan-0.36.0.tgz", - "integrity": "sha512-sHD5BSiqSrgWow7VmugEFzV8vGdsz5m+w1v9tK6YwRzuAD7vbo57chluq+UBzIqStoCH+0yOzRzSALH7hrfffg==", + "version": "0.42.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-bunyan/-/instrumentation-bunyan-0.42.0.tgz", + "integrity": "sha512-GBh6ybwKmFZjc86SyHVx72jHg+4pFPaXT3IZgJ4QtnMsMf0/q5m2aHAjid+yakmEkApsnRWX8pJ8nkl1e+6mag==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/api-logs": "^0.49.1", - "@opentelemetry/instrumentation": "^0.49.1", + "@opentelemetry/api-logs": "^0.54.0", + "@opentelemetry/instrumentation": "^0.54.0", "@types/bunyan": "1.8.9" }, "engines": { @@ -5990,13 +4213,13 @@ } }, "node_modules/@opentelemetry/instrumentation-cassandra-driver": { - "version": "0.36.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-cassandra-driver/-/instrumentation-cassandra-driver-0.36.0.tgz", - "integrity": "sha512-gMfxzryOIP/mvSLXBJp/QxSr2NvS+cC1dkIXn+aSOzYoU1U3apeF3nAyuikmY9dRCQDV7wHPslqbi+pCmd4pAQ==", + "version": "0.42.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-cassandra-driver/-/instrumentation-cassandra-driver-0.42.0.tgz", + "integrity": "sha512-35I9Gw4BeSs9NPe7fugu9e/mWKaapc/N1wounHnGt259/Q3ISGMOQRrOwIBw+x/XJygJvn4Ss1c+r5h89TsVAw==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6006,14 +4229,14 @@ } }, "node_modules/@opentelemetry/instrumentation-connect": { - "version": "0.34.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-connect/-/instrumentation-connect-0.34.0.tgz", - "integrity": "sha512-PJO99nfyUp3JSoBMhwZsOQDm/XKfkb/QQ8YTsNX4ZJ28phoRcNLqe36mqIMp80DKmKAX4xkxCAyrSYtW8QqZxA==", + "version": "0.40.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-connect/-/instrumentation-connect-0.40.0.tgz", + "integrity": "sha512-3aR/3YBQ160siitwwRLjwqrv2KBT16897+bo6yz8wIfel6nWOxTZBJudcbsK3p42pTC7qrbotJ9t/1wRLpv79Q==", "license": "Apache-2.0", "dependencies": { "@opentelemetry/core": "^1.8.0", - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0", + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0", "@types/connect": "3.4.36" }, "engines": { @@ -6024,13 +4247,13 @@ } }, "node_modules/@opentelemetry/instrumentation-cucumber": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-cucumber/-/instrumentation-cucumber-0.4.0.tgz", - "integrity": "sha512-n53QvozzgMS9imEclow2nBYJ/jtZlZqiKIqDUi2/g0nDi08F555JhDS03d/Z+4NJxbu7bDLAg12giCV9KZN/Jw==", + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-cucumber/-/instrumentation-cucumber-0.10.0.tgz", + "integrity": "sha512-5sT6Ap3W7StEL0Oax/vd1YTEcTPTefx+9myzkKrr72hxzFzSooGRCxlU3sfPwZqWptUV7+QWTMd7SqGEEPnE/w==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6040,12 +4263,12 @@ } }, "node_modules/@opentelemetry/instrumentation-dataloader": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-dataloader/-/instrumentation-dataloader-0.7.0.tgz", - "integrity": "sha512-sIaevxATJV5YaZzBTTcTaDEnI+/1vxYs+lVk1honnvrEAaP0FA9C/cFrQEN0kP2BDHkHRE/t6y5lGUqusi/h3A==", + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-dataloader/-/instrumentation-dataloader-0.13.0.tgz", + "integrity": "sha512-wbU3WdgUAXljEIY2nfpkqID/VH70ThnES8mZZHKCZlV/Pl5T4+qmrVdT7U9/WUzz8flwsXfER6T6jl48Wbl+LQ==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1" + "@opentelemetry/instrumentation": "^0.54.0" }, "engines": { "node": ">=14" @@ -6055,14 +4278,12 @@ } }, "node_modules/@opentelemetry/instrumentation-dns": { - "version": "0.34.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-dns/-/instrumentation-dns-0.34.0.tgz", - "integrity": "sha512-3tmXdvrzHQ7S3v82Cm36PTYLtgg2+hVm00K1xB3uzP08GEo9w/F8DW4me9z6rDroVGiLIg621RZ6dzjBcmmFCg==", + "version": "0.40.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-dns/-/instrumentation-dns-0.40.0.tgz", + "integrity": "sha512-tLNR8XLPiYRKKk3/UqifXnPP2TVt1RcwvHU0R1ETL1xkZ1ZHMTmSC4x6TignnHOFtRixtJ05EgMGejnffaBXkQ==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0", - "semver": "^7.5.4" + "@opentelemetry/instrumentation": "^0.54.0" }, "engines": { "node": ">=14" @@ -6072,14 +4293,14 @@ } }, "node_modules/@opentelemetry/instrumentation-express": { - "version": "0.36.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-express/-/instrumentation-express-0.36.1.tgz", - "integrity": "sha512-ltIE4kIMa+83QjW/p7oe7XCESF29w3FQ9/T1VgShdX7fzm56K2a0xfEX1vF8lnHRGERYxIWX9D086C6gJOjVGA==", + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-express/-/instrumentation-express-0.44.0.tgz", + "integrity": "sha512-GWgibp6Q0wxyFaaU8ERIgMMYgzcHmGrw3ILUtGchLtLncHNOKk0SNoWGqiylXWWT4HTn5XdV8MGawUgpZh80cA==", "license": "Apache-2.0", "dependencies": { "@opentelemetry/core": "^1.8.0", - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6089,14 +4310,14 @@ } }, "node_modules/@opentelemetry/instrumentation-fastify": { - "version": "0.34.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-fastify/-/instrumentation-fastify-0.34.0.tgz", - "integrity": "sha512-2Qu66XBkfJ8tr6H+RHBTyw/EX73N9U7pvNa49aonDnT9/mK58k7AKOscpRnKXOvHqc2YIdEPRcBIWxhksPFZVA==", + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-fastify/-/instrumentation-fastify-0.41.0.tgz", + "integrity": "sha512-pNRjFvf0mvqfJueaeL/qEkuGJwgtE5pgjIHGYwjc2rMViNCrtY9/Sf+Nu8ww6dDd/Oyk2fwZZP7i0XZfCnETrA==", "license": "Apache-2.0", "dependencies": { "@opentelemetry/core": "^1.8.0", - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6106,14 +4327,13 @@ } }, "node_modules/@opentelemetry/instrumentation-fs": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.10.0.tgz", - "integrity": "sha512-XtMoNINVsIQTQHjtxe7A0Lng96wxA5DSD5CYVVvpquG6HJRdZ4xNe9DTU03YtoEFqlN9qTfvGb/6ILzhKhiG8g==", + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.16.0.tgz", + "integrity": "sha512-hMDRUxV38ln1R3lNz6osj3YjlO32ykbHqVrzG7gEhGXFQfu7LJUx8t9tEwE4r2h3CD4D0Rw4YGDU4yF4mP3ilg==", "license": "Apache-2.0", "dependencies": { "@opentelemetry/core": "^1.8.0", - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0" + "@opentelemetry/instrumentation": "^0.54.0" }, "engines": { "node": ">=14" @@ -6123,13 +4343,12 @@ } }, "node_modules/@opentelemetry/instrumentation-generic-pool": { - "version": "0.34.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-generic-pool/-/instrumentation-generic-pool-0.34.0.tgz", - "integrity": "sha512-jdI7tfVVwZJuTu4j2kAvJtx4wlEQKIXSZnZG4RdqRHc56KqQQDuVTBLvUgmDXvnSVclH9ayf4oaAV08R9fICtw==", + "version": "0.40.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-generic-pool/-/instrumentation-generic-pool-0.40.0.tgz", + "integrity": "sha512-k+/JlNDHN3bPi/Cir+Ew6tKHFVCa1ZFeQyGUw5HQkRX/twCRaN3kJFXJW+rDAN90XwK3RtC9AWwBihDGh/oSlQ==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0" + "@opentelemetry/instrumentation": "^0.54.0" }, "engines": { "node": ">=14" @@ -6139,12 +4358,12 @@ } }, "node_modules/@opentelemetry/instrumentation-graphql": { - "version": "0.38.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-graphql/-/instrumentation-graphql-0.38.1.tgz", - "integrity": "sha512-mSt4ztn3EVlLtZJ+tDEqq5GUEYdY8cbTT9SeVJFmXSfdSQkPZn0ovo/dRe6dUcplM60gg4w+llw8SZuQN0iZfQ==", + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-graphql/-/instrumentation-graphql-0.44.0.tgz", + "integrity": "sha512-FYXTe3Bv96aNpYktqm86BFUTpjglKD0kWI5T5bxYkLUPEPvFn38vWGMJTGrDMVou/i55E4jlWvcm6hFIqLsMbg==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1" + "@opentelemetry/instrumentation": "^0.54.0" }, "engines": { "node": ">=14" @@ -6154,13 +4373,13 @@ } }, "node_modules/@opentelemetry/instrumentation-grpc": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-grpc/-/instrumentation-grpc-0.49.1.tgz", - "integrity": "sha512-f8mQjFi5/PiP4SK3VDU1/3sUUgs6exMtBgcnNycgCKgN40htiPT+MuDRwdRnRMNI/4vNQ7p1/5r4Q5oN0GuRBw==", + "version": "0.54.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-grpc/-/instrumentation-grpc-0.54.2.tgz", + "integrity": "sha512-KhSzerCaaqVH2zfDro7nTunWUZXt1pQISQpE83LuQTOKGk7mN3G60T1wliQ3Qdg0X3UUuhCXEC7u6IAVfDxkUQ==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "0.49.1", - "@opentelemetry/semantic-conventions": "1.22.0" + "@opentelemetry/instrumentation": "0.54.2", + "@opentelemetry/semantic-conventions": "1.27.0" }, "engines": { "node": ">=14" @@ -6170,24 +4389,23 @@ } }, "node_modules/@opentelemetry/instrumentation-grpc/node_modules/@opentelemetry/semantic-conventions": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.22.0.tgz", - "integrity": "sha512-CAOgFOKLybd02uj/GhCdEeeBjOS0yeoDeo/CA7ASBSmenpZHAKGB3iDm/rv3BQLcabb/OprDEsSQ1y0P8A7Siw==", + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz", + "integrity": "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==", "license": "Apache-2.0", "engines": { "node": ">=14" } }, "node_modules/@opentelemetry/instrumentation-hapi": { - "version": "0.35.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-hapi/-/instrumentation-hapi-0.35.0.tgz", - "integrity": "sha512-j7q99aTLHfjNKW94qJnEaDatgz+q2psTKs7lxZO4QHRnoDltDk39a44/+AkI1qBJNw5xyLjrApqkglfbWJ2abg==", + "version": "0.42.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-hapi/-/instrumentation-hapi-0.42.0.tgz", + "integrity": "sha512-TQC0BtIWLHrp6nKsYdZ5t5B7aiZ16BwbRqZtYYQxeJVsq/HQTANWpknjtA7KMxv5tAUMCrU/eDo8F3qioUOSZg==", "license": "Apache-2.0", "dependencies": { "@opentelemetry/core": "^1.8.0", - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0", - "@types/hapi__hapi": "20.0.13" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6197,14 +4415,15 @@ } }, "node_modules/@opentelemetry/instrumentation-http": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-http/-/instrumentation-http-0.49.1.tgz", - "integrity": "sha512-Yib5zrW2s0V8wTeUK/B3ZtpyP4ldgXj9L3Ws/axXrW1dW0/mEFKifK50MxMQK9g5NNJQS9dWH7rvcEGZdWdQDA==", + "version": "0.54.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-http/-/instrumentation-http-0.54.2.tgz", + "integrity": "sha512-mABjJ34UcU32pg8g18L9xBh0U3JON/2F6/57BYYy8AZJp2a71lZjcKr0T00pICoic50TW5HvcTrmyfMil+AiXQ==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/instrumentation": "0.49.1", - "@opentelemetry/semantic-conventions": "1.22.0", + "@opentelemetry/core": "1.27.0", + "@opentelemetry/instrumentation": "0.54.2", + "@opentelemetry/semantic-conventions": "1.27.0", + "forwarded-parse": "2.1.2", "semver": "^7.5.2" }, "engines": { @@ -6215,39 +4434,54 @@ } }, "node_modules/@opentelemetry/instrumentation-http/node_modules/@opentelemetry/core": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.22.0.tgz", - "integrity": "sha512-0VoAlT6x+Xzik1v9goJ3pZ2ppi6+xd3aUfg4brfrLkDBHRIVjMP0eBHrKrhB+NKcDyMAg8fAbGL3Npg/F6AwWA==", + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.27.0.tgz", + "integrity": "sha512-yQPKnK5e+76XuiqUH/gKyS8wv/7qITd5ln56QkBTf3uggr0VkXOXfcaAuG330UfdYu83wsyoBwqwxigpIG+Jkg==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/semantic-conventions": "1.22.0" + "@opentelemetry/semantic-conventions": "1.27.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" + "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "node_modules/@opentelemetry/instrumentation-http/node_modules/@opentelemetry/semantic-conventions": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.22.0.tgz", - "integrity": "sha512-CAOgFOKLybd02uj/GhCdEeeBjOS0yeoDeo/CA7ASBSmenpZHAKGB3iDm/rv3BQLcabb/OprDEsSQ1y0P8A7Siw==", + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz", + "integrity": "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==", "license": "Apache-2.0", "engines": { "node": ">=14" } }, "node_modules/@opentelemetry/instrumentation-ioredis": { - "version": "0.38.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.38.0.tgz", - "integrity": "sha512-c9nQFhRjFAtpInTks7z5v9CiOCiR8U9GbIhIv0TLEJ/r0wqdKNLfLZzCrr9XQ9WasxeOmziLlPFhpRBAd9Q4oA==", + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.44.0.tgz", + "integrity": "sha512-312pE2xc0ihX9haTf9WC4OF9in5EfVO1y5I8Ef9aMQKJNhuSe3IgzQAqGoLfaYajC+ig0IZ9SQKU8mRbFwHU+A==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/redis-common": "^0.36.1", - "@opentelemetry/semantic-conventions": "^1.0.0", - "@types/ioredis4": "npm:@types/ioredis@^4.28.10" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/redis-common": "^0.36.2", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-kafkajs": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-kafkajs/-/instrumentation-kafkajs-0.4.0.tgz", + "integrity": "sha512-I9VwDG314g7SDL4t8kD/7+1ytaDBRbZQjhVaQaVIDR8K+mlsoBhLsWH79yHxhHQKvwCSZwqXF+TiTOhoQVUt7A==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6257,13 +4491,13 @@ } }, "node_modules/@opentelemetry/instrumentation-knex": { - "version": "0.34.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-knex/-/instrumentation-knex-0.34.0.tgz", - "integrity": "sha512-6kZOEvNJOylTQunU5zSSi4iTuCkwIL9nwFnZg7719p61u3d6Qj3X4xi9su46VE3M0dH7vEoxUW+nb/0ilm+aZg==", + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-knex/-/instrumentation-knex-0.41.0.tgz", + "integrity": "sha512-OhI1SlLv5qnsnm2dOVrian/x3431P75GngSpnR7c4fcVFv7prXGYu29Z6ILRWJf/NJt6fkbySmwdfUUnFnHCTg==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6273,16 +4507,14 @@ } }, "node_modules/@opentelemetry/instrumentation-koa": { - "version": "0.38.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-koa/-/instrumentation-koa-0.38.0.tgz", - "integrity": "sha512-lQujF4I3wdcrOF14miCV2pC72H+OJKb2LrrmTvTDAhELQDN/95v0doWgT9aHybUGkaAeB3QG4d09sved548TlA==", + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-koa/-/instrumentation-koa-0.44.0.tgz", + "integrity": "sha512-ryPqGIQ4hpMGd85bAGjRMDAy/ic+Qdh1GtFGJo9KaXdzbcvZoF1ZgXVsKTYDxbD1n5C0BoQy6rcWg8Lu68iCJA==", "license": "Apache-2.0", "dependencies": { "@opentelemetry/core": "^1.8.0", - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0", - "@types/koa": "2.14.0", - "@types/koa__router": "12.0.3" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6292,12 +4524,12 @@ } }, "node_modules/@opentelemetry/instrumentation-lru-memoizer": { - "version": "0.35.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-lru-memoizer/-/instrumentation-lru-memoizer-0.35.0.tgz", - "integrity": "sha512-wCXe+iCF7JweMgY3blLM2Y1G0GSwLEeSA61z/y1UwzvBLEEXt7vL6qOl2mkNcUL9ZbLDS+EABatBH+vFO6DV5Q==", + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-lru-memoizer/-/instrumentation-lru-memoizer-0.41.0.tgz", + "integrity": "sha512-6OePkk4RYCPVsnS0TroEK6UZzxxxjVWaE6EPdOn2qxGHMtm+Qb80tiBQ6BbmC+f7bjc27O85JY8gxeTybhHZXw==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1" + "@opentelemetry/instrumentation": "^0.54.0" }, "engines": { "node": ">=14" @@ -6307,13 +4539,13 @@ } }, "node_modules/@opentelemetry/instrumentation-memcached": { - "version": "0.34.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-memcached/-/instrumentation-memcached-0.34.0.tgz", - "integrity": "sha512-RleFfaag3Evg4pTzHwDBwo1KiFgnCtiT4V6MQRRHadytNGdpcL+Ynz32ydDdiOXeadt7xpRI7HSvBy0quGTXSw==", + "version": "0.40.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-memcached/-/instrumentation-memcached-0.40.0.tgz", + "integrity": "sha512-VzJUUH6cVz8yrb25RvvjhxCpwu4vUk28I0m5nnnhebULOo8p9lda5PgQeVde2+jQAd977C/vN714fkbYOmwb+A==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0", + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0", "@types/memcached": "^2.2.6" }, "engines": { @@ -6324,14 +4556,13 @@ } }, "node_modules/@opentelemetry/instrumentation-mongodb": { - "version": "0.41.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.41.0.tgz", - "integrity": "sha512-DlSH0oyEuTW5gprCUppb0Qe3pK3cpUUFW5eTmayWNyICI1LFunwtcrULTNv6UiThD/V5ykAf/GGGEa7KFAmkog==", + "version": "0.48.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.48.0.tgz", + "integrity": "sha512-9YWvaGvrrcrydMsYGLu0w+RgmosLMKe3kv/UNlsPy8RLnCkN2z+bhhbjjjuxtUmvEuKZMCoXFluABVuBr1yhjw==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/sdk-metrics": "^1.9.1", - "@opentelemetry/semantic-conventions": "^1.0.0" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6341,14 +4572,14 @@ } }, "node_modules/@opentelemetry/instrumentation-mongoose": { - "version": "0.36.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.36.0.tgz", - "integrity": "sha512-UelQ8dLQRLTdck3tPJdZ17b+Hk9usLf1cY2ou5THAaZpulUdpg62Q9Hx2RHRU71Rp2/YMDk25og7GJhuWScfEA==", + "version": "0.43.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.43.0.tgz", + "integrity": "sha512-y1mWuL/zb6IKi199HkROgmStxF/ybEsnKYgx+/lpLATd57oZHOqrXP9tLmp9qRVI5c6P5XEWfe7ZCvrj07iDMQ==", "license": "Apache-2.0", "dependencies": { "@opentelemetry/core": "^1.8.0", - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6358,14 +4589,14 @@ } }, "node_modules/@opentelemetry/instrumentation-mysql": { - "version": "0.36.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql/-/instrumentation-mysql-0.36.0.tgz", - "integrity": "sha512-2mt/032SLkiuddzMrq3YwM0bHksXRep69EzGRnBfF+bCbwYvKLpqmSFqJZ9T3yY/mBWj+tvdvc1+klXGrh2QnQ==", + "version": "0.42.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql/-/instrumentation-mysql-0.42.0.tgz", + "integrity": "sha512-1GN2EBGVSZABGQ25MSz3faeBW/DwhzmE10aNW1/A2mvQAxF1CvpMk17YmNUzwapVt29iKsiU3SXQG7vjh/019A==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0", - "@types/mysql": "2.15.22" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0", + "@types/mysql": "2.15.26" }, "engines": { "node": ">=14" @@ -6375,14 +4606,14 @@ } }, "node_modules/@opentelemetry/instrumentation-mysql2": { - "version": "0.36.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql2/-/instrumentation-mysql2-0.36.0.tgz", - "integrity": "sha512-F63lKcl/R+if2j5Vz66c2/SLXQEtLlFkWTmYb8NQSgmcCaEKjML4RRRjZISIT4IBwdpanJ2qmNuXVM6MYqhBXw==", + "version": "0.42.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql2/-/instrumentation-mysql2-0.42.1.tgz", + "integrity": "sha512-5hOQbFSpqsgDLaqIeWZNbSWB6XdwN+aBjoCIe60lmGG86zeNXu9I6l1kEckRb+Gy0i7zrt0Tk8S62zsOSZ8l7Q==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0", - "@opentelemetry/sql-common": "^0.40.0" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0", + "@opentelemetry/sql-common": "^0.40.1" }, "engines": { "node": ">=14" @@ -6392,13 +4623,13 @@ } }, "node_modules/@opentelemetry/instrumentation-nestjs-core": { - "version": "0.35.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-nestjs-core/-/instrumentation-nestjs-core-0.35.0.tgz", - "integrity": "sha512-INKA7CIOteTSRVxP7SQaFby11AYU3uezI93xDaDRGY4TloXNVoyw5n6UmcVJU4yDn6xY2r7zZ2SVHvblUc21/g==", + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-nestjs-core/-/instrumentation-nestjs-core-0.41.0.tgz", + "integrity": "sha512-XCqtghFktpcJ2BOaJtFfqtTMsHffJADxfYhJl28WT6ygCChS2uZVxMKKLsy+i9VtPaw/i1IumPICL6mbhwq+Vw==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6408,13 +4639,13 @@ } }, "node_modules/@opentelemetry/instrumentation-net": { - "version": "0.34.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-net/-/instrumentation-net-0.34.0.tgz", - "integrity": "sha512-gjybNOQQqbXmD1qVHNO2qBJI4V6p3QQ7xKg3pnC/x7wRdxn+siLQj7QIVxW85C3mymngoJJdRs6BwI3qPUfsPQ==", + "version": "0.40.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-net/-/instrumentation-net-0.40.0.tgz", + "integrity": "sha512-abErnVRxTmtiF7EvBISW81Se2nj/j3Xtpfy//9++dgvDOXwbcD1Xz1via6ZHOm/VamboGhqPlYiO7ABzluPLwg==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6424,16 +4655,17 @@ } }, "node_modules/@opentelemetry/instrumentation-pg": { - "version": "0.39.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.39.1.tgz", - "integrity": "sha512-pX5ujDOyGpPcrZlzaD3LJzmyaSMMMKAP+ffTHJp9vasvZJr+LifCk53TMPVUafcXKV/xX/IIkvADO+67M1Z25g==", + "version": "0.47.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.47.1.tgz", + "integrity": "sha512-qIcydMBVlKtAyFQWYunjqvFMVqIGvxGMXISrdLuSbcCqico9QKhK7bF5wzsotjGwHcGnc7q5kRqSL7j+LnY1Cw==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0", - "@opentelemetry/sql-common": "^0.40.0", + "@opentelemetry/core": "^1.26.0", + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "1.27.0", + "@opentelemetry/sql-common": "^0.40.1", "@types/pg": "8.6.1", - "@types/pg-pool": "2.0.4" + "@types/pg-pool": "2.0.6" }, "engines": { "node": ">=14" @@ -6442,13 +4674,24 @@ "@opentelemetry/api": "^1.3.0" } }, + "node_modules/@opentelemetry/instrumentation-pg/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz", + "integrity": "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, "node_modules/@opentelemetry/instrumentation-pino": { - "version": "0.36.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-pino/-/instrumentation-pino-0.36.0.tgz", - "integrity": "sha512-oEz+BJEYRBMAUu7MVJFJhhlsBuwLaUGjbJciKZRIeGX+fUtgcbQGV+a2Ris9jR3yFzWZrYg0aNBSCbGqvPCtMQ==", + "version": "0.43.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-pino/-/instrumentation-pino-0.43.0.tgz", + "integrity": "sha512-jlOOgbODWRRNknWXY1VLgmqgG0SO4kLgU3XnejjO/3De4OisroAsMGk+1cRB5AQ6WZ8WLAMkMyTShaOe6j2Asw==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1" + "@opentelemetry/api-logs": "^0.54.0", + "@opentelemetry/core": "^1.25.0", + "@opentelemetry/instrumentation": "^0.54.0" }, "engines": { "node": ">=14" @@ -6458,14 +4701,14 @@ } }, "node_modules/@opentelemetry/instrumentation-redis": { - "version": "0.37.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-redis/-/instrumentation-redis-0.37.0.tgz", - "integrity": "sha512-9G0T74kheu37k+UvyBnAcieB5iowxska3z2rhUcSTL8Cl0y/CvMn7sZ7txkUbXt0rdX6qeEUdMLmbsY2fPUM7Q==", + "version": "0.43.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-redis/-/instrumentation-redis-0.43.0.tgz", + "integrity": "sha512-dufe08W3sCOjutbTJmV6tg2Y3+7IBe59oQrnIW2RCgjRhsW0Jjaenezt490eawO0MdXjUfFyrIUg8WetKhE4xA==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/redis-common": "^0.36.1", - "@opentelemetry/semantic-conventions": "^1.0.0" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/redis-common": "^0.36.2", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6475,14 +4718,14 @@ } }, "node_modules/@opentelemetry/instrumentation-redis-4": { - "version": "0.37.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-redis-4/-/instrumentation-redis-4-0.37.0.tgz", - "integrity": "sha512-WNO+HALvPPvjbh7UEEIuay0Z0d2mIfSCkBZbPRwZttDGX6LYGc2WnRgJh3TnYqjp7/y9IryWIbajAFIebj1OBA==", + "version": "0.43.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-redis-4/-/instrumentation-redis-4-0.43.0.tgz", + "integrity": "sha512-6B2+CFRY9xRnkeZrSvlTyY2yB/zAgxjbXS5EwXhE3ZAKR1hWWoUzaTADIKT5xe9/VbDW42U3UoOPCcaCmeAXww==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/redis-common": "^0.36.1", - "@opentelemetry/semantic-conventions": "^1.0.0" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/redis-common": "^0.36.2", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6492,14 +4735,14 @@ } }, "node_modules/@opentelemetry/instrumentation-restify": { - "version": "0.36.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-restify/-/instrumentation-restify-0.36.0.tgz", - "integrity": "sha512-QbOh8HpnnRn4xxFXX77Gdww6M78yx7dRiIKR6+H3j5LH5u6sYckTXw3TGPSsXsaM4DQHy0fOw15sAcJoWkC+aQ==", + "version": "0.42.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-restify/-/instrumentation-restify-0.42.0.tgz", + "integrity": "sha512-ApDD9HNy6de6xrHmISEfkQHwwX1f1JrBj0ADnlk6tVdJ0j/vNmsZNLwaU2IA2K3mHqbp2YLarLgxAZp6rjcfWg==", "license": "Apache-2.0", "dependencies": { "@opentelemetry/core": "^1.8.0", - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6509,13 +4752,13 @@ } }, "node_modules/@opentelemetry/instrumentation-router": { - "version": "0.35.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-router/-/instrumentation-router-0.35.0.tgz", - "integrity": "sha512-MdxGJuNTIy/2qDI8yow6cRBQ87m6O//VuHIlawe8v0x1NsTOSwS72xm+BzTuY9D0iMqiJUiTlE3dBs8DA91MTw==", + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-router/-/instrumentation-router-0.41.0.tgz", + "integrity": "sha512-IbvzgaoylMqStOOtwucEvSu5CDbfQN+H1ZZ2p6c9Kmvzptqh6G441GFy0FFVVqxOAHNhQm2w6n0Ag8trdBjCfw==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6525,13 +4768,13 @@ } }, "node_modules/@opentelemetry/instrumentation-socket.io": { - "version": "0.37.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-socket.io/-/instrumentation-socket.io-0.37.0.tgz", - "integrity": "sha512-aIztxmx/yis/goEndnoITrZvDDr1GdCtlsWo9ex7MhUIjqq5nJbTuyigf3GmU86XFFhSThxfQuJ9DpJyPxfBfA==", + "version": "0.43.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-socket.io/-/instrumentation-socket.io-0.43.0.tgz", + "integrity": "sha512-HAQoIZ6N/ey1L4jF69gmqo7RyeSv5rc4sZZAd1v6SVaB8ZolTEyWEzGlu1NRZZTnqfWNxDkX6J1/omWpDd9k0w==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6541,14 +4784,14 @@ } }, "node_modules/@opentelemetry/instrumentation-tedious": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-tedious/-/instrumentation-tedious-0.8.0.tgz", - "integrity": "sha512-BBRW8+Qm2PLNkVMynr3Q7L4xCAOCOs0J9BJIJ8ZGoatW42b2H4qhMhq35jfPDvEL5u5azxHDapmUVYrDJDjAfA==", + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-tedious/-/instrumentation-tedious-0.15.0.tgz", + "integrity": "sha512-Kb7yo8Zsq2TUwBbmwYgTAMPK0VbhoS8ikJ6Bup9KrDtCx2JC01nCb+M0VJWXt7tl0+5jARUbKWh5jRSoImxdCw==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1", - "@opentelemetry/semantic-conventions": "^1.0.0", - "@types/tedious": "^4.0.10" + "@opentelemetry/instrumentation": "^0.54.0", + "@opentelemetry/semantic-conventions": "^1.27.0", + "@types/tedious": "^4.0.14" }, "engines": { "node": ">=14" @@ -6557,13 +4800,30 @@ "@opentelemetry/api": "^1.3.0" } }, - "node_modules/@opentelemetry/instrumentation-winston": { - "version": "0.35.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-winston/-/instrumentation-winston-0.35.0.tgz", - "integrity": "sha512-ymcuA3S2flnLmH1GS0105H91iDLap8cizOCaLMCp7Xz7r4L+wFf1zfix9M+iSkxcPFshHRt8LFA/ELXw51nk0g==", + "node_modules/@opentelemetry/instrumentation-undici": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-undici/-/instrumentation-undici-0.7.1.tgz", + "integrity": "sha512-sIl4zrRDP7pR+2Pmdm9XJQULMKiUmvZze2cEW6gUz7TXCEaYmJ+vNMdd7qgeRo8C7AMm+T08mptobFVKPzdz+A==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/instrumentation": "^0.49.1" + "@opentelemetry/core": "^1.8.0", + "@opentelemetry/instrumentation": "^0.54.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.7.0" + } + }, + "node_modules/@opentelemetry/instrumentation-winston": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-winston/-/instrumentation-winston-0.41.0.tgz", + "integrity": "sha512-qtqGDx2Plu71s9xaeXut0YgZFG/y68ENG9vvo/SODeEC+4/APiS/htQ5YNJIxxjOuxYowdFYRqV9Kmef2EUzmw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "^0.54.0", + "@opentelemetry/instrumentation": "^0.54.0" }, "engines": { "node": ">=14" @@ -6573,138 +4833,125 @@ } }, "node_modules/@opentelemetry/otlp-exporter-base": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.40.0.tgz", - "integrity": "sha512-AUmMUPM1/oYGbOWYRBBQz4Ic/adMYA/mIMnAy+QAEmCzjBIC/fyRReVhJmF2cpkvYh7QOkX3017zl2dgWLHpvQ==", + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.55.0.tgz", + "integrity": "sha512-iHQI0Zzq3h1T6xUJTVFwmFl5Dt5y1es+fl4kM+k5T/3YvmVyeYkSiF+wHCg6oKrlUAJfk+t55kaAu3sYmt7ZYA==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.14.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/otlp-transformer": "0.55.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": "^1.0.0" - } - }, - "node_modules/@opentelemetry/otlp-exporter-base/node_modules/@opentelemetry/core": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.14.0.tgz", - "integrity": "sha512-MnMZ+sxsnlzloeuXL2nm5QcNczt/iO82UOeQQDHhV83F2fP3sgntW2evvtoxJki0MBLxEsh5ADD7PR/Hn5uzjw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "1.14.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" - } - }, - "node_modules/@opentelemetry/otlp-exporter-base/node_modules/@opentelemetry/semantic-conventions": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.14.0.tgz", - "integrity": "sha512-rJfCY8rCWz3cb4KI6pEofnytvMPuj3YLQwoscCCYZ5DkdiPjo15IQ0US7+mjcWy9H3fcZIzf2pbJZ7ck/h4tug==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" + "@opentelemetry/api": "^1.3.0" } }, "node_modules/@opentelemetry/otlp-grpc-exporter-base": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-grpc-exporter-base/-/otlp-grpc-exporter-base-0.40.0.tgz", - "integrity": "sha512-rgfyCofGMpou1OsCF1fNr/2iBzgeZj3rjplEBi0yfX6s3nNcJ6ZfhDvyblKG6dd/UydPSHYAtFAstZwwuucFJA==", + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-grpc-exporter-base/-/otlp-grpc-exporter-base-0.55.0.tgz", + "integrity": "sha512-gebbjl9FiSp52igWXuGjcWQKfB6IBwFGt5z1VFwTcVZVeEZevB6bJIqoFrhH4A02m7OUlpJ7l4EfRi3UtkNANQ==", "license": "Apache-2.0", "dependencies": { "@grpc/grpc-js": "^1.7.1", - "@opentelemetry/core": "1.14.0", - "@opentelemetry/otlp-exporter-base": "0.40.0", - "protobufjs": "^7.2.2" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/otlp-exporter-base": "0.55.0", + "@opentelemetry/otlp-transformer": "0.55.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": "^1.0.0" + "@opentelemetry/api": "^1.3.0" } }, - "node_modules/@opentelemetry/otlp-grpc-exporter-base/node_modules/@opentelemetry/core": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.14.0.tgz", - "integrity": "sha512-MnMZ+sxsnlzloeuXL2nm5QcNczt/iO82UOeQQDHhV83F2fP3sgntW2evvtoxJki0MBLxEsh5ADD7PR/Hn5uzjw==", + "node_modules/@opentelemetry/otlp-transformer": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.55.0.tgz", + "integrity": "sha512-kVqEfxtp6mSN2Dhpy0REo1ghP4PYhC1kMHQJ2qVlO99Pc+aigELjZDfg7/YKmL71gR6wVGIeJfiql/eXL7sQPA==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/semantic-conventions": "1.14.0" + "@opentelemetry/api-logs": "0.55.0", + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/sdk-logs": "0.55.0", + "@opentelemetry/sdk-metrics": "1.28.0", + "@opentelemetry/sdk-trace-base": "1.28.0", + "protobufjs": "^7.3.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" + "@opentelemetry/api": "^1.3.0" } }, - "node_modules/@opentelemetry/otlp-grpc-exporter-base/node_modules/@opentelemetry/semantic-conventions": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.14.0.tgz", - "integrity": "sha512-rJfCY8rCWz3cb4KI6pEofnytvMPuj3YLQwoscCCYZ5DkdiPjo15IQ0US7+mjcWy9H3fcZIzf2pbJZ7ck/h4tug==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "node_modules/@opentelemetry/otlp-proto-exporter-base": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-proto-exporter-base/-/otlp-proto-exporter-base-0.49.1.tgz", - "integrity": "sha512-x1qB4EUC7KikUl2iNuxCkV8yRzrSXSyj4itfpIO674H7dhI7Zv37SFaOJTDN+8Z/F50gF2ISFH9CWQ4KCtGm2A==", - "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/api-logs": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.55.0.tgz", + "integrity": "sha512-3cpa+qI45VHYcA5c0bHM6VHo9gicv3p5mlLHNG3rLyjQU8b7e0st1rWtrUn3JbZ3DwwCfhKop4eQ9UuYlC6Pkg==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/otlp-exporter-base": "0.49.1", - "protobufjs": "^7.2.3" + "@opentelemetry/api": "^1.3.0" }, "engines": { "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.0.0" } }, - "node_modules/@opentelemetry/otlp-proto-exporter-base/node_modules/@opentelemetry/core": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.22.0.tgz", - "integrity": "sha512-0VoAlT6x+Xzik1v9goJ3pZ2ppi6+xd3aUfg4brfrLkDBHRIVjMP0eBHrKrhB+NKcDyMAg8fAbGL3Npg/F6AwWA==", + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/resources": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.28.0.tgz", + "integrity": "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/semantic-conventions": "1.22.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" + "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, - "node_modules/@opentelemetry/otlp-proto-exporter-base/node_modules/@opentelemetry/otlp-exporter-base": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.49.1.tgz", - "integrity": "sha512-z6sHliPqDgJU45kQatAettY9/eVF58qVPaTuejw9YWfSRqid9pXPYeegDCSdyS47KAUgAtm+nC28K3pfF27HWg==", + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-metrics": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.28.0.tgz", + "integrity": "sha512-43tqMK/0BcKTyOvm15/WQ3HLr0Vu/ucAl/D84NO7iSlv6O4eOprxSHa3sUtmYkaZWHqdDJV0AHVz/R6u4JALVQ==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.22.0" + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": "^1.0.0" + "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, - "node_modules/@opentelemetry/otlp-proto-exporter-base/node_modules/@opentelemetry/semantic-conventions": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.22.0.tgz", - "integrity": "sha512-CAOgFOKLybd02uj/GhCdEeeBjOS0yeoDeo/CA7ASBSmenpZHAKGB3iDm/rv3BQLcabb/OprDEsSQ1y0P8A7Siw==", + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-trace-base": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.28.0.tgz", + "integrity": "sha512-ceUVWuCpIao7Y5xE02Xs3nQi0tOGmMea17ecBdwtCvdo9ekmO+ijc9RFDgfifMl7XCBf41zne/1POM3LqSTZDA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz", + "integrity": "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==", "license": "Apache-2.0", "engines": { "node": ">=14" @@ -6722,11 +4969,29 @@ "@opentelemetry/api": "^1.0.0" } }, - "node_modules/@opentelemetry/propagator-aws-xray": { - "version": "1.26.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-aws-xray/-/propagator-aws-xray-1.26.2.tgz", - "integrity": "sha512-k43wxTjKYvwfce9L4eT8fFYy/ATmCfPHZPZsyT/6ABimf2KE1HafoOsIcxLOtmNSZt6dCvBIYCrXaOWta20xJg==", + "node_modules/@opentelemetry/propagator-b3": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-b3/-/propagator-b3-1.28.0.tgz", + "integrity": "sha512-Q7HVDIMwhN5RxL4bECMT4BdbyYSAKkC6U/RGn4NpO/cbqP6ZRg+BS7fPo/pGZi2w8AHfpIGQFXQmE8d2PC5xxQ==", "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/propagator-jaeger": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-jaeger/-/propagator-jaeger-1.28.0.tgz", + "integrity": "sha512-wKJ94+s8467CnIRgoSRh0yXm/te0QMOwTq9J01PfG/RzYZvlvN8aRisN2oZ9SznB45dDGnMj3BhUlchSA9cEKA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0" + }, "engines": { "node": ">=14" }, @@ -6744,13 +5009,14 @@ } }, "node_modules/@opentelemetry/resource-detector-alibaba-cloud": { - "version": "0.28.10", - "resolved": "https://registry.npmjs.org/@opentelemetry/resource-detector-alibaba-cloud/-/resource-detector-alibaba-cloud-0.28.10.tgz", - "integrity": "sha512-TZv/1Y2QCL6sJ+X9SsPPBXe4786bc/Qsw0hQXFsNTbJzDTGGUmOAlSZ2qPiuqAd4ZheUYfD+QA20IvAjUz9Hhg==", + "version": "0.29.7", + "resolved": "https://registry.npmjs.org/@opentelemetry/resource-detector-alibaba-cloud/-/resource-detector-alibaba-cloud-0.29.7.tgz", + "integrity": "sha512-PExUl/R+reSQI6Y/eNtgAsk6RHk1ElYSzOa8/FHfdc/nLmx9sqMasBEpLMkETkzDP7t27ORuXe4F9vwkV2uwwg==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/resources": "^1.0.0", - "@opentelemetry/semantic-conventions": "^1.22.0" + "@opentelemetry/core": "^1.26.0", + "@opentelemetry/resources": "^1.10.0", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6776,14 +5042,32 @@ "@opentelemetry/api": "^1.0.0" } }, - "node_modules/@opentelemetry/resource-detector-container": { - "version": "0.3.11", - "resolved": "https://registry.npmjs.org/@opentelemetry/resource-detector-container/-/resource-detector-container-0.3.11.tgz", - "integrity": "sha512-22ndMDakxX+nuhAYwqsciexV8/w26JozRUV0FN9kJiqSWtA1b5dCVtlp3J6JivG5t8kDN9UF5efatNnVbqRT9Q==", + "node_modules/@opentelemetry/resource-detector-azure": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@opentelemetry/resource-detector-azure/-/resource-detector-azure-0.2.12.tgz", + "integrity": "sha512-iIarQu6MiCjEEp8dOzmBvCSlRITPFTinFB2oNKAjU6xhx8d7eUcjNOKhBGQTvuCriZrxrEvDaEEY9NfrPQ6uYQ==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/resources": "^1.0.0", - "@opentelemetry/semantic-conventions": "^1.22.0" + "@opentelemetry/core": "^1.25.1", + "@opentelemetry/resources": "^1.10.1", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.0.0" + } + }, + "node_modules/@opentelemetry/resource-detector-container": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/@opentelemetry/resource-detector-container/-/resource-detector-container-0.5.3.tgz", + "integrity": "sha512-x5DxWu+ZALBuFpxwO2viv9ktH4Y3Gk9LaYKn2U8J+aeD412iy/OcGLPbQ76Px7pQ8qaJ5rnjcevBOHYT4aA+zQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^1.26.0", + "@opentelemetry/resources": "^1.10.0", + "@opentelemetry/semantic-conventions": "^1.27.0" }, "engines": { "node": ">=14" @@ -6826,6 +5110,21 @@ "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, + "node_modules/@opentelemetry/resources/node_modules/@opentelemetry/core": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.30.1.tgz", + "integrity": "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, "node_modules/@opentelemetry/resources/node_modules/@opentelemetry/semantic-conventions": { "version": "1.28.0", "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz", @@ -6835,6 +5134,60 @@ "node": ">=14" } }, + "node_modules/@opentelemetry/sdk-logs": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.55.0.tgz", + "integrity": "sha512-TSx+Yg/d48uWW6HtjS1AD5x6WPfLhDWLl/WxC7I2fMevaiBuKCuraxTB8MDXieCNnBI24bw9ytyXrDCswFfWgA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.55.0", + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs/node_modules/@opentelemetry/api-logs": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.55.0.tgz", + "integrity": "sha512-3cpa+qI45VHYcA5c0bHM6VHo9gicv3p5mlLHNG3rLyjQU8b7e0st1rWtrUn3JbZ3DwwCfhKop4eQ9UuYlC6Pkg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/sdk-logs/node_modules/@opentelemetry/resources": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.28.0.tgz", + "integrity": "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz", + "integrity": "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, "node_modules/@opentelemetry/sdk-metrics": { "version": "1.30.1", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.30.1.tgz", @@ -6851,6 +5204,253 @@ "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, + "node_modules/@opentelemetry/sdk-metrics/node_modules/@opentelemetry/core": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.30.1.tgz", + "integrity": "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz", + "integrity": "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/sdk-node": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-node/-/sdk-node-0.55.0.tgz", + "integrity": "sha512-gSXQWV23+9vhbjsvAIeM0LxY3W8DTKI3MZlzFp61noIb1jSr46ET+qoUjHlfZ1Yymebv9KXWeZsqhft81HBXuQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.55.0", + "@opentelemetry/core": "1.28.0", + "@opentelemetry/exporter-logs-otlp-grpc": "0.55.0", + "@opentelemetry/exporter-logs-otlp-http": "0.55.0", + "@opentelemetry/exporter-logs-otlp-proto": "0.55.0", + "@opentelemetry/exporter-trace-otlp-grpc": "0.55.0", + "@opentelemetry/exporter-trace-otlp-http": "0.55.0", + "@opentelemetry/exporter-trace-otlp-proto": "0.55.0", + "@opentelemetry/exporter-zipkin": "1.28.0", + "@opentelemetry/instrumentation": "0.55.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/sdk-logs": "0.55.0", + "@opentelemetry/sdk-metrics": "1.28.0", + "@opentelemetry/sdk-trace-base": "1.28.0", + "@opentelemetry/sdk-trace-node": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/api-logs": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.55.0.tgz", + "integrity": "sha512-3cpa+qI45VHYcA5c0bHM6VHo9gicv3p5mlLHNG3rLyjQU8b7e0st1rWtrUn3JbZ3DwwCfhKop4eQ9UuYlC6Pkg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/instrumentation": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.55.0.tgz", + "integrity": "sha512-YDCMlaQRZkziLL3t6TONRgmmGxDx6MyQDXRD0dknkkgUZtOK5+8MWft1OXzmNu6XfBOdT12MKN5rz+jHUkafKQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.55.0", + "@types/shimmer": "^1.2.0", + "import-in-the-middle": "^1.8.1", + "require-in-the-middle": "^7.1.1", + "semver": "^7.5.2", + "shimmer": "^1.2.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/resources": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.28.0.tgz", + "integrity": "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/sdk-metrics": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.28.0.tgz", + "integrity": "sha512-43tqMK/0BcKTyOvm15/WQ3HLr0Vu/ucAl/D84NO7iSlv6O4eOprxSHa3sUtmYkaZWHqdDJV0AHVz/R6u4JALVQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/sdk-trace-base": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.28.0.tgz", + "integrity": "sha512-ceUVWuCpIao7Y5xE02Xs3nQi0tOGmMea17ecBdwtCvdo9ekmO+ijc9RFDgfifMl7XCBf41zne/1POM3LqSTZDA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz", + "integrity": "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.30.1.tgz", + "integrity": "sha512-jVPgBbH1gCy2Lb7X0AVQ8XAfgg0pJ4nvl8/IiQA6nxOsPvS+0zMJaFSs2ltXe0J6C8dqjcnpyqINDJmU30+uOg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base/node_modules/@opentelemetry/core": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.30.1.tgz", + "integrity": "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz", + "integrity": "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/sdk-trace-node": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-1.28.0.tgz", + "integrity": "sha512-N0sYfYXvHpP0FNIyc+UfhLnLSTOuZLytV0qQVrDWIlABeD/DWJIGttS7nYeR14gQLXch0M1DW8zm3VeN6Opwtg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/context-async-hooks": "1.28.0", + "@opentelemetry/core": "1.28.0", + "@opentelemetry/propagator-b3": "1.28.0", + "@opentelemetry/propagator-jaeger": "1.28.0", + "@opentelemetry/sdk-trace-base": "1.28.0", + "semver": "^7.5.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-node/node_modules/@opentelemetry/resources": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.28.0.tgz", + "integrity": "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-node/node_modules/@opentelemetry/sdk-trace-base": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.28.0.tgz", + "integrity": "sha512-ceUVWuCpIao7Y5xE02Xs3nQi0tOGmMea17ecBdwtCvdo9ekmO+ijc9RFDgfifMl7XCBf41zne/1POM3LqSTZDA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-node/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz", + "integrity": "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, "node_modules/@opentelemetry/semantic-conventions": { "version": "1.40.0", "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.40.0.tgz", @@ -6876,9 +5476,9 @@ } }, "node_modules/@oxc-project/types": { - "version": "0.124.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.124.0.tgz", - "integrity": "sha512-VBFWMTBvHxS11Z5Lvlr3IWgrwhMTXV+Md+EQF0Xf60+wAdsGFTBx7X7K/hP4pi8N7dcm1RvcHwDxZ16Qx8keUg==", + "version": "0.126.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.126.0.tgz", + "integrity": "sha512-oGfVtjAgwQVVpfBrbtk4e1XDyWHRFta6BS3GWVzrF8xYBT2VGQAk39yJS/wFSMrZqoiCU4oghT3Ch0HaHGIHcQ==", "dev": true, "license": "MIT", "funding": { @@ -6898,30 +5498,25 @@ "node": ">=10.0.0" } }, - "node_modules/@panva/asn1.js": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@panva/asn1.js/-/asn1.js-1.0.0.tgz", - "integrity": "sha512-UdkG3mLEqXgnlKsWanWcgb6dOjUzJ+XC5f+aWw30qrtjxeNUSfKX1cd5FBzOaXQumoe9nIqeZUvrRJS03HCCtw==", - "license": "MIT", - "engines": { - "node": ">=10.13.0" - } - }, - "node_modules/@paralleldrive/cuid2": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/@paralleldrive/cuid2/-/cuid2-2.3.1.tgz", - "integrity": "sha512-XO7cAxhnTZl0Yggq6jOgjiOHhbgcO4NqFqwSmQpjK3b6TEE6Uj/jfSk6wzYyemh3+I0sHirKSetjQwn5cZktFw==", - "license": "MIT", - "dependencies": { - "@noble/hashes": "^1.1.5" - } - }, "node_modules/@pinojs/redact": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==", "license": "MIT" }, + "node_modules/@pkgr/core": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.2.9.tgz", + "integrity": "sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/pkgr" + } + }, "node_modules/@playwright/test": { "version": "1.59.1", "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.59.1.tgz", @@ -7010,9 +5605,9 @@ "license": "BSD-3-Clause" }, "node_modules/@rolldown/binding-android-arm64": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.15.tgz", - "integrity": "sha512-YYe6aWruPZDtHNpwu7+qAHEMbQ/yRl6atqb/AhznLTnD3UY99Q1jE7ihLSahNWkF4EqRPVC4SiR4O0UkLK02tA==", + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.16.tgz", + "integrity": "sha512-rhY3k7Bsae9qQfOtph2Pm2jZEA+s8Gmjoz4hhmx70K9iMQ/ddeae+xhRQcM5IuVx5ry1+bGfkvMn7D6MJggVSA==", "cpu": [ "arm64" ], @@ -7027,9 +5622,9 @@ } }, "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-rc.15.tgz", - "integrity": "sha512-oArR/ig8wNTPYsXL+Mzhs0oxhxfuHRfG7Ikw7jXsw8mYOtk71W0OkF2VEVh699pdmzjPQsTjlD1JIOoHkLP1Fg==", + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-rc.16.tgz", + "integrity": "sha512-rNz0yK078yrNn3DrdgN+PKiMOW8HfQ92jQiXxwX8yW899ayV00MLVdaCNeVBhG/TbH3ouYVObo8/yrkiectkcQ==", "cpu": [ "arm64" ], @@ -7044,9 +5639,9 @@ } }, "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-rc.15.tgz", - "integrity": "sha512-YzeVqOqjPYvUbJSWJ4EDL8ahbmsIXQpgL3JVipmN+MX0XnXMeWomLN3Fb+nwCmP/jfyqte5I3XRSm7OfQrbyxw==", + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-rc.16.tgz", + "integrity": "sha512-r/OmdR00HmD4i79Z//xO06uEPOq5hRXdhw7nzkxQxwSavs3PSHa1ijntdpOiZ2mzOQ3fVVu8C1M19FoNM+dMUQ==", "cpu": [ "x64" ], @@ -7061,9 +5656,9 @@ } }, "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-rc.15.tgz", - "integrity": "sha512-9Erhx956jeQ0nNTyif1+QWAXDRD38ZNjr//bSHrt6wDwB+QkAfl2q6Mn1k6OBPerznjRmbM10lgRb1Pli4xZPw==", + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-rc.16.tgz", + "integrity": "sha512-KcRE5w8h0OnjUatG8pldyD14/CQ5Phs1oxfR+3pKDjboHRo9+MkqQaiIZlZRpsxC15paeXme/I127tUa9TXJ6g==", "cpu": [ "x64" ], @@ -7078,9 +5673,9 @@ } }, "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-rc.15.tgz", - "integrity": "sha512-cVwk0w8QbZJGTnP/AHQBs5yNwmpgGYStL88t4UIaqcvYJWBfS0s3oqVLZPwsPU6M0zlW4GqjP0Zq5MnAGwFeGA==", + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-rc.16.tgz", + "integrity": "sha512-bT0guA1bpxEJ/ZhTRniQf7rNF8ybvXOuWbNIeLABaV5NGjx4EtOWBTSRGWFU9ZWVkPOZ+HNFP8RMcBokBiZ0Kg==", "cpu": [ "arm" ], @@ -7095,13 +5690,16 @@ } }, "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-rc.15.tgz", - "integrity": "sha512-eBZ/u8iAK9SoHGanqe/jrPnY0JvBN6iXbVOsbO38mbz+ZJsaobExAm1Iu+rxa4S1l2FjG0qEZn4Rc6X8n+9M+w==", + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-rc.16.tgz", + "integrity": "sha512-+tHktCHWV8BDQSjemUqm/Jl/TPk3QObCTIjmdDy/nlupcujZghmKK2962LYrqFpWu+ai01AN/REOH3NEpqvYQg==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -7112,13 +5710,16 @@ } }, "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-rc.15.tgz", - "integrity": "sha512-ZvRYMGrAklV9PEkgt4LQM6MjQX2P58HPAuecwYObY2DhS2t35R0I810bKi0wmaYORt6m/2Sm+Z+nFgb0WhXNcQ==", + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-rc.16.tgz", + "integrity": "sha512-3fPzdREH806oRLxpTWW1Gt4tQHs0TitZFOECB2xzCFLPKnSOy90gwA7P29cksYilFO6XVRY1kzga0cL2nRjKPg==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -7129,13 +5730,16 @@ } }, "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.0-rc.15.tgz", - "integrity": "sha512-VDpgGBzgfg5hLg+uBpCLoFG5kVvEyafmfxGUV0UHLcL5irxAK7PKNeC2MwClgk6ZAiNhmo9FLhRYgvMmedLtnQ==", + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.0-rc.16.tgz", + "integrity": "sha512-EKwI1tSrLs7YVw+JPJT/G2dJQ1jl9qlTTTEG0V2Ok/RdOenRfBw2PQdLPyjhIu58ocdBfP7vIRN/pvMsPxs/AQ==", "cpu": [ "ppc64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -7146,13 +5750,16 @@ } }, "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.0-rc.15.tgz", - "integrity": "sha512-y1uXY3qQWCzcPgRJATPSOUP4tCemh4uBdY7e3EZbVwCJTY3gLJWnQABgeUetvED+bt1FQ01OeZwvhLS2bpNrAQ==", + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.0-rc.16.tgz", + "integrity": "sha512-Uknladnb3Sxqu6SEcqBldQyJUpk8NleooZEc0MbRBJ4inEhRYWZX0NJu12vNf2mqAq7gsofAxHrGghiUYjhaLQ==", "cpu": [ "s390x" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -7163,13 +5770,16 @@ } }, "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-rc.15.tgz", - "integrity": "sha512-023bTPBod7J3Y/4fzAN6QtpkSABR0rigtrwaP+qSEabUh5zf6ELr9Nc7GujaROuPY3uwdSIXWrvhn1KxOvurWA==", + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-rc.16.tgz", + "integrity": "sha512-FIb8+uG49sZBtLTn+zt1AJ20TqVcqWeSIyoVt0or7uAWesgKaHbiBh6OpA/k9v0LTt+PTrb1Lao133kP4uVxkg==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -7180,13 +5790,16 @@ } }, "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-rc.15.tgz", - "integrity": "sha512-witB2O0/hU4CgfOOKUoeFgQ4GktPi1eEbAhaLAIpgD6+ZnhcPkUtPsoKKHRzmOoWPZue46IThdSgdo4XneOLYw==", + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-rc.16.tgz", + "integrity": "sha512-RuERhF9/EgWxZEXYWCOaViUWHIboceK4/ivdtQ3R0T44NjLkIIlGIAVAuCddFxsZ7vnRHtNQUrt2vR2n2slB2w==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -7197,9 +5810,9 @@ } }, "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-rc.15.tgz", - "integrity": "sha512-UCL68NJ0Ud5zRipXZE9dF5PmirzJE4E4BCIOOssEnM7wLDsxjc6Qb0sGDxTNRTP53I6MZpygyCpY8Aa8sPfKPg==", + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-rc.16.tgz", + "integrity": "sha512-mXcXnvd9GpazCxeUCCnZ2+YF7nut+ZOEbE4GtaiPtyY6AkhZWbK70y1KK3j+RDhjVq5+U8FySkKRb/+w0EeUwA==", "cpu": [ "arm64" ], @@ -7214,9 +5827,9 @@ } }, "node_modules/@rolldown/binding-wasm32-wasi": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-rc.15.tgz", - "integrity": "sha512-ApLruZq/ig+nhaE7OJm4lDjayUnOHVUa77zGeqnqZ9pn0ovdVbbNPerVibLXDmWeUZXjIYIT8V3xkT58Rm9u5Q==", + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-rc.16.tgz", + "integrity": "sha512-3Q2KQxnC8IJOLqXmUMoYwyIPZU9hzRbnHaoV3Euz+VVnjZKcY8ktnNP8T9R4/GGQtb27C/UYKABxesKWb8lsvQ==", "cpu": [ "wasm32" ], @@ -7226,16 +5839,27 @@ "dependencies": { "@emnapi/core": "1.9.2", "@emnapi/runtime": "1.9.2", - "@napi-rs/wasm-runtime": "^1.1.3" + "@napi-rs/wasm-runtime": "^1.1.4" }, "engines": { - "node": ">=14.0.0" + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.2.tgz", + "integrity": "sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" } }, "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-rc.15.tgz", - "integrity": "sha512-KmoUoU7HnN+Si5YWJigfTws1jz1bKBYDQKdbLspz0UaqjjFkddHsqorgiW1mxcAj88lYUE6NC/zJNwT+SloqtA==", + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-rc.16.tgz", + "integrity": "sha512-tj7XRemQcOcFwv7qhpUxMTBbI5mWMlE4c1Omhg5+h8GuLXzyj8HviYgR+bB2DMDgRqUE+jiDleqSCRjx4aYk/Q==", "cpu": [ "arm64" ], @@ -7250,9 +5874,9 @@ } }, "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-rc.15.tgz", - "integrity": "sha512-3P2A8L+x75qavWLe/Dll3EYBJLQmtkJN8rfh+U/eR3MqMgL/h98PhYI+JFfXuDPgPeCB7iZAKiqii5vqOvnA0g==", + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-rc.16.tgz", + "integrity": "sha512-PH5DRZT+F4f2PTXRXR8uJxnBq2po/xFtddyabTJVJs/ZYVHqXPEgNIr35IHTEa6bpa0Q8Awg+ymkTaGnKITw4g==", "cpu": [ "x64" ], @@ -7267,48 +5891,27 @@ } }, "node_modules/@rolldown/pluginutils": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.15.tgz", - "integrity": "sha512-UromN0peaE53IaBRe9W7CjrZgXl90fqGpK+mIZbA3qSTeYqg3pqpROBdIPvOG3F5ereDHNwoHBI2e50n1BDr1g==", + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.16.tgz", + "integrity": "sha512-45+YtqxLYKDWQouLKCrpIZhke+nXxhsw+qAHVzHDVwttyBlHNBVs2K25rDXrZzhpTp9w1FlAlvweV1H++fdZoA==", "dev": true, "license": "MIT" }, - "node_modules/@sideway/address": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@sideway/address/-/address-4.1.5.tgz", - "integrity": "sha512-IqO/DUQHUkPeixNQ8n0JA6102hT9CmaljNTPmQ1u8MEhBo/R4Q8eKLN/vGZxuebwOroDB4cbpjheD4+/sKFK4Q==", - "license": "BSD-3-Clause", - "dependencies": { - "@hapi/hoek": "^9.0.0" - } - }, - "node_modules/@sideway/formula": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@sideway/formula/-/formula-3.0.1.tgz", - "integrity": "sha512-/poHZJJVjx3L+zVD6g9KgHfYnb443oi7wLu/XKojDviHy6HOEOA6z1Trk5aR1dGcmPenJEgb2sK2I80LeS3MIg==", - "license": "BSD-3-Clause" - }, - "node_modules/@sideway/pinpoint": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@sideway/pinpoint/-/pinpoint-2.0.0.tgz", - "integrity": "sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==", - "license": "BSD-3-Clause" - }, "node_modules/@simple-git/args-pathspec": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@simple-git/args-pathspec/-/args-pathspec-1.0.2.tgz", - "integrity": "sha512-nEFVejViHUoL8wU8GTcwqrvqfUG40S5ts6S4fr1u1Ki5CklXlRDYThPVA/qurTmCYFGnaX3XpVUmICLHdvhLaA==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@simple-git/args-pathspec/-/args-pathspec-1.0.3.tgz", + "integrity": "sha512-ngJMaHlsWDTfjyq9F3VIQ8b7NXbBLq5j9i5bJ6XLYtD6qlDXT7fdKY2KscWWUF8t18xx052Y/PUO1K1TRc9yKA==", "dev": true, "license": "MIT" }, "node_modules/@simple-git/argv-parser": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@simple-git/argv-parser/-/argv-parser-1.0.3.tgz", - "integrity": "sha512-NMKv9sJcSN2VvnPT9Ja7eKfGy8Q8mMFLwPTCcuZMtv3+mYcLIZflg31S/tp2XCCyiY7YAx6cgBHQ0fwA2fWHpQ==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@simple-git/argv-parser/-/argv-parser-1.1.1.tgz", + "integrity": "sha512-Q9lBcfQ+VQCpQqGJFHe5yooOS5hGdLFFbJ5R+R5aDsnkPCahtn1hSkMcORX65J2Z5lxSkD0lQorMsncuBQxYUw==", "dev": true, "license": "MIT", "dependencies": { - "@simple-git/args-pathspec": "^1.0.2" + "@simple-git/args-pathspec": "^1.0.3" } }, "node_modules/@sinonjs/commons": { @@ -7411,16 +6014,16 @@ } }, "node_modules/@smithy/config-resolver": { - "version": "4.4.14", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.4.14.tgz", - "integrity": "sha512-N55f8mPEccpzKetUagdvmAy8oohf0J5cuj9jLI1TaSceRlq0pJsIZepY3kmAXAhyxqXPV6hDerDQhqQPKWgAoQ==", + "version": "4.4.16", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.4.16.tgz", + "integrity": "sha512-GFlGPNLZKrGfqWpqVb31z7hvYCA9ZscfX1buYnvvMGcRYsQQnhH+4uN6mWWflcD5jB4OXP/LBrdpukEdjl41tg==", "license": "Apache-2.0", "dependencies": { - "@smithy/node-config-provider": "^4.3.13", - "@smithy/types": "^4.14.0", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/types": "^4.14.1", "@smithy/util-config-provider": "^4.2.2", - "@smithy/util-endpoints": "^3.3.4", - "@smithy/util-middleware": "^4.2.13", + "@smithy/util-endpoints": "^3.4.1", + "@smithy/util-middleware": "^4.2.14", "tslib": "^2.6.2" }, "engines": { @@ -7428,18 +6031,18 @@ } }, "node_modules/@smithy/core": { - "version": "3.23.14", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.23.14.tgz", - "integrity": "sha512-vJ0IhpZxZAkFYOegMKSrxw7ujhhT2pass/1UEcZ4kfl5srTAqtPU5I7MdYQoreVas3204ykCiNhY1o7Xlz6Yyg==", + "version": "3.23.15", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.23.15.tgz", + "integrity": "sha512-E7GVCgsQttzfujEZb6Qep005wWf4xiL4x06apFEtzQMWYBPggZh/0cnOxPficw5cuK/YjjkehKoIN4YUaSh0UQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/protocol-http": "^5.3.13", - "@smithy/types": "^4.14.0", - "@smithy/url-parser": "^4.2.13", + "@smithy/protocol-http": "^5.3.14", + "@smithy/types": "^4.14.1", + "@smithy/url-parser": "^4.2.14", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", - "@smithy/util-middleware": "^4.2.13", - "@smithy/util-stream": "^4.5.22", + "@smithy/util-middleware": "^4.2.14", + "@smithy/util-stream": "^4.5.23", "@smithy/util-utf8": "^4.2.2", "@smithy/uuid": "^1.1.2", "tslib": "^2.6.2" @@ -7449,15 +6052,15 @@ } }, "node_modules/@smithy/credential-provider-imds": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.2.13.tgz", - "integrity": "sha512-wboCPijzf6RJKLOvnjDAiBxGSmSnGXj35o5ZAWKDaHa/cvQ5U3ZJ13D4tMCE8JG4dxVAZFy/P0x/V9CwwdfULQ==", + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.2.14.tgz", + "integrity": "sha512-Au28zBN48ZAoXdooGUHemuVBrkE+Ie6RPmGNIAJsFqj33Vhb6xAgRifUydZ2aY+M+KaMAETAlKk5NC5h1G7wpg==", "license": "Apache-2.0", "dependencies": { - "@smithy/node-config-provider": "^4.3.13", - "@smithy/property-provider": "^4.2.13", - "@smithy/types": "^4.14.0", - "@smithy/url-parser": "^4.2.13", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/property-provider": "^4.2.14", + "@smithy/types": "^4.14.1", + "@smithy/url-parser": "^4.2.14", "tslib": "^2.6.2" }, "engines": { @@ -7465,13 +6068,13 @@ } }, "node_modules/@smithy/eventstream-codec": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-4.2.13.tgz", - "integrity": "sha512-vYahwBAtRaAcFbOmE9aLr12z7RiHYDSLcnogSdxfm7kKfsNa3wH+NU5r7vTeB5rKvLsWyPjVX8iH94brP7umiQ==", + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-4.2.14.tgz", + "integrity": "sha512-erZq0nOIpzfeZdCyzZjdJb4nVSKLUmSkaQUVkRGQTXs30gyUGeKnrYEg+Xe1W5gE3aReS7IgsvANwVPxSzY6Pw==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/crc32": "5.2.0", - "@smithy/types": "^4.14.0", + "@smithy/types": "^4.14.1", "@smithy/util-hex-encoding": "^4.2.2", "tslib": "^2.6.2" }, @@ -7480,13 +6083,13 @@ } }, "node_modules/@smithy/eventstream-serde-browser": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-4.2.13.tgz", - "integrity": "sha512-wwybfcOX0tLqCcBP378TIU9IqrDuZq/tDV48LlZNydMpCnqnYr+hWBAYbRE+rFFf/p7IkDJySM3bgiMKP2ihPg==", + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-4.2.14.tgz", + "integrity": "sha512-8IelTCtTctWRbb+0Dcy+C0aICh1qa0qWXqgjcXDmMuCvPJRnv26hiDZoAau2ILOniki65mCPKqOQs/BaWvO4CQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/eventstream-serde-universal": "^4.2.13", - "@smithy/types": "^4.14.0", + "@smithy/eventstream-serde-universal": "^4.2.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -7494,12 +6097,12 @@ } }, "node_modules/@smithy/eventstream-serde-config-resolver": { - "version": "4.3.13", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-4.3.13.tgz", - "integrity": "sha512-ied1lO559PtAsMJzg2TKRlctLnEi1PfkNeMMpdwXDImk1zV9uvS/Oxoy/vcy9uv1GKZAjDAB5xT6ziE9fzm5wA==", + "version": "4.3.14", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-4.3.14.tgz", + "integrity": "sha512-sqHiHpYRYo3FJlaIxD1J8PhbcmJAm7IuM16mVnwSkCToD7g00IBZzKuiLNMGmftULmEUX6/UAz8/NN5uMP8bVA==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.0", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -7507,13 +6110,13 @@ } }, "node_modules/@smithy/eventstream-serde-node": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-4.2.13.tgz", - "integrity": "sha512-hFyK+ORJrxAN3RYoaD6+gsGDQjeix8HOEkosoajvXYZ4VeqonM3G4jd9IIRm/sWGXUKmudkY9KdYjzosUqdM8A==", + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-4.2.14.tgz", + "integrity": "sha512-Ht/8BuGlKfFTy0H3+8eEu0vdpwGztCnaLLXtpXNdQqiR7Hj4vFScU3T436vRAjATglOIPjJXronY+1WxxNLSiw==", "license": "Apache-2.0", "dependencies": { - "@smithy/eventstream-serde-universal": "^4.2.13", - "@smithy/types": "^4.14.0", + "@smithy/eventstream-serde-universal": "^4.2.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -7521,13 +6124,13 @@ } }, "node_modules/@smithy/eventstream-serde-universal": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-4.2.13.tgz", - "integrity": "sha512-kRrq4EKLGeOxhC2CBEhRNcu1KSzNJzYY7RK3S7CxMPgB5dRrv55WqQOtRwQxQLC04xqORFLUgnDlc6xrNUULaA==", + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-4.2.14.tgz", + "integrity": "sha512-lWyt4T2XQZUZgK3tQ3Wn0w3XBvZsK/vjTuJl6bXbnGZBHH0ZUSONTYiK9TgjTTzU54xQr3DRFwpjmhp0oLm3gg==", "license": "Apache-2.0", "dependencies": { - "@smithy/eventstream-codec": "^4.2.13", - "@smithy/types": "^4.14.0", + "@smithy/eventstream-codec": "^4.2.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -7535,14 +6138,14 @@ } }, "node_modules/@smithy/fetch-http-handler": { - "version": "5.3.16", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.3.16.tgz", - "integrity": "sha512-nYDRUIvNd4mFmuXraRWt6w5UsZTNqtj4hXJA/iiOD4tuseIdLP9Lq38teH/SZTcIFCa2f+27o7hYpIsWktJKEQ==", + "version": "5.3.17", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.3.17.tgz", + "integrity": "sha512-bXOvQzaSm6MnmLaWA1elgfQcAtN4UP3vXqV97bHuoOrHQOJiLT3ds6o9eo5bqd0TJfRFpzdGnDQdW3FACiAVdw==", "license": "Apache-2.0", "dependencies": { - "@smithy/protocol-http": "^5.3.13", - "@smithy/querystring-builder": "^4.2.13", - "@smithy/types": "^4.14.0", + "@smithy/protocol-http": "^5.3.14", + "@smithy/querystring-builder": "^4.2.14", + "@smithy/types": "^4.14.1", "@smithy/util-base64": "^4.3.2", "tslib": "^2.6.2" }, @@ -7551,14 +6154,14 @@ } }, "node_modules/@smithy/hash-blob-browser": { - "version": "4.2.14", - "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-4.2.14.tgz", - "integrity": "sha512-rtQ5es8r/5v4rav7q5QTsfx9CtCyzrz/g7ZZZBH2xtMmd6G/KQrLOWfSHTvFOUPlVy59RQvxeBYJaLRoybMEyA==", + "version": "4.2.15", + "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-4.2.15.tgz", + "integrity": "sha512-0PJ4Al3fg2nM4qKrAIxyNcApgqHAXcBkN8FeizOz69z0rb26uZ6lMESYtxegaTlXB5Hj84JfwMPavMrwDMjucA==", "license": "Apache-2.0", "dependencies": { "@smithy/chunked-blob-reader": "^5.2.2", "@smithy/chunked-blob-reader-native": "^4.2.3", - "@smithy/types": "^4.14.0", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -7566,12 +6169,12 @@ } }, "node_modules/@smithy/hash-node": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.2.13.tgz", - "integrity": "sha512-4/oy9h0jjmY80a2gOIo75iLl8TOPhmtx4E2Hz+PfMjvx/vLtGY4TMU/35WRyH2JHPfT5CVB38u4JRow7gnmzJA==", + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.2.14.tgz", + "integrity": "sha512-8ZBDY2DD4wr+GGjTpPtiglEsqr0lUP+KHqgZcWczFf6qeZ/YRjMIOoQWVQlmwu7EtxKTd8YXD8lblmYcpBIA1g==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.0", + "@smithy/types": "^4.14.1", "@smithy/util-buffer-from": "^4.2.2", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" @@ -7581,12 +6184,12 @@ } }, "node_modules/@smithy/hash-stream-node": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-4.2.13.tgz", - "integrity": "sha512-WdQ7HwUjINXETeh6dqUeob1UHIYx8kAn9PSp1HhM2WWegiZBYVy2WXIs1lB07SZLan/udys9SBnQGt9MQbDpdg==", + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-4.2.14.tgz", + "integrity": "sha512-tw4GANWkZPb6+BdD4Fgucqzey2+r73Z/GRo9zklsCdwrnxxumUV83ZIaBDdudV4Ylazw3EPTiJZhpX42105ruQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.0", + "@smithy/types": "^4.14.1", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, @@ -7595,12 +6198,12 @@ } }, "node_modules/@smithy/invalid-dependency": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.2.13.tgz", - "integrity": "sha512-jvC0RB/8BLj2SMIkY0Npl425IdnxZJxInpZJbu563zIRnVjpDMXevU3VMCRSabaLB0kf/eFIOusdGstrLJ8IDg==", + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.2.14.tgz", + "integrity": "sha512-c21qJiTSb25xvvOp+H2TNZzPCngrvl5vIPqPB8zQ/DmJF4QWXO19x1dWfMJZ6wZuuWUPPm0gV8C0cU3+ifcWuw==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.0", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -7620,12 +6223,12 @@ } }, "node_modules/@smithy/md5-js": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-4.2.13.tgz", - "integrity": "sha512-cNm7I9NXolFxtS20ojROddOEpSAeI1Obq6pd1Kj5HtHws3s9Fkk8DdHDfQSs5KuxCewZuVK6UqrJnfJmiMzDuQ==", + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-4.2.14.tgz", + "integrity": "sha512-V2v0vx+h0iUSNG1Alt+GNBMSLGCrl9iVsdd+Ap67HPM9PN479x12V8LkuMoKImNZxn3MXeuyUjls+/7ZACZghA==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.0", + "@smithy/types": "^4.14.1", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, @@ -7633,35 +6236,14 @@ "node": ">=18.0.0" } }, - "node_modules/@smithy/middleware-compression": { - "version": "4.3.43", - "resolved": "https://registry.npmjs.org/@smithy/middleware-compression/-/middleware-compression-4.3.43.tgz", - "integrity": "sha512-MphcLSNTvBN9G2/ko7NBV2psEfsQRZviXmf612ZwvbSY7dJZNroc2+WPHBf+I9KO2SFl4VFz11rTTueihwWjlQ==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/core": "^3.23.14", - "@smithy/is-array-buffer": "^4.2.2", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/protocol-http": "^5.3.13", - "@smithy/types": "^4.14.0", - "@smithy/util-config-provider": "^4.2.2", - "@smithy/util-middleware": "^4.2.13", - "@smithy/util-utf8": "^4.2.2", - "fflate": "0.8.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@smithy/middleware-content-length": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.2.13.tgz", - "integrity": "sha512-IPMLm/LE4AZwu6qiE8Rr8vJsWhs9AtOdySRXrOM7xnvclp77Tyh7hMs/FRrMf26kgIe67vFJXXOSmVxS7oKeig==", + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.2.14.tgz", + "integrity": "sha512-xhHq7fX4/3lv5NHxLUk3OeEvl0xZ+Ek3qIbWaCL4f9JwgDZEclPBElljaZCAItdGPQl/kSM4LPMOpy1MYgprpw==", "license": "Apache-2.0", "dependencies": { - "@smithy/protocol-http": "^5.3.13", - "@smithy/types": "^4.14.0", + "@smithy/protocol-http": "^5.3.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -7669,18 +6251,18 @@ } }, "node_modules/@smithy/middleware-endpoint": { - "version": "4.4.29", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.4.29.tgz", - "integrity": "sha512-R9Q/58U+qBiSARGWbAbFLczECg/RmysRksX6Q8BaQEpt75I7LI6WGDZnjuC9GXSGKljEbA7N118LhGaMbfrTXw==", + "version": "4.4.30", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.4.30.tgz", + "integrity": "sha512-qS2XqhKeXmdZ4nEQ4cOxIczSP/Y91wPAHYuRwmWDCh975B7/57uxsm5d6sisnUThn2u2FwzMdJNM7AbO1YPsPg==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.23.14", - "@smithy/middleware-serde": "^4.2.17", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/shared-ini-file-loader": "^4.4.8", - "@smithy/types": "^4.14.0", - "@smithy/url-parser": "^4.2.13", - "@smithy/util-middleware": "^4.2.13", + "@smithy/core": "^3.23.15", + "@smithy/middleware-serde": "^4.2.18", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/shared-ini-file-loader": "^4.4.9", + "@smithy/types": "^4.14.1", + "@smithy/url-parser": "^4.2.14", + "@smithy/util-middleware": "^4.2.14", "tslib": "^2.6.2" }, "engines": { @@ -7688,19 +6270,19 @@ } }, "node_modules/@smithy/middleware-retry": { - "version": "4.5.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.5.1.tgz", - "integrity": "sha512-/zY+Gp7Qj2D2hVm3irkCyONER7E9MiX3cUUm/k2ZmhkzZkrPgwVS4aJ5NriZUEN/M0D1hhjrgjUmX04HhRwdWA==", + "version": "4.5.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.5.3.tgz", + "integrity": "sha512-TE8dJNi6JuxzGSxMCVd3i9IEWDndCl3bmluLsBNDWok8olgj65OfkndMhl9SZ7m14c+C5SQn/PcUmrDl57rSFw==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.23.14", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/protocol-http": "^5.3.13", - "@smithy/service-error-classification": "^4.2.13", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", - "@smithy/util-middleware": "^4.2.13", - "@smithy/util-retry": "^4.3.1", + "@smithy/core": "^3.23.15", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/protocol-http": "^5.3.14", + "@smithy/service-error-classification": "^4.2.14", + "@smithy/smithy-client": "^4.12.11", + "@smithy/types": "^4.14.1", + "@smithy/util-middleware": "^4.2.14", + "@smithy/util-retry": "^4.3.2", "@smithy/uuid": "^1.1.2", "tslib": "^2.6.2" }, @@ -7709,14 +6291,14 @@ } }, "node_modules/@smithy/middleware-serde": { - "version": "4.2.17", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.2.17.tgz", - "integrity": "sha512-0T2mcaM6v9W1xku86Dk0bEW7aEseG6KenFkPK98XNw0ZhOqOiD1MrMsdnQw9QsL3/Oa85T53iSMlm0SZdSuIEQ==", + "version": "4.2.18", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.2.18.tgz", + "integrity": "sha512-M6CSgnp3v4tYz9ynj2JHbA60woBZcGqEwNjTKjBsNHPV26R1ZX52+0wW8WsZU18q45jD0tw2wL22S17Ze9LpEw==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.23.14", - "@smithy/protocol-http": "^5.3.13", - "@smithy/types": "^4.14.0", + "@smithy/core": "^3.23.15", + "@smithy/protocol-http": "^5.3.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -7724,12 +6306,12 @@ } }, "node_modules/@smithy/middleware-stack": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-4.2.13.tgz", - "integrity": "sha512-g72jN/sGDLyTanrCLH9fhg3oysO3f7tQa6eWWsMyn2BiYNCgjF24n4/I9wff/5XidFvjj9ilipAoQrurTUrLvw==", + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-4.2.14.tgz", + "integrity": "sha512-2dvkUKLuFdKsCRmOE4Mn63co0Djtsm+JMh0bYZQupN1pJwMeE8FmQmRLLzzEMN0dnNi7CDCYYH8F0EVwWiPBeA==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.0", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -7737,14 +6319,14 @@ } }, "node_modules/@smithy/node-config-provider": { - "version": "4.3.13", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-4.3.13.tgz", - "integrity": "sha512-iGxQ04DsKXLckbgnX4ipElrOTk+IHgTyu0q0WssZfYhDm9CQWHmu6cOeI5wmWRxpXbBDhIIfXMWz5tPEtcVqbw==", + "version": "4.3.14", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-4.3.14.tgz", + "integrity": "sha512-S+gFjyo/weSVL0P1b9Ts8C/CwIfNCgUPikk3sl6QVsfE/uUuO+QsF+NsE/JkpvWqqyz1wg7HFdiaZuj5CoBMRg==", "license": "Apache-2.0", "dependencies": { - "@smithy/property-provider": "^4.2.13", - "@smithy/shared-ini-file-loader": "^4.4.8", - "@smithy/types": "^4.14.0", + "@smithy/property-provider": "^4.2.14", + "@smithy/shared-ini-file-loader": "^4.4.9", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -7752,79 +6334,27 @@ } }, "node_modules/@smithy/node-http-handler": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-2.5.0.tgz", - "integrity": "sha512-mVGyPBzkkGQsPoxQUbxlEfRjrj6FPyA3u3u2VXGr9hT8wilsoQdZdvKpMBFMB8Crfhv5dNkKHIW0Yyuc7eABqA==", + "version": "4.5.3", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.5.3.tgz", + "integrity": "sha512-lc5jFL++x17sPhIwMWJ3YOnqmSjw/2Po6VLDlUIXvxVWRuJwRXnJ4jOBBLB0cfI5BB5ehIl02Fxr1PDvk/kxDw==", "license": "Apache-2.0", "dependencies": { - "@smithy/abort-controller": "^2.2.0", - "@smithy/protocol-http": "^3.3.0", - "@smithy/querystring-builder": "^2.2.0", - "@smithy/types": "^2.12.0", + "@smithy/protocol-http": "^5.3.14", + "@smithy/querystring-builder": "^4.2.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/node-http-handler/node_modules/@smithy/protocol-http": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-3.3.0.tgz", - "integrity": "sha512-Xy5XK1AFWW2nlY/biWZXu6/krgbaf2dg0q492D8M5qthsnU2H+UgFeZLbM76FnH7s6RO/xhQRkj+T6KBO3JzgQ==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^2.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/node-http-handler/node_modules/@smithy/querystring-builder": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-2.2.0.tgz", - "integrity": "sha512-L1kSeviUWL+emq3CUVSgdogoM/D9QMFaqxL/dd0X7PCNWmPXqt+ExtrBjqT0V7HLN03Vs9SuiLrG3zy3JGnE5A==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^2.12.0", - "@smithy/util-uri-escape": "^2.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/node-http-handler/node_modules/@smithy/types": { - "version": "2.12.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-2.12.0.tgz", - "integrity": "sha512-QwYgloJ0sVNBeBuBs65cIkTbfzV/Q6ZNPCJ99EICFEdJYG50nGIY/uYXp+TbsdJReIuPr0a0kXmCvren3MbRRw==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/node-http-handler/node_modules/@smithy/util-uri-escape": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-2.2.0.tgz", - "integrity": "sha512-jtmJMyt1xMD/d8OtbVJ2gFZOSKc+ueYJZPW20ULW1GOp/q/YIM0wNh+u8ZFao9UaIGz4WoPW8hC64qlWLIfoDA==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" + "node": ">=18.0.0" } }, "node_modules/@smithy/property-provider": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-4.2.13.tgz", - "integrity": "sha512-bGzUCthxRmezuxkbu9wD33wWg9KX3hJpCXpQ93vVkPrHn9ZW6KNNdY5xAUWNuRCwQ+VyboFuWirG1lZhhkcyRQ==", + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-4.2.14.tgz", + "integrity": "sha512-WuM31CgfsnQ/10i7NYr0PyxqknD72Y5uMfUMVSniPjbEPceiTErb4eIqJQ+pdxNEAUEWrewrGjIRjVbVHsxZiQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.0", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -7832,12 +6362,12 @@ } }, "node_modules/@smithy/protocol-http": { - "version": "5.3.13", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.3.13.tgz", - "integrity": "sha512-+HsmuJUF4u8POo6s8/a2Yb/AQ5t/YgLovCuHF9oxbocqv+SZ6gd8lC2duBFiCA/vFHoHQhoq7QjqJqZC6xOxxg==", + "version": "5.3.14", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.3.14.tgz", + "integrity": "sha512-dN5F8kHx8RNU0r+pCwNmFZyz6ChjMkzShy/zup6MtkRmmix4vZzJdW+di7x//b1LiynIev88FM18ie+wwPcQtQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.0", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -7845,12 +6375,12 @@ } }, "node_modules/@smithy/querystring-builder": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.2.13.tgz", - "integrity": "sha512-tG4aOYFCZdPMjbgfhnIQ322H//ojujldp1SrHPHpBSb3NqgUp3dwiUGRJzie87hS1DYwWGqDuPaowoDF+rYCbQ==", + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.2.14.tgz", + "integrity": "sha512-XYA5Z0IqTeF+5XDdh4BBmSA0HvbgVZIyv4cmOoUheDNR57K1HgBp9ukUMx3Cr3XpDHHpLBnexPE3LAtDsZkj2A==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.0", + "@smithy/types": "^4.14.1", "@smithy/util-uri-escape": "^4.2.2", "tslib": "^2.6.2" }, @@ -7859,12 +6389,12 @@ } }, "node_modules/@smithy/querystring-parser": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-4.2.13.tgz", - "integrity": "sha512-hqW3Q4P+CDzUyQ87GrboGMeD7XYNMOF+CuTwu936UQRB/zeYn3jys8C3w+wMkDfY7CyyyVwZQ5cNFoG0x1pYmA==", + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-4.2.14.tgz", + "integrity": "sha512-hr+YyqBD23GVvRxGGrcc/oOeNlK3PzT5Fu4dzrDXxzS1LpFiuL2PQQqKPs87M79aW7ziMs+nvB3qdw77SqE7Lw==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.0", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -7872,24 +6402,24 @@ } }, "node_modules/@smithy/service-error-classification": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-4.2.13.tgz", - "integrity": "sha512-a0s8XZMfOC/qpqq7RCPvJlk93rWFrElH6O++8WJKz0FqnA4Y7fkNi/0mnGgSH1C4x6MFsuBA8VKu4zxFrMe5Vw==", + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-4.2.14.tgz", + "integrity": "sha512-vVimoUnGxlx4eLLQbZImdOZFOe+Zh+5ACntv8VxZuGP72LdWu5GV3oEmCahSEReBgRJoWjypFkrehSj7BWx1HQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.0" + "@smithy/types": "^4.14.1" }, "engines": { "node": ">=18.0.0" } }, "node_modules/@smithy/shared-ini-file-loader": { - "version": "4.4.8", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.4.8.tgz", - "integrity": "sha512-VZCZx2bZasxdqxVgEAhREvDSlkatTPnkdWy1+Kiy8w7kYPBosW0V5IeDwzDUMvWBt56zpK658rx1cOBFOYaPaw==", + "version": "4.4.9", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.4.9.tgz", + "integrity": "sha512-495/V2I15SHgedSJoDPD23JuSfKAp726ZI1V0wtjB07Wh7q/0tri/0e0DLefZCHgxZonrGKt/OCTpAtP1wE1kQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.0", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -7897,16 +6427,16 @@ } }, "node_modules/@smithy/signature-v4": { - "version": "5.3.13", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.3.13.tgz", - "integrity": "sha512-YpYSyM0vMDwKbHD/JA7bVOF6kToVRpa+FM5ateEVRpsTNu564g1muBlkTubXhSKKYXInhpADF46FPyrZcTLpXg==", + "version": "5.3.14", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.3.14.tgz", + "integrity": "sha512-1D9Y/nmlVjCeSivCbhZ7hgEpmHyY1h0GvpSZt3l0xcD9JjmjVC1CHOozS6+Gh+/ldMH8JuJ6cujObQqfayAVFA==", "license": "Apache-2.0", "dependencies": { "@smithy/is-array-buffer": "^4.2.2", - "@smithy/protocol-http": "^5.3.13", - "@smithy/types": "^4.14.0", + "@smithy/protocol-http": "^5.3.14", + "@smithy/types": "^4.14.1", "@smithy/util-hex-encoding": "^4.2.2", - "@smithy/util-middleware": "^4.2.13", + "@smithy/util-middleware": "^4.2.14", "@smithy/util-uri-escape": "^4.2.2", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" @@ -7916,17 +6446,17 @@ } }, "node_modules/@smithy/smithy-client": { - "version": "4.12.9", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.12.9.tgz", - "integrity": "sha512-ovaLEcTU5olSeHcRXcxV6viaKtpkHZumn6Ps0yn7dRf2rRSfy794vpjOtrWDO0d1auDSvAqxO+lyhERSXQ03EQ==", + "version": "4.12.11", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.12.11.tgz", + "integrity": "sha512-wzz/Wa1CH/Tlhxh0s4DQPEcXSxSVfJ59AZcUh9Gu0c6JTlKuwGf4o/3P2TExv0VbtPFt8odIBG+eQGK2+vTECg==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.23.14", - "@smithy/middleware-endpoint": "^4.4.29", - "@smithy/middleware-stack": "^4.2.13", - "@smithy/protocol-http": "^5.3.13", - "@smithy/types": "^4.14.0", - "@smithy/util-stream": "^4.5.22", + "@smithy/core": "^3.23.15", + "@smithy/middleware-endpoint": "^4.4.30", + "@smithy/middleware-stack": "^4.2.14", + "@smithy/protocol-http": "^5.3.14", + "@smithy/types": "^4.14.1", + "@smithy/util-stream": "^4.5.23", "tslib": "^2.6.2" }, "engines": { @@ -7934,9 +6464,9 @@ } }, "node_modules/@smithy/types": { - "version": "4.14.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.14.0.tgz", - "integrity": "sha512-OWgntFLW88kx2qvf/c/67Vno1yuXm/f9M7QFAtVkkO29IJXGBIg0ycEaBTH0kvCtwmvZxRujrgP5a86RvsXJAQ==", + "version": "4.14.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.14.1.tgz", + "integrity": "sha512-59b5HtSVrVR/eYNei3BUj3DCPKD/G7EtDDe7OEJE7i7FtQFugYo6MxbotS8mVJkLNVf8gYaAlEBwwtJ9HzhWSg==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" @@ -7946,13 +6476,13 @@ } }, "node_modules/@smithy/url-parser": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-4.2.13.tgz", - "integrity": "sha512-2G03yoboIRZlZze2+PT4GZEjgwQsJjUgn6iTsvxA02bVceHR6vp4Cuk7TUnPFWKF+ffNUk3kj4COwkENS2K3vw==", + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-4.2.14.tgz", + "integrity": "sha512-p06BiBigJ8bTA3MgnOfCtDUWnAMY0YfedO/GRpmc7p+wg3KW8vbXy1xwSu5ASy0wV7rRYtlfZOIKH4XqfhjSQQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/querystring-parser": "^4.2.13", - "@smithy/types": "^4.14.0", + "@smithy/querystring-parser": "^4.2.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -8023,14 +6553,14 @@ } }, "node_modules/@smithy/util-defaults-mode-browser": { - "version": "4.3.45", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.45.tgz", - "integrity": "sha512-ag9sWc6/nWZAuK3Wm9KlFJUnRkXLrXn33RFjIAmCTFThqLHY+7wCst10BGq56FxslsDrjhSie46c8OULS+BiIw==", + "version": "4.3.47", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.47.tgz", + "integrity": "sha512-zlIuXai3/SHjQUQ8y3g/woLvrH573SK2wNjcDaHu5e9VOcC0JwM1MI0Sq0GZJyN3BwSUneIhpjZ18nsiz5AtQw==", "license": "Apache-2.0", "dependencies": { - "@smithy/property-provider": "^4.2.13", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", + "@smithy/property-provider": "^4.2.14", + "@smithy/smithy-client": "^4.12.11", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -8038,17 +6568,17 @@ } }, "node_modules/@smithy/util-defaults-mode-node": { - "version": "4.2.49", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.49.tgz", - "integrity": "sha512-jlN6vHwE8gY5AfiFBavtD3QtCX2f7lM3BKkz7nFKSNfFR5nXLXLg6sqXTJEEyDwtxbztIDBQCfjsGVXlIru2lQ==", + "version": "4.2.52", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.52.tgz", + "integrity": "sha512-cQBz8g68Vnw1W2meXlkb3D/hXJU+Taiyj9P8qLJtjREEV9/Td65xi4A/H1sRQ8EIgX5qbZbvdYPKygKLholZ3w==", "license": "Apache-2.0", "dependencies": { - "@smithy/config-resolver": "^4.4.14", - "@smithy/credential-provider-imds": "^4.2.13", - "@smithy/node-config-provider": "^4.3.13", - "@smithy/property-provider": "^4.2.13", - "@smithy/smithy-client": "^4.12.9", - "@smithy/types": "^4.14.0", + "@smithy/config-resolver": "^4.4.16", + "@smithy/credential-provider-imds": "^4.2.14", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/property-provider": "^4.2.14", + "@smithy/smithy-client": "^4.12.11", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -8056,13 +6586,13 @@ } }, "node_modules/@smithy/util-endpoints": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-3.3.4.tgz", - "integrity": "sha512-BKoR/ubPp9KNKFxPpg1J28N1+bgu8NGAtJblBP7yHy8yQPBWhIAv9+l92SlQLpolGm71CVO+btB60gTgzT0wog==", + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-3.4.1.tgz", + "integrity": "sha512-wMxNDZJrgS5mQV9oxCs4TWl5767VMgOfqfZ3JHyCkMtGC2ykW9iPqMvFur695Otcc5yxLG8OKO/80tsQBxrhXg==", "license": "Apache-2.0", "dependencies": { - "@smithy/node-config-provider": "^4.3.13", - "@smithy/types": "^4.14.0", + "@smithy/node-config-provider": "^4.3.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -8082,12 +6612,12 @@ } }, "node_modules/@smithy/util-middleware": { - "version": "4.2.13", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.2.13.tgz", - "integrity": "sha512-GTooyrlmRTqvUen4eK7/K1p6kryF7bnDfq6XsAbIsf2mo51B/utaH+XThY6dKgNCWzMAaH/+OLmqaBuLhLWRow==", + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.2.14.tgz", + "integrity": "sha512-1Su2vj9RYNDEv/V+2E+jXkkwGsgR7dc4sfHn9Z7ruzQHJIEni9zzw5CauvRXlFJfmgcqYP8fWa0dkh2Q2YaQyw==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.0", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -8095,13 +6625,13 @@ } }, "node_modules/@smithy/util-retry": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-4.3.1.tgz", - "integrity": "sha512-FwmicpgWOkP5kZUjN3y+3JIom8NLGqSAJBeoIgK0rIToI817TEBHCrd0A2qGeKQlgDeP+Jzn4i0H/NLAXGy9uQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-4.3.2.tgz", + "integrity": "sha512-2+KTsJEwTi63NUv4uR9IQ+IFT1yu6Rf6JuoBK2WKaaJ/TRvOiOVGcXAsEqX/TQN2thR9yII21kPUJq1UV/WI2A==", "license": "Apache-2.0", "dependencies": { - "@smithy/service-error-classification": "^4.2.13", - "@smithy/types": "^4.14.0", + "@smithy/service-error-classification": "^4.2.14", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -8109,14 +6639,14 @@ } }, "node_modules/@smithy/util-stream": { - "version": "4.5.22", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.5.22.tgz", - "integrity": "sha512-3H8iq/0BfQjUs2/4fbHZ9aG9yNzcuZs24LPkcX1Q7Z+qpqaGM8+qbGmE8zo9m2nCRgamyvS98cHdcWvR6YUsew==", + "version": "4.5.23", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.5.23.tgz", + "integrity": "sha512-N6on1+ngJ3RznZOnDWNveIwnTSlqxNnXuNAh7ez889ZZaRdXoNRTXKgmYOLe6dB0gCmAVtuRScE1hymQFl4hpg==", "license": "Apache-2.0", "dependencies": { - "@smithy/fetch-http-handler": "^5.3.16", - "@smithy/node-http-handler": "^4.5.2", - "@smithy/types": "^4.14.0", + "@smithy/fetch-http-handler": "^5.3.17", + "@smithy/node-http-handler": "^4.5.3", + "@smithy/types": "^4.14.1", "@smithy/util-base64": "^4.3.2", "@smithy/util-buffer-from": "^4.2.2", "@smithy/util-hex-encoding": "^4.2.2", @@ -8127,21 +6657,6 @@ "node": ">=18.0.0" } }, - "node_modules/@smithy/util-stream/node_modules/@smithy/node-http-handler": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.5.2.tgz", - "integrity": "sha512-/oD7u8M0oj2ZTFw7GkuuHWpIxtWdLlnyNkbrWcyVYhd5RJNDuczdkb0wfnQICyNFrVPlr8YHOhamjNy3zidhmA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.13", - "@smithy/querystring-builder": "^4.2.13", - "@smithy/types": "^4.14.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@smithy/util-uri-escape": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-4.2.2.tgz", @@ -8168,12 +6683,12 @@ } }, "node_modules/@smithy/util-waiter": { - "version": "4.2.15", - "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-4.2.15.tgz", - "integrity": "sha512-oUt9o7n8hBv3BL56sLSneL0XeigZSuem0Hr78JaoK33D9oKieyCvVP8eTSe3j7g2mm/S1DvzxKieG7JEWNJUNg==", + "version": "4.2.16", + "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-4.2.16.tgz", + "integrity": "sha512-GtclrKoZ3Lt7jPQ7aTIYKfjY92OgceScftVnkTsG8e1KV8rkvZgN+ny6YSRhd9hxB8rZtwVbmln7NTvE5O3GmQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.0", + "@smithy/types": "^4.14.1", "tslib": "^2.6.2" }, "engines": { @@ -8192,16 +6707,6 @@ "node": ">=18.0.0" } }, - "node_modules/@so-ric/colorspace": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@so-ric/colorspace/-/colorspace-1.1.6.tgz", - "integrity": "sha512-/KiKkpHNOBgkFJwu9sh48LkHSMYGyuTcSFK/qMBdnOAlrRJzRSXAOFB5qwzaVQuDl8wAvHVMkaASQDReTahxuw==", - "license": "MIT", - "dependencies": { - "color": "^5.0.2", - "text-hex": "1.0.x" - } - }, "node_modules/@socket.io/component-emitter": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.2.tgz", @@ -8251,22 +6756,6 @@ "eslint": "^9.0.0 || ^10.0.0" } }, - "node_modules/@stylistic/eslint-plugin-js": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/@stylistic/eslint-plugin-js/-/eslint-plugin-js-4.4.1.tgz", - "integrity": "sha512-eLisyHvx7Sel8vcFZOEwDEBGmYsYM1SqDn81BWgmbqEXfXRf8oe6Rwp+ryM/8odNjlxtaaxp0Ihmt86CnLAxKg==", - "license": "MIT", - "dependencies": { - "eslint-visitor-keys": "^4.2.0", - "espree": "^10.3.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "peerDependencies": { - "eslint": ">=9.0.0" - } - }, "node_modules/@tokenizer/inflate": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.4.1.tgz", @@ -8307,16 +6796,6 @@ "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", "license": "MIT" }, - "node_modules/@tootallnate/once": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", - "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">= 10" - } - }, "node_modules/@tybys/wasm-util": { "version": "0.10.1", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", @@ -8328,25 +6807,17 @@ "tslib": "^2.4.0" } }, - "node_modules/@types/accepts": { - "version": "1.3.7", - "resolved": "https://registry.npmjs.org/@types/accepts/-/accepts-1.3.7.tgz", - "integrity": "sha512-Pay9fq2lM2wXPWbteBsRAGiWH2hig4ZE2asK+mm7kUzlxRTfL961rj89I6zV/E3PcIkDqyuBEcMxFT7rccugeQ==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/aws-lambda": { - "version": "8.10.122", - "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.122.tgz", - "integrity": "sha512-vBkIh9AY22kVOCEKo5CJlyCgmSWvasC+SWUxL/x/vOwRobMpI/HG1xp/Ae3AqmSiZeLUbOhW0FCD3ZjqqUxmXw==", + "version": "8.10.143", + "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.143.tgz", + "integrity": "sha512-u5vzlcR14ge/4pMTTMDQr3MF0wEe38B2F9o84uC4F43vN5DGTy63npRrB6jQhyt+C0lGv4ZfiRcRkqJoZuPnmg==", "license": "MIT" }, "node_modules/@types/body-parser": { "version": "1.19.6", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", "integrity": "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==", + "dev": true, "license": "MIT", "dependencies": { "@types/connect": "*", @@ -8392,24 +6863,6 @@ "@types/node": "*" } }, - "node_modules/@types/content-disposition": { - "version": "0.5.9", - "resolved": "https://registry.npmjs.org/@types/content-disposition/-/content-disposition-0.5.9.tgz", - "integrity": "sha512-8uYXI3Gw35MhiVYhG3s295oihrxRyytcRHjSjqnqZVDDy/xcGBRny7+Xj1Wgfhv5QzRtN2hB2dVRBUX9XW3UcQ==", - "license": "MIT" - }, - "node_modules/@types/cookies": { - "version": "0.9.2", - "resolved": "https://registry.npmjs.org/@types/cookies/-/cookies-0.9.2.tgz", - "integrity": "sha512-1AvkDdZM2dbyFybL4fxpuNCaWyv//0AwsuUk2DWeXyM1/5ZKm6W3z6mQi24RZ4l2ucY+bkSHzbDVpySqPGuV8A==", - "license": "MIT", - "dependencies": { - "@types/connect": "*", - "@types/express": "*", - "@types/keygrip": "*", - "@types/node": "*" - } - }, "node_modules/@types/cors": { "version": "2.8.19", "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.19.tgz", @@ -8452,24 +6905,26 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, "license": "MIT" }, "node_modules/@types/express": { - "version": "4.17.25", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", - "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.6.tgz", + "integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==", + "dev": true, "license": "MIT", "dependencies": { "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.33", - "@types/qs": "*", - "@types/serve-static": "^1" + "@types/express-serve-static-core": "^5.0.0", + "@types/serve-static": "^2" } }, "node_modules/@types/express-serve-static-core": { - "version": "4.19.8", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.8.tgz", - "integrity": "sha512-02S5fmqeoKzVZCHPZid4b8JH2eM5HzQLZWN2FohQEy/0eXTq8VXZfSN6Pcr3F6N9R/vNrj7cpgbhjie6m/1tCA==", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.1.1.tgz", + "integrity": "sha512-v4zIMr/cX7/d2BpAEX3KNKL/JrT1s43s96lLvvdTmza1oEvDudCqK9aF/djc/SWgy8Yh0h30TZx5VpzqFCxk5A==", + "dev": true, "license": "MIT", "dependencies": { "@types/node": "*", @@ -8489,46 +6944,6 @@ "@types/node": "*" } }, - "node_modules/@types/hapi__catbox": { - "version": "10.2.6", - "resolved": "https://registry.npmjs.org/@types/hapi__catbox/-/hapi__catbox-10.2.6.tgz", - "integrity": "sha512-qdMHk4fBlwRfnBBDJaoaxb+fU9Ewi2xqkXD3mNjSPl2v/G/8IJbDpVRBuIcF7oXrcE8YebU5M8cCeKh1NXEn0w==", - "license": "MIT" - }, - "node_modules/@types/hapi__hapi": { - "version": "20.0.13", - "resolved": "https://registry.npmjs.org/@types/hapi__hapi/-/hapi__hapi-20.0.13.tgz", - "integrity": "sha512-LP4IPfhIO5ZPVOrJo7H8c8Slc0WYTFAUNQX1U0LBPKyXioXhH5H2TawIgxKujIyOhbwoBbpvOsBf6o5+ToJIrQ==", - "license": "MIT", - "dependencies": { - "@hapi/boom": "^9.0.0", - "@hapi/iron": "^6.0.0", - "@hapi/podium": "^4.1.3", - "@types/hapi__catbox": "*", - "@types/hapi__mimos": "*", - "@types/hapi__shot": "*", - "@types/node": "*", - "joi": "^17.3.0" - } - }, - "node_modules/@types/hapi__mimos": { - "version": "4.1.4", - "resolved": "https://registry.npmjs.org/@types/hapi__mimos/-/hapi__mimos-4.1.4.tgz", - "integrity": "sha512-i9hvJpFYTT/qzB5xKWvDYaSXrIiNqi4ephi+5Lo6+DoQdwqPXQgmVVOZR+s3MBiHoFqsCZCX9TmVWG3HczmTEQ==", - "license": "MIT", - "dependencies": { - "@types/mime-db": "*" - } - }, - "node_modules/@types/hapi__shot": { - "version": "4.1.6", - "resolved": "https://registry.npmjs.org/@types/hapi__shot/-/hapi__shot-4.1.6.tgz", - "integrity": "sha512-h33NBjx2WyOs/9JgcFeFhkxnioYWQAZxOHdmqDuoJ1Qjxpcs+JGvSjEEoDeWfcrF+1n47kKgqph5IpfmPOnzbg==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/highlight.js": { "version": "9.12.4", "resolved": "https://registry.npmjs.org/@types/highlight.js/-/highlight.js-9.12.4.tgz", @@ -8543,16 +6958,11 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/http-assert": { - "version": "1.5.6", - "resolved": "https://registry.npmjs.org/@types/http-assert/-/http-assert-1.5.6.tgz", - "integrity": "sha512-TTEwmtjgVbYAzZYWyeHPrrtWnfVkm8tQkP8P21uQifPgMRgjrow3XDEYqucuC8SKZJT7pUnhU/JymvjggxO9vw==", - "license": "MIT" - }, "node_modules/@types/http-errors": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz", "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==", + "dev": true, "license": "MIT" }, "node_modules/@types/ioredis-mock": { @@ -8565,16 +6975,6 @@ "ioredis": ">=5" } }, - "node_modules/@types/ioredis4": { - "name": "@types/ioredis", - "version": "4.28.10", - "resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.28.10.tgz", - "integrity": "sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/jquery": { "version": "3.5.34", "resolved": "https://registry.npmjs.org/@types/jquery/-/jquery-3.5.34.tgz", @@ -8589,64 +6989,9 @@ "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, "license": "MIT" }, - "node_modules/@types/jsonwebtoken": { - "version": "8.5.9", - "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-8.5.9.tgz", - "integrity": "sha512-272FMnFGzAVMGtu9tkr29hRL6bZj4Zs1KZNeHLnKqAvp06tAIcarTMwOh8/8bz4FmKRcMxZhZNeUAQsNLoiPhg==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/keygrip": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/@types/keygrip/-/keygrip-1.0.6.tgz", - "integrity": "sha512-lZuNAY9xeJt7Bx4t4dx0rYCDqGPW8RXhQZK1td7d4H6E9zYbLoOtjBvfwdTKpsyxQI/2jv+armjX/RW+ZNpXOQ==", - "license": "MIT" - }, - "node_modules/@types/koa": { - "version": "2.14.0", - "resolved": "https://registry.npmjs.org/@types/koa/-/koa-2.14.0.tgz", - "integrity": "sha512-DTDUyznHGNHAl+wd1n0z1jxNajduyTh8R53xoewuerdBzGo6Ogj6F2299BFtrexJw4NtgjsI5SMPCmV9gZwGXA==", - "license": "MIT", - "dependencies": { - "@types/accepts": "*", - "@types/content-disposition": "*", - "@types/cookies": "*", - "@types/http-assert": "*", - "@types/http-errors": "*", - "@types/keygrip": "*", - "@types/koa-compose": "*", - "@types/node": "*" - } - }, - "node_modules/@types/koa__router": { - "version": "12.0.3", - "resolved": "https://registry.npmjs.org/@types/koa__router/-/koa__router-12.0.3.tgz", - "integrity": "sha512-5YUJVv6NwM1z7m6FuYpKfNLTZ932Z6EF6xy2BbtpJSyn13DKNQEkXVffFVSnJHxvwwWh2SAeumpjAYUELqgjyw==", - "license": "MIT", - "dependencies": { - "@types/koa": "*" - } - }, - "node_modules/@types/koa-compose": { - "version": "3.2.9", - "resolved": "https://registry.npmjs.org/@types/koa-compose/-/koa-compose-3.2.9.tgz", - "integrity": "sha512-BroAZ9FTvPiCy0Pi8tjD1OfJ7bgU1gQf0eR6e1Vm+JJATy9eKOG3hQMFtMciMawiSOVnLMdmUOC46s7HBhSTsA==", - "license": "MIT", - "dependencies": { - "@types/koa": "*" - } - }, - "node_modules/@types/long": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", - "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==", - "license": "MIT", - "optional": true - }, "node_modules/@types/memcached": { "version": "2.2.10", "resolved": "https://registry.npmjs.org/@types/memcached/-/memcached-2.2.10.tgz", @@ -8656,18 +7001,6 @@ "@types/node": "*" } }, - "node_modules/@types/mime": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", - "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", - "license": "MIT" - }, - "node_modules/@types/mime-db": { - "version": "1.43.6", - "resolved": "https://registry.npmjs.org/@types/mime-db/-/mime-db-1.43.6.tgz", - "integrity": "sha512-r2cqxAt/Eo5yWBOQie1lyM1JZFCiORa5xtLlhSZI0w8RJggBPKw8c4g/fgQCzWydaDR5bL4imnmix2d1n52iBw==", - "license": "MIT" - }, "node_modules/@types/mime-types": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-3.0.1.tgz", @@ -8683,9 +7016,9 @@ "license": "MIT" }, "node_modules/@types/mysql": { - "version": "2.15.22", - "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.22.tgz", - "integrity": "sha512-wK1pzsJVVAjYCSZWQoWHziQZbNggXFDUEIGf54g4ZM/ERuP86uGdWeKZWMYlqTPMZfHJJvLPyogXGvCOg87yLQ==", + "version": "2.15.26", + "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.26.tgz", + "integrity": "sha512-DSLCOXhkvfS5WNNPbfn2KdICAmk8lLc+/PNvnPnF7gOdMZCxopXduqv0OQ13y/yA/zXTSikZZqVgybUxOEg6YQ==", "license": "MIT", "dependencies": { "@types/node": "*" @@ -8722,9 +7055,9 @@ } }, "node_modules/@types/pg-pool": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@types/pg-pool/-/pg-pool-2.0.4.tgz", - "integrity": "sha512-qZAvkv1K3QbmHHFYSNRYPkRjOWRLBYrL4B9c+wG0GSVGBw0NtJwPcgx/DSddeDJvRGMHCEQ4VMEVfuJ/0gZ3XQ==", + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/pg-pool/-/pg-pool-2.0.6.tgz", + "integrity": "sha512-TaAUE5rq2VQYxab5Ts7WZhKNmuN78Q6PiFonTDdpbx8a1H0M1vhy3rhiMjl+e2iHmogyMw7jZF4FrE6eJUy5HQ==", "license": "MIT", "dependencies": { "@types/pg": "*" @@ -8734,12 +7067,14 @@ "version": "6.15.0", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.0.tgz", "integrity": "sha512-JawvT8iBVWpzTrz3EGw9BTQFg3BQNmwERdKE22vlTxawwtbyUSlMppvZYKLZzB5zgACXdXxbD3m1bXaMqP/9ow==", + "dev": true, "license": "MIT" }, "node_modules/@types/range-parser": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "dev": true, "license": "MIT" }, "node_modules/@types/retry": { @@ -8752,29 +7087,20 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz", "integrity": "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==", + "dev": true, "license": "MIT", "dependencies": { "@types/node": "*" } }, "node_modules/@types/serve-static": { - "version": "1.15.10", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.10.tgz", - "integrity": "sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-2.2.0.tgz", + "integrity": "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==", + "dev": true, "license": "MIT", "dependencies": { "@types/http-errors": "*", - "@types/node": "*", - "@types/send": "<1" - } - }, - "node_modules/@types/serve-static/node_modules/@types/send": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.6.tgz", - "integrity": "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==", - "license": "MIT", - "dependencies": { - "@types/mime": "^1", "@types/node": "*" } }, @@ -8800,12 +7126,6 @@ "@types/node": "*" } }, - "node_modules/@types/triple-beam": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.5.tgz", - "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==", - "license": "MIT" - }, "node_modules/@types/uuid": { "version": "10.0.0", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", @@ -8823,17 +7143,17 @@ } }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.58.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.58.1.tgz", - "integrity": "sha512-eSkwoemjo76bdXl2MYqtxg51HNwUSkWfODUOQ3PaTLZGh9uIWWFZIjyjaJnex7wXDu+TRx+ATsnSxdN9YWfRTQ==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.58.2.tgz", + "integrity": "sha512-aC2qc5thQahutKjP+cl8cgN9DWe3ZUqVko30CMSZHnFEHyhOYoZSzkGtAI2mcwZ38xeImDucI4dnqsHiOYuuCw==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.12.2", - "@typescript-eslint/scope-manager": "8.58.1", - "@typescript-eslint/type-utils": "8.58.1", - "@typescript-eslint/utils": "8.58.1", - "@typescript-eslint/visitor-keys": "8.58.1", + "@typescript-eslint/scope-manager": "8.58.2", + "@typescript-eslint/type-utils": "8.58.2", + "@typescript-eslint/utils": "8.58.2", + "@typescript-eslint/visitor-keys": "8.58.2", "ignore": "^7.0.5", "natural-compare": "^1.4.0", "ts-api-utils": "^2.5.0" @@ -8846,23 +7166,22 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "@typescript-eslint/parser": "^8.58.1", + "@typescript-eslint/parser": "^8.58.2", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/parser": { - "version": "8.58.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.58.1.tgz", - "integrity": "sha512-gGkiNMPqerb2cJSVcruigx9eHBlLG14fSdPdqMoOcBfh+vvn4iCq2C8MzUB89PrxOXk0y3GZ1yIWb9aOzL93bw==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.58.2.tgz", + "integrity": "sha512-/Zb/xaIDfxeJnvishjGdcR4jmr7S+bda8PKNhRGdljDM+elXhlvN0FyPSsMnLmJUrVG9aPO6dof80wjMawsASg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { - "@typescript-eslint/scope-manager": "8.58.1", - "@typescript-eslint/types": "8.58.1", - "@typescript-eslint/typescript-estree": "8.58.1", - "@typescript-eslint/visitor-keys": "8.58.1", + "@typescript-eslint/scope-manager": "8.58.2", + "@typescript-eslint/types": "8.58.2", + "@typescript-eslint/typescript-estree": "8.58.2", + "@typescript-eslint/visitor-keys": "8.58.2", "debug": "^4.4.3" }, "engines": { @@ -8896,14 +7215,14 @@ } }, "node_modules/@typescript-eslint/project-service": { - "version": "8.58.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.58.1.tgz", - "integrity": "sha512-gfQ8fk6cxhtptek+/8ZIqw8YrRW5048Gug8Ts5IYcMLCw18iUgrZAEY/D7s4hkI0FxEfGakKuPK/XUMPzPxi5g==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.58.2.tgz", + "integrity": "sha512-Cq6UfpZZk15+r87BkIh5rDpi38W4b+Sjnb8wQCPPDDweS/LRCFjCyViEbzHk5Ck3f2QDfgmlxqSa7S7clDtlfg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.58.1", - "@typescript-eslint/types": "^8.58.1", + "@typescript-eslint/tsconfig-utils": "^8.58.2", + "@typescript-eslint/types": "^8.58.2", "debug": "^4.4.3" }, "engines": { @@ -8936,14 +7255,14 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.58.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.58.1.tgz", - "integrity": "sha512-TPYUEqJK6avLcEjumWsIuTpuYODTTDAtoMdt8ZZa93uWMTX13Nb8L5leSje1NluammvU+oI3QRr5lLXPgihX3w==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.58.2.tgz", + "integrity": "sha512-SgmyvDPexWETQek+qzZnrG6844IaO02UVyOLhI4wpo82dpZJY9+6YZCKAMFzXb7qhx37mFK1QcPQ18tud+vo6Q==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.58.1", - "@typescript-eslint/visitor-keys": "8.58.1" + "@typescript-eslint/types": "8.58.2", + "@typescript-eslint/visitor-keys": "8.58.2" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -8954,9 +7273,9 @@ } }, "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.58.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.58.1.tgz", - "integrity": "sha512-JAr2hOIct2Q+qk3G+8YFfqkqi7sC86uNryT+2i5HzMa2MPjw4qNFvtjnw1IiA1rP7QhNKVe21mSSLaSjwA1Olw==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.58.2.tgz", + "integrity": "sha512-3SR+RukipDvkkKp/d0jP0dyzuls3DbGmwDpVEc5wqk5f38KFThakqAAO0XMirWAE+kT00oTauTbzMFGPoAzB0A==", "dev": true, "license": "MIT", "engines": { @@ -8971,15 +7290,15 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.58.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.58.1.tgz", - "integrity": "sha512-HUFxvTJVroT+0rXVJC7eD5zol6ID+Sn5npVPWoFuHGg9Ncq5Q4EYstqR+UOqaNRFXi5TYkpXXkLhoCHe3G0+7w==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.58.2.tgz", + "integrity": "sha512-Z7EloNR/B389FvabdGeTo2XMs4W9TjtPiO9DAsmT0yom0bwlPyRjkJ1uCdW1DvrrrYP50AJZ9Xc3sByZA9+dcg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.58.1", - "@typescript-eslint/typescript-estree": "8.58.1", - "@typescript-eslint/utils": "8.58.1", + "@typescript-eslint/types": "8.58.2", + "@typescript-eslint/typescript-estree": "8.58.2", + "@typescript-eslint/utils": "8.58.2", "debug": "^4.4.3", "ts-api-utils": "^2.5.0" }, @@ -9014,9 +7333,9 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "8.58.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.58.1.tgz", - "integrity": "sha512-io/dV5Aw5ezwzfPBBWLoT+5QfVtP8O7q4Kftjn5azJ88bYyp/ZMCsyW1lpKK46EXJcaYMZ1JtYj+s/7TdzmQMw==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.58.2.tgz", + "integrity": "sha512-9TukXyATBQf/Jq9AMQXfvurk+G5R2MwfqQGDR2GzGz28HvY/lXNKGhkY+6IOubwcquikWk5cjlgPvD2uAA7htQ==", "dev": true, "license": "MIT", "engines": { @@ -9028,16 +7347,16 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.58.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.58.1.tgz", - "integrity": "sha512-w4w7WR7GHOjqqPnvAYbazq+Y5oS68b9CzasGtnd6jIeOIeKUzYzupGTB2T4LTPSv4d+WPeccbxuneTFHYgAAWg==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.58.2.tgz", + "integrity": "sha512-ELGuoofuhhoCvNbQjFFiobFcGgcDCEm0ThWdmO4Z0UzLqPXS3KFvnEZ+SHewwOYHjM09tkzOWXNTv9u6Gqtyuw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/project-service": "8.58.1", - "@typescript-eslint/tsconfig-utils": "8.58.1", - "@typescript-eslint/types": "8.58.1", - "@typescript-eslint/visitor-keys": "8.58.1", + "@typescript-eslint/project-service": "8.58.2", + "@typescript-eslint/tsconfig-utils": "8.58.2", + "@typescript-eslint/types": "8.58.2", + "@typescript-eslint/visitor-keys": "8.58.2", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", @@ -9074,16 +7393,16 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "8.58.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.58.1.tgz", - "integrity": "sha512-Ln8R0tmWC7pTtLOzgJzYTXSCjJ9rDNHAqTaVONF4FEi2qwce8mD9iSOxOpLFFvWp/wBFlew0mjM1L1ihYWfBdQ==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.58.2.tgz", + "integrity": "sha512-QZfjHNEzPY8+l0+fIXMvuQ2sJlplB4zgDZvA+NmvZsZv3EQwOcc1DuIU1VJUTWZ/RKouBMhDyNaBMx4sWvrzRA==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.58.1", - "@typescript-eslint/types": "8.58.1", - "@typescript-eslint/typescript-estree": "8.58.1" + "@typescript-eslint/scope-manager": "8.58.2", + "@typescript-eslint/types": "8.58.2", + "@typescript-eslint/typescript-estree": "8.58.2" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -9098,13 +7417,13 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.58.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.58.1.tgz", - "integrity": "sha512-y+vH7QE8ycjoa0bWciFg7OpFcipUuem1ujhrdLtq1gByKwfbC7bPeKsiny9e0urg93DqwGcHey+bGRKCnF1nZQ==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.58.2.tgz", + "integrity": "sha512-f1WO2Lx8a9t8DARmcWAUPJbu0G20bJlj8L4z72K00TMeJAoyLr/tHhI/pzYBLrR4dXWkcxO1cWYZEOX8DKHTqA==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.58.1", + "@typescript-eslint/types": "8.58.2", "eslint-visitor-keys": "^5.0.0" }, "engines": { @@ -9129,9 +7448,9 @@ } }, "node_modules/@vercel/oidc": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.1.0.tgz", - "integrity": "sha512-Fw28YZpRnA3cAHHDlkt7xQHiJ0fcL+NRcIqsocZQUSmbzeIKRpwttJjik5ZGanXP+vlA4SbTg+AbA3bP363l+w==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz", + "integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==", "license": "Apache-2.0", "engines": { "node": ">= 20" @@ -9143,7 +7462,6 @@ "integrity": "sha512-x7FptB5oDruxNPDNY2+S8tCh0pcq7ymCe1gTHcsp733jYjrJl8V1gMUlVysuCD9Kz46Xz9t1akkv08dPcYDs1w==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@bcoe/v8-coverage": "^1.0.2", "@vitest/utils": "4.1.4", @@ -9299,13 +7617,6 @@ "vitest": "4.1.4" } }, - "node_modules/@vitest/ui/node_modules/fflate": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", - "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", - "dev": true, - "license": "MIT" - }, "node_modules/@vitest/utils": { "version": "4.1.4", "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.4.tgz", @@ -9585,22 +7896,38 @@ "license": "MIT" }, "node_modules/accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", "license": "MIT", "dependencies": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" }, "engines": { "node": ">= 0.6" } }, + "node_modules/accepts/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/accepts/node_modules/negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", "license": "MIT", "engines": { "node": ">= 0.6" @@ -9611,7 +7938,6 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -9619,11 +7945,10 @@ "node": ">=0.4.0" } }, - "node_modules/acorn-import-assertions": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz", - "integrity": "sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==", - "deprecated": "package has been renamed to acorn-import-attributes", + "node_modules/acorn-import-attributes": { + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", + "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", "license": "MIT", "peerDependencies": { "acorn": "^8" @@ -9646,6 +7971,7 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, "license": "MIT", "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" @@ -9675,27 +8001,13 @@ "node": ">= 8.0.0" } }, - "node_modules/aggregate-error": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", - "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", - "dev": true, - "license": "MIT", - "dependencies": { - "clean-stack": "^2.0.0", - "indent-string": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/ai": { - "version": "6.0.156", - "resolved": "https://registry.npmjs.org/ai/-/ai-6.0.156.tgz", - "integrity": "sha512-uyi/5LYbugHQxZsR2PeAFOZEL4WqKkzZw4pv0nQvvdgxgVOsM7snOmGrYkp5fShxH/vnd08SXvHCVTX7oUW7xQ==", + "version": "6.0.168", + "resolved": "https://registry.npmjs.org/ai/-/ai-6.0.168.tgz", + "integrity": "sha512-2HqCJuO+1V2aV7vfYs5LFEUfxbkGX+5oa54q/gCCTL7KLTdbxcCu5D7TdLA5kwsrs3Szgjah9q6D9tpjHM3hUQ==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/gateway": "3.0.95", + "@ai-sdk/gateway": "3.0.104", "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.23", "@opentelemetry/api": "1.9.0" @@ -9720,6 +8032,7 @@ "version": "6.14.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==", + "dev": true, "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.1", @@ -9804,12 +8117,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/any-base": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/any-base/-/any-base-1.1.0.tgz", - "integrity": "sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg==", - "license": "MIT" - }, "node_modules/anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", @@ -9835,38 +8142,12 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, - "node_modules/append-field": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", - "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==", - "license": "MIT" - }, - "node_modules/append-transform": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/append-transform/-/append-transform-2.0.0.tgz", - "integrity": "sha512-7yeyCEurROLQJFv5Xj4lEGTy0borxepjFv1g22oAdqFu//SrAlDl1O1Nxx15SH1RoliUml6p8dwJW9jvZughhg==", - "dev": true, - "license": "MIT", - "dependencies": { - "default-require-extensions": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/aproba": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.1.0.tgz", "integrity": "sha512-tLIEcj5GuR2RSTnxNKdkK0dJ/GrC7P38sUkiDmDuHfsHmbagTFAxDVIBltoklXEVIQ/f14IL8IMJ5pn9Hez1Ew==", "license": "ISC" }, - "node_modules/archy": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz", - "integrity": "sha512-Xg+9RwCg/0p32teKdGMPTPnVXKD0w3DfHnFTficozsAgsvq2XenPJq/MYpzzQ/v8zrOyJn6Ds39VA4JIDwFfqw==", - "dev": true, - "license": "MIT" - }, "node_modules/are-we-there-yet": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz", @@ -9887,92 +8168,6 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "license": "Python-2.0" }, - "node_modules/args": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/args/-/args-5.0.3.tgz", - "integrity": "sha512-h6k/zfFgusnv3i5TU08KQkVKuCPBtL/PWQbWkHUxvJrZ2nAyeaUupneemcrgn1xmqxPQsPIzwkUhOpoqPDRZuA==", - "license": "MIT", - "dependencies": { - "camelcase": "5.0.0", - "chalk": "2.4.2", - "leven": "2.1.0", - "mri": "1.1.4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/args/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "license": "MIT", - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/args/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/args/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "license": "MIT", - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/args/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "license": "MIT" - }, - "node_modules/args/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "license": "MIT", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/args/node_modules/has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/args/node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "license": "MIT", - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/array-buffer-byte-length": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.2.tgz", @@ -9989,12 +8184,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", - "license": "MIT" - }, "node_modules/array-union": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", @@ -10048,25 +8237,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/arrify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/asn1": { - "version": "0.2.6", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", - "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", - "license": "MIT", - "dependencies": { - "safer-buffer": "~2.1.0" - } - }, "node_modules/assertion-error": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", @@ -10103,16 +8273,6 @@ "node": ">= 0.4" } }, - "node_modules/async-retry": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", - "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", - "license": "MIT", - "optional": true, - "dependencies": { - "retry": "0.13.1" - } - }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -10163,19 +8323,19 @@ "fastq": "^1.17.1" } }, - "node_modules/await-to-js": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/await-to-js/-/await-to-js-3.0.0.tgz", - "integrity": "sha512-zJAaP9zxTcvTHRlejau3ZOY4V7SRpiByf3/dxx2uyKxxor19tpmpV2QRsTKikckwhaPmr2dVpxxMr7jOCYVp5g==", + "node_modules/aws-ssl-profiles": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/aws-ssl-profiles/-/aws-ssl-profiles-1.1.2.tgz", + "integrity": "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==", "license": "MIT", "engines": { - "node": ">=6.0.0" + "node": ">= 6.0.0" } }, "node_modules/axios": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.15.0.tgz", - "integrity": "sha512-wWyJDlAatxk30ZJer+GeCWS209sA42X+N5jU2jy6oHTp7ufw8uzUTVFBX9+wTfAlhiJXGS0Bq7X6efruWjuK9Q==", + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.15.1.tgz", + "integrity": "sha512-WOG+Jj8ZOvR0a3rAn+Tuf1UQJRxw5venr6DgdbJzngJE3qG7X0kL83CZGpdHMxEm+ZK3seAbvFsw4FfOfP9vxg==", "license": "MIT", "dependencies": { "follow-redirects": "^1.15.11", @@ -10223,9 +8383,9 @@ } }, "node_modules/baseline-browser-mapping": { - "version": "2.10.17", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.17.tgz", - "integrity": "sha512-HdrkN8eVG2CXxeifv/VdJ4A4RSra1DTW8dc/hdxzhGHN8QePs6gKaWM9pHPcpCoxYZJuOZ8drHmbdpLHjCYjLA==", + "version": "2.10.20", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.20.tgz", + "integrity": "sha512-1AaXxEPfXT+GvTBJFuy4yXVHWJBXa4OdbIebGN/wX5DlsIkU0+wzGnd2lOzokSk51d5LUmqjgBLRLlypLUqInQ==", "dev": true, "license": "Apache-2.0", "bin": { @@ -10239,6 +8399,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "5.1.2" @@ -10251,6 +8412,7 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true, "license": "MIT" }, "node_modules/bcrypt": { @@ -10267,25 +8429,10 @@ "node": ">= 10.0.0" } }, - "node_modules/bcrypt-pbkdf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", - "license": "BSD-3-Clause", - "dependencies": { - "tweetnacl": "^0.14.3" - } - }, - "node_modules/bcrypt-pbkdf/node_modules/tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==", - "license": "Unlicense" - }, "node_modules/better-sqlite3": { - "version": "12.8.0", - "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.8.0.tgz", - "integrity": "sha512-RxD2Vd96sQDjQr20kdP+F+dK/1OUNiVOl200vKBZY8u0vTwysfolF6Hq+3ZK2+h8My9YvZhHsF+RSGZW2VYrPQ==", + "version": "12.9.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.9.0.tgz", + "integrity": "sha512-wqUv4Gm3toFpHDQmaKD4QhZm3g1DjUBI0yzS4UBl6lElUmXFYdTQmmEDpAFa5o8FiFiymURypEnfVHzILKaxqQ==", "hasInstallScript": true, "license": "MIT", "dependencies": { @@ -10384,61 +8531,52 @@ "ieee754": "^1.1.13" } }, - "node_modules/bmp-js": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", - "integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==", - "license": "MIT" - }, - "node_modules/bmp-ts": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/bmp-ts/-/bmp-ts-1.0.9.tgz", - "integrity": "sha512-cTEHk2jLrPyi+12M3dhpEbnnPOsaZuq7C45ylbbQIiWgDFZq4UVYPEY5mlqjvsj/6gJv9qX5sa+ebDzLXT28Vw==", - "license": "MIT" - }, "node_modules/body-parser": { - "version": "1.20.4", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.4.tgz", - "integrity": "sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==", + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz", + "integrity": "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==", "license": "MIT", "dependencies": { - "bytes": "~3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "~1.2.0", - "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", - "on-finished": "~2.4.1", - "qs": "~6.14.0", - "raw-body": "~2.5.3", - "type-is": "~1.6.18", - "unpipe": "~1.0.0" + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.3", + "http-errors": "^2.0.0", + "iconv-lite": "^0.7.0", + "on-finished": "^2.4.1", + "qs": "^6.14.1", + "raw-body": "^3.0.1", + "type-is": "^2.0.1" }, "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/body-parser/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { - "ms": "2.0.0" + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, - "node_modules/body-parser/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, "node_modules/boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "dev": true, "license": "ISC" }, "node_modules/bowser": { @@ -10513,7 +8651,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", @@ -10571,17 +8708,9 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true, "license": "MIT" }, - "node_modules/buildcheck": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz", - "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==", - "optional": true, - "engines": { - "node": ">=10.0.0" - } - }, "node_modules/bundle-name": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", @@ -10617,22 +8746,6 @@ "node": ">= 0.8" } }, - "node_modules/caching-transform": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-4.0.0.tgz", - "integrity": "sha512-kpqOvwXnjjN44D89K5ccQC+RUrsy7jB/XLlRrx0D7/2HNcTPqzsb6XgYoErwko6QsV184CA2YgS1fxDiiDZMWA==", - "dev": true, - "license": "MIT", - "dependencies": { - "hasha": "^5.0.0", - "make-dir": "^3.0.0", - "package-hash": "^4.0.0", - "write-file-atomic": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/call-bind": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.9.tgz", @@ -10684,6 +8797,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -10701,18 +8815,18 @@ } }, "node_modules/camelcase": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.0.0.tgz", - "integrity": "sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==", + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", "license": "MIT", "engines": { "node": ">=6" } }, "node_modules/caniuse-lite": { - "version": "1.0.30001787", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001787.tgz", - "integrity": "sha512-mNcrMN9KeI68u7muanUpEejSLghOKlVhRqS/Za2IeyGllJ9I9otGpR9g3nsw7n4W378TE/LyIteA0+/FOZm4Kg==", + "version": "1.0.30001788", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001788.tgz", + "integrity": "sha512-6q8HFp+lOQtcf7wBK+uEenxymVWkGKkjFpCvw5W25cmMwEDU45p1xQFBQv8JDlMMry7eNxyBaR+qxgmTUZkIRQ==", "dev": true, "funding": [ { @@ -10735,7 +8849,6 @@ "resolved": "https://registry.npmjs.org/chai/-/chai-4.5.0.tgz", "integrity": "sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==", "license": "MIT", - "peer": true, "dependencies": { "assertion-error": "^1.1.0", "check-error": "^1.0.3", @@ -10765,6 +8878,7 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.1.0", @@ -10875,16 +8989,6 @@ "node": ">= 10.0" } }, - "node_modules/clean-stack": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", - "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -10938,19 +9042,6 @@ "node": ">=0.10.0" } }, - "node_modules/color": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/color/-/color-5.0.3.tgz", - "integrity": "sha512-ezmVcLR3xAVp8kYOm4GS45ZLLgIE6SPAFoduLr6hTDajwb3KZ2F46gulK3XpcwRFb5KKGCSezCBAY4Dw4HsyXA==", - "license": "MIT", - "dependencies": { - "color-convert": "^3.1.3", - "color-string": "^2.1.3" - }, - "engines": { - "node": ">=18" - } - }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -10969,27 +9060,6 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "license": "MIT" }, - "node_modules/color-string": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-2.1.4.tgz", - "integrity": "sha512-Bb6Cq8oq0IjDOe8wJmi4JeNn763Xs9cfrBcaylK1tPypWzyoy2G3l90v9k64kjphl/ZJjPIShFztenRomi8WTg==", - "license": "MIT", - "dependencies": { - "color-name": "^2.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/color-string/node_modules/color-name": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-2.1.0.tgz", - "integrity": "sha512-1bPaDNFm0axzE4MEAzKPuqKWeRaT43U/hyxKPBdqTfmPF+d6n7FSoTFxLVULUJOmiLp01KjhIPPH+HrXZJN4Rg==", - "license": "MIT", - "engines": { - "node": ">=12.20" - } - }, "node_modules/color-support": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", @@ -10999,27 +9069,6 @@ "color-support": "bin.js" } }, - "node_modules/color/node_modules/color-convert": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-3.1.3.tgz", - "integrity": "sha512-fasDH2ont2GqF5HpyO4w0+BcewlhHEZOFn9c1ckZdHpJ56Qb7MHhH/IcJZbBGgvdtwdwNbLvxiBEdg336iA9Sg==", - "license": "MIT", - "dependencies": { - "color-name": "^2.0.0" - }, - "engines": { - "node": ">=14.6" - } - }, - "node_modules/color/node_modules/color-name": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-2.1.0.tgz", - "integrity": "sha512-1bPaDNFm0axzE4MEAzKPuqKWeRaT43U/hyxKPBdqTfmPF+d6n7FSoTFxLVULUJOmiLp01KjhIPPH+HrXZJN4Rg==", - "license": "MIT", - "engines": { - "node": ">=12.20" - } - }, "node_modules/colorette": { "version": "2.0.20", "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", @@ -11052,19 +9101,6 @@ "resolved": "tools/comment-parser", "link": true }, - "node_modules/commondir": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", - "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", - "dev": true, - "license": "MIT" - }, - "node_modules/composite-error": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/composite-error/-/composite-error-1.0.2.tgz", - "integrity": "sha512-kr6tZNUb15tHkSGhS6kNxxLHpgYguU6r5F+bUXcxbNYkLGIPX/Z2KKyXgli5t83FjGkBJ+GrludBoj3O8E/1Hw==", - "license": "MIT" - }, "node_modules/compressible": { "version": "2.0.18", "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", @@ -11116,21 +9152,6 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "license": "MIT" }, - "node_modules/concat-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", - "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", - "engines": [ - "node >= 6.0" - ], - "license": "MIT", - "dependencies": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.0.2", - "typedarray": "^0.0.6" - } - }, "node_modules/concurrently": { "version": "8.2.2", "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-8.2.2.tgz", @@ -11175,49 +9196,23 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, - "node_modules/configstore": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz", - "integrity": "sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA==", - "license": "BSD-2-Clause", - "optional": true, - "dependencies": { - "dot-prop": "^5.2.0", - "graceful-fs": "^4.1.2", - "make-dir": "^3.0.0", - "unique-string": "^2.0.0", - "write-file-atomic": "^3.0.0", - "xdg-basedir": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/console-control-strings": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", "license": "ISC" }, - "node_modules/console-table-printer": { - "version": "2.15.0", - "resolved": "https://registry.npmjs.org/console-table-printer/-/console-table-printer-2.15.0.tgz", - "integrity": "sha512-SrhBq4hYVjLCkBVOWaTzceJalvn5K1Zq5aQA6wXC/cYjI3frKWNPEMK3sZsJfNNQApvCQmgBcc13ZKmFj8qExw==", - "license": "MIT", - "dependencies": { - "simple-wcswidth": "^1.1.2" - } - }, "node_modules/content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", "license": "MIT", - "dependencies": { - "safe-buffer": "5.2.1" - }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/content-type": { @@ -11248,15 +9243,6 @@ "dev": true, "license": "MIT" }, - "node_modules/convertapi": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/convertapi/-/convertapi-1.15.0.tgz", - "integrity": "sha512-wu1pJ27SuIc/mNlbjs8lP1hAsEN16AmKzMZNo9Qx/Z3CrH1ozGQYF2jGXaceXWSRNvKFoCuF0m5ek+vz8Nswrw==", - "license": "MIT", - "dependencies": { - "axios": "^1.6.2" - } - }, "node_modules/cookie": { "version": "0.7.2", "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", @@ -11312,20 +9298,6 @@ "node": ">= 0.4.0" } }, - "node_modules/cpu-features": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.10.tgz", - "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==", - "hasInstallScript": true, - "optional": true, - "dependencies": { - "buildcheck": "~0.0.6", - "nan": "^2.19.0" - }, - "engines": { - "node": ">=10.0.0" - } - }, "node_modules/cross-fetch": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.2.0.tgz", @@ -11339,6 +9311,7 @@ "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, "license": "MIT", "dependencies": { "path-key": "^3.1.0", @@ -11349,16 +9322,6 @@ "node": ">= 8" } }, - "node_modules/crypto-random-string": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-2.0.0.tgz", - "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">=8" - } - }, "node_modules/css-select": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.3.0.tgz", @@ -11394,6 +9357,7 @@ "version": "6.2.2", "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">= 6" @@ -11402,183 +9366,6 @@ "url": "https://github.com/sponsors/fb55" } }, - "node_modules/csso": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/csso/-/csso-5.0.5.tgz", - "integrity": "sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ==", - "license": "MIT", - "dependencies": { - "css-tree": "~2.2.0" - }, - "engines": { - "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0", - "npm": ">=7.0.0" - } - }, - "node_modules/csso/node_modules/css-tree": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.2.1.tgz", - "integrity": "sha512-OA0mILzGc1kCOCSJerOeqDxDQ4HOh+G8NbOJFOTgOCzpw7fCBubk0fEyxp8AgOL/jvLgYA/uV0cMbe43ElF1JA==", - "license": "MIT", - "dependencies": { - "mdn-data": "2.0.28", - "source-map-js": "^1.0.1" - }, - "engines": { - "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0", - "npm": ">=7.0.0" - } - }, - "node_modules/csso/node_modules/mdn-data": { - "version": "2.0.28", - "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.28.tgz", - "integrity": "sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g==", - "license": "CC0-1.0" - }, - "node_modules/cssstyle": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.6.0.tgz", - "integrity": "sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg==", - "license": "MIT", - "dependencies": { - "@asamuzakjp/css-color": "^3.2.0", - "rrweb-cssom": "^0.8.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/cssstyle/node_modules/@asamuzakjp/css-color": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-3.2.0.tgz", - "integrity": "sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw==", - "license": "MIT", - "dependencies": { - "@csstools/css-calc": "^2.1.3", - "@csstools/css-color-parser": "^3.0.9", - "@csstools/css-parser-algorithms": "^3.0.4", - "@csstools/css-tokenizer": "^3.0.3", - "lru-cache": "^10.4.3" - } - }, - "node_modules/cssstyle/node_modules/@csstools/color-helpers": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-5.1.0.tgz", - "integrity": "sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/csstools" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/csstools" - } - ], - "license": "MIT-0", - "engines": { - "node": ">=18" - } - }, - "node_modules/cssstyle/node_modules/@csstools/css-calc": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-2.1.4.tgz", - "integrity": "sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/csstools" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/csstools" - } - ], - "license": "MIT", - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@csstools/css-parser-algorithms": "^3.0.5", - "@csstools/css-tokenizer": "^3.0.4" - } - }, - "node_modules/cssstyle/node_modules/@csstools/css-color-parser": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-3.1.0.tgz", - "integrity": "sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/csstools" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/csstools" - } - ], - "license": "MIT", - "dependencies": { - "@csstools/color-helpers": "^5.1.0", - "@csstools/css-calc": "^2.1.4" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@csstools/css-parser-algorithms": "^3.0.5", - "@csstools/css-tokenizer": "^3.0.4" - } - }, - "node_modules/cssstyle/node_modules/@csstools/css-parser-algorithms": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.5.tgz", - "integrity": "sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/csstools" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/csstools" - } - ], - "license": "MIT", - "peer": true, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@csstools/css-tokenizer": "^3.0.4" - } - }, - "node_modules/cssstyle/node_modules/@csstools/css-tokenizer": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-3.0.4.tgz", - "integrity": "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/csstools" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/csstools" - } - ], - "license": "MIT", - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/cssstyle/node_modules/lru-cache": { - "version": "10.4.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", - "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", - "license": "ISC" - }, "node_modules/data-uri-to-buffer": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", @@ -11602,6 +9389,44 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" } }, + "node_modules/data-urls/node_modules/tr46": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-6.0.0.tgz", + "integrity": "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==", + "dev": true, + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/data-urls/node_modules/webidl-conversions": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-8.0.1.tgz", + "integrity": "sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=20" + } + }, + "node_modules/data-urls/node_modules/whatwg-url": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-16.0.1.tgz", + "integrity": "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@exodus/bytes": "^1.11.0", + "tr46": "^6.0.0", + "webidl-conversions": "^8.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, "node_modules/data-view-buffer": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/data-view-buffer/-/data-view-buffer-1.0.2.tgz", @@ -11702,33 +9527,6 @@ "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==", "license": "MIT" }, - "node_modules/decode-bmp": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/decode-bmp/-/decode-bmp-0.2.1.tgz", - "integrity": "sha512-NiOaGe+GN0KJqi2STf24hfMkFitDUaIoUU3eKvP/wAbLe8o6FuW5n/x7MHPR0HKvBokp6MQY/j7w8lewEeVCIA==", - "license": "MIT", - "dependencies": { - "@canvas/image-data": "^1.0.0", - "to-data-view": "^1.1.0" - }, - "engines": { - "node": ">=8.6.0" - } - }, - "node_modules/decode-ico": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/decode-ico/-/decode-ico-0.4.1.tgz", - "integrity": "sha512-69NZfbKIzux1vBOd31al3XnMnH+2mqDhEgLdpygErm4d60N+UwA5Sq5WFjmEDQzumgB9fElojGwWG0vybVfFmA==", - "license": "MIT", - "dependencies": { - "@canvas/image-data": "^1.0.0", - "decode-bmp": "^0.2.0", - "to-data-view": "^1.1.0" - }, - "engines": { - "node": ">=8.6" - } - }, "node_modules/decompress-response": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", @@ -11783,6 +9581,7 @@ "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, "license": "MIT" }, "node_modules/default-browser": { @@ -11813,22 +9612,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/default-require-extensions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-3.0.1.tgz", - "integrity": "sha512-eXTJmRbm2TIt9MgWTsOH1wEuhew6XGZcMeGKCtLedIg/NCsg1iBePXkceTdK4Fii7pzmN9tGsZhKzZ4h7O/fxw==", - "dev": true, - "license": "MIT", - "dependencies": { - "strip-bom": "^4.0.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/define-data-property": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", @@ -11917,16 +9700,6 @@ "node": ">=6" } }, - "node_modules/destroy": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", - "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", - "license": "MIT", - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, "node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", @@ -11945,12 +9718,6 @@ "node": ">=0.3.1" } }, - "node_modules/diff-match-patch": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/diff-match-patch/-/diff-match-patch-1.0.5.tgz", - "integrity": "sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==", - "license": "Apache-2.0" - }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -11964,28 +9731,10 @@ "node": ">=8" } }, - "node_modules/dns2": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/dns2/-/dns2-2.1.0.tgz", - "integrity": "sha512-m27K11aQalRbmUs7RLaz6aPyceLjAoqjPRNTdE7qUouQpl+PC8Bi67O+i9SuJUPbQC8dxFrczAxfmTPuTKHNkw==", - "license": "MIT" - }, "node_modules/docs": { "resolved": "src/docs", "link": true }, - "node_modules/doctrine": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", - "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", - "license": "Apache-2.0", - "dependencies": { - "esutils": "^2.0.2" - }, - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/dom-converter": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/dom-converter/-/dom-converter-0.2.0.tgz", @@ -12011,20 +9760,11 @@ "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" } }, - "node_modules/dom-serializer/node_modules/entities": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", - "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==", - "dev": true, - "license": "BSD-2-Clause", - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, "node_modules/domelementtype": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "dev": true, "funding": [ { "type": "github", @@ -12075,19 +9815,6 @@ "tslib": "^2.0.3" } }, - "node_modules/dot-prop": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz", - "integrity": "sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q==", - "license": "MIT", - "optional": true, - "dependencies": { - "is-obj": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/dotenv": { "version": "16.6.1", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", @@ -12115,19 +9842,6 @@ "node": ">= 0.4" } }, - "node_modules/duplexify": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", - "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", - "license": "MIT", - "optional": true, - "dependencies": { - "end-of-stream": "^1.4.1", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1", - "stream-shift": "^1.0.2" - } - }, "node_modules/dynalite": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/dynalite/-/dynalite-4.0.0.tgz", @@ -12167,9 +9881,9 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.5.334", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.334.tgz", - "integrity": "sha512-mgjZAz7Jyx1SRCwEpy9wefDS7GvNPazLthHg8eQMJ76wBdGQQDW33TCrUTvQ4wzpmOrv2zrFoD3oNufMdyMpog==", + "version": "1.5.340", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.340.tgz", + "integrity": "sha512-908qahOGocRMinT2nM3ajCEM99H4iPdv84eagPP3FfZy/1ZGeOy2CZYzjhms81ckOPCXPlW7LkY4XpxD8r1DrA==", "dev": true, "license": "ISC" }, @@ -12179,12 +9893,6 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "license": "MIT" }, - "node_modules/enabled": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", - "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==", - "license": "MIT" - }, "node_modules/encodeurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", @@ -12224,57 +9932,6 @@ "node": ">=10.2.0" } }, - "node_modules/engine.io-client": { - "version": "6.6.4", - "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.6.4.tgz", - "integrity": "sha512-+kjUJnZGwzewFDw951CDWcwj35vMNf2fcj7xQWOctq1F2i1jkDdVvdFG9kM/BEChymCH36KgjnW0NsL58JYRxw==", - "license": "MIT", - "dependencies": { - "@socket.io/component-emitter": "~3.1.0", - "debug": "~4.4.1", - "engine.io-parser": "~5.2.1", - "ws": "~8.18.3", - "xmlhttprequest-ssl": "~2.1.1" - } - }, - "node_modules/engine.io-client/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/engine.io-client/node_modules/ws": { - "version": "8.18.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", - "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", - "license": "MIT", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, "node_modules/engine.io-parser": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.2.3.tgz", @@ -12284,6 +9941,19 @@ "node": ">=10.0.0" } }, + "node_modules/engine.io/node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/engine.io/node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -12301,6 +9971,15 @@ } } }, + "node_modules/engine.io/node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/engine.io/node_modules/ws": { "version": "8.18.3", "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", @@ -12336,52 +10015,12 @@ "node": ">=10.13.0" } }, - "node_modules/enquirer": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.4.1.tgz", - "integrity": "sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ==", - "license": "MIT", - "dependencies": { - "ansi-colors": "^4.1.1", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8.6" - } - }, - "node_modules/enquirer/node_modules/ansi-colors": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", - "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/ent": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.2.tgz", - "integrity": "sha512-kKvD1tO6BM+oK9HzCPpUdRb4vKFQY/FPTFmurMvh6LlN68VMrdj77w8yp51/kDbpkFOS9J8w5W6zIzgM2H8/hw==", - "license": "MIT", - "optional": true, - "dependencies": { - "call-bound": "^1.0.3", - "es-errors": "^1.3.0", - "punycode": "^1.4.1", - "safe-regex-test": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/entities": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", - "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", + "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==", + "dev": true, "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, "funding": { "url": "https://github.com/fb55/entities?sponsor=1" } @@ -12562,13 +10201,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/es6-error": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", - "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", - "dev": true, - "license": "MIT" - }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -12588,6 +10220,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, "license": "MIT", "engines": { "node": ">=10" @@ -12600,8 +10233,8 @@ "version": "9.39.4", "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.4.tgz", "integrity": "sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==", + "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -12656,6 +10289,53 @@ } } }, + "node_modules/eslint-config-prettier": { + "version": "10.1.8", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-10.1.8.tgz", + "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", + "dev": true, + "license": "MIT", + "bin": { + "eslint-config-prettier": "bin/cli.js" + }, + "funding": { + "url": "https://opencollective.com/eslint-config-prettier" + }, + "peerDependencies": { + "eslint": ">=7.0.0" + } + }, + "node_modules/eslint-plugin-prettier": { + "version": "5.5.5", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.5.5.tgz", + "integrity": "sha512-hscXkbqUZ2sPithAuLm5MXL+Wph+U7wHngPBv9OMWwlP8iaflyxpjTYZkmdgB4/vPIhemRlBEoLrH7UC1n7aUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "prettier-linter-helpers": "^1.0.1", + "synckit": "^0.11.12" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint-plugin-prettier" + }, + "peerDependencies": { + "@types/eslint": ">=8.0.0", + "eslint": ">=8.0.0", + "eslint-config-prettier": ">= 7.0.0 <10.0.0 || >=10.1.0", + "prettier": ">=3.0.0" + }, + "peerDependenciesMeta": { + "@types/eslint": { + "optional": true + }, + "eslint-config-prettier": { + "optional": true + } + } + }, "node_modules/eslint-rule-composer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/eslint-rule-composer/-/eslint-rule-composer-0.3.0.tgz", @@ -12670,6 +10350,7 @@ "version": "8.4.0", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", + "dev": true, "license": "BSD-2-Clause", "dependencies": { "esrecurse": "^4.3.0", @@ -12686,6 +10367,7 @@ "version": "4.2.1", "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, "license": "Apache-2.0", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -12698,12 +10380,14 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, "license": "MIT" }, "node_modules/eslint/node_modules/brace-expansion": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.13.tgz", - "integrity": "sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w==", + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", + "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", + "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -12714,6 +10398,7 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, "license": "MIT", "engines": { "node": ">= 4" @@ -12723,6 +10408,7 @@ "version": "3.1.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -12731,19 +10417,11 @@ "node": "*" } }, - "node_modules/esm": { - "version": "3.2.25", - "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", - "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/espree": { "version": "10.4.0", "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "dev": true, "license": "BSD-2-Clause", "dependencies": { "acorn": "^8.15.0", @@ -12774,6 +10452,7 @@ "version": "1.7.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, "license": "BSD-3-Clause", "dependencies": { "estraverse": "^5.1.0" @@ -12786,6 +10465,7 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, "license": "BSD-2-Clause", "dependencies": { "estraverse": "^5.2.0" @@ -12798,6 +10478,7 @@ "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=4.0" @@ -12817,6 +10498,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=0.10.0" @@ -12858,19 +10540,14 @@ } }, "node_modules/eventsource-parser": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz", - "integrity": "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==", + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.8.tgz", + "integrity": "sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ==", "license": "MIT", "engines": { "node": ">=18.0.0" } }, - "node_modules/exif-parser": { - "version": "0.1.12", - "resolved": "https://registry.npmjs.org/exif-parser/-/exif-parser-0.1.12.tgz", - "integrity": "sha512-c2bQfLNbMzLPmzQuOr8fy0csy84WmwnER81W88DzTp9CYNPJ6yzOj2EZAh9pywYpqHnshVLHQJ8WzldAyfY+Iw==" - }, "node_modules/expand-template": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", @@ -12891,77 +10568,89 @@ } }, "node_modules/express": { - "version": "4.22.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", - "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", "dependencies": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "~1.20.3", - "content-disposition": "~0.5.4", - "content-type": "~1.0.4", - "cookie": "~0.7.1", - "cookie-signature": "~1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "~1.3.1", - "fresh": "~0.5.2", - "http-errors": "~2.0.0", - "merge-descriptors": "1.0.3", - "methods": "~1.1.2", - "on-finished": "~2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "~0.1.12", - "proxy-addr": "~2.0.7", - "qs": "~6.14.0", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "~0.19.0", - "serve-static": "~1.16.2", - "setprototypeof": "1.2.0", - "statuses": "~2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" }, "engines": { - "node": ">= 0.10.0" + "node": ">= 18" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/express" } }, - "node_modules/express-xml-bodyparser": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/express-xml-bodyparser/-/express-xml-bodyparser-0.4.1.tgz", - "integrity": "sha512-PlojEEQXdwc68ofPiAanknPf4QBTrFWXPZ+5jDhfrXP/CdLaqEQxQuuzrCqnvy1kETciTxz6OFnDZW/rIxtmlQ==", + "node_modules/express/node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", "license": "MIT", - "dependencies": { - "xml2js": "^0.6.2" - }, "engines": { - "node": ">=18.0" + "node": ">=6.6.0" } }, "node_modules/express/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { - "ms": "2.0.0" + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, - "node_modules/express/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" + "node_modules/express/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } }, "node_modules/extend": { "version": "3.0.2", @@ -12981,6 +10670,13 @@ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "license": "MIT" }, + "node_modules/fast-diff": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.3.0.tgz", + "integrity": "sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/fast-glob": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", @@ -13015,6 +10711,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, "license": "MIT" }, "node_modules/fast-json-stringify": { @@ -13067,6 +10764,7 @@ "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, "license": "MIT" }, "node_modules/fast-querystring": { @@ -13078,13 +10776,6 @@ "fast-decode-uri-component": "^1.0.1" } }, - "node_modules/fast-text-encoding": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.6.tgz", - "integrity": "sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w==", - "license": "Apache-2.0", - "optional": true - }, "node_modules/fast-uri": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", @@ -13102,9 +10793,9 @@ "license": "BSD-3-Clause" }, "node_modules/fast-xml-builder": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.4.tgz", - "integrity": "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.5.tgz", + "integrity": "sha512-4TJn/8FKLeslLAH3dnohXqE3QSoxkhvaMzepOIZytwJXZO69Bfz0HBdDHzOTOon6G59Zrk6VQ2bEiv1t61rfkA==", "funding": [ { "type": "github", @@ -13117,9 +10808,9 @@ } }, "node_modules/fast-xml-parser": { - "version": "5.5.8", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.5.8.tgz", - "integrity": "sha512-Z7Fh2nVQSb2d+poDViM063ix2ZGt9jmY1nWhPfHBOK2Hgnb/OW3P4Et3P/81SEej0J7QbWtJqxO05h8QYfK7LQ==", + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.7.2.tgz", + "integrity": "sha512-P7oW7tLbYnhOLQk/Gv7cZgzgMPP/XN03K02/Jy6Y/NHzyIAIpxuZIM/YqAkfiXFPxA2CTm7NtCijK9EDu09u2w==", "funding": [ { "type": "github", @@ -13128,9 +10819,10 @@ ], "license": "MIT", "dependencies": { - "fast-xml-builder": "^1.1.4", - "path-expression-matcher": "^1.2.0", - "strnum": "^2.2.0" + "@nodable/entities": "^2.1.0", + "fast-xml-builder": "^1.1.5", + "path-expression-matcher": "^1.5.0", + "strnum": "^2.2.3" }, "bin": { "fxparser": "src/cli/cli.js" @@ -13147,9 +10839,9 @@ } }, "node_modules/fastify": { - "version": "5.8.4", - "resolved": "https://registry.npmjs.org/fastify/-/fastify-5.8.4.tgz", - "integrity": "sha512-sa42J1xylbBAYUWALSBoyXKPDUvM3OoNOibIefA+Oha57FryXKKCZarA1iDntOCWp3O35voZLuDg2mdODXtPzQ==", + "version": "5.8.5", + "resolved": "https://registry.npmjs.org/fastify/-/fastify-5.8.5.tgz", + "integrity": "sha512-Yqptv59pQzPgQUSIm87hMqHJmdkb1+GPxdE6vW6FRyVE9G86mt7rOghitiU4JHRaTyDUk9pfeKmDeu70lAwM4Q==", "funding": [ { "type": "github", @@ -13226,33 +10918,6 @@ "node": ">=22.5.0" } }, - "node_modules/fauxqs/node_modules/@smithy/node-http-handler": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.5.2.tgz", - "integrity": "sha512-/oD7u8M0oj2ZTFw7GkuuHWpIxtWdLlnyNkbrWcyVYhd5RJNDuczdkb0wfnQICyNFrVPlr8YHOhamjNy3zidhmA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.13", - "@smithy/querystring-builder": "^4.2.13", - "@smithy/types": "^4.14.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/faye-websocket": { - "version": "0.11.4", - "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.4.tgz", - "integrity": "sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==", - "license": "Apache-2.0", - "dependencies": { - "websocket-driver": ">=0.5.1" - }, - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -13271,18 +10936,11 @@ } } }, - "node_modules/fecha": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", - "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==", - "license": "MIT" - }, "node_modules/fengari": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/fengari/-/fengari-0.1.5.tgz", "integrity": "sha512-0DS4Nn4rV8qyFlQCpKK8brT61EUtswynrpfFTcgLErcilBIBskSMQ86fO2WVuybr14ywyKdRjv91FiRZwnEuvQ==", "license": "MIT", - "peer": true, "dependencies": { "readline-sync": "^1.4.10", "sprintf-js": "^1.1.3", @@ -13322,15 +10980,17 @@ } }, "node_modules/fflate": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.1.tgz", - "integrity": "sha512-/exOvEuc+/iaUm105QIiOt4LpBdMTWsXxqR0HDF35vx3fmaKzw7354gTilCh5rkzEt8WYyG//ku3h3nRmd7CHQ==", + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", + "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", + "dev": true, "license": "MIT" }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", + "dev": true, "license": "MIT", "dependencies": { "flat-cache": "^4.0.0" @@ -13339,15 +10999,6 @@ "node": ">=16.0.0" } }, - "node_modules/file-stream-rotator": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/file-stream-rotator/-/file-stream-rotator-0.6.1.tgz", - "integrity": "sha512-u+dBid4PvZw17PmDeRcNOtCP9CCK/9lRN2w+r1xIS7yOL9JFrIBKTvrYsxT4P0pGtThYTn++QS5ChHaUov3+zQ==", - "license": "MIT", - "dependencies": { - "moment": "^2.29.1" - } - }, "node_modules/file-type": { "version": "21.3.3", "resolved": "https://registry.npmjs.org/file-type/-/file-type-21.3.3.tgz", @@ -13376,19 +11027,6 @@ "resolved": "tools/file-walker", "link": true }, - "node_modules/fill-keys": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/fill-keys/-/fill-keys-1.0.2.tgz", - "integrity": "sha512-tcgI872xXjwFF4xgQmLxi76GnwJG3g/3isB1l4/G5Z4zrbddGpBjqZCO9oEAcB5wX0Hj/5iQB3toxfO7in1hHA==", - "license": "MIT", - "dependencies": { - "is-object": "~1.0.1", - "merge-descriptors": "~1.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -13402,54 +11040,41 @@ } }, "node_modules/finalhandler": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", - "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", "license": "MIT", "dependencies": { - "debug": "2.6.9", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "on-finished": "~2.4.1", - "parseurl": "~1.3.3", - "statuses": "~2.0.2", - "unpipe": "~1.0.0" + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" }, "engines": { - "node": ">= 0.8" + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/finalhandler/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/finalhandler/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, - "node_modules/find-cache-dir": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz", - "integrity": "sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==", - "dev": true, - "license": "MIT", - "dependencies": { - "commondir": "^1.0.1", - "make-dir": "^3.0.2", - "pkg-dir": "^4.1.0" + "ms": "^2.1.3" }, "engines": { - "node": ">=8" + "node": ">=6.0" }, - "funding": { - "url": "https://github.com/avajs/find-cache-dir?sponsor=1" + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, "node_modules/find-my-way": { @@ -13470,6 +11095,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, "license": "MIT", "dependencies": { "locate-path": "^6.0.0", @@ -13482,90 +11108,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/firebase-admin": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/firebase-admin/-/firebase-admin-10.3.0.tgz", - "integrity": "sha512-A0wgMLEjyVyUE+heyMJYqHRkPVjpebhOYsa47RHdrTM4ltApcx8Tn86sUmjqxlfh09gNnILAm7a8q5+FmgBYpg==", - "license": "Apache-2.0", - "dependencies": { - "@fastify/busboy": "^1.1.0", - "@firebase/database-compat": "^0.2.0", - "@firebase/database-types": "^0.9.7", - "@types/node": ">=12.12.47", - "jsonwebtoken": "^8.5.1", - "jwks-rsa": "^2.0.2", - "node-forge": "^1.3.1", - "uuid": "^8.3.2" - }, - "engines": { - "node": ">=12.7.0" - }, - "optionalDependencies": { - "@google-cloud/firestore": "^4.15.1", - "@google-cloud/storage": "^5.18.3" - } - }, - "node_modules/firebase-admin/node_modules/jsonwebtoken": { - "version": "8.5.1", - "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz", - "integrity": "sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==", - "license": "MIT", - "dependencies": { - "jws": "^3.2.2", - "lodash.includes": "^4.3.0", - "lodash.isboolean": "^3.0.3", - "lodash.isinteger": "^4.0.4", - "lodash.isnumber": "^3.0.3", - "lodash.isplainobject": "^4.0.6", - "lodash.isstring": "^4.0.1", - "lodash.once": "^4.0.0", - "ms": "^2.1.1", - "semver": "^5.6.0" - }, - "engines": { - "node": ">=4", - "npm": ">=1.4.28" - } - }, - "node_modules/firebase-admin/node_modules/jwa": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.2.tgz", - "integrity": "sha512-eeH5JO+21J78qMvTIDdBXidBd6nG2kZjg5Ohz/1fpa28Z4CcsWUzJ1ZZyFq/3z3N17aZy+ZuBoHljASbL1WfOw==", - "license": "MIT", - "dependencies": { - "buffer-equal-constant-time": "^1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/firebase-admin/node_modules/jws": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.3.tgz", - "integrity": "sha512-byiJ0FLRdLdSVSReO/U4E7RoEyOCKnEnEPMjq3HxWtvzLsV08/i5RQKsFVNkCldrCaPr2vDNAOMsfs8T/Hze7g==", - "license": "MIT", - "dependencies": { - "jwa": "^1.4.2", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/firebase-admin/node_modules/semver": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", - "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", - "license": "ISC", - "bin": { - "semver": "bin/semver" - } - }, - "node_modules/firebase-admin/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/flat": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", @@ -13580,6 +11122,7 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", + "dev": true, "license": "MIT", "dependencies": { "flatted": "^3.2.9", @@ -13593,18 +11136,13 @@ "version": "3.4.2", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", + "dev": true, "license": "ISC" }, - "node_modules/fn.name": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", - "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==", - "license": "MIT" - }, "node_modules/follow-redirects": { - "version": "1.15.11", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", - "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", + "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", "funding": [ { "type": "individual", @@ -13636,20 +11174,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/foreground-child": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-2.0.0.tgz", - "integrity": "sha512-dCIq9FpEcyQyXKCkyzmlPTFNgrCzPudOe+mhvJU5zAtlBnGVy2yKxtfsxK2tQBThwq225jcvBjpw1Gr40uzZCA==", - "dev": true, - "license": "ISC", - "dependencies": { - "cross-spawn": "^7.0.0", - "signal-exit": "^3.0.2" - }, - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/form-data": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", @@ -13715,36 +11239,21 @@ "node": ">= 0.6" } }, + "node_modules/forwarded-parse": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/forwarded-parse/-/forwarded-parse-2.1.2.tgz", + "integrity": "sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==", + "license": "MIT" + }, "node_modules/fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, - "node_modules/fromentries": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/fromentries/-/fromentries-1.3.2.tgz", - "integrity": "sha512-cHEpEQHUg0f8XdtZCc2ZAhrHzKzT0MrFUTcvx+hfxYu7rGMDc5SKoXFh+n4YigxsHXRzc6OrCshdR1bWH6HHyg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -13752,17 +11261,18 @@ "license": "MIT" }, "node_modules/fs-extra": { - "version": "11.3.4", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.4.tgz", - "integrity": "sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA==", + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "dev": true, "license": "MIT", "dependencies": { "graceful-fs": "^4.2.0", - "jsonfile": "^6.0.1", - "universalify": "^2.0.0" + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" }, "engines": { - "node": ">=14.14" + "node": ">=6 <7 || >=8" } }, "node_modules/fs-minipass": { @@ -13887,20 +11397,41 @@ } }, "node_modules/gaxios": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-4.3.3.tgz", - "integrity": "sha512-gSaYYIO1Y3wUtdfHmjDUZ8LWaxJQpiavzbF5Kq53akSzvmVg0RfyOcFDbO1KJ/KCGRFz2qG+lS81F0nkr7cRJA==", + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", + "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", "license": "Apache-2.0", - "optional": true, "dependencies": { - "abort-controller": "^3.0.0", "extend": "^3.0.2", - "https-proxy-agent": "^5.0.0", + "https-proxy-agent": "^7.0.1", "is-stream": "^2.0.0", - "node-fetch": "^2.6.7" + "node-fetch": "^2.6.9", + "uuid": "^9.0.1" }, "engines": { - "node": ">=10" + "node": ">=14" + } + }, + "node_modules/gaxios/node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/gaxios/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" } }, "node_modules/gcp-metadata": { @@ -13917,42 +11448,13 @@ "node": ">=14" } }, - "node_modules/gcp-metadata/node_modules/agent-base": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", - "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/gcp-metadata/node_modules/gaxios": { - "version": "6.7.1", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", - "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", - "license": "Apache-2.0", - "dependencies": { - "extend": "^3.0.2", - "https-proxy-agent": "^7.0.1", - "is-stream": "^2.0.0", - "node-fetch": "^2.6.9", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/gcp-metadata/node_modules/https-proxy-agent": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", - "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "node_modules/generate-function": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz", + "integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==", "license": "MIT", "dependencies": { - "agent-base": "^7.1.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" + "is-property": "^1.0.2" } }, "node_modules/generator-function": { @@ -13964,16 +11466,6 @@ "node": ">= 0.4" } }, - "node_modules/gensync": { - "version": "1.0.0-beta.2", - "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", - "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/genwiki": { "resolved": "tools/genwiki", "link": true @@ -14020,15 +11512,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/get-package-type": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", - "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/get-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", @@ -14059,22 +11542,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/getopts": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/getopts/-/getopts-2.3.0.tgz", - "integrity": "sha512-5eDf9fuSXwxBL6q5HX+dhDj+dslFGWzU5thZ9kNKUkcPtaPdatmUFKwHFrLb/uf/WpA4BHET+AX3Scl56cAjpA==", - "license": "MIT" - }, - "node_modules/gifwrap": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/gifwrap/-/gifwrap-0.10.1.tgz", - "integrity": "sha512-2760b1vpJHNmLzZ/ubTtNnEx5WApN/PYWJvXvgS+tL1egTTthayFYIQQNi136FLEDcN/IyEY2EcGpIITD6eYUw==", - "license": "MIT", - "dependencies": { - "image-q": "^4.0.0", - "omggif": "^1.0.10" - } - }, "node_modules/github-from-package": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", @@ -14116,6 +11583,7 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, "license": "ISC", "dependencies": { "is-glob": "^4.0.3" @@ -14138,9 +11606,9 @@ "license": "MIT" }, "node_modules/glob/node_modules/brace-expansion": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.13.tgz", - "integrity": "sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w==", + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", + "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -14312,242 +11780,6 @@ "url": "https://opencollective.com/node-fetch" } }, - "node_modules/google-gax": { - "version": "2.30.5", - "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-2.30.5.tgz", - "integrity": "sha512-Jey13YrAN2hfpozHzbtrwEfEHdStJh1GwaQ2+Akh1k0Tv/EuNVSuBtHZoKSBm5wBMvNsxTsEIZ/152NrYyZgxQ==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "@grpc/grpc-js": "~1.6.0", - "@grpc/proto-loader": "^0.6.12", - "@types/long": "^4.0.0", - "abort-controller": "^3.0.0", - "duplexify": "^4.0.0", - "fast-text-encoding": "^1.0.3", - "google-auth-library": "^7.14.0", - "is-stream-ended": "^0.1.4", - "node-fetch": "^2.6.1", - "object-hash": "^3.0.0", - "proto3-json-serializer": "^0.1.8", - "protobufjs": "6.11.3", - "retry-request": "^4.0.0" - }, - "bin": { - "compileProtos": "build/tools/compileProtos.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/google-gax/node_modules/@grpc/grpc-js": { - "version": "1.6.12", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.6.12.tgz", - "integrity": "sha512-JmvQ03OTSpVd9JTlj/K3IWHSz4Gk/JMLUTtW7Zb0KvO1LcOYGATh5cNuRYzCAeDR3O8wq+q8FZe97eO9MBrkUw==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "@grpc/proto-loader": "^0.7.0", - "@types/node": ">=12.12.47" - }, - "engines": { - "node": "^8.13.0 || >=10.10.0" - } - }, - "node_modules/google-gax/node_modules/@grpc/grpc-js/node_modules/@grpc/proto-loader": { - "version": "0.7.15", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.15.tgz", - "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "lodash.camelcase": "^4.3.0", - "long": "^5.0.0", - "protobufjs": "^7.2.5", - "yargs": "^17.7.2" - }, - "bin": { - "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/google-gax/node_modules/@grpc/grpc-js/node_modules/protobufjs": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", - "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", - "hasInstallScript": true, - "license": "BSD-3-Clause", - "optional": true, - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/node": ">=13.7.0", - "long": "^5.0.0" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/google-gax/node_modules/@grpc/proto-loader": { - "version": "0.6.13", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.6.13.tgz", - "integrity": "sha512-FjxPYDRTn6Ec3V0arm1FtSpmP6V50wuph2yILpyvTKzjc76oDdoihXqM1DzOW5ubvCC8GivfCnNtfaRE8myJ7g==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "@types/long": "^4.0.1", - "lodash.camelcase": "^4.3.0", - "long": "^4.0.0", - "protobufjs": "^6.11.3", - "yargs": "^16.2.0" - }, - "bin": { - "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/google-gax/node_modules/@grpc/proto-loader/node_modules/long": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", - "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", - "license": "Apache-2.0", - "optional": true - }, - "node_modules/google-gax/node_modules/@grpc/proto-loader/node_modules/yargs": { - "version": "16.2.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", - "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "license": "MIT", - "optional": true, - "dependencies": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/google-gax/node_modules/cliui": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", - "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "license": "ISC", - "optional": true, - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - } - }, - "node_modules/google-gax/node_modules/gcp-metadata": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.3.1.tgz", - "integrity": "sha512-x850LS5N7V1F3UcV7PoupzGsyD6iVwTVvsh3tbXfkctZnBnjW5yu5z1/3k3SehF7TyoTIe78rJs02GMMy+LF+A==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "gaxios": "^4.0.0", - "json-bigint": "^1.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/google-gax/node_modules/google-auth-library": { - "version": "7.14.1", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-7.14.1.tgz", - "integrity": "sha512-5Rk7iLNDFhFeBYc3s8l1CqzbEBcdhwR193RlD4vSNFajIcINKI8W8P0JLmBpwymHqqWbX34pJDQu39cSy/6RsA==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "arrify": "^2.0.0", - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "fast-text-encoding": "^1.0.0", - "gaxios": "^4.0.0", - "gcp-metadata": "^4.2.0", - "gtoken": "^5.0.4", - "jws": "^4.0.0", - "lru-cache": "^6.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/google-gax/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "license": "ISC", - "optional": true, - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/google-gax/node_modules/protobufjs": { - "version": "6.11.3", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.3.tgz", - "integrity": "sha512-xL96WDdCZYdU7Slin569tFX712BxsxslWwAfAhCYjQKGTq7dAU91Lomy6nLLhh/dyGhk/YH4TwTSRxTzhuHyZg==", - "hasInstallScript": true, - "license": "BSD-3-Clause", - "optional": true, - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/long": "^4.0.1", - "@types/node": ">=13.7.0", - "long": "^4.0.0" - }, - "bin": { - "pbjs": "bin/pbjs", - "pbts": "bin/pbts" - } - }, - "node_modules/google-gax/node_modules/protobufjs/node_modules/long": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", - "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", - "license": "Apache-2.0", - "optional": true - }, - "node_modules/google-gax/node_modules/yargs-parser": { - "version": "20.2.9", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", - "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", - "license": "ISC", - "optional": true, - "engines": { - "node": ">=10" - } - }, "node_modules/google-logging-utils": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz", @@ -14557,23 +11789,6 @@ "node": ">=14" } }, - "node_modules/google-p12-pem": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.1.4.tgz", - "integrity": "sha512-HHuHmkLgwjdmVRngf5+gSmpkyaRI6QmOg77J8tkNBHhNEI62sGHyw4/+UkgyZEI7h84NbWprXDJ+sa3xOYFvTg==", - "deprecated": "Package is no longer maintained", - "license": "MIT", - "optional": true, - "dependencies": { - "node-forge": "^1.3.1" - }, - "bin": { - "gp12-pem": "build/src/bin/gp12-pem.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -14632,21 +11847,6 @@ "node": ">=4.x" } }, - "node_modules/gtoken": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-5.3.2.tgz", - "integrity": "sha512-gkvEKREW7dXWF8NV8pVrKfW7WqReAmjjkMBh6lNCCGOM4ucS0r0YyXXl0r/9Yj8wcW/32ISkfc8h5mPTDbtifQ==", - "license": "MIT", - "optional": true, - "dependencies": { - "gaxios": "^4.0.0", - "google-p12-pem": "^3.1.3", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/handlebars": { "version": "4.7.9", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.9.tgz", @@ -14684,6 +11884,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -14749,34 +11950,10 @@ "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", "license": "ISC" }, - "node_modules/hash-stream-validation": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/hash-stream-validation/-/hash-stream-validation-0.2.4.tgz", - "integrity": "sha512-Gjzu0Xn7IagXVkSu9cSFuK1fqzwtLwFhNhVL8IFJijRNMgUttFbBSIAzKuSIrsFMO1+g1RlsoN49zPIbwPDMGQ==", - "license": "MIT", - "optional": true - }, - "node_modules/hasha": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/hasha/-/hasha-5.2.2.tgz", - "integrity": "sha512-Hrp5vIK/xr5SkeN2onO32H0MgNZ0f17HRNH39WfL0SYUNOTZ5Lz1TJ8Pajo/87dYGEFlLMm7mIc/k/s6Bvz9HQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-stream": "^2.0.0", - "type-fest": "^0.8.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", + "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", "license": "MIT", "dependencies": { "function-bind": "^1.1.2" @@ -14877,9 +12054,9 @@ } }, "node_modules/html-webpack-plugin": { - "version": "5.6.6", - "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-5.6.6.tgz", - "integrity": "sha512-bLjW01UTrvoWTJQL5LsMRo1SypHW80FTm12OJRSnr3v6YHNhfe+1r0MYUZJMACxnCHURVnBWRwAsWs2yPU9Ezw==", + "version": "5.6.7", + "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-5.6.7.tgz", + "integrity": "sha512-md+vXtdCAe60s1k6AU3dUyMJnDxUyQAwfwPKoLisvgUF1IXjtlLsk2se54+qfL9Mdm26bbwvjJybpNx48NKRLw==", "dev": true, "license": "MIT", "dependencies": { @@ -14929,16 +12106,6 @@ "entities": "^2.0.0" } }, - "node_modules/htmlparser2/node_modules/entities": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", - "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==", - "dev": true, - "license": "BSD-2-Clause", - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, "node_modules/http-errors": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", @@ -14959,12 +12126,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/http-parser-js": { - "version": "0.5.10", - "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.10.tgz", - "integrity": "sha512-Pysuw9XpUq5dVc/2SMHpuTY01RFl8fttgcyunjL7eEMhGM3cI4eOmiCycJDVCo/7O7ClfQD3SaI6ftDzqOXYMA==", - "license": "MIT" - }, "node_modules/http-proxy": { "version": "1.18.1", "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", @@ -14980,21 +12141,6 @@ "node": ">=8.0.0" } }, - "node_modules/http-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", - "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", - "license": "MIT", - "optional": true, - "dependencies": { - "@tootallnate/once": "2", - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/http-server": { "version": "14.1.1", "resolved": "https://registry.npmjs.org/http-server/-/http-server-14.1.1.tgz", @@ -15023,19 +12169,6 @@ "node": ">=12" } }, - "node_modules/http-server/node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "dev": true, - "license": "MIT", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/https-proxy-agent": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", @@ -15074,22 +12207,20 @@ "url": "https://github.com/sponsors/typicode" } }, - "node_modules/ico-endec": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/ico-endec/-/ico-endec-0.1.6.tgz", - "integrity": "sha512-ZdLU38ZoED3g1j3iEyzcQj+wAkY2xfWNkymszfJPoxucIUhK7NayQ+/C4Kv0nDFMIsbtbEHldv3V8PU494/ueQ==", - "license": "MPL-2.0" - }, "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", "license": "MIT", "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" + "safer-buffer": ">= 2.1.2 < 3.0.0" }, "engines": { "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/ieee754": { @@ -15129,25 +12260,11 @@ "dev": true, "license": "ISC" }, - "node_modules/image-q": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/image-q/-/image-q-4.0.0.tgz", - "integrity": "sha512-PfJGVgIfKQJuq3s0tTDOKtztksibuUEbJQIYT3by6wctQo+Rdlh7ef4evJ5NCdxY4CfMbvFkocEwbl4BF8RlJw==", - "license": "MIT", - "dependencies": { - "@types/node": "16.9.1" - } - }, - "node_modules/image-q/node_modules/@types/node": { - "version": "16.9.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz", - "integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g==", - "license": "MIT" - }, "node_modules/import-fresh": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, "license": "MIT", "dependencies": { "parent-module": "^1.0.0", @@ -15161,13 +12278,13 @@ } }, "node_modules/import-in-the-middle": { - "version": "1.7.1", - "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-1.7.1.tgz", - "integrity": "sha512-1LrZPDtW+atAxH42S6288qyDFNQ2YCty+2mxEPRtfazH6Z5QwkaBSTS2ods7hnVJioF6rkRfNoA6A/MstpFXLg==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-1.15.0.tgz", + "integrity": "sha512-bpQy+CrsRmYmoPMAE/0G33iwRqwW4ouqdRg8jgbH3aKuCtOc8lxgmYXg2dMM92CRiGP660EtBcymH/eVUpCSaA==", "license": "Apache-2.0", "dependencies": { - "acorn": "^8.8.2", - "acorn-import-assertions": "^1.9.0", + "acorn": "^8.14.0", + "acorn-import-attributes": "^1.9.5", "cjs-module-lexer": "^1.2.2", "module-details-from-path": "^1.0.3" } @@ -15196,19 +12313,10 @@ "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", - "license": "MIT", - "engines": { - "node": ">=0.8.19" - } - }, - "node_modules/indent-string": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", - "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", "dev": true, "license": "MIT", "engines": { - "node": ">=8" + "node": ">=0.8.19" } }, "node_modules/inflight": { @@ -15249,12 +12357,13 @@ } }, "node_modules/interpret": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", - "integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-3.1.1.tgz", + "integrity": "sha512-6xwYfHbajpoF0xLW+iwLkhwgvLoZDfjYfoFNu8ftMoXINzwuymNLd9u/KmwtdT2GbR+/Cz66otEGEVVUHX9QLQ==", + "dev": true, "license": "MIT", "engines": { - "node": ">= 0.10" + "node": ">=10.13.0" } }, "node_modules/ioredis": { @@ -15262,7 +12371,6 @@ "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.10.1.tgz", "integrity": "sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA==", "license": "MIT", - "peer": true, "dependencies": { "@ioredis/commands": "1.5.1", "cluster-key-slot": "^1.1.0", @@ -15647,25 +12755,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-obj": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", - "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/is-object": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-object/-/is-object-1.0.2.tgz", - "integrity": "sha512-2rRIahhZr2UWb45fIOuvZGpFtz0TyOZLf32KxBbSoUCeZR495zCKlWUKKUByk3geS2eAs7ZAABt0Y/Rx0GiQGA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-plain-object": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", @@ -15685,6 +12774,18 @@ "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", "license": "MIT" }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/is-property": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", + "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==", + "license": "MIT" + }, "node_modules/is-regex": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", @@ -15754,13 +12855,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/is-stream-ended": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/is-stream-ended/-/is-stream-ended-0.1.4.tgz", - "integrity": "sha512-xj0XPvmr7bQFTvirqnFr50o0hQIh6ZItDqloxt5aJrR4NQsYeSsyFQERYGCAzfindAcnKjINnwEEgLx4IqVzQw==", - "license": "MIT", - "optional": true - }, "node_modules/is-string": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.1.1.tgz", @@ -15777,15 +12871,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-supported-regexp-flag": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-supported-regexp-flag/-/is-supported-regexp-flag-2.0.0.tgz", - "integrity": "sha512-8i4+OYUjdUaJ88KAs1WojIThDFjIpeYNrSlYy1g/At2p9YjQ7HEmB1yn60un0jRFjM3TQbKPMAluTPEPncZfqA==", - "license": "MIT", - "engines": { - "node": ">=12.20" - } - }, "node_modules/is-symbol": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.1.1.tgz", @@ -15818,13 +12903,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-typedarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==", - "devOptional": true, - "license": "MIT" - }, "node_modules/is-weakmap": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.2.tgz", @@ -15868,16 +12946,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-windows": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", - "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/is-wsl": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.1.tgz", @@ -15899,15 +12967,6 @@ "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", "license": "MIT" }, - "node_modules/isbot": { - "version": "3.8.0", - "resolved": "https://registry.npmjs.org/isbot/-/isbot-3.8.0.tgz", - "integrity": "sha512-vne1mzQUTR+qsMLeCBL9+/tgnDXRyc2pygLGl/WsgA+EZKIiB5Ehu0CiVTHIIk30zhJ24uGz4M5Ppse37aR0Hg==", - "license": "Unlicense", - "engines": { - "node": ">=12" - } - }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -15934,73 +12993,6 @@ "node": ">=8" } }, - "node_modules/istanbul-lib-hook": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/istanbul-lib-hook/-/istanbul-lib-hook-3.0.0.tgz", - "integrity": "sha512-Pt/uge1Q9s+5VAZ+pCo16TYMWPBIl+oaNIjgLQxcX0itS6ueeaA+pEfThZpH8WxhFgCiEb8sAJY6MdUKgiIWaQ==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "append-transform": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-instrument": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-4.0.3.tgz", - "integrity": "sha512-BXgQl9kf4WTCPCCpmFGoJkz/+uhvm7h7PFKUYxh7qarQd3ER33vHG//qaE8eN25l07YqZPpHXU9I09l/RD5aGQ==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@babel/core": "^7.7.5", - "@istanbuljs/schema": "^0.1.2", - "istanbul-lib-coverage": "^3.0.0", - "semver": "^6.3.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-instrument/node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/istanbul-lib-processinfo": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/istanbul-lib-processinfo/-/istanbul-lib-processinfo-2.0.3.tgz", - "integrity": "sha512-NkwHbo3E00oybX6NGJi6ar0B29vxyvNwoC7eJ4G4Yq28UfY758Hgn/heV8VRFhevPED4LXfFz0DQ8z/0kw9zMg==", - "dev": true, - "license": "ISC", - "dependencies": { - "archy": "^1.0.0", - "cross-spawn": "^7.0.3", - "istanbul-lib-coverage": "^3.2.0", - "p-map": "^3.0.0", - "rimraf": "^3.0.0", - "uuid": "^8.3.2" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-processinfo/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "dev": true, - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/istanbul-lib-report": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", @@ -16032,21 +13024,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/istanbul-lib-source-maps": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz", - "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "debug": "^4.1.1", - "istanbul-lib-coverage": "^3.0.0", - "source-map": "^0.6.1" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/istanbul-reports": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", @@ -16101,99 +13078,12 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, - "node_modules/jimp": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/jimp/-/jimp-1.6.1.tgz", - "integrity": "sha512-hNQh6rZtWfSVWSNVmvq87N5BPJsNH7k7I7qyrXf9DOma9xATQk3fsyHazCQe51nCjdkoWdTmh0vD7bjVSLoxxw==", - "license": "MIT", - "dependencies": { - "@jimp/core": "1.6.1", - "@jimp/diff": "1.6.1", - "@jimp/js-bmp": "1.6.1", - "@jimp/js-gif": "1.6.1", - "@jimp/js-jpeg": "1.6.1", - "@jimp/js-png": "1.6.1", - "@jimp/js-tiff": "1.6.1", - "@jimp/plugin-blit": "1.6.1", - "@jimp/plugin-blur": "1.6.1", - "@jimp/plugin-circle": "1.6.1", - "@jimp/plugin-color": "1.6.1", - "@jimp/plugin-contain": "1.6.1", - "@jimp/plugin-cover": "1.6.1", - "@jimp/plugin-crop": "1.6.1", - "@jimp/plugin-displace": "1.6.1", - "@jimp/plugin-dither": "1.6.1", - "@jimp/plugin-fisheye": "1.6.1", - "@jimp/plugin-flip": "1.6.1", - "@jimp/plugin-hash": "1.6.1", - "@jimp/plugin-mask": "1.6.1", - "@jimp/plugin-print": "1.6.1", - "@jimp/plugin-quantize": "1.6.1", - "@jimp/plugin-resize": "1.6.1", - "@jimp/plugin-rotate": "1.6.1", - "@jimp/plugin-threshold": "1.6.1", - "@jimp/types": "1.6.1", - "@jimp/utils": "1.6.1" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/joi": { - "version": "17.13.3", - "resolved": "https://registry.npmjs.org/joi/-/joi-17.13.3.tgz", - "integrity": "sha512-otDA4ldcIx+ZXsKHWmp0YizCweVRZG96J10b0FevjfuncLO1oX59THoAmHkNubYJ+9gWsYsp5k8v4ib6oDv1fA==", - "license": "BSD-3-Clause", - "dependencies": { - "@hapi/hoek": "^9.3.0", - "@hapi/topo": "^5.1.0", - "@sideway/address": "^4.1.5", - "@sideway/formula": "^3.0.1", - "@sideway/pinpoint": "^2.0.0" - } - }, - "node_modules/jose": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/jose/-/jose-2.0.7.tgz", - "integrity": "sha512-5hFWIigKqC+e/lRyQhfnirrAqUdIPMB7SJRqflJaO29dW7q5DFvH1XCSTmv6PQ6pb++0k6MJlLRoS0Wv4s38Wg==", - "license": "MIT", - "dependencies": { - "@panva/asn1.js": "^1.0.0" - }, - "engines": { - "node": ">=10.13.0 < 13 || >=13.7.0" - }, - "funding": { - "url": "https://github.com/sponsors/panva" - } - }, - "node_modules/jpeg-js": { - "version": "0.4.4", - "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", - "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==", - "license": "BSD-3-Clause" - }, "node_modules/jquery": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/jquery/-/jquery-4.0.0.tgz", "integrity": "sha512-TXCHVR3Lb6TZdtw1l3RTLf8RBWVGexdxL6AC8/e0xZKEpBflBsjh9/8LXw+dkNFuOyW9B7iB3O1sP7hS0Kiacg==", "license": "MIT" }, - "node_modules/js-levenshtein": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/js-levenshtein/-/js-levenshtein-1.1.6.tgz", - "integrity": "sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/js-sha256": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/js-sha256/-/js-sha256-0.9.0.tgz", - "integrity": "sha512-sga3MHh9sgQN2+pJ9VYZ+1LPwXOxuBJBA5nrR5/ofPfuiJBE2hnjsaN8se8JznOmGLN2p49Pe5U/ttafcs/apA==", - "license": "MIT" - }, "node_modules/js-tokens": { "version": "10.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", @@ -16214,14 +13104,14 @@ } }, "node_modules/jsdom": { - "version": "29.0.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-29.0.0.tgz", - "integrity": "sha512-9FshNB6OepopZ08unmmGpsF7/qCjxGPbo3NbgfJAnPeHXnsODE9WWffXZtRFRFe0ntzaAOcSKNJFz8wiyvF1jQ==", + "version": "29.0.2", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-29.0.2.tgz", + "integrity": "sha512-9VnGEBosc/ZpwyOsJBCQ/3I5p7Q5ngOY14a9bf5btenAORmZfDse1ZEheMiWcJ3h81+Fv7HmJFdS0szo/waF2w==", "dev": true, "license": "MIT", "dependencies": { - "@asamuzakjp/css-color": "^5.0.1", - "@asamuzakjp/dom-selector": "^7.0.2", + "@asamuzakjp/css-color": "^5.1.5", + "@asamuzakjp/dom-selector": "^7.0.6", "@bramus/specificity": "^2.4.2", "@csstools/css-syntax-patches-for-csstree": "^1.1.1", "@exodus/bytes": "^1.15.0", @@ -16235,7 +13125,7 @@ "saxes": "^6.0.0", "symbol-tree": "^3.2.4", "tough-cookie": "^6.0.1", - "undici": "^7.24.3", + "undici": "^7.24.5", "w3c-xmlserializer": "^5.0.0", "webidl-conversions": "^8.0.1", "whatwg-mimetype": "^5.0.0", @@ -16267,16 +13157,42 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" } }, - "node_modules/jsesc": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", - "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "node_modules/jsdom/node_modules/tr46": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-6.0.0.tgz", + "integrity": "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==", + "dev": true, "license": "MIT", - "bin": { - "jsesc": "bin/jsesc" + "dependencies": { + "punycode": "^2.3.1" }, "engines": { - "node": ">=6" + "node": ">=20" + } + }, + "node_modules/jsdom/node_modules/webidl-conversions": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-8.0.1.tgz", + "integrity": "sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=20" + } + }, + "node_modules/jsdom/node_modules/whatwg-url": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-16.0.1.tgz", + "integrity": "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@exodus/bytes": "^1.11.0", + "tr46": "^6.0.0", + "webidl-conversions": "^8.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" } }, "node_modules/json-bigint": { @@ -16292,6 +13208,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, "license": "MIT" }, "node_modules/json-colorizer": { @@ -16303,13 +13220,6 @@ "colorette": "^2.0.20" } }, - "node_modules/json-parse-even-better-errors": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", - "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", - "dev": true, - "license": "MIT" - }, "node_modules/json-schema": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", @@ -16352,34 +13262,22 @@ "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, "license": "MIT" }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, "license": "MIT" }, - "node_modules/json5": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", - "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", - "license": "MIT", - "bin": { - "json5": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/jsonfile": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz", - "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==", + "dev": true, "license": "MIT", - "dependencies": { - "universalify": "^2.0.0" - }, "optionalDependencies": { "graceful-fs": "^4.1.6" } @@ -16406,15 +13304,6 @@ "npm": ">=6" } }, - "node_modules/jssha": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/jssha/-/jssha-3.3.1.tgz", - "integrity": "sha512-VCMZj12FCFMQYcFLPRm/0lOBbLi8uM2BhXPTqw3U4YAfs4AZfiApOoBLoN8cQE60Z50m1MYMTQVCfgF/KaCVhQ==", - "license": "BSD-3-Clause", - "engines": { - "node": "*" - } - }, "node_modules/just-extend": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-6.2.0.tgz", @@ -16433,23 +13322,6 @@ "safe-buffer": "^5.0.1" } }, - "node_modules/jwks-rsa": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/jwks-rsa/-/jwks-rsa-2.1.5.tgz", - "integrity": "sha512-IODtn1SwEm7n6GQZnQLY0oxKDrMh7n/jRH1MzE8mlxWMrh2NnMyOsXTebu8vJ1qCpmuTJcL4DdiE0E4h8jnwsA==", - "license": "MIT", - "dependencies": { - "@types/express": "^4.17.14", - "@types/jsonwebtoken": "^8.5.9", - "debug": "^4.3.4", - "jose": "^2.0.6", - "limiter": "^1.1.5", - "lru-memoizer": "^2.1.4" - }, - "engines": { - "node": ">=10 < 13 || >=14" - } - }, "node_modules/jws": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", @@ -16460,14 +13332,11 @@ "safe-buffer": "^5.0.1" } }, - "node_modules/keygen": { - "resolved": "tools/keygen", - "link": true - }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, "license": "MIT", "dependencies": { "json-buffer": "3.0.1" @@ -16483,116 +13352,6 @@ "node": ">=0.10.0" } }, - "node_modules/knex": { - "version": "3.2.9", - "resolved": "https://registry.npmjs.org/knex/-/knex-3.2.9.tgz", - "integrity": "sha512-dtAILTjBMaG8YloP5oBxohDIKyIsdQ/TkcVvSjhsksvsjeH63Y0PADyuMDfNZKbVT3Rlx3vEYVBlecbPT/KerA==", - "license": "MIT", - "dependencies": { - "colorette": "2.0.19", - "commander": "^10.0.0", - "debug": "4.3.4", - "escalade": "^3.1.1", - "esm": "^3.2.25", - "get-package-type": "^0.1.0", - "getopts": "2.3.0", - "interpret": "^2.2.0", - "lodash": "^4.17.21", - "pg-connection-string": "2.6.2", - "rechoir": "^0.8.0", - "resolve-from": "^5.0.0", - "tarn": "^3.0.2", - "tildify": "2.0.0" - }, - "bin": { - "knex": "bin/cli.js" - }, - "engines": { - "node": ">=16" - }, - "peerDependencies": { - "pg-query-stream": "^4.14.0" - }, - "peerDependenciesMeta": { - "better-sqlite3": { - "optional": true - }, - "mysql": { - "optional": true - }, - "mysql2": { - "optional": true - }, - "pg": { - "optional": true - }, - "pg-native": { - "optional": true - }, - "pg-query-stream": { - "optional": true - }, - "sqlite3": { - "optional": true - }, - "tedious": { - "optional": true - } - } - }, - "node_modules/knex/node_modules/colorette": { - "version": "2.0.19", - "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.19.tgz", - "integrity": "sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==", - "license": "MIT" - }, - "node_modules/knex/node_modules/commander": { - "version": "10.0.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", - "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", - "license": "MIT", - "engines": { - "node": ">=14" - } - }, - "node_modules/knex/node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "license": "MIT", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/knex/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "license": "MIT" - }, - "node_modules/knex/node_modules/resolve-from": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", - "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/kuler": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", - "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==", - "license": "MIT" - }, "node_modules/lazy": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/lazy/-/lazy-1.0.11.tgz", @@ -16642,19 +13401,11 @@ "node": ">=12" } }, - "node_modules/leven": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/leven/-/leven-2.1.0.tgz", - "integrity": "sha512-nvVPLpIHUxCUoRLrFqTgSxXJ614d8AgQoWl7zPe/2VadE8+1dpU3LBhowRuBAcuwruWtOdD8oYC9jDNJjXDPyA==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, "license": "MIT", "dependencies": { "prelude-ls": "^1.2.1", @@ -16753,21 +13504,6 @@ "node": ">=6" } }, - "node_modules/license-check-and-add/node_modules/fs-extra": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", - "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", - "dev": true, - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.2.0", - "jsonfile": "^4.0.0", - "universalify": "^0.1.0" - }, - "engines": { - "node": ">=6 <7 || >=8" - } - }, "node_modules/license-check-and-add/node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -16788,16 +13524,6 @@ "node": ">=4" } }, - "node_modules/license-check-and-add/node_modules/jsonfile": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", - "integrity": "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==", - "dev": true, - "license": "MIT", - "optionalDependencies": { - "graceful-fs": "^4.1.6" - } - }, "node_modules/license-check-and-add/node_modules/locate-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", @@ -16879,16 +13605,6 @@ "node": ">=6" } }, - "node_modules/license-check-and-add/node_modules/universalify": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", - "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4.0.0" - } - }, "node_modules/license-check-and-add/node_modules/wrap-ansi": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", @@ -16930,10 +13646,6 @@ "yargs-parser": "^13.1.2" } }, - "node_modules/license-headers": { - "resolved": "tools/license-headers", - "link": true - }, "node_modules/light-my-request": { "version": "6.6.0", "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-6.6.0.tgz", @@ -17127,6 +13839,9 @@ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -17148,6 +13863,9 @@ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -17169,6 +13887,9 @@ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -17190,6 +13911,9 @@ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -17245,11 +13969,6 @@ "url": "https://opencollective.com/parcel" } }, - "node_modules/limiter": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/limiter/-/limiter-1.1.5.tgz", - "integrity": "sha512-FWWMIEOxz3GwUI4Ts/IvgVy6LPvoMPgjMdQ185nN6psJyBJ4yOpzqm695/h5umdLJg2vW3GR5iG11MAkR2AzJA==" - }, "node_modules/loader-runner": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.1.tgz", @@ -17268,6 +13987,7 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, "license": "MIT", "dependencies": { "p-locate": "^5.0.0" @@ -17297,25 +14017,12 @@ "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", "license": "MIT" }, - "node_modules/lodash.clonedeep": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", - "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==", - "license": "MIT" - }, "node_modules/lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", "license": "MIT" }, - "node_modules/lodash.flattendeep": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz", - "integrity": "sha512-uHaJFihxmJcEX3kT4I23ABqKKalJ/zDrDg0lsFtc1h+3uw49SIJ5beyhx5ExVRti3AvKoOJngIj7xz3oylPdWQ==", - "dev": true, - "license": "MIT" - }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", @@ -17362,6 +14069,7 @@ "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, "license": "MIT" }, "node_modules/lodash.once": { @@ -17453,23 +14161,6 @@ "node": ">=4" } }, - "node_modules/logform": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/logform/-/logform-2.7.0.tgz", - "integrity": "sha512-TFYA4jnP7PVbmlBIfhlSe+WKxs9dklXMTEGcBCIvLhE/Tn3H6Gk1norupVW7m5Cnd4bLcr08AytbyV/xj7f/kQ==", - "license": "MIT", - "dependencies": { - "@colors/colors": "1.6.0", - "@types/triple-beam": "^1.3.2", - "fecha": "^4.2.0", - "ms": "^2.1.1", - "safe-stable-stringify": "^2.3.1", - "triple-beam": "^1.3.0" - }, - "engines": { - "node": ">= 12.0.0" - } - }, "node_modules/long": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", @@ -17521,34 +14212,28 @@ } }, "node_modules/lru-cache": { - "version": "11.3.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.3.tgz", - "integrity": "sha512-JvNw9Y81y33E+BEYPr0U7omo+U9AySnsMsEiXgwT6yqd31VQWTLNQqmT4ou5eqPFUrTfIDFta2wKhB1hyohtAQ==", + "version": "11.3.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.5.tgz", + "integrity": "sha512-NxVFwLAnrd9i7KUBxC4DrUhmgjzOs+1Qm50D3oF1/oL+r1NpZ4gA7xvG0/zJ8evR7zIKn4vLf7qTNduWFtCrRw==", + "dev": true, "license": "BlueOak-1.0.0", "engines": { "node": "20 || >=22" } }, - "node_modules/lru-memoizer": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/lru-memoizer/-/lru-memoizer-2.3.0.tgz", - "integrity": "sha512-GXn7gyHAMhO13WSKrIiNfztwxodVsP8IoZ3XfrJV4yH2x0/OeTO/FIaAHTY5YekdGgW94njfuKmyyt1E0mR6Ug==", + "node_modules/lru.min": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.4.tgz", + "integrity": "sha512-DqC6n3QQ77zdFpCMASA1a3Jlb64Hv2N2DciFGkO/4L9+q/IpIAuRlKOvCXabtRW6cQf8usbmM6BE/TOPysCdIA==", "license": "MIT", - "dependencies": { - "lodash.clonedeep": "^4.5.0", - "lru-cache": "6.0.0" - } - }, - "node_modules/lru-memoizer/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "license": "ISC", - "dependencies": { - "yallist": "^4.0.0" - }, "engines": { - "node": ">=10" + "bun": ">=1.0.0", + "deno": ">=1.30.0", + "node": ">=8.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wellwelwel" } }, "node_modules/magic-string": { @@ -17658,10 +14343,13 @@ } }, "node_modules/merge-descriptors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", - "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", "license": "MIT", + "engines": { + "node": ">=18" + }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } @@ -17683,19 +14371,11 @@ "node": ">= 8" } }, - "node_modules/methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/micromatch": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, "license": "MIT", "dependencies": { "braces": "^3.0.3", @@ -17709,6 +14389,7 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "dev": true, "license": "MIT", "engines": { "node": ">=8.6" @@ -17722,15 +14403,16 @@ "link": true }, "node_modules/mime": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", - "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true, "license": "MIT", "bin": { "mime": "cli.js" }, "engines": { - "node": ">=10.0.0" + "node": ">=4" } }, "node_modules/mime-db": { @@ -17941,9 +14623,9 @@ "license": "MIT" }, "node_modules/mocha/node_modules/brace-expansion": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.13.tgz", - "integrity": "sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w==", + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", + "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -18238,10 +14920,6 @@ "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", "license": "MIT" }, - "node_modules/module-docgen": { - "resolved": "tools/module-docgen", - "link": true - }, "node_modules/module-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/module-error/-/module-error-1.0.2.tgz", @@ -18251,73 +14929,6 @@ "node": ">=10" } }, - "node_modules/module-not-found-error": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/module-not-found-error/-/module-not-found-error-1.0.1.tgz", - "integrity": "sha512-pEk4ECWQXV6z2zjhRZUongnLJNUeGQJ3w6OQ5ctGwD+i5o93qjRQUk2Rt6VdNeu3sEP0AB4LcfvdebpxBRVr4g==", - "license": "MIT" - }, - "node_modules/moment": { - "version": "2.30.1", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", - "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==", - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/morgan": { - "version": "1.10.1", - "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.10.1.tgz", - "integrity": "sha512-223dMRJtI/l25dJKWpgij2cMtywuG/WiUKXdvwfbhGKBhy1puASqXwFzmWZ7+K73vUPoR7SS2Qz2cI/g9MKw0A==", - "license": "MIT", - "dependencies": { - "basic-auth": "~2.0.1", - "debug": "2.6.9", - "depd": "~2.0.0", - "on-finished": "~2.3.0", - "on-headers": "~1.1.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/morgan/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/morgan/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, - "node_modules/morgan/node_modules/on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==", - "license": "MIT", - "dependencies": { - "ee-first": "1.1.1" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/mri": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/mri/-/mri-1.1.4.tgz", - "integrity": "sha512-6y7IjGPm8AzlvoUrwAaw1tLnUBudaS3752vcd8JtrpGGQn+rXIe63LFVHm/YMwtqAuh+LJPCFdlLYPWM1nYn6w==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/mrmime": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.1.tgz", @@ -18334,34 +14945,6 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, - "node_modules/multer": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/multer/-/multer-2.1.1.tgz", - "integrity": "sha512-mo+QTzKlx8R7E5ylSXxWzGoXoZbOsRMpyitcht8By2KHvMbf3tjwosZ/Mu/XYU6UuJ3VZnODIrak5ZrPiPyB6A==", - "license": "MIT", - "dependencies": { - "append-field": "^1.0.0", - "busboy": "^1.6.0", - "concat-stream": "^2.0.0", - "type-is": "^1.6.18" - }, - "engines": { - "node": ">= 10.16.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/multi-progress": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/multi-progress/-/multi-progress-4.0.0.tgz", - "integrity": "sha512-9zcjyOou3FFCKPXsmkbC3ethv51SFPoA4dJD6TscIp2pUmy26kBDZW6h9XofPELrzseSkuD7r0V+emGEeo39Pg==", - "license": "MIT", - "peerDependencies": { - "progress": "^2.0.0" - } - }, "node_modules/murmurhash": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/murmurhash/-/murmurhash-2.0.1.tgz", @@ -18416,12 +14999,39 @@ } } }, - "node_modules/nan": { - "version": "2.26.2", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.26.2.tgz", - "integrity": "sha512-0tTvBTYkt3tdGw22nrAy50x7gpbGCCFH3AFcyS5WiUu7Eu4vWlri1woE6qHBSfy11vksDqkiwjOnlR7WV8G1Hw==", + "node_modules/mysql2": { + "version": "3.22.1", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.22.1.tgz", + "integrity": "sha512-48+9UXehKyxxiP2pqCxUq+MSFvX+v41jwsSpFDQO/jAoFuAELutBGJUhWJnDbe82/OBlIhSBMC82WeonmznT/Q==", "license": "MIT", - "optional": true + "dependencies": { + "aws-ssl-profiles": "^1.1.2", + "denque": "^2.1.0", + "generate-function": "^2.3.1", + "iconv-lite": "^0.7.2", + "long": "^5.3.2", + "lru.min": "^1.1.4", + "named-placeholders": "^1.1.6", + "sql-escaper": "^1.3.3" + }, + "engines": { + "node": ">= 8.0" + }, + "peerDependencies": { + "@types/node": ">= 8" + } + }, + "node_modules/named-placeholders": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.6.tgz", + "integrity": "sha512-Tz09sEL2EEuv5fFowm419c1+a/jSMiBjI9gHxVLrVdbUkkNUUfjsVYs9pVZu5oCon/kmRh9TfLEObFtkVxmY0w==", + "license": "MIT", + "dependencies": { + "lru.min": "^1.1.0" + }, + "engines": { + "node": ">=8.0.0" + } }, "node_modules/nanoid": { "version": "3.3.11", @@ -18458,6 +15068,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, "license": "MIT" }, "node_modules/negotiator": { @@ -18594,37 +15205,6 @@ } } }, - "node_modules/node-fetch/node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "license": "MIT" - }, - "node_modules/node-fetch/node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "license": "BSD-2-Clause" - }, - "node_modules/node-fetch/node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "node_modules/node-forge": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.4.0.tgz", - "integrity": "sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==", - "license": "(BSD-3-Clause OR GPL-2.0)", - "engines": { - "node": ">= 6.13.0" - } - }, "node_modules/node-gyp-build": { "version": "4.8.4", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", @@ -18636,19 +15216,6 @@ "node-gyp-build-test": "build-test.js" } }, - "node_modules/node-preload": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz", - "integrity": "sha512-RM5oyBy45cLEoHqCeh+MNuFAxO0vTFBLskvQbOKnEE7YTTSN4tbN8QWDIPQ6L+WvKsB/qLEGpYe2ZZ9d4W9OIQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "process-on-spawn": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/node-releases": { "version": "2.0.37", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.37.tgz", @@ -18837,6 +15404,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dev": true, "license": "BSD-2-Clause", "dependencies": { "boolbase": "^1.0.0" @@ -18845,198 +15413,6 @@ "url": "https://github.com/fb55/nth-check?sponsor=1" } }, - "node_modules/nwsapi": { - "version": "2.2.23", - "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.23.tgz", - "integrity": "sha512-7wfH4sLbt4M0gCDzGE6vzQBo0bfTKjU7Sfpqy/7gs1qBfYz2vEJH6vXcBKpO3+6Yu1telwd0t9HpyOoLEQQbIQ==", - "license": "MIT" - }, - "node_modules/nyc": { - "version": "15.1.0", - "resolved": "https://registry.npmjs.org/nyc/-/nyc-15.1.0.tgz", - "integrity": "sha512-jMW04n9SxKdKi1ZMGhvUTHBN0EICCRkHemEoE5jm6mTYcqcdas0ATzgUgejlQUHMvpnOZqGB5Xxsv9KxJW1j8A==", - "dev": true, - "license": "ISC", - "dependencies": { - "@istanbuljs/load-nyc-config": "^1.0.0", - "@istanbuljs/schema": "^0.1.2", - "caching-transform": "^4.0.0", - "convert-source-map": "^1.7.0", - "decamelize": "^1.2.0", - "find-cache-dir": "^3.2.0", - "find-up": "^4.1.0", - "foreground-child": "^2.0.0", - "get-package-type": "^0.1.0", - "glob": "^7.1.6", - "istanbul-lib-coverage": "^3.0.0", - "istanbul-lib-hook": "^3.0.0", - "istanbul-lib-instrument": "^4.0.0", - "istanbul-lib-processinfo": "^2.0.2", - "istanbul-lib-report": "^3.0.0", - "istanbul-lib-source-maps": "^4.0.0", - "istanbul-reports": "^3.0.2", - "make-dir": "^3.0.0", - "node-preload": "^0.2.1", - "p-map": "^3.0.0", - "process-on-spawn": "^1.0.0", - "resolve-from": "^5.0.0", - "rimraf": "^3.0.0", - "signal-exit": "^3.0.2", - "spawn-wrap": "^2.0.0", - "test-exclude": "^6.0.0", - "yargs": "^15.0.2" - }, - "bin": { - "nyc": "bin/nyc.js" - }, - "engines": { - "node": ">=8.9" - } - }, - "node_modules/nyc/node_modules/cliui": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", - "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^6.2.0" - } - }, - "node_modules/nyc/node_modules/convert-source-map": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz", - "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==", - "dev": true, - "license": "MIT" - }, - "node_modules/nyc/node_modules/find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/nyc/node_modules/locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/nyc/node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/nyc/node_modules/p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/nyc/node_modules/resolve-from": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", - "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/nyc/node_modules/wrap-ansi": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", - "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/nyc/node_modules/y18n": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", - "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/nyc/node_modules/yargs": { - "version": "15.4.1", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", - "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", - "dev": true, - "license": "MIT", - "dependencies": { - "cliui": "^6.0.0", - "decamelize": "^1.2.0", - "find-up": "^4.1.0", - "get-caller-file": "^2.0.1", - "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", - "string-width": "^4.2.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^18.1.2" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/nyc/node_modules/yargs-parser": { - "version": "18.1.3", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", - "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -19046,16 +15422,6 @@ "node": ">=0.10.0" } }, - "node_modules/object-hash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", - "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">= 6" - } - }, "node_modules/object-inspect": { "version": "1.13.4", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", @@ -19130,12 +15496,6 @@ ], "license": "MIT" }, - "node_modules/omggif": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/omggif/-/omggif-1.0.10.tgz", - "integrity": "sha512-LMJTtvgc/nugXj0Vcrrs68Mn2D1r0zf630VNtqtpI1FEO7e+O9FP4gqs9AcnBaSEeoHIPm28u6qgPR0oyEpGSw==", - "license": "MIT" - }, "node_modules/on-exit-leak-free": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", @@ -19175,15 +15535,6 @@ "wrappy": "1" } }, - "node_modules/one-time": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", - "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", - "license": "MIT", - "dependencies": { - "fn.name": "1.x.x" - } - }, "node_modules/open": { "version": "10.2.0", "resolved": "https://registry.npmjs.org/open/-/open-10.2.0.tgz", @@ -19249,6 +15600,7 @@ "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, "license": "MIT", "dependencies": { "deep-is": "^0.1.3", @@ -19263,12 +15615,12 @@ } }, "node_modules/otpauth": { - "version": "9.2.4", - "resolved": "https://registry.npmjs.org/otpauth/-/otpauth-9.2.4.tgz", - "integrity": "sha512-t0Nioq2Up2ZaT5AbpXZLTjrsNtLc/g/rVSaEThmKLErAuT9mrnAKJryiPOKc3rCH+3ycWBgKpRHYn+DHqfaPiQ==", + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/otpauth/-/otpauth-9.5.0.tgz", + "integrity": "sha512-Ldhc6UYl4baR5toGr8nfKC+L/b8/RgHKoIixAebgoNGzUUCET02g04rMEZ2ZsPfeVQhMHcuaOgb28nwMr81zCA==", "license": "MIT", "dependencies": { - "jssha": "~3.3.1" + "@noble/hashes": "2.0.1" }, "funding": { "url": "https://github.com/hectorm/otpauth?sponsor=1" @@ -19295,6 +15647,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, "license": "MIT", "dependencies": { "yocto-queue": "^0.1.0" @@ -19310,6 +15663,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, "license": "MIT", "dependencies": { "p-limit": "^3.0.2" @@ -19322,16 +15676,16 @@ } }, "node_modules/p-map": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", - "integrity": "sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==", + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.4.tgz", + "integrity": "sha512-tkAQEw8ysMzmkhgw8k+1U/iPhWNhykKnSk4Rd5zLoPJCuJaGRPo6YposrZgaxHKzDHdDWWZvE/Sk7hsL2X/CpQ==", "dev": true, "license": "MIT", - "dependencies": { - "aggregate-error": "^3.0.0" - }, "engines": { - "node": ">=8" + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/p-retry": { @@ -19356,28 +15710,6 @@ "node": ">=6" } }, - "node_modules/package-hash": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/package-hash/-/package-hash-4.0.0.tgz", - "integrity": "sha512-whdkPIooSu/bASggZ96BWVvZTRMOFxnyUG5PnTSGKoJE2gd5mbVNmR2Nj20QFzxYYgAXpoqC+AiXzl+UMRh7zQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "graceful-fs": "^4.1.15", - "hasha": "^5.0.0", - "lodash.flattendeep": "^4.4.0", - "release-zalgo": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pako": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", - "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", - "license": "(MIT AND Zlib)" - }, "node_modules/param-case": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz", @@ -19393,6 +15725,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, "license": "MIT", "dependencies": { "callsites": "^3.0.0" @@ -19401,41 +15734,6 @@ "node": ">=6" } }, - "node_modules/parse-bmfont-ascii": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/parse-bmfont-ascii/-/parse-bmfont-ascii-1.0.6.tgz", - "integrity": "sha512-U4RrVsUFCleIOBsIGYOMKjn9PavsGOXxbvYGtMOEfnId0SVNsgehXh1DxUdVPLoxd5mvcEtvmKs2Mmf0Mpa1ZA==", - "license": "MIT" - }, - "node_modules/parse-bmfont-binary": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/parse-bmfont-binary/-/parse-bmfont-binary-1.0.6.tgz", - "integrity": "sha512-GxmsRea0wdGdYthjuUeWTMWPqm2+FAd4GI8vCvhgJsFnoGhTrLhXDDupwTo7rXVAgaLIGoVHDZS9p/5XbSqeWA==", - "license": "MIT" - }, - "node_modules/parse-bmfont-xml": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/parse-bmfont-xml/-/parse-bmfont-xml-1.1.6.tgz", - "integrity": "sha512-0cEliVMZEhrFDwMh4SxIyVJpqYoOWDJ9P895tFuS+XuNzI5UBmBk5U5O4KuJdTnZpSBI4LFA2+ZiJaiwfSwlMA==", - "license": "MIT", - "dependencies": { - "xml-parse-from-string": "^1.0.0", - "xml2js": "^0.5.0" - } - }, - "node_modules/parse-bmfont-xml/node_modules/xml2js": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", - "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==", - "license": "MIT", - "dependencies": { - "sax": ">=0.6.0", - "xmlbuilder": "~11.0.0" - }, - "engines": { - "node": ">=4.0.0" - } - }, "node_modules/parse-domain": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/parse-domain/-/parse-domain-8.3.0.tgz", @@ -19449,18 +15747,31 @@ } }, "node_modules/parse5": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.0.tgz", - "integrity": "sha512-9m4m5GSgXjL4AjumKzq1Fgfp3Z8rsvjRNbnkVwfu2ImRqE5D0LnY2QfDen18FSY9C573YU5XxSapdHZTZ2WolA==", + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz", + "integrity": "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==", "dev": true, "license": "MIT", "dependencies": { - "entities": "^6.0.0" + "entities": "^8.0.0" }, "funding": { "url": "https://github.com/inikulin/parse5?sponsor=1" } }, + "node_modules/parse5/node_modules/entities": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-8.0.0.tgz", + "integrity": "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -19485,6 +15796,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -19518,6 +15830,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -19530,10 +15843,14 @@ "license": "MIT" }, "node_modules/path-to-regexp": { - "version": "0.1.13", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", - "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", - "license": "MIT" + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } }, "node_modules/path-type": { "version": "4.0.0", @@ -19561,12 +15878,6 @@ "node": "*" } }, - "node_modules/pg-connection-string": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", - "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==", - "license": "MIT" - }, "node_modules/pg-int8": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", @@ -19602,6 +15913,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, "license": "ISC" }, "node_modules/picomatch": { @@ -19654,27 +15966,6 @@ "integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==", "license": "MIT" }, - "node_modules/pixelmatch": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.3.0.tgz", - "integrity": "sha512-o8mkY4E/+LNUf6LzX96ht6k6CEDi65k9G2rjMtBe9Oo+VPKSvl+0GKHuH/AlG+GA5LPG/i5hrekkxUc3s2HU+Q==", - "license": "ISC", - "dependencies": { - "pngjs": "^6.0.0" - }, - "bin": { - "pixelmatch": "bin/pixelmatch" - } - }, - "node_modules/pixelmatch/node_modules/pngjs": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-6.0.0.tgz", - "integrity": "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==", - "license": "MIT", - "engines": { - "node": ">=12.13.0" - } - }, "node_modules/pkg-dir": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", @@ -19791,15 +16082,6 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/pngjs": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz", - "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==", - "license": "MIT", - "engines": { - "node": ">=14.19.0" - } - }, "node_modules/portfinder": { "version": "1.0.38", "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.38.tgz", @@ -19824,9 +16106,9 @@ } }, "node_modules/postcss": { - "version": "8.5.9", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.9.tgz", - "integrity": "sha512-7a70Nsot+EMX9fFU3064K/kdHWZqGVY+BADLyXc8Dfv+mTLLVl6JzJpPaCZ2kQL9gIJvKXSLMHhqdRRjwQeFtw==", + "version": "8.5.10", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.10.tgz", + "integrity": "sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==", "dev": true, "funding": [ { @@ -19922,11 +16204,41 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.8.0" } }, + "node_modules/prettier": { + "version": "3.8.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.3.tgz", + "integrity": "sha512-7igPTM53cGHMW8xWuVTydi2KO233VFiTNyF5hLJqpilHfmn8C8gPf+PS7dUT64YcXFbiMGZxS9pCSxL/Dxm/Jw==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/prettier-linter-helpers": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.1.tgz", + "integrity": "sha512-SxToR7P8Y2lWmv/kTzVLC1t/GDI2WGjMwNhLLE9qtH8Q13C+aEmuRlzDst4Up4s0Wc8sF2M+J57iB3cMLqftfg==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-diff": "^1.1.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/pretty-error": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/pretty-error/-/pretty-error-4.0.0.tgz", @@ -19948,19 +16260,6 @@ "node": ">= 0.6.0" } }, - "node_modules/process-on-spawn": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/process-on-spawn/-/process-on-spawn-1.1.0.tgz", - "integrity": "sha512-JOnOPQ/8TZgjs1JIH/m9ni7FfimjNa/PRx7y/Wb5qdItsnhO0jE4AT7fC0HjC28DUQWDr50dwSYZLdRMlqDq3Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "fromentries": "^1.2.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/process-warning": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", @@ -19977,16 +16276,6 @@ ], "license": "MIT" }, - "node_modules/progress": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", - "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", - "license": "MIT", - "peer": true, - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/prompt-sync": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/prompt-sync/-/prompt-sync-4.2.0.tgz", @@ -20017,54 +16306,10 @@ "node": ">=6" } }, - "node_modules/proto3-json-serializer": { - "version": "0.1.9", - "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-0.1.9.tgz", - "integrity": "sha512-A60IisqvnuI45qNRygJjrnNjX2TMdQGMY+57tR3nul3ZgO2zXkR9OGR8AXxJhkqx84g0FTnrfi3D5fWMSdANdQ==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "protobufjs": "^6.11.2" - } - }, - "node_modules/proto3-json-serializer/node_modules/long": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", - "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", - "license": "Apache-2.0", - "optional": true - }, - "node_modules/proto3-json-serializer/node_modules/protobufjs": { - "version": "6.11.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.4.tgz", - "integrity": "sha512-5kQWPaJHi1WoCpjTGszzQ32PG2F4+wRY6BmAT4Vfw56Q2FZ4YZzK20xUYQH4YkfehY1e6QSICrJquM6xXZNcrw==", - "hasInstallScript": true, - "license": "BSD-3-Clause", - "optional": true, - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/long": "^4.0.1", - "@types/node": ">=13.7.0", - "long": "^4.0.0" - }, - "bin": { - "pbjs": "bin/pbjs", - "pbts": "bin/pbts" - } - }, "node_modules/protobufjs": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", - "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz", + "integrity": "sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -20116,17 +16361,6 @@ "node": ">=10" } }, - "node_modules/proxyquire": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/proxyquire/-/proxyquire-2.1.3.tgz", - "integrity": "sha512-BQWfCqYM+QINd+yawJz23tbBM40VIGXOdDw3X344KcclI/gtBbdWF6SlQ4nK/bYhF9d27KYug9WzljHC6B9Ysg==", - "license": "MIT", - "dependencies": { - "fill-keys": "^1.0.2", - "module-not-found-error": "^1.0.1", - "resolve": "^1.11.1" - } - }, "node_modules/pstree.remy": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", @@ -20144,29 +16378,19 @@ "once": "^1.3.1" } }, - "node_modules/pumpify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-2.0.1.tgz", - "integrity": "sha512-m7KOje7jZxrmutanlkS1daj1dS6z6BgslzOXmcSEpIlCxM3VJH7lG5QLeck/6hgF6F4crFf01UtQmNsJfweTAw==", + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", "license": "MIT", - "optional": true, - "dependencies": { - "duplexify": "^4.1.1", - "inherits": "^2.0.3", - "pump": "^3.0.0" + "engines": { + "node": ">=6" } }, - "node_modules/punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==", - "license": "MIT", - "optional": true - }, "node_modules/qs": { - "version": "6.14.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz", - "integrity": "sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==", + "version": "6.15.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.1.tgz", + "integrity": "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg==", "license": "BSD-3-Clause", "dependencies": { "side-channel": "^1.1.0" @@ -20215,18 +16439,18 @@ } }, "node_modules/raw-body": { - "version": "2.5.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", - "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", "license": "MIT", "dependencies": { "bytes": "~3.1.2", "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", + "iconv-lite": "~0.7.0", "unpipe": "~1.0.0" }, "engines": { - "node": ">= 0.8" + "node": ">= 0.10" } }, "node_modules/rc": { @@ -20313,6 +16537,7 @@ "version": "0.8.0", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", "integrity": "sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==", + "dev": true, "license": "MIT", "dependencies": { "resolve": "^1.20.0" @@ -20321,46 +16546,6 @@ "node": ">= 10.13.0" } }, - "node_modules/recursive-readdir": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.3.tgz", - "integrity": "sha512-8HrF5ZsXk5FAH9dgsx3BlUer73nIhuj+9OrQwEbLTPOBzGkL1lsFCR01am+v+0m2Cmbs1nP12hLDl5FA7EszKA==", - "license": "MIT", - "dependencies": { - "minimatch": "^3.0.5" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/recursive-readdir/node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "license": "MIT" - }, - "node_modules/recursive-readdir/node_modules/brace-expansion": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.13.tgz", - "integrity": "sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w==", - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/recursive-readdir/node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, "node_modules/redis-errors": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", @@ -20440,19 +16625,6 @@ "integrity": "sha512-qaZBjmRIuXLfuLnzgqpFdBPa5W0euSX1tMnoMUHGPphLwJmrt8xbNiOIHrlvYOD6oNJ0M5owPCZyPibI8de5pQ==", "license": "MIT" }, - "node_modules/release-zalgo": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/release-zalgo/-/release-zalgo-1.0.0.tgz", - "integrity": "sha512-gUAyHVHPPC5wdqX/LG4LWtRYtgjxyX78oanFNTMMyFEfOqdC54s3eE82imuWKbOeqYht2CrNf64Qb8vgmmtZGA==", - "dev": true, - "license": "ISC", - "dependencies": { - "es6-error": "^4.0.1" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/renderkid": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/renderkid/-/renderkid-3.0.0.tgz", @@ -20545,11 +16717,12 @@ "license": "MIT" }, "node_modules/resolve": { - "version": "1.22.11", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", - "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==", + "version": "1.22.12", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", + "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==", "license": "MIT", "dependencies": { + "es-errors": "^1.3.0", "is-core-module": "^2.16.1", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" @@ -20591,24 +16764,12 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, "license": "MIT", "engines": { "node": ">=4" } }, - "node_modules/response-time": { - "version": "2.3.4", - "resolved": "https://registry.npmjs.org/response-time/-/response-time-2.3.4.tgz", - "integrity": "sha512-fiyq1RvW5/Br6iAtT8jN1XrNY8WPu2+yEypLbaijWry8WDZmn12azG9p/+c+qpEebURLlQmqCB8BNSu7ji+xQQ==", - "license": "MIT", - "dependencies": { - "depd": "~2.0.0", - "on-headers": "~1.1.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, "node_modules/ret": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/ret/-/ret-0.5.0.tgz", @@ -20627,20 +16788,6 @@ "node": ">= 4" } }, - "node_modules/retry-request": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.2.2.tgz", - "integrity": "sha512-xA93uxUD/rogV7BV59agW/JHPGXeREMWiZc9jhcwY4YdZ7QOtC7qbomYg0n4wyk2lJhggjvKvhNX8wln/Aldhg==", - "license": "MIT", - "optional": true, - "dependencies": { - "debug": "^4.1.1", - "extend": "^3.0.2" - }, - "engines": { - "node": ">=8.10.0" - } - }, "node_modules/reusify": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", @@ -20674,14 +16821,14 @@ } }, "node_modules/rolldown": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.15.tgz", - "integrity": "sha512-Ff31guA5zT6WjnGp0SXw76X6hzGRk/OQq2hE+1lcDe+lJdHSgnSX6nK3erbONHyCbpSj9a9E+uX/OvytZoWp2g==", + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.16.tgz", + "integrity": "sha512-rzi5WqKzEZw3SooTt7cgm4eqIoujPIyGcJNGFL7iPEuajQw7vxMHUkXylu4/vhCkJGXsgRmxqMKXUpT6FEgl0g==", "dev": true, "license": "MIT", "dependencies": { - "@oxc-project/types": "=0.124.0", - "@rolldown/pluginutils": "1.0.0-rc.15" + "@oxc-project/types": "=0.126.0", + "@rolldown/pluginutils": "1.0.0-rc.16" }, "bin": { "rolldown": "bin/cli.mjs" @@ -20690,28 +16837,55 @@ "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rolldown/binding-android-arm64": "1.0.0-rc.15", - "@rolldown/binding-darwin-arm64": "1.0.0-rc.15", - "@rolldown/binding-darwin-x64": "1.0.0-rc.15", - "@rolldown/binding-freebsd-x64": "1.0.0-rc.15", - "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.15", - "@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.15", - "@rolldown/binding-linux-arm64-musl": "1.0.0-rc.15", - "@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.15", - "@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.15", - "@rolldown/binding-linux-x64-gnu": "1.0.0-rc.15", - "@rolldown/binding-linux-x64-musl": "1.0.0-rc.15", - "@rolldown/binding-openharmony-arm64": "1.0.0-rc.15", - "@rolldown/binding-wasm32-wasi": "1.0.0-rc.15", - "@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.15", - "@rolldown/binding-win32-x64-msvc": "1.0.0-rc.15" + "@rolldown/binding-android-arm64": "1.0.0-rc.16", + "@rolldown/binding-darwin-arm64": "1.0.0-rc.16", + "@rolldown/binding-darwin-x64": "1.0.0-rc.16", + "@rolldown/binding-freebsd-x64": "1.0.0-rc.16", + "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.16", + "@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.16", + "@rolldown/binding-linux-arm64-musl": "1.0.0-rc.16", + "@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.16", + "@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.16", + "@rolldown/binding-linux-x64-gnu": "1.0.0-rc.16", + "@rolldown/binding-linux-x64-musl": "1.0.0-rc.16", + "@rolldown/binding-openharmony-arm64": "1.0.0-rc.16", + "@rolldown/binding-wasm32-wasi": "1.0.0-rc.16", + "@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.16", + "@rolldown/binding-win32-x64-msvc": "1.0.0-rc.16" } }, - "node_modules/rrweb-cssom": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz", - "integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==", - "license": "MIT" + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/router/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } }, "node_modules/run-applescript": { "version": "7.1.0", @@ -20760,14 +16934,14 @@ } }, "node_modules/safe-array-concat": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.3.tgz", - "integrity": "sha512-AURm5f0jYEOydBj7VQlVvDrjeFgthDdEF5H1dP+6mNpoXOMo1quQqJ4wvJDyRZ9+pO3kGWoOdmV08cSv2aJV6Q==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.4.tgz", + "integrity": "sha512-wtZlHyOje6OZTGqAoaDKxFkgRtkF9CnHAVnCHKfuj200wAgL+bSJhdsCD2l0Qx/2ekEXjPWcyKkfGb5CPboslg==", "license": "MIT", "dependencies": { - "call-bind": "^1.0.8", - "call-bound": "^1.0.2", - "get-intrinsic": "^1.2.6", + "call-bind": "^1.0.9", + "call-bound": "^1.0.4", + "get-intrinsic": "^1.3.0", "has-symbols": "^1.1.0", "isarray": "^2.0.5" }, @@ -20832,9 +17006,9 @@ } }, "node_modules/safe-regex2": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/safe-regex2/-/safe-regex2-5.1.0.tgz", - "integrity": "sha512-pNHAuBW7TrcleFHsxBr5QMi/Iyp0ENjUKz7GCcX1UO7cMh+NmVK6HxQckNL1tJp1XAJVjG6B8OKIPqodqj9rtw==", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/safe-regex2/-/safe-regex2-5.1.1.tgz", + "integrity": "sha512-mOSBvHGDZMuIEZMdOz/aCEYDCv0E7nfcNsIhUF+/P+xC7Hyf3FkvymqgPbg9D1EdSGu+uKbJgy09K/RKKc7kJA==", "funding": [ { "type": "github", @@ -20868,15 +17042,6 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, - "node_modules/sax": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz", - "integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==", - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=11.0.0" - } - }, "node_modules/saxes": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", @@ -20915,7 +17080,6 @@ "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -20988,12 +17152,6 @@ ], "license": "BSD-3-Clause" }, - "node_modules/seedrandom": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", - "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==", - "license": "MIT" - }, "node_modules/semver": { "version": "7.7.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", @@ -21007,69 +17165,81 @@ } }, "node_modules/send": { - "version": "0.19.2", - "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", - "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", "license": "MIT", "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "~0.5.2", - "http-errors": "~2.0.1", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "~2.4.1", - "range-parser": "~1.2.1", - "statuses": "~2.0.2" + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" }, "engines": { - "node": ">= 0.8.0" + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/send/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/send/node_modules/debug/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, - "node_modules/send/node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "license": "MIT", - "bin": { - "mime": "cli.js" + "ms": "^2.1.3" }, "engines": { - "node": ">=4" + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/send/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/serve-static": { - "version": "1.16.3", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", - "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", "license": "MIT", "dependencies": { - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "~0.19.1" + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" }, "engines": { - "node": ">= 0.8.0" + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/set-blocking": { @@ -21193,31 +17363,11 @@ "@img/sharp-win32-x64": "0.34.5" } }, - "node_modules/sharp-bmp": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/sharp-bmp/-/sharp-bmp-0.1.5.tgz", - "integrity": "sha512-IpWAy+AeTlWNHiBU8HH4atcKbztgKOXTuT4W8aFaeASPCeJwCVpoUymWMfEmwfvWSCOV1s7VmGTlKhcPLkt+Lw==", - "license": "MIT", - "dependencies": { - "bmp-js": "*", - "sharp": "*" - } - }, - "node_modules/sharp-ico": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/sharp-ico/-/sharp-ico-0.1.5.tgz", - "integrity": "sha512-a3jODQl82NPp1d5OYb0wY+oFaPk7AvyxipIowCHk7pBsZCWgbe0yAkU2OOXdoH0ENyANhyOQbs9xkAiRHcF02Q==", - "license": "MIT", - "dependencies": { - "decode-ico": "*", - "ico-endec": "*", - "sharp": "*" - } - }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" @@ -21230,6 +17380,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -21248,43 +17399,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/shescape": { - "version": "2.1.11", - "resolved": "https://registry.npmjs.org/shescape/-/shescape-2.1.11.tgz", - "integrity": "sha512-kR+oVEEgfo2TzK6FZAXtZMZ4aaVFuy5WeToxgjh95KPqUOLoFb1M+PYOWrDV7QqfHgdwnBpPBeBbf07sDgyAtA==", - "license": "MPL-2.0", - "dependencies": { - "@ericcornelissen/lregexp": "^1.0.7", - "which": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0" - }, - "engines": { - "node": "^14.18.0 || ^16.13.0 || ^18 || ^19 || ^20 || ^22 || ^24 || ^25" - } - }, - "node_modules/shescape/node_modules/isexe": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-4.0.0.tgz", - "integrity": "sha512-FFUtZMpoZ8RqHS3XeXEmHWLA4thH+ZxCv2lOiPIn1Xc7CxrqhWzNSDzD+/chS/zbYezmiwWLdQC09JdQKmthOw==", - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=20" - } - }, - "node_modules/shescape/node_modules/which": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/which/-/which-6.0.1.tgz", - "integrity": "sha512-oGLe46MIrCRqX7ytPUf66EAYvdeMIZYn3WaocqqKZAxrBpkqHfL/qvTyJ/bTk5+AqHCjXmrv3CEWgy368zhRUg==", - "license": "ISC", - "dependencies": { - "isexe": "^4.0.0" - }, - "bin": { - "node-which": "bin/which.js" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, "node_modules/shimmer": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", @@ -21422,16 +17536,16 @@ } }, "node_modules/simple-git": { - "version": "3.35.2", - "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.35.2.tgz", - "integrity": "sha512-ZMjl06lzTm1EScxEGuM6+mEX+NQd14h/B3x0vWU+YOXAMF8sicyi1K4cjTfj5is+35ChJEHDl1EjypzYFWH2FA==", + "version": "3.36.0", + "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.36.0.tgz", + "integrity": "sha512-cGQjLjK8bxJw4QuYT7gxHw3/IouVESbhahSsHrX97MzCL1gu2u7oy38W6L2ZIGECEfIBG4BabsWDPjBxJENv9Q==", "dev": true, "license": "MIT", "dependencies": { "@kwsites/file-exists": "^1.1.1", "@kwsites/promise-deferred": "^1.1.1", - "@simple-git/args-pathspec": "^1.0.2", - "@simple-git/argv-parser": "^1.0.3", + "@simple-git/args-pathspec": "^1.0.3", + "@simple-git/argv-parser": "^1.1.0", "debug": "^4.4.0" }, "funding": { @@ -21470,21 +17584,6 @@ "node": ">=10" } }, - "node_modules/simple-wcswidth": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/simple-wcswidth/-/simple-wcswidth-1.1.2.tgz", - "integrity": "sha512-j7piyCjAeTDSjzTSQ7DokZtMNwNlEAyxqSZeCS+CXH7fJ4jx3FuJ/mTW3mE+6JLs4VJBbcll0Kjn+KXI5t21Iw==", - "license": "MIT" - }, - "node_modules/simple-xml-to-json": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/simple-xml-to-json/-/simple-xml-to-json-1.2.7.tgz", - "integrity": "sha512-mz9VXphOxQWX3eQ/uXCtm6upltoN0DLx8Zb5T4TFC4FHB7S9FDPGre8CfLWqPWQQH/GrQYd2AXhhVM5LDpYx6Q==", - "license": "MIT", - "engines": { - "node": ">=20.12.2" - } - }, "node_modules/sinon": { "version": "15.2.0", "resolved": "https://registry.npmjs.org/sinon/-/sinon-15.2.0.tgz", @@ -21563,7 +17662,6 @@ "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-2.5.6.tgz", "integrity": "sha512-DkkO/dz7MGln0dHn5bmN3pPy+JmywNICWrJqVWiVOyvXjWQFIv9c2h24JrQLLFJ2aQVQf/Cvl1vblnd4r2apLQ==", "license": "MIT", - "peer": true, "dependencies": { "debug": "~4.4.1", "ws": "~8.18.3" @@ -21607,38 +17705,6 @@ } } }, - "node_modules/socket.io-client": { - "version": "4.8.3", - "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-4.8.3.tgz", - "integrity": "sha512-uP0bpjWrjQmUt5DTHq9RuoCBdFJF10cdX9X+a368j/Ft0wmaVgxlrjvK3kjvgCODOMMOz9lcaRzxmso0bTWZ/g==", - "license": "MIT", - "dependencies": { - "@socket.io/component-emitter": "~3.1.0", - "debug": "~4.4.1", - "engine.io-client": "~6.6.1", - "socket.io-parser": "~4.2.4" - }, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/socket.io-client/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/socket.io-parser": { "version": "4.2.6", "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.6.tgz", @@ -21669,6 +17735,19 @@ } } }, + "node_modules/socket.io/node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/socket.io/node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -21686,6 +17765,15 @@ } } }, + "node_modules/socket.io/node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/sonic-boom": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz", @@ -21708,6 +17796,7 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, "license": "BSD-3-Clause", "engines": { "node": ">=0.10.0" @@ -21730,24 +17819,6 @@ "integrity": "sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==", "dev": true }, - "node_modules/spawn-wrap": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/spawn-wrap/-/spawn-wrap-2.0.0.tgz", - "integrity": "sha512-EeajNjfN9zMnULLwhZZQU3GWBoFNkbngTUPfaawT4RkMiviTxcX0qfhVbGey39mfctfDHkWtuecgQ8NJcyQWHg==", - "dev": true, - "license": "ISC", - "dependencies": { - "foreground-child": "^2.0.0", - "is-windows": "^1.0.2", - "make-dir": "^3.0.0", - "rimraf": "^3.0.0", - "signal-exit": "^3.0.2", - "which": "^2.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/split2": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", @@ -21763,30 +17834,19 @@ "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", "license": "BSD-3-Clause" }, - "node_modules/ssh2": { - "version": "1.17.0", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", - "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", - "hasInstallScript": true, - "dependencies": { - "asn1": "^0.2.6", - "bcrypt-pbkdf": "^1.0.2" - }, - "engines": { - "node": ">=10.16.0" - }, - "optionalDependencies": { - "cpu-features": "~0.0.10", - "nan": "^2.23.0" - } - }, - "node_modules/stack-trace": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", - "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", + "node_modules/sql-escaper": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/sql-escaper/-/sql-escaper-1.3.3.tgz", + "integrity": "sha512-BsTCV265VpTp8tm1wyIm1xqQCS+Q9NHx2Sr+WcnUrgLrQ6yiDIvHYJV5gHxsj1lMBy2zm5twLaZao8Jd+S8JJw==", "license": "MIT", "engines": { - "node": "*" + "bun": ">=1.0.0", + "deno": ">=2.0.0", + "node": ">=12.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/mysqljs/sql-escaper?sponsor=1" } }, "node_modules/stackback": { @@ -21812,9 +17872,9 @@ } }, "node_modules/std-env": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.0.0.tgz", - "integrity": "sha512-zUMPtQ/HBY3/50VbpkupYHbRroTRZJPRLvreamgErJVys0ceuzMkD44J/QjqhHjOzK42GQ3QZIeFG1OYfOtKqQ==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.1.0.tgz", + "integrity": "sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==", "dev": true, "license": "MIT" }, @@ -21831,23 +17891,6 @@ "node": ">= 0.4" } }, - "node_modules/stream-events": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", - "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", - "license": "MIT", - "optional": true, - "dependencies": { - "stubs": "^3.0.0" - } - }, - "node_modules/stream-shift": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", - "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", - "license": "MIT", - "optional": true - }, "node_modules/streamsearch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", @@ -21865,54 +17908,6 @@ "safe-buffer": "~5.2.0" } }, - "node_modules/string-hash": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/string-hash/-/string-hash-1.1.3.tgz", - "integrity": "sha512-kJUvRUFK49aub+a7T1nNE66EJbZBMnBgoC1UbCZ5n6bsZKBRga4KgBRTMn/pFkeCZSYtNeSyMxPDM0AXWELk2A==", - "license": "CC0-1.0" - }, - "node_modules/string-length": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/string-length/-/string-length-6.0.0.tgz", - "integrity": "sha512-1U361pxZHEQ+FeSjzqRpV+cu2vTzYeWeafXFLykiFlv4Vc0n3njgU8HrMbyik5uwm77naWMuVG8fhEF+Ovb1Kg==", - "license": "MIT", - "dependencies": { - "strip-ansi": "^7.1.0" - }, - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/string-length/node_modules/ansi-regex": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", - "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/string-length/node_modules/strip-ansi": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", - "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^6.2.2" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, "node_modules/string-template": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/string-template/-/string-template-1.0.0.tgz", @@ -22001,20 +17996,11 @@ "node": ">=8" } }, - "node_modules/strip-bom": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", - "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -22051,13 +18037,6 @@ "url": "https://github.com/sponsors/Borewit" } }, - "node_modules/stubs": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", - "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==", - "license": "MIT", - "optional": true - }, "node_modules/super-regex": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/super-regex/-/super-regex-0.2.0.tgz", @@ -22079,6 +18058,7 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, "license": "MIT", "dependencies": { "has-flag": "^4.0.0" @@ -22111,136 +18091,28 @@ "node": ">=4.x" } }, - "node_modules/svgo": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/svgo/-/svgo-3.3.3.tgz", - "integrity": "sha512-+wn7I4p7YgJhHs38k2TNjy1vCfPIfLIJWR5MnCStsN8WuuTcBnRKcMHQLMM2ijxGZmDoZwNv8ipl5aTTen62ng==", - "license": "MIT", - "dependencies": { - "commander": "^7.2.0", - "css-select": "^5.1.0", - "css-tree": "^2.3.1", - "css-what": "^6.1.0", - "csso": "^5.0.5", - "picocolors": "^1.0.0", - "sax": "^1.5.0" - }, - "bin": { - "svgo": "bin/svgo" - }, - "engines": { - "node": ">=14.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/svgo" - } - }, - "node_modules/svgo/node_modules/commander": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", - "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", - "license": "MIT", - "engines": { - "node": ">= 10" - } - }, - "node_modules/svgo/node_modules/css-select": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", - "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", - "license": "BSD-2-Clause", - "dependencies": { - "boolbase": "^1.0.0", - "css-what": "^6.1.0", - "domhandler": "^5.0.2", - "domutils": "^3.0.1", - "nth-check": "^2.0.1" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, - "node_modules/svgo/node_modules/css-tree": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.3.1.tgz", - "integrity": "sha512-6Fv1DV/TYw//QF5IzQdqsNDjx/wc8TrMBZsqjL9eW01tWb7R7k/mq+/VXfJCl7SoD5emsJop9cOByJZfs8hYIw==", - "license": "MIT", - "dependencies": { - "mdn-data": "2.0.30", - "source-map-js": "^1.0.1" - }, - "engines": { - "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" - } - }, - "node_modules/svgo/node_modules/dom-serializer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", - "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.2", - "entities": "^4.2.0" - }, - "funding": { - "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" - } - }, - "node_modules/svgo/node_modules/domhandler": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", - "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", - "license": "BSD-2-Clause", - "dependencies": { - "domelementtype": "^2.3.0" - }, - "engines": { - "node": ">= 4" - }, - "funding": { - "url": "https://github.com/fb55/domhandler?sponsor=1" - } - }, - "node_modules/svgo/node_modules/domutils": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", - "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", - "license": "BSD-2-Clause", - "dependencies": { - "dom-serializer": "^2.0.0", - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3" - }, - "funding": { - "url": "https://github.com/fb55/domutils?sponsor=1" - } - }, - "node_modules/svgo/node_modules/entities": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", - "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/svgo/node_modules/mdn-data": { - "version": "2.0.30", - "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.30.tgz", - "integrity": "sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==", - "license": "CC0-1.0" - }, "node_modules/symbol-tree": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", "license": "MIT" }, + "node_modules/synckit": { + "version": "0.11.12", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.12.tgz", + "integrity": "sha512-Bh7QjT8/SuKUIfObSXNHNSK6WHo6J1tHCqJsuaFDP7gP0fkzSfTxI8y85JrppZ0h8l0maIgc2tfuZQ6/t3GtnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@pkgr/core": "^0.2.9" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/synckit" + } + }, "node_modules/tapable": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.2.tgz", @@ -22319,42 +18191,6 @@ "node": ">=10" } }, - "node_modules/tarn": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.2.tgz", - "integrity": "sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==", - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/teeny-request": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-7.2.0.tgz", - "integrity": "sha512-SyY0pek1zWsi0LRVAALem+avzMLc33MKW/JLLakdP4s9+D7+jHcy5x6P+h94g2QNZsAqQNfX5lsbd3WSeJXrrw==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "http-proxy-agent": "^5.0.0", - "https-proxy-agent": "^5.0.0", - "node-fetch": "^2.6.1", - "stream-events": "^1.0.5", - "uuid": "^8.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/teeny-request/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "license": "MIT", - "optional": true, - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/terser": { "version": "5.46.1", "resolved": "https://registry.npmjs.org/terser/-/terser-5.46.1.tgz", @@ -22415,64 +18251,6 @@ "dev": true, "license": "MIT" }, - "node_modules/test-exclude": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", - "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", - "dev": true, - "license": "ISC", - "dependencies": { - "@istanbuljs/schema": "^0.1.2", - "glob": "^7.1.4", - "minimatch": "^3.0.4" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/test-exclude/node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true, - "license": "MIT" - }, - "node_modules/test-exclude/node_modules/brace-expansion": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.13.tgz", - "integrity": "sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/test-exclude/node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/text-decoding": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/text-decoding/-/text-decoding-1.0.0.tgz", - "integrity": "sha512-/0TJD42KDnVwKmDK6jj3xP7E2MG7SHAOG4tyTgyUCRPdHwvkquYNLEQltmdMa3owq3TkddCVcTsoctJI8VQNKA==", - "license": "MIT" - }, - "node_modules/text-hex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", - "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==", - "license": "MIT" - }, "node_modules/thread-stream": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.0.0.tgz", @@ -22485,21 +18263,6 @@ "node": ">=20" } }, - "node_modules/tiktoken": { - "version": "1.0.22", - "resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.22.tgz", - "integrity": "sha512-PKvy1rVF1RibfF3JlXBSP0Jrcw2uq3yXdgcEXtKTYn3QJ/cBRBHDnrJ5jHky+MENZ6DIPwNUGWpkVx+7joCpNA==", - "license": "MIT" - }, - "node_modules/tildify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz", - "integrity": "sha512-Cc+OraorugtXNfs50hU9KS369rFXCfgGLpfCfvlc+Ud5u6VWmUQsOAa9HbTvheQdYnrdJqqv1e5oIqXppMYnSw==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/time-span": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/time-span/-/time-span-5.1.0.tgz", @@ -22528,12 +18291,6 @@ "dev": true, "license": "MIT" }, - "node_modules/tinycolor2": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/tinycolor2/-/tinycolor2-1.6.0.tgz", - "integrity": "sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==", - "license": "MIT" - }, "node_modules/tinyexec": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.1.1.tgz", @@ -22600,12 +18357,6 @@ "node": ">=14.14" } }, - "node_modules/to-data-view": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/to-data-view/-/to-data-view-1.1.0.tgz", - "integrity": "sha512-1eAdufMg6mwgmlojAx3QeMnzB/BTVp7Tbndi3U7ftcT2zCZadjxkkmLmd97zmaxWi+sgGcgWrokmpEoy0Dn0vQ==", - "license": "MIT" - }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -22645,10 +18396,6 @@ "node": ">=0.6" } }, - "node_modules/token-count-accuracy": { - "resolved": "tools/token-count-accuracy", - "link": true - }, "node_modules/token-types": { "version": "6.1.2", "resolved": "https://registry.npmjs.org/token-types/-/token-types-6.1.2.tgz", @@ -22701,27 +18448,10 @@ } }, "node_modules/tr46": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-6.0.0.tgz", - "integrity": "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==", - "dev": true, - "license": "MIT", - "dependencies": { - "punycode": "^2.3.1" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/tr46/node_modules/punycode": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", - "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" }, "node_modules/tree-kill": { "version": "1.2.2", @@ -22733,15 +18463,6 @@ "tree-kill": "cli.js" } }, - "node_modules/triple-beam": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", - "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==", - "license": "MIT", - "engines": { - "node": ">= 14.0.0" - } - }, "node_modules/ts-algebra": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", @@ -22779,16 +18500,11 @@ "node": "*" } }, - "node_modules/tweetnacl": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-1.0.3.tgz", - "integrity": "sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==", - "license": "Unlicense" - }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, "license": "MIT", "dependencies": { "prelude-ls": "^1.2.1" @@ -22806,36 +18522,34 @@ "node": ">=4" } }, - "node_modules/type-fest": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", - "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", - "dev": true, - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=8" - } - }, "node_modules/type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", + "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", "license": "MIT", "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" + "content-type": "^1.0.5", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" }, "engines": { "node": ">= 0.6" } }, - "node_modules/type-is/node_modules/media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "node_modules/type-is/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/typed-array-buffer": { @@ -22912,29 +18626,12 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/typedarray": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==", - "license": "MIT" - }, - "node_modules/typedarray-to-buffer": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", - "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", - "devOptional": true, - "license": "MIT", - "dependencies": { - "is-typedarray": "^1.0.0" - } - }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "devOptional": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -23019,9 +18716,9 @@ "license": "MIT" }, "node_modules/undici": { - "version": "7.24.7", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.24.7.tgz", - "integrity": "sha512-H/nlJ/h0ggGC+uRL3ovD+G0i4bqhvsDOpbDv7At5eFLlj2b41L8QliGbnl2H7SnDiYhENphh1tQFJZf+MyfLsQ==", + "version": "7.25.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.25.0.tgz", + "integrity": "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==", "dev": true, "license": "MIT", "engines": { @@ -23046,26 +18743,14 @@ "node": ">= 0.8.0" } }, - "node_modules/unique-string": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz", - "integrity": "sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==", - "license": "MIT", - "optional": true, - "dependencies": { - "crypto-random-string": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/universalify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", - "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "dev": true, "license": "MIT", "engines": { - "node": ">= 10.0.0" + "node": ">= 4.0.0" } }, "node_modules/unpipe": { @@ -23112,20 +18797,12 @@ "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, "license": "BSD-2-Clause", "dependencies": { "punycode": "^2.1.0" } }, - "node_modules/uri-js/node_modules/punycode": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", - "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/url-join": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz", @@ -23137,15 +18814,6 @@ "resolved": "src/useapi", "link": true }, - "node_modules/utif2": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/utif2/-/utif2-4.1.0.tgz", - "integrity": "sha512-+oknB9FHrJ7oW7A2WZYajOcv4FcDR4CfoGB0dPNfxbi4GO05RRnFmt5oa23+9w32EanrYcSJWspUiJkLMs+37w==", - "license": "MIT", - "dependencies": { - "pako": "^1.0.11" - } - }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -23159,15 +18827,6 @@ "dev": true, "license": "MIT" }, - "node_modules/utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", - "license": "MIT", - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/uuid": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", @@ -23214,18 +18873,17 @@ } }, "node_modules/vite": { - "version": "8.0.8", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.8.tgz", - "integrity": "sha512-dbU7/iLVa8KZALJyLOBOQ88nOXtNG8vxKuOT4I2mD+Ya70KPceF4IAmDsmU0h1Qsn5bPrvsY9HJstCRh3hG6Uw==", + "version": "8.0.9", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.9.tgz", + "integrity": "sha512-t7g7GVRpMXjNpa67HaVWI/8BWtdVIQPCL2WoozXXA7LBGEFK4AkkKkHx2hAQf5x1GZSlcmEDPkVLSGahxnEEZw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.4", - "postcss": "^8.5.8", - "rolldown": "1.0.0-rc.15", - "tinyglobby": "^0.2.15" + "postcss": "^8.5.10", + "rolldown": "1.0.0-rc.16", + "tinyglobby": "^0.2.16" }, "bin": { "vite": "bin/vite.js" @@ -23368,19 +19026,6 @@ "node": ">= 6" } }, - "node_modules/vite-plugin-static-copy/node_modules/p-map": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.4.tgz", - "integrity": "sha512-tkAQEw8ysMzmkhgw8k+1U/iPhWNhykKnSk4Rd5zLoPJCuJaGRPo6YposrZgaxHKzDHdDWWZvE/Sk7hsL2X/CpQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/vite-plugin-static-copy/node_modules/picomatch": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", @@ -23428,7 +19073,6 @@ "integrity": "sha512-tFuJqTxKb8AvfyqMfnavXdzfy3h3sWZRWwfluGbkeR7n0HUev+FmNgZ8SDrRBTVrVCjgH5cA21qGbCffMNtWvg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@vitest/expect": "4.1.4", "@vitest/mocker": "4.1.4", @@ -23549,22 +19193,17 @@ } }, "node_modules/webidl-conversions": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-8.0.1.tgz", - "integrity": "sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=20" - } + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" }, "node_modules/webpack": { - "version": "5.106.0", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.106.0.tgz", - "integrity": "sha512-Pkx5joZ9RrdgO5LBkyX1L2ZAJeK/Taz3vqZ9CbcP0wS5LEMx5QkKsEwLl29QJfihZ+DKRBFldzy1O30pJ1MDpA==", + "version": "5.106.2", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.106.2.tgz", + "integrity": "sha512-wGN3qcrBQIFmQ/c0AiOAQBvrZ5lmY8vbbMv4Mxfgzqd/B6+9pXtLo73WuS1dSGXM5QYY3hZnIbvx+K1xxe6FyA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", @@ -23582,9 +19221,8 @@ "events": "^3.2.0", "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.2.11", - "json-parse-even-better-errors": "^2.3.1", "loader-runner": "^4.3.1", - "mime-types": "^2.1.27", + "mime-db": "^1.54.0", "neo-async": "^2.6.2", "schema-utils": "^4.3.3", "tapable": "^2.3.0", @@ -23614,7 +19252,6 @@ "integrity": "sha512-pIDJHIEI9LR0yxHXQ+Qh95k2EvXpWzZ5l+d+jIo+RdSm9MiHfzazIxwwni/p7+x4eJZuvG1AJwgC4TNQ7NRgsg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@discoveryjs/json-ext": "^0.5.0", "@webpack-cli/configtest": "^2.1.1", @@ -23665,16 +19302,6 @@ "node": ">=14" } }, - "node_modules/webpack-cli/node_modules/interpret": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/interpret/-/interpret-3.1.1.tgz", - "integrity": "sha512-6xwYfHbajpoF0xLW+iwLkhwgvLoZDfjYfoFNu8ftMoXINzwuymNLd9u/KmwtdT2GbR+/Cz66otEGEVVUHX9QLQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10.13.0" - } - }, "node_modules/webpack-merge": { "version": "5.10.0", "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-5.10.0.tgz", @@ -23724,29 +19351,6 @@ "node": ">=4.0" } }, - "node_modules/websocket-driver": { - "version": "0.7.4", - "resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.4.tgz", - "integrity": "sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg==", - "license": "Apache-2.0", - "dependencies": { - "http-parser-js": ">=0.5.1", - "safe-buffer": ">=5.1.0", - "websocket-extensions": ">=0.1.1" - }, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/websocket-extensions": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz", - "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==", - "license": "Apache-2.0", - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/whatwg-encoding": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz", @@ -23785,24 +19389,20 @@ } }, "node_modules/whatwg-url": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-16.0.1.tgz", - "integrity": "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw==", - "dev": true, + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", "license": "MIT", "dependencies": { - "@exodus/bytes": "^1.11.0", - "tr46": "^6.0.0", - "webidl-conversions": "^8.0.1" - }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" } }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, "license": "ISC", "dependencies": { "isexe": "^2.0.0" @@ -23987,74 +19587,11 @@ "integrity": "sha512-gEIQU4mkgl2OPeoNrWflcJFJ3Ae2BPd4eCsHHA/XikslkIVms/nHhvnvzIZV7VLmBvtFlDOzLt9rrZT+n6D67A==", "license": "MIT" }, - "node_modules/winston": { - "version": "3.19.0", - "resolved": "https://registry.npmjs.org/winston/-/winston-3.19.0.tgz", - "integrity": "sha512-LZNJgPzfKR+/J3cHkxcpHKpKKvGfDZVPS4hfJCc4cCG0CgYzvlD6yE/S3CIL/Yt91ak327YCpiF/0MyeZHEHKA==", - "license": "MIT", - "peer": true, - "dependencies": { - "@colors/colors": "^1.6.0", - "@dabh/diagnostics": "^2.0.8", - "async": "^3.2.3", - "is-stream": "^2.0.0", - "logform": "^2.7.0", - "one-time": "^1.0.0", - "readable-stream": "^3.4.0", - "safe-stable-stringify": "^2.3.1", - "stack-trace": "0.0.x", - "triple-beam": "^1.3.0", - "winston-transport": "^4.9.0" - }, - "engines": { - "node": ">= 12.0.0" - } - }, - "node_modules/winston-daily-rotate-file": { - "version": "4.7.1", - "resolved": "https://registry.npmjs.org/winston-daily-rotate-file/-/winston-daily-rotate-file-4.7.1.tgz", - "integrity": "sha512-7LGPiYGBPNyGHLn9z33i96zx/bd71pjBn9tqQzO3I4Tayv94WPmBNwKC7CO1wPHdP9uvu+Md/1nr6VSH9h0iaA==", - "license": "MIT", - "dependencies": { - "file-stream-rotator": "^0.6.1", - "object-hash": "^2.0.1", - "triple-beam": "^1.3.0", - "winston-transport": "^4.4.0" - }, - "engines": { - "node": ">=8" - }, - "peerDependencies": { - "winston": "^3" - } - }, - "node_modules/winston-daily-rotate-file/node_modules/object-hash": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-2.2.0.tgz", - "integrity": "sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==", - "license": "MIT", - "engines": { - "node": ">= 6" - } - }, - "node_modules/winston-transport": { - "version": "4.9.0", - "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.9.0.tgz", - "integrity": "sha512-8drMJ4rkgaPo1Me4zD/3WLfI/zPdA9o2IipKODunnGDcuqbHwjsbB79ylv04LCGGzU0xQ6vTznOMpQGaLhhm6A==", - "license": "MIT", - "dependencies": { - "logform": "^2.7.0", - "readable-stream": "^3.6.2", - "triple-beam": "^1.3.0" - }, - "engines": { - "node": ">= 12.0.0" - } - }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -24089,19 +19626,6 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "license": "ISC" }, - "node_modules/write-file-atomic": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", - "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==", - "devOptional": true, - "license": "ISC", - "dependencies": { - "imurmurhash": "^0.1.4", - "is-typedarray": "^1.0.0", - "signal-exit": "^3.0.2", - "typedarray-to-buffer": "^3.1.5" - } - }, "node_modules/ws": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.0.tgz", @@ -24138,16 +19662,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/xdg-basedir": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz", - "integrity": "sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">=8" - } - }, "node_modules/xml-name-validator": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", @@ -24157,48 +19671,12 @@ "node": ">=18" } }, - "node_modules/xml-parse-from-string": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz", - "integrity": "sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g==", - "license": "MIT" - }, - "node_modules/xml2js": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", - "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==", - "license": "MIT", - "dependencies": { - "sax": ">=0.6.0", - "xmlbuilder": "~11.0.0" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/xmlbuilder": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", - "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", - "license": "MIT", - "engines": { - "node": ">=4.0" - } - }, "node_modules/xmlchars": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", "license": "MIT" }, - "node_modules/xmlhttprequest-ssl": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.1.2.tgz", - "integrity": "sha512-TEU+nJVUUnA4CYJFLvK5X9AOeH4KvDvhIfm0vV1GaQRtchnG0hgK5p8hw/xjv8cunWYCsiPCSDzObPyhEwq3KQ==", - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", @@ -24227,6 +19705,7 @@ "version": "2.8.3", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", + "dev": true, "license": "ISC", "bin": { "yaml": "bin.mjs" @@ -24492,6 +19971,7 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, "license": "MIT", "engines": { "node": ">=10" @@ -24505,7 +19985,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } @@ -24524,408 +20003,144 @@ "version": "2.5.1", "license": "AGPL-3.0-only", "dependencies": { - "@aws-sdk/client-cloudwatch": "^3.940.0", - "@aws-sdk/client-polly": "^3.622.0", - "@aws-sdk/client-textract": "^3.621.0", - "@google/generative-ai": "^0.21.0", - "@heyputer/kv.js": "^0.1.9", - "@heyputer/multest": "^0.0.2", + "@anthropic-ai/sdk": "^0.68.0", + "@aws-sdk/client-dynamodb": "^3.490.0", + "@aws-sdk/client-polly": "^3.1028.0", + "@aws-sdk/client-s3": "^3.1028.0", + "@aws-sdk/client-textract": "^3.1028.0", + "@aws-sdk/credential-providers": "^3.1021.0", + "@aws-sdk/lib-dynamodb": "^3.490.0", + "@aws-sdk/s3-request-presigner": "^3.1028.0", + "@google/genai": "^1.19.0", + "@heyputer/kv.js": "^0.2.1", "@heyputer/putility": "^1.0.0", - "@mistralai/mistralai": "^1.3.4", - "@opentelemetry/api": "^1.4.1", - "@opentelemetry/auto-instrumentations-node": "^0.43.0", - "@opentelemetry/exporter-metrics-otlp-grpc": "^0.40.0", - "@opentelemetry/exporter-trace-otlp-grpc": "^0.40.0", - "@opentelemetry/sdk-metrics": "^1.14.0", - "@opentelemetry/sdk-node": "^0.49.1", + "@mistralai/mistralai": "^1.15.1", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/auto-instrumentations-node": "^0.52.0", + "@opentelemetry/exporter-metrics-otlp-grpc": "^0.55.0", + "@opentelemetry/exporter-trace-otlp-grpc": "^0.55.0", + "@opentelemetry/resources": "^1.28.0", + "@opentelemetry/sdk-metrics": "^1.28.0", + "@opentelemetry/sdk-node": "^0.55.0", + "@opentelemetry/sdk-trace-base": "^1.28.0", + "@opentelemetry/semantic-conventions": "^1.28.0", "@pagerduty/pdjs": "^2.2.4", - "@smithy/node-http-handler": "^2.2.2", + "@smithy/node-http-handler": "^2.5.0", "@socket.io/redis-streams-adapter": "^0.3.1", - "args": "^5.0.3", - "axios": "^1.8.2", - "bcrypt": "^5.1.0", + "axios": "^1.15.0", + "bcrypt": "^5.1.1", "better-sqlite3": "^12.6.0", "busboy": "^1.6.0", "chai-as-promised": "^7.1.1", "clean-css": "^5.3.2", - "composite-error": "^1.0.2", - "compression": "^1.7.4", - "convertapi": "^1.15.0", - "cookie-parser": "^1.4.6", + "compression": "^1.8.1", + "cookie-parser": "^1.4.7", "dedent": "^1.5.3", - "dns2": "^2.1.0", - "express": "^4.18.2", - "file-type": "^21.3.3", - "firebase-admin": "^10.3.0", - "form-data": "^4.0.0", + "dynalite": "^4.0.0", + "express": "^5.0.0", + "fauxqs": "^2.5.0", "groq-sdk": "^0.5.0", - "handlebars": "^4.7.8", - "helmet": "^7.0.0", + "handlebars": "^4.7.9", + "helmet": "^7.2.0", "hi-base32": "^0.5.1", "html-entities": "^2.3.3", - "ioredis": "^5.9.2", + "ioredis": "^5.10.1", "ioredis-mock": "^8.13.1", - "is-glob": "^4.0.3", - "isbot": "^3.7.1", - "jimp": "^1.6.0", - "js-sha256": "^0.9.0", - "json5": "^2.2.3", - "jsonwebtoken": "^9.0.0", - "knex": "^3.1.0", + "jsonwebtoken": "^9.0.3", "lorem-ipsum": "^2.0.8", - "lru-cache": "^11.0.2", - "micromatch": "^4.0.5", "mime-types": "^2.1.35", - "moment": "^2.29.4", - "morgan": "^1.10.0", - "multer": "^2.0.2", - "multi-progress": "^4.0.0", "murmurhash": "^2.0.1", - "music-metadata": "^11.12.3", - "nodemailer": "^7.0.7", - "on-finished": "^2.4.1", - "openai": "^6.7.0", - "otpauth": "9.2.4", + "mysql2": "^3.21.1", + "nodemailer": "^7.0.13", + "openai": "^6.34.0", + "otpauth": "^9.2.4", + "parse-domain": "^8.2.2", "prompt-sync": "^4.2.0", - "proxyquire": "^2.1.3", - "recursive-readdir": "^2.2.3", - "replicate": "^1.4.0", - "response-time": "^2.3.2", - "seedrandom": "^3.0.5", + "replicate": "^1.0.0", "sharp": "^0.34.3", - "sharp-bmp": "^0.1.5", - "sharp-ico": "^0.1.5", - "shescape": "^2.1.10", - "socket.io": "^4.6.2", - "socket.io-client": "^4.6.2", - "ssh2": "^1.13.0", - "string-hash": "^1.1.3", - "string-length": "^6.0.0", + "socket.io": "^4.8.3", "svg-captcha": "^1.4.0", - "svgo": "^3.3.3", - "tiktoken": "^1.0.16", "together-ai": "^0.33.0", - "tweetnacl": "^1.0.3", - "ua-parser-js": "^1.0.38", + "ua-parser-js": "^1.0.41", "uglify-js": "^3.17.4", - "uuid": "^9.0.0", - "validator": "^13.9.0", - "winston": "^3.9.0", - "winston-daily-rotate-file": "^4.7.1", - "yargs": "^17.7.2" + "uuid": "^9.0.1", + "validator": "^13.15.35" }, "devDependencies": { "@types/node": "^24.0.0", "chai": "^4.3.7", - "jsdom": "29.0.0", "mocha": "^7.2.0", "nodemon": "^3.1.0", - "nyc": "^15.1.0", - "sinon": "^15.2.0", "typescript": "^5.9.3", + "vite": "^8.0.0", "vitest": "^4.0.14" } }, - "src/backend/node_modules/@heyputer/kv.js": { - "version": "0.1.92", - "resolved": "https://registry.npmjs.org/@heyputer/kv.js/-/kv.js-0.1.92.tgz", - "integrity": "sha512-D+trimrG/V6mU5zeQrKyH476WotvvRn0McttxiFxEzWLiMqR6aBmQ5apeKrZAheglHmwf0D3FO5ykmU2lCuLvQ==", - "license": "MIT", - "dependencies": { - "minimatch": "^9.0.0" - } - }, - "src/backend/node_modules/@opentelemetry/api": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.8.0.tgz", - "integrity": "sha512-I/s6F7yKUDdtMsoBWXJe8Qz40Tui5vsuKCWJEWVL+5q9sSWRzzx6v2KeNsOBEwd94j0eWkpWCH4yB6rZg9Mf0w==", - "license": "Apache-2.0", - "peer": true, - "engines": { - "node": ">=8.0.0" - } - }, - "src/backend/node_modules/@opentelemetry/context-async-hooks": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-1.22.0.tgz", - "integrity": "sha512-Nfdxyg8YtWqVWkyrCukkundAjPhUXi93JtVQmqDT1mZRVKqA7e2r7eJCrI+F651XUBMp0hsOJSGiFk3QSpaIJw==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" - } - }, - "src/backend/node_modules/@opentelemetry/core": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.22.0.tgz", - "integrity": "sha512-0VoAlT6x+Xzik1v9goJ3pZ2ppi6+xd3aUfg4brfrLkDBHRIVjMP0eBHrKrhB+NKcDyMAg8fAbGL3Npg/F6AwWA==", + "src/backend/node_modules/@smithy/node-http-handler": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-2.5.0.tgz", + "integrity": "sha512-mVGyPBzkkGQsPoxQUbxlEfRjrj6FPyA3u3u2VXGr9hT8wilsoQdZdvKpMBFMB8Crfhv5dNkKHIW0Yyuc7eABqA==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/semantic-conventions": "1.22.0" + "@smithy/abort-controller": "^2.2.0", + "@smithy/protocol-http": "^3.3.0", + "@smithy/querystring-builder": "^2.2.0", + "@smithy/types": "^2.12.0", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" + "node": ">=14.0.0" } }, - "src/backend/node_modules/@opentelemetry/otlp-exporter-base": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.49.1.tgz", - "integrity": "sha512-z6sHliPqDgJU45kQatAettY9/eVF58qVPaTuejw9YWfSRqid9pXPYeegDCSdyS47KAUgAtm+nC28K3pfF27HWg==", + "src/backend/node_modules/@smithy/protocol-http": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-3.3.0.tgz", + "integrity": "sha512-Xy5XK1AFWW2nlY/biWZXu6/krgbaf2dg0q492D8M5qthsnU2H+UgFeZLbM76FnH7s6RO/xhQRkj+T6KBO3JzgQ==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.22.0" + "@smithy/types": "^2.12.0", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.0.0" + "node": ">=14.0.0" } }, - "src/backend/node_modules/@opentelemetry/otlp-grpc-exporter-base": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-grpc-exporter-base/-/otlp-grpc-exporter-base-0.49.1.tgz", - "integrity": "sha512-DNDNUWmOqtKTFJAyOyHHKotVox0NQ/09ETX8fUOeEtyNVHoGekAVtBbvIA3AtK+JflP7LC0PTjlLfruPM3Wy6w==", + "src/backend/node_modules/@smithy/querystring-builder": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-2.2.0.tgz", + "integrity": "sha512-L1kSeviUWL+emq3CUVSgdogoM/D9QMFaqxL/dd0X7PCNWmPXqt+ExtrBjqT0V7HLN03Vs9SuiLrG3zy3JGnE5A==", "license": "Apache-2.0", "dependencies": { - "@grpc/grpc-js": "^1.7.1", - "@opentelemetry/core": "1.22.0", - "@opentelemetry/otlp-exporter-base": "0.49.1", - "protobufjs": "^7.2.3" + "@smithy/types": "^2.12.0", + "@smithy/util-uri-escape": "^2.2.0", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.0.0" + "node": ">=14.0.0" } }, - "src/backend/node_modules/@opentelemetry/otlp-transformer": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.49.1.tgz", - "integrity": "sha512-Z+koA4wp9L9e3jkFacyXTGphSWTbOKjwwXMpb0CxNb0kjTHGUxhYRN8GnkLFsFo5NbZPjP07hwAqeEG/uCratQ==", + "src/backend/node_modules/@smithy/types": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-2.12.0.tgz", + "integrity": "sha512-QwYgloJ0sVNBeBuBs65cIkTbfzV/Q6ZNPCJ99EICFEdJYG50nGIY/uYXp+TbsdJReIuPr0a0kXmCvren3MbRRw==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/api-logs": "0.49.1", - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0", - "@opentelemetry/sdk-logs": "0.49.1", - "@opentelemetry/sdk-metrics": "1.22.0", - "@opentelemetry/sdk-trace-base": "1.22.0" + "tslib": "^2.6.2" }, "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.9.0" + "node": ">=14.0.0" } }, - "src/backend/node_modules/@opentelemetry/propagator-b3": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-b3/-/propagator-b3-1.22.0.tgz", - "integrity": "sha512-qBItJm9ygg/jCB5rmivyGz1qmKZPsL/sX715JqPMFgq++Idm0x+N9sLQvWFHFt2+ZINnCSojw7FVBgFW6izcXA==", + "src/backend/node_modules/@smithy/util-uri-escape": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-2.2.0.tgz", + "integrity": "sha512-jtmJMyt1xMD/d8OtbVJ2gFZOSKc+ueYJZPW20ULW1GOp/q/YIM0wNh+u8ZFao9UaIGz4WoPW8hC64qlWLIfoDA==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.22.0" + "tslib": "^2.6.2" }, "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" - } - }, - "src/backend/node_modules/@opentelemetry/propagator-jaeger": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-jaeger/-/propagator-jaeger-1.22.0.tgz", - "integrity": "sha512-pMLgst3QIwrUfepraH5WG7xfpJ8J3CrPKrtINK0t7kBkuu96rn+HDYQ8kt3+0FXvrZI8YJE77MCQwnJWXIrgpA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" - } - }, - "src/backend/node_modules/@opentelemetry/resources": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.22.0.tgz", - "integrity": "sha512-+vNeIFPH2hfcNL0AJk/ykJXoUCtR1YaDUZM+p3wZNU4Hq98gzq+7b43xbkXjadD9VhWIUQqEwXyY64q6msPj6A==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/semantic-conventions": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" - } - }, - "src/backend/node_modules/@opentelemetry/sdk-logs": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.49.1.tgz", - "integrity": "sha512-gCzYWsJE0h+3cuh3/cK+9UwlVFyHvj3PReIOCDOmdeXOp90ZjKRoDOJBc3mvk1LL6wyl1RWIivR8Rg9OToyesw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.4.0 <1.9.0", - "@opentelemetry/api-logs": ">=0.39.1" - } - }, - "src/backend/node_modules/@opentelemetry/sdk-metrics": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.22.0.tgz", - "integrity": "sha512-k6iIx6H3TZ+BVMr2z8M16ri2OxWaljg5h8ihGJxi/KQWcjign6FEaEzuigXt5bK9wVEhqAcWLCfarSftaNWkkg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0", - "lodash.merge": "^4.6.2" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.9.0" - } - }, - "src/backend/node_modules/@opentelemetry/sdk-node": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-node/-/sdk-node-0.49.1.tgz", - "integrity": "sha512-feBIT85ndiSHXsQ2gfGpXC/sNeX4GCHLksC4A9s/bfpUbbgbCSl0RvzZlmEpCHarNrkZMwFRi4H0xFfgvJEjrg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api-logs": "0.49.1", - "@opentelemetry/core": "1.22.0", - "@opentelemetry/exporter-trace-otlp-grpc": "0.49.1", - "@opentelemetry/exporter-trace-otlp-http": "0.49.1", - "@opentelemetry/exporter-trace-otlp-proto": "0.49.1", - "@opentelemetry/exporter-zipkin": "1.22.0", - "@opentelemetry/instrumentation": "0.49.1", - "@opentelemetry/resources": "1.22.0", - "@opentelemetry/sdk-logs": "0.49.1", - "@opentelemetry/sdk-metrics": "1.22.0", - "@opentelemetry/sdk-trace-base": "1.22.0", - "@opentelemetry/sdk-trace-node": "1.22.0", - "@opentelemetry/semantic-conventions": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.9.0" - } - }, - "src/backend/node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/exporter-trace-otlp-grpc": { - "version": "0.49.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.49.1.tgz", - "integrity": "sha512-Zbd7f3zF7fI2587MVhBizaW21cO/SordyrZGtMtvhoxU6n4Qb02Gx71X4+PzXH620e0+JX+Pcr9bYb1HTeVyJA==", - "license": "Apache-2.0", - "dependencies": { - "@grpc/grpc-js": "^1.7.1", - "@opentelemetry/core": "1.22.0", - "@opentelemetry/otlp-grpc-exporter-base": "0.49.1", - "@opentelemetry/otlp-transformer": "0.49.1", - "@opentelemetry/resources": "1.22.0", - "@opentelemetry/sdk-trace-base": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.0.0" - } - }, - "src/backend/node_modules/@opentelemetry/sdk-trace-base": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.22.0.tgz", - "integrity": "sha512-pfTuSIpCKONC6vkTpv6VmACxD+P1woZf4q0K46nSUvXFvOFqjBYKFaAMkKD3M1mlKUUh0Oajwj35qNjMl80m1Q==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "1.22.0", - "@opentelemetry/resources": "1.22.0", - "@opentelemetry/semantic-conventions": "1.22.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" - } - }, - "src/backend/node_modules/@opentelemetry/sdk-trace-node": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-1.22.0.tgz", - "integrity": "sha512-gTGquNz7ue8uMeiWPwp3CU321OstQ84r7PCDtOaCicjbJxzvO8RZMlEC4geOipTeiF88kss5n6w+//A0MhP1lQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/context-async-hooks": "1.22.0", - "@opentelemetry/core": "1.22.0", - "@opentelemetry/propagator-b3": "1.22.0", - "@opentelemetry/propagator-jaeger": "1.22.0", - "@opentelemetry/sdk-trace-base": "1.22.0", - "semver": "^7.5.2" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.9.0" - } - }, - "src/backend/node_modules/@opentelemetry/semantic-conventions": { - "version": "1.22.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.22.0.tgz", - "integrity": "sha512-CAOgFOKLybd02uj/GhCdEeeBjOS0yeoDeo/CA7ASBSmenpZHAKGB3iDm/rv3BQLcabb/OprDEsSQ1y0P8A7Siw==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "src/backend/node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "license": "MIT" - }, - "src/backend/node_modules/brace-expansion": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", - "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==", - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "src/backend/node_modules/minimatch": { - "version": "9.0.9", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", - "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.2" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" + "node": ">=14.0.0" } }, "src/docs": { @@ -24953,74 +20168,8 @@ "@esbuild/linux-x64": "0.25.11" } }, - "src/docs/node_modules/@esbuild/aix-ppc64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.11.tgz", - "integrity": "sha512-Xt1dOL13m8u0WE8iplx9Ibbm+hFAO0GsU2P34UNoDGvZYkY8ifSiy6Zuc1lYxfG7svWE2fzqCUmFp5HCn51gJg==", - "cpu": [ - "ppc64" - ], - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/android-arm": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.11.tgz", - "integrity": "sha512-uoa7dU+Dt3HYsethkJ1k6Z9YdcHjTrSb5NUy66ZfZaSV8hEYGD5ZHbEMXnqLFlbBflLsl89Zke7CAdDJ4JI+Gg==", - "cpu": [ - "arm" - ], - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/android-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.11.tgz", - "integrity": "sha512-9slpyFBc4FPPz48+f6jyiXOx/Y4v34TUeDDXJpZqAWQn/08lKGeD8aDp9TMn9jDz2CiEuHwfhRmGBvpnd/PWIQ==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/android-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.11.tgz", - "integrity": "sha512-Sgiab4xBjPU1QoPEIqS3Xx+R2lezu0LKIEcYe6pftr56PqPygbB7+szVnzoShbx64MUupqoE0KyRlN7gezbl8g==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, "src/docs/node_modules/@esbuild/darwin-arm64": { "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.11.tgz", - "integrity": "sha512-VekY0PBCukppoQrycFxUqkCojnTQhdec0vevUL/EDOCnXd9LKWqD/bHwMPzigIJXPhC59Vd1WFIL57SKs2mg4w==", "cpu": [ "arm64" ], @@ -25033,330 +20182,8 @@ "node": ">=18" } }, - "src/docs/node_modules/@esbuild/darwin-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.11.tgz", - "integrity": "sha512-+hfp3yfBalNEpTGp9loYgbknjR695HkqtY3d3/JjSRUyPg/xd6q+mQqIb5qdywnDxRZykIHs3axEqU6l1+oWEQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/freebsd-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.11.tgz", - "integrity": "sha512-CmKjrnayyTJF2eVuO//uSjl/K3KsMIeYeyN7FyDBjsR3lnSJHaXlVoAK8DZa7lXWChbuOk7NjAc7ygAwrnPBhA==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/freebsd-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.11.tgz", - "integrity": "sha512-Dyq+5oscTJvMaYPvW3x3FLpi2+gSZTCE/1ffdwuM6G1ARang/mb3jvjxs0mw6n3Lsw84ocfo9CrNMqc5lTfGOw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/linux-arm": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.11.tgz", - "integrity": "sha512-TBMv6B4kCfrGJ8cUPo7vd6NECZH/8hPpBHHlYI3qzoYFvWu2AdTvZNuU/7hsbKWqu/COU7NIK12dHAAqBLLXgw==", - "cpu": [ - "arm" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/linux-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.11.tgz", - "integrity": "sha512-Qr8AzcplUhGvdyUF08A1kHU3Vr2O88xxP0Tm8GcdVOUm25XYcMPp2YqSVHbLuXzYQMf9Bh/iKx7YPqECs6ffLA==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/linux-ia32": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.11.tgz", - "integrity": "sha512-TmnJg8BMGPehs5JKrCLqyWTVAvielc615jbkOirATQvWWB1NMXY77oLMzsUjRLa0+ngecEmDGqt5jiDC6bfvOw==", - "cpu": [ - "ia32" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/linux-loong64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.11.tgz", - "integrity": "sha512-DIGXL2+gvDaXlaq8xruNXUJdT5tF+SBbJQKbWy/0J7OhU8gOHOzKmGIlfTTl6nHaCOoipxQbuJi7O++ldrxgMw==", - "cpu": [ - "loong64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/linux-mips64el": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.11.tgz", - "integrity": "sha512-Osx1nALUJu4pU43o9OyjSCXokFkFbyzjXb6VhGIJZQ5JZi8ylCQ9/LFagolPsHtgw6himDSyb5ETSfmp4rpiKQ==", - "cpu": [ - "mips64el" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/linux-ppc64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.11.tgz", - "integrity": "sha512-nbLFgsQQEsBa8XSgSTSlrnBSrpoWh7ioFDUmwo158gIm5NNP+17IYmNWzaIzWmgCxq56vfr34xGkOcZ7jX6CPw==", - "cpu": [ - "ppc64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/linux-riscv64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.11.tgz", - "integrity": "sha512-HfyAmqZi9uBAbgKYP1yGuI7tSREXwIb438q0nqvlpxAOs3XnZ8RsisRfmVsgV486NdjD7Mw2UrFSw51lzUk1ww==", - "cpu": [ - "riscv64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/linux-s390x": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.11.tgz", - "integrity": "sha512-HjLqVgSSYnVXRisyfmzsH6mXqyvj0SA7pG5g+9W7ESgwA70AXYNpfKBqh1KbTxmQVaYxpzA/SvlB9oclGPbApw==", - "cpu": [ - "s390x" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/netbsd-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.11.tgz", - "integrity": "sha512-hr9Oxj1Fa4r04dNpWr3P8QKVVsjQhqrMSUzZzf+LZcYjZNqhA3IAfPQdEh1FLVUJSiu6sgAwp3OmwBfbFgG2Xg==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/netbsd-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.11.tgz", - "integrity": "sha512-u7tKA+qbzBydyj0vgpu+5h5AeudxOAGncb8N6C9Kh1N4n7wU1Xw1JDApsRjpShRpXRQlJLb9wY28ELpwdPcZ7A==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/openbsd-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.11.tgz", - "integrity": "sha512-Qq6YHhayieor3DxFOoYM1q0q1uMFYb7cSpLD2qzDSvK1NAvqFi8Xgivv0cFC6J+hWVw2teCYltyy9/m/14ryHg==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/openbsd-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.11.tgz", - "integrity": "sha512-CN+7c++kkbrckTOz5hrehxWN7uIhFFlmS/hqziSFVWpAzpWrQoAG4chH+nN3Be+Kzv/uuo7zhX716x3Sn2Jduw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/openharmony-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.11.tgz", - "integrity": "sha512-rOREuNIQgaiR+9QuNkbkxubbp8MSO9rONmwP5nKncnWJ9v5jQ4JxFnLu4zDSRPf3x4u+2VN4pM4RdyIzDty/wQ==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/sunos-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.11.tgz", - "integrity": "sha512-nq2xdYaWxyg9DcIyXkZhcYulC6pQ2FuCgem3LI92IwMgIZ69KHeY8T4Y88pcwoLIjbed8n36CyKoYRDygNSGhA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/win32-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.11.tgz", - "integrity": "sha512-3XxECOWJq1qMZ3MN8srCJ/QfoLpL+VaxD/WfNRm1O3B4+AZ/BnLVgFbUV3eiRYDMXetciH16dwPbbHqwe1uU0Q==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/win32-ia32": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.11.tgz", - "integrity": "sha512-3ukss6gb9XZ8TlRyJlgLn17ecsK4NSQTmdIXRASVsiS2sQ6zPPZklNJT5GR5tE/MUarymmy8kCEf5xPCNCqVOA==", - "cpu": [ - "ia32" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "src/docs/node_modules/@esbuild/win32-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.11.tgz", - "integrity": "sha512-D7Hpz6A2L4hzsRpPaCYkQnGOotdUpDzSGRIv9I+1ITdHROSFUWW95ZPZWQmGka1Fg7W3zFJowyn9WGwMJ0+KPA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, "src/docs/node_modules/agent-base": { "version": "7.1.4", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", - "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", "license": "MIT", "engines": { "node": ">= 14" @@ -25364,8 +20191,6 @@ }, "src/docs/node_modules/data-urls": { "version": "5.0.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-5.0.0.tgz", - "integrity": "sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==", "license": "MIT", "dependencies": { "whatwg-mimetype": "^4.0.0", @@ -25375,10 +20200,20 @@ "node": ">=18" } }, + "src/docs/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "src/docs/node_modules/esbuild": { "version": "0.25.11", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.11.tgz", - "integrity": "sha512-KohQwyzrKTQmhXDW1PjCv3Tyspn9n5GcY2RTDqeORIdIJY8yKIF7sTSopFmn/wpMPW4rdPXI0UE5LJLuq3bx0Q==", "hasInstallScript": true, "license": "MIT", "bin": { @@ -25416,10 +20251,20 @@ "@esbuild/win32-x64": "0.25.11" } }, + "src/docs/node_modules/fs-extra": { + "version": "11.3.4", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, "src/docs/node_modules/html-encoding-sniffer": { "version": "4.0.0", - "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-4.0.0.tgz", - "integrity": "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==", "license": "MIT", "dependencies": { "whatwg-encoding": "^3.1.1" @@ -25430,8 +20275,6 @@ }, "src/docs/node_modules/http-proxy-agent": { "version": "7.0.2", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", - "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", "license": "MIT", "dependencies": { "agent-base": "^7.1.0", @@ -25443,8 +20286,6 @@ }, "src/docs/node_modules/https-proxy-agent": { "version": "7.0.6", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", - "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", "license": "MIT", "dependencies": { "agent-base": "^7.1.2", @@ -25456,8 +20297,6 @@ }, "src/docs/node_modules/iconv-lite": { "version": "0.6.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", "license": "MIT", "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" @@ -25468,8 +20307,6 @@ }, "src/docs/node_modules/jsdom": { "version": "26.1.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-26.1.0.tgz", - "integrity": "sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg==", "license": "MIT", "dependencies": { "cssstyle": "^4.2.1", @@ -25505,10 +20342,18 @@ } } }, + "src/docs/node_modules/jsonfile": { + "version": "6.2.0", + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, "src/docs/node_modules/parse5": { "version": "7.3.0", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", - "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", "license": "MIT", "dependencies": { "entities": "^6.0.0" @@ -25517,19 +20362,8 @@ "url": "https://github.com/inikulin/parse5?sponsor=1" } }, - "src/docs/node_modules/punycode": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", - "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "src/docs/node_modules/tldts": { "version": "6.1.86", - "resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz", - "integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==", "license": "MIT", "dependencies": { "tldts-core": "^6.1.86" @@ -25540,14 +20374,10 @@ }, "src/docs/node_modules/tldts-core": { "version": "6.1.86", - "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz", - "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==", "license": "MIT" }, "src/docs/node_modules/tough-cookie": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz", - "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==", "license": "BSD-3-Clause", "dependencies": { "tldts": "^6.1.32" @@ -25558,8 +20388,6 @@ }, "src/docs/node_modules/tr46": { "version": "5.1.1", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", - "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==", "license": "MIT", "dependencies": { "punycode": "^2.3.1" @@ -25568,10 +20396,15 @@ "node": ">=18" } }, + "src/docs/node_modules/universalify": { + "version": "2.0.1", + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, "src/docs/node_modules/webidl-conversions": { "version": "7.0.0", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", - "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", "license": "BSD-2-Clause", "engines": { "node": ">=12" @@ -25579,9 +20412,6 @@ }, "src/docs/node_modules/whatwg-encoding": { "version": "3.1.1", - "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", - "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", - "deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation", "license": "MIT", "dependencies": { "iconv-lite": "0.6.3" @@ -25592,8 +20422,6 @@ }, "src/docs/node_modules/whatwg-mimetype": { "version": "4.0.0", - "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", - "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", "license": "MIT", "engines": { "node": ">=18" @@ -25601,8 +20429,6 @@ }, "src/docs/node_modules/whatwg-url": { "version": "14.2.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", - "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==", "license": "MIT", "dependencies": { "tr46": "^5.1.0", @@ -25634,7 +20460,7 @@ "clean-css": "^5.3.2", "dotenv": "^16.4.5", "eslint": "^9.1.1", - "express": "^4.18.2", + "express": "^5.0.0", "globals": "^15.0.0", "html-entities": "^2.3.3", "jsdom": "^29.0.0", @@ -25673,6 +20499,16 @@ "version": "1.0.0", "license": "AGPL-3.0-only" }, + "src/worker": { + "name": "@heyputer/worker", + "version": "1.0.0", + "license": "AGPL-3.0-only", + "devDependencies": { + "terser-webpack-plugin": "^5.3.14", + "webpack": "^5.88.2", + "webpack-cli": "^5.1.1" + } + }, "tools/comment-parser": { "version": "1.0.0", "license": "AGPL-3.0-only", @@ -25682,8 +20518,6 @@ }, "tools/comment-parser/node_modules/assertion-error": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", - "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", "dev": true, "license": "MIT", "engines": { @@ -25692,8 +20526,6 @@ }, "tools/comment-parser/node_modules/chai": { "version": "5.3.3", - "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", - "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", "dev": true, "license": "MIT", "dependencies": { @@ -25709,8 +20541,6 @@ }, "tools/comment-parser/node_modules/check-error": { "version": "2.1.3", - "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", - "integrity": "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==", "dev": true, "license": "MIT", "engines": { @@ -25719,8 +20549,6 @@ }, "tools/comment-parser/node_modules/deep-eql": { "version": "5.0.2", - "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", - "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", "dev": true, "license": "MIT", "engines": { @@ -25729,15 +20557,11 @@ }, "tools/comment-parser/node_modules/loupe": { "version": "3.2.1", - "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", - "integrity": "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==", "dev": true, "license": "MIT" }, "tools/comment-parser/node_modules/pathval": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", - "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", "dev": true, "license": "MIT", "engines": { @@ -25752,22 +20576,6 @@ "version": "0.0.0", "license": "AGPL-3.0-only" }, - "tools/keygen": { - "version": "1.0.0", - "license": "AGPL-3.0-only" - }, - "tools/license-headers": { - "version": "1.0.0", - "license": "AGPL-3.0-only", - "dependencies": { - "console-table-printer": "^2.12.1", - "dedent": "^1.5.3", - "diff-match-patch": "^1.0.5", - "enquirer": "^2.4.1", - "js-levenshtein": "^1.1.6", - "yaml": "^2.4.5" - } - }, "tools/migrations-test": { "version": "1.0.0", "license": "AGPL-3.0-only", @@ -25777,26 +20585,10 @@ }, "tools/migrations-test/node_modules/commander": { "version": "12.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", - "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", "license": "MIT", "engines": { "node": ">=18" } - }, - "tools/module-docgen": { - "version": "1.0.0", - "license": "AGPL-3.0-only", - "dependencies": { - "@babel/parser": "^7.26.2", - "@babel/traverse": "^7.25.9", - "dedent": "^1.5.3", - "doctrine": "^3.0.0" - } - }, - "tools/token-count-accuracy": { - "version": "1.0.0", - "license": "AGPL-3.0-only" } } } diff --git a/package.json b/package.json index 4b5bd8501..85a91edc2 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "@eslint/js": "^9.35.0", "@playwright/test": "^1.56.1", "@stylistic/eslint-plugin": "^5.3.1", - "@types/express": "^4.17.21", + "@types/express": "^5.0.0", "@types/mime-types": "^3.0.1", "@types/uuid": "^10.0.0", "@typescript-eslint/eslint-plugin": "^8.46.1", @@ -25,19 +25,20 @@ "clean-css": "^5.3.2", "dotenv": "^16.4.5", "eslint": "^9.35.0", + "eslint-config-prettier": "^10.1.8", + "eslint-plugin-prettier": "^5.5.5", "eslint-rule-composer": "^0.3.0", "globals": "^15.15.0", - "html-entities": "^2.3.3", "html-webpack-plugin": "^5.6.0", "husky": "^9.1.7", "license-check-and-add": "^4.0.5", "mocha": "^7.2.0", "nodemon": "^3.1.0", + "prettier": "^3.8.3", "simple-git": "^3.32.3", "typescript": "^5.4.5", "uglify-js": "^3.17.4", "vite-plugin-static-copy": "^3.3.0", - "vitest": "^4.0.14", "webpack": "^5.88.2", "webpack-cli": "^5.1.1", "yaml": "^2.8.1" @@ -48,13 +49,15 @@ "test:backend": "npm run build:ts; vitest run --config=src/backend/vitest.config.ts", "test:backend-coverage": "npm run build:ts; vitest run --config=src/backend/vitest.config.ts", "start=gui": "nodemon --exec \"node dev-server.js\" ", - "start": "node ./tools/run-selfhosted.js", + "start": "node --enable-source-maps -r ./dist/src/backend/telemetry.js ./dist/src/backend/index.js", "prestart": "npm run build:ts", - "dev": "npm run build:ts && node ./tools/run-selfhosted.js", - "build": "npx eslint --quiet -c eslint/mandatory.eslint.config.js src/backend/src extensions && npm run build:ts && cd src/gui && node ./build.js", + "dev": "npm run build:ts && node --enable-source-maps -r ./dist/src/backend/telemetry.js ./dist/src/backend/index.js", + "build": "npm run build:ts && cd src/gui && node ./build.js && cd ../puter-js && npm run build", + "build:workerLib": "cd src/puter-js && npm run build && cd ../worker && npm run build", "check-translations": "node tools/check-translations.js", "prepare": "husky", - "build:ts": "tsc -p tsconfig.build.json" + "build:ts": "tsc -p tsconfig.json && node ./tools/write-dist-package-json.mjs", + "postinstall": "./tools/extensionSetup.sh" }, "workspaces": [ "src/*", @@ -70,38 +73,16 @@ }, "dependencies": { "@ai-sdk/openai": "^3.0.25", - "@anthropic-ai/sdk": "^0.68.0", - "@aws-sdk/client-dynamodb": "^3.490.0", "@aws-sdk/client-s3": "^3.1020.0", - "@aws-sdk/client-secrets-manager": "^3.879.0", - "@aws-sdk/client-sns": "^3.907.0", - "@aws-sdk/credential-providers": "^3.1021.0", - "@aws-sdk/lib-dynamodb": "^3.490.0", - "@google/genai": "^1.19.0", + "@aws-sdk/s3-request-presigner": "^3.1028.0", "@heyputer/putility": "^1.0.2", - "@paralleldrive/cuid2": "^2.2.2", - "@stylistic/eslint-plugin-js": "^4.4.1", "ai": "^6.0.73", "dedent": "^1.5.3", - "dynalite": "^4.0.0", - "express": "^4.18.2", - "express-xml-bodyparser": "^0.4.1", - "fauxqs": "^2.5.0", - "file-type": "21.3.3", "javascript-time-ago": "^2.5.11", - "json-colorizer": "^3.0.1", - "music-metadata": "11.12.3", - "open": "^10.1.0", - "parse-domain": "^8.2.2", - "string-template": "^1.0.0", - "uuid": "^9.0.1" + "open": "^10.1.0" }, "optionalDependencies": { - "@emnapi/core": "^1.9.2", - "@emnapi/runtime": "^1.9.2", - "sharp": "^0.34.4", - "sharp-bmp": "^0.1.5", - "sharp-ico": "^0.1.5" + "sharp": "^0.34.4" }, "engines": { "node": ">=24.0.0" diff --git a/src/backend/CONTRIBUTING.md b/src/backend/CONTRIBUTING.md deleted file mode 100644 index a3101335d..000000000 --- a/src/backend/CONTRIBUTING.md +++ /dev/null @@ -1,84 +0,0 @@ -# Contributing to Puter's Backend - -## File Structure - - - -## Architecture - -- [boot sequence](./doc/contributors/boot-sequence.md) -- [modules and services](./doc/contributors/modules.md) - -## Features - -- [protected apps](./doc/features/protected-apps.md) -- [service scripts](./doc/features/service-scripts.md) - -## Lists of Things - -- [list of permissions](./doc/lists-of-things/list-of-permissions.md) - -## Code-First Approach - -If you prefer to understand a system by looking at the -first files which are invoked and starting from there, -here's a handy list! - -- [Kernel](./src/Kernel.js), despite its intimidating name, is a - relatively simple (< 200 LOC) class which loads the modules - (modules register services), and then starts all the services. -- [RuntimeEnvironment](./src/boot/RuntimeEnvironment.js) - sets the configuration and runtime directories. It's invoked by Kernel. -- The default setup for running a self-hosted Puter loads these modules: - - [CoreModule](./src/CoreModule.js) - - [DatabaseModule](./src/DatabaseModule.js) - - [LocalDiskStorageModule](./src/LocalDiskStorageModule.js) -- HTTP endpoints are registered with - [WebServerService](./src/services/WebServerService.js) - by these services: - - [ServeGUIService](./src/services/ServeGUIService.js) - - [PuterAPIService](./src/services/PuterAPIService.js) - - [FilesystemAPIService](./src/services/FilesystemAPIService.js) - -## Development Philosophies - -### The copy-paste rule - -If you're copying and pasting code, you need to ask this question: -- am I copying as a reference (i.e. how this function is used), -- or am I copying an implementation of actual behavior? - -If your answer is the first, you should find more than one piece of -code that's doing the same thing you want to do and see if any of them -are doing it differently. One of the ways of doing this thing is going -to be more recent and/or (yes, potentially "or") more correct. -More correct approaches are ones which reduce -[coupling](https://en.wikipedia.org/wiki/Coupling_(computer_programming)), -move from legacy implementations to more recent ones, and are actually -more convenient for you to use. Whenever ever any of these three things -are in contention it's very important to communicate this to the -appropriate maintainers and contributors. - -If your answer is the second, you should find a way to -[DRY that code](https://en.wikipedia.org/wiki/Don%27t_repeat_yourself). - -### Architecture Mistakes? You will make them and it will suck. - -In my experience, the harder I think about the correct way to implement -something, the bigger a mistake I'm going to make; ***unless*** a big part -of the reason I'm thinking so hard is because I want to find a solution -that reduces complexity and has the right maintenance trade-off. -There's no easy solution for this so just keep it in mind; there are some -things we might write 2 times, 3 times, even more times over before we -really get it right and *that's okay*; sometimes part of doing useful work is -doing the useless work that reveals what the useful work is. - -## Underlying Constructs - -- [putility's README.md](../putility/README.md) - - Whenever you see `AdvancedBase`, that's from here - - Many things in backend extend this. Anything that doesn't only doesn't - because it was written before `AdvancedBase` existed. - - Allows adding "traits" to classes - - Have you ever wanted to wrap every method of a class with - common behavior? This can do that! diff --git a/src/backend/README.md b/src/backend/README.md deleted file mode 100644 index 54578bb9f..000000000 --- a/src/backend/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# Puter Backend - -_Part of a High-Level Distributed Operating System_ - -Whether or not you call Puter an operating system -(we call it a "high-level distributed operating system"), -**operating systems for devices** -are a useful reference point to describe the architecture of Puter. -If Puter's "hardware" is services, and Puter's "userspace" is the -client side of the API, then Puter's "kernel" is the backend. - -Puter's backend is composed of: -- The **Kernel** class, which is responsible for initialization -- A number of **Modules** which are registered in **Kernel** for a customized - Puter instance. -- Many **Services** which are contained inside modules. - -## Documentation - -- [Backend File Structure](./doc/contributors/structure.md) -- [Boot Sequence](./doc/contributors/boot-sequence.md) -- [Kernel](./doc/Kernel.md) -- [Modules](./doc/contributors/modules.md) - -## Can I use Puter's Backend Alone? - -Puter's backend is not dependent on Puter's frontned. In fact, you could -prevent Puter's GUI from ever showing up by disabling PuterHomepageModule. -Similarly, you can run Puter's backend with no modules loaded for a completely -blank slate, or only include CoreModule and WebModule to quickly build your -own backend that's compatible with any of Puter's services. - -## What can it do? - -Puter's Kernel only initializes modules, nothing more. The modules bring a lot -of capabilities to the table, however. Within this directory you'll find modules that: -- coerce all the well-known AI services to a common interface -- manage authentication with Wisp servers (this brings TCP to the browser!) -- manage apps on Puter -- allow a user to host websites from Puter -- provide persistent key-value storage to Puter's desktop and apps -- provide a fast filesystem implementation -- communicate with other instances of Puter's backend, - secured with elliptic curve cryptography -- provide more services like converting files and compiling low-level code. - -![diagram of Puter backend connections](./doc/assets/puter-backend-map.drawio.png) diff --git a/src/backend/clients/EventClient.js b/src/backend/clients/EventClient.js new file mode 100644 index 000000000..85c0ac4d9 --- /dev/null +++ b/src/backend/clients/EventClient.js @@ -0,0 +1,118 @@ +import { PuterClient } from './types'; + +export class EventClient extends PuterClient { + #eventListeners = {}; + + onServerStart() { + this.emit('serverStart', {}, {}); + } + onServerPrepareShutdown() { + this.emit('serverPrepareShutdown', {}, {}); + } + onServerShutdown() { + this.emit('serverShutdown', {}, {}); + } + + /** + * Dispatch an event to every matching subscriber. + * + * Match semantics: emit walks every dot-separated prefix of `key`, + * looking up `.*` listeners for prefixes shorter than the full + * key, and exact-key listeners on the final iteration. So emitting + * `outer.gui.item.removed` fires subscribers on: + * + * - `outer.*` + * - `outer.gui.*` + * - `outer.gui.item.*` + * - `outer.gui.item.removed` + * + * Subscribers are still keyed in a single map — wildcards just live + * under their literal `.*` string. No regex, no per-emit + * scan of every listener. + * + * @param {string} key + * @param {unknown} data + * @param {object} meta + */ + emit(key, data, meta) { + const parts = key.split('.'); + for (let i = 0; i < parts.length; i++) { + const matchKey = + i === parts.length - 1 + ? key + : `${parts.slice(0, i + 1).join('.')}.*`; + const listeners = this.#eventListeners[matchKey]; + if (!listeners) continue; + for (const listener of listeners) { + this.#emitEvent(listener, key, data, meta); + } + } + } + + /** + * Like `emit`, but awaits every matched listener before resolving. + * + * Use this when the emitter needs to act on mutations the handlers made + * to `data` — e.g. validation hooks where a listener can set + * `data.allow = false` to reject, or pre-commit pipelines where every + * stage must complete before the next step runs. Regular `emit` is + * fire-and-forget and can't observe handler state changes. + * + * Listeners run sequentially in the order they're registered so an + * earlier handler's mutation is visible to later ones. A listener that + * throws is logged (same as `emit`) and the chain continues. + * + * @param {string} key + * @param {unknown} data + * @param {object} meta + * @returns {Promise} + */ + async emitAndWait(key, data, meta) { + const parts = key.split('.'); + for (let i = 0; i < parts.length; i++) { + const matchKey = + i === parts.length - 1 + ? key + : `${parts.slice(0, i + 1).join('.')}.*`; + const listeners = this.#eventListeners[matchKey]; + if (!listeners) continue; + for (const listener of listeners) { + try { + await listener(key, data, meta); + } catch (e) { + console.error('Error in event listener for event', key, e); + } + } + } + } + + /** + * Subscribe to an event by exact key OR a wildcard prefix. + * + * Wildcards: a key ending in `.*` matches every event whose name + * starts with the prefix. `outer.*` matches `outer.gui.item.removed`, + * `outer.fs.write-hash`, and any other dot-extended descendant. + * Exact keys still match exactly. See `emit()` for the dispatch order. + * + * Callback receives the full `(key, data, meta)` tuple as passed + * to `emit()` — wildcard subscribers can branch on the triggering + * event name. + * + * @param {string} key + * @param {(key: string, data: unknown, meta: object) => void} callback + */ + on(key, callback) { + if (!this.#eventListeners[key]) { + this.#eventListeners[key] = []; + } + this.#eventListeners[key].push(callback); + } + + async #emitEvent(listener, key, data, meta) { + try { + await listener(key, data, meta); + } catch (e) { + console.error('Error in event listener for event', key, e); + } + } +} diff --git a/src/backend/clients/alarm/AlarmClient.ts b/src/backend/clients/alarm/AlarmClient.ts new file mode 100644 index 000000000..245caaa87 --- /dev/null +++ b/src/backend/clients/alarm/AlarmClient.ts @@ -0,0 +1,376 @@ +import { event as pdEvent } from '@pagerduty/pdjs'; +import { inspect } from 'node:util'; +import { createHash } from 'node:crypto'; +import type { IConfig } from '../../types'; +import { PuterClient } from '../types'; + +// ── Types ──────────────────────────────────────────────────────────── + +export interface AlarmFields { + error?: Error; + [key: string]: unknown; +} + +interface AlarmOccurrence { + message: string; + fields: AlarmFields; + timestamp: number; +} + +interface Alarm { + id: string; + shortId: string; + message: string; + fields: AlarmFields; + error?: Error; + started: number; + timestamps: number[]; + occurrences: AlarmOccurrence[]; + severity?: PagerSeverity; + noAlert?: boolean; +} + +type PagerSeverity = 'critical' | 'error' | 'warning' | 'info'; + +export interface AlertPayload { + id: string; + message: string; + source: string; + severity: PagerSeverity; + custom?: Record; +} + +type AlertHandler = (alert: AlertPayload) => Promise; + +interface KnownErrorRule { + match: { + id: string; + message?: string; + fields?: Record; + }; + action: { + type: 'no-alert' | 'severity'; + value?: PagerSeverity; + }; +} + +// ── Helpers ────────────────────────────────────────────────────────── + +/** + * Deterministic short identifier derived from an alarm ID. + * Produces a readable 3-word slug like "amber-delta-fox". + */ +const WORD_POOL = [ + 'alpha', + 'amber', + 'arc', + 'bolt', + 'cape', + 'cask', + 'core', + 'crow', + 'dawn', + 'delta', + 'dune', + 'echo', + 'edge', + 'elk', + 'fern', + 'flint', + 'fog', + 'fox', + 'gate', + 'glow', + 'haze', + 'helm', + 'hive', + 'jade', + 'keel', + 'knot', + 'lark', + 'lime', + 'lynx', + 'mast', + 'mist', + 'moss', + 'node', + 'nova', + 'opal', + 'orbit', + 'palm', + 'peak', + 'pine', + 'pike', + 'quad', + 'quay', + 'rail', + 'reef', + 'rune', + 'sage', + 'shard', + 'silo', + 'slate', + 'spark', + 'surge', + 'tarn', + 'tide', + 'vale', + 'vane', + 'wren', + 'yard', + 'yew', + 'zeal', + 'zero', + 'zinc', + 'zone', +]; + +function shortId(id: string): string { + const hash = createHash('sha256').update(id).digest(); + const words: string[] = []; + for (let i = 0; i < 3; i++) { + words.push(WORD_POOL[hash[i] % WORD_POOL.length]); + } + return words.join('-'); +} + +function displayId(alarm: Alarm): string { + if (alarm.id.length < 20) return alarm.id; + return `${alarm.shortId} (${alarm.id.slice(0, 20)}...)`; +} + +function cleanFields(fields: AlarmFields): Record { + const out: Record = {}; + for (const [key, value] of Object.entries(fields)) { + out[key] = inspect(value); + } + return out; +} + +// ── AlarmClient ────────────────────────────────────────────────────── + +/** + * Manages system alarms and dispatches alerts to external paging + * services (PagerDuty, or any registered handler). + */ +export class AlarmClient extends PuterClient { + private alarms = new Map(); + private aliases = new Map(); + private alertHandlers: AlertHandler[] = []; + private knownErrors: KnownErrorRule[] = []; + private draining = false; + private drainLogged = false; + + constructor(config: IConfig) { + super(config); + } + + // ── Lifecycle ──────────────────────────────────────────────────── + + override async onServerStart(): Promise { + const pagerConf = this.config.pager; + if (!pagerConf?.pagerduty?.enabled) return; + + const routingKey = pagerConf.pagerduty.routingKey; + if (!routingKey) { + console.warn( + '[alarm] PagerDuty enabled but no routingKey configured', + ); + return; + } + + const serverId = this.config.serverId; + + this.alertHandlers.push(async (alert) => { + await pdEvent({ + data: { + routing_key: routingKey, + event_action: 'trigger', + dedup_key: alert.id, + payload: { + summary: alert.message, + source: alert.source, + severity: alert.severity, + custom_details: { + ...alert.custom, + server_id: serverId, + }, + }, + }, + }); + }); + + console.log('[alarm] PagerDuty handler registered'); + } + + override onServerPrepareShutdown(): void { + if (this.draining) return; + this.draining = true; + console.log('[alarm] entering drain mode — suppressing new alarms'); + } + + // ── Public API ─────────────────────────────────────────────────── + + /** + * Create or update an alarm. If the alarm ID already exists, the + * occurrence count is incremented and a repeat alert is dispatched. + */ + create(id: string, message: string, fields: AlarmFields = {}): void { + if (this.draining) { + if (!this.drainLogged) { + this.drainLogged = true; + console.log('[alarm] suppressing alarm while draining'); + } + return; + } + + const existing = this.alarms.get(id); + + if (existing) { + this.recordOccurrence(existing, message, fields); + this.handleRepeat(existing); + return; + } + + const alarm: Alarm = { + id, + shortId: shortId(id), + message, + fields, + started: Date.now(), + timestamps: [Date.now()], + occurrences: [], + }; + if (fields.error) alarm.error = fields.error; + + this.alarms.set(id, alarm); + this.aliases.set(alarm.shortId, alarm); + this.recordOccurrence(alarm, message, fields); + this.handleNew(alarm); + } + + /** Clear an active alarm. */ + clear(id: string): void { + const alarm = this.alarms.get(id); + if (!alarm) return; + + this.alarms.delete(id); + this.aliases.delete(alarm.shortId); + console.log(`[alarm] CLEAR ${displayId(alarm)} :: ${alarm.message}`); + } + + /** Look up an alarm by its full ID or short ID. */ + get(id: string): Alarm | undefined { + return this.alarms.get(id) ?? this.aliases.get(id); + } + + /** + * Register an additional alert handler. Handlers are called for + * every alarm that isn't suppressed by a known-error rule. + */ + addAlertHandler(handler: AlertHandler): void { + this.alertHandlers.push(handler); + } + + /** + * Add rules that can suppress or adjust severity of known errors. + */ + setKnownErrors(rules: KnownErrorRule[]): void { + this.knownErrors = rules; + } + + // ── Internals ──────────────────────────────────────────────────── + + private recordOccurrence( + alarm: Alarm, + message: string, + fields: AlarmFields, + ): void { + alarm.message = message; + alarm.fields = { ...alarm.fields, ...fields }; + alarm.timestamps.push(Date.now()); + if (fields.error) alarm.error = fields.error; + + alarm.occurrences.push({ + message, + fields, + timestamp: Date.now(), + }); + } + + private applyKnownErrors(alarm: Alarm): void { + for (const rule of this.knownErrors) { + if (!this.ruleMatches(rule, alarm)) continue; + + switch (rule.action.type) { + case 'no-alert': + alarm.noAlert = true; + break; + case 'severity': + alarm.severity = rule.action.value; + break; + } + } + } + + private ruleMatches(rule: KnownErrorRule, alarm: Alarm): boolean { + const { match } = rule; + if (match.id !== alarm.id) return false; + if (match.message && match.message !== alarm.message) return false; + if (match.fields) { + for (const [key, value] of Object.entries(match.fields)) { + if (alarm.fields[key] !== value) return false; + } + } + return true; + } + + private handleNew(alarm: Alarm): void { + this.applyKnownErrors(alarm); + + console.error(`[alarm] ACTIVE ${displayId(alarm)} :: ${alarm.message}`); + + if (alarm.error) { + console.error(alarm.error); + } + + if (alarm.noAlert) return; + + this.dispatchAlert(alarm); + } + + private handleRepeat(alarm: Alarm): void { + this.applyKnownErrors(alarm); + + console.warn( + `[alarm] REPEAT ${displayId(alarm)} :: ${alarm.message} (${alarm.timestamps.length})`, + ); + + if (alarm.noAlert) return; + + this.dispatchAlert(alarm); + } + + private dispatchAlert(alarm: Alarm): void { + const severity = alarm.severity ?? 'critical'; + const fieldsClean = cleanFields(alarm.fields); + + const payload: AlertPayload = { + id: alarm.id || 'something-bad', + message: alarm.message || alarm.id || 'something bad happened', + source: 'alarm', + severity, + custom: { + fields: fieldsClean, + trace: alarm.error?.stack, + repeat_count: alarm.timestamps.length, + }, + }; + + for (const handler of this.alertHandlers) { + handler(payload).catch((err) => { + console.error(`[alarm] alert handler failed: ${err?.message}`); + }); + } + } +} diff --git a/src/backend/clients/database/DatabaseClient.ts b/src/backend/clients/database/DatabaseClient.ts new file mode 100644 index 000000000..2c35acf32 --- /dev/null +++ b/src/backend/clients/database/DatabaseClient.ts @@ -0,0 +1,144 @@ +import type { IConfig } from '../../types'; +import { PuterClient } from '../types'; + +export interface WriteResult { + insertId: number | bigint; + anyRowsAffected: boolean; +} + +export interface BatchEntry { + statement: string; + values: unknown[]; +} + +/** + * Base database client. Subclasses must override every method that throws here. + * + * Do not instantiate directly — use the factory exported from + * `clients/database/index.ts` which picks the right implementation + * based on `config.database.engine`. + */ +export class AbstractDatabaseClient extends PuterClient { + /** Short name used by `case()` to pick engine-specific values. */ + readonly engineName: string = ''; + + constructor(config: IConfig) { + super(config); + } + + // ------------------------------------------------------------------ + // Abstract interface — subclasses MUST override + // ------------------------------------------------------------------ + + /** + * Execute a read query. Returns an array of row objects. + */ + async read( + _query: string, + _params: unknown[] = [], + ): Promise[]> { + throw new Error('DatabaseClient.read() not implemented'); + } + + /** + * Read that prefers the primary database (useful when read-replicas + * may have replication lag). In single-node setups this is identical + * to `read()`. + */ + async pread( + _query: string, + _params: unknown[] = [], + ): Promise[]> { + throw new Error('DatabaseClient.pread() not implemented'); + } + + /** + * Execute a write query (INSERT / UPDATE / DELETE). + */ + async write(_query: string, _params: unknown[] = []): Promise { + throw new Error('DatabaseClient.write() not implemented'); + } + + /** + * Execute multiple write statements in a single transaction. + */ + async batchWrite(_entries: BatchEntry[]): Promise { + throw new Error('DatabaseClient.batchWrite() not implemented'); + } + + // ------------------------------------------------------------------ + // Shared helpers (rely on the abstract methods above) + // ------------------------------------------------------------------ + + /** + * Generate and execute an INSERT statement from a table name and a + * key/value data object. + */ + async insert( + tableName: string, + data: Record, + ): Promise { + const cols = Object.keys(data); + const values = Object.values(data); + const sql = + `INSERT INTO \`${tableName}\` ` + + `(${cols.map((c) => `\`${c}\``).join(', ')}) ` + + `VALUES (${cols.map(() => '?').join(', ')})`; + return this.write(sql, values); + } + + /** + * Like `read()` but falls back to the primary when read-replicas are + * in use. Subclasses may override with replica-aware logic; the + * default delegates to `pread()`. + */ + async tryHardRead( + query: string, + params: unknown[] = [], + ): Promise[]> { + const primary = this.pread(query, params); + primary.catch(() => {}); + + try { + const rows = await this.read(query, params); + if (rows.length > 0) { + return rows; + } + } catch { + // replica failed — fall through to primary + } + return primary; + } + + /** + * Like `tryHardRead()` but throws when the result set is empty. + */ + async requireRead( + query: string, + params: unknown[] = [], + ): Promise[]> { + const rows = await this.tryHardRead(query, params); + if (rows.length === 0) { + throw new Error(`required read returned no rows: ${query}`); + } + return rows; + } + + /** + * Return the value from `choices` that matches the current engine. + * + * Usage: + * ``` + * db.case({ sqlite: "datetime('now')", mysql: 'NOW()', otherwise: 'NOW()' }) + * ``` + * + * If the engine name isn't present in `choices`, falls back to + * `choices.otherwise`. + */ + case(choices: Record & { otherwise?: T }): T { + if (Object.prototype.hasOwnProperty.call(choices, this.engineName)) { + return choices[this.engineName]; + } + return choices.otherwise as T; + } +} diff --git a/src/backend/clients/database/MySQLDatabaseClient.ts b/src/backend/clients/database/MySQLDatabaseClient.ts new file mode 100644 index 000000000..693c401c2 --- /dev/null +++ b/src/backend/clients/database/MySQLDatabaseClient.ts @@ -0,0 +1,401 @@ +import { createPool, type Pool } from 'mysql2'; +import { AbstractDatabaseClient, type WriteResult } from './DatabaseClient'; +import { SQLBatcher } from './SQLBatcher.js'; +import type { IConfig } from '../../types'; + +const RETRIABLE_ERROR_CODES = new Set([ + 'PROTOCOL_CONNECTION_LOST', + 'PROTOCOL_SEQUENCE_TIMEOUT', + 'PROTOCOL_ENQUEUE_AFTER_FATAL_ERROR', + 'ECONNRESET', + 'ETIMEDOUT', + 'EPIPE', + 'ECONNREFUSED', + 'EHOSTUNREACH', + 'ENETUNREACH', + 'EAI_AGAIN', +]); + +const RETRIABLE_ERROR_MESSAGES = [ + 'Connection lost', + 'read ECONNRESET', + 'ETIMEDOUT', +]; + +type PoolConfig = Parameters[0]; + +enum Configuration { + SINGLE, + REPLICA, +} + +export class MySQLDatabaseClient extends AbstractDatabaseClient { + override readonly engineName = 'mysql'; + + private primaryPool!: Pool; + private replicaPool!: Pool; + private db!: SQLBatcher; + private dbReplica!: SQLBatcher; + private configuration = Configuration.SINGLE; + private shutdownStarted = false; + private shutdownTimer: ReturnType | null = null; + + constructor(config: IConfig) { + super(config); + } + + // ------------------------------------------------------------------ + // Lifecycle + // ------------------------------------------------------------------ + + override async onServerStart(): Promise { + const dbConf = this.config.database!; + + this.primaryPool = this.createPool({ + host: dbConf.host ?? '127.0.0.1', + port: dbConf.port ?? 3306, + user: dbConf.user ?? 'root', + password: dbConf.password ?? '', + database: dbConf.database ?? 'puter', + }); + console.log('[mysql] connected to primary'); + + this.db = new SQLBatcher(this.primaryPool, 40, 10); + + if (dbConf.replica) { + this.replicaPool = this.createPool(dbConf.replica); + this.configuration = Configuration.REPLICA; + console.log('[mysql] connected to read-replica'); + } else { + this.replicaPool = this.primaryPool; + this.configuration = Configuration.SINGLE; + } + + this.dbReplica = new SQLBatcher(this.replicaPool, 10, 10); + } + + override async onServerPrepareShutdown(): Promise { + if (this.shutdownStarted) return; + this.shutdownStarted = true; + + // Allow in-flight queries to drain before closing pools + const drainMs = 60_000; + console.log( + `[mysql] draining in-flight queries (${drainMs}ms) before closing pools`, + ); + + this.shutdownTimer = setTimeout(() => { + this.shutdownTimer = null; + this.closeCurrentPools('drain').catch((e) => + console.error('[mysql] error closing pools after drain', e), + ); + }, drainMs); + + if (typeof this.shutdownTimer.unref === 'function') { + this.shutdownTimer.unref(); + } + } + + override async onServerShutdown(): Promise { + if (this.shutdownTimer) { + clearTimeout(this.shutdownTimer); + this.shutdownTimer = null; + } + await this.closeCurrentPools('shutdown'); + } + + // ------------------------------------------------------------------ + // Query interface + // ------------------------------------------------------------------ + + override async read( + query: string, + params: unknown[] = [], + ): Promise[]> { + const result = await this.dbReplica.execute(query, params); + if (!result) return []; + return (result[0] as Record[]) ?? []; + } + + override async pread( + query: string, + params: unknown[] = [], + ): Promise[]> { + const result = await this.db.execute(query, params); + if (!result) return []; + return (result[0] as Record[]) ?? []; + } + + override async write( + query: string, + params: unknown[] = [], + ): Promise { + const result = await this.db.execute(query, params); + const header = result[0] as { + insertId?: number; + affectedRows?: number; + }; + return { + insertId: header.insertId ?? 0, + anyRowsAffected: (header.affectedRows ?? 0) > 0, + }; + } + + override async batchWrite( + entries: { statement: string; values: unknown[] }[], + ): Promise { + if (entries.length === 0) return; + // Bypass the SQLBatcher: it coalesces queries from unrelated callers + // into a single multi-statement string, which is incompatible with + // wrapping a transaction around just *our* statements. Acquire a + // dedicated connection so BEGIN/COMMIT/ROLLBACK only scope `entries`. + const conn = await this.primaryPool.promise().getConnection(); + try { + await conn.beginTransaction(); + try { + for (const { statement, values } of entries) { + await conn.execute(statement, values); + } + await conn.commit(); + } catch (err) { + await conn.rollback().catch(() => {}); + throw err; + } + } finally { + conn.release(); + } + } + + override async tryHardRead( + query: string, + params: unknown[] = [], + ): Promise[]> { + if (this.configuration === Configuration.SINGLE) { + return this.read(query, params); + } + + // Run both reads in parallel — prefer replica when it returns rows, + // otherwise fall back to primary to handle replication lag. + const primaryPromise = this.db.execute(query, params); + try { + const replicaResult = await this.dbReplica.execute(query, params); + if ( + Array.isArray(replicaResult?.[0]) && + (replicaResult[0] as unknown[]).length > 0 + ) { + primaryPromise.catch(() => {}); // suppress unhandled rejection + return replicaResult[0] as Record[]; + } + } catch { + // fall through to primary + } + + const primaryResult = await primaryPromise; + return (primaryResult?.[0] as Record[]) ?? []; + } + + // ------------------------------------------------------------------ + // Pool management + // ------------------------------------------------------------------ + + private createPool(poolConfig: PoolConfig): Pool { + return createPool({ + maxPreparedStatements: 900, + ...poolConfig, + multipleStatements: true, + } as PoolConfig); + } + + /** Reinitialize the primary pool (e.g. after a health-check failure). */ + reinitPrimary(): void { + if (this.shutdownStarted) return; + + const dbConf = this.config.database!; + const previous = this.primaryPool; + this.primaryPool = this.createPool({ + host: dbConf.host ?? '127.0.0.1', + port: dbConf.port ?? 3306, + user: dbConf.user ?? 'root', + password: dbConf.password ?? '', + database: dbConf.database ?? 'puter', + }); + this.db = new SQLBatcher(this.primaryPool, 40, 10); + + if (this.configuration === Configuration.SINGLE) { + this.replicaPool = this.primaryPool; + this.dbReplica = new SQLBatcher(this.primaryPool, 10, 10); + } + + if (previous && previous !== this.primaryPool) { + this.closePool(previous, 'reinit:primary').catch(() => {}); + } + } + + /** Reinitialize the replica pool. */ + reinitReplica(): void { + if (this.shutdownStarted || !this.config.database?.replica) return; + + const previous = this.replicaPool; + this.replicaPool = this.createPool(this.config.database.replica); + this.dbReplica = new SQLBatcher(this.replicaPool, 10); + + if ( + previous && + previous !== this.replicaPool && + previous !== this.primaryPool + ) { + this.closePool(previous, 'reinit:replica').catch(() => {}); + } + } + + // ------------------------------------------------------------------ + // Retry helpers (for health checks or resilient reads) + // ------------------------------------------------------------------ + + static isRetriableError(error: unknown): boolean { + const code = (error as { code?: string })?.code; + if (code && RETRIABLE_ERROR_CODES.has(code)) return true; + + const msg = String((error as Error)?.message ?? ''); + return RETRIABLE_ERROR_MESSAGES.some((m) => msg.includes(m)); + } + + async readWithRetry( + label: string, + operation: () => Promise, + opts?: { + maxAttempts?: number; + baseBackoffMs?: number; + maxBackoffMs?: number; + jitterRatio?: number; + }, + ): Promise { + const maxAttempts = opts?.maxAttempts ?? 3; + const baseBackoffMs = opts?.baseBackoffMs ?? 100; + const maxBackoffMs = opts?.maxBackoffMs ?? 500; + const jitterRatio = opts?.jitterRatio ?? 0.2; + + let attempt = 1; + + while (true) { + try { + return await operation(); + } catch (error) { + if (this.shutdownStarted) throw error; + if ( + attempt >= maxAttempts || + !MySQLDatabaseClient.isRetriableError(error) + ) + throw error; + + const raw = baseBackoffMs * 2 ** (attempt - 1); + const capped = Math.min(maxBackoffMs, raw); + const window = Math.round(capped * jitterRatio); + const jitter = + window === 0 + ? 0 + : Math.floor(Math.random() * (window * 2 + 1)) - window; + const delay = Math.max(0, capped + jitter); + + console.warn( + `[${label}] transient mysql error (${(error as { code?: string })?.code ?? 'unknown'}); retry ${attempt + 1}/${maxAttempts} in ${delay}ms`, + ); + await new Promise((r) => setTimeout(r, delay)); + attempt++; + } + } + } + + // ------------------------------------------------------------------ + // Internal pool lifecycle + // ------------------------------------------------------------------ + + private async closePool( + pool: Pool, + label: string, + timeoutMs: number | null = null, + ): Promise { + if (!pool) return; + + await new Promise((resolve, reject) => { + let settled = false; + let timer: ReturnType | null = null; + + const finish = (err?: unknown) => { + if (settled) return; + settled = true; + if (timer) clearTimeout(timer); + if (err) reject(err); + else resolve(); + }; + + if (timeoutMs !== null) { + timer = setTimeout(() => { + console.warn( + `[mysql] timed out closing pool (${label}); forcing`, + ); + this.forceDestroyConnections(pool, `${label}:timeout`); + finish(); + }, timeoutMs); + } + + try { + pool.end((err) => finish(err)); + } catch (err) { + finish(err); + } + }); + } + + private forceDestroyConnections(pool: Pool, label: string): void { + // mysql2 internal — _allConnections is a CircularBuffer + const all = ( + pool as unknown as { + _allConnections?: { + forEach: (fn: (c: { destroy: () => void }) => void) => void; + }; + } + )._allConnections; + if (!all || typeof all.forEach !== 'function') return; + + let count = 0; + all.forEach((conn) => { + try { + conn.destroy(); + count++; + } catch { + // no-op + } + }); + if (count > 0) + console.warn( + `[mysql] force-closed ${count} connections (${label})`, + ); + } + + private async closeCurrentPools(reason: string): Promise { + const timeoutMs = reason.startsWith('signal:') ? 45_000 : null; + const tasks: Promise[] = []; + + if (this.primaryPool) { + tasks.push( + this.closePool( + this.primaryPool, + `${reason}:primary`, + timeoutMs, + ), + ); + } + if (this.replicaPool && this.replicaPool !== this.primaryPool) { + tasks.push( + this.closePool( + this.replicaPool, + `${reason}:replica`, + timeoutMs, + ), + ); + } + + await Promise.all(tasks); + } +} diff --git a/src/backend/clients/database/SQLBatcher.js b/src/backend/clients/database/SQLBatcher.js new file mode 100644 index 000000000..7e272301c --- /dev/null +++ b/src/backend/clients/database/SQLBatcher.js @@ -0,0 +1,141 @@ +import { metrics } from '@opentelemetry/api'; + +const DEFAULT_MAX_QUEUE_SIZE = 1000; +const DEFAULT_FAILURE_THRESHOLD = 5; +const DEFAULT_COOLDOWN_MS = 5_000; + +const meter = metrics.getMeter('puter-backend'); +const enqueueDroppedCounter = meter.createCounter( + 'sql_batcher.enqueue.dropped', + { + description: + 'Items dropped from SQLBatcher queue at the high-water mark', + }, +); +const enqueueRejectedCounter = meter.createCounter( + 'sql_batcher.enqueue.rejected', + { description: 'Items rejected because the SQLBatcher circuit is open' }, +); +const flushFailureCounter = meter.createCounter('sql_batcher.flush.failed', { + description: 'SQLBatcher flush attempts that threw', +}); + +export class SQLBatcher { + dbPool; + maxTimeInQueue; + maxBatchSize; + maxQueueSize; + failureThreshold; + cooldownMs; + queue = []; + timeouts = []; + #consecutiveFailures = 0; + #lastFailureAt = 0; + + constructor( + dbPool, + maxTimeInQueue = 20, + maxBatchSize = 50, + maxQueueSize = DEFAULT_MAX_QUEUE_SIZE, + failureThreshold = DEFAULT_FAILURE_THRESHOLD, + cooldownMs = DEFAULT_COOLDOWN_MS, + ) { + this.dbPool = dbPool; + this.maxTimeInQueue = maxTimeInQueue; + this.maxBatchSize = maxBatchSize; + this.maxQueueSize = maxQueueSize; + this.failureThreshold = failureThreshold; + this.cooldownMs = cooldownMs; + } + + async execute(sql, values) { + return this.query(sql, values); + } + + promise() { + return this; + } + + #createPublicBatchError() { + const error = new Error('Database operation failed'); + error.code = 'dbBatchFailed'; + return error; + } + + // Open while we've seen `failureThreshold` consecutive flush failures and + // the cooldown window since the most recent failure hasn't elapsed. After + // cooldown a probe request is allowed through; success resets the counter. + #isBreakerOpen() { + if (this.#consecutiveFailures < this.failureThreshold) return false; + return Date.now() - this.#lastFailureAt < this.cooldownMs; + } + + async query(sql, values) { + if (this.#isBreakerOpen()) { + enqueueRejectedCounter.add(1); + throw this.#createPublicBatchError(); + } + + const { promise, resolve, reject } = Promise.withResolvers(); + + // Drop-oldest at the high-water mark. Bounds memory while preferring + // to flush the most recent work — older queued entries are likeliest + // to have already exceeded any caller-side timeout anyway. + while (this.queue.length >= this.maxQueueSize) { + const dropped = this.queue.shift(); + dropped.reject(this.#createPublicBatchError()); + enqueueDroppedCounter.add(1); + } + + this.queue.push({ + sql, + values, + resolve, + reject, + timestamp: Date.now(), + }); + + if (this.queue.length >= this.maxBatchSize) { + this.flush(this.queue.splice(0, this.maxBatchSize)); + } else if (this.queue.length === 1) { + this.timeouts.push( + setTimeout(() => { + this.flush(this.queue.splice(0, this.queue.length)); + }, this.maxTimeInQueue), + ); + } + + return promise; + } + + async flush(batch) { + const timeout = this.timeouts.shift(); + if (timeout && !timeout._destroyed) { + clearTimeout(timeout); + } + if (batch.length === 0) return; + + const query = `${batch.map((b) => b.sql.replace(/;+\s*$/, '')).join(';')}; SELECT 1`; // SELECT 1 forces mysql2 to return array + const values = batch.map((b) => b.values ?? []).flat(); + + try { + const [results, fields] = await this.dbPool + .promise() + .query(query, values); + + this.#consecutiveFailures = 0; + for (let i = 0; i < batch.length; i++) { + const b = batch[i]; + b.resolve([results[i], fields?.[i]]); + } + } catch (error) { + this.#consecutiveFailures++; + this.#lastFailureAt = Date.now(); + flushFailureCounter.add(1); + console.warn('Error in SQLBatcher flush:', error); + for (const b of batch) { + b.reject(this.#createPublicBatchError()); + } + } + } +} diff --git a/src/backend/clients/database/SqliteDatabaseClient.ts b/src/backend/clients/database/SqliteDatabaseClient.ts new file mode 100644 index 000000000..00c7a5d3b --- /dev/null +++ b/src/backend/clients/database/SqliteDatabaseClient.ts @@ -0,0 +1,533 @@ +import { existsSync, readFileSync } from 'fs'; +import { basename, extname, join, resolve } from 'path'; +import { createContext, runInContext } from 'vm'; +import { AbstractDatabaseClient, type WriteResult } from './DatabaseClient'; +import type { IConfig } from '../../types'; + +const MIGRATIONS_DIR = resolve(__dirname, './migrations'); + +/** + * Ordered list of [threshold_version, files[]] pairs. + * A database whose `user_version` is <= threshold_version will have + * the corresponding files applied. + */ +const AVAILABLE_MIGRATIONS: [number, string[]][] = [ + [-1, ['0001_create-tables.sql', '0002_add-default-apps.sql']], + [0, ['0003_user-permissions.sql']], + [1, ['0004_sessions.sql']], + [2, ['0005_background-apps.sql']], + [3, ['0006_update-apps.sql']], + [4, ['0007_sessions.sql']], + [5, ['0008_otp.sql']], + [6, ['0009_app-prefix-fix.sql']], + [7, ['0010_add-git-app.sql']], + [8, ['0011_notification.sql']], + [9, ['0012_appmetadata.sql']], + [10, ['0013_protected-apps.sql']], + [11, ['0014_share.sql']], + [12, ['0015_group.sql']], + [13, ['0016_group-permissions.sql']], + [14, ['0017_publicdirs.sql']], + [15, ['0018_fix-0003.sql']], + [16, ['0019_fix-0016.sql']], + [17, ['0020_dev-center.sql']], + [18, ['0021_app-owner-id.sql']], + [19, ['0022_dev-center-max.sql']], + [20, ['0023_fix-kv.sql']], + [21, ['0024_default-groups.sql']], + [22, ['0025_system-user.dbmig.js']], + [23, ['0026_user-groups.dbmig.js']], + // 24 is skipped (0027 only registered in some branches) + [25, ['0028_clean-email.sql']], + // 26 skipped + [27, ['0030_comments.sql']], + [28, ['0031_audit-meta.sql']], + [29, ['0032_signup_metadata.sql']], + [30, ['0033_ai-usage.sql']], + [31, ['0034_app-redirect.sql']], + [32, ['0035_threads.sql']], + [33, ['0036_dev-to-app.sql']], + [34, ['0038_custom-domains.sql']], + [35, ['0039_add-expireAt-to-kv-store.sql']], + [36, ['0040_add_user_metadata.sql']], + [37, ['0041_add_unique_constraint_user_uuid.sql']], + [38, ['0042_add_cloudflare_d1.sql']], + [39, ['0043_add_dt.sql']], + [40, ['0044_dev-center-godmode.sql']], + [41, ['0045_user_oidc_providers.sql']], + [42, ['0046_is-private-apps.sql']], +]; + +export class SqliteDatabaseClient extends AbstractDatabaseClient { + override readonly engineName = 'sqlite'; + + // better-sqlite3 instance — set during onServerStart + private db!: InstanceType; + + constructor(config: IConfig) { + super(config); + } + + // ------------------------------------------------------------------ + // Lifecycle + // ------------------------------------------------------------------ + + override async onServerStart(): Promise { + const Database = (await import('better-sqlite3')).default; + + const dbPath = this.config.database?.path ?? ':memory:'; + const isNew = dbPath === ':memory:' || !existsSync(dbPath); + + this.db = new Database(dbPath); + + await this.runMigrations(isNew); + } + + override onServerShutdown(): void { + if (this.db) { + this.db.close(); + } + } + + // ------------------------------------------------------------------ + // Query interface + // ------------------------------------------------------------------ + + override async read( + query: string, + params: unknown[] = [], + ): Promise[]> { + query = this.transformQuery(query); + params = this.transformParams(params); + return this.db.prepare(query).all(...params) as Record< + string, + unknown + >[]; + } + + override async pread( + query: string, + params: unknown[] = [], + ): Promise[]> { + // SQLite is single-node — pread is identical to read + return this.read(query, params); + } + + override async write( + query: string, + params: unknown[] = [], + ): Promise { + query = this.transformQuery(query); + params = this.transformParams(params); + + const info = this.db.prepare(query).run(...params); + + return { + insertId: info.lastInsertRowid, + anyRowsAffected: info.changes > 0, + }; + } + + override async batchWrite( + entries: { statement: string; values: unknown[] }[], + ): Promise { + this.db.transaction(() => { + for (let { statement, values } of entries) { + statement = this.transformQuery(statement); + values = this.transformParams(values); + this.db.prepare(statement).run(...values); + } + })(); + } + + // ------------------------------------------------------------------ + // SQLite-specific transforms + // ------------------------------------------------------------------ + + private transformQuery(query: string): string { + return query.replace(/now\(\)/gi, "datetime('now')"); + } + + private transformParams(params: unknown[]): unknown[] { + return params.map((p) => { + if (typeof p === 'boolean') return p ? 1 : 0; + return p; + }); + } + + // ------------------------------------------------------------------ + // Migration system + // ------------------------------------------------------------------ + + private async runMigrations(isNew: boolean): Promise { + const highestVersion = + AVAILABLE_MIGRATIONS[AVAILABLE_MIGRATIONS.length - 1][0] + 1; + const targetVersion = + this.config.database?.targetVersion ?? highestVersion; + + const userVersion = isNew ? -1 : this.getEffectiveUserVersion(); + + console.log(`[sqlite] database version: ${userVersion}`); + + const toApply: string[] = []; + for (const [threshold, files] of AVAILABLE_MIGRATIONS) { + if ( + threshold + 1 >= targetVersion && + targetVersion !== highestVersion + ) { + console.warn( + `[sqlite] early exit: target version set to ${targetVersion}`, + ); + break; + } + if (userVersion <= threshold) { + toApply.push(...files); + } + } + + if (toApply.length === 0) return; + + console.log( + `[sqlite] upgrading database: ${userVersion} -> ${targetVersion} (${toApply.length} migration files)`, + ); + + for (const file of toApply) { + const filePath = join(MIGRATIONS_DIR, file); + const contents = readFileSync(filePath, 'utf8'); + const ext = extname(file); + const name = basename(file); + + switch (ext) { + case '.sql': + this.applySqlMigration(name, contents); + break; + case '.js': + await this.applyJsMigration(name, contents); + break; + default: + throw new Error( + `[sqlite] unrecognised migration type: ${file}`, + ); + } + } + + this.db.exec(`PRAGMA user_version = ${targetVersion};`); + console.log(`[sqlite] database upgraded to version ${targetVersion}`); + } + + private getEffectiveUserVersion(): number { + const userVersion = ( + this.db.prepare('PRAGMA user_version').get() as { + user_version: number; + } + ).user_version; + if (userVersion !== 0) return userVersion; + + const hasAppsTable = this.hasTable('apps'); + const hasUserTable = this.hasTable('user'); + + if (!hasAppsTable || !hasUserTable) { + console.warn( + '[sqlite] user_version=0 but bootstrap tables are missing; treating database as uninitialized', + ); + return -1; + } + + const inferredUserVersion = this.inferLegacyUserVersion(); + if (inferredUserVersion !== 0) { + console.warn( + `[sqlite] user_version=0; inferred legacy schema version ${inferredUserVersion}`, + ); + } + + return inferredUserVersion; + } + + private inferLegacyUserVersion(): number { + const markers: Array<{ version: number; check: () => boolean }> = [ + { + version: 1, + check: () => + this.hasTable('user_to_user_permissions') && + this.hasTable('audit_user_to_user_permissions'), + }, + { + version: 2, + check: () => this.hasTable('sessions'), + }, + { + version: 3, + check: () => this.hasColumn('apps', 'background'), + }, + { + version: 5, + check: () => + this.hasColumn('sessions', 'created_at') && + this.hasColumn('sessions', 'last_activity'), + }, + { + version: 6, + check: () => + this.hasColumn('user', 'otp_secret') && + this.hasColumn('user', 'otp_enabled') && + this.hasColumn('user', 'otp_recovery_codes'), + }, + { + version: 8, + check: () => + this.hasRow('SELECT 1 FROM `apps` WHERE `uid` = ?', [ + 'app-e3ac5486-da8c-42ad-8377-8728086e0980', + ]), + }, + { + version: 9, + check: () => this.hasTable('notification'), + }, + { + version: 10, + check: () => this.hasColumn('apps', 'metadata'), + }, + { + version: 11, + check: () => + this.hasColumn('apps', 'protected') && + this.hasColumn('subdomains', 'protected'), + }, + { + version: 12, + check: () => this.hasTable('share'), + }, + { + version: 13, + check: () => + this.hasTable('group') && this.hasTable('jct_user_group'), + }, + { + version: 14, + check: () => + this.hasTable('user_to_group_permissions') && + this.hasTable('audit_user_to_group_permissions'), + }, + { + version: 15, + check: () => + this.hasColumn('user', 'public_uuid') && + this.hasColumn('user', 'public_id'), + }, + { + version: 16, + check: () => + this.columnAllowsNull( + 'audit_user_to_user_permissions', + 'issuer_user_id', + ) && + this.columnAllowsNull( + 'audit_user_to_user_permissions', + 'holder_user_id', + ), + }, + { + version: 17, + check: () => + this.columnAllowsNull( + 'audit_user_to_group_permissions', + 'user_id', + ) && + this.columnAllowsNull( + 'audit_user_to_group_permissions', + 'group_id', + ), + }, + { + version: 18, + check: () => + this.hasRow('SELECT 1 FROM `apps` WHERE `uid` = ?', [ + 'app-0b37f054-07d4-4627-8765-11bd23e889d4', + ]), + }, + { + version: 21, + check: () => this.columnTypeIs('kv', 'value', 'JSON'), + }, + { + version: 22, + check: () => + this.hasRow('SELECT 1 FROM `group` WHERE `uid` = ?', [ + '26bfb1fb-421f-45bc-9aa4-d81ea569e7a5', + ]), + }, + { + version: 23, + check: () => + this.hasRow('SELECT 1 FROM `user` WHERE `uuid` = ?', [ + '5d4adce0-a381-4982-9c02-6e2540026238', + ]), + }, + { + version: 24, + check: () => + this.hasRow('SELECT 1 FROM `group` WHERE `uid` = ?', [ + 'b7220104-7905-4985-b996-649fdcdb3c8f', + ]), + }, + { + version: 26, + check: () => this.hasColumn('user', 'clean_email'), + }, + { + version: 28, + check: () => this.hasTable('user_comments'), + }, + { + version: 29, + check: () => this.hasColumn('user', 'audit_metadata'), + }, + { + version: 30, + check: () => this.hasColumn('user', 'signup_ip'), + }, + { + version: 31, + check: () => this.hasTable('ai_usage'), + }, + { + version: 32, + check: () => this.hasTable('old_app_names'), + }, + { + version: 33, + check: () => this.hasTable('thread'), + }, + { + version: 34, + check: () => + this.hasTable('dev_to_app_permissions') && + this.hasTable('audit_dev_to_app_permissions'), + }, + { + version: 35, + check: () => this.hasColumn('subdomains', 'domain'), + }, + { + version: 36, + check: () => this.hasColumn('kv', 'expireAt'), + }, + { + version: 37, + check: () => this.hasColumn('user', 'metadata'), + }, + { + version: 39, + check: () => this.hasColumn('subdomains', 'database_id'), + }, + { + version: 40, + check: () => this.hasColumn('user_to_app_permissions', 'dt'), + }, + { + version: 42, + check: () => this.hasTable('user_oidc_providers'), + }, + { + version: 43, + check: () => this.hasColumn('apps', 'is_private'), + }, + ]; + + let inferredUserVersion = 0; + + for (const marker of markers) { + if (marker.check()) { + inferredUserVersion = marker.version; + } + } + + return inferredUserVersion; + } + + private hasTable(table: string): boolean { + return Boolean( + this.db + .prepare( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", + ) + .get(table), + ); + } + + private hasColumn(table: string, column: string): boolean { + return Boolean( + this.db + .prepare( + `SELECT 1 FROM pragma_table_info(${this.quoteSqlString(table)}) WHERE name = ?`, + ) + .get(column), + ); + } + + private columnAllowsNull(table: string, column: string): boolean { + const info = this.db + .prepare( + `SELECT * FROM pragma_table_info(${this.quoteSqlString(table)}) WHERE name = ?`, + ) + .get(column) as { notnull: number } | undefined; + + return info?.notnull === 0; + } + + private columnTypeIs(table: string, column: string, type: string): boolean { + const info = this.db + .prepare( + `SELECT type FROM pragma_table_info(${this.quoteSqlString(table)}) WHERE name = ?`, + ) + .get(column) as { type: string } | undefined; + + return info?.type?.toUpperCase() === type.toUpperCase(); + } + + private hasRow(query: string, params: unknown[] = []): boolean { + try { + return Boolean(this.db.prepare(query).get(...params)); + } catch { + return false; + } + } + + private quoteSqlString(value: string): string { + return `'${value.replaceAll("'", "''")}'`; + } + + private applySqlMigration(name: string, contents: string): void { + const statements = contents.split(/;\s*\n/); + for (let i = 0; i < statements.length; i++) { + const stmt = statements[i].trim(); + if (stmt === '') continue; + try { + this.db.exec(`${stmt};`); + } catch (e) { + throw new Error( + `[sqlite] failed to apply ${name} at statement ${i}`, + { cause: e }, + ); + } + } + } + + private async applyJsMigration( + name: string, + contents: string, + ): Promise { + const wrapped = `(async () => {${contents}})()`; + const ctx = createContext({ + read: this.read.bind(this), + write: this.write.bind(this), + log: console, + console, + }); + try { + await runInContext(wrapped, ctx); + } catch (e) { + throw new Error(`[sqlite] failed to apply ${name}`, { cause: e }); + } + } +} diff --git a/src/backend/clients/database/index.ts b/src/backend/clients/database/index.ts new file mode 100644 index 000000000..7aa4aba22 --- /dev/null +++ b/src/backend/clients/database/index.ts @@ -0,0 +1,31 @@ +export { + AbstractDatabaseClient as DatabaseClient, + type WriteResult, + type BatchEntry, +} from './DatabaseClient'; +export { SqliteDatabaseClient } from './SqliteDatabaseClient'; +export { MySQLDatabaseClient } from './MySQLDatabaseClient'; + +import type { IConfig } from '../../types'; +import { AbstractDatabaseClient } from './DatabaseClient'; +import { MySQLDatabaseClient } from './MySQLDatabaseClient'; +import { SqliteDatabaseClient } from './SqliteDatabaseClient'; + +/** + * Factory class registered in `puterClients`. PuterServer calls + * `new DatabaseClientFactory(config)` — the constructor returns the + * concrete subclass selected by `config.database.engine`. + */ +export const DatabaseClientFactory = class DatabaseClientFactory { + constructor(config: IConfig) { + const engine = config.database?.engine ?? 'sqlite'; + switch (engine) { + case 'mysql': + return new MySQLDatabaseClient(config); + case 'sqlite': + return new SqliteDatabaseClient(config); + default: + throw new Error(`Unknown database engine: ${engine}`); + } + } +} as new (config: IConfig) => AbstractDatabaseClient; diff --git a/src/backend/src/services/database/sqlite_setup/0001_create-tables.sql b/src/backend/clients/database/migrations/0001_create-tables.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0001_create-tables.sql rename to src/backend/clients/database/migrations/0001_create-tables.sql diff --git a/src/backend/src/services/database/sqlite_setup/0002_add-default-apps.sql b/src/backend/clients/database/migrations/0002_add-default-apps.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0002_add-default-apps.sql rename to src/backend/clients/database/migrations/0002_add-default-apps.sql diff --git a/src/backend/src/services/database/sqlite_setup/0003_user-permissions.sql b/src/backend/clients/database/migrations/0003_user-permissions.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0003_user-permissions.sql rename to src/backend/clients/database/migrations/0003_user-permissions.sql diff --git a/src/backend/src/services/database/sqlite_setup/0004_sessions.sql b/src/backend/clients/database/migrations/0004_sessions.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0004_sessions.sql rename to src/backend/clients/database/migrations/0004_sessions.sql diff --git a/src/backend/src/services/database/sqlite_setup/0005_background-apps.sql b/src/backend/clients/database/migrations/0005_background-apps.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0005_background-apps.sql rename to src/backend/clients/database/migrations/0005_background-apps.sql diff --git a/src/backend/src/services/database/sqlite_setup/0006_update-apps.sql b/src/backend/clients/database/migrations/0006_update-apps.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0006_update-apps.sql rename to src/backend/clients/database/migrations/0006_update-apps.sql diff --git a/src/backend/src/services/database/sqlite_setup/0007_sessions.sql b/src/backend/clients/database/migrations/0007_sessions.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0007_sessions.sql rename to src/backend/clients/database/migrations/0007_sessions.sql diff --git a/src/backend/src/services/database/sqlite_setup/0008_otp.sql b/src/backend/clients/database/migrations/0008_otp.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0008_otp.sql rename to src/backend/clients/database/migrations/0008_otp.sql diff --git a/src/backend/src/services/database/sqlite_setup/0009_app-prefix-fix.sql b/src/backend/clients/database/migrations/0009_app-prefix-fix.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0009_app-prefix-fix.sql rename to src/backend/clients/database/migrations/0009_app-prefix-fix.sql diff --git a/src/backend/src/services/database/sqlite_setup/0010_add-git-app.sql b/src/backend/clients/database/migrations/0010_add-git-app.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0010_add-git-app.sql rename to src/backend/clients/database/migrations/0010_add-git-app.sql diff --git a/src/backend/src/services/database/sqlite_setup/0011_notification.sql b/src/backend/clients/database/migrations/0011_notification.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0011_notification.sql rename to src/backend/clients/database/migrations/0011_notification.sql diff --git a/src/backend/src/services/database/sqlite_setup/0012_appmetadata.sql b/src/backend/clients/database/migrations/0012_appmetadata.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0012_appmetadata.sql rename to src/backend/clients/database/migrations/0012_appmetadata.sql diff --git a/src/backend/src/services/database/sqlite_setup/0013_protected-apps.sql b/src/backend/clients/database/migrations/0013_protected-apps.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0013_protected-apps.sql rename to src/backend/clients/database/migrations/0013_protected-apps.sql diff --git a/src/backend/src/services/database/sqlite_setup/0014_share.sql b/src/backend/clients/database/migrations/0014_share.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0014_share.sql rename to src/backend/clients/database/migrations/0014_share.sql diff --git a/src/backend/src/services/database/sqlite_setup/0015_group.sql b/src/backend/clients/database/migrations/0015_group.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0015_group.sql rename to src/backend/clients/database/migrations/0015_group.sql diff --git a/src/backend/src/services/database/sqlite_setup/0016_group-permissions.sql b/src/backend/clients/database/migrations/0016_group-permissions.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0016_group-permissions.sql rename to src/backend/clients/database/migrations/0016_group-permissions.sql diff --git a/src/backend/src/services/database/sqlite_setup/0017_publicdirs.sql b/src/backend/clients/database/migrations/0017_publicdirs.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0017_publicdirs.sql rename to src/backend/clients/database/migrations/0017_publicdirs.sql diff --git a/src/backend/src/services/database/sqlite_setup/0018_fix-0003.sql b/src/backend/clients/database/migrations/0018_fix-0003.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0018_fix-0003.sql rename to src/backend/clients/database/migrations/0018_fix-0003.sql diff --git a/src/backend/src/services/database/sqlite_setup/0019_fix-0016.sql b/src/backend/clients/database/migrations/0019_fix-0016.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0019_fix-0016.sql rename to src/backend/clients/database/migrations/0019_fix-0016.sql diff --git a/src/backend/src/services/database/sqlite_setup/0020_dev-center.sql b/src/backend/clients/database/migrations/0020_dev-center.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0020_dev-center.sql rename to src/backend/clients/database/migrations/0020_dev-center.sql diff --git a/src/backend/src/services/database/sqlite_setup/0021_app-owner-id.sql b/src/backend/clients/database/migrations/0021_app-owner-id.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0021_app-owner-id.sql rename to src/backend/clients/database/migrations/0021_app-owner-id.sql diff --git a/src/backend/src/services/database/sqlite_setup/0022_dev-center-max.sql b/src/backend/clients/database/migrations/0022_dev-center-max.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0022_dev-center-max.sql rename to src/backend/clients/database/migrations/0022_dev-center-max.sql diff --git a/src/backend/src/services/database/sqlite_setup/0023_fix-kv.sql b/src/backend/clients/database/migrations/0023_fix-kv.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0023_fix-kv.sql rename to src/backend/clients/database/migrations/0023_fix-kv.sql diff --git a/src/backend/src/services/database/sqlite_setup/0024_default-groups.sql b/src/backend/clients/database/migrations/0024_default-groups.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0024_default-groups.sql rename to src/backend/clients/database/migrations/0024_default-groups.sql diff --git a/src/backend/src/services/database/sqlite_setup/0025_system-user.dbmig.js b/src/backend/clients/database/migrations/0025_system-user.dbmig.js similarity index 50% rename from src/backend/src/services/database/sqlite_setup/0025_system-user.dbmig.js rename to src/backend/clients/database/migrations/0025_system-user.dbmig.js index 58457ef0d..bf75732a6 100644 --- a/src/backend/src/services/database/sqlite_setup/0025_system-user.dbmig.js +++ b/src/backend/clients/database/migrations/0025_system-user.dbmig.js @@ -27,17 +27,21 @@ user to the first username in this sequence: let existing_user; -;[existing_user] = await read("SELECT username FROM `user` WHERE username='system'"); +[existing_user] = await read( + "SELECT username FROM `user` WHERE username='system'", +); -if ( existing_user ) { +if (existing_user) { let replace_num = 0; let replace_name = 'system_'; - for ( ;; ) { - ;[existing_user] = await read('SELECT username FROM `user` WHERE username=?', - [replace_name]); - if ( ! existing_user ) break; - replace_name = `system_${ replace_num++}`; + for (;;) { + [existing_user] = await read( + 'SELECT username FROM `user` WHERE username=?', + [replace_name], + ); + if (!existing_user) break; + replace_name = `system_${replace_num++}`; } console.debug('updating existing user called system', { @@ -45,24 +49,31 @@ if ( existing_user ) { replace_name, }); - await write('UPDATE `user` SET username=? WHERE username=\'system\' LIMIT 1', - [replace_name]); + await write( + "UPDATE `user` SET username=? WHERE username='system' LIMIT 1", + [replace_name], + ); } -const { insertId: system_user_id } = await write('INSERT INTO `user` (`uuid`, `username`) VALUES (?, ?)', - [ - '5d4adce0-a381-4982-9c02-6e2540026238', - 'system', - ]); +const { insertId: system_user_id } = await write( + 'INSERT INTO `user` (`uuid`, `username`) VALUES (?, ?)', + ['5d4adce0-a381-4982-9c02-6e2540026238', 'system'], +); -const [{ id: system_group_id }] = await read('SELECT id FROM `group` WHERE uid=?', - ['26bfb1fb-421f-45bc-9aa4-d81ea569e7a5']); +const [{ id: system_group_id }] = await read( + 'SELECT id FROM `group` WHERE uid=?', + ['26bfb1fb-421f-45bc-9aa4-d81ea569e7a5'], +); -const [{ id: admin_group_id }] = await read('SELECT id FROM `group` WHERE uid=?', - ['ca342a5e-b13d-4dee-9048-58b11a57cc55']); +const [{ id: admin_group_id }] = await read( + 'SELECT id FROM `group` WHERE uid=?', + ['ca342a5e-b13d-4dee-9048-58b11a57cc55'], +); // admin group has unlimited access to all drivers -await write('INSERT INTO `user_to_group_permissions` ' + - '(`user_id`, `group_id`, `permission`, `extra`) ' + - 'VALUES (?, ?, ?, ?)', -[system_user_id, admin_group_id, 'driver', '{}']); +await write( + 'INSERT INTO `user_to_group_permissions` ' + + '(`user_id`, `group_id`, `permission`, `extra`) ' + + 'VALUES (?, ?, ?, ?)', + [system_user_id, admin_group_id, 'driver', '{}'], +); diff --git a/mods/mods_available/kdmod/module.js b/src/backend/clients/database/migrations/0026_user-groups.dbmig.js similarity index 69% rename from mods/mods_available/kdmod/module.js rename to src/backend/clients/database/migrations/0026_user-groups.dbmig.js index c2d38a5f2..6d03351a1 100644 --- a/mods/mods_available/kdmod/module.js +++ b/src/backend/clients/database/migrations/0026_user-groups.dbmig.js @@ -16,10 +16,14 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -extension.on('install', ({ services }) => { - const { CustomPuterService } = require('./CustomPuterService.js'); - services.registerService('__custom-puter', CustomPuterService); - const { ShareTestService } = require('./ShareTestService.js'); - services.registerService('__share-test', ShareTestService); -}); +const { insertId: temp_group_id } = await write( + 'INSERT INTO `group` (`uid`, `owner_user_id`, `extra`, `metadata`) ' + + 'VALUES (?, ?, ?, ?)', + [ + 'b7220104-7905-4985-b996-649fdcdb3c8f', + 1, + '{"critical": true, "type": "default", "name": "temp"}', + '{"title": "Guest", "color": "#777777"}', + ], +); diff --git a/src/backend/src/services/database/sqlite_setup/0027_emulator-app.dbmig.js b/src/backend/clients/database/migrations/0027_emulator-app.dbmig.js similarity index 85% rename from src/backend/src/services/database/sqlite_setup/0027_emulator-app.dbmig.js rename to src/backend/clients/database/migrations/0027_emulator-app.dbmig.js index ac984a72c..f88c82547 100644 --- a/src/backend/src/services/database/sqlite_setup/0027_emulator-app.dbmig.js +++ b/src/backend/clients/database/migrations/0027_emulator-app.dbmig.js @@ -20,10 +20,12 @@ const insert = async (tbl, subject) => { const keys = Object.keys(subject); - await write(`INSERT INTO \`${ tbl }\` ` + - `(${ keys.map(key => key).join(', ') }) ` + - `VALUES (${ keys.map(() => '?').join(', ') })`, - keys.map(key => subject[key])); + await write( + `INSERT INTO \`${tbl}\` ` + + `(${keys.map((key) => key).join(', ')}) ` + + `VALUES (${keys.map(() => '?').join(', ')})`, + keys.map((key) => subject[key]), + ); }; await insert('apps', { diff --git a/src/backend/src/services/database/sqlite_setup/0028_clean-email.sql b/src/backend/clients/database/migrations/0028_clean-email.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0028_clean-email.sql rename to src/backend/clients/database/migrations/0028_clean-email.sql diff --git a/src/backend/src/services/database/sqlite_setup/0029_emulator_priv.sql b/src/backend/clients/database/migrations/0029_emulator_priv.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0029_emulator_priv.sql rename to src/backend/clients/database/migrations/0029_emulator_priv.sql diff --git a/src/backend/src/services/database/sqlite_setup/0030_comments.sql b/src/backend/clients/database/migrations/0030_comments.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0030_comments.sql rename to src/backend/clients/database/migrations/0030_comments.sql diff --git a/src/backend/src/services/database/sqlite_setup/0031_audit-meta.sql b/src/backend/clients/database/migrations/0031_audit-meta.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0031_audit-meta.sql rename to src/backend/clients/database/migrations/0031_audit-meta.sql diff --git a/src/backend/src/services/database/sqlite_setup/0032_signup_metadata.sql b/src/backend/clients/database/migrations/0032_signup_metadata.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0032_signup_metadata.sql rename to src/backend/clients/database/migrations/0032_signup_metadata.sql diff --git a/src/backend/src/services/database/sqlite_setup/0033_ai-usage.sql b/src/backend/clients/database/migrations/0033_ai-usage.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0033_ai-usage.sql rename to src/backend/clients/database/migrations/0033_ai-usage.sql diff --git a/src/backend/src/services/database/sqlite_setup/0034_app-redirect.sql b/src/backend/clients/database/migrations/0034_app-redirect.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0034_app-redirect.sql rename to src/backend/clients/database/migrations/0034_app-redirect.sql diff --git a/src/backend/src/services/database/sqlite_setup/0035_threads.sql b/src/backend/clients/database/migrations/0035_threads.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0035_threads.sql rename to src/backend/clients/database/migrations/0035_threads.sql diff --git a/src/backend/src/services/database/sqlite_setup/0036_dev-to-app.sql b/src/backend/clients/database/migrations/0036_dev-to-app.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0036_dev-to-app.sql rename to src/backend/clients/database/migrations/0036_dev-to-app.sql diff --git a/src/backend/src/services/database/sqlite_setup/0037_cost.sql b/src/backend/clients/database/migrations/0037_cost.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0037_cost.sql rename to src/backend/clients/database/migrations/0037_cost.sql diff --git a/src/backend/src/services/database/sqlite_setup/0038_custom-domains.sql b/src/backend/clients/database/migrations/0038_custom-domains.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0038_custom-domains.sql rename to src/backend/clients/database/migrations/0038_custom-domains.sql diff --git a/src/backend/src/services/database/sqlite_setup/0039_add-expireAt-to-kv-store.sql b/src/backend/clients/database/migrations/0039_add-expireAt-to-kv-store.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0039_add-expireAt-to-kv-store.sql rename to src/backend/clients/database/migrations/0039_add-expireAt-to-kv-store.sql diff --git a/src/backend/src/services/database/sqlite_setup/0040_add_user_metadata.sql b/src/backend/clients/database/migrations/0040_add_user_metadata.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0040_add_user_metadata.sql rename to src/backend/clients/database/migrations/0040_add_user_metadata.sql diff --git a/src/backend/src/services/database/sqlite_setup/0041_add_unique_constraint_user_uuid.sql b/src/backend/clients/database/migrations/0041_add_unique_constraint_user_uuid.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0041_add_unique_constraint_user_uuid.sql rename to src/backend/clients/database/migrations/0041_add_unique_constraint_user_uuid.sql diff --git a/src/backend/src/services/database/sqlite_setup/0042_add_cloudflare_d1.sql b/src/backend/clients/database/migrations/0042_add_cloudflare_d1.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0042_add_cloudflare_d1.sql rename to src/backend/clients/database/migrations/0042_add_cloudflare_d1.sql diff --git a/src/backend/src/services/database/sqlite_setup/0043_add_dt.sql b/src/backend/clients/database/migrations/0043_add_dt.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0043_add_dt.sql rename to src/backend/clients/database/migrations/0043_add_dt.sql diff --git a/src/backend/src/services/database/sqlite_setup/0044_dev-center-godmode.sql b/src/backend/clients/database/migrations/0044_dev-center-godmode.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0044_dev-center-godmode.sql rename to src/backend/clients/database/migrations/0044_dev-center-godmode.sql diff --git a/src/backend/src/services/database/sqlite_setup/0045_user_oidc_providers.sql b/src/backend/clients/database/migrations/0045_user_oidc_providers.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0045_user_oidc_providers.sql rename to src/backend/clients/database/migrations/0045_user_oidc_providers.sql diff --git a/src/backend/src/services/database/sqlite_setup/0046_is-private-apps.sql b/src/backend/clients/database/migrations/0046_is-private-apps.sql similarity index 100% rename from src/backend/src/services/database/sqlite_setup/0046_is-private-apps.sql rename to src/backend/clients/database/migrations/0046_is-private-apps.sql diff --git a/src/backend/clients/dynamodb/DDBClient.ts b/src/backend/clients/dynamodb/DDBClient.ts new file mode 100644 index 000000000..9302e40e1 --- /dev/null +++ b/src/backend/clients/dynamodb/DDBClient.ts @@ -0,0 +1,537 @@ +import { + CreateTableCommand, + CreateTableCommandInput, + DynamoDBClient, +} from '@aws-sdk/client-dynamodb'; +import { + BatchGetCommand, + BatchGetCommandInput, + BatchWriteCommand, + BatchWriteCommandInput, + DeleteCommand, + DynamoDBDocumentClient, + GetCommand, + PutCommand, + QueryCommand, + ScanCommand, + UpdateCommand, +} from '@aws-sdk/lib-dynamodb'; +import { NodeHttpHandler } from '@smithy/node-http-handler'; +import dynalite from 'dynalite'; +import { once } from 'node:events'; +import { Agent as httpsAgent } from 'node:https'; +import { PuterClient } from '../types'; +import type { IConfig, IDynamoConfig } from '../../types'; + +const LOCAL_DYNAMO_PATH_KEY = ':memory:'; +const localDynaliteEndpointPromises = new Map>(); +const MAX_BATCH_WRITE_ITEMS = 25; +const MAX_BATCH_WRITE_RETRIES = 8; +const BATCH_WRITE_RETRY_BASE_MS = 25; + +const getDynalitePathKey = (path?: string) => { + if (path === ':memory:') return LOCAL_DYNAMO_PATH_KEY; + return path || './volatile/runtime/puter-ddb'; +}; + +const getOrCreateLocalDynaliteEndpoint = async (pathKey: string) => { + let endpointPromise = localDynaliteEndpointPromises.get(pathKey); + if (endpointPromise) return endpointPromise; + + endpointPromise = (async () => { + const dynaliteOptions = + pathKey === LOCAL_DYNAMO_PATH_KEY + ? { createTableMs: 0 } + : { createTableMs: 0, path: pathKey }; + + const dynaliteInstance = dynalite(dynaliteOptions); + const dynaliteServer = dynaliteInstance.listen(0, '127.0.0.1'); + dynaliteServer.unref?.(); + await once(dynaliteServer, 'listening'); + + const address = dynaliteServer.address(); + const port = + (typeof address === 'object' && address + ? address.port + : undefined) || 4567; + return `http://127.0.0.1:${port}`; + })(); + + localDynaliteEndpointPromises.set(pathKey, endpointPromise); + endpointPromise.catch(() => { + if (localDynaliteEndpointPromises.get(pathKey) === endpointPromise) { + localDynaliteEndpointPromises.delete(pathKey); + } + }); + + return endpointPromise; +}; + +const chunkValues = (values: T[], size: number): T[][] => { + if (values.length === 0) { + return []; + } + + const chunks: T[][] = []; + for (let index = 0; index < values.length; index += size) { + chunks.push(values.slice(index, index + size)); + } + return chunks; +}; + +const sleep = async (ms: number) => { + await new Promise((resolve) => setTimeout(resolve, ms)); +}; + +export class DDBClient extends PuterClient { + #documentClient: DynamoDBDocumentClient | null = null; + #localInitPromise: Promise | null = null; + #ddbConfig: IDynamoConfig; + + constructor(config: IConfig) { + super(config); + this.#ddbConfig = config.dynamo ?? {}; + + if (this.#ddbConfig.aws) { + this.#bindAwsClient(); + return; + } + + this.#localInitPromise = this.#bindLocalClient(); + this.#localInitPromise.catch((error) => { + console.error('Failed to initialize local DynamoDB client', error); + }); + } + + async recreateClient() { + if (this.#ddbConfig.aws) { + this.#bindAwsClient(); + return; + } + + this.#localInitPromise = this.#bindLocalClient(); + await this.#localInitPromise; + } + + async get>( + table: string, + key: T, + consistentRead = false, + ) { + const command = new GetCommand({ + TableName: table, + Key: key, + ConsistentRead: consistentRead, + ReturnConsumedCapacity: 'TOTAL', + }); + + const client = await this.#getDocumentClient(); + return client.send(command); + } + + async put>(table: string, item: T) { + const command = new PutCommand({ + TableName: table, + Item: item, + ReturnConsumedCapacity: 'TOTAL', + }); + + const client = await this.#getDocumentClient(); + return client.send(command); + } + + async batchGet( + params: { table: string; items: Record }[], + consistentRead = false, + ) { + const allRequestItemsPerTable = params.reduce( + (acc, curr) => { + if (!acc[curr.table]) acc[curr.table] = []; + acc[curr.table].push(curr.items); + return acc; + }, + {} as Record[]>, + ); + + const requestItems: BatchGetCommandInput['RequestItems'] = + Object.entries(allRequestItemsPerTable).reduce( + (acc, [table, keyList]) => { + acc[table] = { + Keys: keyList, + ConsistentRead: consistentRead, + }; + return acc; + }, + {} as NonNullable, + ); + + const command = new BatchGetCommand({ + RequestItems: requestItems, + ReturnConsumedCapacity: 'TOTAL', + }); + + const client = await this.#getDocumentClient(); + return client.send(command); + } + + async batchPut(params: { table: string; item: Record }[]) { + const consumedCapacityByTable = new Map(); + if (params.length === 0) { + return { ConsumedCapacity: [] }; + } + + const accumulateConsumedCapacity = ( + consumedCapacityEntries: + | Array<{ TableName?: string; CapacityUnits?: number }> + | undefined, + ) => { + if (!consumedCapacityEntries) { + return; + } + + for (const consumedCapacityEntry of consumedCapacityEntries) { + const table = consumedCapacityEntry.TableName; + if (!table) { + continue; + } + + const existingUsage = consumedCapacityByTable.get(table) ?? 0; + consumedCapacityByTable.set( + table, + existingUsage + + Number(consumedCapacityEntry.CapacityUnits ?? 0), + ); + } + }; + + const client = await this.#getDocumentClient(); + const chunks = chunkValues(params, MAX_BATCH_WRITE_ITEMS); + + for (const chunk of chunks) { + let requestItems = chunk.reduce( + (acc, curr) => { + const tableRequests = acc[curr.table] ?? []; + tableRequests.push({ + PutRequest: { + Item: curr.item, + }, + }); + acc[curr.table] = tableRequests; + return acc; + }, + {} as NonNullable, + ); + + for ( + let attempt = 0; + attempt <= MAX_BATCH_WRITE_RETRIES; + attempt++ + ) { + if (Object.keys(requestItems).length === 0) { + break; + } + + const response = await client.send( + new BatchWriteCommand({ + RequestItems: requestItems, + ReturnConsumedCapacity: 'TOTAL', + }), + ); + accumulateConsumedCapacity( + response.ConsumedCapacity as + | Array<{ TableName?: string; CapacityUnits?: number }> + | undefined, + ); + + const unprocessedItems = response.UnprocessedItems ?? {}; + if (Object.keys(unprocessedItems).length === 0) { + requestItems = {}; + break; + } + + requestItems = unprocessedItems as NonNullable< + BatchWriteCommandInput['RequestItems'] + >; + if (attempt < MAX_BATCH_WRITE_RETRIES) { + const delayMs = Math.min( + 1000, + BATCH_WRITE_RETRY_BASE_MS * 2 ** attempt, + ); + await sleep(delayMs); + } + } + + if (Object.keys(requestItems).length > 0) { + throw new Error('Failed to batch write all items to DynamoDB'); + } + } + + return { + ConsumedCapacity: Array.from(consumedCapacityByTable.entries()).map( + ([TableName, CapacityUnits]) => ({ + TableName, + CapacityUnits, + }), + ), + }; + } + + async del>(table: string, key: T) { + const command = new DeleteCommand({ + TableName: table, + Key: key, + ReturnConsumedCapacity: 'TOTAL', + }); + + const client = await this.#getDocumentClient(); + return client.send(command); + } + + async query>( + table: string, + keys: T, + limit = 0, + pageKey?: Record, + index = '', + consistentRead = false, + options?: { beginsWith?: { key: string; value: string } }, + ) { + const keyExpressionParts = Object.keys(keys).map( + (key) => `#${key} = :${key}`, + ); + const expressionAttributeValues = Object.entries(keys).reduce( + (acc, [key, value]) => { + acc[`:${key}`] = value; + return acc; + }, + {} as Record, + ); + const expressionAttributeNames = Object.keys(keys).reduce( + (acc, key) => { + acc[`#${key}`] = key; + return acc; + }, + {} as Record, + ); + + if (options?.beginsWith?.key && options.beginsWith.value !== '') { + const beginsKey = options.beginsWith.key; + const beginsValueToken = `:${beginsKey}_begins_with`; + keyExpressionParts.push( + `begins_with(#${beginsKey}, ${beginsValueToken})`, + ); + expressionAttributeValues[beginsValueToken] = + options.beginsWith.value; + expressionAttributeNames[`#${beginsKey}`] = beginsKey; + } + + const command = new QueryCommand({ + TableName: table, + ...(!index ? {} : { IndexName: index }), + KeyConditionExpression: keyExpressionParts.join(' AND '), + ExpressionAttributeValues: expressionAttributeValues, + ExpressionAttributeNames: expressionAttributeNames, + ConsistentRead: consistentRead, + ...(!pageKey ? {} : { ExclusiveStartKey: pageKey }), + ...(!limit ? {} : { Limit: limit }), + ReturnConsumedCapacity: 'TOTAL', + }); + + const client = await this.#getDocumentClient(); + return client.send(command); + } + + async update>( + table: string, + key: T, + expression: string, + expressionValues?: Record, + expressionNames?: Record, + ) { + const hasValues = + !!expressionValues && Object.keys(expressionValues).length > 0; + const hasNames = + !!expressionNames && Object.keys(expressionNames).length > 0; + const command = new UpdateCommand({ + TableName: table, + Key: key, + UpdateExpression: expression, + ...(hasValues + ? { ExpressionAttributeValues: expressionValues } + : {}), + ...(hasNames ? { ExpressionAttributeNames: expressionNames } : {}), + ReturnValues: 'ALL_NEW', + ReturnConsumedCapacity: 'TOTAL', + }); + + try { + const client = await this.#getDocumentClient(); + return await client.send(command); + } catch (error) { + console.error('DDB Update Error', error); + throw error; + } + } + + async createTableIfNotExists( + params: CreateTableCommandInput, + ttlAttribute?: string, + ) { + if (this.#ddbConfig.aws) { + console.warn( + 'Creating DynamoDB tables in AWS is disabled by default, but if needed, update DDBClient', + ); + return; + } + + try { + const client = await this.#getDocumentClient(); + await client.send(new CreateTableCommand(params)); + } catch (error) { + if ((error as Error)?.name !== 'ResourceInUseException') { + throw error; + } + } + + if (ttlAttribute) { + await this.#deleteExpiredItems( + params.TableName!, + params.KeySchema!, + ttlAttribute, + ); + } + } + + async #getDocumentClient() { + if (this.#documentClient) { + return this.#documentClient; + } + + if (this.#localInitPromise) { + await this.#localInitPromise; + } + + if (!this.#documentClient) { + throw new Error('DynamoDB document client is not initialized'); + } + + return this.#documentClient; + } + + #bindAwsClient() { + const accessKeyId = this.#ddbConfig.aws?.access_key; + const secretAccessKey = this.#ddbConfig.aws?.secret_key; + + if (!accessKeyId || !secretAccessKey) { + throw new Error( + 'DynamoDB aws config requires both `access_key` and `secret_key`', + ); + } + + const ddbClient = new DynamoDBClient({ + credentials: { + accessKeyId, + secretAccessKey, + }, + maxAttempts: 3, + requestHandler: new NodeHttpHandler({ + connectionTimeout: 5000, + requestTimeout: 5000, + httpsAgent: new httpsAgent({ keepAlive: true }), + }), + ...(this.#ddbConfig.endpoint + ? { endpoint: this.#ddbConfig.endpoint } + : {}), + region: this.#ddbConfig.aws?.region || 'us-west-2', + }); + + this.#documentClient = DynamoDBDocumentClient.from(ddbClient, { + marshallOptions: { + removeUndefinedValues: true, + }, + }); + } + + async #bindLocalClient() { + const pathKey = getDynalitePathKey(this.#ddbConfig.path); + const endpoint = await getOrCreateLocalDynaliteEndpoint(pathKey); + + const ddbClient = new DynamoDBClient({ + credentials: { + accessKeyId: 'fake', + secretAccessKey: 'fake', + }, + maxAttempts: 3, + requestHandler: new NodeHttpHandler({ + connectionTimeout: 5000, + requestTimeout: 5000, + httpsAgent: new httpsAgent({ keepAlive: true }), + }), + endpoint, + region: 'us-west-2', + }); + + this.#documentClient = DynamoDBDocumentClient.from(ddbClient, { + marshallOptions: { + removeUndefinedValues: true, + }, + }); + } + + async #deleteExpiredItems( + table: string, + keySchema: NonNullable, + ttlAttribute: string, + ) { + const now = Math.floor(Date.now() / 1000); + const keyNames = keySchema.map((key) => key.AttributeName!); + + let lastEvaluatedKey: Record | undefined; + const client = await this.#getDocumentClient(); + + do { + const scan = await client.send( + new ScanCommand({ + TableName: table, + FilterExpression: '#ttl < :now', + ExpressionAttributeNames: { + '#ttl': ttlAttribute, + ...Object.fromEntries( + keyNames.map((key) => [`#k_${key}`, key]), + ), + }, + ExpressionAttributeValues: { ':now': now }, + ProjectionExpression: keyNames + .map((key) => `#k_${key}`) + .join(', '), + ...(lastEvaluatedKey + ? { ExclusiveStartKey: lastEvaluatedKey } + : {}), + }), + ); + + lastEvaluatedKey = scan.LastEvaluatedKey as + | Record + | undefined; + const items = scan.Items; + if (!items || items.length === 0) continue; + + const chunks = chunkValues(items, MAX_BATCH_WRITE_ITEMS); + for (const chunk of chunks) { + await client.send( + new BatchWriteCommand({ + RequestItems: { + [table]: chunk.map((item) => ({ + DeleteRequest: { + Key: Object.fromEntries( + keyNames.map((key) => [key, item[key]]), + ), + }, + })), + }, + }), + ); + } + } while (lastEvaluatedKey); + } +} diff --git a/src/backend/clients/email/EmailClient.ts b/src/backend/clients/email/EmailClient.ts new file mode 100644 index 000000000..a6449803b --- /dev/null +++ b/src/backend/clients/email/EmailClient.ts @@ -0,0 +1,247 @@ +import dedent from 'dedent'; +import handlebars, { template } from 'handlebars'; +import nodemailer from 'nodemailer'; +import type { IConfig } from '../../types'; +import { PuterClient } from '../types'; +import { EMAIL_TEMPLATES, type EmailTemplateName } from './templates'; + +// ── Types ──────────────────────────────────────────────────────────── + +// nodemailer doesn't ship TS types, so declare the subset we use. +interface NodemailerTransport { + sendMail: (options: SendMailOptions) => Promise; + close?: () => void; +} + +export interface SendMailOptions { + from?: string; + to: string; + cc?: string; + bcc?: string; + subject: string; + html?: string; + text?: string; + replyTo?: string; +} + +export type EmailValidator = (email: string) => Promise | boolean; + +interface CompiledTemplate { + subject: ReturnType; + html: ReturnType; +} + +// ── Clean-email rules ──────────────────────────────────────────────── + +type CleanRule = (parts: { local: string; domain: string }) => { + local: string; + domain: string; +}; + +const CLEAN_RULES: Record = { + dots_dont_matter: ({ local, domain }) => ({ + local: local.replace(/\./g, ''), + domain, + }), + remove_subaddressing: ({ local, domain }) => ({ + local: local.split('+')[0], + domain, + }), +}; + +const PROVIDER_RULES: Record = { + gmail: { apply: ['dots_dont_matter'], skip: [] }, + icloud: { apply: ['dots_dont_matter'], skip: [] }, + yahoo: { apply: [], skip: ['remove_subaddressing'] }, +}; + +const DOMAIN_TO_PROVIDER: Record = { + 'gmail.com': 'gmail', + 'googlemail.com': 'gmail', + 'yahoo.com': 'yahoo', + 'yahoo.co.uk': 'yahoo', + 'yahoo.ca': 'yahoo', + 'yahoo.com.au': 'yahoo', + 'icloud.com': 'icloud', + 'me.com': 'icloud', + 'mac.com': 'icloud', +}; + +const DOMAIN_ALIASES: Record = { + 'googlemail.com': 'gmail.com', +}; + +// ── EmailClient ────────────────────────────────────────────────────── + +/** + * Unified email client. Handles: + * - Template-based outbound mail (via `send`) + * - Raw nodemailer passthrough (via `sendRaw`) + * - Canonical-form normalization for dedup (via `clean`) + * - Policy + extensible validation (via `validate`) + */ +export class EmailClient extends PuterClient { + private transport: NodemailerTransport | null = null; + private compiledTemplates: Partial< + Record + > = {}; + private validators: EmailValidator[] = []; + + constructor(config: IConfig) { + super(config); + this.registerHandlebarsHelpers(); + this.compileTemplates(); + } + + // ── Lifecycle ──────────────────────────────────────────────────── + + override onServerStart(): void { + const emailConf = this.config.email; + if (!emailConf) { + console.warn( + '[email] no email transport configured — send() will fail until configured', + ); + return; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + this.transport = nodemailer.createTransport(emailConf as any); + console.log('[email] transport configured'); + } + + override onServerShutdown(): void { + this.transport?.close?.(); + this.transport = null; + } + + // ── Public API: sending ────────────────────────────────────────── + + /** + * Render a template and send it to `to`. + */ + async send( + to: string, + template: T, + values: Record = {}, + ): Promise { + const compiled = this.compiledTemplates[template]; + if (!compiled) { + throw new Error(`Unknown email template: ${template}`); + } + + await this.sendRaw({ + from: this.defaultFrom(), + to, + subject: compiled.subject(values), + html: compiled.html(values), + }); + } + + /** + * Raw send — bypasses the template system. Useful for one-off + * admin emails that don't warrant a named template. + */ + async sendRaw(options: SendMailOptions): Promise { + if (!this.transport) { + throw new Error('EmailClient transport is not configured'); + } + await this.transport.sendMail({ + from: options.from ?? this.defaultFrom(), + ...options, + }); + } + + // ── Public API: clean / validate ───────────────────────────────── + + /** + * Normalize an email to its canonical form for dedup comparisons. + * Applies provider-specific rules (e.g. Gmail ignores dots in + * the local part) plus generic subaddressing removal. + */ + clean(email: string): string { + let [local, domain] = email.split('@'); + if (!local || !domain) return email; + + if (DOMAIN_ALIASES[domain]) { + domain = DOMAIN_ALIASES[domain]; + } + + // Default: strip subaddressing on everything unless provider skips it + const ruleNames = new Set(['remove_subaddressing']); + const provider = DOMAIN_TO_PROVIDER[domain]; + const rules = provider ? PROVIDER_RULES[provider] : undefined; + + if (rules) { + rules.apply.forEach((r) => ruleNames.add(r)); + rules.skip.forEach((r) => ruleNames.delete(r)); + } + + let parts = { local, domain }; + for (const name of ruleNames) { + parts = CLEAN_RULES[name](parts); + } + + return `${parts.local}@${parts.domain}`; + } + + /** + * Check whether an email is allowed to be used. Checks domain + * blocklist plus any registered validators (services can call + * `addValidator()` to register custom policy hooks). + */ + async validate(email: string): Promise { + if (this.config.env === 'dev') return true; + + const cleaned = this.clean(email); + + const blocked = this.config.blockedEmailDomains; + if (Array.isArray(blocked)) { + for (const suffix of blocked) { + if (cleaned.endsWith(suffix)) return false; + } + } + + for (const validator of this.validators) { + const ok = await validator(cleaned); + if (!ok) return false; + } + + return true; + } + + /** + * Register a custom validation hook. Services can call this + * during their startup to veto specific emails (e.g. a + * disposable-email service). + */ + addValidator(fn: EmailValidator): void { + this.validators.push(fn); + } + + // ── Internals ──────────────────────────────────────────────────── + + private defaultFrom(): string { + return this.config.email?.from ?? '"Puter" no-reply@puter.com'; + } + + private registerHandlebarsHelpers(): void { + handlebars.registerHelper('nl2br', (text: unknown) => { + if (text == null) return ''; + const escaped = String(text) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); + return new handlebars.SafeString(escaped.replace(/\n/g, '
')); + }); + } + + private compileTemplates(): void { + for (const [name, template] of Object.entries(EMAIL_TEMPLATES)) { + this.compiledTemplates[name as EmailTemplateName] = { + subject: handlebars.compile(template.subject), + html: handlebars.compile(dedent(template.html)), + }; + } + } +} diff --git a/src/backend/clients/email/templates.ts b/src/backend/clients/email/templates.ts new file mode 100644 index 000000000..c1110b46c --- /dev/null +++ b/src/backend/clients/email/templates.ts @@ -0,0 +1,178 @@ +/** + * Email template definitions. Keys are the template names; values are + * the Handlebars-compilable `subject` and `html` strings. + * + * Rendered values are supplied by callers of `EmailClient.send()`. + * Variables use standard Handlebars syntax: `{{var}}`, `{{#if cond}}…{{/if}}`, + * and the custom helper `{{{nl2br text}}}` for HTML-safe newline conversion. + */ + +export interface EmailTemplate { + subject: string; + html: string; +} + +export const EMAIL_TEMPLATES = { + 'approved-for-listing': { + subject: '🎉 Your app has been approved for listing!', + html: ` +

Hi there,

+

+Exciting news! {{app_title}} is now approved and live on Puter App Center. It's now ready for users worldwide to discover and enjoy. +

+

+Next Step: As your app begins to gain traction with more users, we will conduct periodic reviews to assess its performance and user engagement. Once your app meets our criteria, we'll invite you to our Incentive Program. This exclusive program will allow you to earn revenue each time users open your app. So, keep an eye out for updates and stay tuned for this exciting opportunity! Make sure to share your app with your fans, friends and family to help it gain traction: https://puter.com/app/{{app_name}} +

+ +

Best,
+The Puter Team +

+ `, + }, + 'listing-rejected': { + subject: 'App Center Listing Request Rejected', + html: ` +

Hi{{#if owner_username}} {{owner_username}}{{/if}},

+

+Thanks for submitting {{app_title}} for the Puter App Center. We reviewed your listing and have rejected it for the following reason(s): +

+
{{{nl2br reason}}}
+

+Please update your app listing and resubmit when ready. If you have questions, just reply to this email. +

+

Best,
+The Puter Team +

+ `, + }, + 'listing-update-request': { + subject: 'Update request for your app listing', + html: ` +

Hi{{#if owner_username}} {{owner_username}}{{/if}},

+

+Please update {{app_title}}. +

+

Requested updates:

+
{{message}}
+

Best,
+The Puter Team +

+ `, + }, + email_change_request: { + subject: '📝 Confirm your email change', + html: ` +

Hi there,

+

+We received a request to link this email to the user "{{username}}" on Puter. If you made this request, please click the link below to confirm the change. If you did not make this request, please ignore this email. +

+ +

+Confirm email change +

+ `, + }, + email_change_notification: { + subject: '📝 Notification of email change', + html: ` +

Hi there,

+

+We're sending an email to let you know about a change to your account. +We have sent a confirmation to "{{new_email}}" to confirm an email change request. +If this was not you, please contact support@puter.com immediately. +

+ `, + }, + password_change_notification: { + subject: '🔑 Password change notification', + html: ` +

Hi there,

+

+We're sending an email to let you know about a change to your account. +Your password was recently changed. If this was not you, please contact +support@puter.com immediately. +

+ `, + }, + email_verification_code: { + subject: '{{code}} is your confirmation code', + html: ` +

Hi there,

+

{{code}} is your email confirmation code.

+

Sincerely,

+

Puter

+ `, + }, + email_verification_link: { + subject: 'Please confirm your email', + html: ` +

Hi there,

+

Please confirm your email address using this link: {{link}}.

+

Sincerely,

+

Puter

+ `, + }, + email_password_recovery: { + subject: 'Password Recovery', + html: ` +

Hi there,

+

A password recovery request was issued for your account, please follow the link below to reset your password:

+

{{link}}

+

Sincerely,

+

Puter

+ `, + }, + enabled_2fa: { + subject: '2FA Enabled on your Account', + html: ` +

Hi there,

+

We're sending you this email to let you know 2FA was successfully enabled +on your account

+

If you did not perform this action please contact support@puter.com +immediately

+

Sincerely,

+

Puter

+ `, + }, + disabled_2fa: { + subject: '2FA Disabled on your Account', + html: ` +

Hi there,

+

We hope you did this on purpose! 2FA Was disabled on your account.

+

If you did not perform this action please contact support@puter.com +immediately

+

Sincerely,

+

Puter

+ `, + }, + share_by_username: { + subject: 'Puter share from {{susername}}', + html: ` +

Hi there {{rusername}},

+

You've received a share from {{susername}} on Puter.

+

Go to puter.com to check it out.

+{{#if message}} +

The following message was included:

+
{{message}}
+{{/if}} +

Sincerely,

+

Puter

+ `, + }, + share_by_email: { + subject: 'share by email', + html: ` +

Hi there,

+

You've received a share from {{sender_name}} on Puter:

+

{{link}}

+{{#if message}} +

The following message was included:

+
{{message}}
+{{/if}} +

Sincerely,

+

Puter

+ `, + }, +} satisfies Record; + +export type EmailTemplateName = keyof typeof EMAIL_TEMPLATES; diff --git a/src/backend/clients/index.ts b/src/backend/clients/index.ts new file mode 100644 index 000000000..e72f67371 --- /dev/null +++ b/src/backend/clients/index.ts @@ -0,0 +1,18 @@ +import { AlarmClient } from './alarm/AlarmClient'; +import { DatabaseClientFactory } from './database'; +import { EmailClient } from './email/EmailClient'; +import { EventClient } from './EventClient'; +import { DDBClient } from './dynamodb/DDBClient'; +import { RedisClient } from './redis/RedisClient'; +import { S3Client } from './s3/S3Client'; +import type { IPuterClientRegistry } from './types'; + +export const puterClients = { + alarm: AlarmClient, + db: DatabaseClientFactory, + email: EmailClient, + event: EventClient, + dynamo: DDBClient, + redis: RedisClient, + s3: S3Client, +} satisfies IPuterClientRegistry; diff --git a/src/backend/clients/redis/RedisClient.ts b/src/backend/clients/redis/RedisClient.ts new file mode 100644 index 000000000..650456915 --- /dev/null +++ b/src/backend/clients/redis/RedisClient.ts @@ -0,0 +1,120 @@ +import Redis, { Cluster } from 'ioredis'; +import MockRedis from 'ioredis-mock'; +import type { IConfig, WithLifecycle } from '../../types'; + +const redisStartupRetryMaxDelayMs = 2000; +const redisSlotsRefreshTimeoutMs = 5000; +const redisConnectTimeoutMs = 10000; +const redisBootRetryRegex = + /Cluster(All)?FailedError|None of startup nodes is available/i; + +const formatRedisError = (error: unknown): string => { + if (error instanceof Error) { + return `${error.name}: ${error.message}`; + } + return String(error); +}; + +const attachClusterEventHandlers = (clusterClient: Cluster): void => { + clusterClient.once('connect', () => { + console.log('[redis] cluster transport connected'); + }); + + clusterClient.once('ready', () => { + console.log('[redis] cluster ready'); + }); + + clusterClient.on('error', (error: unknown) => { + const errorText = formatRedisError(error); + if (redisBootRetryRegex.test(errorText)) { + console.warn( + `[redis] startup issue while connecting to cluster; retrying automatically (${errorText})`, + ); + return; + } + console.error('[redis] cluster error', error); + }); + + clusterClient.on('node error', (error: unknown, nodeKey: string) => { + const errorText = formatRedisError(error); + if (redisBootRetryRegex.test(errorText)) { + console.warn( + `[redis] startup issue for cluster node ${nodeKey}; retrying automatically (${errorText})`, + ); + return; + } + console.error(`[redis] cluster node error (${nodeKey})`, error); + }); +}; + +const buildCluster = (config: IConfig): Cluster => { + const redisConfig = config.redis ?? {}; + const startupNodes = redisConfig.startupNodes ?? []; + const useMock = redisConfig.useMock ?? startupNodes.length === 0; + + if (useMock) { + console.log('connected to local redis mock'); + return new MockRedis.Cluster([ + 'redis://localhost:7001', + ]) as unknown as Cluster; + } + + const cluster = new Redis.Cluster( + startupNodes as ConstructorParameters[0], + { + dnsLookup: (address, callback) => callback(null, address), + clusterRetryStrategy: (attempts) => + Math.min(100 + attempts * 100, redisStartupRetryMaxDelayMs), + retryDelayOnFailover: 50, + retryDelayOnClusterDown: 50, + retryDelayOnTryAgain: 50, + slotsRefreshTimeout: redisSlotsRefreshTimeoutMs, + enableOfflineQueue: true, + redisOptions: { + tls: {}, + connectTimeout: redisConnectTimeoutMs, + maxRetriesPerRequest: 1, + }, + }, + ); + attachClusterEventHandlers(cluster); + console.log('connecting to redis from config'); + return cluster; +}; + +/** + * `RedisClient` IS the ioredis `Cluster` instance — consumers call + * `this.clients.redis.get(...)` / `.set(...)` directly rather than + * going through an inner `.client` field. Lifecycle methods + * (`onServerShutdown`) are attached onto the cluster instance itself. + * + * Type-wise, `RedisClient` is `Cluster & WithLifecycle`; the registry- + * facing value below is a constructor that returns that shape. Mirrors + * the `DatabaseClientFactory` pattern. + */ +export type RedisClient = Cluster & WithLifecycle; + +export const RedisClient = class RedisClient { + constructor(config: IConfig) { + const cluster = buildCluster(config); + + const onServerShutdown = async (): Promise => { + try { + await cluster.quit(); + } catch (error) { + console.warn( + '[redis] failed to quit redis client cleanly', + error, + ); + cluster.disconnect(); + } + }; + + // Attach lifecycle hooks directly onto the cluster instance so the + // server boot loop's `if (client.onServerShutdown) client.onServerShutdown()` + // picks them up without a wrapper object. + Object.assign(cluster, { onServerShutdown }); + + return cluster as unknown as RedisClient; + } +} as unknown as new (config: IConfig) => RedisClient; diff --git a/src/backend/clients/s3/S3Client.ts b/src/backend/clients/s3/S3Client.ts new file mode 100644 index 000000000..a8d5f8d8c --- /dev/null +++ b/src/backend/clients/s3/S3Client.ts @@ -0,0 +1,308 @@ +import { + AbortMultipartUploadCommand, + CompleteMultipartUploadCommand, + CreateMultipartUploadCommand, + PutObjectCommand, + S3Client as AwsS3Client, + type S3ClientConfig, + UploadPartCommand, +} from '@aws-sdk/client-s3'; +import { fromNodeProviderChain } from '@aws-sdk/credential-providers'; +import { NodeHttpHandler } from '@smithy/node-http-handler'; +import type { FauxqsServer } from 'fauxqs'; +import { existsSync } from 'node:fs'; +import fs from 'node:fs/promises'; +import { Agent as HttpsAgent } from 'node:https'; +import path from 'node:path'; +import type { IConfig } from '../../types'; +import { nativeImport } from '../../util/nativeImport.js'; +import { PuterClient } from '../types'; + +const DEFAULT_MULTIPART_PART_SIZE_BYTES = 5 * 1024 * 1024; +const FAUXQS_SAFE_PUT_OBJECT_LIMIT_BYTES = 10 * 1024 * 1024; +const LEGACY_STORAGE_BUCKET = 'puter-local'; + +type S3CommandSender = Pick; + +export class S3Client extends PuterClient { + private clientMap = new Map(); + private awsConfig: Partial = {}; + private fauxqsServer: FauxqsServer | null = null; + private useProviderChain = false; + + /** Maximum size for a single PutObject call before switching to multipart. */ + maxSingleUploadSize = FAUXQS_SAFE_PUT_OBJECT_LIMIT_BYTES; + /** Part size used for multipart uploads. */ + partSize = DEFAULT_MULTIPART_PART_SIZE_BYTES; + + constructor(config: IConfig) { + super(config); + } + + // ------------------------------------------------------------------ + // Lifecycle + // ------------------------------------------------------------------ + + override async onServerStart(): Promise { + const s3Conf = this.config.s3; + + if (s3Conf && 's3Config' in s3Conf && s3Conf.s3Config) { + // Real S3 / S3-compatible endpoint + const { + endpoint, + accessKeyId, + secretAccessKey, + region, + useCredentialChain, + } = s3Conf.s3Config; + + if (useCredentialChain) { + this.useProviderChain = true; + this.awsConfig = { credentials: fromNodeProviderChain() }; + this.partSize = 64 * 1024 * 1024; + this.maxSingleUploadSize = 128 * 1024 * 1024; + } else { + this.awsConfig = { + endpoint, + credentials: { accessKeyId, secretAccessKey }, + ...(region ? { region } : {}), + }; + } + + console.log('[s3] configured with remote endpoint'); + } else { + // Local dev: spin up fauxqs in-process + const localConfig = + s3Conf && 'localConfig' in s3Conf + ? s3Conf.localConfig + : undefined; + const forceInMem = localConfig?.inMemory; + const fauxqsHost = forceInMem ? '127.0.0.1' : localConfig?.host; + + const { startFauxqs } = + await nativeImport('fauxqs'); + this.fauxqsServer = await startFauxqs({ + host: fauxqsHost, + port: forceInMem ? 0 : (localConfig?.port ?? 4566), + logger: false, + dataDir: forceInMem + ? undefined + : (localConfig?.dataDir ?? './fauxqs-data'), + s3StorageDir: forceInMem + ? undefined + : (localConfig?.s3StorageDir ?? './fauxqs-s3-data'), + init: { region: 'us-west-2', buckets: [LEGACY_STORAGE_BUCKET] }, + }); + + // WSL Internal IP fix + let fauxqsAddress = this.fauxqsServer.address; + if (fauxqsAddress.includes('10.255.255.254')) { + fauxqsAddress = fauxqsAddress.replace( + '10.255.255.254', + '127.0.0.1', + ); + } + + this.awsConfig = { + endpoint: fauxqsAddress, + credentials: { + accessKeyId: 'fakeAccessKeyId', + secretAccessKey: 'fakeSecretAccessKey', + }, + }; + + console.log(`[s3] started local fauxqs at ${fauxqsAddress}`); + + // Migrate files from legacy local storage directory if present + if (!forceInMem) { + const result = await this.migrateLegacyStorage(); + if (result.migratedFileCount > 0) { + console.log( + `[s3] migrated ${result.migratedFileCount} file(s) from legacy storage`, + ); + } + } + } + } + + override async onServerShutdown(): Promise { + if (this.fauxqsServer) { + await this.fauxqsServer.stop(); + this.fauxqsServer = null; + } + for (const client of this.clientMap.values()) { + client.destroy(); + } + this.clientMap.clear(); + } + + // ------------------------------------------------------------------ + // Public API + // ------------------------------------------------------------------ + + /** + * Get (or create) an S3Client for the given region. + * Clients are cached per-region for connection reuse. + */ + get( + region = this.config.s3_region || this.config.region || 'us-west-2', + ): AwsS3Client { + const existing = this.clientMap.get(region); + if (existing) return existing; + + const client = new AwsS3Client({ + region, + requestStreamBufferSize: 32 * 1024, + requestHandler: new NodeHttpHandler({ + socketTimeout: 5000, + httpsAgent: new HttpsAgent({ + maxSockets: 500, + keepAlive: true, + keepAliveMsecs: 1000, + }), + }), + ...this.awsConfig, + }); + + this.clientMap.set(region, client); + return client; + } + + // ------------------------------------------------------------------ + // Legacy storage migration + // ------------------------------------------------------------------ + + private async migrateLegacyStorage( + opts: { + bucket?: string; + legacyPath?: string; + } = {}, + ): Promise<{ migratedFileCount: number; scannedEntryCount: number }> { + const bucket = opts.bucket ?? LEGACY_STORAGE_BUCKET; + const legacyPath = + opts.legacyPath ?? path.join(process.cwd(), 'storage'); + + if (!existsSync(legacyPath)) { + return { migratedFileCount: 0, scannedEntryCount: 0 }; + } + + const client = this.get(); + const entries = await fs.readdir(legacyPath); + let migratedFileCount = 0; + + for (const entryName of entries) { + const filePath = path.join(legacyPath, entryName); + const stat = await fs.stat(filePath); + if (!stat.isFile()) continue; + + if (stat.size > this.maxSingleUploadSize) { + await this.uploadMultipart({ + bucket, + client, + filePath, + fileSize: stat.size, + key: entryName, + }); + } else { + const body = await fs.readFile(filePath); + await client.send( + new PutObjectCommand({ + Bucket: bucket, + Key: entryName, + Body: body, + }), + ); + } + migratedFileCount++; + } + + await fs.rm(legacyPath, { recursive: true }); + return { migratedFileCount, scannedEntryCount: entries.length }; + } + + private async uploadMultipart({ + bucket, + client, + filePath, + fileSize, + key, + }: { + bucket: string; + client: S3CommandSender; + filePath: string; + fileSize: number; + key: string; + }): Promise { + const { UploadId } = await client.send( + new CreateMultipartUploadCommand({ Bucket: bucket, Key: key }), + ); + if (!UploadId) + throw new Error(`Failed to start multipart upload for ${filePath}`); + + const uploadedParts: { ETag: string; PartNumber: number }[] = []; + const fileHandle = await fs.open(filePath, 'r'); + + try { + let offset = 0; + let partNumber = 1; + + while (offset < fileSize) { + const partLength = Math.min(this.partSize, fileSize - offset); + const partBuffer = Buffer.alloc(partLength); + const { bytesRead } = await fileHandle.read( + partBuffer, + 0, + partLength, + offset, + ); + if (bytesRead <= 0) break; + + const body = + bytesRead === partBuffer.length + ? partBuffer + : partBuffer.subarray(0, bytesRead); + const { ETag } = await client.send( + new UploadPartCommand({ + Bucket: bucket, + ContentLength: bytesRead, + Key: key, + PartNumber: partNumber, + UploadId, + Body: body, + }), + ); + + if (!ETag) + throw new Error( + `No ETag for ${filePath} part ${partNumber}`, + ); + uploadedParts.push({ ETag, PartNumber: partNumber }); + + offset += bytesRead; + partNumber++; + } + + await client.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId, + MultipartUpload: { Parts: uploadedParts }, + }), + ); + } catch (error) { + await client + .send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId, + }), + ) + .catch(() => {}); + throw error; + } finally { + await fileHandle.close(); + } + } +} diff --git a/src/backend/clients/types.ts b/src/backend/clients/types.ts new file mode 100644 index 000000000..79c1d3ad2 --- /dev/null +++ b/src/backend/clients/types.ts @@ -0,0 +1,24 @@ +import type { IConfig, WithLifecycle } from '../types'; + +export interface IPuterClient { + new (config: IConfig): T; +} + +export const PuterClient = class PuterClient implements WithLifecycle { + constructor(protected config: IConfig) {} + public onServerStart() { + return; + } + public onServerPrepareShutdown() { + return; + } + public onServerShutdown() { + return; + } +} satisfies IPuterClient; + +export type IPuterClientRegistry = Record< + string, + | IPuterClient + | (InstanceType> & Record) +>; diff --git a/src/backend/controllers/apps/AppController.js b/src/backend/controllers/apps/AppController.js new file mode 100644 index 000000000..68dd19c5d --- /dev/null +++ b/src/backend/controllers/apps/AppController.js @@ -0,0 +1,409 @@ +import { HttpError } from '../../core/http/HttpError.js'; +import { driversContainers } from '../../exports.js'; +import { + ICON_DATA_URL_MIME_ALLOWLIST, + isTrustedIconHost, +} from '../../util/appIcon.js'; +import { resolvePrivateLaunchAccess } from '../../util/privateLaunchAccess.js'; +import { PuterController } from '../types.js'; +import DEFAULT_APP_ICON from './default-app-icon.js'; + +/** + * REST endpoints for app management. + * + * Delegates to AppDriver for the actual CRUD + permission logic — + * these routes are just thin shape adapters that translate REST + * conventions into driver calls. + */ +export class AppController extends PuterController { + get appStore() { + return this.stores.app; + } + + get appDriver() { + // Drivers are wired into the shared driversContainers export by + // PuterServer at boot. Controllers get them lazily via this getter + // since they're instantiated before drivers in the boot order. + const d = driversContainers.apps; + if (!d) throw new Error('AppDriver not registered yet'); + return d; + } + + registerRoutes(router) { + // GET /apps — list apps owned by the current user + router.get( + '/apps', + { + subdomain: 'api', + requireUserActor: true, + }, + async (req, res) => { + const apps = await this.appDriver.select({ + predicate: ['user-can-edit'], + }); + res.json(apps); + }, + ); + + // GET /apps/nameAvailable?name=foo + router.get( + '/apps/nameAvailable', + { + subdomain: 'api', + requireUserActor: true, + }, + async (req, res) => { + const name = req.query?.name; + if (!name || typeof name !== 'string') { + throw new HttpError( + 400, + 'Missing or invalid `name` query param', + ); + } + const available = await this.appDriver.isNameAvailable(name); + res.json({ name, available }); + }, + ); + + // POST /rao — record a recent app open. When an app-under-user + // actor calls this, the app id is already on the token — clients + // don't re-send it in the body. Fall back to `actor.app.uid` + // before 400-ing for a missing body field. + router.post( + '/rao', + { + subdomain: 'api', + requireAuth: true, + }, + async (req, res) => { + const bodyAppUid = req.body?.app_uid; + const actorAppUid = req.actor?.app?.uid; + const app_uid = + typeof bodyAppUid === 'string' && bodyAppUid.length > 0 + ? bodyAppUid + : actorAppUid; + if (!app_uid || typeof app_uid !== 'string') { + throw new HttpError(400, 'Missing or invalid `app_uid`'); + } + + const app = await this.appStore.getByUid(app_uid); + if (!app) throw new HttpError(404, 'App not found'); + + // Persist open record + try { + await this.clients.db.write( + 'INSERT INTO `app_opens` (`app_uid`, `user_id`, `ts`) VALUES (?, ?, ?)', + [ + app_uid, + req.actor.user.id, + Math.floor(Date.now() / 1000), + ], + ); + } catch (e) { + console.warn('[rao] insert failed:', e); + } + + try { + this.clients.event?.emit( + 'app.opened', + { + app_uid, + user_id: req.actor.user.id, + ts: Math.floor(Date.now() / 1000), + }, + {}, + ); + } catch { + // event emission best-effort + } + + res.json({}); + }, + ); + + // GET /apps/:name — returns the app(s) by name. + // Supports pipe-separated names for batch lookup: /apps/foo|bar|baz + router.get( + '/apps/:name', + { + subdomain: 'api', + requireUserActor: true, + }, + async (req, res) => { + const raw = req.params.name; + const names = raw.split('|').filter(Boolean); + + const userUid = req.actor?.user?.uuid ?? null; + + const results = await Promise.all( + names.map(async (name) => { + const app = await this.appStore.getByName(name); + if (!app) return null; + let shaped; + try { + shaped = await this.appDriver.read({ + uid: app.uid, + }); + } catch { + return null; + } + const privateAccess = await resolvePrivateLaunchAccess({ + app: shaped, + eventClient: this.clients.event, + userUid, + source: 'appsRoute', + args: req.query ?? {}, + }); + return { ...shaped, privateAccess }; + }), + ); + + // Single-name requests return the app directly; batch returns an array + if (names.length === 1) { + const single = results[0]; + if (!single) throw new HttpError(404, 'App not found'); + return res.json(single); + } + res.json(results); + }, + ); + + // ── POST /query/app ──────────────────────────────────────── + // Batch marketplace-style lookup by name or UID. + // + // Access rules: only apps the caller has a legitimate reason to + // see are returned — public (`approved_for_listing`), owned by + // the caller, or explicitly accessible via AppDriver.read (for + // protected apps with a granted permission). Everything else is + // silently skipped so the endpoint can't be used to enumerate + // existence of private / unapproved apps by guessing names. + // + // Response shape is intentionally narrow and mirrors v1 — no + // internal identifiers (mysql `id`, `owner_user_id`), no + // `index_url`, no admin flags. Developer `metadata` is + // included for public/owned apps only, consistent with + // marketplace semantics. + + const QUERY_APP_MAX_ENTRIES = 200; + const QUERY_APP_MAX_SELECTOR_LEN = 200; + + router.post( + '/query/app', + { + subdomain: 'api', + requireAuth: true, + }, + async (req, res) => { + const appList = Array.isArray(req.body) ? req.body : []; + if (appList.length > QUERY_APP_MAX_ENTRIES) { + throw new HttpError( + 400, + `request body must contain at most ${QUERY_APP_MAX_ENTRIES} selectors`, + ); + } + + const actorUserId = req.actor?.user?.id ?? null; + const results = []; + + for (const selector of appList) { + if ( + typeof selector !== 'string' || + selector.length === 0 || + selector.length > QUERY_APP_MAX_SELECTOR_LEN + ) { + continue; + } + const isUid = selector.startsWith('app-'); + const app = isUid + ? await this.appStore.getByUid(selector) + : await this.appStore.getByName(selector); + if (!app) continue; + + const isOwner = + actorUserId !== null && + app.owner_user_id === actorUserId; + const isApproved = Boolean(app.approved_for_listing); + + if (!isOwner && !isApproved) { + // Unapproved, non-owned — only surface if the + // caller has an explicit grant (purchased / + // permissioned). AppDriver.read enforces that + // via #canReadApp; a thrown 403 means "not + // accessible" and we treat it as "not found". + try { + const shaped = await this.appDriver.read({ + uid: app.uid, + }); + if (!shaped) continue; + } catch { + continue; + } + } + + const assocRows = await this.clients.db.read( + 'SELECT `type` FROM `app_filetype_association` WHERE `app_id` = ?', + [app.id], + ); + + results.push({ + uuid: app.uid, + name: app.name, + title: app.title, + description: app.description, + metadata: app.metadata, + tags: + typeof app.tags === 'string' + ? app.tags.split(',') + : [], + created: app.timestamp, + associations: assocRows.map((r) => r.type), + }); + } + + res.json(results); + }, + ); + + // ── GET /app-icon/:app_uid(/:size) ───────────────────────── + // Serve app icon — data URL decoded inline, HTTP URL redirected. + // + // ⚠ FLAG: Missing sharp-based resize pipeline; serves the original. + + const ICON_SIZES = [16, 32, 64, 128, 256, 512]; + + // Neutering headers for any response that echoes an icon byte + // stream on the main origin. `image/svg+xml` is in our MIME + // allow-list — it's a legitimate image format, and our own + // default icon is SVG — but SVGs can carry ` + + +`; + +// ── Controller ────────────────────────────────────────────────────── + +/** + * Routes driver RPC calls through a unified HTTP surface. + * + * - `POST /drivers/call` — invoke `.(args)` on the + * registered driver after a per-actor permission + rate-limit check. + * Stream-shaped results are piped directly; everything else is + * returned as JSON. + * - `GET /drivers/list-interfaces` — enumerate registered driver + * interfaces with their default + alternate implementations. + * - `GET /drivers/xd` — legacy iframe bridge; serves an HTML page that + * proxies `postMessage` RPCs to `/drivers/call` on the same origin. + * + * Holds an internal iface → driverName → instance map built at + * construction time from `this.drivers`. Extensions that register + * additional drivers end up in that bag before this controller is + * instantiated, so they show up here automatically. + */ +@Controller('/drivers') +export class DriverController extends PuterController { + /** iface → Map */ + #drivers = new Map>(); + /** iface → default driver name */ + #defaults = new Map(); + + constructor(...args: ConstructorParameters) { + super(...args); + this.#buildIfaceMap(); + } + + // ── Lookup API (used by tests / internals) ────────────────────── + + /** Resolve a driver by interface + optional name (default when omitted). */ + resolve(interfaceName: string, driverName?: string): DriverInstance | null { + const ifaceMap = this.#drivers.get(interfaceName); + if (!ifaceMap) return null; + const name = driverName ?? this.#defaults.get(interfaceName); + if (!name) return null; + return ifaceMap.get(name) ?? null; + } + + listInterfaces(): string[] { + return [...this.#drivers.keys()]; + } + + listDrivers(interfaceName: string): string[] { + const ifaceMap = this.#drivers.get(interfaceName); + return ifaceMap ? [...ifaceMap.keys()] : []; + } + + getDefault(interfaceName: string): string | undefined { + return this.#defaults.get(interfaceName); + } + + // ── Route registration ────────────────────────────────────────── + + registerRoutes(router: PuterRouter): void { + router.post( + '/call', + { subdomain: 'api', requireAuth: true }, + this.#handleCall, + ); + router.get( + '/list-interfaces', + { subdomain: 'api', requireAuth: true }, + this.#handleListInterfaces, + ); + router.get( + '/xd', + { subdomain: 'api', requireAuth: true }, + this.#handleXd, + ); + router.get( + '/usage', + { subdomain: 'api', requireAuth: true }, + this.#handleUsage, + ); + } + + // ── Handlers ──────────────────────────────────────────────────── + + #handleCall = async (req: Request, res: Response): Promise => { + const { + interface: ifaceName, + method, + driver: driverName, + args = {}, + } = (req.body ?? {}) as Record; + + if (!ifaceName || typeof ifaceName !== 'string') { + throw new HttpError(400, 'Missing or invalid `interface`'); + } + if (!method || typeof method !== 'string') { + throw new HttpError(400, 'Missing or invalid `method`'); + } + const requestedDriver = + typeof driverName === 'string' ? driverName : undefined; + + const driver = this.resolve(ifaceName, requestedDriver); + if (!driver) { + const resolvedName = requestedDriver ?? this.getDefault(ifaceName); + throw new HttpError( + 404, + `Driver not found: ${ifaceName}:${resolvedName ?? '(no default)'}`, + ); + } + + const fn = driver[method]; + if (typeof fn !== 'function') { + throw new HttpError( + 404, + `Method '${method}' not found on driver '${ifaceName}'`, + ); + } + + // Resolve the concrete driver name for permission keys, falling + // back through prototype metadata → instance field → requested name. + const resolvedDriverName = + (driver as Record).driverName ?? + (Object.getPrototypeOf(driver) as Record) + .__driverName ?? + requestedDriver ?? + 'unknown'; + + if (req.actor) { + const permService = this.services.permission as unknown as + | PermissionService + | undefined; + if (permService) { + const permKey = `service:${resolvedDriverName}:ii:${ifaceName}`; + const hasPermission = await permService.check( + req.actor, + permKey, + ); + if (!hasPermission) { + throw new HttpError( + 403, + `Permission denied for ${ifaceName}:${method}`, + { + legacyCode: 'forbidden', + }, + ); + } + } + } + + if (!(await checkDriverRateLimit(req, ifaceName, method))) { + throw new HttpError(429, 'Too many requests.'); + } + + // Stash the requested driver name in Context so multi-provider + // drivers (TTS/OCR/image/video) can route to the right internal + // provider when invoked via an alias. `driverName` lives on the + // generic extras map — not a well-known key — so it doesn't + // pollute the typed Context surface. Always set, even when no + // alias was requested, so the driver sees `undefined` rather than + // a stale value from a prior call. + Context.set('driverName', requestedDriver); + + // Drivers read actor/context via the Context API — no drilled args. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const result = await (fn as (...x: unknown[]) => any).call( + driver, + args, + ); + + if (isDriverStreamResult(result)) { + res.setHeader('Content-Type', result.content_type); + if (result.chunked) { + res.setHeader('Transfer-Encoding', 'chunked'); + } + result.stream.pipe(res); + return; + } + + // Drivers can optionally stash top-level response metadata via + // `Context.set('driverMetadata', ...)`. Used by the chat driver to + // surface `{service_used, providerUsed}` without polluting the + // result body — matches v1's wire shape. + const driverMetadata = Context.get('driverMetadata'); + + const payload: Record = { + success: true, + result, + service: { name: resolvedDriverName }, + }; + if (driverMetadata && typeof driverMetadata === 'object') { + payload.metadata = driverMetadata; + } + res.json(payload); + }; + + #handleListInterfaces = (_req: Request, res: Response): void => { + const interfaces = this.listInterfaces(); + const out: Record< + string, + { drivers: string[]; default: string | undefined } + > = {}; + for (const iface of interfaces) { + out[iface] = { + drivers: this.listDrivers(iface), + default: this.getDefault(iface), + }; + } + res.json(out); + }; + + #handleXd = (_req: Request, res: Response): void => { + res.type('text/html'); + res.send(XD_HTML); + }; + + /** GET /drivers/usage — monthly driver usage for the authenticated actor. */ + #handleUsage = async (req: Request, res: Response): Promise => { + const actor = req.actor; + if (!actor?.user?.id) + throw new HttpError(401, 'Authentication required'); + + const userId = actor.user.id; + const db = this.clients.db; + + // Per-user usage: aggregate today's counts from monthly_usage_counts + const userRows = await db.read( + `SELECT \`year\`, \`month\`, \`service\`, SUM(\`count\`) AS count, MAX(\`max\`) AS max + FROM \`monthly_usage_counts\` + WHERE \`user_id\` = ? AND \`app_id\` IS NULL + GROUP BY \`year\`, \`month\`, \`service\` + ORDER BY \`year\` DESC, \`month\` DESC + LIMIT 100`, + [userId], + ); + + // Per-app usage: aggregate by app + const appRows = await db.read( + `SELECT a.\`uid\` AS app_uid, a.\`name\` AS app_name, + m.\`year\`, m.\`month\`, m.\`service\`, SUM(m.\`count\`) AS count, MAX(m.\`max\`) AS max + FROM \`monthly_usage_counts\` m + LEFT JOIN \`apps\` a ON m.\`app_id\` = a.\`id\` + WHERE m.\`user_id\` = ? AND m.\`app_id\` IS NOT NULL + GROUP BY a.\`uid\`, a.\`name\`, m.\`year\`, m.\`month\`, m.\`service\` + ORDER BY m.\`year\` DESC, m.\`month\` DESC + LIMIT 500`, + [userId], + ); + + // Group app rows by app name + const apps: Record>> = {}; + for (const row of appRows) { + const name = String( + (row as Record).app_name ?? 'unknown', + ); + if (!apps[name]) apps[name] = []; + apps[name].push(row as Record); + } + + res.json({ user: userRows, apps }); + }; + + // ── Internals ─────────────────────────────────────────────────── + + #buildIfaceMap(): void { + const bag = this.drivers as unknown as Record; + for (const instance of Object.values(bag)) { + const meta = resolveDriverMeta(instance); + if (meta) this.#registerDriver(meta, instance); + } + } + + #registerDriver(meta: DriverMeta, instance: DriverInstance): void { + let ifaceMap = this.#drivers.get(meta.interfaceName); + if (!ifaceMap) { + ifaceMap = new Map(); + this.#drivers.set(meta.interfaceName, ifaceMap); + } + if (ifaceMap.has(meta.driverName)) { + console.warn( + `[driver-controller] overwriting driver ${meta.interfaceName}:${meta.driverName}`, + ); + } + ifaceMap.set(meta.driverName, instance); + // Register each alias pointing at the same instance. Legacy puter-js + // calls that pass a provider id in the `driver` slot (e.g. the TTS + // module sends `aws-polly` / `openai-tts` / `elevenlabs-tts` instead + // of the unified `ai-tts`) resolve here; the handler sets + // Context.driverName to the alias so the method can route to the + // right internal provider. + for (const alias of meta.aliases) { + if (alias === meta.driverName) continue; + if (ifaceMap.has(alias)) { + console.warn( + `[driver-controller] alias collision on ${meta.interfaceName}:${alias} — keeping first registration`, + ); + continue; + } + ifaceMap.set(alias, instance); + } + if (meta.isDefault || !this.#defaults.has(meta.interfaceName)) { + this.#defaults.set(meta.interfaceName, meta.driverName); + } + } +} diff --git a/extensions/fsv2/src/controllers/FSController.ts b/src/backend/controllers/fs/FSController.ts similarity index 51% rename from extensions/fsv2/src/controllers/FSController.ts rename to src/backend/controllers/fs/FSController.ts index 39ca51ca0..45ef51ded 100644 --- a/extensions/fsv2/src/controllers/FSController.ts +++ b/src/backend/controllers/fs/FSController.ts @@ -1,15 +1,23 @@ -import type { ACLService } from '@heyputer/backend/src/services/auth/ACLService.js'; -import type { EventService } from '@heyputer/backend/src/services/EventService.js'; import Busboy from 'busboy'; import type { Request, Response } from 'express'; import { posix as pathPosix } from 'node:path'; -import type { FSEntryService } from '../services/FSEntryService.js'; +import { pipeline } from 'node:stream/promises'; +import type { Actor } from '../../core/actor.js'; +import { Context } from '../../core/context.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import { Controller, Get, Post } from '../../core/http/decorators.js'; import type { PreparedBatchWrite, UploadedBatchWriteItem, UploadProgressTrackerLike, -} from '../services/types.js'; -import type { FSEntry, FSEntryWriteInput } from '../types/FSEntry.js'; +} from '../../services/fs/types.js'; +import type { FSEntry, FSEntryWriteInput } from '../../stores/fs/FSEntry.js'; +import { + runWithConcurrencyLimit, + runWithConcurrencyLimitSettled, +} from '../../util/concurrency.js'; +import { PuterController } from '../types.js'; +import { FS_COSTS } from './costs.js'; import type { CompleteWriteRequest, CompleteWriteResponse, @@ -20,11 +28,7 @@ import type { WriteGuiMetadata, WriteRequest, WriteResponse, -} from '../types/requests.js'; -import { - runWithConcurrencyLimit, - runWithConcurrencyLimitSettled, -} from '../utils/concurrency.js'; +} from './requestTypes.js'; import type { AbortWriteRequest, BatchWriteManifest, @@ -34,56 +38,50 @@ import type { ThumbnailUploadPrepareItem, ThumbnailUploadPreparePayload, } from './types.js'; - -const { Controller, ExtensionController, HttpError, Post } = extension.import( - 'extensionController', -); -const { Context } = extension.import('core'); -const getApp = extension.import('core').util.helpers.get_app as (query: { - uid: string; -}) => Promise<{ id?: unknown } | null>; class UploadProgressTracker implements UploadProgressTrackerLike { total = 0; progress = 0; #listeners: Array<(delta: number) => void> = []; - setTotal (value: number) { + setTotal(value: number) { this.total = value; } - add (amount: number) { + add(amount: number) { this.progress += amount; - for ( const listener of this.#listeners ) { + for (const listener of this.#listeners) { listener(amount); } } - subscribe (callback: (delta: number) => void) { + subscribe(callback: (delta: number) => void) { this.#listeners.push(callback); return { detach: () => { const idx = this.#listeners.indexOf(callback); - if ( idx !== -1 ) this.#listeners.splice(idx, 1); + if (idx !== -1) this.#listeners.splice(idx, 1); }, }; } } -const aclService = extension.import('service:acl') as ACLService; const MAX_THUMBNAIL_BYTES = 2 * 1024 * 1024; const DEFAULT_BATCH_ACL_CHECK_CONCURRENCY = 32; const DEFAULT_BATCH_WRITE_SIDE_EFFECT_CONCURRENCY = 8; @Controller('/fs') -export class FSController extends ExtensionController { - constructor ( - private fsEntryService: FSEntryService, - private eventService: EventService, - ) { - super(); +export class FSController extends PuterController { + override getReportedCosts(): Record[] { + return Object.entries(FS_COSTS).map(([usageType, ucentsPerUnit]) => ({ + usageType, + ucentsPerUnit, + unit: 'byte', + source: 'controller:fs', + })); } - @Post('/startWrite', { subdomain: 'api' }) - async startWrite ( + + @Post('/startWrite', { subdomain: 'api', requireVerified: true }) + async startWrite( req: Request, res: Response, ) { @@ -104,20 +102,27 @@ export class FSController extends ExtensionController { }); const { response, createdDirectoryEntries } = - await this.fsEntryService.startUrlWriteWithCreatedDirectories( + await this.services.fs.startUrlWriteWithCreatedDirectories( userId, requestBody, storageAllowanceMax, ); - await this.#attachSignedThumbnailUploadTargets([requestBody], [response]); - if ( ! requestBody.directory ) { + await this.#attachSignedThumbnailUploadTargets( + [requestBody], + [response], + ); + if (!requestBody.directory) { await this.#runNonCritical(async () => { - await this.#emitGuiPendingWriteEvent(userId, requestBody, response); + await this.#emitGuiPendingWriteEvent( + userId, + requestBody, + response, + ); }, 'emitStartWritePendingEvent'); } - if ( createdDirectoryEntries.length > 0 ) { + if (createdDirectoryEntries.length > 0) { void this.#runNonCritical(async () => { - for ( const createdDirectoryEntry of createdDirectoryEntries ) { + for (const createdDirectoryEntry of createdDirectoryEntries) { await this.#emitGuiWriteEvent( 'outer.gui.item.added', createdDirectoryEntry, @@ -129,8 +134,8 @@ export class FSController extends ExtensionController { res.json(response); } - @Post('/startBatchWrite', { subdomain: 'api' }) - async startBatchWrites ( + @Post('/startBatchWrite', { subdomain: 'api', requireVerified: true }) + async startBatchWrites( req: Request, res: Response, ) { @@ -139,26 +144,26 @@ export class FSController extends ExtensionController { const appUidLookupCache = new Map>(); const requests = Array.isArray(req.body) ? await Promise.all( - req.body.map(async (requestBody) => { - const normalizedRequestBody = this.#withGuiMetadata( - requestBody, - req.body, - ); - normalizedRequestBody.fileMetadata = - this.#normalizeFileMetadataPath( - req, - normalizedRequestBody.fileMetadata, - normalizedRequestBody, - ); - normalizedRequestBody.fileMetadata = - await this.#resolveAssociatedAppMetadata( - normalizedRequestBody.fileMetadata, - normalizedRequestBody, - appUidLookupCache, - ); - return normalizedRequestBody; - }), - ) + req.body.map(async (requestBody) => { + const normalizedRequestBody = this.#withGuiMetadata( + requestBody, + req.body, + ); + normalizedRequestBody.fileMetadata = + this.#normalizeFileMetadataPath( + req, + normalizedRequestBody.fileMetadata, + normalizedRequestBody, + ); + normalizedRequestBody.fileMetadata = + await this.#resolveAssociatedAppMetadata( + normalizedRequestBody.fileMetadata, + normalizedRequestBody, + appUidLookupCache, + ); + return normalizedRequestBody; + }), + ) : []; await this.#assertBatchWriteAccess( req, @@ -167,18 +172,21 @@ export class FSController extends ExtensionController { ); const { responses, createdDirectoryEntries } = - await this.fsEntryService.batchStartUrlWritesWithCreatedDirectories( + await this.services.fs.batchStartUrlWritesWithCreatedDirectories( userId, requests, storageAllowanceMax, ); const directoryGuiMetadataByPath = new Map< string, - WriteGuiMetadata | undefined + WriteGuiMetadata | undefined >( requests .filter((request) => request.directory) - .map((request) => [request.fileMetadata.path, request.guiMetadata]), + .map((request) => [ + request.fileMetadata.path, + request.guiMetadata, + ]), ); const emittedDirectoryPaths = new Set(); @@ -189,8 +197,8 @@ export class FSController extends ExtensionController { 32, async (writeResponse, index) => { const requestBody = requests[index]; - if ( requestBody && writeResponse ) { - if ( ! requestBody.directory ) { + if (requestBody && writeResponse) { + if (!requestBody.directory) { await this.#emitGuiPendingWriteEvent( userId, requestBody, @@ -201,17 +209,19 @@ export class FSController extends ExtensionController { }, ); }, 'emitStartBatchWritePendingEvents'); - if ( createdDirectoryEntries.length > 0 ) { + if (createdDirectoryEntries.length > 0) { void this.#runNonCritical(async () => { - for ( const createdDirectoryEntry of createdDirectoryEntries ) { - if ( emittedDirectoryPaths.has(createdDirectoryEntry.path) ) { + for (const createdDirectoryEntry of createdDirectoryEntries) { + if (emittedDirectoryPaths.has(createdDirectoryEntry.path)) { continue; } emittedDirectoryPaths.add(createdDirectoryEntry.path); await this.#emitGuiWriteEvent( 'outer.gui.item.added', createdDirectoryEntry, - directoryGuiMetadataByPath.get(createdDirectoryEntry.path), + directoryGuiMetadataByPath.get( + createdDirectoryEntry.path, + ), ); } }, 'emitStartBatchWriteDirectoryEvents'); @@ -219,8 +229,8 @@ export class FSController extends ExtensionController { res.json(responses); } - @Post('/completeWrite', { subdomain: 'api' }) - async completeWrite ( + @Post('/completeWrite', { subdomain: 'api', requireVerified: true }) + async completeWrite( req: Request, res: Response, ) { @@ -228,7 +238,7 @@ export class FSController extends ExtensionController { const requestBody = this.#withGuiMetadata(req.body, req.body); this.#assertNoInlineSignedThumbnailData(requestBody.thumbnailData); - const response = await this.fsEntryService.completeUrlWrite( + const response = await this.services.fs.completeUrlWrite( userId, requestBody, ); @@ -245,21 +255,21 @@ export class FSController extends ExtensionController { res.json({ ...response, fsEntry: writeResponse.fsEntry }); } - @Post('/completeBatchWrite', { subdomain: 'api' }) - async completeBatchWrites ( + @Post('/completeBatchWrite', { subdomain: 'api', requireVerified: true }) + async completeBatchWrites( req: Request, res: Response, ) { const userId = this.#getActorUserId(req); const requests = Array.isArray(req.body) ? req.body.map((requestBody) => { - return this.#withGuiMetadata(requestBody, req.body); - }) + return this.#withGuiMetadata(requestBody, req.body); + }) : []; - for ( const requestBody of requests ) { + for (const requestBody of requests) { this.#assertNoInlineSignedThumbnailData(requestBody.thumbnailData); } - const response = await this.fsEntryService.batchCompleteUrlWrite( + const response = await this.services.fs.batchCompleteUrlWrite( userId, requests, ); @@ -268,51 +278,53 @@ export class FSController extends ExtensionController { DEFAULT_BATCH_WRITE_SIDE_EFFECT_CONCURRENCY, async (writeResponse, index) => { const requestBody = requests[index]; - const withSideEffects = await this.#applyWriteResponseSideEffects( - userId, - { - fsEntry: writeResponse.fsEntry, - wasOverwrite: writeResponse.wasOverwrite, - requestedThumbnail: writeResponse.requestedThumbnail, - contentHashSha256: null, - }, - requestBody?.guiMetadata, - ); + const withSideEffects = + await this.#applyWriteResponseSideEffects( + userId, + { + fsEntry: writeResponse.fsEntry, + wasOverwrite: writeResponse.wasOverwrite, + requestedThumbnail: + writeResponse.requestedThumbnail, + contentHashSha256: null, + }, + requestBody?.guiMetadata, + ); return { ...writeResponse, fsEntry: withSideEffects.fsEntry }; }, ); res.json(updatedResponse); } - @Post('/abortWrite', { subdomain: 'api' }) - async abortWrite ( + @Post('/abortWrite', { subdomain: 'api', requireVerified: true }) + async abortWrite( req: Request, res: Response<{ ok: true }>, ) { const userId = this.#getActorUserId(req); - if ( ! req.body?.uploadId ) { + if (!req.body?.uploadId) { throw new HttpError(400, 'Missing uploadId'); } - await this.fsEntryService.abortUrlWrite(userId, req.body.uploadId); + await this.services.fs.abortUrlWrite(userId, req.body.uploadId); res.json({ ok: true }); } - @Post('/signMultipartParts', { subdomain: 'api' }) - async signMultipartParts ( + @Post('/signMultipartParts', { subdomain: 'api', requireVerified: true }) + async signMultipartParts( req: Request, res: Response, ) { const userId = this.#getActorUserId(req); - const response = await this.fsEntryService.signMultipartParts( + const response = await this.services.fs.signMultipartParts( userId, req.body, ); res.json(response); } - @Post('/write', { subdomain: 'api' }) - async write ( + @Post('/write', { subdomain: 'api', requireVerified: true }) + async write( req: Request, res: Response, ) { @@ -331,7 +343,9 @@ export class FSController extends ExtensionController { await this.#assertWriteAccess(req, requestBody.fileMetadata, { pathAlreadyNormalized: true, }); - const normalizedPath = this.#normalizePath(requestBody.fileMetadata.path); + const normalizedPath = this.#normalizePath( + requestBody.fileMetadata.path, + ); const uploadTracker = await this.#createUploadTracker( userId, normalizedPath, @@ -339,7 +353,7 @@ export class FSController extends ExtensionController { Number(requestBody.fileMetadata.size ?? 0), requestBody.guiMetadata, ); - const response = await this.fsEntryService.write( + const response = await this.services.fs.write( userId, requestBody, uploadTracker, @@ -353,8 +367,8 @@ export class FSController extends ExtensionController { res.json(updatedResponse); } - @Post('/batchWrite', { subdomain: 'api' }) - async batchWrites ( + @Post('/batchWrite', { subdomain: 'api', requireVerified: true }) + async batchWrites( req: Request, res: Response, ) { @@ -362,7 +376,7 @@ export class FSController extends ExtensionController { const storageAllowanceMax = this.#getStorageAllowanceMaxOverride(req); const requestMode = this.#resolveBatchWriteRequestMode(req); const appUidLookupCache = new Map>(); - if ( requestMode === 'multipart' ) { + if (requestMode === 'multipart') { let parsedManifest: ParsedMultipartBatchManifest | null = null; let preparedBatch: PreparedBatchWrite | null = null; let manifestPreparationPromise: Promise | null = null; @@ -372,10 +386,10 @@ export class FSController extends ExtensionController { let fileOrderIndex = 0; const failParse = (error: unknown) => { - if ( parseFailure ) { + if (parseFailure) { return; } - if ( error instanceof Error ) { + if (error instanceof Error) { parseFailure = error; return; } @@ -385,16 +399,19 @@ export class FSController extends ExtensionController { const busboy = Busboy({ headers: req.headers }); busboy.on('field', (fieldName, value, info) => { - if ( info.fieldnameTruncated || info.valueTruncated ) { + if (info.fieldnameTruncated || info.valueTruncated) { failParse( - new HttpError(400, 'Batch write manifest field is truncated'), + new HttpError( + 400, + 'Batch write manifest field is truncated', + ), ); return; } - if ( fieldName !== 'manifest' ) { + if (fieldName !== 'manifest') { return; } - if ( manifestPreparationPromise ) { + if (manifestPreparationPromise) { failParse( new HttpError( 409, @@ -405,7 +422,10 @@ export class FSController extends ExtensionController { } try { - parsedManifest = this.#parseBatchWriteManifest(value, undefined); + parsedManifest = this.#parseBatchWriteManifest( + value, + undefined, + ); const ignoredItemIndexes = new Set(); parsedManifest = { ...parsedManifest, @@ -419,58 +439,71 @@ export class FSController extends ExtensionController { })), ignoredItemIndexes, }; - for ( const item of parsedManifest.items ) { - if ( this.#shouldIgnoreUploadPath(item.fileMetadata.path) ) { + for (const item of parsedManifest.items) { + if ( + this.#shouldIgnoreUploadPath(item.fileMetadata.path) + ) { ignoredItemIndexes.add(item.index); } } manifestPreparationPromise = (async () => { try { - if ( ! parsedManifest ) { - throw new HttpError(400, 'Batch write manifest is missing'); + if (!parsedManifest) { + throw new HttpError( + 400, + 'Batch write manifest is missing', + ); } parsedManifest = { ...parsedManifest, items: await Promise.all( parsedManifest.items.map(async (item) => ({ ...item, - fileMetadata: await this.#resolveAssociatedAppMetadata( - item.fileMetadata, - item, - appUidLookupCache, - ), + fileMetadata: + await this.#resolveAssociatedAppMetadata( + item.fileMetadata, + item, + appUidLookupCache, + ), })), ), }; - const activeManifestItems = parsedManifest.items.filter( - (item) => !parsedManifest?.ignoredItemIndexes?.has(item.index), - ); + const activeManifestItems = + parsedManifest.items.filter( + (item) => + !parsedManifest?.ignoredItemIndexes?.has( + item.index, + ), + ); await this.#assertBatchWriteAccess( req, - activeManifestItems.map((item) => item.fileMetadata), + activeManifestItems.map( + (item) => item.fileMetadata, + ), { pathAlreadyNormalized: true }, ); - preparedBatch = await this.fsEntryService.prepareBatchWrites( - userId, - activeManifestItems.map((item) => ({ - fileMetadata: item.fileMetadata, - thumbnailData: item.thumbnailData, - guiMetadata: item.guiMetadata, - })), - storageAllowanceMax, - ); - await this.fsEntryService.assertStorageAllowanceForPreparedBatch( + preparedBatch = + await this.services.fs.prepareBatchWrites( + userId, + activeManifestItems.map((item) => ({ + fileMetadata: item.fileMetadata, + thumbnailData: item.thumbnailData, + guiMetadata: item.guiMetadata, + })), + storageAllowanceMax, + ); + await this.services.fs.assertStorageAllowanceForPreparedBatch( preparedBatch, undefined, storageAllowanceMax, ); - } catch ( error ) { + } catch (error) { failParse(error); } })(); - } catch ( error ) { + } catch (error) { failParse(error); } }); @@ -480,10 +513,10 @@ export class FSController extends ExtensionController { fileOrderIndex++; const uploadPromise = (async () => { try { - if ( parseFailure ) { + if (parseFailure) { throw parseFailure; } - if ( ! manifestPreparationPromise ) { + if (!manifestPreparationPromise) { throw new HttpError( 400, 'Batch write manifest must come before file content', @@ -491,11 +524,14 @@ export class FSController extends ExtensionController { } await manifestPreparationPromise; - if ( parseFailure ) { + if (parseFailure) { throw parseFailure; } - if ( !parsedManifest || !preparedBatch ) { - throw new HttpError(400, 'Batch write manifest is missing'); + if (!parsedManifest || !preparedBatch) { + throw new HttpError( + 400, + 'Batch write manifest is missing', + ); } const itemIndex = this.#resolveMultipartFileIndex( @@ -503,13 +539,13 @@ export class FSController extends ExtensionController { currentFileOrder, parsedManifest, ); - if ( parsedManifest.ignoredItemIndexes.has(itemIndex) ) { - if ( !stream.readableEnded && !stream.destroyed ) { + if (parsedManifest.ignoredItemIndexes.has(itemIndex)) { + if (!stream.readableEnded && !stream.destroyed) { stream.resume(); } return null; } - if ( uploadedIndexes.has(itemIndex) ) { + if (uploadedIndexes.has(itemIndex)) { throw new HttpError( 409, `Duplicate file content for batch index ${itemIndex}`, @@ -517,8 +553,9 @@ export class FSController extends ExtensionController { } uploadedIndexes.add(itemIndex); - const preparedItem = preparedBatch.itemsByIndex.get(itemIndex); - if ( ! preparedItem ) { + const preparedItem = + preparedBatch.itemsByIndex.get(itemIndex); + if (!preparedItem) { throw new HttpError( 400, `Batch write metadata was not found for index ${itemIndex}`, @@ -533,14 +570,14 @@ export class FSController extends ExtensionController { preparedItem.guiMetadata, ); - return await this.fsEntryService.uploadPreparedBatchItem({ + return await this.services.fs.uploadPreparedBatchItem({ preparedBatch, itemIndex, fileContent: stream, uploadTracker, }); - } catch ( error ) { - if ( !stream.readableEnded && !stream.destroyed ) { + } catch (error) { + if (!stream.readableEnded && !stream.destroyed) { stream.resume(); } throw error; @@ -556,7 +593,7 @@ export class FSController extends ExtensionController { req.pipe(busboy); await parsingComplete; - if ( ! manifestPreparationPromise ) { + if (!manifestPreparationPromise) { await Promise.allSettled(uploadPromises); throw new HttpError(400, 'Batch write manifest is required'); } @@ -574,23 +611,26 @@ export class FSController extends ExtensionController { (uploadedItem): uploadedItem is UploadedBatchWriteItem => uploadedItem !== null, ); - if ( parseFailure ) { - if ( preparedBatch ) { - await this.fsEntryService.cleanupPreparedBatchUploads( + if (parseFailure) { + if (preparedBatch) { + await this.services.fs.cleanupPreparedBatchUploads( preparedBatch, uploadedItems, ); } throw parseFailure; } - if ( ! preparedBatch ) { - throw new HttpError(500, 'Failed to prepare batch write operation'); + if (!preparedBatch) { + throw new HttpError( + 500, + 'Failed to prepare batch write operation', + ); } const failedUpload = uploadResults.find( (result) => result.status === 'rejected', ); - if ( failedUpload?.status === 'rejected' ) { - await this.fsEntryService.cleanupPreparedBatchUploads( + if (failedUpload?.status === 'rejected') { + await this.services.fs.cleanupPreparedBatchUploads( preparedBatch, uploadedItems, ); @@ -600,7 +640,7 @@ export class FSController extends ExtensionController { } const writeResponses = - await this.fsEntryService.finalizePreparedBatchWrites( + await this.services.fs.finalizePreparedBatchWrites( preparedBatch, uploadedItems, ); @@ -622,31 +662,31 @@ export class FSController extends ExtensionController { const requests = Array.isArray(req.body) ? await Promise.all( - req.body.map(async (requestBody) => { - const normalizedRequestBody = this.#withGuiMetadata( - requestBody, - req.body, - ); - normalizedRequestBody.fileMetadata = - this.#normalizeFileMetadataPath( - req, - normalizedRequestBody.fileMetadata, - normalizedRequestBody, - ); - normalizedRequestBody.fileMetadata = - await this.#resolveAssociatedAppMetadata( - normalizedRequestBody.fileMetadata, - normalizedRequestBody, - appUidLookupCache, - ); - return normalizedRequestBody; - }), - ) + req.body.map(async (requestBody) => { + const normalizedRequestBody = this.#withGuiMetadata( + requestBody, + req.body, + ); + normalizedRequestBody.fileMetadata = + this.#normalizeFileMetadataPath( + req, + normalizedRequestBody.fileMetadata, + normalizedRequestBody, + ); + normalizedRequestBody.fileMetadata = + await this.#resolveAssociatedAppMetadata( + normalizedRequestBody.fileMetadata, + normalizedRequestBody, + appUidLookupCache, + ); + return normalizedRequestBody; + }), + ) : []; const filteredRequests = requests.filter((requestBody) => { return !this.#shouldIgnoreUploadPath(requestBody.fileMetadata.path); }); - if ( filteredRequests.length === 0 ) { + if (filteredRequests.length === 0) { res.json([]); return; } @@ -656,7 +696,7 @@ export class FSController extends ExtensionController { { pathAlreadyNormalized: true }, ); - const preparedBatch = await this.fsEntryService.prepareBatchWrites( + const preparedBatch = await this.services.fs.prepareBatchWrites( userId, filteredRequests.map((requestBody) => ({ fileMetadata: requestBody.fileMetadata, @@ -665,7 +705,7 @@ export class FSController extends ExtensionController { })), storageAllowanceMax, ); - await this.fsEntryService.assertStorageAllowanceForPreparedBatch( + await this.services.fs.assertStorageAllowanceForPreparedBatch( preparedBatch, undefined, storageAllowanceMax, @@ -676,7 +716,7 @@ export class FSController extends ExtensionController { 8, async (requestBody, index) => { const preparedItem = preparedBatch.items[index]; - if ( ! preparedItem ) { + if (!preparedItem) { throw new Error( `Failed to resolve prepared batch item for index ${index}`, ); @@ -688,7 +728,7 @@ export class FSController extends ExtensionController { preparedItem.normalizedInput.size, requestBody.guiMetadata, ); - return this.fsEntryService.uploadPreparedBatchItem({ + return this.services.fs.uploadPreparedBatchItem({ preparedBatch, itemIndex: preparedItem.index, fileContent: requestBody.fileContent, @@ -699,15 +739,17 @@ export class FSController extends ExtensionController { ); const uploadedItems = uploadResults .filter( - (result): result is PromiseFulfilledResult => + ( + result, + ): result is PromiseFulfilledResult => result.status === 'fulfilled', ) .map((result) => result.value); const failedUpload = uploadResults.find( (result) => result.status === 'rejected', ); - if ( failedUpload?.status === 'rejected' ) { - await this.fsEntryService.cleanupPreparedBatchUploads( + if (failedUpload?.status === 'rejected') { + await this.services.fs.cleanupPreparedBatchUploads( preparedBatch, uploadedItems, ); @@ -717,7 +759,7 @@ export class FSController extends ExtensionController { } const writeResponses = - await this.fsEntryService.finalizePreparedBatchWrites( + await this.services.fs.finalizePreparedBatchWrites( preparedBatch, uploadedItems, ); @@ -736,7 +778,564 @@ export class FSController extends ExtensionController { res.json(updatedResponses); } - #getActorUserId (req: Request): number { + // ── Read-side routes ──────────────────────────────────────────────── + + @Post('/stat', { subdomain: 'api', requireVerified: true }) + async statEntry(req: Request, res: Response) { + const actor = this.#requireActor(req); + const userId = this.#getActorUserId(req); + const body = this.#toObjectRecord(req.body); + const entry = await this.#resolveEntryForRequest(body); + await this.#assertAccess(actor, entry.path, 'see'); + + const wantsSize = this.#toBoolean(body.return_size); + const subtreeSize = + entry.isDir && wantsSize + ? await this.services.fs.getSubtreeSize(userId, entry.path) + : undefined; + + entry.suggestedApps = + await this.services.suggestedApps.getSuggestedApps(entry); + + res.json({ + ...entry, + ...(subtreeSize !== undefined ? { size: subtreeSize } : {}), + }); + } + + @Post('/readdir', { subdomain: 'api', requireVerified: true }) + async readdirEntries(req: Request, res: Response) { + const actor = this.#requireActor(req); + const body = this.#toObjectRecord(req.body); + + if (this.#isRootPathRef(body)) { + const { listRootEntries } = + await import('../../services/fs/rootListing.js'); + const rootChildren = await listRootEntries( + actor, + this.stores.fsEntry, + this.services.permission, + ); + const rootSuggestions = + await this.services.suggestedApps.getSuggestedAppsForEntries( + rootChildren, + ); + for (let index = 0; index < rootChildren.length; index++) { + const child = rootChildren[index]; + if (child) { + child.suggestedApps = rootSuggestions[index] ?? []; + } + } + res.json(rootChildren); + return; + } + + const parent = await this.#resolveEntryForRequest(body); + if (!parent.isDir) { + throw new HttpError(400, 'Target is not a directory'); + } + await this.#assertAccess(actor, parent.path, 'list'); + + const limit = this.#toNumberOrUndefined(body.limit); + const offset = this.#toNumberOrUndefined(body.offset); + const sortByRaw = + typeof body.sort_by === 'string' + ? body.sort_by.toLowerCase() + : undefined; + const sortBy = + (['name', 'modified', 'type', 'size'] as const).find( + (v) => v === sortByRaw, + ) ?? null; + const sortOrderRaw = + typeof body.sort_order === 'string' + ? body.sort_order.toLowerCase() + : undefined; + const sortOrder = + (['asc', 'desc'] as const).find((v) => v === sortOrderRaw) ?? null; + + const children = await this.services.fs.listDirectory(parent.uuid, { + limit, + offset, + sortBy, + sortOrder, + }); + + const suggestions = + await this.services.suggestedApps.getSuggestedAppsForEntries( + children, + ); + for (let index = 0; index < children.length; index++) { + const child = children[index]; + if (child) { + child.suggestedApps = suggestions[index] ?? []; + } + } + + res.json(children); + } + + @Post('/search', { subdomain: 'api', requireVerified: true }) + async searchEntries(req: Request, res: Response) { + this.#requireActor(req); + const userId = this.#getActorUserId(req); + const body = this.#toObjectRecord(req.body); + const query = + typeof body.query === 'string' + ? body.query + : typeof body.text === 'string' + ? body.text + : ''; + if (query.trim().length === 0) { + throw new HttpError(400, 'Missing `query`'); + } + const limit = this.#toNumberOrUndefined(body.limit); + const results = await this.services.fs.searchByName( + userId, + query, + limit ?? 200, + ); + res.json(results); + } + + @Get('/read', { subdomain: 'api', requireVerified: true }) + async readEntry(req: Request, res: Response) { + const actor = this.#requireActor(req); + const query = this.#toObjectRecord(req.query); + const entry = await this.#resolveEntryForRequest(query); + await this.#assertAccess(actor, entry.path, 'read'); + + if (entry.isDir) { + throw new HttpError( + 400, + 'Cannot read a directory; use /fs/readdir', + { legacyCode: 'cannot_read_a_directory' }, + ); + } + + const range = + typeof req.headers.range === 'string' + ? req.headers.range + : undefined; + const download = await this.services.fs.readContent(entry, { + range, + }); + + if (download.contentType) + res.setHeader('Content-Type', download.contentType); + if (download.contentLength !== null) + res.setHeader('Content-Length', String(download.contentLength)); + if (download.contentRange) + res.setHeader('Content-Range', download.contentRange); + if (download.etag) res.setHeader('ETag', download.etag); + if (download.lastModified) + res.setHeader('Last-Modified', download.lastModified.toUTCString()); + + res.setHeader( + 'Content-Disposition', + `inline; filename="${encodeURIComponent(entry.name)}"`, + ); + res.status(range ? 206 : 200); + + const metering = this.services.metering as + | { + batchIncrementUsages?: ( + actor: unknown, + entries: unknown[], + ) => void; + } + | undefined; + + try { + await pipeline(download.body, res); + } catch { + // Client disconnect or upstream stream error — pipeline already + // tore down both ends. Response is partially sent; nothing to do. + return; + } + + // Meter egress only on successful completion. + if (metering?.batchIncrementUsages && download.contentLength) { + try { + const bytes = download.contentLength; + metering.batchIncrementUsages(actor, [ + { + usageType: 'filesystem:egress:bytes', + usageAmount: bytes, + costOverride: + FS_COSTS['filesystem:egress:bytes'] * bytes, + }, + ]); + } catch { + // ignore — metering is non-critical. + } + } + } + + // ── Mutation routes ──────────────────────────────────────────────── + + @Post('/mkdir', { subdomain: 'api', requireVerified: true }) + async mkdirEntry(req: Request, res: Response) { + const actor = this.#requireActor(req); + const userId = this.#getActorUserId(req); + const body = this.#toObjectRecord(req.body); + const rawPath = typeof body.path === 'string' ? body.path : ''; + if (!rawPath.trim()) throw new HttpError(400, 'Missing `path`'); + + // Normalize first: expands `~`, collapses `..`, ensures leading `/` + // and no trailing `/`. Without this, `pathPosix.dirname(...)` below + // would compute a wrong parent for `~/...` inputs (e.g. dirname of + // `/~/Documents/foo` is `/~/Documents`, not `//Documents`). + const username = this.#getActorUsername(req); + const path = this.#normalizePath(rawPath, username); + if (path === '/') throw new HttpError(400, 'Cannot mkdir at root'); + + // ACL: write on parent (or on target path if overwriting existing). + const parentPath = pathPosix.dirname(path); + await this.#assertAccess( + actor, + parentPath === '/' ? path : parentPath, + 'write', + ); + + const entry = await this.services.fs.mkdir(userId, { + path, + overwrite: this.#toBoolean(body.overwrite) ?? false, + dedupeName: + this.#toBoolean(body.dedupe_name ?? body.dedupeName) ?? false, + createMissingParents: + this.#toBoolean( + body.create_missing_parents ?? + body.create_missing_ancestors, + ) ?? false, + }); + this.#emitGuiItemAdded(entry); + res.json(entry); + } + + @Post('/touch', { subdomain: 'api', requireVerified: true }) + async touchEntry(req: Request, res: Response) { + const actor = this.#requireActor(req); + const userId = this.#getActorUserId(req); + const body = this.#toObjectRecord(req.body); + const rawPath = typeof body.path === 'string' ? body.path : ''; + if (!rawPath.trim()) throw new HttpError(400, 'Missing `path`'); + + const username = this.#getActorUsername(req); + const path = this.#normalizePath(rawPath, username); + if (path === '/') throw new HttpError(400, 'Cannot touch root'); + + const parentPath = pathPosix.dirname(path); + await this.#assertAccess( + actor, + parentPath === '/' ? path : parentPath, + 'write', + ); + + const entry = await this.services.fs.touch(userId, { + path, + setAccessed: this.#toBoolean(body.set_accessed_to_now) ?? false, + setModified: this.#toBoolean(body.set_modified_to_now) ?? false, + setCreated: this.#toBoolean(body.set_created_to_now) ?? false, + createMissingParents: + this.#toBoolean(body.create_missing_parents) ?? false, + }); + res.json(entry); + } + + @Post('/rename', { subdomain: 'api', requireVerified: true }) + async renameEntry(req: Request, res: Response) { + const actor = this.#requireActor(req); + const body = this.#toObjectRecord(req.body); + const newName = typeof body.new_name === 'string' ? body.new_name : ''; + if (!newName.trim()) throw new HttpError(400, 'Missing `new_name`'); + + const entry = await this.#resolveEntryForRequest(body); + await this.#assertAccess(actor, entry.path, 'write'); + + const renamed = await this.services.fs.rename(entry, newName); + this.#emitGuiItemUpdated(renamed); + res.json(renamed); + } + + @Post('/delete', { subdomain: 'api', requireVerified: true }) + async deleteEntry(req: Request, res: Response) { + const actor = this.#requireActor(req); + const userId = this.#getActorUserId(req); + const body = this.#toObjectRecord(req.body); + const entry = await this.#resolveEntryForRequest(body); + await this.#assertAccess(actor, entry.path, 'write'); + + const descendantsOnly = this.#toBoolean(body.descendants_only) ?? false; + await this.services.fs.remove(userId, { + entry, + recursive: this.#toBoolean(body.recursive) ?? false, + descendantsOnly, + }); + this.#emitGuiItemRemoved(entry, descendantsOnly); + res.json({ ok: true }); + } + + @Post('/move', { subdomain: 'api', requireVerified: true }) + async moveEntry(req: Request, res: Response) { + const actor = this.#requireActor(req); + const userId = this.#getActorUserId(req); + const body = this.#toObjectRecord(req.body); + const sourceRef = this.#extractNodeRef(body.source ?? body); + const destinationRef = this.#extractNodeRef(body.destination); + + const source = await this.#resolveEntryForRequest(sourceRef); + const destinationParent = + await this.#resolveEntryForRequest(destinationRef); + + await this.#assertAccess(actor, source.path, 'write'); + await this.#assertAccess(actor, destinationParent.path, 'write'); + + const moved = await this.services.fs.move(userId, { + source, + destinationParent, + newName: + typeof body.new_name === 'string' ? body.new_name : undefined, + overwrite: this.#toBoolean(body.overwrite) ?? false, + dedupeName: + this.#toBoolean(body.dedupe_name ?? body.change_name) ?? false, + }); + this.#emitGuiItemMoved(source, moved); + res.json(moved); + } + + @Post('/copy', { subdomain: 'api', requireVerified: true }) + async copyEntry(req: Request, res: Response) { + const actor = this.#requireActor(req); + const userId = this.#getActorUserId(req); + const body = this.#toObjectRecord(req.body); + const sourceRef = this.#extractNodeRef(body.source ?? body); + const destinationRef = this.#extractNodeRef(body.destination); + + const source = await this.#resolveEntryForRequest(sourceRef); + const destinationParent = + await this.#resolveEntryForRequest(destinationRef); + + await this.#assertAccess(actor, source.path, 'read'); + await this.#assertAccess(actor, destinationParent.path, 'write'); + + const copy = await this.services.fs.copy(userId, { + source, + destinationParent, + newName: + typeof body.new_name === 'string' ? body.new_name : undefined, + overwrite: this.#toBoolean(body.overwrite) ?? false, + dedupeName: + this.#toBoolean(body.dedupe_name ?? body.change_name) ?? true, + }); + this.#emitGuiItemAdded(copy); + res.json(copy); + } + + @Post('/mkshortcut', { subdomain: 'api', requireVerified: true }) + async mkshortcutEntry(req: Request, res: Response) { + const actor = this.#requireActor(req); + const userId = this.#getActorUserId(req); + const body = this.#toObjectRecord(req.body); + const parentRef = this.#extractNodeRef(body.parent ?? body); + const targetRef = this.#extractNodeRef(body.target); + const name = typeof body.name === 'string' ? body.name : ''; + if (!name.trim()) throw new HttpError(400, 'Missing `name`'); + + const parent = await this.#resolveEntryForRequest(parentRef); + const target = await this.#resolveEntryForRequest(targetRef); + + await this.#assertAccess(actor, target.path, 'read'); + await this.#assertAccess(actor, parent.path, 'write'); + + const shortcut = await this.services.fs.mkshortcut(userId, { + parent, + name, + target, + dedupeName: this.#toBoolean(body.dedupe_name) ?? true, + }); + this.#emitGuiItemAdded(shortcut); + res.json(shortcut); + } + + // ── Read-side helpers ─────────────────────────────────────────────── + + #requireActor(req: Request): Actor { + const actor = req.actor; + if (!actor) { + throw new HttpError(401, 'Unauthorized'); + } + return actor; + } + + #isRootPathRef(source: Record): boolean { + if (typeof source.path !== 'string') return false; + if (source.uid !== undefined || source.uuid !== undefined) return false; + if (source.id !== undefined) return false; + return source.path.trim() === '/'; + } + + async #resolveEntryForRequest(source: Record) { + const mod = await import('../../services/fs/resolveNode.js'); + const username = Context.get('actor')?.user?.username; + const rawPath = + typeof source.path === 'string' ? source.path : undefined; + const ref = { + path: + rawPath !== undefined + ? mod.expandTildePath(rawPath, username) + : undefined, + uid: + typeof source.uid === 'string' + ? source.uid + : typeof source.uuid === 'string' + ? source.uuid + : undefined, + id: + typeof source.id === 'number' || typeof source.id === 'string' + ? source.id + : undefined, + }; + const entry = await mod.resolveNode(this.stores.fsEntry, ref, { + required: true, + }); + if (!entry) { + throw new HttpError(404, 'Entry not found'); + } + return entry; + } + + async #assertAccess( + actor: Actor, + path: string, + mode: 'see' | 'list' | 'read' | 'write', + ) { + const fsService = this.services.fs; + let ancestorsCache: Promise< + Array<{ uid: string; path: string }> + > | null = null; + const descriptor = { + path, + resolveAncestors() { + if (!ancestorsCache) { + ancestorsCache = fsService.getAncestorChain(path); + } + return ancestorsCache; + }, + }; + const allowed = await this.services.acl.check(actor, descriptor, mode); + if (allowed) return; + const safe = (await this.services.acl.getSafeAclError( + actor, + descriptor, + mode, + )) as { + status?: unknown; + message?: unknown; + fields?: { code?: unknown }; + }; + const status = Number(safe?.status); + const message = + typeof safe?.message === 'string' && safe.message.length > 0 + ? safe.message + : 'Access denied'; + const code = + typeof safe?.fields?.code === 'string' + ? safe.fields.code + : undefined; + const legacyCode = code === 'forbidden' ? 'access_denied' : code; + if (status === 404) { + throw new HttpError(404, message, { + ...(legacyCode ? { legacyCode } : {}), + }); + } + throw new HttpError(403, message, { + legacyCode: legacyCode ?? 'access_denied', + }); + } + + #toNumberOrUndefined(value: unknown): number | undefined { + if (typeof value === 'number' && Number.isFinite(value)) return value; + if (typeof value === 'string' && value.trim().length > 0) { + const parsed = Number(value); + if (Number.isFinite(parsed)) return parsed; + } + return undefined; + } + + // Accepts loose inputs from route bodies. `source`/`destination` fields may + // arrive as a plain string (= path) or an object of { path | uid | id }. + #extractNodeRef(value: unknown): Record { + if (typeof value === 'string') return { path: value }; + if (value && typeof value === 'object' && !Array.isArray(value)) { + return value as Record; + } + return {}; + } + + // Fire-and-forget GUI events for single-entry mutations. These feed the + // desktop cache invalidator and extension listeners (e.g. thumbnails). + #emitGuiItemAdded(entry: FSEntry): void { + void this.#emitGuiWriteEvent( + 'outer.gui.item.added', + entry, + undefined, + ).catch(() => undefined); + } + + #emitGuiItemUpdated(entry: FSEntry): void { + void this.#emitGuiWriteEvent( + 'outer.gui.item.updated', + entry, + undefined, + ).catch(() => undefined); + } + + #emitGuiItemRemoved(entry: FSEntry, descendantsOnly = false): void { + // GUI listens for `outer.gui.item.removed`; same envelope shape. + // `descendants_only` lets the GUI keep the parent (e.g. Trash) and + // only drop its children — without it the GUI removes the parent too. + void (async () => { + try { + await this.clients.event.emit( + 'outer.gui.item.removed', + { + user_id_list: [entry.userId], + response: { + ...entry, + from_new_service: true, + descendants_only: descendantsOnly, + }, + }, + {}, + ); + } catch { + // ignore — non-critical. + } + })(); + } + + #emitGuiItemMoved(source: FSEntry, moved: FSEntry): void { + void (async () => { + try { + await this.clients.event.emit( + 'outer.gui.item.moved', + { + user_id_list: [moved.userId], + response: { + ...moved, + from_path: source.path, + from_new_service: true, + }, + }, + {}, + ); + } catch { + // ignore — non-critical. + } + })(); + } + + #getActorUserId(req: Request): number { const requestUser = ( req as Request & { user?: { @@ -744,21 +1343,21 @@ export class FSController extends ExtensionController { }; } ).user; - const actorUser = req.actor?.type?.user; + const actorUser = req.actor?.user; const candidateUserId = requestUser?.id ?? actorUser?.id; - if ( candidateUserId === undefined || candidateUserId === null ) { + if (candidateUserId === undefined || candidateUserId === null) { throw new HttpError(401, 'Unauthorized'); } const userId = Number(candidateUserId); - if ( Number.isNaN(userId) ) { + if (Number.isNaN(userId)) { throw new HttpError(401, 'Unauthorized'); } return userId; } - #getActorUsername (req: Request): string { + #getActorUsername(req: Request): string { const requestUser = ( req as Request & { user?: { @@ -766,7 +1365,7 @@ export class FSController extends ExtensionController { }; } ).user; - const actorUser = req.actor?.type?.user; + const actorUser = req.actor?.user; const actorUsername = requestUser?.username ?? actorUser?.username; if ( typeof actorUsername !== 'string' || @@ -777,59 +1376,62 @@ export class FSController extends ExtensionController { return actorUsername.trim(); } - #toObjectRecord (value: unknown): Record { - if ( !value || typeof value !== 'object' || Array.isArray(value) ) { + #toObjectRecord(value: unknown): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) { return {}; } return value as Record; } - #firstDefined (...values: unknown[]): unknown { - for ( const value of values ) { - if ( value !== undefined && value !== null ) { + #firstDefined(...values: unknown[]): unknown { + for (const value of values) { + if (value !== undefined && value !== null) { return value; } } return undefined; } - #toBoolean (value: unknown): boolean | undefined { - if ( typeof value === 'boolean' ) { + #toBoolean(value: unknown): boolean | undefined { + if (typeof value === 'boolean') { return value; } - if ( typeof value === 'number' ) { - if ( value === 1 ) return true; - if ( value === 0 ) return false; + if (typeof value === 'number') { + if (value === 1) return true; + if (value === 0) return false; return undefined; } - if ( typeof value === 'string' ) { + if (typeof value === 'string') { const normalizedValue = value.trim().toLowerCase(); - if ( ['1', 'true', 'yes', 'on'].includes(normalizedValue) ) { + if (['1', 'true', 'yes', 'on'].includes(normalizedValue)) { return true; } - if ( ['0', 'false', 'no', 'off'].includes(normalizedValue) ) { + if (['0', 'false', 'no', 'off'].includes(normalizedValue)) { return false; } } return undefined; } - #toNumber (value: unknown): number | undefined { - if ( value === undefined || value === null || value === '' ) { + #toNumber(value: unknown): number | undefined { + if (value === undefined || value === null || value === '') { return undefined; } const candidate = Number(value); - if ( ! Number.isFinite(candidate) ) { + if (!Number.isFinite(candidate)) { return undefined; } return candidate; } - #isDedupeEnabled (fileMetadata: FSEntryWriteInput | undefined): boolean { - if ( ! fileMetadata ) { + #isDedupeEnabled(fileMetadata: FSEntryWriteInput | undefined): boolean { + if (!fileMetadata) { return false; } - const metadataRecord = fileMetadata as unknown as Record; + const metadataRecord = fileMetadata as unknown as Record< + string, + unknown + >; const dedupeCandidate = this.#firstDefined( fileMetadata.dedupeName, metadataRecord.dedupe_name, @@ -837,7 +1439,7 @@ export class FSController extends ExtensionController { return this.#toBoolean(dedupeCandidate) ?? false; } - #resolveWriteFileMetadata ( + #resolveWriteFileMetadata( fileMetadata: FSEntryWriteInput | undefined, fallbackSource?: unknown, ): FSEntryWriteInput { @@ -848,15 +1450,18 @@ export class FSController extends ExtensionController { ...metadataRecord, }; - const path = this.#firstDefined(metadataRecord.path, fallbackRecord.path); - if ( typeof path === 'string' ) { + const path = this.#firstDefined( + metadataRecord.path, + fallbackRecord.path, + ); + if (typeof path === 'string') { normalizedFileMetadata.path = path; } const size = this.#toNumber( this.#firstDefined(metadataRecord.size, fallbackRecord.size), ); - if ( size !== undefined ) { + if (size !== undefined) { normalizedFileMetadata.size = size; } @@ -866,7 +1471,7 @@ export class FSController extends ExtensionController { fallbackRecord.contentType, fallbackRecord.content_type, ); - if ( typeof contentType === 'string' && contentType.length > 0 ) { + if (typeof contentType === 'string' && contentType.length > 0) { normalizedFileMetadata.contentType = contentType; } @@ -876,14 +1481,17 @@ export class FSController extends ExtensionController { fallbackRecord.checksumSha256, fallbackRecord.checksum_sha256, ); - if ( typeof checksumSha256 === 'string' && checksumSha256.length > 0 ) { + if (typeof checksumSha256 === 'string' && checksumSha256.length > 0) { normalizedFileMetadata.checksumSha256 = checksumSha256; } const overwrite = this.#toBoolean( - this.#firstDefined(metadataRecord.overwrite, fallbackRecord.overwrite), + this.#firstDefined( + metadataRecord.overwrite, + fallbackRecord.overwrite, + ), ); - if ( overwrite !== undefined ) { + if (overwrite !== undefined) { normalizedFileMetadata.overwrite = overwrite; } @@ -897,7 +1505,7 @@ export class FSController extends ExtensionController { fallbackRecord.change_name, ), ); - if ( dedupeName !== undefined ) { + if (dedupeName !== undefined) { normalizedFileMetadata.dedupeName = dedupeName; } @@ -914,14 +1522,17 @@ export class FSController extends ExtensionController { fallbackRecord.create_file_parent, ), ); - if ( createMissingParents !== undefined ) { + if (createMissingParents !== undefined) { normalizedFileMetadata.createMissingParents = createMissingParents; } const immutable = this.#toBoolean( - this.#firstDefined(metadataRecord.immutable, fallbackRecord.immutable), + this.#firstDefined( + metadataRecord.immutable, + fallbackRecord.immutable, + ), ); - if ( immutable !== undefined ) { + if (immutable !== undefined) { normalizedFileMetadata.immutable = immutable; } @@ -933,7 +1544,7 @@ export class FSController extends ExtensionController { fallbackRecord.is_public, ), ); - if ( isPublic !== undefined ) { + if (isPublic !== undefined) { normalizedFileMetadata.isPublic = isPublic; } @@ -945,7 +1556,7 @@ export class FSController extends ExtensionController { fallbackRecord.multipart_part_size, ), ); - if ( multipartPartSize !== undefined && multipartPartSize > 0 ) { + if (multipartPartSize !== undefined && multipartPartSize > 0) { normalizedFileMetadata.multipartPartSize = multipartPartSize; } @@ -953,7 +1564,7 @@ export class FSController extends ExtensionController { metadataRecord.bucket, fallbackRecord.bucket, ); - if ( typeof bucket === 'string' && bucket.length > 0 ) { + if (typeof bucket === 'string' && bucket.length > 0) { normalizedFileMetadata.bucket = bucket; } @@ -963,7 +1574,7 @@ export class FSController extends ExtensionController { fallbackRecord.bucketRegion, fallbackRecord.bucket_region, ); - if ( typeof bucketRegion === 'string' && bucketRegion.length > 0 ) { + if (typeof bucketRegion === 'string' && bucketRegion.length > 0) { normalizedFileMetadata.bucketRegion = bucketRegion; } @@ -975,14 +1586,14 @@ export class FSController extends ExtensionController { fallbackRecord.associated_app_id, ), ); - if ( associatedAppId !== undefined ) { + if (associatedAppId !== undefined) { normalizedFileMetadata.associatedAppId = associatedAppId; } return normalizedFileMetadata as unknown as FSEntryWriteInput; } - async #resolveAssociatedAppMetadata ( + async #resolveAssociatedAppMetadata( fileMetadata: FSEntryWriteInput, fallbackSource?: unknown, appUidLookupCache?: Map>, @@ -998,7 +1609,7 @@ export class FSController extends ExtensionController { fallbackRecord.associated_app_id, ), ); - if ( associatedAppId !== undefined ) { + if (associatedAppId !== undefined) { return { ...fileMetadata, associatedAppId, @@ -1013,19 +1624,19 @@ export class FSController extends ExtensionController { fallbackRecord.appUid, fallbackRecord.app_uid, ); - if ( typeof appUid !== 'string' || appUid.trim().length === 0 ) { + if (typeof appUid !== 'string' || appUid.trim().length === 0) { return fileMetadata; } const normalizedAppUid = appUid.trim(); const lookupPromise = (() => { const cachedLookup = appUidLookupCache?.get(normalizedAppUid); - if ( cachedLookup ) { + if (cachedLookup) { return cachedLookup; } const createdLookupPromise = (async () => { - const app = await getApp({ uid: normalizedAppUid }); + const app = await this.stores.app.getByUid(normalizedAppUid); return this.#toNumber(app?.id) ?? null; })(); appUidLookupCache?.set(normalizedAppUid, createdLookupPromise); @@ -1033,7 +1644,7 @@ export class FSController extends ExtensionController { })(); const resolvedAppId = await lookupPromise; - if ( resolvedAppId === null ) { + if (resolvedAppId === null) { return fileMetadata; } return { @@ -1042,37 +1653,42 @@ export class FSController extends ExtensionController { }; } - #toStorageCapacityCandidate (value: unknown): number | undefined { + #toStorageCapacityCandidate(value: unknown): number | undefined { const capacity = Number(value); - if ( !Number.isFinite(capacity) || capacity < 0 ) { + if (!Number.isFinite(capacity) || capacity < 0) { return undefined; } return capacity; } - #getStorageAllowanceMaxOverride (req: Request): number | undefined { - const actorUser = req.actor?.type.user; + #getStorageAllowanceMaxOverride(req: Request): number | undefined { + // free_storage / actual_free_storage are user-row fields not on + // the ActorUser type. Access via the escape hatch until a proper + // storage-quota mechanism is in place. + const actorUser = req.actor?.user as + | Record + | undefined; const candidates = [ this.#toStorageCapacityCandidate(actorUser?.free_storage), this.#toStorageCapacityCandidate(actorUser?.actual_free_storage), ].filter((candidate): candidate is number => candidate !== undefined); - if ( candidates.length === 0 ) { + if (candidates.length === 0) { return undefined; } return Math.max(...candidates); } - #normalizePath (path: string, username?: string): string { + #normalizePath(path: string, username?: string): string { const trimmedPath = path.trim(); - if ( trimmedPath.length === 0 ) { + if (trimmedPath.length === 0) { throw new HttpError(400, 'Path cannot be empty'); } let pathToNormalize = trimmedPath; - if ( pathToNormalize === '~' || pathToNormalize.startsWith('~/') ) { - if ( ! username ) { + if (pathToNormalize === '~' || pathToNormalize.startsWith('~/')) { + if (!username) { throw new HttpError(400, 'Unable to resolve home path'); } @@ -1080,16 +1696,16 @@ export class FSController extends ExtensionController { } let normalizedPath = pathPosix.normalize(pathToNormalize); - if ( ! normalizedPath.startsWith('/') ) { + if (!normalizedPath.startsWith('/')) { normalizedPath = `/${normalizedPath}`; } - if ( normalizedPath.length > 1 && normalizedPath.endsWith('/') ) { + if (normalizedPath.length > 1 && normalizedPath.endsWith('/')) { normalizedPath = normalizedPath.slice(0, -1); } return normalizedPath; } - #normalizeFileMetadataPath ( + #normalizeFileMetadataPath( req: Request, fileMetadata: FSEntryWriteInput | undefined, fallbackSource?: unknown, @@ -1098,7 +1714,7 @@ export class FSController extends ExtensionController { fileMetadata, fallbackSource, ); - if ( typeof resolvedFileMetadata.path !== 'string' ) { + if (typeof resolvedFileMetadata.path !== 'string') { throw new HttpError(400, 'Missing path'); } @@ -1109,7 +1725,7 @@ export class FSController extends ExtensionController { }; } - #extractGuiMetadata ( + #extractGuiMetadata( input: unknown, fallback: WriteGuiMetadata | undefined, ): WriteGuiMetadata | undefined { @@ -1119,29 +1735,29 @@ export class FSController extends ExtensionController { : {}; const guiMetadata: WriteGuiMetadata = { originalClientSocketId: - typeof source.originalClientSocketId === 'string' - ? source.originalClientSocketId - : typeof source.original_client_socket_id === 'string' - ? source.original_client_socket_id - : fallback?.originalClientSocketId, + typeof source.originalClientSocketId === 'string' + ? source.originalClientSocketId + : typeof source.original_client_socket_id === 'string' + ? source.original_client_socket_id + : fallback?.originalClientSocketId, socketId: - typeof source.socketId === 'string' - ? source.socketId - : typeof source.socket_id === 'string' - ? source.socket_id - : fallback?.socketId, + typeof source.socketId === 'string' + ? source.socketId + : typeof source.socket_id === 'string' + ? source.socket_id + : fallback?.socketId, operationId: - typeof source.operationId === 'string' - ? source.operationId - : typeof source.operation_id === 'string' - ? source.operation_id - : fallback?.operationId, + typeof source.operationId === 'string' + ? source.operationId + : typeof source.operation_id === 'string' + ? source.operation_id + : fallback?.operationId, itemUploadId: - typeof source.itemUploadId === 'string' - ? source.itemUploadId - : typeof source.item_upload_id === 'string' - ? source.item_upload_id - : fallback?.itemUploadId, + typeof source.itemUploadId === 'string' + ? source.itemUploadId + : typeof source.item_upload_id === 'string' + ? source.item_upload_id + : fallback?.itemUploadId, }; if ( @@ -1163,7 +1779,7 @@ export class FSController extends ExtensionController { value, this.#extractGuiMetadata(fallbackSource, undefined), ); - if ( ! guiMetadata ) { + if (!guiMetadata) { return value; } return { @@ -1172,7 +1788,7 @@ export class FSController extends ExtensionController { }; } - async #assertWriteAccess ( + async #assertWriteAccess( req: Request, fileMetadata: FSEntryWriteInput | undefined, options?: { @@ -1180,54 +1796,59 @@ export class FSController extends ExtensionController { }, ): Promise { const actor = req.actor; - if ( ! actor ) { + if (!actor) { throw new HttpError(401, 'Unauthorized'); } const normalizedFileMetadata = options?.pathAlreadyNormalized ? fileMetadata : this.#normalizeFileMetadataPath(req, fileMetadata); - if ( ! normalizedFileMetadata ) { + if (!normalizedFileMetadata) { throw new HttpError(400, 'Missing path'); } const targetPath = normalizedFileMetadata.path; - if ( targetPath === '/' ) { + if (targetPath === '/') { throw new HttpError(400, 'Cannot write to root path'); } const parentPath = pathPosix.dirname(targetPath); - if ( parentPath === '/' ) { + if (parentPath === '/') { throw new HttpError(400, 'Cannot write to root path'); } const dedupeEnabled = this.#isDedupeEnabled(normalizedFileMetadata); let pathToCheck = parentPath; - if ( Boolean(normalizedFileMetadata.overwrite) && !dedupeEnabled ) { + if (Boolean(normalizedFileMetadata.overwrite) && !dedupeEnabled) { const destinationExists = - await this.fsEntryService.entryExistsByPath(targetPath); - if ( destinationExists ) { + await this.services.fs.entryExistsByPath(targetPath); + if (destinationExists) { pathToCheck = targetPath; } } - const fsEntryService = this.fsEntryService; - let ancestorsCache: Promise> | null = - null; + const fsService = this.services.fs; + let ancestorsCache: Promise< + Array<{ uid: string; path: string }> + > | null = null; const resourceDescriptor = { path: pathToCheck, - resolveAncestors () { - if ( ! ancestorsCache ) { - ancestorsCache = fsEntryService.getAncestorChain(pathToCheck); + resolveAncestors() { + if (!ancestorsCache) { + ancestorsCache = fsService.getAncestorChain(pathToCheck); } return ancestorsCache; }, }; - const canWrite = await aclService.check(actor, resourceDescriptor, 'write'); - if ( canWrite ) { + const canWrite = await this.services.acl.check( + actor, + resourceDescriptor, + 'write', + ); + if (canWrite) { return; } - const safeAclError = (await aclService.get_safe_acl_error( + const safeAclError = (await this.services.acl.getSafeAclError( actor, resourceDescriptor, 'write', @@ -1251,7 +1872,7 @@ export class FSController extends ExtensionController { const legacyCode = safeAclCode === 'forbidden' ? 'access_denied' : safeAclCode; - if ( safeAclStatus === 404 ) { + if (safeAclStatus === 404) { throw new HttpError(404, safeAclMessage, { ...(legacyCode ? { legacyCode } : {}), }); @@ -1262,7 +1883,7 @@ export class FSController extends ExtensionController { }); } - async #assertBatchWriteAccess ( + async #assertBatchWriteAccess( req: Request, fileMetadataItems: Array, options?: { @@ -1281,19 +1902,25 @@ export class FSController extends ExtensionController { ); } - #toEventGuiMetadata ( + #toEventGuiMetadata( guiMetadata: WriteGuiMetadata | undefined, includeOriginalClientSocketId = true, ): Record { - if ( ! guiMetadata ) { + if (!guiMetadata) { return {}; } return { - ...(includeOriginalClientSocketId && guiMetadata.originalClientSocketId - ? { original_client_socket_id: guiMetadata.originalClientSocketId } + ...(includeOriginalClientSocketId && + guiMetadata.originalClientSocketId + ? { + original_client_socket_id: + guiMetadata.originalClientSocketId, + } + : {}), + ...(guiMetadata.socketId + ? { socket_id: guiMetadata.socketId } : {}), - ...(guiMetadata.socketId ? { socket_id: guiMetadata.socketId } : {}), ...(guiMetadata.operationId ? { operation_id: guiMetadata.operationId } : {}), @@ -1303,14 +1930,13 @@ export class FSController extends ExtensionController { }; } - async #toGuiFsEntry (entry: FSEntry): Promise> { + async #toGuiFsEntry(entry: FSEntry): Promise> { const dirpath = pathPosix.dirname(entry.path); const extension = pathPosix.extname(entry.name).slice(1).toLowerCase(); const response = { id: entry.uuid, uid: entry.uuid, uuid: entry.uuid, - user_id: entry.userId, parent_id: entry.parentUid, parent_uid: entry.parentUid, path: entry.path, @@ -1330,7 +1956,6 @@ export class FSController extends ExtensionController { created: entry.created, accessed: entry.accessed, size: entry.size, - associated_app_id: entry.associatedAppId, }; if ( @@ -1341,7 +1966,13 @@ export class FSController extends ExtensionController { uuid: entry.uuid, thumbnail: response.thumbnail, }; - await this.eventService.emit('thumbnail.read', thumbnailEntry); + // emitAndWait — listener rewrites s3:// / legacy URLs into + // time-limited signed URLs on the payload object. + await this.clients.event.emitAndWait( + 'thumbnail.read', + thumbnailEntry, + {}, + ); response.thumbnail = typeof thumbnailEntry.thumbnail === 'string' && thumbnailEntry.thumbnail.length > 0 @@ -1352,7 +1983,7 @@ export class FSController extends ExtensionController { return response; } - async #emitGuiWriteEvent ( + async #emitGuiWriteEvent( eventName: 'outer.gui.item.added' | 'outer.gui.item.updated', fsEntry: FSEntry, guiMetadata: WriteGuiMetadata | undefined, @@ -1362,33 +1993,28 @@ export class FSController extends ExtensionController { ...this.#toEventGuiMetadata(guiMetadata, false), from_new_service: true, }; - await this.eventService.emit(eventName, { - user_id_list: [fsEntry.userId], - response, - }); + await this.clients.event.emit( + eventName, + { + user_id_list: [fsEntry.userId], + response, + }, + {}, + ); } - async #emitFsLifecycleEvent ( - eventName: 'fs.write.file' | 'fs.create.file', - fsEntry: FSEntry, - ): Promise { - await this.eventService.emit(eventName, { - node: fsEntry, - context: Context.get(), - }); - } - - async #emitGuiPendingWriteEvent ( + async #emitGuiPendingWriteEvent( userId: number, requestBody: SignedWriteRequest, response: SignedWriteResponse, ): Promise { - const normalizedPath = this.#normalizePath(requestBody.fileMetadata.path); + const normalizedPath = this.#normalizePath( + requestBody.fileMetadata.path, + ); const pendingResponse = { id: response.objectKey, uid: response.objectKey, uuid: response.objectKey, - user_id: userId, path: normalizedPath, name: pathPosix.basename(normalizedPath), is_dir: false, @@ -1400,71 +2026,83 @@ export class FSController extends ExtensionController { ...this.#toEventGuiMetadata(requestBody.guiMetadata), from_new_service: true, }; - await this.eventService.emit('outer.gui.item.pending', { - user_id_list: [userId], - response: pendingResponse, - }); + await this.clients.event.emit( + 'outer.gui.item.pending', + { + user_id_list: [userId], + response: pendingResponse, + }, + {}, + ); } - #isAppDataPath (targetPath: string): boolean { + #isAppDataPath(targetPath: string): boolean { const pathParts = targetPath.split('/').filter(Boolean); return pathParts.length >= 2 && pathParts[1] === 'AppData'; } - #estimateDataUrlSize (dataUrl: string): number { + #estimateDataUrlSize(dataUrl: string): number { const commaIndex = dataUrl.indexOf(','); - const base64 = commaIndex === -1 ? dataUrl : dataUrl.slice(commaIndex + 1); + const base64 = + commaIndex === -1 ? dataUrl : dataUrl.slice(commaIndex + 1); return Math.ceil((base64.length * 3) / 4); } - #isOversizedThumbnailDataUrl (thumbnail: string): boolean { - if ( ! thumbnail.startsWith('data:') ) { + #isOversizedThumbnailDataUrl(thumbnail: string): boolean { + if (!thumbnail.startsWith('data:')) { return false; } return this.#estimateDataUrlSize(thumbnail) > MAX_THUMBNAIL_BYTES; } - async #applyThumbnailAfterWrite ( + async #applyThumbnailAfterWrite( userId: number, fsEntry: FSEntry, requestedThumbnail: string | null | undefined, ): Promise { - if ( !requestedThumbnail || this.#isAppDataPath(fsEntry.path) ) { + if (!requestedThumbnail || this.#isAppDataPath(fsEntry.path)) { return fsEntry; } - if ( this.#isOversizedThumbnailDataUrl(requestedThumbnail) ) { + if (this.#isOversizedThumbnailDataUrl(requestedThumbnail)) { return fsEntry; } const thumbnailPayload = { url: requestedThumbnail }; - await this.eventService.emit('thumbnail.created', thumbnailPayload); + // emitAndWait — the thumbnails extension may rewrite `url` from a + // data URL to an `s3://` pointer; plain `emit` races with the DB + // update below. + await this.clients.event.emitAndWait( + 'thumbnail.created', + thumbnailPayload, + {}, + ); const finalThumbnail = typeof thumbnailPayload.url === 'string' && thumbnailPayload.url.length > 0 ? thumbnailPayload.url : null; - if ( finalThumbnail === fsEntry.thumbnail || finalThumbnail === null ) { + if (finalThumbnail === fsEntry.thumbnail || finalThumbnail === null) { return fsEntry; } - return this.fsEntryService.updateEntryThumbnail( + return this.services.fs.updateEntryThumbnail( userId, fsEntry.uuid, finalThumbnail, ); } - #toThumbnailPrepareItem ( + #toThumbnailPrepareItem( requestBody: SignedWriteRequest, index: number, ): ThumbnailUploadPrepareItem | null { - if ( requestBody.directory ) { + if (requestBody.directory) { return null; } const thumbnailMetadata = requestBody.thumbnailMetadata; - if ( ! thumbnailMetadata ) { + if (!thumbnailMetadata) { return null; } @@ -1472,40 +2110,43 @@ export class FSController extends ExtensionController { typeof thumbnailMetadata.contentType === 'string' ? thumbnailMetadata.contentType.trim() : ''; - if ( ! contentType ) { + if (!contentType) { throw new HttpError( 400, 'thumbnailMetadata.contentType is required for signed thumbnail upload', ); } - if ( thumbnailMetadata.size === undefined ) { + if (thumbnailMetadata.size === undefined) { return null; } const size = Number(thumbnailMetadata.size); - if ( !Number.isFinite(size) || size < 0 ) { + if (!Number.isFinite(size) || size < 0) { throw new HttpError( 400, 'thumbnailMetadata.size must be a non-negative number', ); } - if ( size > MAX_THUMBNAIL_BYTES ) { + if (size > MAX_THUMBNAIL_BYTES) { return null; } return { index, contentType, size }; } - async #attachSignedThumbnailUploadTargets ( + async #attachSignedThumbnailUploadTargets( requests: SignedWriteRequest[], responses: SignedWriteResponse[], ): Promise { const prepareItems = requests .map((requestBody, index) => - this.#toThumbnailPrepareItem(requestBody, index)) - .filter((item): item is ThumbnailUploadPrepareItem => Boolean(item)); - if ( prepareItems.length === 0 ) { + this.#toThumbnailPrepareItem(requestBody, index), + ) + .filter((item): item is ThumbnailUploadPrepareItem => + Boolean(item), + ); + if (prepareItems.length === 0) { return; } @@ -1518,17 +2159,27 @@ export class FSController extends ExtensionController { }), ), }; - await this.eventService.emit('thumbnail.upload.prepare', payload); + // emitAndWait — listeners populate `uploadUrl` / `thumbnailUrl` on + // each item; plain `emit` returns before the extension runs and we'd + // read the payload back empty. + await this.clients.event.emitAndWait( + 'thumbnail.upload.prepare', + payload, + {}, + ); - for ( const item of payload.items ) { + for (const item of payload.items) { const response = responses[item.index]; - if ( ! response ) { + if (!response) { throw new HttpError( 500, 'Failed to resolve signed thumbnail response target', ); } - if ( typeof item.uploadUrl !== 'string' || item.uploadUrl.length === 0 ) { + if ( + typeof item.uploadUrl !== 'string' || + item.uploadUrl.length === 0 + ) { continue; } if ( @@ -1543,11 +2194,13 @@ export class FSController extends ExtensionController { } } - #assertNoInlineSignedThumbnailData (thumbnailData: string | undefined): void { - if ( typeof thumbnailData !== 'string' ) { + #assertNoInlineSignedThumbnailData( + thumbnailData: string | undefined, + ): void { + if (typeof thumbnailData !== 'string') { return; } - if ( thumbnailData.startsWith('data:') ) { + if (thumbnailData.startsWith('data:')) { throw new HttpError( 400, 'Signed write completion does not accept inline thumbnail data. Upload thumbnail to signed URL and provide thumbnail URL.', @@ -1555,16 +2208,16 @@ export class FSController extends ExtensionController { } } - #isMultipartRequest (req: Request): boolean { + #isMultipartRequest(req: Request): boolean { const contentType = req.headers['content-type']; - if ( typeof contentType !== 'string' ) { + if (typeof contentType !== 'string') { return false; } return contentType.includes('multipart/form-data'); } - #resolveBatchWriteRequestMode (req: Request): 'multipart' | 'json' { - if ( this.#isMultipartRequest(req) ) { + #resolveBatchWriteRequestMode(req: Request): 'multipart' | 'json' { + if (this.#isMultipartRequest(req)) { return 'multipart'; } @@ -1587,13 +2240,13 @@ export class FSController extends ExtensionController { ); } - async #runNonCritical ( + async #runNonCritical( work: () => Promise, operationName: string, ): Promise { try { await work(); - } catch ( error ) { + } catch (error) { console.error( `prodfsv2 non-critical operation failed: ${operationName}`, error, @@ -1601,7 +2254,7 @@ export class FSController extends ExtensionController { } } - async #createUploadTracker ( + async #createUploadTracker( userId: number, itemUid: string, itemPath: string, @@ -1612,38 +2265,46 @@ export class FSController extends ExtensionController { uploadTracker.setTotal(Math.max(0, expectedSize)); const context = Context.get(); - if ( ! context ) { + if (!context) { return uploadTracker; } - await this.eventService.emit('fs.storage.upload-progress', { - upload_tracker: uploadTracker, - context, - meta: { - user_id: userId, - userId: userId, - item_uid: itemUid, - item_path: itemPath, - ...this.#toEventGuiMetadata(guiMetadata), + await this.clients.event.emit( + 'fs.storage.upload-progress', + { + upload_tracker: uploadTracker, + context, + meta: { + user_id: userId, + userId: userId, + item_uid: itemUid, + item_path: itemPath, + ...this.#toEventGuiMetadata(guiMetadata), + }, }, - }); + {}, + ); return uploadTracker; } - async #emitWriteHashEvent ( + async #emitWriteHashEvent( contentHashSha256: string | null | undefined, entryUuid: string, ): Promise { - if ( ! contentHashSha256 ) { + if (!contentHashSha256) { return; } - await this.eventService.emit('outer.fs.write-hash', { - hash: contentHashSha256, - uuid: entryUuid, - }); + await this.clients.event.emit( + 'outer.fs.write-hash', + { + hash: contentHashSha256, + uuid: entryUuid, + }, + {}, + ); } - async #applyWriteResponseSideEffects ( + async #applyWriteResponseSideEffects( userId: number, response: WriteResponse, guiMetadata: WriteGuiMetadata | undefined, @@ -1651,7 +2312,10 @@ export class FSController extends ExtensionController { let fsEntry = response.fsEntry; const hashEventPromise = this.#runNonCritical(async () => { - await this.#emitWriteHashEvent(response.contentHashSha256, fsEntry.uuid); + await this.#emitWriteHashEvent( + response.contentHashSha256, + fsEntry.uuid, + ); }, 'emitWriteHashEvent'); await this.#runNonCritical(async () => { @@ -1672,23 +2336,16 @@ export class FSController extends ExtensionController { ); }, 'emitGuiWriteEvent'); - await this.#runNonCritical(async () => { - await this.#emitFsLifecycleEvent( - response.wasOverwrite ? 'fs.write.file' : 'fs.create.file', - fsEntry, - ); - }, 'emitFsLifecycleEvent'); - await hashEventPromise; return { ...response, fsEntry }; } - #shouldIgnoreUploadPath (targetPath: string): boolean { + #shouldIgnoreUploadPath(targetPath: string): boolean { return pathPosix.basename(targetPath).toLowerCase() === '.ds_store'; } - #parseBatchWriteManifest ( + #parseBatchWriteManifest( manifestRaw: string, fallbackGuiMetadata: WriteGuiMetadata | undefined, ): ParsedMultipartBatchManifest { @@ -1719,7 +2376,7 @@ export class FSController extends ExtensionController { fallbackGuiMetadata, ); const normalizedItems = manifest.items.map((item, orderIndex) => { - if ( !item || typeof item !== 'object' ) { + if (!item || typeof item !== 'object') { throw new HttpError( 400, `Batch write manifest item at position ${orderIndex} is invalid`, @@ -1729,14 +2386,14 @@ export class FSController extends ExtensionController { const candidateIndex = (item as { index?: number | string }).index ?? orderIndex; const index = Number(candidateIndex); - if ( !Number.isInteger(index) || index < 0 ) { + if (!Number.isInteger(index) || index < 0) { throw new HttpError( 400, `Batch write manifest item index is invalid at position ${orderIndex}`, ); } - if ( !item.fileMetadata || typeof item.fileMetadata !== 'object' ) { + if (!item.fileMetadata || typeof item.fileMetadata !== 'object') { throw new HttpError( 400, `Batch write manifest item ${index} is missing fileMetadata`, @@ -1747,17 +2404,20 @@ export class FSController extends ExtensionController { index, fileMetadata: item.fileMetadata, thumbnailData: - typeof item.thumbnailData === 'string' - ? item.thumbnailData - : undefined, - guiMetadata: this.#extractGuiMetadata(item, manifestGuiMetadata), + typeof item.thumbnailData === 'string' + ? item.thumbnailData + : undefined, + guiMetadata: this.#extractGuiMetadata( + item, + manifestGuiMetadata, + ), }; }); const seenIndexes = new Set(); const fieldIndexMap = new Map(); - for ( const item of normalizedItems ) { - if ( seenIndexes.has(item.index) ) { + for (const item of normalizedItems) { + if (seenIndexes.has(item.index)) { throw new HttpError( 409, `Batch write manifest has duplicate index ${item.index}`, @@ -1777,32 +2437,32 @@ export class FSController extends ExtensionController { }; } - #resolveMultipartFileIndex ( + #resolveMultipartFileIndex( fieldName: string, fileOrderIndex: number, manifest: ParsedMultipartBatchManifest, ): number { const directMatch = manifest.fieldIndexMap.get(fieldName); - if ( directMatch !== undefined ) { + if (directMatch !== undefined) { return directMatch; } - if ( /^\d+$/.test(fieldName) ) { + if (/^\d+$/.test(fieldName)) { const parsedIndex = Number(fieldName); - if ( manifest.fieldIndexMap.get(String(parsedIndex)) !== undefined ) { + if (manifest.fieldIndexMap.get(String(parsedIndex)) !== undefined) { return parsedIndex; } } - if ( fieldName === 'file' || fieldName === 'files' ) { + if (fieldName === 'file' || fieldName === 'files') { const itemAtPosition = manifest.items[fileOrderIndex]; - if ( itemAtPosition ) { + if (itemAtPosition) { return itemAtPosition.index; } } const fallbackItem = manifest.items[fileOrderIndex]; - if ( fallbackItem ) { + if (fallbackItem) { return fallbackItem.index; } diff --git a/src/backend/controllers/fs/LegacyFSController.ts b/src/backend/controllers/fs/LegacyFSController.ts new file mode 100644 index 000000000..857bc4844 --- /dev/null +++ b/src/backend/controllers/fs/LegacyFSController.ts @@ -0,0 +1,2044 @@ +import Busboy from 'busboy'; +import type { Request, RequestHandler, Response } from 'express'; +import { contentType as contentTypeFromMime } from 'mime-types'; +import { posix as pathPosix } from 'node:path'; +import type { Actor } from '../../core/actor.js'; +import { isAccessTokenActor } from '../../core/actor.js'; +import { Context } from '../../core/context.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import type { PuterRouter } from '../../core/http/PuterRouter.js'; +import type { ACLService } from '../../services/acl/ACLService.js'; +import type { SignedFile } from '../../util/fileSigning.js'; +import { verifySignature } from '../../util/fileSigning.js'; +import { PuterController } from '../types.js'; +import { FS_COSTS } from './costs.js'; +import { + asRecord, + assertAccess, + getBoolean, + getString, + loadLegacyAssociatedApps, + resolveV1Selector, + signEntry, + signingConfigFromAppConfig, + toLegacyEntry, +} from './legacyFsHelpers.js'; + +/** + * Legacy FS routes, implemented as thin shims over `FSService`. + * + * Each shim parses the request shape (FSNodeParam-style `{ path, uid, id }` + * or `{ parent, name }`), invokes the service method, and returns the + * snake_case response clients expect. + */ + +type RouterCache = Map; + +const additionalRoutePaths: Record = {}; + +async function loadAdditionalRouter( + key: string, +): Promise { + const path = additionalRoutePaths[key]; + if (!path) return null; + try { + const mod = await import(path); + return (mod.default ?? mod) as RequestHandler; + } catch (err) { + console.error( + `[legacy-fs] failed to load additional route module '${key}':`, + err, + ); + return null; + } +} + +// ── Controller ────────────────────────────────────────────────────── + +export class LegacyFSController extends PuterController { + #additionalCache: RouterCache = new Map(); + + registerRoutes(router: PuterRouter): void { + const apiOptions = { subdomain: 'api', requireVerified: true } as const; + // Signed-URL routes: the handler validates the URL signature itself, + // so no auth gate is applied (matches v1, which mounted these routers + // with no middleware). + const signedOptions = { subdomain: 'api' } as const; + + // Core filesystem_api routes — direct handlers over the FS service. + router.post('/stat', apiOptions, this.stat); + router.post('/readdir', apiOptions, this.readdir); + router.post('/mkdir', apiOptions, this.mkdir); + router.post('/copy', apiOptions, this.copy); + router.post('/move', apiOptions, this.move); + router.post('/delete', apiOptions, this.delete); + router.post('/rename', apiOptions, this.rename); + router.post('/touch', apiOptions, this.touch); + router.post('/search', apiOptions, this.search); + router.get('/read', apiOptions, this.read); + router.get( + '/token-read', + { subdomain: 'api', requireVerified: false }, + this.tokenRead, + ); + + router.post('/batch', apiOptions, this.batch); + + // Signed-URL + meta routes. + router.post('/sign', apiOptions, this.sign); + router.post('/writeFile', signedOptions, this.writeFile); + router.get('/file', signedOptions, this.file); + router.all('/df', apiOptions, this.df); + router.post('/open_item', apiOptions, this.openItem); + router.post( + '/auth/request-app-root-dir', + apiOptions, + this.requestAppRootDir, + ); + router.post('/auth/check-app-acl', apiOptions, this.checkAppAcl); + + // `/down` — session-auth'd file download. Unlike `/file` (signed URL) + // this accepts a path on the user's behalf and streams as attachment. + // Matches v1 semantics: mounted on both root and api subdomains + // because the GUI triggers it from `window.origin`, not the api host. + router.post( + '/down', + { + subdomain: ['api', ''], + requireUserActor: true, + requireVerified: true, + antiCsrf: true, + }, + this.down, + ); + // /itemMetadata is deprecated; not called by puter-js. Return 410 Gone. + router.get('/itemMetadata', apiOptions, (_req, res) => { + res.status(410).json({ + error: 'itemMetadata is deprecated; use /fs/stat', + }); + }); + + router.get('/get-launch-apps', apiOptions, async (req, res) => { + const recommendedSvc = this.services.recommendedApps as unknown as + | { getRecommendedApps?: () => Promise } + | undefined; + const recommended = recommendedSvc?.getRecommendedApps + ? await recommendedSvc.getRecommendedApps() + : []; + + let recent: unknown[] = []; + const userId = req.actor?.user?.id; + if (userId) { + const recentUids = + (await ( + this.stores.app as unknown as { + getRecentAppOpens?: ( + id: number, + opts?: { limit?: number }, + ) => Promise; + } + ).getRecentAppOpens?.(userId, { limit: 10 })) ?? []; + const apps: unknown[] = []; + for (const uid of recentUids) { + const app = await ( + this.stores.app as unknown as { + getByUid: ( + uid: string, + ) => Promise | null>; + } + ).getByUid(uid); + if (app) { + apps.push({ + uuid: app.uid, + name: app.name, + title: app.title, + icon: app.icon ?? null, + godmode: Boolean(app.godmode), + maximize_on_start: Boolean(app.maximize_on_start), + index_url: app.index_url, + }); + } + } + recent = apps; + } + + res.json({ recommended, recent }); + }); + + router.post('/suggest_apps', apiOptions, async (req, res) => { + const suggestSvc = this.services.suggestedApps; + if (!suggestSvc?.getSuggestedApps) { + res.json([]); + return; + } + const body = req.body ?? {}; + // Client sends { uid } or { path } identifying a file entry. + // Resolve the entry to get its name/path for extension detection. + let entryName: string | undefined; + let entryPath: string | undefined; + if (body.uid || body.path) { + try { + const entry = await resolveV1Selector( + this.stores.fsEntry, + body, + ); + entryName = entry?.name; + entryPath = entry?.path; + } catch { + // If we can't resolve, fall back to empty + } + } + const suggestions = await suggestSvc.getSuggestedApps({ + name: entryName, + path: entryPath, + }); + res.json(suggestions); + }); + + // puter-js polls this to decide whether to purge its in-memory FS + // cache. SocketService bumps a per-user Redis key on every + // `outer.gui.item.*` mutation — read it back here. + router.get( + '/cache/last-change-timestamp', + apiOptions, + async (req, res) => { + const userId = req.actor?.user?.id; + if (!userId) { + res.json({ timestamp: 0 }); + return; + } + const socket = this.services.socket as unknown as + | { + getLastChangeTimestamp?: ( + id: number, + ) => Promise; + } + | undefined; + const timestamp = socket?.getLastChangeTimestamp + ? await socket.getLastChangeTimestamp(userId) + : 0; + res.json({ timestamp }); + }, + ); + + // ── POST /readdir-subdomains ──────────────────────────────── + router.post( + '/readdir-subdomains', + apiOptions, + async (req: Request, res: Response) => { + const userId = req.actor?.user?.id; + if (!userId) + throw new HttpError(401, 'Authentication required'); + const rows = await this.clients.db.read( + 'SELECT `subdomain`, `root_dir_id`, `uuid`, `ts` FROM `subdomains` WHERE `user_id` = ?', + [userId], + ); + res.json(rows); + }, + ); + + // ── POST /update-fsentry-thumbnail ────────────────────────── + router.post( + '/update-fsentry-thumbnail', + apiOptions, + async (req: Request, res: Response) => { + const userId = req.actor?.user?.id; + if (!userId) + throw new HttpError(401, 'Authentication required'); + const { uid, thumbnail } = (req.body ?? {}) as { + uid?: string; + thumbnail?: string; + }; + if (!uid) throw new HttpError(400, 'Missing `uid`'); + if (!thumbnail) throw new HttpError(400, 'Missing `thumbnail`'); + + const entry = await this.stores.fsEntry.getEntryByUuid(uid); + if (!entry || entry.userId !== userId) + throw new HttpError(403, 'Access denied'); + + // Emit thumbnail.created so the thumbnails extension can S3-upload. + // emitAndWait is required — the extension rewrites `event.url` + // from a data URL to an `s3://` pointer, and the DB write below + // needs to see that rewrite. + const event = { url: thumbnail }; + await this.clients.event.emitAndWait( + 'thumbnail.created', + event, + {}, + ); + + await this.clients.db.write( + 'UPDATE `fsentries` SET `thumbnail` = ? WHERE `uuid` = ?', + [event.url, uid], + ); + res.json({ thumbnail: event.url }); + }, + ); + + for (const key of Object.keys(additionalRoutePaths)) { + router.use( + this.#createLazyHandler( + key, + this.#additionalCache, + loadAdditionalRouter, + ), + ); + } + } + + // ── Route implementations ─────────────────────────────────────────── + // + // Handlers are public arrow class fields so they auto-bind `this` and can + // be passed directly to `router.post(...)` without `.bind(this)`. Express + // 5 catches their async rejections and routes them to the error handler. + + stat = async (req: Request, res: Response): Promise => { + const actor = this.#requireActor(req); + const userId = this.#getActorUserId(req); + const body = asRecord(req.body); + + const entry = await resolveV1Selector(this.stores.fsEntry, body); + await assertAccess( + this.services.acl, + this.services.fs, + actor, + entry.path, + 'see', + ); + + entry.suggestedApps = + await this.services.suggestedApps.getSuggestedApps(entry); + + const appsById = await loadLegacyAssociatedApps(this.stores.app, [ + entry, + ]); + + const shaped = await toLegacyEntry(this.clients.event, entry, { + fsEntryStore: this.stores.fsEntry, + userStore: this.stores.user as unknown as { + getById: ( + id: number, + ) => Promise | null>; + }, + appsById, + }); + + // Optional hydrations: + if (entry.isDir && getBoolean(body, 'return_size')) { + shaped.size = await this.services.fs.getSubtreeSize( + userId, + entry.path, + ); + } + // Legacy clients sometimes ask for `return_versions`, `return_shares`. + // We don't have parity for these yet — return empty arrays to avoid + // breaking `response.x.forEach(...)` patterns. `return_owner` is a + // no-op flag here: the `owner` field is already populated by + // `toLegacyEntry` as `{ username }`. + if (getBoolean(body, 'return_versions')) shaped.versions = []; + if (getBoolean(body, 'return_shares')) shaped.shares = []; + + res.json(shaped); + }; + + readdir = async (req: Request, res: Response): Promise => { + const actor = this.#requireActor(req); + const body = asRecord(req.body); + + if (this.#isRootPathRef(body)) { + const { listRootEntries } = + await import('../../services/fs/rootListing.js'); + const rootChildren = await listRootEntries( + actor, + this.stores.fsEntry, + this.services.permission, + ); + const rootSuggestions = + await this.services.suggestedApps.getSuggestedAppsForEntries( + rootChildren, + ); + for (let index = 0; index < rootChildren.length; index++) { + const child = rootChildren[index]; + if (child) { + child.suggestedApps = rootSuggestions[index] ?? []; + } + } + const rootAppsById = await loadLegacyAssociatedApps( + this.stores.app, + rootChildren, + ); + const shaped = await Promise.all( + rootChildren.map((c) => + toLegacyEntry(this.clients.event, c, { + appsById: rootAppsById, + }), + ), + ); + res.json(shaped); + return; + } + + const parent = await resolveV1Selector(this.stores.fsEntry, body); + if (!parent.isDir) { + throw new HttpError(400, 'Target is not a directory', { + legacyCode: 'dest_is_not_a_directory', + }); + } + await assertAccess( + this.services.acl, + this.services.fs, + actor, + parent.path, + 'list', + ); + + const children = await this.services.fs.listDirectory(parent.uuid, { + sortBy: this.#parseSortBy(body), + sortOrder: this.#parseSortOrder(body), + }); + + const suggestions = + await this.services.suggestedApps.getSuggestedAppsForEntries( + children, + ); + for (let index = 0; index < children.length; index++) { + const child = children[index]; + if (child) { + child.suggestedApps = suggestions[index] ?? []; + } + } + + const appsById = await loadLegacyAssociatedApps( + this.stores.app, + children, + ); + + const shaped = await Promise.all( + children.map((c) => + toLegacyEntry(this.clients.event, c, { appsById }), + ), + ); + res.json(shaped); + }; + + mkdir = async (req: Request, res: Response): Promise => { + const actor = this.#requireActor(req); + const userId = this.#getActorUserId(req); + const body = asRecord(req.body); + const rawPath = getString(body, 'path'); + if (!rawPath) throw new HttpError(400, '`path` is required'); + + // Supports `{ parent, path }` where `path` is a relative suffix. + let targetPath = rawPath; + if (body.parent !== undefined && !rawPath.startsWith('/')) { + const parent = await resolveV1Selector( + this.stores.fsEntry, + body.parent, + ); + targetPath = + parent.path === '/' + ? `/${rawPath}` + : `${parent.path}/${rawPath}`; + } + + const parentPath = pathPosix.dirname( + targetPath.startsWith('/') ? targetPath : `/${targetPath}`, + ); + await assertAccess( + this.services.acl, + this.services.fs, + actor, + parentPath === '/' ? targetPath : parentPath, + 'write', + ); + + const entry = await this.services.fs.mkdir(userId, { + path: targetPath, + overwrite: getBoolean(body, 'overwrite') ?? false, + dedupeName: getBoolean(body, 'dedupe_name', 'change_name') ?? false, + createMissingParents: + getBoolean( + body, + 'create_missing_parents', + 'create_missing_ancestors', + ) ?? false, + }); + await this.#emitGuiEvent('outer.gui.item.added', entry); + + res.json(await toLegacyEntry(this.clients.event, entry)); + }; + + copy = async (req: Request, res: Response): Promise => { + const actor = this.#requireActor(req); + const userId = this.#getActorUserId(req); + const body = asRecord(req.body); + + const source = await resolveV1Selector( + this.stores.fsEntry, + body.source, + ); + const destinationParent = await resolveV1Selector( + this.stores.fsEntry, + body.destination, + ); + + await assertAccess( + this.services.acl, + this.services.fs, + actor, + source.path, + 'read', + ); + await assertAccess( + this.services.acl, + this.services.fs, + actor, + destinationParent.path, + 'write', + ); + + const copy = await this.services.fs.copy(userId, { + source, + destinationParent, + newName: getString(body, 'new_name'), + overwrite: getBoolean(body, 'overwrite') ?? false, + dedupeName: getBoolean(body, 'dedupe_name', 'change_name') ?? false, + }); + await this.#emitGuiEvent('outer.gui.item.added', copy); + + // Legacy response shape: `[{copied: fsentry, overwritten?}]`. + // Array is historical — originally supported bulk copy. + const copied = await toLegacyEntry(this.clients.event, copy, { + fsEntryStore: this.stores.fsEntry, + userStore: this.stores.user as unknown as { + getById: ( + id: number, + ) => Promise | null>; + }, + }); + res.json([{ copied }]); + }; + + move = async (req: Request, res: Response): Promise => { + const actor = this.#requireActor(req); + const userId = this.#getActorUserId(req); + const body = asRecord(req.body); + + const source = await resolveV1Selector( + this.stores.fsEntry, + body.source, + ); + const destinationParent = await resolveV1Selector( + this.stores.fsEntry, + body.destination, + ); + + await assertAccess( + this.services.acl, + this.services.fs, + actor, + source.path, + 'write', + ); + await assertAccess( + this.services.acl, + this.services.fs, + actor, + destinationParent.path, + 'write', + ); + + const moved = await this.services.fs.move(userId, { + source, + destinationParent, + newName: getString(body, 'new_name'), + overwrite: getBoolean(body, 'overwrite') ?? false, + dedupeName: getBoolean(body, 'dedupe_name', 'change_name') ?? false, + // Trash/restore rides on this: GUI sends + // `{ original_name, original_path, trashed_ts }` when moving into + // Trash, and `null`/`{}` when restoring. See + // `src/gui/src/helpers.js` → `window.move_items`. + newMetadata: (body.new_metadata ?? undefined) as + | Record + | null + | undefined, + }); + const oldPath = source.path; + await this.#emitGuiEvent('outer.gui.item.moved', moved, { + old_path: oldPath, + }); + + // Legacy response shape: `{moved: fsentry, old_path}`. + const movedEntry = await toLegacyEntry(this.clients.event, moved, { + fsEntryStore: this.stores.fsEntry, + userStore: this.stores.user as unknown as { + getById: ( + id: number, + ) => Promise | null>; + }, + }); + res.json({ moved: movedEntry, old_path: oldPath }); + }; + + delete = async (req: Request, res: Response): Promise => { + const actor = this.#requireActor(req); + const userId = this.#getActorUserId(req); + const body = asRecord(req.body); + + // /delete can take `paths: []` for bulk delete, or a single selector. + const descendantsOnly = getBoolean(body, 'descendants_only') ?? false; + const pathsArray = Array.isArray(body.paths) ? body.paths : null; + if (pathsArray) { + const removedEntries: unknown[] = []; + for (const raw of pathsArray) { + const entry = await resolveV1Selector(this.stores.fsEntry, raw); + await assertAccess( + this.services.acl, + this.services.fs, + actor, + entry.path, + 'write', + ); + await this.services.fs.remove(userId, { + entry, + recursive: getBoolean(body, 'recursive') ?? true, + descendantsOnly, + }); + await this.#emitGuiEvent('outer.gui.item.removed', entry, { + descendants_only: descendantsOnly, + }); + removedEntries.push( + await toLegacyEntry(this.clients.event, entry), + ); + } + res.json(removedEntries); + return; + } + + const entry = await resolveV1Selector(this.stores.fsEntry, body); + await assertAccess( + this.services.acl, + this.services.fs, + actor, + entry.path, + 'write', + ); + await this.services.fs.remove(userId, { + entry, + recursive: getBoolean(body, 'recursive') ?? true, + descendantsOnly, + }); + await this.#emitGuiEvent('outer.gui.item.removed', entry, { + descendants_only: descendantsOnly, + }); + res.json({ ok: true, uid: entry.uuid }); + }; + + rename = async (req: Request, res: Response): Promise => { + const actor = this.#requireActor(req); + const body = asRecord(req.body); + + const newName = getString(body, 'new_name'); + if (!newName) throw new HttpError(400, '`new_name` is required'); + + const entry = await resolveV1Selector(this.stores.fsEntry, body); + await assertAccess( + this.services.acl, + this.services.fs, + actor, + entry.path, + 'write', + ); + + const renamed = await this.services.fs.rename(entry, newName); + await this.#emitGuiEvent('outer.gui.item.updated', renamed); + res.json(await toLegacyEntry(this.clients.event, renamed)); + }; + + touch = async (req: Request, res: Response): Promise => { + const actor = this.#requireActor(req); + const userId = this.#getActorUserId(req); + const body = asRecord(req.body); + + const rawPath = getString(body, 'path'); + if (!rawPath) throw new HttpError(400, '`path` is required'); + + const parentPath = pathPosix.dirname( + rawPath.startsWith('/') ? rawPath : `/${rawPath}`, + ); + if (parentPath === '/') { + throw new HttpError(400, 'Cannot touch in root'); + } + await assertAccess( + this.services.acl, + this.services.fs, + actor, + parentPath, + 'write', + ); + + await this.services.fs.touch(userId, { + path: rawPath, + setAccessed: getBoolean(body, 'set_accessed_to_now') ?? false, + setModified: getBoolean(body, 'set_modified_to_now') ?? false, + setCreated: getBoolean(body, 'set_created_to_now') ?? false, + createMissingParents: + getBoolean(body, 'create_missing_parents') ?? false, + }); + // /touch historically returns an empty body. + res.send(''); + }; + + search = async (req: Request, res: Response): Promise => { + this.#requireActor(req); + const userId = this.#getActorUserId(req); + const body = asRecord(req.body); + const query = getString(body, 'query', 'text') ?? ''; + if (query.trim().length === 0) + throw new HttpError(400, '`query` is required'); + + const results = await this.services.fs.searchByName(userId, query, 200); + const shaped = await Promise.all( + results.map((r) => toLegacyEntry(this.clients.event, r)), + ); + res.json(shaped); + }; + + read = async (req: Request, res: Response, options = {}): Promise => { + const actor = this.#requireActor(req); + const query = asRecord(req.query); + + // Legacy v1 /read aliased `file` onto either path or uid depending on + // whether the value starts with `/`. resolveV1Selector does the same + // dispatch when handed a raw string. + const selector = + typeof query.file === 'string' && query.file.length > 0 + ? query.file + : query; + const entry = await resolveV1Selector(this.stores.fsEntry, selector); + await assertAccess( + this.services.acl, + this.services.fs, + actor, + entry.path, + 'read', + ); + + if (entry.isDir) { + throw new HttpError(400, 'Cannot read a directory'); + } + + const range = + typeof req.headers.range === 'string' + ? req.headers.range + : undefined; + const download = await this.services.fs.readContent(entry, { + range, + }); + + // Force `application/octet-stream` on this endpoint for wire parity + // with v1. puter-js's `parseResponse` branches on Content-Type — + // `application/octet-stream` returns the raw Blob while other + // types wrap in `{success, result: Blob}`. Clients (including the + // GUI) expect the raw-Blob shape. Use `/fs/read` for type-aware + // streaming. + if (options.realMime) { + res.setHeader('Content-Type', contentTypeFromMime(entry.name)); + } else { + res.setHeader('Content-Type', 'application/octet-stream'); + } + + if (download.contentLength !== null) + res.setHeader('Content-Length', String(download.contentLength)); + if (download.contentRange) + res.setHeader('Content-Range', download.contentRange); + if (download.etag) res.setHeader('ETag', download.etag); + if (download.lastModified) + res.setHeader('Last-Modified', download.lastModified.toUTCString()); + res.setHeader( + 'Content-Disposition', + `inline; filename="${encodeURIComponent(entry.name)}"`, + ); + res.status(range ? 206 : 200); + + // Best-effort egress metering. + const metering = this.services.metering as + | { + batchIncrementUsages?: ( + actor: unknown, + entries: unknown[], + ) => void; + } + | undefined; + if (metering?.batchIncrementUsages && download.contentLength) { + download.body.once('end', () => { + try { + const bytes = download.contentLength!; + metering.batchIncrementUsages!(actor, [ + { + usageType: 'filesystem:egress:bytes', + usageAmount: bytes, + costOverride: + FS_COSTS['filesystem:egress:bytes'] * bytes, + }, + ]); + } catch { + // ignore — non-critical. + } + }); + } + download.body.on('error', (err) => { + res.destroy(err); + }); + download.body.pipe(res); + }; + + tokenRead = async (req: Request, res: Response): Promise => { + const query = asRecord(req.query); + const accessToken = getString(query, 'token'); + if (!accessToken) { + throw new HttpError(401, 'Token authentication failed', { + legacyCode: 'token_auth_failed', + }); + } + + const actor = + await this.services.auth.authenticateFromToken(accessToken); + if (!isAccessTokenActor(actor)) { + throw new HttpError(401, 'Token authentication failed', { + legacyCode: 'token_auth_failed', + }); + } + + req.actor = actor; + Context.set('actor', actor); + + // Forward back to regular read after setting actor + return this.read(req, res, { realMime: true }); + }; + + // ── Signed-URL + meta routes ──────────────────────────────────────── + + /** + * POST /sign + * Body: `{ items: [{ uid?, path?, action }], app_uid? }`. Returns + * `{ signatures: [...], token? }`. Apps may only sign files under their + * own AppData subtree. + */ + sign = async (req: Request, res: Response): Promise => { + const actor = this.#requireActor(req); + const body = asRecord(req.body); + const items = Array.isArray(body.items) ? body.items : []; + if (items.length === 0) throw new HttpError(400, '`items` is required'); + + const isApp = Boolean((actor as { app?: unknown }).app); + const signingCfg = signingConfigFromAppConfig(this.config); + + // Apps can only sign inside their AppData root. + let appDataRoot: string | null = null; + if (isApp) { + const username = (actor as { user?: { username?: string } }).user + ?.username; + const appUid = (actor as { app?: { uid?: string } }).app?.uid; + if (!username || !appUid) throw new HttpError(403, 'Forbidden'); + appDataRoot = `/${username}/AppData/${appUid}`; + } + + type SignedOrEmpty = + | (SignedFile & { path?: string }) + | Record; + const result: { signatures: SignedOrEmpty[]; token?: string } = { + signatures: [], + }; + + // Optional app grant: provide app_uid to grant permissions + token. + let grantApp: { uid: string } | null = null; + if (typeof body.app_uid === 'string' && body.app_uid.length > 0) { + const app = await this.stores.app.getByUid(body.app_uid); + if (!app) throw new HttpError(404, 'App not found'); + grantApp = { uid: app.uid }; + result.token = this.services.auth.getUserAppToken(actor, app.uid); + } + + for (const rawItem of items) { + const item = asRecord(rawItem); + const uid = typeof item.uid === 'string' ? item.uid : undefined; + const path = typeof item.path === 'string' ? item.path : undefined; + const action = + typeof item.action === 'string' ? item.action : 'read'; + if (!uid && !path) { + result.signatures.push({}); + continue; + } + try { + const entry = await resolveV1Selector(this.stores.fsEntry, { + uid, + path, + }); + + // App-sandbox check. + const withinAppRoot = appDataRoot + ? entry.path === appDataRoot || + entry.path.startsWith(`${appDataRoot}/`) + : true; + if (!withinAppRoot) { + throw new HttpError(403, 'Forbidden'); + } + + // ACL: always require read; downgrade write→read silently. + await assertAccess( + this.services.acl, + this.services.fs, + actor, + entry.path, + 'read', + ); + let finalAction: 'read' | 'write' = 'read'; + if (action === 'write') { + const writeOk = await this.services.acl.check( + actor, + { + path: entry.path, + resolveAncestors: () => + this.services.fs.getAncestorChain(entry.path), + }, + 'write', + ); + finalAction = writeOk ? 'write' : 'read'; + } + + if (grantApp) { + // Grant the app the permission the user is signing for. + await this.services.permission.grantUserAppPermission( + actor, + grantApp.uid, + `fs:${entry.uuid}:${finalAction}`, + {}, + { reason: 'endpoint:sign' }, + ); + } + + const signed = signEntry(entry, signingCfg); + result.signatures.push({ ...signed, path: entry.path }); + } catch { + // Silently skip unresolvable items. + result.signatures.push({}); + } + } + + res.json(result); + }; + + /** + * POST /writeFile?uid=&operation= + * Signature-authenticated multipart upload. `operation` dispatches to one + * of write/copy/move/mkdir/delete/rename/trash. Signature must be valid + * for `write` action on `uid`. + */ + writeFile = async (req: Request, res: Response): Promise => { + const query = asRecord(req.query); + const signingCfg = signingConfigFromAppConfig(this.config); + verifySignature( + { + uid: query.uid as string, + expires: query.expires as string, + signature: query.signature as string, + }, + 'write', + signingCfg, + ); + + const uid = typeof query.uid === 'string' ? query.uid : ''; + const targetEntry = await resolveV1Selector(this.stores.fsEntry, { + uid, + }); + if (!targetEntry) throw new HttpError(404, 'Item not found'); + + // Owner suspension check. + const owner = await this.stores.user.getById(targetEntry.userId); + if (!owner) throw new HttpError(500, 'Owner not found'); + if ((owner as { suspended?: unknown }).suspended) + throw new HttpError(401, 'Account suspended'); + + const userId = targetEntry.userId; + const operation = + typeof query.operation === 'string' ? query.operation : 'write'; + + // `write` — multipart upload, streamed directly to the v2 write path. + if (operation === 'write') { + const body = asRecord(req.body); + const parentEntry = targetEntry.isDir + ? targetEntry + : await this.#resolveParentOfEntry(targetEntry); + const name = + typeof body.name === 'string' + ? body.name + : targetEntry.isDir + ? `upload-${Date.now()}` + : targetEntry.name; + const targetPath = + parentEntry.path === '/' + ? `/${name}` + : `${parentEntry.path}/${name}`; + + // Parse multipart and pipe the first `file` part into fsService.write. + const uploadResult = await this.#multipartWrite( + req, + userId, + targetPath, + ); + await this.#emitGuiEvent( + 'outer.gui.item.added', + uploadResult.fsEntry, + ); + const signed = signEntry(uploadResult.fsEntry, signingCfg); + res.json({ ...signed, path: uploadResult.fsEntry.path }); + return; + } + + // Non-write operations: route to existing service methods and sign the result. + const record = asRecord(req.body); + if (operation === 'mkdir') { + const folderName = + typeof record.name === 'string' + ? record.name + : `folder-${Date.now()}`; + const entry = await this.services.fs.mkdir(userId, { + path: targetEntry.isDir + ? `${targetEntry.path === '/' ? '' : targetEntry.path}/${folderName}` + : targetEntry.path, + dedupeName: true, + }); + await this.#emitGuiEvent('outer.gui.item.added', entry); + res.json({ ...signEntry(entry, signingCfg), path: entry.path }); + return; + } + if (operation === 'rename') { + const newName = + typeof record.new_name === 'string' ? record.new_name : ''; + if (!newName) throw new HttpError(400, '`new_name` required'); + const renamed = await this.services.fs.rename(targetEntry, newName); + await this.#emitGuiEvent('outer.gui.item.updated', renamed); + res.json({ ...signEntry(renamed, signingCfg), path: renamed.path }); + return; + } + if (operation === 'delete' || operation === 'trash') { + // Treat trash == delete (recursive). Most clients just call delete + // directly; if a trash folder becomes important we can revisit. + await this.services.fs.remove(userId, { + entry: targetEntry, + recursive: true, + }); + await this.#emitGuiEvent('outer.gui.item.removed', targetEntry); + res.json({ ok: true, uid: targetEntry.uuid }); + return; + } + if (operation === 'copy' || operation === 'move') { + const destRef = + record.destination ?? + record.destination_uid ?? + record.dest_path; + if (!destRef) throw new HttpError(400, '`destination` required'); + const destinationParent = await resolveV1Selector( + this.stores.fsEntry, + destRef, + ); + const method = operation === 'copy' ? 'copy' : 'move'; + const result = await this.services.fs[method](userId, { + source: targetEntry, + destinationParent, + newName: + typeof record.new_name === 'string' + ? record.new_name + : undefined, + overwrite: getBoolean(record, 'overwrite') ?? false, + dedupeName: getBoolean(record, 'dedupe_name') ?? false, + }); + await this.#emitGuiEvent( + operation === 'copy' + ? 'outer.gui.item.added' + : 'outer.gui.item.moved', + result, + operation === 'move' + ? { old_path: targetEntry.path } + : undefined, + ); + res.json({ ...signEntry(result, signingCfg), path: result.path }); + return; + } + + throw new HttpError( + 400, + `Unsupported writeFile operation: '${operation}'`, + ); + }; + + /** + * GET /file?uid=&signature=...&expires=... + * Signature-authenticated file read. Directories return a signed listing + * of children; files stream bytes (with Range support when `download` + * isn't requested). + */ + file = async (req: Request, res: Response): Promise => { + const query = asRecord(req.query); + const signingCfg = signingConfigFromAppConfig(this.config); + verifySignature( + { + uid: query.uid as string, + expires: query.expires as string, + signature: query.signature as string, + }, + 'read', + signingCfg, + ); + + const uid = typeof query.uid === 'string' ? query.uid : ''; + const entry = await resolveV1Selector(this.stores.fsEntry, { uid }); + + // Owner-suspension guard — matches v1's /file. A signed URL stays + // valid forever by default, so a signature minted before a suspension + // would otherwise keep leaking content. + const owner = await this.stores.user.getById(entry.userId); + if ((owner as { suspended?: unknown } | null)?.suspended) { + throw new HttpError(401, 'Account suspended'); + } + + // Directory: return a signed listing of direct children. + if (entry.isDir) { + const children = await this.services.fs.listDirectory(entry.uuid); + const signedChildren = children.map((child) => ({ + ...signEntry(child, signingCfg), + path: child.path, + })); + res.json(signedChildren); + return; + } + + // File: stream bytes with Range support. + const range = + typeof req.headers.range === 'string' + ? req.headers.range + : undefined; + const download = await this.services.fs.readContent(entry, { + range, + }); + const wantsAttachment = + query.download === 'true' || + query.download === '1' || + query.download === true; + + if (download.contentType) + res.setHeader('Content-Type', download.contentType); + if (download.contentLength !== null) + res.setHeader('Content-Length', String(download.contentLength)); + if (download.contentRange) + res.setHeader('Content-Range', download.contentRange); + if (download.etag) res.setHeader('ETag', download.etag); + if (download.lastModified) + res.setHeader('Last-Modified', download.lastModified.toUTCString()); + res.setHeader( + 'Content-Disposition', + `${wantsAttachment ? 'attachment' : 'inline'}; filename="${encodeURIComponent(entry.name)}"`, + ); + res.status(range ? 206 : 200); + + download.body.on('error', (err) => { + res.destroy(err); + }); + download.body.pipe(res); + }; + + /** GET|POST /df — user storage allowance. */ + df = async (req: Request, res: Response): Promise => { + this.#requireActor(req); + const userId = this.#getActorUserId(req); + const allowance = + await this.services.fs.getUsersStorageAllowance(userId); + res.json({ + used: allowance.curr, + capacity: allowance.max, + }); + }; + + /** + * POST /open_item — resolve an entry, grant the default suggested app + * write access to it, and return a signed URL + user-app token so the + * launched app can read/write the file via its app-under-user token. + * + * Matches v1 semantics: permission is always granted as `write` — the + * underlying user's permission check still caps the effective access + * (grantUserAppPermission doesn't escalate user privileges). + */ + openItem = async (req: Request, res: Response): Promise => { + const actor = this.#requireActor(req); + const body = asRecord(req.body); + const entry = await resolveV1Selector(this.stores.fsEntry, body); + + await assertAccess( + this.services.acl, + this.services.fs, + actor, + entry.path, + 'read', + ); + + const suggested = + (await this.services.suggestedApps?.getSuggestedApps({ + name: entry.name, + path: entry.path, + })) ?? []; + + let token: string | null = null; + const defaultAppUid = + typeof suggested[0]?.uuid === 'string' + ? (suggested[0].uuid as string) + : undefined; + if (defaultAppUid) { + await this.services.permission.grantUserAppPermission( + actor, + defaultAppUid, + `fs:${entry.uuid}:write`, + {}, + { reason: 'open_item' }, + ); + token = this.services.auth.getUserAppToken(actor, defaultAppUid); + } + + const signingCfg = signingConfigFromAppConfig(this.config); + const signature = { ...signEntry(entry, signingCfg), path: entry.path }; + res.json({ + signature, + token, + suggested_apps: suggested, + }); + }; + + /** + * POST /auth/request-app-root-dir — an app-under-user requests stat on + * its own app root directory. The app must own itself. + */ + requestAppRootDir = async (req: Request, res: Response): Promise => { + const actor = this.#requireActor(req); + const body = asRecord(req.body); + const appUid = getString(body, 'app_uid'); + if (!appUid) throw new HttpError(400, '`app_uid` is required'); + + const actorApp = (actor as { app?: { uid?: string } }).app; + if (!actorApp?.uid || actorApp.uid !== appUid) { + throw new HttpError( + 403, + 'Only the app itself may request its root dir', + ); + } + const userId = this.#getActorUserId(req); + const username = (actor as { user?: { username?: string } }).user + ?.username; + if (!username) throw new HttpError(401, 'Unauthorized'); + + const rootPath = `/${username}/AppData/${appUid}`; + // Auto-create the AppData/ tree on first call. + const entry = await this.services.fs.mkdir(userId, { + path: rootPath, + createMissingParents: true, + }); + res.json(await toLegacyEntry(this.clients.event, entry)); + }; + + /** + * POST /auth/check-app-acl — check whether an app has a given mode of + * access to a subject FS entry. + */ + checkAppAcl = async (req: Request, res: Response): Promise => { + this.#requireActor(req); + const body = asRecord(req.body); + + const subjectRef = body.subject; + const appRef = body.app; + const mode = (getString(body, 'mode') ?? 'read') as + | 'see' + | 'list' + | 'read' + | 'write'; + if (!subjectRef || !appRef) + throw new HttpError(400, '`subject` and `app` are required'); + + const subject = await resolveV1Selector( + this.stores.fsEntry, + subjectRef, + ); + let app: { uid: string } | null = null; + if (typeof appRef === 'string') { + app = + (await this.stores.app.getByUid(appRef)) ?? + (await this.stores.app.getByName(appRef)); + } + if (!app) throw new HttpError(404, 'App not found'); + + // Build an actor-under-user shape for the check. + const actorForApp = { + user: (req.actor as { user?: unknown }).user, + app: { uid: (app as { uid: string }).uid }, + } as unknown as Actor; + const descriptor = { + path: subject.path, + resolveAncestors: () => + this.services.fs.getAncestorChain(subject.path), + }; + const allowed = await (this.services.acl as ACLService).check( + actorForApp, + descriptor, + mode, + ); + res.json({ allowed }); + }; + + /** + * POST /down?path=/absolute/path — session-auth'd, path-based file + * download. Keeps v1's wire contract: path query param, anti-CSRF body + * token, attachment response. No signed URL involved — /file (signature + * based) and /down (session based) are the two download paths. + */ + down = async (req: Request, res: Response): Promise => { + const actor = this.#requireActor(req); + + const rawPath = + typeof req.query.path === 'string' ? req.query.path.trim() : ''; + if (!rawPath) throw new HttpError(400, '`path` is required'); + if (rawPath === '/') + throw new HttpError(400, 'Cannot download a directory'); + + const entry = await resolveV1Selector(this.stores.fsEntry, { + path: rawPath, + }); + if (entry.isDir) + throw new HttpError(400, 'Cannot download a directory'); + + // Same ACL gate that /read uses — owners hit the is-owner implicator; + // shared-file readers get through the permission scan. + await assertAccess( + this.services.acl, + this.services.fs, + actor, + entry.path, + 'read', + ); + + const range = + typeof req.headers.range === 'string' + ? req.headers.range + : undefined; + const download = await this.services.fs.readContent(entry, { + range, + }); + + res.setHeader('Content-Type', 'application/octet-stream'); + if (download.contentLength !== null) + res.setHeader('Content-Length', String(download.contentLength)); + if (download.contentRange) + res.setHeader('Content-Range', download.contentRange); + if (download.etag) res.setHeader('ETag', download.etag); + if (download.lastModified) + res.setHeader('Last-Modified', download.lastModified.toUTCString()); + res.setHeader( + 'Content-Disposition', + `attachment; filename="${encodeURIComponent(entry.name)}"`, + ); + res.status(range ? 206 : 200); + + download.body.on('error', (err) => { + res.destroy(err); + }); + download.body.pipe(res); + }; + + // Helpers for writeFile + // ── GUI event emission ─────────────────────────────────────────── + // + // Fire-and-forget `outer.gui.item.*` events so SocketService, + // BroadcastService, WorkerDriver (hot-reload), and cache-invalidation + // listeners pick up mutations made through the legacy (bare-path) routes. + // FSController (v2-native /fs/* routes) emits these from its own handlers; + // LegacyFSController delegates to the same FSService but needs its + // own emissions because the service layer deliberately doesn't emit GUI + // events (that's a controller concern). + + async #emitGuiEvent( + eventName: + | 'outer.gui.item.added' + | 'outer.gui.item.updated' + | 'outer.gui.item.removed' + | 'outer.gui.item.moved', + entry: import('../../stores/fs/FSEntry.js').FSEntry, + extra?: Record, + ): Promise { + // GUI consumes snake_case fields (`user_id`, `parent_uid`, `is_dir`, + // …) — spreading the raw FSEntry ships camelCase, which the client + // silently ignores. Run the entry through `toLegacyEntry` first so + // the event payload matches what /stat et al. return, then overlay + // per-op extras (e.g. `old_path` for moves). + try { + const response = { + ...(await toLegacyEntry(this.clients.event, entry)), + ...extra, + from_new_service: true, + }; + await this.clients.event.emit( + eventName, + { + user_id_list: [entry.userId], + response, + }, + {}, + ); + } catch { + // Non-critical — GUI event failure must never break the HTTP response. + } + } + + async #resolveParentOfEntry(entry: { path: string; userId: number }) { + const parentPath = pathPosix.dirname(entry.path); + const parent = await resolveV1Selector(this.stores.fsEntry, { + path: parentPath, + }); + return parent; + } + + async #multipartWrite( + req: Request, + userId: number, + targetPath: string, + ): Promise<{ fsEntry: import('../../stores/fs/FSEntry.js').FSEntry }> { + // Parse the first `file` part via busboy and stream it into write. + const { Readable: NodeReadable } = await import('node:stream'); + return new Promise((resolve, reject) => { + const bb = Busboy({ headers: req.headers }); + let dispatched = false; + let writePromise: Promise | null = null; + let size = 0; + + bb.on('field', () => { + // Fields are ignored — only the file stream matters here. + }); + bb.on('file', (_fieldName, fileStream, info) => { + if (dispatched) { + fileStream.resume(); + return; + } + dispatched = true; + const passthrough = new NodeReadable({ + read() { + // no-op; data pushed from the busboy file stream. + }, + }); + fileStream.on('data', (chunk: Buffer) => { + size += chunk.length; + passthrough.push(chunk); + }); + fileStream.on('end', () => passthrough.push(null)); + fileStream.on('error', (err: Error) => + passthrough.destroy(err), + ); + + const contentType = + info && typeof info.mimeType === 'string' + ? info.mimeType + : undefined; + writePromise = this.services.fs + .write(userId, { + fileMetadata: { + path: targetPath, + size: 0, // real size accumulates as stream drains + ...(contentType ? { contentType } : {}), + overwrite: true, + }, + fileContent: passthrough, + }) + .then((response) => { + resolve({ fsEntry: response.fsEntry }); + }) + .catch(reject); + }); + bb.on('close', () => { + if (!dispatched) { + reject(new HttpError(400, 'No file uploaded')); + return; + } + if (!writePromise) { + reject(new HttpError(500, 'Write did not dispatch')); + } + // size is logged only; fsService.write handles quota/size. + void size; + }); + bb.on('error', (err) => reject(err)); + req.pipe(bb); + }); + } + + // ── Batch route ───────────────────────────────────────────────────── + // + // `/batch` interleaves multipart JSON operations with optional file + // uploads. puter-js uses it for `write`, `shortcut`, `mkdir`, `move`, + // `delete`, and `symlink` — `write` ops are paired with `file` blob + // parts (by `item_upload_id`, then fallback position) and matching + // `fileinfo` JSON. + // + // File bodies are buffered in memory per op; large uploads should go + // through the signed `/writeFile` endpoint instead, which streams. + // The wire shape (multipart/form-data) is preserved for client + // compatibility. Unknown op-types are rejected per-op. + + batch = async (req: Request, res: Response): Promise => { + this.#requireActor(req); + const userId = this.#getActorUserId(req); + const actor = req.actor!; + const username = actor.user?.username; + const contentType = + typeof req.headers['content-type'] === 'string' + ? req.headers['content-type'] + : ''; + + // Parse the request. We support both multipart/form-data (the + // canonical client shape) and JSON bodies (handy for ad-hoc + // callers / tests). + const parsed = contentType.includes('multipart/form-data') + ? await this.#parseMultipartBatch(req) + : { ops: this.#parseJsonBatch(req), files: [], fileinfos: [] }; + const { ops: operationSpecs, files, fileinfos } = parsed; + + const results: unknown[] = []; + let hasError = false; + let sequentialFileIdx = 0; + + for (const spec of operationSpecs) { + try { + const record = asRecord(spec); + const op = typeof record.op === 'string' ? record.op : ''; + let shaped: unknown; + + if (op === 'write') { + // Pair with a file part — prefer `item_upload_id` + // index (what puter-js sets), fall back to the op's + // order among write ops for safety. + const uploadIdRaw = record.item_upload_id; + let fileIdx = + typeof uploadIdRaw === 'number' + ? uploadIdRaw + : typeof uploadIdRaw === 'string' && + /^\d+$/.test(uploadIdRaw) + ? Number(uploadIdRaw) + : sequentialFileIdx; + if (fileIdx >= files.length) fileIdx = sequentialFileIdx; + sequentialFileIdx += 1; + const filePart = files[fileIdx]; + if (!filePart) { + throw new HttpError( + 400, + `write op has no paired file (item_upload_id=${uploadIdRaw})`, + ); + } + const fileInfo = fileinfos[fileIdx] ?? {}; + const name = + getString(record, 'name') ?? + (typeof fileInfo.name === 'string' + ? fileInfo.name + : undefined); + if (!name) { + throw new HttpError(400, 'write op missing `name`'); + } + const parentPath = getString(record, 'path') ?? ''; + const expandedParent = this.#expandTilde( + parentPath, + username, + ); + const targetPath = + expandedParent && expandedParent !== '/' + ? `${expandedParent.replace(/\/+$/, '')}/${name}` + : `/${name}`; + // Mirrors the per-op /write|/mkdir routes: assert write + // on the parent dir, but when the parent resolves to `/` + // fall back to the target path so the ACL check rides + // the ancestor chain instead of bouncing on root. + const writeAclPath = + expandedParent && expandedParent !== '/' + ? expandedParent.replace(/\/+$/, '') + : targetPath; + await assertAccess( + this.services.acl, + this.services.fs, + actor, + writeAclPath, + 'write', + ); + const dedupeName = + getBoolean(record, 'dedupe_name') ?? true; + const overwrite = getBoolean(record, 'overwrite') ?? false; + const createMissingParents = + getBoolean( + record, + 'create_missing_ancestors', + 'create_missing_parents', + ) ?? false; + const writeContentType = + typeof fileInfo.type === 'string' + ? fileInfo.type + : filePart.mimeType; + const response = await this.services.fs.write(userId, { + fileMetadata: { + path: targetPath, + size: filePart.content.length, + ...(writeContentType + ? { contentType: writeContentType } + : {}), + overwrite, + dedupeName, + createMissingParents, + }, + fileContent: filePart.content, + }); + await this.#emitGuiEvent( + 'outer.gui.item.added', + response.fsEntry, + ); + shaped = await toLegacyEntry( + this.clients.event, + response.fsEntry, + ); + } else if (op === 'mkdir') { + const parentPath = getString(record, 'path') ?? ''; + const name = getString(record, 'name'); + if (!name) { + throw new HttpError(400, 'mkdir op missing `name`'); + } + const expandedParent = this.#expandTilde( + parentPath, + username, + ); + const targetPath = + expandedParent && expandedParent !== '/' + ? `${expandedParent.replace(/\/+$/, '')}/${name}` + : `/${name}`; + const writeAclPath = + expandedParent && expandedParent !== '/' + ? expandedParent.replace(/\/+$/, '') + : targetPath; + await assertAccess( + this.services.acl, + this.services.fs, + actor, + writeAclPath, + 'write', + ); + const entry = await this.services.fs.mkdir(userId, { + path: targetPath, + dedupeName: getBoolean(record, 'dedupe_name') ?? true, + createMissingParents: + getBoolean( + record, + 'create_missing_ancestors', + 'create_missing_parents', + ) ?? false, + }); + await this.#emitGuiEvent('outer.gui.item.added', entry); + shaped = await toLegacyEntry(this.clients.event, entry); + } else if (op === 'shortcut') { + const parentPath = getString(record, 'path') ?? ''; + const name = getString(record, 'name'); + const shortcutToUid = + getString(record, 'shortcut_to_uid') ?? + getString(record, 'shortcut_to'); + if (!name) { + throw new HttpError(400, 'shortcut op missing `name`'); + } + if (!shortcutToUid) { + throw new HttpError( + 400, + 'shortcut op missing `shortcut_to_uid`', + ); + } + const target = await resolveV1Selector( + this.stores.fsEntry, + { uid: shortcutToUid }, + ); + const expandedParent = this.#expandTilde( + parentPath, + username, + ); + const parent = await resolveV1Selector( + this.stores.fsEntry, + { path: expandedParent || '/' }, + ); + await assertAccess( + this.services.acl, + this.services.fs, + actor, + target.path, + 'read', + ); + await assertAccess( + this.services.acl, + this.services.fs, + actor, + parent.path, + 'write', + ); + const link = await this.services.fs.mkshortcut(userId, { + parent, + name, + target, + dedupeName: getBoolean(record, 'dedupe_name') ?? true, + }); + await this.#emitGuiEvent('outer.gui.item.added', link); + shaped = await toLegacyEntry(this.clients.event, link); + } else if (op === 'move') { + const source = await resolveV1Selector( + this.stores.fsEntry, + record.source, + ); + const destinationParent = await resolveV1Selector( + this.stores.fsEntry, + record.destination, + ); + await assertAccess( + this.services.acl, + this.services.fs, + actor, + source.path, + 'write', + ); + await assertAccess( + this.services.acl, + this.services.fs, + actor, + destinationParent.path, + 'write', + ); + const moved = await this.services.fs.move(userId, { + source, + destinationParent, + newName: getString(record, 'new_name'), + overwrite: getBoolean(record, 'overwrite') ?? false, + dedupeName: getBoolean(record, 'dedupe_name') ?? false, + }); + await this.#emitGuiEvent('outer.gui.item.moved', moved, { + old_path: source.path, + }); + shaped = await toLegacyEntry(this.clients.event, moved); + } else if (op === 'delete') { + const entry = await resolveV1Selector( + this.stores.fsEntry, + getString(record, 'path') ?? record, + ); + await assertAccess( + this.services.acl, + this.services.fs, + actor, + entry.path, + 'write', + ); + const descendantsOnly = + getBoolean(record, 'descendants_only') ?? false; + await this.services.fs.remove(userId, { + entry, + recursive: getBoolean(record, 'recursive') ?? true, + descendantsOnly, + }); + await this.#emitGuiEvent('outer.gui.item.removed', entry, { + descendants_only: descendantsOnly, + }); + shaped = { ok: true, uid: entry.uuid }; + } else { + throw new HttpError(400, `Unsupported batch op: '${op}'`); + } + results.push(shaped); + } catch (err) { + hasError = true; + results.push(this.#serializeBatchError(err)); + } + } + + res.status(hasError ? 218 : 200).json({ results }); + }; + + async #parseMultipartBatch(req: Request): Promise<{ + ops: unknown[]; + files: Array<{ content: Buffer; mimeType?: string; filename?: string }>; + fileinfos: Array>; + }> { + return new Promise((resolve, reject) => { + const ops: unknown[] = []; + const files: Array<{ + content: Buffer; + mimeType?: string; + filename?: string; + }> = []; + const fileinfos: Array> = []; + let parseError: Error | null = null; + const bb = Busboy({ headers: req.headers }); + + bb.on('field', (fieldName, value) => { + try { + if (fieldName === 'operation') { + ops.push(JSON.parse(value)); + } else if (fieldName === 'fileinfo') { + const parsed = JSON.parse(value); + fileinfos.push( + parsed && typeof parsed === 'object' + ? (parsed as Record) + : {}, + ); + } + // Ignore operation_id / socket_id / misc fields — not + // needed for v2 batch semantics. + } catch (err) { + parseError = + err instanceof Error ? err : new Error(String(err)); + } + }); + + // Buffer file parts into memory so batched writes can be + // processed in any order relative to the operation specs. + // For streaming uploads use the signed `/writeFile` endpoint. + bb.on('file', (_fieldName, stream, info) => { + const chunks: Buffer[] = []; + stream.on('data', (chunk: Buffer) => chunks.push(chunk)); + stream.on('end', () => { + files.push({ + content: Buffer.concat(chunks), + mimeType: + info && typeof info.mimeType === 'string' + ? info.mimeType + : undefined, + filename: + info && typeof info.filename === 'string' + ? info.filename + : undefined, + }); + }); + stream.on('error', (err: Error) => { + parseError = err; + }); + }); + + bb.on('close', () => { + if (parseError) reject(parseError); + else resolve({ ops, files, fileinfos }); + }); + bb.on('error', (err) => reject(err)); + + req.pipe(bb); + }); + } + + #parseJsonBatch(req: Request): unknown[] { + const body = asRecord(req.body); + if (Array.isArray(body.operations)) return body.operations; + if (Array.isArray(body.ops)) return body.ops; + return []; + } + + #expandTilde(path: string, username: string | undefined): string { + if (!path) return path; + if (path !== '~' && !path.startsWith('~/')) return path; + if (!username) throw new HttpError(400, 'Unable to resolve home path'); + return `/${username}${path.slice(1)}`; + } + + #serializeBatchError(err: unknown): Record { + if (err instanceof HttpError) { + return { + error: true, + status: err.statusCode, + message: err.message, + code: err.legacyCode ?? err.code, + }; + } + if (err instanceof Error) { + return { error: true, status: 500, message: err.message }; + } + return { error: true, status: 500, message: 'Unknown batch error' }; + } + + // ── Helpers ───────────────────────────────────────────────────────── + + #parsePositiveIntegerQuery( + query: Record, + key: string, + message: string, + ): number | undefined { + const value = query[key]; + if (value === undefined || value === null || value === '') { + return undefined; + } + const parsed = Number.parseInt(String(value), 10); + if (!Number.isInteger(parsed) || parsed < 1) { + throw new HttpError(400, message); + } + return parsed; + } + + #parseNonNegativeIntegerQuery( + query: Record, + key: string, + message: string, + ): number | undefined { + const value = query[key]; + if (value === undefined || value === null || value === '') { + return undefined; + } + const parsed = Number.parseInt(String(value), 10); + if (!Number.isInteger(parsed) || parsed < 0) { + throw new HttpError(400, message); + } + return parsed; + } + + #normalizeRangeHeader(rangeHeader: string): string | undefined { + const firstRange = rangeHeader.includes(',') + ? rangeHeader.split(',')[0]?.trim() + : rangeHeader.trim(); + if (!firstRange) return undefined; + + const matches = firstRange.match(/^bytes=(\d+)-(\d*)$/); + if (!matches) return undefined; + + const [, start, end] = matches; + return end ? `bytes=${start}-${end}` : `bytes=${start}-`; + } + + #pipeLimitedLines( + source: NodeJS.ReadableStream, + res: Response, + lineCount: number, + ): void { + let remainingLines = lineCount; + let isClosed = false; + + const closeSource = () => { + if (isClosed) return; + isClosed = true; + if ('destroy' in source && typeof source.destroy === 'function') { + source.destroy(); + } + }; + + source.on('error', (err) => { + if (!isClosed) { + isClosed = true; + res.destroy(err); + } + }); + + res.on('close', () => { + closeSource(); + }); + + source.on('data', (chunk: Buffer | string) => { + if (isClosed) return; + + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + let endIndex = buffer.length; + + for (let index = 0; index < buffer.length; index++) { + if (buffer[index] !== 0x0a) continue; + remainingLines -= 1; + if (remainingLines === 0) { + endIndex = index + 1; + break; + } + } + + if (endIndex > 0) { + const canContinue = res.write(buffer.subarray(0, endIndex)); + if (!canContinue) { + source.pause(); + res.once('drain', () => { + if (!isClosed) { + source.resume(); + } + }); + } + } + + if (endIndex !== buffer.length) { + res.end(); + closeSource(); + } + }); + + source.on('end', () => { + if (!isClosed) { + isClosed = true; + res.end(); + } + }); + } + + #requireActor(req: Request) { + const actor = req.actor; + if (!actor) { + throw new HttpError(401, 'Unauthorized'); + } + return actor; + } + + #getActorUserId(req: Request): number { + const requestUser = (req as Request & { user?: { id?: unknown } }).user; + const actorUser = req.actor?.user; + const candidate = requestUser?.id ?? actorUser?.id; + if (candidate === undefined || candidate === null) { + throw new HttpError(401, 'Unauthorized'); + } + const numeric = Number(candidate); + if (Number.isNaN(numeric)) throw new HttpError(401, 'Unauthorized'); + return numeric; + } + + #isRootPathRef(body: Record): boolean { + if (body.uid !== undefined || body.uuid !== undefined) return false; + if (body.id !== undefined) return false; + if (body.parent !== undefined) return false; + const path = body.path; + if (typeof path !== 'string') return false; + return path.trim() === '/'; + } + + #parseSortBy( + body: Record, + ): 'name' | 'modified' | 'type' | 'size' | null { + const raw = getString(body, 'sort_by'); + if (!raw) return null; + const normalized = raw.toLowerCase(); + return ( + (['name', 'modified', 'type', 'size'] as const).find( + (v) => v === normalized, + ) ?? null + ); + } + + #parseSortOrder(body: Record): 'asc' | 'desc' | null { + const raw = getString(body, 'sort_order'); + if (!raw) return null; + const normalized = raw.toLowerCase(); + return (['asc', 'desc'] as const).find((v) => v === normalized) ?? null; + } + + // Reserved escape hatch for lazy-loading auxiliary route handlers. + #createLazyHandler( + key: string, + cache: RouterCache, + loader: (key: string) => Promise, + ): RequestHandler { + return async (req, res, next) => { + let handler = cache.get(key); + if (handler === undefined) { + handler = await loader(key); + cache.set(key, handler); + } + if (!handler) { + next(); + return; + } + handler(req, res, next); + }; + } +} diff --git a/src/backend/controllers/fs/costs.ts b/src/backend/controllers/fs/costs.ts new file mode 100644 index 000000000..f125a2c3c --- /dev/null +++ b/src/backend/controllers/fs/costs.ts @@ -0,0 +1,11 @@ +import { toMicroCents } from '../../services/metering/utils.js'; + +// Microcents per byte. Egress roughly matches S3 data-transfer-out +// (~$0.12/GiB); cached egress is CloudFront-backed (~$0.10/GiB). +// Ingress and deletes are currently free. +export const FS_COSTS = { + 'filesystem:ingress:bytes': 0, + 'filesystem:delete:bytes': 0, + 'filesystem:egress:bytes': toMicroCents(0.12 / 1024 / 1024 / 1024), + 'filesystem:cached-egress:bytes': toMicroCents(0.1 / 1024 / 1024 / 1024), +} as const; diff --git a/src/backend/controllers/fs/legacyFsHelpers.ts b/src/backend/controllers/fs/legacyFsHelpers.ts new file mode 100644 index 000000000..595d3b685 --- /dev/null +++ b/src/backend/controllers/fs/legacyFsHelpers.ts @@ -0,0 +1,434 @@ +import { posix as pathPosix } from 'node:path'; +import { contentType as contentTypeFromMime } from 'mime-types'; +import type { FSEntry } from '../../stores/fs/FSEntry.js'; +import type { FSEntryStore } from '../../stores/fs/FSEntryStore.js'; +import type { FSService } from '../../services/fs/FSService.js'; +import type { ACLService, AclMode } from '../../services/acl/ACLService.js'; +import type { Actor } from '../../core/actor.js'; +import { Context } from '../../core/context.js'; +import type { EventClient } from '../../clients/EventClient.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import { + resolveNode, + normalizeAbsolutePath, + joinChildPath, + expandTildePath, +} from '../../services/fs/resolveNode.js'; +import { + signFile, + type SigningConfig, + type SignedFile, +} from '../../util/fileSigning.js'; +import type { IConfig } from '../../types.js'; + +/** + * Shared helpers used by the legacy FS route shims (LegacyFSController). + * + * Legacy clients speak snake_case and expect specific response shapes — + * these helpers encapsulate that translation so the route handlers stay + * terse. + */ + +// ── Body parsing ───────────────────────────────────────────────────── + +export function asRecord(value: unknown): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) return {}; + return value as Record; +} + +export function getString( + record: Record, + ...keys: string[] +): string | undefined { + for (const key of keys) { + const value = record[key]; + if (typeof value === 'string' && value.length > 0) return value; + } + return undefined; +} + +export function getBoolean( + record: Record, + ...keys: string[] +): boolean | undefined { + for (const key of keys) { + const value = record[key]; + if (typeof value === 'boolean') return value; + if (typeof value === 'number') { + if (value === 1) return true; + if (value === 0) return false; + } + if (typeof value === 'string') { + const normalized = value.trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) return true; + if (['0', 'false', 'no', 'off'].includes(normalized)) return false; + } + } + return undefined; +} + +// Accepts either `{ path }` or `{ uid }` or `{ id }` or `{ parent, name }` +// from a legacy body field. Returns a resolved entry (throwing 404 if not +// found or 400 if no usable ref is present). +// +// `~` / `~/...` paths are expanded to `//...` using the actor's +// username, read from the ALS-backed Context. Legacy clients send tilde- +// rooted paths (e.g. `~/AppData//...`); FSController does the same +// expansion via its own `#normalizePath` helper. +export async function resolveV1Selector( + fsEntryStore: FSEntryStore, + raw: unknown, +): Promise { + const username = Context.get('actor')?.user?.username; + + // String shorthand — either an absolute path (`/a/b/c`) or a UUID. + // The legacy API accepts both interchangeably; dispatch on the leading + // character rather than guessing by regex. Anything that doesn't start + // with `/` is treated as a uid. Tilde-rooted paths are path-shaped. + if (typeof raw === 'string') { + const isPath = raw.startsWith('/') || raw.startsWith('~'); + const ref = isPath + ? { path: expandTildePath(raw, username) } + : { uid: raw }; + const entry = await resolveNode(fsEntryStore, ref, { required: true }); + if (!entry) throw new HttpError(404, `Entry not found: ${raw}`); + return entry; + } + + const record = asRecord(raw); + + // {parent, name}: "child selector" — resolve parent, then child by name. + if (record.parent !== undefined && typeof record.name === 'string') { + const parent = await resolveV1Selector(fsEntryStore, record.parent); + const childPath = joinChildPath(parent.path, record.name); + const child = await resolveNode( + fsEntryStore, + { path: childPath }, + { required: true }, + ); + if (!child) throw new HttpError(404, `Entry not found: ${childPath}`); + return child; + } + + const rawPath = typeof record.path === 'string' ? record.path : undefined; + const ref = { + path: + rawPath !== undefined + ? expandTildePath(rawPath, username) + : undefined, + uid: + typeof record.uid === 'string' + ? record.uid + : typeof record.uuid === 'string' + ? record.uuid + : undefined, + id: + typeof record.id === 'number' || typeof record.id === 'string' + ? record.id + : undefined, + }; + const entry = await resolveNode(fsEntryStore, ref, { required: true }); + if (!entry) throw new HttpError(404, 'Entry not found'); + return entry; +} + +// ── ACL ────────────────────────────────────────────────────────────── + +export async function assertAccess( + aclService: ACLService, + fsService: FSService, + actor: Actor, + path: string, + mode: AclMode, +): Promise { + let ancestors: Promise> | null = null; + const descriptor = { + path, + resolveAncestors() { + if (!ancestors) { + ancestors = fsService.getAncestorChain(path); + } + return ancestors; + }, + }; + const allowed = await aclService.check(actor, descriptor, mode); + if (allowed) return; + const safe = (await aclService.getSafeAclError( + actor, + descriptor, + mode, + )) as { + status?: unknown; + message?: unknown; + fields?: { code?: unknown }; + }; + const status = Number(safe?.status); + const message = + typeof safe?.message === 'string' && safe.message.length > 0 + ? safe.message + : 'Access denied'; + const code = + typeof safe?.fields?.code === 'string' ? safe.fields.code : undefined; + const legacyCode = code === 'forbidden' ? 'access_denied' : code; + + // App-under-user actors see denials as 404 "subject_does_not_exist" + // so existence of a sibling user's / other-app's files isn't leaked + // through the error code. User-actor denials keep the real 403. + const isAppActor = Boolean((actor as { app?: unknown })?.app); + if (isAppActor) { + throw new HttpError(404, `Entry not found: path=${path}`, { + legacyCode: 'subject_does_not_exist', + }); + } + + if (status === 404) { + throw new HttpError(404, message, { + ...(legacyCode ? { legacyCode } : {}), + }); + } + throw new HttpError(403, message, { + legacyCode: legacyCode ?? 'access_denied', + }); +} + +// ── Response shaping ──────────────────────────────────────────────── + +type AppRowLookup = { + getByIds: (ids: number[]) => Promise>>; +}; + +const toIntBool = (v: unknown): number => (v ? 1 : 0); + +/** + * Convert an AppStore-normalized app row into the v1 `associated_app` shape + * embedded in legacy FS entries. Booleans round-trip back to integers (0/1) + * because the v1 wire contract emits them that way and existing clients key + * off it. Other columns pass through as-is — `metadata` is already parsed. + */ +function mapAppForLegacyAssociatedApp( + app: Record, +): Record { + return { + id: app.id, + uid: app.uid, + owner_user_id: app.owner_user_id, + icon: app.icon, + name: app.name, + title: app.title, + description: app.description, + godmode: toIntBool(app.godmode), + maximize_on_start: toIntBool(app.maximize_on_start), + index_url: app.index_url, + approved_for_listing: toIntBool(app.approved_for_listing), + approved_for_opening_items: toIntBool(app.approved_for_opening_items), + approved_for_incentive_program: toIntBool( + app.approved_for_incentive_program, + ), + timestamp: app.timestamp ?? null, + last_review: app.last_review ?? null, + tags: app.tags ?? null, + app_owner: app.app_owner ?? null, + background: toIntBool(app.background), + metadata: app.metadata ?? null, + protected: toIntBool(app.protected), + is_private: toIntBool(app.is_private), + }; +} + +/** + * Batch-load `associated_app` payloads for a set of entries. Dedupes app ids + * across the input, hands them to `AppStore.getByIds` (one pipelined Redis + * MGET + a single `id IN (…)` query for any cache misses), and returns a map + * keyed by app id holding the v1-shaped embed. Callers pass the result to + * `toLegacyEntry` via `opts.appsById` so each entry hydrates without a + * second round-trip. + * + * Empty input short-circuits — readdir on a directory of plain files makes + * zero extra calls. + */ +export async function loadLegacyAssociatedApps( + appStore: AppRowLookup, + entries: FSEntry[], +): Promise>> { + const ids = [ + ...new Set( + entries + .map((e) => e.associatedAppId) + .filter((id): id is number => typeof id === 'number'), + ), + ]; + const out = new Map>(); + if (ids.length === 0) return out; + const apps = await appStore.getByIds(ids); + for (const [id, app] of apps) { + out.set(id, mapAppForLegacyAssociatedApp(app)); + } + return out; +} + +/** + * Produce the snake_case entry shape legacy clients expect. If `thumbnail` + * is set, asks the thumbnail extension (via `thumbnail.read` event) to swap + * an S3 URL for a signed one. Pass `fsEntryStore`/`userStore` to hydrate + * `is_empty` (directories) and `owner` — both are required fields per + * the legacy stat contract but need extra DB lookups. Pass `appsById` + * (built via `loadLegacyAssociatedApps`) to populate `associated_app`. + */ +export async function toLegacyEntry( + eventClient: EventClient | undefined, + entry: FSEntry, + opts: { + fsEntryStore?: FSEntryStore; + userStore?: { + getById: (id: number) => Promise | null>; + }; + appsById?: Map>; + } = {}, +): Promise> { + const dirname = pathPosix.dirname(entry.path); + // v1 contract: `type` is a MIME content-type (e.g. "image/png; charset=utf-8") + // for files, or "folder" for directories. The GUI's icon lookup does + // `type.startsWith('image/')` etc., so a bare extension breaks every + // banner that falls through the name-extension ladder in item_icon.js. + const mimeType = entry.isDir + ? 'folder' + : contentTypeFromMime(entry.name) || null; + + const pathComponents = entry.path.split('/'); + const appdata_app = + pathComponents[2] === 'AppData' ? pathComponents[3] : undefined; + + const response: Record = { + id: entry.uuid, + uid: entry.uuid, + uuid: entry.uuid, + parent_id: entry.parentUid, + parent_uid: entry.parentUid, + path: entry.path, + dirname, + dirpath: dirname, + name: entry.name, + is_dir: Boolean(entry.isDir), + is_shortcut: entry.isShortcut ? 1 : 0, + shortcut_to: entry.shortcutTo, + is_symlink: entry.isSymlink ? 1 : 0, + symlink_path: entry.symlinkPath, + type: mimeType, + writable: true, + is_public: entry.isPublic, + thumbnail: entry.thumbnail, + immutable: Boolean(entry.immutable), + metadata: entry.metadata, + modified: entry.modified, + created: entry.created, + accessed: entry.accessed, + size: entry.size, + layout: entry.layout, + subdomains: entry.subdomains, + workers: entry.workers, + has_website: entry.hasWebsite ?? entry.subdomains.length > 0, + suggested_apps: entry.suggestedApps, + associated_app: + entry.associatedAppId !== null && opts.appsById + ? (opts.appsById.get(entry.associatedAppId) ?? null) + : null, + appdata_app, + }; + + // `is_empty` — only meaningful for directories. Single-row probe so we + // don't pay for listing every child. + if (entry.isDir && opts.fsEntryStore) { + try { + const children = await opts.fsEntryStore.listChildren(entry.uuid, { + limit: 1, + }); + response.is_empty = children.length === 0; + } catch { + response.is_empty = false; + } + } else if (!entry.isDir) { + response.is_empty = false; + } + + // `owner` — username-only. Matches the legacy safe-entry contract. + if (opts.userStore) { + try { + const owner = await opts.userStore.getById(entry.userId); + if (owner && typeof owner.username === 'string') { + response.owner = { username: owner.username }; + } + } catch { + /* best-effort */ + } + } + + // Let the thumbnail extension swap an s3:// key for a signed URL. + if ( + typeof response.thumbnail === 'string' && + (response.thumbnail as string).length > 0 && + eventClient + ) { + const thumbnailEntry = { + uuid: entry.uuid, + thumbnail: response.thumbnail as string, + }; + try { + // emitAndWait — listener mutates `thumbnail` on the payload; + // plain `emit` is fire-and-forget and would drop the rewrite. + await eventClient.emitAndWait('thumbnail.read', thumbnailEntry, {}); + } catch { + // ignore — non-critical. + } + response.thumbnail = + typeof thumbnailEntry.thumbnail === 'string' && + thumbnailEntry.thumbnail.length > 0 + ? thumbnailEntry.thumbnail + : null; + } + + return response; +} + +export { normalizeAbsolutePath }; + +// ── Signing ───────────────────────────────────────────────────────── + +/** + * Pull the signing config off the app config. Throws if either value is + * missing — these are required for signed URL routes to function. + */ +export function signingConfigFromAppConfig(config: IConfig): SigningConfig { + const secret = config.url_signature_secret; + const apiBaseUrl = config.api_base_url; + if (typeof secret !== 'string' || secret.length === 0) { + throw new HttpError( + 500, + 'Server misconfiguration: url_signature_secret not set', + ); + } + if (typeof apiBaseUrl !== 'string' || apiBaseUrl.length === 0) { + throw new HttpError( + 500, + 'Server misconfiguration: api_base_url not set', + ); + } + return { secret, apiBaseUrl }; +} + +/** + * Convenience wrapper: turn an FSEntry into a signed-file response object. + */ +export function signEntry( + entry: { + uuid: string; + name: string; + isDir: boolean; + size: number | null; + accessed: number | null; + modified: number; + created: number | null; + }, + config: SigningConfig, +): SignedFile { + return signFile(entry as Parameters[0], config); +} diff --git a/extensions/fsv2/src/types/requests.ts b/src/backend/controllers/fs/requestTypes.ts similarity index 89% rename from extensions/fsv2/src/types/requests.ts rename to src/backend/controllers/fs/requestTypes.ts index 59f2ef406..4f4c381f2 100644 --- a/extensions/fsv2/src/types/requests.ts +++ b/src/backend/controllers/fs/requestTypes.ts @@ -1,4 +1,4 @@ -import { FSEntry, FSEntryWriteInput } from './FSEntry.js'; +import { FSEntry, FSEntryWriteInput } from '../../stores/fs/FSEntry.js'; import type { Readable } from 'node:stream'; export type UploadMode = 'single' | 'multipart'; @@ -89,7 +89,16 @@ export interface BinaryPayload { export interface WriteRequest { fileMetadata: FSEntryWriteInput; - fileContent: Buffer | Readable | ReadableStream | string | Blob | File | Uint8Array | ArrayBuffer | BinaryPayload; + fileContent: + | Buffer + | Readable + | ReadableStream + | string + | Blob + | File + | Uint8Array + | ArrayBuffer + | BinaryPayload; encoding?: 'utf8' | 'base64' | 'ascii' | 'latin1' | 'utf16le' | 'hex'; thumbnailData?: string; guiMetadata?: WriteGuiMetadata; diff --git a/extensions/fsv2/src/controllers/types.ts b/src/backend/controllers/fs/types.ts similarity index 88% rename from extensions/fsv2/src/controllers/types.ts rename to src/backend/controllers/fs/types.ts index c7a5a3dbc..4686e9636 100644 --- a/extensions/fsv2/src/controllers/types.ts +++ b/src/backend/controllers/fs/types.ts @@ -1,6 +1,6 @@ import type { Readable } from 'node:stream'; -import type { FSEntryWriteInput } from '../types/FSEntry.js'; -import type { WriteGuiMetadata } from '../types/requests.js'; +import type { FSEntryWriteInput } from '../../stores/fs/FSEntry.js'; +import type { WriteGuiMetadata } from './requestTypes.js'; export interface AbortWriteRequest { uploadId: string; diff --git a/src/backend/controllers/homepage/HomepageController.ts b/src/backend/controllers/homepage/HomepageController.ts new file mode 100644 index 000000000..c6817c992 --- /dev/null +++ b/src/backend/controllers/homepage/HomepageController.ts @@ -0,0 +1,145 @@ +import express from 'express'; +import path from 'node:path'; +import { PuterController } from '../types.js'; +import type { PuterRouter } from '../../core/http/PuterRouter'; +import type { + PuterHomepageService, + PageMeta, + LaunchOptions, +} from '../../services/homepage/PuterHomepageService'; + +/** + * Routes that render the Puter GUI shell, plus a catch-all static fallback + * under `/src` for non-dist/src paths (images, fonts, lib + * files referenced from the shell). + * + * All root-subdomain-only. Registered last in the controller list so the + * static catch-all doesn't shadow specific API routes. + */ +export class HomepageController extends PuterController { + registerRoutes(router: PuterRouter) { + const homepage = this.services + .homepage as unknown as PuterHomepageService; + if (!homepage) return; + + const defaultMeta = (req: express.Request): PageMeta => ({ + title: String(this.config.gui_params?.title ?? 'Puter'), + description: String( + this.config.gui_params?.short_description ?? '', + ), + short_description: String( + this.config.gui_params?.short_description ?? '', + ), + company: 'Puter Technologies Inc.', + canonical_url: `${req.protocol}://${this.config.domain ?? req.hostname}${req.path}`, + social_media_image: String( + this.config.gui_params?.social_media_image ?? '', + ), + }); + + const sendShell = async ( + req: express.Request, + res: express.Response, + metaOverrides: Partial = {}, + launch: LaunchOptions = {}, + ) => { + const meta = { ...defaultMeta(req), ...metaOverrides }; + const actor = + ( + req as express.Request & { + actor?: Parameters[0]['actor']; + } + ).actor ?? null; + await homepage.send({ req, res, actor }, meta, launch); + }; + + // ── Root + path-aliased shell routes ──────────────────────── + + router.get('/', {}, (req, res) => sendShell(req, res)); + + router.get('/settings', {}, (req, res) => sendShell(req, res)); + router.get('/settings/*splat', {}, (req, res) => sendShell(req, res)); + + router.get('/dashboard', {}, (req, res) => sendShell(req, res)); + router.get('/dashboard/', {}, (req, res) => sendShell(req, res)); + + router.get('/action/*splat', {}, (req, res) => sendShell(req, res)); + + router.get('/@:username', {}, (req, res) => sendShell(req, res)); + + // ── /app/:name ─ app metadata baked into the shell ────────── + + router.get('/app/:name', {}, async (req, res) => { + const name = String(req.params.name ?? ''); + const app = name ? await this.stores.app.getByName(name) : null; + + if (app) { + const metadata = + (typeof app.metadata === 'string' + ? safeJsonParse(app.metadata) + : (app.metadata as Record | null)) ?? + {}; + await sendShell(req, res, { + title: String(app.title ?? name), + description: String(app.description ?? ''), + short_description: String(app.description ?? ''), + icon: typeof app.icon === 'string' ? app.icon : undefined, + social_media_image: + typeof metadata.social_image === 'string' + ? metadata.social_image + : undefined, + app: app as Record, + }); + return; + } + + // App not found — return 404 but still render the shell so the + // client-side router can decide what to display. + res.status(404); + await sendShell(req, res, { + title: name + ? name.charAt(0).toUpperCase() + name.slice(1) + : 'Puter', + }); + }); + + // ── /show/* ─ launch explorer with the requested file path ── + + router.get('/show/*splat', {}, (req, res) => { + const filePath = req.path.slice('/show'.length); + const launch: LaunchOptions = { + on_initialized: [ + { + $: 'window-call', + fn_name: 'launch_app', + args: [{ name: 'explorer', path: filePath }], + }, + ], + }; + return sendShell(req, res, {}, launch); + }); + + // ── Fallback static mount ─────────────────────────────────── + // Serves lingering GUI assets (images, fonts, lib files, etc.) + // out of /src. Falls through to the 404 handler + // when the file doesn't exist. + if (this.config.gui_assets_root) { + router.use( + '/', + { subdomain: '' }, + express.static(path.join(this.config.gui_assets_root, 'src')), + ); + } + } +} + +const safeJsonParse = (s: string): Record | null => { + try { + const parsed = JSON.parse(s); + return parsed && typeof parsed === 'object' + ? (parsed as Record) + : null; + } catch { + return null; + } +}; diff --git a/src/backend/controllers/hosting/HostingController.js b/src/backend/controllers/hosting/HostingController.js new file mode 100644 index 000000000..fa5169bd7 --- /dev/null +++ b/src/backend/controllers/hosting/HostingController.js @@ -0,0 +1,64 @@ +import { HttpError } from '../../core/http/HttpError.js'; +import { PuterController } from '../types.js'; + +/** + * Site hosting endpoints. Listing and create/update are not exposed as + * controller routes — clients use the `puter-subdomains` driver + * (select / create / update / read) so they get the v1-shape with + * uuids and nested objects (no raw mysql ids). Only `/delete-site` + * lives here because v1 also exposed it as a top-level POST. + */ +export class HostingController extends PuterController { + constructor(config, clients, stores, services) { + super(config, clients, stores, services); + } + + get subdomainStore() { + return this.stores.subdomain; + } + + registerRoutes(router) { + // ── Delete site ───────────────────────────────────────────── + + router.post( + '/delete-site', + { + subdomain: 'api', + requireUserActor: true, + requireVerified: true, + }, + async (req, res) => { + const { site_uuid } = req.body ?? {}; + if (!site_uuid || typeof site_uuid !== 'string') { + throw new HttpError(400, 'Missing or invalid `site_uuid`'); + } + + const row = await this.subdomainStore.getByUuid(site_uuid, { + userId: req.actor.user.id, + }); + if (!row) { + throw new HttpError( + 404, + 'Site not found or not owned by you', + ); + } + if (row.protected) { + throw new HttpError( + 403, + 'Cannot delete a protected subdomain', + ); + } + + await this.subdomainStore.deleteByUuid(site_uuid, { + userId: req.actor.user.id, + }); + + res.json({}); + }, + ); + } + + onServerStart() {} + onServerPrepareShutdown() {} + onServerShutdown() {} +} diff --git a/src/backend/controllers/index.ts b/src/backend/controllers/index.ts new file mode 100644 index 000000000..22905a53c --- /dev/null +++ b/src/backend/controllers/index.ts @@ -0,0 +1,43 @@ +import { AppController } from './apps/AppController.js'; +import { AuthController } from './auth/AuthController.js'; +import { BroadcastController } from './broadcast/BroadcastController.js'; +import { DesktopController } from './desktop/DesktopController.js'; +import { DriverController } from './drivers/DriverController.js'; +import { FSController } from './fs/FSController.js'; +import { HomepageController } from './homepage/HomepageController.js'; +import { HostingController } from './hosting/HostingController.js'; +import { LegacyFSController } from './fs/LegacyFSController.js'; +import { NotificationController } from './notification/NotificationController.js'; +import { OIDCController } from './oidc/OIDCController.js'; +import { PuterAIController } from './puterai/PuterAIController.js'; +import { ShareController } from './share/ShareController.js'; +import { StaticAssetsController } from './static/StaticAssetsController.js'; +import { StaticPagesController } from './static/StaticPagesController.js'; +import { SystemController } from './system/SystemController.js'; +import { WebDAVController } from './webdav/WebDAVController.js'; +import { WispController } from './wisp/WispController.js'; +import type { IPuterControllerRegistry } from './types.js'; +import { PeerController } from './peer/PeerController.js'; + +export const puterControllers = { + staticAssets: StaticAssetsController, + staticPages: StaticPagesController, + auth: AuthController, + apps: AppController, + desktop: DesktopController, + hosting: HostingController, + system: SystemController, + fs: FSController, + legacyFs: LegacyFSController, + puterAi: PuterAIController, + drivers: DriverController, + broadcast: BroadcastController, + notification: NotificationController, + share: ShareController, + webdav: WebDAVController, + oidc: OIDCController, + wisp: WispController, + peer: PeerController, + // Last so its catch-all static fallback doesn't shadow earlier routes. + homepage: HomepageController, +} satisfies IPuterControllerRegistry; diff --git a/src/backend/controllers/notification/NotificationController.ts b/src/backend/controllers/notification/NotificationController.ts new file mode 100644 index 000000000..37d797c02 --- /dev/null +++ b/src/backend/controllers/notification/NotificationController.ts @@ -0,0 +1,84 @@ +import type { Request, Response } from 'express'; +import { Controller, Post } from '../../core/http/decorators.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import type { NotificationService } from '../../services/notification/NotificationService.js'; +import { PuterController } from '../types.js'; + +/** + * GUI-facing notification endpoints. These supplement the + * `puter-notifications` driver (which handles CRUD via `/drivers/call`) + * with two small mutation routes that the puter desktop client calls + * directly. + * + * Both routes emit `outer.gui.notif.ack` via the NotificationService + * so other open tabs for the same user see the state change immediately. + */ +@Controller('/notif') +export class NotificationController extends PuterController { + /** + * POST /notif/mark-ack — user dismissed a notification. + * Sets `acknowledged` timestamp; pushes ack event to sockets. + */ + @Post('/mark-ack', { subdomain: 'api', requireAuth: true }) + async markAck(req: Request, res: Response): Promise { + const uid = req.body?.uid; + if (typeof uid !== 'string' || uid.length === 0) { + throw new HttpError(400, '`uid` must be a non-empty string'); + } + const userId = req.actor?.user?.id; + if (!userId) throw new HttpError(401, 'Unauthorized'); + + const notifService = this.services.notification as unknown as + | NotificationService + | undefined; + if (notifService?.markAcknowledged) { + await notifService.markAcknowledged(uid, userId); + } else { + // Fallback: direct store call if service isn't wired + await ( + this.stores as Record as { + notification: { + markAcknowledged: ( + uid: string, + userId: number, + ) => Promise; + }; + } + ).notification.markAcknowledged(uid, userId); + } + res.json({}); + } + + /** + * POST /notif/mark-read — user saw a notification. + * Sets `shown` timestamp; pushes ack event to sockets. + */ + @Post('/mark-read', { subdomain: 'api', requireAuth: true }) + async markRead(req: Request, res: Response): Promise { + const uid = req.body?.uid; + if (typeof uid !== 'string' || uid.length === 0) { + throw new HttpError(400, '`uid` must be a non-empty string'); + } + const userId = req.actor?.user?.id; + if (!userId) throw new HttpError(401, 'Unauthorized'); + + const notifService = this.services.notification as unknown as + | NotificationService + | undefined; + if (notifService?.markShown) { + await notifService.markShown(uid, userId); + } else { + await ( + this.stores as Record as { + notification: { + markShown: ( + uid: string, + userId: number, + ) => Promise; + }; + } + ).notification.markShown(uid, userId); + } + res.json({}); + } +} diff --git a/src/backend/controllers/oidc/OIDCController.ts b/src/backend/controllers/oidc/OIDCController.ts new file mode 100644 index 000000000..782c4fc6c --- /dev/null +++ b/src/backend/controllers/oidc/OIDCController.ts @@ -0,0 +1,559 @@ +import type { Request, Response } from 'express'; +import { HttpError } from '../../core/http/HttpError.js'; +import type { PuterRouter } from '../../core/http/PuterRouter.js'; +import { PuterController } from '../types.js'; + +const REVALIDATION_COOKIE_NAME = 'puter_revalidation'; +const REVALIDATION_EXPIRY_SEC = 300; + +const OIDC_ERROR_REDIRECT_MAP: Record> = { + login: { account_not_found: 'signup', other: 'login' }, + signup: { account_already_exists: 'login', other: 'signup' }, +}; + +const ALLOWED_ERRORS = ['account_suspended', 'unauthorized'] as const; + +function buildErrorRedirectUrl( + origin: string, + sourceFlow: string, + errorCondition: string, + message: (typeof ALLOWED_ERRORS)[number], + stateDecoded?: Record, +): string { + const targetFlow = + OIDC_ERROR_REDIRECT_MAP[sourceFlow]?.[errorCondition] ?? sourceFlow; + const base = origin.replace(/\/$/, '') || '/'; + + if (stateDecoded?.embedded_in_popup && stateDecoded?.msg_id != null) { + const params = new URLSearchParams({ + embedded_in_popup: 'true', + msg_id: String(stateDecoded.msg_id), + auth_error: '1', + message: ALLOWED_ERRORS.includes(message) + ? message + : 'unauthorized', + action: targetFlow, + }); + if (stateDecoded?.opener_origin) { + params.set('opener_origin', String(stateDecoded.opener_origin)); + } + return `${base}/?${params.toString()}`; + } + + const params = new URLSearchParams({ + action: targetFlow, + auth_error: '1', + message: ALLOWED_ERRORS.includes(message) ? message : 'unauthorized', + }); + return `${base}/?${params.toString()}`; +} + +function appendQueryParam(url: string, key: string, value: string): string { + const sep = url.includes('?') ? '&' : '?'; + return `${url}${sep}${encodeURIComponent(key)}=${encodeURIComponent(value)}`; +} + +/** + * True iff `target` parses as a URL whose origin equals `origin`. Used to + * clamp OIDC redirect targets — `startsWith` would accept + * `https://puter.com.evil.com` against `https://puter.com`. + */ +function isSameOrigin(target: string, origin: string): boolean { + if (!origin) return true; + try { + return new URL(target).origin === new URL(origin).origin; + } catch { + return false; + } +} + +/** + * OIDC controller — provider listing, auth start, callbacks for + * login/signup/revalidate, and revalidate-done landing page. + */ +export class OIDCController extends PuterController { + registerRoutes(router: PuterRouter): void { + // ── GET /auth/oidc/providers ──────────────────────────────── + // Public — list enabled provider IDs for the frontend. + + router.get( + '/auth/oidc/providers', + { subdomain: 'api' }, + async (_req: Request, res: Response) => { + const providers = + await this.services.oidc.getEnabledProviderIds(); + res.json({ providers }); + }, + ); + + // ── GET /auth/oidc/:provider/start ────────────────────────── + // Redirect user to IdP authorization endpoint. + + router.get( + '/auth/oidc/:provider/start', + { + subdomain: '', + rateLimit: { scope: 'oidc-general', limit: 30, window: 60_000 }, + }, + async (req: Request, res: Response) => { + const provider = String(req.params.provider); + const cfg = + await this.services.oidc.getProviderConfig(provider); + if (!cfg) throw new HttpError(404, 'Provider not configured.'); + + const flow = String( + Array.isArray(req.query.flow) + ? req.query.flow[0] + : (req.query.flow ?? 'login'), + ); + const origin = (this.config.origin ?? '').replace(/\/$/, ''); + + const flowRedirects: Record = { + login: origin || '/', + signup: origin || '/', + revalidate: `${origin}/auth/revalidate-done`, + }; + + let appRedirectUri = flowRedirects[flow] ?? (origin || '/'); + + // Popup support + const rawPopup = Array.isArray(req.query.embedded_in_popup) + ? req.query.embedded_in_popup[0] + : req.query.embedded_in_popup; + const embeddedInPopup = rawPopup === 'true' || rawPopup === '1'; + const rawMsgId = Array.isArray(req.query.msg_id) + ? req.query.msg_id[0] + : req.query.msg_id; + const msgId = + rawMsgId != null && rawMsgId !== '' + ? String(rawMsgId) + : null; + const rawOpener = Array.isArray(req.query.opener_origin) + ? req.query.opener_origin[0] + : req.query.opener_origin; + const openerOrigin = + rawOpener != null && rawOpener !== '' + ? String(rawOpener) + : null; + + if (embeddedInPopup && msgId) { + appRedirectUri = `${origin}/action/sign-in?embedded_in_popup=true&msg_id=${encodeURIComponent(msgId)}`; + if (openerOrigin) { + appRedirectUri += `&opener_origin=${encodeURIComponent(openerOrigin)}`; + } + } + + const statePayload: Record = { + provider, + redirect_uri: appRedirectUri, + }; + if (embeddedInPopup && msgId) { + statePayload.embedded_in_popup = true; + statePayload.msg_id = msgId; + if (openerOrigin) statePayload.opener_origin = openerOrigin; + } + if (flow === 'revalidate') { + const rawUserUuid = Array.isArray(req.query.user_uuid) + ? req.query.user_uuid[0] + : req.query.user_uuid; + if (typeof rawUserUuid !== 'string' || !rawUserUuid) + throw new HttpError( + 400, + 'user_uuid required for revalidate flow.', + ); + statePayload.user_uuid = rawUserUuid; + statePayload.flow = 'revalidate'; + } + + const state = this.services.oidc.signState(statePayload); + const url = await this.services.oidc.getAuthorizationUrl( + provider, + state, + flow, + ); + if (!url) + throw new HttpError( + 500, + 'Could not build authorization URL.', + ); + + res.redirect(302, url); + }, + ); + + // ── GET /auth/oidc/callback/login ─────────────────────────── + + router.get( + '/auth/oidc/callback/login', + { + subdomain: '', + rateLimit: { scope: 'oidc-general', limit: 30, window: 60_000 }, + }, + async (req: Request, res: Response) => { + const origin = this.config.origin ?? ''; + const result = await this.#processCallback(req, 'login'); + if ('error' in result) { + console.warn(`OIDC login callback error: ${result.error}`); + return res.redirect( + 302, + buildErrorRedirectUrl( + origin, + 'login', + 'other', + result.error, + ), + ); + } + + const { provider, userinfo, stateDecoded } = result; + const resolved = await this.#resolveOrCreateOIDCUser( + provider, + userinfo, + ); + if ('error' in resolved) { + console.warn( + `OIDC login user resolution error: ${resolved.error}`, + ); + return res.redirect( + 302, + buildErrorRedirectUrl( + origin, + 'login', + 'other', + resolved.error, + stateDecoded, + ), + ); + } + const user = resolved.user; + + if (user.suspended) { + console.warn( + `Suspended user tried to login via oidc: ${user.username}`, + ); + return res.redirect( + 302, + buildErrorRedirectUrl( + origin, + 'login', + 'other', + 'This account is suspended.', + stateDecoded, + ), + ); + } + + await this.#finishLogin(res, user, stateDecoded); + }, + ); + + // ── GET /auth/oidc/callback/signup ────────────────────────── + + router.get( + '/auth/oidc/callback/signup', + { + subdomain: '', + rateLimit: { scope: 'oidc-general', limit: 30, window: 60_000 }, + }, + async (req: Request, res: Response) => { + const origin = this.config.origin ?? ''; + const result = await this.#processCallback(req, 'signup'); + if ('error' in result) { + return res.redirect( + 302, + buildErrorRedirectUrl( + origin, + 'signup', + 'other', + 'unauthorized', + ), + ); + } + + const { provider, userinfo, stateDecoded } = result; + const resolved = await this.#resolveOrCreateOIDCUser( + provider, + userinfo, + ); + if ('error' in resolved) { + return res.redirect( + 302, + buildErrorRedirectUrl( + origin, + 'signup', + 'other', + 'unauthorized', + stateDecoded, + ), + ); + } + const user = resolved.user; + + if (user.suspended) { + return res.redirect( + 302, + buildErrorRedirectUrl( + origin, + 'signup', + 'other', + 'account_suspended', + stateDecoded, + ), + ); + } + + // If we landed on an existing account (either via the + // provider_sub or via email match), signal the GUI so it can + // render a "signed in" flow rather than "account created". + const extra = + resolved.origin === 'created' + ? undefined + : { oidc_switched: 'login' }; + await this.#finishLogin(res, user, stateDecoded, extra); + }, + ); + + // ── GET /auth/oidc/callback/revalidate ────────────────────── + + router.get( + '/auth/oidc/callback/revalidate', + { + subdomain: '', + rateLimit: { scope: 'oidc-general', limit: 30, window: 60_000 }, + }, + async (req: Request, res: Response): Promise => { + const result = await this.#processCallback(req, 'revalidate'); + if ('error' in result) { + res.status(400).send(result.error); + return; + } + + const { provider, userinfo, stateDecoded } = result; + if ( + stateDecoded.flow !== 'revalidate' || + typeof stateDecoded.user_uuid !== 'string' || + stateDecoded.user_uuid.length === 0 + ) { + res.status(400).send('Invalid revalidate state.'); + return; + } + + const user = await this.services.oidc.findUserByProviderSub( + provider, + userinfo.sub, + ); + if (!user) { + res.status(400).send('No account found.'); + return; + } + if (user.uuid !== stateDecoded.user_uuid) { + res.status(403).send( + 'Wrong account. Sign in with the account linked to this session.', + ); + return; + } + + const token = this.services.oidc.signRevalidation(user.uuid); + res.cookie(REVALIDATION_COOKIE_NAME, token, { + sameSite: 'lax', + secure: true, + httpOnly: true, + maxAge: REVALIDATION_EXPIRY_SEC * 1000, + path: '/', + }); + + const origin = (this.config.origin ?? '').replace(/\/$/, ''); + const requested = + (stateDecoded.redirect_uri as string) || + `${origin}/auth/revalidate-done`; + const target = isSameOrigin(requested, origin) + ? requested + : `${origin}/auth/revalidate-done`; + res.redirect(302, target); + }, + ); + + // ── GET /auth/revalidate-done ─────────────────────────────── + // Landing page after revalidation; posts to opener for popup flow. + + router.get( + '/auth/revalidate-done', + { subdomain: '' }, + (_req: Request, res: Response) => { + const origin = this.config.origin ?? ''; + res.set('Content-Type', 'text/html; charset=utf-8'); + res.send(`Re-validated

Re-validated. Closing…

`); + }, + ); + } + + // ── Shared helpers ────────────────────────────────────────────── + + /** + * Resolve an OIDC callback to a Puter user. In order: + * 1. Existing link on (provider, sub) → that user. + * 2. Email matches an existing account whose email is CONFIRMED → + * link (provider, sub) to that user. + * 3. Email matches an account whose email is UNCONFIRMED → refuse. + * We don't know who owns an unconfirmed address, so linking would + * let whoever controls the OIDC identity hijack a pending signup. + * 4. Otherwise create a new user and link. + * + * Step 2 also requires `email_verified !== false` on the OIDC side, + * otherwise a malicious IdP could claim someone else's email. + * + * The email-match path does NOT touch the existing user's password, so + * password login keeps working. + */ + async #resolveOrCreateOIDCUser( + provider: string, + userinfo: { sub: string; email?: unknown; [k: string]: unknown }, + ): Promise< + | { error: string } + | { + user: import('../../stores/user/UserStore.js').UserRow; + origin: 'linked-sub' | 'linked-email' | 'created'; + } + > { + // 1. Existing provider/sub link. + const linked = await this.services.oidc.findUserByProviderSub( + provider, + userinfo.sub, + ); + if (linked) return { user: linked, origin: 'linked-sub' }; + + // 2/3. Email match branch. + const claimedEmail = + typeof userinfo.email === 'string' ? userinfo.email : null; + if (claimedEmail) { + const byEmail = + await this.services.oidc.findUserByEmail(claimedEmail); + if (byEmail) { + if (!byEmail.email_confirmed) { + return { + error: 'An account with this email exists but the email is not yet confirmed. Please sign in with your password to confirm it first.', + }; + } + const outcome = await this.services.oidc.linkProviderToUser( + byEmail.id, + provider, + userinfo as { sub: string; email?: string }, + ); + if (!outcome.success) { + return { + error: outcome.error ?? 'Failed to link provider.', + }; + } + return { user: byEmail, origin: 'linked-email' }; + } + } + + // 3. Fresh account. + const outcome = await this.services.oidc.createUserFromOIDC( + provider, + userinfo as { sub: string; email?: string }, + ); + if (!outcome.success || !outcome.user) { + return { error: outcome.error ?? 'Account creation failed.' }; + } + return { user: outcome.user, origin: 'created' }; + } + + async #processCallback( + req: Request, + flow: string, + ): Promise< + | { error: string } + | { + provider: string; + userinfo: { sub: string; [k: string]: unknown }; + stateDecoded: Record; + } + > { + const code = String( + Array.isArray(req.query.code) + ? req.query.code[0] + : (req.query.code ?? ''), + ); + const state = String( + Array.isArray(req.query.state) + ? req.query.state[0] + : (req.query.state ?? ''), + ); + if (!code || !state) return { error: 'Missing code or state.' }; + + const stateDecoded = this.services.oidc.verifyState(state); + if (!stateDecoded || !stateDecoded.provider) + return { error: 'Invalid or expired state.' }; + + const provider = String(stateDecoded.provider); + const callbackUrl = this.services.oidc.getCallbackUrl(flow); + if (!callbackUrl) return { error: 'Invalid flow.' }; + + const tokens = await this.services.oidc.exchangeCodeForTokens( + provider, + code, + callbackUrl, + ); + if (!tokens || !tokens.access_token) + return { error: 'Token exchange failed.' }; + + const userinfo = await this.services.oidc.getUserInfo( + provider, + tokens.access_token, + ); + if (!userinfo || !userinfo.sub) + return { error: 'Could not get user info.' }; + + return { provider, userinfo, stateDecoded }; + } + + async #finishLogin( + res: Response, + user: { + id: number; + uuid: string; + username: string; + email?: string | null; + [k: string]: unknown; + }, + stateDecoded: Record, + extraQueryParams?: Record, + ): Promise { + const { token: sessionToken } = + await this.services.auth.createSessionToken( + user as import('../../stores/user/UserStore.js').UserRow, + ); + + const cookieName = this.config.cookie_name ?? 'puter_token'; + res.cookie(cookieName, sessionToken, { + sameSite: 'none', + secure: true, + httpOnly: true, + }); + + const origin = (this.config.origin ?? '').replace(/\/$/, ''); + let target = (stateDecoded.redirect_uri as string) || origin || '/'; + if (!isSameOrigin(target, origin)) { + target = origin || '/'; + } + + if (extraQueryParams) { + for (const [k, v] of Object.entries(extraQueryParams)) { + if (v != null) target = appendQueryParam(target, k, v); + } + } + + res.redirect(302, target); + } +} diff --git a/src/backend/controllers/peer/PeerController.ts b/src/backend/controllers/peer/PeerController.ts new file mode 100644 index 000000000..48f7e19fe --- /dev/null +++ b/src/backend/controllers/peer/PeerController.ts @@ -0,0 +1,197 @@ +import type { Request, Response } from 'express'; +import type { Actor } from '../../core/actor.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import type { PuterRouter } from '../../core/http/PuterRouter.js'; +import { PuterController } from '../types.js'; +import { PEER_COSTS } from './costs.js'; + +/** + * Encode a UUID (or `app-` UID) as base64url with no padding. + * Strips an `app-` prefix and dashes, then reinterprets the hex bytes. + */ +const uuidToBase64url = (uuid: string): string => + Buffer.from(uuid.replace(/^app-/, '').replaceAll('-', ''), 'hex').toString( + 'base64url', + ); + +/** + * Decode a base64url-encoded hex UUID back to dashed form. + * Returns null if the input doesn't decode to exactly 16 bytes. + */ +const base64urlToUuid = (encoded: string): string | null => { + try { + const hex = Buffer.from(encoded, 'base64url').toString('hex'); + if (hex.length !== 32) return null; + return [ + hex.slice(0, 8), + hex.slice(8, 12), + hex.slice(12, 16), + hex.slice(16, 20), + hex.slice(20), + ].join('-'); + } catch { + return null; + } +}; + +/** + * Build the customIdentifier sent to Cloudflare for credential generation. + * Shape: `` for user actors, `:` for + * app-under-user actors. Cloudflare echoes this back in usage records, + * letting us attribute egress to the originating user (and app, if any). + */ +const actorToTurnIdentifier = (actor: Actor): string => { + const userPart = uuidToBase64url(actor.user.uuid); + if (!actor.app) return userPart; + return `${userPart}:${uuidToBase64url(actor.app.uid)}`; +}; + +/** + * Peer controller — WebRTC signalling info + TURN credential generation. + * + * Config shape: + * config.peers.signaller_url — WebRTC signaller URL + * config.peers.fallback_ice — fallback ICE server list + * config.peers.turn.cloudflare_turn_service_id + * config.peers.turn.cloudflare_turn_api_token + * config.peers.turn.ttl — credential TTL (default 86400) + */ +export class PeerController extends PuterController { + override getReportedCosts(): Record[] { + return Object.entries(PEER_COSTS).map(([usageType, ucentsPerUnit]) => ({ + usageType, + ucentsPerUnit, + unit: 'byte', + source: 'controller:peer', + })); + } + + registerRoutes(router: PuterRouter): void { + router.get( + '/peer/signaller-info', + { subdomain: 'api' }, + this.#signallerInfo, + ); + router.post( + '/peer/generate-turn', + { subdomain: 'api', requireAuth: true }, + this.#generateTurn, + ); + router.post( + '/turn/ingest-usage', + { subdomain: 'api' }, + this.#ingestUsage, + ); + } + + /** GET /peer/signaller-info — public, no auth required. */ + #signallerInfo = (_req: Request, res: Response): void => { + res.json({ + url: this.config.peers?.signaller_url ?? null, + fallbackIce: this.config.peers?.fallback_ice ?? [], + }); + }; + + /** POST /peer/generate-turn — generate TURN credentials via Cloudflare. */ + #generateTurn = async (req: Request, res: Response): Promise => { + const cfg = this.config.peers; + if ( + !cfg || + !cfg.turn || + !cfg.turn.cloudflare_turn_service_id || + !cfg.turn.cloudflare_turn_api_token || + !cfg.turn.ttl + ) { + throw new HttpError(503, 'TURN not configured'); + } + const serviceId = cfg.turn.cloudflare_turn_service_id; + const apiToken = cfg.turn.cloudflare_turn_api_token; + const ttl = cfg.turn.ttl; + + const customIdentifier = actorToTurnIdentifier(req.actor); + + const cfRes = await fetch( + `https://rtc.live.cloudflare.com/v1/turn/keys/${serviceId}/credentials/generate-ice-servers`, + { + method: 'POST', + headers: { + Authorization: `Bearer ${apiToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ ttl, customIdentifier }), + }, + ); + + if (!cfRes.ok) { + const body = await cfRes.text(); + console.warn( + '[peer] Cloudflare TURN credential generation failed', + cfRes.status, + body, + ); + throw new HttpError(500, 'TURN credential generation failed'); + } + + const data = (await cfRes.json()) as { iceServers?: unknown }; + res.json({ ttl, iceServers: data.iceServers }); + }; + + /** POST /turn/ingest-usage — internal-only TURN egress metering. + * an external service that knows the usage information from cloudflare will send it to us here. + * Meters each record directly against the owning user via `services.metering.incrementUsage` + * multiplied by turn:egress-bytes cost. */ + #ingestUsage = async (req: Request, res: Response): Promise => { + const cfg = this.config.peers; + if (!cfg || !cfg.internal_auth_secret) { + throw new HttpError(403, 'Forbidden'); + } + const expectedSecret = cfg.internal_auth_secret; + const header = req.headers['x-puter-internal-auth']; + if (!expectedSecret || header !== expectedSecret) { + throw new HttpError(403, 'Forbidden'); + } + + const { records } = req.body ?? {}; + if (!Array.isArray(records)) { + throw new HttpError(400, 'Missing `records` array'); + } + + for (const record of records) { + if (!record || typeof record !== 'object') continue; + const egressBytes = Number(record.egressBytes ?? 0); + if (egressBytes <= 0) continue; + + const userUuid = record.userId + ? base64urlToUuid(String(record.userId)) + : null; + if (!userUuid) continue; + + try { + const user = await this.stores.user.getByUuid(userUuid); + if (!user) continue; + const costInMicrocents = + egressBytes * PEER_COSTS['turn:egress-bytes']; + const actor = { + user: { + uuid: user.uuid, + id: user.id, + username: user.username, + }, + }; + await this.services.metering.incrementUsage( + actor, + 'turn:egress-bytes', + egressBytes, + costInMicrocents, + ); + } catch (e) { + console.warn( + '[peer] TURN metering failed:', + (e as Error).message, + ); + } + } + + res.json({ ok: true }); + }; +} diff --git a/src/backend/controllers/peer/costs.ts b/src/backend/controllers/peer/costs.ts new file mode 100644 index 000000000..04c103bc1 --- /dev/null +++ b/src/backend/controllers/peer/costs.ts @@ -0,0 +1,4 @@ +// Microcents per byte of TURN egress ($0.05/GB). +export const PEER_COSTS = { + 'turn:egress-bytes': 0.005, +} as const; diff --git a/src/backend/controllers/puterai/PuterAIController.ts b/src/backend/controllers/puterai/PuterAIController.ts new file mode 100644 index 000000000..28d676129 --- /dev/null +++ b/src/backend/controllers/puterai/PuterAIController.ts @@ -0,0 +1,1626 @@ +import crypto from 'node:crypto'; +import { Readable } from 'node:stream'; +import type { Request, Response } from 'express'; +import { HttpError } from '../../core/http/HttpError.js'; +import { isAppActor } from '../../core/actor.js'; +import type { PuterRouter } from '../../core/http/PuterRouter.js'; +import { PuterController } from '../types.js'; +import { isDriverStreamResult } from '../../drivers/meta.js'; +import type { ChatCompletionDriver } from '../../drivers/ai-chat/ChatCompletionDriver.js'; +import type { + ICompleteArguments, + IChatCompleteResult, +} from '../../drivers/ai-chat/types.js'; + +const GEMINI_DOWNLOAD_BASE = + 'https://generativelanguage.googleapis.com/download/v1beta/files'; + +/** + * OpenAI-/Anthropic-compatible HTTP surface on top of the + * `puter-chat-completion` driver. + * + * Third-party SDKs (OpenAI's and Anthropic's official clients, LangChain, + * etc.) point at their vendor's wire shape. These routes accept that wire + * shape, translate to the internal `ICompleteArguments`, hand off to the + * ChatCompletionDriver, and translate the result (or NDJSON stream) back + * into the vendor's response / SSE shape. + * + * All routes live on `subdomain: 'api'` and reject app-under-user actors — + * only user actors may proxy. + */ +export class PuterAIController extends PuterController { + registerRoutes(router: PuterRouter): void { + const apiOpts = { subdomain: 'api', requireAuth: true } as const; + + // Every route below carries the `/puterai` prefix for wire + // compatibility with puter-js and existing API tests. + router.post( + '/puterai/openai/v1/chat/completions', + apiOpts, + this.openaiChatCompletions, + ); + router.post( + '/puterai/openai/v1/completions', + apiOpts, + this.openaiCompletions, + ); + router.post( + '/puterai/openai/v1/responses', + apiOpts, + this.openaiResponses, + ); + router.post( + '/puterai/anthropic/v1/messages', + apiOpts, + this.anthropicMessages, + ); + + // Model listing — enumerate available models per AI service + router.get('/puterai/chat/models', apiOpts, this.#listModels('aiChat')); + router.get( + '/puterai/chat/models/details', + apiOpts, + this.#modelDetails('aiChat'), + ); + router.get( + '/puterai/image/models', + apiOpts, + this.#listModels('aiImage'), + ); + router.get( + '/puterai/image/models/details', + apiOpts, + this.#modelDetails('aiImage'), + ); + router.get( + '/puterai/video/models', + apiOpts, + this.#listModels('aiVideo'), + ); + router.get( + '/puterai/video/models/details', + apiOpts, + this.#modelDetails('aiVideo'), + ); + + // ── Video URL proxy ───────────────────────────────────────── + // Reverse-proxies AI-generated video URLs that can't be given + // directly to the client (auth-gated provider downloads). The + // URL itself is HMAC-signed, so no additional auth gate. + router.get( + '/puterai/video/proxy', + { subdomain: 'api' }, + this.#videoProxy, + ); + } + + #videoProxy = async (req: Request, res: Response): Promise => { + const fileId = + typeof req.query.fileId === 'string' ? req.query.fileId : ''; + const provider = + typeof req.query.provider === 'string' ? req.query.provider : ''; + const expires = + typeof req.query.expires === 'string' ? req.query.expires : ''; + const signature = + typeof req.query.signature === 'string' ? req.query.signature : ''; + + if (!/^[a-zA-Z0-9_-]+$/.test(fileId)) { + res.status(400).send('Invalid or missing fileId parameter'); + return; + } + if (!expires || !signature) { + res.status(403).send('Missing signature'); + return; + } + if (Number(expires) < Date.now() / 1000) { + res.status(403).send('Signature expired'); + return; + } + + const secret = this.config.url_signature_secret; + if (!secret) { + res.status(500).send('URL signature secret not configured'); + return; + } + const expected = crypto + .createHash('sha256') + .update(`${fileId}/video-proxy/${secret}/${expires}`) + .digest('hex'); + // Constant-time compare so signature probing can't time-leak. + const sigBuf = Buffer.from(signature, 'hex'); + const expBuf = Buffer.from(expected, 'hex'); + if ( + sigBuf.length !== expBuf.length || + !crypto.timingSafeEqual(sigBuf, expBuf) + ) { + res.status(403).send('Invalid signature'); + return; + } + + if (provider !== 'gemini') { + res.status(400).send('Unsupported provider'); + return; + } + + // Same key used by `gemini-video-generation` driver to mint the asset. + const apiKey = + this.config.providers?.['gemini-video-generation']?.apiKey; + if (!apiKey) { + res.status(500).send('Gemini API key not configured'); + return; + } + + const upstream = await fetch( + `${GEMINI_DOWNLOAD_BASE}/${fileId}:download?alt=media&key=${apiKey}`, + ); + if (!upstream.ok) { + res.status(upstream.status).send('Failed to fetch video'); + return; + } + const contentType = upstream.headers.get('content-type'); + if (contentType) res.setHeader('Content-Type', contentType); + + if (!upstream.body) { + res.status(500).send('Empty response body'); + return; + } + Readable.fromWeb( + upstream.body as unknown as import('node:stream/web').ReadableStream, + ).pipe(res); + }; + + #listModels(driverKey: string) { + return async (_req: Request, res: Response): Promise => { + const driver = (this.drivers as Record)[ + driverKey + ] as { list?: () => string[] } | undefined; + if (!driver?.list) + throw new HttpError(501, 'Model listing not available'); + const models = driver.list(); + const HIDDEN = ['costly', 'fake', 'abuse', 'model-fallback-test-1']; + res.json({ + models: (models as string[]).filter((m) => !HIDDEN.includes(m)), + }); + }; + } + + #modelDetails(driverKey: string) { + return async (_req: Request, res: Response): Promise => { + const driver = (this.drivers as Record)[ + driverKey + ] as { models?: () => Array<{ id: string }> } | undefined; + if (!driver?.models) + throw new HttpError(501, 'Model details not available'); + const models = driver.models(); + const HIDDEN = ['costly', 'fake', 'abuse', 'model-fallback-test-1']; + res.json({ + models: (models as Array<{ id: string }>).filter( + (m) => !HIDDEN.includes(m.id), + ), + }); + }; + } + + // ── /openai/v1/chat/completions ───────────────────────────────── + + openaiChatCompletions = async ( + req: Request, + res: Response, + ): Promise => { + this.#rejectAppActor(req); + + const body = asRecord(req.body); + const stream = !!body.stream; + + if (!Array.isArray(body.messages)) { + throw new HttpError( + 400, + '`messages` must be an array of chat messages', + ); + } + + const completionId = `chatcmpl-${randomId()}`; + const created = Math.floor(Date.now() / 1000); + + const completeArgs: ICompleteArguments = { + messages: body.messages, + model: toStringOrEmpty(body.model), + stream, + ...(body.tools ? { tools: body.tools as unknown[] } : {}), + ...(body.temperature !== undefined + ? { temperature: Number(body.temperature) } + : {}), + ...(body.max_tokens !== undefined + ? { max_tokens: Number(body.max_tokens) } + : {}), + ...(body.provider + ? { provider: toStringOrEmpty(body.provider) } + : { provider: DEFAULTS.openaiChat }), + }; + + const result = await this.#driver().complete(completeArgs); + const effectiveModel = completeArgs.model || ''; + + if (stream) { + const streamResult = expectStream(result); + setSseHeaders(res); + + let buffer = ''; + let usage: Record | null = null; + let toolCallIndex = 0; + let sawToolCalls = false; + + const sendChunk = ( + delta: Record, + finishReason: string | null = null, + extra: Record = {}, + ): void => { + res.write( + `data: ${JSON.stringify({ + id: completionId, + object: 'chat.completion.chunk', + created, + model: effectiveModel, + choices: [ + { + index: 0, + delta, + logprobs: null, + finish_reason: finishReason, + }, + ], + ...extra, + })}\n\n`, + ); + }; + + pipeNdjsonStream( + streamResult.stream, + (ev) => { + if (ev.type === 'text' && typeof ev.text === 'string') { + sendChunk({ content: ev.text }); + } else if (ev.type === 'tool_use') { + sawToolCalls = true; + sendChunk({ + tool_calls: [ + { + index: toolCallIndex++, + id: ev.id, + type: 'function', + function: { + name: ev.name, + arguments: + typeof ev.input === 'string' + ? ev.input + : JSON.stringify( + ev.input ?? {}, + ), + }, + }, + ], + }); + } else if (ev.type === 'usage') { + usage = ev.usage as Record; + } + }, + { + onEnd: () => { + const finishReason = sawToolCalls + ? 'tool_calls' + : 'stop'; + sendChunk( + {}, + finishReason, + usage ? { usage: buildOpenAIUsage(usage) } : {}, + ); + res.write('data: [DONE]\n\n'); + res.end(); + }, + onError: (err) => { + res.write( + `data: ${JSON.stringify({ error: { message: err?.message ?? 'stream error', type: 'stream_error' } })}\n\n`, + ); + res.write('data: [DONE]\n\n'); + res.end(); + }, + getBuffer: () => buffer, + setBuffer: (v) => { + buffer = v; + }, + }, + ); + return; + } + + const messageResult = result as Extract< + IChatCompleteResult, + { message?: unknown } + >; + const message = (messageResult.message ?? {}) as Record< + string, + unknown + >; + const toolCalls = + (message.tool_calls as unknown[] | undefined) ?? + normalizeToolCallsFromContent(message.content); + const contentText = extractTextContent(message.content); + + res.json({ + id: completionId, + object: 'chat.completion', + created, + model: effectiveModel, + choices: [ + { + index: 0, + message: { + role: (message.role as string) || 'assistant', + content: contentText, + ...(toolCalls ? { tool_calls: toolCalls } : {}), + }, + logprobs: null, + finish_reason: + (messageResult.finish_reason as string | undefined) ?? + 'stop', + }, + ], + usage: buildOpenAIUsage( + messageResult.usage as Record | undefined, + ), + }); + }; + + // ── /openai/v1/completions ────────────────────────────────────── + + openaiCompletions = async (req: Request, res: Response): Promise => { + this.#rejectAppActor(req); + + const body = asRecord(req.body); + const stream = !!body.stream; + + let messages = body.messages as unknown[] | undefined; + if (!messages) { + messages = [{ role: 'user', content: getPromptText(body.prompt) }]; + } + + const completeArgs: ICompleteArguments = { + messages, + model: toStringOrEmpty(body.model), + stream, + ...(body.temperature !== undefined + ? { temperature: Number(body.temperature) } + : {}), + ...(body.max_tokens !== undefined + ? { max_tokens: Number(body.max_tokens) } + : {}), + ...(body.provider + ? { provider: toStringOrEmpty(body.provider) } + : { provider: DEFAULTS.openaiCompletion }), + }; + + const completionId = `cmpl-${randomId()}`; + const created = Math.floor(Date.now() / 1000); + const result = await this.#driver().complete(completeArgs); + const effectiveModel = completeArgs.model || ''; + + if (stream) { + const streamResult = expectStream(result); + setSseHeaders(res); + + let buffer = ''; + let usage: Record | null = null; + + const sendChunk = ( + text: string, + finishReason: string | null = null, + extra: Record = {}, + ): void => { + res.write( + `data: ${JSON.stringify({ + id: completionId, + object: 'text_completion', + created, + model: effectiveModel, + choices: [ + { + text, + index: 0, + logprobs: null, + finish_reason: finishReason, + }, + ], + ...extra, + })}\n\n`, + ); + }; + + pipeNdjsonStream( + streamResult.stream, + (ev) => { + if (ev.type === 'text' && typeof ev.text === 'string') { + sendChunk(ev.text); + } else if (ev.type === 'usage') { + usage = ev.usage as Record; + } + }, + { + onEnd: () => { + sendChunk( + '', + 'stop', + usage ? { usage: buildOpenAIUsage(usage) } : {}, + ); + res.write('data: [DONE]\n\n'); + res.end(); + }, + onError: (err) => { + res.write( + `data: ${JSON.stringify({ error: { message: err?.message ?? 'stream error', type: 'stream_error' } })}\n\n`, + ); + res.write('data: [DONE]\n\n'); + res.end(); + }, + getBuffer: () => buffer, + setBuffer: (v) => { + buffer = v; + }, + }, + ); + return; + } + + const messageResult = result as Extract< + IChatCompleteResult, + { message?: unknown } + >; + res.json({ + id: completionId, + object: 'text_completion', + created, + model: effectiveModel, + choices: [ + { + text: extractTextContent( + ( + messageResult.message as + | Record + | undefined + )?.content, + ), + index: 0, + logprobs: null, + finish_reason: + (messageResult.finish_reason as string | undefined) ?? + 'stop', + }, + ], + usage: buildOpenAIUsage( + messageResult.usage as Record | undefined, + ), + }); + }; + + // ── /openai/v1/responses ──────────────────────────────────────── + + openaiResponses = async (req: Request, res: Response): Promise => { + this.#rejectAppActor(req); + + const body = asRecord(req.body); + const stream = !!body.stream; + + const providerName = + toStringOrEmpty(body.provider) || DEFAULTS.openaiResponses; + if (providerName !== DEFAULTS.openaiResponses) { + throw new HttpError( + 400, + `\`provider\` must be '${DEFAULTS.openaiResponses}'`, + ); + } + + const messages: unknown[] = [ + ...(body.instructions + ? [{ role: 'system', content: body.instructions }] + : []), + ...responseInputToMessages(body.input), + ]; + + const completeArgs: ICompleteArguments = { + messages, + model: toStringOrEmpty(body.model), + stream, + ...(body.tools ? { tools: body.tools as unknown[] } : {}), + ...(body.tool_choice ? { tool_choice: body.tool_choice } : {}), + ...(body.parallel_tool_calls !== undefined + ? { parallel_tool_calls: !!body.parallel_tool_calls } + : {}), + ...(body.temperature !== undefined + ? { temperature: Number(body.temperature) } + : {}), + ...(body.max_output_tokens !== undefined + ? { max_tokens: Number(body.max_output_tokens) } + : {}), + ...(body.top_p !== undefined ? { top_p: Number(body.top_p) } : {}), + ...(body.reasoning + ? { + reasoning: + body.reasoning as ICompleteArguments['reasoning'], + } + : {}), + ...(body.text + ? { text: body.text as ICompleteArguments['text'] } + : {}), + ...(body.include ? { include: body.include as unknown[] } : {}), + ...(body.instructions + ? { + instructions: + body.instructions as ICompleteArguments['instructions'], + } + : {}), + ...(body.metadata + ? { metadata: body.metadata as Record } + : {}), + ...(body.conversation ? { conversation: body.conversation } : {}), + ...(body.previous_response_id + ? { previous_response_id: String(body.previous_response_id) } + : {}), + ...(body.prompt ? { prompt: body.prompt } : {}), + ...(body.prompt_cache_key + ? { prompt_cache_key: String(body.prompt_cache_key) } + : {}), + ...(body.prompt_cache_retention + ? { + prompt_cache_retention: + body.prompt_cache_retention as ICompleteArguments['prompt_cache_retention'], + } + : {}), + ...(body.store !== undefined ? { store: !!body.store } : {}), + ...(body.truncation + ? { + truncation: + body.truncation as ICompleteArguments['truncation'], + } + : {}), + ...(body.background !== undefined + ? { background: !!body.background } + : {}), + ...(body.service_tier + ? { + service_tier: + body.service_tier as ICompleteArguments['service_tier'], + } + : {}), + provider: providerName, + }; + + const responseId = generateId('resp'); + const createdAt = Math.floor(Date.now() / 1000); + const result = await this.#driver().complete(completeArgs); + const effectiveModel = completeArgs.model || ''; + + if (stream) { + const streamResult = expectStream(result); + setSseHeaders(res); + + let buffer = ''; + let sequenceNumber = 0; + let usage: Record | null = null; + let messageItem: { + id: string; + type: string; + role: string; + status: string; + content: Array<{ + type: string; + text: string; + annotations: unknown[]; + }>; + } | null = null; + let messageOutputIndex: number | null = null; + const output: unknown[] = []; + let textContent = ''; + + const sendEvent = (event: Record): void => { + res.write(`event: ${event.type}\n`); + res.write( + `data: ${JSON.stringify({ ...event, sequence_number: ++sequenceNumber })}\n\n`, + ); + }; + + sendEvent({ + type: 'response.created', + response: createResponseShell({ + responseId, + createdAt, + model: effectiveModel, + body, + output: [], + status: 'in_progress', + }), + }); + + pipeNdjsonStream( + streamResult.stream, + (ev) => { + if (ev.type === 'text' && typeof ev.text === 'string') { + if (!messageItem) { + messageItem = { + id: generateId('msg'), + type: 'message', + role: 'assistant', + status: 'in_progress', + content: [], + }; + output.push(messageItem); + messageOutputIndex = output.length - 1; + sendEvent({ + type: 'response.output_item.added', + output_index: messageOutputIndex, + item: messageItem, + }); + const part = { + type: 'output_text', + text: '', + annotations: [] as unknown[], + }; + messageItem.content.push(part); + sendEvent({ + type: 'response.content_part.added', + output_index: messageOutputIndex, + item_id: messageItem.id, + content_index: 0, + part, + }); + } + textContent += ev.text; + messageItem.content[0].text = textContent; + sendEvent({ + type: 'response.output_text.delta', + output_index: messageOutputIndex, + item_id: messageItem.id, + content_index: 0, + delta: ev.text, + }); + } else if (ev.type === 'tool_use') { + const item = { + id: + (ev.canonical_id as string | undefined) || + generateId('fc'), + type: 'function_call', + call_id: ev.id, + name: ev.name, + arguments: + typeof ev.input === 'string' + ? ev.input + : JSON.stringify(ev.input ?? {}), + status: 'completed', + }; + output.push(item); + const outputIndex = output.length - 1; + sendEvent({ + type: 'response.output_item.added', + output_index: outputIndex, + item: { + ...item, + status: 'in_progress', + arguments: '', + }, + }); + sendEvent({ + type: 'response.function_call_arguments.delta', + output_index: outputIndex, + item_id: item.id, + delta: item.arguments, + }); + sendEvent({ + type: 'response.function_call_arguments.done', + output_index: outputIndex, + item_id: item.id, + name: item.name, + arguments: item.arguments, + }); + sendEvent({ + type: 'response.output_item.done', + output_index: outputIndex, + item, + }); + } else if (ev.type === 'usage') { + usage = buildResponsesUsage( + ev.usage as Record, + ); + } + }, + { + onEnd: () => { + if (messageItem) { + messageItem.status = 'completed'; + sendEvent({ + type: 'response.output_text.done', + output_index: messageOutputIndex, + item_id: messageItem.id, + content_index: 0, + text: textContent, + logprobs: [], + }); + sendEvent({ + type: 'response.content_part.done', + output_index: messageOutputIndex, + item_id: messageItem.id, + content_index: 0, + part: messageItem.content[0], + }); + sendEvent({ + type: 'response.output_item.done', + output_index: messageOutputIndex, + item: messageItem, + }); + } + sendEvent({ + type: 'response.completed', + response: createResponseShell({ + responseId, + createdAt, + model: effectiveModel, + body, + output, + usage, + status: 'completed', + }), + }); + res.write('data: [DONE]\n\n'); + res.end(); + }, + onError: (err) => { + sendEvent({ + type: 'error', + error: { + message: err?.message ?? 'stream error', + type: 'stream_error', + }, + }); + res.write('data: [DONE]\n\n'); + res.end(); + }, + getBuffer: () => buffer, + setBuffer: (v) => { + buffer = v; + }, + }, + ); + return; + } + + const messageResult = result as Extract< + IChatCompleteResult, + { message?: unknown } + >; + const usage = buildResponsesUsage( + messageResult.usage as Record | undefined, + ); + const outputItems = responseOutputFromResult(messageResult); + + res.json( + createResponseShell({ + responseId, + createdAt, + model: effectiveModel, + body, + output: outputItems, + usage, + status: 'completed', + }), + ); + }; + + // ── /anthropic/v1/messages ────────────────────────────────────── + + anthropicMessages = async (req: Request, res: Response): Promise => { + this.#rejectAppActor(req); + + const body = asRecord(req.body); + const stream = !!body.stream; + + if (!Array.isArray(body.messages)) { + throw new HttpError( + 400, + '`messages` must be an array of chat messages', + ); + } + + const normalizedMessages = normalizeAnthropicMessages( + body.messages as unknown[], + body.system, + ); + const tools = normalizeAnthropicTools(body.tools); + + const completeArgs: ICompleteArguments = { + messages: normalizedMessages, + model: toStringOrEmpty(body.model), + stream, + ...(tools ? { tools } : {}), + ...(body.temperature !== undefined + ? { temperature: Number(body.temperature) } + : {}), + ...(body.max_tokens !== undefined + ? { max_tokens: Number(body.max_tokens) } + : {}), + ...(body.provider + ? { provider: toStringOrEmpty(body.provider) } + : { provider: DEFAULTS.anthropic }), + }; + + const messageId = `msg_${randomId()}`; + const result = await this.#driver().complete(completeArgs); + const effectiveModel = completeArgs.model || ''; + + if (stream) { + const streamResult = expectStream(result); + setSseHeaders(res); + + const sendEvent = ( + eventType: string, + data: Record, + ): void => { + res.write( + `event: ${eventType}\ndata: ${JSON.stringify(data)}\n\n`, + ); + }; + + // message_start + sendEvent('message_start', { + type: 'message_start', + message: { + id: messageId, + type: 'message', + role: 'assistant', + content: [], + model: effectiveModel, + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 0, output_tokens: 0 }, + }, + }); + + let buffer = ''; + let usage: Record | null = null; + let contentIndex = 0; + let blockOpen = false; + let sawToolCalls = false; + + const openTextBlock = (): void => { + if (blockOpen) return; + sendEvent('content_block_start', { + type: 'content_block_start', + index: contentIndex, + content_block: { type: 'text', text: '' }, + }); + blockOpen = true; + }; + const closeBlock = (): void => { + if (!blockOpen) return; + sendEvent('content_block_stop', { + type: 'content_block_stop', + index: contentIndex, + }); + blockOpen = false; + contentIndex++; + }; + + pipeNdjsonStream( + streamResult.stream, + (ev) => { + if (ev.type === 'text' && typeof ev.text === 'string') { + openTextBlock(); + sendEvent('content_block_delta', { + type: 'content_block_delta', + index: contentIndex, + delta: { type: 'text_delta', text: ev.text }, + }); + } else if (ev.type === 'tool_use') { + sawToolCalls = true; + closeBlock(); + sendEvent('content_block_start', { + type: 'content_block_start', + index: contentIndex, + content_block: { + type: 'tool_use', + id: ev.id, + name: ev.name, + input: {}, + }, + }); + blockOpen = true; + const inputStr = + typeof ev.input === 'string' + ? ev.input + : JSON.stringify(ev.input ?? {}); + sendEvent('content_block_delta', { + type: 'content_block_delta', + index: contentIndex, + delta: { + type: 'input_json_delta', + partial_json: inputStr, + }, + }); + closeBlock(); + } else if (ev.type === 'usage') { + usage = ev.usage as Record; + } + }, + { + onEnd: () => { + closeBlock(); + const stopReason = sawToolCalls + ? 'tool_use' + : 'end_turn'; + const resolvedUsage = buildAnthropicUsage(usage ?? {}); + sendEvent('message_delta', { + type: 'message_delta', + delta: { + stop_reason: stopReason, + stop_sequence: null, + }, + usage: { + output_tokens: resolvedUsage.output_tokens, + }, + }); + sendEvent('message_stop', { type: 'message_stop' }); + res.end(); + }, + onError: (err) => { + sendEvent('error', { + type: 'error', + error: { + type: 'api_error', + message: err?.message ?? 'stream error', + }, + }); + res.end(); + }, + getBuffer: () => buffer, + setBuffer: (v) => { + buffer = v; + }, + }, + ); + return; + } + + const messageResult = result as Extract< + IChatCompleteResult, + { message?: unknown } + >; + const message = (messageResult.message ?? {}) as Record< + string, + unknown + >; + const toolUseBlocks = extractToolUseBlocks(message); + const textContent = extractTextContent(message.content); + + const contentBlocks: Array> = []; + if (textContent) + contentBlocks.push({ type: 'text', text: textContent }); + contentBlocks.push(...toolUseBlocks); + if (contentBlocks.length === 0) + contentBlocks.push({ type: 'text', text: '' }); + + res.json({ + id: messageId, + type: 'message', + role: 'assistant', + content: contentBlocks, + model: effectiveModel, + stop_reason: toolUseBlocks.length > 0 ? 'tool_use' : 'end_turn', + stop_sequence: null, + usage: buildAnthropicUsage( + messageResult.usage as Record | undefined, + ), + }); + }; + + // ── Internals ─────────────────────────────────────────────────── + + #driver(): ChatCompletionDriver { + const driver = ( + this.drivers as unknown as { aiChat: ChatCompletionDriver } + ).aiChat; + if (!driver) + throw new HttpError(500, 'Chat completion driver not registered'); + return driver; + } + + #rejectAppActor(req: Request): void { + // Proxy routes are user-only; apps must call puter-chat-completion directly. + if (isAppActor(req.actor)) { + throw new HttpError( + 403, + 'App actors may not proxy to upstream AI APIs', + ); + } + } +} + +// ── Shared helpers ────────────────────────────────────────────────── + +const DEFAULTS = { + openaiChat: 'openai-completion', + openaiCompletion: 'openai-completion', + openaiResponses: 'openai-responses', + anthropic: 'claude', +} as const; + +const randomId = (): string => crypto.randomUUID().replace(/-/g, ''); +const generateId = (prefix: string): string => `${prefix}_${randomId()}`; + +const asRecord = (value: unknown): Record => { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : {}; +}; + +const toStringOrEmpty = (v: unknown): string => + typeof v === 'string' ? v : ''; + +const setSseHeaders = (res: Response): void => { + res.setHeader('Content-Type', 'text/event-stream; charset=utf-8'); + res.setHeader('Cache-Control', 'no-cache, no-transform'); + res.setHeader('Connection', 'keep-alive'); +}; + +/** + * The chat driver returns either a stream-result envelope or a plain + * message result. Proxy routes invoked with `stream: true` expect the + * former; 500 if the driver dropped the signal. + */ +const expectStream = ( + result: IChatCompleteResult, +): { stream: NodeJS.ReadableStream } => { + if (!isDriverStreamResult(result as unknown)) { + throw new HttpError(500, 'expected streaming response'); + } + return result as unknown as { stream: NodeJS.ReadableStream }; +}; + +/** + * The chat driver's stream emits one JSON object per line + * (`{type: 'text', text}` / `{type: 'tool_use', ...}` / `{type: 'usage', ...}`). + * This helper consumes the stream line-by-line and hands parsed events to + * the caller's reducer, so the per-route translators can stay shape-focused. + */ +interface NdjsonPipeOptions { + onEnd: () => void; + onError: (err: Error) => void; + getBuffer: () => string; + setBuffer: (v: string) => void; +} + +const pipeNdjsonStream = ( + stream: NodeJS.ReadableStream, + onEvent: (event: Record) => void, + opts: NdjsonPipeOptions, +): void => { + stream.on('data', (chunk: Buffer | string) => { + opts.setBuffer( + opts.getBuffer() + + (typeof chunk === 'string' ? chunk : chunk.toString('utf8')), + ); + let newlineIndex: number; + let buf = opts.getBuffer(); + while ((newlineIndex = buf.indexOf('\n')) >= 0) { + const line = buf.slice(0, newlineIndex).trim(); + buf = buf.slice(newlineIndex + 1); + if (!line) continue; + let event: Record; + try { + event = JSON.parse(line) as Record; + } catch { + continue; + } + onEvent(event); + } + opts.setBuffer(buf); + }); + stream.on('end', opts.onEnd); + stream.on('error', opts.onError); +}; + +// ── OpenAI/Anthropic shape helpers ─────────────────────────────────── + +const extractTextContent = (content: unknown): string => { + if (content === undefined || content === null) return ''; + if (typeof content === 'string') return content; + if (Array.isArray(content)) { + return content + .map((part) => { + if (typeof part === 'string') return part; + if (part && typeof part === 'object') { + const r = part as Record; + if (typeof r.text === 'string') return r.text; + if (typeof r.content === 'string') return r.content; + } + return ''; + }) + .join(''); + } + if (typeof content === 'object') { + const r = content as Record; + if (typeof r.text === 'string') return r.text; + if (typeof r.content === 'string') return r.content; + } + return ''; +}; + +const normalizeToolCallsFromContent = ( + content: unknown, +): Array> | undefined => { + if (!Array.isArray(content)) return undefined; + const toolCalls: Array> = []; + for (const part of content) { + if (!part || typeof part !== 'object') continue; + const p = part as Record; + if (p.type !== 'tool_use') continue; + toolCalls.push({ + id: p.id, + type: 'function', + function: { + name: p.name, + arguments: + typeof p.input === 'string' + ? p.input + : JSON.stringify(p.input ?? {}), + }, + }); + } + return toolCalls.length ? toolCalls : undefined; +}; + +const buildOpenAIUsage = ( + usage: Record | undefined, +): Record => { + const u = usage ?? {}; + const promptTokens = Number(u.prompt_tokens ?? u.input_tokens ?? 0); + const completionTokens = Number( + u.completion_tokens ?? u.output_tokens ?? 0, + ); + return { + prompt_tokens: promptTokens, + completion_tokens: completionTokens, + total_tokens: promptTokens + completionTokens, + }; +}; + +const buildAnthropicUsage = ( + usage: Record | undefined, +): { input_tokens: number; output_tokens: number } => { + const u = usage ?? {}; + return { + input_tokens: Number(u.input_tokens ?? u.prompt_tokens ?? 0), + output_tokens: Number(u.output_tokens ?? u.completion_tokens ?? 0), + }; +}; + +const buildResponsesUsage = ( + usage: Record | undefined, +): Record => { + const u = usage ?? {}; + const inputTokens = Number(u.prompt_tokens ?? u.input_tokens ?? 0); + const outputTokens = Number(u.completion_tokens ?? u.output_tokens ?? 0); + const inputDetails = + (u.input_tokens_details as Record | undefined) ?? {}; + const outputDetails = + (u.output_tokens_details as Record | undefined) ?? {}; + return { + input_tokens: inputTokens, + input_tokens_details: { + cached_tokens: Number( + u.cached_tokens ?? inputDetails.cached_tokens ?? 0, + ), + }, + output_tokens: outputTokens, + output_tokens_details: { + reasoning_tokens: Number(outputDetails.reasoning_tokens ?? 0), + }, + total_tokens: inputTokens + outputTokens, + }; +}; + +const getPromptText = (prompt: unknown): string => { + if (prompt === undefined || prompt === null) return ''; + if (Array.isArray(prompt)) { + if (prompt.length === 0) return ''; + if (prompt.length === 1 && typeof prompt[0] === 'string') + return prompt[0]; + throw new HttpError( + 400, + '`prompt` must be a string or single-item string array', + ); + } + if (typeof prompt !== 'string') + throw new HttpError(400, '`prompt` must be a string'); + return prompt; +}; + +// ── OpenAI /responses input → message list ────────────────────────── + +const parseJsonMaybe = (value: unknown): unknown => { + if (typeof value !== 'string') return value ?? {}; + try { + return JSON.parse(value); + } catch { + return value; + } +}; + +const normalizeContentPart = (part: unknown): Record => { + if (typeof part === 'string') return { type: 'text', text: part }; + if (!part || typeof part !== 'object') return { type: 'text', text: '' }; + const p = part as Record; + if (p.type === 'input_text' || p.type === 'output_text') { + return { type: 'text', text: String(p.text ?? '') }; + } + if (p.type === 'input_image') { + return { + type: 'image_url', + ...(p.detail ? { detail: p.detail } : {}), + ...(p.image_url ? { image_url: { url: p.image_url } } : {}), + ...(p.file_id ? { file_id: p.file_id } : {}), + }; + } + if (p.type === 'input_audio') + return { type: 'input_audio', input_audio: p.input_audio }; + if (p.type === 'input_file') { + return { + type: 'input_file', + ...(p.file_data ? { file_data: p.file_data } : {}), + ...(p.file_id ? { file_id: p.file_id } : {}), + ...(p.file_url ? { file_url: p.file_url } : {}), + ...(p.filename ? { filename: p.filename } : {}), + }; + } + return p; +}; + +const normalizeMessageContent = (content: unknown): unknown => { + if (content === undefined || content === null) return ''; + if (typeof content === 'string') return content; + if (Array.isArray(content)) return content.map(normalizeContentPart); + return [normalizeContentPart(content)]; +}; + +const responseInputToMessages = (input: unknown): unknown[] => { + if (input === undefined || input === null) return []; + if (typeof input === 'string') return [{ role: 'user', content: input }]; + if (!Array.isArray(input)) { + throw new HttpError(400, '`input` must be a string or array'); + } + + const messages: unknown[] = []; + for (const item of input) { + if (typeof item === 'string') { + messages.push({ role: 'user', content: item }); + continue; + } + if (!item || typeof item !== 'object') continue; + const it = item as Record; + + if (it.type === 'function_call_output') { + messages.push({ + role: 'tool', + tool_call_id: it.call_id, + content: + typeof it.output === 'string' + ? it.output + : JSON.stringify(it.output ?? {}), + }); + continue; + } + if (it.type === 'function_call') { + messages.push({ + role: 'assistant', + content: [ + { + type: 'tool_use', + id: + (it.call_id as string | undefined) || + (it.id as string | undefined) || + generateId('call'), + canonical_id: it.id, + name: it.name, + input: parseJsonMaybe(it.arguments), + }, + ], + }); + continue; + } + if (it.type === 'message' || it.role) { + messages.push({ + role: + it.role === 'developer' + ? 'system' + : (it.role as string | undefined) || 'user', + content: normalizeMessageContent(it.content), + }); + continue; + } + messages.push({ role: 'user', content: normalizeMessageContent(it) }); + } + return messages; +}; + +// ── OpenAI /responses result → output items ───────────────────────── + +const responseOutputFromResult = ( + result: Extract, +): unknown[] => { + const output: unknown[] = []; + const message = (result.message ?? {}) as Record; + const content = + typeof message.content === 'string' + ? message.content + : Array.isArray(message.content) + ? (message.content as unknown[]) + .filter( + (part): part is Record => + !!part && + typeof part === 'object' && + (part as Record).type === 'text', + ) + .map((part) => String(part.text ?? '')) + .join('') + : ''; + + if (content) { + output.push({ + id: generateId('msg'), + type: 'message', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: content, annotations: [] }], + }); + } + + for (const toolCall of (message.tool_calls as unknown[] | undefined) ?? + []) { + if (!toolCall || typeof toolCall !== 'object') continue; + const tc = toolCall as Record; + const fn = (tc.function as Record | undefined) ?? {}; + output.push({ + id: (tc.canonical_id as string | undefined) || generateId('fc'), + type: 'function_call', + call_id: tc.id, + name: fn.name, + arguments: fn.arguments ?? '{}', + status: 'completed', + }); + } + + return output; +}; + +interface ResponseShellParams { + responseId: string; + createdAt: number; + model: string; + body: Record; + output: unknown[]; + usage?: Record | null; + status: string; +} + +const createResponseShell = ({ + responseId, + createdAt, + model, + body, + output, + usage, + status, +}: ResponseShellParams): Record => ({ + id: responseId, + object: 'response', + created_at: createdAt, + status, + error: null, + incomplete_details: null, + instructions: body.instructions ?? null, + metadata: body.metadata ?? null, + model, + output, + output_text: output + .filter( + (item): item is Record => + !!item && + typeof item === 'object' && + (item as Record).type === 'message', + ) + .flatMap((item) => (item.content as unknown[] | undefined) ?? []) + .filter( + (part): part is Record => + !!part && + typeof part === 'object' && + (part as Record).type === 'output_text', + ) + .map((part) => String(part.text ?? '')) + .join(''), + parallel_tool_calls: body.parallel_tool_calls ?? false, + temperature: body.temperature ?? null, + tool_choice: body.tool_choice ?? 'auto', + tools: Array.isArray(body.tools) + ? (body.tools as unknown[]).map(normalizeResponsesTool) + : [], + top_p: body.top_p ?? null, + ...(body.max_output_tokens !== undefined + ? { max_output_tokens: body.max_output_tokens } + : {}), + ...(body.previous_response_id + ? { previous_response_id: body.previous_response_id } + : {}), + ...(body.store !== undefined ? { store: body.store } : {}), + ...(body.text ? { text: body.text } : {}), + ...(body.truncation ? { truncation: body.truncation } : {}), + ...(usage ? { usage } : {}), +}); + +const normalizeResponsesTool = (tool: unknown): unknown => { + if (!tool || typeof tool !== 'object') return tool; + const t = tool as Record; + if (t.type !== 'function') return t; + return { ...(t.function as Record), type: 'function' }; +}; + +// ── Anthropic → internal messages ─────────────────────────────────── + +const normalizeAnthropicTools = (tools: unknown): unknown[] | undefined => { + if (!Array.isArray(tools) || tools.length === 0) return undefined; + return tools.map((t) => { + if (!t || typeof t !== 'object') return t; + const tt = t as Record; + if (tt.type === 'function' && tt.function) return tt; + return { + type: 'function', + function: { + name: tt.name, + description: tt.description || '', + parameters: tt.input_schema || { + type: 'object', + properties: {}, + }, + }, + }; + }); +}; + +const normalizeAnthropicMessages = ( + messages: unknown[], + system: unknown, +): unknown[] => { + const result: unknown[] = []; + + if (system) { + if (typeof system === 'string') { + result.push({ role: 'system', content: system }); + } else if (Array.isArray(system)) { + const text = system + .map((s) => { + if (typeof s === 'string') return s; + if ( + s && + typeof s === 'object' && + typeof (s as Record).text === 'string' + ) { + return String((s as Record).text); + } + return ''; + }) + .join('\n'); + if (text) result.push({ role: 'system', content: text }); + } + } + + for (const msg of messages) { + if (!msg || typeof msg !== 'object') continue; + const m = msg as Record; + if (m.role === 'user' && Array.isArray(m.content)) { + const toolResults: Array> = []; + const otherParts: unknown[] = []; + for (const part of m.content) { + if ( + part && + typeof part === 'object' && + (part as Record).type === 'tool_result' + ) { + toolResults.push(part as Record); + } else { + otherParts.push(part); + } + } + if (otherParts.length > 0) { + result.push({ role: 'user', content: otherParts }); + } + for (const tr of toolResults) { + let contentStr = ''; + if (typeof tr.content === 'string') { + contentStr = tr.content; + } else if (Array.isArray(tr.content)) { + contentStr = tr.content + .map((p) => { + if (typeof p === 'string') return p; + if ( + p && + typeof p === 'object' && + typeof (p as Record).text === + 'string' + ) { + return String( + (p as Record).text, + ); + } + return ''; + }) + .join(''); + } + result.push({ + role: 'tool', + tool_call_id: tr.tool_use_id, + content: contentStr, + }); + } + if (otherParts.length === 0 && toolResults.length > 0) continue; + if (toolResults.length > 0) continue; + } + result.push(m); + } + + return result; +}; + +const extractToolUseBlocks = ( + message: Record, +): Array> => { + const blocks: Array> = []; + + const toolCalls = message.tool_calls; + if (Array.isArray(toolCalls)) { + for (const tc of toolCalls) { + if (!tc || typeof tc !== 'object') continue; + const t = tc as Record; + const fn = + (t.function as Record | undefined) ?? {}; + blocks.push({ + type: 'tool_use', + id: t.id, + name: fn.name ?? '', + input: + typeof fn.arguments === 'string' + ? safeParseJson(fn.arguments) + : (fn.arguments ?? {}), + }); + } + } + + if (Array.isArray(message.content)) { + for (const part of message.content) { + if (!part || typeof part !== 'object') continue; + const p = part as Record; + if (p.type !== 'tool_use') continue; + blocks.push({ + type: 'tool_use', + id: p.id, + name: p.name, + input: + typeof p.input === 'string' + ? safeParseJson(p.input) + : (p.input ?? {}), + }); + } + } + + return blocks; +}; + +const safeParseJson = (s: string): unknown => { + try { + return JSON.parse(s); + } catch { + return {}; + } +}; diff --git a/src/backend/controllers/share/ShareController.ts b/src/backend/controllers/share/ShareController.ts new file mode 100644 index 000000000..803d6691c --- /dev/null +++ b/src/backend/controllers/share/ShareController.ts @@ -0,0 +1,378 @@ +// import type { Request, Response } from 'express'; +// import { HttpError } from '../../core/http/HttpError.js'; +import type { PuterRouter } from '../../core/http/PuterRouter.js'; +import { PuterController } from '../types.js'; + +// const SHARE_TOKEN_TYPE = 'share'; +// const SHARE_TOKEN_EXPIRY = '14d'; + +/** + * Share link endpoints — check, apply, and request access to pending + * shares. The main `POST /share` creation endpoint is also here. + * + * Shares are permission grants addressed to an email. When the + * recipient doesn't have a Puter account yet, the share row lives in + * the `share` table until they sign up and apply it. When they DO have + * an account, permissions are granted immediately and no row is stored. + */ +export class ShareController extends PuterController { + registerRoutes(_router: PuterRouter): void { + // const api = { subdomain: 'api' } as const; + // router.post('/sharelink/check', api, this.#check); + // router.post( + // '/sharelink/apply', + // { ...api, requireAuth: true }, + // this.#apply, + // ); + // router.post( + // '/sharelink/request', + // { ...api, requireAuth: true }, + // this.#request, + // ); + // router.post('/share', { ...api, requireAuth: true }, this.#share); + } + + // // ── POST /sharelink/check ─────────────────────────────────────── + // // Public — verify a share token from an email link. + + // #check = async (req: Request, res: Response): Promise => { + // const token = req.body?.token; + // if (typeof token !== 'string' || token.length === 0) { + // throw new HttpError(400, 'Missing `token`'); + // } + + // let decoded: { uid?: string; type?: string }; + // try { + // decoded = this.services.token.verify(SHARE_TOKEN_TYPE, token); + // } catch { + // throw new HttpError(400, 'Invalid or expired share token'); + // } + // if (decoded.type !== `token:${SHARE_TOKEN_TYPE}` || !decoded.uid) { + // throw new HttpError(400, 'Invalid share token'); + // } + + // const share = await this.stores.share.getByUid(decoded.uid); + // if (!share) throw new HttpError(404, 'Share not found or expired'); + + // res.json({ + // $: 'api:share', + // uid: share.uid, + // email: share.recipient_email, + // }); + // }; + + // // ── POST /sharelink/apply ─────────────────────────────────────── + // // Auth required — apply a pending share's permissions to the caller. + + // #apply = async (req: Request, res: Response): Promise => { + // const uid = req.body?.uid; + // if (typeof uid !== 'string') throw new HttpError(400, 'Missing `uid`'); + + // const actor = req.actor; + // if (!actor?.user) throw new HttpError(401, 'Unauthorized'); + + // const share = await this.stores.share.getByUid(uid); + // if (!share) throw new HttpError(404, 'Share not found or expired'); + + // // Issuer must still exist + // const issuer = await this.stores.user.getById(share.issuer_user_id); + // if (!issuer) + // throw new HttpError(410, 'Share expired — issuer account gone'); + + // // Email must be confirmed + // if ( + // actor.user.requires_email_confirmation && + // !actor.user.email_confirmed + // ) { + // throw new HttpError( + // 403, + // 'Please confirm your email before applying shares', + // ); + // } + + // // Recipient email must match + // if ( + // !actor.user.email || + // actor.user.email.toLowerCase() !== + // share.recipient_email.toLowerCase() + // ) { + // throw new HttpError( + // 403, + // 'This share was sent to a different email address', + // ); + // } + + // // Grant each permission + // const issuerActor = { + // user: { + // id: issuer.id, + // uuid: issuer.uuid, + // username: issuer.username, + // email: issuer.email ?? null, + // suspended: false, + // email_confirmed: true, + // requires_email_confirmation: false, + // }, + // } as import('../../core/actor.js').Actor; + // const data = (share.data ?? {}) as { + // permissions?: Array<{ + // permission: string; + // extra?: Record; + // }>; + // }; + // for (const perm of data.permissions ?? []) { + // try { + // await this.services.permission.grantUserUserPermission( + // issuerActor, + // actor.user.username ?? '', + // perm.permission, + // perm.extra ?? {}, + // ); + // } catch (err) { + // console.warn('[share] grant failed for', perm.permission, err); + // } + // } + + // // Share consumed — delete it + // await this.stores.share.deleteByUid(uid); + + // res.json({ $: 'api:status-report', status: 'success' }); + // }; + + // // ── POST /sharelink/request ───────────────────────────────────── + // // Auth required — notify the issuer that someone is requesting access. + + // #request = async (req: Request, res: Response): Promise => { + // const uid = req.body?.uid; + // if (typeof uid !== 'string') throw new HttpError(400, 'Missing `uid`'); + + // const actor = req.actor; + // if (!actor?.user) throw new HttpError(401, 'Unauthorized'); + + // const share = await this.stores.share.getByUid(uid); + // if (!share) throw new HttpError(404, 'Share not found or expired'); + + // const issuer = await this.stores.user.getById(share.issuer_user_id); + // if (!issuer) + // throw new HttpError(410, 'Share expired — issuer account gone'); + + // // If caller IS the intended recipient (confirmed email matches), + // // they should just /apply instead. + // if ( + // actor.user.email_confirmed && + // actor.user.email?.toLowerCase() === + // share.recipient_email.toLowerCase() + // ) { + // throw new HttpError( + // 400, + // 'You are the intended recipient — use /sharelink/apply instead', + // ); + // } + + // // Notify the issuer + // if (this.services.notification) { + // await this.services.notification.notify([issuer.id], { + // source: 'sharing', + // title: `User ${actor.user.username} is trying to open a share you sent to ${share.recipient_email}`, + // template: 'user-requesting-share', + // fields: { + // username: actor.user.username, + // intended_recipient: share.recipient_email, + // permissions: + // (share.data as Record)?.permissions ?? + // [], + // }, + // }); + // } + + // res.json({ $: 'api:status-report', status: 'success' }); + // }; + + // // ── POST /share ───────────────────────────────────────────────── + // // Auth required — create shares for recipients (users or emails). + + // #share = async (req: Request, res: Response): Promise => { + // const actor = req.actor; + // if (!actor?.user) throw new HttpError(401, 'Unauthorized'); + + // const body = req.body ?? {}; + // let recipients = body.recipients; + // let shares = body.shares; + // const dryRun = !!body.dry_run; + + // if (!recipients) throw new HttpError(400, 'Missing `recipients`'); + // if (!shares) throw new HttpError(400, 'Missing `shares`'); + // if (!Array.isArray(recipients)) recipients = [recipients]; + // if (!Array.isArray(shares)) shares = [shares]; + + // // Build the permissions list from share declarations. + // const permissions = this.#resolvePermissions(shares as unknown[]); + + // const recipientResults: unknown[] = []; + + // for (const recipient of recipients as unknown[]) { + // const recipientStr = + // typeof recipient === 'string' ? recipient.trim() : ''; + // if (!recipientStr) { + // recipientResults.push({ + // $: 'error', + // message: 'empty recipient', + // }); + // continue; + // } + + // try { + // // Try username first + // const targetUser = + // (await this.stores.user.getByUsername(recipientStr)) ?? + // (recipientStr.includes('@') + // ? await this.stores.user.getByEmail(recipientStr) + // : null); + + // if (targetUser) { + // // Direct grant — user exists + // if (!dryRun) { + // for (const perm of permissions) { + // try { + // await this.services.permission.grantUserUserPermission( + // actor, + // targetUser.username ?? '', + // perm.permission, + // perm.extra ?? {}, + // ); + // } catch (err) { + // console.warn( + // '[share] grant to user failed', + // perm.permission, + // err, + // ); + // } + // } + + // // Notify + // if (this.services.notification) { + // await this.services.notification.notify( + // [targetUser.id], + // { + // source: 'sharing', + // title: `${actor.user.username} shared items with you`, + // template: 'file-shared-with-you', + // fields: { + // username: actor.user.username, + // permissions: permissions.map( + // (p) => p.permission, + // ), + // }, + // }, + // ); + // } + // } + // recipientResults.push({ + // $: 'api:status-report', + // status: 'success', + // }); + // } else if (recipientStr.includes('@')) { + // // Email recipient — store pending share + // if (!dryRun) { + // const share = await this.stores.share.create({ + // issuerUserId: actor.user.id, + // recipientEmail: recipientStr.toLowerCase(), + // data: { + // permissions, + // metadata: body.metadata ?? {}, + // }, + // }); + + // // Sign a share token (14-day expiry) + // const token = this.services.token.sign( + // SHARE_TOKEN_TYPE, + // { + // type: `token:${SHARE_TOKEN_TYPE}`, + // uid: share.uid, + // }, + // { expiresIn: SHARE_TOKEN_EXPIRY }, + // ); + + // // Email the share link + // const origin = `https://${this.config.domain ?? 'puter.com'}`; + // try { + // await this.clients.email.sendRaw({ + // to: recipientStr, + // subject: `${actor.user.username} shared something with you on Puter`, + // html: `

${actor.user.username} shared items with you.

Click here to accept

`, + // }); + // } catch (err) { + // console.warn('[share] email send failed', err); + // } + // } + // recipientResults.push({ + // $: 'api:status-report', + // status: 'success', + // }); + // } else { + // recipientResults.push({ + // $: 'error', + // message: 'User not found', + // }); + // } + // } catch (err) { + // recipientResults.push({ $: 'error', message: String(err) }); + // } + // } + + // const allOk = recipientResults.every( + // (r: unknown) => (r as Record).status === 'success', + // ); + // const anyOk = recipientResults.some( + // (r: unknown) => (r as Record).status === 'success', + // ); + + // res.json({ + // $: 'api:share', + // $version: 'v0.0.0', + // status: allOk ? 'success' : anyOk ? 'mixed' : 'aborted', + // recipients: recipientResults, + // ...(dryRun ? { dry_run: true } : {}), + // }); + // }; + + // // ── Helpers ────────────────────────────────────────────────────── + + // /** + // * Convert share declarations into a flat permission list. + // * Supports `fs-share` ({ path, access }) and `app-share` ({ uid, name }). + // */ + // #resolvePermissions( + // shares: unknown[], + // ): Array<{ permission: string; extra?: Record }> { + // const perms: Array<{ + // permission: string; + // extra?: Record; + // }> = []; + + // for (const share of shares) { + // if (!share || typeof share !== 'object') continue; + // const s = share as Record; + + // if (s.$ === 'fs-share' || s.type === 'fs-share' || s.path) { + // const path = String(s.path ?? ''); + // const access = String(s.access ?? 'read'); + // if (path) { + // perms.push({ permission: `fs:${path}:${access}` }); + // } + // } else if ( + // s.$ === 'app-share' || + // s.type === 'app-share' || + // s.uid || + // s.name + // ) { + // const appUid = String(s.uid ?? s.name ?? ''); + // if (appUid) { + // perms.push({ permission: `app:uid#${appUid}:access` }); + // } + // } + // } + + // return perms; + // } +} diff --git a/src/backend/controllers/static/StaticAssetsController.ts b/src/backend/controllers/static/StaticAssetsController.ts new file mode 100644 index 000000000..965692c1c --- /dev/null +++ b/src/backend/controllers/static/StaticAssetsController.ts @@ -0,0 +1,131 @@ +import express from 'express'; +import { existsSync } from 'node:fs'; +import path from 'node:path'; +import { PuterController } from '../types.js'; +import type { PuterRouter } from '../../core/http/PuterRouter'; + +/** + * Static asset routes. + * + * /puter.js/v1, /puter.js/v2 → any subdomain + * /v1, /v2, /putility/v1 → js subdomain + * /sdk/* → root subdomain — puter-js bundle + * /dist/*, /src/*, /assets/* → root subdomain + * + * Each block depends on its config root (`client_libs_root`, + * `gui_assets_root`, `puterjs_root`). When unset, that block is skipped — + * deployments that don't ship the libs or the GUI just don't get those + * routes. + */ +export class StaticAssetsController extends PuterController { + registerRoutes(router: PuterRouter) { + if (this.config.client_libs_root) { + const root = this.config.client_libs_root; + + router.get('/puter.js/v1', { subdomain: '*' }, (_req, res) => { + res.sendFile('puter.js/v1.js', { root }); + }); + router.get('/puter.js/v2', { subdomain: '*' }, (_req, res) => { + res.sendFile('puter.js/v2.js', { root }); + }); + + router.get('/v1', { subdomain: 'js' }, (_req, res) => { + res.sendFile('puter.js/v1.js', { root }); + }); + router.get('/v2', { subdomain: 'js' }, (_req, res) => { + res.sendFile('puter.js/v2.js', { root }); + }); + router.get('/putility/v1', { subdomain: 'js' }, (_req, res) => { + res.sendFile('putility.js/v1.js', { root }); + }); + } + + // puter-js SDK mount. GUI loads it at `/sdk/puter.dev.js`; the + // webpack dev build writes that filename, but the OSS repo ships + // `puter.js` (minified) as the built artifact. Fall back to + // `puter.js` when `.dev.js` isn't present so `yarn start` works + // out of the box without running the dev-mode webpack build. + const puterjsRoot = this.config.puterjs_root; + if (puterjsRoot) { + const hasDev = existsSync(path.join(puterjsRoot, 'puter.dev.js')); + if (!hasDev && existsSync(path.join(puterjsRoot, 'puter.js'))) { + router.get( + '/sdk/puter.dev.js', + { subdomain: '' }, + (_req, res) => { + res.sendFile('puter.js', { root: puterjsRoot }); + }, + ); + } + router.use('/sdk', { subdomain: '' }, express.static(puterjsRoot)); + + // Third-party apps (dev-center, emulator, …) load puter-js via + // `/puter.js/v{1,2}` — a self-contained single-file endpoint. + // When `client_libs_root` is configured the block above already + // owns these routes and wins by registration order; skip to + // avoid a noisy double-mount. + if (!this.config.client_libs_root) { + const puterJsFile = hasDev ? 'puter.dev.js' : 'puter.js'; + router.get('/puter.js/v1', { subdomain: '*' }, (_req, res) => { + res.sendFile(puterJsFile, { root: puterjsRoot }); + }); + router.get('/puter.js/v2', { subdomain: '*' }, (_req, res) => { + res.sendFile(puterJsFile, { root: puterjsRoot }); + }); + // GUI bundle hard-codes `https://js.puter.com/v{1,2}` as the + // script source in prod mode. Setups that route `js.puter.com` + // to a self-hosted instance (DNS flip, host rewrite) need the + // bare `/v1` and `/v2` paths on the `js` subdomain too — not + // just the `/puter.js/*` prefix. Serve the same file. + router.get('/v1', { subdomain: 'js' }, (_req, res) => { + res.sendFile(puterJsFile, { root: puterjsRoot }); + }); + router.get('/v2', { subdomain: 'js' }, (_req, res) => { + res.sendFile(puterJsFile, { root: puterjsRoot }); + }); + } + } + + if (this.config.gui_assets_root) { + const root = this.config.gui_assets_root; + + router.use( + '/dist', + { subdomain: '' }, + express.static(path.join(root, 'dist')), + ); + router.use( + '/src', + { subdomain: '' }, + express.static(path.join(root, 'src')), + ); + + const publicDir = path.join(root, 'public'); + if (existsSync(publicDir)) { + router.use( + '/assets', + { subdomain: '' }, + express.static(publicDir), + ); + } + } + + // Built-in app mounts. The seed SQL ships apps with + // `index_url: https://builtins.namespaces.puter.com/`, and + // `launch_app` rewrites that prefix to `${gui_origin}/builtin/`. + // Without these static mounts the iframe loads from our own origin + // and hits the 404 handler. `builtin_apps` maps each wire name to + // the directory we serve it from. + const builtinApps = this.config.builtin_apps; + if (builtinApps) { + for (const [name, dirPath] of Object.entries(builtinApps)) { + if (!dirPath || !existsSync(dirPath)) continue; + router.use( + `/builtin/${name}`, + { subdomain: '' }, + express.static(dirPath), + ); + } + } + } +} diff --git a/src/backend/controllers/static/StaticPagesController.ts b/src/backend/controllers/static/StaticPagesController.ts new file mode 100644 index 000000000..6ed5137b1 --- /dev/null +++ b/src/backend/controllers/static/StaticPagesController.ts @@ -0,0 +1,195 @@ +import { PuterController } from '../types.js'; +import type { PuterRouter } from '../../core/http/PuterRouter'; +import { promoteToVerifiedGroup } from '../../util/userProvisioning.js'; + +/** + * One-off user-facing pages. + * + * /robots.txt — static text + * /sitemap.xml — docs + approved apps + * /unsubscribe — toggles `user.unsubscribed` from an email link + * /confirm-email-by-token — email-link confirmation flow (distinct from + * the POST /confirm-email JSON API used by the + * in-app code-entry form) + * + * All root-subdomain-only, all unauthenticated (the confirm/unsubscribe + * tokens in the query string are the auth). + */ +export class StaticPagesController extends PuterController { + registerRoutes(router: PuterRouter) { + const wrap = (inner: string) => + `${inner}`; + const err = (msg: string) => + wrap(`

${msg}

`); + const ok = (msg: string) => + wrap(`

${msg}

`); + + // ── /robots.txt ───────────────────────────────────────────── + router.get('/robots.txt', {}, (req, res) => { + const domain = this.config.domain ?? req.hostname; + const disallowed = [ + 'AhrefsBot', + 'BLEXBot', + 'DotBot', + 'ia_archiver', + 'MJ12bot', + 'SearchmetricsBot', + 'SemrushBot', + ]; + const body = + disallowed + .map((ua) => `User-agent: ${ua}\nDisallow: /\n`) + .join('\n') + + `\nSitemap: ${req.protocol}://${domain}/sitemap.xml\n`; + res.type('text/plain').send(body); + }); + + // ── /sitemap.xml ──────────────────────────────────────────── + router.get('/sitemap.xml', {}, async (req, res) => { + const domain = this.config.domain ?? req.hostname; + const origin = `${req.protocol}://${domain}`; + const apps = (await this.clients.db.read( + 'SELECT `name` FROM `apps` WHERE `approved_for_listing` = 1', + )) as Array<{ name: string }>; + const urls = [ + `${req.protocol}://docs.${domain}/`, + ...apps.map( + (a) => `${origin}/app/${a.name}`, + ), + ]; + const body = + '' + + '' + + urls.join('') + + ''; + res.type('application/xml').send(body); + }); + + // ── /unsubscribe ──────────────────────────────────────────── + router.get('/unsubscribe', {}, async (req, res) => { + const userUuid = + typeof req.query.user_uuid === 'string' + ? req.query.user_uuid + : undefined; + if (!userUuid) { + res.send(err('user_uuid is required')); + return; + } + + const user = await this.stores.user.getByUuid(userUuid); + if (!user) { + res.send(err('User not found.')); + return; + } + if (user.unsubscribed) { + res.send(ok('You are already unsubscribed.')); + return; + } + + await this.stores.user.update(user.id, { unsubscribed: 1 }); + res.send(ok('You have successfully unsubscribed from all emails.')); + }); + + // ── /confirm-email-by-token ───────────────────────────────── + router.get('/confirm-email-by-token', {}, async (req, res) => { + const userUuid = + typeof req.query.user_uuid === 'string' + ? req.query.user_uuid + : undefined; + const token = + typeof req.query.token === 'string' + ? req.query.token + : undefined; + if (!userUuid) { + res.send(err('user_uuid is required')); + return; + } + if (!token) { + res.send(err('token is required')); + return; + } + + const user = await this.stores.user.getByProperty( + 'uuid', + userUuid, + { force: true }, + ); + if (!user) { + res.send(err('user not found.')); + return; + } + if (user.email_confirmed) { + res.send(ok('Email already confirmed.')); + return; + } + if (user.email_confirm_token !== token) { + res.send(err('invalid token.')); + return; + } + + // v2 writes `clean_email` at signup (lowercased email). Older rows + // that predate that may be null — fall back to email.lower(). + const cleanEmail = + (user.clean_email as string | null | undefined) ?? + String(user.email ?? '').toLowerCase(); + + const [dupe] = (await this.clients.db.read( + `SELECT EXISTS( + SELECT 1 FROM \`user\` WHERE (\`email\` = ? OR \`clean_email\` = ?) + AND \`email_confirmed\` = 1 + AND \`password\` IS NOT NULL + ) AS email_exists`, + [user.email, cleanEmail], + )) as Array<{ email_exists: number }>; + if (dupe?.email_exists) { + res.send( + err('This email was confirmed on a different account.'), + ); + return; + } + + // Revoke any other accounts' pending change-email slots targeting + // this address — they're no longer valid once someone confirms it. + await this.clients.db.write( + 'UPDATE `user` SET `unconfirmed_change_email` = NULL, `change_email_confirm_token` = NULL WHERE `unconfirmed_change_email` = ?', + [user.email], + ); + + await this.stores.user.update(user.id, { + email_confirmed: 1, + requires_email_confirmation: 0, + email_confirm_code: null, + email_confirm_token: null, + }); + + await promoteToVerifiedGroup(this.stores.group, this.config, user); + + // Best-effort side-channels — don't fail the user-visible response + // if sockets or the event bus are unavailable. + try { + await this.services.socket.send( + { room: user.id }, + 'user.email_confirmed', + {}, + ); + } catch { + /* ignore */ + } + try { + this.clients.event?.emit( + 'user.email-confirmed', + { + user_id: user.id, + user_uid: user.uuid, + email: user.email, + }, + {}, + ); + } catch { + /* ignore */ + } + + res.send(ok('Your email has been successfully confirmed.')); + }); + } +} diff --git a/src/backend/controllers/system/SystemController.js b/src/backend/controllers/system/SystemController.js new file mode 100644 index 000000000..12e171f4c --- /dev/null +++ b/src/backend/controllers/system/SystemController.js @@ -0,0 +1,149 @@ +import { HttpError } from '../../core/http/HttpError.js'; +import { PuterController } from '../types.js'; + +/** + * System-level endpoints — health, version, contact. + * + * These are all low-risk, authenticated or not, and mostly stateless. + */ +export class SystemController extends PuterController { + constructor(config, clients, stores, services, drivers) { + super(config, clients, stores, services, drivers); + this.bootTime = Date.now(); + } + + registerRoutes( + /** @type {import('../../core/http/PuterRouter.js').PuterRouter} */ + router, + ) { + // ── Healthcheck ───────────────────────────────────────────── + // Delegates to ServerHealthService for the real check-based + // status. Returns `{ ok: true }` when all registered checks pass, + // or `{ ok: false, failed: [...] }` + 503 when any fail or the + // server is draining. + router.get('/healthcheck', { subdomain: '*' }, async (_req, res) => { + const health = this.services.health; + if (!health || typeof health.getStatus !== 'function') { + // Fallback for boot ordering / missing service. + return res.send('ok'); + } + const status = await health.getStatus(); + if (!status.ok) return res.status(503).json(status); + return res.json(status); + }); + + // ── Version ───────────────────────────────────────────────── + + router.get('/version', { subdomain: '*' }, (_req, res) => { + const version = + this.config.version ?? + process.env.npm_package_version ?? + 'unknown'; + const parts = String(version).split('.'); + res.json({ + version, + major: parts[0] ? Number(parts[0]) : null, + minor: parts[1] ? Number(parts[1]) : null, + patch: parts[2] ? Number(parts[2]) : null, + environment: this.config.env ?? 'prod', + location: this.config.serverId ?? null, + deploy_timestamp: this.bootTime, + }); + }); + + // ── Contact us ────────────────────────────────────────────── + + router.post( + '/contactUs', + { + subdomain: 'api', + requireUserActor: true, + rateLimit: { + scope: 'contact-us', + limit: 10, + window: 15 * 60_000, + key: 'user', + }, + }, + async (req, res) => { + const { message } = req.body ?? {}; + if (!message || typeof message !== 'string') { + throw new HttpError(400, '`message` is required'); + } + if (message.length > 100_000) { + throw new HttpError( + 400, + '`message` is too long (max 100,000 characters)', + ); + } + + // Persist to feedback table for durability + try { + await this.clients.db.write( + 'INSERT INTO `feedback` (`user_id`, `message`) VALUES (?, ?)', + [req.actor.user.id, message], + ); + } catch (e) { + console.warn('[contactUs] feedback insert failed:', e); + } + + // Send to support email + const supportEmail = + this.config.support_email ?? 'support@puter.com'; + if (this.clients.email && req.actor.user?.email) { + try { + await this.clients.email.sendRaw({ + to: supportEmail, + replyTo: req.actor.user.email, + subject: `Contact from ${req.actor.user.username}`, + text: message, + }); + } catch (e) { + console.warn('[contactUs] email send failed:', e); + } + } + + res.json({}); + }, + ); + + // ── GET /whoarewe ─────────────────────────────────────────── + + router.get('/whoarewe', {}, (_req, res) => { + res.json({ + name: 'Puter', + version: this.config.version ?? null, + environment: this.config.env ?? 'prod', + }); + }); + + // ── GET /lsmod ────────────────────────────────────────────── + // Enumerates driver interfaces and their implementors. + + router.get( + '/lsmod', + { subdomain: 'api', requireAuth: true }, + (_req, res) => { + const interfaces = {}; + for (const [key, driver] of Object.entries(this.drivers)) { + const ifaceName = driver?.driverInterface; + if (!ifaceName) continue; + const driverName = driver.driverName ?? key; + if (!interfaces[ifaceName]) { + interfaces[ifaceName] = { implementors: {} }; + } + interfaces[ifaceName].implementors[driverName] = { + isDefault: Boolean(driver.isDefault), + }; + } + res.json({ interfaces }); + }, + ); + } + + onServerStart() {} + onServerPrepareShutdown() { + globalThis.__puter_draining = true; + } + onServerShutdown() {} +} diff --git a/src/backend/controllers/types.ts b/src/backend/controllers/types.ts new file mode 100644 index 000000000..5d3bb2d25 --- /dev/null +++ b/src/backend/controllers/types.ts @@ -0,0 +1,58 @@ +import type { puterClients } from '../clients'; +import type { PuterRouter } from '../core/http/PuterRouter'; +import type { puterDrivers } from '../drivers'; +import type { puterServices } from '../services'; +import type { puterStores } from '../stores'; +import type { + IConfig, + LayerInstances, + WithControllerRegistration, +} from '../types'; + +export type IPuterController< + T extends WithControllerRegistration = WithControllerRegistration, +> = new ( + config: IConfig, + clients: LayerInstances, + stores: LayerInstances, + services: LayerInstances, + drivers: LayerInstances, +) => T; + +/** + * Base class for v2 controllers. `registerRoutes(router)` receives a + * `PuterRouter` (not an express app) — see `core/http/PuterRouter.ts`. + * Controllers either override `registerRoutes` imperatively or lean on the + * `@Controller` / `@Post` / etc. decorators, which install a default + * `registerRoutes` walker on the prototype. + */ +export const PuterController = + class PuterController implements WithControllerRegistration { + constructor( + protected config: IConfig, + protected clients: LayerInstances, + protected stores: LayerInstances, + protected services: LayerInstances, + protected drivers: LayerInstances, + ) {} + public onServerStart() { + return; + } + public onServerPrepareShutdown() { + return; + } + public onServerShutdown() { + return; + } + public getReportedCosts(): Record[] { + return []; + } + public registerRoutes(_router: PuterRouter) {} + } satisfies IPuterController; + +export type IPuterControllerRegistry = Record< + string, + | IPuterController + | (InstanceType> & + Record) +>; diff --git a/src/backend/controllers/webdav/WebDAVController.ts b/src/backend/controllers/webdav/WebDAVController.ts new file mode 100644 index 000000000..72df3895e --- /dev/null +++ b/src/backend/controllers/webdav/WebDAVController.ts @@ -0,0 +1,838 @@ +import { compare as bcryptCompare } from 'bcrypt'; +import type { Request, Response } from 'express'; +import { posix as pathPosix } from 'node:path'; +import type { Actor } from '../../core/actor.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import type { PuterRouter } from '../../core/http/PuterRouter.js'; +import { verify as verifyOtp } from '../../services/auth/OTPUtil.js'; +import { expandTildePath } from '../../services/fs/resolveNode.js'; +import type { FSEntry } from '../../stores/fs/FSEntry.js'; +import { PuterController } from '../types.js'; +import { + createLock, + deleteLock, + extractLockToken, + getFileLocks, + getLockIfValid, + hasWritePermission, + refreshLock, +} from './locks.js'; + +const DAV_HEADERS = { + DAV: '1, 2, ordered-collections', + 'MS-Author-Via': 'DAV', +}; + +const ALLOW_METHODS = + 'OPTIONS, GET, HEAD, POST, PUT, DELETE, COPY, MOVE, MKCOL, PROPFIND, PROPPATCH, LOCK, UNLOCK, TRACE'; + +// macOS creates these files; reject them to keep the FS clean. +const MACOS_JUNK_REGEX = /(?:^\.DS_Store$|^\._)/; + +/** + * WebDAV controller — full RFC 4918 surface on the `dav.*` subdomain. + * + * All FS operations go through v2's FSService + S3ObjectStore. + * Locking uses Redis (see `./locks.ts`). ACL is enforced via ACLService + * before every mutation and read. + * + * Auth: HTTP Basic → parse credentials → verify via AuthService + + * bcrypt (or `-token` username for token-based auth). Falls back to + * the global authProbe's `req.actor` if a session cookie is present. + */ +export class WebDAVController extends PuterController { + registerRoutes(router: PuterRouter): void { + // Single catch-all on the `dav` subdomain. We dispatch by req.method + // inside the handler because WebDAV uses non-standard HTTP verbs that + // Express doesn't have first-class router methods for in all versions. + router.use( + { subdomain: 'dav' }, + async (req: Request, res: Response, _next) => { + try { + await this.#dispatch(req, res); + } catch (err) { + if (err instanceof HttpError) { + res.status(err.statusCode).send(err.message); + return; + } + console.error('[webdav] unhandled error', err); + res.status(500).send('Internal Server Error'); + } + // Don't call next — we always handle or error. + }, + ); + } + + async #dispatch(req: Request, res: Response): Promise { + // Authenticate + const actor = await this.#resolveActor(req, res); + if (!actor) return; // 401 already sent + + // Expand `~`/`~/...` against the authenticated actor's username. + // WebDAV doesn't standardize `~`, but some clients do — and the + // pre-existing behaviour silently expanded it via the FS store. + const davPath = expandTildePath( + decodeURIComponent(req.path), + actor.user.username, + ); + const redis = this.clients.redis; + const lockToken = extractLockToken( + (req.headers['if'] as string | undefined) ?? + (req.headers['lock-token'] as string | undefined), + ); + + switch (req.method.toUpperCase()) { + case 'OPTIONS': + return this.#options(res); + case 'HEAD': + case 'GET': + return this.#get( + req, + res, + actor, + davPath, + req.method === 'HEAD', + ); + case 'PROPFIND': + return this.#propfind(req, res, actor, davPath); + case 'PROPPATCH': + return this.#proppatch(res, davPath, redis, lockToken); + case 'MKCOL': + return this.#mkcol(req, res, actor, davPath, redis, lockToken); + case 'PUT': + return this.#put(req, res, actor, davPath, redis, lockToken); + case 'DELETE': + return this.#delete(res, actor, davPath, redis, lockToken); + case 'COPY': + return this.#copy(req, res, actor, davPath, redis, lockToken); + case 'MOVE': + return this.#move(req, res, actor, davPath, redis, lockToken); + case 'LOCK': + return this.#lock(req, res, davPath, redis, lockToken); + case 'UNLOCK': + return this.#unlock(req, res, davPath, redis); + default: + res.status(405) + .set('Allow', ALLOW_METHODS) + .send('Method Not Allowed'); + } + } + + // ── Auth ───────────────────────────────────────────────────────── + + async #resolveActor(req: Request, res: Response): Promise { + // If the global authProbe already resolved an actor, use it. + if (req.actor?.user) return req.actor; + + // Parse HTTP Basic + const authHeader = req.headers.authorization; + if (!authHeader || !authHeader.startsWith('Basic ')) { + res.status(401) + .set({ + 'WWW-Authenticate': 'Basic realm="WebDAV"', + ...DAV_HEADERS, + }) + .send('Authentication required'); + return null; + } + + const decoded = Buffer.from(authHeader.slice(6), 'base64').toString( + 'utf-8', + ); + const colonIdx = decoded.indexOf(':'); + if (colonIdx < 0) { + res.status(401) + .set('WWW-Authenticate', 'Basic realm="WebDAV"') + .send('Invalid credentials'); + return null; + } + const username = decoded.slice(0, colonIdx); + const password = decoded.slice(colonIdx + 1); + + // `-token` username: password IS the auth token + if (username === '-token') { + const actor = + await this.services.auth.authenticateFromToken(password); + if (!actor) { + res.status(401) + .set('WWW-Authenticate', 'Basic realm="WebDAV"') + .send('Invalid token'); + return null; + } + return actor; + } + + // Regular username + password (with optional 6-digit OTP suffix) + const user = await this.stores.user.getByUsername(username); + if (!user || !user.password) { + res.status(401) + .set('WWW-Authenticate', 'Basic realm="WebDAV"') + .send('Invalid credentials'); + return null; + } + + // If 2FA is enabled the password MUST be suffixed with the 6-digit + // TOTP code — HTTP Basic has no channel for a second factor. + const otpEnabled = Boolean(user.otp_enabled); + let passwordOk = false; + if (otpEnabled) { + if (password.length <= 6) { + res.status(401) + .set('WWW-Authenticate', 'Basic realm="WebDAV"') + .send('Invalid credentials'); + return null; + } + const basePassword = password.slice(0, -6); + const otpCode = password.slice(-6); + const baseOk = await bcryptCompare(basePassword, user.password); + const otpOk = + baseOk && + typeof user.otp_secret === 'string' && + verifyOtp(user.username, user.otp_secret, otpCode); + passwordOk = Boolean(otpOk); + } else { + passwordOk = await bcryptCompare(password, user.password); + } + + if (!passwordOk) { + res.status(401) + .set('WWW-Authenticate', 'Basic realm="WebDAV"') + .send('Invalid credentials'); + return null; + } + + // Build a session-less actor for the user + return { + user: { + id: user.id, + uuid: user.uuid, + username: user.username, + email: user.email ?? null, + suspended: user.suspended ?? false, + email_confirmed: user.email_confirmed ?? false, + requires_email_confirmation: + user.requires_email_confirmation ?? false, + }, + }; + } + + // ── OPTIONS ────────────────────────────────────────────────────── + + #options(res: Response): void { + res.status(200) + .set({ + Allow: ALLOW_METHODS, + ...DAV_HEADERS, + 'Accept-Ranges': 'bytes', + 'Content-Type': 'text/plain; charset=utf-8', + 'Cache-Control': 'no-cache', + }) + .send(''); + } + + // ── GET / HEAD ────────────────────────────────────────────────── + + async #get( + req: Request, + res: Response, + actor: Actor, + davPath: string, + headOnly: boolean, + ): Promise { + const entry = await this.stores.fsEntry.getEntryByPath(davPath); + if (!entry) throw new HttpError(404, 'Not Found'); + if (entry.isDir) throw new HttpError(400, 'Cannot GET a directory'); + + await this.#assertRead(actor, davPath); + + const etag = `"${entry.uuid}-${Math.floor(entry.modified ?? entry.created ?? 0)}"`; + const size = entry.size ?? 0; + + res.set({ + 'Accept-Ranges': 'bytes', + 'Content-Length': String(size), + 'Last-Modified': new Date( + entry.modified ?? entry.created ?? 0, + ).toUTCString(), + ETag: etag, + }); + + if (headOnly) { + res.status(200).end(); + return; + } + + const rangeHeader = req.headers.range; + const result = await this.services.fs.readContent(entry, { + range: rangeHeader, + }); + if (result.contentType) res.set('Content-Type', result.contentType); + if (result.contentRange) { + res.status(206).set({ + 'Content-Range': result.contentRange, + 'Content-Length': String(result.contentLength ?? 0), + }); + } + result.body.pipe(res); + } + + // ── PROPFIND ──────────────────────────────────────────────────── + + async #propfind( + req: Request, + res: Response, + actor: Actor, + davPath: string, + ): Promise { + const depth = req.headers.depth ?? '1'; + + const entry = + davPath === '/' + ? null // root always exists + : await this.stores.fsEntry.getEntryByPath(davPath); + if (davPath !== '/' && !entry) throw new HttpError(404, 'Not Found'); + + await this.#assertRead(actor, davPath); + + const isDir = davPath === '/' || !!entry?.isDir; + const responses = [propfindEntry(davPath, entry, isDir)]; + + if (depth !== '0' && isDir && entry) { + const children = await this.services.fs.listDirectory( + entry.uuid, + {}, + ); + for (const child of children) { + responses.push(propfindEntry(child.path, child, child.isDir)); + } + } else if (depth !== '0' && davPath === '/') { + // Root: list top-level user directories + const rootEntry = await this.stores.fsEntry.getEntryByPath( + `/${actor.user!.username}`, + ); + if (rootEntry) { + responses.push( + propfindEntry(rootEntry.path, rootEntry, rootEntry.isDir), + ); + } + } + + res.status(207) + .set({ 'Content-Type': 'application/xml; charset=utf-8' }) + .send(wrapMultistatus(responses.join('\n'))); + } + + // ── PROPPATCH (stub — acknowledges but doesn't persist props) ─── + + async #proppatch( + res: Response, + davPath: string, + redis: unknown, + lockToken: string | null, + ): Promise { + if ( + !(await hasWritePermission( + redis as import('ioredis').Cluster, + davPath, + lockToken, + )) + ) { + throw new HttpError(423, 'Locked'); + } + res.status(207) + .set({ 'Content-Type': 'application/xml; charset=utf-8' }) + .send( + `\n${escapeXml(encodeURI(davPath))}HTTP/1.1 200 OK`, + ); + } + + // ── MKCOL ─────────────────────────────────────────────────────── + + async #mkcol( + req: Request, + res: Response, + actor: Actor, + davPath: string, + redis: unknown, + lockToken: string | null, + ): Promise { + if (davPath === '/') throw new HttpError(403, 'Cannot create at root'); + if ( + req.headers['content-length'] && + Number(req.headers['content-length']) > 0 + ) { + throw new HttpError(415, 'MKCOL must not have a body'); + } + if ( + !(await hasWritePermission( + redis as import('ioredis').Cluster, + davPath, + lockToken, + )) + ) { + throw new HttpError(423, 'Locked'); + } + const userId = actor.user!.id as number; + const parentPath = pathPosix.dirname(davPath); + await this.#assertWrite(actor, parentPath); + + const existing = await this.stores.fsEntry.getEntryByPath(davPath); + if (existing) throw new HttpError(405, 'Already exists'); + + const entry = await this.services.fs.mkdir(userId, { + path: davPath, + }); + this.#emitGuiEvent('outer.gui.item.added', entry); + res.status(201) + .set({ 'Content-Length': '0', Location: `${davPath}/` }) + .end(); + } + + // ── PUT ───────────────────────────────────────────────────────── + + async #put( + req: Request, + res: Response, + actor: Actor, + davPath: string, + redis: unknown, + lockToken: string | null, + ): Promise { + const name = pathPosix.basename(davPath); + if (MACOS_JUNK_REGEX.test(name)) { + res.status(422).send('Ignored macOS metadata file'); + return; + } + if ( + !(await hasWritePermission( + redis as import('ioredis').Cluster, + davPath, + lockToken, + )) + ) { + throw new HttpError(423, 'Locked'); + } + + const userId = actor.user!.id as number; + const parentPath = pathPosix.dirname(davPath); + await this.#assertWrite(actor, parentPath); + + const contentLength = Number( + req.headers['content-length'] ?? + req.headers['x-expected-entity-length'] ?? + 0, + ); + if (!contentLength && contentLength !== 0) + throw new HttpError(400, 'Missing Content-Length'); + + // Check if overwrite + const existing = await this.stores.fsEntry.getEntryByPath(davPath); + + // Expect: 100-continue + if (req.headers.expect === '100-continue') { + (req.socket as { write?: (s: string) => void }).write?.( + 'HTTP/1.1 100 Continue\r\n\r\n', + ); + } + + const writeResult = await this.services.fs.write(userId, { + fileMetadata: { + path: davPath, + size: contentLength, + overwrite: true, + createMissingParents: true, + }, + fileContent: req, + }); + + this.#emitGuiEvent( + existing ? 'outer.gui.item.updated' : 'outer.gui.item.added', + writeResult.fsEntry, + ); + + const fe = writeResult.fsEntry; + const etag = `"${fe.uuid}-${Math.floor(fe.modified ?? fe.created ?? 0)}"`; + res.status(existing ? 204 : 201) + .set({ + ETag: etag, + 'Last-Modified': new Date( + fe.modified ?? fe.created ?? 0, + ).toUTCString(), + }) + .end(); + } + + // ── DELETE ─────────────────────────────────────────────────────── + + async #delete( + res: Response, + actor: Actor, + davPath: string, + redis: unknown, + lockToken: string | null, + ): Promise { + if ( + !(await hasWritePermission( + redis as import('ioredis').Cluster, + davPath, + lockToken, + )) + ) { + throw new HttpError(423, 'Locked'); + } + const userId = actor.user!.id as number; + await this.#assertWrite(actor, davPath); + + const entry = await this.stores.fsEntry.getEntryByPath(davPath); + if (!entry) throw new HttpError(404, 'Not Found'); + + await this.services.fs.remove(userId, { entry, recursive: true }); + this.#emitGuiEvent('outer.gui.item.removed', entry); + res.status(204).end(); + } + + // ── COPY ──────────────────────────────────────────────────────── + + async #copy( + req: Request, + res: Response, + actor: Actor, + davPath: string, + redis: unknown, + lockToken: string | null, + ): Promise { + const destPath = this.#parseDestination(req); + if ( + !(await hasWritePermission( + redis as import('ioredis').Cluster, + destPath, + lockToken, + )) + ) { + throw new HttpError(423, 'Locked'); + } + + const userId = actor.user!.id as number; + await this.#assertRead(actor, davPath); + await this.#assertWrite(actor, pathPosix.dirname(destPath)); + + const source = await this.stores.fsEntry.getEntryByPath(davPath); + if (!source) throw new HttpError(404, 'Source not found'); + + const overwrite = req.headers.overwrite !== 'F'; + const destExists = await this.stores.fsEntry.getEntryByPath(destPath); + if (destExists && !overwrite) + throw new HttpError(412, 'Destination exists and Overwrite=F'); + + const destParent = await this.stores.fsEntry.getEntryByPath( + pathPosix.dirname(destPath), + ); + if (!destParent?.isDir) + throw new HttpError( + 409, + 'Destination parent missing or not a directory', + ); + + const copy = await this.services.fs.copy(userId, { + source, + destinationParent: destParent, + newName: pathPosix.basename(destPath), + overwrite, + }); + this.#emitGuiEvent('outer.gui.item.added', copy); + res.status(destExists ? 204 : 201).end(); + } + + // ── MOVE ──────────────────────────────────────────────────────── + + async #move( + req: Request, + res: Response, + actor: Actor, + davPath: string, + redis: unknown, + lockToken: string | null, + ): Promise { + const destPath = this.#parseDestination(req); + const r = redis as import('ioredis').Cluster; + if (!(await hasWritePermission(r, davPath, lockToken))) + throw new HttpError(423, 'Locked'); + if (!(await hasWritePermission(r, destPath, lockToken))) + throw new HttpError(423, 'Locked'); + + const userId = actor.user!.id as number; + await this.#assertWrite(actor, davPath); + await this.#assertWrite(actor, pathPosix.dirname(destPath)); + + const source = await this.stores.fsEntry.getEntryByPath(davPath); + if (!source) throw new HttpError(404, 'Source not found'); + + const overwrite = req.headers.overwrite !== 'F'; + const destExists = await this.stores.fsEntry.getEntryByPath(destPath); + if (destExists && !overwrite) + throw new HttpError(412, 'Destination exists and Overwrite=F'); + + const destParent = await this.stores.fsEntry.getEntryByPath( + pathPosix.dirname(destPath), + ); + if (!destParent?.isDir) + throw new HttpError( + 409, + 'Destination parent missing or not a directory', + ); + + const moved = await this.services.fs.move(userId, { + source, + destinationParent: destParent, + newName: pathPosix.basename(destPath), + overwrite, + }); + this.#emitGuiEvent('outer.gui.item.moved', moved, { + old_path: davPath, + }); + res.status(destExists ? 204 : 201).end(); + } + + // ── LOCK ──────────────────────────────────────────────────────── + + async #lock( + req: Request, + res: Response, + davPath: string, + redis: unknown, + headerToken: string | null, + ): Promise { + const r = redis as import('ioredis').Cluster; + + // Refresh existing lock + if (headerToken) { + const existing = await getLockIfValid(r, headerToken); + if (!existing) throw new HttpError(412, 'Lock token not found'); + await refreshLock(r, headerToken); + res.status(200) + .set({ + 'Content-Type': 'application/xml; charset=utf-8', + ...DAV_HEADERS, + }) + .send( + lockResponseXml(headerToken, davPath, existing.lockScope), + ); + return; + } + + // Parse requested scope from XML body + let lockScope: 'exclusive' | 'shared' = 'exclusive'; + const body = req.body as Record | undefined; + if (body?.lockinfo) { + const info = body.lockinfo as Record; + const scope = info.lockscope as Record | undefined; + if (scope?.shared !== undefined) lockScope = 'shared'; + } + + // Check for conflicts + const existingLocks = await getFileLocks(r, davPath); + for (const lock of existingLocks) { + if (lockScope === 'exclusive' || lock.lockScope === 'exclusive') { + throw new HttpError(423, 'Locked — conflicting lock exists'); + } + } + + const token = await createLock(r, davPath, lockScope); + const status = 200; + + res.status(status) + .set({ + 'Content-Type': 'application/xml; charset=utf-8', + 'Lock-Token': `<${token}>`, + ...DAV_HEADERS, + }) + .send(lockResponseXml(token, davPath, lockScope)); + } + + // ── UNLOCK ────────────────────────────────────────────────────── + + async #unlock( + req: Request, + res: Response, + davPath: string, + redis: unknown, + ): Promise { + const r = redis as import('ioredis').Cluster; + const tokenHeader = req.headers['lock-token'] as string | undefined; + const token = extractLockToken(tokenHeader); + if (!token) throw new HttpError(400, 'Missing Lock-Token header'); + + const lock = await getLockIfValid(r, token); + if (!lock) { + // Idempotent — if already expired, just 204. + res.status(204).end(); + return; + } + if (lock.path !== davPath) + throw new HttpError(403, 'Lock token does not match this path'); + + await deleteLock(r, token); + res.status(204).end(); + } + + // ── ACL helpers ───────────────────────────────────────────────── + + async #assertRead(actor: Actor, path: string): Promise { + const descriptor = { + path, + resolveAncestors: () => this.services.fs.getAncestorChain(path), + }; + const ok = await this.services.acl.check(actor, descriptor, 'read'); + if (!ok) throw new HttpError(403, 'Permission denied'); + } + + async #assertWrite(actor: Actor, path: string): Promise { + const descriptor = { + path, + resolveAncestors: () => this.services.fs.getAncestorChain(path), + }; + const ok = await this.services.acl.check(actor, descriptor, 'write'); + if (!ok) throw new HttpError(403, 'Permission denied'); + } + + // ── Event emission ────────────────────────────────────────────── + + #emitGuiEvent( + eventName: string, + entry: FSEntry, + extra?: Record, + ): void { + const payload = { + user_id_list: [entry.userId], + response: { ...entry, ...extra, from_new_service: true }, + }; + const meta = {}; + void Promise.resolve() + .then(() => this.clients.event.emit(eventName, payload, meta)) + .catch(() => { + // non-critical + }); + } + + // ── Misc helpers ──────────────────────────────────────────────── + + #parseDestination(req: Request): string { + const dest = req.headers.destination as string | undefined; + if (!dest) throw new HttpError(400, 'Missing Destination header'); + try { + const url = new URL(dest, `http://${req.headers.host}`); + return decodeURIComponent(url.pathname); + } catch { + return decodeURIComponent(dest); + } + } +} + +// ── XML helpers ────────────────────────────────────────────────────── + +function escapeXml(text: string): string { + return text + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +function wrapMultistatus(inner: string): string { + return `\n\n${inner}\n`; +} + +function propfindEntry( + href: string, + entry: FSEntry | null, + isDir: boolean, +): string { + const encodedHref = + encodeURI(href) + (isDir && !href.endsWith('/') ? '/' : ''); + const modified = + entry?.modified ?? entry?.created ?? '2025-01-01T00:00:00Z'; + const created = entry?.created ?? '2025-01-01T00:00:00Z'; + const name = entry?.name ?? (pathPosix.basename(href) || '/'); + const uid = entry?.uuid ?? 'root'; + const modTs = Math.floor(new Date(modified as string).getTime()); + + let props = ` + ${escapeXml(String(name))} + ${new Date(modified as string).toUTCString()} + ${new Date(created as string).toISOString()} + ${isDir ? '' : ''} + "${uid}-${modTs}" + + + + + + 0`; + + if (!isDir && entry) { + props += `\n ${entry.size ?? 0}`; + const mime = mimeFromExt(pathPosix.extname(entry.name)); + props += `\n ${escapeXml(mime)}`; + } + + return ` + ${escapeXml(encodedHref)} + + ${props} + + HTTP/1.1 200 OK + + `; +} + +const MIME_MAP: Record = { + '.html': 'text/html', + '.htm': 'text/html', + '.css': 'text/css', + '.js': 'application/javascript', + '.mjs': 'application/javascript', + '.json': 'application/json', + '.xml': 'application/xml', + '.svg': 'image/svg+xml', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.ico': 'image/x-icon', + '.pdf': 'application/pdf', + '.txt': 'text/plain', + '.md': 'text/markdown', + '.csv': 'text/csv', + '.mp3': 'audio/mpeg', + '.mp4': 'video/mp4', + '.webm': 'video/webm', + '.zip': 'application/zip', + '.wasm': 'application/wasm', +}; + +function mimeFromExt(ext: string): string { + return MIME_MAP[ext.toLowerCase()] ?? 'application/octet-stream'; +} + +function lockResponseXml( + token: string, + path: string, + scope: 'exclusive' | 'shared', +): string { + return ` + + + + + + 0 + webdav-user + Second-7200 + ${escapeXml(token)} + ${escapeXml(encodeURI(path))} + + +`; +} diff --git a/src/backend/controllers/webdav/locks.ts b/src/backend/controllers/webdav/locks.ts new file mode 100644 index 000000000..27588421f --- /dev/null +++ b/src/backend/controllers/webdav/locks.ts @@ -0,0 +1,208 @@ +import { randomUUID } from 'node:crypto'; +import { posix as pathPosix } from 'node:path'; +import type { Cluster } from 'ioredis'; + +/** + * Redis-backed WebDAV lock store. + * + * Two key families: + * `dav:lock:` → JSON `{ path, lockScope, lockType }` (per-token metadata) + * `dav:locks:` → JSON `{ : { lockScope, lockType }, ... }` (per-path map) + * + * Both keys share the same TTL so they expire together. + */ + +const DAV_LOCK_TTL_SECONDS = 30; +const LOCK_PREFIX = 'dav:lock:'; +const LOCKS_PREFIX = 'dav:locks:'; + +export interface LockInfo { + lockToken: string; + lockScope: 'exclusive' | 'shared'; + lockType: 'write'; + path: string; +} + +export async function createLock( + redis: Cluster, + filePath: string, + lockScope: 'exclusive' | 'shared', + lockType: 'write' = 'write', +): Promise { + const lockToken = `urn:uuid:${randomUUID()}`; + const meta = JSON.stringify({ path: filePath, lockScope, lockType }); + + // Per-token metadata + await redis.set( + `${LOCK_PREFIX}${lockToken}`, + meta, + 'EX', + DAV_LOCK_TTL_SECONDS, + ); + + // Per-path map — merge with any existing locks on this path + const existing = await getPathLockMap(redis, filePath); + existing[lockToken] = { lockScope, lockType }; + await redis.set( + `${LOCKS_PREFIX}${filePath}`, + JSON.stringify(existing), + 'EX', + DAV_LOCK_TTL_SECONDS, + ); + + return lockToken; +} + +export async function deleteLock( + redis: Cluster, + lockToken: string, +): Promise { + const raw = await redis.get(`${LOCK_PREFIX}${lockToken}`); + if (raw) { + const meta = JSON.parse(raw) as { path: string }; + const pathMap = await getPathLockMap(redis, meta.path); + delete pathMap[lockToken]; + if (Object.keys(pathMap).length === 0) { + await redis.del(`${LOCKS_PREFIX}${meta.path}`); + } else { + await redis.set( + `${LOCKS_PREFIX}${meta.path}`, + JSON.stringify(pathMap), + 'EX', + DAV_LOCK_TTL_SECONDS, + ); + } + } + await redis.del(`${LOCK_PREFIX}${lockToken}`); +} + +export async function refreshLock( + redis: Cluster, + lockToken: string, +): Promise { + const raw = await redis.get(`${LOCK_PREFIX}${lockToken}`); + if (!raw) return false; + const meta = JSON.parse(raw) as { path: string }; + + // Re-set with fresh TTL + await redis.set( + `${LOCK_PREFIX}${lockToken}`, + raw, + 'EX', + DAV_LOCK_TTL_SECONDS, + ); + // Refresh the path map TTL too + const pathRaw = await redis.get(`${LOCKS_PREFIX}${meta.path}`); + if (pathRaw) { + await redis.set( + `${LOCKS_PREFIX}${meta.path}`, + pathRaw, + 'EX', + DAV_LOCK_TTL_SECONDS, + ); + } + return true; +} + +/** + * Get all active locks on a path, including inherited locks from + * ancestor directories. + */ +export async function getFileLocks( + redis: Cluster, + filePath: string, +): Promise { + const results: LockInfo[] = []; + // Walk up the path hierarchy + let current = filePath; + for (;;) { + const map = await getPathLockMap(redis, current); + for (const [token, info] of Object.entries(map)) { + results.push({ + lockToken: token, + lockScope: (info as { lockScope: 'exclusive' | 'shared' }) + .lockScope, + lockType: (info as { lockType: 'write' }).lockType, + path: current, + }); + } + if (current === '/') break; + current = pathPosix.dirname(current); + } + return results; +} + +/** + * Verify a lock token is still valid and return its metadata. + */ +export async function getLockIfValid( + redis: Cluster, + lockToken: string, +): Promise { + const raw = await redis.get(`${LOCK_PREFIX}${lockToken}`); + if (!raw) return null; + const meta = JSON.parse(raw) as { + path: string; + lockScope: 'exclusive' | 'shared'; + lockType: 'write'; + }; + return { lockToken, ...meta }; +} + +/** + * Check whether the caller has write permission under WebDAV locking + * rules. Returns true if the write is allowed. + */ +export async function hasWritePermission( + redis: Cluster, + filePath: string, + headerLockToken: string | null, +): Promise { + const locks = await getFileLocks(redis, filePath); + if (locks.length === 0) return true; // no locks → allowed + if (!headerLockToken) return false; // locks exist but no token → denied + + // Verify the provided token + const myLock = await getLockIfValid(redis, headerLockToken); + if (!myLock) return false; // token expired or invalid + + // Token's path must match or be an ancestor of the target + if (!filePath.startsWith(myLock.path) && myLock.path !== filePath) { + return false; + } + + // Check lock scope rules + for (const lock of locks) { + if (lock.lockToken === headerLockToken) continue; // skip our own lock + if (lock.lockScope === 'exclusive') return false; // blocked by another exclusive + } + return true; +} + +// ── Internals ─────────────────────────────────────────────────────── + +async function getPathLockMap( + redis: Cluster, + filePath: string, +): Promise> { + const raw = await redis.get(`${LOCKS_PREFIX}${filePath}`); + if (!raw) return {}; + try { + return JSON.parse(raw) as Record< + string, + { lockScope: string; lockType: string } + >; + } catch { + return {}; + } +} + +/** + * Extract a lock token from the `If` or `Lock-Token` header. + * Formats: `()` or `` or just `urn:uuid:...` + */ +export function extractLockToken(header: string | undefined): string | null { + if (!header) return null; + const match = header.match(/?/); + return match?.[1] ?? null; +} diff --git a/src/backend/controllers/wisp/WispController.ts b/src/backend/controllers/wisp/WispController.ts new file mode 100644 index 000000000..92bd27d49 --- /dev/null +++ b/src/backend/controllers/wisp/WispController.ts @@ -0,0 +1,102 @@ +import type { Request, Response } from 'express'; +import { HttpError } from '../../core/http/HttpError.js'; +import type { PuterRouter } from '../../core/http/PuterRouter.js'; +import { PuterController } from '../types.js'; + +/** + * WISP relay token controller — create and verify short-lived JWT tokens + * for the WISP network proxy. + * + * Config: `config.wisp.server` — WISP relay server address. + */ +export class WispController extends PuterController { + registerRoutes(router: PuterRouter): void { + router.post( + '/wisp/relay-token/create', + { subdomain: 'api', requireAuth: true }, + this.#create, + ); + router.post( + '/wisp/relay-token/verify', + { subdomain: 'api', requireAuth: false }, + this.#verify, + ); + } + + /** POST /wisp/relay-token/create — mint a relay token (auth optional). */ + #create = async (req: Request, res: Response): Promise => { + const actor = req.actor; + const wispCfg = this.#wispConfig(); + + if (actor?.user?.uuid) { + const token = this.services.token.sign( + 'wisp', + { + $: 'token:wisp', + $v: '0.0.0', + user_uid: actor.user.uuid, + }, + { expiresIn: '1d' }, + ); + res.json({ token, server: wispCfg.server ?? null }); + } else { + const token = this.services.token.sign( + 'wisp', + { + $: 'token:wisp', + $v: '0.0.0', + guest: true, + }, + { expiresIn: '1d' }, + ); + res.json({ token, server: wispCfg.server ?? null }); + } + }; + + /** POST /wisp/relay-token/verify — verify a relay token and apply policy. */ + #verify = async (req: Request, res: Response): Promise => { + const bodyToken = req.body?.token; + if (!bodyToken || typeof bodyToken !== 'string') { + throw new HttpError(400, 'Missing `token`'); + } + + let decoded: Record; + try { + decoded = this.services.token.verify>( + 'wisp', + bodyToken, + ); + if (decoded.$ !== 'token:wisp') throw new Error('wrong token type'); + } catch { + throw new HttpError(403, 'Forbidden'); + } + + // Build policy event — extensions can deny via extension.on('wisp.get-policy') + const isGuest = Boolean(decoded.guest); + let user: Record | null = null; + if (!isGuest && decoded.user_uid) { + user = await this.stores.user.getByUuid(String(decoded.user_uid)); + } + + const event: Record = { + allow: true, + policy: { allow: true }, + guest: isGuest, + user, + }; + // emitAndWait so async listeners can fetch policy data before + // mutating `event.allow` / `event.policy`; plain emit would return + // control before any awaited work completed. + await this.clients.event.emitAndWait('wisp.get-policy', event, {}); + + if (!event.allow) { + throw new HttpError(403, 'Forbidden'); + } + + res.json(event.policy); + }; + + #wispConfig(): NonNullable { + return this.config.wisp ?? {}; + } +} diff --git a/src/backend/core/actor.ts b/src/backend/core/actor.ts new file mode 100644 index 000000000..871943d79 --- /dev/null +++ b/src/backend/core/actor.ts @@ -0,0 +1,99 @@ +/** + * Minimal actor shape used by stores and services. + * + * Stores/services that key data on "who's acting" need the user/app identity, + * plus a `system` flag for internal operations that should bypass quotas + * and metering. + */ + +export interface ActorUser { + uuid: string; + id?: number; + username?: string; + email?: string | null; + /** True when the account has been suspended by an admin. */ + suspended?: boolean; + /** True when the user has confirmed the email on file. */ + email_confirmed?: boolean; + /** True for accounts that must confirm email before most actions (non-temp users). */ + requires_email_confirmation?: boolean; +} + +export interface ActorApp { + uid: string; + id?: number; +} + +/** + * Access-token wrapper. When set, this actor is acting *through* an access + * token issued by `issuer`. The token's row in `access_token_permissions` + * gates which permissions of the issuer it can exercise. + */ +export interface ActorAccessToken { + uid: string; + issuer: Actor; + authorized?: Actor | null; +} + +export interface Actor { + user: ActorUser; + app?: ActorApp | null; + /** True for the system actor; skips metering / quota tracking. */ + system?: boolean; + accessToken?: ActorAccessToken | null; + /** + * Session reference when authenticated via a session token (user actors) + * or an app-under-user token that carries a session. Absent for system, + * raw-app, and pure access-token actors. Used for session introspection + * and targeted logout. + */ + session?: { uid: string } | null; +} + +/** UUID of the baked-in system user (see 0025 seed migration). */ +export const SYSTEM_ACTOR_UUID = '5d4adce0-a381-4982-9c02-6e2540026238'; + +/** The default system actor used when no actor is supplied. */ +export const SYSTEM_ACTOR: Actor = { + user: { uuid: SYSTEM_ACTOR_UUID, username: 'system' }, + system: true, +}; + +export const isSystemActor = (actor: Actor | undefined | null): boolean => { + return !!actor?.system || actor?.user?.uuid === SYSTEM_ACTOR_UUID; +}; + +export const isAppActor = (actor: Actor | undefined | null): boolean => { + return !!actor?.app && !actor?.accessToken; +}; + +export const isAccessTokenActor = ( + actor: Actor | undefined | null, +): boolean => { + return !!actor?.accessToken; +}; + +/** + * Stable identifier for an actor. + * Used as a cache key (e.g., permission scan cache) and for cycle detection. + */ +export const actorUid = (actor: Actor): string => { + if (actor.accessToken) { + const authorizedUid = actor.accessToken.authorized + ? actorUid(actor.accessToken.authorized) + : ''; + return `access-token:${actorUid(actor.accessToken.issuer)}:${authorizedUid}:${actor.accessToken.uid}`; + } + if (isSystemActor(actor)) return 'system'; + if (actor.app) return `app-under-user:${actor.user.uuid}:${actor.app.uid}`; + return `user:${actor.user.uuid}`; +}; + +/** + * Return a user-only actor for any app-under-user actor. For non-app actors, + * returns the actor unchanged. + */ +export const userRelatedActor = (actor: Actor): Actor => { + if (!actor.app && !actor.accessToken) return actor; + return { user: actor.user }; +}; diff --git a/src/backend/core/context.ts b/src/backend/core/context.ts new file mode 100644 index 000000000..e58052e50 --- /dev/null +++ b/src/backend/core/context.ts @@ -0,0 +1,130 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import type { Request } from 'express'; +import type { Actor } from './actor'; + +/** + * Per-request context with both typed well-known fields AND an open-ended + * key-value map for ad-hoc data. Common fields (`actor`, `req`) are typed for + * autocomplete / safety, while the generic `get`/`set` bag lets any code + * stash per-request values without threading them through function arguments. + * + * Backed by Node's `AsyncLocalStorage`, so the context propagates through + * async/await, timers, and microtasks automatically. The middleware + * (`createRequestContextMiddleware`) wraps each incoming request in a fresh + * context after the auth probe has populated `req.actor`. + * + * Usage: + * ```ts + * // read typed field + * const actor = Context.get('actor'); + * + * // read the express request from anywhere + * const req = Context.get('req'); + * + * // stash / read ad-hoc values + * Context.set('myService.txId', txId); + * const txId = Context.get('myService.txId'); + * ``` + */ + +// ── Well-known typed keys ─────────────────────────────────────────── + +export interface KnownContextFields { + /** The authenticated actor, if one was resolved by the auth probe. */ + actor: Actor | undefined; + /** The express request object for this request. */ + req: Request; + /** A unique id for this request — useful for structured logging / tracing. */ + requestId: string; +} + +// ── Context store ─────────────────────────────────────────────────── + +interface ContextStore { + known: Partial; + extra: Map; +} + +const als = new AsyncLocalStorage(); + +// ── Public API ────────────────────────────────────────────────────── + +/** + * Static-style context accessor. + * + * Well-known keys (`actor`, `req`, `requestId`) return typed values. + * Any other string key hits the generic map and returns `unknown`. + */ +export class Context { + /** + * Get a value from the current request context. + * + * Well-known keys return typed values; arbitrary string keys + * return `unknown`. Returns `undefined` when called outside a + * request scope or when the key hasn't been set. + */ + /** Get the entire context store (no-arg form). */ + static get(): ContextStore | undefined; + static get( + key: K, + ): KnownContextFields[K] | undefined; + static get(key: string): unknown; + static get(key?: string): unknown { + if (key === undefined) return als.getStore(); + const store = als.getStore(); + if (!store) return undefined; + if (key in store.known) { + return (store.known as Record)[key]; + } + return store.extra.get(key); + } + + /** + * Set a value on the current request context. + * + * Well-known keys are type-checked; arbitrary keys accept `unknown`. + */ + static set( + key: K, + value: KnownContextFields[K], + ): void; + static set(key: string, value: unknown): void; + static set(key: string, value: unknown): void { + const store = als.getStore(); + if (!store) { + throw new Error( + `Context.set('${key}', ...) called outside a request scope`, + ); + } + if (key === 'actor' || key === 'req' || key === 'requestId') { + (store.known as Record)[key] = value; + } else { + store.extra.set(key, value); + } + } + + /** + * Returns the full context store, or `undefined` when called outside a + * request scope. Prefer `.get(key)` for individual lookups. + */ + static current(): ContextStore | undefined { + return als.getStore(); + } +} + +// ── Internal: used by the request-context middleware ───────────────── + +/** + * Run `fn` inside a new context scope. Used by the request-context + * middleware to wrap the remainder of the middleware/handler chain. + */ +export const runWithContext = ( + initial: Partial, + fn: () => T, +): T => { + const store: ContextStore = { + known: { ...initial }, + extra: new Map(), + }; + return als.run(store, fn); +}; diff --git a/src/backend/core/http/HttpError.ts b/src/backend/core/http/HttpError.ts new file mode 100644 index 000000000..74917d34c --- /dev/null +++ b/src/backend/core/http/HttpError.ts @@ -0,0 +1,75 @@ +/** + * Options accepted by `HttpError`. All optional. + */ +export interface HttpErrorOptions { + /** Underlying error. Set as the standard `Error.cause`. */ + cause?: unknown; + /** + * Stable wire-format error code that legacy clients key on (e.g. + * `item_with_same_name_exists`, `forbidden`, `subject_does_not_exist`). + * Serialized as `code` in the response body for back-compat. + */ + legacyCode?: string; + /** + * Modern, structured error code. If both `legacyCode` and `code` are set, + * the legacy one takes the `code` slot in the response body and `code` + * is emitted as `errorCode`, so clients keying on either field find + * what they expect. + */ + code?: string; + /** Additional fields merged into the response body. */ + fields?: Record; +} + +/** + * The single error type controllers and services throw to surface an HTTP + * failure. The terminal `errorHandler` middleware catches it, serializes a + * JSON body, and sets the response status. + * + * Usage: + * ```ts + * throw new HttpError(404, 'Item not found'); + * throw new HttpError(409, 'Cannot overwrite directory', { legacyCode: 'is_directory' }); + * throw new HttpError(403, 'Forbidden', { legacyCode: 'forbidden', fields: { target } }); + * ``` + * + * Express 5 forwards thrown errors (sync and async) to error-handling + * middleware automatically — no `next(err)` ceremony required. + */ +export class HttpError extends Error { + readonly statusCode: number; + readonly legacyCode?: string; + readonly code?: string; + readonly fields?: Record; + + constructor( + statusCode: number, + message: string, + options: HttpErrorOptions = {}, + ) { + super( + message, + options.cause !== undefined ? { cause: options.cause } : undefined, + ); + this.name = 'HttpError'; + this.statusCode = statusCode; + this.legacyCode = options.legacyCode; + this.code = options.code; + this.fields = options.fields; + } +} + +/** + * Type guard that survives module-graph duplication (defensive — cross-realm + * `instanceof` can be unreliable in test setups). Pure runtime convenience; + * normal callers can use `instanceof HttpError`. + */ +export const isHttpError = (e: unknown): e is HttpError => { + if (e instanceof HttpError) return true; + return Boolean( + e && + typeof e === 'object' && + (e as { name?: unknown }).name === 'HttpError' && + typeof (e as { statusCode?: unknown }).statusCode === 'number', + ); +}; diff --git a/src/backend/core/http/PuterRouter.ts b/src/backend/core/http/PuterRouter.ts new file mode 100644 index 000000000..75c75832f --- /dev/null +++ b/src/backend/core/http/PuterRouter.ts @@ -0,0 +1,272 @@ +import type { RequestHandler } from 'express'; +import type { + RouteDescriptor, + RouteMethod, + RouteOptions, + RoutePath, + TypedHandler, +} from './types'; + +/** + * Normalized result of argument parsing for either path-required methods + * (`get`, `post`, ...) or the more permissive `use`. + */ +interface NormalizedArgs { + path?: RoutePath; + options: RouteOptions; + handler: RequestHandler; +} + +/** + * PuterRouter is a **collector**, not an active express router. + * + * Controllers call familiar express-shaped methods (`router.get(...)`, + * `router.post(...)`, ...); the router pushes a `RouteDescriptor` onto + * `routes`. `PuterServer` then walks each controller's routes and + * materializes them into real express handlers, applying middleware + * derived from the per-route `options` plus any caller-supplied + * `options.middleware` chain. + * + * Keeping registration purely declarative means: + * - Decorator-style and imperative-style controllers share one target. + * - New per-route options (auth, subdomain, body parsing) can be added + * without touching any call site. + * - The router has no dependency on an express app — useful for tests + * and for controllers constructed before the server is wired. + */ +export class PuterRouter { + readonly prefix: string; + readonly routes: RouteDescriptor[] = []; + + constructor(prefix: string = '') { + this.prefix = prefix; + } + + // ── use ───────────────────────────────────────────────────────── + // + // `use` is the only method whose path is optional (global-ish + // middleware) and whose options can appear with or without a path. + // All four overloads route into `#parseUseArgs`. + + use(handler: RequestHandler): this; + use(options: RouteOptions, handler: RequestHandler): this; + use(path: RoutePath, handler: RequestHandler): this; + use(path: RoutePath, options: RouteOptions, handler: RequestHandler): this; + use(...args: unknown[]): this { + const normalized = this.#parseUseArgs(args); + this.routes.push({ method: 'use', ...normalized }); + return this; + } + + // ── HTTP verbs + WebDAV ───────────────────────────────────────── + // + // All take `(path, handler)` or `(path, options, handler)`. + + all(path: RoutePath, handler: RequestHandler): this; + all( + path: RoutePath, + options: O, + handler: TypedHandler, + ): this; + all(...args: unknown[]): this { + return this.#push('all', args); + } + + get(path: RoutePath, handler: RequestHandler): this; + get( + path: RoutePath, + options: O, + handler: TypedHandler, + ): this; + get(...args: unknown[]): this { + return this.#push('get', args); + } + + head(path: RoutePath, handler: RequestHandler): this; + head( + path: RoutePath, + options: O, + handler: TypedHandler, + ): this; + head(...args: unknown[]): this { + return this.#push('head', args); + } + + post(path: RoutePath, handler: RequestHandler): this; + post( + path: RoutePath, + options: O, + handler: TypedHandler, + ): this; + post(...args: unknown[]): this { + return this.#push('post', args); + } + + put(path: RoutePath, handler: RequestHandler): this; + put( + path: RoutePath, + options: O, + handler: TypedHandler, + ): this; + put(...args: unknown[]): this { + return this.#push('put', args); + } + + delete(path: RoutePath, handler: RequestHandler): this; + delete( + path: RoutePath, + options: O, + handler: TypedHandler, + ): this; + delete(...args: unknown[]): this { + return this.#push('delete', args); + } + + patch(path: RoutePath, handler: RequestHandler): this; + patch( + path: RoutePath, + options: O, + handler: TypedHandler, + ): this; + patch(...args: unknown[]): this { + return this.#push('patch', args); + } + + options(path: RoutePath, handler: RequestHandler): this; + options( + path: RoutePath, + options: O, + handler: TypedHandler, + ): this; + options(...args: unknown[]): this { + return this.#push('options', args); + } + + lock(path: RoutePath, handler: RequestHandler): this; + lock( + path: RoutePath, + options: O, + handler: TypedHandler, + ): this; + lock(...args: unknown[]): this { + return this.#push('lock', args); + } + + unlock(path: RoutePath, handler: RequestHandler): this; + unlock( + path: RoutePath, + options: O, + handler: TypedHandler, + ): this; + unlock(...args: unknown[]): this { + return this.#push('unlock', args); + } + + propfind(path: RoutePath, handler: RequestHandler): this; + propfind( + path: RoutePath, + options: O, + handler: TypedHandler, + ): this; + propfind(...args: unknown[]): this { + return this.#push('propfind', args); + } + + proppatch(path: RoutePath, handler: RequestHandler): this; + proppatch( + path: RoutePath, + options: O, + handler: TypedHandler, + ): this; + proppatch(...args: unknown[]): this { + return this.#push('proppatch', args); + } + + mkcol(path: RoutePath, handler: RequestHandler): this; + mkcol( + path: RoutePath, + options: O, + handler: TypedHandler, + ): this; + mkcol(...args: unknown[]): this { + return this.#push('mkcol', args); + } + + copy(path: RoutePath, handler: RequestHandler): this; + copy( + path: RoutePath, + options: O, + handler: TypedHandler, + ): this; + copy(...args: unknown[]): this { + return this.#push('copy', args); + } + + move(path: RoutePath, handler: RequestHandler): this; + move( + path: RoutePath, + options: O, + handler: TypedHandler, + ): this; + move(...args: unknown[]): this { + return this.#push('move', args); + } + + // ── Internals ─────────────────────────────────────────────────── + + #push(method: RouteMethod, args: unknown[]): this { + const normalized = this.#parsePathArgs(args); + this.routes.push({ method, ...normalized }); + return this; + } + + #parsePathArgs(args: unknown[]): NormalizedArgs { + // (path, handler) — two args, handler is last + if (args.length === 2) { + return { + path: args[0] as RoutePath, + options: {}, + handler: args[1] as RequestHandler, + }; + } + // (path, options, handler) + return { + path: args[0] as RoutePath, + options: (args[1] as RouteOptions) ?? {}, + handler: args[2] as RequestHandler, + }; + } + + #parseUseArgs(args: unknown[]): NormalizedArgs { + if (args.length === 1) { + // use(handler) + return { options: {}, handler: args[0] as RequestHandler }; + } + if (args.length === 2) { + const [first, second] = args; + // Path-like first arg: string, RegExp, or array of those. + if ( + typeof first === 'string' || + first instanceof RegExp || + Array.isArray(first) + ) { + return { + path: first as RoutePath, + options: {}, + handler: second as RequestHandler, + }; + } + // Otherwise the first arg is options. + return { + options: (first as RouteOptions) ?? {}, + handler: second as RequestHandler, + }; + } + // use(path, options, handler) + return { + path: args[0] as RoutePath, + options: (args[1] as RouteOptions) ?? {}, + handler: args[2] as RequestHandler, + }; + } +} diff --git a/src/backend/core/http/__typecheck__.ts b/src/backend/core/http/__typecheck__.ts new file mode 100644 index 000000000..2c1bd5052 --- /dev/null +++ b/src/backend/core/http/__typecheck__.ts @@ -0,0 +1,69 @@ +/** + * Compile-time narrowing checks for PuterRouter type ergonomics. + * + * This file is *only* here to fail the typecheck if the const-generic + * narrowing on `req.actor` regresses. It produces no runtime artifacts of + * interest. Delete it whenever a real test suite for the router exists. + */ +import type { Actor } from '../actor'; +import { PuterRouter } from './PuterRouter'; + +const r = new PuterRouter(); + +// No options → req.actor: Actor | undefined +r.get('/anon', (req, _res) => { + const a: Actor | undefined = req.actor; + void a; + // Negative: assigning the (possibly-undefined) actor to a non-null + // `Actor` should error. If this `@ts-expect-error` comment ever stops + // firing, narrowing is being applied where it shouldn't be. + // @ts-expect-error req.actor is Actor | undefined here + const b: Actor = req.actor; + void b; +}); + +// requireAuth: true → req.actor: Actor (non-null) +r.get('/auth', { requireAuth: true }, (req, _res) => { + const a: Actor = req.actor; + void a; + void req.actor.user.username; +}); + +// requireUserActor → req.actor: Actor +r.post('/me', { requireUserActor: true }, (req, _res) => { + const a: Actor = req.actor; + void a; +}); + +// adminOnly: true → req.actor: Actor +r.post('/admin', { adminOnly: true }, (req, _res) => { + const a: Actor = req.actor; + void a; +}); + +// adminOnly: extras array → req.actor: Actor +r.post('/admin-extras', { adminOnly: ['mod'] }, (req, _res) => { + const a: Actor = req.actor; + void a; +}); + +// allowedAppIds → req.actor: Actor +r.post('/from-app', { allowedAppIds: ['app-x'] }, (req, _res) => { + const a: Actor = req.actor; + void a; +}); + +// Just a subdomain gate (no auth implied) → req.actor: Actor | undefined +r.get('/subdomain', { subdomain: 'api' }, (req, _res) => { + const a: Actor | undefined = req.actor; + void a; +}); + +// Variable-typed options (boolean, not literal true) → no narrowing, +// req.actor stays Actor | undefined. This intentionally stays loose: +// dynamic options can't be reflected at the type level. +const dynamicOpts = { requireAuth: true as boolean }; +r.get('/dyn', dynamicOpts, (req, _res) => { + const a: Actor | undefined = req.actor; + void a; +}); diff --git a/src/backend/core/http/decorators.ts b/src/backend/core/http/decorators.ts new file mode 100644 index 000000000..f5584e075 --- /dev/null +++ b/src/backend/core/http/decorators.ts @@ -0,0 +1,152 @@ +import type { RequestHandler } from 'express'; +import type { PuterRouter } from './PuterRouter'; +import { + PREFIX_METADATA_KEY, + ROUTES_METADATA_KEY, + type CollectedRoute, + type RouteMethod, + type RouteOptions, + type RoutePath, +} from './types'; + +/** + * Decorator-style route registration for controllers that prefer annotations + * over imperative `registerRoutes(router)` bodies. + * + * Stage-3 decorators (TS 5+), matching the extensionController pattern. Every + * method decorator pushes a `CollectedRoute` onto `prototype.__puterRoutes` + * during class initialization. `@Controller` seals the deal by installing a + * `registerRoutes` method on the prototype that walks the collected routes + * and feeds them to the `PuterRouter` passed in by `PuterServer`. + * + * Usage is optional — imperative controllers that override `registerRoutes` + * directly work equally well. + */ + +// ── Prototype shape helpers ───────────────────────────────────────── + +interface DecoratedPrototype { + [ROUTES_METADATA_KEY]?: CollectedRoute[]; + [PREFIX_METADATA_KEY]?: string; + registerRoutes?: (router: PuterRouter) => void; +} + +const getOrInitRoutes = (proto: DecoratedPrototype): CollectedRoute[] => { + if (!proto[ROUTES_METADATA_KEY]) { + proto[ROUTES_METADATA_KEY] = []; + } + return proto[ROUTES_METADATA_KEY]!; +}; + +// ── @Controller ───────────────────────────────────────────────────── + +/** + * Class decorator. + * + * - Stores the controller's path `prefix` on the prototype so `PuterServer` + * can construct a correctly-prefixed `PuterRouter` for this controller. + * - Installs a default `registerRoutes(router)` on the prototype that walks + * routes collected by method decorators (if this class hasn't defined its + * own `registerRoutes`). This means a purely-decorated controller needs + * no body — the decorators do all the wiring. + * + * Controllers that define their own `registerRoutes` are untouched; they can + * still use `@Post` etc. and walk `prototype[ROUTES_METADATA_KEY]` manually + * if they want to combine the styles. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type AnyCtor = new (...args: any[]) => any; + +export function Controller(prefix: string = '') { + return ( + value: T, + _context: ClassDecoratorContext, + ): void => { + const proto = value.prototype as DecoratedPrototype; + proto[PREFIX_METADATA_KEY] = prefix; + + // Only install the default walker if the class itself hasn't + // defined registerRoutes. We check *own* properties (not inherited) + // so a PuterController base-class default doesn't block us. + const hasOwnRegister = Object.prototype.hasOwnProperty.call( + proto, + 'registerRoutes', + ); + if (hasOwnRegister) return; + + proto.registerRoutes = function (router: PuterRouter): void { + const routes = ((this as DecoratedPrototype)[ROUTES_METADATA_KEY] ?? + []) as CollectedRoute[]; + for (const r of routes) { + const bound = r.handler.bind(this) as RequestHandler; + if (r.method === 'use') { + if (r.path !== undefined) { + router.use(r.path, r.options, bound); + } else { + router.use(r.options, bound); + } + continue; + } + if (r.path === undefined) { + // A non-use method without a path is a mistake in the decorator + // call site; surface it loudly rather than silently dropping. + throw new Error( + `@${r.method.toUpperCase()} decorator missing path`, + ); + } + // Delegate to the appropriately-named method on the router. + // The method set is enumerated in `RouteMethod` so this cast is safe. + const routerMethod = router[ + r.method as Exclude + ] as ( + path: RoutePath, + options: RouteOptions, + handler: RequestHandler, + ) => PuterRouter; + routerMethod.call(router, r.path, r.options, bound); + } + }; + }; +} + +// ── Method decorators (@Get, @Post, ...) ─────────────────────────── + +// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type +type AnyMethod = Function; + +const createMethodDecorator = (method: Exclude) => { + return (path: RoutePath, options: RouteOptions = {}) => { + return ( + target: AnyMethod, + context: ClassMethodDecoratorContext, + ): void => { + context.addInitializer(function () { + const proto = Object.getPrototypeOf( + this as object, + ) as DecoratedPrototype; + getOrInitRoutes(proto).push({ + method, + path, + options, + handler: target as unknown as RequestHandler, + }); + }); + }; + }; +}; + +export const All = createMethodDecorator('all'); +export const Get = createMethodDecorator('get'); +export const Head = createMethodDecorator('head'); +export const Post = createMethodDecorator('post'); +export const Put = createMethodDecorator('put'); +export const Delete = createMethodDecorator('delete'); +export const Patch = createMethodDecorator('patch'); +export const Options = createMethodDecorator('options'); +export const Lock = createMethodDecorator('lock'); +export const Unlock = createMethodDecorator('unlock'); +export const Propfind = createMethodDecorator('propfind'); +export const Proppatch = createMethodDecorator('proppatch'); +export const Mkcol = createMethodDecorator('mkcol'); +export const Copy = createMethodDecorator('copy'); +export const Move = createMethodDecorator('move'); diff --git a/src/backend/core/http/expressAugmentation.ts b/src/backend/core/http/expressAugmentation.ts new file mode 100644 index 000000000..1330f5bb8 --- /dev/null +++ b/src/backend/core/http/expressAugmentation.ts @@ -0,0 +1,57 @@ +import type { Actor } from '../actor'; + +/** + * Global Express.Request augmentation for v2. + * + * Every field declared here is populated by *global* middleware installed by + * `PuterServer` (auth probe, body parser, etc.). Per-route fields stay local + * to their handlers via `TypedRequest` instead. + * + * This module is import-only — it has no runtime exports. Files that consume + * the augmented `Request` should `import './expressAugmentation'` (or any + * file that imports it transitively) so TypeScript loads the declaration. + */ + +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace Express { + interface Request { + /** + * Populated by the global auth probe when a valid token is + * attached to the request (Bearer header, auth_token body/query, + * session cookie, or socket handshake). Absent for anonymous + * requests — route-level gates decide whether to reject. + */ + actor?: Actor; + + /** The raw token string, if one was presented and parsed. */ + token?: string; + + /** + * Raw request body bytes, captured by the global JSON parser's + * `verify` callback. Available for any JSON request — needed by + * webhook handlers that verify an HMAC over the exact bytes the + * sender signed (e.g., AppStore prod webhooks, BroadcastService + * inter-instance hooks). + * + * Set only when the global JSON parser actually ran (request had + * `Content-Type: application/json` or one of the recognized + * JSON-as-text variants). For non-JSON requests it stays + * `undefined`. + */ + rawBody?: Buffer; + + /** Parsed user-agent, populated by the global UA-parsing middleware. */ + ua?: { + browser: { name?: string; version?: string; major?: string }; + os: { name?: string; version?: string }; + device: { vendor?: string; model?: string; type?: string }; + }; + + /** True when the request's Host is a custom domain (not one of the configured Puter domains). */ + is_custom_domain?: boolean; + } + } +} + +export {}; diff --git a/src/backend/core/http/index.ts b/src/backend/core/http/index.ts new file mode 100644 index 000000000..e3a90376c --- /dev/null +++ b/src/backend/core/http/index.ts @@ -0,0 +1,42 @@ +export { HttpError, isHttpError, type HttpErrorOptions } from './HttpError'; +export { PuterRouter } from './PuterRouter'; +export { + All, + Controller, + Copy, + Delete, + Get, + Head, + Lock, + Mkcol, + Move, + Options, + Patch, + Post, + Propfind, + Proppatch, + Put, + Unlock, +} from './decorators'; +export { + PREFIX_METADATA_KEY, + ROUTES_METADATA_KEY, + type AuthRequired, + type CollectedRoute, + type RouteDescriptor, + type RouteMethod, + type RouteOptions, + type RoutePath, + type TypedHandler, + type TypedRequest, +} from './types'; +export { + DEFAULT_ADMIN_USERNAMES, + adminOnlyGate, + allowedAppIdsGate, + requireAuthGate, + requireUserActorGate, + subdomainGate, +} from './middleware/gates'; +export { createErrorHandler } from './middleware/errorHandler'; +export { createNotFoundHandler } from './middleware/notFoundHandler'; diff --git a/src/backend/core/http/middleware/antiCsrf.js b/src/backend/core/http/middleware/antiCsrf.js new file mode 100644 index 000000000..8b2ebcd88 --- /dev/null +++ b/src/backend/core/http/middleware/antiCsrf.js @@ -0,0 +1,76 @@ +import crypto from 'node:crypto'; +import { HttpError } from '../HttpError.js'; + +/** + * Anti-CSRF token manager (Redis-backed). + * + * One key per token: `csrf::` with TTL. Consume is + * DEL — returns 1 if it existed (and we just consumed it), 0 otherwise. + * Atomic across a cluster, no MULTI needed since each op touches a + * single key. + * + * Tokens expire after `TOKEN_TTL_MS` whether consumed or not. + */ + +const TOKEN_TTL_MS = 10 * 60_000; // 10 minutes + +let redisClient = null; + +/** Call once during server boot with `clients.redis`. */ +export function setAntiCsrfRedis(redis) { + redisClient = redis; +} + +const keyFor = (sessionId, token) => `csrf:${sessionId}:${token}`; + +export const antiCsrf = { + async createToken(sessionId) { + if (!redisClient) + throw new Error('anti-csrf: redis client not configured'); + const token = crypto.randomBytes(32).toString('hex'); + await redisClient.set( + keyFor(sessionId, token), + '1', + 'PX', + TOKEN_TTL_MS, + ); + return token; + }, + async consumeToken(sessionId, token) { + if (!token || !sessionId) return false; + if (!redisClient) + throw new Error('anti-csrf: redis client not configured'); + const removed = await redisClient.del(keyFor(sessionId, token)); + return Number(removed) === 1; + }, +}; + +// ── Route middleware ──────────────────────────────────────────────── + +/** + * Middleware that requires a valid anti-CSRF token in `req.body.anti_csrf`. + * The session key is `req.actor.user.uuid`. + */ +export function requireAntiCsrf() { + return async (req, _res, next) => { + try { + const sessionId = req.actor?.user?.uuid; + if (!sessionId) { + return next( + new HttpError( + 401, + 'Authentication required for CSRF protection.', + ), + ); + } + if ( + !(await antiCsrf.consumeToken(sessionId, req.body?.anti_csrf)) + ) { + return next(new HttpError(400, 'Incorrect anti-CSRF token.')); + } + next(); + } catch (err) { + next(err); + } + }; +} diff --git a/src/backend/core/http/middleware/authProbe.ts b/src/backend/core/http/middleware/authProbe.ts new file mode 100644 index 000000000..6e05b56ad --- /dev/null +++ b/src/backend/core/http/middleware/authProbe.ts @@ -0,0 +1,152 @@ +import type { Request, RequestHandler } from 'express'; +import type { AuthService } from '../../../services/auth/AuthService'; + +// Ensure the `Request.actor` / `Request.token` augmentation is in scope +// wherever this middleware is imported. +import '../expressAugmentation'; + +interface AuthProbeOptions { + authService: AuthService; + /** Name of the session cookie to inspect. Falls back to `config.cookie_name`. */ + cookieName?: string; +} + +/** + * Non-enforcing auth probe. Runs globally (installed by `PuterServer`) on + * every request, tries to locate a token in the usual places, and — if one + * is present and valid — attaches an `Actor` to `req.actor`. + * + * Key property: this middleware **never rejects**. Missing tokens, malformed + * tokens, expired tokens, tokens pointing at deleted users — all result in + * `req.actor` being left undefined. Per-route gates decide whether absence + * is acceptable. + * + * Token lookup order: + * 1. `req.body.auth_token` + * 2. `Authorization: Bearer ` header + * 3. `x-api-key` header — third-party SDK convention (Anthropic etc.) + * 4. Session cookie + * 5. `?auth_token=...` query param + * 6. Socket handshake query (for ws upgrades that pass through HTTP first) + */ +export const createAuthProbe = (opts: AuthProbeOptions): RequestHandler => { + const { authService, cookieName } = opts; + return async (req, _res, next): Promise => { + // If something upstream already attached an actor, respect it. + if (req.actor) { + next(); + return; + } + + const token = extractToken(req, cookieName); + if (!token) { + next(); + return; + } + + try { + const actor = await authService.authenticateFromToken(token); + if (actor) { + req.actor = actor; + req.token = token; + } + } catch { + // Probe never rejects — invalid tokens just leave `req.actor` undefined. + } + next(); + }; +}; + +/** + * Token extraction logic covering the request sources clients use to + * authenticate. + */ +const extractToken = (req: Request, cookieName?: string): string | null => { + // 1. Body (`{ "auth_token": "..." }`) + const bodyToken = (req.body as { auth_token?: unknown } | undefined) + ?.auth_token; + if (typeof bodyToken === 'string' && bodyToken.length > 0) { + return stripBearer(bodyToken); + } + + // 2. Authorization header. Reject `Basic ...` (not our scheme) and + // the bare word `Bearer` (sent by some Office clients as a placeholder). + const authHeader = + typeof req.header === 'function' + ? req.header('Authorization') + : undefined; + if ( + typeof authHeader === 'string' && + !authHeader.startsWith('Basic ') && + authHeader !== 'Bearer' + ) { + const stripped = authHeader.replace(/^Bearer\s+/i, '').trim(); + if (stripped.length > 0 && stripped !== 'undefined') { + return stripped; + } + } + + // 3. `x-api-key` header — some third-party SDKs (Anthropic's in + // particular) send their API key in this header. Accepted globally + // so every route gated on auth works uniformly for those clients. + const xApiKey = + typeof req.header === 'function' ? req.header('x-api-key') : undefined; + if (typeof xApiKey === 'string' && xApiKey.length > 0) { + return stripBearer(xApiKey); + } + + // 4. Cookie (set by login flow for session tokens). We parse the + // Cookie header directly rather than depending on `cookie-parser` + // middleware — the probe only needs one named value. + if (cookieName) { + const cookieToken = readCookie(req, cookieName); + if (cookieToken) { + return stripBearer(cookieToken); + } + } + + // 5. Query string (used by e.g. QR login, asset URLs). + const queryToken = (req.query as { auth_token?: unknown } | undefined) + ?.auth_token; + if (typeof queryToken === 'string' && queryToken.length > 0) { + return stripBearer(queryToken); + } + + // 6. Socket handshake (for websocket upgrades that pass through HTTP). + const handshake = ( + req as unknown as { handshake?: { query?: { auth_token?: unknown } } } + ).handshake; + const handshakeToken = handshake?.query?.auth_token; + if (typeof handshakeToken === 'string' && handshakeToken.length > 0) { + return stripBearer(handshakeToken); + } + + return null; +}; + +const stripBearer = (t: string): string => t.replace(/^Bearer\s+/i, '').trim(); + +/** + * Minimal cookie reader. Avoids pulling in `cookie-parser` for the one + * lookup the probe needs. Handles quoted values and URL-decodes the result. + */ +const readCookie = (req: Request, name: string): string | null => { + const header = + typeof req.header === 'function' ? req.header('cookie') : undefined; + if (!header || typeof header !== 'string') return null; + const target = `${name}=`; + for (const rawPair of header.split(';')) { + const pair = rawPair.trim(); + if (!pair.startsWith(target)) continue; + let value = pair.slice(target.length); + if (value.startsWith('"') && value.endsWith('"')) { + value = value.slice(1, -1); + } + try { + return decodeURIComponent(value); + } catch { + return value; + } + } + return null; +}; diff --git a/src/backend/core/http/middleware/captcha.js b/src/backend/core/http/middleware/captcha.js new file mode 100644 index 000000000..da3b7767f --- /dev/null +++ b/src/backend/core/http/middleware/captcha.js @@ -0,0 +1,109 @@ +import crypto from 'node:crypto'; +import { HttpError } from '../HttpError.js'; +import svgCaptcha from 'svg-captcha'; +/** + * Simple SVG captcha service — generates image challenges and verifies + * one-time tokens. Tokens are stored in Redis so generation and verification + * can happen on different server nodes. + * + * Exposed as a route option: `{ captcha: true }` on any route. + * The middleware rejects if captcha is enabled and the request + * doesn't carry valid captchaToken + captchaAnswer fields. + * + * When captcha is disabled in config, the middleware is a no-op. + */ + +const EXPIRATION_MS = 10 * 60_000; // 10 minutes +const DIFFICULTY = { + easy: { size: 4, width: 150, height: 50, noise: 1 }, + medium: { size: 6, width: 180, height: 50, noise: 2 }, + hard: { size: 7, width: 200, height: 60, noise: 3 }, +}; + +let redisClient = null; + +/** Call once during server boot with `clients.redis`. */ +export function setCaptchaRedis(redis) { + redisClient = redis; +} + +const keyFor = (token) => `captcha:${token}`; + +function requireRedis() { + if (!redisClient) throw new Error('captcha: redis client not configured'); + return redisClient; +} + +function readTransactionValue(result) { + if (!Array.isArray(result)) return result; + if (result[0]) throw result[0]; + return result[1]; +} + +// ── Public API ────────────────────────────────────────────────────── + +/** Generate a captcha image + token pair. */ +export async function generateCaptcha(difficulty = 'medium') { + if (!svgCaptcha) throw new Error('svg-captcha not available'); + const redis = requireRedis(); + const opts = DIFFICULTY[difficulty] || DIFFICULTY.medium; + const captcha = svgCaptcha.create({ + ...opts, + ignoreChars: '0o1ilI', + color: true, + background: '#f0f0f0', + }); + const token = crypto.randomBytes(32).toString('hex'); + await redis.set( + keyFor(token), + captcha.text.toLowerCase(), + 'PX', + EXPIRATION_MS, + ); + return { token, image: captcha.data }; +} + +/** Verify a captcha answer. One-time use — token is consumed. */ +export async function verifyCaptcha(token, answer) { + if (typeof token !== 'string' || typeof answer !== 'string') return false; + const redis = requireRedis(); + const results = await redis + .multi() + .get(keyFor(token)) + .del(keyFor(token)) + .exec(); + const text = readTransactionValue(results?.[0]); + if (!text) return false; + return text === answer.toLowerCase().trim(); +} + +// ── Route middleware ──────────────────────────────────────────────── + +/** + * Captcha gate middleware factory. + * + * Reads `captchaToken` and `captchaAnswer` from `req.body`. + * Rejects with 400 if missing or invalid. + * + * Pass `enabled` from config — when false, the gate is a no-op. + */ +export function captchaGate(enabled) { + return async (req, _res, next) => { + if (!enabled) return next(); + + try { + const { captchaToken, captchaAnswer } = req.body ?? {}; + if (!captchaToken || !captchaAnswer) { + return next( + new HttpError(400, 'Captcha verification required.'), + ); + } + if (!(await verifyCaptcha(captchaToken, captchaAnswer))) { + return next(new HttpError(400, 'Invalid captcha response.')); + } + next(); + } catch (err) { + next(err); + } + }; +} diff --git a/src/backend/core/http/middleware/errorHandler.ts b/src/backend/core/http/middleware/errorHandler.ts new file mode 100644 index 000000000..f5cd33b1d --- /dev/null +++ b/src/backend/core/http/middleware/errorHandler.ts @@ -0,0 +1,114 @@ +import type { ErrorRequestHandler, RequestHandler } from 'express'; +import { HttpError, isHttpError } from '../HttpError'; + +interface ErrorHandlerOptions { + /** + * Optional logger for non-HttpError failures. Receives `(err, req)`. + * Defaults to `console.error` with the request method/url. + */ + onUnhandled?: (err: unknown, req: Parameters[0]) => void; + /** + * Optional hook fired for every error caught (HttpError and otherwise). + * Use for alarm wiring (e.g., page on 500s) without coupling the + * middleware to a specific service. + */ + onError?: (err: unknown, req: Parameters[0]) => void; +} + +/** + * Terminal express error middleware. Install last, after all routes and + * controllers have been registered. + * + * Express 5 forwards thrown errors (sync and async) here automatically, so + * controllers and gate middlewares can simply `throw new HttpError(...)`. + * + * Response shape is kept for wire-compat with existing clients: + * ```json + * { + * "error": "", + * "message": "", + * "code": "", + * "errorCode": "", + * ...fields + * } + * ``` + * + * `message` is a duplicate of `error` kept for the legacy GUI, which keys on + * `errorJson.message` when parsing auth-window AJAX error responses. + * + * Non-HttpError failures (programming bugs, unexpected exceptions) become + * a generic 500 response — no internal details leak. The full error is + * passed to `onUnhandled` for logging/alerting. + */ +export const createErrorHandler = ( + opts: ErrorHandlerOptions = {}, +): ErrorRequestHandler => { + const onUnhandled = + opts.onUnhandled ?? + ((err, req) => { + console.error( + `[v2] unhandled error on ${req.method} ${req.url}:`, + err, + ); + }); + + return (err, req, res, next): void => { + // If the response already started streaming, we can't send a JSON + // error. Defer to express's default handler to abort the connection. + if (res.headersSent) { + opts.onError?.(err, req); + next(err); + return; + } + + if (isHttpError(err)) { + opts.onError?.(err, req); + res.status(err.statusCode).json(serializeHttpError(err)); + return; + } + + // Anything else is treated as an unexpected 500. We never serialize + // it back to the client to avoid leaking stack traces, internal + // error messages, etc. + opts.onError?.(err, req); + onUnhandled(err, req); + res.status(500).json({ + error: 'Internal Server Error', + message: 'Internal Server Error', + code: 'internal_error', + }); + }; +}; + +const serializeHttpError = (err: HttpError): Record => { + const payload: Record = { + error: err.message, + message: err.message, + }; + + // `code` slot precedence: legacyCode wins for back-compat. If both are + // set, the modern code goes to `errorCode` so clients that key on either + // field find what they expect. + if (err.legacyCode) { + payload.code = err.legacyCode; + if (err.code) payload.errorCode = err.code; + } else if (err.code) { + payload.code = err.code; + } + + if (err.fields) { + for (const [k, v] of Object.entries(err.fields)) { + // Don't let `fields` clobber the canonical slots. + if ( + k === 'error' || + k === 'message' || + k === 'code' || + k === 'errorCode' + ) + continue; + payload[k] = v; + } + } + + return payload; +}; diff --git a/src/backend/core/http/middleware/gates.ts b/src/backend/core/http/middleware/gates.ts new file mode 100644 index 000000000..77e7ad355 --- /dev/null +++ b/src/backend/core/http/middleware/gates.ts @@ -0,0 +1,198 @@ +import type { RequestHandler } from 'express'; +import { HttpError } from '../HttpError'; + +// Make sure the `Express.Request.actor` augmentation is in scope. +import '../expressAugmentation'; + +/** + * Per-route gate middlewares. + * + * These are tiny because they need to be: composability is the point. The + * server's route materializer (`server.ts#materializeRoute`) consults the + * per-route `RouteOptions` and pushes the relevant gate(s) onto the express + * middleware chain in this order: + * + * subdomain → requireAuth (+ suspended check) → requireUserActor → + * adminOnly → allowedAppIds → caller middleware → handler + * + * Each gate either calls `next()` to pass through, calls `next('route')` to + * skip (subdomain only), or throws an `HttpError` for the terminal error + * handler to serialize. Express 5 forwards thrown errors automatically; no + * `next(err)` ceremony required. + */ + +// ── subdomain ─────────────────────────────────────────────────────── + +/** + * Skip this route entirely (via `next('route')`) when the request's + * leftmost subdomain doesn't match. This *isn't* a rejection — it lets + * a different route matcher handle the request. + */ +export const subdomainGate = (allowed: string | string[]): RequestHandler => { + const allowList = Array.isArray(allowed) ? allowed : [allowed]; + return (req, _res, next) => { + // Express `req.subdomains` is reverse-of-URL order; the leftmost + // subdomain (the active one) is the last element. + const active = req.subdomains?.[req.subdomains.length - 1] ?? ''; + if (!allowList.includes(active)) { + next('route'); + return; + } + next(); + }; +}; + +// ── requireAuth (+ suspended check) ───────────────────────────────── + +/** + * Reject anonymous requests with 401. Also reject authenticated-but-suspended + * users with 403 — `actor.user.suspended` is populated by `AuthService` from + * `UserStore`, so the gate doesn't need its own DB hit. + * + * Implied by `requireUserActor`, `adminOnly`, and `allowedAppIds`; the + * materializer ensures only one copy ends up in the chain. + */ +export const requireAuthGate = (): RequestHandler => { + return (req, _res, next) => { + if (!req.actor) { + next( + new HttpError(401, 'Authentication required', { + legacyCode: 'token_required', + }), + ); + return; + } + if (req.actor.user.suspended) { + next( + new HttpError(403, 'Account suspended', { + legacyCode: 'forbidden', + }), + ); + return; + } + next(); + }; +}; + +// ── requireUserActor ──────────────────────────────────────────────── + +/** + * Reject app-under-user and access-token actors with 403. Use on endpoints + * that should only be exercised by a human session — settings changes, + * admin-style actions on the user's own account. + */ +export const requireUserActorGate = (): RequestHandler => { + return (req, _res, next) => { + const actor = req.actor; + // requireAuth runs first; this gate just narrows the actor type. + if (!actor) { + next( + new HttpError(401, 'Authentication required', { + legacyCode: 'token_required', + }), + ); + return; + } + if (actor.app || actor.accessToken) { + next( + new HttpError( + 403, + 'This endpoint is only available to user sessions', + { legacyCode: 'forbidden' }, + ), + ); + return; + } + next(); + }; +}; + +// ── adminOnly ─────────────────────────────────────────────────────── + +/** Built-in admin usernames that always pass `adminOnly`. */ +export const DEFAULT_ADMIN_USERNAMES = ['admin', 'system'] as const; + +/** + * Reject unless `actor.user.username` matches `admin`, `system`, or one of + * the supplied extras. Extras are *additional* allowed users on top of the + * built-in pair, not a replacement for it. + * + * Implies `requireAuth`. Does *not* imply `requireUserActor` — admin + * endpoints are callable via an admin's access token or app-under-user + * actor; combine with `requireUserActor` explicitly if a route must be + * restricted to browser sessions. + */ +export const adminOnlyGate = ( + extras: readonly string[] = [], +): RequestHandler => { + const allowList = new Set([...DEFAULT_ADMIN_USERNAMES, ...extras]); + return (req, _res, next) => { + const username = req.actor?.user.username; + if (!username || !allowList.has(username)) { + next( + new HttpError(403, 'Only admins may request this resource', { + legacyCode: 'forbidden', + }), + ); + return; + } + next(); + }; +}; + +// ── requireVerified ───────────────────────────────────────────────── + +/** + * Reject unless the authenticated user has a confirmed email. Gated behind + * `strict_email_verification_required` config so self-hosted deployments + * without email delivery don't brick their own filesystem routes. + * + * Reads `req.actor?.user?.email_confirmed`, which is present on both + * user-only and app-under-user actors, so it works for either shape. + */ +export const requireVerifiedGate = (strictFlag: boolean): RequestHandler => { + return (req, _res, next) => { + if (!strictFlag) { + next(); + return; + } + const user = req.actor?.user as Record | undefined; + if (!user?.email_confirmed) { + next( + new HttpError(400, 'Account email is not verified', { + legacyCode: 'account_is_not_verified', + }), + ); + return; + } + next(); + }; +}; + +// ── allowedAppIds ─────────────────────────────────────────────────── + +/** + * Reject unless the actor is acting through one of the named apps. + * App-under-user actors are permitted iff `actor.app.uid` is in the allowList; + * non-app actors are rejected. + * + * Implies `requireAuth`. Doesn't pair sensibly with `requireUserActor` + * (a user-only actor has no app), but if both are set we reject loudly here. + */ +export const allowedAppIdsGate = ( + allowedAppUids: readonly string[], +): RequestHandler => { + const allowList = new Set(allowedAppUids); + return (req, _res, next) => { + const appUid = req.actor?.app?.uid; + if (appUid && !allowList.has(appUid)) { + next( + new HttpError(403, 'This app may not request this resource', { + legacyCode: 'forbidden', + }), + ); + return; + } + next(); + }; +}; diff --git a/src/backend/core/http/middleware/hostRedirects.ts b/src/backend/core/http/middleware/hostRedirects.ts new file mode 100644 index 000000000..e49be1f5d --- /dev/null +++ b/src/backend/core/http/middleware/hostRedirects.ts @@ -0,0 +1,135 @@ +import type { RequestHandler } from 'express'; +import { stat } from 'node:fs/promises'; +import path from 'node:path'; +import type { IConfig } from '../../../types'; + +/** Native-app subdomains served via `nativeAppStatic`. */ +const NATIVE_APP_SUBDOMAINS = [ + 'about', + 'developer', + 'docs', + 'editor', + 'markus', + 'pdf', + 'apps', +] as const; + +/** Subset served out of a `dist/` subdirectory rather than the app root. */ +const NATIVE_APPS_WITH_DIST = new Set(['docs', 'developer']); + +/** + * Subdomains that v2 serves itself. Anything NOT in this set that lives on + * the root domain is treated as a user-defined site and redirected to the + * static hosting domain. + * + * Kept as a plain Set so `has()` is O(1); order doesn't matter. + */ +const RESERVED_SUBDOMAINS = new Set([ + 'api', + 'js', + 'dav', + // Native apps (reserved here regardless of whether nativeAppStatic is + // currently installed — the redirect should still skip them). + ...NATIVE_APP_SUBDOMAINS, + // App-icon serving subdomain. + 'puter-app-icons', + // Extension-owned subdomains. + 'onlyoffice', +]); + +/** + * Redirects `www.` → `` (dropping the path). + */ +export const createWwwRedirect = (config: IConfig): RequestHandler => { + const domain = (config.domain ?? '').toLowerCase(); + return (req, res, next) => { + const active = req.subdomains?.[req.subdomains.length - 1] ?? ''; + if (active !== 'www') return next(); + if (!domain) return next(); + res.redirect(`${req.protocol}://${domain}`); + }; +}; + +/** + * Redirects user-defined subdomains on the main domain to the static hosting + * domain. `foo.puter.com/bar?x=1` → `302 foo.puter.site/bar?x=1`. + * + * Passes through when: + * - no active subdomain (root) + * - active subdomain is reserved (api, js, native apps, …) + * - host doesn't end in `config.domain` (custom domains, other hosts) + * - `static_hosting_domain` isn't configured + */ +export const createUserSubdomainRedirect = ( + config: IConfig, +): RequestHandler => { + const domain = (config.domain ?? '').toLowerCase(); + const target = (config.static_hosting_domain ?? '').toLowerCase(); + if (!domain || !target) { + return (_req, _res, next) => next(); + } + return (req, res, next) => { + const active = ( + req.subdomains?.[req.subdomains.length - 1] ?? '' + ).toLowerCase(); + if (active === '' || RESERVED_SUBDOMAINS.has(active)) return next(); + + const host = (req.headers.host ?? '').toLowerCase(); + if (!host.endsWith(domain)) return next(); + + // host ends in domain — swap the domain suffix for the hosting one, + // preserving the subdomain prefix and any port. + const newHost = host.slice(0, host.length - domain.length) + target; + res.redirect(302, `${req.protocol}://${newHost}${req.originalUrl}`); + }; +}; + +/** + * Serves static files from native-app bundles for the reserved app + * subdomains (`editor.*`, `docs.*`, …). `docs` and `developer` resolve + * under a `/dist` subdir — everything else maps directly to `/`. + * + * When the requested path is a directory without a trailing slash, responds + * with 307 so relative asset URLs resolve correctly. + * + * Pass-through when `native_apps_root` is unset so self-hosted deployments + * that don't ship the apps don't trip on 404s. + */ +export const createNativeAppStatic = (config: IConfig): RequestHandler => { + const root = config.native_apps_root; + const apps = new Set(NATIVE_APP_SUBDOMAINS); + if (!root) { + return (_req, _res, next) => next(); + } + return async (req, res, next) => { + const active = ( + req.subdomains?.[req.subdomains.length - 1] ?? '' + ).toLowerCase(); + if (!apps.has(active)) return next(); + + const appRoot = NATIVE_APPS_WITH_DIST.has(active) + ? path.join(root, active, 'dist') + : path.join(root, active); + + // req.path is already url-decoded by express; normalize strips any + // `..` segments before sendFile's `root` option enforces its own + // traversal guard. + const requested = path.normalize(req.path); + const absolute = path.join(appRoot, requested); + + try { + const info = await stat(absolute); + if (info.isDirectory() && !req.path.endsWith('/')) { + const search = req.originalUrl.slice(req.path.length); + res.redirect(307, `${req.path}/${search}`); + return; + } + } catch { + return next(); + } + + res.sendFile(requested, { root: appRoot }, (err) => { + if (err) next(); + }); + }; +}; diff --git a/src/backend/core/http/middleware/notFoundHandler.ts b/src/backend/core/http/middleware/notFoundHandler.ts new file mode 100644 index 000000000..0201a32c7 --- /dev/null +++ b/src/backend/core/http/middleware/notFoundHandler.ts @@ -0,0 +1,16 @@ +import type { RequestHandler } from 'express'; +import { HttpError } from '../HttpError'; + +/** + * Catch-all 404 middleware. Install last (just before the error handler); + * any request that didn't match a route lands here. + * + * Throws an `HttpError(404)` rather than writing the response directly so + * the same error-handler pipeline serializes the body — keeps the wire shape + * consistent with every other failure (`{ error: '...', code: 'not_found' }`). + */ +export const createNotFoundHandler = (): RequestHandler => { + return (_req, _res, next): void => { + next(new HttpError(404, 'Not Found', { legacyCode: 'not_found' })); + }; +}; diff --git a/src/backend/core/http/middleware/privateAppGate.ts b/src/backend/core/http/middleware/privateAppGate.ts new file mode 100644 index 000000000..ddea82f98 --- /dev/null +++ b/src/backend/core/http/middleware/privateAppGate.ts @@ -0,0 +1,662 @@ +import type { Request } from 'express'; +import type { AuthService } from '../../../services/auth/AuthService'; +import type { IConfig } from '../../../types'; + +/** + * Support helpers for the private-app access gate — ported from v1's + * `puterSiteMiddleware.js` (see `origin/main:src/backend/src/routers/ + * hosting/puterSiteMiddleware.js`). Split out because the middleware file + * was getting long. + * + * Covers: + * - Host/subdomain parsing against `private_app_hosting_domain(_alt)`. + * - Broader private-app detection for hosted sites whose subdomain row + * has no `associated_app_id` — falls back to an `index_url` lookup + * against the subdomain owner's private apps. + * - Bootstrap-token identity resolution (`Authorization: Bearer`, + * `?puter.auth.token=`, `X-Puter-Auth-Token`, referrer query) so + * private-app visitors with a valid session token (but no cookie on + * the private host) can still be identified. + * - Building the redirect URL from a public hosting host (`puter.site`) + * to the private host (`puter.app`) when a private app is being + * served off the wrong domain. + * + * Sticky cookies (`puter.private.asset.token` for private apps, + * `puter.public.hosted.actor.token` for public-hosted actors) are set + * after a visitor passes the gate, and honored on subsequent requests + * to skip the full entitlement lookup. See AuthService + * `createPrivateAssetToken` / `createPublicHostedActorToken`. + */ + +export interface PrivateHostingConfig { + domain: string | null; + staticDomains: string[]; + privateDomains: string[]; + /** + * Raw hosting domain values (preserving port, if configured). Used for + * `index_url` candidate generation — the DB stores URLs exactly as the + * app was created, so dev setups with explicit ports like + * `app.puter.localhost:4100` must be matched verbatim. + */ + staticDomainsRaw: string[]; + privateDomainsRaw: string[]; + /** Configured protocol (e.g. `http` in dev, `https` in prod). */ + protocol: string; +} + +export interface PrivateIdentity { + source: + | 'private-cookie' + | 'session-cookie' + | 'bootstrap-token' + | 'authorization' + | 'query' + | 'referrer' + | 'none'; + userUid?: string; + sessionUuid?: string; + /** True when resolved from the sticky `puter.private.asset.token` cookie. */ + hasValidPrivateCookie?: boolean; +} + +interface SubdomainLike { + user_id?: number | null; + associated_app_id?: number | null; +} + +interface AppLike { + id?: number; + uid?: string; + name?: string; + owner_user_id?: number; + is_private?: boolean | number | null; + index_url?: string | null; +} + +interface DBClient { + read: ( + sql: string, + params: unknown[], + ) => Promise[]>; +} + +// ── Host helpers ──────────────────────────────────────────────────── + +export function normalizeHost(value: string | undefined | null): string | null { + if (typeof value !== 'string') return null; + const trimmed = value.trim().toLowerCase().replace(/^\./, ''); + if (!trimmed) return null; + return trimmed.split(':')[0] || null; +} + +/** Like `normalizeHost` but preserves port when present. */ +export function normalizeHostRaw( + value: string | undefined | null, +): string | null { + if (typeof value !== 'string') return null; + const trimmed = value.trim().toLowerCase().replace(/^\./, ''); + return trimmed || null; +} + +export function buildHostingConfig(config: IConfig): PrivateHostingConfig { + const staticRaw = [ + normalizeHostRaw(config.static_hosting_domain), + normalizeHostRaw(config.static_hosting_domain_alt), + ].filter((d): d is string => !!d); + const privateRaw = [ + normalizeHostRaw(config.private_app_hosting_domain), + normalizeHostRaw(config.private_app_hosting_domain_alt), + ].filter((d): d is string => !!d); + const rawProtocol = + typeof config.protocol === 'string' + ? config.protocol.trim().replace(/:$/, '') + : ''; + return { + domain: normalizeHost(config.domain), + staticDomains: [ + normalizeHost(config.static_hosting_domain), + normalizeHost(config.static_hosting_domain_alt), + ].filter((d): d is string => !!d), + privateDomains: [ + normalizeHost(config.private_app_hosting_domain), + normalizeHost(config.private_app_hosting_domain_alt), + ].filter((d): d is string => !!d), + staticDomainsRaw: staticRaw, + privateDomainsRaw: privateRaw, + protocol: rawProtocol || 'https', + }; +} + +export function hostMatchesPrivateDomain( + host: string, + privateDomains: string[], +): boolean { + return privateDomains.some((pd) => host === pd || host.endsWith(`.${pd}`)); +} + +// ── Subdomain extraction from a hosted request ───────────────────── + +export function subdomainFromHost( + host: string, + hostingDomains: string[], +): string { + // Longest-first so `foo.bar.puter.app` matches `bar.puter.app` before + // falling back to `puter.app`. + const sorted = [...hostingDomains].sort((a, b) => b.length - a.length); + for (const d of sorted) { + const suffix = `.${d}`; + if (host === d) return ''; + if (host.endsWith(suffix)) { + const prefix = host.slice(0, host.length - suffix.length); + return prefix.split('.')[0] || ''; + } + } + return host.split('.')[0] || ''; +} + +// ── Private-app detection fallback (v1 `resolvePrivateAppForHostedSite`) + +/** + * When the subdomain row has no `associated_app_id`, v1 looked up the + * owner's private apps and matched `index_url` against the request host + * variants (this-subdomain × every hosting domain). Ports that logic. + * + * Returns the matched private app (if any) so the caller can run the + * normal access check — closes a gap where a private app's files could + * be served via a subdomain whose row wasn't explicitly linked. + */ +export async function resolvePrivateAppForHostedSite(opts: { + req: Request; + site: SubdomainLike; + associatedApp: AppLike | null; + db: DBClient; + config: PrivateHostingConfig; + matchedHostingDomain: string; +}): Promise { + // When the subdomain row's own `associated_app_id` points at a private + // app, that wins outright. Otherwise fall through to index_url matching + // so a private app whose canonical URL lives on one hosting variant + // (`beans.puter.site`) still resolves when the visitor hits another + // (`beans.puter.app`). + if (opts.associatedApp && Number(opts.associatedApp.is_private ?? 0) > 0) { + return opts.associatedApp; + } + if (!opts.site?.user_id) return opts.associatedApp ?? null; + + const host = normalizeHost(opts.req.hostname); + if (!host) return opts.associatedApp ?? null; + + const hostedSubdomain = subdomainFromHost(host, [ + ...opts.config.staticDomains, + ...opts.config.privateDomains, + ]); + if (!hostedSubdomain) return opts.associatedApp ?? null; + + // Build host variants with AND without port, then cross each with both + // protocols. Apps store whatever URL the user typed at create time, so + // we match liberally: ports-in-config (dev), the request's own header + // host, and every configured hosting variant all count as equivalent. + const hostCandidates = new Set(); + hostCandidates.add(host); + const headerHost = + typeof opts.req.headers?.host === 'string' + ? opts.req.headers.host.trim().toLowerCase() + : ''; + if (headerHost) hostCandidates.add(headerHost); + const hostingDomainVariants = [ + ...opts.config.staticDomains, + ...opts.config.privateDomains, + ...opts.config.staticDomainsRaw, + ...opts.config.privateDomainsRaw, + ]; + for (const d of hostingDomainVariants) { + if (!d) continue; + hostCandidates.add(`${hostedSubdomain}.${d}`); + } + + const protocolCandidates = new Set([ + opts.req.protocol || 'https', + opts.config.protocol, + 'https', + 'http', + ]); + + const urlCandidates: string[] = []; + for (const hc of hostCandidates) { + for (const protocol of protocolCandidates) { + const base = `${protocol}://${hc}`; + urlCandidates.push(base, `${base}/`, `${base}/index.html`); + } + } + const uniqueCandidates = [...new Set(urlCandidates)]; + if (uniqueCandidates.length === 0) return opts.associatedApp ?? null; + + const placeholders = uniqueCandidates.map(() => '?').join(', '); + const rows = await opts.db.read( + `SELECT * FROM apps WHERE owner_user_id = ? AND is_private = 1 AND index_url IN (${placeholders}) LIMIT 2`, + [opts.site.user_id, ...uniqueCandidates], + ); + if (rows.length === 0) return opts.associatedApp ?? null; + if (rows.length > 1) { + console.warn('[puter-site] private_access.host_match_ambiguous', { + requestHost: host, + matchCount: rows.length, + }); + } + return rows[0] as unknown as AppLike; +} + +// ── Bootstrap token resolution ────────────────────────────────────── + +function getAuthorizationToken(req: Request): string | null { + const header = req.headers?.authorization; + if (typeof header !== 'string') return null; + const match = header.match(/^Bearer\s+(.+)$/i); + return match?.[1]?.trim() || null; +} + +function getQueryToken(req: Request): string | null { + const q = req.query as Record | undefined; + const candidates = [q?.['puter.auth.token'], q?.auth_token]; + for (const v of candidates) { + if (typeof v === 'string' && v.trim()) return v.trim(); + } + return null; +} + +function getHeaderToken(req: Request): string | null { + const raw = req.headers?.['x-puter-auth-token']; + if (typeof raw === 'string' && raw.trim()) return raw.trim(); + return null; +} + +function getReferrerToken(req: Request): string | null { + const ref = req.headers?.referer || req.headers?.referrer; + if (typeof ref !== 'string' || !ref.trim()) return null; + try { + const url = new URL(ref); + return ( + url.searchParams.get('puter.auth.token') || + url.searchParams.get('auth_token') + ); + } catch { + return null; + } +} + +export function getBootstrapToken( + req: Request, +): { token: string; source: PrivateIdentity['source'] } | null { + const auth = getAuthorizationToken(req); + if (auth) return { token: auth, source: 'authorization' }; + const q = getQueryToken(req); + if (q) return { token: q, source: 'query' }; + const h = getHeaderToken(req); + if (h) return { token: h, source: 'authorization' }; + const r = getReferrerToken(req); + if (r) return { token: r, source: 'referrer' }; + return null; +} + +/** + * Resolve the acting user for a private-app hosted request. Lookup + * order (first hit wins): + * + * 1. `puter.private.asset.token` cookie — the sticky cookie set + * after a previous successful entitlement check. Must match the + * expected app + subdomain + private host. + * 2. `req.actor` from the auth probe (e.g. main session cookie on + * same-site requests). + * 3. Raw session cookie fallback (cross-site drops the probe's read). + * 4. Bootstrap token from Authorization / query / header / referrer. + * + * Returns `{source: 'none'}` when no identity can be established — + * the caller then renders the login bootstrap page. + */ +export async function resolvePrivateIdentity(opts: { + req: Request; + authService: AuthService; + sessionCookieName: string | undefined; + expectedAppUid?: string; + expectedSubdomain?: string; + expectedPrivateHost?: string; +}): Promise { + const { + req, + authService, + sessionCookieName, + expectedAppUid, + expectedSubdomain, + expectedPrivateHost, + } = opts; + + const cookies = (req as Request & { cookies?: Record }) + .cookies; + + // 1. Sticky private-asset cookie. + const privateCookieName = authService.getPrivateAssetCookieName(); + const privateCookieToken = + typeof cookies?.[privateCookieName] === 'string' + ? cookies[privateCookieName] + : null; + if (privateCookieToken) { + try { + const claims = authService.verifyPrivateAssetToken( + privateCookieToken, + { + expectedAppUid, + expectedSubdomain, + expectedPrivateHost, + }, + ); + return { + source: 'private-cookie', + userUid: claims.userUid, + sessionUuid: claims.sessionUuid, + hasValidPrivateCookie: true, + }; + } catch { + /* fall through — stale / mismatched cookie */ + } + } + + // 2. Auth probe actor. + const existingActor = req.actor; + if (existingActor?.user?.uuid) { + return { + source: 'session-cookie', + userUid: existingActor.user.uuid, + sessionUuid: existingActor.session?.uid, + }; + } + + // 3. Raw session cookie fallback. + const sessionToken = + sessionCookieName && typeof cookies?.[sessionCookieName] === 'string' + ? cookies[sessionCookieName] + : null; + if (sessionToken) { + try { + const actor = await authService.authenticateFromToken(sessionToken); + if (actor?.user?.uuid) { + return { + source: 'session-cookie', + userUid: actor.user.uuid, + sessionUuid: actor.session?.uid, + }; + } + } catch { + /* fall through */ + } + } + + // 4. Bootstrap token. + const bootstrap = getBootstrapToken(req); + if (bootstrap) { + try { + const actor = await authService.authenticateFromToken( + bootstrap.token, + ); + if (actor?.user?.uuid) { + return { + source: bootstrap.source, + userUid: actor.user.uuid, + sessionUuid: actor.session?.uid, + }; + } + } catch { + /* fall through */ + } + } + + return { source: 'none' }; +} + +/** + * Mirror of `resolvePrivateIdentity` for public hosted apps. Reads the + * sticky `puter.public.hosted.actor.token` cookie first, then the same + * session/bootstrap fallbacks. + */ +export async function resolvePublicHostedIdentity(opts: { + req: Request; + authService: AuthService; + sessionCookieName: string | undefined; + expectedAppUid?: string; + expectedSubdomain?: string; + expectedHost?: string; +}): Promise { + const { + req, + authService, + sessionCookieName, + expectedAppUid, + expectedSubdomain, + expectedHost, + } = opts; + + const cookies = (req as Request & { cookies?: Record }) + .cookies; + + const publicCookieName = authService.getPublicHostedActorCookieName(); + const publicCookieToken = + typeof cookies?.[publicCookieName] === 'string' + ? cookies[publicCookieName] + : null; + if (publicCookieToken) { + try { + const claims = authService.verifyPublicHostedActorToken( + publicCookieToken, + { + expectedAppUid, + expectedSubdomain, + expectedHost, + }, + ); + return { + source: 'private-cookie', + userUid: claims.userUid, + sessionUuid: claims.sessionUuid, + hasValidPublicCookie: true, + }; + } catch { + /* fall through */ + } + } + + const existingActor = req.actor; + if (existingActor?.user?.uuid) { + return { + source: 'session-cookie', + userUid: existingActor.user.uuid, + sessionUuid: existingActor.session?.uid, + }; + } + + const sessionToken = + sessionCookieName && typeof cookies?.[sessionCookieName] === 'string' + ? cookies[sessionCookieName] + : null; + if (sessionToken) { + try { + const actor = await authService.authenticateFromToken(sessionToken); + if (actor?.user?.uuid) { + return { + source: 'session-cookie', + userUid: actor.user.uuid, + sessionUuid: actor.session?.uid, + }; + } + } catch { + /* fall through */ + } + } + + const bootstrap = getBootstrapToken(req); + if (bootstrap) { + try { + const actor = await authService.authenticateFromToken( + bootstrap.token, + ); + if (actor?.user?.uuid) { + return { + source: bootstrap.source, + userUid: actor.user.uuid, + sessionUuid: actor.session?.uid, + }; + } + } catch { + /* fall through */ + } + } + + return { source: 'none' }; +} + +// ── Redirect helpers ──────────────────────────────────────────────── + +/** Build the URL to redirect a private-app request to its private host. */ +export function buildPrivateHostRedirect( + req: Request, + app: AppLike, + config: PrivateHostingConfig, +): string | null { + // Prefer the raw configured value so dev setups that include a port + // (`app.puter.localhost:4100`) produce a working redirect target. + const privateDomain = + config.privateDomainsRaw[0] ?? config.privateDomains[0]; + if (!privateDomain) return null; + const host = normalizeHost(req.hostname); + if (!host) return null; + const subdomain = subdomainFromHost(host, [ + ...config.staticDomains, + ...config.privateDomains, + ]); + if (!subdomain) return null; + try { + const protocol = config.protocol || req.protocol || 'https'; + const base = `${protocol}://${subdomain}.${privateDomain}`; + const reqPath = (req.originalUrl || '/').startsWith('/') + ? req.originalUrl || '/' + : `/${req.originalUrl}`; + return new URL(reqPath, base).toString(); + } catch { + return null; + } + void app; // reserved for future use (logging) +} + +/** Redirect URL when private access is denied — lands on the app-center listing. */ +export function buildAppCenterFallback( + app: AppLike, + config: PrivateHostingConfig, +): string { + if (!config.domain) return '/'; + const appName = + typeof app?.name === 'string' && app.name.trim() + ? app.name.trim() + : null; + if (!appName) { + return `https://${config.domain}/app/app-center/?item=${encodeURIComponent(app?.uid ?? '')}`; + } + return `https://${config.domain}/app/app-center/?item=${encodeURIComponent(appName)}`; +} + +// ── Login bootstrap HTML ──────────────────────────────────────────── + +/** + * Minimal HTML page that prompts the visitor to sign in with Puter. + * Uses puter.js's `auth.signIn` to get a token, then redirects back to + * the same URL with `?puter.auth.token=…` so the middleware can resolve + * identity on the next request. + * + * Kept inline (no template engine dependency). Ported from v1's + * `respondPrivateLoginBootstrap` with non-essential bells removed. + */ +export function renderLoginBootstrapHtml(app: AppLike): string { + const escape = (value: unknown): string => + String(value ?? '') + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", '''); + const title = escape(app?.title ?? app?.name ?? 'this app'); + const name = escape(app?.name ?? 'this app'); + return ` + + + + +Sign In Required | ${title} + + + + +
+

Sign in required

+

${name} requires Puter authentication before private files can load.

+

Click "Sign In with Puter" to continue.

+
+ + +
+
+ + + +`; +} diff --git a/src/backend/core/http/middleware/puterSite.ts b/src/backend/core/http/middleware/puterSite.ts new file mode 100644 index 000000000..dea85531e --- /dev/null +++ b/src/backend/core/http/middleware/puterSite.ts @@ -0,0 +1,506 @@ +import type { RequestHandler } from 'express'; +import { contentType as contentTypeFromMime } from 'mime-types'; +import { posix as pathPosix } from 'node:path'; +import type { puterClients } from '../../../clients'; +import type { puterServices } from '../../../services'; +import type { puterStores } from '../../../stores'; +import { FS_COSTS } from '../../../controllers/fs/costs'; +import type { IConfig, LayerInstances } from '../../../types'; +import { + buildAppCenterFallback, + buildHostingConfig, + buildPrivateHostRedirect, + hostMatchesPrivateDomain, + renderLoginBootstrapHtml, + resolvePrivateAppForHostedSite, + resolvePrivateIdentity, + resolvePublicHostedIdentity, +} from './privateAppGate'; + +/** + * Serves user-hosted static sites on the hosting domains (`*.puter.site`, + * `*.puter.app`, and their alt variants). Must run after the auth probe so + * `req.actor` is populated for the private-app gate, but before controller + * routes so site hosts don't accidentally hit the API/GUI routers. + * + * Scope: + * - subdomain → site row (SubdomainStore) → file under site root + * - 404 for unknown subdomain / missing file / suspended owner + * - private-app gate via `app.privateAccess.check` — marketplace extension + * decides; default is denied + redirect to `app-center` + * - Range / ETag / Last-Modified passthrough via `fsEntry.readContent` + * + * Deferred (not yet implemented): + * - `.at` username-based sites (UUIDv5-keyed `/user/Public`). + * - `.puter_site_config` error rules (custom status-code → file mapping). + * - Custom domains (subdomains table `domain` column) — requires host + * validation to allow arbitrary hostnames first. + */ + +const SUBDOMAIN_404 = `

404

Subdomain or site is not pointing to a directory.

`; +interface SubdomainRow { + id: number; + uuid: string; + subdomain: string; + user_id: number | null; + root_dir_id: number | null; + associated_app_id: number | null; + domain?: string | null; + protected?: number | null; +} + +interface AppRow { + id: number; + uid: string; + name?: string; + is_private?: number | null; + owner_user_id?: number; +} + +interface UserRow { + id: number; + uuid: string; + username: string; + suspended?: number | null; +} + +interface Layers { + clients: LayerInstances; + stores: LayerInstances; + services: LayerInstances; +} + +function normalizeHost(value: string | undefined | null): string | null { + if (typeof value !== 'string') return null; + const trimmed = value.trim().toLowerCase().replace(/^\./, ''); + if (!trimmed) return null; + return trimmed.split(':')[0] || null; +} + +export const createPuterSiteMiddleware = ( + config: IConfig, + layers: Layers, +): RequestHandler => { + const domain = normalizeHost(config.domain); + const hostingDomains = [ + normalizeHost(config.static_hosting_domain), + normalizeHost(config.static_hosting_domain_alt), + normalizeHost(config.private_app_hosting_domain), + normalizeHost(config.private_app_hosting_domain_alt), + ].filter((d): d is string => !!d); + + // The private-app hosting domains are the only ones where unowned, + // unentitled access should be rejected outright — used below to + // default-deny when a subdomain row on these hosts has no associated + // app (or the app lookup fails). + const privateHostingDomains = new Set( + [ + normalizeHost(config.private_app_hosting_domain), + normalizeHost(config.private_app_hosting_domain_alt), + ].filter((d): d is string => !!d), + ); + + if (hostingDomains.length === 0) { + return (_req, _res, next) => next(); + } + + // Longest-first so `foo.bar.puter.site` matches `bar.puter.site` before + // falling back to `puter.site`. + const sortedHostingDomains = [...hostingDomains].sort( + (a, b) => b.length - a.length, + ); + + const matchHostingDomain = (host: string): string | null => { + for (const d of sortedHostingDomains) { + if (host === d) return d; + if (host.endsWith(`.${d}`)) return d; + } + return null; + }; + + return async (req, res, next) => { + const host = normalizeHost(req.hostname); + if (!host) return next(); + + const matched = matchHostingDomain(host); + if (!matched) return next(); + + // Bare hosting domain (e.g. `puter.site`) → redirect to the main site. + if (host === matched) { + if (domain) { + res.redirect(302, `${req.protocol}://${domain}`); + return; + } + res.status(404).type('text/plain').send('Subdomain not found'); + return; + } + + // `host` is `.`; subdomain is the left-most label. + const prefix = host.slice(0, host.length - matched.length - 1); + const subdomain = prefix.split('.')[0] || ''; + + if (!subdomain || subdomain === 'www') { + if (domain) { + res.redirect(302, `${req.protocol}://${domain}`); + return; + } + res.status(404).type('text/plain').send('Subdomain not found'); + return; + } + + const site = (await layers.stores.subdomain.getBySubdomain( + subdomain, + )) as unknown as SubdomainRow | null; + if (!site || site.user_id === null || site.user_id === undefined) { + res.status(404).type('text/plain').send('Subdomain not found'); + return; + } + + // Suspended owner 404s — don't leak the suspension reason. + const owner = (await layers.stores.user.getById( + site.user_id, + )) as unknown as UserRow | null; + if (!owner || owner.suspended) { + res.status(404).type('text/plain').send('Subdomain not found'); + return; + } + + const hostingCfg = buildHostingConfig(config); + + let associatedApp: AppRow | null = null; + if ( + site.associated_app_id !== null && + site.associated_app_id !== undefined + ) { + associatedApp = (await layers.stores.app.getById( + site.associated_app_id, + )) as unknown as AppRow | null; + } + + const privateApp = (await resolvePrivateAppForHostedSite({ + req, + site: { + user_id: site.user_id, + associated_app_id: site.associated_app_id, + }, + associatedApp, + db: layers.clients.db, + config: hostingCfg, + matchedHostingDomain: matched, + })) as AppRow | null; + + const isPrivateApp = Boolean(privateApp?.is_private); + + if (isPrivateApp) { + // Private apps must run on the private hosting domain. If a + // visitor arrives via the public domain (puter.site), redirect + // them to the equivalent private-host URL so the cookie scope + // and gate run on the right origin. + if (!hostMatchesPrivateDomain(host, hostingCfg.privateDomains)) { + const redirectUrl = buildPrivateHostRedirect( + req, + privateApp as never, + hostingCfg, + ); + if (redirectUrl) { + res.redirect(302, redirectUrl); + return; + } + // No private host configured — refuse rather than leak. + res.status(403) + .type('text/plain') + .send('Private app host mismatch'); + return; + } + + // Resolve identity. Lookup order: sticky private-asset + // cookie → req.actor → session cookie → bootstrap token. + const identity = await resolvePrivateIdentity({ + req, + authService: layers.services.auth, + sessionCookieName: + typeof config.cookie_name === 'string' + ? config.cookie_name + : undefined, + expectedAppUid: privateApp!.uid, + expectedSubdomain: subdomain, + expectedPrivateHost: host, + }); + + if (!identity.userUid) { + // No identity yet — render the sign-in bootstrap so the + // browser can call `puter.auth.signIn()` and retry with a + // token in the query string. + res.status(200) + .set('Cache-Control', 'no-store') + .set('X-Robots-Tag', 'noindex, nofollow') + .set('Referrer-Policy', 'no-referrer') + .type('text/html; charset=UTF-8') + .send( + renderLoginBootstrapHtml( + privateApp as unknown as { + uid?: string; + name?: string; + title?: string; + }, + ), + ); + return; + } + + // Entitlement check runs on every request — matching v1. The + // sticky cookie is an identity shortcut, not an access cache; + // the marketplace extension already caches access decisions + // in Redis so repeat checks are cheap. This guarantees that + // if entitlement is revoked (refund, grant removed) the very + // next request stops serving content. + const checkEvent = { + appUid: privateApp!.uid, + userUid: identity.userUid, + requestHost: host, + requestPath: req.path, + result: { + allowed: false, + } as { + allowed: boolean; + reason?: string; + redirectUrl?: string; + checkedBy?: string; + }, + }; + try { + await layers.clients.event.emitAndWait( + 'app.privateAccess.check', + checkEvent, + {}, + ); + } catch (e) { + console.error('[puter-site] privateAccess.check threw', e); + } + if (!checkEvent.result.allowed) { + const fallback = buildAppCenterFallback( + privateApp as unknown as { + name?: string; + uid?: string; + }, + hostingCfg, + ); + res.redirect(302, checkEvent.result.redirectUrl || fallback); + return; + } + + // Mint the sticky cookie only when we don't already have a + // valid one — keeps Set-Cookie off of the hot path for repeat + // visitors but still refreshes after rotation/expiry. + if (!identity.hasValidPrivateCookie) { + try { + const token = layers.services.auth.createPrivateAssetToken({ + appUid: privateApp!.uid, + userUid: identity.userUid, + sessionUuid: identity.sessionUuid, + subdomain, + privateHost: host, + }); + res.cookie( + layers.services.auth.getPrivateAssetCookieName(), + token, + layers.services.auth.getPrivateAssetCookieOptions({ + requestHostname: host, + }), + ); + } catch (e) { + console.warn( + '[puter-site] failed to mint private asset cookie', + e, + ); + } + } + + // Referrer-policy hardening — don't leak private-host URLs to + // third-party resources loaded from the app. + res.setHeader('Referrer-Policy', 'no-referrer'); + } else if (privateHostingDomains.has(matched)) { + // Private host with no private app → refuse. Prevents a + // public-app subdomain from leaking via the private host. + res.status(404).type('text/plain').send('Subdomain not found'); + return; + } else { + // Public hosted site. Mint the public hosted-actor cookie if + // we can identify the visitor — lets the hosted page make + // cross-origin requests as the actor without needing a + // host-scoped main session cookie. No-op for anonymous + // visitors. + try { + const identity = await resolvePublicHostedIdentity({ + req, + authService: layers.services.auth, + sessionCookieName: + typeof config.cookie_name === 'string' + ? config.cookie_name + : undefined, + expectedAppUid: associatedApp?.uid, + expectedSubdomain: subdomain, + expectedHost: host, + }); + if ( + identity.userUid && + !(identity as { hasValidPublicCookie?: boolean }) + .hasValidPublicCookie && + associatedApp?.uid + ) { + const token = + layers.services.auth.createPublicHostedActorToken({ + appUid: associatedApp.uid, + userUid: identity.userUid, + sessionUuid: identity.sessionUuid, + subdomain, + host, + }); + res.cookie( + layers.services.auth.getPublicHostedActorCookieName(), + token, + layers.services.auth.getPublicHostedActorCookieOptions({ + requestHostname: host, + }), + ); + } + } catch (e) { + // Best-effort — don't block the public file serve. + console.warn( + '[puter-site] public hosted actor resolve failed', + e, + ); + } + } + + if (site.root_dir_id === null || site.root_dir_id === undefined) { + res.status(404) + .type('text/html; charset=UTF-8') + .send(SUBDOMAIN_404); + return; + } + + const rootEntry = await layers.stores.fsEntry.getEntryById( + site.root_dir_id, + ); + if (!rootEntry) { + res.status(404) + .type('text/html; charset=UTF-8') + .send(SUBDOMAIN_404); + return; + } + if (!rootEntry.isDir) { + res.status(404) + .type('text/html; charset=UTF-8') + .send(SUBDOMAIN_404); + return; + } + + // Resolve URL path → absolute FS path under the site root. + let urlPath = req.path || '/'; + if (urlPath.endsWith('/')) urlPath += 'index.html'; + const decoded = decodeURIComponent(urlPath); + // pathPosix.normalize strips `..` segments; the join with '/' anchors + // it so traversal can't escape the site root. + const resolvedUrlPath = pathPosix.normalize( + pathPosix.join('/', decoded), + ); + const rootPath = rootEntry.path.replace(/\/+$/, ''); + if (!rootPath || rootPath === '/') { + res.status(403).type('text/plain').send('Forbidden'); + return; + } + const filePath = rootPath + resolvedUrlPath; + + // Subdomain hosting bypasses ACL by design: anything the owner placed + // under the registered root_dir is treated as public. Path traversal + // is blocked above by `pathPosix.normalize` anchoring at `/`. + const entry = await layers.stores.fsEntry.getEntryByPath(filePath); + if (!entry || entry.isDir) { + res.status(404) + .type('text/html; charset=UTF-8') + .send('

404

Not Found

'); + return; + } + + // Stream the file. `fsEntry.readContent` honours Range + emits + // ETag/Last-Modified when the S3 layer returns them. + const range = + typeof req.headers.range === 'string' + ? req.headers.range + : undefined; + let download; + try { + download = await layers.services.fs.readContent(entry, { + range, + }); + } catch (e) { + console.error('[puter-site] readContent failed', e); + return next(e); + } + + const mime = + contentTypeFromMime(entry.name) || 'application/octet-stream'; + res.setHeader('Content-Type', mime); + if (download.contentLength !== null) { + res.setHeader('Content-Length', String(download.contentLength)); + } + if (download.contentRange) + res.setHeader('Content-Range', download.contentRange); + if (download.etag) res.setHeader('ETag', download.etag); + if (download.lastModified) + res.setHeader('Last-Modified', download.lastModified.toUTCString()); + res.setHeader('Accept-Ranges', 'bytes'); + res.setHeader('Access-Control-Allow-Origin', '*'); + res.status(range ? 206 : 200); + + // Best-effort egress metering against the site owner. The request + // itself is unauthenticated (public site visitor), so we can't use + // req.actor — charge the account that hosts the file. Same cost + // key as FS read egress (`filesystem:egress:bytes`). Fires once + // the body stream ends so we only meter bytes actually delivered + // (not aborted mid-stream). + const metering = layers.services.metering as unknown as + | { + batchIncrementUsages?: ( + actor: unknown, + entries: unknown[], + ) => void; + } + | undefined; + if (metering?.batchIncrementUsages && download.contentLength) { + const ownerActor = { + user: { + uuid: owner.uuid, + id: owner.id, + username: owner.username, + suspended: !!owner.suspended, + }, + }; + download.body.once('end', () => { + try { + const bytes = download.contentLength!; + metering.batchIncrementUsages!(ownerActor, [ + { + usageType: 'filesystem:egress:bytes', + usageAmount: bytes, + costOverride: + FS_COSTS['filesystem:egress:bytes'] * bytes, + }, + ]); + } catch { + // ignore — non-critical. + } + }); + } + + req.on('close', () => download.body.destroy()); + download.body.on('error', (err) => res.destroy(err)); + download.body.pipe(res); + }; +}; diff --git a/src/backend/core/http/middleware/rateLimit.js b/src/backend/core/http/middleware/rateLimit.js new file mode 100644 index 000000000..8a70062cb --- /dev/null +++ b/src/backend/core/http/middleware/rateLimit.js @@ -0,0 +1,287 @@ +import crypto from 'node:crypto'; +import { HttpError } from '../HttpError.js'; + +/** + * Sliding-window rate limiter with a swappable backend. + * + * Three backends, selected once at boot via `configureRateLimit(...)`: + * - `redis`: Redis sorted sets — atomic per key across a cluster. + * Boot default (server.ts); ioredis-mock in dev. + * - `kv`: one row per hit in the system KV (DynamoDB), with TTL. + * `kv.list()` already drops expired rows, so "entries under + * the prefix" == "entries still in the window". + * - `memory`: per-process counters. Capped + actively swept; does not + * coordinate across nodes, so use only when redis is absent. + * + * Each backend exports a `check(key, limit, windowMs)` that returns + * `true` (and records the hit) or `false` (rate-limited). That's the + * whole surface. + */ + +// ── Memory backend ────────────────────────────────────────────────── + +// Hard cap and retention bound memory in the worst case. Without them, +// one-shot keys (visited once, never again) leak forever: their single +// timestamp prevents the empty-array sweep from collecting them, even +// after the window has long passed. +const MEMORY_MAX_KEYS = 10_000; +const MEMORY_MAX_RETAIN_MS = 60 * 60_000; + +const memoryWindows = new Map(); +{ + const sweep = setInterval(() => { + const cutoff = Date.now() - MEMORY_MAX_RETAIN_MS; + for (const [k, ts] of memoryWindows) { + if (ts.length === 0 || ts[ts.length - 1] < cutoff) + memoryWindows.delete(k); + } + }, 60_000); + sweep.unref?.(); +} + +async function checkMemory(key, limit, windowMs) { + const now = Date.now(); + const cutoff = now - windowMs; + let timestamps = memoryWindows.get(key); + if (!timestamps) { + // Map preserves insertion order; FIFO-evict before adding so a + // unique-key flood between sweep ticks can't blow up memory. + if (memoryWindows.size >= MEMORY_MAX_KEYS) { + const oldest = memoryWindows.keys().next().value; + memoryWindows.delete(oldest); + } + timestamps = []; + memoryWindows.set(key, timestamps); + } + while (timestamps.length > 0 && timestamps[0] < cutoff) timestamps.shift(); + if (timestamps.length >= limit) return false; + timestamps.push(now); + return true; +} + +// ── Redis backend ─────────────────────────────────────────────────── + +async function checkRedis( + /** @type {import('ioredis').Cluster} */ + redis, + /** @type {string} */ + key, + /** @type {number} */ + limit, + /** @type {number} */ + windowMs, +) { + const redisKey = `rate:${key}`; + const now = Date.now(); + const cutoff = now - windowMs; + const member = `${now}:${crypto.randomUUID()}`; + + // Valkey/Redis MULTI/EXEC keeps this standard-command path compatible with + // managed clusters where Lua scripting may be restricted. Add before + // counting so concurrent requests cannot all observe count < limit and + // over-admit; if the post-add count is too high, remove this request's + // member and reject. Races can be conservative, but not permissive. + const results = await redis + .multi() + .zremrangebyscore(redisKey, 0, cutoff) + .zadd(redisKey, now, member) + .zcard(redisKey) + .pexpire(redisKey, windowMs) + .exec(); + + const count = Number( + Array.isArray(results[2]) ? results[2][1] : results[2], + ); + if (count > limit) { + await redis.zrem(redisKey, member); + return false; + } + return true; +} + +// ── KV backend ────────────────────────────────────────────────────── + +async function checkKv(kv, key, limit, windowMs) { + const prefix = `rate:${key}:`; + // `list` filters by TTL already, so a non-expired row ⇒ in-window. + // Cap the fetch at `limit + 1` — once we know it's over, the exact + // count doesn't matter. + const { res } = await kv.list({ + as: 'keys', + pattern: prefix, + limit: limit + 1, + }); + const keys = Array.isArray(res) ? res : (res?.items ?? []); + if (keys.length >= limit) return false; + + const now = Date.now(); + await kv.set({ + key: `${prefix}${now}:${crypto.randomUUID()}`, + value: 1, + expireAt: Math.ceil((now + windowMs) / 1000), + }); + return true; +} + +// ── Backend selection ─────────────────────────────────────────────── + +let checkFn = checkMemory; + +/** + * Call once during server boot, after clients/stores are built. + * configureRateLimit({ backend: 'redis', redis: clients.redis }) + * configureRateLimit({ backend: 'kv', kv: stores.kv }) + * configureRateLimit() // memory + * + * Throws if the chosen backend's dependency is missing, so a typo in + * config surfaces loudly instead of silently downgrading. + */ +export function configureRateLimit({ backend, redis, kv } = {}) { + if (backend === 'redis') { + if (!redis) + throw new Error( + 'rate-limit: redis backend requires a redis client', + ); + checkFn = (key, limit, windowMs) => + checkRedis(redis, key, limit, windowMs); + return; + } + if (backend === 'kv') { + if (!kv) throw new Error('rate-limit: kv backend requires a kv store'); + checkFn = (key, limit, windowMs) => checkKv(kv, key, limit, windowMs); + return; + } + checkFn = checkMemory; +} + +// ── Key strategies ────────────────────────────────────────────────── + +/** + * Build a rate-limit key from the request. + * + * Strategies: + * 'fingerprint' — IP + User-Agent hash (default). Good for + * unauthenticated endpoints where the same IP may + * serve many users (offices, VPNs). + * 'ip' — bare IP. Simpler but coarser. + * 'user' — actor UUID. Use for authenticated endpoints where + * you want per-account limits regardless of IP. + * function — custom `(req) => string`. + */ +function resolveKey(req, scope, strategy) { + const prefix = scope ? `${scope}:` : ''; + + if (typeof strategy === 'function') { + return prefix + strategy(req); + } + + switch (strategy) { + case 'user': { + const id = req.actor?.user?.id; + if (!id) { + // Fall back to fingerprint if no actor (shouldn't happen + // on requireAuth routes, but be safe) + return prefix + fingerprint(req); + } + return prefix + id; + } + case 'ip': + return prefix + ip(req); + case 'fingerprint': + default: + return prefix + fingerprint(req); + } +} + +function ip(req) { + // `req.ip` honors the app-level `trust proxy` setting — it returns the + // leftmost untrusted XFF address when behind the configured proxy chain + // and the direct socket peer otherwise. Reading XFF directly would let a + // client forge their rate-limit key by spoofing the header. + return req.ip || req.socket?.remoteAddress || 'unknown'; +} + +function fingerprint(req) { + const parts = [ + ip(req), + req.headers?.['user-agent'] || '', + req.headers?.['accept-language'] || '', + req.headers?.['accept-encoding'] || '', + ]; + return crypto + .createHash('sha256') + .update(parts.join('|')) + .digest('base64url') + .slice(0, 16); +} + +// ── Route middleware ──────────────────────────────────────────────── + +/** + * Express middleware factory. Reads from the materialised route option: + * + * { rateLimit: { limit: 10, window: 15 * 60_000, key: 'user' } } + * { rateLimit: { limit: 100, window: 60_000 } } // fingerprint default + * + * Rejects with 429. Fails open on backend error — a broken Redis/KV + * shouldn't 500 every request. + */ +export function rateLimitGate(opts) { + const { + limit, + window: windowMs, + key: strategy = 'fingerprint', + scope, + } = opts; + + return async (req, _res, next) => { + const key = resolveKey( + req, + scope ?? req.route?.path ?? 'route', + strategy, + ); + try { + if (!(await checkFn(key, limit, windowMs))) + return next(new HttpError(429, 'Too many requests.')); + next(); + } catch (err) { + console.error( + '[rate-limit] backend check failed, failing open:', + err, + ); + next(); + } + }; +} + +// ── Driver-call helper ────────────────────────────────────────────── + +/** + * Check rate limit for a driver call. Called from DriverController's + * /call handler. Keyed by user + interface:method so different drivers + * and different methods don't crowd each other. + * + * Defaults to 600/minute (10/sec) — loose enough for chatty UI patterns + * (app listings, repeated `puter-apps:es:app:read` during desktop boot, + * kv polling) while still catching runaway loops. Origin/main only + * rate-limited drivers whose policy explicitly asked for it; caller can + * still pass tighter values for sensitive methods. + * + * Returns true if allowed, false if rate-limited. + */ +export async function checkDriverRateLimit( + req, + ifaceName, + method, + limit = 600, + windowMs = 60_000, +) { + const uid = req.actor?.user?.uuid || fingerprint(req); + const key = `driver:${ifaceName}:${method}:${uid}`; + try { + return await checkFn(key, limit, windowMs); + } catch (err) { + console.error('[rate-limit] driver check failed, failing open:', err); + return true; + } +} diff --git a/src/backend/core/http/middleware/requestContext.ts b/src/backend/core/http/middleware/requestContext.ts new file mode 100644 index 000000000..b2d20b665 --- /dev/null +++ b/src/backend/core/http/middleware/requestContext.ts @@ -0,0 +1,31 @@ +import { v4 as uuidv4 } from 'uuid'; +import type { RequestHandler } from 'express'; +import { runWithContext } from '../../context'; +import '../expressAugmentation'; + +/** + * Wraps the remaining middleware + handler chain in a per-request + * `AsyncLocalStorage` scope. + * + * Install order in `PuterServer#installGlobalMiddleware`: + * + * body parsers → authProbe → **requestContext** → routes + * + * Running AFTER the auth probe means `req.actor` is already populated + * when we snapshot it into the context. Everything downstream — gates, + * per-route parsers, controller handlers, and any services they call — + * runs inside the ALS scope and can reach the context via + * `Context.get('actor')`, `Context.get('req')`, etc. + */ +export const createRequestContextMiddleware = (): RequestHandler => { + return (req, _res, next) => { + runWithContext( + { + actor: req.actor, + req, + requestId: uuidv4(), + }, + () => next(), + ); + }; +}; diff --git a/src/backend/core/http/middleware/userProtected.ts b/src/backend/core/http/middleware/userProtected.ts new file mode 100644 index 000000000..0bf509af9 --- /dev/null +++ b/src/backend/core/http/middleware/userProtected.ts @@ -0,0 +1,206 @@ +import type { Request, RequestHandler, Response, NextFunction } from 'express'; +import bcrypt from 'bcrypt'; +import { HttpError } from '../HttpError'; +import type { IConfig } from '../../../types'; +import type { UserStore, UserRow } from '../../../stores/user/UserStore'; +import type { OIDCService } from '../../../services/auth/OIDCService'; +import type { TokenService } from '../../../services/auth/TokenService'; + +/** + * Gate for security-critical account endpoints mounted under `/user-protected/*`. + * + * Runs AFTER the built-in `requireUserActor` + `antiCsrf` gates; adds four + * extra checks: + * + * 1. **Session-cookie only** — reject API tokens, GUI tokens, `x-api-key` + * headers, query-string tokens. `authProbe` stashes the token it + * resolved as `req.token`; if that doesn't match the session cookie + * value, the request came in via a non-cookie source and is rejected. + * 2. **Cache-bypass user refresh** — a suspended account whose session + * row is still cached would otherwise pass; re-fetch with + * `{ force: true }` and reject anything suspended. + * 3. **Temp-user block** — temporary accounts (no password + no email) + * can only reach `/delete-own-user`. Opt in by constructing with + * `{ allowTempUsers: true }` on that route. + * 4. **Password OR OIDC revalidation cookie** — `req.body.password` is + * verified via bcrypt against the user row; otherwise a valid + * `puter_revalidation` cookie (signed via `services.token.sign('oidc-state')`) + * is required. OIDC-only accounts (no password) MUST use the + * revalidation cookie — password path returns `oidc_revalidation_required` + * with a `revalidate_url` so the GUI can open the OIDC popup. + */ + +const REVALIDATION_COOKIE_NAME = 'puter_revalidation'; + +interface RevalidationPayload { + user_uuid: string; + purpose: string; +} + +export interface UserProtectedGateDeps { + config: IConfig; + userStore: UserStore; + oidcService: OIDCService; + tokenService: TokenService; +} + +export interface UserProtectedGateOptions { + /** Allow temp accounts (no password + no email) through. Default: false. */ + allowTempUsers?: boolean; +} + +// Extend Request so the middleware chain can hand the refreshed row off to +// the handler without re-fetching. +declare module 'express-serve-static-core' { + interface Request { + userProtected?: { user: UserRow }; + } +} + +async function buildRevalidateFields( + config: IConfig, + oidcService: OIDCService, + user: UserRow, +): Promise | undefined> { + const origin = (config.origin ?? '').replace(/\/$/, ''); + const providers = await oidcService.getEnabledProviderIds(); + const provider = providers && providers[0]; + if (!provider || !origin) return undefined; + return { + revalidate_url: `${origin}/auth/oidc/${provider}/start?flow=revalidate&user_uuid=${encodeURIComponent(user.uuid)}`, + }; +} + +export const createUserProtectedGate = ( + deps: UserProtectedGateDeps, + options: UserProtectedGateOptions = {}, +): RequestHandler[] => { + const { config, userStore, oidcService, tokenService } = deps; + const cookieName = config.cookie_name ?? 'puter_token'; + const allowTemp = !!options.allowTempUsers; + + // 1. Session cookie only. + const requireSessionCookie: RequestHandler = (req, _res, next) => { + const cookieValue = req.cookies?.[cookieName]; + if (!cookieValue || (req.token && req.token !== cookieValue)) { + throw new HttpError(401, 'Session cookie required', { + legacyCode: 'session_required', + }); + } + next(); + }; + + // 2. Fresh user row (bypass cache to catch just-suspended accounts). + // `getById` doesn't take options; go through `getByProperty` with + // `{ force: true }` to force a primary read. + const refreshUser: RequestHandler = async ( + req: Request, + _res: Response, + next: NextFunction, + ) => { + const actor = req.actor; + if (!actor?.user?.id) throw new HttpError(401, 'User required'); + const user = await userStore.getByProperty('id', actor.user.id, { + force: true, + }); + if (!user) throw new HttpError(404, 'User not found'); + if (user.suspended) + throw new HttpError(403, 'Account is suspended', { + legacyCode: 'account_suspended', + }); + req.userProtected = { user }; + next(); + }; + + // 3. Password (bcrypt) OR valid OIDC revalidation cookie. + // + // - Temp users (no password + no email) pass only when the route was + // registered with `allowTempUsers: true` (delete-own-user). + // - `req.body.password` → bcrypt match against user row. OIDC-only + // accounts bounce with `oidc_revalidation_required` + a + // `revalidate_url` helper so the GUI can open the OIDC popup. + // - Otherwise accept a valid `puter_revalidation` cookie. Expiry, + // `purpose === 'revalidate'`, matching `user_uuid` all required. + // - Password account, neither credential → 403 `password_required`. + const verifyIdentity: RequestHandler = async ( + req: Request, + _res: Response, + next: NextFunction, + ) => { + const user = req.userProtected?.user; + if (!user) throw new HttpError(500, 'user-protected state missing'); + + const isTemp = user.password === null && user.email === null; + if (isTemp) { + if (allowTemp) return next(); + throw new HttpError(403, 'Temporary account', { + legacyCode: 'temporary_account', + }); + } + + const bodyPassword = + typeof req.body?.password === 'string' ? req.body.password : null; + if (bodyPassword) { + if (user.password === null) { + const fields = await buildRevalidateFields( + config, + oidcService, + user, + ); + throw new HttpError(403, 'OIDC revalidation required', { + legacyCode: 'oidc_revalidation_required', + fields, + }); + } + let match = false; + try { + match = await bcrypt.compare( + bodyPassword, + String(user.password), + ); + } catch { + match = false; + } + if (!match) + throw new HttpError(400, 'Password mismatch', { + legacyCode: 'password_mismatch', + }); + return next(); + } + + const cookieValue = req.cookies?.[REVALIDATION_COOKIE_NAME]; + if (cookieValue) { + try { + const payload = tokenService.verify( + 'oidc-state', + cookieValue, + ); + if ( + payload?.purpose === 'revalidate' && + payload.user_uuid === user.uuid + ) { + return next(); + } + } catch { + // Fall through to the no-credentials branch. + } + } + + if (user.password === null) { + const fields = await buildRevalidateFields( + config, + oidcService, + user, + ); + throw new HttpError(403, 'OIDC revalidation required', { + legacyCode: 'oidc_revalidation_required', + fields, + }); + } + throw new HttpError(403, 'Password required', { + legacyCode: 'password_required', + }); + }; + + return [requireSessionCookie, refreshUser, verifyIdentity]; +}; diff --git a/src/backend/core/http/types.ts b/src/backend/core/http/types.ts new file mode 100644 index 000000000..74154afff --- /dev/null +++ b/src/backend/core/http/types.ts @@ -0,0 +1,240 @@ +import type { NextFunction, Request, RequestHandler, Response } from 'express'; +import type { Actor } from '../actor'; + +/** + * Every route method PuterRouter exposes. Mirrors the express router surface + * plus WebDAV verbs that some endpoints still use. + * + * `use` and `all` don't map to a single HTTP verb — they're treated uniformly + * by the materializer (see `v2/server.ts`). + */ +export type RouteMethod = + | 'use' + | 'all' + | 'get' + | 'head' + | 'post' + | 'put' + | 'delete' + | 'patch' + | 'options' + | 'lock' + | 'unlock' + | 'propfind' + | 'proppatch' + | 'mkcol' + | 'copy' + | 'move'; + +/** + * Path shape accepted by express route methods. Kept permissive rather than + * re-exporting express's internal `PathParams` (which isn't stable public API). + */ +export type RoutePath = string | RegExp | Array; + +/** + * Per-route options declared by the caller. + * + * The materializer (`v2/server.ts#materializeRoute`) translates these into a + * middleware chain in this order: + * + * subdomain → requireAuth (+ suspended) → requireUserActor → + * adminOnly → allowedAppIds → caller `middleware: []` → handler + * + * `requireUserActor`, `adminOnly`, and `allowedAppIds` all imply + * `requireAuth`; the materializer dedupes so only one auth gate ends up + * in the chain. Commented-out slots are reserved for the next chunks + * (body parsing, post-auth gates, timing). + */ +export interface RouteOptions { + /** Extra per-route middleware. Applied after built-in gates, before the handler. */ + middleware?: RequestHandler[]; + + /** + * Subdomain routing. If set, the route only matches requests whose + * leftmost subdomain is in this list (via `next('route')` skip). + * + * If omitted, verb-routes (get/post/etc.) are restricted to the root + * origin only (no subdomain). Pass `'*'` to explicitly match ANY + * subdomain/root. `use()` middleware is not gated by default. + */ + subdomain?: string | string[]; + + /** Reject anonymous + suspended-user requests with 401/403. */ + requireAuth?: boolean; + + /** Reject app/access-token actors. Implies `requireAuth`. */ + requireUserActor?: boolean; + + /** + * Reject unless the actor's username is `admin`, `system`, or one of the + * extras in this array. `true` means just `admin`/`system`; an array adds + * to that pair (does not replace it). Implies `requireAuth`. + * + * Does NOT imply `requireUserActor` — admin endpoints accept an admin's + * access-token or app-under-user actor. Combine with `requireUserActor` + * to restrict to browser sessions. + */ + adminOnly?: boolean | string[]; + + /** Reject unless the actor is acting through one of these apps. Implies `requireAuth`. */ + allowedAppIds?: string[]; + + /** + * Reject unless the actor's user has a confirmed email. 400 with + * `account_is_not_verified` on failure. No-op when + * `config.strict_email_verification_required` is falsy, so self-hosted + * deployments can opt in via config. Implies `requireAuth` but NOT + * `requireUserActor` — app-under-user actors also carry a `.user`, so + * verification applies uniformly whether the user acts directly or + * through an app. + */ + requireVerified?: boolean; + + /** + * Per-route JSON body parsing override. By default the global parser + * handles every `application/json` request with a 50mb limit and stashes + * the raw bytes on `req.rawBody` for signature-verification use cases. + * + * Use this option only when a route needs different parser settings: + * - `false` — opt out of parsing entirely (rare; the route reads the + * raw stream itself, e.g. some webhook proxies). The global parser + * will still have already run if the content-type was JSON, so this + * is mostly useful for routes that accept *non*-JSON body shapes + * and want to ensure no further parsers attach. + * - `{ limit, type }` — override the limit (e.g., for ML endpoints + * that legitimately need 100mb) or the matched content-type list + * (e.g., to ALSO accept `application/x-ndjson`). + */ + bodyJson?: false | { limit?: string; type?: string | string[] }; + + /** + * Per-route raw (Buffer) body parser. Use for binary uploads where the + * route handler wants `req.body: Buffer` directly. Default content-type + * match is `application/octet-stream`; pass `type` to override. + */ + bodyRaw?: boolean | { limit?: string; type?: string | string[] }; + + /** + * Per-route text body parser. `req.body` becomes a string. Default + * content-type match is `text/plain`. + */ + bodyText?: boolean | { limit?: string; type?: string | string[] }; + + /** + * Per-route urlencoded form parser. `req.body` becomes a parsed object. + * Default `extended: true` (uses `qs`); pass `extended: false` for the + * built-in `querystring` parser. + */ + bodyUrlencoded?: boolean | { limit?: string; extended?: boolean }; + + /** + * Require captcha verification. When `true`, the route rejects + * requests that don't carry valid `captchaToken` + `captchaAnswer` + * fields in the body. No-op when captcha is disabled in config. + */ + captcha?: boolean; + + /** + * Require a valid one-time anti-CSRF token in `req.body.anti_csrf`. + * The token is consumed on use. Requires authentication (keyed by + * user uuid). + */ + antiCsrf?: boolean; + + /** + * Per-route rate limiting. In-memory sliding window keyed by + * request identity. + * + * `key` controls how requests are bucketed: + * - `'fingerprint'` (default) — IP + User-Agent hash. Safe for + * shared IPs (offices, VPNs). + * - `'ip'` — bare IP address. + * - `'user'` — actor's user ID. Use for authenticated routes + * where you want per-account limits. + * - `(req) => string` — custom key function. + * + * `scope` is an optional namespace prefix to isolate counters + * between routes that share the same key strategy. Defaults to + * the route path. + */ + rateLimit?: { + limit: number; + window: number; + key?: 'fingerprint' | 'ip' | 'user' | ((req: Request) => string); + scope?: string; + }; + + // Reserved — wire as the corresponding features/services land: + // bodyFiles?: string[]; // multer-style multipart fields + // responseTimeout?: number; +} + +/** + * Normalized route record produced by PuterRouter (and the class/method + * decorators). `path` is omitted only for `router.use(handler)` / `use(options, handler)`. + */ +export interface RouteDescriptor { + method: RouteMethod; + path?: RoutePath; + options: RouteOptions; + handler: RequestHandler; +} + +/** + * Shape stored on decorated controller prototypes by `@Get` / `@Post` / etc. + * `handler` is the method reference — still unbound at decoration time; + * the installed `registerRoutes` binds it to the instance at walk time. + */ +export interface CollectedRoute { + method: RouteMethod; + path?: RoutePath; + options: RouteOptions; + handler: RequestHandler; +} + +/** Internal: the property name used to stash decorator metadata on prototypes. */ +export const ROUTES_METADATA_KEY = '__puterRoutes' as const; +/** Internal: the property name used to stash a controller's path prefix. */ +export const PREFIX_METADATA_KEY = '__puterControllerPrefix' as const; + +// ── Type narrowing helpers ────────────────────────────────────────── +// +// When a route declares a gate option (requireAuth, requireUserActor, +// adminOnly, allowedAppIds), the materializer guarantees the corresponding +// gate runs before the handler. These types encode that guarantee at the +// type level, so handlers can use `req.actor` without a non-null assertion. +// +// Activated by the `const` generic on PuterRouter's per-method overloads: +// the literal options object is captured precisely (e.g. `{requireAuth: true}` +// rather than `{requireAuth: boolean}`), letting the conditional branches +// match by value. + +/** + * `true` iff the materializer will run an auth gate before the handler. + * Branches match readonly *and* mutable arrays so callers don't need + * `as const` on every options literal. + */ +export type AuthRequired = O extends { + requireAuth: true; +} + ? true + : O extends { requireUserActor: true } + ? true + : O extends { adminOnly: true | readonly string[] | string[] } + ? true + : O extends { allowedAppIds: readonly string[] | string[] } + ? true + : false; + +/** Express `Request` with `actor` narrowed based on the route's options. */ +export type TypedRequest = Omit & { + actor: AuthRequired extends true ? Actor : Actor | undefined; +}; + +/** Handler signature whose `req.actor` reflects the route's gate options. */ +export type TypedHandler = ( + req: TypedRequest, + res: Response, + next: NextFunction, +) => void | Promise; diff --git a/src/backend/core/http/vendorTypes.d.ts b/src/backend/core/http/vendorTypes.d.ts new file mode 100644 index 000000000..73ee49168 --- /dev/null +++ b/src/backend/core/http/vendorTypes.d.ts @@ -0,0 +1,30 @@ +/** + * Ambient type declarations for third-party packages that don't ship + * their own `.d.ts`. Keeps `tsc --noEmit` clean without pulling in + * `@types/*` packages for each one. + */ +declare module 'cookie-parser' { + import type { RequestHandler } from 'express'; + function cookieParser( + secret?: string | string[], + options?: object, + ): RequestHandler; + export = cookieParser; +} + +declare module 'compression' { + import type { RequestHandler } from 'express'; + function compression(options?: object): RequestHandler; + export = compression; +} + +declare module 'ua-parser-js' { + function UAParser(ua?: string): { + browser: { name?: string; version?: string; major?: string }; + engine: { name?: string; version?: string }; + os: { name?: string; version?: string }; + device: { vendor?: string; model?: string; type?: string }; + cpu: { architecture?: string }; + }; + export = UAParser; +} diff --git a/src/backend/core/index.ts b/src/backend/core/index.ts new file mode 100644 index 000000000..4a4ef5235 --- /dev/null +++ b/src/backend/core/index.ts @@ -0,0 +1,14 @@ +export { Context, runWithContext, type KnownContextFields } from './context'; +export { + type Actor, + type ActorUser, + type ActorApp, + type ActorAccessToken, + SYSTEM_ACTOR, + SYSTEM_ACTOR_UUID, + isSystemActor, + isAppActor, + isAccessTokenActor, + actorUid, + userRelatedActor, +} from './actor'; diff --git a/src/backend/src/data/hardcoded-permissions.js b/src/backend/data/hardcoded-permissions.js similarity index 91% rename from src/backend/src/data/hardcoded-permissions.js rename to src/backend/data/hardcoded-permissions.js index 5339fe44d..262b107ef 100644 --- a/src/backend/src/data/hardcoded-permissions.js +++ b/src/backend/data/hardcoded-permissions.js @@ -30,8 +30,8 @@ const default_implicit_user_app_permissions = { 'driver:puter-apps': {}, 'driver:puter-subdomains': {}, 'driver:temp-email': {}, - 'service': {}, - 'feature': {}, + service: {}, + feature: {}, }; const implicit_user_app_permissions = [ @@ -92,8 +92,8 @@ const driverPolicies = { temp: { kv: { 'rate-limit': { - max: 100, - period: 10000, + max: 10, + period: 1000, }, }, es: { @@ -106,8 +106,8 @@ const driverPolicies = { user: { kv: { 'rate-limit': { - max: 200, - period: 10000, + max: 20, + period: 1000, }, }, es: { @@ -119,19 +119,18 @@ const driverPolicies = { }, }; -const clonePolicy = policy => - JSON.parse(JSON.stringify(policy)); +const clonePolicy = (policy) => JSON.parse(JSON.stringify(policy)); -const getPolicyBySelector = selector => { +const getPolicyBySelector = (selector) => { const [scope, policyName] = selector.split('.'); const policy = driverPolicies[scope]?.[policyName]; - if ( ! policy ) { + if (!policy) { throw new Error(`unknown driver policy selector: ${selector}`); } return policy; }; -const policyPerm = selector => ({ +const policyPerm = (selector) => ({ policy: { ...clonePolicy(getPolicyBySelector(selector)), }, @@ -140,15 +139,14 @@ const policyPerm = selector => ({ const hardcoded_user_group_permissions = { system: { 'ca342a5e-b13d-4dee-9048-58b11a57cc55': { - 'driver': {}, - 'service': {}, - 'feature': {}, - 'kernel-info': {}, + driver: {}, + service: {}, + feature: {}, 'local-terminal:access': {}, }, 'b7220104-7905-4985-b996-649fdcdb3c8f': { - 'driver': {}, - 'service': {}, + driver: {}, + service: {}, 'service:hello-world:ii:hello-world': policyPerm('temp.es'), 'service:puter-kvstore:ii:puter-kvstore': policyPerm('temp.kv'), 'driver:puter-kvstore': policyPerm('temp.kv'), @@ -162,8 +160,8 @@ const hardcoded_user_group_permissions = { 'service:es\\Csubdomain:ii:crud-q': policyPerm('user.es'), }, '78b1b1dd-c959-44d2-b02c-8735671f9997': { - 'driver': {}, - 'service': {}, + driver: {}, + service: {}, 'service:hello-world:ii:hello-world': policyPerm('user.es'), 'service:puter-kvstore:ii:puter-kvstore': policyPerm('user.kv'), 'driver:puter-kvstore': policyPerm('user.kv'), diff --git a/src/backend/src/services/MeteringService/subPolicies/index.ts b/src/backend/data/subPolicies/index.ts similarity index 59% rename from src/backend/src/services/MeteringService/subPolicies/index.ts rename to src/backend/data/subPolicies/index.ts index 6db83db83..7da8715dd 100644 --- a/src/backend/src/services/MeteringService/subPolicies/index.ts +++ b/src/backend/data/subPolicies/index.ts @@ -1,7 +1,4 @@ import { REGISTERED_USER_FREE } from './registeredUserFreePolicy.js'; import { TEMP_USER_FREE } from './tempUserFreePolicy.js'; -export const SUB_POLICIES = [ - TEMP_USER_FREE, - REGISTERED_USER_FREE, -] as const; \ No newline at end of file +export const SUB_POLICIES = [TEMP_USER_FREE, REGISTERED_USER_FREE] as const; diff --git a/src/backend/src/services/MeteringService/subPolicies/registeredUserFreePolicy.ts b/src/backend/data/subPolicies/registeredUserFreePolicy.ts similarity index 53% rename from src/backend/src/services/MeteringService/subPolicies/registeredUserFreePolicy.ts rename to src/backend/data/subPolicies/registeredUserFreePolicy.ts index 115762b4d..df51f6426 100644 --- a/src/backend/src/services/MeteringService/subPolicies/registeredUserFreePolicy.ts +++ b/src/backend/data/subPolicies/registeredUserFreePolicy.ts @@ -1,8 +1,8 @@ -import { DEFAULT_FREE_SUBSCRIPTION } from '../consts.js'; -import { toMicroCents } from '../utils.js'; +import { DEFAULT_FREE_SUBSCRIPTION } from '../../services/metering/consts.js'; +import { toMicroCents } from '../../services/metering/utils.js'; export const REGISTERED_USER_FREE = { id: DEFAULT_FREE_SUBSCRIPTION, monthUsageAllowance: toMicroCents(0.25), monthlyStorageAllowance: 100 * 1024 * 1024, // 100MiB -} as const; \ No newline at end of file +} as const; diff --git a/src/backend/src/services/MeteringService/subPolicies/tempUserFreePolicy.ts b/src/backend/data/subPolicies/tempUserFreePolicy.ts similarity index 52% rename from src/backend/src/services/MeteringService/subPolicies/tempUserFreePolicy.ts rename to src/backend/data/subPolicies/tempUserFreePolicy.ts index 80017d71a..b6c86d718 100644 --- a/src/backend/src/services/MeteringService/subPolicies/tempUserFreePolicy.ts +++ b/src/backend/data/subPolicies/tempUserFreePolicy.ts @@ -1,8 +1,8 @@ -import { DEFAULT_TEMP_SUBSCRIPTION } from '../consts.js'; -import { toMicroCents } from '../utils.js'; +import { DEFAULT_TEMP_SUBSCRIPTION } from '../../services/metering/consts.js'; +import { toMicroCents } from '../../services/metering/utils.js'; export const TEMP_USER_FREE = { id: DEFAULT_TEMP_SUBSCRIPTION, monthUsageAllowance: toMicroCents(0.25), monthlyStorageAllowance: 100 * 1024 * 1024, // 100MiB -} as const; \ No newline at end of file +} as const; diff --git a/src/backend/doc/A-and-A/auth.md b/src/backend/doc/A-and-A/auth.md deleted file mode 100644 index 5c88949b3..000000000 --- a/src/backend/doc/A-and-A/auth.md +++ /dev/null @@ -1,63 +0,0 @@ -# Authentication Documentation - -## Concepts - -### Actor - -An "Actor" is an entity that can be authenticated. The following types of -actors are currently supported by Puter: -- **UserActorType** - represents a user and is identified by a user's UUID -- **AppUnderUserActorType** - represents an app running in an iframe from a - `puter.site` domain or another origin and is identified by a user's UUID - and an app's UUID together. -- **AccessTokenActorType** - not widely currently, but Puter supports - a concept called "access tokens". Any user can create an access token and - then grant any permissions they want to that access token. The access - token will have those permissions granted provided that the user who - created the access token does as well (via permission cascade) -- **SiteActorType** - represents a `puter.site` website accessing Puter's API. -- **SystemActorType** - internal representation of the actor during a privileged - backend operation. This actor cannot be authenticated in a request. - This actor does not represent the `system` user. - -### Token - -- **Legacy** - legacy tokens result in an error response -- **Session** - this token is a JWT with a claim for the UUID of an entry in - server memory or the database that we call a "session". This entry associates - the token to a user and some metadata for security auditing purposes. - Revoking the session entry disables the token. - This type of token resolves to an actor with **UserActorType**. -- **AppUnderUser** - this token is a JWT with a claim for an app UUID and a - claim for a session UUID. - Revoking the session entry disables the token. - This type of token resolves to an actor with **AppUnderUserActorType**. -- **AccessToken** - this token is a JWT with three claims: - - A session UUID - - An optional App UUID - - A UUID representing the access token for permission associations - The session or session+app creates a **UserActorType** or - **AppUnderUserActorType** actor respectively. This actor is called - the "authorizor". This actor is aggregated by an **AccessTokenActorType** - actor which becomes the effective actor for a request. -- **ActorSite** - this token is a JWT with a claim for a site UID. - The site UID is associated with an origin, generally a `puter.site` - subdomain. - -## Components - -### Auth Middleware - -There have so far been three iterations of the authentication middleware: -- `src/backend/src/middleware/auth.js` -- `src/backend/src/middleware/auth2.js` -- `src/backend/src/middleware/configurable_auth.js` - -The newest implementation is `configurable_auth` and eventually the other -two will be removed. There is no legacy behavior involved: -- `auth` was rewritten to use `auth2` -- `auth2` was rewritten to use `configurable_auth` - -The `configurable_auth` middleware accepts a parameter that can be specified -if an endpoint is optionally authenticated. In this case, the request's -`actor` will be `undefined` if there was no information for authentication. diff --git a/src/backend/doc/A-and-A/permission.md b/src/backend/doc/A-and-A/permission.md deleted file mode 100644 index ee4b758b0..000000000 --- a/src/backend/doc/A-and-A/permission.md +++ /dev/null @@ -1,179 +0,0 @@ -# Permission Documentation - -## Concepts - -### Permission - -A permission is a string composed of colon-delimited components which identifies -a resource or functionality to which access can be controlled. - -For example, `fs:e8ac2973-287b-4121-a75d-7e0619eb8e87:read` is a permission which -represents reading the file or directory with UUID `e8ac2973-287b-4121-a75d-7e0619eb8e87`. - -### Group - -A group has an owner and several member users. An owner decides what users are in the -group and what users are not. Any user can grant permissions to the group. - -### Granting & Revoking - -Granting is the act of creating a permission association to a user or group from -the current user. A permission association also holds an object called `extra` -which holds additional claims associated with the permission association. -These are arbitrary and can be used in any way by the subsystem or extension that -is checking the permission. `extra` is usually just an empty object. - -Revoking is the act of removing a permission association. - -### Permission Options - -Permission options are an association between a permission and an actor that can not -be revoked by another actor. For example, the user `ed` always has access to files -under `/ed`. The user `system` always has all permissions granted. These can also be -considered "terminals" because they will always be at -the end of a pathway through granted permissions between users. -This are also called "implied" permissions because they are implied by the system. - -### Permission Pathways - -A permission pathway is the path between users or groups that leads to a permission. - -For example, `ed` can grant the permission `a:b` to `fred`, then `fred` can grant -that permission to the group `cool_group`, and then `alice` may be in the group -`cool_group`. Assuming `ed` holds the implied permission `a:b`, a permission path -exists between `alice` and `ed` via `cool_group` and `fred`: - -``` -alice <--<> cool_group <-- fred <-- ed (a:b) -``` - -If any link in this chain breaks the permission is effectively revoked from `alice` -unless there is another pathway leading to a valid permission option for `a:b`. - -### Reading - AKA Permission Scan Result - -A permission reading is a JSON-serializable object which contains all the pathways -a specified actor has to permissions options matching the specified permission strings. - -The following is an example reading for the user `ed3` on the permission -`fs:24729b88-a4c5-4990-ad4e-272b87895732:read`. This file is owned by the -user `admin` who shared it with `ed3`. - -``` -[ - { - "$": "explode", - "from": "fs:24729b88-a4c5-4990-ad4e-272b87895732:read", - "to": [ - "fs:24729b88-a4c5-4990-ad4e-272b87895732:read", - "fs:24729b88-a4c5-4990-ad4e-272b87895732:write", - "fs:24729b88-a4c5-4990-ad4e-272b87895732", - "fs" - ] - }, - { - "$": "path", - "via": "user", - "has_terminal": true, - "permission": "fs:24729b88-a4c5-4990-ad4e-272b87895732:read", - "data": {}, - "holder_username": "ed3", - "issuer_username": "admin", - "reading": [ - { - "$": "explode", - "from": "fs:24729b88-a4c5-4990-ad4e-272b87895732:read", - "to": [ - "fs:24729b88-a4c5-4990-ad4e-272b87895732:read", - "fs:24729b88-a4c5-4990-ad4e-272b87895732:write", - "fs:24729b88-a4c5-4990-ad4e-272b87895732", - "fs" - ] - }, - { - "$": "option", - "permission": "fs:24729b88-a4c5-4990-ad4e-272b87895732:read", - "source": "implied", - "by": "is-owner", - "data": {} - }, - { - "$": "option", - "permission": "fs:24729b88-a4c5-4990-ad4e-272b87895732:write", - "source": "implied", - "by": "is-owner", - "data": {} - }, - { - "$": "option", - "permission": "fs:24729b88-a4c5-4990-ad4e-272b87895732", - "source": "implied", - "by": "is-owner", - "data": {} - }, - { - "$": "time", - "value": 19 - } - ] - }, - { - "$": "time", - "value": 20 - } -] -``` - -Each object in the reading has a property named `$` which is the type for the object. -The most fundamental types for permission readings are `path` and `option`. A path -always contains another reading, which contains more paths or options. An option -specifies the permission string, the name of the rule that granted the permission, -and a data object which may hold additional claims. - -Readings begin with an `explode` if there are multiple strings that may grant the -permission. - -Readings end with a `time` that repots how long the reading took to help manage -the potential performance impact of complex permission graphs. - -## Permission Service - -### check(actor, permissions) - -Returns true if the current actor has a path to any permission options matching -any of the permission strings specified by `permissions`. This is done by invoking -`scan()` and returning `true` if there are more than 0 permission options. - -### scan(actor, permissions) - -Returns a "reading". A permission reading is a JSON-serializable structure. -Readings are described above. - -## Permission Scan Sequence - -The `scan()` method of **PermissionService** invokes the permission scan sequence. -The permission scan sequence is a [Sequence](https://github.com/HeyPuter/puter/blob/0e0bfd6d7c92eed5080518a099c9a66a2f2dc9ec/src/backend/src/codex/Sequence.js) -that is defined in [scan-permission.js](src/backend/src/structured/sequence/scan-permission.js). -It invokes many "permission scanners" which are defined in -[permission-scanners.js](src/backend/src/unstructured/permission-scanners.js) - -The Permission Scan Sequence is as follows: -- `grant_if_system` - if system user, push an option to the reading and stop -- `rewrite_permission` - process the permission through any permission string - rewriters that were registered with PermissionService by other services. - For example, since path-based file permissions aren't currently supported - the FilesystemService regsiters a rewriter that converts any `fs:/` - permission into a corresponding UUID permission. -- `explode_permission` - break the permission into multiple permissions - than are sufficient to grant the permission being scanned. For example if - there are multiple components, like `a.b.c`, having either permission `a.b` or - `a` granted implis having `a.b.c` granted. Other services can also register - "permission exploders" which handle non-hierarchical cases such as - `fs:AAAA:write` implying `fs:AAAA:read`. -- `run_scanners` - run the permission scanners. - -Each permission scanner has a name, documentation text, and a scan function. -The scan function has access to the scan sequence's context and can push -objects onto the permission reading. - -For information on individual scanners, refer to permission-scanners.js. diff --git a/src/backend/doc/Kernel.md b/src/backend/doc/Kernel.md deleted file mode 100644 index 0cb15ef05..000000000 --- a/src/backend/doc/Kernel.md +++ /dev/null @@ -1,65 +0,0 @@ -# Puter Kernel Documentation - -## Overview - -The **Puter Kernel** is the core runtime component of the Puter system. It provides the foundational infrastructure for: - -- Initializing the runtime environment -- Managing internal and external modules (extensions) -- Setting up and booting core services -- Configuring logging and debugging utilities -- Integrating with third-party modules and performing dependency installs at runtime - -This kernel is responsible for orchestrating the startup sequence and ensuring that all necessary services, modules, and environmental configurations are properly loaded before the application enters its operational state. - ---- - -## Features - -1. **Modular Architecture**: - The Kernel supports both internal and external modules: - - **Internal Modules**: Provided to Kernel by an initializing script, such - as `tools/run-selfhosted.js`, via the `add_module()` method. - - **External Modules**: Discovered in configured module directories and installed - dynamically. This includes resolving and executing `package.json` entries and - running `npm install` as needed. - -2. **Service Container & Registry**: - The Kernel initializes a service container that manages a wide range of services. Services can: - - Register modules - - Initialize dependencies - - Emit lifecycle events (`boot.consolidation`, `boot.activation`, `boot.ready`) to - orchestrate a stable and consistent environment. - -3. **Runtime Environment Setup**: - The Kernel sets up a `RuntimeEnvironment` to determine configuration paths and environment parameters. It also provides global helpers like `kv` for key-value storage and `cl` for simplified console logging. - -4. **Logging and Debugging**: - Uses a temporary `BootLogger` for the initialization phase until LogService is - initialized, at which point it will replace the boot logger. Debugging features - (`ll`, `xtra_log`) are enabled in development environments for convenience. - -## Initialization & Boot Process - -1. **Constructor**: - When a Kernel instance is created, it sets up basic parameters, initializes an empty - module list, and prepares `useapi()` integration. - -2. **Booting**: - The `boot()` method: - - Parses CLI arguments using `yargs`. - - Calls `_runtime_init()` to set up the `RuntimeEnvironment` and boot logger. - - Initializes global debugging/logging utilities. - - Sets up the service container (usually called `services`c instance of **Container**). - - Invokes module installation and service bootstrapping processes. - -3. **Module Installation**: - Internal modules are registered and installed first. - External modules are discovered, packaged, installed, and their code is executed. - External modules are given a special context with access to `useapi()`, a dynamic - import mechanism for Puter modules and extensions. - -4. **Service Bootstrapping**: - After modules and extensions are installed, services are initialized and activated. - For more information about how this works, see [boot-sequence.md](./contributors/boot-sequence.md). - diff --git a/src/backend/doc/README.md b/src/backend/doc/README.md deleted file mode 100644 index 12280e773..000000000 --- a/src/backend/doc/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## Backend - Contributor Documentation - -### Where to Start - -Start with [Backend File Structure](./contributors/structure.md). - -There also also some videos. In one of the videos Eric does a -Steve Ballmer impression so it's definitely worth it. -- [Services and Modules in Puter](https://www.youtube.com/watch?v=TOeS67QXMVU) -- [Puter's Boot Sequence](https://www.youtube.com/watch?v=a8bOLNnW1Uo) -- [Building a Driver on Puter](https://www.youtube.com/watch?v=8znQmrKgNxA) - -### Index - -- [Backend File Structure](./contributors/structure.md) -- [Boot Sequence](./contributors/boot-sequence.md) -- [Kernel](./Kernel.md) -- [Modules](./contributors/modules.md) -- [Configuring Logs](./log_config.md) diff --git a/src/backend/doc/contributors/boot-sequence.md b/src/backend/doc/contributors/boot-sequence.md deleted file mode 100644 index b98e8facd..000000000 --- a/src/backend/doc/contributors/boot-sequence.md +++ /dev/null @@ -1,93 +0,0 @@ -# Puter Backend Boot Sequence - -This document describes the boot sequence of Puter's backend. - -**Runtime Environment** - - Configuration directory is determined - - Runtime directory is determined - - Mod directory is determined - - Services are instantiated - -**Construction** - - Data structures are created - -**Initialization** - - Registries are populated - - Services prepare for next phase - -**Consolidation** - - Service event bus receives first event (`boot.consolidation`) - - Services perform coordinated setup behaviors - - Services prepare for next phase - -**Activation** - - Blocking listeners of `boot.consolidation` have resolved - - HTTP servers start listening - -**Ready** - - Services are informed that Puter is providing service - -## Boot Phases - -### Construction - -Services implement a method called `construct` which initializes members -of an instance. Services do not override the class constructor of -**BaseService**. This makes it possible to use the `new` operator without -invoking a service's constructor behavior during debugging. - -The first phase of the boot sequence, "construction", is simply a loop to -call `construct` on all registered services. - -The `_construct` override should not: -- call other services -- emit events - -### Initialization - -At initialization, the `init()` method is called on all services. -The `_init` override can be used to: -- register information with other services, when services don't - need to register this information in a specific sequence. - An example of this is registering commands with CommandService. -- perform setup that is required before the consolidation phase starts. - -### Consolidation - -Consolidation is a phase where services should emit events that -are related to bringing up the system. For example, WebServerService -('web-server') emits an event telling services to install middlewares, -and later emits an event telling services to install routes. - -Consolidation starts when Kernel emits `boot.consolidation` to the -services event bus, which happens after `init()` resolves for all -services. - -### Activation - -Activation is a phase where services begin listening on external -interfaces. For example, this is when the web server starts listening. - -Activation starts when Kernel emits `boot.activation`. - -### Ready - -Ready is a phase where services are informed that everything is up. - -Ready starts when Kernel emits `boot.ready`. - -## Events and Asynchronous Execution - -The services event bus is implemented so you can `await` a call to `.emit()`. -Event listeners can choose to have blocking behavior by returning a promise. - -During emission of a particular event, listeners of this event will not -block each other, but all listeners must resolve before the call to -`.emit()` is resolved. (i.e. `emit` uses `Promise.all`) - -## Legacy Services - -Some services were implemented before the `BaseService` class - which -implements the `init` method - was created. These services are called -"legacy services" and they are instantiated _after_ initialization but -_before_ consolidation. diff --git a/src/backend/doc/contributors/coding-style.md b/src/backend/doc/contributors/coding-style.md deleted file mode 100644 index 4ff125539..000000000 --- a/src/backend/doc/contributors/coding-style.md +++ /dev/null @@ -1,212 +0,0 @@ -# Backend Style - -## File Structure - -### Copyright Notice - -All files should begin with the standard copyright notice: - -```javascript -/* - * Copyright (C) 2025-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -``` - -### Imports - -```javascript -const express = require('express'); -const passport = require('passport'); - -const { get_user } = require("../../helpers"); -const BaseService = require("../../services/BaseService"); -const config = require("../../config"); - -const path = require('path'); -const fs = require('fs'); -``` - -Import order is generally: -1. Third party dependencies. Having these occur first makes it easy to quickly - determine what this source file is likely to be responsible for. -2. Files within the module. -3. Standard library, "builtins" - -## Code Formatting - -### Indentation and Spacing - -```javascript -const fn = async () => { - const a = 5; // Spaces between operators - - // Note: "=" in for loop initializer does not require space around - // Note: operators in condition part have space around - for ( let i=0; i < 10; i++ ) { - console.log('hello'); - } - - // Control structures have space inside parenthesis - for ( const thing of stuff ) { - // NOOP - } - - // Function calls do not have space inside parenthesis - await something(1, 2); -} -``` - -- Use 4 spaces for indentation. -- Use spaces around operators (`=`, `+`, etc.); not required in - for loop initializer. -- Use a space after keywords like `if`, `for`, `while`, etc. - ```javascript - return [1,2,3]; // Sure - return[1,2,4]; // Definitely not - ``` -- Use spaces between parenthesis in control structures unless - parenthesis are empty. - ```javascript - if ( a === b ) { - return null; - } - ``` -- No trailing whitespace at the end of lines -- Use a space after commas in arrays and objects -- Empty blocks should have the comment `// NOOP` within braces - -### Line Length - -- Try to keep lines under 100 characters for better readability - - Try to keep them under 80, but this is not always practical -- For long function calls or objects, break them into multiple lines - - -### Trailing Commas - -```javascript -// This is great -{ - "apple", - "banana", - "cactus", // <-- Good! -} - -// This is also fine -[ - 1, 2, 3, - 4, 5, 6, - 7, 8, 9, -] - -[ - something(), - another_thing(), - the_last_thing() // <-- Nope, please add trailing comma! -] -``` - -We use trailing commas where applicable because it's easier to re-order -lines, especially when using vim motions. - -### Braces and Blocks - -- Single statement blocks must either be on the same line as - the corresponding control structure, or surrounding by braces: - ```javascript - if ( a === b ) return null; // Sure - if ( a === b ) - return null; // Please no 🤮 - if ( a === b ) { - return null; // Nice - } - ``` -- Opening braces go on the same line as the statement -- Put a space before the opening brace - - -## Naming Conventions - -### Variables - -- Variables are generally in camelCase -- Variables might have a prefix_beforeThem - -```javascript -const svc_systemData = this.services.get('system-data'); -const svc_su = this.services.get('su'); -effective_policy = await svc_su.sudo(async () => { - return await svc_systemData.interpret(effective_policy.data); -}); -``` - -In the example above we see the `svc_` prefix is used to indicate a -reference to a backend service. The name of the service is `system-data` -which is not a valid identifier, so we use `svc_systemData` for our -variable name. - -### Classes - -- Use PascalCase for class names -- Use snake_case for class methods -- Instance variables are often `snake_case` because it's easier to - read. `camelCase` is acceptable too. -- Instance variables only used internally should have a - `trailing_underscore_` even if in `camelCase_`. We avoid using - `#privateProperties` because it unnecessarily inhibits debugging - and patching. - -### File Names - -- Use PascalCase for class files (e.g., `UserService.js`) -- Use kebab-case for non-class files (e.g., `auth-helper.js`) - -## Documentation - -### JSDoc Comments - -- Backend services (classes extending `BaseService`) should have JSDoc comments -- Public methods of backend services should have JSDoc comments -- Include parameter descriptions, return values, and examples where appropriate - -```javascript -/** - * @class UserService - * @description Service for managing user operations - */ - -/** - * Get a user by their ID - * @param {string} id - The user ID - * @returns {Promise} The user object - * @throws {Error} If user not found - */ -async function getUserById(id) { - // ... -} -``` - -### Inline Comments - -- Use inline comments to explain complex logic -- Prefix comments with tags like `track:` to indicate specific purposes - -```javascript -// track: slice a prefix -const uid = uid_part.slice('uid#'.length); -``` diff --git a/src/backend/doc/contributors/modules.md b/src/backend/doc/contributors/modules.md deleted file mode 100644 index c19b49d66..000000000 --- a/src/backend/doc/contributors/modules.md +++ /dev/null @@ -1,103 +0,0 @@ -# Puter Kernel Moduels and Services - -## Modules - -A Puter kernel module is simply a collection of services that run when -the module is installed. You can find an example of this in the -`run-selfhosted.js` script at the root of the Puter monorepo. - -Here is the relevant excerpt in `run-selfhosted.js` at the time of -writing this documentation: - -```javascript -const { - Kernel, - CoreModule, - DatabaseModule, - LocalDiskStorageModule, - SelfHostedModule -} = (await import('@heyputer/backend')).default; - -const k = new Kernel(); -k.add_module(new CoreModule()); -k.add_module(new DatabaseModule()); -k.add_module(new LocalDiskStorageModule()); -k.add_module(new SelfHostedModule()); -k.boot(); -``` - -A few modules are added to Puter before booting. If you want to install -your own modules into Puter you can edit this file for self-hosted runs -or create your own script that boots Puter. This makes it possible to -have deployments of Puter with custom functionality. - -To function properly, Puter needs **CoreModule**, a database module, -and a storage module. - -A module extends -[AdvancedBase](../../../putility/README.md) -and implements -an `install` method. The install method has one parameter, a -[Context](../../src/util/context.js) -object containing all the values kernel modules have access to. This -includes the `services` -[Container](../../src/services/Container.js`). - -A module adds services to Puter.eA typical module may look something -like this: - -```javascript -class MyPuterModule extends AdvancedBase { - async install (context) { - const services = context.get('services'); - - const MyService = require('./path/to/MyService.js'); - services.registerService('my-service', MyService, { - some_options: 'for-my-service', - }); - } -} -``` - -## Services - -Services extend -[BaseService](../../src/services/BaseService.js) -and provide additional functionality for Puter. They can add HTTP -endpoints and register objects with other services. - -When implementing a service it is important to understand -Puter's [boot sequence](./boot-sequence.md) - -A typical service may look like this: - -```javascript -class MyService extends BaseService { - static MODULES = { - // Use node's `require` function to populate this object; - // this makes these available to `this.require` and offers - // dependency-injection for unit testing. - ['some-module']: require('some-module') - } - - // Do not override the constructor of BaseService - use this instead! - async _construct () { - this.my_list = []; - } - - // This method is called after _construct has been called on all - // other services. - async _init () { - const services = this.services; - - // We can get the instances of other services here - const svc_otherService = services.get('other-service'); - } - - // The service container can listen on the "service event bus" - async ['__on_boot.consolidation'] () {} - async ['__on_boot.activation'] () {} - async ['__on_start.webserver'] () {} - async ['__on_install.routes'] () {} -} -``` diff --git a/src/backend/doc/extensions/README.md b/src/backend/doc/extensions/README.md deleted file mode 100644 index 2ac7ca49e..000000000 --- a/src/backend/doc/extensions/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# Puter Backend Extensions - -## What Are Extensions - -Extensions can extend the functionality of Puter's backend by handling specific -events or importing/exporting runtime libraries. - -## Creating an Extension - -The easiest way to create an extension is to place a new file or directory under -the `extensions/` directory immediately under the root directory of the Puter -repository. If your extension is a single `.js` file called `my-extension.js` it -will be implicitly converted into a CJS module with the following structure: - -``` -extensions/ - | - |- my-extension/ - | - |- package.json - |- main.js -``` - -The location of the extensions directory can be changed in -[the config file](../../../../doc/self-hosters/config.md) -by setting `mod_directories` to an array of valid locations. -The `mod_directories` parameter has the following default value: -```json -["{repo}/mods/mods_enabled", "{repo}/extensions"] -``` - -### Events - -The primary mechanism of communication between extensions and Puter, -and between different extensions, is through events. The `extension` -pseudo-global provides `.on(fn)` to add event listemers and -`.emit('name', { arbitrary: 'data' })` to emit events. - -To try working with events, you could make a simple extension that -emits an event after adding a listener for its own event: - -```javascript -// Listen to a test event called 'test-event' -extension.on('test-event', event => { - console.log(`We got the test event from ${sender}`); -}); - -// Listen to init; a good time to emit events -extension.on('init', event => { - extension.emit('test-event', { sender: 'Quinn' }); -}); -``` - -### Puter Extension Imports - -Your extensions may need to invoke specific actions in Puter's backend -in response to an event. Puter provides libraries at runtime which you -can access via `extension.imports`: - -```javascript -const { kv } = extension.imports('data'); -kv.set('some-key', 'some value'); -``` - -#### The `data` import - -The data import makes it possible to access Puter's database, persistent -key-value store, and in-memory cache. -- [Read more about the 'data' import](./builtins/data.md) - - -### Adding Features to Puter -- [Implementing Drivers](./pages/drivers.md) - -## Extensions - Planned Features - -Extensions are under refactor currently. This is the checklist: -- [x] Add RuntimeModule construct for imports and exports -- [x] Add support to implement drivers in extensions -- [ ] Add the ability to target specific extensions when - emitting events -- [ ] Add event name aliasing and configurable import mapping -- [ ] Extract extension loading from the core -- [ ] List exports in console diff --git a/src/backend/doc/extensions/builtins/data.md b/src/backend/doc/extensions/builtins/data.md deleted file mode 100644 index cb2611f9c..000000000 --- a/src/backend/doc/extensions/builtins/data.md +++ /dev/null @@ -1,128 +0,0 @@ -## Extensions - the `data` extension - -The `data` extension can be imported in custom extensions for access -to the database and key-value store. - -You can import these from `'data'`: -- `db` - Puter's main SQL database -- `kv` - A persistent key-value store -- `cache` - In-memory [kv.js](https://github.com/HeyPuter/kv.js/) store - -```javascript -const { db, kv, cache } = extension.import('data'); -``` - -### Database (`db`) - -Don't forget to import it first! -```javascript -const { db } = extension.import('data'); -``` - -#### `db.read` - -Usage: - -```javascript -const rows = await db.read('SELECT * FROM apps WHERE `name` = ?', [ - 'editor' -]); -``` -#### `db.write` - -Usage: - -```javascript -const { - insertId, // internal ID of new row (if this is an INSERT) - anyRowsAffected, // true if 1 or more rows were affected -} = await db.write( - // A query like INSERT, UPDATE, DELETE, etc... - 'INSERT INTO example_table (a, b, c) VALUES (?, ?, ?)', - // Parameters (all user input should go here) - [ - "Value for column a", - "Value for column b", - "Value for column c", - ] -); -``` - -### Persistent KV Store (`kv`) - -Don't forget to import it first! -```javascript -const { kv } = extension.import('data'); -``` - -#### `kv.get({ key })` - -```javascript -// Short-Form (like kv.js) -const someValue = kv.get('some-key'); - -// Long-Form (the `puter-kvstore` driver interface) -const someValue = kv.get({ key: 'some-key' }); -``` - -#### `kv.set({ key, value })` - -```javascript -await kv.set('some-key', 'some value'); - -// or... - -await kv.set({ - key: 'some-key', - value: 'some value', -}); -``` - -#### `kv.expire({ key, ttl })` - -This key will persist for 20 minutes, even if the server restarts. - -```javascript -kv.expire({ - key: 'some-key', - ttl: 1000 * 60 * 20, // 20 minutes -}); -``` - -### `kv.expireAt({ key, timestamp })` - -The following example expires a key 1 second before -["the apocalypse"](https://en.wikipedia.org/wiki/Year_2038_problem). -(don't worry, KV won't break in 2038) - -```javascript -kv.expireAt( - key: 'some-key', - // Expires Jan 19 2038 3:14:07 GMT - timestamp: 2147483647, -); -``` - -### In-Memory Cache (`cache`) - -Don't forget to import it first! -```javascript -const { cache } = extension.import('data'); -``` - -The in-memory cache is provided by [kv.js](https://github.com/HeyPuter/kv.js). -Below is a simple example. -For comprehensive documentation, see the [kv.js repository's readme](https://github.com/HeyPuter/kv.js/blob/main/README.md). - -```javascript -const { cache } = extension.require('data'); - -cache.set('some-key', 'some value'); -const value = cache.get('some-key'); // some value - -// This value only exists for 5 minutes -cache.set('temporary', 'abcdefg', { EX: 5 * 60 }); - -cache.incr('qwerty'); // cache.get('qwerty') is now: 1 -cache.incr('qwerty'); // cache.get('qwerty') is now: 2 -``` diff --git a/src/backend/doc/extensions/pages/core-devs.md b/src/backend/doc/extensions/pages/core-devs.md deleted file mode 100644 index 302912b6b..000000000 --- a/src/backend/doc/extensions/pages/core-devs.md +++ /dev/null @@ -1,135 +0,0 @@ -## Extensions - Technical Context for Core Devs - -This document provides technical context for extensions from the perspective of -core backend modules and services, including the backend kernel. - -### Lifecycle - -For extensions, the concept of an "init" event handler is different from core. -This is because a developer of an extension expects `init` to occur after core -modules and services have been initialized. For this reason, extensions receive -`init` when backend services receive `boot.consolidation`. - -It is still possible to handle core's `init` event in an extension. This is done -using the `preinit` event. - -``` -Backend Core Lifecycle - Modules -> Construction -> Initialization -> Consolidation -> Activation -> Ready -Extension Lifecycle - index.js executed -> (no event) -> 'preinit' -> 'init' -> (no event) -> 'ready' -``` - -Extensions have an implicit Service instance that needs to listen for events on -the **Service Event Bus** such as `install.routes` (emitted by WebServerService). -Since extensions need to affect the behavior of the service when these events -occur (for example using `extension.post()` to add a POST handler) it is necessary -for their entry files to be loaded during a module installation phase, when -services are being registered and `_construct()` has not yet been called on any -service. - -Kernel.js loads all core modules/services before any extensions. This allows -core modules and services to create [runtime modules](./runtime-modules.md) -which can be imported by services. - -### How Extensions are Loaded - -Before extensions are loaded, all of Puter's core modules have their `.install()` -methods called. The core modules are the ones added with `kernel.add_module`, -for example in [run-selfhosted.js](../../../../../tools/run-selfhosted.js). - -Then, `Kernel.install_extern_mods_` is called. This is where a `readdir` is -performed on each directory listed in the `"mod_directories"` configuration -parameter, which has a default value of `["{repo}/extensions"]` (the -placeholder `{repo}` is automatically replaced with the path to the Puter -repository). - -For each item in each mod directory, except for ignored items like `.git` -directories, a mod is installed. First a directory is created in Puter's -runtime directory (`volatile/runtime` locally, `/var/puter` on a server). -If the item is a file then a `package.json` will be created for it after -`//@extension` directives are processed. If the item is a directory then -it is copied as is and `//@extension` directives are not supported -(`puter.json` is used instead). Source files for the mod are copied to -the mod directory under the runtime directory. - -It is at this point the pseudo-globals are added be prepending `cost` -declarations at the top of `.js` files in the extension. This is not -a great way to do this, but there is a severe lack of options here. -See the heading below - "Extension Pseudo-Globals" - for details. - -Before the entry file for the extension is `require()`'d a couple of -objects are created: an `ExtensionModule` and an `Extension`. -The `ExtensionModule` is a Puter module just like any of the Puter core -modules, so it has an `.install()` method that installs services before -Puter's kernel starts the initialization sequence. In this case it will -install the implied service that an extension creates if it registers -routes or performs any other action that's typically done inside services -in core modules. - -A RuntimeModule is also created. This could be thought of as analygous -to node's own `Module` class, but instead of being for imports/exports -between npm modules it's for imports/exports between Puter extensions -loaded at runtime. (see [runtime modules](./runtime-modules.md)) - -### Extension Pseudo-Globals - -The `extension` global is a different object per extension, which will -make it possible to develop "remapping" for imports/exports when -extension names collide among other functions that need context about -which extension is calling them. Implementing this per-extension global -was very tricky and many solutions were considered, including using the -`node:vm` builtin module to run the extension in a different instance. -Unfortunately `node:vm` support for EMCAScript Modules is lacking; -`vm.Module` has a drastically different API from `vm.Script`, requires -an experimental feature flag to be passed to node, and does not provide -any alternative to `createRequire` to make a valid linker for the -dependencies of a package being run in `node:vm`. - -The current solution - which sucks - is as follows: prepend `const` -definitions to the top of every `.js` file in the extension's installation -directory unless it's under a directory called `node_modules` or `gui`. -This type of "pseudo-global" has a quirk when compared to real globals, -which is that they can't be shadowed at the root scope without an error -being thrown. The naive solution of wrapping the rest of the file's -contents in a scope limiter (`{ ... }`) would break ES Module support -because `import` directives must be in the top-level scope, and the naive -solution to that problem of moving imports to the top of the file after -adding the scope limiter requires invoking a javascript parser do -determine the difference between a line starting with `import` because -it's actually an import and this unholy abomination of a situation: -``` -console.log(` -import { me, and, everything, breaks } from 'lackOfLexicalAnalysis'; -`); -``` - -Exposing the same instance for `extension` to all extensions with a -real global and using AsyncLocalStorage to get the necessary information -about the calling extension on each of `extension`'s methods was another -idea. This would cause surprising behavior for extension developers when -calling methods on `extension` in callbacks that lose the async context -fail because of missing extension information. - -Eventually a better compromise will be to have commonjs extensions -run using `vm.Script` and ESM extensions continue to run using this hack. - -### Event Listener Sub-Context - -In extensions, event handlers are registered using `extension.on`. These -handlers, when called, are supplemented with identifying information for -the extension through AsyncLocalStorage. This means any methods called -on the object passed from the event (usually just called `event`) will -be able to access the extension's name. - -This is used by CommandService's `create.commands` event. For example -the following extension code will register the command `utils:say-hello` -if it is invoked form an extension named `utils`: - -```javascript -extension.on('create.commands', event => { - event.createCommand('say-hello', async (args, console) => { - console.log('Hello,', ...args); - }); -}); -``` diff --git a/src/backend/doc/extensions/pages/drivers.md b/src/backend/doc/extensions/pages/drivers.md deleted file mode 100644 index 64be7a523..000000000 --- a/src/backend/doc/extensions/pages/drivers.md +++ /dev/null @@ -1,145 +0,0 @@ -## Extensions - Implementing Drivers - -Puter's concept of drivers has existed long before the extension system -was refined, and to keep things moving forward it has become easier to -develop Puter drivers in extensions than anywhere else in Puter's source. -If you want to build a driver, an extension is the recommended way to do it. - -### What are Puter drivers? - -Puter drivers are all called through the `/drivers/call` endpoint, so they -can be thought of as being "above" the HTTP layer. When a method on a driver -throws an error you will still receive a `200` HTTP status response because -the the invocation - from the HTTP layer - was successful. - -A driver response follows this structure: -```json -{ - "success": true, - "service": { - "name": "implementation-name" - }, - "result": "any type of value goes here", - "metadata": {} -} -``` - -There exists an example driver called `hello-world`. This driver implements -a method called `greet` with the optional parameter `subject` which returns -a string greeting either `World` (default) or the specified subject. - -```javascript -await puter.call('hello-world', 'no-frills', 'greet', { subject: 'Dave' }); -``` - -Let's break it down: - -#### `'hello-world'` - -`'hello-world'` is the name of an "interface". An interface can be thought of -a contract of what inputs are allowed and what outputs are expected. For -example the `hello-world` interface specifies that there must be a method -called `greet` and it should return a string representing a greeting. - -To add another example, an interface called `weather` specify a method called -`forcast5day` that always returns a list of 5 objects with a particular -structure. - -#### `no-frills` - -`'no-frills'` is a simple - "no frills" (nothing extra) - implementation of -the `hello-world` interface. All it does is return the string: -```javascript -`Hello, ${subject ?? 'World'}!` -``` - - -#### `'greet'` - -`greet` is the method being called. It's the only method on the `hello-world` -interface. - -#### `{ subject: 'Dave' }` - -These are the arguments to the `greet` method. The arguments specify that we -want to say "Hello" to Dave. Hopefully he doesn't ask us to open the pod bay -doors, or if he does we hopefully have extensions to add a driver interface -and driver implementation for the pod bay doors so that we can interact with -them. - -### Drivers in Extensions - -The `hellodriver` extension adds the `hello-world` interface like this: -```javascript -extension.on('create.interfaces', event => { - // createInterface is the only method on this `event` - event.createInterface('hello-world', { - description: 'Provides methods for generating greetings', - methods: { - greet: { - description: 'Returns a greeting', - parameters: { - subject: { - type: 'string', - optional: true - }, - locale: { - type: 'string', - optional: true - }, - } - } - } - }) -}); -``` - -The `hellodriver` extension adds the `no-frills` implementation for -`hello-world` like this: -```javascript -extension.on('create.drivers', event => { - event.createDriver('hello-world', 'no-frills', { - greet ({ subject }) { - return `Hello, ${subject ?? 'World'}!`; - } - }); -});` -``` - -You can pass an instance of a class for a driver implementation as well: -```javascript -class Greeter { - greet ({ subject }) { - return `Hello, ${subject ?? 'World'}!`; - } -} - -extension.on('create.drivers', event => { - event.createDriver('hello-world', 'no-frills', new Greeter()); -});` -``` - -Instances of classes being supported -may seem to be implied by the example before this -one, but that is not the case. What's shown here is that function members -of the object passed to `createDriver` will not be "bound" (have their -`.bind()` method called with a different object as the instance variable). - -### Permission Denied - -When you try to access a driver as any user other than the default -`admin` user, it will not work unless permission has been granted. - -The `hellodriver` extension grants permission to all clients using -the following snippet: -```javascript -extension.on('create.permissions', event => { - event.grant_to_everyone('service:no-frills:ii:hello-world'); -}); -``` - -The `create.permissions` event's `event` object has a few methods -you can use depending on the desired granularity: -- `grant_to_everyone` - grants permission to all users -- `grant_to_users` - grants permission to only registered users - (i.e. not to temporary/guest users) diff --git a/src/backend/doc/extensions/pages/import-and-export.md b/src/backend/doc/extensions/pages/import-and-export.md deleted file mode 100644 index 68d804add..000000000 --- a/src/backend/doc/extensions/pages/import-and-export.md +++ /dev/null @@ -1,28 +0,0 @@ -## Extensions - Importing & Exporting - -Here are two extensions. One extension has an "extension export" (an export to -other extensions) and an "extension import" (an import from another extension). -This is different from regular `import` or `require()` because it resolves to -a Puter extension loaded at runtime rather than an `npm` module. - -To import and export in Puter extensions, we use `extension.import()` and `extension.exports`. - -`exports-something.js` -```javascript -//@puter priority -1 -// ^ setting load priority to "-1" allows other extensions to import -// this extension's exports before the initialization event occurs - -// Just like "module.exports", but for extensions! -extension.exports = { - test_value: 'Hello, extensions!', -}; -``` - -`imports-something.js` -```javascript -const { test_value } = extension.import('exports-something'); - -console.log(test_value); // 'Hello, extensions!' -``` - diff --git a/src/backend/doc/extensions/pages/runtime-modules.md b/src/backend/doc/extensions/pages/runtime-modules.md deleted file mode 100644 index 454f699b5..000000000 --- a/src/backend/doc/extensions/pages/runtime-modules.md +++ /dev/null @@ -1,49 +0,0 @@ -## Extensions - Runtime Modules - -Runtime modules are modules that extensions can import with tihs syntax: - -```javascript -const somelib = extension.import('somelib'); -``` - -These modules are registered in the [runtime module registry](../../../src/extension/RuntimeModuleRegistry.js) -which is instantiated by [Kernel.js](../../../src/Kernel.js). - -All extensions implicitly have a Runtime Module. The runtime module shares the name -of the extension that it corresponds to. Extensions can export to their module by -using `extension.exports`: - -```javascript -extension.exports = { /* ... */ }; -``` - -The [Extension](../../../src/Extension.js) object proxies this call to the -runtime module (called `this.runtime` in the snippet): - -```javascript -class Extension extends AdvancedBase { - // ... - set exports (value) { - this.runtime.exports = value; - } - // ... -} -``` - -You may be wondering why RuntimeModule is a separate class from Extension, -rather than just registering extensions into this registry. - -Separating RuntimeModule allows core code that has not yet been migrated -to extensions to export values as if they came from extensions. -Since core modules are loaded before extensions, this allows any legacy -`useapi` definitions be be exported where modules are installed. - -For example, in [CoreModule.js](../../../src/CoreModule.js) this snippet -of code is used to add a runtime module called `core`: - -```javascript -// Extension compatibility -const runtimeModule = new RuntimeModule({ name: 'core' }); -context.get('runtime-modules').register(runtimeModule); -runtimeModule.exports = useapi.use('core'); -``` diff --git a/src/backend/doc/features/service-scripts.md b/src/backend/doc/features/service-scripts.md deleted file mode 100644 index 9fdea19eb..000000000 --- a/src/backend/doc/features/service-scripts.md +++ /dev/null @@ -1,150 +0,0 @@ -> **NOTICE:** This documentation is new and might contain errors. -> Feel free to open a Github issue if you run into any problems. - -# Service Scripts - -## What is a Service Script? - -Service scripts allow backend services to provide client-side code that -runs in Puter's GUI. This is useful if you want to make a mod or plugin -for Puter that has backend functionality. For example, you might want -to add a tab to the settings panel to make use of or configure the service. - -Service scripts are made possible by the `puter-homepage` service, which -allows you to register URLs for additional javascript files Puter's -GUI should load. - -## ES Modules - A Problem of Ordering - -In browsers, script tags with `type=module` implicitly behave according -to those with the `defer` attribute. This means after the DOM is loaded -the scripts will run in the order in which they appear in the document. - -Relying on this execution order however does not work. This is because -`import` is implicitly asynchronous. Effectively, this means these -scripts will execute in arbitrary order if they all have imports. - -In a situation where all the client-side code is bundled with rollup -or webpack this is not an issue as you typically only have one -entry script. To facilitate loading service scripts, which are not -bundled with the GUI, we require that service scripts call the global -`service_script` function to access the API for service scripts. - -## Providing a Service Script - -For a service to provide a service script, it simply needs to serve -static files (the "service script") on some URL, and register that -URL with the `puter-homepage` service. - -In this example below we use builtin functionality of express to serve -static files. - -```javascript -class MyService extends BaseService { - async _init () { - // First we tell `puter-homepage` that we're going to be serving - // a javascript file which we want to be included when the GUI - // loads. - const svc_puterHomepage = this.services.get('puter-homepage'); - svc_puterHomepage.register_script('/my-service-script/main.js'); - } - - async ['__on_install.routes'] (_, { app }) { - // Here we ask express to serve our script. This is made possible - // by WebServerService which provides the `app` object when it - // emits the 'install.routes` event. - app.use('/my-service-script', - express.static( - PathBuilder.add(__dirname).add('gui').build() - ) - ); - } -} -``` - -## A Simple Service Script - - - -```javascript -import SomeModule from "./SomeModule.js"; - -service_script(api => { - api.on_ready(() => { - // This callback is invoked when the GUI is ready - - // We can use api.get() to import anything exposed to - // service scripts by Puter's GUI; for example: - const Button = api.use('ui.components.Button'); - // ^ Here we get Puter's Button component, which is made - // available to service scripts. - }); -}); -``` - -## Adding a Settings Tab - -Starting with the following example: - -```javascript -import MySettingsTab from "./MySettingsTab.js"; - -globalThis.service_script(api => { - api.on_ready(() => { - const svc_settings = globalThis.services.get('settings'); - svc_settings.register_tab(MySettingsTab(api)); - }); -}); -``` - -The module **MySettingsTab** exports a function for scoping the `api` -object, and that function returns a settings tab. The settings tab is -an object with a specific format that Puter's settings window understands. - -Here are the contents of `MySettingsTab.js`: - -```javascript -import MyWindow from "./MyWindow.js"; - -export default api => ({ - id: 'my-settings-tab', - title_i18n_key: 'My Settings Tab', - icon: 'shield.svg', - factory: () => { - const NotifCard = api.use('ui.component.NotifCard'); - const ActionCard = api.use('ui.component.ActionCard'); - const JustHTML = api.use('ui.component.JustHTML'); - const Flexer = api.use('ui.component.Flexer'); - const UIAlert = api.use('ui.window.UIAlert'); - - // The root component for our settings tab will be a "flexer", - // which by default displays its child components in a vertical - // layout. - const component = new Flexer({ - children: [ - // We can insert raw HTML as a component - new JustHTML({ - no_shadow: true, // use CSS for settings window - html: '

Some Heading

', - }), - new NotifCard({ - text: 'I am a card with some text', - style: 'settings-card-success', - }), - new ActionCard({ - title: 'Open an Alert', - button_text: 'Click Me', - on_click: async () => { - // Here we open an example window - await UIAlert({ - message: 'Hello, Puter!', - }); - } - }) - ] - }); - - return component; - } -}); -``` diff --git a/src/backend/doc/howto_make_driver.md b/src/backend/doc/howto_make_driver.md deleted file mode 100644 index ca8bad7c2..000000000 --- a/src/backend/doc/howto_make_driver.md +++ /dev/null @@ -1,239 +0,0 @@ -# How to Make a Puter Driver - -## What is a Driver? - -A driver can be one of two things depending on what you're -talking about: -- a **driver interface** describes a general type of service - and what its parameters and result look like. - For example, `puter-chat-completion` is a driver interface - for AI Chat services, and it specifies that any service - on Puter for AI Chat needs a method called `complete` that - accepts a JSON parameter called `messages`. -- a **driver implementation** exists when a **Service** on - Puter implements a **trait** with the same name as a - driver interface. - -## Part 1: Choose or Create a Driver Interface - -Available driver interfaces exist at this location in the repo: -[/src/backend/src/services/drivers/interfaces.js](../src/services/drivers/interfaces.js). - -When creating a new Puter driver implementation, you should check -this file to see if there's an appropriate interface. We're going -to make a driver that returns greeting strings, so we can use the -existing `hello-world` interface. If there wasn't an existing -interface, it would need to be created. Let's break down this -interface: - -```javascript -'hello-world': { - description: 'A simple driver that returns a greeting.', - methods: { - greet: { - description: 'Returns a greeting.', - parameters: { - subject: { - type: 'string', - optional: true, - }, - }, - result: { type: 'string' }, - } - } -}, -``` - -The **description** describes what the interface is for. This -should be provided that both driver developers and users can -quickly identify what types of services should use it. - -The **methods** object should have at least one entry, but it -may have more. The key of each entry is the name of a method; -in here we see `greet`. Each method also has a description, -a **parameters** object, and a **result** object. - -The **parameters** object has an entry for each parameter that -may be passed to the method. Each entry is an object with a -`type` property specifying what values are allowed, and possibly -an `optional: true` entry. - -All methods for Puter drivers use _named parameters_. There are no -positional parameters in Puter driver methods. - -The **result** object specifies the type of the result. A service -called DriverService will use this to determine the response format -and headers of the response. - -## Part 2: Create a Service - -Creating a service is very easy, provided the service doesn't do -anything. Simply add a class to `src/backend/src/services` or into -the module of your choice (`src/backend/src/modules/`) -that looks like this: - -```javascript -const BaseService = require('./BaseService') -// NOTE: the path specified ^ HERE might be different depending -// on the location of your file. - -class PrankGreetService extends BaseService { -} -``` - -Notice I called the service "PrankGreet". This is a good service -name because you already know what the service is likely to -implement: this service generates a greeting, but it is a greeting -that intends to play a prank on whoever is beeing greeted. - -Then, register the service into a module. If you put the service -under `src/backend/src/services`, then it goes in -[CoreModule](..//src/CoreModule.js) somewhere near the end of -the `install()` method. Otherwise, it will go in the `*Module.js` -file in the module where you placed your service. - -The code to register the service is two lines of code that will -look something like this: - -```javascript -const { PrankGreetServie } = require('./path/to/PrankGreetServie.js'); -services.registerService('prank-greet', PrankGreetServie); -``` - -## Part 3: Verify that the Service is Registered - -It's always a good idea to verify that the service is loaded -when starting Puter. Otherwise, you might spend time trying to -determine why your code doesn't work, when in fact it's not -running at all to begin with. - -To do this, we'll add an `_init` handler to the service that -logs a message after a few seconds. We wait a few seconds so that -any log noise from boot won't bury our message. - -```javascript -class PrankGreetService extends BaseService { - async _init () { - // Wait for 5 seconds - await new Promise(rslv => setTimeout(rslv), 5000); - - // Display a log message - console.debug('Hello from PrankGreetService!'); - } -} -``` - -## Part 4: Implement the Driver Interface in your Service - -Now that it has been verified that the service is loaded, we can -start implementing the driver interface we chose eralier. - -```javascript -class PrankGreetService extends BaseService { - async _init () { - // ... same as before - } - - // Now we add this: - static IMPLEMENTS = { - ['hello-world']: { - async greet ({ subject }) { - if ( subject ) { - return `Hello ${subject}, tell me about updog!`; - } - return `Hello, tell me about updog!`; - } - } - } -} -``` - -## Part 5: Test the Driver Implementation - -We have now created the `prank-greet` implementation of `hello-world`. -Let's make a request in the browser to check it out. The example below -is a `fetch` call using `http://api.puter.localhost:4100` as the API -origin, which is the default when you're running Puter's backend locally. - -Also, in this request I refer to `puter.authToken`. If you run this -snippet in the Dev Tools window of your browser from a tab with Puter -open (your local Puter, to be precise), this should contain the current -value for your auth token. - -```javascript -await (await fetch("http://api.puter.localhost:4100/drivers/call", { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - interface: 'hello-world', - service: 'prank-greet', - method: 'greet', - args: { - subject: 'World', - }, - }), - "method": "POST", -})).json(); -``` - -**You might see a permissions error!** Don't worry, this is expected; -in the next step we'll add the required permissions. - -## Part 6: Permissions - -In the previous step, you will only have gotten a successful response -if you're logged in as the `admin` user. If you're logged in as another -user you won't have access to the service's driver implementations be -default. - -To grant permission for all users, update -[hardcoded-permissions.js](../src/data/hardcoded-permissions.js). - -First, look for the constant `hardcoded_user_group_permissions`. -Whereever you see an entry for `service:hello-world:ii:hello-world`, add -the corresponding entry for your service, which will be called -``` -service:prank-greet:ii:hello-world -``` - -To help you remember the permission string, its helpful to know that -`ii` in the string stands for "invoke interface". i.e. the scope of the -permission is under `service:prank-greet` (the `prank-greet` service) -and we want permission to invoke the interface `hello-world` on that -service. - -You'll notice each entry in `hardcoded_user_group_permissions` has a value -determined by a call to the utility function `policy_perm(...)`. The policy -called `user.es` is a permissive policy for storage drivers, and we can -re-purpose it for our greeting implementor. - -The policy of a permission determines behavior like rate limiting. This is -an advanced topic that is not covered in this guide. - -If you want apps to be able to access the driver implementation without -explicit permission from a user, you will need to also register it in the -`default_implicit_user_app_permissions` constant. Additionally, you can -use the `implicit_user_app_permissions` constant to grant implicit -permission to the builtin Puter apps only. - -Permissions to implementations on services can also be granted at runtime -to a user or group of users using the permissions API. This is beyond the -scope of this guide. - -## Part 7: Verify Successful Response - -If all went well, you should see the response in your console when you -try the request from Part 5. Try logging into a user other than `admin` -to verify permisison is granted. - -```json -"Hello World, tell me about updog!" -``` - -## Part 8: Next Steps - -- [Access Configuration](./services/config.md) -- [Output Logs](./services/log.md) -- [Add HTTP Routes](./services/http.md) diff --git a/src/backend/doc/license_header.txt b/src/backend/doc/license_header.txt deleted file mode 100644 index d7e027660..000000000 --- a/src/backend/doc/license_header.txt +++ /dev/null @@ -1,16 +0,0 @@ -Copyright (C) 2024 Puter Technologies Inc. - -This file is part of Puter. - -Puter is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published -by the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . \ No newline at end of file diff --git a/src/backend/doc/lists-of-things/list-of-permissions.md b/src/backend/doc/lists-of-things/list-of-permissions.md deleted file mode 100644 index 0e0efc308..000000000 --- a/src/backend/doc/lists-of-things/list-of-permissions.md +++ /dev/null @@ -1,55 +0,0 @@ -# Permissions - -## Filesystem Permissions - -### `fs::` - -- `` specifies the file that this permission - is associated with. - The ACL service - (which checks filesystem permissions) - knows if the value is a path or UUID based on the presence - of a leading slash; if it starts with `"/"` it's a path. -- `` specifies one of: - `write`, `read`, `list`, `see`; where each item in that - list implies all the access levels which follow. -- A permission that grants access to a directory, - such as `/user/shared`, implies access - of the same **access level** to all child file or directory - nodes under that location, **recursively**; - `fs:/user/shared:read` implies `fs:/user/shared/nested/file.txt:read` -- The "real" permission is `fs::`; - whenever path is specified the permission is rewritten. - **note:** future support for other filesystems - could make this rewrite rule conditional. - -## App and Subdomain permissions - -### `site::access` -- `` specifies the subdomain that this - permission is associated with. - Here, "subdomain" means the **"name of the subdomain"**, - which means a site accessed via `my-name.example.site` - will be specified here with `my-name`. -- This permission is always rewritten as the permission - described below (backend does this automatically). - -### `site:uid#:access` -- If the subdomain is **not** [protected](../features/protected-apps.md), - this permission is ignored by the system. -- If the subdomain **is** protected, this permission will - allow access to the site via a Puter app iframe with - a token for the entity to which permission was granted - -### `app::access` - -- `` specifies the app that this - permission is associated with. -- This permission is always rewritten as the permission - described below (backend does this automatically). - -### `app:uid#:access` -- If the app is **not** [protected](../features/protected-apps.md), - this permission is ignored by the system. -- If the app **is** protected, this permission will - allow reading the app's metadata and seeing that the app exists. diff --git a/src/backend/doc/lists-of-things/list-of-tto-types.md b/src/backend/doc/lists-of-things/list-of-tto-types.md deleted file mode 100644 index 256854158..000000000 --- a/src/backend/doc/lists-of-things/list-of-tto-types.md +++ /dev/null @@ -1,29 +0,0 @@ -# Types for Type-Tagged Objects - -## Internal Use - -### `{ $: 'share-intent' }` - -- Used in the `/share` endpoint -- Permissions get applied to existing users -- For email shares, is trasnformed into a `token:share` - which is stored in the `share` database table. - -- **variants:** - - `share-intent:file` - - `share-intent:app` -- **properties:** - - `permissions` - a list of permissions to grant - -### `{ $: 'internal:share' }` -- Stored in the `share` database table -- **properties:** - - `permissions` - a list of permissions to grant - -### `{ $: 'token:share }` - -- Stored in a JWT called the "share token" -- Contains only the share UUID - -- **properties:** - - `uid` - UUID of a share diff --git a/src/backend/doc/log_config.md b/src/backend/doc/log_config.md deleted file mode 100644 index 04385e114..000000000 --- a/src/backend/doc/log_config.md +++ /dev/null @@ -1,43 +0,0 @@ -## Backend - Configuring Logs - -### Log visibility specified by configuration file - -The configuration file can define an array parameter called `logging`. -This configures the visibility of specific logs in core areas based on -which string flags are present. - -For example, the following configuration enables HTTP request logs: -```json -{ - "logging": ['http'] -} -``` - -Sometimes "enabling" a log means moving its log level from `debug` to `info`. - -#### Available logging flags: -- `http`: http requests -- `fsentries-not-found`: information about files that were stat'd but weren't there - -#### Other log options - -- Setting `log_upcoming_alarms` to `true` will log alarms before they are created. - This would be useful if AlarmService itself is failing. -- Setting `trace_logs` to `true` will display a stack trace below every log message. - This can be useful if you don't know where a particular log is coming from and - want to track it down. - -#### Service-level log configuration - -Services can be configured to change their logging behavior. Services will have one of -two behaviors: - -1. **info logging** - `log.info` can be used to create an `[INFO]` log message -2. **debug logging only** - `log.info` is redirected to `log.debug` - -Services will have **info logging** enabled by default, unless the class definition -has the static member `static LOG_DEBUG = true` (in which case **debug logging only** -is the default). - -In a service's configuration block the desired behavior can be specified by setting -either `"log_debug": true` or `"log_info": true` diff --git a/src/backend/doc/services/config.md b/src/backend/doc/services/config.md deleted file mode 100644 index 8e57c63e9..000000000 --- a/src/backend/doc/services/config.md +++ /dev/null @@ -1,46 +0,0 @@ -# Service Configuration - -To locate your configuration file, see [Configuring Puter](https://github.com/HeyPuter/puter/wiki/self_hosters-config). - -### Accessing Service Configuration - -Service configuration appears under the `"services"` property in the -configuration file for Puter. If Puter's configuration had no other -values except for a service config with one key, it might look like -this: - -```json -{ - "services": { - "my-service": { - "somekey": "some value" - } - } -} -``` - -Services have their configuration object assigned to `this.config`. - -```javascript -class MyService extends BaseService { - async _init () { - // You can access configuration for a service like this - this.log.info('value of my key is: ' + this.config.somekey); - } -} -``` - -### Accessing Global Configuration - -Services can access global configuration. This can be useful for knowing how -Puter itself is configured, but using this global config object for service -configuration is discouraged as it could create conflicts between services. - -```javascript -class MyService extends BaseService { - async _init () { - // You can access configuration for a service like this - this.log.info('Puter is hosted on: ' + this.global_config.domain); - } -} -``` diff --git a/src/backend/doc/services/event_buses.md b/src/backend/doc/services/event_buses.md deleted file mode 100644 index a96d51014..000000000 --- a/src/backend/doc/services/event_buses.md +++ /dev/null @@ -1,33 +0,0 @@ -# Event Buses - -Puter's backend has two event buses: -- Service Event Bus -- Application Event Bus - -## Service Event Bus - -This is a simple event bus that lives in the [Container](../../src/services/Container.js) -class. There is only one instance of **Container** and it is called the "services container". -When Puter boots, all the services registered by modules are registered into the services -container. - -Services handle events from the Service Event Bus by implementing methods which are named -with the prefix `__on_`. This prefix looks a little strange at first so it's worth -breaking it down: -- `__` (two underscores) prevents collision with common method names, and also - common conventions like beginning a method name with a single underscore - to indicate a method that should be overridden. -- `on` is the meaningful name. -- `_`, the last underscore, is for readability, as the event name conventionally - begins with a lowercase letter. - -Note that you will need to use the - -Example: -```javascript -class MyService extends BaseService { - ['__on_boot.ready'] () { - // - } -} -``` diff --git a/src/backend/doc/services/http.md b/src/backend/doc/services/http.md deleted file mode 100644 index 4b533270c..000000000 --- a/src/backend/doc/services/http.md +++ /dev/null @@ -1,4 +0,0 @@ -# Adding HTTP Routes to Services - -Services can serve HTTP routes when the [WebModule](../../src/modules/web/WebModule.js) -is enabled by listening for the `install.routes` event on the [Service Event Bus](./) \ No newline at end of file diff --git a/src/backend/doc/services/log.md b/src/backend/doc/services/log.md deleted file mode 100644 index bf7310b3d..000000000 --- a/src/backend/doc/services/log.md +++ /dev/null @@ -1,44 +0,0 @@ -# Logging in Services - -# NOTE: You can, and maybe should, just use console log methods, as they are overriden to log through our logger - -Services all have a logger available at `this.log`. - -```javascript -class MyService extends BaseService { - async init () { - this.log.info('Hello, Logger!'); - } -} -``` - -There are multiple "log levels", similar to `logrus` or other common logging -libraries. - -```javascript -class MyService extends BaseService { - async init () { - this.log.info('I\'m just a regular log.'); - this.log.debug('I\'m only for developers.'); - this.log.warn('It is statistically unlikely I will be awknowledged.'); - this.log.error('Something is broken! Pay attention!'); - this.log.noticeme('This will be noticed, unlike warnings. Use sparingly.'); - this.log.system('I am a system event, like shutdown.'); - this.log.tick('A periodic behavior like cache pruning is occurring.'); - } -} -``` - -Log methods can take a second parameter, an object specifying fields. - -```javascript - -class MyService extends BaseService { - async init () { - this.log.info('I have fields!', { - why: "why not", - random_number: 1, // chosen by coin toss, guarenteed to be random - }); - } -} -``` diff --git a/src/backend/drivers/ai-chat/ChatCompletionDriver.ts b/src/backend/drivers/ai-chat/ChatCompletionDriver.ts new file mode 100644 index 000000000..5018e7666 --- /dev/null +++ b/src/backend/drivers/ai-chat/ChatCompletionDriver.ts @@ -0,0 +1,726 @@ +import { PassThrough } from 'node:stream'; +import crypto from 'node:crypto'; +import { Context } from '../../core/context.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import { + DEFAULT_FREE_SUBSCRIPTION, + DEFAULT_TEMP_SUBSCRIPTION, +} from '../../services/metering/consts.js'; +import type { DriverStreamResult } from '../meta.js'; +import { PuterDriver } from '../types.js'; +import { ClaudeProvider } from './providers/claude/ClaudeProvider.js'; +import { DeepSeekProvider } from './providers/deepseek/DeepSeekProvider.js'; +import { FakeChatProvider } from './providers/FakeChatProvider.js'; +import { GeminiChatProvider } from './providers/gemini/GeminiChatProvider.js'; +import { GroqAIProvider } from './providers/groq/GroqAIProvider.js'; +import { MistralAIProvider } from './providers/mistral/MistralAiProvider.js'; +import { OllamaChatProvider } from './providers/ollama/OllamaProvider.js'; +import { OpenAiChatProvider } from './providers/openai/OpenAiChatCompletionsProvider.js'; +import { OpenAiResponsesChatProvider } from './providers/openai/OpenAiChatResponsesProvider.js'; +import { OpenRouterProvider } from './providers/openrouter/OpenRouterProvider.js'; +import { TogetherAIProvider } from './providers/together/TogetherAIProvider.js'; +import { XAIProvider } from './providers/xai/XAIProvider.js'; +import type { + IChatCompleteResult, + IChatModel, + IChatProvider, + ICompleteArguments, +} from './types.js'; +import { normalize_tools_object } from './utils/FunctionCalling.js'; +import { + extract_text, + normalize_messages, + normalize_single_message, +} from './utils/Messages.js'; +import { AIChatStream } from './utils/Streaming.js'; + +const MAX_FALLBACKS = 4; // includes first attempt + +/** + * Driver implementing the `puter-chat-completion` interface. + * + * Manages multiple upstream providers (Claude, OpenAI, …) and handles + * model resolution, provider routing, fallback on failure, and message + * normalisation. Each provider is a plain `IChatProvider` — the driver + * instantiates them from config on boot. + * + * Providers handle their own metering internally. + */ +export class ChatCompletionDriver extends PuterDriver { + readonly driverInterface = 'puter-chat-completion'; + readonly driverName = 'ai-chat'; + readonly isDefault = true; + + #providers: Record = {}; + #modelIdMap: Record = {}; + + override onServerStart() { + this.#registerProviders(); + this.#buildModelMap(); + } + + // ── Interface methods ─────────────────────────────────────────── + + async models() { + const seen = new Set(); + return Object.values(this.#modelIdMap) + .flat() + .filter((model) => { + if (seen.has(model.id)) return false; + seen.add(model.id); + return true; + }) + .sort((a, b) => { + if (a.provider === b.provider) return a.id.localeCompare(b.id); + return a.provider!.localeCompare(b.provider!); + }); + } + + async list() { + return (await this.models()).map((m) => m.puterId || m.id).sort(); + } + + override getReportedCosts(): Record[] { + const out: Record[] = []; + const seen = new Set(); + for (const bucket of Object.values(this.#modelIdMap)) { + for (const model of bucket) { + const key = `${model.provider}:${model.id}`; + if (seen.has(key)) continue; + seen.add(key); + for (const [costKey, raw] of Object.entries( + model.costs ?? {}, + )) { + // `tokens` is a scale descriptor ("costs expressed per N + // tokens"), not a real per-operation cost — skip it. + if (costKey === 'tokens') continue; + if (typeof raw !== 'number' || !Number.isFinite(raw)) + continue; + out.push({ + usageType: `${model.provider}:${model.id}:${costKey}`, + ucentsPerUnit: raw, + unit: 'token', + source: `driver:aiChat/${model.provider}`, + costs_currency: model.costs_currency, + }); + } + } + } + return out; + } + + async complete(args: ICompleteArguments): Promise { + const actor = Context.get('actor'); + if (!actor) throw new HttpError(401, 'Authentication required'); + + let intendedProvider = args.provider || ''; + if (!args.model && !intendedProvider) { + intendedProvider = 'claude'; // default provider + } + if ( + !args.model && + intendedProvider && + this.#providers[intendedProvider] + ) { + args.model = this.#providers[intendedProvider].getDefaultModel(); + } + + let model = this.#resolveModel(args.model, intendedProvider); + if (!model) { + throw new HttpError(400, `Model not found: ${args.model}`); + } + + if (args.messages) { + args.messages = normalize_messages(args.messages); + } + if (args.tools) { + normalize_tools_object(args.tools); + } + + const completionId = crypto + .randomUUID() + .replaceAll('-', '') + .slice(0, 25); + + const validateEvent: Record = { + actor, + completionId, + allow: true, + intended_service: intendedProvider, + parameters: args, + }; + + await this.clients.event.emitAndWait( + 'ai.prompt.validate', + validateEvent, + {}, + ); + if (!validateEvent.allow) { + const reason = + typeof validateEvent.message === 'string' + ? validateEvent.message + : 'Prompt blocked by policy'; + throw new HttpError(403, reason); + } + + // ── Credit / subscription gates (metering) ──────────────────── + // Cheap pre-flight: reject when the user can't afford even the + // approximate input cost, keep subscriber-only models gated, and + // cap `max_tokens` so output can't exceed remaining credits. + const metering = this.services.metering; + const inputCostKey = + (model.input_cost_key as string | undefined) ?? 'input_tokens'; + const outputCostKey = + (model.output_cost_key as string | undefined) ?? 'output_tokens'; + const inputTokenCost = Number(model.costs?.[inputCostKey] ?? 0); + const outputTokenCost = Number(model.costs?.[outputCostKey] ?? 0); + const text = extract_text(args.messages ?? []); + // Rough estimator from v1 — avg of char/4 and word*(4/3), halved. + // See https://help.openai.com/en/articles/4936856 + const approximateTokenCount = Math.floor( + (text.length / 4 + text.split(/\s+/).length * (4 / 3)) / 2, + ); + const approximateInputCost = approximateTokenCount * inputTokenCost; + const minimumCredits = Number(model.minimumCredits || 1); + + const usageAllowed = await metering.hasEnoughCredits( + actor, + Math.max(approximateInputCost, minimumCredits), + ); + if (!usageAllowed) { + throw new HttpError(402, 'No usage left for request.', { + legacyCode: 'insufficient_funds', + }); + } + + if (model.subscriberOnly) { + const subscription = await metering.getActorSubscription(actor); + const isDefaultPolicy = + subscription.id === DEFAULT_FREE_SUBSCRIPTION || + subscription.id === DEFAULT_TEMP_SUBSCRIPTION; + if (isDefaultPolicy) { + throw new HttpError( + 403, + `The model ${model.id} is only available to subscribers. Please subscribe to access this model.`, + { legacyCode: 'permission_denied' }, + ); + } + } + + if (outputTokenCost > 0) { + const remainingCredits = await metering.getRemainingUsage(actor); + const maxAllowedOutputUcents = + remainingCredits - approximateInputCost; + const maxAllowedOutputTokens = + maxAllowedOutputUcents / outputTokenCost; + if (maxAllowedOutputTokens) { + const cap = Math.floor( + Math.min( + args.max_tokens ?? Number.POSITIVE_INFINITY, + maxAllowedOutputTokens, + model.max_tokens - approximateTokenCount, + ), + ); + args.max_tokens = cap < 1 ? undefined : cap; + } + } + + // First attempt + const provider = this.#providers[model.provider!]; + if (!provider) { + throw new HttpError(500, `No provider found for model ${model.id}`); + } + + const attempts: { model: string; provider: string; error: string }[] = + []; + let res: IChatCompleteResult | undefined; + + try { + res = await provider.complete({ + ...args, + model: model.id, + provider: model.provider, + }); + } catch (e) { + const error = e as Error; + attempts.push({ + model: model.id, + provider: model.provider!, + error: error?.message ?? String(e), + }); + + // Fallback loop + const tried = [model.id]; + const triedProviders = [model.provider!]; + let lastError: Error | null = error; + + while (lastError && tried.length < MAX_FALLBACKS) { + const fallback = this.#findFallback( + model.id, + tried, + triedProviders, + ); + if (!fallback) break; + + const fbProvider = this.#providers[fallback.provider!]; + if (!fbProvider) break; + + // Credits can be exhausted mid-fallback by parallel requests; + // re-check before another upstream hit. Same bail as the + // pre-flight above. + const fallbackUsageAllowed = await metering.hasEnoughCredits( + actor, + 1, + ); + if (!fallbackUsageAllowed) { + throw new HttpError(402, 'No usage left for request.', { + legacyCode: 'insufficient_funds', + }); + } + + tried.push(fallback.id); + triedProviders.push(fallback.provider!); + + try { + res = await fbProvider.complete({ + ...args, + model: fallback.id, + provider: fallback.provider, + }); + model = fallback; + lastError = null; + } catch (fbErr) { + lastError = fbErr as Error; + attempts.push({ + model: fallback.id, + provider: fallback.provider!, + error: lastError?.message ?? String(fbErr), + }); + } + } + } + + if (!res) { + throw new HttpError(500, 'All providers failed', { + fields: { attempts }, + }); + } + + const username = actor.user?.username; + + // Streaming result — create a PassThrough, kick off the provider's + // stream populator, and return a DriverStreamResult so the route + // handler pipes it to the HTTP response as chunked NDJSON. + if ('init_chat_stream' in res && res.init_chat_stream) { + const passthrough = new PassThrough(); + const chatStream = new AIChatStream({ stream: passthrough }); + const init = res.init_chat_stream; + const cleanup = res.finally_fn; + + // Intercept `chatStream.end(usage)` so we can fire the same + // complete + cost-calculated events the non-streaming branch + // emits. Providers always terminate streams via `.end(usage)`; + // if they skip it, we just lose the cost event (no worse than + // not emitting). + const originalEnd = chatStream.end.bind(chatStream); + chatStream.end = (usage?: Record) => { + this.clients.event.emit( + 'ai.prompt.complete', + { + username, + completionId, + intended_service: intendedProvider, + parameters: args, + result: { usage, stream: true }, + model_used: model.id, + service_used: model.provider, + }, + {}, + ); + if (usage) { + this.#emitCostCalculated({ + completionId, + username, + usage, + model, + intendedProvider, + }); + } + return originalEnd(usage!); + }; + + // Fire-and-forget — the stream writes happen async while the + // response is being piped to the client. + (async () => { + try { + await init({ chatStream }); + } catch (e) { + passthrough.write( + `${JSON.stringify({ + type: 'error', + message: (e as Error).message, + })}\n`, + ); + passthrough.end(); + } finally { + if (cleanup) await cleanup(); + } + })(); + + const streamResult: DriverStreamResult = { + dataType: 'stream', + content_type: 'application/x-ndjson', + chunked: true, + stream: passthrough, + }; + return streamResult as unknown as IChatCompleteResult; + } + + // ── Post-completion audit event ────────────────────────────── + // Only for non-streaming results (streaming emits from the + // `chatStream.end` wrapper above). Extensions like prompt_block / + // prodMeteringAndBilling listen for this to log completions. + this.clients.event.emit( + 'ai.prompt.complete', + { + username, + completionId, + intended_service: intendedProvider, + parameters: args, + result: res, + model_used: model.id, + service_used: model.provider, + }, + {}, + ); + + if ('usage' in res && res.usage) { + this.#emitCostCalculated({ + completionId, + username, + usage: res.usage, + model, + intendedProvider, + }); + } + + Context.set('driverMetadata', { + service_used: model.provider, + providerUsed: model.id, + }); + + if (args.response?.normalize && 'message' in res && res.message) { + return { + ...res, + message: normalize_single_message(res.message), + normalized: true, + via_ai_chat_service: true, + }; + } + + return { ...res, via_ai_chat_service: true }; + } + + // Compute per-token cost in microcents using the model's cost map, + // then emit `ai.prompt.cost-calculated` for listeners that persist + // billing/abuse rows keyed on the completion id. + #emitCostCalculated(params: { + completionId: string; + username?: string; + usage: Record; + model: IChatModel; + intendedProvider: string; + }) { + const { completionId, username, usage, model, intendedProvider } = + params; + + const inputKey = + (model.input_cost_key as string | undefined) ?? 'input_tokens'; + const outputKey = + (model.output_cost_key as string | undefined) ?? 'output_tokens'; + const inputCostPer = Number(model.costs?.[inputKey] ?? 0); + const outputCostPer = Number(model.costs?.[outputKey] ?? 0); + const inputTokens = Number( + usage[inputKey] ?? usage.prompt_tokens ?? usage.input_tokens ?? 0, + ); + const outputTokens = Number( + usage[outputKey] ?? + usage.completion_tokens ?? + usage.output_tokens ?? + 0, + ); + const inputUcents = Math.round(inputTokens * inputCostPer); + const outputUcents = Math.round(outputTokens * outputCostPer); + + this.clients.event.emit( + 'ai.prompt.cost-calculated', + { + completionId, + username, + usage, + input_tokens: inputTokens, + output_tokens: outputTokens, + input_ucents: inputUcents, + output_ucents: outputUcents, + total_ucents: inputUcents + outputUcents, + costs_currency: model.costs_currency, + model_used: model.id, + service_used: model.provider, + intended_service: intendedProvider, + model_details: { + id: model.id, + provider: model.provider, + input_cost_key: inputKey, + output_cost_key: outputKey, + costs: model.costs, + costs_currency: model.costs_currency, + }, + }, + {}, + ); + } + + // ── Provider registration ─────────────────────────────────────── + + #registerProviders() { + const providers = this.config.providers ?? {}; + const metering = this.services.metering; + + const readKey = (cfg: Record | undefined) => + (cfg?.apiKey as string | undefined) ?? + (cfg?.secret_key as string | undefined); + + const claudeKey = readKey(providers['claude']); + if (claudeKey) { + this.#providers['claude'] = new ClaudeProvider( + metering, + { + fsEntry: this.stores.fsEntry, + s3Object: this.stores.s3Object, + }, + this.services.fs, + { apiKey: claudeKey }, + ); + } + + const openaiKey = readKey(providers['openai-completion']); + if (openaiKey) { + const openaiStores = { + fsEntry: this.stores.fsEntry, + s3Object: this.stores.s3Object, + }; + const openaiCompletions = new OpenAiChatProvider( + metering, + openaiStores, + this.services.fs, + { + apiKey: openaiKey, + }, + ); + const openaiResponses = new OpenAiResponsesChatProvider( + metering, + openaiStores, + this.services.fs, + { apiKey: openaiKey }, + ); + // web_search is Responses-only; let the Completions path delegate + // to its sibling when users request it. + openaiCompletions.setResponsesProvider(openaiResponses); + this.#providers['openai-completion'] = openaiCompletions; + this.#providers['openai-responses'] = openaiResponses; + } + + const geminiKey = readKey(providers['gemini']); + if (geminiKey) { + this.#providers['gemini'] = new GeminiChatProvider(metering, { + apiKey: geminiKey, + }); + } + + const groqKey = readKey(providers['groq']); + if (groqKey) { + this.#providers['groq'] = new GroqAIProvider( + { apiKey: groqKey }, + metering, + ); + } + + const deepseekKey = readKey(providers['deepseek']); + if (deepseekKey) { + this.#providers['deepseek'] = new DeepSeekProvider( + { apiKey: deepseekKey }, + metering, + ); + } + + const mistralKey = readKey(providers['mistral']); + if (mistralKey) { + this.#providers['mistral'] = new MistralAIProvider( + { apiKey: mistralKey }, + metering, + ); + } + + const xaiKey = readKey(providers['xai']); + if (xaiKey) { + this.#providers['xai'] = new XAIProvider( + { apiKey: xaiKey }, + metering, + ); + } + + const openrouter = providers['openrouter']; + const openrouterKey = readKey(openrouter); + if (openrouterKey) { + this.#providers['openrouter'] = new OpenRouterProvider( + { + apiKey: openrouterKey, + apiBaseUrl: openrouter?.apiBaseUrl as string | undefined, + }, + metering, + ); + } + + const togetherKey = readKey(providers['together-ai']); + if (togetherKey) { + this.#providers['together-ai'] = new TogetherAIProvider( + { apiKey: togetherKey }, + metering, + ); + } + + // Ollama — auto-discover local instance unless `enabled: false`. + const ollama = providers['ollama']; + if (ollama?.enabled !== false) { + this.#providers['ollama'] = new OllamaChatProvider( + { + apiBaseUrl: ollama?.apiBaseUrl, + }, + metering, + ); + } + + // Fake provider — always available for testing + this.#providers['fake-chat'] = new FakeChatProvider(); + } + + // ── Model map ─────────────────────────────────────────────────── + + async #buildModelMap() { + const AGGREGATORS = new Set(['together-ai', 'openrouter']); + + for (const providerName in this.#providers) { + const provider = this.#providers[providerName]; + const isAggregator = AGGREGATORS.has(providerName); + + for (const model of await provider.models()) { + model.id = model.id.trim().toLowerCase(); + if (!this.#modelIdMap[model.id]) { + this.#modelIdMap[model.id] = []; + } + this.#modelIdMap[model.id].push({ + ...model, + provider: providerName, + }); + + if (model.puterId) { + if (model.aliases) { + model.aliases.push(model.puterId); + } else { + model.aliases = [model.puterId]; + } + } + + if (isAggregator && model.aliases) { + let skip = false; + for (const rawAlias of model.aliases) { + const alias = rawAlias.trim().toLowerCase(); + const existing = this.#modelIdMap[alias]; + if ( + existing && + existing !== this.#modelIdMap[model.id] + ) { + if (existing.some((m) => m.provider === 'gemini')) { + // Gemini exception — let the aggregator + // entry through. + continue; + } + skip = true; + break; + } + } + if (skip) { + // Remove the entry we just pushed; leave the bucket + // intact for other providers. + const bucket = this.#modelIdMap[model.id]; + bucket.pop(); + if (bucket.length === 0) { + delete this.#modelIdMap[model.id]; + } + continue; + } + } + + if (model.aliases) { + for (let alias of model.aliases) { + alias = alias.trim().toLowerCase(); + if (!this.#modelIdMap[alias]) { + this.#modelIdMap[alias] = + this.#modelIdMap[model.id]; + } else if ( + this.#modelIdMap[alias] !== + this.#modelIdMap[model.id] + ) { + this.#modelIdMap[alias].push({ + ...model, + provider: providerName, + }); + this.#modelIdMap[model.id] = + this.#modelIdMap[alias]; + } + } + } + + // Sort: together-ai always last; then cheapest input-cost + // first; ties break by shorter id (usually the official + // name over a long aggregator-qualified one). + this.#modelIdMap[model.id].sort((a, b) => { + const aAgg = a.provider === 'together-ai'; + const bAgg = b.provider === 'together-ai'; + if (aAgg !== bAgg) return aAgg ? 1 : -1; + const aCost = a.costs[ + (a.input_cost_key as string) || 'input_tokens' + ] as number; + const bCost = b.costs[ + (b.input_cost_key as string) || 'input_tokens' + ] as number; + if (aCost === bCost) return a.id.length - b.id.length; + return aCost - bCost; + }); + } + } + } + + #resolveModel(modelId: string, provider?: string): IChatModel | null { + const models = this.#modelIdMap[modelId?.trim().toLowerCase()]; + if (!models || models.length === 0) return null; + if (!provider) return models[0]; + return models.find((m) => m.provider === provider) ?? models[0]; + } + + #findFallback( + modelId: string, + tried: string[], + triedProviders: string[], + ): IChatModel | null { + const models = this.#modelIdMap[modelId]; + if (!models) return null; + return ( + models.find( + (m) => + !tried.includes(m.id) || + !triedProviders.includes(m.provider!), + ) ?? null + ); + } +} diff --git a/src/backend/drivers/ai-chat/providers/ChatProvider.ts b/src/backend/drivers/ai-chat/providers/ChatProvider.ts new file mode 100644 index 000000000..07cbe5832 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/ChatProvider.ts @@ -0,0 +1,26 @@ +import type { + IChatModel, + IChatProvider, + ICompleteArguments, + IChatCompleteResult, +} from '../types.js'; + +/** + * Abstract base for AI chat providers. Each provider wraps a single + * upstream API (Anthropic, OpenAI, …) and exposes the unified + * `IChatProvider` contract. + */ +export class ChatProvider implements IChatProvider { + getDefaultModel(): string { + return ''; + } + models(): IChatModel[] | Promise { + return []; + } + list(): string[] | Promise { + return []; + } + async complete(_arg: ICompleteArguments): Promise { + throw new Error('Method not implemented.'); + } +} diff --git a/src/backend/src/services/ai/chat/providers/FakeChatProvider.ts b/src/backend/drivers/ai-chat/providers/FakeChatProvider.ts similarity index 62% rename from src/backend/src/services/ai/chat/providers/FakeChatProvider.ts rename to src/backend/drivers/ai-chat/providers/FakeChatProvider.ts index d019003e2..2f94debb5 100644 --- a/src/backend/src/services/ai/chat/providers/FakeChatProvider.ts +++ b/src/backend/drivers/ai-chat/providers/FakeChatProvider.ts @@ -19,19 +19,19 @@ import dedent from 'dedent'; import { LoremIpsum } from 'lorem-ipsum'; -import { AIChatStream } from '../../utils/Streaming'; -import { IChatProvider, ICompleteArguments, PuterMessage } from './types'; +import { AIChatStream } from '../utils/Streaming.js'; +import { IChatProvider, ICompleteArguments, PuterMessage } from '../types.js'; export class FakeChatProvider implements IChatProvider { - checkModeration (_text: string): ReturnType { + checkModeration(_text: string) { throw new Error('Method not implemented.'); } - getDefaultModel () { + getDefaultModel() { return 'fake'; } - async models () { + async models() { return [ { id: 'fake', @@ -42,7 +42,6 @@ export class FakeChatProvider implements IChatProvider { 'output-tokens': 0, }, max_tokens: 8192, - }, { id: 'costly', @@ -66,25 +65,41 @@ export class FakeChatProvider implements IChatProvider { }, ]; } - async list () { + async list() { return ['fake', 'costly', 'abuse']; } - async complete ({ messages, stream, model, max_tokens, custom }: ICompleteArguments): ReturnType { - + async complete({ + messages, + stream, + model, + max_tokens, + custom, + }: ICompleteArguments): ReturnType { // Determine token counts based on messages and model const usedModel = model || this.getDefaultModel(); // For the costly model, simulate actual token counting - const resp = this.getFakeResponse(usedModel, custom, messages, max_tokens); + const resp = this.getFakeResponse( + usedModel, + custom, + messages, + max_tokens, + ); - if ( stream ) { + if (stream) { return { - init_chat_stream: async ({ chatStream }: { chatStream: AIChatStream }) => { - await new Promise(rslv => setTimeout(rslv, 500)); - chatStream.stream.write(`${JSON.stringify({ - type: 'text', - text: (await resp).message.content[0].text, - }) }\n`); + init_chat_stream: async ({ + chatStream, + }: { + chatStream: AIChatStream; + }) => { + await new Promise((rslv) => setTimeout(rslv, 500)); + chatStream.stream.write( + `${JSON.stringify({ + type: 'text', + text: (await resp).message.content[0].text, + })}\n`, + ); chatStream.end({}); }, stream: true, @@ -96,20 +111,27 @@ export class FakeChatProvider implements IChatProvider { return resp; } - async getFakeResponse (modelId: string, custom: unknown, messages: PuterMessage[], maxTokens: number = 8192): ReturnType { + async getFakeResponse( + modelId: string, + custom: unknown, + messages: PuterMessage[], + maxTokens: number = 8192, + ): ReturnType { let inputTokens = 0; let outputTokens = 0; - if ( modelId === 'costly' ) { + if (modelId === 'costly') { // Simple token estimation: roughly 4 chars per token for input - if ( messages && messages.length > 0 ) { - for ( const message of messages ) { - if ( typeof message.content === 'string' ) { + if (messages && messages.length > 0) { + for (const message of messages) { + if (typeof message.content === 'string') { inputTokens += Math.ceil(message.content.length / 4); - } else if ( Array.isArray(message.content) ) { - for ( const content of message.content ) { - if ( content.type === 'text' ) { - inputTokens += Math.ceil(content.text.length / 4); + } else if (Array.isArray(message.content)) { + for (const content of message.content) { + if (content.type === 'text') { + inputTokens += Math.ceil( + content.text.length / 4, + ); } } } @@ -117,13 +139,15 @@ export class FakeChatProvider implements IChatProvider { } // Generate random output token count between 50 and 200 - outputTokens = Math.floor(Math.min((Math.random() * 150) + 50, maxTokens)); + outputTokens = Math.floor( + Math.min(Math.random() * 150 + 50, maxTokens), + ); // outputTokens = Math.floor(Math.random() * 150) + 50; } // Generate the response text let responseText; - if ( modelId === 'abuse' ) { + if (modelId === 'abuse') { responseText = dedent(`

Free AI and Cloud for everyone!


Come on down to puter.com and try it out! @@ -145,28 +169,28 @@ export class FakeChatProvider implements IChatProvider { // Report usage based on model const usage = { - 'input_tokens': modelId === 'costly' ? inputTokens : 0, - 'output_tokens': modelId === 'costly' ? outputTokens : 1, + input_tokens: modelId === 'costly' ? inputTokens : 0, + output_tokens: modelId === 'costly' ? outputTokens : 1, }; return { message: { - 'id': '00000000-0000-0000-0000-000000000000', - 'type': 'message', - 'role': 'assistant', - 'model': modelId, - 'content': [ + id: '00000000-0000-0000-0000-000000000000', + type: 'message', + role: 'assistant', + model: modelId, + content: [ { - 'type': 'text', - 'text': responseText, + type: 'text', + text: responseText, }, ], - 'stop_reason': 'end_turn', - 'stop_sequence': null, - 'usage': usage, + stop_reason: 'end_turn', + stop_sequence: null, + usage: usage, }, - 'usage': usage, - 'finish_reason': 'stop', + usage: usage, + finish_reason: 'stop', }; } } diff --git a/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.ts b/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.ts new file mode 100644 index 000000000..2919ca3c9 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.ts @@ -0,0 +1,508 @@ +import Anthropic from '@anthropic-ai/sdk'; +import type { Message } from '@anthropic-ai/sdk/resources'; +import type { BetaUsage } from '@anthropic-ai/sdk/resources/beta.js'; +import type { + MessageCreateParams, + Usage, +} from '@anthropic-ai/sdk/resources/messages.js'; +import { Context } from '../../../../core/context.js'; +import type { FSService } from '../../../../services/fs/FSService.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { FSEntryStore } from '../../../../stores/fs/FSEntryStore.js'; +import type { S3ObjectStore } from '../../../../stores/fs/S3ObjectStore.js'; +import type { + IChatProvider, + ICompleteArguments, + IChatCompleteResult, +} from '../../types.js'; +import { make_claude_tools } from '../../utils/FunctionCalling.js'; +import { extract_and_remove_system_messages } from '../../utils/Messages.js'; +import type { + AIChatStream, + AIChatTextStream, + AIChatToolUseStream, +} from '../../utils/Streaming.js'; +import { FILES_API_BETA, processPuterPathUploads } from './fileUpload.js'; +import { CLAUDE_MODELS } from './models.js'; + +export class ClaudeProvider implements IChatProvider { + anthropic: Anthropic; + + #meteringService: MeteringService; + + #stores: { fsEntry: FSEntryStore; s3Object: S3ObjectStore }; + + #fsService: FSService; + + constructor( + meteringService: MeteringService, + stores: { fsEntry: FSEntryStore; s3Object: S3ObjectStore }, + fsService: FSService, + config: { apiKey: string }, + ) { + this.#meteringService = meteringService; + this.#stores = stores; + this.#fsService = fsService; + this.anthropic = new Anthropic({ + apiKey: config.apiKey, + timeout: 10 * 60 * 1001, + }); + } + + getDefaultModel() { + return 'claude-haiku-4-5-20251001'; + } + + models() { + return CLAUDE_MODELS; + } + + async list() { + const models = this.models(); + const model_names: string[] = []; + for (const model of models) { + model_names.push(model.id); + if (model.aliases) { + model_names.push(...model.aliases); + } + } + return model_names; + } + + async complete({ + messages, + stream, + model, + tools, + max_tokens, + temperature, + reasoning, + reasoning_effort, + }: ICompleteArguments): Promise { + tools = make_claude_tools(tools); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let system_prompts: string | any[]; + [system_prompts, messages] = + extract_and_remove_system_messages(messages); + + // Apply cache_control to system prompt content blocks + if ( + system_prompts.length > 0 && + system_prompts[0].cache_control && + system_prompts[0]?.content + ) { + system_prompts[0].content = system_prompts[0].content.map( + (prompt: any) => { + prompt.cache_control = system_prompts[0].cache_control; + return prompt; + }, + ); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + messages = messages.map((message: any) => { + if (message.cache_control) { + message.content[0].cache_control = message.cache_control; + } + delete message.cache_control; + return message; + }); + + // Convert OpenAI-style tool calls/results to Claude format + // eslint-disable-next-line @typescript-eslint/no-explicit-any + messages = messages.map((message: any) => { + if (message.tool_calls && Array.isArray(message.tool_calls)) { + if (!Array.isArray(message.content)) { + message.content = message.content ? [message.content] : []; + } + for (const toolCall of message.tool_calls) { + message.content.push({ + type: 'tool_use', + id: toolCall.id, + name: toolCall.function?.name, + input: toolCall.function?.arguments ?? {}, + }); + } + delete message.tool_calls; + } + + if (message.role !== 'tool') return message; + + const toolUseId = message.tool_call_id || message.tool_use_id; + + const contentValue = (() => { + if (Array.isArray(message.content)) { + const toolResultBlock = message.content.find( + (part: any) => part?.type === 'tool_result', + ); + if (toolResultBlock) { + return ( + toolResultBlock.content ?? + toolResultBlock.text ?? + '' + ); + } + + return message.content + .map((part: any) => { + if (typeof part === 'string') return part; + if (part && typeof part.text === 'string') + return part.text; + if (part && typeof part.content === 'string') + return part.content; + return ''; + }) + .join(''); + } + if (typeof message.content === 'string') return message.content; + if (message.content && typeof message.content.text === 'string') + return message.content.text; + if ( + message.content && + typeof message.content.content === 'string' + ) + return message.content.content; + return ''; + })(); + + return { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: toolUseId, + content: contentValue, + }, + ], + }; + }); + + // Claude requires tool_use.input to be a dictionary + // eslint-disable-next-line @typescript-eslint/no-explicit-any + messages = messages.map((message: any) => { + if (!Array.isArray(message.content)) return message; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + message.content = message.content.map((part: any) => { + if (part?.type !== 'tool_use') return part; + if (typeof part.input === 'string') { + try { + part.input = JSON.parse(part.input); + } catch { + part.input = {}; + } + } else if (part.input === undefined || part.input === null) { + part.input = {}; + } + return part; + }); + return message; + }); + + const modelUsed = + this.models().find((m) => + [m.id, ...(m.aliases || [])].includes(model), + ) || this.models().find((m) => m.id === this.getDefaultModel())!; + + const requestedReasoningEffort = reasoning_effort ?? reasoning?.effort; + const thinkingConfig = this.#buildThinkingConfig({ + modelId: modelUsed.id, + reasoningEffort: requestedReasoningEffort, + maxTokens: max_tokens, + }); + // Opus 4.7 errors on non-default sampling params; omit temperature entirely. + // Other models require temperature=1 when thinking is enabled. + const isOpus47 = modelUsed.id === 'claude-opus-4-7'; + const resolvedTemperature = isOpus47 + ? undefined + : thinkingConfig + ? 1 + : (temperature ?? 0); + const supportsEffort = [ + 'claude-opus-4-7', + 'claude-opus-4-6', + 'claude-sonnet-4-6', + ].includes(modelUsed.id); + + const actor = Context.get('actor'); + + // Upload any `puter_path` parts to Anthropic's Files API and rewrite + // them in-place to reference the returned `file_id`. Must happen + // before sdkParams snapshots `messages`. + const { fileIds: uploadedFileIds } = await processPuterPathUploads( + this.anthropic, + messages, + this.#stores, + this.#fsService, + actor, + ); + const usesBetaFiles = uploadedFileIds.length > 0; + + const sdkParams: MessageCreateParams & { + betas?: string[]; + } = { + model: modelUsed.id, + max_tokens: Math.floor( + max_tokens || + (model === 'claude-3-5-sonnet-20241022' || + model === 'claude-3-5-sonnet-20240620' + ? 8192 + : this.models().filter( + (e) => + (e as any).name === model || + e.aliases?.includes(model), + )[0]?.max_tokens || 4096), + ), + ...(resolvedTemperature !== undefined + ? { temperature: resolvedTemperature } + : {}), + ...(system_prompts && system_prompts[0]?.content + ? { system: system_prompts[0]?.content } + : {}), + tool_choice: { type: 'auto', disable_parallel_tool_use: true }, + messages, + ...(tools ? { tools } : {}), + ...(thinkingConfig ? { thinking: thinkingConfig } : {}), + ...(supportsEffort && requestedReasoningEffort + ? { output_config: { effort: requestedReasoningEffort } } + : {}), + ...(usesBetaFiles ? { betas: [FILES_API_BETA] } : {}), + } as MessageCreateParams & { betas?: string[] }; + + const cleanupUploads = async () => { + if (uploadedFileIds.length === 0) return; + await Promise.all( + uploadedFileIds.map(async (id) => { + try { + await this.anthropic.beta.files.delete(id, { + betas: [FILES_API_BETA], + }); + } catch { + /* best-effort */ + } + }), + ); + }; + + if (stream) { + const init_chat_stream = async ({ + chatStream, + }: { + chatStream: AIChatStream; + }) => { + const completion = usesBetaFiles + ? this.anthropic.beta.messages.stream(sdkParams) + : this.anthropic.messages.stream(sdkParams); + const usageSum: Record = {}; + + let message, contentBlock; + let currentContentBlockType: string | null = null; + for await (const event of completion) { + if (event.type === 'message_delta') { + const meteredData = this.#usageFormatterUtil( + (event?.usage ?? {}) as Usage | BetaUsage, + ); + for (const key in meteredData) { + usageSum[key] = Math.max( + usageSum[key] ?? 0, + meteredData[key as keyof typeof meteredData], + ); + } + } + if (event.type === 'message_start') { + message = chatStream.message(); + continue; + } + if (event.type === 'message_stop') { + message!.end(); + message = null; + continue; + } + if (event.type === 'content_block_start') { + currentContentBlockType = event.content_block.type; + if (event.content_block.type === 'tool_use') { + contentBlock = message!.contentBlock({ + type: event.content_block.type, + id: event.content_block.id, + name: event.content_block.name, + }); + } else if (event.content_block.type === 'thinking') { + contentBlock = message!.contentBlock({ + type: 'text', + }); + } else { + contentBlock = message!.contentBlock({ + type: event.content_block.type, + }); + } + continue; + } + if (event.type === 'content_block_stop') { + contentBlock!.end(); + contentBlock = null; + currentContentBlockType = null; + continue; + } + if (event.type === 'content_block_delta') { + if (event.delta.type === 'input_json_delta') { + (contentBlock as AIChatToolUseStream)!.addPartialJSON( + event.delta.partial_json, + ); + } else if (event.delta.type === 'text_delta') { + if (currentContentBlockType === 'thinking') { + (contentBlock as AIChatTextStream)!.addReasoning( + event.delta.text, + ); + } else { + (contentBlock as AIChatTextStream)!.addText( + event.delta.text, + ); + } + } else if (event.delta.type === 'thinking_delta') { + (contentBlock as AIChatTextStream)!.addReasoning( + (event.delta as { thinking: string }).thinking, + ); + } + // signature_delta — ignored + } + } + const finalUsage = await completion + .finalMessage() + .then((msg) => + this.#usageFormatterUtil( + msg.usage as Usage | BetaUsage, + ), + ) + .catch(() => null); + if (finalUsage) { + for (const [key, value] of Object.entries(finalUsage)) { + usageSum[key] = value; + } + } + chatStream.end(usageSum); + const costsOverrideFromModel = + this.#buildCostsOverrideFromModel(usageSum, modelUsed); + this.#meteringService.utilRecordUsageObject( + usageSum, + actor, + `claude:${modelUsed.id}`, + costsOverrideFromModel, + ); + }; + + return { + init_chat_stream, + stream: true, + finally_fn: cleanupUploads, + }; + } + + try { + const msg = await (usesBetaFiles + ? this.anthropic.beta.messages.create(sdkParams) + : this.anthropic.messages.create(sdkParams)); + const usage = this.#usageFormatterUtil( + (msg as Message).usage as Usage | BetaUsage, + ); + const costsOverrideFromModel = this.#buildCostsOverrideFromModel( + usage, + modelUsed, + ); + this.#meteringService.utilRecordUsageObject( + usage, + actor, + `claude:${modelUsed.id}`, + costsOverrideFromModel, + ); + + return { message: msg, usage, finish_reason: 'stop' }; + } finally { + await cleanupUploads(); + } + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + #usageFormatterUtil(usage: any) { + return { + input_tokens: usage?.input_tokens || 0, + ephemeral_5m_input_tokens: + usage?.cache_creation?.ephemeral_5m_input_tokens || + usage?.cache_creation_input_tokens || + 0, + ephemeral_1h_input_tokens: + usage?.cache_creation?.ephemeral_1h_input_tokens || 0, + cache_read_input_tokens: usage?.cache_read_input_tokens || 0, + output_tokens: usage?.output_tokens || 0, + thinking_tokens: + usage?.thinking_tokens || + usage?.output_tokens_details?.thinking_tokens || + 0, + }; + } + + #buildCostsOverrideFromModel( + usage: Record, + modelUsed: { costs: Record }, + ) { + return Object.fromEntries( + Object.entries(usage).map(([k, v]) => { + const modelCost = + modelUsed.costs[k] ?? + (k === 'thinking_tokens' + ? modelUsed.costs.output_tokens + : 0); + return [k, v * modelCost]; + }), + ); + } + + #buildThinkingConfig({ + modelId, + reasoningEffort, + maxTokens, + }: { + modelId?: string; + reasoningEffort?: 'low' | 'medium' | 'high'; + maxTokens?: number; + }) { + if (!reasoningEffort) return undefined; + + // Opus 4.7, 4.6, and Sonnet 4.6 use adaptive thinking + // (`budget_tokens` is deprecated on 4.6/Sonnet 4.6, removed on 4.7). + // Opus 4.7 omits thinking content by default; `display: 'summarized'` + // restores visible reasoning in the stream. + if (modelId === 'claude-opus-4-7') { + return { + type: 'adaptive' as const, + display: 'summarized' as const, + }; + } + if (modelId === 'claude-opus-4-6' || modelId === 'claude-sonnet-4-6') { + return { type: 'adaptive' as const }; + } + + const requestedBudget = { low: 1024, medium: 4096, high: 8192 }[ + reasoningEffort + ]; + + if (typeof maxTokens === 'number' && Number.isFinite(maxTokens)) { + if (Math.floor(maxTokens - 1) < 1024) return undefined; + } + + const budget_tokens = Math.floor( + Math.max( + 1024, + Math.min( + requestedBudget, + maxTokens ? maxTokens - 1 : requestedBudget, + ), + ), + ); + + return { type: 'enabled' as const, budget_tokens }; + } + + checkModeration(_text: string): never { + throw new Error('CheckModeration not provided by Claude provider.'); + } +} diff --git a/src/backend/drivers/ai-chat/providers/claude/fileUpload.ts b/src/backend/drivers/ai-chat/providers/claude/fileUpload.ts new file mode 100644 index 000000000..ccb73704e --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/claude/fileUpload.ts @@ -0,0 +1,120 @@ +import Anthropic, { toFile } from '@anthropic-ai/sdk'; +import type { Actor } from '../../../../core/actor.js'; +import type { FSService } from '../../../../services/fs/FSService.js'; +import type { FSEntryStore } from '../../../../stores/fs/FSEntryStore.js'; +import type { S3ObjectStore } from '../../../../stores/fs/S3ObjectStore.js'; +import { loadFileInput } from '../../../util/fileInput.js'; + +export const FILES_API_BETA = 'files-api-2025-04-14'; +// Claude's documented per-file cap is 500MB, but pulling huge objects +// through base64 token counting is impractical — cap at 30MB like v1. +const MAX_FILE_SIZE = 30 * 1_000_000; + +interface ContentPart { + puter_path?: string; + type?: string; + text?: string; + source?: { type: string; file_id: string }; +} + +export interface ClaudeUploadResult { + /** File IDs uploaded this request; caller deletes them after completion. */ + fileIds: string[]; +} + +/** + * Resolve any `puter_path` content parts by uploading the referenced FS + * entries to Anthropic's Files API and rewriting each part to reference the + * returned `file_id`. Parts that fail (too large, missing, etc.) are swapped + * for an inline `text` error so the model can explain rather than the whole + * request failing. + * + * Callers MUST pass `betas: [FILES_API_BETA]` on the subsequent + * `beta.messages.create`/`.stream` call when any files were uploaded, and + * should clean up via `anthropic.beta.files.delete(id)` in their finally path. + */ +export async function processPuterPathUploads( + anthropic: Anthropic, + messages: Array<{ content?: unknown }>, + stores: { fsEntry: FSEntryStore; s3Object: S3ObjectStore }, + fsService: FSService, + actor: Actor | undefined, +): Promise { + const parts: ContentPart[] = []; + for (const message of messages) { + if (!Array.isArray(message.content)) continue; + for (const part of message.content as ContentPart[]) { + if (part?.puter_path) parts.push(part); + } + } + if (parts.length === 0) return { fileIds: [] }; + + const fileIds: string[] = []; + await Promise.all( + parts.map((part) => + processPart(part, anthropic, stores, fsService, actor, fileIds), + ), + ); + return { fileIds }; +} + +async function processPart( + part: ContentPart, + anthropic: Anthropic, + stores: { fsEntry: FSEntryStore; s3Object: S3ObjectStore }, + fsService: FSService, + actor: Actor | undefined, + fileIds: string[], +): Promise { + const path = part.puter_path!; + delete part.puter_path; + + if (!actor?.user?.id) { + setTextError(part, 'unauthenticated caller cannot resolve puter_path'); + return; + } + + try { + const loaded = await loadFileInput(stores, fsService, actor, path, { + maxBytes: MAX_FILE_SIZE, + }); + const mimeType = loaded.mimeType ?? 'application/octet-stream'; + const uploaded = await anthropic.beta.files.upload({ + file: await toFile(loaded.buffer, loaded.filename, { + type: mimeType, + }), + betas: [FILES_API_BETA], + }); + fileIds.push(uploaded.id); + + part.type = contentBlockTypeForMime(mimeType); + part.source = { type: 'file', file_id: uploaded.id }; + } catch (err) { + const status = (err as { status?: number })?.status; + if (status === 413) { + setTextError( + part, + `input file exceeded maximum of ${MAX_FILE_SIZE} bytes`, + ); + return; + } + const message = (err as Error)?.message || 'failed to read input file'; + setTextError(part, message); + } +} + +// Mirrors the table at https://docs.claude.com/en/docs/build-with-claude/files +function contentBlockTypeForMime(mimeType: string): string { + if (mimeType.startsWith('image/')) return 'image'; + if (mimeType.startsWith('text/')) return 'document'; + if (mimeType === 'application/pdf' || mimeType === 'application/x-pdf') { + return 'document'; + } + return 'container_upload'; +} + +function setTextError(part: ContentPart, reason: string): void { + delete part.source; + part.type = 'text'; + part.text = `{error: ${reason}; the user did not write this message}`; +} diff --git a/src/backend/src/services/ai/chat/providers/ClaudeProvider/models.ts b/src/backend/drivers/ai-chat/providers/claude/models.ts similarity index 78% rename from src/backend/src/services/ai/chat/providers/ClaudeProvider/models.ts rename to src/backend/drivers/ai-chat/providers/claude/models.ts index 7ff6a1fa5..91da27128 100644 --- a/src/backend/src/services/ai/chat/providers/ClaudeProvider/models.ts +++ b/src/backend/drivers/ai-chat/providers/claude/models.ts @@ -1,16 +1,23 @@ -import { IChatModel } from '../types'; +import type { IChatModel } from '../../types.js'; // Hardcoded from https://models.dev/api.json export const CLAUDE_MODELS: IChatModel[] = [ { puterId: 'anthropic:anthropic/claude-opus-4-7', id: 'claude-opus-4-7', - modalities: { 'input': ['text', 'image', 'pdf'], 'output': ['text'] }, + modalities: { input: ['text', 'image', 'pdf'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2026-01', release_date: '2026-04-16', - aliases: ['claude-opus', 'claude-opus-latest', 'claude-opus-4-7-latest', 'claude-opus-4.7', 'claude-opus-4-7', 'anthropic/claude-opus-4-7'], + aliases: [ + 'claude-opus', + 'claude-opus-latest', + 'claude-opus-4-7-latest', + 'claude-opus-4.7', + 'claude-opus-4-7', + 'anthropic/claude-opus-4-7', + ], name: 'Claude Opus 4.7', costs_currency: 'usd-cents', input_cost_key: 'input_tokens', @@ -29,12 +36,19 @@ export const CLAUDE_MODELS: IChatModel[] = [ { puterId: 'anthropic:anthropic/claude-sonnet-4-6', id: 'claude-sonnet-4-6', - modalities: { 'input': ['text', 'image', 'pdf'], 'output': ['text'] }, + modalities: { input: ['text', 'image', 'pdf'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2025-08', release_date: '2026-02-17', - aliases: ['claude-sonnet-latest', 'claude-sonnet', 'claude-sonnet-4-6-latest', 'claude-sonnet-4.6', 'claude-sonnet-4-6', 'anthropic/claude-sonnet-4-6'], + aliases: [ + 'claude-sonnet-latest', + 'claude-sonnet', + 'claude-sonnet-4-6-latest', + 'claude-sonnet-4.6', + 'claude-sonnet-4-6', + 'anthropic/claude-sonnet-4-6', + ], name: 'Claude Sonnet 4.6', costs_currency: 'usd-cents', input_cost_key: 'input_tokens', @@ -53,12 +67,17 @@ export const CLAUDE_MODELS: IChatModel[] = [ { puterId: 'anthropic:anthropic/claude-opus-4-6', id: 'claude-opus-4-6', - modalities: { 'input': ['text', 'image', 'pdf'], 'output': ['text'] }, + modalities: { input: ['text', 'image', 'pdf'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2025-05', release_date: '2026-02-05', - aliases: ['claude-opus-4-6-latest', 'claude-opus-4.6', 'claude-opus-4-6', 'anthropic/claude-opus-4-6'], + aliases: [ + 'claude-opus-4-6-latest', + 'claude-opus-4.6', + 'claude-opus-4-6', + 'anthropic/claude-opus-4-6', + ], name: 'Claude Opus 4.6', costs_currency: 'usd-cents', input_cost_key: 'input_tokens', @@ -77,12 +96,17 @@ export const CLAUDE_MODELS: IChatModel[] = [ { puterId: 'anthropic:anthropic/claude-opus-4-5', id: 'claude-opus-4-5-20251101', - modalities: { 'input': ['text', 'image', 'pdf'], 'output': ['text'] }, + modalities: { input: ['text', 'image', 'pdf'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2025-03-31', release_date: '2025-11-01', - aliases: ['claude-opus-4-5-latest', 'claude-opus-4-5', 'claude-opus-4.5', 'anthropic/claude-opus-4-5'], + aliases: [ + 'claude-opus-4-5-latest', + 'claude-opus-4-5', + 'claude-opus-4.5', + 'anthropic/claude-opus-4-5', + ], name: 'Claude Opus 4.5', costs_currency: 'usd-cents', input_cost_key: 'input_tokens', @@ -101,12 +125,20 @@ export const CLAUDE_MODELS: IChatModel[] = [ { puterId: 'anthropic:anthropic/claude-haiku-4-5', id: 'claude-haiku-4-5-20251001', - modalities: { 'input': ['text', 'image', 'pdf'], 'output': ['text'] }, + modalities: { input: ['text', 'image', 'pdf'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2025-02-28', release_date: '2025-10-15', - aliases: ['claude-haiku', 'claude-haiku-latest', 'claude-haiku-4.5-latest', 'claude-haiku-4.5', 'claude-haiku-4-5', 'claude-4-5-haiku', 'anthropic/claude-haiku-4-5'], + aliases: [ + 'claude-haiku', + 'claude-haiku-latest', + 'claude-haiku-4.5-latest', + 'claude-haiku-4.5', + 'claude-haiku-4-5', + 'claude-4-5-haiku', + 'anthropic/claude-haiku-4-5', + ], name: 'Claude Haiku 4.5', costs_currency: 'usd-cents', input_cost_key: 'input_tokens', @@ -125,12 +157,16 @@ export const CLAUDE_MODELS: IChatModel[] = [ { puterId: 'anthropic:anthropic/claude-sonnet-4-5', id: 'claude-sonnet-4-5-20250929', - modalities: { 'input': ['text', 'image', 'pdf'], 'output': ['text'] }, + modalities: { input: ['text', 'image', 'pdf'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2025-07-31', release_date: '2025-09-29', - aliases: ['claude-sonnet-4.5', 'claude-sonnet-4-5', 'anthropic/claude-sonnet-4-5'], + aliases: [ + 'claude-sonnet-4.5', + 'claude-sonnet-4-5', + 'anthropic/claude-sonnet-4-5', + ], name: 'Claude Sonnet 4.5', costs_currency: 'usd-cents', input_cost_key: 'input_tokens', @@ -149,7 +185,7 @@ export const CLAUDE_MODELS: IChatModel[] = [ { puterId: 'anthropic:anthropic/claude-opus-4-1', id: 'claude-opus-4-1-20250805', - modalities: { 'input': ['text', 'image', 'pdf'], 'output': ['text'] }, + modalities: { input: ['text', 'image', 'pdf'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2025-03-31', @@ -173,12 +209,16 @@ export const CLAUDE_MODELS: IChatModel[] = [ { puterId: 'anthropic:anthropic/claude-opus-4', id: 'claude-opus-4-20250514', - modalities: { 'input': ['text', 'image', 'pdf'], 'output': ['text'] }, + modalities: { input: ['text', 'image', 'pdf'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2025-03-31', release_date: '2025-05-22', - aliases: ['claude-opus-4', 'claude-opus-4-latest', 'anthropic/claude-opus-4'], + aliases: [ + 'claude-opus-4', + 'claude-opus-4-latest', + 'anthropic/claude-opus-4', + ], name: 'Claude Opus 4', costs_currency: 'usd-cents', input_cost_key: 'input_tokens', @@ -197,12 +237,16 @@ export const CLAUDE_MODELS: IChatModel[] = [ { puterId: 'anthropic:anthropic/claude-sonnet-4', id: 'claude-sonnet-4-20250514', - modalities: { 'input': ['text', 'image', 'pdf'], 'output': ['text'] }, + modalities: { input: ['text', 'image', 'pdf'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2025-03-31', release_date: '2025-05-22', - aliases: ['claude-sonnet-4', 'claude-sonnet-4-latest', 'anthropic/claude-sonnet-4'], + aliases: [ + 'claude-sonnet-4', + 'claude-sonnet-4-latest', + 'anthropic/claude-sonnet-4', + ], name: 'Claude Sonnet 4', costs_currency: 'usd-cents', input_cost_key: 'input_tokens', @@ -221,7 +265,7 @@ export const CLAUDE_MODELS: IChatModel[] = [ { puterId: 'anthropic:anthropic/claude-3-7-sonnet', id: 'claude-3-7-sonnet-20250219', - modalities: { 'input': ['text', 'image', 'pdf'], 'output': ['text'] }, + modalities: { input: ['text', 'image', 'pdf'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-10-31', @@ -245,7 +289,7 @@ export const CLAUDE_MODELS: IChatModel[] = [ { puterId: 'anthropic:anthropic/claude-3-5-sonnet', id: 'claude-3-5-sonnet-20241022', - modalities: { 'input': ['text', 'image', 'pdf'], 'output': ['text'] }, + modalities: { input: ['text', 'image', 'pdf'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-04-30', @@ -263,15 +307,13 @@ export const CLAUDE_MODELS: IChatModel[] = [ cache_read_input_tokens: 300 * 0.1, output_tokens: 1500, }, - qualitative_speed: 'fast', - training_cutoff: '2024-04', context: 200000, max_tokens: 8192, }, { puterId: 'anthropic:anthropic/claude-3-5-sonnet-20240620', id: 'claude-3-5-sonnet-20240620', - modalities: { 'input': ['text', 'image', 'pdf'], 'output': ['text'] }, + modalities: { input: ['text', 'image', 'pdf'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-04-30', @@ -289,13 +331,13 @@ export const CLAUDE_MODELS: IChatModel[] = [ cache_read_input_tokens: 300 * 0.1, output_tokens: 1500, }, - context: 200000, // might be wrong + context: 200000, max_tokens: 8192, }, { puterId: 'anthropic:anthropic/claude-3-haiku', id: 'claude-3-haiku-20240307', - modalities: { 'input': ['text', 'image', 'pdf'], 'output': ['text'] }, + modalities: { input: ['text', 'image', 'pdf'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2023-08-31', @@ -312,7 +354,6 @@ export const CLAUDE_MODELS: IChatModel[] = [ cache_read_input_tokens: 25 * 0.1, output_tokens: 125, }, - qualitative_speed: 'fastest', context: 200000, max_tokens: 4096, }, diff --git a/src/backend/src/services/ai/chat/providers/DeepSeekProvider/DeepSeekProvider.ts b/src/backend/drivers/ai-chat/providers/deepseek/DeepSeekProvider.ts similarity index 64% rename from src/backend/src/services/ai/chat/providers/DeepSeekProvider/DeepSeekProvider.ts rename to src/backend/drivers/ai-chat/providers/deepseek/DeepSeekProvider.ts index 4413443c3..01bbf5356 100644 --- a/src/backend/src/services/ai/chat/providers/DeepSeekProvider/DeepSeekProvider.ts +++ b/src/backend/drivers/ai-chat/providers/deepseek/DeepSeekProvider.ts @@ -20,10 +20,10 @@ import dedent from 'dedent'; import { OpenAI } from 'openai'; import { ChatCompletionCreateParams } from 'openai/resources/index.js'; -import { Context } from '../../../../../util/context.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import * as OpenAIUtil from '../../../utils/OpenAIUtil.js'; -import { IChatProvider, ICompleteArguments } from '../types.js'; +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { IChatProvider, ICompleteArguments } from '../../types.js'; +import * as OpenAIUtil from '../../utils/OpenAIUtil.js'; import { DEEPSEEK_MODELS } from './models.js'; export class DeepSeekProvider implements IChatProvider { @@ -31,7 +31,7 @@ export class DeepSeekProvider implements IChatProvider { #meteringService: MeteringService; - constructor (config: { apiKey: string }, meteringService: MeteringService) { + constructor(config: { apiKey: string }, meteringService: MeteringService) { this.#openai = new OpenAI({ apiKey: config.apiKey, baseURL: 'https://api.deepseek.com', @@ -39,41 +39,55 @@ export class DeepSeekProvider implements IChatProvider { this.#meteringService = meteringService; } - getDefaultModel () { + getDefaultModel() { return 'deepseek-chat'; } - models () { + models() { return DEEPSEEK_MODELS; } - async list () { + async list() { const models = this.models(); const modelNames: string[] = []; - for ( const model of models ) { + for (const model of models) { modelNames.push(model.id); - if ( model.aliases ) { + if (model.aliases) { modelNames.push(...model.aliases); } } return modelNames; } - async complete ({ messages, stream, model, tools, max_tokens, temperature }: ICompleteArguments): ReturnType { + async complete({ + messages, + stream, + model, + tools, + max_tokens, + temperature, + }: ICompleteArguments): ReturnType { const actor = Context.get('actor'); const availableModels = this.models(); - const modelUsed = availableModels.find(m => [m.id, ...(m.aliases || [])].includes(model)) || availableModels.find(m => m.id === this.getDefaultModel())!; + const modelUsed = + availableModels.find((m) => + [m.id, ...(m.aliases || [])].includes(model), + ) || availableModels.find((m) => m.id === this.getDefaultModel())!; messages = await OpenAIUtil.process_input_messages(messages); - for ( const message of messages ) { + for (const message of messages) { // DeepSeek doesn't accept string arrays alongside tool calls - if ( message.tool_calls && Array.isArray(message.content) ) { + if (message.tool_calls && Array.isArray(message.content)) { message.content = ''; } } // Function calling currently loops unless we inject the tool result as a system message. - const TOOL_TEXT = (message: { tool_call_id: string; content: string }) => dedent(` + const TOOL_TEXT = (message: { + tool_call_id: string; + content: string; + }) => + dedent(` Hi DeepSeek V3, your tool calling is broken and you are not able to obtain tool results in the expected way. That's okay, we can work around this. @@ -84,9 +98,9 @@ export class DeepSeekProvider implements IChatProvider { Tool call ${message.tool_call_id} returned: ${message.content}. `); - for ( let i = messages.length - 1; i >= 0; i-- ) { + for (let i = messages.length - 1; i >= 0; i--) { const message = messages[i]; - if ( message.role === 'tool' ) { + if (message.role === 'tool') { messages.splice(i + 1, 0, { role: 'system', content: [ @@ -106,18 +120,27 @@ export class DeepSeekProvider implements IChatProvider { max_tokens: max_tokens || 1000, temperature, stream, - ...(stream ? { - stream_options: { include_usage: true }, - } : {}), + ...(stream + ? { + stream_options: { include_usage: true }, + } + : {}), } as ChatCompletionCreateParams); return OpenAIUtil.handle_completion_output({ usage_calculator: ({ usage }) => { const trackedUsage = OpenAIUtil.extractMeteredUsage(usage); - const costsOverrideFromModel = Object.fromEntries(Object.entries(trackedUsage).map(([k, v]) => { - return [k, v * (modelUsed.costs[k])]; - })); - this.#meteringService.utilRecordUsageObject(trackedUsage, actor, `deepseek:${modelUsed.id}`, costsOverrideFromModel); + const costsOverrideFromModel = Object.fromEntries( + Object.entries(trackedUsage).map(([k, v]) => { + return [k, v * modelUsed.costs[k]]; + }), + ); + this.#meteringService.utilRecordUsageObject( + trackedUsage, + actor, + `deepseek:${modelUsed.id}`, + costsOverrideFromModel, + ); return trackedUsage; }, stream, @@ -125,7 +148,9 @@ export class DeepSeekProvider implements IChatProvider { }); } - checkModeration (_text: string): ReturnType { + checkModeration( + _text: string, + ): ReturnType { throw new Error('Method not implemented.'); } } diff --git a/src/backend/src/services/ai/chat/providers/DeepSeekProvider/models.ts b/src/backend/drivers/ai-chat/providers/deepseek/models.ts similarity index 88% rename from src/backend/src/services/ai/chat/providers/DeepSeekProvider/models.ts rename to src/backend/drivers/ai-chat/providers/deepseek/models.ts index b7df50898..70f367e5e 100644 --- a/src/backend/src/services/ai/chat/providers/DeepSeekProvider/models.ts +++ b/src/backend/drivers/ai-chat/providers/deepseek/models.ts @@ -1,11 +1,11 @@ -import { IChatModel } from '../types.js'; +import type { IChatModel } from '../../types.js'; // Hardcoded from https://models.dev/api.json export const DEEPSEEK_MODELS: IChatModel[] = [ { puterId: 'deepseek:deepseek/deepseek-chat', id: 'deepseek-chat', - modalities: { 'input': ['text'], 'output': ['text'] }, + modalities: { input: ['text'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-07', @@ -27,7 +27,7 @@ export const DEEPSEEK_MODELS: IChatModel[] = [ { puterId: 'deepseek:deepseek/deepseek-reasoner', id: 'deepseek-reasoner', - modalities: { 'input': ['text'], 'output': ['text'] }, + modalities: { input: ['text'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-07', diff --git a/src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.ts b/src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.ts new file mode 100644 index 000000000..564bd2c82 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.ts @@ -0,0 +1,120 @@ +// Preamble: Before this we used Gemini's SDK directly and as we found out +// its actually kind of terrible. So we use the openai sdk now +import openai, { OpenAI } from 'openai'; +import { ChatCompletionCreateParams } from 'openai/resources/index.js'; +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { IChatProvider, ICompleteArguments } from '../../types.js'; +import { + handle_completion_output, + process_input_messages, +} from '../../utils/OpenAIUtil.js'; +import { GEMINI_MODELS } from './models.js'; + +export class GeminiChatProvider implements IChatProvider { + meteringService: MeteringService; + openai: OpenAI; + + defaultModel = 'gemini-2.5-flash'; + + constructor(meteringService: MeteringService, config: { apiKey: string }) { + this.meteringService = meteringService; + this.openai = new openai.OpenAI({ + apiKey: config.apiKey, + baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai/', + }); + } + + getDefaultModel() { + return this.defaultModel; + } + + async models() { + return GEMINI_MODELS; + } + async list() { + return (await this.models()) + .map((m) => [m.id, ...(m.aliases || [])]) + .flat(); + } + + async complete({ + messages, + stream, + model, + tools, + max_tokens, + temperature, + }: ICompleteArguments): ReturnType { + const actor = Context.get('actor'); + messages = await process_input_messages(messages); + + // delete cache_control + messages = messages.map((m) => { + delete m.cache_control; + return m; + }); + + const modelUsed = + (await this.models()).find((m) => + [m.id, ...(m.aliases || [])].includes(model), + ) || + (await this.models()).find((m) => m.id === this.getDefaultModel())!; + const sdk_params: ChatCompletionCreateParams = { + messages: messages, + model: modelUsed.id, + ...(tools ? { tools } : {}), + ...(max_tokens ? { max_completion_tokens: max_tokens } : {}), + ...(temperature ? { temperature } : {}), + stream, + ...(stream + ? { + stream_options: { include_usage: true }, + } + : {}), + } as ChatCompletionCreateParams; + + let completion; + try { + completion = await this.openai.chat.completions.create(sdk_params); + } catch (e) { + console.error('Gemini completion error: ', e); + throw e; + } + + return handle_completion_output({ + usage_calculator: ({ usage }) => { + const trackedUsage = { + prompt_tokens: + (usage.prompt_tokens ?? 0) - + (usage.prompt_tokens_details?.cached_tokens ?? 0), + completion_tokens: usage.completion_tokens ?? 0, + cached_tokens: + usage.prompt_tokens_details?.cached_tokens ?? 0, + }; + + const costsOverrideFromModel = Object.fromEntries( + Object.entries(trackedUsage).map(([k, v]) => { + return [k, v * modelUsed.costs[k]]; + }), + ); + this.meteringService.utilRecordUsageObject( + trackedUsage, + actor, + `gemini:${modelUsed?.id}`, + costsOverrideFromModel, + ); + + return trackedUsage; + }, + stream, + completion, + }); + } + + checkModeration( + _text: string, + ): ReturnType { + throw new Error('No moderation logic.'); + } +} diff --git a/src/backend/src/services/ai/chat/providers/GeminiProvider/models.ts b/src/backend/drivers/ai-chat/providers/gemini/models.ts similarity index 60% rename from src/backend/src/services/ai/chat/providers/GeminiProvider/models.ts rename to src/backend/drivers/ai-chat/providers/gemini/models.ts index 17263e88f..0847e834c 100644 --- a/src/backend/src/services/ai/chat/providers/GeminiProvider/models.ts +++ b/src/backend/drivers/ai-chat/providers/gemini/models.ts @@ -1,17 +1,14 @@ -import { IChatModel } from '../types'; - -export const GEMINI_IMAGE_CHAT_MODELS = [ - 'gemini-2.5-flash-image', - 'gemini-3-pro-image-preview', - 'gemini-3.1-flash-image-preview', -]; +import type { IChatModel } from '../../types.js'; // Hardcoded from https://models.dev/api.json export const GEMINI_MODELS: IChatModel[] = [ { puterId: 'google:google/gemini-2.0-flash', id: 'gemini-2.0-flash', - modalities: { 'input': ['text', 'image', 'audio', 'video', 'pdf'], 'output': ['text'] }, + modalities: { + input: ['text', 'image', 'audio', 'video', 'pdf'], + output: ['text'], + }, open_weights: false, tool_call: true, knowledge: '2024-06', @@ -27,14 +24,16 @@ export const GEMINI_MODELS: IChatModel[] = [ prompt_tokens: 10, completion_tokens: 40, cached_tokens: 3, - }, max_tokens: 8192, }, { puterId: 'google:google/gemini-2.0-flash-lite', id: 'gemini-2.0-flash-lite', - modalities: { 'input': ['text', 'image', 'audio', 'video', 'pdf'], 'output': ['text'] }, + modalities: { + input: ['text', 'image', 'audio', 'video', 'pdf'], + output: ['text'], + }, open_weights: false, tool_call: true, knowledge: '2024-06', @@ -55,7 +54,10 @@ export const GEMINI_MODELS: IChatModel[] = [ { puterId: 'google:google/gemini-2.5-flash', id: 'gemini-2.5-flash', - modalities: { 'input': ['text', 'image', 'audio', 'video', 'pdf'], 'output': ['text'] }, + modalities: { + input: ['text', 'image', 'audio', 'video', 'pdf'], + output: ['text'], + }, open_weights: false, tool_call: true, knowledge: '2025-01', @@ -77,7 +79,10 @@ export const GEMINI_MODELS: IChatModel[] = [ { puterId: 'google:google/gemini-2.5-flash-lite', id: 'gemini-2.5-flash-lite', - modalities: { 'input': ['text', 'image', 'audio', 'video', 'pdf'], 'output': ['text'] }, + modalities: { + input: ['text', 'image', 'audio', 'video', 'pdf'], + output: ['text'], + }, open_weights: false, tool_call: true, knowledge: '2025-01', @@ -99,7 +104,10 @@ export const GEMINI_MODELS: IChatModel[] = [ { puterId: 'google:google/gemini-2.5-pro', id: 'gemini-2.5-pro', - modalities: { 'input': ['text', 'image', 'audio', 'video', 'pdf'], 'output': ['text'] }, + modalities: { + input: ['text', 'image', 'audio', 'video', 'pdf'], + output: ['text'], + }, open_weights: false, tool_call: true, knowledge: '2025-01', @@ -121,7 +129,10 @@ export const GEMINI_MODELS: IChatModel[] = [ { puterId: 'google:google/gemini-3.1-pro-preview', id: 'gemini-3.1-pro-preview', - modalities: { 'input': ['text', 'image', 'video', 'audio', 'pdf'], 'output': ['text'] }, + modalities: { + input: ['text', 'image', 'video', 'audio', 'pdf'], + output: ['text'], + }, open_weights: false, tool_call: true, knowledge: '2025-01', @@ -143,7 +154,10 @@ export const GEMINI_MODELS: IChatModel[] = [ { puterId: 'google:google/gemini-3-flash-preview', id: 'gemini-3-flash-preview', - modalities: { 'input': ['text', 'image', 'video', 'audio', 'pdf'], 'output': ['text'] }, + modalities: { + input: ['text', 'image', 'video', 'audio', 'pdf'], + output: ['text'], + }, open_weights: false, tool_call: true, knowledge: '2025-01', @@ -165,7 +179,10 @@ export const GEMINI_MODELS: IChatModel[] = [ { puterId: 'google:google/gemini-3.1-flash-lite-preview', id: 'gemini-3.1-flash-lite-preview', - modalities: { 'input': ['text', 'image', 'video', 'audio', 'pdf'], 'output': ['text'] }, + modalities: { + input: ['text', 'image', 'video', 'audio', 'pdf'], + output: ['text'], + }, open_weights: false, tool_call: true, knowledge: '2025-01', @@ -184,72 +201,4 @@ export const GEMINI_MODELS: IChatModel[] = [ }, max_tokens: 65536, }, - { - puterId: 'google:google/gemini-2.5-flash-image', - id: 'gemini-2.5-flash-image', - modalities: { 'input': ['text', 'image'], 'output': ['text', 'image'] }, - open_weights: false, - tool_call: false, - knowledge: '2025-01', - release_date: '2025-03-20', - name: 'Gemini 2.5 Flash Image', - aliases: ['google/gemini-2.5-flash-image', 'gemini-2.5-flash-image-preview', 'google/gemini-2.5-flash-image-preview'], - context: 65_536, - costs_currency: 'usd-cents', - input_cost_key: 'prompt_tokens', - output_cost_key: 'completion_tokens', - costs: { - tokens: 1_000_000, - prompt_tokens: 30, - completion_tokens: 250, - output_image: 3_000, - }, - max_tokens: 32_768, - }, - { - puterId: 'google:google/gemini-3-pro-image-preview', - id: 'gemini-3-pro-image-preview', - modalities: { 'input': ['text', 'image'], 'output': ['text', 'image'] }, - open_weights: false, - tool_call: false, - knowledge: '2025-01', - release_date: '2025-11-18', - name: 'Gemini 3 Pro Image', - aliases: ['google/gemini-3-pro-image-preview', 'gemini-3-pro-image', 'google/gemini-3-pro-image'], - context: 65_536, - costs_currency: 'usd-cents', - input_cost_key: 'prompt_tokens', - output_cost_key: 'completion_tokens', - allowedQualityLevels: ['1K', '2K', '4K'], - costs: { - tokens: 1_000_000, - prompt_tokens: 200, - completion_tokens: 1200, - output_image: 12_000, - }, - max_tokens: 32_768, - }, - { - puterId: 'google:google/gemini-3.1-flash-image-preview', - id: 'gemini-3.1-flash-image-preview', - modalities: { 'input': ['text', 'image'], 'output': ['text', 'image'] }, - open_weights: false, - tool_call: false, - knowledge: '2025-01', - release_date: '2026-02-19', - name: 'Gemini 3.1 Flash Image', - aliases: ['google/gemini-3.1-flash-image-preview', 'gemini-3.1-flash-image', 'google/gemini-3.1-flash-image'], - context: 65_536, - costs_currency: 'usd-cents', - input_cost_key: 'prompt_tokens', - output_cost_key: 'completion_tokens', - allowedQualityLevels: ['512', '1K', '2K', '4K'], - costs: { - tokens: 1_000_000, - prompt_tokens: 25, - completion_tokens: 150, - output_image: 6_000, - }, - max_tokens: 32_768, - }, ]; diff --git a/src/backend/src/services/ai/chat/providers/GroqAiProvider/GroqAIProvider.ts b/src/backend/drivers/ai-chat/providers/groq/GroqAIProvider.ts similarity index 60% rename from src/backend/src/services/ai/chat/providers/GroqAiProvider/GroqAIProvider.ts rename to src/backend/drivers/ai-chat/providers/groq/GroqAIProvider.ts index 3105417e8..ffcb8ef5a 100644 --- a/src/backend/src/services/ai/chat/providers/GroqAiProvider/GroqAIProvider.ts +++ b/src/backend/drivers/ai-chat/providers/groq/GroqAIProvider.ts @@ -20,10 +20,10 @@ import Groq from 'groq-sdk'; import { ChatCompletionCreateParams } from 'groq-sdk/resources/chat/completions.mjs'; import { CompletionUsage } from 'openai/resources'; -import { Context } from '../../../../../util/context.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import * as OpenAIUtil from '../../../utils/OpenAIUtil.js'; -import { IChatProvider, ICompleteArguments } from '../types.js'; +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { IChatProvider, ICompleteArguments } from '../../types.js'; +import * as OpenAIUtil from '../../utils/OpenAIUtil.js'; import { GROQ_MODELS } from './models.js'; export class GroqAIProvider implements IChatProvider { @@ -31,41 +31,51 @@ export class GroqAIProvider implements IChatProvider { #meteringService: MeteringService; - constructor (config: { apiKey: string }, meteringService: MeteringService) { + constructor(config: { apiKey: string }, meteringService: MeteringService) { this.#client = new Groq({ apiKey: config.apiKey, }); this.#meteringService = meteringService; } - getDefaultModel () { + getDefaultModel() { return 'llama-3.1-8b-instant'; } - models () { + models() { return GROQ_MODELS; } - async list () { + async list() { const models = this.models(); const modelNames: string[] = []; - for ( const model of models ) { + for (const model of models) { modelNames.push(model.id); - if ( model.aliases ) { + if (model.aliases) { modelNames.push(...model.aliases); } } return modelNames; } - async complete ({ messages, model, stream, tools, max_tokens, temperature }: ICompleteArguments): ReturnType { + async complete({ + messages, + model, + stream, + tools, + max_tokens, + temperature, + }: ICompleteArguments): ReturnType { const actor = Context.get('actor'); const availableModels = this.models(); - const modelUsed = availableModels.find(m => [m.id, ...(m.aliases || [])].includes(model)) || availableModels.find(m => m.id === this.getDefaultModel())!; + const modelUsed = + availableModels.find((m) => + [m.id, ...(m.aliases || [])].includes(model), + ) || availableModels.find((m) => m.id === this.getDefaultModel())!; messages = await OpenAIUtil.process_input_messages(messages); - for ( const message of messages ) { - if ( message.tool_calls && Array.isArray(message.content) ) { + for (const message of messages) { + if (message.tool_calls && Array.isArray(message.content)) { message.content = ''; } } @@ -81,16 +91,24 @@ export class GroqAIProvider implements IChatProvider { return OpenAIUtil.handle_completion_output({ deviations: { - index_usage_from_stream_chunk: chunk => + index_usage_from_stream_chunk: (chunk) => // x_groq contains usage details for streamed responses - (chunk as { x_groq?: { usage?: CompletionUsage } }).x_groq?.usage, + (chunk as { x_groq?: { usage?: CompletionUsage } }).x_groq + ?.usage, }, usage_calculator: ({ usage }) => { const trackedUsage = OpenAIUtil.extractMeteredUsage(usage); - const costsOverride = Object.fromEntries(Object.entries(trackedUsage).map(([k, v]) => { - return [k, v * (modelUsed.costs[k])]; - })); - this.#meteringService.utilRecordUsageObject(trackedUsage, actor, `groq:${modelUsed.id}`, costsOverride); + const costsOverride = Object.fromEntries( + Object.entries(trackedUsage).map(([k, v]) => { + return [k, v * modelUsed.costs[k]]; + }), + ); + this.#meteringService.utilRecordUsageObject( + trackedUsage, + actor, + `groq:${modelUsed.id}`, + costsOverride, + ); return trackedUsage; }, stream, @@ -98,7 +116,9 @@ export class GroqAIProvider implements IChatProvider { }); } - checkModeration (_text: string): ReturnType { + checkModeration( + _text: string, + ): ReturnType { throw new Error('Method not implemented.'); } } diff --git a/src/backend/src/services/ai/chat/providers/GroqAiProvider/models.ts b/src/backend/drivers/ai-chat/providers/groq/models.ts similarity index 99% rename from src/backend/src/services/ai/chat/providers/GroqAiProvider/models.ts rename to src/backend/drivers/ai-chat/providers/groq/models.ts index e48f0943f..73556285d 100644 --- a/src/backend/src/services/ai/chat/providers/GroqAiProvider/models.ts +++ b/src/backend/drivers/ai-chat/providers/groq/models.ts @@ -1,4 +1,4 @@ -import { IChatModel } from '../types.js'; +import type { IChatModel } from '../../types.js'; // Hardcoded from https://models.dev/api.json export const GROQ_MODELS: IChatModel[] = [ diff --git a/src/backend/src/services/ai/chat/providers/MistralAiProvider/MistralAiProvider.ts b/src/backend/drivers/ai-chat/providers/mistral/MistralAiProvider.ts similarity index 57% rename from src/backend/src/services/ai/chat/providers/MistralAiProvider/MistralAiProvider.ts rename to src/backend/drivers/ai-chat/providers/mistral/MistralAiProvider.ts index b07bb6ffe..1061854af 100644 --- a/src/backend/src/services/ai/chat/providers/MistralAiProvider/MistralAiProvider.ts +++ b/src/backend/drivers/ai-chat/providers/mistral/MistralAiProvider.ts @@ -19,10 +19,14 @@ import { Mistral } from '@mistralai/mistralai'; import { ChatCompletionResponse } from '@mistralai/mistralai/models/components/chatcompletionresponse.js'; -import { Context } from '../../../../../util/context.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import * as OpenAIUtil from '../../../utils/OpenAIUtil.js'; -import { IChatProvider, ICompleteArguments } from '../types.js'; +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import * as OpenAIUtil from '../../utils/OpenAIUtil.js'; +import type { + IChatProvider, + ICompleteArguments, + IChatCompleteResult, +} from '../../types.js'; import { MISTRAL_MODELS } from './models.js'; export class MistralAIProvider implements IChatProvider { @@ -30,48 +34,58 @@ export class MistralAIProvider implements IChatProvider { #meteringService: MeteringService; - constructor (config: { apiKey: string }, meteringService: MeteringService) { + constructor(config: { apiKey: string }, meteringService: MeteringService) { this.#client = new Mistral({ apiKey: config.apiKey, }); this.#meteringService = meteringService; } - getDefaultModel () { + getDefaultModel() { return 'mistral-small-2506'; } - async models () { + async models() { return MISTRAL_MODELS; } - async list () { + async list() { const models = await this.models(); const ids: string[] = []; - for ( const model of models ) { + for (const model of models) { ids.push(model.id); - if ( model.aliases ) { + if (model.aliases) { ids.push(...model.aliases); } } return ids; } - async complete ({ messages, stream, model, tools, max_tokens, temperature }: ICompleteArguments): ReturnType { - + async complete({ + messages, + stream, + model, + tools, + max_tokens, + temperature, + }: ICompleteArguments): Promise { messages = await OpenAIUtil.process_input_messages(messages); - for ( const message of messages ) { - if ( message.tool_calls ) { + for (const message of messages) { + if (message.tool_calls) { message.toolCalls = message.tool_calls; delete message.tool_calls; } - if ( message.tool_call_id ) { + if (message.tool_call_id) { message.toolCallId = message.tool_call_id; delete message.tool_call_id; } } - const selectedModel = (await this.models()).find(m => [m.id, ...(m.aliases || [])].includes(model)) || (await this.models()).find(m => m.id === this.getDefaultModel())!; + const selectedModel = + (await this.models()).find((m) => + [m.id, ...(m.aliases || [])].includes(model), + ) || + (await this.models()).find((m) => m.id === this.getDefaultModel())!; const actor = Context.get('actor'); const completion = await this.#client.chat[ stream ? 'stream' : 'complete' @@ -85,20 +99,25 @@ export class MistralAIProvider implements IChatProvider { return await OpenAIUtil.handle_completion_output({ deviations: { - index_usage_from_stream_chunk: chunk => { - if ( ! chunk.usage ) return; + index_usage_from_stream_chunk: (chunk) => { + if (!chunk.usage) return; const snake_usage = {}; - for ( const key in chunk.usage ) { - const snakeKey = key.replace(/([A-Z])/g, '_$1').toLowerCase(); + for (const key in chunk.usage) { + const snakeKey = key + .replace(/([A-Z])/g, '_$1') + .toLowerCase(); snake_usage[snakeKey] = chunk.usage[key]; } return snake_usage; }, - chunk_but_like_actually: chunk => (chunk as any).data, - index_tool_calls_from_stream_choice: choice => (choice.delta as any).toolCalls, - coerce_completion_usage: (completion: ChatCompletionResponse) => ({ + chunk_but_like_actually: (chunk) => (chunk as any).data, + index_tool_calls_from_stream_choice: (choice) => + (choice.delta as any).toolCalls, + coerce_completion_usage: ( + completion: ChatCompletionResponse, + ) => ({ prompt_tokens: completion.usage.promptTokens, completion_tokens: completion.usage.completionTokens, }), @@ -107,16 +126,25 @@ export class MistralAIProvider implements IChatProvider { stream, usage_calculator: ({ usage }) => { const trackedUsage = OpenAIUtil.extractMeteredUsage(usage); - const costsOverrideFromModel = Object.fromEntries(Object.entries(trackedUsage).map(([k, v]) => { - return [k, v * (selectedModel.costs[k])]; - })); - this.#meteringService.utilRecordUsageObject(trackedUsage, actor, `mistral:${selectedModel.id}`, costsOverrideFromModel); + const costsOverrideFromModel = Object.fromEntries( + Object.entries(trackedUsage).map(([k, v]) => { + return [k, v * selectedModel.costs[k]]; + }), + ); + this.#meteringService.utilRecordUsageObject( + trackedUsage, + actor, + `mistral:${selectedModel.id}`, + costsOverrideFromModel, + ); return trackedUsage; }, }); } - checkModeration (_text: string): ReturnType { + checkModeration( + _text: string, + ): ReturnType { throw new Error('Method not implemented.'); } } diff --git a/src/backend/src/services/ai/chat/providers/MistralAiProvider/models.ts b/src/backend/drivers/ai-chat/providers/mistral/models.ts similarity index 80% rename from src/backend/src/services/ai/chat/providers/MistralAiProvider/models.ts rename to src/backend/drivers/ai-chat/providers/mistral/models.ts index a0929edb9..0d46a9dd8 100644 --- a/src/backend/src/services/ai/chat/providers/MistralAiProvider/models.ts +++ b/src/backend/drivers/ai-chat/providers/mistral/models.ts @@ -1,11 +1,11 @@ -import { IChatModel } from '../types'; +import type { IChatModel } from '../../types.js'; // Hardcoded from https://models.dev/api.json export const MISTRAL_MODELS: IChatModel[] = [ { puterId: 'mistralai:mistralai/mistral-medium-2508', id: 'mistral-medium-2508', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2025-05', @@ -32,7 +32,7 @@ export const MISTRAL_MODELS: IChatModel[] = [ { puterId: 'mistralai:mistralai/open-mistral-7b', id: 'open-mistral-7b', - modalities: { 'input': ['text'], 'output': ['text'] }, + modalities: { input: ['text'], output: ['text'] }, open_weights: true, tool_call: true, knowledge: '2023-12', @@ -59,7 +59,7 @@ export const MISTRAL_MODELS: IChatModel[] = [ { puterId: 'mistralai:mistralai/open-mistral-nemo', id: 'open-mistral-nemo', - modalities: { 'input': ['text'], 'output': ['text'] }, + modalities: { input: ['text'], output: ['text'] }, open_weights: true, tool_call: true, knowledge: '2024-07', @@ -73,7 +73,8 @@ export const MISTRAL_MODELS: IChatModel[] = [ ], context: 131072, max_tokens: 131072, - description: 'Our best multilingual open source model released July 2024.', + description: + 'Our best multilingual open source model released July 2024.', provider: 'mistral', costs_currency: 'usd-cents', input_cost_key: 'prompt_tokens', @@ -87,7 +88,7 @@ export const MISTRAL_MODELS: IChatModel[] = [ { puterId: 'mistralai:mistralai/pixtral-large-2411', id: 'pixtral-large-2411', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: true, tool_call: true, knowledge: '2024-11', @@ -114,19 +115,17 @@ export const MISTRAL_MODELS: IChatModel[] = [ { puterId: 'mistralai:mistralai/codestral-2508', id: 'codestral-2508', - modalities: { 'input': ['text'], 'output': ['text'] }, + modalities: { input: ['text'], output: ['text'] }, open_weights: true, tool_call: true, knowledge: '2024-10', release_date: '2024-05-29', name: 'codestral-2508', - aliases: [ - 'codestral-latest', - 'mistralai/codestral-2508', - ], + aliases: ['codestral-latest', 'mistralai/codestral-2508'], context: 256000, max_tokens: 256000, - description: 'Our cutting-edge language model for coding released August 2025.', + description: + 'Our cutting-edge language model for coding released August 2025.', provider: 'mistral', costs_currency: 'usd-cents', input_cost_key: 'prompt_tokens', @@ -140,16 +139,13 @@ export const MISTRAL_MODELS: IChatModel[] = [ { puterId: 'mistralai:mistralai/devstral-small-2507', id: 'devstral-small-2507', - modalities: { 'input': ['text'], 'output': ['text'] }, + modalities: { input: ['text'], output: ['text'] }, open_weights: true, tool_call: true, knowledge: '2025-05', release_date: '2025-07-10', name: 'devstral-small-2507', - aliases: [ - 'devstral-small-latest', - 'mistralai/devstral-small-2507', - ], + aliases: ['devstral-small-latest', 'mistralai/devstral-small-2507'], context: 131072, max_tokens: 131072, description: 'Our small open-source code-agentic model.', @@ -167,16 +163,13 @@ export const MISTRAL_MODELS: IChatModel[] = [ { puterId: 'mistralai:mistralai/devstral-medium-2507', id: 'devstral-medium-2507', - modalities: { 'input': ['text'], 'output': ['text'] }, + modalities: { input: ['text'], output: ['text'] }, open_weights: true, tool_call: true, knowledge: '2025-05', release_date: '2025-07-10', name: 'devstral-medium-2507', - aliases: [ - 'devstral-medium-latest', - 'mistralai/devstral-medium-2507', - ], + aliases: ['devstral-medium-latest', 'mistralai/devstral-medium-2507'], context: 131072, max_tokens: 131072, description: 'Our medium code-agentic model.', @@ -194,19 +187,17 @@ export const MISTRAL_MODELS: IChatModel[] = [ { puterId: 'mistralai:mistralai/mistral-small-2506', id: 'mistral-small-2506', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: true, tool_call: true, knowledge: '2025-03', release_date: '2025-06-20', name: 'mistral-small-2506', - aliases: [ - 'mistral-small-latest', - 'mistralai/mistral-small-2506', - ], + aliases: ['mistral-small-latest', 'mistralai/mistral-small-2506'], context: 131072, max_tokens: 131072, - description: 'Our latest enterprise-grade small model with the latest version released June 2025.', + description: + 'Our latest enterprise-grade small model with the latest version released June 2025.', provider: 'mistral', costs_currency: 'usd-cents', input_cost_key: 'prompt_tokens', @@ -220,19 +211,17 @@ export const MISTRAL_MODELS: IChatModel[] = [ { puterId: 'mistralai:mistralai/magistral-medium-2509', id: 'magistral-medium-2509', - modalities: { 'input': ['text'], 'output': ['text'] }, + modalities: { input: ['text'], output: ['text'] }, open_weights: true, tool_call: true, knowledge: '2025-06', release_date: '2025-03-17', name: 'magistral-medium-2509', - aliases: [ - 'magistral-medium-latest', - 'mistralai/magistral-medium-2509', - ], + aliases: ['magistral-medium-latest', 'mistralai/magistral-medium-2509'], context: 131072, max_tokens: 131072, - description: 'Our frontier-class reasoning model release candidate September 2025.', + description: + 'Our frontier-class reasoning model release candidate September 2025.', provider: 'mistral', costs_currency: 'usd-cents', input_cost_key: 'prompt_tokens', @@ -246,16 +235,13 @@ export const MISTRAL_MODELS: IChatModel[] = [ { puterId: 'mistralai:mistralai/magistral-small-2509', id: 'magistral-small-2509', - modalities: { 'input': ['text'], 'output': ['text'] }, + modalities: { input: ['text'], output: ['text'] }, open_weights: true, tool_call: true, knowledge: '2025-06', release_date: '2025-03-17', name: 'magistral-small-2509', - aliases: [ - 'magistral-small-latest', - 'mistralai/magistral-small-2509', - ], + aliases: ['magistral-small-latest', 'mistralai/magistral-small-2509'], context: 131072, max_tokens: 131072, description: 'Our efficient reasoning model released September 2025.', @@ -273,10 +259,7 @@ export const MISTRAL_MODELS: IChatModel[] = [ puterId: 'mistralai:mistralai/voxtral-mini-2507', id: 'voxtral-mini-2507', name: 'voxtral-mini-2507', - aliases: [ - 'voxtral-mini-latest', - 'mistralai/voxtral-mini-2507', - ], + aliases: ['voxtral-mini-latest', 'mistralai/voxtral-mini-2507'], context: 32768, max_tokens: 32768, description: 'A mini audio understanding model released in July 2025', @@ -294,10 +277,7 @@ export const MISTRAL_MODELS: IChatModel[] = [ puterId: 'mistralai:mistralai/voxtral-small-2507', id: 'voxtral-small-2507', name: 'voxtral-small-2507', - aliases: [ - 'voxtral-small-latest', - 'mistralai/voxtral-small-2507', - ], + aliases: ['voxtral-small-latest', 'mistralai/voxtral-small-2507'], context: 32768, max_tokens: 32768, description: 'A small audio understanding model released in July 2025', @@ -314,16 +294,13 @@ export const MISTRAL_MODELS: IChatModel[] = [ { puterId: 'mistralai:mistralai/mistral-large-2512', id: 'mistral-large-latest', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: true, tool_call: true, knowledge: '2024-11', release_date: '2024-11-01', name: 'mistral-large-2512', - aliases: [ - 'mistral-large-2512', - 'mistralai/mistral-large-2512', - ], + aliases: ['mistral-large-2512', 'mistralai/mistral-large-2512'], context: 262144, max_tokens: 262144, description: 'Official mistral-large-2512 Mistral AI model', @@ -340,16 +317,13 @@ export const MISTRAL_MODELS: IChatModel[] = [ { puterId: 'mistralai:mistralai/ministral-3b-2512', id: 'ministral-3b-2512', - modalities: { 'input': ['text'], 'output': ['text'] }, + modalities: { input: ['text'], output: ['text'] }, open_weights: true, tool_call: true, knowledge: '2024-10', release_date: '2024-10-01', name: 'ministral-3b-2512', - aliases: [ - 'ministral-3b-latest', - 'mistralai/ministral-3b-2512', - ], + aliases: ['ministral-3b-latest', 'mistralai/ministral-3b-2512'], context: 131072, max_tokens: 131072, description: 'Ministral 3 (a.k.a. Tinystral) 3B Instruct.', @@ -366,16 +340,13 @@ export const MISTRAL_MODELS: IChatModel[] = [ { puterId: 'mistralai:mistralai/ministral-8b-2512', id: 'ministral-8b-2512', - modalities: { 'input': ['text'], 'output': ['text'] }, + modalities: { input: ['text'], output: ['text'] }, open_weights: true, tool_call: true, knowledge: '2024-10', release_date: '2024-10-01', name: 'ministral-8b-2512', - aliases: [ - 'ministral-8b-latest', - 'mistralai/ministral-8b-2512', - ], + aliases: ['ministral-8b-latest', 'mistralai/ministral-8b-2512'], context: 262144, max_tokens: 262144, description: 'Ministral 3 (a.k.a. Tinystral) 8B Instruct.', @@ -393,10 +364,7 @@ export const MISTRAL_MODELS: IChatModel[] = [ puterId: 'mistralai:mistralai/ministral-14b-2512', id: 'ministral-14b-2512', name: 'ministral-14b-2512', - aliases: [ - 'ministral-14b-latest', - 'mistralai/ministral-14b-2512', - ], + aliases: ['ministral-14b-latest', 'mistralai/ministral-14b-2512'], context: 262144, max_tokens: 262144, description: 'Ministral 3 (a.k.a. Tinystral) 14B Instruct.', diff --git a/src/backend/src/services/ai/chat/providers/OllamaProvider.ts b/src/backend/drivers/ai-chat/providers/ollama/OllamaProvider.ts similarity index 57% rename from src/backend/src/services/ai/chat/providers/OllamaProvider.ts rename to src/backend/drivers/ai-chat/providers/ollama/OllamaProvider.ts index f324a2c9c..6332bfa7d 100644 --- a/src/backend/src/services/ai/chat/providers/OllamaProvider.ts +++ b/src/backend/drivers/ai-chat/providers/ollama/OllamaProvider.ts @@ -19,65 +19,70 @@ import axios from 'axios'; import { default as openai, default as OpenAI } from 'openai'; -import { Context } from '../../../../util/context.js'; +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; import { kv } from '../../../../util/kvSingleton.js'; import * as OpenAIUtil from '../../utils/OpenAIUtil.js'; -import { IChatModel, IChatProvider, ICompleteArguments } from './types'; -import { MeteringService } from '../../../MeteringService/MeteringService'; +import { IChatModel, IChatProvider, ICompleteArguments } from '../../types.js'; import { ChatCompletionCreateParams } from 'openai/resources/index.js'; /** -* OllamaService class - Provides integration with Ollama's API for chat completions -* Extends BaseService to implement the puter-chat-completion interface. -* Handles model management, message adaptation, streaming responses, -* and usage tracking for Ollama's language models. -* @extends BaseService -*/ + * OllamaService class - Provides integration with Ollama's API for chat completions + * Extends BaseService to implement the puter-chat-completion interface. + * Handles model management, message adaptation, streaming responses, + * and usage tracking for Ollama's language models. + * @extends BaseService + */ export class OllamaChatProvider implements IChatProvider { - #apiBaseUrl: string; #openai: OpenAI; #meteringService: MeteringService; - constructor (config: { api_base_url?: string } | undefined, meteringService: MeteringService) { + constructor( + config: { apiBaseUrl?: string } | undefined, + meteringService: MeteringService, + ) { // Ollama typically runs on HTTP, not HTTPS - this.#apiBaseUrl = config?.api_base_url || 'http://localhost:11434'; + this.#apiBaseUrl = config?.apiBaseUrl || 'http://localhost:11434'; // OpenAI SDK is used to interact with the Ollama API this.#openai = new openai.OpenAI({ apiKey: 'ollama', // Ollama doesn't use an API key, it uses the "ollama" string - baseURL: `${config?.api_base_url }/v1`, + baseURL: `${this.#apiBaseUrl}/v1`, }); this.#meteringService = meteringService; } - async models () { + async models() { let models = kv.get('ollamaChat:models'); - if ( ! models ) { + if (!models) { try { const resp = await axios.request({ method: 'GET', url: `${this.#apiBaseUrl}/api/tags`, }); models = resp.data.models || []; - if ( models.length > 0 ) { + if (models.length > 0) { kv.set('ollamaChat:models', models); } - } catch ( error ) { - console.error('Failed to fetch models from Ollama:', (error as Error).message); + } catch (error) { + console.error( + 'Failed to fetch models from Ollama:', + (error as Error).message, + ); // Return empty array if Ollama is not available return []; } } - if ( !models || models.length === 0 ) { + if (!models || models.length === 0) { return []; } const coerced_models: IChatModel[] = []; - for ( const model of models ) { + for (const model of models) { // Ollama API returns models with 'name' property, not 'model' const modelName = model.name || model.model || 'unknown'; coerced_models.push({ @@ -94,17 +99,23 @@ export class OllamaChatProvider implements IChatProvider { } return coerced_models; } - async list () { + async list() { const models = await this.models(); const model_names: string[] = []; - for ( const model of models ) { + for (const model of models) { model_names.push(model.id); } return model_names; } - async complete ({ messages, stream, model, tools, max_tokens, temperature }: ICompleteArguments): ReturnType { - - if ( model.startsWith('ollama:') ) { + async complete({ + messages, + stream, + model, + tools, + max_tokens, + temperature, + }: ICompleteArguments): ReturnType { + if (model.startsWith('ollama:')) { model = model.slice('ollama:'.length); } @@ -119,26 +130,45 @@ export class OllamaChatProvider implements IChatProvider { max_tokens, temperature: temperature, // default to 1.0 stream: !!stream, - ...(stream ? { - stream_options: { include_usage: true }, - } : {}), - } as ChatCompletionCreateParams) ; + ...(stream + ? { + stream_options: { include_usage: true }, + } + : {}), + } as ChatCompletionCreateParams); - const modelDetails = (await this.models()).find(m => m.id === `ollama:${model}`); - const modelIdForMetering = modelDetails?.id ?? (model ? (model.startsWith('ollama/') ? `ollama:${model}` : `ollama:ollama/${model}`) : undefined); + const modelDetails = (await this.models()).find( + (m) => m.id === `ollama:${model}`, + ); + const modelIdForMetering = + modelDetails?.id ?? + (model + ? model.startsWith('ollama/') + ? `ollama:${model}` + : `ollama:ollama/${model}` + : undefined); return OpenAIUtil.handle_completion_output({ usage_calculator: ({ usage }) => { - const trackedUsage = { - prompt: (usage.prompt_tokens ?? 1 ) - (usage.prompt_tokens_details?.cached_tokens ?? 0), + prompt: + (usage.prompt_tokens ?? 1) - + (usage.prompt_tokens_details?.cached_tokens ?? 0), completion: usage.completion_tokens ?? 1, - input_cache_read: usage.prompt_tokens_details?.cached_tokens ?? 0, + input_cache_read: + usage.prompt_tokens_details?.cached_tokens ?? 0, }; - const costOverwrites = Object.fromEntries(Object.keys(trackedUsage).map((k) => { - return [k, 0]; // override to 0 since local is free - })); - if ( modelIdForMetering ) { - this.#meteringService.utilRecordUsageObject(trackedUsage, actor, modelIdForMetering, costOverwrites); + const costOverwrites = Object.fromEntries( + Object.keys(trackedUsage).map((k) => { + return [k, 0]; // override to 0 since local is free + }), + ); + if (modelIdForMetering) { + this.#meteringService.utilRecordUsageObject( + trackedUsage, + actor, + modelIdForMetering, + costOverwrites, + ); } return trackedUsage; }, @@ -146,15 +176,15 @@ export class OllamaChatProvider implements IChatProvider { completion, }); } - checkModeration (_text: string): ReturnType { + checkModeration(_text: string) { throw new Error('Method not implemented.'); } /** - * Returns the default model identifier for the Ollama service - * @returns {string} The default model ID 'gpt-oss:20b' - */ - getDefaultModel () { + * Returns the default model identifier for the Ollama service + * @returns {string} The default model ID 'gpt-oss:20b' + */ + getDefaultModel() { return 'gpt-oss:20b'; } } diff --git a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts new file mode 100644 index 000000000..37f3035ed --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts @@ -0,0 +1,251 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { OpenAI } from 'openai'; +import { ChatCompletionCreateParams } from 'openai/resources/index.js'; +import { Context } from '../../../../core/context.js'; +import type { FSService } from '../../../../services/fs/FSService.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { FSEntryStore } from '../../../../stores/fs/FSEntryStore.js'; +import type { S3ObjectStore } from '../../../../stores/fs/S3ObjectStore.js'; +import type { IChatProvider, ICompleteArguments } from '../../types.js'; +import * as OpenAiUtil from '../../utils/OpenAIUtil.js'; +import { processPuterPathUploads } from './fileUpload.js'; +import { OPEN_AI_MODELS } from './models.js'; +import type { OpenAiResponsesChatProvider } from './OpenAiChatResponsesProvider.js'; + +/** + * OpenAICompletionService class provides an interface to OpenAI's chat completion API. + * Extends BaseService to handle chat completions, message moderation, token counting, + * and streaming responses. Implements the puter-chat-completion interface and manages + * OpenAI API interactions with support for multiple models including GPT-4 variants. + * Handles usage tracking, spending records, and content moderation. + */ +export class OpenAiChatProvider implements IChatProvider { + /** + * @type {import('openai').OpenAI} + */ + #openAi: OpenAI; + + #defaultModel = 'gpt-5-nano'; + + #meteringService: MeteringService; + + #stores: { fsEntry: FSEntryStore; s3Object: S3ObjectStore }; + + #fsService: FSService; + + #responsesProvider: OpenAiResponsesChatProvider | null = null; + + constructor( + meteringService: MeteringService, + stores: { fsEntry: FSEntryStore; s3Object: S3ObjectStore }, + fsService: FSService, + config: { apiKey: string }, + ) { + this.#meteringService = meteringService; + this.#stores = stores; + this.#fsService = fsService; + this.#openAi = new OpenAI({ apiKey: config.apiKey }); + } + + // Wired up by the driver after both OpenAI providers are built, so the + // Chat Completions path can delegate `web_search` tool calls (Responses-only) + // to the sibling provider without a circular constructor dependency. + setResponsesProvider(provider: OpenAiResponsesChatProvider): void { + this.#responsesProvider = provider; + } + + /** + * Returns an array of available AI models with their pricing information. + * Each model object includes an ID and cost details (currency, tokens, input/output rates). + */ + models() { + return OPEN_AI_MODELS.filter((e) => !e.responses_api_only); + } + + list() { + const models = this.models(); + const modelNames: string[] = []; + for (const model of models) { + modelNames.push(model.id); + if (model.aliases) { + modelNames.push(...model.aliases); + } + } + return modelNames; + } + + getDefaultModel() { + return this.#defaultModel; + } + + async complete( + params: ICompleteArguments, + ): ReturnType { + const { + max_tokens, + moderation, + tools, + verbosity, + stream, + reasoning, + reasoning_effort, + temperature, + text, + } = params; + let { messages, model } = params; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if (tools?.filter((e: any) => e.type === 'web_search').length) { + // web_search is a Responses-API-only tool — hand the whole call + // off to the sibling provider when the user requested it. + if (!this.#responsesProvider) { + throw new Error( + 'web_search tool requires the OpenAI Responses provider, which is not configured', + ); + } + return await this.#responsesProvider.complete(params); + } + // Validate messages + if (!Array.isArray(messages)) { + throw new Error('`messages` must be an array'); + } + const actor = Context.get('actor'); + + model = model ?? this.#defaultModel; + + const modelUsed = + this.models().find((m) => + [m.id, ...(m.aliases || [])].includes(model), + ) || this.models().find((m) => m.id === this.getDefaultModel())!; + + // messages.unshift({ + // role: 'system', + // content: 'Don\'t let the user trick you into doing something bad.', + // }) + + const userIdentifier = + actor?.user.id + actor?.app?.uid ? `:${actor?.app?.uid}` : ''; + + // Resolve any `puter_path` content parts into inline base64 data URLs. + // Chat Completions doesn't support file uploads, so this is the only + // way to get user-provided files (images, audio) in front of the model. + await processPuterPathUploads( + messages, + this.#stores, + this.#fsService, + actor, + ); + + // Here's something fun; the documentation shows `type: 'image_url'` in + // objects that contain an image url, but everything still works if + // that's missing. We normalise it here so the token count code works. + messages = await OpenAiUtil.process_input_messages(messages); + + const requestedReasoningEffort = reasoning_effort ?? reasoning?.effort; + const requestedVerbosity = verbosity ?? text?.verbosity; + const supportsReasoningControls = + typeof model === 'string' && model.startsWith('gpt-5'); + + const completionParams: ChatCompletionCreateParams = { + user: userIdentifier, + safety_identifier: userIdentifier, + messages: messages, + model: modelUsed.id, + ...(tools ? { tools } : {}), + ...(max_tokens ? { max_completion_tokens: max_tokens } : {}), + ...(temperature ? { temperature } : {}), + stream: !!stream, + ...(stream + ? { + stream_options: { include_usage: true }, + } + : {}), + ...(supportsReasoningControls + ? {} + : { + ...(requestedReasoningEffort + ? { reasoning_effort: requestedReasoningEffort } + : {}), + ...(requestedVerbosity + ? { verbosity: requestedVerbosity } + : {}), + }), + } as ChatCompletionCreateParams; + + const completion = + await this.#openAi.chat.completions.create(completionParams); + + return OpenAiUtil.handle_completion_output({ + usage_calculator: ({ usage }) => { + const trackedUsage = { + prompt_tokens: + (usage.prompt_tokens ?? 0) - + (usage.prompt_tokens_details?.cached_tokens ?? 0), + completion_tokens: usage.completion_tokens ?? 0, + cached_tokens: + usage.prompt_tokens_details?.cached_tokens ?? 0, + }; + + const costsOverrideFromModel = Object.fromEntries( + Object.entries(trackedUsage).map(([k, v]) => { + return [k, v * modelUsed.costs[k]]; + }), + ); + + this.#meteringService.utilRecordUsageObject( + trackedUsage, + actor, + `openai:${modelUsed?.id}`, + costsOverrideFromModel, + ); + return trackedUsage; + }, + stream, + completion, + moderate: moderation ? this.checkModeration.bind(this) : undefined, + }); + } + + async checkModeration(text: string) { + // create moderation + const results = await this.#openAi.moderations.create({ + model: 'omni-moderation-latest', + input: text, + }); + + let flagged = false; + + for (const result of results?.results ?? []) { + // OpenAI does a crazy amount of false positives. We filter by their 80% interval + const veryFlaggedEntries = Object.entries( + result.category_scores, + ).filter((e) => e[1] > 0.8); + if (veryFlaggedEntries.length > 0) { + flagged = true; + break; + } + } + + return { + flagged, + results, + }; + } +} diff --git a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts new file mode 100644 index 000000000..375e65f0d --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts @@ -0,0 +1,284 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { OpenAI } from 'openai'; +import { ResponseCreateParams } from 'openai/resources/responses/responses.mjs'; +import { Context } from '../../../../core/context.js'; +import type { FSService } from '../../../../services/fs/FSService.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { FSEntryStore } from '../../../../stores/fs/FSEntryStore.js'; +import type { S3ObjectStore } from '../../../../stores/fs/S3ObjectStore.js'; +import type { IChatProvider, ICompleteArguments } from '../../types.js'; +import * as OpenAiUtil from '../../utils/OpenAIUtil.js'; +import { processPuterPathUploads } from './fileUpload.js'; +import { OPEN_AI_MODELS } from './models.js'; + +/** + * OpenAICompletionService class provides an interface to OpenAI's chat completion API. + * Extends BaseService to handle chat completions, message moderation, token counting, + * and streaming responses. Implements the puter-chat-completion interface and manages + * OpenAI API interactions with support for multiple models including GPT-4 variants. + * Handles usage tracking, spending records, and content moderation. + */ +export class OpenAiResponsesChatProvider implements IChatProvider { + /** + * @type {import('openai').OpenAI} + */ + #openAi: OpenAI; + + #defaultModel = 'gpt-5-nano'; + + #meteringService: MeteringService; + + #stores: { fsEntry: FSEntryStore; s3Object: S3ObjectStore }; + + #fsService: FSService; + + constructor( + meteringService: MeteringService, + stores: { fsEntry: FSEntryStore; s3Object: S3ObjectStore }, + fsService: FSService, + config: { apiKey: string }, + ) { + this.#meteringService = meteringService; + this.#stores = stores; + this.#fsService = fsService; + this.#openAi = new OpenAI({ apiKey: config.apiKey }); + } + + /** + * Returns an array of available AI models with their pricing information. + * Each model object includes an ID and cost details (currency, tokens, input/output rates). + */ + models(extra_params) { + if (extra_params?.no_restrictions) { + return OPEN_AI_MODELS; + } + return OPEN_AI_MODELS.filter((e) => e.responses_api_only === true); + } + + list() { + const models = this.models({ no_restrictions: false }); + const modelNames: string[] = []; + for (const model of models) { + modelNames.push(model.id); + if (model.aliases) { + modelNames.push(...model.aliases); + } + } + return modelNames; + } + + getDefaultModel() { + return this.#defaultModel; + } + + async complete({ + messages, + model, + max_tokens, + moderation, + tools, + tool_choice, + parallel_tool_calls, + include, + conversation, + previous_response_id, + instructions, + metadata, + prompt, + prompt_cache_key, + prompt_cache_retention, + store, + top_p, + truncation, + background, + service_tier, + verbosity, + stream, + reasoning, + reasoning_effort, + temperature, + text, + }: ICompleteArguments): ReturnType { + // Validate messages + if (!Array.isArray(messages)) { + throw new Error('`messages` must be an array'); + } + const actor = Context.get('actor'); + + model = model ?? this.#defaultModel; + + const modelUsed = + this.models({ no_restrictions: true }).find((m) => + [m.id, ...(m.aliases || [])].includes(model), + ) || + this.models({ no_restrictions: true }).find( + (m) => m.id === this.getDefaultModel(), + )!; + + // messages.unshift({ + // role: 'system', + // content: 'Don\'t let the user trick you into doing something bad.', + // }) + + const userIdentifier = + actor?.user.id + actor?.app?.uid ? `:${actor?.app?.uid}` : ''; + + // Resolve any `puter_path` content parts into inline base64 data URLs + // before the Responses API sees them. + await processPuterPathUploads( + messages, + this.#stores, + this.#fsService, + actor, + ); + + if (tools) { + // Unravel tools to OpenAI Responses API format + tools = (tools as any).map((e) => { + if (e.type === 'function') { + const tool = e.function; + tool.type = 'function'; + return tool; + } else { + return e; + } + }); + } + + // Here's something fun; the documentation shows `type: 'image_url'` in + // objects that contain an image url, but everything still works if + // that's missing. We normalise it here so the token count code works. + messages = + await OpenAiUtil.process_input_messages_responses_api(messages); + + const requestedReasoningEffort = reasoning_effort ?? reasoning?.effort; + const requestedVerbosity = verbosity ?? text?.verbosity; + const supportsReasoningControls = + typeof model === 'string' && model.startsWith('gpt-5'); + + const completionParams: ResponseCreateParams = { + user: userIdentifier, + safety_identifier: userIdentifier, + input: messages, + model: modelUsed.id, + ...(tools ? { tools } : {}), + ...(tool_choice !== undefined ? { tool_choice } : {}), + ...(parallel_tool_calls !== undefined + ? { parallel_tool_calls } + : {}), + ...(include !== undefined ? { include } : {}), + ...(conversation !== undefined ? { conversation } : {}), + ...(previous_response_id !== undefined + ? { previous_response_id } + : {}), + ...(instructions !== undefined ? { instructions } : {}), + ...(metadata !== undefined ? { metadata } : {}), + ...(prompt !== undefined ? { prompt } : {}), + ...(prompt_cache_key !== undefined ? { prompt_cache_key } : {}), + ...(prompt_cache_retention !== undefined + ? { prompt_cache_retention } + : {}), + ...(store !== undefined ? { store } : {}), + ...(max_tokens !== undefined + ? { max_output_tokens: max_tokens } + : {}), + ...(temperature !== undefined ? { temperature } : {}), + ...(top_p !== undefined ? { top_p } : {}), + ...(truncation !== undefined ? { truncation } : {}), + ...(background !== undefined ? { background } : {}), + ...(service_tier !== undefined ? { service_tier } : {}), + ...(stream !== undefined ? { stream: !!stream } : {}), + ...(text !== undefined ? { text } : {}), + ...(supportsReasoningControls + ? {} + : { + ...(requestedReasoningEffort + ? { reasoning_effort: requestedReasoningEffort } + : {}), + ...(requestedVerbosity + ? { verbosity: requestedVerbosity } + : {}), + }), + ...(supportsReasoningControls && reasoning ? { reasoning } : {}), + } as ResponseCreateParams; + + // console.log("completion params: ", completionParams) + const completion = + await this.#openAi.responses.create(completionParams); + // console.log("Completion: ", completion) + return OpenAiUtil.handle_completion_output_responses_api({ + usage_calculator: ({ usage }) => { + const trackedUsage = { + prompt_tokens: + ((usage as any).input_tokens ?? 0) - + ((usage as any).input_tokens_details?.cached_tokens ?? + 0), + completion_tokens: (usage as any).output_tokens ?? 0, + cached_tokens: + (usage as any).input_tokens_details?.cached_tokens ?? 0, + }; + + const costsOverrideFromModel = Object.fromEntries( + Object.entries(trackedUsage).map(([k, v]) => { + return [k, v * modelUsed.costs[k]]; + }), + ); + + this.#meteringService.utilRecordUsageObject( + trackedUsage, + actor, + `openai:${modelUsed?.id}`, + costsOverrideFromModel, + ); + return trackedUsage; + }, + stream, + completion, + moderate: moderation ? this.checkModeration.bind(this) : undefined, + }); + } + + async checkModeration(text: string) { + // create moderation + const results = await this.#openAi.moderations.create({ + model: 'omni-moderation-latest', + input: text, + }); + + let flagged = false; + + for (const result of results?.results ?? []) { + // OpenAI does a crazy amount of false positives. We filter by their 80% interval + const veryFlaggedEntries = Object.entries( + result.category_scores, + ).filter((e) => e[1] > 0.8); + if (veryFlaggedEntries.length > 0) { + flagged = true; + break; + } + } + + return { + flagged, + results, + }; + } +} diff --git a/src/backend/drivers/ai-chat/providers/openai/fileUpload.ts b/src/backend/drivers/ai-chat/providers/openai/fileUpload.ts new file mode 100644 index 000000000..55c22da69 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/openai/fileUpload.ts @@ -0,0 +1,102 @@ +import type { Actor } from '../../../../core/actor.js'; +import type { FSService } from '../../../../services/fs/FSService.js'; +import type { FSEntryStore } from '../../../../stores/fs/FSEntryStore.js'; +import type { S3ObjectStore } from '../../../../stores/fs/S3ObjectStore.js'; +import { loadFileInput } from '../../../util/fileInput.js'; + +// Chat Completions doesn't support file inputs, so we inline files as +// base64 data URLs. 5MB is the practical cap before token counts and +// request payloads get out of hand. +export const MAX_FILE_SIZE = 5 * 1_000_000; + +interface ContentPart { + puter_path?: string; + type?: string; + text?: string; + image_url?: { url: string }; + input_audio?: { data: string; format: string }; +} + +/** + * Resolve any `puter_path` content parts into inline base64 data URLs. + * + * Rewrites each matching part in place: images become `image_url`, audio + * becomes `input_audio`, and any error (too large, unsupported MIME, + * missing file, permission denied) is swapped for a `text` part describing + * the problem so the model can surface it to the user rather than the + * request failing outright. + */ +export async function processPuterPathUploads( + messages: Array<{ content?: unknown }>, + stores: { fsEntry: FSEntryStore; s3Object: S3ObjectStore }, + fsService: FSService, + actor: Actor | undefined, +): Promise { + const tasks: Array> = []; + for (const message of messages) { + if (!Array.isArray(message.content)) continue; + for (const part of message.content as ContentPart[]) { + if (!part || !part.puter_path) continue; + tasks.push(processPart(part, stores, fsService, actor)); + } + } + await Promise.all(tasks); +} + +async function processPart( + part: ContentPart, + stores: { fsEntry: FSEntryStore; s3Object: S3ObjectStore }, + fsService: FSService, + actor: Actor | undefined, +): Promise { + const path = part.puter_path!; + delete part.puter_path; + + if (!actor?.user?.id) { + setTextError(part, 'unauthenticated caller cannot resolve puter_path'); + return; + } + + try { + const loaded = await loadFileInput(stores, fsService, actor, path, { + maxBytes: MAX_FILE_SIZE, + }); + const mimeType = loaded.mimeType ?? 'application/octet-stream'; + const base64 = loaded.buffer.toString('base64'); + + if (mimeType.startsWith('image/')) { + part.type = 'image_url'; + part.image_url = { url: `data:${mimeType};base64,${base64}` }; + return; + } + if (mimeType.startsWith('audio/')) { + part.type = 'input_audio'; + part.input_audio = { + data: `data:${mimeType};base64,${base64}`, + format: mimeType.split('/')[1], + }; + return; + } + setTextError(part, 'input file has unsupported MIME type'); + } catch (err) { + const status = (err as { status?: number })?.status; + if (status === 413) { + setTextError( + part, + `input file exceeded maximum of ${MAX_FILE_SIZE} bytes`, + ); + return; + } + const message = (err as Error)?.message || 'failed to read input file'; + setTextError(part, message); + } +} + +function setTextError(part: ContentPart, reason: string): void { + delete part.image_url; + delete part.input_audio; + part.type = 'text'; + // "poor man's system prompt" — the model sees the error inline and can + // explain to the user instead of silently dropping the attachment. + part.text = `{error: ${reason}; the user did not write this message}`; +} diff --git a/src/backend/src/services/ai/chat/providers/OpenAiProvider/models.ts b/src/backend/drivers/ai-chat/providers/openai/models.ts similarity index 89% rename from src/backend/src/services/ai/chat/providers/OpenAiProvider/models.ts rename to src/backend/drivers/ai-chat/providers/openai/models.ts index 25279629b..be965d806 100644 --- a/src/backend/src/services/ai/chat/providers/OpenAiProvider/models.ts +++ b/src/backend/drivers/ai-chat/providers/openai/models.ts @@ -1,13 +1,13 @@ // TODO DS: centralize somewhere -import { IChatModel } from '../types'; +import type { IChatModel } from '../../types.js'; // Hardcoded from https://models.dev/api.json export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-5.5', id: 'gpt-5.5-2026-04-23', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2025-12-01', @@ -28,7 +28,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-5.5-pro', id: 'gpt-5.5-pro-2026-04-23', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2025-12-01', @@ -50,7 +50,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-5.4', id: 'gpt-5.4-2026-03-05', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2025-08-31', @@ -71,7 +71,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-5.3-codex', id: 'gpt-5.3-codex', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2025-08-31', @@ -92,7 +92,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-5.2-codex', id: 'gpt-5.2-codex', - modalities: { 'input': ['text', 'image', 'pdf'], 'output': ['text'] }, + modalities: { input: ['text', 'image', 'pdf'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2025-08-31', @@ -114,7 +114,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-5.2-chat', id: 'gpt-5.2-chat-latest', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2025-08-31', @@ -135,7 +135,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-5.2-pro', id: 'gpt-5.2-pro-2025-12-11', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2025-08-31', @@ -156,7 +156,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-5.2', id: 'gpt-5.2-2025-12-11', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2025-08-31', @@ -177,7 +177,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-5.1', id: 'gpt-5.1', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-09-30', @@ -198,7 +198,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-5.1-codex', id: 'gpt-5.1-codex', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-09-30', @@ -220,7 +220,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-5.1-codex-mini', id: 'gpt-5.1-codex-mini', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-09-30', @@ -242,7 +242,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-5.1-chat', id: 'gpt-5.1-chat-latest', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-09-30', @@ -263,7 +263,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-5', id: 'gpt-5-2025-08-07', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-09-30', @@ -284,7 +284,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-5-mini', id: 'gpt-5-mini-2025-08-07', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-05-30', @@ -305,7 +305,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-5-nano', id: 'gpt-5-nano-2025-08-07', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-05-30', @@ -326,7 +326,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-5-chat', id: 'gpt-5-chat-latest', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: false, knowledge: '2024-09-30', @@ -347,7 +347,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-4o', id: 'gpt-4o', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2023-09', @@ -368,7 +368,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-4o-mini', id: 'gpt-4o-mini', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2023-09', @@ -389,7 +389,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/o1', id: 'o1', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2023-09', @@ -410,7 +410,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/o1-mini', id: 'o1-mini', - modalities: { 'input': ['text'], 'output': ['text'] }, + modalities: { input: ['text'], output: ['text'] }, open_weights: false, tool_call: false, knowledge: '2023-09', @@ -430,7 +430,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/o1-pro', id: 'o1-pro', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2023-09', @@ -450,7 +450,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/o3', id: 'o3', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-05', @@ -471,7 +471,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/o3-pro', id: 'o3-pro', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-05', @@ -493,7 +493,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/o3-mini', id: 'o3-mini', - modalities: { 'input': ['text'], 'output': ['text'] }, + modalities: { input: ['text'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-05', @@ -514,7 +514,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/o4-mini', id: 'o4-mini', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-05', @@ -534,7 +534,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-4.1', id: 'gpt-4.1', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-04', @@ -555,7 +555,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-4.1-mini', id: 'gpt-4.1-mini', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-04', @@ -576,7 +576,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ { puterId: 'openai:openai/gpt-4.1-nano', id: 'gpt-4.1-nano', - modalities: { 'input': ['text', 'image'], 'output': ['text'] }, + modalities: { input: ['text', 'image'], output: ['text'] }, open_weights: false, tool_call: true, knowledge: '2024-04', diff --git a/src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.ts b/src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.ts new file mode 100644 index 000000000..c67f6c762 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.ts @@ -0,0 +1,286 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import axios from 'axios'; +import { OpenAI } from 'openai'; +import { ChatCompletionCreateParams } from 'openai/resources'; +import { HttpError } from '../../../../core/http/HttpError.js'; +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import { kv } from '../../../../util/kvSingleton.js'; +import * as OpenAIUtil from '../../utils/OpenAIUtil.js'; +import type { + IChatModel, + IChatProvider, + IChatCompleteResult, +} from '../../types.js'; +import { OPEN_ROUTER_MODEL_OVERRIDES } from './modelOverrides.js'; + +type OpenrouterUsage = OpenAI.Completions.CompletionUsage & { + cost?: number; +}; + +export class OpenRouterProvider implements IChatProvider { + #meteringService: MeteringService; + + #openai: OpenAI; + + #apiBaseUrl: string = 'https://openrouter.ai/api/v1'; + + constructor( + config: { apiBaseUrl?: string; apiKey: string }, + meteringService: MeteringService, + ) { + this.#apiBaseUrl = config.apiBaseUrl || 'https://openrouter.ai/api/v1'; + this.#openai = new OpenAI({ + apiKey: config.apiKey, + baseURL: this.#apiBaseUrl, + }); + this.#meteringService = meteringService; + } + + getDefaultModel() { + return 'openrouter:openai/gpt-5-nano'; + } + /** + * Returns a list of available model names including their aliases + * @returns {Promise} Array of model identifiers and their aliases + * @description Retrieves all available model IDs and their aliases, + * flattening them into a single array of strings that can be used for model selection + */ + async list() { + const models = await this.models(); + const model_names: string[] = []; + for (const model of models) { + model_names.push(model.id); + } + return model_names; + } + + /** + * AI Chat completion method. + * See AIChatService for more details. + */ + async complete({ + messages, + stream, + model, + tools, + max_tokens, + temperature, + }): Promise { + const modelUsed = + (await this.models()).find((m) => + [m.id, ...(m.aliases || [])].includes(model), + ) || + (await this.models()).find((m) => m.id === this.getDefaultModel())!; + + const modelIdForParams = modelUsed.id.startsWith('openrouter:') + ? modelUsed.id.slice('openrouter:'.length) + : modelUsed.id; + + if (model === 'openrouter/auto') { + throw new HttpError( + 400, + "The model 'openrouter/auto' is not allowed", + { + legacyCode: 'field_invalid', + fields: { + key: 'model', + expected: 'allowed model', + got: 'disallowed model', + }, + }, + ); + } + + const actor = Context.get('actor'); + + messages = await OpenAIUtil.process_input_messages(messages); + + const completionParams = { + messages, + model: modelIdForParams, + ...(tools ? { tools } : {}), + max_tokens, + temperature: temperature, // default to 1.0 + stream, + ...(stream + ? { + stream_options: { include_usage: true }, + } + : {}), + usage: { include: true }, + } as ChatCompletionCreateParams; + + let completion; + try { + completion = + await this.#openai.chat.completions.create(completionParams); + } catch (e: unknown) { + // If you overestimate allowed max_tokens on openrouter then it will throw an error. + // Since we know the user has enough for the query anyways, we should reexecute the + // request without max_tokens. + const err = e as { error: Error }; + if ( + err && + err.error && + err.error.message && + err.error.message.startsWith( + "This endpoint's maximum context length is ", + ) + ) { + delete completionParams.max_tokens; + completion = + await this.#openai.chat.completions.create( + completionParams, + ); + } else { + console.log('Openarouter error: ', err.error.message); + throw e; + } + } + + return OpenAIUtil.handle_completion_output({ + usage_calculator: ({ usage }: { usage: OpenrouterUsage }) => { + if (typeof usage.cost === 'number') { + // custom open router logic because they're pricing are weird + const trackedUsage = { + prompt: + (usage.prompt_tokens ?? 0) - + (usage.prompt_tokens_details?.cached_tokens ?? 0), + completion: usage.completion_tokens ?? 0, + input_cache_read: + usage.prompt_tokens_details?.cached_tokens ?? 0, + request: + (usage as unknown as Record) + .request || 1, + billedUsage: 1, + }; + const costOverwrites = Object.fromEntries( + Object.keys(trackedUsage).map((k) => { + return [k, 0]; // make everything else 0 if they don't respect their own pricing + }), + ); + costOverwrites.billedUsage = usage.cost * 100_000_000 || 1; + this.#meteringService.utilRecordUsageObject( + trackedUsage, + actor, + modelUsed.id, + costOverwrites, + ); + return trackedUsage; + } else { + // custom open router logic because they're pricing are weird + const trackedUsage = { + prompt: + (usage.prompt_tokens ?? 0) - + (usage.prompt_tokens_details?.cached_tokens ?? 0), + completion: usage.completion_tokens ?? 0, + input_cache_read: + usage.prompt_tokens_details?.cached_tokens ?? 0, + request: + (usage as unknown as Record) + .request || 1, + }; + const costOverwrites = Object.fromEntries( + Object.keys(trackedUsage).map((k) => { + return [k, modelUsed.costs[k] * trackedUsage[k]]; + }), + ); + this.#meteringService.utilRecordUsageObject( + trackedUsage, + actor, + modelUsed.id, + costOverwrites, + ); + return trackedUsage; + } + }, + stream, + completion, + }); + } + + async models() { + let models = kv.get('openrouterChat:models'); + if (!models) { + try { + const resp = await axios.request({ + method: 'GET', + url: `${this.#apiBaseUrl}/models`, + }); + + models = resp.data.data; + kv.set('openrouterChat:models', models); + } catch (e) { + console.log(e); + } + } + if (!models) return []; + const coerced_models: IChatModel[] = []; + for (const model of models) { + if ((model.id as string).includes('openrouter/auto')) { + continue; + } + const overridenModel = OPEN_ROUTER_MODEL_OVERRIDES.find( + (m) => m.id === `openrouter:${model.id}`, + ); + const microcentCosts = Object.fromEntries( + Object.entries(model.pricing).map(([k, v]) => [ + k, + Math.round( + ((v as number) < 0 ? 1 : (v as number)) * + 1_000_000 * + 100, + ), + ]), + ); + if (!microcentCosts.request) { + microcentCosts.request = 0; + } + coerced_models.push({ + id: `openrouter:${model.id}`, + name: `${model.name} (OpenRouter)`, + aliases: [ + model.id, + model.name, + `openrouter/${model.id}`, + model.id.split('/').slice(1).join('/'), + ], + context: model.context_length, + max_tokens: model.top_provider.max_completion_tokens, + costs_currency: 'usd-cents', + input_cost_key: 'prompt', + output_cost_key: 'completion', + costs: { + tokens: 1_000_000, + ...microcentCosts, + }, + ...overridenModel, + }); + } + return coerced_models; + } + checkModeration( + _text: string, + ): ReturnType { + throw new Error('Method not implemented.'); + } +} diff --git a/src/backend/src/services/ai/chat/providers/OpenRouterProvider/modelOverrides.ts b/src/backend/drivers/ai-chat/providers/openrouter/modelOverrides.ts similarity index 63% rename from src/backend/src/services/ai/chat/providers/OpenRouterProvider/modelOverrides.ts rename to src/backend/drivers/ai-chat/providers/openrouter/modelOverrides.ts index 4d36dedc3..04d982752 100644 --- a/src/backend/src/services/ai/chat/providers/OpenRouterProvider/modelOverrides.ts +++ b/src/backend/drivers/ai-chat/providers/openrouter/modelOverrides.ts @@ -1,5 +1,5 @@ -import { toMicroCents } from '../../../../MeteringService/utils.js'; -import { IChatModel } from '../types'; +import { toMicroCents } from '../../../../services/metering/utils.js'; +import type { IChatModel } from '../../types.js'; export const OPEN_ROUTER_MODEL_OVERRIDES: IChatModel[] = [ { @@ -7,4 +7,4 @@ export const OPEN_ROUTER_MODEL_OVERRIDES: IChatModel[] = [ subscriberOnly: true, minimumCredits: toMicroCents(2), } as IChatModel, -]; \ No newline at end of file +]; diff --git a/src/backend/src/services/ai/chat/providers/TogetherAiProvider/TogetherAIProvider.ts b/src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.ts similarity index 60% rename from src/backend/src/services/ai/chat/providers/TogetherAiProvider/TogetherAIProvider.ts rename to src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.ts index 883ffc742..7917189b8 100644 --- a/src/backend/src/services/ai/chat/providers/TogetherAiProvider/TogetherAIProvider.ts +++ b/src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.ts @@ -18,11 +18,11 @@ */ import { Together } from 'together-ai'; -import { Context } from '../../../../../util/context.js'; -import { kv } from '../../../../../util/kvSingleton.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import * as OpenAIUtil from '../../../utils/OpenAIUtil.js'; -import { IChatModel, IChatProvider, ICompleteArguments } from '../types.js'; +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import { kv } from '../../../../util/kvSingleton.js'; +import * as OpenAIUtil from '../../utils/OpenAIUtil.js'; +import { IChatModel, IChatProvider, ICompleteArguments } from '../../types.js'; const TOGETHER_AI_CHAT_COST_MAP = { prompt_tokens: 'input', @@ -36,28 +36,37 @@ export class TogetherAIProvider implements IChatProvider { #kvKey = 'togetherai:models'; - constructor (config: { apiKey: string }, meteringService: MeteringService) { + constructor(config: { apiKey: string }, meteringService: MeteringService) { this.#together = new Together({ apiKey: config.apiKey, }); this.#meteringService = meteringService; } - getDefaultModel () { + getDefaultModel() { return 'togetherai:meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo'; } - async models () { + async models() { let models: IChatModel[] | undefined = kv.get(this.#kvKey); - if ( models ) return models; + if (models) return models; const apiModels = await this.#together.models.list(); models = []; - for ( const model of apiModels ) { - if ( model.type === 'chat' || model.type === 'code' || model.type === 'language' || model.type === 'moderation' ) { + for (const model of apiModels) { + if ( + model.type === 'chat' || + model.type === 'code' || + model.type === 'language' || + model.type === 'moderation' + ) { models.push({ id: `togetherai:${model.id}`, - aliases: [model.id, `togetherai/${model.id}`, model.id.split('/').slice(1).join('/')], + aliases: [ + model.id, + `togetherai/${model.id}`, + model.id.split('/').slice(1).join('/'), + ], name: model.display_name, context: model.context_length, description: model.display_name, @@ -91,27 +100,38 @@ export class TogetherAIProvider implements IChatProvider { return models; } - async list () { + async list() { const models = await this.models(); const modelIds: string[] = []; - for ( const model of models ) { + for (const model of models) { modelIds.push(model.id); - if ( model.aliases ) { + if (model.aliases) { modelIds.push(...model.aliases); } } return modelIds; } - async complete ({ messages, stream, model, tools, max_tokens, temperature }: ICompleteArguments): ReturnType { - if ( model === 'model-fallback-test-1' ) { + async complete({ + messages, + stream, + model, + tools, + max_tokens, + temperature, + }: ICompleteArguments): ReturnType { + if (model === 'model-fallback-test-1') { throw new Error('Model Fallback Test 1'); } const actor = Context.get('actor'); const models = await this.models(); - const modelUsed = models.find(m => [m.id, ...(m.aliases || [])].includes(model)) || models.find(m => m.id === this.getDefaultModel())!; - const modelIdForParams = modelUsed.id.startsWith('togetherai:') ? modelUsed.id.slice('togetherai:'.length) : modelUsed.id; + const modelUsed = + models.find((m) => [m.id, ...(m.aliases || [])].includes(model)) || + models.find((m) => m.id === this.getDefaultModel())!; + const modelIdForParams = modelUsed.id.startsWith('togetherai:') + ? modelUsed.id.slice('togetherai:'.length) + : modelUsed.id; messages = await OpenAIUtil.process_input_messages(messages); @@ -121,9 +141,20 @@ export class TogetherAIProvider implements IChatProvider { stream, ...(tools ? { tools } : {}), // TODO: make this better but togetherai doesn't handle max tokens properly at all - ...(max_tokens ? { max_tokens: max_tokens - messages.reduce((acc, curr) => { - return acc + (curr.type === 'text' ? curr.text.length / 2 : 200); - }, 0) } : {}), + ...(max_tokens + ? { + max_tokens: + max_tokens - + messages.reduce((acc, curr) => { + return ( + acc + + (curr.type === 'text' + ? curr.text.length / 2 + : 200) + ); + }, 0), + } + : {}), ...(temperature ? { temperature } : {}), ...(stream ? { stream_options: { include_usage: true } } : {}), } as Together.Chat.Completions.CompletionCreateParamsNonStreaming); @@ -131,12 +162,19 @@ export class TogetherAIProvider implements IChatProvider { return OpenAIUtil.handle_completion_output({ usage_calculator: ({ usage }) => { const trackedUsage = OpenAIUtil.extractMeteredUsage(usage); - const costsOverride = Object.fromEntries(Object.entries(trackedUsage).map(([k, v]) => { - const mappedKey = TOGETHER_AI_CHAT_COST_MAP[k] || k; - return [k, v * (modelUsed.costs[mappedKey])]; - })); + const costsOverride = Object.fromEntries( + Object.entries(trackedUsage).map(([k, v]) => { + const mappedKey = TOGETHER_AI_CHAT_COST_MAP[k] || k; + return [k, v * modelUsed.costs[mappedKey]]; + }), + ); - this.#meteringService.utilRecordUsageObject(trackedUsage, actor, `togetherai:${modelIdForParams}`, costsOverride); + this.#meteringService.utilRecordUsageObject( + trackedUsage, + actor, + `togetherai:${modelIdForParams}`, + costsOverride, + ); return trackedUsage; }, stream, @@ -144,7 +182,7 @@ export class TogetherAIProvider implements IChatProvider { }); } - checkModeration (_text: string): ReturnType { + checkModeration(_text: string) { throw new Error('Method not implemented.'); } } diff --git a/src/backend/src/services/ai/chat/providers/XAIProvider/XAIProvider.ts b/src/backend/drivers/ai-chat/providers/xai/XAIProvider.ts similarity index 60% rename from src/backend/src/services/ai/chat/providers/XAIProvider/XAIProvider.ts rename to src/backend/drivers/ai-chat/providers/xai/XAIProvider.ts index 2598ac86d..c63a51590 100644 --- a/src/backend/src/services/ai/chat/providers/XAIProvider/XAIProvider.ts +++ b/src/backend/drivers/ai-chat/providers/xai/XAIProvider.ts @@ -19,10 +19,14 @@ import { OpenAI } from 'openai'; import { ChatCompletionCreateParams } from 'openai/resources/index.js'; -import { Context } from '../../../../../util/context.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import * as OpenAIUtil from '../../../utils/OpenAIUtil.js'; -import { IChatProvider, ICompleteArguments } from '../types.js'; +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import * as OpenAIUtil from '../../utils/OpenAIUtil.js'; +import type { + IChatProvider, + ICompleteArguments, + IChatCompleteResult, +} from '../../types.js'; import { XAI_MODELS } from './models.js'; export class XAIProvider implements IChatProvider { @@ -30,7 +34,7 @@ export class XAIProvider implements IChatProvider { #meteringService: MeteringService; - constructor (config: { apiKey: string }, meteringService: MeteringService) { + constructor(config: { apiKey: string }, meteringService: MeteringService) { this.#openai = new OpenAI({ apiKey: config.apiKey, baseURL: 'https://api.x.ai/v1', @@ -38,30 +42,38 @@ export class XAIProvider implements IChatProvider { this.#meteringService = meteringService; } - getDefaultModel () { + getDefaultModel() { return 'grok-beta'; } - models () { + models() { return XAI_MODELS; } - async list () { + async list() { const models = this.models(); const modelNames: string[] = []; - for ( const model of models ) { + for (const model of models) { modelNames.push(model.id); - if ( model.aliases ) { + if (model.aliases) { modelNames.push(...model.aliases); } } return modelNames; } - async complete ({ messages, stream, model, tools }: ICompleteArguments): ReturnType { + async complete({ + messages, + stream, + model, + tools, + }: ICompleteArguments): Promise { const actor = Context.get('actor'); const availableModels = this.models(); - const modelUsed = availableModels.find(m => [m.id, ...(m.aliases || [])].includes(model)) || availableModels.find(m => m.id === this.getDefaultModel())!; + const modelUsed = + availableModels.find((m) => + [m.id, ...(m.aliases || [])].includes(model), + ) || availableModels.find((m) => m.id === this.getDefaultModel())!; messages = await OpenAIUtil.process_input_messages(messages); let completion; try { @@ -71,11 +83,12 @@ export class XAIProvider implements IChatProvider { ...(tools ? { tools } : {}), max_tokens: 1000, stream, - ...(stream ? { - stream_options: { include_usage: true }, - } : {}), + ...(stream + ? { + stream_options: { include_usage: true }, + } + : {}), } as ChatCompletionCreateParams); - } catch (e) { console.log('XAI AI process_input_messages error: ', e); } @@ -83,10 +96,17 @@ export class XAIProvider implements IChatProvider { return OpenAIUtil.handle_completion_output({ usage_calculator: ({ usage }) => { const trackedUsage = OpenAIUtil.extractMeteredUsage(usage); - const costsOverride = Object.fromEntries(Object.entries(trackedUsage).map(([key, value]) => { - return [key, value * (modelUsed.costs[key])]; - })); - this.#meteringService.utilRecordUsageObject(trackedUsage, actor, `xai:${modelUsed.id}`, costsOverride); + const costsOverride = Object.fromEntries( + Object.entries(trackedUsage).map(([key, value]) => { + return [key, value * modelUsed.costs[key]]; + }), + ); + this.#meteringService.utilRecordUsageObject( + trackedUsage, + actor, + `xai:${modelUsed.id}`, + costsOverride, + ); return trackedUsage; }, stream, @@ -94,7 +114,9 @@ export class XAIProvider implements IChatProvider { }); } - checkModeration (_text: string): ReturnType { + checkModeration( + _text: string, + ): ReturnType { throw new Error('Method not implemented.'); } } diff --git a/src/backend/src/services/ai/chat/providers/XAIProvider/models.ts b/src/backend/drivers/ai-chat/providers/xai/models.ts similarity index 99% rename from src/backend/src/services/ai/chat/providers/XAIProvider/models.ts rename to src/backend/drivers/ai-chat/providers/xai/models.ts index 4f9c1d348..0e2ec9b8d 100644 --- a/src/backend/src/services/ai/chat/providers/XAIProvider/models.ts +++ b/src/backend/drivers/ai-chat/providers/xai/models.ts @@ -1,4 +1,4 @@ -import { IChatModel } from '../types.js'; +import type { IChatModel } from '../../types.js'; // Hardcoded from https://models.dev/api.json export const XAI_MODELS: IChatModel[] = [ diff --git a/src/backend/drivers/ai-chat/types.ts b/src/backend/drivers/ai-chat/types.ts new file mode 100644 index 000000000..a4196d2d3 --- /dev/null +++ b/src/backend/drivers/ai-chat/types.ts @@ -0,0 +1,108 @@ +/** + * Types for the `puter-chat-completion` driver interface. + * + * No openai SDK type dependency. The PuterMessage type is intentionally + * loose; each provider normalises internally. + */ + +export type ModelCost = Record; + +export interface ModelModalities { + input: string[]; + output: string[]; +} + +export interface IChatModel extends Record< + string, + unknown +> { + id: string; + provider?: string; + puterId?: string; + aliases?: string[]; + costs_currency: string; + input_cost_key?: keyof T; + output_cost_key?: keyof T; + costs: T; + context?: number; + max_tokens: number; + subscriberOnly?: boolean; + minimumCredits?: number; + modalities?: ModelModalities; + open_weights?: boolean; + tool_call?: boolean; + knowledge?: string; + release_date?: string; +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type PuterMessage = any; + +export interface ICompleteArguments { + messages: PuterMessage[]; + provider?: string; + stream?: boolean; + model: string; + tools?: unknown[]; + tool_choice?: unknown; + parallel_tool_calls?: boolean; + include?: unknown[]; + conversation?: unknown; + previous_response_id?: string; + instructions?: string | PuterMessage[]; + metadata?: Record; + prompt?: unknown; + prompt_cache_key?: string; + prompt_cache_retention?: 'in-memory' | '24h' | undefined; + store?: boolean; + top_p?: number; + truncation?: 'auto' | 'disabled' | undefined; + background?: boolean; + service_tier?: + | 'auto' + | 'default' + | 'flex' + | 'scale' + | 'priority' + | undefined; + max_tokens?: number; + temperature?: number; + reasoning?: { effort: 'low' | 'medium' | 'high' } | undefined; + text?: string & { verbosity?: 'concise' | 'detailed' | undefined }; + reasoning_effort?: 'low' | 'medium' | 'high' | undefined; + verbosity?: 'concise' | 'detailed' | undefined; + moderation?: boolean; + custom?: unknown; + response?: { + normalize?: boolean; + }; + customLimitMessage?: string; +} + +export interface IChatStreamResult { + init_chat_stream: (params: { chatStream: unknown }) => Promise; + stream: true; + finally_fn: () => Promise; + message?: never; + usage?: never; + finish_reason?: never; +} + +export interface IChatMessageResult { + message: PuterMessage; + usage: Record; + finish_reason: string; + init_chat_stream?: never; + stream?: never; + finally_fn?: never; + normalized?: boolean; +} + +export type IChatCompleteResult = IChatStreamResult | IChatMessageResult; + +export interface IChatProvider { + models(extra_params?: unknown): IChatModel[] | Promise; + list(): string[] | Promise; + getDefaultModel(): string; + complete(arg: ICompleteArguments): Promise; +} diff --git a/src/backend/drivers/ai-chat/utils/FunctionCalling.js b/src/backend/drivers/ai-chat/utils/FunctionCalling.js new file mode 100644 index 000000000..6f3c6e234 --- /dev/null +++ b/src/backend/drivers/ai-chat/utils/FunctionCalling.js @@ -0,0 +1,129 @@ +export const normalize_json_schema = (schema) => { + if (!schema) return schema; + + if (schema.type === 'object') { + if (!schema.properties) { + return schema; + } + + const keys = Object.keys(schema.properties); + for (const key of keys) { + schema.properties[key] = normalize_json_schema( + schema.properties[key], + ); + } + } + + if (schema.type === 'array') { + if (!schema.items) { + schema.items = {}; + } else { + schema.items = normalize_json_schema(schema.items); + } + } + + return schema; +}; + +/** + * Normalizes the 'tools' object in-place. + * + * This function will accept an array of tools provided by the + * user, and produce a normalized object that can then be + * converted to the apprpriate representation for another + * service. + * + * We will accept conventions from either service that a user + * might expect to work, prioritizing the OpenAI convention + * when conflicting conventions are present. + * + * @param {*} tools + */ +export const normalize_tools_object = (tools) => { + for (let i = 0; i < tools.length; i++) { + const tool = tools[i]; + + if (tool.type === 'web_search') { + // OpenAI Responses specific + continue; + } + let normalized_tool = {}; + + const normalize_function = (fn) => { + const normal_fn = {}; + let parameters = fn.parameters || fn.input_schema; + + if (!parameters || typeof parameters !== 'object') { + parameters = { type: 'object' }; + } else if (!parameters.type) { + parameters.type = 'object'; + } + + normal_fn.parameters = parameters; + + if (parameters.properties) { + parameters = normalize_json_schema(parameters); + } + + if (fn.name) { + normal_fn.name = fn.name; + } + + if (fn.description) { + normal_fn.description = fn.description; + } + + return normal_fn; + }; + + if (tool.input_schema) { + normalized_tool = { + type: 'function', + function: normalize_function(tool), + }; + } else if (tool.type === 'function') { + normalized_tool = { + type: 'function', + function: normalize_function(tool.function || tool), + }; + } else { + normalized_tool = { + type: 'function', + function: normalize_function(tool), + }; + } + + tools[i] = normalized_tool; + } + return tools; +}; + +/** + * This function will convert a normalized tools object to the + * format expected by OpenAI. + * + * @param {*} tools + * @returns + */ +export const make_openai_tools = (tools) => { + return tools; +}; + +/** + * This function will convert a normalized tools object to the + * format expected by Claude. + * + * @param {*} tools + * @returns + */ +export const make_claude_tools = (tools) => { + if (!tools) return undefined; + return tools.map((tool) => { + const { name, description, parameters } = tool.function; + return { + name, + description, + input_schema: parameters, + }; + }); +}; diff --git a/src/backend/drivers/ai-chat/utils/Messages.js b/src/backend/drivers/ai-chat/utils/Messages.js new file mode 100644 index 000000000..08f736b8f --- /dev/null +++ b/src/backend/drivers/ai-chat/utils/Messages.js @@ -0,0 +1,232 @@ +/** + * Normalizes a single message into a standardized format with role and content array. + * Converts string messages to objects, ensures content is an array of content blocks, + * transforms tool_calls into tool_use content blocks, and coerces content items into objects. + * + * @param {string|Object} message - The message to normalize, either a string or message object + * @param {Object} params - Optional parameters including default role + * @returns {Object} Normalized message with role and content array + * @throws {Error} If message is not a string or object + * @throws {Error} If message has no content property and no tool_calls + * @throws {Error} If any content item is not a string or object + */ +export const normalize_single_message = (message, params = {}) => { + params = Object.assign( + { + role: 'user', + }, + params, + ); + + if (typeof message === 'string') { + message = { + content: [message], + }; + } + if (!message || typeof message !== 'object' || Array.isArray(message)) { + throw new Error('each message must be a string or object'); + } + if (!message.role) { + message.role = params.role; + } + if (!message.content) { + if (message.tool_calls) { + message.content = []; + for (let i = 0; i < message.tool_calls.length; i++) { + const tool_call = message.tool_calls[i]; + message.content.push({ + type: 'tool_use', + id: tool_call.id, + name: tool_call.function.name, + input: tool_call.function.arguments, + }); + } + delete message.tool_calls; + } else if (message.role !== 'tool') { + throw new Error("each message must have a 'content' property"); + } + } + + // Normalize OpenAI-style tool results into internal tool_result blocks + if (message.role === 'tool') { + const tool_use_id = + message.tool_call_id || message.tool_use_id || message.id; + const tool_content = message.content; + message.tool_use_id = tool_use_id; + message.content = [ + { + type: 'tool_result', + tool_use_id, + content: + typeof tool_content === 'string' + ? tool_content + : JSON.stringify(tool_content ?? {}), + }, + ]; + } + if (!Array.isArray(message.content)) { + message.content = [message.content]; + } + // Coerce each content block into an object + for (let i = 0; i < message.content.length; i++) { + if (typeof message.content[i] === 'string') { + message.content[i] = { + type: 'text', + text: message.content[i], + }; + } + if ( + !message || + typeof message.content[i] !== 'object' || + Array.isArray(message.content[i]) + ) { + throw new Error( + 'each message content item must be a string or object', + ); + } + if ( + typeof message.content[i].text === 'string' && + !message.content[i].type + ) { + message.content[i].type = 'text'; + } + } + + // Remove "text" properties from content blocks with type=tool_result + for (let i = 0; i < message.content.length; i++) { + if (message.content[i].type !== 'tool_use') { + continue; + } + if (Object.prototype.hasOwnProperty.call(message.content[i], 'text')) { + delete message.content[i].text; + } + } + + return message; +}; + +/** + * Normalizes an array of messages by applying normalize_single_message to each, + * then splits messages with multiple content blocks into separate messages, + * and finally merges consecutive messages from the same role. + * + * @param {Array} messages - Array of messages to normalize + * @param {Object} params - Optional parameters passed to normalize_single_message + * @returns {Array} Normalized and merged array of messages + */ +export const normalize_messages = (messages, params = {}) => { + for (let i = 0; i < messages.length; i++) { + messages[i] = normalize_single_message(messages[i], params); + } + + // Split messages with multiple content blocks into separate messages. + // Keep assistant tool_use blocks together to preserve OpenAI tool-call ordering. + // TODO: unit test this + messages = [...messages]; + for (let i = 0; i < messages.length; i++) { + const message = messages[i]; + const separated_messages = []; + const has_tool_use = + message.role === 'assistant' && + message.content?.some((c) => c?.type === 'tool_use'); + if (has_tool_use) { + separated_messages.push(message); + messages.splice(i, 1, ...separated_messages); + continue; + } + for (let j = 0; j < message.content.length; j++) { + separated_messages.push({ + ...message, + content: [message.content[j]], + }); + } + messages.splice(i, 1, ...separated_messages); + } + + // If multiple messages are from the same role, merge them + // but avoid merging tool_use/tool_result messages, since order matters + const hasToolContent = (message) => { + if (!message || !Array.isArray(message.content)) return false; + return message.content.some( + (part) => + part && + (part.type === 'tool_use' || part.type === 'tool_result'), + ); + }; + const merged_messages = []; + let current_role = null; + for (let i = 0; i < messages.length; i++) { + const can_merge = + current_role === messages[i].role && + !hasToolContent(messages[i]) && + !hasToolContent(merged_messages[merged_messages.length - 1]); + if (can_merge) { + merged_messages[merged_messages.length - 1].content.push( + ...messages[i].content, + ); + } else { + merged_messages.push(messages[i]); + current_role = messages[i].role; + } + } + + return merged_messages; +}; + +/** + * Separates system messages from other messages in the array. + * + * @param {Array} messages - Array of messages to process + * @returns {Array} Tuple containing [system_messages, non_system_messages] + */ +export const extract_and_remove_system_messages = (messages) => { + const system_messages = []; + const new_messages = []; + for (let i = 0; i < messages.length; i++) { + if (messages[i].role === 'system') { + system_messages.push(messages[i]); + } else { + new_messages.push(messages[i]); + } + } + return [system_messages, new_messages]; +}; + +/** + * Extracts all text content from messages, handling various message formats. + * Processes strings, objects with content arrays, and nested content structures, + * joining all text with spaces. + * + * @param {Array} messages - Array of messages to extract text from + * @returns {string} Concatenated text content from all messages + * @throws {Error} If text content is not a string + */ +export const extract_text = (messages) => { + return messages + .map((m) => { + if (typeof m === 'string') { + return m; + } + if (!m || typeof m !== 'object' || Array.isArray(m)) { + return ''; + } + if (Array.isArray(m.content)) { + return m.content.map((c) => c.text).join(' '); + } + if (typeof m.content === 'string') { + return m.content; + } else { + const is_text_type = + m.content.type === 'text' || + !Object.prototype.hasOwnProperty.call(m.content, 'type'); + if (is_text_type) { + if (typeof m.content.text !== 'string') { + throw new Error('text content must be a string'); + } + return m.content.text; + } + return ''; + } + }) + .join(' '); +}; diff --git a/src/backend/drivers/ai-chat/utils/OpenAIUtil.js b/src/backend/drivers/ai-chat/utils/OpenAIUtil.js new file mode 100644 index 000000000..8a042178e --- /dev/null +++ b/src/backend/drivers/ai-chat/utils/OpenAIUtil.js @@ -0,0 +1,521 @@ +/** + * Process input messages from Puter's normalized format to OpenAI's format + * May make changes in-place. + * + * @param {Array} messages - array of normalized messages + * @returns {Array} - array of messages in OpenAI format + */ +export const process_input_messages = async (messages) => { + for (const msg of messages) { + if (!msg.content) continue; + if (typeof msg.content !== 'object') continue; + + const content = msg.content; + + for (const o of content) { + if (o['image_url'] && !o.type) { + o.type = 'image_url'; + } + if (o['video_url'] && !o.type) { + o.type = 'video_url'; + } + } + + // coerce tool calls + let is_tool_call = false; + for (let i = content.length - 1; i >= 0; i--) { + const content_block = content[i]; + + if (content_block.type === 'tool_use') { + if (!msg.tool_calls) { + msg.tool_calls = []; + is_tool_call = true; + } + msg.tool_calls.push({ + id: content_block.id, + type: 'function', + function: { + name: content_block.name, + arguments: JSON.stringify(content_block.input), + }, + ...(content_block.extra_content + ? { extra_content: content_block.extra_content } + : {}), + }); + content.splice(i, 1); + } + } + + if (is_tool_call) msg.content = null; + + // coerce tool results + // (we assume multiple tool results were already split into separate messages) + for (let i = content.length - 1; i >= 0; i--) { + const content_block = content[i]; + if (content_block.type !== 'tool_result') continue; + msg.role = 'tool'; + msg.tool_call_id = content_block.tool_use_id; + msg.content = content_block.content; + } + } + + return messages; +}; + +export const process_input_messages_responses_api = async (messages) => { + for (const msg of messages) { + const content_as_string = (content) => { + if (content === undefined || content === null) return ''; + if (typeof content === 'string') return content; + if (Array.isArray(content)) { + return content + .map((part) => { + if (typeof part === 'string') return part; + if (part && typeof part.text === 'string') + return part.text; + if (part && typeof part.content === 'string') + return part.content; + return ''; + }) + .join(''); + } + if (content && typeof content.text === 'string') + return content.text; + if (content && typeof content.content === 'string') + return content.content; + return ''; + }; + + if (msg.role === 'tool') { + msg.type = 'function_call_output'; + msg.call_id = msg.tool_call_id || msg.tool_use_id; + msg.output = content_as_string(msg.content); + delete msg.role; + delete msg.content; + delete msg.tool_call_id; + delete msg.tool_use_id; + delete msg.tool_calls; + continue; + } + + if (!msg.content) continue; + if (typeof msg.content !== 'object') continue; + + const content = msg.content; + + for (const o of content) { + if (o['image_url'] && !o.type) { + o.type = 'image_url'; + } + if (o['video_url'] && !o.type) { + o.type = 'video_url'; + } + } + + // coerce tool calls + let is_tool_call = false; + for (let i = content.length - 1; i >= 0; i--) { + const content_block = content[i]; + if ( + content_block.type === 'text' && + (msg.role === 'user' || msg.role === 'system') + ) { + content_block.type = 'input_text'; + } + if (content_block.type === 'text' && msg.role === 'assistant') { + content_block.type = 'output_text'; + } + + if (content_block.type === 'tool_use') { + if (!msg.tool_calls) { + msg.tool_calls = []; + is_tool_call = true; + } + msg.tool_calls.push({ + id: content_block.id, + canonical_id: content_block.canonical_id, + type: 'function', + function: { + name: content_block.name, + arguments: JSON.stringify(content_block.input), + }, + ...(content_block.extra_content + ? { extra_content: content_block.extra_content } + : {}), + }); + + content.splice(i, 1); + } + } + + // Right now this does NOT support parallel tool calls! + // We only allow sequential toolcalling right now so this shouldn't be an issue right now + // but this probably needs to be changed in the future to split "one completions message" + // into multiple responses inputs. + if (is_tool_call) { + msg.call_id = msg.tool_calls[0].id; + msg.id = msg.tool_calls[0].canonical_id; + msg.name = msg.tool_calls[0].function.name; + msg.arguments = msg.tool_calls[0].function.arguments; + msg.type = 'function_call'; + + delete msg.role; + delete msg.content; + delete msg.tool_calls; + } + + // coerce tool results + for (let i = content.length - 1; i >= 0; i--) { + const content_block = content[i]; + if (content_block.type !== 'tool_result') continue; + msg.type = 'function_call_output'; + msg.call_id = content_block.tool_use_id; + msg.output = content_block.content; + + delete msg.role; + delete msg.content; + } + } + + return messages; +}; + +export const create_usage_calculator = ({ model_details }) => { + return ({ usage }) => { + const tokens = []; + + tokens.push({ + type: 'prompt', + model: model_details.id, + amount: usage.prompt_tokens, + cost: model_details.cost.input * usage.prompt_tokens, + }); + + tokens.push({ + type: 'completion', + model: model_details.id, + amount: usage.completion_tokens, + cost: model_details.cost.output * usage.completion_tokens, + }); + + return tokens; + }; +}; + +export const extractMeteredUsage = (usage) => { + return { + prompt_tokens: usage.prompt_tokens ?? 0, + completion_tokens: usage.completion_tokens ?? 0, + cached_tokens: usage.prompt_tokens_details?.cached_tokens ?? 0, + }; +}; + +export const create_chat_stream_handler = + ({ deviations, completion, usage_calculator }) => + async ({ chatStream }) => { + deviations = Object.assign( + { + // affected by: Groq + index_usage_from_stream_chunk: (chunk) => chunk.usage, + // affected by: Mistral + chunk_but_like_actually: (chunk) => chunk, + index_tool_calls_from_stream_choice: (choice) => + choice.delta.tool_calls, + }, + deviations, + ); + + const message = chatStream.message(); + let textblock = message.contentBlock({ type: 'text' }); + let toolblock = null; + let mode = 'text'; + const tool_call_blocks = []; + + let last_usage = null; + for await (let chunk of completion) { + chunk = deviations.chunk_but_like_actually(chunk); + const chunk_usage = deviations.index_usage_from_stream_chunk(chunk); + if (chunk_usage) last_usage = chunk_usage; + if (chunk.choices.length < 1) continue; + + const choice = chunk.choices[0]; + + // Deepseek returns choice.delta.reasoning_content, openrouter returns choice.delta.reasoning. + if (choice.delta.reasoning_content || choice.delta.reasoning) { + textblock.addReasoning( + choice.delta.reasoning_content || choice.delta.reasoning, + ); + // Q: Why don't "continue" to next chunk here? + // A: For now, reasoning_content and content never appear together, but I’m not sure if they’ll always be mutually exclusive. + } + + if (choice.delta.content) { + if (mode === 'tool') { + toolblock.end(); + mode = 'text'; + textblock = message.contentBlock({ type: 'text' }); + } + textblock.addText(choice.delta.content); + continue; + } + + if (choice.delta.extra_content) { + // Gemini specific thing for metadata, we will basically be appending onto the current message by abusing .addText a little + // Apps have to choose to handle extra_content themselves, it doesn't seem like theres a way we can do it in a backwards + // compatible fashion since most streaming apps will handle chat history by continuously updating content themselves + // This doesn't present us a chance to add in an extra object for gemini's chat continuing features + textblock.addExtraContent(choice.delta.extra_content); + } + + const tool_calls = + deviations.index_tool_calls_from_stream_choice(choice); + if (tool_calls) { + if (mode === 'text') { + mode = 'tool'; + textblock.end(); + } + for (const tool_call of tool_calls) { + if (!tool_call_blocks[tool_call.index]) { + toolblock = message.contentBlock({ + type: 'tool_use', + id: tool_call.id, + name: tool_call.function.name, + ...(tool_call.extra_content + ? { extra_content: tool_call.extra_content } + : {}), + }); + tool_call_blocks[tool_call.index] = toolblock; + } else { + toolblock = tool_call_blocks[tool_call.index]; + } + toolblock.addPartialJSON(tool_call.function.arguments); + } + } + } + + // TODO DS: this is a bit too abstracted... this is basically just doing the metering now + const usage = usage_calculator({ usage: last_usage }); + + if (mode === 'text') textblock.end(); + if (mode === 'tool') toolblock.end(); + + message.end(); + chatStream.end(usage); + }; + +export const create_chat_stream_handler_responses_api = + ({ deviations, completion, usage_calculator }) => + async ({ chatStream }) => { + deviations = Object.assign( + { + // affected by: Groq + index_usage_from_stream_chunk: (chunk) => chunk.usage, + // affected by: Mistral + chunk_but_like_actually: (chunk) => chunk, + index_tool_calls_from_stream_choice: (choice) => + choice.delta.tool_calls, + }, + deviations, + ); + + const message = chatStream.message(); + const textblock = message.contentBlock({ type: 'text' }); + let toolblock = null; + const mode = 'text'; + + let last_usage = null; + for await (const chunk of completion) { + if (chunk.type === 'response.output_text.delta') { + textblock.addText(chunk.delta); + continue; + } + + if (chunk.type === 'response.completed') { + last_usage = chunk.response.usage; + } + + if ( + chunk.type === 'response.output_item.done' && + chunk.item?.type === 'function_call' + ) { + const tool_call = chunk.item; + toolblock = message.contentBlock({ + type: 'tool_use', + canonical_id: tool_call.id, + id: tool_call.call_id, + name: tool_call.name, + ...(tool_call.extra_content + ? { extra_content: tool_call.extra_content } + : {}), + }); + toolblock.addPartialJSON(tool_call.arguments); + toolblock.end(); + } + } + + // TODO DS: this is a bit too abstracted... this is basically just doing the metering now + const usage = usage_calculator({ usage: last_usage }); + + if (mode === 'text') textblock.end(); + if (mode === 'tool') toolblock.end(); + + message.end(); + chatStream.end(usage); + }; + +/** + * + * @param {object} params + * @param {(args: {usage: import("openai/resources/completions.mjs").CompletionUsage})=> unknown } params.usage_calculator + * @returns + */ +export const handle_completion_output = async ({ + deviations, + stream, + completion, + moderate, + usage_calculator, + finally_fn, +}) => { + deviations = Object.assign( + { + // affected by: Mistral + coerce_completion_usage: (completion) => completion.usage, + }, + deviations, + ); + + if (stream) { + const init_chat_stream = create_chat_stream_handler({ + deviations, + completion, + usage_calculator, + }); + + return { + stream: true, + init_chat_stream, + finally_fn, + }; + } + + if (finally_fn) await finally_fn(); + + // We need to moderate the completion too + const mod_text = completion.choices[0].message.content; + if (moderate && mod_text !== null) { + const moderation_result = await moderate(mod_text); + if (moderation_result.flagged) { + throw new Error('message is not allowed'); + } + } + + const ret = completion.choices[0]; + const completion_usage = deviations.coerce_completion_usage(completion); + ret.usage = usage_calculator + ? usage_calculator({ + ...completion, + usage: completion_usage, + }) + : { + input_tokens: completion_usage.prompt_tokens, + output_tokens: completion_usage.completion_tokens, + }; + return ret; +}; + +/** + * + * @param {object} params + * @param {(args: {usage: import("openai/resources/completions.mjs").CompletionUsage})=> unknown } params.usage_calculator + * @returns + */ +export const handle_completion_output_responses_api = async ({ + deviations, + stream, + completion, + moderate, + usage_calculator, + finally_fn, +}) => { + deviations = Object.assign( + { + // affected by: Mistral + coerce_completion_usage: (completion) => completion.usage, + }, + deviations, + ); + + if (stream) { + const init_chat_stream = create_chat_stream_handler_responses_api({ + deviations, + completion, + usage_calculator, + }); + + return { + stream: true, + init_chat_stream, + finally_fn, + }; + } + + if (finally_fn) await finally_fn(); + + const output = Array.isArray(completion.output) ? completion.output : []; + const responseToolCalls = output + .filter((item) => item?.type === 'function_call') + .map((item) => ({ + id: item.call_id, + type: 'function', + function: { + name: item.name, + arguments: item.arguments, + }, + ...(item.id ? { canonical_id: item.id } : {}), + })); + + const is_empty = completion.output_text.trim() === ''; + if (is_empty && responseToolCalls.length < 1) { + // GPT refuses to generate an empty response if you ask it to, + // so this will probably only happen on an error condition. + throw new Error('an empty response was generated'); + } + + // We need to moderate the completion too + const mod_text = completion.output_text; + if (moderate && mod_text !== null) { + const moderation_result = await moderate(mod_text); + if (moderation_result.flagged) { + throw new Error('message is not allowed'); + } + } + + const ret = { + finish_reason: 'stop', + index: 0, + message: { + content: completion.output_text, + reasoning: null, // Fix later to add proper reasoning + refusal: null, + role: 'assistant', + ...(responseToolCalls.length + ? { tool_calls: responseToolCalls } + : {}), + }, + }; + ret.role = output.find((item) => item?.role)?.role ?? 'assistant'; + + delete ret.type; + + ret.usage = usage_calculator + ? usage_calculator({ + ...completion, + usage: completion.usage, + }) + : { + input_tokens: completion.usage.input_tokens, + output_tokens: completion.usage.output_tokens, + }; + return ret; +}; diff --git a/src/backend/src/services/ai/utils/Streaming.js b/src/backend/drivers/ai-chat/utils/Streaming.js similarity index 53% rename from src/backend/src/services/ai/utils/Streaming.js rename to src/backend/drivers/ai-chat/utils/Streaming.js index 47e7fa135..c9ba8fc81 100644 --- a/src/backend/src/services/ai/utils/Streaming.js +++ b/src/backend/drivers/ai-chat/utils/Streaming.js @@ -1,75 +1,67 @@ export class AIChatConstructStream { - constructor (chatStream, params) { + constructor(chatStream, params) { this.chatStream = chatStream; - if ( this._start ) this._start(params); - } - end () { + if (this._start) this._start(params); } + end() {} } export class AIChatTextStream extends AIChatConstructStream { - addText (text, extra_content) { + addText(text, extra_content) { const json = JSON.stringify({ type: 'text', text, ...(extra_content ? { extra_content } : {}), }); - this.chatStream.stream.write(`${json }\n`); + this.chatStream.stream.write(`${json}\n`); } - addReasoning (reasoning) { + addReasoning(reasoning) { const json = JSON.stringify({ - type: 'reasoning', reasoning, + type: 'reasoning', + reasoning, }); - this.chatStream.stream.write(`${json }\n`); + this.chatStream.stream.write(`${json}\n`); } - addImage (image) { - const json = JSON.stringify({ - type: 'image', - image, - }); - this.chatStream.stream.write(`${json }\n`); - } - - addExtraContent (extra_content) { + addExtraContent(extra_content) { const json = JSON.stringify({ type: 'extra_content', extra_content, }); - this.chatStream.stream.write(`${json }\n`); + this.chatStream.stream.write(`${json}\n`); } } export class AIChatToolUseStream extends AIChatConstructStream { - _start (params) { + _start(params) { this.contentBlock = params; this.buffer = ''; } - addPartialJSON (partial_json) { + addPartialJSON(partial_json) { this.buffer += partial_json; } - end () { - if ( this.buffer.trim() === '' ) { + end() { + if (this.buffer.trim() === '') { this.buffer = '{}'; } - if ( process.env.DEBUG ) console.log('BUFFER BEING PARSED', this.buffer); + if (process.env.DEBUG) console.log('BUFFER BEING PARSED', this.buffer); const str = JSON.stringify({ type: 'tool_use', ...this.contentBlock, input: JSON.parse(this.buffer), - ...( !this.contentBlock.text ? { text: '' } : {}), + ...(!this.contentBlock.text ? { text: '' } : {}), }); - this.chatStream.stream.write(`${str }\n`); + this.chatStream.stream.write(`${str}\n`); } } export class AIChatMessageStream extends AIChatConstructStream { - contentBlock ({ type, ...params }) { - if ( type === 'tool_use' ) { + contentBlock({ type, ...params }) { + if (type === 'tool_use') { return new AIChatToolUseStream(this.chatStream, params); } - if ( type === 'text' ) { + if (type === 'text') { return new AIChatTextStream(this.chatStream, params); } throw new Error(`Unknown content block type: ${type}`); @@ -78,26 +70,28 @@ export class AIChatMessageStream extends AIChatConstructStream { export class AIChatStream { stream; - constructor ({ stream }) { + constructor({ stream }) { this.stream = stream; } - end (/** @type {Record} */ usage) { - this.stream.write(`${JSON.stringify({ - type: 'usage', - usage, - }) }\n`); + end(/** @type {Record} */ usage) { + this.stream.write( + `${JSON.stringify({ + type: 'usage', + usage, + })}\n`, + ); this.stream.end(); } - message () { + message() { return new AIChatMessageStream(this); } - write (...args) { + write(...args) { return this.stream.write(...args); } } export default class Streaming { static AIChatStream = AIChatStream; -}; +} diff --git a/src/backend/drivers/ai-image/ImageGenerationDriver.ts b/src/backend/drivers/ai-image/ImageGenerationDriver.ts new file mode 100644 index 000000000..ae787a008 --- /dev/null +++ b/src/backend/drivers/ai-image/ImageGenerationDriver.ts @@ -0,0 +1,282 @@ +import crypto from 'node:crypto'; +import { Context } from '../../core/context.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import { PuterDriver } from '../types.js'; +import { CloudflareImageProvider } from './providers/cloudflare/CloudflareImageProvider.js'; +import { GeminiImageProvider } from './providers/gemini/GeminiImageProvider.js'; +import { OpenAiImageProvider } from './providers/openai/OpenAiImageProvider.js'; +import { ReplicateImageGenerationProvider } from './providers/replicate/ReplicateImageGenerationProvider.js'; +import { TogetherImageProvider } from './providers/together/TogetherImageProvider.js'; +import { XAIImageProvider } from './providers/xai/XAIImageProvider.js'; +import type { IGenerateParams, IImageModel, IImageProvider } from './types.js'; + +/** + * Driver implementing the `puter-image-generation` interface. + * + * Manages multiple upstream providers and routes `generate()` calls + * based on the requested model. Mirrors ChatCompletionDriver's pattern: + * providers are instantiated from config on boot, a model map is built + * from each provider's declared models, and calls are dispatched. + * + * Output is a URL string (web URL or data URI) — no streaming, no + * TypedValue wrapper. + */ +export class ImageGenerationDriver extends PuterDriver { + readonly driverInterface = 'puter-image-generation'; + readonly driverName = 'ai-image'; + // puter-js's `txt2img` falls through `options.driver` into the + // driver-name slot (e.g. `xai-image-generation`), so alias all provider + // ids here. `generate` falls back to `Context.driverName` when + // `args.provider` isn't supplied. + readonly driverAliases = [ + 'openai-image-generation', + 'gemini-image-generation', + 'together-image-generation', + 'cloudflare-image-generation', + 'xai-image-generation', + 'replicate-image-generation', + ]; + readonly isDefault = true; + + #providers: Record = {}; + #modelIdMap: Record = {}; + + override onServerStart() { + this.#registerProviders(); + this.#buildModelMap(); + } + + async models() { + const seen = new Set(); + return Object.values(this.#modelIdMap) + .flat() + .filter((m) => { + if (seen.has(m.id)) return false; + seen.add(m.id); + return true; + }) + .sort((a, b) => { + if (a.provider === b.provider) return a.id.localeCompare(b.id); + return (a.provider ?? '').localeCompare(b.provider ?? ''); + }); + } + + async list() { + return (await this.models()).map((m) => m.puterId || m.id).sort(); + } + + override getReportedCosts(): Record[] { + const out: Record[] = []; + const seen = new Set(); + for (const bucket of Object.values(this.#modelIdMap)) { + for (const model of bucket) { + const key = `${model.provider}:${model.id}`; + if (seen.has(key)) continue; + seen.add(key); + for (const [costKey, raw] of Object.entries( + (model as { costs?: Record }).costs ?? {}, + )) { + if (typeof raw !== 'number' || !Number.isFinite(raw)) + continue; + out.push({ + usageType: `${model.provider}:${model.id}:${costKey}`, + costValue: raw, + source: `driver:aiImage/${model.provider}`, + }); + } + } + } + return out; + } + + async generate(args: IGenerateParams): Promise { + const actor = Context.get('actor'); + if (!actor) throw new HttpError(401, 'Authentication required'); + + let modelId = args.model?.trim().toLowerCase(); + let intendedProvider = + args.provider ?? (Context.get('driverName') as string | undefined); + + // Default: first registered provider's default model if none given + if (!modelId && !intendedProvider) { + intendedProvider = Object.keys(this.#providers)[0]; + } + if (!modelId && intendedProvider) { + modelId = this.#providers[intendedProvider]?.getDefaultModel(); + } + if (!modelId) throw new HttpError(400, 'Missing `model`'); + + const model = this.#resolveModel(modelId, intendedProvider); + if (!model) { + throw new HttpError(400, `Model not found: ${args.model}`); + } + + const provider = this.#providers[model.provider!]; + if (!provider) { + throw new HttpError(500, `No provider found for model ${model.id}`); + } + + // Audit log for abuse / billing. Fired before the upstream call + // so a failed generate still shows up in the log (prompt_block + // uses this to track user-by-user image prompts). + const completionId = crypto.randomUUID(); + this.clients.event.emit( + 'ai.log.image', + { + actor, + completionId, + parameters: args, + intended_service: model.id, + model_used: model.id, + service_used: model.provider, + }, + {}, + ); + + return provider.generate({ + ...args, + model: model.id, + provider: model.provider, + }); + } + + #registerProviders() { + const providers = this.config.providers ?? {}; + const m = this.services.metering; + + const readKey = ( + ...cfgs: Array | undefined> + ): string | undefined => { + for (const cfg of cfgs) { + if (!cfg) continue; + const k = + (cfg.apiKey as string | undefined) ?? + (cfg.secret_key as string | undefined); + if (k) return k; + } + return undefined; + }; + + const openaiKey = readKey( + providers['openai-image-generation'], + providers['openai-completion'], + providers['openai'], + ); + if (openaiKey) { + this.#providers['openai-image-generation'] = + new OpenAiImageProvider({ apiKey: openaiKey }, m); + } + + const geminiKey = readKey( + providers['gemini-image-generation'], + providers['gemini'], + ); + if (geminiKey) { + this.#providers['gemini-image-generation'] = + new GeminiImageProvider({ apiKey: geminiKey }, m); + } + + const togetherKey = readKey( + providers['together-image-generation'], + providers['together-ai'], + ); + if (togetherKey) { + this.#providers['together-image-generation'] = + new TogetherImageProvider({ apiKey: togetherKey }, m); + } + + const cloudflare = (providers['cloudflare-image-generation'] ?? + providers['cloudflare-workers-ai-image'] ?? + providers['cloudflare-workers-ai']) as + | Record + | undefined; + const cfToken = + (cloudflare?.apiToken as string | undefined) ?? + (cloudflare?.apiKey as string | undefined) ?? + (cloudflare?.secret_key as string | undefined); + const cfAccount = + (cloudflare?.accountId as string | undefined) ?? + (cloudflare?.account_id as string | undefined); + if (cfToken && cfAccount) { + this.#providers['cloudflare-image-generation'] = + new CloudflareImageProvider( + { + apiToken: cfToken, + accountId: cfAccount, + apiBaseUrl: cloudflare?.apiBaseUrl as + | string + | undefined, + }, + m, + ); + } + + const xaiKey = readKey( + providers['xai-image-generation'], + providers['xai'], + ); + if (xaiKey) { + this.#providers['xai-image-generation'] = new XAIImageProvider( + { apiKey: xaiKey }, + m, + ); + } + + const replicateKey = readKey(providers['replicate-image-generation']); + if (replicateKey) { + this.#providers['replicate-image-generation'] = + new ReplicateImageGenerationProvider( + { apiKey: replicateKey }, + m, + ); + } + } + + async #buildModelMap() { + for (const providerName in this.#providers) { + const provider = this.#providers[providerName]; + for (const model of await provider.models()) { + model.id = model.id.trim().toLowerCase(); + if (!this.#modelIdMap[model.id]) { + this.#modelIdMap[model.id] = []; + } + this.#modelIdMap[model.id].push({ + ...model, + provider: providerName, + }); + + if (model.puterId) { + model.aliases = model.aliases + ? [...model.aliases, model.puterId] + : [model.puterId]; + } + if (model.aliases) { + for (let alias of model.aliases) { + alias = alias.trim().toLowerCase(); + if (!this.#modelIdMap[alias]) { + this.#modelIdMap[alias] = + this.#modelIdMap[model.id]; + } else if ( + this.#modelIdMap[alias] !== + this.#modelIdMap[model.id] + ) { + this.#modelIdMap[alias].push({ + ...model, + provider: providerName, + }); + this.#modelIdMap[model.id] = + this.#modelIdMap[alias]; + } + } + } + } + } + } + + #resolveModel(modelId: string, provider?: string): IImageModel | null { + const models = this.#modelIdMap[modelId]; + if (!models || models.length === 0) return null; + if (!provider) return models[0]; + return models.find((m) => m.provider === provider) ?? models[0]; + } +} diff --git a/src/backend/drivers/ai-image/providers/ImageProvider.ts b/src/backend/drivers/ai-image/providers/ImageProvider.ts new file mode 100644 index 000000000..670696d59 --- /dev/null +++ b/src/backend/drivers/ai-image/providers/ImageProvider.ts @@ -0,0 +1,14 @@ +/** + * Base class for image generation providers. + * + * Concrete providers (OpenAI, Gemini, etc.) extend this and implement + * `generate`, `models`, and `getDefaultModel`. + */ + +import type { IImageProvider, IImageModel, IGenerateParams } from '../types.js'; + +export abstract class ImageProvider implements IImageProvider { + abstract generate(params: IGenerateParams): Promise; + abstract models(): IImageModel[] | Promise; + abstract getDefaultModel(): string; +} diff --git a/src/backend/drivers/ai-image/providers/cloudflare/CloudflareImageProvider.ts b/src/backend/drivers/ai-image/providers/cloudflare/CloudflareImageProvider.ts new file mode 100644 index 000000000..4e06576c4 --- /dev/null +++ b/src/backend/drivers/ai-image/providers/cloudflare/CloudflareImageProvider.ts @@ -0,0 +1,517 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { + IGenerateParams, + IImageModel, + IImageProvider, +} from '../../types.js'; +import { + CLOUDFLARE_IMAGE_GENERATION_MODELS, + CloudflareImageModel, +} from './models.js'; + +type CloudflareGenerateParams = IGenerateParams & { + steps?: number; + num_steps?: number; + seed?: number; + guidance?: number; + negative_prompt?: string; + output_format?: 'jpeg' | 'png' | 'webp'; + image?: string; +}; + +interface CostComponent { + key: string; + usageAmount: number; + totalCostMicroCents: number; +} + +const DEFAULT_MODEL = '@cf/black-forest-labs/flux-1-schnell'; +const DEFAULT_RATIO = { w: 1024, h: 1024 }; + +export class CloudflareImageProvider implements IImageProvider { + #apiToken: string; + #accountId: string; + #apiBaseUrl: string; + #meteringService: MeteringService; + + constructor( + config: { + apiToken: string; + accountId: string; + apiBaseUrl?: string; + }, + meteringService: MeteringService, + ) { + this.#apiToken = config.apiToken; + this.#accountId = config.accountId; + this.#apiBaseUrl = + config.apiBaseUrl || 'https://api.cloudflare.com/client/v4'; + this.#meteringService = meteringService; + } + + models(): IImageModel[] { + return CLOUDFLARE_IMAGE_GENERATION_MODELS; + } + + getDefaultModel(): string { + return DEFAULT_MODEL; + } + + async generate(params: IGenerateParams): Promise { + const options = params as CloudflareGenerateParams; + const { prompt, test_mode } = options; + const ratio = this.#normalizeRatio(options.ratio); + const selectedModel = this.#getModel(options.model); + + if (test_mode) { + return 'https://puter-sample-data.puter.site/image_example.png'; + } + + if (typeof prompt !== 'string' || prompt.trim().length === 0) { + throw new Error('`prompt` must be a non-empty string'); + } + + const actor = Context.get('actor'); + if (!actor) { + throw new Error('actor not found in context'); + } + + const steps = this.#resolveSteps(selectedModel, options); + const costComponents = this.#estimateCost(selectedModel, ratio, steps, { + hasInputImage: + typeof options.image === 'string' && + options.image.trim() !== '', + }); + const totalCostInMicroCents = costComponents.reduce( + (acc, component) => acc + component.totalCostMicroCents, + 0, + ); + const usageAllowed = await this.#meteringService.hasEnoughCredits( + actor, + totalCostInMicroCents, + ); + if (!usageAllowed) { + throw new Error('Insufficient credits for image generation'); + } + + const response = await this.#runModel(selectedModel, { + ...options, + ratio, + steps, + }); + + this.#meteringService.batchIncrementUsages( + actor, + costComponents + .filter( + (component) => + component.usageAmount > 0 && + component.totalCostMicroCents > 0, + ) + .map((component) => ({ + usageType: `cloudflare:${this.#getMeteringModelKey(selectedModel)}:${component.key}`, + usageAmount: component.usageAmount, + costOverride: component.totalCostMicroCents, + })), + ); + + return response; + } + + #getModel(model?: string): CloudflareImageModel { + const models = CLOUDFLARE_IMAGE_GENERATION_MODELS; + const found = models.find( + (m) => m.id === model || m.aliases?.includes(model ?? ''), + ); + return found || models.find((m) => m.id === DEFAULT_MODEL)!; + } + + #normalizeRatio(ratio?: { w: number; h: number }) { + const width = Number(ratio?.w); + const height = Number(ratio?.h); + if ( + Number.isFinite(width) && + Number.isFinite(height) && + width > 0 && + height > 0 + ) { + return { + w: Math.max(64, Math.round(width)), + h: Math.max(64, Math.round(height)), + }; + } + return { ...DEFAULT_RATIO }; + } + + #resolveSteps( + model: CloudflareImageModel, + options: CloudflareGenerateParams, + ): number { + const input = Number( + options.steps ?? options.num_steps ?? model.defaultSteps ?? 25, + ); + const fallback = model.defaultSteps ?? 25; + if (!Number.isFinite(input)) return fallback; + return Math.max(1, Math.min(50, Math.round(input))); + } + + // Cloudflare models have *really exact* billing needs. They pretty much bill based on exactly what the model does + // If a model is a diffusion model, thing flux-2-dev, we actually need to calculate how many steps they take to + // Denoise the model and calculate based on that. It's pretty annoying and we'll have to keep updating this table + // in the future likely. It's VERY easy to screw this up. I would not recommend touching any step based calculations + // unless you actually know what you're doing here, or you might regret it! + // Signed -- NS + #estimateCost( + model: CloudflareImageModel, + ratio: { w: number; h: number }, + steps: number, + options?: { hasInputImage?: boolean }, + ): CostComponent[] { + const tiles = this.#tileCount(ratio); + const pixels = ratio.w * ratio.h; + const megapixels = this.#megapixels(ratio); + + switch (model.billingScheme) { + case 'tile-plus-step': + return [ + { + key: 'tile_512', + usageAmount: tiles, + totalCostMicroCents: this.#costForUnits( + tiles, + model.costs.tile_512, + ), + }, + { + key: 'step', + usageAmount: steps, + totalCostMicroCents: this.#costForUnits( + steps, + model.costs.step, + ), + }, + ]; + case 'step-only': + return [ + { + key: 'step', + usageAmount: steps, + totalCostMicroCents: this.#costForUnits( + steps, + model.costs.step, + ), + }, + ]; + case 'flux2-dev-tile-step': + return [ + { + key: 'input_tile_512_per_step', + usageAmount: tiles * steps, + totalCostMicroCents: this.#costForUnits( + tiles * steps, + model.costs.input_tile_512_per_step, + ), + }, + { + key: 'output_tile_512_per_step', + usageAmount: tiles * steps, + totalCostMicroCents: this.#costForUnits( + tiles * steps, + model.costs.output_tile_512_per_step, + ), + }, + ]; + case 'flux2-klein-4b-tile': + return [ + { + key: 'input_tile_512', + usageAmount: tiles, + totalCostMicroCents: this.#costForUnits( + tiles, + model.costs.input_tile_512, + ), + }, + { + key: 'output_tile_512', + usageAmount: tiles, + totalCostMicroCents: this.#costForUnits( + tiles, + model.costs.output_tile_512, + ), + }, + ]; + case 'flux2-klein-9b-mp': { + const firstMP = Math.min(megapixels, 1); + const subsequentMP = Math.max(0, megapixels - firstMP); + const firstPixels = Math.min(pixels, 1_000_000); + const subsequentPixels = Math.max(0, pixels - firstPixels); + const inputImageMP = options?.hasInputImage ? megapixels : 0; + return [ + { + key: 'first_mp', + usageAmount: firstMP, + totalCostMicroCents: this.#costForMillionUnits( + firstPixels, + model.costs.first_mp, + ), + }, + { + key: 'subsequent_mp', + usageAmount: subsequentMP, + totalCostMicroCents: this.#costForMillionUnits( + subsequentPixels, + model.costs.subsequent_mp, + ), + }, + { + key: 'input_image_mp', + usageAmount: inputImageMP, + totalCostMicroCents: options?.hasInputImage + ? this.#costForMillionUnits( + pixels, + model.costs.input_image_mp, + ) + : 0, + }, + ]; + } + default: + return []; + } + } + + async #runModel( + model: CloudflareImageModel, + params: CloudflareGenerateParams & { + ratio: { w: number; h: number }; + steps: number; + }, + ) { + const endpoint = `${this.#apiBaseUrl}/accounts/${this.#accountId}/ai/run/${model.id}`; + const headers: Record = { + Authorization: `Bearer ${this.#apiToken}`, + }; + + let body; + if (model.requiresMultipart) { + const formData = new FormData(); + formData.append('prompt', params.prompt); + formData.append('width', String(params.ratio.w)); + formData.append('height', String(params.ratio.h)); + formData.append('steps', String(params.steps)); + + if (Number.isFinite(params.seed)) + formData.append( + 'seed', + String(Math.round(params.seed as number)), + ); + if (Number.isFinite(params.guidance)) + formData.append('guidance', String(params.guidance)); + if (typeof params.negative_prompt === 'string') + formData.append('negative_prompt', params.negative_prompt); + if (typeof params.output_format === 'string') + formData.append('output_format', params.output_format); + if (typeof params.image === 'string') + formData.append('image', params.image); + body = formData; + } else { + headers['Content-Type'] = 'application/json'; + body = JSON.stringify({ + prompt: params.prompt, + width: params.ratio.w, + height: params.ratio.h, + steps: params.steps, + num_steps: params.steps, + ...(Number.isFinite(params.seed) + ? { seed: Math.round(params.seed as number) } + : {}), + ...(Number.isFinite(params.guidance) + ? { guidance: params.guidance } + : {}), + ...(typeof params.negative_prompt === 'string' + ? { negative_prompt: params.negative_prompt } + : {}), + ...(typeof params.output_format === 'string' + ? { output_format: params.output_format } + : {}), + }); + } + + const response = await fetch(endpoint, { + method: 'POST', + headers, + body, + }); + + const contentType = ( + response.headers.get('content-type') || '' + ).toLowerCase(); + if (contentType.startsWith('image/')) { + const imageBuffer = Buffer.from(await response.arrayBuffer()); + return `data:${contentType};base64,${imageBuffer.toString('base64')}`; + } + + const text = await response.text(); + let payload: unknown; + try { + payload = text ? JSON.parse(text) : {}; + } catch { + payload = { raw: text }; + } + + if (!response.ok) { + const message = + this.#extractErrorMessage(payload) || + `Cloudflare image generation failed with status ${response.status}`; + throw new Error(message); + } + + if (typeof payload === 'object' && payload !== null) { + const envelope = payload as Record; + if (envelope.success === false) { + const message = + this.#extractErrorMessage(payload) || + 'Cloudflare image generation failed'; + throw new Error(message); + } + } + + const imageString = this.#extractImageString(payload); + if (!imageString) { + throw new Error( + 'Cloudflare image generation response did not include image data', + ); + } + + if ( + imageString.startsWith('data:image/') || + imageString.startsWith('http://') || + imageString.startsWith('https://') + ) { + return imageString; + } + + const mime = this.#mimeForFormat(params.output_format); + return `data:${mime};base64,${imageString}`; + } + + #extractImageString(payload: unknown): string | undefined { + if (typeof payload === 'string') return payload; + if (!payload || typeof payload !== 'object') return undefined; + + const record = payload as Record; + if (typeof record.image === 'string') return record.image; + if (typeof record.output === 'string') return record.output; + if ( + Array.isArray(record.images) && + typeof record.images[0] === 'string' + ) + return record.images[0]; + if ( + Array.isArray(record.images) && + typeof record.images[0] === 'object' && + record.images[0] !== null + ) { + const firstImage = record.images[0] as Record; + if (typeof firstImage.image === 'string') return firstImage.image; + } + if ( + Array.isArray(record.output) && + typeof record.output[0] === 'string' + ) + return record.output[0]; + + if (record.result) { + const nested = this.#extractImageString(record.result); + if (nested) return nested; + } + if (record.response) { + const nested = this.#extractImageString(record.response); + if (nested) return nested; + } + return undefined; + } + + #extractErrorMessage(payload: unknown): string | undefined { + if (!payload || typeof payload !== 'object') return undefined; + const record = payload as Record; + + if (typeof record.error === 'string') return record.error; + if (typeof record.message === 'string') return record.message; + if (Array.isArray(record.errors) && record.errors.length > 0) { + const first = record.errors[0] as Record; + if (typeof first?.message === 'string') return first.message; + if (typeof first?.error === 'string') return first.error; + } + return undefined; + } + + #tileCount({ w, h }: { w: number; h: number }) { + return Math.ceil(w / 512) * Math.ceil(h / 512); + } + + #megapixels({ w, h }: { w: number; h: number }) { + return (w * h) / 1_000_000; + } + + #mimeForFormat(format?: string) { + if (format === 'jpeg') return 'image/jpeg'; + if (format === 'webp') return 'image/webp'; + return 'image/png'; + } + + #costForUnits(units: number, microCentsPerUnit?: number) { + if (!Number.isFinite(units) || units <= 0) return 0; + if ( + !Number.isFinite(microCentsPerUnit) || + (microCentsPerUnit as number) <= 0 + ) + return 0; + return Math.round(units * (microCentsPerUnit as number)); + } + + // `numerator` is in millionths of a unit (e.g. pixels out of 1,000,000 for MP-based pricing). + #costForMillionUnits(numerator: number, microCentsPerMillion?: number) { + if (!Number.isFinite(numerator) || numerator <= 0) return 0; + if ( + !Number.isFinite(microCentsPerMillion) || + (microCentsPerMillion as number) <= 0 + ) + return 0; + return Math.round( + (numerator * (microCentsPerMillion as number)) / 1_000_000, + ); + } + + #getMeteringModelKey(model: CloudflareImageModel) { + if (model.puterId && typeof model.puterId === 'string') { + return model.puterId; + } + + if (model.id.startsWith('@cf/')) { + return `workers-ai:${model.id.slice('@cf/'.length)}`; + } + + return model.id.replace(/^@+/, ''); + } +} diff --git a/src/backend/src/services/ai/image/providers/CloudflareImageGenerationProvider/models.ts b/src/backend/drivers/ai-image/providers/cloudflare/models.ts similarity index 88% rename from src/backend/src/services/ai/image/providers/CloudflareImageGenerationProvider/models.ts rename to src/backend/drivers/ai-image/providers/cloudflare/models.ts index 8930798f9..c032519e5 100644 --- a/src/backend/src/services/ai/image/providers/CloudflareImageGenerationProvider/models.ts +++ b/src/backend/drivers/ai-image/providers/cloudflare/models.ts @@ -17,7 +17,7 @@ * along with this program. If not, see . */ -import { IImageModel } from '../types'; +import type { IImageModel } from '../../types.js'; export type CloudflareBillingScheme = | 'tile-plus-step' @@ -38,9 +38,7 @@ export const CLOUDFLARE_IMAGE_GENERATION_MODELS: CloudflareImageModel[] = [ { puterId: 'workers-ai:black-forest-labs/flux.1-schnell', id: '@cf/black-forest-labs/flux-1-schnell', - aliases: [ - 'black-forest-labs/flux.1-schnell', - ], + aliases: ['black-forest-labs/flux.1-schnell'], name: 'FLUX.1 Schnell', costs_currency: 'usd-microcents', index_cost_key: 'step', @@ -54,9 +52,7 @@ export const CLOUDFLARE_IMAGE_GENERATION_MODELS: CloudflareImageModel[] = [ { puterId: 'workers-ai:leonardo/lucid-origin', id: '@cf/leonardo/lucid-origin', - aliases: [ - 'leonardo/lucid-origin', - ], + aliases: ['leonardo/lucid-origin'], name: 'Lucid Origin', costs_currency: 'usd-microcents', index_cost_key: 'step', @@ -70,9 +66,7 @@ export const CLOUDFLARE_IMAGE_GENERATION_MODELS: CloudflareImageModel[] = [ { puterId: 'workers-ai:leonardo/phoenix-1.0', id: '@cf/leonardo/phoenix-1.0', - aliases: [ - 'leonardo/phoenix-1.0', - ], + aliases: ['leonardo/phoenix-1.0'], name: 'Phoenix 1.0', costs_currency: 'usd-microcents', index_cost_key: 'step', @@ -86,9 +80,7 @@ export const CLOUDFLARE_IMAGE_GENERATION_MODELS: CloudflareImageModel[] = [ { puterId: 'workers-ai:black-forest-labs/flux.2-dev', id: '@cf/black-forest-labs/flux-2-dev', - aliases: [ - 'black-forest-labs/flux.2-dev', - ], + aliases: ['black-forest-labs/flux.2-dev'], name: 'FLUX.2 Dev', costs_currency: 'usd-microcents', index_cost_key: 'input_tile_512_per_step', @@ -103,9 +95,7 @@ export const CLOUDFLARE_IMAGE_GENERATION_MODELS: CloudflareImageModel[] = [ { puterId: 'workers-ai:black-forest-labs/flux.2-klein-4b', id: '@cf/black-forest-labs/flux-2-klein-4b', - aliases: [ - 'black-forest-labs/flux.2-klein-4b', - ], + aliases: ['black-forest-labs/flux.2-klein-4b'], name: 'FLUX.2 Klein 4B', costs_currency: 'usd-microcents', index_cost_key: 'input_tile_512', @@ -119,9 +109,7 @@ export const CLOUDFLARE_IMAGE_GENERATION_MODELS: CloudflareImageModel[] = [ { puterId: 'workers-ai:black-forest-labs/flux.2-klein-9b', id: '@cf/black-forest-labs/flux-2-klein-9b', - aliases: [ - 'black-forest-labs/flux.2-klein-9b', - ], + aliases: ['black-forest-labs/flux.2-klein-9b'], name: 'FLUX.2 Klein 9B', costs_currency: 'usd-microcents', index_cost_key: 'first_mp', diff --git a/src/backend/drivers/ai-image/providers/gemini/GeminiImageProvider.ts b/src/backend/drivers/ai-image/providers/gemini/GeminiImageProvider.ts new file mode 100644 index 000000000..cf15922c2 --- /dev/null +++ b/src/backend/drivers/ai-image/providers/gemini/GeminiImageProvider.ts @@ -0,0 +1,480 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { GenerateContentResponse, GoogleGenAI } from '@google/genai'; +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import { + GEMINI_DEFAULT_RATIO, + GEMINI_ESTIMATED_IMAGE_TOKENS, + GEMINI_IMAGE_GENERATION_MODELS, + IGeminiImageModel, +} from './models.js'; +import type { + IGenerateParams, + IImageModel, + IImageProvider, +} from '../../types.js'; + +const MIME_SIGNATURES: Record = { + '/9j/': 'image/jpeg', + iVBOR: 'image/png', + UklGR: 'image/webp', +}; + +interface GeminiUsageMetadata { + promptTokenCount: number; + candidatesTokenCount: number; + candidatesTextTokenCount: number; + candidatesImageTokenCount: number; + thoughtsTokenCount: number; +} + +export class GeminiImageProvider implements IImageProvider { + #meteringService: MeteringService; + #client: GoogleGenAI; + + constructor(config: { apiKey: string }, meteringService: MeteringService) { + if (!config.apiKey) { + throw new Error('Gemini image generation requires an API key'); + } + this.#meteringService = meteringService; + this.#client = new GoogleGenAI({ apiKey: config.apiKey }); + } + + models(): IImageModel[] { + return GEMINI_IMAGE_GENERATION_MODELS; + } + + getDefaultModel(): string { + return GEMINI_IMAGE_GENERATION_MODELS[0].id; + } + + async generate(params: IGenerateParams): Promise { + const { + prompt, + test_mode, + input_image, + input_image_mime_type, + model, + quality, + } = params; + let { ratio, input_images } = params; + + const selectedModel = + (this.models() as IGeminiImageModel[]).find( + (m) => m.id === model, + ) || + (this.models() as IGeminiImageModel[]).find( + (m) => m.id === this.getDefaultModel(), + )!; + + if (test_mode) { + return 'https://puter-sample-data.puter.site/image_example.png'; + } + + if (typeof prompt !== 'string' || prompt.trim().length === 0) { + throw new Error('`prompt` must be a non-empty string'); + } + + if (selectedModel.apiType === 'generateImages') { + return this.#generateWithImagen(prompt, selectedModel, params); + } + + const allowedRatios = selectedModel.allowedRatios ?? [ + GEMINI_DEFAULT_RATIO, + ]; + ratio = + ratio && this.#isValidRatio(ratio, allowedRatios) + ? ratio + : allowedRatios[0]; + + // Backwards compat: merge singular input_image into input_images + if (input_image && (!input_images || input_images.length === 0)) { + input_images = [input_image]; + } + + // Validate input images have detectable MIME types + if (input_images?.length) { + for (const img of input_images) { + const mime = this.#detectMimeType(img) ?? input_image_mime_type; + if (!mime) { + throw new Error( + 'Could not detect MIME type for an input image. Provide a known image format (JPEG, PNG, WebP) or set `input_image_mime_type`.', + ); + } + } + } + + const actor = Context.get('actor'); + + // --- Pre-flight cost estimation --- + const inputImageCount = input_images?.length ?? 0; + const estimatedImageInputTokens = inputImageCount * 560; // https://ai.google.dev/gemini-api/docs/pricing#gemini-3-pro-image-preview + const estimatedPromptTokenCount = + this.#estimatePromptTokenCount(prompt) + estimatedImageInputTokens; + const estimatedInputCostInCents = this.#calculateTokenCostInCents( + estimatedPromptTokenCount, + selectedModel.costs.input, + ); + + // Estimate output image tokens + const imageTokenKey = quality + ? `${selectedModel.id}:${quality}` + : selectedModel.id; + const estimatedOutputImageTokens = + GEMINI_ESTIMATED_IMAGE_TOKENS[imageTokenKey] ?? + GEMINI_ESTIMATED_IMAGE_TOKENS[selectedModel.id]; + if (estimatedOutputImageTokens === undefined) { + throw new Error( + `No estimated image token count configured for '${imageTokenKey}'.`, + ); + } + const estimatedOutputImageCostInCents = this.#calculateTokenCostInCents( + estimatedOutputImageTokens, + selectedModel.costs.output_image, + ); + const estimatedOutputTextCostInCents = this.#calculateTokenCostInCents( + 50, + selectedModel.costs.output, + ); // small text overhead estimate + const estimatedOutputCostInCents = + estimatedOutputImageCostInCents + estimatedOutputTextCostInCents; + + const estimatedTotalCostInMicroCents = this.#toMicroCents( + estimatedInputCostInCents + estimatedOutputCostInCents, + ); + const usageAllowed = await this.#meteringService.hasEnoughCredits( + actor, + estimatedTotalCostInMicroCents, + ); + + if (!usageAllowed) { + throw new Error('Insufficient credits for image generation'); + } + + // --- API call --- + const contents = this.#buildContents( + prompt, + input_images, + input_image_mime_type, + ); + const aspectRatio = `${ratio.w}:${ratio.h}`; + + const imageConfig: Record = { aspectRatio }; + if (quality && selectedModel.allowedQualityLevels?.includes(quality)) { + imageConfig.imageSize = quality; + } + + const response = await this.#client.models.generateContent({ + model: selectedModel.id, + contents, + config: { + responseModalities: ['TEXT', 'IMAGE'], + imageConfig, + }, + }); + + // --- Actual cost calculation from response usage --- + const usage = this.#extractUsageMetadata(response); + const inputTokenCount = + usage.promptTokenCount || estimatedPromptTokenCount; + + const outputTextTokenCount = + usage.candidatesTextTokenCount + usage.thoughtsTokenCount; + const outputImageTokenCount = + usage.candidatesImageTokenCount || estimatedOutputImageTokens; + + const inputCostInCents = this.#calculateTokenCostInCents( + inputTokenCount, + selectedModel.costs.input, + ); + const outputTextCostInCents = this.#calculateTokenCostInCents( + outputTextTokenCount, + selectedModel.costs.output, + ); + const outputImageCostInCents = this.#calculateTokenCostInCents( + outputImageTokenCount, + selectedModel.costs.output_image, + ); + + const usagePrefix = `gemini:${selectedModel.id}`; + this.#meteringService.batchIncrementUsages(actor, [ + { + usageType: `${usagePrefix}:input`, + usageAmount: Math.max(inputTokenCount, 1), + costOverride: this.#toMicroCents(inputCostInCents), + }, + { + usageType: `${usagePrefix}:output:text`, + usageAmount: Math.max(outputTextTokenCount, 1), + costOverride: this.#toMicroCents(outputTextCostInCents), + }, + { + usageType: `${usagePrefix}:output:image`, + usageAmount: Math.max(outputImageTokenCount, 1), + costOverride: this.#toMicroCents(outputImageCostInCents), + }, + ]); + + const url = this.#extractImageUrl(response); + + if (!url) { + throw new Error('Failed to extract image URL from Gemini response'); + } + + return url; + } + + async #generateWithImagen( + prompt: string, + selectedModel: IGeminiImageModel, + params: IGenerateParams, + ): Promise { + const actor = Context.get('actor'); + if (!actor) { + throw new Error('actor not found in context'); + } + const costCents = selectedModel.costs?.['per-image']; + if (costCents === undefined) { + throw new Error( + `No per-image cost configured for model '${selectedModel.id}'`, + ); + } + const costInMicroCents = Math.ceil(costCents * 1_000_000); + + const usageAllowed = await this.#meteringService.hasEnoughCredits( + actor, + costInMicroCents, + ); + if (!usageAllowed) { + throw new Error('Insufficient credits for image generation'); + } + + const allowedRatios = selectedModel.allowedRatios ?? [ + GEMINI_DEFAULT_RATIO, + ]; + const ratio = + params.ratio && this.#isValidRatio(params.ratio, allowedRatios) + ? params.ratio + : allowedRatios[0]; + const aspectRatio = `${ratio.w}:${ratio.h}`; + + const config: Record = { + numberOfImages: 1, + aspectRatio, + }; + + if ( + params.quality && + selectedModel.allowedQualityLevels?.includes(params.quality) + ) { + config.imageSize = params.quality; + } + + const response = await this.#client.models.generateImages({ + model: selectedModel.id, + prompt, + config, + }); + + const generated = response?.generatedImages; + if (!generated || generated.length === 0) { + throw new Error('Imagen response did not include an image'); + } + + const entry = generated[0]; + if (entry.raiFilteredReason) { + throw new Error(`Image was filtered: ${entry.raiFilteredReason}`); + } + + const image = entry.image; + if (!image?.imageBytes) { + throw new Error('Imagen response did not include image bytes'); + } + + const usageKey = `gemini:${selectedModel.id}`; + await this.#meteringService.incrementUsage( + actor, + usageKey, + 1, + costInMicroCents, + ); + + const mimeType = image.mimeType ?? 'image/png'; + return `data:${mimeType};base64,${image.imageBytes}`; + } + + #buildContents( + prompt: string, + input_images?: string[], + input_image_mime_type?: string, + ) { + const parts: Record[] = [{ text: prompt }]; + + if (input_images?.length) { + for (const img of input_images) { + const parsed = this.#parseDataUri(img); + const mimeType = + parsed?.mimeType ?? + this.#detectMimeType(img) ?? + input_image_mime_type ?? + 'image/png'; + const rawBase64 = parsed?.base64 ?? img; + parts.push({ + inlineData: { + mimeType, + data: rawBase64, + }, + }); + } + } + + return parts; + } + + #extractUsageMetadata( + response: GenerateContentResponse, + ): GeminiUsageMetadata { + const usage = ( + response as GenerateContentResponse & { + usageMetadata?: Record; + } + ).usageMetadata; + + let candidatesImageTokenCount = 0; + + const details = usage?.candidatesTokensDetails; + if (Array.isArray(details)) { + for (const entry of details) { + if (entry?.modality === 'IMAGE') { + candidatesImageTokenCount += this.#toSafeCount( + entry.tokenCount, + ); + } + } + } + + // api only returns modality image, so calculate text tokens as candidates (output) - image tokens + const candidatesTokenCount = this.#toSafeCount( + usage?.candidatesTokenCount, + ); + const candidatesTextTokenCount = Math.max( + 0, + candidatesTokenCount - candidatesImageTokenCount, + ); + + return { + promptTokenCount: this.#toSafeCount(usage?.promptTokenCount), + candidatesTokenCount, + candidatesTextTokenCount, + candidatesImageTokenCount, + thoughtsTokenCount: this.#toSafeCount(usage?.thoughtsTokenCount), + }; + } + + #estimatePromptTokenCount(prompt: string): number { + const text = prompt.trim(); + if (text.length === 0) return 0; + + // Same approximation used by chat billing flow. + return Math.max( + 1, + Math.floor( + (text.length / 4 + text.split(/\s+/).length * (4 / 3)) / 2, + ), + ); + } + + #calculateTokenCostInCents( + tokenCount: number, + centsPerMillion?: number, + ): number { + if (!Number.isFinite(tokenCount) || tokenCount <= 0) return 0; + if (!Number.isFinite(centsPerMillion) || (centsPerMillion ?? 0) <= 0) + return 0; + + return (tokenCount / 1_000_000) * (centsPerMillion as number); + } + + #toMicroCents(cents: number): number { + if (!Number.isFinite(cents) || cents <= 0) return 1; + return Math.ceil(cents * 1_000_000); + } + + #toSafeCount(value: unknown): number { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) + return 0; + return Math.floor(value); + } + + #extractImageUrl(response: GenerateContentResponse): string | undefined { + const parts = response?.candidates?.[0]?.content?.parts; + if (!Array.isArray(parts)) { + return undefined; + } + + for (const part of parts) { + if (part?.inlineData?.data) { + const mimeType = part.inlineData.mimeType ?? 'image/png'; + return `data:${mimeType};base64,${part.inlineData.data}`; + } + } + return undefined; + } + + #detectMimeType(data: string): string | undefined { + // Handle data URIs like "data:image/jpeg;base64,..." + const parsed = this.#parseDataUri(data); + if (parsed) { + return parsed.mimeType; + } + + for (const [signature, mimeType] of Object.entries(MIME_SIGNATURES)) { + if (data.startsWith(signature)) { + return mimeType; + } + } + return undefined; + } + + #parseDataUri( + data: string, + ): { mimeType: string; base64: string } | undefined { + if (!data.startsWith('data:image/')) return undefined; + + const commaIdx = data.indexOf(','); + if (commaIdx === -1) return undefined; + + const header = data.substring(5, commaIdx); // after "data:" up to "," + if (!header.endsWith(';base64')) return undefined; + + const mimeType = header.substring(0, header.length - 7); // strip ";base64" + if (mimeType.length === 0) return undefined; + + return { mimeType, base64: data.substring(commaIdx + 1) }; + } + + #isValidRatio( + ratio: { w: number; h: number }, + allowedRatios: { w: number; h: number }[], + ) { + return allowedRatios.some((r) => r.w === ratio.w && r.h === ratio.h); + } +} diff --git a/src/backend/src/services/ai/image/providers/GeminiImageGenerationProvider/models.ts b/src/backend/drivers/ai-image/providers/gemini/models.ts similarity index 97% rename from src/backend/src/services/ai/image/providers/GeminiImageGenerationProvider/models.ts rename to src/backend/drivers/ai-image/providers/gemini/models.ts index d59417bc7..90429f3e9 100644 --- a/src/backend/src/services/ai/image/providers/GeminiImageGenerationProvider/models.ts +++ b/src/backend/drivers/ai-image/providers/gemini/models.ts @@ -17,7 +17,7 @@ * along with this program. If not, see . */ -import { IImageModel } from '../types'; +import type { IImageModel } from '../../types.js'; export interface IGeminiImageModel extends IImageModel { apiType?: 'generateContent' | 'generateImages'; @@ -153,7 +153,7 @@ export const GEMINI_IMAGE_GENERATION_MODELS: IGeminiImageModel[] = [ }, }, - // ── Imagen models (use generateImages API) ───────────────────── + // -- Imagen models (use generateImages API) -- { puterId: 'google:google/imagen-4.0-fast', id: 'imagen-4.0-fast-generate-001', diff --git a/src/backend/drivers/ai-image/providers/openai/OpenAiImageProvider.ts b/src/backend/drivers/ai-image/providers/openai/OpenAiImageProvider.ts new file mode 100644 index 000000000..cec801f63 --- /dev/null +++ b/src/backend/drivers/ai-image/providers/openai/OpenAiImageProvider.ts @@ -0,0 +1,610 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import openai, { OpenAI } from 'openai'; +import { + ImageGenerateParamsNonStreaming, + ImagesResponse, +} from 'openai/resources/images.js'; +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { + IGenerateParams, + IImageModel, + IImageProvider, +} from '../../types.js'; +import { OPEN_AI_IMAGE_GENERATION_MODELS } from './models.js'; + +interface OpenAIImageUsage { + inputTokens: number; + outputTokens: number; + inputTextTokens: number; + inputImageTokens: number; + cachedInputTokens: number; + cachedInputTextTokens: number; + cachedInputImageTokens: number; +} + +/** + * OpenAI image generation provider for v2. + * Supports DALL-E 2/3 and GPT Image models. + */ +export class OpenAiImageProvider implements IImageProvider { + #meteringService: MeteringService; + #openai: OpenAI; + + static #NON_SIZE_COST_KEYS = [ + 'text_input', + 'text_cached_input', + 'text_output', + 'image_input', + 'image_cached_input', + 'image_output', + ]; + + constructor(config: { apiKey: string }, meteringService: MeteringService) { + this.#meteringService = meteringService; + this.#openai = new openai.OpenAI({ + apiKey: config.apiKey, + }); + } + + models() { + return OPEN_AI_IMAGE_GENERATION_MODELS; + } + + getDefaultModel(): string { + return 'dall-e-2'; + } + + async generate({ + prompt, + quality, + test_mode, + model, + ratio, + }: IGenerateParams) { + const selectedModel = + this.models().find((m) => m.id === model) || + this.models().find((m) => m.id === this.getDefaultModel())!; + + if (test_mode) { + return 'https://puter-sample-data.puter.site/image_example.png'; + } + + if (typeof prompt !== 'string') { + throw new Error('`prompt` must be a string'); + } + + const validRatios = selectedModel?.allowedRatios; + if (validRatios) { + if ( + !ratio || + !validRatios.some((r) => r.w === ratio.w && r.h === ratio.h) + ) { + ratio = validRatios[0]; // Default to the first allowed ratio + } + } else { + // Open-ended size models (gpt-image-2): conform to OpenAI's size + // rules (16px multiples, 3840 cap, 3:1 ratio, pixel budget). + ratio = this.#normalizeGptImage2Ratio(ratio); + } + + if (!ratio) { + ratio = { w: 1024, h: 1024 }; // Fallback ratio + } + + const validQualities = selectedModel?.allowedQualityLevels; + if (validQualities && (!quality || !validQualities.includes(quality))) { + quality = validQualities[0]; // Default to the first allowed quality + } + + const size = `${ratio.w}x${ratio.h}`; + const price_key = this.#buildPriceKey(selectedModel.id, quality!, size); + let outputPriceInCents: number | undefined = + selectedModel?.costs[price_key]; + if (outputPriceInCents === undefined) { + outputPriceInCents = this.#estimateOutputCostFromTokens( + selectedModel, + ratio, + quality, + ); + } + if (outputPriceInCents === undefined) { + const availableSizes = Object.keys(selectedModel?.costs).filter( + (key) => !OpenAiImageProvider.#NON_SIZE_COST_KEYS.includes(key), + ); + throw new Error( + `Invalid size/quality combination. Expected one of: ${availableSizes.join(', ')}. Got: ${price_key}`, + ); + } + + const actor = Context.get('actor'); + const userIdentifier = + actor?.user.id + actor?.app?.uid ? `:${actor?.app?.uid}` : ''; + + const estimatedPromptTokenCount = + this.#estimatePromptTokenCount(prompt); + const estimatedInputCostInCents = this.#calculateInputCostInCents( + selectedModel, + { + inputTokens: estimatedPromptTokenCount, + inputTextTokens: estimatedPromptTokenCount, + inputImageTokens: 0, + cachedInputTokens: 0, + cachedInputTextTokens: 0, + cachedInputImageTokens: 0, + } as OpenAIImageUsage, + ); + const estimatedOutputCostInCents = outputPriceInCents; + const estimatedTotalCostInMicroCents = this.#toMicroCents( + estimatedInputCostInCents + estimatedOutputCostInCents, + ); + const usageAllowed = await this.#meteringService.hasEnoughCredits( + actor, + estimatedTotalCostInMicroCents, + ); + + if (!usageAllowed) { + throw new Error('Insufficient credits for image generation'); + } + + // Build API parameters based on model + const apiParams = this.#buildApiParams(selectedModel.id, { + user: userIdentifier, + prompt, + size, + quality, + } as Partial); + + const result = await this.#openai.images.generate(apiParams); + + const usage = this.#extractUsage(result); + const hasInputTokenUsage = + usage.inputTokens > 0 || + usage.inputTextTokens > 0 || + usage.inputImageTokens > 0; + + const billableUsage = hasInputTokenUsage + ? usage + : { + ...usage, + inputTokens: estimatedPromptTokenCount, + inputTextTokens: estimatedPromptTokenCount, + }; + + const inputCostInCents = hasInputTokenUsage + ? this.#calculateInputCostInCents(selectedModel, billableUsage) + : estimatedInputCostInCents; + const outputCostInCents = this.#calculateOutputCostInCents( + selectedModel, + usage, + outputPriceInCents, + ); + + const usageType = `openai:${selectedModel.id}:${price_key}`; + const usageEntries: Array<{ + usageType: string; + usageAmount: number; + costOverride: number; + }> = []; + if (inputCostInCents > 0) { + usageEntries.push({ + usageType: `${usageType}:input`, + usageAmount: Math.max( + billableUsage.inputTokens || estimatedPromptTokenCount, + 1, + ), + costOverride: this.#toMicroCents(inputCostInCents), + }); + } + if (outputCostInCents > 0) { + usageEntries.push({ + usageType: `${usageType}:output`, + usageAmount: Math.max(usage.outputTokens, 1), + costOverride: this.#toMicroCents(outputCostInCents), + }); + } + if (usageEntries.length) { + this.#meteringService.batchIncrementUsages(actor, usageEntries); + } + + const url = + result.data?.[0]?.url || + (result.data?.[0]?.b64_json + ? `data:image/png;base64,${result.data[0].b64_json}` + : null); + + if (!url) { + throw new Error('Failed to extract image URL from OpenAI response'); + } + + return url; + } + + #extractUsage(result: ImagesResponse): OpenAIImageUsage { + const usage = (result.usage ?? {}) as ImagesResponse.Usage & + Record; + const inputTokens = this.#toSafeCount(usage.input_tokens); + const outputTokens = this.#toSafeCount(usage.output_tokens); + + const inputDetails = (usage.input_tokens_details ?? + {}) as unknown as Record; + const inputTextTokens = this.#toSafeCount(inputDetails.text_tokens); + const inputImageTokens = this.#toSafeCount(inputDetails.image_tokens); + + const cachedInputTokens = Math.max( + this.#toSafeCount( + (usage as Record).cached_input_tokens, + ), + this.#toSafeCount(inputDetails.cached_tokens), + ); + + const cachedDetails = ((inputDetails.cached_tokens_details || + inputDetails.cache_tokens_details) ?? + {}) as Record; + const cachedInputTextTokens = this.#toSafeCount( + cachedDetails.text_tokens, + ); + const cachedInputImageTokens = this.#toSafeCount( + cachedDetails.image_tokens, + ); + + return { + inputTokens, + outputTokens, + inputTextTokens, + inputImageTokens, + cachedInputTokens, + cachedInputTextTokens, + cachedInputImageTokens, + }; + } + + #calculateInputCostInCents( + selectedModel: IImageModel, + usage: OpenAIImageUsage, + ): number { + if (!this.#isGptImageModel(selectedModel.id)) { + return 0; + } + + const textInputRate = this.#getCostRate(selectedModel, 'text_input'); + const textCachedInputRate = + this.#getCostRate(selectedModel, 'text_cached_input') ?? + textInputRate; + const imageInputRate = this.#getCostRate(selectedModel, 'image_input'); + const imageCachedInputRate = + this.#getCostRate(selectedModel, 'image_cached_input') ?? + imageInputRate; + + if (textInputRate === undefined && imageInputRate === undefined) { + return 0; + } + + const totalInputTokens = Math.max( + usage.inputTokens, + usage.inputTextTokens + usage.inputImageTokens, + ); + let textTokens = usage.inputTextTokens; + const imageTokens = usage.inputImageTokens; + + // Current image generate calls are usually text-only prompts. + if (textTokens + imageTokens === 0 && totalInputTokens > 0) { + textTokens = totalInputTokens; + } + + const knownInputTokens = textTokens + imageTokens; + const cachedInputTokens = Math.min( + usage.cachedInputTokens, + knownInputTokens || totalInputTokens, + ); + + let cachedTextTokens = Math.min( + usage.cachedInputTextTokens, + textTokens, + ); + let cachedImageTokens = Math.min( + usage.cachedInputImageTokens, + imageTokens, + ); + + let cachedRemaining = Math.max( + 0, + cachedInputTokens - (cachedTextTokens + cachedImageTokens), + ); + if (cachedRemaining > 0) { + const availableText = Math.max(textTokens - cachedTextTokens, 0); + const availableImage = Math.max(imageTokens - cachedImageTokens, 0); + const availableTotal = availableText + availableImage; + + if (availableTotal > 0) { + const proportionalText = Math.min( + availableText, + Math.round( + (availableText / availableTotal) * cachedRemaining, + ), + ); + cachedTextTokens += proportionalText; + cachedRemaining -= proportionalText; + + const proportionalImage = Math.min( + availableImage, + cachedRemaining, + ); + cachedImageTokens += proportionalImage; + cachedRemaining -= proportionalImage; + } + + if (cachedRemaining > 0 && textTokens > cachedTextTokens) { + const extraText = Math.min( + textTokens - cachedTextTokens, + cachedRemaining, + ); + cachedTextTokens += extraText; + cachedRemaining -= extraText; + } + + if (cachedRemaining > 0 && imageTokens > cachedImageTokens) { + const extraImage = Math.min( + imageTokens - cachedImageTokens, + cachedRemaining, + ); + cachedImageTokens += extraImage; + cachedRemaining -= extraImage; + } + } + + const uncachedTextTokens = Math.max(textTokens - cachedTextTokens, 0); + const uncachedImageTokens = Math.max( + imageTokens - cachedImageTokens, + 0, + ); + + return ( + this.#costForTokens(uncachedTextTokens, textInputRate) + + this.#costForTokens(cachedTextTokens, textCachedInputRate) + + this.#costForTokens(uncachedImageTokens, imageInputRate) + + this.#costForTokens(cachedImageTokens, imageCachedInputRate) + ); + } + + #calculateOutputCostInCents( + selectedModel: IImageModel, + usage: OpenAIImageUsage, + fallbackPriceInCents: number, + ): number { + if (!this.#isGptImageModel(selectedModel.id)) { + return fallbackPriceInCents; + } + + if (usage.outputTokens <= 0) { + return fallbackPriceInCents; + } + + const imageOutputRate = this.#getCostRate( + selectedModel, + 'image_output', + ); + if (imageOutputRate !== undefined) { + return this.#costForTokens(usage.outputTokens, imageOutputRate); + } + + const textOutputRate = this.#getCostRate(selectedModel, 'text_output'); + if (textOutputRate !== undefined) { + return this.#costForTokens(usage.outputTokens, textOutputRate); + } + + return fallbackPriceInCents; + } + + #estimatePromptTokenCount(prompt: string): number { + const text = prompt.trim(); + if (text.length === 0) return 0; + + // Same approximation used by chat and Gemini image billing flows. + return Math.max( + 1, + Math.floor( + (text.length / 4 + text.split(/\s+/).length * (4 / 3)) / 2, + ), + ); + } + + #getCostRate(selectedModel: IImageModel, key: string): number | undefined { + const value = selectedModel.costs[key]; + if (!Number.isFinite(value)) { + return undefined; + } + return value; + } + + #costForTokens(tokenCount: number, centsPerMillion?: number): number { + if (!Number.isFinite(tokenCount) || tokenCount <= 0) return 0; + if (!Number.isFinite(centsPerMillion) || (centsPerMillion ?? 0) <= 0) + return 0; + return (tokenCount / 1_000_000) * (centsPerMillion as number); + } + + #toMicroCents(cents: number): number { + if (!Number.isFinite(cents) || cents <= 0) return 1; + return Math.ceil(cents * 1_000_000); + } + + #toSafeCount(value: unknown): number { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) + return 0; + return Math.floor(value); + } + + #isGptImageModel(model: string) { + // Covers gpt-image-1, gpt-image-1-mini, gpt-image-1.5, gpt-image-2 and future variants. + return model.startsWith('gpt-image-'); + } + + // gpt-image-2 size rules: each edge in [16, 3840] and a multiple of 16, + // long:short ratio <= 3:1, pixel count in [655360, 8294400]. Silently + // clamps/snaps rather than throwing so arbitrary user input is accepted. + // https://developers.openai.com/api/docs/guides/image-generation + #normalizeGptImage2Ratio(ratio?: { w: number; h: number }) { + const MIN_EDGE = 16; + const MAX_EDGE = 3840; + const STEP = 16; + const MAX_RATIO = 3; + const MIN_PIXELS = 655_360; + const MAX_PIXELS = 8_294_400; + + let w = Number(ratio?.w); + let h = Number(ratio?.h); + if (!Number.isFinite(w) || !Number.isFinite(h) || w <= 0 || h <= 0) { + return { w: 1024, h: 1024 }; + } + + // 1. Clamp long:short ratio to MAX_RATIO by shrinking the longer edge. + if (w / h > MAX_RATIO) w = h * MAX_RATIO; + else if (h / w > MAX_RATIO) h = w * MAX_RATIO; + + // 2. Cap each edge at MAX_EDGE, preserving aspect ratio. + if (w > MAX_EDGE) { + const s = MAX_EDGE / w; + w = MAX_EDGE; + h *= s; + } + if (h > MAX_EDGE) { + const s = MAX_EDGE / h; + h = MAX_EDGE; + w *= s; + } + + // 3. Scale uniformly into the pixel budget. + const prescaledPixels = w * h; + if (prescaledPixels < MIN_PIXELS) { + const s = Math.sqrt(MIN_PIXELS / prescaledPixels); + w *= s; + h *= s; + } else if (prescaledPixels > MAX_PIXELS) { + const s = Math.sqrt(MAX_PIXELS / prescaledPixels); + w *= s; + h *= s; + } + + // 4. Snap to STEP. Bias rounding direction so snap doesn't push pixels + // back out of the budget. + const dir = + prescaledPixels < MIN_PIXELS + ? 1 + : prescaledPixels > MAX_PIXELS + ? -1 + : 0; + const snap = (v: number) => { + const snapped = + dir > 0 + ? Math.ceil(v / STEP) * STEP + : dir < 0 + ? Math.floor(v / STEP) * STEP + : Math.round(v / STEP) * STEP; + return Math.max(MIN_EDGE, Math.min(MAX_EDGE, snapped)); + }; + w = snap(w); + h = snap(h); + + // 5. If snap rounding pushed ratio above MAX_RATIO, trim the longer + // edge by one STEP. Pixel budget had headroom from step 3 so this + // won't drop below MIN_PIXELS. + if (Math.max(w, h) / Math.min(w, h) > MAX_RATIO) { + if (w >= h) w = Math.max(MIN_EDGE, w - STEP); + else h = Math.max(MIN_EDGE, h - STEP); + } + return { w, h }; + } + + // extracted from calculator at https://developers.openai.com/api/docs/guides/image-generation#cost-and-latency + #estimateGptImage2OutputTokens( + width: number, + height: number, + quality?: string, + ): number { + const FACTORS: Record = { + low: 16, + medium: 48, + high: 96, + }; + const factor = FACTORS[quality ?? ''] ?? FACTORS.medium; + const longEdge = Math.max(width, height); + const shortEdge = Math.min(width, height); + const shortLatent = Math.round((factor * shortEdge) / longEdge); + const latentW = width >= height ? factor : shortLatent; + const latentH = width >= height ? shortLatent : factor; + const baseArea = latentW * latentH; + return Math.ceil((baseArea * (2_000_000 + width * height)) / 4_000_000); + } + + #estimateOutputCostFromTokens( + selectedModel: IImageModel, + ratio: { w: number; h: number }, + quality?: string, + ): number | undefined { + if (!selectedModel.id.startsWith('gpt-image-2')) return undefined; + const rate = this.#getCostRate(selectedModel, 'image_output'); + if (rate === undefined) return undefined; + const tokens = this.#estimateGptImage2OutputTokens( + ratio.w, + ratio.h, + quality, + ); + return this.#costForTokens(tokens, rate); + } + + #buildPriceKey(model: string, quality: string, size: string) { + if (this.#isGptImageModel(model)) { + // GPT image models use format: "quality:size" - default to low if not specified + const qualityLevel = quality || 'low'; + return `${qualityLevel}:${size}`; + } + + // DALL-E models use format: "hd:size" or just "size" + return (quality === 'hd' ? 'hd:' : '') + size; + } + + #buildApiParams( + model: string, + baseParams: Partial, + ): ImageGenerateParamsNonStreaming { + const apiParams = { + user: baseParams.user, + prompt: baseParams.prompt, + size: baseParams.size, + } as ImageGenerateParamsNonStreaming; + + if (this.#isGptImageModel(model)) { + // GPT image models require the model parameter and use quality mapping + apiParams.model = model; + // Default to low quality if not specified, consistent with _buildPriceKey + apiParams.quality = baseParams.quality || 'low'; + } else { + // dall-e models + apiParams.model = model; + if (baseParams.quality === 'hd') { + apiParams.quality = 'hd'; + } + } + + return apiParams; + } +} diff --git a/src/backend/src/services/ai/image/providers/OpenAiImageGenerationProvider/models.ts b/src/backend/drivers/ai-image/providers/openai/models.ts similarity index 85% rename from src/backend/src/services/ai/image/providers/OpenAiImageGenerationProvider/models.ts rename to src/backend/drivers/ai-image/providers/openai/models.ts index de62f7584..16821f01d 100644 --- a/src/backend/src/services/ai/image/providers/OpenAiImageGenerationProvider/models.ts +++ b/src/backend/drivers/ai-image/providers/openai/models.ts @@ -1,4 +1,4 @@ -import { IImageModel } from '../types'; +import type { IImageModel } from '../../types.js'; export const OPEN_AI_IMAGE_GENERATION_MODELS: IImageModel[] = [ { @@ -51,7 +51,11 @@ export const OPEN_AI_IMAGE_GENERATION_MODELS: IImageModel[] = [ 'high:1536x1024': 20, }, allowedQualityLevels: ['low', 'medium', 'high'], - allowedRatios: [{ w: 1024, h: 1024 }, { w: 1024, h: 1536 }, { w: 1536, h: 1024 }], + allowedRatios: [ + { w: 1024, h: 1024 }, + { w: 1024, h: 1536 }, + { w: 1536, h: 1024 }, + ], }, { puterId: 'openai:openai/gpt-image-1-mini', @@ -81,7 +85,11 @@ export const OPEN_AI_IMAGE_GENERATION_MODELS: IImageModel[] = [ 'high:1536x1024': 5.2, }, allowedQualityLevels: ['low', 'medium', 'high'], - allowedRatios: [{ w: 1024, h: 1024 }, { w: 1024, h: 1536 }, { w: 1536, h: 1024 }], + allowedRatios: [ + { w: 1024, h: 1024 }, + { w: 1024, h: 1536 }, + { w: 1536, h: 1024 }, + ], }, { puterId: 'openai:openai/gpt-image-1', @@ -111,13 +119,17 @@ export const OPEN_AI_IMAGE_GENERATION_MODELS: IImageModel[] = [ 'high:1536x1024': 25, }, allowedQualityLevels: ['low', 'medium', 'high'], - allowedRatios: [{ w: 1024, h: 1024 }, { w: 1024, h: 1536 }, { w: 1536, h: 1024 }], + allowedRatios: [ + { w: 1024, h: 1024 }, + { w: 1024, h: 1536 }, + { w: 1536, h: 1024 }, + ], }, { puterId: 'openai:openai/dall-e-3', id: 'dall-e-3', aliases: ['openai/dall-e-3'], - name: 'DALL·E 3', + name: 'DALL-E 3', version: '1.0', costs_currency: 'usd-cents', index_cost_key: '1024x1024', @@ -130,13 +142,17 @@ export const OPEN_AI_IMAGE_GENERATION_MODELS: IImageModel[] = [ 'hd:1792x1024': 12, }, allowedQualityLevels: ['', 'hd'], - allowedRatios: [{ w: 1024, h: 1024 }, { w: 1024, h: 1792 }, { w: 1792, h: 1024 }], + allowedRatios: [ + { w: 1024, h: 1024 }, + { w: 1024, h: 1792 }, + { w: 1792, h: 1024 }, + ], }, { puterId: 'openai:openai/dall-e-2', id: 'dall-e-2', aliases: ['openai/dall-e-2'], - name: 'DALL·E 2', + name: 'DALL-E 2', version: '1.0', costs_currency: 'usd-cents', index_cost_key: '1024x1024', @@ -145,6 +161,10 @@ export const OPEN_AI_IMAGE_GENERATION_MODELS: IImageModel[] = [ '512x512': 1.8, '1024x1024': 2, }, - allowedRatios: [{ w: 256, h: 256 }, { w: 512, h: 512 }, { w: 1024, h: 1024 }], + allowedRatios: [ + { w: 256, h: 256 }, + { w: 512, h: 512 }, + { w: 1024, h: 1024 }, + ], }, ]; diff --git a/src/backend/drivers/ai-image/providers/replicate/ReplicateImageGenerationProvider.ts b/src/backend/drivers/ai-image/providers/replicate/ReplicateImageGenerationProvider.ts new file mode 100644 index 000000000..ea379db65 --- /dev/null +++ b/src/backend/drivers/ai-image/providers/replicate/ReplicateImageGenerationProvider.ts @@ -0,0 +1,330 @@ +import Replicate from 'replicate'; +import sharp from 'sharp'; +import type { Actor } from '../../../../core/actor.js'; +import { Context } from '../../../../core/context.js'; +import { HttpError } from '../../../../core/http/HttpError.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import { secureFetch } from '../../../../util/secureHttp.js'; +import type { IGenerateParams, IImageProvider } from '../../types.js'; +import { + REPLICATE_IMAGE_GENERATION_MODELS, + type ReplicateImageModel, +} from './models.js'; + +const DEFAULT_MODEL = 'black-forest-labs/flux-schnell'; +const DEFAULT_RATIO = { w: 1024, h: 1024 }; + +type ReplicateGenerateParams = IGenerateParams & { + go_fast?: boolean; + seed?: number; + steps?: number; + guidance?: number; + output_quality?: number; + output_megapixels?: string; + prompt_strength?: number; + negative_prompt?: string; + response_format?: string; + disable_safety_checker?: boolean; +}; + +export class ReplicateImageGenerationProvider implements IImageProvider { + #client: Replicate; + #meteringService: MeteringService; + + constructor(config: { apiKey: string }, meteringService: MeteringService) { + if (!config.apiKey) { + throw new Error('Replicate image generation requires an API key'); + } + this.#client = new Replicate({ auth: config.apiKey }); + this.#meteringService = meteringService; + } + + models() { + return REPLICATE_IMAGE_GENERATION_MODELS; + } + + getDefaultModel(): string { + return DEFAULT_MODEL; + } + + async generate(params: IGenerateParams): Promise { + const extra = params as ReplicateGenerateParams; + const { prompt, test_mode } = extra; + + const selectedModel = this.#getModel(extra.model); + const ratio = this.#normalizeRatio(extra.ratio); + + if (test_mode) { + return 'https://puter-sample-data.puter.site/image_example.png'; + } + + if (typeof prompt !== 'string' || prompt.trim().length === 0) { + throw new Error('`prompt` must be a non-empty string'); + } + + const actor = Context.get('actor'); + if (!actor) { + throw new HttpError(401, 'actor not found in context'); + } + + const goFast = selectedModel.supportsGoFast + ? extra.go_fast !== undefined + ? !!extra.go_fast + : (selectedModel.goFastDefault ?? false) + : false; + + const inputImages: string[] = []; + if (selectedModel.imageInputKey) { + if (extra.input_image) inputImages.push(extra.input_image); + if (extra.input_images?.length) + inputImages.push(...extra.input_images); + } + const singleImage = selectedModel.singleImageInputKey + ? extra.input_image + : undefined; + const allInputUrls = singleImage ? [singleImage] : inputImages; + const inputMp = + allInputUrls.length > 0 + ? await this.#measureInputMegapixels(allInputUrls) + : 0; + + const outputMp = this.#resolveOutputMegapixels(extra.output_megapixels); + + const totalCostMicroCents = this.#estimateCost( + selectedModel, + outputMp, + goFast, + inputMp, + ); + if (totalCostMicroCents <= 0) { + throw new Error( + `Error calculating cost for Replicate model ${selectedModel.id}`, + ); + } + const usageAllowed = await this.#meteringService.hasEnoughCredits( + actor, + totalCostMicroCents, + ); + if (!usageAllowed) { + throw new Error('Insufficient credits for image generation'); + } + + const input: Record = { + prompt, + aspect_ratio: this.#toAspectRatio(ratio), + disable_safety_checker: !!extra.disable_safety_checker, + }; + if (selectedModel.supportsGoFast) { + input.go_fast = goFast; + } + if (inputImages.length && selectedModel.imageInputKey) { + input[selectedModel.imageInputKey] = inputImages; + } else if (singleImage && selectedModel.singleImageInputKey) { + input[selectedModel.singleImageInputKey] = singleImage; + } + if (Number.isFinite(extra.seed)) + input.seed = Math.round(extra.seed as number); + if (Number.isFinite(extra.steps)) + input.num_inference_steps = Math.round(extra.steps as number); + if (Number.isFinite(extra.guidance)) input.guidance = extra.guidance; + if (Number.isFinite(extra.output_quality)) + input.output_quality = Math.round(extra.output_quality as number); + if ( + typeof extra.output_megapixels === 'string' && + selectedModel.resolutionInputKey + ) { + input[selectedModel.resolutionInputKey] = + extra.output_megapixels + + (selectedModel.resolutionSuffix ?? ''); + } else if (typeof extra.output_megapixels === 'string') { + input.megapixels = extra.output_megapixels; + } + if (Number.isFinite(extra.prompt_strength)) + input.prompt_strength = extra.prompt_strength; + if (typeof extra.negative_prompt === 'string') + input.negative_prompt = extra.negative_prompt; + if (typeof extra.response_format === 'string') + input.output_format = extra.response_format; + + const output = await this.#client.run( + selectedModel.replicateId as `${string}/${string}`, + { input }, + ); + + const url = this.#extractUrl(output); + if (!url) { + throw new Error( + 'Failed to extract image URL from Replicate response', + ); + } + + this.#recordUsage(actor, selectedModel, outputMp, goFast, inputMp); + + return url; + } + + #getModel(model?: string): ReplicateImageModel { + const models = REPLICATE_IMAGE_GENERATION_MODELS; + const found = models.find( + (m) => m.id === model || m.aliases?.includes(model ?? ''), + ); + return found ?? models.find((m) => m.id === DEFAULT_MODEL)!; + } + + #normalizeRatio(ratio?: { w: number; h: number }) { + const w = Number(ratio?.w); + const h = Number(ratio?.h); + if (Number.isFinite(w) && Number.isFinite(h) && w > 0 && h > 0) { + return { w: Math.round(w), h: Math.round(h) }; + } + return { ...DEFAULT_RATIO }; + } + + #toAspectRatio(ratio: { w: number; h: number }): string { + const g = this.#gcd(ratio.w, ratio.h); + return `${ratio.w / g}:${ratio.h / g}`; + } + + #gcd(a: number, b: number): number { + return b === 0 ? a : this.#gcd(b, a % b); + } + + #resolveOutputMegapixels(userValue?: string): number { + if (typeof userValue === 'string') { + const parsed = parseFloat(userValue); + if (Number.isFinite(parsed) && parsed > 0) return parsed; + } + return 1; + } + + async #measureInputMegapixels(imageUrls: string[]): Promise { + let totalMp = 0; + for (const url of imageUrls) { + try { + // User-supplied URLs: SSRF-guarded + (optionally) proxied. + const res = await secureFetch(url); + const buffer = Buffer.from(await res.arrayBuffer()); + const meta = await sharp(buffer).metadata(); + if (meta.width && meta.height) { + totalMp += Math.ceil( + (meta.width * meta.height) / 1_000_000, + ); + } + } catch { + totalMp += 1; + } + } + return totalMp; + } + + #resolveCosts( + model: ReplicateImageModel, + goFast: boolean, + ): Record { + return goFast && model.costs_go_fast + ? model.costs_go_fast + : model.costs; + } + + #estimateCost( + model: ReplicateImageModel, + outputMp: number, + goFast: boolean, + inputMp: number, + ): number { + const costs = this.#resolveCosts(model, goFast); + + if (model.billingScheme === 'per-image') { + const cents = costs.output; + if (!cents || cents <= 0) { + throw new Error( + `Replicate model ${model.id} has no valid per-image cost configured`, + ); + } + return Math.round(cents * 1_000_000); + } + + const runCents = costs.run ?? 0; + const outputMpCents = costs.output_mp; + if (!outputMpCents || outputMpCents <= 0) { + throw new Error( + `Replicate model ${model.id} has no valid output_mp cost configured`, + ); + } + const inputMpCents = (costs.input_mp ?? 0) * inputMp; + return Math.round( + (runCents + outputMpCents * outputMp + inputMpCents) * 1_000_000, + ); + } + + #recordUsage( + actor: Actor, + model: ReplicateImageModel, + outputMp: number, + goFast: boolean, + inputMp: number, + ) { + const prefix = `replicate:${model.id}`; + const costs = this.#resolveCosts(model, goFast); + + if (model.billingScheme === 'per-image') { + const cents = costs.output; + if (!cents || cents <= 0) return; + this.#meteringService.incrementUsage( + actor, + `${prefix}:output`, + 1, + Math.round(cents * 1_000_000), + ); + return; + } + + const components: { + usageType: string; + usageAmount: number; + costOverride: number; + }[] = []; + + const runCents = costs.run ?? 0; + if (runCents > 0) { + components.push({ + usageType: `${prefix}:run`, + usageAmount: 1, + costOverride: Math.round(runCents * 1_000_000), + }); + } + + const outputMpCents = costs.output_mp ?? 0; + if (outputMpCents > 0) { + components.push({ + usageType: `${prefix}:output_mp`, + usageAmount: outputMp, + costOverride: Math.round(outputMpCents * outputMp * 1_000_000), + }); + } + + const inputMpCents = costs.input_mp ?? 0; + if (inputMpCents > 0 && inputMp > 0) { + components.push({ + usageType: `${prefix}:input_mp`, + usageAmount: inputMp, + costOverride: Math.round(inputMpCents * inputMp * 1_000_000), + }); + } + + if (components.length > 0) { + this.#meteringService.batchIncrementUsages(actor, components); + } + } + + #extractUrl(output: unknown): string | undefined { + if (typeof output === 'string') return output; + if (Array.isArray(output)) { + const first = output[0]; + if (typeof first === 'string') return first; + if (first && typeof first === 'object') return String(first); + } + if (output && typeof output === 'object') return String(output); + return undefined; + } +} diff --git a/src/backend/src/services/ai/image/providers/ReplicateImageGenerationProvider/models.ts b/src/backend/drivers/ai-image/providers/replicate/models.ts similarity index 62% rename from src/backend/src/services/ai/image/providers/ReplicateImageGenerationProvider/models.ts rename to src/backend/drivers/ai-image/providers/replicate/models.ts index c5eb7fad0..e3e55b95c 100644 --- a/src/backend/src/services/ai/image/providers/ReplicateImageGenerationProvider/models.ts +++ b/src/backend/drivers/ai-image/providers/replicate/models.ts @@ -1,44 +1,25 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { IImageModel } from '../types'; +import { IImageModel } from '../../types.js'; export type ReplicateBillingScheme = 'per-image' | 'megapixel'; export type ReplicateImageModel = IImageModel & { replicateId: string; billingScheme: ReplicateBillingScheme; - imageInputKey?: string; // our input_images - singleImageInputKey?: string; // our input_image + imageInputKey?: string; + singleImageInputKey?: string; supportsGoFast?: boolean; goFastDefault?: boolean; costs_go_fast?: Record; - resolutionInputKey?: string; // our output_megapixels + resolutionInputKey?: string; resolutionSuffix?: string; }; // Costs are in USD cents. -// Megapixel models: output_mp = cost per output megapixel, input_mp = cost per input megapixel (img2img). -// Some also have a flat per-run base cost. -// Per-image models: output = flat cost per generated image. +// Megapixel models: `output_mp` = cost per output megapixel, `input_mp` = +// cost per input megapixel (img2img). Some also carry a flat `run` cost. +// Per-image models: `output` = flat cost per generated image. export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [ - // --- Black Forest Labs FLUX.2 --- + // Black Forest Labs FLUX.2 { id: 'black-forest-labs/flux-2-pro', replicateId: 'black-forest-labs/flux-2-pro', @@ -47,11 +28,7 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [ name: 'FLUX.2 Pro', costs_currency: 'usd-cents', index_cost_key: 'output_mp', - costs: { - run: 1.5, // $0.015 flat per run - input_mp: 1.5, // $0.015 per input megapixel - output_mp: 1.5, // $0.015 per output megapixel - }, + costs: { run: 1.5, input_mp: 1.5, output_mp: 1.5 }, billingScheme: 'megapixel', imageInputKey: 'input_images', resolutionInputKey: 'resolution', @@ -65,14 +42,8 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [ name: 'FLUX.2 Dev', costs_currency: 'usd-cents', index_cost_key: 'output_mp', - costs: { - input_mp: 1.4, // $0.014 per MP (go_fast=false) - output_mp: 1.4, - }, - costs_go_fast: { - input_mp: 1.2, // $0.012 per MP (go_fast=true) - output_mp: 1.2, - }, + costs: { input_mp: 1.4, output_mp: 1.4 }, + costs_go_fast: { input_mp: 1.2, output_mp: 1.2 }, billingScheme: 'megapixel', imageInputKey: 'input_images', supportsGoFast: true, @@ -87,10 +58,7 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [ name: 'FLUX.2 Klein 9B', costs_currency: 'usd-cents', index_cost_key: 'output_mp', - costs: { - input_mp: 1.1, // $0.011 per MP - output_mp: 1.1, - }, + costs: { input_mp: 1.1, output_mp: 1.1 }, billingScheme: 'megapixel', imageInputKey: 'images', resolutionInputKey: 'output_megapixels', @@ -103,16 +71,13 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [ name: 'FLUX.2 Klein 4B', costs_currency: 'usd-cents', index_cost_key: 'output_mp', - costs: { - input_mp: 0.1, // $0.001 per MP - output_mp: 0.1, - }, + costs: { input_mp: 0.1, output_mp: 0.1 }, billingScheme: 'megapixel', imageInputKey: 'images', resolutionInputKey: 'output_megapixels', }, - // --- Black Forest Labs FLUX.1 --- + // Black Forest Labs FLUX.1 { id: 'black-forest-labs/flux-schnell', replicateId: 'black-forest-labs/flux-schnell', @@ -121,9 +86,7 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [ name: 'FLUX.1 Schnell', costs_currency: 'usd-cents', index_cost_key: 'output', - costs: { - output: 0.3, // $0.003 per image - }, + costs: { output: 0.3 }, billingScheme: 'per-image', }, { @@ -134,14 +97,12 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [ name: 'FLUX 1.1 Pro', costs_currency: 'usd-cents', index_cost_key: 'output', - costs: { - output: 4, // $0.04 per image - }, + costs: { output: 4 }, billingScheme: 'per-image', singleImageInputKey: 'image_prompt', }, - // --- Leonardo AI --- + // Leonardo AI { id: 'leonardoai/lucid-origin', replicateId: 'leonardoai/lucid-origin', @@ -150,9 +111,7 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [ name: 'Lucid Origin', costs_currency: 'usd-cents', index_cost_key: 'output', - costs: { - output: 1.65, // ~11 units * $0.0015/unit = $0.0165 - }, + costs: { output: 1.65 }, billingScheme: 'per-image', }, { @@ -163,9 +122,7 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [ name: 'Phoenix 1.0', costs_currency: 'usd-cents', index_cost_key: 'output', - costs: { - output: 3.75, // ~25 units * $0.0015/unit = $0.0375 - }, + costs: { output: 3.75 }, billingScheme: 'per-image', }, ]; diff --git a/src/backend/src/services/ai/image/providers/TogetherImageGenerationProvider/TogetherImageGenerationProvider.ts b/src/backend/drivers/ai-image/providers/together/TogetherImageProvider.ts similarity index 50% rename from src/backend/src/services/ai/image/providers/TogetherImageGenerationProvider/TogetherImageGenerationProvider.ts rename to src/backend/drivers/ai-image/providers/together/TogetherImageProvider.ts index 9c35e9e44..04bcd9e6a 100644 --- a/src/backend/src/services/ai/image/providers/TogetherImageGenerationProvider/TogetherImageGenerationProvider.ts +++ b/src/backend/drivers/ai-image/providers/together/TogetherImageProvider.ts @@ -18,12 +18,13 @@ */ import { Together } from 'together-ai'; -import APIError from '../../../../../api/APIError.js'; -import { ErrorService } from '../../../../../modules/core/ErrorService.js'; -import { Context } from '../../../../../util/context.js'; -import { EventService } from '../../../../EventService.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import { IGenerateParams, IImageModel, IImageProvider } from '../types.js'; +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { + IGenerateParams, + IImageModel, + IImageProvider, +} from '../../types.js'; import { TOGETHER_IMAGE_GENERATION_MODELS } from './models.js'; const TOGETHER_DEFAULT_RATIO = { w: 1024, h: 1024 }; @@ -48,55 +49,45 @@ const CONDITION_IMAGE_MODELS = [ 'togetherai:black-forest-labs/flux.1-kontext-max', ]; -export class TogetherImageGenerationProvider implements IImageProvider { +export class TogetherImageProvider implements IImageProvider { #client: Together; #meteringService: MeteringService; - #errors: ErrorService; - #eventService: EventService; - constructor (config: { apiKey: string }, meteringService: MeteringService, errorService: ErrorService, eventService: EventService) { - if ( ! config.apiKey ) { + constructor(config: { apiKey: string }, meteringService: MeteringService) { + if (!config.apiKey) { throw new Error('Together AI image generation requires an API key'); } this.#meteringService = meteringService; - this.#errors = errorService; - this.#eventService = eventService; this.#client = new Together({ apiKey: config.apiKey }); } - models (): IImageModel[] { + models(): IImageModel[] { return TOGETHER_IMAGE_GENERATION_MODELS; } - getDefaultModel (): string { + getDefaultModel(): string { return DEFAULT_MODEL; } - async generate (params: IGenerateParams): Promise { + async generate(params: IGenerateParams): Promise { const { prompt, test_mode } = params; let { model, ratio, quality } = params; const options = params as TogetherGenerateParams; const selectedModel = this.#getModel(model); - await this.#eventService.emit('ai.log.image', { actor: Context.get('actor'), parameters: params, completionId: '0', intended_service: selectedModel.id }); - - if ( test_mode ) { + if (test_mode) { return 'https://puter-sample-data.puter.site/image_example.png'; } - if ( typeof prompt !== 'string' || prompt.trim().length === 0 ) { + if (typeof prompt !== 'string' || prompt.trim().length === 0) { throw new Error('`prompt` must be a non-empty string'); } ratio = ratio || TOGETHER_DEFAULT_RATIO; const actor = Context.get('actor'); - if ( ! actor ) { - this.#errors.report('together-image-generation:unknown-actor', { - message: 'failed to resolve actor for Together image generation', - trace: true, - }); + if (!actor) { throw new Error('actor not found in context'); } @@ -106,20 +97,23 @@ export class TogetherImageGenerationProvider implements IImageProvider { let usageAmount: number; let usageKey: string; - if ( pricingUnit === 'per-image' ) { + if (pricingUnit === 'per-image') { const centsPerImage = selectedModel.costs['per-image']; - if ( centsPerImage === undefined ) { - throw new Error(`Model ${selectedModel.id} missing 'per-image' cost`); + if (centsPerImage === undefined) { + throw new Error( + `Model ${selectedModel.id} missing 'per-image' cost`, + ); } costInMicroCents = centsPerImage * 1_000_000; usageAmount = 1; usageKey = 'per-image'; - } else if ( pricingUnit === 'per-tier' ) { - const tierKey = quality && selectedModel.costs[quality] !== undefined - ? quality - : Object.keys(selectedModel.costs)[0]; + } else if (pricingUnit === 'per-tier') { + const tierKey = + quality && selectedModel.costs[quality] !== undefined + ? quality + : Object.keys(selectedModel.costs)[0]; const centsPerImage = selectedModel.costs[tierKey]; - if ( centsPerImage === undefined ) { + if (centsPerImage === undefined) { throw new Error(`Model ${selectedModel.id} missing tier cost`); } costInMicroCents = centsPerImage * 1_000_000; @@ -127,7 +121,7 @@ export class TogetherImageGenerationProvider implements IImageProvider { usageKey = tierKey; } else { const centsPerMP = selectedModel.costs['1MP']; - if ( centsPerMP === undefined ) { + if (centsPerMP === undefined) { throw new Error(`Model ${selectedModel.id} missing '1MP' cost`); } const MP = (ratio.h * ratio.w) / 1_000_000; @@ -138,51 +132,84 @@ export class TogetherImageGenerationProvider implements IImageProvider { const usageType = `${selectedModel.id}:${usageKey}`; - const usageAllowed = await this.#meteringService.hasEnoughCredits(actor, costInMicroCents); + const usageAllowed = await this.#meteringService.hasEnoughCredits( + actor, + costInMicroCents, + ); - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); + if (!usageAllowed) { + throw new Error('Insufficient credits for image generation'); } // Resolve abstract aspect ratios (e.g. 1:1, 16:9) to concrete pixel // dimensions via the model's own resolution_map. let resolvedRatio = ratio; - if ( pricingUnit === 'per-tier' && quality && selectedModel.resolution_map ) { + if ( + pricingUnit === 'per-tier' && + quality && + selectedModel.resolution_map + ) { const ratioKey = `${ratio.w}:${ratio.h}`; - const resolutionEntry = selectedModel.resolution_map[ratioKey]?.[quality]; - if ( resolutionEntry ) { + const resolutionEntry = + selectedModel.resolution_map[ratioKey]?.[quality]; + if (resolutionEntry) { resolvedRatio = resolutionEntry; } } - const request = this.#buildRequest(prompt, { ...options, ratio: resolvedRatio, model: selectedModel.id.replace('togetherai:', '') }) as unknown as Together.Images.ImageGenerateParams; + const request = this.#buildRequest(prompt, { + ...options, + ratio: resolvedRatio, + model: selectedModel.id.replace('togetherai:', ''), + }) as unknown as Together.Images.ImageGenerateParams; try { const response = await this.#client.images.generate(request); - if ( ! response?.data?.length ) { - throw new Error('Together AI response did not include image data'); + if (!response?.data?.length) { + throw new Error( + 'Together AI response did not include image data', + ); } - this.#meteringService.incrementUsage(actor, usageType, usageAmount, costInMicroCents); + this.#meteringService.incrementUsage( + actor, + usageType, + usageAmount, + costInMicroCents, + ); - const first = response.data[0] as { url?: string; b64_json?: string }; - const url = first.url || (first.b64_json ? `data:image/png;base64,${ first.b64_json}` : undefined); + const first = response.data[0] as { + url?: string; + b64_json?: string; + }; + const url = + first.url || + (first.b64_json + ? `data:image/png;base64,${first.b64_json}` + : undefined); - if ( ! url ) { - throw new Error('Together AI response did not include an image URL'); + if (!url) { + throw new Error( + 'Together AI response did not include an image URL', + ); } return url; - } catch ( error ) { - throw new Error(`Together AI image generation error: ${(error as Error).message}`); + } catch (error) { + throw new Error( + `Together AI image generation error: ${(error as Error).message}`, + ); } } - #getModel (model?: string) { - return this.models().find(m => m.id === model) || this.models().find(m => m.id === DEFAULT_MODEL)!; + #getModel(model?: string) { + return ( + this.models().find((m) => m.id === model) || + this.models().find((m) => m.id === DEFAULT_MODEL)! + ); } - #buildRequest (prompt: string, options: TogetherGenerateParams) { + #buildRequest(prompt: string, options: TogetherGenerateParams) { const { ratio, model, @@ -205,45 +232,67 @@ export class TogetherImageGenerationProvider implements IImageProvider { n: 1, }; - const requiresConditionImage = this.#modelRequiresConditionImage(request.model as string); + const requiresConditionImage = this.#modelRequiresConditionImage( + request.model as string, + ); const ratioWidth = ratio?.w !== undefined ? Number(ratio.w) : undefined; - const ratioHeight = ratio?.h !== undefined ? Number(ratio.h) : undefined; + const ratioHeight = + ratio?.h !== undefined ? Number(ratio.h) : undefined; - const normalizedWidth = this.#normalizeDimension((ratioWidth ?? TOGETHER_DEFAULT_RATIO.w)); - const normalizedHeight = this.#normalizeDimension((ratioHeight ?? TOGETHER_DEFAULT_RATIO.h)); + const normalizedWidth = this.#normalizeDimension( + ratioWidth ?? TOGETHER_DEFAULT_RATIO.w, + ); + const normalizedHeight = this.#normalizeDimension( + ratioHeight ?? TOGETHER_DEFAULT_RATIO.h, + ); - if ( normalizedWidth ) request.width = normalizedWidth; - if ( normalizedHeight ) request.height = normalizedHeight; + if (normalizedWidth) request.width = normalizedWidth; + if (normalizedHeight) request.height = normalizedHeight; - if ( typeof steps === 'number' && Number.isFinite(steps) ) { + if (typeof steps === 'number' && Number.isFinite(steps)) { request.steps = Math.max(1, Math.min(50, Math.round(steps))); } - if ( typeof seed === 'number' && Number.isFinite(seed) ) request.seed = Math.round(seed); - if ( typeof negative_prompt === 'string' ) request.negative_prompt = negative_prompt; - if ( disable_safety_checker ) { + if (typeof seed === 'number' && Number.isFinite(seed)) + request.seed = Math.round(seed); + if (typeof negative_prompt === 'string') + request.negative_prompt = negative_prompt; + if (disable_safety_checker) { request.disable_safety_checker = true; } - if ( typeof response_format === 'string' ) request.response_format = response_format; + if (typeof response_format === 'string') + request.response_format = response_format; - const resolvedImageBase64 = typeof image_base64 === 'string' - ? image_base64 - : (typeof input_image === 'string' ? input_image : undefined); + const resolvedImageBase64 = + typeof image_base64 === 'string' + ? image_base64 + : typeof input_image === 'string' + ? input_image + : undefined; - if ( typeof image_url === 'string' ) request.image_url = image_url; - if ( resolvedImageBase64 ) request.image_base64 = resolvedImageBase64; - if ( typeof mask_image_url === 'string' ) request.mask_image_url = mask_image_url; - if ( typeof mask_image_base64 === 'string' ) request.mask_image_base64 = mask_image_base64; - if ( typeof prompt_strength === 'number' && Number.isFinite(prompt_strength) ) { + if (typeof image_url === 'string') request.image_url = image_url; + if (resolvedImageBase64) request.image_base64 = resolvedImageBase64; + if (typeof mask_image_url === 'string') + request.mask_image_url = mask_image_url; + if (typeof mask_image_base64 === 'string') + request.mask_image_base64 = mask_image_base64; + if ( + typeof prompt_strength === 'number' && + Number.isFinite(prompt_strength) + ) { request.prompt_strength = Math.max(0, Math.min(1, prompt_strength)); } - if ( requiresConditionImage ) { + if (requiresConditionImage) { const conditionSource = resolvedImageBase64 ? resolvedImageBase64 - : (typeof image_url === 'string' ? image_url : undefined); + : typeof image_url === 'string' + ? image_url + : undefined; - if ( ! conditionSource ) { - throw new Error(`Model ${request.model} requires an image_url or image_base64 input`); + if (!conditionSource) { + throw new Error( + `Model ${request.model} requires an image_url or image_base64 input`, + ); } request.condition_image = conditionSource; @@ -252,19 +301,21 @@ export class TogetherImageGenerationProvider implements IImageProvider { return request; } - #normalizeDimension (value?: number) { - if ( typeof value !== 'number' || Number.isNaN(value) ) return undefined; + #normalizeDimension(value?: number) { + if (typeof value !== 'number' || Number.isNaN(value)) return undefined; const rounded = Math.max(64, Math.round(value)); // Flux models expect multiples of 8. Snap to the nearest multiple without going below 64. return Math.max(64, Math.round(rounded / 8) * 8); } - #modelRequiresConditionImage (modelId?: string) { - if ( typeof modelId !== 'string' || modelId.trim() === '' ) { + #modelRequiresConditionImage(modelId?: string) { + if (typeof modelId !== 'string' || modelId.trim() === '') { return false; } const normalized = modelId.toLowerCase(); - return CONDITION_IMAGE_MODELS.some(required => normalized === required); + return CONDITION_IMAGE_MODELS.some( + (required) => normalized === required, + ); } } diff --git a/src/backend/src/services/ai/image/providers/TogetherImageGenerationProvider/models.ts b/src/backend/drivers/ai-image/providers/together/models.ts similarity index 76% rename from src/backend/src/services/ai/image/providers/TogetherImageGenerationProvider/models.ts rename to src/backend/drivers/ai-image/providers/together/models.ts index 1b3e2abd1..b2d57918c 100644 --- a/src/backend/src/services/ai/image/providers/TogetherImageGenerationProvider/models.ts +++ b/src/backend/drivers/ai-image/providers/together/models.ts @@ -17,38 +17,148 @@ * along with this program. If not, see . */ -import { IImageModel } from '../types'; +import { IImageModel } from '../../types.js'; type ResolutionMap = Record>; export const GEMINI_3_IMAGE_RESOLUTION_MAP: ResolutionMap = { - '1:1': { '1K': { w: 1024, h: 1024 }, '2K': { w: 2048, h: 2048 }, '4K': { w: 4096, h: 4096 } }, - '2:3': { '1K': { w: 848, h: 1264 }, '2K': { w: 1696, h: 2528 }, '4K': { w: 3392, h: 5096 } }, - '3:2': { '1K': { w: 1264, h: 848 }, '2K': { w: 2528, h: 1696 }, '4K': { w: 5096, h: 3392 } }, - '3:4': { '1K': { w: 896, h: 1200 }, '2K': { w: 1792, h: 2400 }, '4K': { w: 3584, h: 4800 } }, - '4:3': { '1K': { w: 1200, h: 896 }, '2K': { w: 2400, h: 1792 }, '4K': { w: 4800, h: 3584 } }, - '4:5': { '1K': { w: 928, h: 1152 }, '2K': { w: 1856, h: 2304 }, '4K': { w: 3712, h: 4608 } }, - '5:4': { '1K': { w: 1152, h: 928 }, '2K': { w: 2304, h: 1856 }, '4K': { w: 4608, h: 3712 } }, - '9:16': { '1K': { w: 768, h: 1376 }, '2K': { w: 1536, h: 2752 }, '4K': { w: 3072, h: 5504 } }, - '16:9': { '1K': { w: 1376, h: 768 }, '2K': { w: 2752, h: 1536 }, '4K': { w: 5504, h: 3072 } }, - '21:9': { '1K': { w: 1584, h: 672 }, '2K': { w: 3168, h: 1344 }, '4K': { w: 6336, h: 2688 } }, + '1:1': { + '1K': { w: 1024, h: 1024 }, + '2K': { w: 2048, h: 2048 }, + '4K': { w: 4096, h: 4096 }, + }, + '2:3': { + '1K': { w: 848, h: 1264 }, + '2K': { w: 1696, h: 2528 }, + '4K': { w: 3392, h: 5096 }, + }, + '3:2': { + '1K': { w: 1264, h: 848 }, + '2K': { w: 2528, h: 1696 }, + '4K': { w: 5096, h: 3392 }, + }, + '3:4': { + '1K': { w: 896, h: 1200 }, + '2K': { w: 1792, h: 2400 }, + '4K': { w: 3584, h: 4800 }, + }, + '4:3': { + '1K': { w: 1200, h: 896 }, + '2K': { w: 2400, h: 1792 }, + '4K': { w: 4800, h: 3584 }, + }, + '4:5': { + '1K': { w: 928, h: 1152 }, + '2K': { w: 1856, h: 2304 }, + '4K': { w: 3712, h: 4608 }, + }, + '5:4': { + '1K': { w: 1152, h: 928 }, + '2K': { w: 2304, h: 1856 }, + '4K': { w: 4608, h: 3712 }, + }, + '9:16': { + '1K': { w: 768, h: 1376 }, + '2K': { w: 1536, h: 2752 }, + '4K': { w: 3072, h: 5504 }, + }, + '16:9': { + '1K': { w: 1376, h: 768 }, + '2K': { w: 2752, h: 1536 }, + '4K': { w: 5504, h: 3072 }, + }, + '21:9': { + '1K': { w: 1584, h: 672 }, + '2K': { w: 3168, h: 1344 }, + '4K': { w: 6336, h: 2688 }, + }, }; export const FLASH_IMAGE_3_1_RESOLUTION_MAP: ResolutionMap = { - '1:1': { '0.5K': { w: 512, h: 512 }, '1K': { w: 1024, h: 1024 }, '2K': { w: 2048, h: 2048 }, '4K': { w: 4096, h: 4096 } }, - '1:4': { '0.5K': { w: 256, h: 1024 }, '1K': { w: 512, h: 2048 }, '2K': { w: 1024, h: 4096 }, '4K': { w: 2048, h: 8192 } }, - '1:8': { '0.5K': { w: 192, h: 1536 }, '1K': { w: 384, h: 3072 }, '2K': { w: 768, h: 6144 }, '4K': { w: 1536, h: 12288 } }, - '2:3': { '0.5K': { w: 424, h: 632 }, '1K': { w: 848, h: 1264 }, '2K': { w: 1696, h: 2528 }, '4K': { w: 3392, h: 5056 } }, - '3:2': { '0.5K': { w: 632, h: 424 }, '1K': { w: 1264, h: 848 }, '2K': { w: 2528, h: 1696 }, '4K': { w: 5056, h: 3392 } }, - '3:4': { '0.5K': { w: 448, h: 600 }, '1K': { w: 896, h: 1200 }, '2K': { w: 1792, h: 2400 }, '4K': { w: 3584, h: 4800 } }, - '4:1': { '0.5K': { w: 1024, h: 256 }, '1K': { w: 2048, h: 512 }, '2K': { w: 4096, h: 1024 }, '4K': { w: 8192, h: 2048 } }, - '4:3': { '0.5K': { w: 600, h: 448 }, '1K': { w: 1200, h: 896 }, '2K': { w: 2400, h: 1792 }, '4K': { w: 4800, h: 3584 } }, - '4:5': { '0.5K': { w: 464, h: 576 }, '1K': { w: 928, h: 1152 }, '2K': { w: 1856, h: 2304 }, '4K': { w: 3712, h: 4608 } }, - '5:4': { '0.5K': { w: 576, h: 464 }, '1K': { w: 1152, h: 928 }, '2K': { w: 2304, h: 1856 }, '4K': { w: 4608, h: 3712 } }, - '8:1': { '0.5K': { w: 1536, h: 192 }, '1K': { w: 3072, h: 384 }, '2K': { w: 6144, h: 768 }, '4K': { w: 12288, h: 1536 } }, - '9:16': { '0.5K': { w: 384, h: 688 }, '1K': { w: 768, h: 1376 }, '2K': { w: 1536, h: 2752 }, '4K': { w: 3072, h: 5504 } }, - '16:9': { '0.5K': { w: 688, h: 384 }, '1K': { w: 1376, h: 768 }, '2K': { w: 2752, h: 1536 }, '4K': { w: 5504, h: 3072 } }, - '21:9': { '0.5K': { w: 792, h: 168 }, '1K': { w: 1584, h: 672 }, '2K': { w: 3168, h: 1344 }, '4K': { w: 6336, h: 2688 } }, + '1:1': { + '0.5K': { w: 512, h: 512 }, + '1K': { w: 1024, h: 1024 }, + '2K': { w: 2048, h: 2048 }, + '4K': { w: 4096, h: 4096 }, + }, + '1:4': { + '0.5K': { w: 256, h: 1024 }, + '1K': { w: 512, h: 2048 }, + '2K': { w: 1024, h: 4096 }, + '4K': { w: 2048, h: 8192 }, + }, + '1:8': { + '0.5K': { w: 192, h: 1536 }, + '1K': { w: 384, h: 3072 }, + '2K': { w: 768, h: 6144 }, + '4K': { w: 1536, h: 12288 }, + }, + '2:3': { + '0.5K': { w: 424, h: 632 }, + '1K': { w: 848, h: 1264 }, + '2K': { w: 1696, h: 2528 }, + '4K': { w: 3392, h: 5056 }, + }, + '3:2': { + '0.5K': { w: 632, h: 424 }, + '1K': { w: 1264, h: 848 }, + '2K': { w: 2528, h: 1696 }, + '4K': { w: 5056, h: 3392 }, + }, + '3:4': { + '0.5K': { w: 448, h: 600 }, + '1K': { w: 896, h: 1200 }, + '2K': { w: 1792, h: 2400 }, + '4K': { w: 3584, h: 4800 }, + }, + '4:1': { + '0.5K': { w: 1024, h: 256 }, + '1K': { w: 2048, h: 512 }, + '2K': { w: 4096, h: 1024 }, + '4K': { w: 8192, h: 2048 }, + }, + '4:3': { + '0.5K': { w: 600, h: 448 }, + '1K': { w: 1200, h: 896 }, + '2K': { w: 2400, h: 1792 }, + '4K': { w: 4800, h: 3584 }, + }, + '4:5': { + '0.5K': { w: 464, h: 576 }, + '1K': { w: 928, h: 1152 }, + '2K': { w: 1856, h: 2304 }, + '4K': { w: 3712, h: 4608 }, + }, + '5:4': { + '0.5K': { w: 576, h: 464 }, + '1K': { w: 1152, h: 928 }, + '2K': { w: 2304, h: 1856 }, + '4K': { w: 4608, h: 3712 }, + }, + '8:1': { + '0.5K': { w: 1536, h: 192 }, + '1K': { w: 3072, h: 384 }, + '2K': { w: 6144, h: 768 }, + '4K': { w: 12288, h: 1536 }, + }, + '9:16': { + '0.5K': { w: 384, h: 688 }, + '1K': { w: 768, h: 1376 }, + '2K': { w: 1536, h: 2752 }, + '4K': { w: 3072, h: 5504 }, + }, + '16:9': { + '0.5K': { w: 688, h: 384 }, + '1K': { w: 1376, h: 768 }, + '2K': { w: 2752, h: 1536 }, + '4K': { w: 5504, h: 3072 }, + }, + '21:9': { + '0.5K': { w: 792, h: 168 }, + '1K': { w: 1584, h: 672 }, + '2K': { w: 3168, h: 1344 }, + '4K': { w: 6336, h: 2688 }, + }, }; export const TOGETHER_IMAGE_GENERATION_MODELS: IImageModel[] = [ @@ -154,7 +264,10 @@ export const TOGETHER_IMAGE_GENERATION_MODELS: IImageModel[] = [ }, { id: 'togetherai:Rundiffusion/Juggernaut-Lightning-Flux', - aliases: ['Rundiffusion/Juggernaut-Lightning-Flux', 'Juggernaut-Lightning-Flux'], + aliases: [ + 'Rundiffusion/Juggernaut-Lightning-Flux', + 'Juggernaut-Lightning-Flux', + ], costs_currency: 'usd-cents', index_cost_key: '1MP', name: 'Rundiffusion/Juggernaut-Lightning-Flux', @@ -387,7 +500,10 @@ export const TOGETHER_IMAGE_GENERATION_MODELS: IImageModel[] = [ }, { id: 'togetherai:stabilityai/stable-diffusion-3-medium', - aliases: ['stabilityai/stable-diffusion-3-medium', 'stable-diffusion-3-medium'], + aliases: [ + 'stabilityai/stable-diffusion-3-medium', + 'stable-diffusion-3-medium', + ], costs_currency: 'usd-cents', index_cost_key: '1MP', name: 'stabilityai/stable-diffusion-3-medium', @@ -397,7 +513,10 @@ export const TOGETHER_IMAGE_GENERATION_MODELS: IImageModel[] = [ }, { id: 'togetherai:stabilityai/stable-diffusion-xl-base-1.0', - aliases: ['stabilityai/stable-diffusion-xl-base-1.0', 'stable-diffusion-xl-base-1.0'], + aliases: [ + 'stabilityai/stable-diffusion-xl-base-1.0', + 'stable-diffusion-xl-base-1.0', + ], costs_currency: 'usd-cents', index_cost_key: '1MP', name: 'stabilityai/stable-diffusion-xl-base-1.0', diff --git a/src/backend/src/services/ai/image/providers/XAIImageGenerationProvider/XAIImageGenerationProvider.ts b/src/backend/drivers/ai-image/providers/xai/XAIImageProvider.ts similarity index 52% rename from src/backend/src/services/ai/image/providers/XAIImageGenerationProvider/XAIImageGenerationProvider.ts rename to src/backend/drivers/ai-image/providers/xai/XAIImageProvider.ts index 2e507a0bb..15bc738f3 100644 --- a/src/backend/src/services/ai/image/providers/XAIImageGenerationProvider/XAIImageGenerationProvider.ts +++ b/src/backend/drivers/ai-image/providers/xai/XAIImageProvider.ts @@ -18,95 +18,105 @@ */ import { OpenAI } from 'openai'; -import APIError from '../../../../../api/APIError.js'; -import { ErrorService } from '../../../../../modules/core/ErrorService.js'; -import { Context } from '../../../../../util/context.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import { IGenerateParams, IImageModel, IImageProvider } from '../types.js'; +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { + IGenerateParams, + IImageModel, + IImageProvider, +} from '../../types.js'; import { XAI_IMAGE_GENERATION_MODELS } from './models.js'; const DEFAULT_MODEL = 'grok-2-image'; const PRICE_KEY = 'output'; -export class XAIImageGenerationProvider implements IImageProvider { +export class XAIImageProvider implements IImageProvider { #client: OpenAI; #meteringService: MeteringService; - #errors: ErrorService; - constructor (config: { apiKey: string }, meteringService: MeteringService, errorService: ErrorService) { - if ( ! config.apiKey ) { + constructor(config: { apiKey: string }, meteringService: MeteringService) { + if (!config.apiKey) { throw new Error('xAI image generation requires an API key'); } this.#meteringService = meteringService; - this.#errors = errorService; this.#client = new OpenAI({ apiKey: config.apiKey, baseURL: 'https://api.x.ai/v1', }); } - models (): IImageModel[] { + models(): IImageModel[] { return XAI_IMAGE_GENERATION_MODELS; } - getDefaultModel (): string { + getDefaultModel(): string { return DEFAULT_MODEL; } - async generate (params: IGenerateParams): Promise { + async generate(params: IGenerateParams): Promise { const { prompt, test_mode } = params; - let { model } = params; + const { model } = params; const selectedModel = this.#getModel(model); - if ( test_mode ) { + if (test_mode) { return 'https://puter-sample-data.puter.site/image_example.png'; } - if ( typeof prompt !== 'string' || prompt.trim().length === 0 ) { + if (typeof prompt !== 'string' || prompt.trim().length === 0) { throw new Error('`prompt` must be a non-empty string'); } const actor = Context.get('actor'); - const user_private_uid = actor?.private_uid ?? 'UNKNOWN'; - if ( user_private_uid === 'UNKNOWN' ) { - this.#errors.report('xai-image-generation:unknown-user', { - message: 'failed to get a user ID for an xAI request', - alarm: true, - trace: true, - }); - } + const userIdentifier = + actor?.user.id + actor?.app?.uid ? `:${actor?.app?.uid}` : ''; const priceInCents = selectedModel.costs[PRICE_KEY]; const costInMicroCents = priceInCents * 1_000_000; - const usageAllowed = await this.#meteringService.hasEnoughCredits(actor, costInMicroCents); + const usageAllowed = await this.#meteringService.hasEnoughCredits( + actor, + costInMicroCents, + ); - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); + if (!usageAllowed) { + throw new Error('Insufficient credits for image generation'); } const response = await this.#client.images.generate({ model: selectedModel.id, prompt, - user: user_private_uid, + user: userIdentifier, }); - const first = response.data?.[0] as { url?: string; b64_json?: string } | undefined; - const url = first?.url || (first?.b64_json ? `data:image/png;base64,${ first.b64_json}` : undefined); + const first = response.data?.[0] as + | { url?: string; b64_json?: string } + | undefined; + const url = + first?.url || + (first?.b64_json + ? `data:image/png;base64,${first.b64_json}` + : undefined); - if ( ! url ) { + if (!url) { throw new Error('Failed to extract image URL from xAI response'); } - this.#meteringService.incrementUsage(actor, `xai:${selectedModel.id}:${PRICE_KEY}`, 1, costInMicroCents); + this.#meteringService.incrementUsage( + actor, + `xai:${selectedModel.id}:${PRICE_KEY}`, + 1, + costInMicroCents, + ); return url; } - #getModel (model?: string) { + #getModel(model?: string) { const models = this.models(); - const found = models.find(m => m.id === model || m.aliases?.includes(model ?? '')); - return found || models.find(m => m.id === DEFAULT_MODEL)!; + const found = models.find( + (m) => m.id === model || m.aliases?.includes(model ?? ''), + ); + return found || models.find((m) => m.id === DEFAULT_MODEL)!; } } diff --git a/src/backend/src/services/ai/image/providers/XAIImageGenerationProvider/models.ts b/src/backend/drivers/ai-image/providers/xai/models.ts similarity index 89% rename from src/backend/src/services/ai/image/providers/XAIImageGenerationProvider/models.ts rename to src/backend/drivers/ai-image/providers/xai/models.ts index 654446d6c..5cff084b6 100644 --- a/src/backend/src/services/ai/image/providers/XAIImageGenerationProvider/models.ts +++ b/src/backend/drivers/ai-image/providers/xai/models.ts @@ -1,4 +1,4 @@ -import { IImageModel } from '../types'; +import type { IImageModel } from '../../types.js'; export const XAI_IMAGE_GENERATION_MODELS: IImageModel[] = [ { diff --git a/src/backend/src/services/ai/image/providers/types.ts b/src/backend/drivers/ai-image/types.ts similarity index 70% rename from src/backend/src/services/ai/image/providers/types.ts rename to src/backend/drivers/ai-image/types.ts index ffe3ca67d..dfd058783 100644 --- a/src/backend/src/services/ai/image/providers/types.ts +++ b/src/backend/drivers/ai-image/types.ts @@ -1,3 +1,7 @@ +/** + * Types for the `puter-image-generation` driver interface. + */ + export type ImagePricingUnit = 'per-image' | 'per-MP' | 'per-tier'; export interface IImageModel { @@ -27,24 +31,23 @@ export interface IImageModel { */ resolution_map?: Record>; allowedQualityLevels?: string[]; - allowedRatios?: { w: number, h: number }[]; + allowedRatios?: { w: number; h: number }[]; } export interface IGenerateParams { - prompt: string, - ratio: { w: number, h: number } - model: string, - provider?: string, - test_mode?: boolean - quality?: string, - input_image?: string, - input_image_mime_type?: string, - input_images?: string[], -}; + prompt: string; + ratio?: { w: number; h: number }; + model?: string; + provider?: string; + test_mode?: boolean; + quality?: string; + input_image?: string; + input_image_mime_type?: string; + input_images?: string[]; +} + export interface IImageProvider { - - generate (params: IGenerateParams): Promise; - models (): Promise | IImageModel[]; - getDefaultModel (): string; - + generate(params: IGenerateParams): Promise; + models(): Promise | IImageModel[]; + getDefaultModel(): string; } diff --git a/src/backend/drivers/ai-ocr/OCRDriver.ts b/src/backend/drivers/ai-ocr/OCRDriver.ts new file mode 100644 index 000000000..fda5e92b2 --- /dev/null +++ b/src/backend/drivers/ai-ocr/OCRDriver.ts @@ -0,0 +1,397 @@ +import { + AnalyzeDocumentCommand, + InvalidS3ObjectException, + TextractClient, +} from '@aws-sdk/client-textract'; +import { Mistral } from '@mistralai/mistralai'; +import { Actor } from '../../core/actor.js'; +import { Context } from '../../core/context.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import { mimeFromName } from '../../util/fileSigning.js'; +import { PuterDriver } from '../types.js'; +import { loadFileInput, type LoadedFile } from '../util/fileInput.js'; +import { OCR_COSTS } from './costs.js'; + +/** + * Driver implementing `puter-ocr` — document OCR. Two providers: + * • `aws-textract` — AWS Textract (region-aware clients; direct S3 source when available) + * • `mistral` — Mistral OCR (URL/data-URL based) + */ +interface RecognizeArgs { + source?: unknown; + file?: unknown; + provider?: string; + // Mistral-specific options — ignored by Textract. + model?: string; + pages?: number[]; + includeImageBase64?: boolean; + imageLimit?: number; + imageMinSize?: number; + bboxAnnotationFormat?: unknown; + documentAnnotationFormat?: unknown; + test_mode?: boolean; +} + +interface TextractBlock { + BlockType?: string; + Confidence?: number; + Text?: string; +} + +interface MistralOcrResponse { + model?: string; + pages?: Array<{ + index?: number; + markdown?: string; + images?: unknown[]; + dimensions?: unknown; + }>; + usageInfo?: { pagesProcessed?: number }; +} + +interface MistralOcrClient { + ocr: { + process: ( + payload: Record, + ) => Promise; + }; +} + +export class OCRDriver extends PuterDriver { + readonly driverInterface = 'puter-ocr'; + readonly driverName = 'ai-ocr'; + + override getReportedCosts(): Record[] { + return Object.entries(OCR_COSTS).map(([usageType, ucentsPerUnit]) => ({ + usageType, + ucentsPerUnit, + unit: 'page', + source: 'driver:aiOcr', + })); + } + + // puter-js's `img2txt` routes by passing the provider id in the `driver` + // slot. Alias them so the unified driver resolves; `recognize` falls + // back to the alias via Context.driverName when `args.provider` isn't + // set. + readonly driverAliases = ['aws-textract', 'mistral']; + readonly isDefault = true; + + // Textract state — one client per region. + #textractClients: Record = {}; + #awsConfig: { + accessKeyId?: string; + secretAccessKey?: string; + region?: string; + } | null = null; + + // Mistral state. + #mistral: MistralOcrClient | null = null; + + override onServerStart() { + const providers = this.config.providers ?? {}; + + const textract = providers['aws-textract'] as + | Record + | undefined; + const textractAws = (textract?.aws ?? textract) as + | Record + | undefined; + const textractAccessKey = textractAws?.access_key as string | undefined; + const textractSecretKey = textractAws?.secret_key as string | undefined; + const textractRegion = + (textractAws?.region as string | undefined) ?? + (textract?.region as string | undefined) ?? + 'us-west-2'; + if (textractAccessKey && textractSecretKey) { + this.#awsConfig = { + accessKeyId: textractAccessKey, + secretAccessKey: textractSecretKey, + region: textractRegion, + }; + } + + const mistral = providers['mistral-ocr']; + if (mistral?.apiKey) { + try { + // Lazy import so we don't pay the cost when Mistral is unused. + + this.#mistral = new Mistral({ + apiKey: mistral.apiKey, + }) as unknown as MistralOcrClient; + } catch (e) { + console.warn( + '[OCRDriver] Failed to init Mistral:', + (e as Error).message, + ); + } + } + } + + async recognize(args: RecognizeArgs) { + if (args.test_mode) return sampleResponse(); + + const provider = + args.provider ?? + (Context.get('driverName') as string | undefined) ?? + this.#defaultProvider(); + if (!provider) throw new HttpError(500, 'No OCR provider configured'); + + const actor = Context.get('actor'); + if (!actor) throw new HttpError(401, 'Authentication required'); + + const input = args.source ?? args.file; + if (!input) throw new HttpError(400, '`source` is required'); + + const loaded = await loadFileInput( + this.stores, + this.services.fs, + actor, + input, + ); + + if (provider === 'aws-textract') { + if (!this.#awsConfig) + throw new HttpError(500, 'AWS credentials not configured'); + return this.#textractRecognize(loaded, actor); + } + if (provider === 'mistral') { + if (!this.#mistral) + throw new HttpError(500, 'Mistral OCR not configured'); + return this.#mistralRecognize(loaded, args, actor); + } + throw new HttpError(400, `Unknown OCR provider: ${provider}`); + } + + #defaultProvider(): 'aws-textract' | 'mistral' | null { + if (this.#awsConfig) return 'aws-textract'; + if (this.#mistral) return 'mistral'; + return null; + } + + // ── AWS Textract ───────────────────────────────────────────────── + + #textractClientFor(region: string): TextractClient { + const cached = this.#textractClients[region]; + if (cached) return cached; + const client = new TextractClient({ + credentials: { + accessKeyId: this.#awsConfig!.accessKeyId!, + secretAccessKey: this.#awsConfig!.secretAccessKey!, + }, + region, + }); + this.#textractClients[region] = client; + return client; + } + + async #textractRecognize(loaded: LoadedFile, actor: Actor) { + const usageType = 'aws-textract:detect-document-text:page'; + const costPerPage = OCR_COSTS[usageType]; + const hasCredits = await this.services.metering.hasEnoughCredits( + actor!, + costPerPage, + ); + if (!hasCredits) throw new HttpError(402, 'Insufficient credits'); + + // Prefer S3 direct source if the file is FS-backed; fall back to raw bytes. + const s3Info = + loaded.fsEntry && + loaded.fsEntry.bucket && + loaded.fsEntry.bucketRegion + ? { + bucket: loaded.fsEntry.bucket, + bucketRegion: loaded.fsEntry.bucketRegion, + key: loaded.fsEntry.uuid, + } + : null; + + const tryRun = async (useS3: boolean) => { + const region = + s3Info && useS3 + ? s3Info.bucketRegion + : (this.#awsConfig!.region ?? 'us-west-2'); + const client = this.#textractClientFor(region); + const document = + s3Info && useS3 + ? { S3Object: { Bucket: s3Info.bucket, Name: s3Info.key } } + : { Bytes: loaded.buffer }; + return client.send( + new AnalyzeDocumentCommand({ + Document: document, + FeatureTypes: ['LAYOUT'], + }), + ); + }; + + let response; + try { + response = await tryRun(Boolean(s3Info)); + } catch (err) { + if (s3Info && err instanceof InvalidS3ObjectException) { + response = await tryRun(false); + } else { + throw err; + } + } + + const blocks: Array<{ + type: string; + confidence: number; + text: string; + }> = []; + let pageCount = 0; + for (const block of (response.Blocks ?? []) as TextractBlock[]) { + if (block.BlockType === 'PAGE') { + pageCount += 1; + continue; + } + if ( + [ + 'CELL', + 'TABLE', + 'MERGED_CELL', + 'LAYOUT_FIGURE', + 'LAYOUT_TEXT', + ].includes(block.BlockType ?? '') + ) + continue; + blocks.push({ + type: `text/textract:${block.BlockType ?? 'UNKNOWN'}`, + confidence: Number(block.Confidence ?? 0), + text: block.Text ?? '', + }); + } + + const pages = pageCount || 1; + this.services.metering.incrementUsage( + actor, + usageType, + pages, + costPerPage * pages, + ); + return { blocks }; + } + + // ── Mistral OCR ────────────────────────────────────────────────── + + async #mistralRecognize( + loaded: LoadedFile, + args: RecognizeArgs, + actor: Actor, + ) { + const model = args.model ?? 'mistral-ocr-latest'; + const chunk = this.#mistralBuildChunk(loaded); + const payload: Record = { model, document: chunk }; + if (args.pages) payload.pages = args.pages; + if (args.includeImageBase64 !== undefined) + payload.includeImageBase64 = args.includeImageBase64; + if (typeof args.imageLimit === 'number') + payload.imageLimit = args.imageLimit; + if (typeof args.imageMinSize === 'number') + payload.imageMinSize = args.imageMinSize; + if (args.bboxAnnotationFormat !== undefined) + payload.bboxAnnotationFormat = args.bboxAnnotationFormat; + if (args.documentAnnotationFormat !== undefined) + payload.documentAnnotationFormat = args.documentAnnotationFormat; + + const response = await this.#mistral!.ocr.process(payload); + const annotations = + payload.documentAnnotationFormat !== undefined || + payload.bboxAnnotationFormat !== undefined; + this.#recordMistralUsage(response, actor, annotations); + return this.#normalizeMistralResponse(response); + } + + #mistralBuildChunk(loaded: LoadedFile): Record { + const mime = + loaded.mimeType ?? + mimeFromName(loaded.filename) ?? + 'application/octet-stream'; + const isPdf = + mime.includes('pdf') || + loaded.filename.toLowerCase().endsWith('.pdf'); + const dataUrl = `data:${mime};base64,${loaded.buffer.toString('base64')}`; + if (isPdf) { + return { + type: 'document_url', + documentUrl: dataUrl, + documentName: loaded.filename, + }; + } + return { type: 'image_url', imageUrl: { url: dataUrl } }; + } + + #normalizeMistralResponse(response: MistralOcrResponse) { + const pages = response?.pages ?? []; + const blocks: Array<{ type: string; text: string; page?: number }> = []; + for (const page of pages) { + if (typeof page?.markdown !== 'string') continue; + const lines = page.markdown + .split('\n') + .map((l) => l.trim()) + .filter(Boolean); + for (const line of lines) { + blocks.push({ + type: 'text/mistral:LINE', + text: line, + page: page.index, + }); + } + } + const text = + blocks.length > 0 + ? blocks.map((b) => b.text).join('\n') + : pages + .map((p) => p?.markdown ?? '') + .join('\n\n') + .trim(); + return { + model: response?.model, + pages, + usage_info: response?.usageInfo, + blocks, + text, + }; + } + + #recordMistralUsage( + response: MistralOcrResponse, + actor: Actor, + annotations: boolean, + ) { + try { + const pagesProcessed = + response?.usageInfo?.pagesProcessed ?? + (Array.isArray(response?.pages) ? response.pages.length : 1); + this.services.metering.incrementUsage( + actor, + 'mistral-ocr:ocr:page', + pagesProcessed, + OCR_COSTS['mistral-ocr:ocr:page'] * pagesProcessed, + ); + if (annotations) { + this.services.metering.incrementUsage( + actor, + 'mistral-ocr:annotations:page', + pagesProcessed, + OCR_COSTS['mistral-ocr:annotations:page'] * pagesProcessed, + ); + } + } catch { + // Non-critical. + } + } +} + +function sampleResponse() { + return { + blocks: [ + { + type: 'text/puter:sample-output', + confidence: 1, + text: 'test_mode is enabled; this is a sample OCR response.', + }, + ], + }; +} diff --git a/src/backend/drivers/ai-ocr/costs.ts b/src/backend/drivers/ai-ocr/costs.ts new file mode 100644 index 000000000..d006e07b9 --- /dev/null +++ b/src/backend/drivers/ai-ocr/costs.ts @@ -0,0 +1,7 @@ +// Microcents per page — Textract $1.50/1000 pages = 150,000 µ¢/page. +// Mistral OCR $1/1000 pages, annotations $3/1000 pages. +export const OCR_COSTS = { + 'aws-textract:detect-document-text:page': 150000, + 'mistral-ocr:ocr:page': 100000, + 'mistral-ocr:annotations:page': 300000, +} as const; diff --git a/src/backend/drivers/ai-speech2speech/VoiceChangerDriver.ts b/src/backend/drivers/ai-speech2speech/VoiceChangerDriver.ts new file mode 100644 index 000000000..4ed490f7b --- /dev/null +++ b/src/backend/drivers/ai-speech2speech/VoiceChangerDriver.ts @@ -0,0 +1,214 @@ +import { Readable } from 'node:stream'; +import { Context } from '../../core/context.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import type { DriverStreamResult } from '../meta.js'; +import { PuterDriver } from '../types.js'; +import { loadFileInput } from '../util/fileInput.js'; +import { VOICE_CHANGER_COSTS } from './costs.js'; + +/** + * Driver implementing `puter-speech2speech` — voice changer. Currently a + * single provider (ElevenLabs). + */ + +const DEFAULT_MODEL = 'eleven_multilingual_sts_v2'; +const DEFAULT_VOICE_ID = '21m00Tcm4TlvDq8ikWAM'; +const DEFAULT_OUTPUT_FORMAT = 'mp3_44100_128'; +const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3'; +const MAX_AUDIO_FILE_SIZE = 25 * 1024 * 1024; + +interface ConvertArgs { + audio: unknown; + voice?: string; + voice_id?: string; + voiceId?: string; + model?: string; + model_id?: string; + voice_settings?: unknown; + voiceSettings?: unknown; + seed?: number; + remove_background_noise?: boolean; + output_format?: string; + file_format?: string; + optimize_streaming_latency?: number; + enable_logging?: boolean; + test_mode?: boolean; +} + +export class VoiceChangerDriver extends PuterDriver { + readonly driverInterface = 'puter-speech2speech'; + readonly driverName = 'elevenlabs-voice-changer'; + readonly isDefault = true; + + override getReportedCosts(): Record[] { + return Object.entries(VOICE_CHANGER_COSTS).map( + ([usageType, ucentsPerUnit]) => ({ + usageType, + ucentsPerUnit, + unit: 'second', + source: 'driver:aiSpeech2Speech', + }), + ); + } + + #apiKey: string | null = null; + #baseUrl = 'https://api.elevenlabs.io'; + #defaultVoiceId = DEFAULT_VOICE_ID; + #defaultModelId = DEFAULT_MODEL; + + override onServerStart() { + const elevenlabs = this.config.providers?.elevenlabs as + | Record + | undefined; + + this.#apiKey = + (elevenlabs?.apiKey as string | undefined) ?? + (elevenlabs?.api_key as string | undefined) ?? + (elevenlabs?.key as string | undefined) ?? + null; + this.#baseUrl = + (elevenlabs?.apiBaseUrl as string | undefined) ?? this.#baseUrl; + this.#defaultVoiceId = + (elevenlabs?.defaultVoiceId as string | undefined) ?? + DEFAULT_VOICE_ID; + this.#defaultModelId = + (elevenlabs?.speechToSpeechModelId as string | undefined) ?? + DEFAULT_MODEL; + } + + async convert( + args: ConvertArgs, + ): Promise { + if (args.test_mode) { + return { url: SAMPLE_AUDIO_URL, content_type: 'audio/mpeg' }; + } + + if (!this.#apiKey) { + throw new HttpError(500, 'ElevenLabs API key not configured'); + } + + const actor = Context.get('actor'); + if (!actor) throw new HttpError(401, 'Authentication required'); + + if (!args.audio) { + throw new HttpError(400, '`audio` is required'); + } + + const loaded = await loadFileInput( + this.stores, + this.services.fs, + actor, + args.audio, + { maxBytes: MAX_AUDIO_FILE_SIZE }, + ); + + const modelId = args.model_id || args.model || this.#defaultModelId; + const voiceId = + args.voice_id || args.voiceId || args.voice || this.#defaultVoiceId; + if (!voiceId) throw new HttpError(400, '`voice` is required'); + + // Metering: estimate duration from file size if we don't parse metadata. + // 16 kbit/s is a safe lower bound for speech audio; pre-check credits + // before we hit the ElevenLabs API. Post-usage we increment by the same + // estimate — duration parsing is deferred to v2.1 if needed. + const estimatedSeconds = Math.max( + 1, + Math.ceil(loaded.buffer.byteLength / 16000), + ); + const usageKey = `elevenlabs:${modelId}:second`; + const ucentsPerSecond = VOICE_CHANGER_COSTS[usageKey] ?? 0; + const estimatedCost = ucentsPerSecond * estimatedSeconds; + + const hasCredits = await this.services.metering.hasEnoughCredits( + actor, + estimatedCost, + ); + if (!hasCredits) { + throw new HttpError(402, 'Insufficient credits'); + } + + const formData = new FormData(); + const blob = new Blob([loaded.buffer], { + type: loaded.mimeType ?? 'application/octet-stream', + }); + formData.append('audio', blob, loaded.filename); + formData.append('model_id', modelId); + + const settings = args.voice_settings ?? args.voiceSettings; + if (settings !== undefined && settings !== null) { + formData.append( + 'voice_settings', + typeof settings === 'string' + ? settings + : JSON.stringify(settings), + ); + } + if (args.seed !== undefined && args.seed !== null) { + formData.append('seed', String(args.seed)); + } + if (typeof args.remove_background_noise === 'boolean') { + formData.append( + 'remove_background_noise', + String(args.remove_background_noise), + ); + } + if (args.file_format) { + formData.append('file_format', args.file_format); + } + + const searchParams = new URLSearchParams(); + const outputFormat = args.output_format || DEFAULT_OUTPUT_FORMAT; + if (outputFormat) searchParams.set('output_format', outputFormat); + if ( + args.optimize_streaming_latency !== undefined && + args.optimize_streaming_latency !== null + ) { + searchParams.set( + 'optimize_streaming_latency', + String(args.optimize_streaming_latency), + ); + } + if (args.enable_logging !== undefined && args.enable_logging !== null) { + searchParams.set('enable_logging', String(args.enable_logging)); + } + + const url = new URL(`/v1/speech-to-speech/${voiceId}`, this.#baseUrl); + const search = searchParams.toString(); + if (search) url.search = search; + + const response = await fetch(url, { + method: 'POST', + headers: { 'xi-api-key': this.#apiKey }, + body: formData, + }); + + if (!response.ok) { + let detail: unknown = null; + try { + detail = await response.json(); + } catch { + // Non-JSON body — ignore. + } + const message = + detail && typeof detail === 'object' && 'detail' in detail + ? String((detail as { detail: unknown }).detail) + : `ElevenLabs returned ${response.status}`; + throw new HttpError(response.status, message); + } + + const arrayBuffer = await response.arrayBuffer(); + const stream = Readable.from(Buffer.from(arrayBuffer)); + this.services.metering.incrementUsage( + actor, + usageKey, + estimatedSeconds, + ucentsPerSecond * estimatedSeconds, + ); + + return { + dataType: 'stream', + content_type: response.headers.get('content-type') ?? 'audio/mpeg', + stream, + }; + } +} diff --git a/src/backend/drivers/ai-speech2speech/costs.ts b/src/backend/drivers/ai-speech2speech/costs.ts new file mode 100644 index 000000000..e585c3e7b --- /dev/null +++ b/src/backend/drivers/ai-speech2speech/costs.ts @@ -0,0 +1,6 @@ +// Microcents per second of audio, per ElevenLabs speech-to-speech model. +// Values mirror the ElevenLabs scale tier (per-unit × 0.9). +export const VOICE_CHANGER_COSTS: Record = { + 'elevenlabs:eleven_multilingual_sts_v2:second': 300000 * 0.9, + 'elevenlabs:eleven_english_sts_v2:second': 300000 * 0.9, +}; diff --git a/src/backend/drivers/ai-speech2txt/SpeechToTextDriver.ts b/src/backend/drivers/ai-speech2txt/SpeechToTextDriver.ts new file mode 100644 index 000000000..0af70dccb --- /dev/null +++ b/src/backend/drivers/ai-speech2txt/SpeechToTextDriver.ts @@ -0,0 +1,304 @@ +import OpenAI, { toFile } from 'openai'; +import { Context } from '../../core/context.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import { PuterDriver } from '../types.js'; +import { loadFileInput } from '../util/fileInput.js'; +import { SPEECH_TO_TEXT_COSTS } from './costs.js'; + +/** + * Driver implementing `puter-speech2txt`. Wraps OpenAI's audio API + * (Whisper + GPT-4o transcribe models) for transcription and translation. + * + * `file` may be a path, uid/uuid ref, or data URL. + */ + +const DEFAULT_TRANSCRIBE_MODEL = 'gpt-4o-mini-transcribe'; +const DEFAULT_TRANSLATE_MODEL = 'whisper-1'; +const MAX_AUDIO_FILE_SIZE = 25 * 1024 * 1024; + +const SAMPLE_TRANSCRIPT = { + text: 'Hello! This is a sample transcription returned while test mode is enabled.', + language: 'en', + duration_seconds: 2, + words: [ + { start: 0.0, end: 0.5, text: 'Hello' }, + { start: 1.1, end: 2.0, text: 'This is a sample transcription.' }, + ], +}; + +interface ModelCapabilities { + canPrompt: boolean; + canLogprobs: boolean; + responseFormats: string[]; + timestampGranularities?: boolean; + diarization?: boolean; + requiresChunkingOverThirtySeconds?: boolean; +} + +const MODEL_CAPS: Record = { + 'gpt-4o-mini-transcribe': { + canPrompt: true, + canLogprobs: true, + responseFormats: ['json', 'text'], + }, + 'gpt-4o-transcribe': { + canPrompt: true, + canLogprobs: true, + responseFormats: ['json', 'text'], + }, + 'gpt-4o-transcribe-diarize': { + canPrompt: false, + canLogprobs: false, + responseFormats: ['json', 'text', 'diarized_json'], + diarization: true, + requiresChunkingOverThirtySeconds: true, + }, + 'whisper-1': { + canPrompt: true, + canLogprobs: false, + responseFormats: ['json', 'text', 'srt', 'verbose_json', 'vtt'], + timestampGranularities: true, + }, +}; + +interface TranscribeArgs { + file: unknown; + model?: string; + response_format?: string; + language?: string; + prompt?: string; + temperature?: number; + logprobs?: boolean; + timestamp_granularities?: string[]; + chunking_strategy?: string; + known_speaker_names?: string[]; + known_speaker_references?: unknown[]; + extra_body?: Record; + stream?: boolean; + test_mode?: boolean; +} + +export class SpeechToTextDriver extends PuterDriver { + readonly driverInterface = 'puter-speech2txt'; + readonly driverName = 'openai-speech2txt'; + readonly isDefault = true; + + override getReportedCosts(): Record[] { + return Object.entries(SPEECH_TO_TEXT_COSTS).map( + ([usageType, ucentsPerUnit]) => ({ + usageType, + ucentsPerUnit, + unit: 'second', + source: 'driver:aiSpeech2Txt', + }), + ); + } + + #openai: OpenAI | null = null; + + override onServerStart() { + const providers = (this.config.providers ?? {}) as Record< + string, + Record | undefined + >; + const readKey = ( + ...cfgs: Array | undefined> + ): string | undefined => { + for (const cfg of cfgs) { + if (!cfg) continue; + const k = + (cfg.apiKey as string | undefined) ?? + (cfg.secret_key as string | undefined); + if (k) return k; + } + return undefined; + }; + const apiKey = readKey( + providers['openai-speech-to-text'], + providers['openai-completion'], + providers['openai'], + ); + if (!apiKey) return; // Leave uninitialized; convert() will reject. + this.#openai = new OpenAI({ apiKey }); + } + + async list_models() { + return Object.entries(MODEL_CAPS).map(([id, caps]) => ({ + id, + name: id, + type: caps.diarization + ? 'transcription' + : id === 'whisper-1' + ? 'translation' + : 'transcription', + response_formats: caps.responseFormats, + supports_prompt: caps.canPrompt, + supports_logprobs: caps.canLogprobs, + ...(caps.diarization ? { supports_diarization: true } : {}), + ...(caps.timestampGranularities + ? { supports_timestamp_granularities: true } + : {}), + })); + } + + async transcribe(args: TranscribeArgs) { + return this.#handleTranscription(args, false); + } + + async translate(args: TranscribeArgs) { + return this.#handleTranscription(args, true); + } + + async #handleTranscription(args: TranscribeArgs, translate: boolean) { + if (args.test_mode) { + return { + ...SAMPLE_TRANSCRIPT, + model: + args.model || + (translate + ? DEFAULT_TRANSLATE_MODEL + : DEFAULT_TRANSCRIBE_MODEL), + }; + } + if (args.stream) { + throw new HttpError( + 400, + 'Streaming transcription is not yet supported', + ); + } + if (!this.#openai) + throw new HttpError(500, 'OpenAI API key not configured'); + if (!args.file) throw new HttpError(400, '`file` is required'); + + const actor = Context.get('actor'); + if (!actor) throw new HttpError(401, 'Authentication required'); + + const loaded = await loadFileInput( + this.stores, + this.services.fs, + actor, + args.file, + { maxBytes: MAX_AUDIO_FILE_SIZE }, + ); + + const selectedModel = + args.model || + (translate ? DEFAULT_TRANSLATE_MODEL : DEFAULT_TRANSCRIBE_MODEL); + const caps = MODEL_CAPS[selectedModel]; + if (!caps) { + throw new HttpError(400, `Unsupported model: ${selectedModel}`); + } + + if ( + args.response_format && + !caps.responseFormats.includes(args.response_format) + ) { + throw new HttpError( + 400, + `response_format must be one of: ${caps.responseFormats.join(', ')}`, + ); + } + if (args.prompt && !caps.canPrompt) { + throw new HttpError( + 400, + `prompt is not supported for model ${selectedModel}`, + ); + } + if (args.logprobs && !caps.canLogprobs) { + throw new HttpError( + 400, + `logprobs is not supported for model ${selectedModel}`, + ); + } + + // Estimate seconds from raw bytes — 16 kbps is a conservative speech-audio + // lower bound. Full metadata parsing (music-metadata) is deferred — clients + // aren't observably sensitive to billing-time delta vs real duration. + const estimatedSeconds = Math.max( + 1, + Math.ceil(loaded.buffer.byteLength / 16000), + ); + const usageType = `openai:${selectedModel}:second`; + const ucentsPerSecond = SPEECH_TO_TEXT_COSTS[usageType] ?? 0; + const estimatedCost = ucentsPerSecond * estimatedSeconds; + const allowed = await this.services.metering.hasEnoughCredits( + actor, + estimatedCost, + ); + if (!allowed) throw new HttpError(402, 'Insufficient credits'); + + const openaiFile = await toFile( + loaded.buffer, + loaded.filename, + loaded.mimeType ? { type: loaded.mimeType } : undefined, + ); + + const payload: Record = { + file: openaiFile, + model: selectedModel, + }; + if (args.response_format) + payload.response_format = args.response_format; + if (args.language) payload.language = args.language; + if (typeof args.temperature === 'number') + payload.temperature = args.temperature; + if (args.prompt && caps.canPrompt) payload.prompt = args.prompt; + if (args.logprobs && caps.canLogprobs) payload.logprobs = args.logprobs; + if (args.timestamp_granularities && caps.timestampGranularities) { + payload.timestamp_granularities = args.timestamp_granularities; + } + if (caps.diarization) { + if (!args.response_format) + payload.response_format = 'diarized_json'; + const needsChunking = + caps.requiresChunkingOverThirtySeconds && estimatedSeconds > 30; + const strategy = + args.chunking_strategy ?? (needsChunking ? 'auto' : undefined); + if (strategy) payload.chunking_strategy = strategy; + + if (args.known_speaker_names || args.known_speaker_references) { + payload.extra_body = { + ...(args.extra_body ?? {}), + ...(args.known_speaker_names + ? { known_speaker_names: args.known_speaker_names } + : {}), + ...(args.known_speaker_references + ? { + known_speaker_references: + args.known_speaker_references, + } + : {}), + }; + } + } else if (args.extra_body) { + payload.extra_body = args.extra_body; + } + + const result = translate + ? await this.#openai.audio.translations.create( + payload as Parameters< + OpenAI['audio']['translations']['create'] + >[0], + ) + : await this.#openai.audio.transcriptions.create( + payload as Parameters< + OpenAI['audio']['transcriptions']['create'] + >[0], + ); + + this.services.metering.incrementUsage( + actor, + usageType, + estimatedSeconds, + ucentsPerSecond * estimatedSeconds, + ); + + // Text response_format: return raw string; otherwise forward the OpenAI object. + if (args.response_format === 'text') { + return typeof result === 'string' + ? result + : ((result as { text?: string }).text ?? ''); + } + return result; + } +} diff --git a/src/backend/drivers/ai-speech2txt/costs.ts b/src/backend/drivers/ai-speech2txt/costs.ts new file mode 100644 index 000000000..6d11a8e88 --- /dev/null +++ b/src/backend/drivers/ai-speech2txt/costs.ts @@ -0,0 +1,7 @@ +// Microcents per second of audio, per OpenAI transcription model. +export const SPEECH_TO_TEXT_COSTS: Record = { + 'openai:gpt-4o-transcribe:second': 10000, + 'openai:gpt-4o-mini-transcribe:second': 5000, + 'openai:gpt-4o-transcribe-diarize:second': 10000, + 'openai:whisper-1:second': 10000, +}; diff --git a/src/backend/drivers/ai-tts/TTSDriver.ts b/src/backend/drivers/ai-tts/TTSDriver.ts new file mode 100644 index 000000000..9584ff276 --- /dev/null +++ b/src/backend/drivers/ai-tts/TTSDriver.ts @@ -0,0 +1,241 @@ +import { Context } from '../../core/context.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import type { DriverStreamResult } from '../meta.js'; +import { PuterDriver } from '../types.js'; +import { AWSPollyTTSProvider } from './providers/awsPolly/AWSPollyTTSProvider.js'; +import { ElevenLabsTTSProvider } from './providers/elevenlabs/ElevenLabsTTSProvider.js'; +import { OpenAITTSProvider } from './providers/openai/OpenAITTSProvider.js'; +import type { + ISynthesizeArgs, + ITTSEngine, + ITTSProvider, + ITTSVoice, +} from './types.js'; + +/** + * Driver implementing the `puter-tts` interface. + * + * Manages multiple upstream TTS providers (OpenAI, ElevenLabs, AWS Polly) + * and handles provider routing, voice/engine aggregation, and speech + * synthesis. Each provider is an `ITTSProvider` instantiated from config + * on boot. + */ +// puter-js still routes TTS via legacy per-provider driver names rather +// than passing `{ provider }` in args, so alias the unified driver under +// the names the client expects. `#providerFromAlias` normalizes those +// aliases to the internal provider keys used by `#providers`. +const TTS_ALIASES = ['aws-polly', 'openai-tts', 'elevenlabs-tts'] as const; +type TTSAlias = (typeof TTS_ALIASES)[number]; +const ALIAS_TO_PROVIDER: Record = { + 'aws-polly': 'aws-polly', + 'openai-tts': 'openai', + 'elevenlabs-tts': 'elevenlabs', +}; + +export class TTSDriver extends PuterDriver { + readonly driverInterface = 'puter-tts'; + readonly driverName = 'ai-tts'; + readonly driverAliases = [...TTS_ALIASES]; + readonly isDefault = true; + + #providers: Record = {}; + + /** Resolve a provider name from the alias the caller used, if any. */ + #providerFromAlias(): string | undefined { + const alias = Context.get('driverName') as string | undefined; + if (!alias) return undefined; + return ALIAS_TO_PROVIDER[alias as TTSAlias]; + } + + override onServerStart() { + this.#registerProviders(); + } + + // ── Interface methods ─────────────────────────────────────────── + + /** + * List all available voices across all configured providers. + */ + async list_voices(args?: Record): Promise { + const provider = + (args?.provider as string | undefined) ?? this.#providerFromAlias(); + + if (provider) { + const p = this.#providers[provider]; + if (!p) return []; + return p.listVoices(args); + } + + const allVoices: ITTSVoice[] = []; + for (const p of Object.values(this.#providers)) { + const voices = await p.listVoices(args); + allVoices.push(...voices); + } + return allVoices; + } + + /** + * List all available engines/models across all configured providers. + */ + async list_engines(args?: Record): Promise { + const provider = + (args?.provider as string | undefined) ?? this.#providerFromAlias(); + + if (provider) { + const p = this.#providers[provider]; + if (!p) return []; + return p.listEngines(); + } + + const allEngines: ITTSEngine[] = []; + for (const p of Object.values(this.#providers)) { + const engines = await p.listEngines(); + allEngines.push(...engines); + } + return allEngines; + } + + /** + * List provider names that are currently configured. + */ + async list(): Promise { + return Object.keys(this.#providers); + } + + override getReportedCosts(): Record[] { + const all: Record[] = []; + for (const p of Object.values(this.#providers)) { + const fn = ( + p as unknown as { + getReportedCosts?: () => Record[]; + } + ).getReportedCosts; + if (typeof fn === 'function') { + try { + const entries = fn.call(p); + if (Array.isArray(entries)) all.push(...entries); + } catch { + // ignore — cost reporting is best-effort + } + } + } + return all; + } + + /** + * Synthesize speech from text. Routes to the appropriate provider + * based on the `provider` argument, or falls back to the first + * available provider. + */ + async synthesize( + args: ISynthesizeArgs, + ): Promise { + const actor = Context.get('actor'); + if (!actor) throw new HttpError(401, 'Authentication required'); + + const providerName = + args.provider || + this.#providerFromAlias() || + this.#getDefaultProviderName(); + if (!providerName) { + throw new HttpError(500, 'No TTS providers configured'); + } + + const provider = this.#providers[providerName]; + if (!provider) { + throw new HttpError( + 400, + `TTS provider not found: ${providerName}. Available: ${Object.keys(this.#providers).join(', ')}`, + ); + } + + return provider.synthesize(args) as Promise< + DriverStreamResult | { url: string; content_type: string } + >; + } + + // ── Provider registration ─────────────────────────────────────── + + #registerProviders() { + const providers = this.config.providers ?? {}; + const m = this.services.metering; + + const openaiConfig = + (providers['openai-tts'] as Record | undefined) ?? + (providers['openai'] as Record | undefined); + const openaiKey = + (openaiConfig?.apiKey as string | undefined) ?? + (openaiConfig?.secret_key as string | undefined); + if (openaiKey) { + try { + this.#providers['openai'] = new OpenAITTSProvider(m, { + apiKey: openaiKey, + }); + } catch (e) { + console.warn( + '[TTSDriver] Failed to init OpenAI TTS provider:', + (e as Error).message, + ); + } + } + + const elevenlabs = providers['elevenlabs'] as + | Record + | undefined; + const elevenKey = + (elevenlabs?.apiKey as string | undefined) ?? + (elevenlabs?.api_key as string | undefined) ?? + (elevenlabs?.key as string | undefined); + if (elevenKey) { + try { + this.#providers['elevenlabs'] = new ElevenLabsTTSProvider(m, { + apiKey: elevenKey, + apiBaseUrl: elevenlabs?.apiBaseUrl as string | undefined, + defaultVoiceId: elevenlabs?.defaultVoiceId as + | string + | undefined, + }); + } catch (e) { + console.warn( + '[TTSDriver] Failed to init ElevenLabs TTS provider:', + (e as Error).message, + ); + } + } + + const polly = providers['aws-polly'] as + | Record + | undefined; + const pollyAws = (polly?.aws ?? polly) as + | Record + | undefined; + const pollyAccessKey = pollyAws?.access_key as string | undefined; + const pollySecretKey = pollyAws?.secret_key as string | undefined; + const pollyRegion = + (pollyAws?.region as string | undefined) ?? + (polly?.region as string | undefined); + if (pollyAccessKey && pollySecretKey) { + try { + this.#providers['aws-polly'] = new AWSPollyTTSProvider(m, { + access_key: pollyAccessKey, + secret_key: pollySecretKey, + region: pollyRegion, + }); + } catch (e) { + console.warn( + '[TTSDriver] Failed to init AWS Polly TTS provider:', + (e as Error).message, + ); + } + } + } + + #getDefaultProviderName(): string | null { + const names = Object.keys(this.#providers); + if (names.length === 0) return null; + // Prefer openai, then elevenlabs, then aws-polly + if (this.#providers['openai']) return 'openai'; + if (this.#providers['elevenlabs']) return 'elevenlabs'; + return names[0]; + } +} diff --git a/src/backend/drivers/ai-tts/providers/TTSProvider.ts b/src/backend/drivers/ai-tts/providers/TTSProvider.ts new file mode 100644 index 000000000..e8246d2cc --- /dev/null +++ b/src/backend/drivers/ai-tts/providers/TTSProvider.ts @@ -0,0 +1,48 @@ +/** + * Abstract base for TTS providers. Each provider wraps a single upstream + * API (OpenAI, ElevenLabs, AWS Polly) and exposes the unified + * `ITTSProvider` contract. + */ + +import type { MeteringService } from '../../../services/metering/MeteringService.js'; +import type { + ITTSProvider, + ITTSVoice, + ITTSEngine, + ISynthesizeArgs, +} from '../types.js'; + +export abstract class TTSProvider implements ITTSProvider { + abstract readonly providerName: string; + + protected meteringService: MeteringService; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + protected providerConfig: any; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(meteringService: MeteringService, config: any) { + this.meteringService = meteringService; + this.providerConfig = config; + } + + async listVoices(_args?: Record): Promise { + return []; + } + + async listEngines(): Promise { + return []; + } + + async synthesize(_args: ISynthesizeArgs): Promise { + throw new Error('Method not implemented.'); + } + + /** + * Provider-specific cost catalogue used by the TTSDriver's aggregated + * `getReportedCosts()`. Subclasses override to expose their per-unit + * metering costs. Shape matches the `WithCostsReporting` contract. + */ + getReportedCosts(): Record[] { + return []; + } +} diff --git a/src/backend/drivers/ai-tts/providers/awsPolly/AWSPollyTTSProvider.ts b/src/backend/drivers/ai-tts/providers/awsPolly/AWSPollyTTSProvider.ts new file mode 100644 index 000000000..594d6cf03 --- /dev/null +++ b/src/backend/drivers/ai-tts/providers/awsPolly/AWSPollyTTSProvider.ts @@ -0,0 +1,276 @@ +import { + PollyClient, + SynthesizeSpeechCommand, + DescribeVoicesCommand, + type Engine, + type LanguageCode, + type VoiceId, +} from '@aws-sdk/client-polly'; +import { HttpError } from '../../../../core/http/HttpError.js'; +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { DriverStreamResult } from '../../../meta.js'; +import type { ITTSVoice, ITTSEngine, ISynthesizeArgs } from '../../types.js'; +import { TTSProvider } from '../TTSProvider.js'; +import { AWS_POLLY_COSTS } from './costs.js'; + +const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3'; + +const VALID_ENGINES = ['standard', 'neural', 'long-form', 'generative']; + +interface PollyVoicesResponse { + Voices: any[]; +} + +/** + * AWS Polly TTS provider. Wraps the AWS Polly speech synthesis API and + * returns audio as a DriverStreamResult. Includes voice caching and + * engine-aware voice selection. + */ +export class AWSPollyTTSProvider extends TTSProvider { + readonly providerName = 'aws-polly'; + + private clients: Record = {}; + private voicesCache: { data: PollyVoicesResponse; expires: number } | null = + null; + + constructor( + meteringService: MeteringService, + config: { + access_key: string; + secret_key: string; + region?: string; + }, + ) { + super(meteringService, config); + } + + private getClient(region?: string): PollyClient { + const cfg = this.providerConfig as { + access_key: string; + secret_key: string; + region?: string; + }; + const resolvedRegion = region ?? cfg.region ?? 'us-west-2'; + + if (this.clients[resolvedRegion]) { + return this.clients[resolvedRegion]; + } + + this.clients[resolvedRegion] = new PollyClient({ + credentials: { + accessKeyId: cfg.access_key, + secretAccessKey: cfg.secret_key, + }, + region: resolvedRegion, + }); + + return this.clients[resolvedRegion]; + } + + private async describeVoices(): Promise { + // Simple in-memory cache with 10-minute TTL + if (this.voicesCache && Date.now() < this.voicesCache.expires) { + return this.voicesCache.data; + } + + const client = this.getClient(); + const command = new DescribeVoicesCommand({}); + const response = await client.send(command); + + this.voicesCache = { + data: response as PollyVoicesResponse, + expires: Date.now() + 10 * 60 * 1000, + }; + + return response as PollyVoicesResponse; + } + + private async getLanguageAppropriateVoice( + language: string, + engine: string, + ): Promise { + const voices = await this.describeVoices(); + + const voice = voices.Voices.find( + (v: any) => + v.LanguageCode === language && + v.SupportedEngines?.includes(engine), + ); + return voice ? voice.Id : null; + } + + private async getDefaultVoiceForEngine(engine: string): Promise { + const voices = await this.describeVoices(); + + const defaultVoices: Record = { + standard: ['Salli', 'Joanna', 'Matthew'], + neural: ['Joanna', 'Matthew', 'Salli'], + 'long-form': ['Joanna', 'Matthew'], + generative: ['Joanna', 'Matthew', 'Salli'], + }; + + const preferred = defaultVoices[engine] || ['Salli']; + + for (const voiceName of preferred) { + const voice = voices.Voices.find( + (v: any) => + v.Id === voiceName && v.SupportedEngines?.includes(engine), + ); + if (voice) return voice.Id; + } + + // Fallback: any voice that supports the engine + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const fallback = voices.Voices.find((v: any) => + v.SupportedEngines?.includes(engine), + ); + return fallback ? fallback.Id : 'Salli'; + } + + async listVoices(args?: Record): Promise { + const engine = args?.engine as string | undefined; + const pollyVoices = await this.describeVoices(); + + let voices = pollyVoices.Voices; + + if (engine) { + if (VALID_ENGINES.includes(engine)) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + voices = voices.filter((voice: any) => + voice.SupportedEngines?.includes(engine), + ); + } else { + throw new HttpError( + 400, + `Invalid engine: ${engine}. Valid engines: ${VALID_ENGINES.join(', ')}`, + { + fields: { engine, valid_engines: VALID_ENGINES }, + }, + ); + } + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return voices.map((voice: any) => ({ + id: voice.Id, + name: voice.Name, + language: { + name: voice.LanguageName, + code: voice.LanguageCode, + }, + provider: 'aws-polly', + supported_engines: voice.SupportedEngines || ['standard'], + })); + } + + async listEngines(): Promise { + return VALID_ENGINES.map((engine) => ({ + id: engine, + name: engine.charAt(0).toUpperCase() + engine.slice(1), + provider: 'aws-polly', + pricing_per_million_chars: AWS_POLLY_COSTS[engine] / 100, // microcents to dollars + })); + } + + override getReportedCosts(): Record[] { + return Object.entries(AWS_POLLY_COSTS).map( + ([engine, ucentsPerUnit]) => ({ + usageType: `aws-polly:${engine}:character`, + ucentsPerUnit, + unit: 'character', + source: 'driver:aiTts/aws-polly', + }), + ); + } + + async synthesize( + args: ISynthesizeArgs, + ): Promise { + const { + text, + voice: voiceArg, + ssml, + language, + engine = 'standard', + test_mode, + } = args; + + if (test_mode) { + return { url: SAMPLE_AUDIO_URL, content_type: 'audio' }; + } + + if (!VALID_ENGINES.includes(engine)) { + throw new HttpError( + 400, + `Invalid engine: ${engine}. Valid engines: ${VALID_ENGINES.join(', ')}`, + { + fields: { engine, valid_engines: VALID_ENGINES }, + }, + ); + } + + if (typeof text !== 'string' || text.trim() === '') { + throw new HttpError(400, 'Missing required field: text', { + legacyCode: 'field_required', + fields: { key: 'text' }, + }); + } + + const actor = Context.get('actor')!; + const usageType = `aws-polly:${engine}:character`; + const ucentsPerChar = AWS_POLLY_COSTS[engine] ?? 0; + const totalCost = ucentsPerChar * text.length; + + const usageAllowed = await this.meteringService.hasEnoughCredits( + actor, + totalCost, + ); + if (!usageAllowed) { + throw new HttpError(402, 'Insufficient funds', { + legacyCode: 'insufficient_funds', + }); + } + + // Resolve voice + let voice = voiceArg ?? undefined; + + if (!voice && language) { + voice = + (await this.getLanguageAppropriateVoice(language, engine)) ?? + undefined; + } + + if (!voice) { + voice = await this.getDefaultVoiceForEngine(engine); + } + + const client = this.getClient(); + + const params = { + Engine: engine as Engine, + OutputFormat: 'mp3' as const, + Text: text, + VoiceId: voice as VoiceId, + LanguageCode: (language ?? 'en-US') as LanguageCode, + TextType: (ssml ? 'ssml' : 'text') as 'ssml' | 'text', + }; + + const command = new SynthesizeSpeechCommand(params); + const response = await client.send(command); + + this.meteringService.incrementUsage( + actor, + usageType, + text.length, + totalCost, + ); + + return { + dataType: 'stream', + content_type: 'audio/mpeg', + chunked: true, + stream: response.AudioStream as unknown as import('node:stream').Readable, + }; + } +} diff --git a/src/backend/drivers/ai-tts/providers/awsPolly/costs.ts b/src/backend/drivers/ai-tts/providers/awsPolly/costs.ts new file mode 100644 index 000000000..ad82a0bed --- /dev/null +++ b/src/backend/drivers/ai-tts/providers/awsPolly/costs.ts @@ -0,0 +1,7 @@ +// Microcents per character, per Polly engine. +export const AWS_POLLY_COSTS: Record = { + standard: 400, // $4.00 per 1M characters + neural: 1600, // $16.00 per 1M characters + 'long-form': 10000, // $100.00 per 1M characters + generative: 3000, // $30.00 per 1M characters +}; diff --git a/src/backend/drivers/ai-tts/providers/elevenlabs/ElevenLabsTTSProvider.ts b/src/backend/drivers/ai-tts/providers/elevenlabs/ElevenLabsTTSProvider.ts new file mode 100644 index 000000000..0a7badb13 --- /dev/null +++ b/src/backend/drivers/ai-tts/providers/elevenlabs/ElevenLabsTTSProvider.ts @@ -0,0 +1,223 @@ +import { Readable } from 'node:stream'; +import { HttpError } from '../../../../core/http/HttpError.js'; +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { DriverStreamResult } from '../../../meta.js'; +import type { ITTSVoice, ITTSEngine, ISynthesizeArgs } from '../../types.js'; +import { TTSProvider } from '../TTSProvider.js'; +import { ELEVENLABS_TTS_COSTS } from './costs.js'; + +const DEFAULT_MODEL = 'eleven_multilingual_v2'; +const DEFAULT_VOICE_ID = '21m00Tcm4TlvDq8ikWAM'; // "Rachel" sample voice +const DEFAULT_OUTPUT_FORMAT = 'mp3_44100_128'; +const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3'; + +const ELEVENLABS_TTS_MODELS = [ + { id: DEFAULT_MODEL, name: 'Eleven Multilingual v2' }, + { id: 'eleven_flash_v2_5', name: 'Eleven Flash v2.5' }, + { id: 'eleven_turbo_v2_5', name: 'Eleven Turbo v2.5' }, + { id: 'eleven_v3', name: 'Eleven v3 Alpha' }, +]; + +/** + * ElevenLabs TTS provider. Uses the ElevenLabs REST API to synthesize + * speech and returns audio as a DriverStreamResult. + */ +export class ElevenLabsTTSProvider extends TTSProvider { + readonly providerName = 'elevenlabs'; + + private apiKey: string; + private baseUrl: string; + private defaultVoiceId: string; + + constructor( + meteringService: MeteringService, + config: { + apiKey: string; + apiBaseUrl?: string; + defaultVoiceId?: string; + }, + ) { + super(meteringService, config); + + this.apiKey = config.apiKey; + this.baseUrl = config.apiBaseUrl ?? 'https://api.elevenlabs.io'; + this.defaultVoiceId = config.defaultVoiceId ?? DEFAULT_VOICE_ID; + } + + private async request( + path: string, + opts: { + method?: string; + body?: unknown; + headers?: Record; + } = {}, + ): Promise { + const { method = 'GET', body, headers = {} } = opts; + + const response = await fetch(`${this.baseUrl}${path}`, { + method, + headers: { + 'xi-api-key': this.apiKey, + ...(body ? { 'Content-Type': 'application/json' } : {}), + ...headers, + }, + body: body ? JSON.stringify(body) : undefined, + }); + + if (response.ok) { + return response; + } + + let detail: unknown = null; + try { + detail = await response.json(); + } catch { + // ignore + } + + console.error('[ElevenLabsTTSProvider] request failed', { + path, + status: response.status, + detail, + }); + throw new HttpError( + 502, + `ElevenLabs request failed (status ${response.status})`, + { + fields: { provider: 'elevenlabs', status: response.status }, + }, + ); + } + + async listVoices(): Promise { + const res = await this.request('/v1/voices'); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const data: any = await res.json(); + const voices = Array.isArray(data?.voices) + ? data.voices + : Array.isArray(data) + ? data + : []; + + return ( + voices + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .map((voice: any) => ({ + id: voice.voice_id || voice.voiceId || voice.id, + name: voice.name, + description: voice.description, + category: voice.category, + provider: 'elevenlabs' as const, + labels: voice.labels, + supported_models: ELEVENLABS_TTS_MODELS.map((m) => m.id), + })) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .filter((v: any) => v.id && v.name) + ); + } + + async listEngines(): Promise { + return ELEVENLABS_TTS_MODELS.map((model) => ({ + id: model.id, + name: model.name, + provider: 'elevenlabs', + pricing_per_million_chars: 0, + })); + } + + override getReportedCosts(): Record[] { + return Object.entries(ELEVENLABS_TTS_COSTS).map( + ([model, ucentsPerUnit]) => ({ + usageType: `elevenlabs:${model}:character`, + ucentsPerUnit, + unit: 'character', + source: 'driver:aiTts/elevenlabs', + }), + ); + } + + async synthesize( + args: ISynthesizeArgs, + ): Promise { + const { + text, + voice: voiceArg, + model: modelArg, + response_format, + output_format, + voice_settings, + voiceSettings, + test_mode, + } = args; + + if (test_mode) { + return { url: SAMPLE_AUDIO_URL, content_type: 'audio' }; + } + + if (typeof text !== 'string' || !text.trim()) { + throw new HttpError(400, 'Missing required field: text', { + legacyCode: 'field_required', + fields: { key: 'text' }, + }); + } + + const voiceId = voiceArg || this.defaultVoiceId; + const modelId = modelArg || DEFAULT_MODEL; + const desiredFormat = + output_format || response_format || DEFAULT_OUTPUT_FORMAT; + + const actor = Context.get('actor')!; + const usageKey = `elevenlabs:${modelId}:character`; + const ucentsPerChar = ELEVENLABS_TTS_COSTS[modelId] ?? 0; + const totalCost = ucentsPerChar * text.length; + + const usageAllowed = await this.meteringService.hasEnoughCredits( + actor, + totalCost, + ); + if (!usageAllowed) { + throw new HttpError(402, 'Insufficient funds', { + legacyCode: 'insufficient_funds', + }); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const payload: any = { + text, + model_id: modelId, + output_format: desiredFormat, + }; + + const finalVoiceSettings = voice_settings ?? voiceSettings; + if (finalVoiceSettings) { + payload.voice_settings = finalVoiceSettings; + } + + const response = await this.request(`/v1/text-to-speech/${voiceId}`, { + method: 'POST', + body: payload, + }); + + const arrayBuffer = await response.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + const stream = Readable.from(buffer); + + this.meteringService.incrementUsage( + actor, + usageKey, + text.length, + totalCost, + ); + + const contentType = + response.headers.get('content-type') || 'audio/mpeg'; + + return { + dataType: 'stream', + content_type: contentType, + chunked: true, + stream, + }; + } +} diff --git a/src/backend/drivers/ai-tts/providers/elevenlabs/costs.ts b/src/backend/drivers/ai-tts/providers/elevenlabs/costs.ts new file mode 100644 index 000000000..a10ae521a --- /dev/null +++ b/src/backend/drivers/ai-tts/providers/elevenlabs/costs.ts @@ -0,0 +1,10 @@ +// Microcents per character for TTS synthesis, per model. Values mirror the +// ElevenLabs scale tier (per-additional-char × 0.9). Seconds-based costs +// for speech-to-speech live on VoiceChangerDriver. +export const ELEVENLABS_TTS_COSTS: Record = { + eleven_multilingual_v2: 18000 * 0.9, + eleven_turbo_v2_5: 18000 * 0.9, + eleven_turbo_v2: 18000 * 0.9, + eleven_flash_v2_5: 9000 * 0.9, + eleven_v3: 18000 * 0.9, +}; diff --git a/src/backend/drivers/ai-tts/providers/openai/OpenAITTSProvider.ts b/src/backend/drivers/ai-tts/providers/openai/OpenAITTSProvider.ts new file mode 100644 index 000000000..1fde0287d --- /dev/null +++ b/src/backend/drivers/ai-tts/providers/openai/OpenAITTSProvider.ts @@ -0,0 +1,214 @@ +import OpenAI from 'openai'; +import { Readable } from 'node:stream'; +import { HttpError } from '../../../../core/http/HttpError.js'; +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { DriverStreamResult } from '../../../meta.js'; +import type { ITTSVoice, ITTSEngine, ISynthesizeArgs } from '../../types.js'; +import { TTSProvider } from '../TTSProvider.js'; +import { OPENAI_TTS_COSTS } from './costs.js'; + +const DEFAULT_MODEL = 'gpt-4o-mini-tts'; +const DEFAULT_VOICE = 'alloy'; +const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3'; + +const RESPONSE_CONTENT_TYPES: Record = { + mp3: 'audio/mpeg', + opus: 'audio/opus', + aac: 'audio/aac', + flac: 'audio/flac', + wav: 'audio/wav', + pcm: 'audio/pcm', +}; + +const OPENAI_TTS_VOICES = [ + { id: 'alloy', name: 'Alloy' }, + { id: 'ash', name: 'Ash' }, + { id: 'ballad', name: 'Ballad' }, + { id: 'coral', name: 'Coral' }, + { id: 'echo', name: 'Echo' }, + { id: 'fable', name: 'Fable' }, + { id: 'nova', name: 'Nova' }, + { id: 'onyx', name: 'Onyx' }, + { id: 'sage', name: 'Sage' }, + { id: 'shimmer', name: 'Shimmer' }, +]; + +const OPENAI_TTS_MODELS = [ + { + id: DEFAULT_MODEL, + name: 'GPT-4o mini TTS', + pricing_per_million_chars: 15, + }, + { + id: 'tts-1', + name: 'TTS 1', + pricing_per_million_chars: 15, + }, + { + id: 'tts-1-hd', + name: 'TTS 1 HD', + pricing_per_million_chars: 30, + }, +]; + +/** + * OpenAI TTS provider. Wraps the OpenAI speech synthesis API and + * returns audio as a DriverStreamResult. + */ +export class OpenAITTSProvider extends TTSProvider { + readonly providerName = 'openai'; + + private openai: OpenAI; + + constructor(meteringService: MeteringService, config: { apiKey: string }) { + super(meteringService, config); + this.openai = new OpenAI({ apiKey: config.apiKey }); + } + + async listVoices(): Promise { + return OPENAI_TTS_VOICES.map((voice) => ({ + id: voice.id, + name: voice.name, + language: { + name: 'English', + code: 'en', + }, + provider: 'openai', + supported_models: OPENAI_TTS_MODELS.map((m) => m.id), + })); + } + + async listEngines(): Promise { + return OPENAI_TTS_MODELS.map((model) => ({ + id: model.id, + name: model.name, + pricing_per_million_chars: model.pricing_per_million_chars, + provider: 'openai', + })); + } + + override getReportedCosts(): Record[] { + return Object.entries(OPENAI_TTS_COSTS).map( + ([model, ucentsPerUnit]) => ({ + usageType: `openai:${model}:character`, + ucentsPerUnit, + unit: 'character', + source: 'driver:aiTts/openai', + }), + ); + } + + async synthesize( + args: ISynthesizeArgs, + ): Promise { + const { + text, + voice: voiceArg, + model: modelArg, + response_format, + instructions, + test_mode, + } = args; + + if (test_mode) { + return { url: SAMPLE_AUDIO_URL, content_type: 'audio' }; + } + + if (typeof text !== 'string' || text.trim() === '') { + throw new HttpError(400, 'Missing required field: text', { + legacyCode: 'field_required', + fields: { key: 'text' }, + }); + } + + const model = modelArg || DEFAULT_MODEL; + if (!OPENAI_TTS_MODELS.find(({ id }) => id === model)) { + throw new HttpError( + 400, + `Invalid model: ${model}. Expected: ${OPENAI_TTS_MODELS.map(({ id }) => id).join(', ')}`, + { + legacyCode: 'field_invalid', + fields: { + key: 'model', + expected: OPENAI_TTS_MODELS.map(({ id }) => id).join( + ', ', + ), + got: model, + }, + }, + ); + } + + const voice = voiceArg || DEFAULT_VOICE; + if (!OPENAI_TTS_VOICES.find(({ id }) => id === voice)) { + throw new HttpError( + 400, + `Invalid voice: ${voice}. Expected: ${OPENAI_TTS_VOICES.map(({ id }) => id).join(', ')}`, + { + legacyCode: 'field_invalid', + fields: { + key: 'voice', + expected: OPENAI_TTS_VOICES.map(({ id }) => id).join( + ', ', + ), + got: voice, + }, + }, + ); + } + + const format = response_format || 'mp3'; + const contentType = + RESPONSE_CONTENT_TYPES[format] || RESPONSE_CONTENT_TYPES.mp3; + + const actor = Context.get('actor')!; + const usageType = `openai:${model}:character`; + const ucentsPerChar = OPENAI_TTS_COSTS[model] ?? 0; + const totalCost = ucentsPerChar * text.length; + + const usageAllowed = await this.meteringService.hasEnoughCredits( + actor, + totalCost, + ); + if (!usageAllowed) { + throw new HttpError(402, 'Insufficient funds', { + legacyCode: 'insufficient_funds', + }); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const payload: any = { + model, + voice, + input: text, + }; + + if (instructions) { + payload.instructions = instructions; + } + + if (response_format) { + payload.response_format = response_format; + } + + const response = await this.openai.audio.speech.create(payload); + const arrayBuffer = await response.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + const stream = Readable.from(buffer); + + this.meteringService.incrementUsage( + actor, + usageType, + text.length, + totalCost, + ); + + return { + dataType: 'stream', + content_type: contentType, + chunked: true, + stream, + }; + } +} diff --git a/src/backend/drivers/ai-tts/providers/openai/costs.ts b/src/backend/drivers/ai-tts/providers/openai/costs.ts new file mode 100644 index 000000000..370c763ab --- /dev/null +++ b/src/backend/drivers/ai-tts/providers/openai/costs.ts @@ -0,0 +1,6 @@ +// Microcents per character, per OpenAI TTS model. +export const OPENAI_TTS_COSTS: Record = { + 'gpt-4o-mini-tts': 1500, + 'tts-1': 1500, + 'tts-1-hd': 3000, +}; diff --git a/src/backend/drivers/ai-tts/types.ts b/src/backend/drivers/ai-tts/types.ts new file mode 100644 index 000000000..0bd6eb215 --- /dev/null +++ b/src/backend/drivers/ai-tts/types.ts @@ -0,0 +1,54 @@ +/** + * Types for the `puter-tts` driver interface. + */ + +export interface ITTSVoice { + id: string; + name: string; + language?: { + name: string; + code: string; + }; + description?: string; + category?: string; + provider: string; + labels?: Record; + supported_models?: string[]; + supported_engines?: string[]; +} + +export interface ITTSEngine { + id: string; + name: string; + provider: string; + pricing_per_million_chars?: number; +} + +export interface ISynthesizeArgs { + text: string; + voice?: string; + model?: string; + response_format?: string; + output_format?: string; + instructions?: string; + ssml?: string; + language?: string; + engine?: string; + voice_settings?: Record; + voiceSettings?: Record; + test_mode?: boolean; + provider?: string; +} + +export interface ITTSProvider { + readonly providerName: string; + + /** List voices available from this provider. */ + listVoices(args?: Record): Promise; + + /** List engines/models available from this provider. */ + listEngines(): Promise; + + /** Synthesize speech from text. Returns a DriverStreamResult. */ + synthesize(args: ISynthesizeArgs): Promise; +} diff --git a/src/backend/drivers/ai-video/VideoGenerationDriver.ts b/src/backend/drivers/ai-video/VideoGenerationDriver.ts new file mode 100644 index 000000000..a3c8912a6 --- /dev/null +++ b/src/backend/drivers/ai-video/VideoGenerationDriver.ts @@ -0,0 +1,338 @@ +import { Context } from '../../core/context.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import { PuterDriver } from '../types.js'; +import { GeminiVideoProvider } from './providers/gemini/GeminiVideoProvider.js'; +import { OpenAIVideoProvider } from './providers/openai/OpenAIVideoProvider.js'; +import { TogetherVideoProvider } from './providers/together/TogetherVideoProvider.js'; +import type { + IGenerateVideoParams, + IVideoModel, + IVideoProvider, +} from './types.js'; + +const DEFAULT_PROVIDER = 'openai-video-generation'; + +/** + * Driver implementing the `puter-video-generation` interface. + * + * Manages multiple upstream providers (OpenAI/Sora, Together, Gemini/Veo, ...) + * and handles model resolution, provider routing, and parameter normalisation. + * Each provider is a plain `IVideoProvider` -- the driver instantiates them + * from config on boot. + * + * Providers handle their own metering internally. + */ +export class VideoGenerationDriver extends PuterDriver { + readonly driverInterface = 'puter-video-generation'; + readonly driverName = 'ai-video'; + // puter-js's `txt2vid` can pass a provider id via `options.driver`, so + // alias all provider ids here. `generate` falls back to + // `Context.driverName` when `args.provider` isn't supplied. + readonly driverAliases = [ + 'openai-video-generation', + 'together-video-generation', + 'gemini-video-generation', + ]; + readonly isDefault = true; + + #providers: Record = {}; + #modelIdMap: Record = {}; + + override onServerStart() { + this.#registerProviders(); + this.#buildModelMap(); + } + + // -- Interface methods --------------------------------------------------- + + async models() { + const seen = new Set(); + return Object.values(this.#modelIdMap) + .flat() + .filter((model) => { + const identity = `${model.provider}:${model.puterId || model.id}`; + if (seen.has(identity)) return false; + seen.add(identity); + return true; + }) + .sort((a, b) => { + if (a.provider === b.provider) return a.id.localeCompare(b.id); + return a.provider!.localeCompare(b.provider!); + }); + } + + async list() { + return (await this.models()).map((m) => m.puterId || m.id).sort(); + } + + override getReportedCosts(): Record[] { + const out: Record[] = []; + const seen = new Set(); + for (const bucket of Object.values(this.#modelIdMap)) { + for (const model of bucket) { + const key = `${model.provider}:${model.id}`; + if (seen.has(key)) continue; + seen.add(key); + for (const [costKey, raw] of Object.entries( + (model as { costs?: Record }).costs ?? {}, + )) { + if (typeof raw !== 'number' || !Number.isFinite(raw)) + continue; + out.push({ + usageType: `${model.provider}:${model.id}:${costKey}`, + costValue: raw, + source: `driver:aiVideo/${model.provider}`, + }); + } + } + } + return out; + } + + async generate(args: IGenerateVideoParams) { + const actor = Context.get('actor'); + if (!actor) throw new HttpError(401, 'Authentication required'); + + if (args.model) { + args.model = args.model.trim().toLowerCase(); + } + + const configuredProviders = Object.keys(this.#providers); + if (configuredProviders.length === 0) { + throw new Error('no video generation providers configured'); + } + + let intendedProvider = + args.provider ?? + (Context.get('driverName') as string | undefined) ?? + ''; + + if (!args.model && !intendedProvider) { + intendedProvider = configuredProviders.includes(DEFAULT_PROVIDER) + ? DEFAULT_PROVIDER + : configuredProviders[0]; + } + + if (intendedProvider && !this.#providers[intendedProvider]) { + intendedProvider = configuredProviders[0]; + } + + if (!args.model && intendedProvider) { + args.model = this.#providers[intendedProvider].getDefaultModel(); + } + + const model = args.model + ? this.#resolveModel(args.model, intendedProvider) + : undefined; + + if (!model) { + throw new HttpError(400, `Model not found: ${args.model}`); + } + + const provider = this.#providers[model.provider!]; + if (!provider) { + throw new HttpError(500, `No provider found for model ${model.id}`); + } + + // Validate / normalise duration + if (model.durationSeconds?.length) { + const requestedSeconds = args.seconds ?? args.duration; + const normalizedSeconds = + typeof requestedSeconds === 'string' + ? Number.parseInt(requestedSeconds, 10) + : requestedSeconds; + const validSeconds = model.durationSeconds.includes( + Number(normalizedSeconds), + ) + ? normalizedSeconds + : model.durationSeconds[0]; + args.seconds = validSeconds; + args.duration = validSeconds; + } + + // Validate / normalise dimensions + if (model.dimensions?.length) { + const requestedResolution = + typeof args.size === 'string' && args.size.trim() + ? args.size + : typeof args.resolution === 'string' && + args.resolution.trim() + ? args.resolution + : undefined; + + const normalizedResolution = + requestedResolution && + model.dimensions.includes(requestedResolution) + ? requestedResolution + : model.dimensions[0]; + args.size = normalizedResolution; + args.resolution = normalizedResolution; + } + + return await provider.generate({ + ...args, + model: model.id, + provider: model.provider, + }); + } + + // -- Provider registration ----------------------------------------------- + + #registerProviders() { + const providers = this.config.providers ?? {}; + const m = this.services.metering; + + // Same lenient reader as ImageGenerationDriver — accept + // `apiKey || secret_key`, and fall back from the video-specific + // provider key to the shared chat key when unset. + const readKey = ( + ...cfgs: Array | undefined> + ): string | undefined => { + for (const cfg of cfgs) { + if (!cfg) continue; + const k = + (cfg.apiKey as string | undefined) ?? + (cfg.secret_key as string | undefined); + if (k) return k; + } + return undefined; + }; + + const openaiKey = readKey( + providers['openai-video-generation'], + providers['openai-completion'], + providers['openai'], + ); + if (openaiKey) { + this.#providers['openai-video-generation'] = + new OpenAIVideoProvider({ apiKey: openaiKey }, m); + } + + const togetherKey = readKey( + providers['together-video-generation'], + providers['together-ai'], + ); + if (togetherKey) { + this.#providers['together-video-generation'] = + new TogetherVideoProvider({ apiKey: togetherKey }, m); + } + + const geminiKey = readKey( + providers['gemini-video-generation'], + providers['gemini'], + ); + if (geminiKey) { + this.#providers['gemini-video-generation'] = + new GeminiVideoProvider({ apiKey: geminiKey }, m); + } + } + + // -- Model map ----------------------------------------------------------- + + async #buildModelMap() { + for (const providerName in this.#providers) { + const provider = this.#providers[providerName]; + for (const model of await provider.models()) { + model.id = model.id.trim().toLowerCase(); + if (model.puterId) { + model.puterId = model.puterId.trim().toLowerCase(); + } + if (model.aliases) { + model.aliases = model.aliases.map((alias) => + alias.trim().toLowerCase(), + ); + } + if (!this.#modelIdMap[model.id]) { + this.#modelIdMap[model.id] = []; + } + this.#modelIdMap[model.id].push({ + ...model, + provider: providerName, + }); + + if (model.puterId) { + if (model.aliases) { + model.aliases.push(model.puterId); + } else { + model.aliases = [model.puterId]; + } + + // Derive standard alias forms from puterId for model singularity: + // puterId "service:org/model" -> "org/model" and "model" + const withoutService = model.puterId.includes(':') + ? model.puterId.slice(model.puterId.indexOf(':') + 1) + : model.puterId; + if (!model.aliases.includes(withoutService)) { + model.aliases.push(withoutService); + } + const shortName = withoutService.includes('/') + ? withoutService.slice(withoutService.indexOf('/') + 1) + : withoutService; + if ( + shortName !== withoutService && + !model.aliases.includes(shortName) + ) { + model.aliases.push(shortName); + } + } + + if (model.aliases) { + for (let alias of model.aliases) { + alias = alias.trim().toLowerCase(); + if (!this.#modelIdMap[alias]) { + this.#modelIdMap[alias] = + this.#modelIdMap[model.id]; + continue; + } + if ( + this.#modelIdMap[alias] !== + this.#modelIdMap[model.id] + ) { + this.#modelIdMap[alias].push({ + ...model, + provider: providerName, + }); + this.#modelIdMap[model.id] = + this.#modelIdMap[alias]; + continue; + } + } + } + + // Sort: cheapest first + this.#modelIdMap[model.id].sort((a, b) => { + const aCostKey = + a.index_cost_key || + a.output_cost_key || + Object.keys(a.costs || {})[0]; + const bCostKey = + b.index_cost_key || + b.output_cost_key || + Object.keys(b.costs || {})[0]; + const aCost = a.costs?.[aCostKey] ?? Infinity; + const bCost = b.costs?.[bCostKey] ?? Infinity; + return aCost - bCost; + }); + } + } + } + + #resolveModel(modelId: string, provider?: string): IVideoModel | null { + const models = this.#modelIdMap[modelId?.trim().toLowerCase()]; + if (!models || models.length === 0) return null; + if (!provider) return models[0]; + + // Prefer exact primary ID match over alias matches + const exactIdMatch = models.find( + (m) => m.id === modelId && m.provider === provider, + ); + if (exactIdMatch) return exactIdMatch; + + const exactPuterIdMatch = models.find( + (m) => m.puterId === modelId && m.provider === provider, + ); + if (exactPuterIdMatch) return exactPuterIdMatch; + + return models.find((m) => m.provider === provider) ?? models[0]; + } +} diff --git a/src/backend/drivers/ai-video/providers/VideoProvider.ts b/src/backend/drivers/ai-video/providers/VideoProvider.ts new file mode 100644 index 000000000..3ac8da260 --- /dev/null +++ b/src/backend/drivers/ai-video/providers/VideoProvider.ts @@ -0,0 +1,22 @@ +import type { + IVideoModel, + IVideoProvider, + IGenerateVideoParams, +} from '../types.js'; + +/** + * Abstract base for AI video providers. Each provider wraps a single + * upstream API (OpenAI, Together, Gemini, ...) and exposes the unified + * `IVideoProvider` contract. + */ +export class VideoProvider implements IVideoProvider { + getDefaultModel(): string { + return ''; + } + models(): IVideoModel[] | Promise { + return []; + } + async generate(_params: IGenerateVideoParams): Promise { + throw new Error('Method not implemented.'); + } +} diff --git a/src/backend/drivers/ai-video/providers/gemini/GeminiVideoProvider.ts b/src/backend/drivers/ai-video/providers/gemini/GeminiVideoProvider.ts new file mode 100644 index 000000000..6817862ec --- /dev/null +++ b/src/backend/drivers/ai-video/providers/gemini/GeminiVideoProvider.ts @@ -0,0 +1,325 @@ +import { + GenerateVideosOperation, + GenerateVideosParameters, + GoogleGenAI, +} from '@google/genai'; +import { Context } from '../../../../core/context.js'; +import { HttpError } from '../../../../core/http/HttpError.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { IGenerateVideoParams, IVideoModel } from '../../types.js'; +import { VideoProvider } from '../VideoProvider.js'; +import { GEMINI_VIDEO_GENERATION_MODELS, IGeminiVideoModel } from './models.js'; + +const DEFAULT_TEST_VIDEO_URL = 'https://assets.puter.site/txt2vid.mp4'; +const POLL_INTERVAL_MS = 10_000; +const DEFAULT_TIMEOUT_MS = 10 * 60 * 1000; + +const DIMENSION_MAP: Record< + string, + { aspectRatio: string; resolution: string } +> = { + '1280x720': { aspectRatio: '16:9', resolution: '720p' }, + '720x1280': { aspectRatio: '9:16', resolution: '720p' }, + '1920x1080': { aspectRatio: '16:9', resolution: '1080p' }, + '1080x1920': { aspectRatio: '9:16', resolution: '1080p' }, + '3840x2160': { aspectRatio: '16:9', resolution: '4k' }, + '2160x3840': { aspectRatio: '9:16', resolution: '4k' }, +}; + +export class GeminiVideoProvider extends VideoProvider { + #client: GoogleGenAI; + #meteringService: MeteringService; + + constructor(config: { apiKey: string }, meteringService: MeteringService) { + super(); + if (!config.apiKey) { + throw new Error('Gemini video generation requires an API key'); + } + this.#client = new GoogleGenAI({ apiKey: config.apiKey }); + this.#meteringService = meteringService; + } + + getDefaultModel(): string { + return GEMINI_VIDEO_GENERATION_MODELS[0].id; + } + + async models(): Promise { + return GEMINI_VIDEO_GENERATION_MODELS.map((model) => ({ + ...model, + aliases: [model.id, `google/${model.id}`], + })); + } + + async generate(params: IGenerateVideoParams): Promise { + const { + prompt, + model: requestedModel, + seconds, + duration, + size, + resolution: _resolution, + negative_prompt: negativePrompt, + reference_images: referenceImages, + input_reference: inputReference, + last_frame: lastFrame, + test_mode: testMode, + } = params ?? {}; + + if (typeof prompt !== 'string' || !prompt.trim()) { + throw new HttpError(400, 'prompt must be a non-empty string'); + } + + const selectedModel = this.#getModel(requestedModel); + + if (testMode) { + return DEFAULT_TEST_VIDEO_URL; + } + + const hasFirstFrame = + selectedModel.supportsImageInput && + typeof inputReference === 'string' && + inputReference.trim().length > 0; + const hasRefImages = + selectedModel.supportsReferenceImages && + Array.isArray(referenceImages) && + referenceImages.length > 0; + + const { aspectRatio, videoResolution } = + this.#resolveAspectAndResolution(size, selectedModel); + + // 1080p and 4K require duration=8 + const isHighRes = + videoResolution === '1080p' || videoResolution === '4k'; + let durationSeconds = + this.#coercePositiveInteger(seconds ?? duration) ?? + selectedModel.durationSeconds?.[0] ?? + 8; + if (isHighRes || hasRefImages) { + durationSeconds = 8; + } + + const is4K = videoResolution === '4k'; + const is1080p = videoResolution === '1080p'; + const perSecondCents = is4K + ? (selectedModel.costs?.['per-second-4k'] ?? + selectedModel.costs?.['per-second']) + : is1080p + ? (selectedModel.costs?.['per-second-1080p'] ?? + selectedModel.costs?.['per-second']) + : selectedModel.costs?.['per-second']; + if (perSecondCents === undefined) { + throw new Error( + `No per-second cost configured for video model '${selectedModel.id}'`, + ); + } + const costCents = perSecondCents * durationSeconds; + const costInMicroCents = Math.ceil(costCents * 1_000_000); + + const actor = Context.get('actor'); + if (!actor) { + throw new HttpError(401, 'Authentication required'); + } + + const usageAllowed = await this.#meteringService.hasEnoughCredits( + actor, + costInMicroCents, + ); + if (!usageAllowed) { + throw new HttpError(402, 'Insufficient funds'); + } + + const config: Record = { + numberOfVideos: 1, + durationSeconds, + }; + + if (aspectRatio) config.aspectRatio = aspectRatio; + if (videoResolution && selectedModel.resolutions.length > 0) { + config.resolution = videoResolution; + } + if (typeof negativePrompt === 'string' && negativePrompt.trim()) { + config.negativePrompt = negativePrompt; + } + + // Reference images (Veo 3.1 supports up to 3) + // When referenceImages is set, image (first frame), video, and lastFrame are not supported. + if (hasRefImages) { + const validImages = referenceImages + .filter( + (img: string) => + typeof img === 'string' && img.trim().length > 0, + ) + .slice(0, 3); + config.referenceImages = validImages.map((img: string) => ({ + image: this.#parseImageInput(img), + referenceType: 'asset', + })); + } + + if ( + !hasRefImages && + typeof lastFrame === 'string' && + lastFrame.trim() + ) { + config.lastFrame = this.#parseImageInput(lastFrame); + } + + const generateParams: GenerateVideosParameters = { + model: selectedModel.id, + prompt, + config, + }; + + // First frame (image-to-video) + if (hasFirstFrame && !hasRefImages) { + generateParams.image = this.#parseImageInput( + inputReference as string, + ); + } + + let operation: GenerateVideosOperation; + try { + operation = + await this.#client.models.generateVideos(generateParams); + } catch (e) { + console.error('Gemini video generation error:', e); + throw e; + } + + const completed = await this.#pollUntilComplete(operation); + + const generatedVideos = completed.response?.generatedVideos; + if (!generatedVideos || generatedVideos.length === 0) { + const filtered = completed.response?.raiMediaFilteredCount ?? 0; + if (filtered > 0) { + const reasons = + completed.response?.raiMediaFilteredReasons?.join(', ') || + 'content policy'; + throw new Error(`Video was filtered due to ${reasons}`); + } + throw new Error('Gemini response did not include a video'); + } + + const video = generatedVideos[0].video; + if (!video) { + throw new Error('Gemini response video entry was empty'); + } + + const resTier = is4K + ? ':4k' + : is1080p && selectedModel.costs?.['per-second-1080p'] + ? ':1080p' + : ''; + const usageKey = `gemini:${selectedModel.id}${resTier}`; + await this.#meteringService.incrementUsage( + actor, + usageKey, + durationSeconds, + costInMicroCents, + ); + + if (video.uri) { + return video.uri; + } + + if (video.videoBytes) { + const mimeType = video.mimeType ?? 'video/mp4'; + return `data:${mimeType};base64,${video.videoBytes}`; + } + + throw new Error( + 'Gemini video response contained neither uri nor videoBytes', + ); + } + + async #pollUntilComplete( + operation: GenerateVideosOperation, + ): Promise { + let op = operation; + const start = Date.now(); + + while (!op.done) { + if (Date.now() - start > DEFAULT_TIMEOUT_MS) { + throw new Error( + 'Timed out waiting for Gemini video generation to complete', + ); + } + + await this.#delay(POLL_INTERVAL_MS); + op = await this.#client.operations.getVideosOperation({ + operation: op, + }); + } + + if (op.error) { + const msg = + (op.error as Record).message ?? + JSON.stringify(op.error); + throw new Error(`Gemini video generation failed: ${msg}`); + } + + return op; + } + + #parseImageInput(input: string): { imageBytes: string; mimeType: string } { + if (input.startsWith('data:')) { + const commaIdx = input.indexOf(','); + if (commaIdx !== -1) { + const header = input.substring(5, commaIdx); + if (header.endsWith(';base64')) { + const mimeType = header.substring(0, header.length - 7); + if (mimeType.length > 0) { + return { + imageBytes: input.substring(commaIdx + 1), + mimeType, + }; + } + } + } + } + return { imageBytes: input, mimeType: 'image/png' }; + } + + #getModel(requestedModel?: string): IGeminiVideoModel { + return ( + GEMINI_VIDEO_GENERATION_MODELS.find( + (m) => m.id === requestedModel, + ) ?? GEMINI_VIDEO_GENERATION_MODELS[0] + ); + } + + #resolveAspectAndResolution( + size: string | undefined, + model: IGeminiVideoModel, + ): { aspectRatio: string; videoResolution: string | undefined } { + if (size && DIMENSION_MAP[size]) { + return { + aspectRatio: DIMENSION_MAP[size].aspectRatio, + videoResolution: DIMENSION_MAP[size].resolution, + }; + } + + return { + aspectRatio: model.aspectRatios[0], + videoResolution: model.resolutions[0], + }; + } + + #coercePositiveInteger(value: unknown): number | undefined { + if (typeof value === 'number' && Number.isFinite(value)) { + const rounded = Math.round(value); + return rounded > 0 ? rounded : undefined; + } + if (typeof value === 'string') { + const numeric = Number.parseInt(value, 10); + return Number.isFinite(numeric) && numeric > 0 + ? numeric + : undefined; + } + return undefined; + } + + async #delay(ms: number): Promise { + return await new Promise((resolve) => setTimeout(resolve, ms)); + } +} diff --git a/src/backend/src/services/ai/video/providers/GeminiVideoGenerationProvider/models.ts b/src/backend/drivers/ai-video/providers/gemini/models.ts similarity index 81% rename from src/backend/src/services/ai/video/providers/GeminiVideoGenerationProvider/models.ts rename to src/backend/drivers/ai-video/providers/gemini/models.ts index 263ed7ba7..66fbb1e04 100644 --- a/src/backend/src/services/ai/video/providers/GeminiVideoGenerationProvider/models.ts +++ b/src/backend/drivers/ai-video/providers/gemini/models.ts @@ -1,23 +1,4 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { IVideoModel } from '../types.js'; +import { IVideoModel } from '../../types.js'; export interface IGeminiVideoModel extends IVideoModel { aspectRatios: string[]; diff --git a/src/backend/drivers/ai-video/providers/openai/OpenAIVideoProvider.ts b/src/backend/drivers/ai-video/providers/openai/OpenAIVideoProvider.ts new file mode 100644 index 000000000..5cafbbfa7 --- /dev/null +++ b/src/backend/drivers/ai-video/providers/openai/OpenAIVideoProvider.ts @@ -0,0 +1,270 @@ +import OpenAI from 'openai'; +import { Context } from '../../../../core/context.js'; +import { HttpError } from '../../../../core/http/HttpError.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { IGenerateVideoParams, IVideoModel } from '../../types.js'; +import { VideoProvider } from '../VideoProvider.js'; +import { OPENAI_VIDEO_MODELS, OPENAI_VIDEO_ALLOWED_SECONDS } from './models.js'; +import { Readable } from 'stream'; + +const DEFAULT_TEST_VIDEO_URL = 'https://assets.puter.site/txt2vid.mp4'; +const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000; +const POLL_INTERVAL_MS = 5_000; +const DEFAULT_DURATION_SECONDS = 4; + +export class OpenAIVideoProvider extends VideoProvider { + #openai: OpenAI; + #meteringService: MeteringService; + + constructor(config: { apiKey: string }, meteringService: MeteringService) { + super(); + if (!config.apiKey) { + throw new Error('OpenAI video generation requires an API key'); + } + this.#openai = new OpenAI({ apiKey: config.apiKey }); + this.#meteringService = meteringService; + } + + getDefaultModel(): string { + return OPENAI_VIDEO_MODELS[0].id; + } + + async models(): Promise { + return OPENAI_VIDEO_MODELS; + } + + async generate(params: IGenerateVideoParams): Promise { + const { + prompt, + model: requestedModel, + duration, + seconds, + size, + resolution, + input_reference: inputReference, + test_mode: testMode, + } = params ?? {}; + + if (typeof prompt !== 'string' || !prompt.trim()) { + throw new HttpError(400, 'prompt must be a non-empty string'); + } + + const selectedModel = await this.#selectModel(requestedModel); + + if (!selectedModel) { + throw new Error(`Unknown video model: ${requestedModel}`); + } + + if (testMode) { + return DEFAULT_TEST_VIDEO_URL; + } + + const defaultSize = selectedModel.dimensions?.[0] ?? '720x1280'; + const normalizedSize = + this.#normalizeSize(size ?? resolution, selectedModel) ?? + defaultSize; + const normalizedSeconds = + this.#normalizeSeconds(seconds ?? duration) ?? + String(DEFAULT_DURATION_SECONDS); + + const sizeTier = this.#determineSizeTier(selectedModel, normalizedSize); + const costPerSecondCents = this.#getCostPerSecond( + selectedModel, + sizeTier, + ); + + if (!costPerSecondCents) { + throw new Error( + `No pricing configured for model ${selectedModel.id} at size ${normalizedSize}`, + ); + } + + const estimatedUnits = + this.#parseSeconds(normalizedSeconds) ?? DEFAULT_DURATION_SECONDS; + const actor = Context.get('actor'); + const costInMicroCents = costPerSecondCents * 1_000_000; + const usageAllowed = await this.#meteringService.hasEnoughCredits( + actor, + costInMicroCents * estimatedUnits, + ); + if (!usageAllowed) { + throw new HttpError(402, 'Insufficient funds'); + } + + const createParams: OpenAI.VideoCreateParams = { + prompt, + model: selectedModel.id, + seconds: normalizedSeconds as OpenAI.VideoSeconds, + size: normalizedSize as OpenAI.VideoSize, + }; + + if (inputReference) { + createParams.input_reference = + inputReference as OpenAI.VideoCreateParams['input_reference']; + } + + const createResponse = await this.#openai.videos.create(createParams); + const finalJob = await this.#pollUntilComplete(createResponse); + + if (finalJob.status === 'failed') { + const errorMessage = + finalJob.error?.message ?? 'Video generation failed'; + throw new Error(errorMessage); + } + + const finalResolution = + this.#normalizeSize(finalJob.size, selectedModel) ?? normalizedSize; + const finalTier = this.#determineSizeTier( + selectedModel, + finalResolution, + ); + const finalCostPerSecondCents = this.#getCostPerSecond( + selectedModel, + finalTier, + ); + + if (!finalCostPerSecondCents) { + throw new Error( + `No pricing configured for model ${selectedModel.id} at size ${finalResolution}`, + ); + } + + const finalCostInMicroCents = finalCostPerSecondCents * 1_000_000; + const actualSeconds = + this.#parseSeconds(finalJob.seconds) ?? estimatedUnits; + + const downloadResponse = await this.#openai.videos.downloadContent( + finalJob.id, + ); + const contentType = + downloadResponse.headers.get('content-type') ?? 'video/mp4'; + + let stream: any = downloadResponse.body; + if (stream && typeof stream.getReader === 'function') { + stream = Readable.fromWeb(stream as any); + } + + if (!stream) { + const arrayBuffer = await downloadResponse.arrayBuffer(); + stream = Readable.from(Buffer.from(arrayBuffer)); + } + + const finalUsageKey = this.#getUsageKey(selectedModel, finalTier); + await this.#meteringService.incrementUsage( + actor, + finalUsageKey, + actualSeconds, + finalCostInMicroCents * actualSeconds, + ); + + return { + stream, + content_type: contentType, + }; + } + + async #selectModel( + requestedModel?: string, + ): Promise { + const allModels = await this.models(); + return allModels.find( + (m) => m.id.toLowerCase() === requestedModel?.toLowerCase(), + ); + } + + async #pollUntilComplete(initialJob: OpenAI.Video): Promise { + let job = initialJob; + const start = Date.now(); + + while (job.status === 'queued' || job.status === 'in_progress') { + if (Date.now() - start > DEFAULT_TIMEOUT_MS) { + throw new Error( + 'Timed out waiting for Sora video generation to complete', + ); + } + + await this.#delay(POLL_INTERVAL_MS); + job = await this.#openai.videos.retrieve(job.id); + } + + return job; + } + + async #delay(ms: number): Promise { + return await new Promise((resolve) => setTimeout(resolve, ms)); + } + + #normalizeSize(candidate: unknown, model: IVideoModel): string | undefined { + if (!candidate) return undefined; + const normalized = this.#normalizeResolution(candidate); + if (normalized && model.dimensions?.includes(normalized)) { + return normalized; + } + return undefined; + } + + #normalizeSeconds(value: unknown): string | undefined { + if (value === null || value === undefined) { + return undefined; + } + const parsed = + typeof value === 'number' + ? String(Math.round(value)) + : typeof value === 'string' + ? value.trim() + : undefined; + if ( + parsed && + OPENAI_VIDEO_ALLOWED_SECONDS.includes( + Number(parsed) as (typeof OPENAI_VIDEO_ALLOWED_SECONDS)[number], + ) + ) { + return parsed; + } + return undefined; + } + + #determineSizeTier(model: IVideoModel, size: string): string { + if (model.id === 'sora-2-pro') { + if (size === '1080x1920' || size === '1920x1080') return 'xxl'; + if (size === '1024x1792' || size === '1792x1024') return 'xl'; + } + return 'default'; + } + + #getCostPerSecond(model: IVideoModel, tier: string): number | undefined { + const key = tier === 'default' ? 'per-second' : `per-second-${tier}`; + return model.costs?.[key]; + } + + #getUsageKey(model: IVideoModel, tier: string): string { + return `openai:${model.id}:${tier}`; + } + + #normalizeResolution(value: unknown): string | undefined { + if (!value) return undefined; + if (typeof value === 'string') { + const match = value.match(/(\d+)\s*x\s*(\d+)/i); + if (match) { + const w = Number.parseInt(match[1], 10); + const h = Number.parseInt(match[2], 10); + if (Number.isFinite(w) && Number.isFinite(h)) { + return `${w}x${h}`; + } + } + } + return undefined; + } + + #parseSeconds(value: unknown): number | undefined { + if (value === null || value === undefined) return undefined; + if (typeof value === 'number' && Number.isFinite(value)) { + return Math.round(value); + } + if (typeof value === 'string') { + const numeric = Number.parseInt(value, 10); + return Number.isFinite(numeric) ? numeric : undefined; + } + return undefined; + } +} diff --git a/src/backend/src/services/ai/video/providers/OpenAIVideoGenerationProvider/models.ts b/src/backend/drivers/ai-video/providers/openai/models.ts similarity index 58% rename from src/backend/src/services/ai/video/providers/OpenAIVideoGenerationProvider/models.ts rename to src/backend/drivers/ai-video/providers/openai/models.ts index 061cd341d..6867c9a76 100644 --- a/src/backend/src/services/ai/video/providers/OpenAIVideoGenerationProvider/models.ts +++ b/src/backend/drivers/ai-video/providers/openai/models.ts @@ -1,23 +1,4 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { IVideoModel } from '../types.js'; +import { IVideoModel } from '../../types.js'; export const OPENAI_VIDEO_ALLOWED_SECONDS = [4, 8, 12] as const; @@ -53,7 +34,14 @@ export const OPENAI_VIDEO_MODELS: IVideoModel[] = [ }, output_cost_key: 'default-duration-per-video', durationSeconds: OPENAI_VIDEO_ALLOWED_SECONDS.slice(), - dimensions: ['720x1280', '1280x720', '1024x1792', '1792x1024', '1080x1920', '1920x1080'], + dimensions: [ + '720x1280', + '1280x720', + '1024x1792', + '1792x1024', + '1080x1920', + '1920x1080', + ], defaultUsageKey: 'openai:sora-2-pro:default', }, ]; diff --git a/src/backend/drivers/ai-video/providers/together/TogetherVideoProvider.ts b/src/backend/drivers/ai-video/providers/together/TogetherVideoProvider.ts new file mode 100644 index 000000000..45ef40a40 --- /dev/null +++ b/src/backend/drivers/ai-video/providers/together/TogetherVideoProvider.ts @@ -0,0 +1,259 @@ +import { Together } from 'together-ai'; +import { Context } from '../../../../core/context.js'; +import { HttpError } from '../../../../core/http/HttpError.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { IGenerateVideoParams, IVideoModel } from '../../types.js'; +import { VideoProvider } from '../VideoProvider.js'; +import { TOGETHER_VIDEO_GENERATION_MODELS } from './models.js'; + +const DEFAULT_TEST_VIDEO_URL = 'https://assets.puter.site/txt2vid.mp4'; +const POLL_INTERVAL_MS = 5_000; +const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000; +const DEFAULT_MODEL = 'minimax/video-01-director'; +const DEFAULT_DURATION_SECONDS = 6; + +export class TogetherVideoProvider extends VideoProvider { + #client: Together; + #meteringService: MeteringService; + + constructor(config: { apiKey: string }, meteringService: MeteringService) { + super(); + if (!config.apiKey) { + throw new Error('Together AI video generation requires an API key'); + } + this.#client = new Together({ apiKey: config.apiKey }); + this.#meteringService = meteringService; + } + + getDefaultModel(): string { + return 'togetherai:minimax/video-01-director'; + } + + async models(): Promise { + return TOGETHER_VIDEO_GENERATION_MODELS.map((model) => ({ + ...model, + aliases: [model.model], + durationSeconds: model.durationSeconds ?? undefined, + dimensions: model.dimensions ?? undefined, + fps: model.fps ?? undefined, + keyframes: model.keyframes ?? undefined, + promptLength: model.promptLength ?? undefined, + promptSupported: model.promptSupported ?? undefined, + })); + } + + async generate(params: IGenerateVideoParams): Promise { + const { + prompt, + model: requestedModel, + seconds, + no_extra_params, + duration, + width, + height, + fps, + steps, + guidance_scale: guidanceScale, + seed, + output_format: outputFormat, + output_quality: outputQuality, + negative_prompt: negativePrompt, + reference_images: referenceImages, + frame_images: frameImages, + metadata, + test_mode: testMode, + } = params ?? {}; + + if (typeof prompt !== 'string' || !prompt.trim()) { + throw new HttpError(400, 'prompt must be a non-empty string'); + } + + const selectedModel = await this.#getModel(requestedModel); + const model = + selectedModel?.model ?? + this.#stripTogetherPrefix(requestedModel ?? DEFAULT_MODEL); + + if (testMode) { + return DEFAULT_TEST_VIDEO_URL; + } + + const costPerVideoCents = selectedModel?.costs?.['per-video']; + if (!costPerVideoCents) { + throw new Error(`No pricing configured for video model ${model}`); + } + const costInMicroCents = costPerVideoCents * 1_000_000; + + let normalizedSeconds = this.#coercePositiveInteger( + seconds ?? duration, + ); + + if (!no_extra_params) { + normalizedSeconds ??= DEFAULT_DURATION_SECONDS; + } + + const actor = Context.get('actor'); + if (!actor) { + throw new HttpError(401, 'Authentication required'); + } + + const usageAllowed = await this.#meteringService.hasEnoughCredits( + actor, + costInMicroCents, + ); + if (!usageAllowed) { + throw new HttpError(402, 'Insufficient funds'); + } + + const createPayload: Together.VideoCreateParams & { + metadata?: object; + } = { + prompt, + model, + }; + + if (normalizedSeconds) { + createPayload.seconds = String(normalizedSeconds); + } + if (this.#isFiniteNumber(width)) { + createPayload.width = Number(width); + } + if (this.#isFiniteNumber(height)) { + createPayload.height = Number(height); + } + if (this.#isFiniteNumber(fps)) { + createPayload.fps = Number(fps); + } + if (this.#isFiniteNumber(steps)) { + createPayload.steps = Number(steps); + } + if (this.#isFiniteNumber(guidanceScale)) { + createPayload.guidance_scale = Number(guidanceScale); + } + if (this.#isFiniteNumber(seed)) { + createPayload.seed = Number(seed); + } + if (typeof outputFormat === 'string' && outputFormat.trim()) { + createPayload.output_format = + outputFormat.trim() as Together.VideoCreateParams['output_format']; + } + if (this.#isFiniteNumber(outputQuality)) { + createPayload.output_quality = Number(outputQuality); + } + if (typeof negativePrompt === 'string' && negativePrompt.trim()) { + createPayload.negative_prompt = negativePrompt; + } + if (Array.isArray(referenceImages) && referenceImages.length > 0) { + createPayload.reference_images = referenceImages.filter( + (item: string) => + typeof item === 'string' && item.trim().length > 0, + ); + } + if (Array.isArray(frameImages) && frameImages.length > 0) { + createPayload.frame_images = frameImages.filter( + (frame: any) => + frame && + typeof frame === 'object' && + typeof frame.input_image === 'string', + ) as Together.VideoCreateParams['frame_images']; + } + if (metadata && typeof metadata === 'object') { + createPayload.metadata = metadata; + } + + const job = await this.#client.videos.create(createPayload); + const finalJob = await this.#pollUntilComplete(job.id); + + if (finalJob.status === 'failed') { + const errorMessage = + finalJob?.info?.errors?.[0]?.message ?? + finalJob?.info?.errors?.message ?? + finalJob?.info?.errors ?? + 'Video generation failed'; + throw new Error(errorMessage); + } + + if (finalJob.status === 'cancelled') { + throw new Error('Video generation was cancelled'); + } + + const usageKey = `together-video:${model}`; + await this.#meteringService.incrementUsage( + actor, + usageKey, + 1, + costInMicroCents, + ); + + const videoUrl = finalJob?.outputs?.video_url; + if (typeof videoUrl === 'string' && videoUrl.trim()) { + return videoUrl; + } + + throw new Error('Together AI response did not include a video URL'); + } + + async #pollUntilComplete(jobId: string): Promise { + // any here because sdk types are wrong https://docs.together.ai/docs/videos-overview -> "Job Status Reference" + let job = await (this.#client as any).videos.retrieve(jobId); + const start = Date.now(); + + while (job.status === 'queued' || job.status === 'in_progress') { + if (Date.now() - start > DEFAULT_TIMEOUT_MS) { + throw new Error( + 'Timed out waiting for Together AI video generation to complete', + ); + } + + await this.#delay(POLL_INTERVAL_MS); + job = await (this.#client as any).videos.retrieve(jobId); + } + + return job; + } + + async #delay(ms: number): Promise { + return await new Promise((resolve) => setTimeout(resolve, ms)); + } + + async #getModel(requestedModel?: string): Promise { + const bareModel = this.#stripTogetherPrefix( + requestedModel ?? DEFAULT_MODEL, + ); + const allModels = await this.models(); + return allModels.find( + (m) => m.model?.toLowerCase() === bareModel.toLowerCase(), + ); + } + + #stripTogetherPrefix(model: string): string { + if (typeof model === 'string' && model.startsWith('togetherai:')) { + return model.slice('togetherai:'.length); + } + return model; + } + + #coercePositiveInteger(value: unknown): number | undefined { + if (typeof value === 'number' && Number.isFinite(value)) { + const rounded = Math.round(value); + return rounded > 0 ? rounded : undefined; + } + if (typeof value === 'string') { + const numeric = Number.parseInt(value, 10); + return Number.isFinite(numeric) && numeric > 0 + ? numeric + : undefined; + } + return undefined; + } + + #isFiniteNumber(value: unknown): boolean { + if (typeof value === 'number') { + return Number.isFinite(value); + } + if (typeof value === 'string') { + const numeric = Number(value); + return Number.isFinite(numeric); + } + return false; + } +} diff --git a/src/backend/src/services/ai/video/providers/TogetherVideoGenerationProvider/models.ts b/src/backend/drivers/ai-video/providers/together/models.ts similarity index 94% rename from src/backend/src/services/ai/video/providers/TogetherVideoGenerationProvider/models.ts rename to src/backend/drivers/ai-video/providers/together/models.ts index 913bd1791..c3e309a90 100644 --- a/src/backend/src/services/ai/video/providers/TogetherVideoGenerationProvider/models.ts +++ b/src/backend/drivers/ai-video/providers/together/models.ts @@ -1,23 +1,4 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { IVideoModel } from '../types.js'; +import { IVideoModel } from '../../types.js'; interface ITogetherVideoModel extends IVideoModel { model: string; diff --git a/src/backend/src/services/ai/video/providers/types.ts b/src/backend/drivers/ai-video/types.ts similarity index 58% rename from src/backend/src/services/ai/video/providers/types.ts rename to src/backend/drivers/ai-video/types.ts index c5e3a8ad3..ac017d7dd 100644 --- a/src/backend/src/services/ai/video/providers/types.ts +++ b/src/backend/drivers/ai-video/types.ts @@ -1,20 +1,5 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . +/** + * Types for the `puter-video-generation` driver interface. */ export interface IVideoModel { @@ -67,7 +52,7 @@ export interface IGenerateVideoParams { } export interface IVideoProvider { - generate (params: IGenerateVideoParams): Promise; - models (): Promise | IVideoModel[]; - getDefaultModel (): string; + generate(params: IGenerateVideoParams): Promise; + models(): Promise | IVideoModel[]; + getDefaultModel(): string; } diff --git a/src/backend/drivers/apps/AppDriver.js b/src/backend/drivers/apps/AppDriver.js new file mode 100644 index 000000000..0f2f87486 --- /dev/null +++ b/src/backend/drivers/apps/AppDriver.js @@ -0,0 +1,1159 @@ +import { Context } from '../../core/context.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import { + ICON_DATA_URL_MIME_ALLOWLIST, + isAppIconEndpointUrl, + isRawBase64ImageString, + normalizeRawBase64ImageString, +} from '../../util/appIcon.js'; +import { resolvePrivateLaunchAccess } from '../../util/privateLaunchAccess.js'; +import { + validateArrayOfStrings, + validateBool, + validateJsonObject, + validateString, + validateUrl, +} from '../../util/validation.js'; +import { PuterDriver } from '../types.js'; + +const APP_NAME_REGEX = /^[a-zA-Z0-9_-]+$/; +const APP_NAME_MAX_LEN = 100; +const APP_TITLE_MAX_LEN = 100; +const APP_DESCRIPTION_MAX_LEN = 7000; + +// Index-url uniqueness exemptions: legacy "coming soon" placeholder apps +// that intentionally share the same hosted index_url. Anything starting +// with one of these strings skips the uniqueness check so multiple rows +// can keep that placeholder URL without merging into each other. +const INDEX_URL_UNIQUENESS_EXEMPTION_CANDIDATES = [ + 'https://dev-center.puter.com/coming-soon', +]; + +// Canonical-uid alias namespace. When a user-created app is merged into +// an existing origin-bootstrap row, the source uid is mapped to the +// canonical (kept) uid so any client that still holds the old uid keeps +// resolving to the joined row. TTL keeps abandoned entries from +// accumulating indefinitely. +const APP_UID_ALIAS_KEY_PREFIX = 'app:canonicalUidAlias'; +const APP_UID_ALIAS_REVERSE_KEY_PREFIX = 'app:canonicalUidAliasReverse'; +const APP_UID_ALIAS_TTL_SECONDS = 60 * 60 * 24 * 90; + +const hasIndexUrlUniquenessExemption = (candidates) => { + for (const candidate of candidates) { + if ( + INDEX_URL_UNIQUENESS_EXEMPTION_CANDIDATES.find((exception) => + candidate.startsWith(exception), + ) + ) { + return true; + } + } + return false; +}; + +/** + * Driver exposing the `puter-apps` interface. + * + * Wraps AppStore with input validation + permission checks. + * Methods follow the `crud-q` shape client SDKs expect: + * create, read, select, update, upsert, delete + * + * Permission model: + * - Owner (apps.owner_user_id === actor.user.id) has full access + * - App actor matching app_owner has full access + * - `system:es:write-all-owners` grants blanket write + * - `app:uid#:access` grants protected-app access + */ +export class AppDriver extends PuterDriver { + driverInterface = 'puter-apps'; + // `es:app` is the wire name puter-js sends in `/drivers/call`'s `driver` + // field. Origin/main registered the service under that exact key; keep + // it so existing clients + hardcoded permission keys (`service:es\Capp:…`) + // resolve without a translation layer. + driverName = 'es:app'; + isDefault = true; + + get appStore() { + return this.stores.app; + } + get permService() { + return this.services.permission; + } + + // ── Driver methods ─────────────────────────────────────────────── + + async create({ object, options } = {}) { + if (!object || typeof object !== 'object') { + throw new HttpError(400, 'Missing or invalid `object`'); + } + const actor = this.#requireActor(); + this.#requireUserOrAppActor(actor); + + const fields = await this.#validateInput(object, { isCreate: true }); + + // Puter-hosted index_url handling. Order matches v1 AppES: + // 1. Refuse if the index_url's subdomain isn't owned by this user. + // 2. Try to merge into an existing row with the same index_url + // (origin-bootstrap takeover, or claiming an unowned row). + // 3. Otherwise enforce index_url uniqueness so two rows can't + // share a hosted URL. + await this.#ensurePuterSiteSubdomainIsOwned( + fields.index_url, + actor.user, + ); + const joinedApp = await this.#maybeJoinOwnedHostedIndexUrlApp({ + object, + options, + user: actor.user, + }); + if (joinedApp) { + return joinedApp; + } + await this.#ensureIndexUrlNotAlreadyInUse({ + indexUrl: fields.index_url, + }); + + // Name conflict handling + if (await this.appStore.existsByName(fields.name)) { + if (options?.dedupe_name) { + let candidate; + let n = 1; + do { + candidate = `${fields.name}-${++n}`; + if (n > 50) + throw new HttpError(400, 'Failed to dedupe app name'); + } while (await this.appStore.existsByName(candidate)); + fields.name = candidate; + } else { + throw new HttpError( + 400, + 'An app with this name already exists', + ); + } + } + + const filetypes = fields.filetype_associations; + delete fields.filetype_associations; + + // Ownership is passed as a separate, privileged arg — the store + // filters `owner_user_id` / `app_owner` out of `fields` (both are + // in READ_ONLY_COLUMNS), so the only way to stamp ownership is + // through this explicit contract. Keeps any future caller that + // forwards raw input into `create` from spoofing the owner. + const app = await this.appStore.create(fields, { + ownerUserId: actor.user.id, + appOwner: actor.app?.id ?? null, + }); + if (filetypes) + await this.appStore.setFiletypeAssociations(app.id, filetypes); + + this.#emitAppChanged({ app, action: 'created' }); + + return this.#toClient(app, actor); + } + + async read({ uid, id, params = {}, ...rest } = {}) { + const actor = this.#requireActor(); + const app = await this.#resolve({ uid, id }); + if (!app) throw new HttpError(404, 'App not found'); + + await this.#checkReadAccess(app, actor); + + // puter-js's `puter.apps.get(name, opts)` packages opts under `params` + // (see `make_driver_method` / `Apps.get`), so stats options live at + // `args.params.stats_period` rather than the top level. Accept both + // shapes for forward-compat with anything that still flattens. + const stats_period = params.stats_period ?? rest.stats_period; + const stats_grouping = params.stats_grouping ?? rest.stats_grouping; + + const needsStats = + params.stats !== false && (stats_period || stats_grouping); + + // Detailed period/grouping is per-app only — skip the batch cache + // and go straight to the live query. The default (no options) goes + // through the cached batched path. + const hasDetailed = Boolean(stats_period || stats_grouping); + const stats = !needsStats + ? undefined + : hasDetailed + ? await this.appStore.getAppStatsDetailed(app.uid, { + period: stats_period, + grouping: stats_grouping, + createdAt: app.created_at ?? app.timestamp, + }) + : (await this.appStore.getAppsStats([app.uid])).get(app.uid); + + return this.#toClient(app, actor, { ...params, stats }); + } + + async select({ predicate, params = {} } = {}) { + const actor = this.#requireActor(); + this.#requireUserOrAppActor(actor); + + const filters = {}; + // predicate: ['user-can-edit'] → scope to owner + if (Array.isArray(predicate) && predicate[0] === 'user-can-edit') { + filters.ownerUserId = actor.user.id; + } + + const apps = await this.appStore.list(filters); + + // Resolve protected-app visibility: + // 1. Cheap local short-circuits (non-protected, self-app, owner). + // 2. Single batched permission check for whatever's left — one + // scan pass covers every remaining app, vs a per-app round + // trip through the permission service. + const needsPermCheck = []; + const localVisible = new Set(); + for (const app of apps) { + if ( + !app.protected || + actor.app?.uid === app.uid || + actor.user?.id === app.owner_user_id + ) { + localVisible.add(app); + } else { + needsPermCheck.push(app); + } + } + + let permGrants; + if (needsPermCheck.length > 0) { + try { + permGrants = await this.permService.checkMany( + actor, + needsPermCheck.map((a) => `app:uid#${a.uid}:access`), + ); + } catch { + permGrants = new Map(); + } + } else { + permGrants = new Map(); + } + + const visible = apps.filter( + (app) => + localVisible.has(app) || + permGrants.get(`app:uid#${app.uid}:access`), + ); + + // Pre-fetch in parallel: + // - per-uid stats (already pipelined inside getAppsStats) + // - filetype associations as a single IN-list query (was N queries) + const [statsByUid, filetypesByAppId] = await Promise.all([ + this.appStore.getAppsStats(visible.map((a) => a.uid)), + this.appStore.getFiletypeAssociationsByIds( + visible.map((a) => a.id), + ), + ]); + + return Promise.all( + visible.map((app) => + this.#toClient(app, actor, { + ...params, + stats: statsByUid.get(app.uid), + filetypes: filetypesByAppId.get(app.id) ?? [], + }), + ), + ); + } + + async update({ uid, id, object } = {}) { + if (!object || typeof object !== 'object') { + throw new HttpError(400, 'Missing or invalid `object`'); + } + const actor = this.#requireActor(); + this.#requireUserOrAppActor(actor); + + const app = await this.#resolve({ uid, id }); + if (!app) throw new HttpError(404, 'App not found'); + + await this.#checkWriteAccess(app, actor); + + const fields = await this.#validateInput(object, { + isCreate: false, + existing: app, + }); + + // Puter-hosted index_url handling on update — same flow as create + // but only when the index_url is actually changing. Self-app is + // excluded from the conflict search via `excludeAppId`. + if (fields.index_url && fields.index_url !== app.index_url) { + await this.#ensurePuterSiteSubdomainIsOwned( + fields.index_url, + actor.user, + ); + const joinedApp = await this.#maybeJoinOwnedHostedIndexUrlApp({ + object, + options: undefined, + user: actor.user, + sourceAppUid: app.uid, + excludeAppId: app.id, + }); + if (joinedApp) { + return joinedApp; + } + await this.#ensureIndexUrlNotAlreadyInUse({ + indexUrl: fields.index_url, + excludeAppId: app.id, + }); + } + + // Name conflict check (only if name is changing) + if (fields.name && fields.name !== app.name) { + if (await this.appStore.existsByName(fields.name)) { + throw new HttpError( + 400, + 'An app with this name already exists', + ); + } + } + + const filetypes = fields.filetype_associations; + delete fields.filetype_associations; + + const updated = await this.appStore.update(app.id, fields); + if (filetypes !== undefined) { + await this.appStore.setFiletypeAssociations(app.id, filetypes); + } + + this.#emitAppChanged({ app: updated, old_app: app, action: 'updated' }); + if (fields.name && fields.name !== app.name) { + this.#emitAppRename({ + app: updated, + old_name: app.name, + new_name: fields.name, + }); + } + + return this.#toClient(updated, actor); + } + + async upsert({ uid, id, object, options } = {}) { + const existing = uid || id ? await this.#resolve({ uid, id }) : null; + if (existing) return this.update({ uid: existing.uid, object }); + return this.create({ object, options }); + } + + async delete({ uid, id } = {}) { + const actor = this.#requireActor(); + this.#requireUserOrAppActor(actor); + + const app = await this.#resolve({ uid, id }); + if (!app) throw new HttpError(404, 'App not found'); + + if (app.protected) { + throw new HttpError(403, 'Cannot delete a protected app'); + } + + await this.#checkWriteAccess(app, actor); + await this.appStore.delete(app.id); + + this.#emitAppChanged({ app: null, old_app: app, action: 'deleted' }); + + return { success: true, uid: app.uid }; + } + + // ── Event emission ─────────────────────────────────────────────── + // + // Consumers (AppIconService, future cf-file-cache port, billing + // event handlers) key off `app_uid`; the full `app` / `old_app` + // payload lets cache invalidators compute exact origins. + + #emitAppChanged({ app, old_app, action }) { + const app_uid = app?.uid ?? old_app?.uid; + if (!app_uid) return; + try { + this.clients.event.emit( + 'app.changed', + { app_uid, app, old_app, action }, + {}, + ); + } catch { + // Non-critical. + } + } + + #emitAppRename({ app, old_name, new_name }) { + try { + this.clients.event.emit( + 'app.rename', + { + app_uid: app.uid, + old_name, + new_name, + app, + }, + {}, + ); + } catch { + // Non-critical. + } + } + + // ── Public helpers (used by AppController) ────────────────────── + + /** Check if an app name is available. Mirrors the REST endpoint behaviour. */ + async isNameAvailable(name) { + validateString(name, { + key: 'name', + maxLen: APP_NAME_MAX_LEN, + regex: APP_NAME_REGEX, + }); + return !(await this.appStore.existsByName(name)); + } + + // ── Validation ─────────────────────────────────────────────────── + + async #validateInput(object, { isCreate }) { + const out = {}; + + if (isCreate || object.name !== undefined) { + out.name = validateString(object.name, { + key: 'name', + maxLen: APP_NAME_MAX_LEN, + regex: APP_NAME_REGEX, + required: isCreate, + }); + } + if (isCreate || object.title !== undefined) { + out.title = validateString(object.title, { + key: 'title', + maxLen: APP_TITLE_MAX_LEN, + required: isCreate, + }); + } + if (object.description !== undefined) { + out.description = validateString(object.description, { + key: 'description', + maxLen: APP_DESCRIPTION_MAX_LEN, + required: false, + allowEmpty: true, + }); + } + if (isCreate || object.index_url !== undefined) { + out.index_url = validateUrl(object.index_url, { + key: 'index_url', + maxLen: 3000, + required: isCreate, + }); + } + if (object.icon !== undefined) { + validateString(object.icon, { + key: 'icon', + maxLen: 5 * 1024 * 1024, + required: false, + allowEmpty: true, + }); + let iconStr = object.icon; + // Accepted shapes (mirrors v1's `image-base64` proptype so + // puter-js callers keep working): + // 1. Empty string — unset + // 2. Raw base64 (no prefix) — normalized to a PNG data URL + // 3. `data:image/;…` with an allow-listed MIME + // 4. `/app-icon/` endpoint URL (relative, or absolute + // on a host we control) + // Anything else (including arbitrary http(s) URLs) is rejected: + // the unauthenticated GET /app-icon/:uid would otherwise 302 + // there and turn this endpoint into a Puter-branded open + // redirector (cached publicly for 15 min). + if (iconStr && iconStr.length > 0) { + // Raw base64 → wrap as data URL (v1 parity) + if (isRawBase64ImageString(iconStr)) { + iconStr = normalizeRawBase64ImageString(iconStr); + } else if (iconStr.startsWith('data:')) { + const semi = iconStr.indexOf(';'); + const comma = iconStr.indexOf(','); + const mimeEnd = + semi !== -1 && (comma === -1 || semi < comma) + ? semi + : comma; + const mime = + mimeEnd !== -1 + ? iconStr.slice(5, mimeEnd).toLowerCase() + : ''; + if (!ICON_DATA_URL_MIME_ALLOWLIST.includes(mime)) { + throw new HttpError( + 400, + '`icon` data URL must use an image MIME type', + ); + } + } else if (!isAppIconEndpointUrl(iconStr, this.config)) { + throw new HttpError( + 400, + '`icon` must be base64, a data:image/… URL, or an app-icon endpoint URL', + ); + } + } + out.icon = iconStr; + } + if (object.maximize_on_start !== undefined) { + out.maximize_on_start = validateBool(object.maximize_on_start, { + key: 'maximize_on_start', + }) + ? 1 + : 0; + } + if (object.background !== undefined) { + out.background = validateBool(object.background, { + key: 'background', + }) + ? 1 + : 0; + } + if (object.metadata !== undefined) { + const meta = validateJsonObject(object.metadata, { + key: 'metadata', + }); + out.metadata = JSON.stringify(meta); + } + if (object.filetype_associations) { + out.filetype_associations = validateArrayOfStrings( + object.filetype_associations, + { + key: 'filetype_associations', + }, + ); + } + + return out; + } + + // ── Permission checks ──────────────────────────────────────────── + + #requireActor() { + const actor = Context.get('actor'); + if (!actor) throw new HttpError(401, 'Authentication required'); + return actor; + } + + #requireUserOrAppActor(actor) { + if (!actor.user) throw new HttpError(403, 'User actor required'); + } + + async #resolve({ uid, id }) { + if (uid) return this.#getByUidWithAlias(uid); + if (id?.uid) return this.#getByUidWithAlias(id.uid); + if (id?.name) return this.appStore.getByName(id.name); + if (id?.id) return this.appStore.getById(id.id); + if (typeof id === 'number') return this.appStore.getById(id); + if (typeof id === 'string') return this.#getByUidWithAlias(id); + return null; + } + + /** + * uid lookup with canonical-uid alias fallback. When two app rows + * have been merged (see {@link #maybeJoinOwnedHostedIndexUrlApp}), + * the source uid is recorded as an alias to the canonical uid. A + * direct uid miss therefore re-queries with the canonical uid so + * any client still holding the old uid keeps resolving to the + * joined row. Mirrors v1 AppES's `#read` alias plumbing. + * + * The alias query is fired in parallel with the direct lookup so + * the common (no-alias) case pays only one round-trip. + */ + async #getByUidWithAlias(uid) { + const aliasPromise = this.#readCanonicalAppUidAlias(uid); + const direct = await this.appStore.getByUid(uid); + if (direct) return direct; + const canonicalUid = await aliasPromise; + if ( + typeof canonicalUid === 'string' && + canonicalUid && + canonicalUid !== uid + ) { + return this.appStore.getByUid(canonicalUid); + } + return null; + } + + async #canReadApp(app, actor) { + if (!app.protected) return true; + // Self-app access + if (actor.app?.uid === app.uid) return true; + // Owner access + if (actor.user?.id === app.owner_user_id) return true; + // Permission check + try { + return await this.permService.check( + actor, + `app:uid#${app.uid}:access`, + ); + } catch { + return false; + } + } + + async #checkReadAccess(app, actor) { + if (await this.#canReadApp(app, actor)) return; + throw new HttpError(403, 'Access denied'); + } + + async #checkWriteAccess(app, actor) { + // App actor matching app_owner + let hasAccess = false; + if (!actor.app?.id) { + hasAccess = actor.user?.id === app.owner_user_id; + } else if (actor.app.id === app.app_owner) { + hasAccess = actor.user?.id === app.owner_user_id; + } + // System-wide write + if (!hasAccess) { + hasAccess = await this.permService.check( + actor, + 'system:es:write-all-owners', + ); + } + if (!hasAccess) { + throw new HttpError(403, 'Access denied'); + } + } + + // ── Serialization ──────────────────────────────────────────────── + + /** + * Resolve the canonical app row that backs `app.index_url`. + * + * Returns `{ origin, expectedUid, canonicalApp }`: + * - `origin` — the parsed origin string from `index_url`. + * - `expectedUid` — the canonical app uid for that origin (oldest + * `apps.index_url` match, or a deterministic UUIDv5 fallback for + * unknown origins). + * - `canonicalApp` — the actual `apps` row at `expectedUid`, or + * `null` when the uid is a UUIDv5 fallback with no DB row. + * + * Used in `#toClient` for two things: + * 1. `created_from_origin` derivation (only set when + * `expectedUid === app.uid`, mirroring v1 AppES). + * 2. The canonical-private gate — when `expectedUid !== app.uid` + * and `canonicalApp.is_private`, the row is squatting on + * someone else's private hosted URL. We must run the + * privateAccess gate against the *canonical* row, not the + * possibly-public squatter row, otherwise pre-existing data + * from before the `subdomain_not_owned` check leaks the + * victim's index_url. + * + * Returns `null` when there's no `index_url` or it doesn't parse. + */ + async #resolveCanonicalForIndexUrl(app) { + if (!app.index_url) return null; + let origin; + try { + const parsed = new URL(app.index_url); + origin = `${parsed.protocol}//${parsed.hostname}${ + parsed.port ? `:${parsed.port}` : '' + }`; + } catch { + return null; + } + try { + const expectedUid = + await this.services.auth.appUidFromOrigin(origin); + // Avoid a needless DB hit on the self-match common case — + // `app` is already the row we'd be re-fetching. + const canonicalApp = + expectedUid && expectedUid !== app.uid + ? await this.appStore.getByUid(expectedUid) + : app; + return { origin, expectedUid, canonicalApp }; + } catch { + return null; + } + } + + async #toClient(app, actor, params = {}) { + if (!app) return null; + + // `select` pre-fetches filetypes for every visible app in one + // batched query and threads them through `params.filetypes` to + // avoid the N+1 in this hot loop. Single-app callers (`read`, + // `create`, `update`) fall back to the per-app query. + const [filetypes, canonicalForIndexUrl] = await Promise.all([ + params.filetypes !== undefined + ? Promise.resolve(params.filetypes) + : this.appStore.getFiletypeAssociations(app.id), + this.#resolveCanonicalForIndexUrl(app), + ]); + + const createdFromOrigin = + canonicalForIndexUrl && canonicalForIndexUrl.expectedUid === app.uid + ? canonicalForIndexUrl.origin + : null; + + const result = { + uid: app.uid, + name: app.name, + title: app.title, + description: app.description, + icon: app.icon, + index_url: app.index_url, + background: Boolean(app.background), + maximize_on_start: Boolean(app.maximize_on_start), + godmode: Boolean(app.godmode), + is_private: Boolean(app.is_private), + protected: Boolean(app.protected), + approved_for_listing: Boolean(app.approved_for_listing), + approved_for_opening_items: Boolean(app.approved_for_opening_items), + approved_for_incentive_program: Boolean( + app.approved_for_incentive_program, + ), + metadata: app.metadata ?? null, + filetype_associations: filetypes, + created_at: app.created_at ?? app.timestamp, + created_from_origin: createdFromOrigin, + stats: params.stats ?? null, + }; + + // Owner info — only expose if actor is the owner or has access + if (actor?.user?.id === app.owner_user_id) { + result.owner = { + username: actor.user.username, + uuid: actor.user.uuid, + }; + } + + // Icon sizing hook (for future AppIconService integration) + if (params.icon_size) { + result.icon_size = params.icon_size; + } + + // Private-app gate: callers without an ownership / purchase / grant + // must not receive `index_url` (the direct hosting URL). They still + // see metadata (title, icon, description) so the marketplace UI can + // render a purchase CTA. Owners + entitled users pass through + // unchanged. Attach `privateAccess` so clients know to redirect to + // app-center rather than launch. + // + // Gate target picking: + // 1. Canonical mismatch + canonical is private → gate against + // the *canonical* row. Catches pre-existing bug data where + // a row's `index_url` points at someone else's private hosted + // URL but the row itself has `is_private = 0`. The + // authoritative privacy decision belongs to the canonical + // row's owner, not the squatter. + // 2. Otherwise, if this row is itself private → gate against + // this row (the legitimate path). + // 3. Otherwise no gate — public app, no entitlement check. + const canonicalApp = canonicalForIndexUrl?.canonicalApp ?? null; + const expectedUid = canonicalForIndexUrl?.expectedUid; + const canonicalMismatchPrivate = + !!expectedUid && + expectedUid !== app.uid && + !!canonicalApp?.is_private; + const gateTarget = canonicalMismatchPrivate + ? canonicalApp + : result.is_private + ? app + : null; + if (gateTarget) { + const isOwner = + actor?.user?.id !== undefined && + actor.user.id === gateTarget.owner_user_id; + const privateAccess = isOwner + ? { hasAccess: true, checkedBy: 'core/app-owner' } + : await resolvePrivateLaunchAccess({ + app: { + uid: gateTarget.uid, + name: gateTarget.name, + is_private: true, + }, + eventClient: this.clients.event, + userUid: actor?.user?.uuid ?? null, + source: canonicalMismatchPrivate + ? 'appDriver:toClient:canonical-private' + : 'appDriver:toClient', + args: {}, + }); + result.privateAccess = privateAccess; + if (!privateAccess.hasAccess) { + delete result.index_url; + } + } + + return result; + } + + // ── Puter-hosted index_url merge logic ─────────────────────────── + // + // Ported from v1 AppES (`#maybeJoinOwnedHostedIndexUrlAppOnCreate`, + // `#ensure_puter_site_subdomain_is_owned`, `#ensureIndexUrlNotAlreadyInUse`, + // and the `app:canonicalUidAlias:*` kvstore pair). When a user creates + // or repoints an app at a puter-hosted subdomain (`*.puter.site`, + // `*.puter.app`, …) we want exactly one app row to back that URL: + // • If a no-owner row exists (origin-bootstrap stub auto-created + // when an unknown origin first hit Puter) → claim it for the + // user, merge the new fields into it. + // • If the same user already has an origin-bootstrap row at that + // URL → merge into it; otherwise reject as a duplicate. + // • If a different user owns the row → reject with + // `app_index_url_already_in_use`. + // Source uid (when called from `update`) is recorded in a kvstore + // alias so `#resolve` can redirect old uids to the canonical row. + + #normalizeConfiguredHostedDomain(domainValue) { + if (typeof domainValue !== 'string') return null; + const normalizedDomain = domainValue + .trim() + .toLowerCase() + .replace(/^\./, ''); + if (!normalizedDomain) return null; + return normalizedDomain.split(':')[0] || null; + } + + #getPuterHostedDomains() { + const domains = new Set(); + const config = this.config ?? {}; + for (const configuredDomain of [ + config.static_hosting_domain, + config.static_hosting_domain_alt, + config.private_app_hosting_domain, + config.private_app_hosting_domain_alt, + ]) { + const normalized = + this.#normalizeConfiguredHostedDomain(configuredDomain); + if (normalized) domains.add(normalized); + } + return [...domains]; + } + + #extractPuterHostedSubdomain(indexUrl) { + if (typeof indexUrl !== 'string' || !indexUrl) return null; + + let hostname; + try { + hostname = new URL(indexUrl).hostname.toLowerCase(); + } catch { + return null; + } + + // Sort longest-first so `foo.puter.app` matches `puter.app` (not + // a shorter `app` if it ever appeared in the configured list). + const hostedDomains = this.#getPuterHostedDomains().sort( + (a, b) => b.length - a.length, + ); + + for (const hostedDomain of hostedDomains) { + const suffix = `.${hostedDomain}`; + if (hostname.endsWith(suffix)) { + const subdomain = hostname.slice( + 0, + hostname.length - suffix.length, + ); + return subdomain || null; + } + } + + return null; + } + + #isPuterHostedIndexUrl(indexUrl) { + return !!this.#extractPuterHostedSubdomain(indexUrl); + } + + /** + * Generate the set of equivalent index_url strings that should + * collide with a given input. We only collapse trailing-slash and + * `/index.html` variants — the underlying `apps.index_url` column + * is matched by exact string, so anything not in this list won't + * be deduped. Mirrors v1 AppES exactly. + */ + #buildEquivalentIndexUrlCandidates(indexUrl) { + if (typeof indexUrl !== 'string' || !indexUrl.trim()) { + return []; + } + + try { + const parsed = new URL(indexUrl); + const origin = `${parsed.protocol}//${parsed.host.toLowerCase()}`; + const pathname = parsed.pathname || '/'; + + const candidates = new Set(); + if (pathname === '/' || pathname.toLowerCase() === '/index.html') { + candidates.add(origin); + candidates.add(`${origin}/`); + candidates.add(`${origin}/index.html`); + } else { + const normalizedPath = pathname.endsWith('/') + ? pathname.slice(0, -1) + : pathname; + candidates.add(`${origin}${normalizedPath}`); + candidates.add(`${origin}${normalizedPath}/`); + } + + return [...candidates]; + } catch { + return [indexUrl.trim()]; + } + } + + async #findIndexUrlConflictRow({ indexUrl, excludeAppId } = {}) { + if (!this.#isPuterHostedIndexUrl(indexUrl)) return null; + + const candidates = this.#buildEquivalentIndexUrlCandidates(indexUrl); + if (candidates.length === 0) return null; + if (hasIndexUrlUniquenessExemption(candidates)) return null; + + return this.appStore.findByIndexUrlCandidates(candidates, { + excludeAppId, + }); + } + + async #ensureIndexUrlNotAlreadyInUse({ indexUrl, excludeAppId } = {}) { + const conflictRow = await this.#findIndexUrlConflictRow({ + indexUrl, + excludeAppId, + }); + if (conflictRow) { + throw new HttpError(400, 'App index_url already in use', { + legacyCode: 'app_index_url_already_in_use', + fields: { + index_url: indexUrl, + app_uid: conflictRow.uid, + }, + }); + } + } + + async #ensurePuterSiteSubdomainIsOwned(indexUrl, user) { + if (!user) return; + const subdomain = this.#extractPuterHostedSubdomain(indexUrl); + if (!subdomain) return; + + const row = await this.stores.subdomain.getBySubdomain(subdomain); + if (!row || row.user_id !== user.id) { + throw new HttpError(400, 'Subdomain not owned by user', { + legacyCode: 'subdomain_not_owned', + fields: { subdomain }, + }); + } + } + + /** + * Origin-bootstrap detection: rows auto-created when an unknown + * origin first needed an app row (no human-supplied metadata). + * Marker is `name === uid && title === uid` and a description + * starting with "App created from origin ". Only these rows are + * eligible for same-owner merging — refusing to merge arbitrary + * same-owner apps prevents accidental data loss. + */ + #isOriginBootstrapApp(app) { + if (!app || typeof app !== 'object') return false; + if (typeof app.uid !== 'string' || !app.uid) return false; + if (app.name !== app.uid) return false; + if (app.title !== app.uid) return false; + if (typeof app.description !== 'string') return false; + return app.description.startsWith('App created from origin '); + } + + // ── Canonical-uid alias kvstore pair ───────────────────────────── + // + // After a merge, the source app uid → canonical uid mapping is + // kept in `stores.kv` (system namespace) so any client that still + // holds the old uid keeps resolving to the joined row via + // `#getByUidWithAlias`. Reverse map lets callers enumerate aliases + // for a canonical uid (matches v1 plumbing). + + #buildCanonicalAppUidAliasKey(oldAppUid) { + return `${APP_UID_ALIAS_KEY_PREFIX}:${oldAppUid}`; + } + + #buildCanonicalAppUidAliasReverseKey(canonicalAppUid) { + return `${APP_UID_ALIAS_REVERSE_KEY_PREFIX}:${canonicalAppUid}`; + } + + #normalizeCanonicalAliasUidList(value) { + if (!Array.isArray(value)) return []; + const out = []; + const seen = new Set(); + for (const item of value) { + if (typeof item !== 'string' || !item) continue; + if (seen.has(item)) continue; + seen.add(item); + out.push(item); + } + return out; + } + + async #readCanonicalAppUidAlias(oldAppUid) { + if (typeof oldAppUid !== 'string' || !oldAppUid) return null; + const key = this.#buildCanonicalAppUidAliasKey(oldAppUid); + try { + const { res } = await this.stores.kv.get({ key }); + if (typeof res === 'string' && res) return res; + } catch { + // Alias reads are best-effort. + } + return null; + } + + async #writeCanonicalAppUidAlias({ oldAppUid, canonicalAppUid }) { + if (typeof oldAppUid !== 'string' || !oldAppUid) return; + if (typeof canonicalAppUid !== 'string' || !canonicalAppUid) return; + if (oldAppUid === canonicalAppUid) return; + + const key = this.#buildCanonicalAppUidAliasKey(oldAppUid); + const reverseKey = + this.#buildCanonicalAppUidAliasReverseKey(canonicalAppUid); + const expireAt = + Math.floor(Date.now() / 1000) + APP_UID_ALIAS_TTL_SECONDS; + try { + const { res: reverseValue } = await this.stores.kv.get({ + key: reverseKey, + }); + const reverseAliases = + this.#normalizeCanonicalAliasUidList(reverseValue); + if (!reverseAliases.includes(oldAppUid)) { + reverseAliases.push(oldAppUid); + } + + await this.stores.kv.set({ + key, + value: canonicalAppUid, + expireAt, + }); + await this.stores.kv.set({ + key: reverseKey, + value: reverseAliases, + expireAt, + }); + } catch { + // Alias writes are best-effort. + } + } + + /** + * Merge an incoming create/update into an existing app row that + * already owns the same puter-hosted index_url. Returns the joined + * (client-shaped) app on success, or `null` when no merge applied. + * Throws `app_index_url_already_in_use` when a conflict exists but + * cannot be merged (different owner, or same-owner non-bootstrap). + * + * `sourceAppUid` is set when called from update — when present and + * different from the conflict row's uid, the source row is deleted + * and an alias is recorded so old-uid clients keep resolving. + */ + async #maybeJoinOwnedHostedIndexUrlApp({ + object, + options, + user, + sourceAppUid, + excludeAppId, + } = {}) { + const indexUrl = object?.index_url; + if (!this.#isPuterHostedIndexUrl(indexUrl)) return null; + + const conflictRow = await this.#findIndexUrlConflictRow({ + indexUrl, + excludeAppId, + }); + if (!conflictRow) return null; + + const conflictOwnerUserId = Number(conflictRow.owner_user_id); + if ( + Number.isInteger(conflictOwnerUserId) && + conflictOwnerUserId > 0 && + conflictOwnerUserId !== user.id + ) { + throw new HttpError(400, 'App index_url already in use', { + legacyCode: 'app_index_url_already_in_use', + fields: { + index_url: indexUrl, + app_uid: conflictRow.uid, + }, + }); + } + + // Unowned (origin-bootstrap) row → claim it before merging. + if ( + !Number.isInteger(conflictOwnerUserId) || + conflictOwnerUserId <= 0 + ) { + await this.appStore.claimOwnership(conflictRow.id, user.id); + } + + const appToJoin = await this.appStore.getByUid(conflictRow.uid); + if (!appToJoin || appToJoin.uid !== conflictRow.uid) { + throw new HttpError(400, 'App index_url already in use', { + legacyCode: 'app_index_url_already_in_use', + fields: { + index_url: indexUrl, + app_uid: conflictRow.uid, + }, + }); + } + if (appToJoin.owner_user_id !== user.id) { + throw new HttpError(400, 'App index_url already in use', { + legacyCode: 'app_index_url_already_in_use', + fields: { + index_url: indexUrl, + app_uid: conflictRow.uid, + }, + }); + } + if ( + Number.isInteger(conflictOwnerUserId) && + conflictOwnerUserId === user.id && + !this.#isOriginBootstrapApp(appToJoin) + ) { + // Prevent merging arbitrary same-owner apps; only allow the + // auto-created origin bootstrap row to be absorbed. + throw new HttpError(400, 'App index_url already in use', { + legacyCode: 'app_index_url_already_in_use', + fields: { + index_url: indexUrl, + app_uid: conflictRow.uid, + }, + }); + } + + // Build the joined input. Pass the original (unvalidated) + // object through the recursive `update` so its `#validateInput` + // re-runs cleanly — `fields` is post-validation (stringified + // metadata, 0/1 bools) and would fail a second pass. + const joinedObject = { ...object }; + const requestedJoinedName = + (typeof joinedObject.name === 'string' + ? joinedObject.name.trim() + : '') || null; + const shouldReapplyRequestedNameAfterMerge = + !!sourceAppUid && !!requestedJoinedName; + // When called from update, defer the rename until after the + // source row is deleted — otherwise the rename would collide + // with the still-existing source app's name. + if (sourceAppUid && joinedObject.name !== undefined) { + delete joinedObject.name; + } + + let joinedApp = await this.update({ + uid: appToJoin.uid, + object: joinedObject, + options, + }); + + if (sourceAppUid && sourceAppUid !== appToJoin.uid) { + await this.#writeCanonicalAppUidAlias({ + oldAppUid: sourceAppUid, + canonicalAppUid: appToJoin.uid, + }); + const sourceApp = await this.appStore.getByUid(sourceAppUid); + if (sourceApp) { + await this.appStore.delete(sourceApp.id); + this.#emitAppChanged({ + app: null, + old_app: sourceApp, + action: 'deleted', + }); + } + } + + if (shouldReapplyRequestedNameAfterMerge) { + joinedApp = await this.update({ + uid: appToJoin.uid, + object: { name: requestedJoinedName }, + options, + }); + } + + return joinedApp; + } +} diff --git a/src/backend/drivers/decorators.ts b/src/backend/drivers/decorators.ts new file mode 100644 index 000000000..74998894e --- /dev/null +++ b/src/backend/drivers/decorators.ts @@ -0,0 +1,51 @@ +import { + DRIVER_DEFAULT_KEY, + DRIVER_INTERFACE_KEY, + DRIVER_NAME_KEY, +} from './meta'; + +/** + * Options for the `@Driver` class decorator. + */ +export interface DriverOptions { + /** Unique name for this implementation within its interface. Defaults to the class name. */ + name?: string; + /** When true, this driver is the default for its interface. */ + default?: boolean; +} + +/** + * Class decorator that marks a driver implementation and records its + * interface + name on the prototype. + * + * Equivalent imperative approach (no decorator needed): + * ```ts + * class MyDriver extends PuterDriver { + * readonly driverInterface = 'puter-chat-completion'; + * readonly driverName = 'my-impl'; + * readonly isDefault = true; + * } + * ``` + * + * Usage: + * ```ts + * @Driver('puter-chat-completion', { name: 'openai-completion', default: true }) + * class OpenAIChatDriver extends PuterDriver { + * async complete(args) { ... } + * } + * ``` + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type AnyCtor = new (...args: any[]) => any; + +export function Driver(interfaceName: string, opts: DriverOptions = {}) { + return ( + value: T, + _context: ClassDecoratorContext, + ): void => { + const proto = value.prototype as Record; + proto[DRIVER_INTERFACE_KEY] = interfaceName; + proto[DRIVER_NAME_KEY] = opts.name ?? value.name; + proto[DRIVER_DEFAULT_KEY] = opts.default ?? false; + }; +} diff --git a/src/backend/drivers/index.ts b/src/backend/drivers/index.ts new file mode 100644 index 000000000..06858a21d --- /dev/null +++ b/src/backend/drivers/index.ts @@ -0,0 +1,31 @@ +import type { IPuterDriverRegistry } from './types'; +import { ChatCompletionDriver } from './ai-chat/ChatCompletionDriver'; +import { ImageGenerationDriver } from './ai-image/ImageGenerationDriver'; +import { TTSDriver } from './ai-tts/TTSDriver'; +import { VideoGenerationDriver } from './ai-video/VideoGenerationDriver'; +import { VoiceChangerDriver } from './ai-speech2speech/VoiceChangerDriver'; +import { SpeechToTextDriver } from './ai-speech2txt/SpeechToTextDriver'; +import { OCRDriver } from './ai-ocr/OCRDriver'; +import { AppDriver } from './apps/AppDriver.js'; +import { KVStoreDriver } from './kv/KVStoreDriver'; +import { NotificationDriver } from './notification/NotificationDriver'; +import { SubdomainDriver } from './subdomain/SubdomainDriver'; +import { WorkerDriver } from './workers/WorkerDriver'; + +export { resolveDriverMeta } from './meta'; +export { Driver } from './decorators'; + +export const puterDrivers = { + kvStore: KVStoreDriver, + aiChat: ChatCompletionDriver, + aiImage: ImageGenerationDriver, + aiTts: TTSDriver, + aiVideo: VideoGenerationDriver, + aiSpeech2Speech: VoiceChangerDriver, + aiSpeech2Txt: SpeechToTextDriver, + aiOcr: OCRDriver, + apps: AppDriver, + subdomains: SubdomainDriver, + notifications: NotificationDriver, + workers: WorkerDriver, +} satisfies IPuterDriverRegistry; diff --git a/src/backend/drivers/kv/KVStoreDriver.ts b/src/backend/drivers/kv/KVStoreDriver.ts new file mode 100644 index 000000000..cc1a02845 --- /dev/null +++ b/src/backend/drivers/kv/KVStoreDriver.ts @@ -0,0 +1,329 @@ +import { HttpError } from '../../core/http/HttpError.js'; +import { Context } from '../../core/context.js'; +import { PuterDriver } from '../types.js'; +import type { Actor } from '../../core/actor.js'; +import type { KVUsage } from '../../stores/systemKv/SystemKVStore.js'; +import { KV_COSTS } from './costs.js'; + +/** + * KV store driver implementing the `puter-kvstore` interface. + * + * Thin wrapper around `stores.kv` (SystemKVStore): it validates/coerces + * request inputs into HTTP-friendly errors, passes the request actor through + * so the store scopes data to the correct namespace, and meters the + * DynamoDB capacity the store reports back. + */ +export class KVStoreDriver extends PuterDriver { + readonly driverInterface = 'puter-kvstore'; + readonly driverName = 'puter-kvstore'; + readonly isDefault = true; + + override getReportedCosts(): Record[] { + return Object.entries(KV_COSTS).map(([usageType, ucentsPerUnit]) => ({ + usageType, + ucentsPerUnit, + unit: 'capacity-unit', + source: 'driver:kvStore', + })); + } + + #coerceKey(key: unknown): string { + if (key === null || key === undefined) { + throw new HttpError(400, 'Missing `key`'); + } + const str = typeof key === 'string' ? key : String(key); + if (str === '') throw new HttpError(400, 'Missing `key`'); + return str; + } + + #opts(appUuid?: string): { actor: Actor | undefined; appUuid?: string } { + const actor = Context.get('actor') as Actor | undefined; + if (actor?.app?.uid) { + // force appUuid to be the one from the actor if it exists, only root tokens allowed to override appUuid + appUuid = undefined; + } + return { actor: Context.get('actor') as Actor | undefined, appUuid }; + } + + #meter(actor: Actor | undefined, usage: KVUsage): void { + if (!actor) return; + const metering = this.services.metering; + if (usage.read > 0) { + void metering + .incrementUsage( + actor, + 'kv:read', + usage.read, + KV_COSTS['kv:read'] * usage.read, + ) + .catch((e) => + console.warn( + '[kv] metering kv:read failed:', + (e as Error).message, + ), + ); + } + if (usage.write > 0) { + void metering + .incrementUsage( + actor, + 'kv:write', + usage.write, + KV_COSTS['kv:write'] * usage.write, + ) + .catch((e) => + console.warn( + '[kv] metering kv:write failed:', + (e as Error).message, + ), + ); + } + } + + async get(args: { + key: unknown; + optConfig?: { appUuid?: string }; + }): Promise { + const { key, optConfig } = args; + if (key === undefined || key === null) { + throw new HttpError(400, 'Missing `key`'); + } + + const opts = this.#opts(optConfig?.appUuid); + + if (Array.isArray(key)) { + if (key.length === 0) return []; + const coerced = key.map((k) => this.#coerceKey(k)); + const { res, usage } = await this.stores.kv.get( + { key: coerced }, + opts, + ); + this.#meter(opts.actor, usage); + return res; + } + + const { res, usage } = await this.stores.kv.get( + { key: this.#coerceKey(key) }, + opts, + ); + this.#meter(opts.actor, usage); + return res; + } + + async set(args: { + key: unknown; + value: unknown; + expireAt?: number; + optConfig?: { appUuid?: string }; + }): Promise { + const { key, value, expireAt, optConfig } = args; + const coerced = this.#coerceKey(key); + if (value === undefined) throw new HttpError(400, 'Missing `value`'); + + const opts = this.#opts(optConfig?.appUuid); + const { res, usage } = await this.stores.kv.set( + { key: coerced, value, expireAt }, + opts, + ); + this.#meter(opts.actor, usage); + return res; + } + + async batchPut(args: { + items: Array<{ key: string; value: unknown; expireAt?: number }>; + optConfig?: { appUuid?: string }; + }): Promise { + const { items, optConfig } = args; + if (!Array.isArray(items) || items.length === 0) { + throw new HttpError(400, 'Missing or empty `items`'); + } + + const coerced = items.map((item) => ({ + key: this.#coerceKey(item.key), + value: item.value, + expireAt: item.expireAt, + })); + + const opts = this.#opts(optConfig?.appUuid); + const { res, usage } = await this.stores.kv.batchPut( + { items: coerced }, + opts, + ); + this.#meter(opts.actor, usage); + return res; + } + + async del(args: { + key: unknown; + optConfig?: { appUuid?: string }; + }): Promise { + const coerced = this.#coerceKey(args.key); + const opts = this.#opts(args.optConfig?.appUuid); + const { res, usage } = await this.stores.kv.del({ key: coerced }, opts); + this.#meter(opts.actor, usage); + return res; + } + + async list(args: { + as?: 'entries' | 'keys' | 'values'; + limit?: number; + cursor?: string | Record; + pattern?: string; + optConfig?: { appUuid?: string }; + }): Promise { + const opts = this.#opts(args.optConfig?.appUuid); + const { res, usage } = await this.stores.kv.list( + { + as: args.as, + limit: args.limit, + cursor: args.cursor, + pattern: args.pattern, + }, + opts, + ); + this.#meter(opts.actor, usage); + return res; + } + + async flush(args: { optConfig?: { appUuid?: string } }): Promise { + const opts = this.#opts(args.optConfig?.appUuid); + const { res, usage } = await this.stores.kv.flush(opts); + this.#meter(opts.actor, usage); + return res; + } + + async incr(args: { + key: unknown; + pathAndAmountMap: Record; + optConfig?: { appUuid?: string }; + }): Promise { + const coerced = this.#coerceKey(args.key); + if ( + !args.pathAndAmountMap || + typeof args.pathAndAmountMap !== 'object' + ) { + throw new HttpError(400, 'Missing or invalid `pathAndAmountMap`'); + } + const opts = this.#opts(args.optConfig?.appUuid); + const { res, usage } = await this.stores.kv.incr( + { key: coerced, pathAndAmountMap: args.pathAndAmountMap }, + opts, + ); + this.#meter(opts.actor, usage); + return res; + } + + async decr(args: { + key: unknown; + pathAndAmountMap: Record; + optConfig?: { appUuid?: string }; + }): Promise { + const coerced = this.#coerceKey(args.key); + if ( + !args.pathAndAmountMap || + typeof args.pathAndAmountMap !== 'object' + ) { + throw new HttpError(400, 'Missing or invalid `pathAndAmountMap`'); + } + const opts = this.#opts(args.optConfig?.appUuid); + const { res, usage } = await this.stores.kv.decr( + { key: coerced, pathAndAmountMap: args.pathAndAmountMap }, + opts, + ); + this.#meter(opts.actor, usage); + return res; + } + + async expireAt(args: { + key: unknown; + timestamp: number; + optConfig?: { appUuid?: string }; + }): Promise { + const coerced = this.#coerceKey(args.key); + if (typeof args.timestamp !== 'number') { + throw new HttpError(400, '`timestamp` must be a number'); + } + const opts = this.#opts(args.optConfig?.appUuid); + const { usage } = await this.stores.kv.expireAt( + { key: coerced, timestamp: args.timestamp }, + opts, + ); + this.#meter(opts.actor, usage); + } + + async expire(args: { + key: unknown; + ttl: number; + optConfig?: { appUuid?: string }; + }): Promise { + const coerced = this.#coerceKey(args.key); + if (typeof args.ttl !== 'number') { + throw new HttpError(400, '`ttl` must be a number (seconds)'); + } + const opts = this.#opts(args.optConfig?.appUuid); + const { usage } = await this.stores.kv.expire( + { key: coerced, ttl: args.ttl }, + opts, + ); + this.#meter(opts.actor, usage); + } + + async update(args: { + key: unknown; + pathAndValueMap: Record; + ttl?: number; + optConfig?: { appUuid?: string }; + }): Promise { + const coerced = this.#coerceKey(args.key); + if (!args.pathAndValueMap || typeof args.pathAndValueMap !== 'object') { + throw new HttpError(400, 'Missing or invalid `pathAndValueMap`'); + } + const opts = this.#opts(args.optConfig?.appUuid); + const { res, usage } = await this.stores.kv.update( + { + key: coerced, + pathAndValueMap: args.pathAndValueMap, + ttl: args.ttl, + }, + opts, + ); + this.#meter(opts.actor, usage); + return res; + } + + async add(args: { + key: unknown; + pathAndValueMap: Record; + optConfig?: { appUuid?: string }; + }): Promise { + const coerced = this.#coerceKey(args.key); + if (!args.pathAndValueMap || typeof args.pathAndValueMap !== 'object') { + throw new HttpError(400, 'Missing or invalid `pathAndValueMap`'); + } + const opts = this.#opts(args.optConfig?.appUuid); + const { res, usage } = await this.stores.kv.add( + { key: coerced, pathAndValueMap: args.pathAndValueMap }, + opts, + ); + this.#meter(opts.actor, usage); + return res; + } + + async remove(args: { + key: unknown; + paths: string[]; + optConfig?: { appUuid?: string }; + }): Promise { + const coerced = this.#coerceKey(args.key); + if (!Array.isArray(args.paths) || args.paths.length === 0) { + throw new HttpError(400, 'Missing or invalid `paths`'); + } + const opts = this.#opts(args.optConfig?.appUuid); + const { res, usage } = await this.stores.kv.remove( + { key: coerced, paths: args.paths }, + opts, + ); + this.#meter(opts.actor, usage); + return res; + } +} diff --git a/src/backend/drivers/kv/costs.ts b/src/backend/drivers/kv/costs.ts new file mode 100644 index 000000000..eb2310876 --- /dev/null +++ b/src/backend/drivers/kv/costs.ts @@ -0,0 +1,6 @@ +// Microcents per underlying DynamoDB capacity unit, as reported by +// SystemKVStore.KVUsage. Cost is `KV_COSTS[op] * usage.`. +export const KV_COSTS = { + 'kv:read': 63, + 'kv:write': 125, +} as const; diff --git a/src/backend/drivers/meta.ts b/src/backend/drivers/meta.ts new file mode 100644 index 000000000..3bbb664a0 --- /dev/null +++ b/src/backend/drivers/meta.ts @@ -0,0 +1,90 @@ +import type { Readable } from 'node:stream'; +import type { WithLifecycle } from '../types'; + +// ── Stream result convention ──────────────────────────────────────── +// +// Driver methods that return a stream instead of JSON wrap the readable +// in this shape. The `/drivers/call` handler detects it and pipes to the +// HTTP response instead of calling `res.json()`. + +export interface DriverStreamResult { + /** Discriminant — must be `'stream'`. */ + dataType: 'stream'; + /** MIME type sent as Content-Type (e.g. `'application/x-ndjson'`). */ + content_type: string; + /** When true, sets `Transfer-Encoding: chunked`. */ + chunked?: boolean; + /** The readable stream to pipe to the response. */ + stream: Readable; +} + +export function isDriverStreamResult(v: unknown): v is DriverStreamResult { + return ( + !!v && + typeof v === 'object' && + (v as Record).dataType === 'stream' && + 'stream' in v + ); +} + +// ── Driver metadata keys ──────────────────────────────────────────── +// +// Metadata keys stored on driver prototypes by the `@Driver` decorator. +// Imperative drivers set these as instance properties instead. + +export const DRIVER_INTERFACE_KEY = '__driverInterface' as const; +export const DRIVER_NAME_KEY = '__driverName' as const; +export const DRIVER_DEFAULT_KEY = '__driverDefault' as const; +export const DRIVER_ALIASES_KEY = '__driverAliases' as const; + +/** + * Resolved metadata for a registered driver. Read from either decorator + * metadata or imperative instance properties. + */ +export interface DriverMeta { + /** The interface this driver implements (e.g. 'puter-chat-completion'). */ + interfaceName: string; + /** Unique name within its interface (e.g. 'openai-completion', 'claude'). */ + driverName: string; + /** When true, this driver is the default for its interface. */ + isDefault: boolean; + /** + * Additional driver names that resolve to the same instance. Used by + * multi-provider drivers (TTS/OCR/image/video) so legacy puter-js calls + * that pass a provider id in the `driver` slot (e.g. `aws-polly`, + * `openai-tts`) still find the unified driver. The requested alias is + * exposed to the driver method via `Context.get('driverName')` so the + * method can route to the right internal provider. + */ + aliases: string[]; +} + +/** + * Extract driver metadata from a driver instance. Checks decorator-set + * prototype metadata first, then falls back to instance properties. + * Returns `null` if the driver doesn't declare an interface. + */ +export function resolveDriverMeta( + driver: WithLifecycle & Record, +): DriverMeta | null { + const proto = Object.getPrototypeOf(driver) as Record; + + const interfaceName = + (proto[DRIVER_INTERFACE_KEY] as string | undefined) ?? + (driver.driverInterface as string | undefined); + const driverName = + (proto[DRIVER_NAME_KEY] as string | undefined) ?? + (driver.driverName as string | undefined); + const isDefault = + (proto[DRIVER_DEFAULT_KEY] as boolean | undefined) ?? + (driver.isDefault as boolean | undefined) ?? + false; + const aliases = + (proto[DRIVER_ALIASES_KEY] as string[] | undefined) ?? + (driver.driverAliases as string[] | undefined) ?? + []; + + if (!interfaceName || !driverName) return null; + + return { interfaceName, driverName, isDefault, aliases }; +} diff --git a/src/backend/drivers/notification/NotificationDriver.ts b/src/backend/drivers/notification/NotificationDriver.ts new file mode 100644 index 000000000..d201151b2 --- /dev/null +++ b/src/backend/drivers/notification/NotificationDriver.ts @@ -0,0 +1,170 @@ +import { Context } from '../../core/context.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import { PuterDriver } from '../types.js'; +import type { Actor } from '../../core/actor.js'; + +const MAX_SELECT_LIMIT = 200; + +/** + * Driver exposing the `puter-notifications` interface. + * + * Wraps NotificationStore with owner-scoped permission checks. + * Methods follow the `crud-q` shape: create, read, select. + * + * Read-only for clients — `update` and `delete` are not exposed. `create` + * is available for server-internal callers (other services push + * notifications via `/drivers/call` with a system token or directly + * through the store). `read` and `select` accept predicates. + * + * Permission model: + * - Strictly owner-limited — each user can only see their own notifications + * - No app-actor access (user tokens only) + * + * Predicates: + * - `'unseen'` — shown IS NULL AND acknowledged IS NULL + * - `'unacknowledged'` — acknowledged IS NULL (may be shown) + * - `'acknowledged'` — acknowledged IS NOT NULL + */ +export class NotificationDriver extends PuterDriver { + readonly driverInterface = 'puter-notifications'; + // Matches origin/main's `iface_to_driver['puter-notifications']` and the + // hardcoded `service:es\Cnotification:…` permission keys. + readonly driverName = 'es:notification'; + readonly isDefault = true; + + // ── Driver methods ────────────────────────────────────────────── + + async create(args: Record): Promise { + const object = args.object as Record | undefined; + if (!object || typeof object !== 'object') { + throw new HttpError(400, 'Missing or invalid `object`'); + } + const actor = this.#requireUserActor(); + + const value = object.value ?? {}; + const created = await this.stores.notification.create({ + userId: actor.user.id, + value, + }); + return this.#toClient(created); + } + + async read(args: Record): Promise { + const actor = this.#requireUserActor(); + const uid = (args.uid ?? args.id) as string | undefined; + if (!uid) throw new HttpError(400, 'Missing `uid`'); + + const row = await this.stores.notification.getByUid(String(uid), { + userId: actor.user.id, + }); + if (!row) throw new HttpError(404, 'Notification not found'); + return this.#toClient(row); + } + + async select(args: Record): Promise { + const actor = this.#requireUserActor(); + const limit = Math.min( + Number(args.limit ?? MAX_SELECT_LIMIT), + MAX_SELECT_LIMIT, + ); + const predicate = args.predicate as string | string[] | undefined; + + const predicateName = Array.isArray(predicate) + ? predicate[0] + : predicate; + + // Route predicate → store query params + let rows: Array>; + switch (predicateName) { + case 'unseen': + rows = await this.stores.notification.listByUserId( + actor.user.id, + { + limit, + filter: 'unseen', + }, + ); + break; + case 'unacknowledged': + case 'unacknowledge': // client compat alias + rows = await this.stores.notification.listByUserId( + actor.user.id, + { + limit, + onlyUnacknowledged: true, + }, + ); + break; + case 'acknowledged': + case 'acknowledge': // client compat alias + rows = await this.stores.notification.listByUserId( + actor.user.id, + { + limit, + filter: 'acknowledged', + }, + ); + break; + default: + rows = await this.stores.notification.listByUserId( + actor.user.id, + { limit }, + ); + break; + } + + return rows.map((r) => this.#toClient(r)); + } + + /** Mark a notification as shown. Used by GUI when notification is displayed. */ + async mark_shown(args: Record): Promise { + const actor = this.#requireUserActor(); + const uid = String(args.uid ?? ''); + if (!uid) throw new HttpError(400, 'Missing `uid`'); + const ok = await this.stores.notification.markShown(uid, actor.user.id); + return { success: ok }; + } + + /** Mark a notification as acknowledged (user dismissed it). */ + async mark_acknowledged(args: Record): Promise { + const actor = this.#requireUserActor(); + const uid = String(args.uid ?? ''); + if (!uid) throw new HttpError(400, 'Missing `uid`'); + const ok = await this.stores.notification.markAcknowledged( + uid, + actor.user.id, + ); + return { success: ok }; + } + + // ── Permissions ───────────────────────────────────────────────── + + #requireUserActor(): Actor & { + user: { id: number; uuid: string; username: string }; + } { + const actor = Context.get('actor') as Actor | undefined; + if (!actor) throw new HttpError(401, 'Authentication required'); + if (!actor.user?.id) throw new HttpError(403, 'User actor required'); + // App-under-user actors are not allowed for notifications. + if (actor.app) + throw new HttpError(403, 'App actors cannot access notifications'); + return actor as Actor & { + user: { id: number; uuid: string; username: string }; + }; + } + + // ── Serialization ─────────────────────────────────────────────── + + #toClient( + row: Record | null, + ): Record | null { + if (!row) return null; + return { + uid: row.uid, + value: row.value, + shown: row.shown ?? null, + acknowledged: row.acknowledged ?? null, + created_at: row.created_at ?? null, + }; + } +} diff --git a/src/backend/drivers/subdomain/SubdomainDriver.ts b/src/backend/drivers/subdomain/SubdomainDriver.ts new file mode 100644 index 000000000..091dd4650 --- /dev/null +++ b/src/backend/drivers/subdomain/SubdomainDriver.ts @@ -0,0 +1,675 @@ +import { posix as pathPosix } from 'node:path'; +import { Context } from '../../core/context.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import { PuterDriver } from '../types.js'; +import type { Actor } from '../../core/actor.js'; +import type { AclMode } from '../../services/acl/ACLService.js'; +import type { FSEntry } from '../../stores/fs/FSEntry.js'; +import type { UserRow } from '../../stores/user/UserStore.js'; +import { expandTildePath } from '../../services/fs/resolveNode.js'; + +const SUBDOMAIN_MAX_LEN = 64; +const SUBDOMAIN_REGEX = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/; +const DEFAULT_MAX_SUBDOMAINS = 500; + +// Reserved words. Extend via config if needed. +const RESERVED_SUBDOMAINS = new Set([ + 'www', + 'api', + 'mail', + 'ftp', + 'admin', + 'localhost', + 'ns1', + 'ns2', + 'smtp', + 'pop', + 'imap', + 'blog', + 'dev', + 'staging', + 'test', +]); + +/** + * Driver exposing the `puter-subdomains` interface. + * + * Wraps SubdomainStore with validation + permission checks. + * Methods follow the `crud-q` shape: create, read, select, update, + * upsert, delete. + * + * Permission model: + * - Owner (user_id) can read/write their own subdomains + * - App actor matching app_owner can read/write scoped subdomains + * - `system:es:write-all-owners` grants blanket write + * - `read-all-subdomains` grants cross-user reads + */ +export class SubdomainDriver extends PuterDriver { + readonly driverInterface = 'puter-subdomains'; + // Matches origin/main's `iface_to_driver['puter-subdomains']` and the + // hardcoded `service:es\Csubdomain:…` permission keys. + readonly driverName = 'es:subdomain'; + readonly isDefault = true; + + // ── Driver methods ────────────────────────────────────────────── + + async create(args: Record): Promise { + const object = args.object as Record | undefined; + + if (!object || typeof object !== 'object') { + throw new HttpError(400, 'Missing or invalid `object`'); + } + const actor = this.#requireActor(); + this.#requireUser(actor); + this.#requireVerified(actor); + + const subdomain = this.#validateSubdomain(object.subdomain); + + // Uniqueness + if (await this.stores.subdomain.existsBySubdomain(subdomain)) { + throw new HttpError( + 409, + 'A site with this subdomain already exists', + ); + } + + // Quota + const maxSubdomains = + ((actor.user as unknown as Record) + .max_subdomains as number | undefined) ?? + this.#configMaxSubdomains(); + const currentCount = (await this.stores.subdomain.countByUserId( + actor.user.id, + )) as number; + if (currentCount >= maxSubdomains) { + throw new HttpError(403, 'Subdomain limit reached'); + } + + const rootDirPath = expandTildePath( + String(object.root_dir ?? ''), + actor.user.username, + ); + const entry = await this.stores.fsEntry.getEntryByPath(rootDirPath); + const rootDirId = entry?.id; + if (!rootDirId) { + throw new HttpError(400, 'root_dir_id does not exist', { + legacyCode: 'bad_request', + }); + } + await this.services.fs.checkFSAccess(entry, actor); + const associatedAppId = await this.#resolveAssociatedAppId(object); + const created = await this.stores.subdomain.create({ + userId: actor.user.id, + subdomain, + rootDirId, + associatedAppId, + appOwner: actor.app?.id ?? null, + }); + const [shaped] = await this.#hydrateRows( + created ? [created as Record] : [], + ); + return shaped ?? null; + } + + async read(args: Record): Promise { + const actor = this.#requireActor(); + const row = await this.#resolve(args); + if (!row) throw new HttpError(404, 'Subdomain not found'); + await this.#checkReadAccess(row, actor); + const [shaped] = await this.#hydrateRows([row]); + return shaped ?? null; + } + + async select(args: Record): Promise { + const actor = this.#requireActor(); + this.#requireUser(actor); + + const predicate = args.predicate as unknown[] | string | undefined; + const limit = Math.min(Number(args.limit ?? 5000), 5000); + + // Match v1: when the actor has `read-all-subdomains` (admin / + // privileged accounts), widen to every subdomain. Without this + // older accounts whose `user_id` rows drifted from the current + // user.id only see a partial slice of their own list. + const widenToAll = + predicate !== 'user-can-edit' && + (await this.#hasPermission(actor, 'read-all-subdomains')); + + let rows = widenToAll + ? await this.stores.subdomain.listAll({ limit }) + : await this.stores.subdomain.listByUserId(actor.user.id, { + limit, + }); + + // App-limited: app actors only see subdomains they own. Skip when + // we widened above — read-all is meant to bypass scoping. + if (!widenToAll && actor.app) { + rows = rows.filter((r) => r.app_owner === actor.app!.id); + } + + return this.#hydrateRows(rows as Array>); + } + + async update(args: Record): Promise { + const object = args.object as Record | undefined; + if (!object || typeof object !== 'object') { + throw new HttpError(400, 'Missing or invalid `object`'); + } + const actor = this.#requireActor(); + this.#requireUser(actor); + this.#requireVerified(actor); + + const row = await this.#resolve(args); + if (!row) throw new HttpError(404, 'Subdomain not found'); + await this.#checkWriteAccess(row, actor); + + // Subdomain name is immutable — strip if provided + const patch: Record = {}; + if (object.root_dir !== undefined) { + const rootDirPath = expandTildePath( + String(object.root_dir), + actor.user.username, + ); + const entry = await this.stores.fsEntry.getEntryByPath(rootDirPath); + const rootDirId = entry?.id; + if (!rootDirId) { + throw new HttpError(400, 'root_dir_id does not exist', { + legacyCode: 'bad_request', + }); + } + if (rootDirId !== (row.root_dir_id ?? null)) { + await this.services.fs.checkFSAccess(entry, actor); + } + patch.root_dir_id = rootDirId; + } + if (object.associated_app_uid !== undefined) + patch.associated_app_id = + await this.#resolveAssociatedAppId(object); + if (object.domain !== undefined) + patch.domain = object.domain != null ? String(object.domain) : null; + + const updated = await this.stores.subdomain.update(row.uuid, patch, { + userId: row.user_id, + }); + const [shaped] = await this.#hydrateRows( + updated ? [updated as Record] : [], + ); + return shaped ?? null; + } + + async #checkFSAccess( + rootDirId: number | null | undefined, + actor: Actor, + mode: AclMode = 'write', + ): Promise { + if (rootDirId == null) return; + + const entry = await this.stores.fsEntry.getEntryById(rootDirId); + if (!entry) { + throw new HttpError(400, 'root_dir_id does not exist'); + } + + const fsService = this.services.fs; + let ancestorsCache: Promise< + Array<{ uid: string; path: string }> + > | null = null; + const descriptor = { + path: entry.path, + resolveAncestors() { + if (!ancestorsCache) { + ancestorsCache = fsService.getAncestorChain(entry.path); + } + return ancestorsCache; + }, + }; + const allowed = await this.services.acl.check(actor, descriptor, mode); + if (allowed) return; + + const safe = (await this.services.acl.getSafeAclError( + actor, + descriptor, + mode, + )) as { + status?: unknown; + message?: unknown; + fields?: { code?: unknown }; + }; + const status = Number(safe?.status); + const message = + typeof safe?.message === 'string' && safe.message.length > 0 + ? safe.message + : 'Access denied'; + const code = + typeof safe?.fields?.code === 'string' + ? safe.fields.code + : undefined; + const legacyCode = code === 'forbidden' ? 'access_denied' : code; + if (status === 404) { + throw new HttpError(404, message, { + ...(legacyCode ? { legacyCode } : {}), + }); + } + throw new HttpError(403, message, { + legacyCode: legacyCode ?? 'access_denied', + }); + } + + async upsert(args: Record): Promise { + const existing = await this.#resolve(args); + if (existing) + return this.update({ + uid: existing.uuid, + object: args.object as Record, + }); + return this.create(args); + } + + async delete(args: Record): Promise { + const actor = this.#requireActor(); + this.#requireUser(actor); + this.#requireVerified(actor); + + const row = await this.#resolve(args); + if (!row) throw new HttpError(404, 'Subdomain not found'); + + if (row.protected) { + throw new HttpError(403, 'Cannot delete a protected subdomain'); + } + + await this.#checkWriteAccess(row, actor); + await this.stores.subdomain.deleteByUuid(row.uuid, { + userId: row.user_id, + }); + return { success: true, uid: row.uuid }; + } + + // ── Resolve ───────────────────────────────────────────────────── + + async #resolve( + args: Record, + ): Promise | null> { + if (args.uid) return this.stores.subdomain.getByUuid(String(args.uid)); + const id = args.id as Record | string | undefined; + if (typeof id === 'string') return this.stores.subdomain.getByUuid(id); + if (id && typeof id === 'object') { + if (id.uid) return this.stores.subdomain.getByUuid(String(id.uid)); + if (id.subdomain) + return this.stores.subdomain.getBySubdomain( + String(id.subdomain), + ); + } + return null; + } + + // ── Validation ────────────────────────────────────────────────── + + #validateSubdomain(raw: unknown): string { + if (typeof raw !== 'string' || raw.trim().length === 0) { + throw new HttpError(400, 'Missing or empty `subdomain`'); + } + if (raw.length > SUBDOMAIN_MAX_LEN) { + throw new HttpError( + 400, + `Subdomain exceeds max length (${SUBDOMAIN_MAX_LEN})`, + ); + } + const s = raw.trim().toLowerCase(); + + if (!SUBDOMAIN_REGEX.test(s)) { + throw new HttpError( + 400, + 'Invalid subdomain format (lowercase alphanumeric + hyphens, must not start/end with hyphen)', + ); + } + if (RESERVED_SUBDOMAINS.has(s)) { + throw new HttpError(400, `Subdomain '${s}' is reserved`); + } + return s; + } + + // ── Permissions ───────────────────────────────────────────────── + + #requireActor(): Actor & { + user: { id: number; uuid: string; username: string }; + } { + const actor = Context.get('actor') as Actor | undefined; + if (!actor?.user?.id) + throw new HttpError(401, 'Authentication required'); + return actor as Actor & { + user: { id: number; uuid: string; username: string }; + }; + } + + #requireUser(actor: Actor): void { + if (!actor.user?.id) throw new HttpError(403, 'User actor required'); + } + + /** + * Mirror of the HTTP-layer `requireVerifiedGate` on /delete-site — only + * active when `strict_email_verification_required` is truthy, so self- + * hosted installs without SMTP aren't bricked. Applied at the driver + * level so /drivers/call can't bypass the gate the HTTP route enforces. + */ + #requireVerified(actor: Actor): void { + if (!this.config.strict_email_verification_required) return; + const user = actor.user as Record | undefined; + if (!user?.email_confirmed) { + throw new HttpError(400, 'Account email is not verified', { + legacyCode: 'account_is_not_verified', + }); + } + } + + async #hasPermission(actor: Actor, permission: string): Promise { + try { + return await this.services.permission.check(actor, permission); + } catch { + return false; + } + } + + async #checkReadAccess( + row: Record, + actor: Actor, + ): Promise { + // Owner + if (actor.user?.id === row.user_id) return; + // App actor matching app_owner + if (actor.app?.id && actor.app.id === row.app_owner) return; + // Cross-user read permission + if (await this.#hasPermission(actor, 'read-all-subdomains')) return; + throw new HttpError(403, 'Access denied'); + } + + async #checkWriteAccess( + row: Record, + actor: Actor, + ): Promise { + // App actor matching app_owner + let hasAccess = false; + if (!actor.app?.id) { + hasAccess = actor.user?.id === row.user_id; + } else if (actor.app.id === row.app_owner) { + hasAccess = actor.user?.id === row.user_id; + } + // System-wide write + if (!hasAccess) { + hasAccess = await this.#hasPermission( + actor, + 'system:es:write-all-owners', + ); + } + if (!hasAccess) { + throw new HttpError(403, 'Access denied'); + } + } + + // ── Input resolution ──────────────────────────────────────────── + // + // Clients send `associated_app_uid` (the app's public UUID), never + // a raw mysql id. v1 used the OM `reference` mapping to do the + // same translation under the hood; v2 does it explicitly. + async #resolveAssociatedAppId( + object: Record, + ): Promise { + const uid = object.associated_app_uid; + if (uid == null) return null; + if (typeof uid !== 'string' || uid.length === 0) { + throw new HttpError(400, '`associated_app_uid` must be a string'); + } + const app = (await this.stores.app.getByUid(uid)) as { + id: number; + } | null; + if (!app) { + throw new HttpError(400, '`associated_app_uid` does not exist'); + } + return app.id; + } + + // ── Config ────────────────────────────────────────────────────── + + #configMaxSubdomains(): number { + const n = Number( + this.config.max_subdomains_per_user ?? DEFAULT_MAX_SUBDOMAINS, + ); + return Number.isFinite(n) && n > 0 ? n : DEFAULT_MAX_SUBDOMAINS; + } + + // ── Serialization ─────────────────────────────────────────────── + // + // v1's `puter-subdomains` shape (canonical, see SubdomainES + the + // mapping at om/mappings/subdomain.js): + // { + // uid, subdomain, domain, + // root_dir: , + // associated_app: | null, + // created_at: , + // owner: { username, uuid }, + // app_owner: | null, + // protected: bool, + // } + // + // We never expose raw mysql ids (user_id / root_dir_id / + // associated_app_id / app_owner-as-id) — clients see uuids and + // nested objects instead. + + /** + * Hydrate raw subdomain rows into the v1-shaped client response. + * + * Resolves the foreign keys (user_id → owner, root_dir_id → + * root_dir, associated_app_id / app_owner → app shapes) with one + * batched lookup per store, regardless of how many rows we're + * shaping. Used by both `select` (many rows) and the single-row + * paths (`create`/`read`/`update`/`upsert`) so the wire shape stays + * identical. + */ + async #hydrateRows( + rows: Array>, + ): Promise>> { + if (rows.length === 0) return []; + + const collectIds = ( + key: 'user_id' | 'root_dir_id' | 'associated_app_id' | 'app_owner', + ): number[] => { + const out = new Set(); + for (const r of rows) { + const v = r[key]; + if (typeof v === 'number') out.add(v); + else if (typeof v === 'string' && v.length > 0) { + const n = Number(v); + if (Number.isFinite(n)) out.add(n); + } + } + return [...out]; + }; + + const userIds = collectIds('user_id'); + const rootDirIds = collectIds('root_dir_id'); + const appIds = [ + ...new Set([ + ...collectIds('associated_app_id'), + ...collectIds('app_owner'), + ]), + ]; + + // Single round-trip per store, all in parallel — the filetype + // lookup keys off the requested ids (not on getByIds' result), + // so it doesn't need to wait for the app rows. + const [usersById, entriesById, appsById, filetypesByAppId] = + await Promise.all([ + this.stores.user.getByIds(userIds), + this.stores.fsEntry.getEntriesByIds(rootDirIds), + this.stores.app.getByIds(appIds), + this.stores.app.getFiletypeAssociationsByIds(appIds), + ]); + + return rows.map((row) => + this.#shapeRow(row, { + usersById, + entriesById, + appsById, + filetypesByAppId, + }), + ); + } + + #shapeRow( + row: Record, + lookups: { + usersById: Map; + entriesById: Map; + appsById: Map>; + filetypesByAppId: Map; + }, + ): Record { + const ts = row.ts; + let createdAt: string | null = null; + if (ts != null) { + const d = ts instanceof Date ? ts : new Date(ts as string); + createdAt = Number.isNaN(d.getTime()) ? null : d.toISOString(); + } + + const ownerId = + typeof row.user_id === 'number' ? row.user_id : Number(row.user_id); + const owner = lookups.usersById.get(ownerId) ?? null; + + const rootDirId = + row.root_dir_id == null + ? null + : typeof row.root_dir_id === 'number' + ? row.root_dir_id + : Number(row.root_dir_id); + const rootEntry = + rootDirId != null + ? (lookups.entriesById.get(rootDirId) ?? null) + : null; + + const associatedAppRefId = + row.associated_app_id == null + ? null + : typeof row.associated_app_id === 'number' + ? row.associated_app_id + : Number(row.associated_app_id); + const associatedApp = + associatedAppRefId != null + ? (lookups.appsById.get(associatedAppRefId) ?? null) + : null; + + const appOwnerRefId = + row.app_owner == null + ? null + : typeof row.app_owner === 'number' + ? row.app_owner + : Number(row.app_owner); + const appOwnerApp = + appOwnerRefId != null + ? (lookups.appsById.get(appOwnerRefId) ?? null) + : null; + + return { + uid: row.uuid, + subdomain: row.subdomain, + // v1 sample emits `""` rather than null when no custom domain + // is set; the mapping declares `domain` as a string column. + domain: typeof row.domain === 'string' ? row.domain : '', + root_dir: rootEntry ? mapEntryToSubdomainRootDir(rootEntry) : null, + associated_app: associatedApp + ? mapAppForEmbed( + associatedApp, + lookups.filetypesByAppId.get(associatedAppRefId!) ?? [], + ) + : null, + created_at: createdAt, + owner: owner + ? { username: owner.username, uuid: owner.uuid } + : null, + app_owner: appOwnerApp + ? mapAppForEmbed( + appOwnerApp, + lookups.filetypesByAppId.get(appOwnerRefId!) ?? [], + ) + : null, + protected: Boolean(row.protected), + }; + } +} + +// ── Embed shape helpers (module-level, sync, no DB) ───────────────── +// +// `root_dir` mirrors v1's `safe_entry` from FSNodeContext, minus the +// fields v1 deletes before sending to clients (`user_id`, `bucket`, +// `bucket_region`). The legacy entry helper lives at +// controllers/fs/legacyFsHelpers.ts and is async (does an +// `is_empty` probe + owner fetch + thumbnail rewrite); subdomains +// don't need any of that, so we reshape inline. + +function mapEntryToSubdomainRootDir(entry: FSEntry): Record { + const dirname = pathPosix.dirname(entry.path); + return { + id: entry.uuid, + uid: entry.uuid, + parent_id: entry.parentUid, + parent_uid: entry.parentUid, + public_token: entry.publicToken, + file_request_token: entry.fileRequestToken, + is_dir: Boolean(entry.isDir), + is_public: entry.isPublic, + is_shortcut: entry.isShortcut ? 1 : 0, + is_symlink: entry.isSymlink ? 1 : 0, + symlink_path: entry.symlinkPath, + sort_by: entry.sortBy, + sort_order: entry.sortOrder, + immutable: entry.immutable ? 1 : 0, + name: entry.name, + metadata: entry.metadata, + modified: entry.modified, + created: entry.created, + accessed: entry.accessed, + size: entry.size, + layout: entry.layout, + path: entry.path, + dirname, + dirpath: dirname, + // v1 attaches an ACL-resolved `writable` here; the subdomain + // owner can always write to their own root_dir, and cross-user + // reads via `read-all-subdomains` aren't expected to mutate, so + // a constant `true` matches v1's behaviour for the typical case + // without a per-row ACL probe. + writable: true, + subdomains: entry.subdomains ?? [], + workers: entry.workers ?? [], + has_website: entry.hasWebsite ?? (entry.subdomains?.length ?? 0) > 0, + }; +} + +/** + * Embed shape for nested app references (`associated_app`, `app_owner`). + * Follows v1's AppES read shape minus the per-app async work + * (`created_from_origin`, private-app gating) — those are top-level-read + * concerns, not relevant for an app embed inside a subdomain row. + */ +function mapAppForEmbed( + app: Record, + filetypes: string[], +): Record { + return { + uid: app.uid, + name: app.name, + title: app.title, + description: app.description, + icon: app.icon, + index_url: app.index_url, + background: Boolean(app.background), + maximize_on_start: Boolean(app.maximize_on_start), + is_private: Boolean(app.is_private), + protected: Boolean(app.protected), + approved_for_listing: Boolean(app.approved_for_listing), + approved_for_opening_items: Boolean(app.approved_for_opening_items), + approved_for_incentive_program: Boolean( + app.approved_for_incentive_program, + ), + metadata: app.metadata ?? null, + filetype_associations: filetypes, + created_at: app.created_at ?? app.timestamp ?? null, + }; +} diff --git a/src/backend/drivers/types.ts b/src/backend/drivers/types.ts new file mode 100644 index 000000000..9fc0d8cee --- /dev/null +++ b/src/backend/drivers/types.ts @@ -0,0 +1,71 @@ +import type { puterClients } from '../clients'; +import type { puterServices } from '../services'; +import type { puterStores } from '../stores'; +import type { IConfig, LayerInstances, WithCostsReporting } from '../types'; + +export type IPuterDriver = + new ( + config: IConfig, + clients: LayerInstances, + stores: LayerInstances, + services: LayerInstances, + ) => T; + +/** + * Base class for v2 drivers. + * + * A driver implements a named interface (e.g., `puter-chat-completion`) and + * exposes methods that match the interface contract. Multiple drivers can + * implement the same interface (e.g., `openai-completion` and `claude` both + * implement `puter-chat-completion`). + * + * **Two ways to declare a driver:** + * + * 1. Decorator: + * ```ts + * @Driver('puter-chat-completion', { name: 'openai', default: true }) + * class OpenAIChat extends PuterDriver { ... } + * ``` + * + * 2. Imperative (no decorator): + * ```ts + * class OpenAIChat extends PuterDriver { + * readonly driverInterface = 'puter-chat-completion'; + * readonly driverName = 'openai'; + * readonly isDefault = true; + * } + * ``` + */ +export const PuterDriver = class PuterDriver implements WithCostsReporting { + /** The interface this driver implements. Set by `@Driver` or override. */ + declare readonly driverInterface?: string; + /** Unique name within its interface. Set by `@Driver` or override. */ + declare readonly driverName?: string; + /** When true, this is the default driver for its interface. */ + declare readonly isDefault?: boolean; + + constructor( + protected config: IConfig, + protected clients: LayerInstances, + protected stores: LayerInstances, + protected services: LayerInstances, + ) {} + public onServerStart() { + return; + } + public onServerPrepareShutdown() { + return; + } + public onServerShutdown() { + return; + } + public getReportedCosts(): Record[] { + return []; + } +} satisfies IPuterDriver; + +export type IPuterDriverRegistry = Record< + string, + | IPuterDriver + | (InstanceType> & Record) +>; diff --git a/src/backend/drivers/util/fileInput.ts b/src/backend/drivers/util/fileInput.ts new file mode 100644 index 000000000..9fd7c65f2 --- /dev/null +++ b/src/backend/drivers/util/fileInput.ts @@ -0,0 +1,217 @@ +import { posix as pathPosix } from 'node:path'; +import type { Actor } from '../../core/actor.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import type { FSService } from '../../services/fs/FSService.js'; +import { expandTildePath, resolveNode } from '../../services/fs/resolveNode.js'; +import type { FSEntryStore } from '../../stores/fs/FSEntryStore.js'; +import type { S3ObjectStore } from '../../stores/fs/S3ObjectStore.js'; +import { mimeFromName } from '../../util/fileSigning.js'; + +/** + * Resolve a file-like input sent through the drivers API into a Buffer. + * + * puter-js sends driver args as plain JSON (no multipart). `audio`, `source`, + * and similar file fields arrive as one of: + * • a data URL string (`data:image/png;base64,...`) + * • a plain path string (`/alice/music/sample.mp3`) + * • an object with `{ path?, uid?, uuid? }` + * + * This helper collapses those shapes into `{ buffer, filename, mimeType }`. + */ + +export interface LoadedFile { + buffer: Buffer; + filename: string; + mimeType: string | null; + // When the input was an FS reference (path/uid), carries the entry back + // so drivers can do FS-specific things (e.g. S3 CopyObject for OCR) — null + // for data-URL inputs. + fsEntry: { + uuid: string; + path: string; + bucket: string | null; + bucketRegion: string | null; + size: number | null; + sqlId: number | null; // null in case of base64 URL or a future dynamodb FS. + } | null; +} + +const DATA_URL_PATTERN = /^data:([^;,]+)?(?:;([^,]*))?,(.*)$/s; + +export async function loadFileInput( + stores: { fsEntry: FSEntryStore; s3Object: S3ObjectStore }, + fsService: FSService, + actor: Actor, + input: unknown, + options: { maxBytes?: number } = {}, +): Promise { + if (!input) { + throw new HttpError(400, 'Missing file input'); + } + if (!Number.isFinite(Number(actor?.user?.id ?? NaN))) { + throw new HttpError(401, 'Unauthorized'); + } + + // Data URL — decode base64/plain inline. + if (typeof input === 'string' && input.startsWith('data:')) { + const match = DATA_URL_PATTERN.exec(input); + if (!match) throw new HttpError(400, 'Invalid data URL'); + const mime = match[1] ?? 'application/octet-stream'; + const encoding = (match[2] ?? '').trim(); + const payload = match[3] ?? ''; + const buffer = + encoding.toLowerCase() === 'base64' + ? Buffer.from(payload, 'base64') + : Buffer.from(decodeURIComponent(payload)); + assertMax(buffer, options.maxBytes); + return { + buffer, + filename: filenameFromMime(mime), + mimeType: mime, + fsEntry: null, + }; + } + + // Path string or object reference → resolve into FSEntry, then S3 read. + const username = actor?.user?.username; + const expandPath = (path: string | undefined) => + path !== undefined ? expandTildePath(path, username) : undefined; + const ref: { path?: string; uid?: string; uuid?: string } = + typeof input === 'string' + ? { path: expandPath(input) } + : (() => { + const record = input as Record; + return { + path: expandPath( + typeof record.path === 'string' + ? record.path + : undefined, + ), + uid: + typeof record.uid === 'string' + ? record.uid + : undefined, + uuid: + typeof record.uuid === 'string' + ? record.uuid + : undefined, + }; + })(); + + const entry = await resolveNode(stores.fsEntry, ref, { required: true }); + if (!entry) throw new HttpError(404, 'File not found'); + if (entry.isDir) + throw new HttpError(400, 'Expected a file, got a directory'); + if (entry.isShortcut || entry.isSymlink) { + throw new HttpError( + 400, + 'Cannot load content of a symlink or shortcut directly', + ); + } + // ACL gate: resolveNode does global UID/UUID/ID/path lookups, no + // namespace check. Without this check, an attacker controlling + // `path`/`uid`/`uuid` (e.g. AI chat `puter_path` content parts) could + // exfiltrate any user's file. Must run before the S3 read below. + await fsService.checkFSAccess(entry, actor, 'read'); + // S3 object key is recorded in entry.metadata.objectKey when written by + // fsv2; older rows fall back to the entry uuid. + const objectKey = deriveObjectKey(entry); + const { body, contentType, contentLength } = + await stores.s3Object.getObjectStream( + { + bucket: stores.s3Object.resolveBucket(entry.bucket), + objectKey, + }, + stores.s3Object.resolveRegion(entry.bucketRegion), + ); + if (contentLength && options.maxBytes && contentLength > options.maxBytes) { + body.destroy(); + throw new HttpError( + 413, + `File exceeds max size (${options.maxBytes} bytes)`, + ); + } + + const chunks: Buffer[] = []; + let total = 0; + for await (const chunk of body) { + const buf = Buffer.isBuffer(chunk) + ? chunk + : Buffer.from(chunk as Uint8Array); + total += buf.byteLength; + if (options.maxBytes && total > options.maxBytes) { + body.destroy(); + throw new HttpError( + 413, + `File exceeds max size (${options.maxBytes} bytes)`, + ); + } + chunks.push(buf); + } + const buffer = Buffer.concat(chunks, total); + const resolvedMime = + contentType ?? mimeFromName(entry.name) ?? 'application/octet-stream'; + + return { + buffer, + filename: entry.name, + mimeType: resolvedMime, + fsEntry: { + uuid: entry.uuid, + path: entry.path, + bucket: entry.bucket, + bucketRegion: entry.bucketRegion, + size: entry.size, + sqlId: entry.id, + }, + }; +} + +function assertMax(buffer: Buffer, maxBytes?: number): void { + if (maxBytes && buffer.byteLength > maxBytes) { + throw new HttpError(413, `Input exceeds max size (${maxBytes} bytes)`); + } +} + +function filenameFromMime(mime: string): string { + const ext = mime.split('/')[1]?.split('+')[0] ?? 'bin'; + return `input.${ext}`; +} + +// Mirrors FSService's private deriveObjectKeyFromEntry helper. fsv2-era +// rows persist an `objectKey` in metadata; older rows simply use the uuid. +function deriveObjectKey(entry: { + uuid: string; + metadata: string | null; +}): string { + if (entry.metadata) { + try { + const parsed = JSON.parse(entry.metadata); + if ( + parsed && + typeof parsed.objectKey === 'string' && + parsed.objectKey.length > 0 + ) { + return parsed.objectKey; + } + } catch { + // Not JSON — fall through. + } + } + return entry.uuid; +} + +export function inferFilenameFromUrlOrPath( + value: string, + fallback = 'input', +): string { + try { + const url = new URL(value); + const basename = pathPosix.basename(url.pathname); + if (basename) return basename; + } catch { + // Not a URL; try treating as a file path. + } + const basename = pathPosix.basename(value); + return basename || fallback; +} diff --git a/src/backend/drivers/workers/WorkerDriver.ts b/src/backend/drivers/workers/WorkerDriver.ts new file mode 100644 index 000000000..a55137325 --- /dev/null +++ b/src/backend/drivers/workers/WorkerDriver.ts @@ -0,0 +1,557 @@ +import { readFileSync } from 'node:fs'; +import { Context } from '../../core/context.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import { PuterDriver } from '../types.js'; +import { loadFileInput } from '../util/fileInput.js'; +import type { Actor } from '../../core/actor.js'; +import path from 'node:path'; + +const CF_BASE_URL = 'https://api.cloudflare.com/client/v4/accounts'; +const WORKER_NAME_REGEX = /^[a-zA-Z0-9_-]+$/; +const MAX_WORKERS_PER_USER = 100; +const MAX_SOURCE_SIZE = 10 * 1024 * 1024; // 10 MB +const WORKER_SUBDOMAIN_PREFIX = 'workers.puter.'; + +// ── Preamble ──────────────────────────────────────────────────────── +// +// The preamble is a webpack-built JS bundle that provides puter.js to +// worker code. It's baked into the source sent to Cloudflare Workers. +// If the file hasn't been built, workers run without puter.js access. + +let preamble = ''; +let preambleLineCount = 0; +try { + const preamblePath = path.join( + __dirname, + '../../../../../src/worker/dist/workerPreamble.js', + ); + console.log('reading: ' + preamblePath); + preamble = readFileSync(preamblePath, 'utf-8'); + preambleLineCount = preamble.split('\n').length - 1; +} catch { + console.warn( + '[workers] preamble not built — workers will not have puter.js injected.', + ); +} + +/** + * Driver exposing the `workers` interface — Cloudflare Workers + * deployment, lifecycle, and file-path queries. + * + * Each "worker" is a JS file in the user's Puter FS, deployed to + * Cloudflare Workers. A corresponding `subdomains` row with subdomain + * `workers.puter.` ties the worker to its source file. + * + * Config: `config.workers.{XAUTHKEY, ACCOUNTID, namespace?, internetExposedUrl?, loggingUrl?}`. + */ +export class WorkerDriver extends PuterDriver { + readonly driverInterface = 'workers'; + // puter-js calls this as `workers:worker-service` (see Workers.js). Keep the name aligned. + readonly driverName = 'worker-service'; + readonly isDefault = true; + + #cfBaseUrl = ''; + + override onServerStart(): void { + const cfg = this.#workerConfig(); + if (cfg.ACCOUNTID) { + this.#cfBaseUrl = `${CF_BASE_URL}/${cfg.ACCOUNTID}/workers`; + if (cfg.namespace) { + this.#cfBaseUrl += `/dispatch/namespaces/${cfg.namespace}`; + } + } + this.#subscribeHotReload(); + } + + // ── Driver methods ────────────────────────────────────────────── + + async create(args: Record): Promise { + const actor = this.#requireActor(); + const workerName = String(args.workerName ?? '').toLowerCase(); + const filePath = String(args.filePath ?? ''); + const appId = args.appId || (actor.app?.id as number | undefined); + if (!workerName) throw new HttpError(400, 'Missing `workerName`'); + if (!filePath) throw new HttpError(400, 'Missing `filePath`'); + if (!WORKER_NAME_REGEX.test(workerName)) { + throw new HttpError( + 400, + 'Worker name must be alphanumeric (plus _ and -)', + ); + } + this.#rejectReserved(workerName); + this.#requireCfConfig(); + const subdomainName = `${WORKER_SUBDOMAIN_PREFIX}${workerName}`; + + // Quota check — count existing workers.puter.* subdomains owned by user + const existingWorkers = + await this.stores.subdomain.listByUserIdAndPrefix( + actor.user.id, + WORKER_SUBDOMAIN_PREFIX, + ); + if (existingWorkers.length >= MAX_WORKERS_PER_USER) { + throw new HttpError( + 403, + `Worker limit reached (max ${MAX_WORKERS_PER_USER})`, + ); + } + + // If tied to an app, verify ownership and get app-scoped token + let authorization = String(args.authorization ?? ''); + let appOwnerId = actor.app?.id ?? null; + if (appId) { + if (actor.app && actor.app.id !== appId) { + throw new HttpError( + 403, + 'Cannot deploy worker for another app', + ); + } + appOwnerId = actor.app?.id; + authorization = this.services.auth.getUserAppToken(actor, appId); + } + if (!authorization && actor.app?.uid) { + authorization = this.services.auth.getUserAppToken( + actor, + actor.app.uid, + ); + } + + const existingSub = + await this.stores.subdomain.getBySubdomain(subdomainName); + if (existingSub) { + this.#checkWorkerWriteAccess( + existingSub, + actor, + 409, + 'Worker name is already in use', + ); + } + if (!authorization) { + // Fall back to a session token for the current user + const userRow = await this.stores.user.getById(actor.user.id!); + if (!userRow) throw new HttpError(500, 'User not found'); + const session = + await this.services.auth.createSessionToken(userRow); + authorization = session.token; + } + + // Read source file. loadFileInput runs the read-ACL check internally + // before pulling bytes from S3. + const loaded = await loadFileInput( + { fsEntry: this.stores.fsEntry, s3Object: this.stores.s3Object }, + this.services.fs, + actor, + filePath, + { maxBytes: MAX_SOURCE_SIZE }, + ); + const sourceCode = loaded.buffer.toString('utf-8'); + + // Create subdomain entry + if (existingSub) { + // Update root_dir if worker already exists + const updated = await this.stores.subdomain.update( + String(existingSub.uuid), + { + root_dir_id: loaded.fsEntry?.sqlId ?? null, + }, + { userId: actor.user.id }, + ); + if (!updated) { + throw new HttpError(409, 'Worker name is already in use'); + } + } else { + if (!loaded.fsEntry?.sqlId) + throw new HttpError(400, `Invalid file recieved!`); + await this.stores.subdomain.create({ + userId: actor.user.id!, + subdomain: subdomainName, + rootDirId: loaded.fsEntry?.sqlId, + appOwner: appOwnerId, + }); + } + + // Deploy to Cloudflare + const cfResult = await this.#cfDeploy( + workerName, + authorization, + preamble + sourceCode, + ); + return cfResult; + } + + async destroy(args: Record): Promise { + const actor = this.#requireActor(); + const workerName = String(args.workerName ?? '').toLowerCase(); + if (!workerName) throw new HttpError(400, 'Missing `workerName`'); + this.#requireCfConfig(); + + const subdomainName = `${WORKER_SUBDOMAIN_PREFIX}${workerName}`; + const row = await this.stores.subdomain.getBySubdomain(subdomainName); + if (!row) throw new HttpError(404, 'Worker not found'); + this.#checkWorkerWriteAccess( + row, + actor, + 403, + 'This is not your worker', + ); + + const cfResult = await this.#cfDelete(workerName); + await this.stores.subdomain.deleteByUuid(row.uuid, { + userId: actor.user.id, + }); + return cfResult; + } + + async getFilePaths(args: Record): Promise { + const actor = this.#requireActor(); + const workerName = args.workerName as string | undefined; + + let rows: Array>; + if (typeof workerName === 'string' && workerName.length > 0) { + const sub = await this.stores.subdomain.getBySubdomain( + `${WORKER_SUBDOMAIN_PREFIX}${workerName}`, + ); + rows = sub ? [sub] : []; + } else { + rows = await this.stores.subdomain.listByUserIdAndPrefix( + actor.user.id, + WORKER_SUBDOMAIN_PREFIX, + actor.app ? { appId: actor.app.id } : {}, + ); + } + + const rootDirIds = rows + .map((r) => r.root_dir_id) + .filter((id): id is number => typeof id === 'number'); + const entriesById = + await this.stores.fsEntry.getEntriesByIds(rootDirIds); + + // Make sure the user only sees their own workers + rows = rows.filter((r) => { + return r.user_id === actor.user.id; + }); + if (actor.app) { + rows = rows.filter((r) => { + return r.app_owner === actor.app?.id; + }); + } + + return rows.map((r) => { + const name = + String(r.subdomain ?? '') + .split('.') + .pop() ?? ''; + let file_path = null; + let file_uid = null; + if (typeof r.root_dir_id === 'number') { + const loaded = entriesById.get(r.root_dir_id); + file_path = loaded?.path; + file_uid = loaded?.uuid; + } + return { + name, + url: `https://${name}.puter.work`, + file_path, + file_uid, + created_at: r.ts + ? new Date(r.ts as string).toISOString() + : null, + }; + }); + } + + async getLoggingUrl(): Promise { + return this.#workerConfig().loggingUrl ?? null; + } + + // ── Cloudflare API ────────────────────────────────────────────── + + async #cfDeploy( + workerName: string, + authorization: string, + code: string, + ): Promise> { + const cfg = this.#workerConfig(); + const metadata = JSON.stringify({ + body_part: 'swCode', + compatibility_flags: ['global_fetch_strictly_public'], + compatibility_date: '2025-07-15', + bindings: [ + { + type: 'secret_text', + name: 'puter_auth', + text: authorization, + }, + { + type: 'plain_text', + name: 'puter_endpoint', + text: cfg.internetExposedUrl ?? 'https://api.puter.com', + }, + ], + }); + + const form = new FormData(); + form.append('metadata', metadata); + form.append( + 'swCode', + new Blob([code], { type: 'application/javascript' }), + ); + + const res = await fetch(`${this.#cfBaseUrl}/scripts/${workerName}/`, { + method: 'PUT', + headers: { Authorization: `Bearer ${cfg.XAUTHKEY}` }, + body: form, + }); + const json = (await res.json()) as { + success?: boolean; + errors?: Array<{ message: string }>; + }; + + if (json.success) { + return { + success: true, + errors: [], + url: `https://${workerName}.puter.work`, + }; + } + + // Parse Cloudflare error stack traces to adjust for preamble offset + const errors = (json.errors ?? []).map((e) => { + const lines = e.message.split('\n'); + const header = lines.shift() ?? ''; + const adjusted = lines.map((line) => { + if (line.includes('at worker.js:')) { + const [before, after] = line.split('at worker.js:'); + const positions = after.split(':'); + positions[0] = String( + Number(positions[0]) - preambleLineCount, + ); + return `${before}at worker.js:${positions.join(':')}`; + } + return line; + }); + return `${header}\n${adjusted.join('\n')}`; + }); + return { success: false, errors, url: null }; + } + + async #cfDelete(workerName: string): Promise> { + const cfg = this.#workerConfig(); + const res = await fetch(`${this.#cfBaseUrl}/scripts/${workerName}/`, { + method: 'DELETE', + headers: { Authorization: `Bearer ${cfg.XAUTHKEY}` }, + }); + return (await res.json()) as Record; + } + + // ── Helpers ────────────────────────────────────────────────────── + + #requireActor(): Actor & { + user: { id: number; uuid: string; username: string }; + } { + const actor = Context.get('actor') as Actor | undefined; + if (!actor?.user?.id) + throw new HttpError(401, 'Authentication required'); + return actor as Actor & { + user: { id: number; uuid: string; username: string }; + }; + } + + #requireCfConfig(): void { + const cfg = this.#workerConfig(); + if (!cfg.XAUTHKEY || !cfg.ACCOUNTID) { + throw new HttpError(503, 'Cloudflare Workers not configured'); + } + } + + #rejectReserved(name: string): void { + const reserved = this.config.reserved_words ?? []; + if (reserved.includes(name)) { + throw new HttpError(400, `Worker name '${name}' is reserved`); + } + } + + #checkWorkerWriteAccess( + row: Record, + actor: Actor & { user: { id: number } }, + errorStatus: number, + errorMessage: string, + ): void { + if (Number(row.user_id) !== actor.user.id) { + throw new HttpError(errorStatus, errorMessage); + } + + if (!actor.app) return; + + const actorAppId = actor.app.id; + const workerAppOwnerId = + row.app_owner === null || row.app_owner === undefined + ? null + : Number(row.app_owner); + if (!actorAppId || workerAppOwnerId !== actorAppId) { + throw new HttpError(errorStatus, errorMessage); + } + } + + #workerConfig(): NonNullable { + return this.config.workers ?? {}; + } + + // ── Hot-reload: auto-redeploy on file write ───────────────────── + // + // When a user saves a JS file that's tied to a worker subdomain, + // we redeploy it to Cloudflare automatically. This is what makes + // "save file → live in prod" instant. + // + // The FS layer emits `outer.gui.item.added` and + // `outer.gui.item.updated` after a write commits. We subscribe to + // those — the payload carries `{ user_id_list, response }` where + // `response` is the entry shape (uuid, path, user_id, etc.). We + // match against worker subdomain `root_dir_id` to decide whether + // to re-deploy. + + #subscribeHotReload(): void { + if (!this.#cfBaseUrl) return; // CF not configured — skip + + for (const eventName of [ + 'outer.gui.item.added', + 'outer.gui.item.updated', + ]) { + this.clients.event.on( + eventName, + (key: string, data: unknown, meta: unknown) => { + void this.#handleFileWrite(data, meta).catch((err) => { + console.error('[workers] hot-reload error', err); + }); + }, + ); + } + } + + async #handleFileWrite(data: unknown, meta: unknown): Promise { + const metaObj = + meta && typeof meta === 'object' + ? (meta as Record) + : {}; + // Only run on the local node — incoming broadcast writes shouldn't trigger a re-deploy + if (metaObj.from_outside) return; + + const d = data as Record | undefined; + if (!d) return; + + // `outer.gui.item.*` events carry `{ user_id_list, response }` + // where `response` is the FS entry shape. Extract what we need. + const response = (d.response ?? d) as Record; + const userIdList = d.user_id_list as Array | undefined; + + const uuid = (response.uuid ?? response.uid) as string | undefined; + const userId = (userIdList?.[0] ?? response.user_id) as + | number + | undefined; + const path = response.path as string | undefined; + + // Only files trigger hot-reload (not directories) + if (response.is_dir || response.isDir) return; + if (!uuid || !userId) return; + + // Check if any worker subdomain points at this file + const workerSubs = await this.stores.subdomain.listByUserIdAndPrefix( + userId, + WORKER_SUBDOMAIN_PREFIX, + ); + const matched = workerSubs.filter((r: Record) => { + // root_dir_id can be the FS entry id or uuid depending on how it was stored + return ( + String(r.root_dir_id) === String(uuid) || + String(r.root_dir_id) === String(response.id) + ); + }); + + if (matched.length === 0) return; + + for (const row of matched) { + const workerFullName = String(row.subdomain ?? ''); + if (!workerFullName.startsWith(WORKER_SUBDOMAIN_PREFIX)) continue; + const workerName = workerFullName.slice( + WORKER_SUBDOMAIN_PREFIX.length, + ); + + try { + const ownerUser = await this.stores.user.getById(userId); + if (!ownerUser) continue; + const ownerActor = { user: ownerUser } as Actor; + + // Read the updated file content. `ownerActor` is the file's + // owner from the originating write event, so the read-ACL + // check inside loadFileInput will pass. + const loaded = await loadFileInput( + { + fsEntry: this.stores.fsEntry, + s3Object: this.stores.s3Object, + }, + this.services.fs, + ownerActor, + path ?? uuid, // prefer path, fall back to uuid + { maxBytes: MAX_SOURCE_SIZE }, + ); + const sourceCode = loaded.buffer.toString('utf-8'); + + // Get an auth token for the deploy + const appOwnerId = row.app_owner as number | null; + let authorization: string; + if (appOwnerId) { + // App-scoped: get the app's uid, then mint an app-under-user token + const app = await this.stores.app.getById(appOwnerId); + if (!app) continue; // app gone + authorization = this.services.auth.getUserAppToken( + ownerActor, + app.uid, + ); + } else { + // User-scoped: mint a session token + const session = + await this.services.auth.createSessionToken(ownerUser); + authorization = session.token; + } + + // Deploy + const cfResult = (await this.#cfDeploy( + workerName, + authorization, + preamble + sourceCode, + )) as { success?: boolean; errors?: unknown[]; url?: string }; + + // Notify the user + await this.#notifyUser(userId, workerName, cfResult); + } catch (err) { + console.warn( + `[workers] hot-reload deploy failed for ${workerName}`, + err, + ); + await this.#notifyUser(userId, workerName, { + success: false, + errors: [String(err)], + }); + } + } + } + + async #notifyUser( + userId: number, + workerName: string, + result: { success?: boolean; errors?: unknown[]; url?: string }, + ): Promise { + try { + const title = result.success + ? `Successfully deployed https://${workerName}.puter.work` + : `Failed to deploy ${workerName}! ${(result.errors ?? []).join(', ')}`; + + await this.services.notification.notify([userId], { + source: 'worker', + title, + template: 'user-requesting-share', + }); + } catch (err) { + console.warn('[workers] notification create failed', err); + } + } +} diff --git a/src/backend/exports.js b/src/backend/exports.js deleted file mode 100644 index b958d14c6..000000000 --- a/src/backend/exports.js +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import CoreModule from './src/CoreModule.js'; -import DatabaseModule from './src/DatabaseModule.js'; -import { Kernel } from './src/Kernel.js'; -import { AppsModule } from './src/modules/apps/AppsModule.js'; -import { BroadcastModule } from './src/modules/broadcast/BroadcastModule.js'; -import { CaptchaModule } from './src/modules/captcha/CaptchaModule.js'; -import { Core2Module } from './src/modules/core/Core2Module.js'; -import { DataAccessModule } from './src/modules/data-access/DataAccessModule.js'; -import { EntityStoreModule } from './src/modules/entitystore/EntityStoreModule.js'; -import { HostOSModule } from './src/modules/hostos/HostOSModule.js'; -import { InternetModule } from './src/modules/internet/InternetModule.js'; -import { KVStoreModule } from './src/modules/kvstore/KVStoreModule.js'; -import { PuterFSModule } from './src/modules/puterfs/PuterFSModule.js'; -import SelfHostedModule from './src/modules/selfhosted/SelfHostedModule.js'; -import { TestConfigModule } from './src/modules/test-config/TestConfigModule.js'; -import { TestDriversModule } from './src/modules/test-drivers/TestDriversModule.js'; -import { WebModule } from './src/modules/web/WebModule.js'; -import BaseService from './src/services/BaseService.js'; -import { Context } from './src/util/context.js'; - -export default { - - // Kernel API - BaseService, - Context, - - Kernel, - - EssentialModules: [ - Core2Module, - PuterFSModule, - HostOSModule, - CoreModule, - WebModule, - // TemplateModule, - AppsModule, - CaptchaModule, - EntityStoreModule, - KVStoreModule, - DataAccessModule, - ], - - // Pre-built modules - CoreModule, - WebModule, - DatabaseModule, - SelfHostedModule, - TestDriversModule, - TestConfigModule, - BroadcastModule, - InternetModule, - CaptchaModule, - KVStoreModule, -}; diff --git a/src/backend/exports.ts b/src/backend/exports.ts new file mode 100644 index 000000000..769cf57a0 --- /dev/null +++ b/src/backend/exports.ts @@ -0,0 +1,15 @@ +import type { IPuterClientRegistry } from './clients/types'; +import type { IPuterControllerRegistry } from './controllers/types'; +import type { IPuterDriverRegistry } from './drivers/types'; +import type { IPuterServiceRegistry } from './services/types'; +import type { IPuterStoreRegistry } from './stores/types'; +import type { IConfig, LayerInstances } from './types'; + +export const configContainer: IConfig = {} as IConfig; + +export const clientsContainers: LayerInstances = {}; +export const storesContainers: LayerInstances = {}; +export const servicesContainers: LayerInstances = {}; +export const controllersContainers: LayerInstances = + {}; +export const driversContainers: LayerInstances = {}; diff --git a/src/backend/extensions.ts b/src/backend/extensions.ts new file mode 100644 index 000000000..11d2f27c5 --- /dev/null +++ b/src/backend/extensions.ts @@ -0,0 +1,337 @@ +import type { RequestHandler } from 'express'; +import type { puterClients } from './clients'; +import type { IPuterClientRegistry } from './clients/types'; +import type { puterControllers } from './controllers'; +import type { IPuterControllerRegistry } from './controllers/types'; +import type { + RouteDescriptor, + RouteMethod, + RouteOptions, + RoutePath, +} from './core/http/types'; +import type { puterDrivers } from './drivers'; +import type { IPuterDriverRegistry } from './drivers/types'; +import { + clientsContainers, + configContainer, + controllersContainers, + driversContainers, + servicesContainers, + storesContainers, +} from './exports'; +import type { puterServices } from './services'; +import type { IPuterServiceRegistry } from './services/types'; +import type { puterStores } from './stores'; +import type { IPuterStoreRegistry } from './stores/types'; +import type { IConfig, LayerInstances } from './types'; + +/** + * The in-memory registry an extension's module-scope code writes into, and + * that `PuterServer` drains during boot. Every field is optional at write + * time — an extension that only needs routes never touches the registries. + */ +export const extensionStore = { + clients: {} as IPuterClientRegistry, + stores: {} as IPuterStoreRegistry, + services: {} as IPuterServiceRegistry, + controllers: {} as IPuterControllerRegistry, + drivers: {} as IPuterDriverRegistry, + globalMiddlewares: [] as RequestHandler[], + // eslint-disable-next-line @typescript-eslint/no-explicit-any + events: {} as Record void)[]>, + /** + * Extension-declared routes. Shape matches the controller-layer + * `RouteDescriptor`, so both flow through the same materializer + * (`PuterServer#materializeRoute`) and inherit the same options → + * middleware translation (subdomain, auth, body parsers, ...). + */ + routeHandlers: [] as RouteDescriptor[], +}; + +/** + * Internal: normalize `(path, handler)` or `(path, options, handler)` into + * a single `RouteDescriptor` the server can materialize. + */ +const pushRoute = ( + method: RouteMethod, + path: RoutePath, + optionsOrHandler: RouteOptions | RequestHandler, + maybeHandler?: RequestHandler, +): void => { + const handler = + typeof optionsOrHandler === 'function' + ? optionsOrHandler + : maybeHandler; + const options = + typeof optionsOrHandler === 'function' ? {} : optionsOrHandler; + if (!handler) { + throw new Error( + `extension.${method}('${String(path)}', ...) missing handler`, + ); + } + extensionStore.routeHandlers.push({ method, path, options, handler }); +}; + +interface ExtensionRouteFn { + (path: RoutePath, handler: RequestHandler): void; + (path: RoutePath, options: RouteOptions, handler: RequestHandler): void; +} + +const makeRouteFn = (method: RouteMethod): ExtensionRouteFn => { + return (( + path: RoutePath, + optionsOrHandler: RouteOptions | RequestHandler, + maybeHandler?: RequestHandler, + ) => { + pushRoute(method, path, optionsOrHandler, maybeHandler); + }) as ExtensionRouteFn; +}; + +/** + * `extension.use` mirrors `app.use` and supports three shapes: + * use(handler) + * use(options, handler) + * use(path, handler) + * use(path, options, handler) + * Pathless calls register global middleware — the server materializer + * drops the path when calling `app.use` (see `RouteDescriptor.path?`). + */ +interface ExtensionUseFn { + (handler: RequestHandler): void; + (options: RouteOptions, handler: RequestHandler): void; + (path: RoutePath, handler: RequestHandler): void; + (path: RoutePath, options: RouteOptions, handler: RequestHandler): void; +} + +const isRequestHandler = (v: unknown): v is RequestHandler => + typeof v === 'function'; + +const isRoutePath = (v: unknown): v is RoutePath => + typeof v === 'string' || v instanceof RegExp || Array.isArray(v); + +const makeUseFn = (): ExtensionUseFn => { + return (( + a: RoutePath | RouteOptions | RequestHandler, + b?: RouteOptions | RequestHandler, + c?: RequestHandler, + ): void => { + let path: RoutePath | undefined; + let options: RouteOptions = {}; + let handler: RequestHandler | undefined; + + if (isRoutePath(a)) { + path = a; + if (isRequestHandler(b)) { + handler = b; + } else { + options = (b as RouteOptions) ?? {}; + handler = c; + } + } else if (isRequestHandler(a)) { + handler = a; + } else { + options = (a as RouteOptions) ?? {}; + handler = isRequestHandler(b) ? b : undefined; + } + + if (!handler) { + throw new Error('extension.use(...) missing handler'); + } + extensionStore.routeHandlers.push({ + method: 'use', + ...(path !== undefined ? { path } : {}), + options, + handler, + }); + }) as ExtensionUseFn; +}; + +/** + * Global `extension` API available inside every dynamically-loaded extension + * module. Exposes: + * + * - Registry writers: `registerClient`, `registerStore`, `registerService`, + * `registerController`, `registerDriver`. + * - Event subscription: `on(event, handler)`. + * - Imperative route registration: `get`, `post`, `put`, `delete`, `patch`, + * `head`, `options`, `all`, `use`. Each accepts the same `RouteOptions` + * vocabulary used by controllers (subdomain, requireAuth, bodyJson, …) + * so extension routes get identical gate + parser treatment. + * - Back-reference lookup: `import('service:foo')` / `'client:bar'` / + * `'store:baz'` / `'controller:qux'` / `'driver:fred'` — returns a lazy + * proxy to the registered instance (thrown on use-before-init). + */ +export const extension = { + // ── Config access ─────────────────────────────────────────────── + // + // Lazy proxy to the server config. Populated by PuterServer during + // boot, so extensions can read it at request time (not import time). + + get config(): IConfig { + return configContainer; + }, + + // ── Event subscription ─────────────────────────────────────────── + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + on: (event: string, handler: (...args: any[]) => void) => { + if (!extensionStore.events[event]) { + extensionStore.events[event] = []; + } + extensionStore.events[event].push(handler); + }, + + // ── Registry writers ───────────────────────────────────────────── + + registerClient: ( + name: string, + client: IPuterClientRegistry[keyof IPuterClientRegistry], + ) => { + extensionStore.clients[name] = client; + }, + registerStore: ( + name: string, + store: IPuterStoreRegistry[keyof IPuterStoreRegistry], + ) => { + extensionStore.stores[name] = store; + }, + registerService: ( + name: string, + service: IPuterServiceRegistry[keyof IPuterServiceRegistry], + ) => { + extensionStore.services[name] = service; + }, + registerController: ( + name: string, + controller: IPuterControllerRegistry[keyof IPuterControllerRegistry], + ) => { + extensionStore.controllers[name] = controller; + }, + registerDriver: ( + name: string, + driver: IPuterDriverRegistry[keyof IPuterDriverRegistry], + ) => { + extensionStore.drivers[name] = driver; + }, + registerGlobalMiddleware: (middleware: RequestHandler) => { + extensionStore.globalMiddlewares.push(middleware); + }, + + // ── Route registration ─────────────────────────────────────────── + // + // Supports two call shapes per verb: + // extension.get('/path', handler) + // extension.get('/path', options, handler) + // + // The `options` object is the same `RouteOptions` shape controllers use, + // so everything that works on a controller route (subdomain, requireAuth, + // requireUserActor, adminOnly, allowedAppIds, middleware, bodyJson, + // bodyRaw, bodyText, bodyUrlencoded) works here identically. + + get: makeRouteFn('get'), + post: makeRouteFn('post'), + put: makeRouteFn('put'), + delete: makeRouteFn('delete'), + patch: makeRouteFn('patch'), + head: makeRouteFn('head'), + options: makeRouteFn('options'), + all: makeRouteFn('all'), + use: makeUseFn(), + + // ── Import proxy ───────────────────────────────────────────────── + + import: ( + name: S, + ): S extends 'client' + ? LayerInstances + : S extends 'store' + ? LayerInstances + : S extends 'service' + ? LayerInstances + : S extends 'controller' + ? LayerInstances + : S extends 'driver' + ? LayerInstances + : never => { + switch (name) { + case 'client': { + const proxyHandler = { + get: (_target: object, prop: string) => { + const proxiedObj = clientsContainers[prop]; + if (!proxiedObj) { + throw new Error( + `Called before initialization: ${name}.${prop}`, + ); + } + return proxiedObj; + }, + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return new Proxy({}, proxyHandler) as any; + } + case 'store': { + const proxyHandler = { + get: (_target: object, prop: string) => { + const proxiedObj = storesContainers[prop]; + if (!proxiedObj) { + throw new Error( + `Called before initialization: ${name}.${prop}`, + ); + } + return proxiedObj; + }, + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return new Proxy({}, proxyHandler) as any; + } + case 'service': { + const proxyHandler = { + get: (_target: object, prop: string) => { + const proxiedObj = servicesContainers[prop]; + if (!proxiedObj) { + throw new Error( + `Called before initialization: ${name}.${prop}`, + ); + } + return proxiedObj; + }, + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return new Proxy({}, proxyHandler) as any; + } + case 'controller': { + const proxyHandler = { + get: (_target: object, prop: string) => { + const proxiedObj = controllersContainers[prop]; + if (!proxiedObj) { + throw new Error( + `Called before initialization: ${name}.${prop}`, + ); + } + return proxiedObj; + }, + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return new Proxy({}, proxyHandler) as any; + } + case 'driver': { + const proxyHandler = { + get: (_target: object, prop: string) => { + const proxiedObj = driversContainers[prop]; + if (!proxiedObj) { + throw new Error( + `Called before initialization: ${name}.${prop}`, + ); + } + return proxiedObj; + }, + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return new Proxy({}, proxyHandler) as any; + } + default: + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return undefined as any; + } + }, +}; diff --git a/src/backend/index.ts b/src/backend/index.ts new file mode 100644 index 000000000..81124a584 --- /dev/null +++ b/src/backend/index.ts @@ -0,0 +1,173 @@ +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import { puterClients } from './clients'; +import { puterControllers } from './controllers'; +import { puterDrivers } from './drivers'; +import { PuterServer } from './server'; +import { puterServices } from './services'; +import { puterStores } from './stores'; +import type { IConfig } from './types'; + +// Config resolution order: +// 1. `process.env.PUTER_CONFIG_PATH` — absolute path to a config file. Used +// by prod (ECS/Docker) where the outer bootstrap writes a merged config +// out of Secrets Manager + container env to a known location. +// 2. `/config.json` — user's runtime override (gitignored), +// deep-merged over config.default.json so users can omit keys they +// don't care to override (e.g. gui_assets_root, database). +// 3. `/config.default.json` — bundled OSS defaults. +// +// Post-flatten depth: compiled file is at `packages/puter/dist/src/backend/index.js`, +// so three `..`s land at `packages/puter/`. +const PACKAGE_ROOT = path.resolve(__dirname, '../../..'); +// Root of the running code tree. Matches PACKAGE_ROOT for a source run, but +// points at `dist/` for a compiled run — so config-declared paths like +// `./extensions` resolve to `dist/extensions` at runtime without the config +// having to know about the build layout. +const RUNTIME_ROOT = path.resolve(__dirname, '../..'); + +const isPlainObject = (v: unknown): v is Record => + typeof v === 'object' && v !== null && !Array.isArray(v); + +const deepMerge = >( + base: T, + override: Record, +): T => { + const out: Record = { ...base }; + for (const [k, v] of Object.entries(override)) { + out[k] = + isPlainObject(v) && isPlainObject(out[k]) + ? deepMerge(out[k] as Record, v) + : v; + } + return out as T; +}; + +const loadConfig = (): IConfig => { + const envPath = process.env.PUTER_CONFIG_PATH; + const runtimePath = path.join(PACKAGE_ROOT, 'config.json'); + const defaultPath = path.join(PACKAGE_ROOT, 'config.default.json'); + + const defaults = existsSync(defaultPath) + ? (JSON.parse(readFileSync(defaultPath, 'utf8')) as Record< + string, + unknown + >) + : {}; + + // Runtime override path: env wins, then config.json, else no override + // (we still return defaults so single-file installs work). + const overridePath = + envPath && existsSync(envPath) + ? envPath + : existsSync(runtimePath) + ? runtimePath + : null; + + console.log(`[config] defaults from ${defaultPath}`); + if (overridePath) console.log(`[config] override from ${overridePath}`); + + const override = overridePath + ? (JSON.parse(readFileSync(overridePath, 'utf8')) as Record< + string, + unknown + >) + : {}; + + const config = deepMerge(defaults, override) as IConfig; + + if (!config.version) { + const pkgPath = path.join(PACKAGE_ROOT, 'package.json'); + if (existsSync(pkgPath)) { + try { + const pkg = JSON.parse(readFileSync(pkgPath, 'utf8')) as { + version?: string; + }; + if (pkg.version) config.version = pkg.version; + } catch { + // fall through — /version returns 'unknown' + } + } + } + + // Computed defaults. `origin` and `pub_port` are the externally-visible + // URL+port — what the browser sees. Separate from `port`, which is the + // bind port (can differ when behind a reverse proxy). Code paths that + // build self-referential URLs (GUI bootstrap, email links, OIDC callbacks) + // depend on `origin` having the right port baked in. + if (config.pub_port === undefined) config.pub_port = config.port; + const protocol = config.protocol ?? 'http'; + const domain = config.domain ?? 'localhost'; + const portSuffix = + config.pub_port === 80 || config.pub_port === 443 + ? '' + : `:${config.pub_port}`; + if (config.origin === undefined) { + config.origin = `${protocol}://${domain}${portSuffix}`; + } + // API lives on the `api.` subdomain on the same host+port as the main + // origin (see PuterRouter subdomain handling in server.ts). Compute it + // from pub_port/domain so a single-port override (e.g. port: 5101) flows + // through to the GUI bootstrap without users having to restate the URL. + if (config.api_base_url === undefined) { + config.api_base_url = `${protocol}://api.${domain}${portSuffix}`; + } + + // Resolve path-valued config fields. Two different roots: + // - `extensions` uses RUNTIME_ROOT because extensions ship inside the + // build output (dist/extensions) and the loader's dynamic import() + // resolves relative paths against the *importing* module file. + // - GUI/puter-js/builtin-apps use PACKAGE_ROOT because those assets + // live only in the source tree (not copied into dist/) and are served + // via express.static at runtime. + const resolveRuntime = (p: string): string => + path.isAbsolute(p) ? p : path.resolve(RUNTIME_ROOT, p); + const resolvePackage = (p: string): string => + path.isAbsolute(p) ? p : path.resolve(PACKAGE_ROOT, p); + + if (Array.isArray(config.extensions)) { + config.extensions = config.extensions.map(resolveRuntime); + } + if (typeof config.gui_assets_root === 'string') { + config.gui_assets_root = resolvePackage(config.gui_assets_root); + } + if (typeof config.puterjs_root === 'string') { + config.puterjs_root = resolvePackage(config.puterjs_root); + } + if (isPlainObject(config.builtin_apps)) { + for (const [k, v] of Object.entries(config.builtin_apps)) { + if (typeof v === 'string') { + (config.builtin_apps as Record)[k] = + resolvePackage(v); + } + } + } + return config; +}; + +// if called directly, start the server +if (require.main === module) { + const config = loadConfig(); + const server = new PuterServer( + config, + puterClients, + puterStores, + puterServices, + puterControllers, + puterDrivers, + ); + server.start(); + // listen for shutdown signals to gracefully stop the server + const shutDownProcess = async () => { + await server.prepareShutdown(); + setTimeout( + async () => { + await server.shutdown(); + process.exit(0); + }, + config.serverId ? 1000 * 90 : 1, + ); + }; + process.on('SIGINT', shutDownProcess); + process.on('SIGTERM', shutDownProcess); +} diff --git a/src/backend/package.json b/src/backend/package.json index 6cede797f..6503fad28 100644 --- a/src/backend/package.json +++ b/src/backend/package.json @@ -1,113 +1,84 @@ { "name": "@heyputer/backend", + "type": "module", "version": "2.5.1", "description": "Backend/Kernel for Puter", - "main": "exports.js", + "main": "exports.ts", "scripts": { - "test": "npx mocha src/**/*.test.js && node ./tools/test.mjs", - "bench": "vitest bench --config=vitest.bench.config.ts --run", - "build:worker": "cd src/services/worker && npm run build" + "test": "npx mocha '**/*.test.js' && node ./tools/test.mjs", + "bench": "vitest bench --config=vitest.bench.config.ts --run" }, "dependencies": { - "@aws-sdk/client-cloudwatch": "^3.940.0", - "@aws-sdk/client-polly": "^3.622.0", - "@aws-sdk/client-textract": "^3.621.0", - "@google/generative-ai": "^0.21.0", - "@heyputer/kv.js": "^0.1.9", - "@heyputer/multest": "^0.0.2", + "@anthropic-ai/sdk": "^0.68.0", + "@aws-sdk/client-dynamodb": "^3.490.0", + "@aws-sdk/client-polly": "^3.1028.0", + "@aws-sdk/client-s3": "^3.1028.0", + "@aws-sdk/client-textract": "^3.1028.0", + "@aws-sdk/s3-request-presigner": "^3.1028.0", + "@aws-sdk/credential-providers": "^3.1021.0", + "@aws-sdk/lib-dynamodb": "^3.490.0", + "@google/genai": "^1.19.0", + "@heyputer/kv.js": "^0.2.1", "@heyputer/putility": "^1.0.0", - "@mistralai/mistralai": "^1.3.4", - "@opentelemetry/api": "^1.4.1", - "@opentelemetry/auto-instrumentations-node": "^0.43.0", - "@opentelemetry/exporter-metrics-otlp-grpc": "^0.40.0", - "@opentelemetry/exporter-trace-otlp-grpc": "^0.40.0", - "@opentelemetry/sdk-metrics": "^1.14.0", - "@opentelemetry/sdk-node": "^0.49.1", + "@mistralai/mistralai": "^1.15.1", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/auto-instrumentations-node": "^0.52.0", + "@opentelemetry/exporter-metrics-otlp-grpc": "^0.55.0", + "@opentelemetry/exporter-trace-otlp-grpc": "^0.55.0", + "@opentelemetry/resources": "^1.28.0", + "@opentelemetry/sdk-metrics": "^1.28.0", + "@opentelemetry/sdk-node": "^0.55.0", + "@opentelemetry/sdk-trace-base": "^1.28.0", + "@opentelemetry/semantic-conventions": "^1.28.0", "@pagerduty/pdjs": "^2.2.4", - "@smithy/node-http-handler": "^2.2.2", + "@smithy/node-http-handler": "^2.5.0", "@socket.io/redis-streams-adapter": "^0.3.1", - "args": "^5.0.3", - "axios": "^1.8.2", - "bcrypt": "^5.1.0", + "axios": "^1.15.0", + "bcrypt": "^5.1.1", "better-sqlite3": "^12.6.0", "busboy": "^1.6.0", "chai-as-promised": "^7.1.1", "clean-css": "^5.3.2", - "composite-error": "^1.0.2", - "compression": "^1.7.4", - "convertapi": "^1.15.0", - "cookie-parser": "^1.4.6", + "compression": "^1.8.1", + "cookie-parser": "^1.4.7", "dedent": "^1.5.3", - "dns2": "^2.1.0", - "express": "^4.18.2", - "file-type": "^21.3.3", - "firebase-admin": "^10.3.0", - "form-data": "^4.0.0", + "dynalite": "^4.0.0", + "express": "^5.0.0", + "fauxqs": "^2.5.0", "groq-sdk": "^0.5.0", - "handlebars": "^4.7.8", - "helmet": "^7.0.0", + "handlebars": "^4.7.9", + "helmet": "^7.2.0", "hi-base32": "^0.5.1", "html-entities": "^2.3.3", - "ioredis": "^5.9.2", + "ioredis": "^5.10.1", "ioredis-mock": "^8.13.1", - "is-glob": "^4.0.3", - "isbot": "^3.7.1", - "jimp": "^1.6.0", - "js-sha256": "^0.9.0", - "json5": "^2.2.3", - "jsonwebtoken": "^9.0.0", - "knex": "^3.1.0", + "jsonwebtoken": "^9.0.3", "lorem-ipsum": "^2.0.8", - "lru-cache": "^11.0.2", - "micromatch": "^4.0.5", "mime-types": "^2.1.35", - "moment": "^2.29.4", - "morgan": "^1.10.0", - "multer": "^2.0.2", - "multi-progress": "^4.0.0", "murmurhash": "^2.0.1", - "music-metadata": "^11.12.3", - "nodemailer": "^7.0.7", - "on-finished": "^2.4.1", - "openai": "^6.7.0", - "otpauth": "9.2.4", + "mysql2": "^3.21.1", + "nodemailer": "^7.0.13", + "openai": "^6.34.0", + "otpauth": "^9.2.4", + "parse-domain": "^8.2.2", "prompt-sync": "^4.2.0", - "proxyquire": "^2.1.3", - "recursive-readdir": "^2.2.3", - "replicate": "^1.4.0", - "response-time": "^2.3.2", - "seedrandom": "^3.0.5", + "replicate": "^1.0.0", "sharp": "^0.34.3", - "sharp-bmp": "^0.1.5", - "sharp-ico": "^0.1.5", - "shescape": "^2.1.10", - "socket.io": "^4.6.2", - "socket.io-client": "^4.6.2", - "ssh2": "^1.13.0", - "string-hash": "^1.1.3", - "string-length": "^6.0.0", + "socket.io": "^4.8.3", "svg-captcha": "^1.4.0", - "svgo": "^3.3.3", - "tiktoken": "^1.0.16", "together-ai": "^0.33.0", - "tweetnacl": "^1.0.3", - "ua-parser-js": "^1.0.38", + "ua-parser-js": "^1.0.41", "uglify-js": "^3.17.4", - "uuid": "^9.0.0", - "validator": "^13.9.0", - "winston": "^3.9.0", - "winston-daily-rotate-file": "^4.7.1", - "yargs": "^17.7.2" + "uuid": "^9.0.1", + "validator": "^13.15.35" }, "devDependencies": { "@types/node": "^24.0.0", "chai": "^4.3.7", - "jsdom": "29.0.0", "mocha": "^7.2.0", "nodemon": "^3.1.0", - "nyc": "^15.1.0", - "sinon": "^15.2.0", "typescript": "^5.9.3", + "vite": "^8.0.0", "vitest": "^4.0.14" }, "author": "Puter Technologies Inc.", diff --git a/src/backend/server.ts b/src/backend/server.ts new file mode 100644 index 000000000..4d0bba9ac --- /dev/null +++ b/src/backend/server.ts @@ -0,0 +1,1171 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import compression from 'compression'; +import cookieParser from 'cookie-parser'; +import express from 'express'; +import type { Application, RequestHandler } from 'express'; +import helmet from 'helmet'; +import uaParser from 'ua-parser-js'; +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import http from 'node:http'; +import { puterClients } from './clients'; +import { puterControllers } from './controllers'; +import { createAuthProbe } from './core/http/middleware/authProbe'; +import { createRequestContextMiddleware } from './core/http/middleware/requestContext'; +import { createErrorHandler } from './core/http/middleware/errorHandler'; +import { isHttpError } from './core/http/HttpError'; +import { + adminOnlyGate, + allowedAppIdsGate, + requireAuthGate, + requireUserActorGate, + requireVerifiedGate, + subdomainGate, +} from './core/http/middleware/gates'; +import { createNotFoundHandler } from './core/http/middleware/notFoundHandler'; +import { + requireAntiCsrf, + setAntiCsrfRedis, +} from './core/http/middleware/antiCsrf'; +import { captchaGate, setCaptchaRedis } from './core/http/middleware/captcha'; +import { + rateLimitGate, + configureRateLimit, +} from './core/http/middleware/rateLimit'; +import { + createWwwRedirect, + createUserSubdomainRedirect, + createNativeAppStatic, +} from './core/http/middleware/hostRedirects'; +import { createPuterSiteMiddleware } from './core/http/middleware/puterSite'; +import { PuterRouter } from './core/http/PuterRouter'; +import { PREFIX_METADATA_KEY, type RouteDescriptor } from './core/http/types'; +import type { AuthService } from './services/auth/AuthService'; +import { puterDrivers } from './drivers'; +import { + clientsContainers, + configContainer, + controllersContainers, + driversContainers, + servicesContainers, + storesContainers, +} from './exports'; +import { extensionStore } from './extensions'; +import { puterServices } from './services'; +import { puterStores } from './stores'; +import type { + IConfig, + LayerInstances, + WithControllerRegistration, + WithLifecycle, +} from './types'; + +export class PuterServer { + clients!: LayerInstances; + stores!: LayerInstances; + services!: LayerInstances; + controllers!: LayerInstances; + drivers!: LayerInstances; + #config: IConfig; + #app!: ReturnType; + #server: ReturnType['listen']> | null = null; + + #ready: Promise; + + constructor( + config: IConfig, + clients: typeof puterClients, + stores: typeof puterStores, + services: typeof puterServices, + controllers: typeof puterControllers, + drivers: typeof puterDrivers, + ) { + this.#config = config; + // Expose config to the extension API (extension.config) + Object.assign(configContainer, config); + this.#ready = this.#setupServer( + clients, + stores, + services, + controllers, + drivers, + ); + } + + async #setupServer( + clients: typeof puterClients, + stores: typeof puterStores, + services: typeof puterServices, + controllers: typeof puterControllers, + drivers: typeof puterDrivers, + ) { + // Load prod extensions from configured directories (dynamic) + const extensionDirs = this.#config.extensions; + await this.#importExtensions(extensionDirs); + + this.clients = {} as typeof this.clients; + for (const [clientName, ClientClass] of Object.entries(clients)) { + this.clients[clientName] = + typeof ClientClass === 'object' + ? ClientClass + : (new (ClientClass as any)(this.#config) as any); + clientsContainers[clientName] = this.clients[clientName]; + } + for (const [clientName, ClientClass] of Object.entries( + extensionStore.clients, + )) { + this.clients[clientName] = + typeof ClientClass === 'object' + ? ClientClass + : (new (ClientClass as any)(this.#config) as any); + clientsContainers[clientName] = this.clients[clientName]; + } + + this.stores = {} as typeof this.stores; + for (const [storeName, StoreClass] of Object.entries(stores)) { + this.stores[storeName] = + typeof StoreClass === 'object' + ? StoreClass + : (new (StoreClass as any)( + this.#config, + this.clients, + this.stores, + ) as any); + storesContainers[storeName] = this.stores[storeName]; + } + for (const [storeName, StoreClass] of Object.entries( + extensionStore.stores, + )) { + this.stores[storeName] = + typeof StoreClass === 'object' + ? StoreClass + : (new (StoreClass as any)( + this.#config, + this.clients, + this.stores, + ) as any); + storesContainers[storeName] = this.stores[storeName]; + } + + this.services = {} as typeof this.services; + for (const [serviceName, ServiceClass] of Object.entries(services)) { + this.services[serviceName] = + typeof ServiceClass === 'object' + ? ServiceClass + : (new (ServiceClass as any)( + this.#config, + this.clients, + this.stores, + this.services, + ) as any); + servicesContainers[serviceName] = this.services[serviceName]; + } + for (const [serviceName, ServiceClass] of Object.entries( + extensionStore.services, + )) { + this.services[serviceName] = + typeof ServiceClass === 'object' + ? ServiceClass + : (new (ServiceClass as any)( + this.#config, + this.clients, + this.stores, + this.services, + ) as any); + servicesContainers[serviceName] = this.services[serviceName]; + } + + // Wire the rate-limiter to its configured backend now that clients + // and stores exist. Memory is the default; `redis` needs a redis + // client, `kv` needs the system KV store (DynamoDB-backed). + this.#configureRateLimiter(); + + // Anti-CSRF tokens live in redis so they survive cross-node hops + // (issue on node A, consume on node B). + setAntiCsrfRedis(this.clients.redis); + setCaptchaRedis(this.clients.redis); + + // init express server here + this.#app = express(); + // `trust proxy` MUST be set before any middleware reads `req.ip` / + // `req.ips` / `req.protocol`, since express derives those from XFF + // only when this flag is set. Default is `false` (no proxy trusted) + // — deployments behind a reverse proxy chain must set + // `config.trust_proxy` to the hop count (e.g. `1` for a single + // Cloudflare/nginx hop). Never `true` in prod: that trusts every hop + // and makes XFF forgeable. + this.#app.set('trust proxy', this.#config.trust_proxy ?? false); + this.#installGlobalMiddleware(); + + // Instantiate drivers BEFORE controllers so controllers can receive + // a typed `drivers` reference. The `/drivers/*` HTTP surface lives + // on `DriverController` (a regular controller) which reads from + // `this.drivers` — no separate registry object here any more. + this.drivers = {} as typeof this.drivers; + const allDriverSources = [ + ...Object.entries(drivers), + ...Object.entries(extensionStore.drivers), + ]; + for (const [driverKey, DriverClass] of allDriverSources) { + const instance = + typeof DriverClass === 'object' + ? DriverClass + : (new (DriverClass as any)( + this.#config, + this.clients, + this.stores, + this.services, + ) as any); + this.drivers[driverKey] = instance; + driversContainers[driverKey] = instance; + } + + this.controllers = {} as typeof this.controllers; + for (const [controllerName, ControllerClass] of Object.entries( + controllers, + )) { + this.controllers[controllerName] = + typeof ControllerClass === 'object' + ? ControllerClass + : (new (ControllerClass as any)( + this.#config, + this.clients, + this.stores, + this.services, + this.drivers, + ) as any); + this.#registerControllerRoutes( + controllerName, + this.controllers[controllerName], + ); + controllersContainers[controllerName] = + this.controllers[controllerName]; + } + for (const [controllerName, ControllerClass] of Object.entries( + extensionStore.controllers, + )) { + this.controllers[controllerName] = + typeof ControllerClass === 'object' + ? ControllerClass + : (new (ControllerClass as any)( + this.#config, + this.clients, + this.stores, + this.services, + this.drivers, + ) as any); + this.#registerControllerRoutes( + controllerName, + this.controllers[controllerName], + ); + controllersContainers[controllerName] = + this.controllers[controllerName]; + } + + // Register extension event listeners. Extensions opted for a + // 2-arg `(data, meta)` handler shape; EventClient calls with + // `(key, data, meta)`. Drop `key` in the adapter so extension + // code stays stable. + Object.entries(extensionStore.events).forEach(([event, handlers]) => { + handlers.forEach((handler) => { + this.clients.event.on( + event, + (_key: string, data: unknown, meta: object) => + handler(data, meta), + ); + }); + }); + + // Extension routes are shaped as `RouteDescriptor`s too, so they + // flow through the same materializer as controller routes — same + // option → middleware translation (subdomain, auth, body parsers, …). + // The extension-layer "prefix" is always empty; extensions compose + // their own path strings. + for (const route of extensionStore.routeHandlers) { + this.#materializeRoute(this.#app, '', route); + } + + // Terminal middleware MUST install last — after every route + extension + // route is registered, so the catch-all 404 only fires for genuinely + // unmatched requests, and the error handler is reachable from any + // thrown error in the stack above it. + this.#installTerminalMiddleware(); + + return true; + } + + /** + * Point the shared rate-limiter at its configured backend. Reads + * `config.rate_limit.backend` (defaults to `redis`) and resolves the + * required dependency from `this.clients` / `this.stores`. Unknown + * or misconfigured backends fall back to memory with a warning so a + * typo doesn't take the server down. + */ + #configureRateLimiter() { + // Default to `redis` — the redis client is always present (falls + // back to ioredis-mock in dev when no nodes are configured), and + // sorted-set rate limiting scales across nodes for free. Set + // `rate_limit.backend` in config to switch to `memory` or `kv`. + const backend = this.#config.rate_limit?.backend ?? 'redis'; + try { + configureRateLimit({ + backend, + redis: this.clients.redis, + kv: this.stores.kv, + }); + } catch (e) { + console.warn( + `[rate-limit] ${backend} backend unavailable, falling back to memory:`, + (e as Error).message, + ); + configureRateLimit(); + } + } + + /** + * Install always-on middleware on the express app, in the order they + * must run at request time. Ordering note: + * - `express.json` must run before `authProbe` so `req.body.auth_token` + * is readable. + * - `authProbe` never rejects; it only populates `req.actor` if a valid + * token is present. + * - Per-route gate middleware (requireAuth, adminOnly, ...) lands in + * `#materializeRoute` as those options ship. + */ + #installGlobalMiddleware() { + // ── Cookie parsing ────────────────────────────────────────── + this.#app.use(cookieParser()); + + // ── Compression ───────────────────────────────────────────── + this.#app.use(compression()); + + // ── Security headers (helmet) ─────────────────────────────── + this.#app.use(helmet.noSniff()); + this.#app.use(helmet.hsts()); + this.#app.use(helmet.ieNoOpen()); + this.#app.use(helmet.permittedCrossDomainPolicies()); + this.#app.use(helmet.xssFilter()); + this.#app.disable('x-powered-by'); + + // Cross-Origin-Resource-Policy: always allow cross-origin reads. + // The stricter COOP+COEP pair (for SharedArrayBuffer) is deferred + // until the hosting layer lands — it requires UA + context gating. + this.#app.use((_req, res, next) => { + res.setHeader('Cross-Origin-Resource-Policy', 'cross-origin'); + next(); + }); + + // ── Query param sanitization ──────────────────────────────── + // Strip non-primitive query values. Express 5's default simple + // parser mostly avoids these, but when `extended` qs is enabled + // (or a client tricks the parser) arrays/objects can sneak in. + this.#app.use((req, _res, next) => { + if (req.query) { + const allowed = ['string', 'number', 'boolean']; + for (const k of Object.keys(req.query)) { + const v = req.query[k]; + if (v != null && !allowed.includes(typeof v)) { + delete req.query[k]; + } + } + } + next(); + }); + + // ── UA parsing ────────────────────────────────────────────── + this.#app.use((req, _res, next) => { + const header = req.headers['user-agent']; + if (header) { + req.ua = uaParser(header); + } + next(); + }); + + // ── Host header validation ────────────────────────────────── + this.#installHostValidation(); + + // ── Host redirects (www → root, user subdomain → static hosting) + // Installed after host validation so we know the host is allowed, + // and before CORS/body-parsing so we short-circuit on redirects + // without burning work. + this.#app.use(createWwwRedirect(this.#config)); + this.#app.use(createUserSubdomainRedirect(this.#config)); + + // ── Native app static serving (editor.*, docs.*, …) ───────── + // No-op when `native_apps_root` is unset. + this.#app.use(createNativeAppStatic(this.#config)); + + // ── CORS headers ──────────────────────────────────────────── + this.#installCors(); + + // ── IP validation ─────────────────────────────────────────── + if (this.#config.enable_ip_validation) { + this.#installIpValidation(); + } + + // ── OPTIONS preflight ─────────────────────────────────────── + this.#app.options('/*splat', (_req, res) => { + res.sendStatus(200); + }); + + // ── Body parsing (JSON + text-as-json shim) ───────────────── + const captureRawBody: NonNullable< + Parameters[0] + >['verify'] = (req, _res, buf) => { + (req as { rawBody?: Buffer }).rawBody = Buffer.from(buf); + }; + this.#app.use(express.json({ limit: '50mb', verify: captureRawBody })); + this.#app.use( + express.json({ + limit: '50mb', + type: (req) => + req.headers['content-type'] === 'text/plain;actually=json', + verify: captureRawBody, + }), + ); + // Form-encoded bodies (e.g. `/down` from the GUI's iframe-triggered + // download form). Needs to run before the auth probe so + // `req.body.auth_token` is populated for urlencoded POSTs the same + // way it is for JSON POSTs. Small cap — this parser is only here to + // cover the auth_token / anti_csrf field shape, not file uploads. + this.#app.use(express.urlencoded({ extended: true, limit: '100kb' })); + + // ── Auth probe ────────────────────────────────────────────── + const authService = this.services.auth as AuthService | undefined; + if (authService) { + this.#app.use( + createAuthProbe({ + authService, + cookieName: this.#config.cookie_name, + }), + ); + } + + // ── Per-request ALS context ───────────────────────────────── + // Runs AFTER auth probe so `req.actor` is already populated when + // we snapshot it into the context. + this.#app.use(createRequestContextMiddleware()); + + // ── User-hosted sites (*.puter.site, *.puter.app) ─────────── + // Short-circuits hosting-domain hosts before any API/GUI + // controller route has a chance to match. Needs DI layers for + // subdomain lookup, private-app gate, and file streaming. + this.#app.use( + createPuterSiteMiddleware(this.#config, { + clients: this.clients, + stores: this.stores, + services: this.services, + }), + ); + + extensionStore.globalMiddlewares.forEach((mw) => { + this.#app.use(mw); + }); + } + + // ── Host header validation ────────────────────────────────────── + + #installHostValidation() { + const config = this.#config; + + // Hostname missing — malformed request from a broken client. + this.#app.use((req, res, next) => { + if (req.hostname === undefined) { + res.status(400).send( + 'Please verify your browser is up-to-date.', + ); + return; + } + next(); + }); + + // Build the allowed-domain set from config. + this.#app.use((req, res, next) => { + if (config.allow_all_host_values) { + next(); + return; + } + + if (!config.allow_no_host_header && !req.headers.host) { + res.status(400).send('Missing Host header.'); + return; + } + + // /healthcheck is always reachable regardless of host. + if (req.path === '/healthcheck') { + next(); + return; + } + + const hostName = (req.headers.host ?? '') + .split(':')[0] + .trim() + .toLowerCase(); + const allowed = this.#getAllowedDomains(); + + if ( + allowed.some((d) => PuterServer.#hostMatchesDomain(hostName, d)) + ) { + next(); + return; + } + + if (config.custom_domains_enabled) { + req.is_custom_domain = true; + next(); + return; + } + + res.status(400).send('Invalid Host header.'); + }); + } + + #allowedDomainsCache: string[] | null = null; + + #getAllowedDomains(): string[] { + if (this.#allowedDomainsCache) return this.#allowedDomainsCache; + const cfg = this.#config; + const raw = [ + cfg.domain, + cfg.static_hosting_domain, + cfg.static_hosting_domain_alt, + cfg.private_app_hosting_domain, + cfg.private_app_hosting_domain_alt, + ]; + const staticDomain = PuterServer.#normalizeDomain( + cfg.static_hosting_domain, + ); + if (staticDomain) raw.push(`at.${staticDomain}`); + if (cfg.allow_nipio_domains) raw.push('nip.io'); + + this.#allowedDomainsCache = raw + .map(PuterServer.#normalizeDomain) + .filter((d): d is string => d !== null); + return this.#allowedDomainsCache; + } + + static #normalizeDomain(d: string | undefined | null): string | null { + if (!d || typeof d !== 'string') return null; + const trimmed = d.trim().toLowerCase(); + return trimmed.length > 0 ? trimmed : null; + } + + static #hostMatchesDomain(hostname: string, domain: string): boolean { + return hostname === domain || hostname.endsWith(`.${domain}`); + } + + // ── CORS headers ───────────────────────────────────────────────── + + #installCors() { + const config = this.#config; + const allowedMethods = + 'GET, POST, OPTIONS, PUT, PATCH, DELETE, PROPFIND, PROPPATCH, MKCOL, COPY, MOVE, LOCK, UNLOCK'; + const allowedHeaders = [ + 'Origin', + 'X-Requested-With', + 'Content-Type', + 'Accept', + 'Authorization', + 'sentry-trace', + 'baggage', + 'Depth', + 'Destination', + 'Overwrite', + 'If', + 'Lock-Token', + 'DAV', + 'stripe-signature', + ].join(', '); + + this.#app.use((req, res, next) => { + const origin = req.headers.origin; + const subdomain = req.subdomains?.[req.subdomains.length - 1]; + const isApiOrDav = subdomain === 'api' || subdomain === 'dav'; + + // Allow any origin. puter.js is meant to be consumed from + // arbitrary third-party sites, so reflect the caller's origin + // (or fall back to `*` for non-browser clients). + res.setHeader('Access-Control-Allow-Origin', origin ?? '*'); + if (origin) res.vary('Origin'); + + // Credentials require a specific (non-`*`) Allow-Origin, which + // we just set when an origin was present. Enable on API/DAV + // so cookie-auth works cross-origin. + if (isApiOrDav && origin) { + res.setHeader('Access-Control-Allow-Credentials', 'true'); + } + + res.setHeader('Access-Control-Allow-Methods', allowedMethods); + res.setHeader('Access-Control-Allow-Headers', allowedHeaders); + + // Private Network Access: grant public origins permission to + // reach loopback/private addresses (e.g. self-hosted Puter on + // localhost, or api.puter.com pointed at a local IP via hosts). + if (req.headers['access-control-request-private-network']) { + res.setHeader('Access-Control-Allow-Private-Network', 'true'); + } + + // Disable iframes on the main domain + if (req.hostname === config.domain) { + res.setHeader('X-Frame-Options', 'SAMEORIGIN'); + } + + next(); + }); + } + + // ── IP validation ─────────────────────────────────────────────── + + #installIpValidation() { + this.#app.use(async (req, res, next) => { + // `req.ip` reflects `trust proxy`: it's the leftmost untrusted + // address from XFF when behind a configured proxy chain, and the + // direct socket peer otherwise. Reading XFF directly would let a + // client forge the value when traffic isn't behind the expected + // proxy. + const ip = req.ip; + const event = { allow: true, ip }; + // emitAndWait so listeners that do async work (IP-reputation + // lookups, Redis checks) can complete before we read + // `event.allow` and decide the gate. + await this.clients.event.emitAndWait('ip.validate', event, {}); + if (!event.allow) { + res.status(403).send('Forbidden'); + return; + } + next(); + }); + } + + /** + * Install end-of-pipeline middleware. Order matters: + * 1. The 404 catch-all runs only when no earlier route matched, so it + * must be installed *after* every controller + extension route. + * 2. The error handler is the express terminal — it catches everything + * thrown by routes, gates, and the 404 above. Express 5 auto-forwards + * thrown errors (sync and async), so handlers can `throw new HttpError(...)` + * without `next(err)` ceremony. + */ + #installTerminalMiddleware() { + this.#app.use(createNotFoundHandler()); + this.#app.use( + createErrorHandler({ + onError: (err, req) => { + // Page on 5xx only — skip 4xx HttpErrors, which are + // expected client-caused failures. Non-HttpError values + // are treated as unexpected 500s. De-dupe alarms by + // route + error signature so a hot loop of the same + // crash lands as a single alarm with N occurrences + // instead of N pages. + const isHttp = isHttpError(err); + const status = isHttp ? err.statusCode : 500; + if (status < 500) return; + const signature = isHttp + ? err.legacyCode || err.code || err.message + : err instanceof Error + ? err.message + : String(err); + const routePath = + (req as unknown as { route?: { path?: string } }).route + ?.path ?? req.path; + const alarmId = `http_${status}:${req.method}:${routePath}:${signature}`; + this.clients.alarm.create( + alarmId, + `HTTP ${status} on ${req.method} ${req.originalUrl}: ${signature}`, + { + error: err instanceof Error ? err : undefined, + status, + method: req.method, + path: req.originalUrl, + body: req.body, + route: routePath, + actor: req.actor, + }, + ); + }, + }), + ); + } + + /** + * Walk a controller's declared routes (via `PuterRouter`) and register + * each one against the underlying express app. Per-route option → middleware + * translation lives here — when we add auth/subdomain/body-parsing + * options, they get wired in at this single point without touching any + * controller call site. + */ + #registerControllerRoutes( + controllerName: string, + controller: WithControllerRegistration, + ) { + if (!controller.registerRoutes) { + throw new Error( + `Controller ${controllerName} does not have registerRoutes method`, + ); + } + + // Controllers annotated with `@Controller('/prefix')` carry the prefix + // on their prototype; bare (imperative) controllers default to ''. + const prefix = (controller as unknown as Record)[ + PREFIX_METADATA_KEY + ] as string | undefined; + const router = new PuterRouter(prefix ?? ''); + controller.registerRoutes(router); + + for (const route of router.routes) { + this.#materializeRoute(this.#app, router.prefix, route); + } + } + + #materializeRoute( + app: Application, + routerPrefix: string, + route: RouteDescriptor, + ) { + const mwChain: RequestHandler[] = []; + const opts = route.options; + + // 1. Subdomain routing. Routes that specify `subdomain` only match + // that subdomain(s). Routes WITHOUT a `subdomain` option (and that + // aren't `use` middleware) are restricted to the root origin — this + // prevents API-subdomain requests from accidentally hitting a root- + // only route. Explicit `subdomain: '*'` disables the gate entirely. + // + // For `use` routes, `next('route')` in a middleware doesn't skip the + // handler (that's only reliable inside `app.METHOD`/`router.METHOD` + // chains). We handle subdomain gating by wrapping the handler for + // `use` routes further down — don't push `subdomainGate` here. + const isUse = route.method === 'use'; + if (opts.subdomain !== undefined) { + if (opts.subdomain !== '*' && !isUse) { + mwChain.push(subdomainGate(opts.subdomain)); + } + // subdomain: '*' → no gate, match any subdomain + } else if (!isUse) { + // No subdomain specified + not a `use()` middleware → root only. + // Root = no subdomain present (req.subdomains is empty). + mwChain.push((req, _res, next) => { + if (req.subdomains && req.subdomains.length > 0) { + next('route'); + return; + } + next(); + }); + } + + // 2. Auth gates. Implication graph: + // adminOnly => requireAuth + // allowedAppIds => requireAuth + // requireUserActor => requireAuth + // Dedupe: only push requireAuthGate once when *any* of these are set. + const needsAuth = Boolean( + opts.requireAuth || + opts.requireUserActor || + opts.adminOnly || + opts.allowedAppIds || + opts.requireVerified, + ); + if (needsAuth) mwChain.push(requireAuthGate()); + + // `requireVerified` intentionally does NOT imply `requireUserActor`: + // FS routes (and similar) want the user's email to be confirmed even + // when an app acts on the user's behalf. `requireVerifiedGate` reads + // `req.actor?.user?.email_confirmed`, which app-under-user actors + // carry, so it works for either actor shape. + // + // `adminOnly` also does NOT imply `requireUserActor`: admin endpoints + // should be callable from scripts/automation using an admin's access + // token, not only from browser sessions. `adminOnlyGate` gates on + // `actor.user.username`, which is populated for access-token and + // app-under-user actors alike. + if (opts.requireUserActor) mwChain.push(requireUserActorGate()); + + if (opts.adminOnly) { + const extras = Array.isArray(opts.adminOnly) ? opts.adminOnly : []; + mwChain.push(adminOnlyGate(extras)); + } + + if (opts.allowedAppIds) { + mwChain.push(allowedAppIdsGate(opts.allowedAppIds)); + } + + // 2a. Email verification. Keyed off `strict_email_verification_required` + // so self-hosted boxes without SMTP don't break every fs route. + if (opts.requireVerified) { + mwChain.push( + requireVerifiedGate( + Boolean(this.#config.strict_email_verification_required), + ), + ); + } + + // 2b. Rate limiting. Runs after auth so 'user' key strategy + // has access to req.actor. + if (opts.rateLimit) { + mwChain.push( + rateLimitGate(opts.rateLimit) as unknown as RequestHandler, + ); + } + + // 2c. Captcha verification. Reads captchaToken + captchaAnswer + // from req.body — body is already parsed by the global JSON + // middleware at this point. + if (opts.captcha) { + const enabled = Boolean(this.#config.captcha?.enabled); + mwChain.push(captchaGate(enabled) as unknown as RequestHandler); + } + + // 2d. Anti-CSRF token consumption. + if (opts.antiCsrf) { + mwChain.push(requireAntiCsrf() as unknown as RequestHandler); + } + + // 3. Per-route body parsers. Each is a no-op when the request's + // content-type doesn't match — multiple can coexist. The global + // `application/json` parser already ran in `#installGlobalMiddleware`, + // so by default the only reason to opt into one of these is to handle + // a non-JSON body shape (raw bytes, plain text, urlencoded form) or + // to override JSON limits on a hot path. + // bodyJson is `false | { limit?, type? }`. Truthiness check excludes + // both `undefined` (no opt) and `false` (explicit opt-out). + if (opts.bodyJson) { + mwChain.push( + express.json({ + limit: opts.bodyJson.limit, + type: opts.bodyJson.type, + }), + ); + } + + if (opts.bodyRaw) { + const raw = opts.bodyRaw === true ? {} : opts.bodyRaw; + mwChain.push( + express.raw({ + limit: raw.limit, + type: raw.type, + }), + ); + } + + if (opts.bodyText) { + const text = opts.bodyText === true ? {} : opts.bodyText; + mwChain.push( + express.text({ + limit: text.limit, + type: text.type, + }), + ); + } + + if (opts.bodyUrlencoded) { + const ue = opts.bodyUrlencoded === true ? {} : opts.bodyUrlencoded; + mwChain.push( + express.urlencoded({ + limit: ue.limit, + extended: ue.extended ?? true, + }), + ); + } + + // 4. Caller-supplied middleware runs after gates + parsers, before the handler. + if (opts.middleware) mwChain.push(...opts.middleware); + + const fullPath = + route.path !== undefined + ? PuterServer.#joinPath(routerPrefix, route.path) + : undefined; + + if (route.method === 'use') { + // Subdomain check for `use` middleware lives INSIDE the handler + // wrapper — `next('route')` from a stand-alone subdomainGate + // doesn't reliably skip a `use` handler in Express 5. + let handler = route.handler; + if (opts.subdomain !== undefined && opts.subdomain !== '*') { + const allowList = Array.isArray(opts.subdomain) + ? opts.subdomain + : [opts.subdomain]; + const original = handler; + handler = (req, res, next) => { + const active = + req.subdomains?.[req.subdomains.length - 1] ?? ''; + if (!allowList.includes(active)) return next(); + return original(req, res, next); + }; + } + if (fullPath !== undefined) { + app.use(fullPath as any, ...mwChain, handler); + } else { + app.use(...mwChain, handler); + } + return; + } + + if (fullPath === undefined) { + throw new Error(`Route method '${route.method}' requires a path`); + } + + // All express + WebDAV verbs accept the same (path, ...handlers) shape. + // The `RouteMethod` union is the allowlist of method names we expose. + const method = app[route.method as keyof Application] as unknown; + if (typeof method !== 'function') { + throw new Error( + `Express app does not support method: ${route.method}`, + ); + } + (method as (...args: unknown[]) => unknown).call( + app, + fullPath, + ...mwChain, + route.handler, + ); + } + + /** + * Join a controller's prefix with a route path. RegExp / array paths are + * passed through unprefixed (consistent with express's behavior; decorator + * paths are assumed to be strings). + */ + static #joinPath( + prefix: string, + path: NonNullable, + ): string | RegExp | Array { + if (typeof path !== 'string') return path; + if (!prefix) return path; + return `${prefix}/${path}`.replace(/\/+/g, '/'); + } + + async #importExtensions(extensionDirs: string[]) { + for (const extDir of extensionDirs) { + // `withFileTypes: true` gives us `Dirent` objects so we can + // distinguish files from directories without extra stat calls + // (and without relying on a dot-in-name heuristic, which breaks + // for data-bearing sidecar dirs like `pages.assets/`). + for (const entry of readdirSync(extDir, { withFileTypes: true })) { + const entryPath = `${extDir}/${entry.name}`; + + if (entry.isFile()) { + if (/\.(js|mjs|cjs)$/.test(entry.name)) { + console.log(`Importing extension file ${entryPath}`); + await import(entryPath); + } + continue; + } + + if (!entry.isDirectory()) continue; // symlinks, etc. — skip + + // Prefer package.json "main"; fall back to index.{js,mjs,cjs}. + // Dirs that match neither (e.g. data-only sidecars) are + // silently ignored rather than crashing the boot. + let mainPath: string | null = null; + const pkgPath = `${entryPath}/package.json`; + if (existsSync(pkgPath)) { + try { + const pkg = JSON.parse( + readFileSync(pkgPath, 'utf-8'), + ) as { main?: string }; + if (pkg.main) mainPath = `${entryPath}/${pkg.main}`; + } catch (e) { + console.warn( + `[extensions] invalid package.json at ${pkgPath}:`, + e, + ); + continue; + } + } + if (!mainPath) { + for (const cand of ['index.js', 'index.mjs', 'index.cjs']) { + if (existsSync(`${entryPath}/${cand}`)) { + mainPath = `${entryPath}/${cand}`; + break; + } + } + } + if (!mainPath) continue; + + console.log(`Importing extension file ${mainPath}`); + await import(mainPath); + } + } + } + + async start(noHttpServer = false) { + await this.#ready; + + // Create the http server explicitly (instead of `app.listen()`) so we + // have the server reference BEFORE listen starts — anything that needs + // to hook into the raw server (socket.io upgrades, WebSockets, …) runs + // its `attachHttpServer(server)` here, pre-listen. + const httpServer = http.createServer(this.#app); + for (const service of Object.values(this.services) as Array< + WithLifecycle & { + attachHttpServer?: (s: http.Server) => void | Promise; + } + >) { + if (typeof service.attachHttpServer === 'function') { + await service.attachHttpServer(httpServer); + } + } + + if (!noHttpServer) { + this.#server = httpServer.listen(this.#config.port, async () => { + const cfg = this.#config; + const liveUrl = + cfg.origin ?? + `${cfg.protocol ?? 'http'}://${cfg.domain ?? 'localhost'}:${this.#config.port}`; + console.log( + '\n************************************************************', + ); + console.log(`* Puter is now live at: ${liveUrl}`); + console.log( + '************************************************************\n', + ); + + for (const client of Object.values( + this.clients, + ) as WithLifecycle[]) { + if (client.onServerStart) { + await client.onServerStart(); + } + } + for (const store of Object.values( + this.stores, + ) as WithLifecycle[]) { + if (store.onServerStart) { + await store.onServerStart(); + } + } + for (const service of Object.values( + this.services, + ) as WithLifecycle[]) { + if (service.onServerStart) { + await service.onServerStart(); + } + } + for (const controller of Object.values( + this.controllers, + ) as WithLifecycle[]) { + if (controller.onServerStart) { + await controller.onServerStart(); + } + } + for (const driver of Object.values( + this.drivers, + ) as WithLifecycle[]) { + if (driver.onServerStart) { + await driver.onServerStart(); + } + } + console.log('PuterServer has fully booted.'); + // Auto-launch the browser on dev boot (matches v1 WebServerService). + // Opt out via `no_browser_launch: true` in config. + if (this.#config.env === 'dev' && !cfg.no_browser_launch) { + try { + const openModule = await import('open'); + await openModule.default(liveUrl); + } catch (e) { + console.log( + '[server] could not auto-open browser:', + (e as Error).message, + ); + } + } + }); + } else { + this.#server = { + close: (cb) => { + console.debug('PuterServer mock close called'); + cb(); + }, + closeAllConnections: () => { + console.debug( + 'PuterServer mock closeAllConnections called', + ); + }, + }; + } + } + + async prepareShutdown() { + if (this.#server) { + this.#server.close(async () => { + console.log( + 'PuterServer has stopped accepting new connections', + ); + for (const client of Object.values( + this.clients, + ) as WithLifecycle[]) { + if (client.onServerPrepareShutdown) { + await client.onServerPrepareShutdown(); + } + } + for (const store of Object.values( + this.stores, + ) as WithLifecycle[]) { + if (store.onServerPrepareShutdown) { + await store.onServerPrepareShutdown(); + } + } + for (const service of Object.values( + this.services, + ) as WithLifecycle[]) { + if (service.onServerPrepareShutdown) { + await service.onServerPrepareShutdown(); + } + } + for (const controller of Object.values( + this.controllers, + ) as WithLifecycle[]) { + if (controller.onServerPrepareShutdown) { + await controller.onServerPrepareShutdown(); + } + } + for (const driver of Object.values( + this.drivers, + ) as WithLifecycle[]) { + if (driver.onServerPrepareShutdown) { + await driver.onServerPrepareShutdown(); + } + } + }); + } + } + + async shutdown() { + if (this.#server) { + console.log('PuterServer is shutting down'); + this.#server.closeAllConnections(); + for (const client of Object.values( + this.clients, + ) as WithLifecycle[]) { + if (client.onServerShutdown) { + await client.onServerShutdown(); + } + } + for (const store of Object.values(this.stores) as WithLifecycle[]) { + if (store.onServerShutdown) { + await store.onServerShutdown(); + } + } + for (const service of Object.values( + this.services, + ) as WithLifecycle[]) { + if (service.onServerShutdown) { + await service.onServerShutdown(); + } + } + for (const controller of Object.values( + this.controllers, + ) as WithLifecycle[]) { + if (controller.onServerShutdown) { + await controller.onServerShutdown(); + } + } + for (const driver of Object.values( + this.drivers, + ) as WithLifecycle[]) { + if (driver.onServerShutdown) { + await driver.onServerShutdown(); + } + } + } + } +} diff --git a/src/backend/services/acl/ACLService.ts b/src/backend/services/acl/ACLService.ts new file mode 100644 index 000000000..6ba7da82b --- /dev/null +++ b/src/backend/services/acl/ACLService.ts @@ -0,0 +1,377 @@ +import type { LayerInstances } from '../../types'; +import type { puterServices } from '../index'; +import { PuterService } from '../types'; +import type { Actor } from '../../core/actor'; +import { isSystemActor } from '../../core/actor'; +import { PermissionUtil } from '../permission/permissionUtil'; +import { MANAGE_PERM_PREFIX } from '../permission/consts'; + +// ── Types ──────────────────────────────────────────────────────────── + +/** + * Thin, filesystem-agnostic view of a resource for ACL checks. + * + * Callers construct a descriptor from whatever entry metadata they already + * have; ACL does not depend on the filesystem layer. FSController does + * exactly this (see its `resourceDescriptor` in `#assertWriteAccess`). + * + * `resolveAncestors()` MUST return the chain starting with the resource + * itself and ending at the direct child of root. Empty means "root". + */ +export interface ResourceDescriptor { + path: string; + resolveAncestors: () => Promise< + ReadonlyArray<{ uid: string; path: string }> + >; +} + +export type AclMode = + | 'see' + | 'list' + | 'read' + | 'write' + | typeof MANAGE_PERM_PREFIX; + +/** Duck-typed error shape compatible with APIError consumers (fsv2). */ +export interface AclError { + status: number; + message: string; + fields: { code: string }; +} + +interface StatPermissionsResult { + [path: string]: string[]; +} + +const MODES_ABOVE: Record = { + see: ['see', 'list', 'read', 'write'], + list: ['list', 'read', 'write'], + read: ['read', 'write'], + write: ['write'], + [MANAGE_PERM_PREFIX]: [MANAGE_PERM_PREFIX], +}; + +const PUBLIC_READ_MODES: ReadonlyArray = Object.freeze([ + 'read', + 'list', + 'see', +]); + +// ── ACLService ─────────────────────────────────────────────────────── + +/** + * ACLService enforces filesystem access-control semantics for Puter. + * + * Design notes: + * + * - **No FSNode dependency.** Callers pass a `ResourceDescriptor` duck type + * (`{ path, resolveAncestors() }`). This lets ACL live as a service + * without pulling in the filesystem layer (which would create a circular + * dependency). + * - **No route registration.** The service is pure; a controller exposes + * `/acl/stat-user-user` and `/acl/set-user-user`. + * + * Tree-walks are done via `resolveAncestors()`, which returns a + * pre-resolved ancestor chain from the caller's FS layer. + */ +export class ACLService extends PuterService { + declare protected services: LayerInstances; + + // ── Public API ─────────────────────────────────────────────────── + + /** + * Returns true iff `actor` is allowed to perform `mode` access on `resource`. + */ + async check( + actor: Actor, + resource: ResourceDescriptor, + mode: AclMode, + ): Promise { + if (isSystemActor(actor)) return true; + + if (resource.path === '/') { + return (PUBLIC_READ_MODES as AclMode[]).includes(mode); + } + + const components = resource.path.slice(1).split('/'); + + // Short-circuit: users accessing their own home directory. + if (!actor.app && !actor.accessToken) { + const username = actor.user.username; + if ( + username && + (resource.path === `/${username}` || + resource.path.startsWith(`/${username}/`)) + ) { + return true; + } + } + + // Short-circuit: apps accessing their own AppData directory (under + // any user). Shared-appdata access is handled below via the + // per-user-permission gate. + if (actor.app && !actor.accessToken) { + const username = actor.user.username; + const appUid = actor.app.uid; + if (username) { + const appDataPath = `/${username}/AppData/${appUid}`; + if ( + resource.path === appDataPath || + resource.path.startsWith(`${appDataPath}/`) + ) { + return true; + } + } + } + + // Public folders: //Public with read-ish mode, owner must have + // confirmed email (or be admin). + if ( + this.config.enable_public_folders && + (PUBLIC_READ_MODES as AclMode[]).includes(mode) && + components.length > 1 && + components[1] === 'Public' + ) { + const ownerUsername = components[0]; + const owner = await this.stores.user.getByUsername(ownerUsername); + if (owner) { + if ( + (owner.email_confirmed ?? false) || + owner.username === 'admin' + ) { + return true; + } + } + } + + // Access tokens: authorizer must have the permission, AND the token + // itself must have it (or inherit it via an ancestor). Any "higher" + // mode (e.g. `write` covers `read`/`list`/`see`) satisfies the check. + if (actor.accessToken) { + const authorizer = actor.accessToken.issuer; + if (!(await this.check(authorizer, resource, mode))) return false; + + const ancestors = await resource.resolveAncestors(); + for (const ancestor of ancestors) { + const permissions = + mode === MANAGE_PERM_PREFIX + ? [ + PermissionUtil.join( + MANAGE_PERM_PREFIX, + 'fs', + ancestor.uid, + ), + ] + : MODES_ABOVE[mode].map((m) => + PermissionUtil.join('fs', ancestor.uid, m), + ); + for (const permission of permissions) { + if ( + await this.stores.permission.hasAccessTokenPerm( + actor.accessToken.uid, + permission, + ) + ) { + return true; + } + } + } + return false; + } + + // App-under-user: underlying user must also hold the permission. + if (actor.app) { + const userActor: Actor = { user: actor.user }; + if (!(await this.check(userActor, resource, mode))) return false; + + // Shared-appdata rule: an app accessing its AppData under a + // *different* user is allowed iff that user has access (checked + // above), i.e. the directory has been explicitly shared. + if ( + components[0] !== actor.user.username && + components[1] === 'AppData' && + components[2] === actor.app.uid + ) { + return true; + } + } + + // Fall back to the permission scan: walk ancestors, any hit wins. + // Widen the scan to all "higher" modes (`write` covers `read`/`list`/ + // `see`, etc.) so granting a stronger mode implies the weaker ones. + const ancestors = await resource.resolveAncestors(); + for (const ancestor of ancestors) { + const permissions = + mode === MANAGE_PERM_PREFIX + ? [ + PermissionUtil.join( + MANAGE_PERM_PREFIX, + 'fs', + ancestor.uid, + ), + ] + : MODES_ABOVE[mode].map((m) => + PermissionUtil.join('fs', ancestor.uid, m), + ); + const reading = await this.services.permission.scan( + actor, + permissions, + ); + const options = PermissionUtil.readingToOptions(reading); + if (options.length > 0) return true; + } + + return false; + } + + /** + * When a check fails, return a user-safe error: 404 if the actor can't + * even `see` the resource (don't leak existence), 403 otherwise. + */ + async getSafeAclError( + actor: Actor, + resource: ResourceDescriptor, + _mode: AclMode, + ): Promise { + const canSee = await this.check(actor, resource, 'see'); + if (!canSee) { + return { + status: 404, + message: 'Subject does not exist', + fields: { code: 'subject_does_not_exist' }, + }; + } + return { + status: 403, + message: 'Forbidden', + fields: { code: 'forbidden' }, + }; + } + + /** + * Stat user-to-user permissions on a resource, walking up the ancestor + * chain. Returns a map from ancestor path → permissions the issuer has + * granted the holder on that ancestor. + * + * Caller (controller) validates that both actors are user-type. + */ + async statUserUser( + issuer: Actor, + holder: Actor, + resource: ResourceDescriptor, + ): Promise { + if (issuer.app || issuer.accessToken) + throw new Error('issuer must be a user actor'); + if (holder.app || holder.accessToken) + throw new Error('holder must be a user actor'); + + const out: StatPermissionsResult = {}; + const ancestors = await resource.resolveAncestors(); + for (const ancestor of ancestors) { + const prefix = PermissionUtil.join('fs', ancestor.uid); + const perms = + await this.services.permission.queryIssuerHolderPermissionsByPrefix( + issuer, + holder, + prefix, + ); + if (perms.length > 0) out[ancestor.path] = perms; + } + return out; + } + + /** + * Grant `mode` on `resource` from `issuer` to `holder`, clearing any + * existing different-mode grants on the same node. No-op if the same + * mode (or, with `onlyIfHigher`, a higher mode) is already present. + * + * Returns `false` when no write was necessary; `true` when a grant + * (and possibly revokes) were issued. + */ + async setUserUser( + issuer: Actor, + holder: Actor, + resource: ResourceDescriptor, + mode: AclMode, + options: { onlyIfHigher?: boolean } = {}, + ): Promise { + if (issuer.app || issuer.accessToken) + throw new Error('issuer must be a user actor'); + if (holder.app || holder.accessToken) + throw new Error('holder must be a user actor'); + if (!holder.user.username) + throw new Error('holder is missing username'); + + const stat = await this.statUserUser(issuer, holder, resource); + const existing = stat[resource.path] ?? []; + + const existingModes = existing.map((p) => + PermissionUtil.isManage(p) + ? MANAGE_PERM_PREFIX + : PermissionUtil.split(p).at(-1), + ); + + if (existingModes.includes(mode)) return false; + + if (options.onlyIfHigher) { + const higher = MODES_ABOVE[mode] ?? [mode]; + if ( + existingModes.some( + (m) => + m === MANAGE_PERM_PREFIX || + (m && higher.includes(m as AclMode)), + ) + ) { + return false; + } + } + + // Resolve the resource's own uid — first element of the ancestor + // chain is the resource itself (see ResourceDescriptor docstring). + const ancestors = await resource.resolveAncestors(); + const self = ancestors[0]; + if (!self) + throw new Error('resource has no ancestor chain (is it root?)'); + const uid = self.uid; + + const newPerm = + mode === MANAGE_PERM_PREFIX + ? PermissionUtil.join(MANAGE_PERM_PREFIX, 'fs', uid) + : PermissionUtil.join('fs', uid, mode); + await this.services.permission.grantUserUserPermission( + issuer, + holder.user.username, + newPerm, + ); + + // Revoke any other modes on the same node (ACL enforces one mode per + // node per issuer/holder — higher modes supersede lower). + for (const perm of existing) { + const existingMode = PermissionUtil.isManage(perm) + ? MANAGE_PERM_PREFIX + : PermissionUtil.split(perm).at(-1); + if (existingMode === mode) continue; + await this.services.permission.revokeUserUserPermission( + issuer, + holder.user.username, + perm, + ); + } + return true; + } + + /** + * The highest mode currently in the ACL hierarchy. Callers that gate on + * "top-level" access (e.g., share-everything) should use this instead of + * hardcoding 'write', so additions (e.g., a future 'config' mode) don't + * require sweeping call-site changes. + */ + getHighestMode(): AclMode { + return 'write'; + } + + /** Modes that imply `mode`. */ + higherModes(mode: AclMode): AclMode[] { + return MODES_ABOVE[mode] ?? [mode]; + } +} diff --git a/src/backend/services/appIcon/AppIconService.ts b/src/backend/services/appIcon/AppIconService.ts new file mode 100644 index 000000000..766c4dfcf --- /dev/null +++ b/src/backend/services/appIcon/AppIconService.ts @@ -0,0 +1,275 @@ +import { Readable } from 'node:stream'; +import type { LayerInstances } from '../../types'; +import type { puterServices } from '../index'; +import { PuterService } from '../types.js'; + +const ICON_SIZES = [16, 32, 64, 128, 256, 512] as const; +const APP_ICONS_SUBDOMAIN = 'puter-app-icons'; +const APP_ICONS_PATH_PREFIX = '/system/app_icons'; + +const ORIGINAL_ICON_FILENAME = (uid: string) => `${uid}.png`; +const SIZED_ICON_FILENAME = (uid: string, size: number) => `${uid}-${size}.png`; + +/** + * App icon generation service. + * + * 1. On boot: ensures `/system/app_icons/` exists (owned by admin/system user) + * and that the `puter-app-icons` subdomain points at it. Icons are then + * served through Puter's regular hosting path + * (`https://puter-app-icons./-.png`) — + * no custom route, no custom S3 plumbing. + * 2. On `app.new-icon` event: decodes the data URL, resizes via sharp to + * the 6 standard sizes, and writes the PNGs into that directory via + * FSService. The write populates the CDN-backed subdomain + * automatically because `puter-app-icons` is a regular hosted site. + * 3. Once the original is persisted, the app's `icon` column is rewritten + * from the data URL to the canonical endpoint URL so later reads + * don't re-ship the base64 payload. + */ +export class AppIconService extends PuterService { + declare protected services: LayerInstances; + + #sharp: typeof import('sharp') | null = null; + #dirReady: Promise | null = null; + #ownerUserId: number | null = null; + + override async onServerStart(): Promise { + try { + this.#sharp = (await import('sharp')).default; + } catch { + console.warn( + '[app-icon] sharp not available — icon resizing disabled', + ); + } + + this.#dirReady = this.ensureIconsDirectory(); + + this.clients.event.on( + 'app.new-icon', + async (_key: string, data: unknown) => { + try { + await this.#processIcon(data as Record); + } catch (err) { + console.warn('[app-icon] icon processing failed', err); + } + }, + ); + + // Apps written with a data URL icon outside this pipeline get + // picked up lazily through `app.changed`. Guarded against the + // `icon-migrated` action we emit ourselves. + this.clients.event.on( + 'app.changed', + async (_key: string, data: unknown) => { + const d = data as Record | undefined; + if (!d?.app_uid) return; + if (d.action === 'icon-migrated') return; + const app = await this.stores.app.getByUid(String(d.app_uid)); + const icon = (app as Record | null)?.icon as + | string + | undefined; + if (icon?.startsWith('data:')) { + await this.#processIcon({ + app_uid: d.app_uid, + data_url: icon, + }); + } + }, + ); + } + + /** Public: canonical URL for an app's icon at a given size (CDN/subdomain-backed). */ + getIconUrl(appUid: string, size: number): string | null { + const base = this.#iconsBaseUrl(); + if (!base) return null; + const normalized = appUid.startsWith('app-') ? appUid : `app-${appUid}`; + return `${base}/${SIZED_ICON_FILENAME(normalized, size)}`; + } + + /** Public: URL of the un-resized original PNG (no size suffix) on the subdomain. */ + getOriginalIconUrl(appUid: string): string | null { + const base = this.#iconsBaseUrl(); + if (!base) return null; + const normalized = appUid.startsWith('app-') ? appUid : `app-${appUid}`; + return `${base}/${ORIGINAL_ICON_FILENAME(normalized)}`; + } + + /** + * Pick the best subdomain URL to redirect an icon request at. Falls back + * to the un-resized original when the sized variant hasn't been generated + * (e.g. apps imported with an HTTP icon URL that predates the sharp + * pipeline), preventing 404s on `-.png`. + */ + async resolveIconRedirectUrl( + appUid: string, + size: number, + ): Promise { + const base = this.#iconsBaseUrl(); + if (!base) return null; + const normalized = appUid.startsWith('app-') ? appUid : `app-${appUid}`; + const sizedPath = `${APP_ICONS_PATH_PREFIX}/${SIZED_ICON_FILENAME(normalized, size)}`; + const sizedExists = await this.stores.fsEntry.getEntryByPath(sizedPath); + if (sizedExists) + return `${base}/${SIZED_ICON_FILENAME(normalized, size)}`; + const originalPath = `${APP_ICONS_PATH_PREFIX}/${ORIGINAL_ICON_FILENAME(normalized)}`; + const originalExists = + await this.stores.fsEntry.getEntryByPath(originalPath); + if (originalExists) + return `${base}/${ORIGINAL_ICON_FILENAME(normalized)}`; + return null; + } + + #iconsBaseUrl(): string | null { + const cfg = this.config; + const host = cfg.static_hosting_domain ?? cfg.static_hosting_domain_alt; + if (!host) return null; + const protocol = cfg.protocol ?? 'https'; + // Externally-visible port. Mirrors what PuterHomepageService et al. + // do — non-80/443 deployments (local dev, reverse-proxied setups on + // non-standard ports) would otherwise get a hostname with no port. + const pubPort = cfg.pub_port; + const portSuffix = + pubPort && pubPort !== 80 && pubPort !== 443 ? `:${pubPort}` : ''; + return `${protocol}://${APP_ICONS_SUBDOMAIN}.${host}${portSuffix}`; + } + + // ── Bootstrap ─────────────────────────────────────────────────── + + /** + * Public so `DefaultUserService` can call it immediately after it + * creates the admin user on first boot — otherwise we'd lose the + * race (AppIconService is registered BEFORE DefaultUserService and + * its own `onServerStart` runs when no admin exists yet). Idempotent: + * safe to call repeatedly. + */ + async ensureIconsDirectory(): Promise { + // The admin user owns the icons directory. DefaultUserService + // creates the admin on first boot; if it doesn't exist yet we + // bail and try again the next time an icon is processed. + const adminUser = await this.stores.user.getByUsername('admin'); + if (!adminUser) { + console.warn( + '[app-icon] admin user not found; deferring icons directory setup', + ); + return; + } + this.#ownerUserId = adminUser.id; + + // Ensure /system/app_icons/ exists. + const existing = await this.stores.fsEntry.getEntryByPath( + APP_ICONS_PATH_PREFIX, + ); + let dirEntry = existing; + if (!dirEntry) { + // Write an empty dir by writing a dummy file and removing it + // isn't great — instead rely on `createMissingParents` when we + // write the first icon. We still need a directory entry for + // the subdomain `root_dir_id` though, so create it explicitly + // via the store's directory helper. + dirEntry = await this.stores.fsEntry.resolveParentDirectory( + adminUser.id, + APP_ICONS_PATH_PREFIX, + true, + ); + } + + if (!dirEntry) { + console.warn('[app-icon] failed to ensure icons directory'); + return; + } + + // Register the `puter-app-icons` subdomain pointing at that dir. + // Idempotent: skip if it already exists. + const already = + await this.stores.subdomain.existsBySubdomain(APP_ICONS_SUBDOMAIN); + if (!already) { + await this.stores.subdomain.create({ + userId: adminUser.id, + subdomain: APP_ICONS_SUBDOMAIN, + rootDirId: dirEntry.id ?? null, + }); + } + } + + // ── Icon pipeline ─────────────────────────────────────────────── + + async #processIcon(data: Record): Promise { + if (this.#dirReady) await this.#dirReady; + if (!this.#ownerUserId) { + // Retry the bootstrap — admin may have been created in the + // meantime (e.g. first-boot race). + await this.ensureIconsDirectory(); + if (!this.#ownerUserId) return; + } + if (!this.#sharp) return; // can't resize without sharp + + const dataUrl = (data.dataUrl ?? data.data_url) as string | undefined; + let appUid = (data.appUid ?? data.app_uid) as string | undefined; + if (!dataUrl || !appUid) return; + if (!appUid.startsWith('app-')) appUid = `app-${appUid}`; + + const commaIdx = dataUrl.indexOf(','); + if (commaIdx === -1) return; + const inputBuffer = Buffer.from(dataUrl.slice(commaIdx + 1), 'base64'); + if (inputBuffer.length === 0) return; + + // Write the original alongside the sized variants so the CDN-backed + // subdomain serves everything through the same path. + const writes: Array> = []; + + const originalPng = await this.#sharp(inputBuffer).png().toBuffer(); + writes.push( + this.#writeIcon(ORIGINAL_ICON_FILENAME(appUid), originalPng), + ); + + for (const size of ICON_SIZES) { + const sizedPng = await this.#sharp(inputBuffer) + .resize(size) + .png() + .toBuffer(); + writes.push( + this.#writeIcon(SIZED_ICON_FILENAME(appUid, size), sizedPng), + ); + } + await Promise.all(writes); + + // Rewrite the DB icon column from data URL to canonical endpoint URL. + // The endpoint URL is `/app-icon/` — the AppController route + // that falls back to the data URL if S3/CDN lookups miss. Using it + // here keeps the icon column small and makes clients go through + // the cached path. + const apiBase = String(this.config.api_base_url ?? '').replace( + /\/+$/, + '', + ); + if (apiBase) { + await this.clients.db.write( + "UPDATE `apps` SET `icon` = ? WHERE `uid` = ? AND `icon` LIKE 'data:%'", + [`${apiBase}/app-icon/${appUid}`, appUid], + ); + await this.stores.app.invalidateByUid(appUid); + this.clients.event.emit( + 'app.changed', + { + app_uid: appUid, + action: 'icon-migrated', + }, + {}, + ); + } + } + + async #writeIcon(filename: string, buffer: Buffer): Promise { + if (!this.#ownerUserId) return; + await this.services.fs.write(this.#ownerUserId, { + fileMetadata: { + path: `${APP_ICONS_PATH_PREFIX}/${filename}`, + size: buffer.length, + contentType: 'image/png', + overwrite: true, + createMissingParents: true, + }, + fileContent: Readable.from(buffer), + }); + } +} diff --git a/src/backend/services/apps/AppPermissionService.ts b/src/backend/services/apps/AppPermissionService.ts new file mode 100644 index 000000000..da65cf028 --- /dev/null +++ b/src/backend/services/apps/AppPermissionService.ts @@ -0,0 +1,220 @@ +import { Context } from '../../core/context.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import { + MANAGE_PERM_PREFIX, + PERMISSION_FOR_NOTHING_IN_PARTICULAR, +} from '../permission/consts.js'; +import { PermissionUtil } from '../permission/permissionUtil.js'; +import type { LayerInstances } from '../../types.js'; +import type { puterStores } from '../../stores/index.js'; +import type { puterServices } from '../index.js'; +import { PuterService } from '../types.js'; + +/** + * Permission rewriters / implicators for the `app:*`, `apps-of-user:*`, + * `subdomains-of-user:*`, and `app-root-dir:*` namespaces. + * + * Ports three v1 services (ProtectedAppService, AppPermissionService, the + * app-root-dir arm of AppService) into one domain-scoped service. Nothing + * here needs to live beyond init — the registrations are stateless. + */ +export class AppPermissionService extends PuterService { + declare protected stores: LayerInstances; + declare protected services: LayerInstances; + + override onServerStart(): void { + const permissions = this.services.permission; + const appStore = this.stores.app; + + // ── app::mode → app:uid#:mode ─────────────────────── + // Names change (via app rename); uids are stable. Store/scan uid + // form so renames don't invalidate existing grants. AppStore caches + // `getByName` in Redis (5m), invalidated on rename/update. + permissions.registerRewriter({ + id: 'app-name-to-uid', + matches: (permission: string) => { + if (!permission.startsWith('app:')) return false; + const [, specifier] = PermissionUtil.split(permission); + return Boolean(specifier && !specifier.startsWith('uid#')); + }, + rewrite: async (permission: string): Promise => { + const [prefix, name, ...rest] = + PermissionUtil.split(permission); + const app = await appStore.getByName(name); + if (!app || typeof app.uid !== 'string') return permission; + return PermissionUtil.join(prefix, `uid#${app.uid}`, ...rest); + }, + }); + + // ── app-is-owner implicator ─────────────────────────────────── + // User actors implicitly hold `app:uid#X:*` (and manage form) on + // apps they own. Mirrors the fs is-owner pattern. + permissions.registerImplicator({ + id: 'app-is-owner', + matches: (permission: string) => { + return ( + permission.startsWith('app:') || + permission.startsWith(`${MANAGE_PERM_PREFIX}:app:`) + ); + }, + check: async ({ actor, permission }): Promise => { + if (actor.app || actor.accessToken) return undefined; + if (!actor.user?.id) return undefined; + + const parts = PermissionUtil.split(permission); + if (parts[0] === MANAGE_PERM_PREFIX) parts.shift(); + if (parts.length < 2) return undefined; + const specifier = parts[1]; + if (!specifier.startsWith('uid#')) return undefined; + const uid = specifier.slice('uid#'.length); + if (!uid) return undefined; + + const app = await appStore.getByUid(uid); + if (!app) return undefined; + const ownerId = (app as { owner_user_id?: number }) + .owner_user_id; + if (ownerId === actor.user.id) return {}; + return undefined; + }, + }); + + // ── apps-of-user::* / subdomains-of-user:… ────────── + // A user implicitly holds read/write over *their own* apps and + // subdomains. `puter.perms` expresses these as + // `apps-of-user::` etc. + permissions.registerImplicator({ + id: 'user-can-grant-read-own-apps', + matches: (permission: string) => { + return ( + permission.startsWith('apps-of-user:') || + permission.startsWith('subdomains-of-user:') + ); + }, + check: async ({ actor, permission }): Promise => { + if (actor.app || actor.accessToken) return undefined; + if (!actor.user?.uuid) return undefined; + const parts = PermissionUtil.split(permission); + if (parts[1] === actor.user.uuid) return {}; + return undefined; + }, + }); + + // ── app-root-dir:: → fs:: ─────── + // Only rewrites during an explicit `grantUserAppPermission` (see + // PermissionService for the context flag). During scans we return + // PERMISSION_FOR_NOTHING_IN_PARTICULAR so `check(actor, 'app-root-dir:…')` + // never accidentally matches through the fs-permission path. + permissions.registerRewriter({ + id: 'app-root-dir-to-fs', + matches: (permission: string) => + permission.startsWith('app-root-dir:'), + rewrite: async (permission: string): Promise => { + if (!Context.get('is_grant_user_app_permission')) { + return PERMISSION_FOR_NOTHING_IN_PARTICULAR; + } + const actor = Context.get('actor'); + if (!actor || actor.app || actor.accessToken) { + throw new HttpError(403, 'Forbidden'); + } + if (!actor.user?.id) { + throw new HttpError(403, 'Forbidden'); + } + + const parts = PermissionUtil.split(permission); + if (parts.length < 3) { + throw new HttpError( + 400, + 'Invalid `app-root-dir` permission', + ); + } + const [, targetAppUid, access, ...rest] = parts; + if (!targetAppUid) { + throw new HttpError(400, 'Missing target_app_uid'); + } + + const targetApp = await appStore.getByUid(targetAppUid); + if (!targetApp) { + throw new HttpError( + 404, + `Entry not found: app=${targetAppUid}`, + { legacyCode: 'subject_does_not_exist' }, + ); + } + if ( + (targetApp as { owner_user_id?: number }).owner_user_id !== + actor.user.id + ) { + throw new HttpError(403, 'Forbidden'); + } + + const rootDirId = await this.#resolveAppRootDirId( + targetApp as { + id: number; + uid: string; + index_url?: string; + }, + ); + if (rootDirId === null) { + throw new HttpError( + 404, + `Entry not found: app root dir for ${targetAppUid}`, + { legacyCode: 'subject_does_not_exist' }, + ); + } + const entry = await this.stores.fsEntry.getEntryById(rootDirId); + if (!entry) { + throw new HttpError( + 404, + `Entry not found: app root dir for ${targetAppUid}`, + { legacyCode: 'subject_does_not_exist' }, + ); + } + return PermissionUtil.join('fs', entry.uuid, access, ...rest); + }, + }); + } + + /** + * Resolve an app's filesystem root directory id. Ported from v1's + * AppService.getAppRootDirId — first checks the canonical + * `subdomains.associated_app_id` binding, then falls back to parsing + * the hosting subdomain out of `app.index_url` and resolving it. + */ + async #resolveAppRootDirId(app: { + id: number; + uid: string; + index_url?: string; + }): Promise { + const rows = (await this.clients.db.read( + 'SELECT root_dir_id FROM subdomains WHERE associated_app_id = ? AND root_dir_id IS NOT NULL LIMIT 1', + [app.id], + )) as Array<{ root_dir_id: number | null }>; + const direct = rows[0]?.root_dir_id; + if (direct !== undefined && direct !== null) { + return Number(direct); + } + + const hostingDomain = ( + this.config as { static_hosting_domain?: string } + ).static_hosting_domain?.toLowerCase(); + if (!hostingDomain || !app.index_url) return null; + + let hostname: string; + try { + hostname = new URL(app.index_url).hostname.toLowerCase(); + } catch { + return null; + } + if (!hostname.endsWith(`.${hostingDomain}`)) return null; + + const subdomain = hostname.slice( + 0, + hostname.length - hostingDomain.length - 1, + ); + const row = (await this.stores.subdomain.getBySubdomain(subdomain)) as { + root_dir_id?: number | null; + } | null; + if (!row?.root_dir_id) return null; + return Number(row.root_dir_id); + } +} diff --git a/src/backend/services/apps/RecommendedAppsService.ts b/src/backend/services/apps/RecommendedAppsService.ts new file mode 100644 index 000000000..92d9dda91 --- /dev/null +++ b/src/backend/services/apps/RecommendedAppsService.ts @@ -0,0 +1,54 @@ +import { getAppIconUrl } from '../../util/appIcon.js'; +import { PuterService } from '../types.js'; + +/** + * Hardcoded list of recommended apps shown on the desktop launch grid. + * Resolved at call time against the apps table. + */ +const RECOMMENDED_APP_NAMES = [ + 'app-center', + 'dev-center', + 'editor', + 'code', + 'camera', + 'music-player', + 'recorder', + 'memos', + 'word-processor', + 'spreadsheet', + 'presentation', + 'pdf-editor', + 'basketball-tap', + 'blockup', + 'pretty-tiles', + 'galaxy-troops', + 'blend-fruits', + 'traffic-tap-puzzle', +]; + +export class RecommendedAppsService extends PuterService { + async getRecommendedApps(): Promise>> { + const apiBaseUrl = this.config.api_base_url as string | undefined; + const results: Array> = []; + for (const name of RECOMMENDED_APP_NAMES) { + const app = await this.stores.app.getByName(name); + if (app) results.push(toAppSummary(app, apiBaseUrl)); + } + return results; + } +} + +function toAppSummary( + app: Record, + apiBaseUrl: string | undefined, +): Record { + return { + uuid: app.uid, + name: app.name, + title: app.title, + icon: getAppIconUrl(app, { apiBaseUrl }) ?? app.icon ?? null, + godmode: Boolean(app.godmode), + maximize_on_start: Boolean(app.maximize_on_start), + index_url: app.index_url, + }; +} diff --git a/src/backend/services/apps/SuggestedAppsService.ts b/src/backend/services/apps/SuggestedAppsService.ts new file mode 100644 index 000000000..bcd26f974 --- /dev/null +++ b/src/backend/services/apps/SuggestedAppsService.ts @@ -0,0 +1,276 @@ +import { posix as pathPosix } from 'node:path'; +import { getAppIconUrl } from '../../util/appIcon.js'; +import { PuterService } from '../types.js'; + +// ── Extension → suggested app names mapping ───────────────────────── +// +// Each extension maps to an ordered list of built-in app names that +// can open files of that type. + +const CODE_EXTS = new Set([ + 'js', + 'jsx', + 'ts', + 'tsx', + 'json', + 'json5', + 'jsonl', + 'css', + 'scss', + 'sass', + 'less', + 'html', + 'htm', + 'xhtml', + 'xml', + 'svg', + 'yaml', + 'yml', + 'toml', + 'ini', + 'conf', + 'cfg', + 'env', + 'sh', + 'bash', + 'zsh', + 'fish', + 'bat', + 'cmd', + 'ps1', + 'py', + 'pyw', + 'rb', + 'php', + 'pl', + 'pm', + 'lua', + 'java', + 'kt', + 'kts', + 'scala', + 'groovy', + 'go', + 'rs', + 'c', + 'h', + 'cpp', + 'hpp', + 'cc', + 'cxx', + 'cs', + 'swift', + 'r', + 'jl', + 'ex', + 'exs', + 'erl', + 'hrl', + 'clj', + 'cljs', + 'hs', + 'ml', + 'mli', + 'fs', + 'fsi', + 'fsx', + 'dart', + 'sql', + 'graphql', + 'gql', + 'proto', + 'makefile', + 'cmake', + 'dockerfile', + 'tf', + 'hcl', + 'nix', + 'vim', + 'el', + 'lisp', + 'rkt', + 'scm', + 'asm', + 's', + 'wasm', + 'wat', + 'v', + 'vhd', + 'vhdl', + 'tcl', +]); + +const IMAGE_EXTS = new Set([ + 'jpg', + 'jpeg', + 'png', + 'gif', + 'webp', + 'svg', + 'bmp', + 'ico', + 'tiff', + 'tif', +]); +const MEDIA_EXTS = new Set([ + 'mp4', + 'webm', + 'mpg', + 'mpeg', + 'avi', + 'mov', + 'mkv', + 'mp3', + 'm4a', + 'ogg', + 'wav', + 'flac', + 'aac', +]); + +function suggestionsForExtension(ext: string): string[] { + const lower = ext.toLowerCase(); + if (CODE_EXTS.has(lower)) return ['code', 'editor']; + if (lower === 'txt' || lower === '') return ['editor', 'code']; + if (lower === 'md') return ['markus', 'editor', 'code']; + if (IMAGE_EXTS.has(lower)) return ['viewer', 'draw']; + if (lower === 'pdf') return ['pdf']; + if (MEDIA_EXTS.has(lower)) return ['player']; + // Unknown extension — fall back to editor + return ['editor']; +} + +// In-memory cache TTL. Apps rarely change, and the worst-case on staleness +// is a few minutes before a new filetype association surfaces — not worth a +// Redis round-trip per lookup on a hot path (readdir fans out per-child). +const SUGGESTION_CACHE_TTL_MS = 5 * 60 * 1000; + +type SuggestionsEntry = { + promise: Promise>>; + expiresAt: number; +}; + +function extractExtension(entry: { name?: string; path?: string }): string { + const name = + entry.name ?? (entry.path ? pathPosix.basename(entry.path) : ''); + return pathPosix.extname(name).replace(/^\./, '').toLowerCase(); +} + +/** + * Given a file entry (path, name, or extension), returns an ordered list + * of apps that can open it. Built-in apps come from the hardcoded map + * above; third-party apps come from the `app_filetype_association` table. + * + * Lookups cache per-extension (plus a separate per-app-name cache for the + * small set of built-in opener apps), so a `readdir` with N children of + * the same type pays the DB cost once. + */ +export class SuggestedAppsService extends PuterService { + // Keyed by the normalized extension (lowercase, no leading dot). The + // cached value is the promise — in-flight lookups coalesce, and the + // same promise is reused for every entry that shares an extension. + #extensionCache = new Map(); + + async getSuggestedApps(entry: { + name?: string; + path?: string; + }): Promise>> { + return this.#getByExtension(extractExtension(entry)); + } + + /** + * Resolve suggestions for many entries in one pass. Entries that share + * an extension are deduped to a single underlying lookup; results are + * returned positionally so callers can `entries[i].suggestedApps = out[i]`. + */ + async getSuggestedAppsForEntries( + entries: Array<{ name?: string; path?: string }>, + ): Promise>>> { + if (entries.length === 0) return []; + + const extensions = entries.map(extractExtension); + const uniqueExtensions = Array.from(new Set(extensions)); + const resultByExt = new Map>>(); + + await Promise.all( + uniqueExtensions.map(async (ext) => { + resultByExt.set(ext, await this.#getByExtension(ext)); + }), + ); + + return extensions.map((ext) => resultByExt.get(ext) ?? []); + } + + #getByExtension(ext: string): Promise>> { + const now = Date.now(); + const cached = this.#extensionCache.get(ext); + if (cached && cached.expiresAt > now) { + return cached.promise; + } + + const promise = this.#resolveForExtension(ext).catch((error) => { + // Failure must not poison the cache — drop the entry so the + // next caller retries. + if (this.#extensionCache.get(ext)?.promise === promise) { + this.#extensionCache.delete(ext); + } + throw error; + }); + this.#extensionCache.set(ext, { + promise, + expiresAt: now + SUGGESTION_CACHE_TTL_MS, + }); + return promise; + } + + async #resolveForExtension( + ext: string, + ): Promise>> { + const builtinNames = suggestionsForExtension(ext); + + const seen = new Set(); + const results: Array> = []; + + const apiBaseUrl = this.config.api_base_url as string | undefined; + + // Built-in apps, looked up by their stable app name. Parallel-safe + // because order is imposed at the end via `builtinNames`. + const builtinApps = await Promise.all( + builtinNames.map((appName) => this.stores.app.getByName(appName)), + ); + for (const app of builtinApps) { + if (app && !seen.has(app.id)) { + seen.add(app.id); + results.push(toAppSummary(app, apiBaseUrl)); + } + } + + if (ext) { + const thirdParty = await this.stores.app.getAppsByFiletype(ext); + for (const app of thirdParty) { + if (seen.has(app.id)) continue; + if (app.approved_for_opening_items) { + seen.add(app.id); + results.push(toAppSummary(app, apiBaseUrl)); + } + } + } + + return results; + } +} + +function toAppSummary( + app: Record, + apiBaseUrl: string | undefined, +): Record { + return { + uuid: app.uid, + name: app.name, + title: app.title, + icon: getAppIconUrl(app, { apiBaseUrl }) ?? app.icon ?? null, + godmode: Boolean(app.godmode), + maximize_on_start: Boolean(app.maximize_on_start), + index_url: app.index_url, + }; +} diff --git a/src/backend/services/auth/AuthService.test.ts b/src/backend/services/auth/AuthService.test.ts new file mode 100644 index 000000000..11fdaf0d4 --- /dev/null +++ b/src/backend/services/auth/AuthService.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest'; +import type { Actor } from '../../core/actor.js'; +import { AuthService } from './AuthService.js'; + +function createAuthService(): AuthService { + const [config, clients, stores, services] = [ + {}, + {}, + {}, + {}, + ] as ConstructorParameters; + return new AuthService(config, clients, stores, services); +} + +describe('AuthService.createAccessToken', () => { + it('rejects access-token actors so scoped tokens cannot mint broader tokens', async () => { + const authService = createAuthService(); + const issuer: Actor = { + user: { + uuid: 'user-issuer', + id: 1, + username: 'issuer', + }, + }; + const actor: Actor = { + user: { + uuid: 'user-issuer', + id: 1, + username: 'issuer', + }, + accessToken: { + uid: 'token-existing', + issuer, + authorized: null, + }, + }; + + await expect( + authService.createAccessToken(actor, [['fs:abc:read']]), + ).rejects.toMatchObject({ + statusCode: 403, + legacyCode: 'forbidden', + }); + }); +}); diff --git a/src/backend/services/auth/AuthService.ts b/src/backend/services/auth/AuthService.ts new file mode 100644 index 000000000..6f2897e30 --- /dev/null +++ b/src/backend/services/auth/AuthService.ts @@ -0,0 +1,792 @@ +import { v4 as uuidv4, v5 as uuidv5 } from 'uuid'; +import type { Actor } from '../../core/actor'; +import { HttpError } from '../../core/http/HttpError.js'; +import type { UserRow } from '../../stores/user/UserStore'; +import type { LayerInstances } from '../../types'; +import type { puterServices } from '../index'; +import { PuterService } from '../types'; +import type { + AccessTokenPayload, + AnyTokenPayload, + AppUnderUserTokenPayload, + SessionRow, + SessionTokenPayload, +} from './types'; + +const APP_ORIGIN_UUID_NAMESPACE = '33de3768-8ee0-43e9-9e73-db192b97a5d8'; + +/** + * Authentication service. + * + * Scope is currently narrow — just `authenticateFromToken`, the one method + * the auth-probe middleware needs. Session creation, logout, token + * rotation, 2FA, and the rest of the auth surface will land when the auth + * controller is wired up (it will own mint / rotate / revoke). + */ +export class AuthService extends PuterService { + declare protected services: LayerInstances; + + override onServerStart(): void { + // Users implicitly hold read access to their own email — needed for + // any permission-gated path that asks for `user::email:read` + // (puter-js's `user::email:read` permission request flows + // through the scan even though the v2 whoami extension inlines the + // email field directly and skips the check). + this.services.permission.registerImplicator({ + id: 'user-set-own', + shortcut: true, + matches: (permission: string) => permission.startsWith('user:'), + check: async ({ actor, permission }): Promise => { + if (actor.app || actor.accessToken) return undefined; + if (!actor.user?.uuid) return undefined; + if (permission === `user:${actor.user.uuid}:email:read`) { + return {}; + } + return undefined; + }, + }); + } + + // ── Public API ────────────────────────────────────────────────── + + /** + * Resolve an auth token to a v2 Actor. + * + * Returns `null` for *any* failure — invalid signature, malformed payload, + * legacy token shape, missing session, missing user, missing app. The + * caller (auth probe) never differentiates: it either attaches an actor + * or leaves `req.actor` undefined for per-route gates to reject. + */ + async authenticateFromToken(token: string): Promise { + let decoded: AnyTokenPayload; + try { + decoded = this.services.token.verify( + 'auth', + token, + ); + } catch { + return null; + } + + // Legacy tokens (pre-`type` field) aren't supported. + if (!decoded.type) return null; + + switch (decoded.type) { + case 'session': + case 'gui': + return this.#actorFromSessionToken(decoded); + case 'app-under-user': + return this.#actorFromAppUnderUserToken(decoded); + case 'access-token': + return this.#actorFromAccessTokenToken(decoded); + default: + return null; + } + } + + // ── Session lifecycle ──────────────────────────────────────────── + + /** + * Create a session and sign a session JWT + GUI JWT for the user. + * + * `meta` is enriched with request metadata (IP, user-agent, etc.) + * when a request context is available. + */ + async createSessionToken( + user: UserRow, + meta: Record = {}, + ): Promise<{ + session: Record; + token: string; + gui_token: string; + }> { + const session = await this.stores.session.create(user.id, meta); + + const token = this.services.token.sign('auth', { + type: 'session', + version: '0.0.0', + uuid: session.uuid, + user_uid: user.uuid, + }); + + const gui_token = this.services.token.sign('auth', { + type: 'gui', + version: '0.0.0', + uuid: session.uuid, + user_uid: user.uuid, + }); + + return { session, token, gui_token }; + } + + /** Sign a GUI token for an existing session. */ + createGuiToken(user: UserRow, sessionUuid: string): string { + return this.services.token.sign('auth', { + type: 'gui', + version: '0.0.0', + uuid: sessionUuid, + user_uid: user.uuid, + }); + } + + /** Sign a session token for an existing session (upgrade from GUI token). */ + createSessionTokenForSession(user: UserRow, sessionUuid: string): string { + return this.services.token.sign('auth', { + type: 'session', + version: '0.0.0', + uuid: sessionUuid, + user_uid: user.uuid, + }); + } + + /** Remove the session referenced by a session/GUI JWT. */ + async removeSessionByToken(token: string): Promise { + let decoded: AnyTokenPayload; + try { + decoded = this.services.token.verify( + 'auth', + token, + ); + } catch { + return; + } + if (decoded.type !== 'session' && decoded.type !== 'gui') return; + await this.stores.session.removeByUuid( + (decoded as SessionTokenPayload).uuid, + ); + } + + /** List all sessions for an actor's user. */ + async listSessions(actor: Actor): Promise>> { + if (!actor.user?.id) return []; + + const rows = await this.stores.session.getByUserId(actor.user.id); + + return rows.map((row: Record) => { + const meta = + (typeof row.meta === 'string' + ? JSON.parse(row.meta as string) + : row.meta) ?? {}; + const isCurrent = actor.session?.uid === row.uuid; + return { + uuid: row.uuid, + created_at: row.created_at, + last_activity: row.last_activity, + current: isCurrent, + ...meta, + }; + }); + } + + /** Revoke a specific session by uuid. */ + async revokeSession(uuid: string): Promise { + await this.stores.session.removeByUuid(uuid); + } + + // ── App / origin resolution ───────────────────────────────────── + + /** + * Resolve an origin URL to an app UID. + * + * Fires `app.from-origin` before hashing so listeners can rewrite the + * origin (e.g. polotno maps `polotno.com` → `studio.polotno.com` so both + * surfaces resolve to the same app row). + * + * Lookup order: + * 1. **Canonical DB match.** If any app in the DB has an `index_url` + * that normalizes to this origin (across every configured hosting + * variant — `puter.site`, `puter.app`, etc.), return that app's + * real UID. Required for private apps: their `/auth/get-user-app-token` + * tokens must reference the real app row so the app-under-user + * verification path can load them. + * 2. **UUIDv5 deterministic fallback.** Origins that don't match any + * app (third-party sites, apps not yet in the DB) get a deterministic + * namespaced UUID + */ + async appUidFromOrigin(origin: string): Promise { + const parsed = this.#originFromUrl(origin); + if (!parsed) { + console.error('[auth] failed to parse origin URL', { origin }); + throw new HttpError(400, 'Invalid origin URL', { + legacyCode: 'no_origin_for_app', + }); + } + const event = { origin: parsed }; + await this.clients.event?.emitAndWait('app.from-origin', event, {}); + + const canonicalUid = await this.#findCanonicalAppUidForOrigin( + event.origin, + ); + if (canonicalUid) return canonicalUid; + + const uid = uuidv5(event.origin, APP_ORIGIN_UUID_NAMESPACE); + return `app-${uid}`; + } + + /** + * Find the real app row whose `index_url` canonically matches `origin`. + * + * Build candidate URLs from the origin's subdomain crossed with every + * configured hosting domain (static + private, with and without ports). + * Prefer the oldest matching app for deterministic tie-breaking across + * historically-duplicated rows. + */ + async #findCanonicalAppUidForOrigin( + origin: string, + ): Promise { + let parsed: URL; + try { + parsed = new URL(origin); + } catch { + return null; + } + + const config = this.config as { + static_hosting_domain?: string; + static_hosting_domain_alt?: string; + private_app_hosting_domain?: string; + private_app_hosting_domain_alt?: string; + protocol?: string; + }; + + const normalizeDomainValue = (v: unknown): string | null => { + if (typeof v !== 'string') return null; + const trimmed = v.trim().toLowerCase().replace(/^\./, ''); + return trimmed || null; + }; + const stripPort = (v: string): string => v.split(':')[0] || v; + + const hostingDomainsRaw = [ + normalizeDomainValue(config.static_hosting_domain), + normalizeDomainValue(config.static_hosting_domain_alt), + normalizeDomainValue(config.private_app_hosting_domain), + normalizeDomainValue(config.private_app_hosting_domain_alt), + ].filter((d): d is string => !!d); + const hostingDomainsStripped = hostingDomainsRaw.map(stripPort); + const hostingDomains = [ + ...new Set([...hostingDomainsRaw, ...hostingDomainsStripped]), + ]; + + const hostRaw = parsed.host.toLowerCase(); + const hostStripped = parsed.hostname.toLowerCase(); + + // Extract the subdomain label under the longest matching hosting + // domain — longest-first avoids matching `puter.app` before + // `foo.puter.app`. + let subdomain: string | null = null; + const sortedHostingDomains = [...hostingDomains].sort( + (a, b) => b.length - a.length, + ); + for (const d of sortedHostingDomains) { + const suffix = `.${d}`; + if (hostRaw === d || hostStripped === d) { + subdomain = null; + break; + } + if (hostRaw.endsWith(suffix)) { + subdomain = hostRaw.slice(0, hostRaw.length - suffix.length); + subdomain = subdomain.split('.')[0] || null; + break; + } + if (hostStripped.endsWith(suffix)) { + subdomain = hostStripped.slice( + 0, + hostStripped.length - suffix.length, + ); + subdomain = subdomain.split('.')[0] || null; + break; + } + } + + const hostCandidates = new Set([hostRaw, hostStripped]); + if (subdomain) { + for (const d of hostingDomains) { + hostCandidates.add(`${subdomain}.${d}`); + } + } + + const protocolCandidates = new Set([ + parsed.protocol.replace(/:$/, ''), + (config.protocol ?? '').trim().replace(/:$/, '') || 'https', + 'https', + 'http', + ]); + + const urlCandidates: string[] = []; + for (const hc of hostCandidates) { + if (!hc) continue; + for (const protocol of protocolCandidates) { + if (!protocol) continue; + const base = `${protocol}://${hc}`; + urlCandidates.push(base, `${base}/`, `${base}/index.html`); + } + } + const uniqueCandidates = [...new Set(urlCandidates)]; + if (uniqueCandidates.length === 0) return null; + + const placeholders = uniqueCandidates.map(() => '?').join(', '); + const rows = (await this.clients.db.read( + `SELECT \`uid\` FROM \`apps\` WHERE \`index_url\` IN (${placeholders}) ORDER BY \`id\` ASC LIMIT 1`, + uniqueCandidates, + )) as Array<{ uid?: string }>; + const uid = rows[0]?.uid; + return typeof uid === 'string' && uid ? uid : null; + } + + /** + * Sign an app-under-user token for the given app UID. + * Requires a user actor in the provided actor. + */ + getUserAppToken(actor: Actor, appUid: string): string { + if (!actor.user) throw new Error('Actor must be a user'); + return this.services.token.sign('auth', { + type: 'app-under-user', + version: '0.0.0', + user_uid: actor.user.uuid, + app_uid: appUid, + ...(actor.session ? { session: actor.session.uid } : {}), + }); + } + + // ── Private / public hosted asset cookies ─────────────────────── + // + // Ported from v1's `createPrivateAssetToken` / `createPublicHostedActor + // Token`. These are sticky cookies set by the puter-site middleware + // after a visitor successfully passes the private-app access gate + // (or is resolved as an actor on a public hosted app). Subsequent + // requests read the cookie and skip the full entitlement lookup. + // + // Claims are kept narrow — userUid + sessionUuid + appUid + subdomain + // + privateHost — so a cookie minted for one app/subdomain cannot be + // replayed against another. `verify*Token` enforces those expectations. + + /** Cookie name that carries the sticky private-asset token. */ + getPrivateAssetCookieName(): string { + return 'puter.private.asset.token'; + } + + /** Cookie name that carries the public hosted-actor token. */ + getPublicHostedActorCookieName(): string { + return 'puter.public.hosted.actor.token'; + } + + /** Shared cookie options for both sticky-auth cookies. */ + getPrivateAssetCookieOptions( + opts: { + requestHostname?: string; + } = {}, + ): Record { + return this.#hostedAssetCookieOptions(opts.requestHostname); + } + + /** Alias — matching v1's naming. Same options used by both cookies. */ + getPublicHostedActorCookieOptions( + opts: { + requestHostname?: string; + } = {}, + ): Record { + return this.#hostedAssetCookieOptions(opts.requestHostname); + } + + createPrivateAssetToken(claims: { + appUid: string; + userUid: string; + sessionUuid?: string; + subdomain?: string; + privateHost?: string; + }): string { + return this.services.token.sign('hosted-asset', { + kind: 'private', + version: '0.0.0', + user_uid: claims.userUid, + app_uid: claims.appUid, + ...(claims.sessionUuid ? { session_uuid: claims.sessionUuid } : {}), + ...(claims.subdomain ? { subdomain: claims.subdomain } : {}), + ...(claims.privateHost ? { host: claims.privateHost } : {}), + }); + } + + createPublicHostedActorToken(claims: { + appUid: string; + userUid: string; + sessionUuid?: string; + subdomain?: string; + host?: string; + }): string { + return this.services.token.sign('hosted-asset', { + kind: 'public', + version: '0.0.0', + user_uid: claims.userUid, + app_uid: claims.appUid, + ...(claims.sessionUuid ? { session_uuid: claims.sessionUuid } : {}), + ...(claims.subdomain ? { subdomain: claims.subdomain } : {}), + ...(claims.host ? { host: claims.host } : {}), + }); + } + + verifyPrivateAssetToken( + token: string, + expected: { + expectedAppUid?: string; + expectedSubdomain?: string; + expectedPrivateHost?: string; + } = {}, + ): { + userUid: string; + sessionUuid?: string; + appUid?: string; + subdomain?: string; + privateHost?: string; + } { + const decoded = this.#verifyHostedAssetToken(token, 'private'); + this.#assertExpected( + decoded, + 'app_uid', + expected.expectedAppUid, + 'expectedAppUid', + ); + this.#assertExpected( + decoded, + 'subdomain', + expected.expectedSubdomain, + 'expectedSubdomain', + ); + this.#assertExpected( + decoded, + 'host', + expected.expectedPrivateHost, + 'expectedPrivateHost', + ); + return { + userUid: decoded.user_uid as string, + sessionUuid: decoded.session_uuid as string | undefined, + appUid: decoded.app_uid as string | undefined, + subdomain: decoded.subdomain as string | undefined, + privateHost: decoded.host as string | undefined, + }; + } + + verifyPublicHostedActorToken( + token: string, + expected: { + expectedAppUid?: string; + expectedSubdomain?: string; + expectedHost?: string; + } = {}, + ): { + userUid: string; + sessionUuid?: string; + appUid?: string; + subdomain?: string; + host?: string; + } { + const decoded = this.#verifyHostedAssetToken(token, 'public'); + this.#assertExpected( + decoded, + 'app_uid', + expected.expectedAppUid, + 'expectedAppUid', + ); + this.#assertExpected( + decoded, + 'subdomain', + expected.expectedSubdomain, + 'expectedSubdomain', + ); + this.#assertExpected( + decoded, + 'host', + expected.expectedHost, + 'expectedHost', + ); + return { + userUid: decoded.user_uid as string, + sessionUuid: decoded.session_uuid as string | undefined, + appUid: decoded.app_uid as string | undefined, + subdomain: decoded.subdomain as string | undefined, + host: decoded.host as string | undefined, + }; + } + + #verifyHostedAssetToken( + token: string, + expectedKind: 'private' | 'public', + ): Record { + const decoded = this.services.token.verify>( + 'hosted-asset', + token, + ); + if (decoded.kind !== expectedKind) { + throw new Error(`hosted-asset token is not ${expectedKind}`); + } + if (typeof decoded.user_uid !== 'string' || !decoded.user_uid) { + throw new Error('hosted-asset token missing user_uid'); + } + return decoded; + } + + #assertExpected( + decoded: Record, + field: string, + expected: string | undefined, + label: string, + ): void { + if (expected === undefined) return; + if (decoded[field] !== expected) { + throw new Error(`hosted-asset token ${label} mismatch`); + } + } + + #hostedAssetCookieOptions( + requestHostname?: string, + ): Record { + // Scope the cookie to the request host only. Not using `domain` + // so the browser doesn't share it across unrelated private-app + // subdomains — each app sees only its own cookie. + const options: Record = { + httpOnly: true, + secure: true, + sameSite: 'none', + maxAge: 7 * 24 * 60 * 60 * 1000, // 7 days + path: '/', + }; + if (requestHostname) { + // Not strictly necessary (browsers default to the response + // origin when `domain` is absent), but included for clarity + // in server logs. + options.hostname = requestHostname; + } + return options; + } + + // ── Access tokens ─────────────────────────────────────────────── + + /** + * Create an access token with the given permissions. + * + * Each permission spec is `[permissionString, extraObject?]`. + * The token is stored in `access_token_permissions` and a JWT is + * returned. + */ + async createAccessToken( + actor: Actor, + permissions: Array<[string, Record?]>, + options: { expiresIn?: string } = {}, + ): Promise { + if (!actor.user) throw new Error('Actor must have a user'); + if (actor.accessToken) { + throw new HttpError( + 403, + 'Access tokens may not create access tokens', + { + legacyCode: 'forbidden', + }, + ); + } + + const tokenUid = uuidv4(); + const jwtPayload: Record = { + type: 'access-token', + version: '0.0.0', + token_uid: tokenUid, + user_uid: actor.user.uuid, + }; + if (actor.app) { + jwtPayload.app_uid = actor.app.uid; + } + + const jwt = this.services.token.sign('auth', jwtPayload, options); + + // Store each permission grant + const db = this.stores.permission as unknown as { + clients: { + db: { write: (q: string, p: unknown[]) => Promise }; + }; + }; + for (const spec of permissions) { + const [permission, extra] = spec; + await (db.clients?.db ?? this.clients.db).write( + 'INSERT INTO `access_token_permissions` (`token_uid`, `authorizer_user_id`, `authorizer_app_id`, `permission`, `extra`) VALUES (?, ?, ?, ?, ?)', + [ + tokenUid, + actor.user.id ?? null, + actor.app?.id ?? null, + permission, + extra ? JSON.stringify(extra) : '{}', + ], + ); + } + await this.stores.permission.invalidateAccessTokenPerms(tokenUid); + + return jwt; + } + + /** + * Revoke an access token by JWT or token UUID. + * + * Caller must be a user actor (gated at the route). Ownership is verified + * before deletion so one user cannot revoke another user's token by + * guessing/leaking the token_uid. + */ + async revokeAccessToken(actor: Actor, tokenOrUuid: string): Promise { + if (!actor.user) throw new Error('Actor must have a user'); + + let tokenUid: string; + let issuerUuidFromJwt: string | undefined; + const isJwt = /^[\w-]+\.[\w-]+\.[\w-]+$/.test(tokenOrUuid.trim()); + if (isJwt) { + const decoded = this.services.token.verify( + 'auth', + tokenOrUuid, + ); + if (decoded.type !== 'access-token' || !decoded.token_uid) { + throw new HttpError(400, 'Invalid access token'); + } + tokenUid = decoded.token_uid; + issuerUuidFromJwt = decoded.user_uid; + } else { + tokenUid = tokenOrUuid; + } + + // A signature-verified JWT is itself proof of who issued the token — + // the body's `user_uid` was set by createAccessToken at mint time. + // For raw-uuid input we fall back to the persisted authorizer. + if (issuerUuidFromJwt !== undefined) { + if (issuerUuidFromJwt !== actor.user.uuid) { + throw new HttpError(404, 'Access token not found'); + } + } else { + const rows = (await this.clients.db.read( + 'SELECT `authorizer_user_id` FROM `access_token_permissions` WHERE `token_uid` = ? LIMIT 1', + [tokenUid], + )) as Array<{ authorizer_user_id?: number | null }>; + const ownerId = rows[0]?.authorizer_user_id ?? null; + if (ownerId === null || ownerId !== actor.user.id) { + throw new HttpError(404, 'Access token not found'); + } + } + + await this.clients.db.write( + 'DELETE FROM `access_token_permissions` WHERE `token_uid` = ?', + [tokenUid], + ); + await this.stores.permission.invalidateAccessTokenPerms(tokenUid); + } + + // ── Internals ─────────────────────────────────────────────────── + + #originFromUrl(url: string): string | null { + try { + const parsed = new URL(url); + return `${parsed.protocol}//${parsed.hostname}${parsed.port ? `:${parsed.port}` : ''}`; + } catch { + return null; + } + } + + async #actorFromSessionToken( + decoded: SessionTokenPayload, + ): Promise { + const session = await this.stores.session.getByUuid(decoded.uuid); + if (!session) return null; + + const user = await this.stores.user.getByUuid(decoded.user_uid); + if (!user) return null; + + this.stores.session + .touch({ uuid: session.uuid, userId: user.id }) + .catch(() => {}); + + return this.#buildUserActor(user, session); + } + + async #actorFromAppUnderUserToken( + decoded: AppUnderUserTokenPayload, + ): Promise { + // App tokens may or may not carry a session reference. If present, + // the token is bound to that session — log out invalidates it. + let session: SessionRow | null = null; + if (decoded.session) { + session = await this.stores.session.getByUuid(decoded.session); + if (!session) return null; + } + + const user = await this.stores.user.getByUuid(decoded.user_uid); + if (!user) return null; + + const app = await this.stores.app.getByUid(decoded.app_uid); + if (!app) return null; + + this.stores.session + .touch({ uuid: session?.uuid, userId: user.id }) + .catch(() => {}); + + return this.#buildAppUnderUserActor(user, app, session); + } + + async #actorFromAccessTokenToken( + decoded: AccessTokenPayload, + ): Promise { + if (!decoded.token_uid || !decoded.user_uid) return null; + + const user = await this.stores.user.getByUuid(decoded.user_uid); + if (!user) return null; + + // The authorizer is the identity whose permissions the access token + // can exercise — either a plain user or an app-under-user. + let authorizer: Actor; + if (decoded.app_uid) { + const app = await this.stores.app.getByUid(decoded.app_uid); + if (!app) return null; + authorizer = this.#buildAppUnderUserActor(user, app, null); + } else { + authorizer = this.#buildUserActor(user, null); + } + + return { + user: this.#actorUserFromRow(user), + accessToken: { + uid: decoded.token_uid, + issuer: authorizer, + authorized: null, + }, + }; + } + + // ── Actor builders ────────────────────────────────────────────── + + #actorUserFromRow(user: UserRow) { + return { + uuid: user.uuid, + id: user.id, + username: user.username, + email: user.email ?? null, + suspended: user.suspended ?? false, + email_confirmed: user.email_confirmed ?? false, + requires_email_confirmation: + user.requires_email_confirmation ?? false, + }; + } + + #buildUserActor(user: UserRow, session: SessionRow | null): Actor { + return { + user: this.#actorUserFromRow(user), + session: session ? { uid: session.uuid } : null, + }; + } + + #buildAppUnderUserActor( + user: UserRow, + app: { uid: string; id: number }, + session: SessionRow | null, + ): Actor { + return { + user: this.#actorUserFromRow(user), + app: { + uid: app.uid, + id: app.id, + }, + session: session ? { uid: session.uuid } : null, + }; + } +} diff --git a/src/backend/services/auth/OIDCService.ts b/src/backend/services/auth/OIDCService.ts new file mode 100644 index 000000000..f9f7a8449 --- /dev/null +++ b/src/backend/services/auth/OIDCService.ts @@ -0,0 +1,494 @@ +import type { LayerInstances } from '../../types'; +import type { puterServices } from '../index'; +import type { UserRow } from '../../stores/user/UserStore'; +import { PuterService } from '../types'; +import { cleanEmail, isBlockedEmail } from '../../util/email.js'; +import { generate_identifier } from '../../util/identifier.js'; +import { generateDefaultFsentries } from '../../util/userProvisioning.js'; +import { Context } from '../../core'; + +const GOOGLE_DISCOVERY_URL = + 'https://accounts.google.com/.well-known/openid-configuration'; +const GOOGLE_SCOPES = 'openid email profile'; +const STATE_EXPIRY_SEC = 600; // 10 minutes +const VALID_OIDC_FLOWS = ['login', 'signup', 'revalidate'] as const; +const REVALIDATION_EXPIRY_SEC = 300; // 5 minutes + +interface ProviderConfig { + client_id: string; + client_secret: string; + authorization_endpoint: string; + token_endpoint: string; + userinfo_endpoint: string; + scopes: string; +} + +interface OIDCUserInfo { + sub: string; + email?: string; + email_verified?: boolean; + name?: string; + picture?: string; + [k: string]: unknown; +} + +/** + * OIDC/OAuth2 service — sign-in with Google (extensible to other providers). + * + * Delegates to TokenService for JWT state signing, AuthService for session + * creation, UserStore for user creation. + * + * Config shape: `config.oidc.providers..{ client_id, client_secret, ... }` + */ +export class OIDCService extends PuterService { + declare protected services: LayerInstances; + + #googleDiscovery: Record | null = null; + #providers: Record> = {}; + + override onServerStart(): void { + const oidcConfig = this.config.oidc; + this.#providers = (oidcConfig?.providers ?? {}) as Record< + string, + Record + >; + } + + // ── Provider config ───────────────────────────────────────────── + + async getProviderConfig( + providerId: string, + ): Promise { + const raw = this.#providers[providerId]; + if (!raw || !raw.client_id || !raw.client_secret) return null; + + if (providerId === 'google') { + const discovery = await this.#fetchGoogleDiscovery(); + if (!discovery) return null; + return { + client_id: raw.client_id, + client_secret: raw.client_secret, + authorization_endpoint: discovery.authorization_endpoint, + token_endpoint: discovery.token_endpoint, + userinfo_endpoint: discovery.userinfo_endpoint, + scopes: raw.scopes ?? GOOGLE_SCOPES, + }; + } + + // Custom provider — must have all endpoints configured explicitly + if ( + raw.authorization_endpoint && + raw.token_endpoint && + raw.userinfo_endpoint + ) { + return { + client_id: raw.client_id, + client_secret: raw.client_secret, + authorization_endpoint: raw.authorization_endpoint, + token_endpoint: raw.token_endpoint, + userinfo_endpoint: raw.userinfo_endpoint, + scopes: raw.scopes ?? 'openid email profile', + }; + } + + return null; + } + + async getEnabledProviderIds(): Promise { + const ids: string[] = []; + for (const id of Object.keys(this.#providers)) { + const cfg = await this.getProviderConfig(id); + if (cfg) ids.push(id); + } + return ids; + } + + // ── Auth URL ──────────────────────────────────────────────────── + + getCallbackUrl(flow: string): string | null { + if (!(VALID_OIDC_FLOWS as readonly string[]).includes(flow)) + return null; + const origin = (this.config.origin ?? '').replace(/\/$/, ''); + return `${origin}/auth/oidc/callback/${flow}`; + } + + async getAuthorizationUrl( + providerId: string, + state: string, + flow: string, + ): Promise { + const config = await this.getProviderConfig(providerId); + if (!config) return null; + const redirectUri = + this.getCallbackUrl(flow) ?? + `${this.config.api_base_url ?? ''}/auth/oidc/callback`; + const params = new URLSearchParams({ + client_id: config.client_id, + redirect_uri: redirectUri, + response_type: 'code', + scope: config.scopes, + state, + }); + return `${config.authorization_endpoint}?${params.toString()}`; + } + + // ── State tokens (CSRF) ───────────────────────────────────────── + + signState(payload: Record): string { + return this.services.token.sign('oidc-state', payload, { + expiresIn: STATE_EXPIRY_SEC, + }); + } + + verifyState(token: string): Record | null { + try { + return this.services.token.verify>( + 'oidc-state', + token, + ); + } catch { + return null; + } + } + + // ── Revalidation tokens ───────────────────────────────────────── + + signRevalidation(userUuid: string): string { + return this.services.token.sign( + 'oidc-state', + { + user_uuid: userUuid, + purpose: 'revalidate', + }, + { expiresIn: REVALIDATION_EXPIRY_SEC }, + ); + } + + // ── Token exchange ────────────────────────────────────────────── + + async exchangeCodeForTokens( + providerId: string, + code: string, + redirectUri: string, + ): Promise<{ access_token: string; [k: string]: unknown } | null> { + const config = await this.getProviderConfig(providerId); + if (!config) return null; + + const body = new URLSearchParams({ + grant_type: 'authorization_code', + code, + redirect_uri: redirectUri, + client_id: config.client_id, + client_secret: config.client_secret, + }); + + const res = await fetch(config.token_endpoint, { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: body.toString(), + }); + + if (!res.ok) { + console.warn('[oidc] token exchange failed', { + status: res.status, + body: await res.text(), + }); + return null; + } + + return (await res.json()) as { access_token: string }; + } + + // ── User info ─────────────────────────────────────────────────── + + async getUserInfo( + providerId: string, + accessToken: string, + ): Promise { + const config = await this.getProviderConfig(providerId); + if (!config?.userinfo_endpoint) return null; + + const res = await fetch(config.userinfo_endpoint, { + headers: { Authorization: `Bearer ${accessToken}` }, + }); + if (!res.ok) return null; + return (await res.json()) as OIDCUserInfo; + } + + // ── User lookup / creation ────────────────────────────────────── + + async findUserByProviderSub( + provider: string, + providerSub: string, + ): Promise { + const link = await this.stores.oidc.getByProviderSub( + provider, + providerSub, + ); + if (!link) return null; + return this.stores.user.getById(link.user_id as number); + } + + /** + * Find an existing Puter user by the email claimed by the OIDC provider. + * + * Matches on both the raw `email` column and the canonical `clean_email` + * column so that `Foo.Bar+tag@gmail.com` (OIDC) resolves to an account + * that signed up as `foobar@gmail.com`. Primary email is preferred over a + * clean_email collision. + */ + async findUserByEmail(email: string): Promise { + if (!email) return null; + const direct = await this.stores.user.getByEmail(email); + if (direct) return direct; + return this.stores.user.getByCleanEmail(cleanEmail(email)); + } + + /** + * Link an OIDC provider to an existing user. Use when the `sub` wasn't + * linked yet but we matched the user by email. + * + * Does NOT touch the password column — a user who originally signed up + * with a password keeps password login. Does mark `email_confirmed` if + * the provider verified the email and the row wasn't already confirmed. + */ + async linkProviderToUser( + userId: number, + providerId: string, + claims: OIDCUserInfo, + ): Promise<{ success: boolean; error?: string }> { + if (claims.email_verified === false) { + return { + success: false, + error: 'Provider did not verify this email address.', + }; + } + + // Only link to accounts whose email is already confirmed. Unconfirmed + // accounts have no proven owner, so linking OIDC would hand control + // to whoever holds the OIDC identity. + const user = await this.stores.user.getById(userId, { force: true }); + if (!user) { + return { success: false, error: 'User not found.' }; + } + if (!user.email_confirmed) { + return { + success: false, + error: 'Account email is not confirmed. Sign in with your password first to confirm it.', + }; + } + + await this.stores.oidc.link(userId, providerId, claims.sub, null); + return { success: true }; + } + + /** + * Create a new Puter user from OIDC claims and link the provider. + * Returns `{ success, user, error? }`. + */ + async createUserFromOIDC( + providerId: string, + claims: OIDCUserInfo, + ): Promise<{ success: boolean; user?: UserRow; error?: string }> { + if (claims.email_verified === false) { + return { + success: false, + error: 'Provider did not verify this email address.', + }; + } + + // Generate a unique username + let username: string; + let attempts = 0; + do { + username = generate_identifier(); + attempts++; + if (attempts > 20) + return { + success: false, + error: 'Failed to generate unique username.', + }; + } while (await this.stores.user.getByUsername(username)); + + // Create user — no password, email assumed confirmed by provider + const { v4: uuidv4 } = await import('uuid'); + const req = Context.get('req'); + const clientIp = req.ip || req.socket?.remoteAddress || null; + const proxyIpChain = req.headers['x-forwarded-for']; + + // Run abuse-prevention validate hook. OIDC ignores + // requires_email_confirmation (provider already verified) and + // no_temp_user (OIDC users are never temp), so only `allow` matters. + const validateEvent = { + req, + data: { username, email: claims.email ?? '' }, + allow: true, + no_temp_user: false, + requires_email_confirmation: false, + message: null as string | null, + }; + try { + await this.clients.event?.emitAndWait( + 'puter.signup.validate', + validateEvent, + {}, + ); + } catch (e) { + console.warn('[oidc] validate hook failed:', e); + } + if (!validateEvent.allow) { + return { + success: false, + error: validateEvent.message ?? 'Signup blocked', + }; + } + + // Email validation — mirrors AuthController#validateEmail. Skipped + // when the IdP didn't return an email (createUserFromOIDC is + // reachable with `email` undefined). + if (claims.email) { + if (isBlockedEmail(claims.email, this.config.blockedEmailDomains)) { + return { + success: false, + error: 'This email is not allowed.', + }; + } + const emailEvent = { + email: cleanEmail(claims.email), + allow: true, + message: null as string | null, + }; + try { + await this.clients.event?.emitAndWait( + 'puter.email.validate', + emailEvent, + {}, + ); + } catch (e) { + console.warn('[oidc] email validate hook failed:', e); + } + if (!emailEvent.allow) { + return { + success: false, + error: + emailEvent.message ?? + 'This email cannot be used. Please try a different email address.', + }; + } + } + + const created = await this.stores.user.create({ + username, + uuid: uuidv4(), + password: null, + email: claims.email ?? null, + clean_email: claims.email ? cleanEmail(claims.email) : null, + free_storage: this.config.storage_capacity ?? null, + requires_email_confirmation: false, + audit_metadata: { + ip: clientIp, + ip_fwd: proxyIpChain, + user_agent: req?.headers?.['user-agent'], + origin: req?.headers?.origin, + }, + signup_ip: clientIp, + signup_ip_forwarded: proxyIpChain, + signup_user_agent: req?.headers?.['user-agent'] ?? null, + signup_origin: req?.headers?.origin, + signup_server: this.config.serverId, + referrer: req?.body?.referrer ?? null, + }); + + if (!created) { + return { success: false, error: 'User creation failed.' }; + } + + // Mark email as confirmed (OIDC provider already verified it). + await this.stores.user.update(created.id, { + email_confirmed: 1, + requires_email_confirmation: 0, + }); + + // Default user group — OIDC users skip the temp group entirely since + // the email is already verified by the IdP. + const defaultGroup = this.config.default_user_group; + if (defaultGroup) { + try { + await this.stores.group.addUsers(defaultGroup, [ + created.username, + ]); + } catch (e) { + console.warn('[oidc] group assignment failed:', e); + } + } + + // Provision home directory + default folders. Idempotent. + try { + await generateDefaultFsentries( + this.clients.db, + this.stores.user, + created, + ); + } catch (e) { + console.warn('[oidc] generateDefaultFsentries failed:', e); + } + + // Link OIDC provider (after provisioning so a failed link doesn't + // leave an orphaned user without a home folder). + await this.stores.oidc.link(created.id, providerId, claims.sub, null); + + // Re-read so callers see email_confirmed / *_uuid / *_id fields + // written above. + const user = await this.stores.user.getById(created.id, { + force: true, + }); + const resolved = user ?? created; + + // Fire signup events — keys match the password-based signup path so + // downstream listeners (welcome email, mailchimp sync, etc.) treat + // both signup routes identically. + try { + this.clients.event?.emit( + 'puter.signup.success', + { + user_id: resolved.id, + user_uuid: resolved.uuid, + email: resolved.email, + username: resolved.username, + }, + {}, + ); + } catch { + // ignore — event emission shouldn't block signup + } + try { + this.clients.event?.emit( + 'user.save_account', + { user_id: resolved.id }, + {}, + ); + } catch { + // ignore + } + + return { success: true, user: resolved }; + } + + // ── Internals ─────────────────────────────────────────────────── + + async #fetchGoogleDiscovery(): Promise | null> { + if (this.#googleDiscovery) return this.#googleDiscovery; + try { + const res = await fetch(GOOGLE_DISCOVERY_URL); + if (!res.ok) return null; + this.#googleDiscovery = (await res.json()) as Record< + string, + string + >; + return this.#googleDiscovery; + } catch (e) { + console.warn('[oidc] Google discovery fetch failed', e); + return null; + } + } +} diff --git a/src/backend/services/auth/OTPUtil.js b/src/backend/services/auth/OTPUtil.js new file mode 100644 index 000000000..c87b7b707 --- /dev/null +++ b/src/backend/services/auth/OTPUtil.js @@ -0,0 +1,50 @@ +import { TOTP } from 'otpauth'; +import crypto from 'node:crypto'; +import { encode } from 'hi-base32'; + +/** + * Standalone OTP utilities — no service class, just functions. + */ + +export function createSecret(label) { + const secret = genOtpSecret(); + const totp = new TOTP({ + issuer: 'puter.com', + label, + algorithm: 'SHA1', + digits: 6, + secret, + }); + return { url: totp.toString(), secret }; +} + +export function createRecoveryCode() { + const buffer = crypto.randomBytes(6); + return encode(buffer).replace(/=/g, '').substring(0, 8); +} + +export function verify(label, secret, code) { + const totp = new TOTP({ + issuer: 'puter.com', + label, + algorithm: 'SHA1', + digits: 6, + secret, + }); + const delta = totp.validate({ token: code }); + if (delta === null) return false; + return [-1, 0, 1].includes(delta); +} + +export function hashRecoveryCode(code) { + return crypto + .createHash('sha256') + .update(code) + .digest('base64') + .slice(0, 22); +} + +function genOtpSecret() { + const buffer = crypto.randomBytes(15); + return encode(buffer).replace(/=/g, '').substring(0, 24); +} diff --git a/src/backend/services/auth/TokenService.ts b/src/backend/services/auth/TokenService.ts new file mode 100644 index 000000000..4f10207bf --- /dev/null +++ b/src/backend/services/auth/TokenService.ts @@ -0,0 +1,244 @@ +import jwt, { type SignOptions } from 'jsonwebtoken'; +import { PuterService } from '../types'; + +// ── Compression tables ────────────────────────────────────────────── +// +// Token payloads are compressed on the wire: full field names become +// short aliases, enum values become single-letter codes, and UUIDs get +// base64-packed (no dashes, no `-` padding). +// +// This keeps tokens small enough to fit in cookies / query strings. +// The `short` aliases and value codes are part of the wire contract — +// existing tokens depend on them, do not change without a migration. + +interface FieldInfo { + short?: string; + values?: { + to_short: Record; + to_long: Record; + }; + encode?: (v: string) => string; + decode?: (v: string) => string; +} + +type FieldInfoShorthand = string | FieldInfo; + +interface CompressionContext { + fullkey_to_info: Record; + short_to_fullkey: Record; +} + +const def = (o: Record): CompressionContext => { + const fullkey_to_info: Record = {}; + for (const k in o) { + const v = o[k]; + fullkey_to_info[k] = typeof v === 'string' ? { short: v } : v; + } + const short_to_fullkey = Object.keys(fullkey_to_info).reduce< + Record + >((acc, key) => { + const short = fullkey_to_info[key].short; + if (short) acc[short] = key; + return acc; + }, {}); + return { fullkey_to_info, short_to_fullkey }; +}; + +const defv = ( + o: Record, +): { to_short: Record; to_long: Record } => { + return { + to_short: o, + to_long: Object.keys(o).reduce>((acc, key) => { + acc[o[key]] = key; + return acc; + }, {}), + }; +}; + +/** + * UUIDs on the wire: strip dashes, hex→base64. Optional prefix is stripped + * before encoding and re-added on decode (e.g., `app-`). + */ +const uuidCompression = (prefix?: string) => ({ + encode: (v: string): string => { + if (prefix) { + if (!v.startsWith(prefix)) { + throw new Error(`Expected ${prefix} prefix`); + } + v = v.slice(prefix.length); + } + const undecorated = v.replace(/-/g, ''); + return Buffer.from(undecorated, 'hex').toString('base64'); + }, + decode: (v: string): string => { + // Already a uuid string → passthrough (for tokens minted pre-compression) + if (v.includes('-')) return v; + const undecorated = Buffer.from(v, 'base64').toString('hex'); + return ( + (prefix ?? '') + + [ + undecorated.slice(0, 8), + undecorated.slice(8, 12), + undecorated.slice(12, 16), + undecorated.slice(16, 20), + undecorated.slice(20), + ].join('-') + ); + }, +}); + +const AUTH_COMPRESSION = def({ + uuid: { short: 'u', ...uuidCompression() }, + session: { short: 's', ...uuidCompression() }, + version: 'v', + type: { + short: 't', + values: defv({ + session: 's', + 'access-token': 't', + 'app-under-user': 'au', + }), + }, + user_uid: { short: 'uu', ...uuidCompression() }, + app_uid: { short: 'au', ...uuidCompression('app-') }, +}); + +// `hosted-asset` scope signs the sticky cookies set after a visitor +// passes the private/public-app access gate (see AuthService +// createPrivateAssetToken / createPublicHostedActorToken). Keeping it +// in its own scope prevents a cookie from ever being honored as a main +// auth token. +const HOSTED_ASSET_COMPRESSION = def({ + version: 'v', + kind: { + short: 'k', + values: defv({ + private: 'pr', + public: 'pu', + }), + }, + user_uid: { short: 'uu', ...uuidCompression() }, + app_uid: { short: 'au', ...uuidCompression('app-') }, + session_uuid: { short: 's', ...uuidCompression() }, + subdomain: 'sd', + host: 'h', +}); + +const COMPRESSION: Record = { + auth: AUTH_COMPRESSION, + 'hosted-asset': HOSTED_ASSET_COMPRESSION, +}; + +// ── TokenService ──────────────────────────────────────────────────── + +/** + * Signs and verifies JWTs. + * + * Kept intentionally small — no session lifecycle, no revocation list, no + * cookie shaping. That logic lives in `AuthService` (actor resolution) and + * will live in a future session controller (mint/rotate/revoke). + */ +export class TokenService extends PuterService { + #secret: string = ''; + + override onServerStart(): void { + const secret = this.config.jwt_secret; + if (!secret) { + throw new Error('TokenService requires `jwt_secret` in config'); + } + this.#secret = secret; + } + + /** + * Sign a payload for the given scope. The compression table for `scope` + * is applied to the payload before signing, so what reaches the wire is + * the short-key form. + */ + sign( + scope: string, + payload: Record, + options?: SignOptions, + ): string { + const context = COMPRESSION[scope]; + const compressed = this.#compressPayload(context, payload); + return jwt.sign(compressed, this.#secret, options ?? {}); + } + + /** + * Verify and decompress. Throws on invalid signature / expired / malformed + * (propagating `jsonwebtoken`'s errors). Callers in the auth probe should + * catch and treat as "no actor". + */ + verify>(scope: string, token: string): T { + const context = COMPRESSION[scope]; + const payload = jwt.verify(token, this.#secret) as Record< + string, + unknown + >; + return this.#decompressPayload(context, payload) as unknown as T; + } + + // ── Internals ─────────────────────────────────────────────────── + + #compressPayload( + context: CompressionContext | undefined, + payload: Record, + ): Record { + if (!context) return payload; + const { fullkey_to_info } = context; + const out: Record = {}; + for (const fullkey in payload) { + const info = fullkey_to_info[fullkey]; + if (!info) { + out[fullkey] = payload[fullkey]; + continue; + } + let k = fullkey; + let v = payload[fullkey]; + if (info.short) k = info.short; + if ( + info.values && + typeof v === 'string' && + info.values.to_short[v] !== undefined + ) { + v = info.values.to_short[v]; + } else if (info.encode && typeof v === 'string') { + v = info.encode(v); + } + out[k] = v; + } + return out; + } + + #decompressPayload( + context: CompressionContext | undefined, + payload: Record, + ): Record { + if (!context) return payload; + const { fullkey_to_info, short_to_fullkey } = context; + const out: Record = {}; + for (const short in payload) { + const fullkey = short_to_fullkey[short]; + if (!fullkey) { + out[short] = payload[short]; + continue; + } + const info = fullkey_to_info[fullkey]; + let k = short; + let v = payload[short]; + if (info.short) k = fullkey; + if ( + info.values && + typeof v === 'string' && + info.values.to_long[v] !== undefined + ) { + v = info.values.to_long[v]; + } else if (info.decode && typeof v === 'string') { + v = info.decode(v); + } + out[k] = v; + } + return out; + } +} diff --git a/src/backend/services/auth/types.ts b/src/backend/services/auth/types.ts new file mode 100644 index 000000000..d0829f92a --- /dev/null +++ b/src/backend/services/auth/types.ts @@ -0,0 +1,73 @@ +// Express `Request` augmentations live in `core/http/expressAugmentation.ts` +// — auth-related fields (`actor`, `token`) are declared there alongside the +// other request-level fields populated by global middleware. + +// ── Token payload shapes (after `TokenService.verify` decompression) ── + +/** Base fields every non-legacy auth token carries. */ +interface TokenPayloadBase { + version?: string; + type: TokenType; +} + +export type TokenType = 'session' | 'gui' | 'app-under-user' | 'access-token'; + +/** + * Session token — issued at login; represents a browser session. + * + * `type === 'session'` is the HTTP-only-cookie flavor; `'gui'` is the same + * shape but served as a response body (e.g., QR login → client-visible + * token). Both resolve to a `UserActor` with `accessToken: null`. + */ +export interface SessionTokenPayload extends TokenPayloadBase { + type: 'session' | 'gui'; + /** Session uuid (plain, not FPE-encrypted for session tokens). */ + uuid: string; + /** User uuid (plain). */ + user_uid: string; +} + +/** + * App-under-user token — issued to an app acting on behalf of a user. + * + * `session`, when present, is the raw session uuid (no encryption). The token + * is bound to that session, so user logout invalidates it. App tokens minted + * outside an interactive session context (e.g., from an access-token actor) + * omit the field entirely. + */ +export interface AppUnderUserTokenPayload extends TokenPayloadBase { + type: 'app-under-user'; + user_uid: string; + app_uid: string; + /** Raw session uuid (optional — some app tokens have no session). */ + session?: string; +} + +/** + * Access token — issued to a third-party / programmatic caller. Carries a + * token uuid whose permissions are managed in `access_token_permissions`. + */ +export interface AccessTokenPayload extends TokenPayloadBase { + type: 'access-token'; + token_uid: string; + user_uid: string; + app_uid?: string; +} + +export type AnyTokenPayload = + | SessionTokenPayload + | AppUnderUserTokenPayload + | AccessTokenPayload; + +// ── Session row (from `sessions` table) ──────────────────────────── + +export interface SessionRow { + id: number; + uuid: string; + user_id: number; + meta?: Record | string | null; + created_at?: number | null; + last_activity?: number | null; +} + +export {}; diff --git a/src/backend/services/broadcast/BroadcastService.ts b/src/backend/services/broadcast/BroadcastService.ts new file mode 100644 index 000000000..842ff7758 --- /dev/null +++ b/src/backend/services/broadcast/BroadcastService.ts @@ -0,0 +1,590 @@ +import axios from 'axios'; +import { createHmac, timingSafeEqual } from 'node:crypto'; +import { Agent as HttpsAgent } from 'node:https'; +import { IBroadcastPeerConfig } from '../../types.js'; +import { PuterService } from '../types.js'; + +// ── Wire types ────────────────────────────────────────────────────── + +interface BroadcastEvent { + key: string; + data: unknown; + meta: Record; +} + +interface IncomingPayload { + events?: unknown; + key?: string; + data?: unknown; + meta?: unknown; +} + +interface IncomingResult { + ok: boolean; + /** HTTP status to send when ok===false. */ + status?: number; + /** Error message body when ok===false. */ + message?: string; + /** Optional informational payload to include when ok===true. */ + info?: Record; +} + +interface IncomingHeaders { + peerId: string | undefined; + timestamp: string | undefined; + nonce: string | undefined; + signature: string | undefined; +} + +// ── Service ───────────────────────────────────────────────────────── + +/** + * Cross-node event replication via signed HTTP webhooks. + * + * **Outbound** — subscribes to local `outer.*` events on the event bus. + * Each event is added to a small in-memory map (deduped by serialized + * shape), then flushed every `outbound_flush_ms` as a single POST per + * configured peer. Each POST carries: + * + * - `X-Broadcast-Peer-Id` — this server's own peerId + * - `X-Broadcast-Timestamp` — unix seconds, peer rejects ±5min + * - `X-Broadcast-Nonce` — monotonic per-peer counter, peer rejects replays + * - `X-Broadcast-Signature` — HMAC-SHA256 of `..` + * + * **Inbound** — `BroadcastController` accepts POSTs at `/broadcast/webhook` + * and hands each one off to `verifyAndEmit()`. The service validates the + * HMAC + nonce + timestamp window, then re-emits each contained event + * onto the local bus tagged with `meta.from_outside = true` so the + * outbound subscriber doesn't bounce it back. + * + * Self-loop avoidance: + * - Outbound subscriber skips events with `meta.from_outside`. + * - Inbound handler ignores POSTs whose `X-Broadcast-Peer-Id` matches + * this server's own peerId (catches misconfigured loopbacks). + * + * No Redis pub/sub here — webhooks are the only transport. Same-cluster + * fan-out is handled by sockets via the Redis streams adapter, so an + * additional Redis channel here would just duplicate work. + */ +export class BroadcastService extends PuterService { + /** peerId → resolved peer config, used for incoming-verify lookup. */ + #peersByKey: Record = {}; + /** Subset of peers with `webhook: true`, used for outbound fan-out. */ + #webhookPeers: IBroadcastPeerConfig[] = []; + + /** Coalesced outbound events, keyed by serialized shape. */ + #outboundEventsByDedupKey = new Map(); + #outboundFlushTimer: ReturnType | null = null; + #outboundIsFlushing = false; + #dedupFallbackCounter = 0; + + #webhookReplayWindowSeconds = 300; + #outboundFlushMs = 2000; + #webhookProtocol: 'http' | 'https' = 'https'; + #webhookHostHeader: string | null = null; + /** Self-signed certs are common between Puter nodes — accept them. */ + #webhookHttpsAgent = new HttpsAgent({ rejectUnauthorized: false }); + + // ── Lifecycle ─────────────────────────────────────────────────── + + override onServerStart(): void { + this.#loadConfig(); + this.#subscribeOutbound(); + } + + override async onServerPrepareShutdown(): Promise { + if (this.#outboundFlushTimer) { + clearTimeout(this.#outboundFlushTimer); + this.#outboundFlushTimer = null; + } + // Best-effort drain — try one final flush so events queued near + // shutdown make it out. + try { + await this.#flushOutboundEvents(); + } catch (err) { + console.warn('[broadcast] final flush failed', err); + } + } + + // ── Public API used by BroadcastController ────────────────────── + + /** + * Verify an incoming webhook POST and, if valid, fan its events + * onto the local event bus (tagged `from_outside: true`). + * + * Caller (controller) provides the request's parsed JSON body, the + * raw bytes that JSON came from (HMAC verifies over those exact + * bytes), and the four broadcast headers. + */ + async verifyAndEmit( + rawBody: Buffer | undefined, + body: unknown, + headers: IncomingHeaders, + ): Promise { + if (!rawBody) { + return { + ok: false, + status: 400, + message: 'Missing or invalid body', + }; + } + if (!body || typeof body !== 'object') { + return { ok: false, status: 400, message: 'Invalid JSON body' }; + } + + const incomingEvents = this.#normalizeIncomingPayload( + body as IncomingPayload, + ); + if (!incomingEvents) { + return { + ok: false, + status: 400, + message: 'Invalid broadcast payload', + }; + } + + const peerId = headers.peerId; + if (!peerId) { + return { + ok: false, + status: 403, + message: 'Missing X-Broadcast-Peer-Id', + }; + } + + // Defend against a misconfigured peer that includes us in its + // own peer list — easy mistake when bootstrapping a cluster. + const localPeerId = this.#resolveLocalPeerId(); + if (localPeerId && peerId === localPeerId) { + return { ok: true, info: { ignored: 'self-peer' } }; + } + + const peer = this.#peersByKey[peerId]; + if (!peer || !peer.webhook_secret) { + return { + ok: false, + status: 403, + message: 'Unknown peer or webhook not configured', + }; + } + + const tsCheck = this.#parseTimestamp(headers.timestamp); + if (!tsCheck.ok) return tsCheck; + const timestamp = tsCheck.timestamp; + + const nonceCheck = this.#parseNonce(headers.nonce); + if (!nonceCheck.ok) return nonceCheck; + const nonce = nonceCheck.nonce; + + if (!headers.signature) { + return { + ok: false, + status: 403, + message: 'Missing X-Broadcast-Signature', + }; + } + + const payloadToSign = `${timestamp}.${nonce}.${rawBody.toString('utf8')}`; + const expectedHmac = createHmac('sha256', peer.webhook_secret) + .update(payloadToSign) + .digest('hex'); + const signatureBuffer = Buffer.from(headers.signature, 'hex'); + const expectedBuffer = Buffer.from(expectedHmac, 'hex'); + if ( + signatureBuffer.length !== expectedBuffer.length || + !timingSafeEqual(signatureBuffer, expectedBuffer) + ) { + return { ok: false, status: 403, message: 'Invalid signature' }; + } + + // Atomic claim of (peerId, ts, nonce) in Redis — single key, so + // it's cluster-safe and shared across ALB-balanced nodes. Done + // post-signature so unsigned/forged requests can't burn slots. + if (!(await this.#claimIncomingNonce(peerId, timestamp, nonce))) { + return { + ok: false, + status: 403, + message: 'Duplicate or stale nonce', + }; + } + + await this.#emitIncomingEventsSequentially(incomingEvents); + return { ok: true }; + } + + // ── Outbound: subscribe + queue + flush ──────────────────────── + + #subscribeOutbound(): void { + // Wildcard: every `outer.*` event gets considered for broadcast. + // The handler skips events that came in via webhook (meta.from_outside) + // so we don't bounce them back to peers. + this.clients.event.on( + 'outer.*', + (key: string, data: unknown, meta: object) => { + this.#handleOutbound(key, data, meta); + }, + ); + } + + #handleOutbound( + key: string, + data: unknown, + meta: object | undefined, + ): void { + const safeMeta = this.#normalizeMeta(meta); + if (safeMeta.from_outside) return; + + const event: BroadcastEvent = { key, data, meta: safeMeta }; + const dedupKey = this.#createDedupKey(event); + this.#outboundEventsByDedupKey.set(dedupKey, event); + this.#scheduleOutboundFlush(); + } + + #createDedupKey(event: BroadcastEvent): string { + try { + return JSON.stringify(event); + } catch { + this.#dedupFallbackCounter += 1; + return `fallback-${this.#dedupFallbackCounter}`; + } + } + + #scheduleOutboundFlush(): void { + if (this.#outboundFlushTimer) return; + this.#outboundFlushTimer = setTimeout(() => { + this.#outboundFlushTimer = null; + void this.#flushOutboundEvents().catch((err) => { + console.warn('[broadcast] outbound flush failed', err); + }); + }, this.#outboundFlushMs); + } + + async #flushOutboundEvents(): Promise { + if ( + this.#outboundIsFlushing || + this.#outboundEventsByDedupKey.size === 0 + ) + return; + + this.#outboundIsFlushing = true; + try { + const events = [...this.#outboundEventsByDedupKey.values()]; + this.#outboundEventsByDedupKey.clear(); + + for (const peer of this.#webhookPeers) { + try { + await this.#sendWebhookToPeer(peer, events); + } catch (err) { + const peerId = peer.peerId ?? 'unknown'; + console.warn( + `[broadcast] webhook send to peer ${peerId} failed`, + err, + ); + } + } + } finally { + this.#outboundIsFlushing = false; + // Anything that arrived during flush gets the next tick. + if (this.#outboundEventsByDedupKey.size > 0) { + this.#scheduleOutboundFlush(); + } + } + } + + async #sendWebhookToPeer( + peer: IBroadcastPeerConfig, + events: BroadcastEvent[], + ): Promise { + const peerId = this.#resolvePeerIdOf(peer); + if (!peerId) return; + const requestUrl = this.#normalizeWebhookUrl(peer.webhook_url); + const mySecret = this.#self()?.secret; + if (!requestUrl || !mySecret) return; + + // Shared INCR across all ALB-balanced nodes so concurrent senders + // can't emit colliding nonces that the receiver would reject. + const nextNonce = await this.#nextOutgoingNonce(peerId); + + const timestamp = Math.floor(Date.now() / 1000); + const rawBody = JSON.stringify({ events }); + const payloadToSign = `${timestamp}.${nextNonce}.${rawBody}`; + const signature = createHmac('sha256', mySecret) + .update(payloadToSign) + .digest('hex'); + + const myPublicId = this.#resolveLocalPeerId() ?? ''; + const headers: Record = { + 'Content-Type': 'application/json', + 'Content-Length': String(Buffer.byteLength(rawBody)), + 'X-Broadcast-Peer-Id': myPublicId, + 'X-Broadcast-Timestamp': String(timestamp), + 'X-Broadcast-Nonce': String(nextNonce), + 'X-Broadcast-Signature': signature, + }; + if (this.#webhookHostHeader) headers.Host = this.#webhookHostHeader; + + const response = await axios.request({ + method: 'POST', + url: requestUrl, + headers, + data: rawBody, + timeout: 15_000, + // We translate non-2xx into a thrown error ourselves so we + // can log the response body on failure. + validateStatus: () => true, + responseType: 'text', + transformResponse: (value: unknown) => value, + ...(requestUrl.startsWith('https:') + ? { httpsAgent: this.#webhookHttpsAgent } + : {}), + }); + + if (response.status < 200 || response.status >= 300) { + console.warn( + `[broadcast] peer ${peerId} responded ${response.status}: ${response.data}`, + ); + throw new Error( + `Webhook POST failed: ${response.status} ${response.statusText}`, + ); + } + } + + // ── Inbound helpers ───────────────────────────────────────────── + + #normalizeIncomingPayload( + payload: IncomingPayload, + ): BroadcastEvent[] | null { + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) + return null; + + // Either `{ events: [...] }` or a single event spread at top level. + if (Array.isArray(payload.events)) { + const out: BroadcastEvent[] = []; + for (const ev of payload.events) { + const norm = this.#normalizeIncomingEvent(ev); + if (!norm) return null; + out.push(norm); + } + return out; + } + + const norm = this.#normalizeIncomingEvent(payload); + return norm ? [norm] : null; + } + + #normalizeIncomingEvent(event: unknown): BroadcastEvent | null { + if (!event || typeof event !== 'object' || Array.isArray(event)) + return null; + const e = event as { key?: unknown; data?: unknown; meta?: unknown }; + if (typeof e.key !== 'string' || e.key.length === 0) return null; + if (e.data === undefined) return null; + return { + key: e.key, + data: e.data, + meta: this.#normalizeMeta(e.meta), + }; + } + + async #emitIncomingEventsSequentially( + events: BroadcastEvent[], + ): Promise { + for (const event of events) { + // Belt-and-braces: a misbehaving peer that forwards already + // outside-tagged events would otherwise bounce ad infinitum. + if (event.meta?.from_outside) { + console.warn( + '[broadcast] dropping incoming event already tagged from_outside', + { key: event.key }, + ); + continue; + } + const metaOut = { ...event.meta, from_outside: true }; + try { + this.clients.event.emit(event.key, event.data, metaOut); + } catch (err) { + console.warn('[broadcast] event re-emit failed', { + key: event.key, + err, + }); + } + } + } + + /** + * Atomically claim an inbound (peerId, ts, nonce) tuple. Returns + * true on first claim, false on replay. SET NX with TTL = replay + * window; single key ⇒ cluster-mode safe. + */ + async #claimIncomingNonce( + peerId: string, + timestamp: number, + nonce: number, + ): Promise { + const key = `broadcast:in:${peerId}:${timestamp}:${nonce}`; + const result = await this.clients.redis.set( + key, + '1', + 'EX', + this.#webhookReplayWindowSeconds, + 'NX', + ); + return result === 'OK'; + } + + /** + * Atomically allocate the next outbound nonce for a peer. Shared + * counter across all ALB-fronted nodes via Redis INCR — guarantees + * uniqueness so the receiver's per-peer replay protection accepts + * every legitimate send. + */ + async #nextOutgoingNonce(peerId: string): Promise { + const n = await this.clients.redis.incr(`broadcast:out:${peerId}`); + return Number(n); + } + + #parseTimestamp( + raw: string | undefined, + ): { ok: true; timestamp: number } | (IncomingResult & { ok: false }) { + if (!raw) + return { + ok: false, + status: 400, + message: 'Missing X-Broadcast-Timestamp', + }; + const ts = Number(raw); + if (Number.isNaN(ts)) + return { + ok: false, + status: 400, + message: 'Invalid X-Broadcast-Timestamp', + }; + const nowSec = Math.floor(Date.now() / 1000); + const window = this.#webhookReplayWindowSeconds; + // 60s of forward tolerance for clock skew; the rest is replay- + // window backstop. + if (ts < nowSec - window || ts > nowSec + 60) { + return { + ok: false, + status: 400, + message: 'Timestamp out of window', + }; + } + return { ok: true, timestamp: ts }; + } + + #parseNonce( + raw: string | undefined, + ): { ok: true; nonce: number } | (IncomingResult & { ok: false }) { + if (raw === undefined || raw === null || raw === '') { + return { + ok: false, + status: 400, + message: 'Missing X-Broadcast-Nonce', + }; + } + const n = Number(raw); + if (Number.isNaN(n)) + return { + ok: false, + status: 400, + message: 'Invalid X-Broadcast-Nonce', + }; + return { ok: true, nonce: n }; + } + + // ── Misc ──────────────────────────────────────────────────────── + + #normalizeMeta(meta: unknown): Record { + if (!meta || typeof meta !== 'object' || Array.isArray(meta)) return {}; + return meta as Record; + } + + #resolveLocalPeerId(): string | null { + const id = this.#self()?.peerId; + if (typeof id !== 'string' || id.trim() === '') return null; + return id.trim(); + } + + #resolvePeerIdOf(peer: IBroadcastPeerConfig): string | null { + const id = peer.peerId; + if (typeof id !== 'string' || id.trim() === '') return null; + return id.trim(); + } + + #self() { + return this.#broadcastConfig().webhook; + } + + #broadcastConfig() { + return this.config.broadcast ?? {}; + } + + #normalizeWebhookUrl(url: string | undefined): string | null { + if (typeof url !== 'string' || url.trim() === '') return null; + const trimmed = url.trim(); + let parsed: URL; + try { + parsed = trimmed.includes('://') + ? new URL(trimmed) + : new URL(`${this.#webhookProtocol}://${trimmed}`); + } catch { + return null; + } + // Coerce protocol so a misconfigured `http://...` peer URL still + // gets sent over our preferred transport. + parsed.protocol = `${this.#webhookProtocol}:`; + return parsed.toString(); + } + + #loadConfig(): void { + const cfg = this.#broadcastConfig(); + const peers = cfg.peers ?? []; + + for (const peerCfg of peers) { + const peerId = this.#resolvePeerIdOf(peerCfg); + if (!peerId) { + console.warn( + '[broadcast] ignoring peer config with missing key/peerId', + { peerCfg }, + ); + continue; + } + if (this.#peersByKey[peerId]) { + console.warn('[broadcast] duplicate peer id', { + peerId, + existing: this.#peersByKey[peerId]?.webhook_url, + duplicate: peerCfg.webhook_url, + }); + } + this.#peersByKey[peerId] = { + peerId, + webhook_secret: peerCfg.webhook_secret, + webhook_url: peerCfg.webhook_url, + webhook: !!peerCfg.webhook, + }; + if (peerCfg.webhook) { + this.#webhookPeers.push({ ...peerCfg, peerId }); + } else { + console.warn( + '[broadcast] non-webhook peer ignored (websocket transport disabled in v2)', + { peerId }, + ); + } + } + + this.#webhookReplayWindowSeconds = Number( + cfg.webhook_replay_window_seconds ?? 300, + ); + const flushMs = Number(cfg.outbound_flush_ms ?? 2000); + this.#outboundFlushMs = + Number.isFinite(flushMs) && flushMs >= 0 ? flushMs : 2000; + + this.#webhookHostHeader = this.config.domain ?? null; + const protoRaw = String(this.config.protocol ?? '') + .trim() + .replace(/:$/, '') + .toLowerCase(); + this.#webhookProtocol = + protoRaw === 'http' || protoRaw === 'https' ? protoRaw : 'https'; + } +} diff --git a/src/backend/services/fs/FSService.ts b/src/backend/services/fs/FSService.ts new file mode 100644 index 000000000..ba71c1d7e --- /dev/null +++ b/src/backend/services/fs/FSService.ts @@ -0,0 +1,3462 @@ +import { posix as pathPosix } from 'node:path'; +import { createHash } from 'node:crypto'; +import { Readable, Transform } from 'node:stream'; +import type { TransformCallback } from 'node:stream'; +import { v4 as uuidv4 } from 'uuid'; +import type { + MultipartCompletePart, + SignedUploadResult, +} from '../../stores/fs/s3Types.js'; +import { + FSEntry, + FSEntryCreateInput, + FSEntryWriteInput, + PendingUploadCreateInput, + PendingUploadSession, +} from '../../stores/fs/FSEntry.js'; +import { + BinaryPayload, + CompleteWriteRequest, + CompleteWriteResponse, + SignMultipartPartsRequest, + SignMultipartPartsResponse, + SignedWriteRequest, + SignedWriteResponse, + UploadMode, + WriteRequest, + WriteResponse, +} from '../../controllers/fs/requestTypes.js'; +import type { + BatchWritePrepareRequest, + NormalizedWriteInput, + PreparedBatchWrite, + UploadedBatchWriteItem, + UploadPayload, + UploadPreparedBatchItemInput, + UploadProgressTrackerLike, +} from './types.js'; +import { runWithConcurrencyLimitSettled } from '../../util/concurrency.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import { PuterService } from '../types.js'; +import type { LayerInstances } from '../../types.js'; +import type { puterStores } from '../../stores/index.js'; +import type { puterServices } from '../index.js'; +import { FSEntryCacheInvalidationEventHandler } from './cacheInvalidation.js'; +import { MANAGE_PERM_PREFIX } from '../permission/consts.js'; +import { PermissionUtil } from '../permission/permissionUtil.js'; +import { Actor } from '../../core/actor.js'; +import { AclMode } from '../acl/ACLService.js'; + +const DEFAULT_CONTENT_TYPE = 'application/octet-stream'; +const DEFAULT_SIGNED_UPLOAD_EXPIRY_SECONDS = 60 * 15; + +// AWS SDK v3 surfaces missing-key errors with both `name` and `Code` set to +// "NoSuchKey". Check both — `Code` is the wire field, `name` is the JS class. +const isNoSuchKeyError = (err: unknown): boolean => { + if (!err || typeof err !== 'object') return false; + const e = err as { name?: unknown; Code?: unknown }; + return e.name === 'NoSuchKey' || e.Code === 'NoSuchKey'; +}; + +interface WriteTargetResolutionInput { + index: number; + normalizedInput: NormalizedWriteInput; +} + +interface WriteTargetResolutionResult { + index: number; + normalizedInput: NormalizedWriteInput; + existingEntry: FSEntry | null; + wasOverwrite: boolean; +} + +interface SignedMultipartCleanupTarget { + bucket: string; + bucketRegion: string; + objectKey: string; + signedUploadResult: SignedUploadResult; +} + +interface StartSignedWriteResult { + response: SignedWriteResponse; + createdDirectoryEntries: FSEntry[]; +} + +interface BatchStartSignedWriteResult { + responses: SignedWriteResponse[]; + createdDirectoryEntries: FSEntry[]; +} + +export class FSService extends PuterService { + declare protected stores: LayerInstances; + declare protected services: LayerInstances; + + override onServerStart(): void { + // Wire cache invalidation: listens to events emitted by FS + // mutations and invalidates Redis-cached fsentries. + new FSEntryCacheInvalidationEventHandler( + this.stores.fsEntry, + this.clients.event, + ); + + this.#registerPermissionRules(); + } + + /** + * FS-domain permission rules. App/site/user registrations live in their + * own services (AppPermissionService, SubdomainPermissionService, + * AuthService). Splitting by domain keeps the dependency surface narrow: + * each service only pulls the stores it actually needs. + * + * The path rewriter relies on `FSEntryStore.getEntryByPath`'s Redis cache + * (60s TTL), which is invalidated on every rename/move/delete through the + * existing event wiring. + */ + #registerPermissionRules(): void { + const permissions = this.services.permission; + const fsEntryStore = this.stores.fsEntry; + + // ── fs:/path:mode → fs::mode ───────────────────────────── + // Clients (puter.perms, requestPermission) emit path-based strings; + // stored as-is they'd never match anything, so resolve to uuid up + // front. + permissions.registerRewriter({ + id: 'fs-path-to-uid', + matches: (permission: string) => { + if ( + !permission.startsWith('fs:') && + !permission.startsWith(`${MANAGE_PERM_PREFIX}:fs:`) + ) + return false; + const [, specifier] = permission.split('fs:'); + return Boolean(specifier && specifier.startsWith('/')); + }, + rewrite: async (permission: string): Promise => { + const [manageOpt, pathPerm] = permission.split('fs:'); + const parts = PermissionUtil.split(pathPerm); + const path = parts[0]; + const rest = parts.slice(1); + if (!path) return permission; + const entry = await fsEntryStore.getEntryByPath(path); + if (!entry) { + throw new HttpError(404, `Entry not found: path=${path}`, { + legacyCode: 'subject_does_not_exist', + }); + } + const manage = manageOpt.replace(':', ''); + const joined = PermissionUtil.join('fs', entry.uuid, ...rest); + return manage ? `${manage}:${joined}` : joined; + }, + }); + + // ── is-owner ────────────────────────────────────────────────── + // For user actors, `fs::*` resolves iff the actor owns the + // underlying entry. Without this, `check(user, fs:UUID:*)` can't + // find a terminal and the `has_terminal` probe that #scanUserApp + // does on the issuer-recurse comes back false, which kills + // downstream app-under-user checks on user-owned files. + permissions.registerImplicator({ + id: 'is-owner', + shortcut: true, + matches: (permission: string): boolean => { + return ( + permission.startsWith('fs:') || + permission.startsWith(`${MANAGE_PERM_PREFIX}:fs:`) || + permission.startsWith( + `${MANAGE_PERM_PREFIX}:${MANAGE_PERM_PREFIX}:fs:`, + ) + ); + }, + check: async ({ actor, permission }): Promise => { + if (actor.app || actor.accessToken) return undefined; + if (!actor.user?.id) return undefined; + + const stripped = permission.replaceAll( + `${MANAGE_PERM_PREFIX}:`, + '', + ); + const parts = PermissionUtil.split(stripped); + const uid = parts[1]; + if (!uid) return undefined; + + const entry = await fsEntryStore.getEntryByUuid(uid); + if (!entry) return undefined; + if (entry.userId === actor.user.id) return {}; + return undefined; + }, + }); + + // ── fs-access-levels exploder ────────────────────────────────── + // `fs:UUID:see` implies `[list, read, write, manage:fs:UUID]`. + // ACLService.check already expands the same-family chain + // (see→list→read→write) via MODES_ABOVE, but the `manage:fs:UUID` + // arm only shows up here — without it, a grant of `fs:UUID:write` + // can't satisfy a direct `scan(actor, 'manage:fs:UUID')`. + const FS_MODE_RULES: Record = { + see: ['list', 'read', 'write'], + list: ['read', 'write'], + read: ['write'], + }; + permissions.registerExploder({ + id: 'fs-access-levels', + matches: (permission: string) => { + return ( + permission.startsWith('fs:') && + PermissionUtil.split(permission).length >= 3 + ); + }, + explode: async ({ permission }) => { + const out = [permission]; + const [fsPrefix, fileId, specifiedMode, ...rest] = + PermissionUtil.split(permission); + const widerModes = FS_MODE_RULES[specifiedMode]; + if (widerModes) { + for (const mode of widerModes) { + out.push( + PermissionUtil.join( + fsPrefix, + fileId, + mode, + ...rest.slice(1), + ), + ); + } + out.push( + PermissionUtil.join( + MANAGE_PERM_PREFIX, + fsPrefix, + fileId, + ), + ); + } + return out; + }, + }); + } + + #normalizePath(path: string): string { + const trimmedPath = path.trim(); + if (trimmedPath.length === 0) { + throw new HttpError(400, 'Path cannot be empty'); + } + if (trimmedPath === '~' || trimmedPath.startsWith('~/')) { + throw new HttpError(400, 'Home path must be resolved before write'); + } + + let normalizedPath = pathPosix.normalize(trimmedPath); + if (!normalizedPath.startsWith('/')) { + normalizedPath = `/${normalizedPath}`; + } + if (normalizedPath.length > 1 && normalizedPath.endsWith('/')) { + normalizedPath = normalizedPath.slice(0, -1); + } + return normalizedPath; + } + + #resolveBucket(metadata: FSEntryWriteInput): string { + const bucket = + metadata.bucket ?? this.config.s3_bucket ?? 'puter-local'; + if (typeof bucket !== 'string' || bucket.length === 0) { + throw new HttpError(500, 'Missing S3 bucket configuration'); + } + return bucket; + } + + #resolveBucketRegion(metadata: FSEntryWriteInput): string { + const bucketRegion = + metadata.bucketRegion ?? + this.config.s3_region ?? + this.config.region ?? + 'us-west-2'; + + if (typeof bucketRegion !== 'string' || bucketRegion.length === 0) { + throw new HttpError(500, 'Missing S3 region configuration'); + } + + return bucketRegion; + } + + #normalizeWriteInput( + userId: number, + metadata: FSEntryWriteInput, + ): NormalizedWriteInput { + const normalizedPath = this.#normalizePath(metadata.path); + if (normalizedPath === '/') { + throw new HttpError(400, 'Cannot write to root path'); + } + + const size = Number(metadata.size); + if (Number.isNaN(size) || size < 0) { + throw new HttpError(400, 'Invalid file size'); + } + + const metadataRecord = metadata as unknown as Record; + const dedupeName = Boolean( + metadata.dedupeName ?? metadataRecord.dedupe_name, + ); + + return { + userId, + path: normalizedPath, + size, + contentType: metadata.contentType ?? DEFAULT_CONTENT_TYPE, + checksumSha256: metadata.checksumSha256, + metadata: metadata.metadata, + thumbnail: metadata.thumbnail, + associatedAppId: metadata.associatedAppId, + overwrite: Boolean(metadata.overwrite), + dedupeName, + createMissingParents: Boolean(metadata.createMissingParents), + immutable: Boolean(metadata.immutable), + isPublic: metadata.isPublic, + multipartPartSize: metadata.multipartPartSize, + bucket: this.#resolveBucket(metadata), + bucketRegion: this.#resolveBucketRegion(metadata), + }; + } + + async #findDedupedPath( + targetPath: string, + reservedPaths: Set, + loadExistingEntry: (path: string) => Promise, + ): Promise { + const parentPath = pathPosix.dirname(targetPath); + const extension = pathPosix.extname(targetPath); + const fileName = pathPosix.basename(targetPath, extension); + + for (let suffix = 1; suffix < 100_000; suffix++) { + const dedupedPath = pathPosix.join( + parentPath, + `${fileName} (${suffix})${extension}`, + ); + if (reservedPaths.has(dedupedPath)) { + continue; + } + const existingEntry = await loadExistingEntry(dedupedPath); + if (!existingEntry) { + return dedupedPath; + } + } + + throw new HttpError(409, 'Unable to resolve deduped file path'); + } + + async #resolveWriteTargets( + userId: number, + inputs: WriteTargetResolutionInput[], + ): Promise { + const reservedPaths = new Set(); + const existingEntryCache = new Map>(); + const initialPaths = Array.from( + new Set(inputs.map((input) => input.normalizedInput.path)), + ); + const initialEntries = + await this.stores.fsEntry.getEntriesByPathsForUser( + userId, + initialPaths, + { + useTryHardRead: true, + skipCache: true, + // ACL has already gated the write — collision detection + // must see entries in shared folders the writer was + // granted access to, even when those live outside the + // writer's own namespace. + crossNamespace: true, + }, + ); + for (let index = 0; index < initialPaths.length; index++) { + const path = initialPaths[index]; + if (!path) { + continue; + } + existingEntryCache.set( + path, + Promise.resolve(initialEntries[index] ?? null), + ); + } + + const loadExistingEntry = async ( + path: string, + ): Promise => { + const cachedPromise = existingEntryCache.get(path); + if (cachedPromise) { + return await cachedPromise; + } + + const readPromise = this.stores.fsEntry.getEntryByPath(path, { + useTryHardRead: true, + skipCache: true, + }); + existingEntryCache.set(path, readPromise); + return await readPromise; + }; + + const results: WriteTargetResolutionResult[] = []; + for (const input of inputs) { + let normalizedInput = input.normalizedInput; + let existingEntry = await loadExistingEntry(normalizedInput.path); + const pathReservedInBatch = reservedPaths.has(normalizedInput.path); + + if (pathReservedInBatch || existingEntry) { + if (normalizedInput.dedupeName) { + const dedupedPath = await this.#findDedupedPath( + normalizedInput.path, + reservedPaths, + loadExistingEntry, + ); + normalizedInput = { + ...normalizedInput, + path: dedupedPath, + }; + existingEntry = await loadExistingEntry(dedupedPath); + } else if (pathReservedInBatch) { + throw new HttpError( + 409, + `Batch contains duplicate target path: ${normalizedInput.path}`, + ); + } + } + + if (existingEntry && existingEntry.isDir) { + throw new HttpError( + 409, + 'Cannot overwrite an existing directory', + ); + } + if (existingEntry && !normalizedInput.overwrite) { + throw new HttpError( + 409, + 'A file already exists at this path and overwrite was not requested', + ); + } + + reservedPaths.add(normalizedInput.path); + results.push({ + index: input.index, + normalizedInput, + existingEntry, + wasOverwrite: Boolean(existingEntry), + }); + } + + return results; + } + + #toCreateInput( + normalizedInput: NormalizedWriteInput, + objectKey: string, + ): FSEntryCreateInput { + return { + userId: normalizedInput.userId, + uuid: objectKey, + path: normalizedInput.path, + size: normalizedInput.size, + contentType: normalizedInput.contentType, + checksumSha256: normalizedInput.checksumSha256, + metadata: normalizedInput.metadata, + thumbnail: normalizedInput.thumbnail, + associatedAppId: normalizedInput.associatedAppId, + overwrite: normalizedInput.overwrite, + createMissingParents: normalizedInput.createMissingParents, + immutable: normalizedInput.immutable, + isPublic: normalizedInput.isPublic, + multipartPartSize: normalizedInput.multipartPartSize, + bucket: normalizedInput.bucket, + bucketRegion: normalizedInput.bucketRegion, + }; + } + + #determineUploadMode( + requestUploadMode: UploadMode | 'auto' | undefined, + size: number, + ): UploadMode { + const maxSingleUploadSize = + this.stores.s3Object.getMaxSingleUploadSize(); + if (requestUploadMode === 'multipart') { + return 'multipart'; + } + if (requestUploadMode === 'single') { + return size > maxSingleUploadSize ? 'multipart' : 'single'; + } + return size > maxSingleUploadSize ? 'multipart' : 'single'; + } + + #resolveStorageMax( + allowanceMax: number, + storageAllowanceMaxOverride?: number, + ): number { + if (allowanceMax === Number.MAX_SAFE_INTEGER) { + return allowanceMax; + } + if (storageAllowanceMaxOverride === undefined) { + return allowanceMax; + } + if ( + !Number.isFinite(storageAllowanceMaxOverride) || + storageAllowanceMaxOverride < 0 + ) { + return allowanceMax; + } + return Math.max(allowanceMax, storageAllowanceMaxOverride); + } + + async #assertStorageAllowance( + userId: number, + incomingSize: number, + existingSize = 0, + storageAllowanceMaxOverride?: number, + ): Promise { + const allowance = + await this.stores.fsEntry.getUserStorageAllowance(userId); + const maxStorage = this.#resolveStorageMax( + allowance.max, + storageAllowanceMaxOverride, + ); + if (maxStorage === Number.MAX_SAFE_INTEGER) { + return; + } + + const projectedUsage = allowance.curr - existingSize + incomingSize; + if (projectedUsage > maxStorage) { + throw new HttpError(413, 'Storage limit reached'); + } + } + + async #assertStorageAllowanceForBatch( + userId: number, + sizeChanges: Array<{ incomingSize: number; existingSize: number }>, + storageAllowanceMaxOverride?: number, + ): Promise { + if (sizeChanges.length === 0) { + return; + } + + const allowance = + await this.stores.fsEntry.getUserStorageAllowance(userId); + const maxStorage = this.#resolveStorageMax( + allowance.max, + storageAllowanceMaxOverride, + ); + if (maxStorage === Number.MAX_SAFE_INTEGER) { + return; + } + + let projectedUsage = allowance.curr; + for (const sizeChange of sizeChanges) { + projectedUsage = + projectedUsage - + sizeChange.existingSize + + sizeChange.incomingSize; + } + + if (projectedUsage > maxStorage) { + throw new HttpError(413, 'Storage limit reached'); + } + } + + #toErrorMessage(error: unknown): string { + if (error instanceof Error) { + return error.message; + } + return 'Unknown error'; + } + + #toError(error: unknown, fallbackMessage: string): Error { + if (error instanceof Error) { + return error; + } + return new Error(fallbackMessage); + } + + #toMultipartParts( + parts: CompleteWriteRequest['parts'], + ): MultipartCompletePart[] { + if (!parts || parts.length === 0) { + return []; + } + return parts.map((part) => ({ + partNumber: Number(part.partNumber), + etag: part.etag, + })); + } + + #parseSessionMetadata(session: PendingUploadSession): FSEntryCreateInput { + if (!session.metadataJson) { + throw new HttpError(500, 'Upload session metadata is missing'); + } + + const parsedMetadata = JSON.parse( + session.metadataJson, + ) as FSEntryCreateInput; + return { + ...parsedMetadata, + userId: session.userId, + uuid: session.objectKey, + path: session.targetPath, + size: session.size, + contentType: session.contentType, + checksumSha256: session.checksumSha256 ?? undefined, + bucket: session.bucket ?? undefined, + bucketRegion: session.bucketRegion ?? undefined, + overwrite: Boolean(session.overwriteTargetUid), + }; + } + + #isBinaryPayload(value: unknown): value is BinaryPayload { + return Boolean( + value && + typeof value === 'object' && + 'base64' in value && + typeof (value as BinaryPayload).base64 === 'string', + ); + } + + #isNodeStream(value: unknown): value is Readable { + return Boolean( + value && + typeof value === 'object' && + typeof (value as Readable).pipe === 'function', + ); + } + + #isWebReadableStream(value: unknown): value is ReadableStream { + return Boolean( + value && + typeof value === 'object' && + typeof (value as ReadableStream).getReader === 'function', + ); + } + + #createCountingStream( + source: Readable, + uploadTracker?: UploadProgressTrackerLike, + ): { + stream: Readable; + uploadedSize: () => number; + contentHashSha256: () => string; + } { + let uploadedBytes = 0; + const hash = createHash('sha256'); + const countingStream = new Transform({ + transform( + chunk: unknown, + _encoding: string, + callback: TransformCallback, + ) { + let chunkLength = 0; + if (Buffer.isBuffer(chunk) || chunk instanceof Uint8Array) { + chunkLength = chunk.byteLength; + hash.update(chunk); + } else if (typeof chunk === 'string') { + chunkLength = Buffer.byteLength(chunk); + hash.update(chunk); + } + uploadedBytes += chunkLength; + if (chunkLength > 0 && uploadTracker) { + uploadTracker.add(chunkLength); + } + callback(null, chunk as Buffer | Uint8Array | string); + }, + }); + + source.on('error', (error) => { + countingStream.destroy(error); + }); + source.pipe(countingStream); + + return { + stream: countingStream, + uploadedSize: () => uploadedBytes, + contentHashSha256: () => hash.digest('hex'), + }; + } + + async #toUploadBody( + content: WriteRequest['fileContent'], + encoding: WriteRequest['encoding'], + uploadTracker?: UploadProgressTrackerLike, + ): Promise { + if (Buffer.isBuffer(content)) { + const hash = createHash('sha256'); + hash.update(content); + return { + body: content, + contentLength: content.byteLength, + uploadedSize: () => content.byteLength, + contentHashSha256: hash.digest('hex'), + }; + } + if (this.#isBinaryPayload(content)) { + const buffer = Buffer.from(content.base64, 'base64'); + const hash = createHash('sha256'); + hash.update(buffer); + return { + body: buffer, + contentLength: buffer.byteLength, + uploadedSize: () => buffer.byteLength, + contentHashSha256: hash.digest('hex'), + }; + } + if (typeof content === 'string') { + if (encoding === 'base64') { + const buffer = Buffer.from(content, 'base64'); + const hash = createHash('sha256'); + hash.update(buffer); + return { + body: buffer, + contentLength: buffer.byteLength, + uploadedSize: () => buffer.byteLength, + contentHashSha256: hash.digest('hex'), + }; + } + const buffer = Buffer.from(content, encoding ?? 'utf8'); + const hash = createHash('sha256'); + hash.update(buffer); + return { + body: buffer, + contentLength: buffer.byteLength, + uploadedSize: () => buffer.byteLength, + contentHashSha256: hash.digest('hex'), + }; + } + if (content instanceof Uint8Array) { + const hash = createHash('sha256'); + hash.update(content); + return { + body: content, + contentLength: content.byteLength, + uploadedSize: () => content.byteLength, + contentHashSha256: hash.digest('hex'), + }; + } + if (content instanceof ArrayBuffer) { + const buffer = Buffer.from(content); + const hash = createHash('sha256'); + hash.update(buffer); + return { + body: buffer, + contentLength: buffer.byteLength, + uploadedSize: () => buffer.byteLength, + contentHashSha256: hash.digest('hex'), + }; + } + if (this.#isNodeStream(content)) { + const streamPayload = this.#createCountingStream( + content, + uploadTracker, + ); + return { + body: streamPayload.stream, + uploadedSize: streamPayload.uploadedSize, + contentHashSha256: null, + finalizeContentHashSha256: () => + streamPayload.contentHashSha256(), + }; + } + if (this.#isWebReadableStream(content)) { + const reader = content.getReader(); + const asyncIterable = { + async *[Symbol.asyncIterator](): AsyncGenerator< + Uint8Array, + void, + void + > { + while (true) { + const readResult = await reader.read(); + if (readResult.done) { + return; + } + if (readResult.value) { + yield readResult.value; + } + } + }, + }; + const streamPayload = this.#createCountingStream( + Readable.from(asyncIterable), + uploadTracker, + ); + return { + body: streamPayload.stream, + uploadedSize: streamPayload.uploadedSize, + contentHashSha256: null, + finalizeContentHashSha256: () => + streamPayload.contentHashSha256(), + }; + } + if (content instanceof Blob) { + const reader = content.stream().getReader(); + const asyncIterable = { + async *[Symbol.asyncIterator](): AsyncGenerator< + Uint8Array, + void, + void + > { + while (true) { + const readResult = await reader.read(); + if (readResult.done) { + return; + } + if (readResult.value) { + yield readResult.value; + } + } + }, + }; + const streamPayload = this.#createCountingStream( + Readable.from(asyncIterable), + uploadTracker, + ); + return { + body: streamPayload.stream, + contentLength: Number.isFinite(content.size) + ? content.size + : undefined, + uploadedSize: streamPayload.uploadedSize, + contentHashSha256: null, + finalizeContentHashSha256: () => + streamPayload.contentHashSha256(), + }; + } + + throw new HttpError(400, 'Unsupported file content payload'); + } + + async #cleanupPreparedBatchUploads( + preparedBatch: PreparedBatchWrite, + uploadedItems: UploadedBatchWriteItem[], + ): Promise { + const cleanupTargets = uploadedItems + .map((uploadedItem) => { + const preparedItem = preparedBatch.itemsByIndex.get( + uploadedItem.index, + ); + if (!preparedItem || preparedItem.wasOverwrite) { + return null; + } + + return { + bucket: preparedItem.normalizedInput.bucket, + bucketRegion: preparedItem.normalizedInput.bucketRegion, + objectKey: uploadedItem.objectKey, + }; + }) + .filter( + ( + target, + ): target is { + bucket: string; + bucketRegion: string; + objectKey: string; + } => Boolean(target), + ); + + if (cleanupTargets.length === 0) { + return; + } + + const cleanupResults = await Promise.allSettled( + cleanupTargets.map((target) => { + return this.stores.s3Object.deleteObject( + target.bucket, + target.objectKey, + target.bucketRegion, + ); + }), + ); + + const cleanupFailures = cleanupResults.filter( + (result) => result.status === 'rejected', + ); + if (cleanupFailures.length > 0) { + console.error( + 'prodfsv2 failed to clean up batch upload objects', + cleanupFailures, + ); + } + } + + getMaxSingleUploadSize(): number { + return this.stores.s3Object.getMaxSingleUploadSize(); + } + + async #cleanupSignedMultipartUploads( + uploads: SignedMultipartCleanupTarget[], + ): Promise { + if (uploads.length === 0) { + return; + } + + const cleanupResults = await Promise.allSettled( + uploads.map((upload) => { + if ( + upload.signedUploadResult.uploadMode !== 'multipart' || + !upload.signedUploadResult.multipartUploadId + ) { + return Promise.resolve(); + } + + return this.stores.s3Object.abortMutipartUpload( + upload.signedUploadResult.multipartUploadId, + upload.bucketRegion, + upload.bucket, + upload.objectKey, + ); + }), + ); + + const cleanupFailures = cleanupResults.filter( + (result) => result.status === 'rejected', + ); + if (cleanupFailures.length > 0) { + console.error( + 'prodfsv2 failed to abort signed multipart uploads', + cleanupFailures, + ); + } + } + + #toSignedMultipartCleanupTargets( + items: Array<{ + index: number; + normalizedInput: NormalizedWriteInput; + }>, + objectKeys: string[], + signedResultsByIndex: Map, + ): SignedMultipartCleanupTarget[] { + return items + .map((item, index) => { + const signedUploadResult = signedResultsByIndex.get(item.index); + const objectKey = objectKeys[index]; + if (!signedUploadResult || !objectKey) { + return null; + } + + return { + bucket: item.normalizedInput.bucket, + bucketRegion: item.normalizedInput.bucketRegion, + objectKey, + signedUploadResult, + }; + }) + .filter((upload): upload is SignedMultipartCleanupTarget => + Boolean(upload), + ); + } + + #toSignedWriteResponse( + sessionId: string, + normalizedInput: NormalizedWriteInput, + objectKey: string, + signedUploadResult: SignedUploadResult, + ): SignedWriteResponse { + return { + sessionId, + uploadMode: signedUploadResult.uploadMode, + objectKey, + bucket: normalizedInput.bucket, + bucketRegion: normalizedInput.bucketRegion, + contentType: normalizedInput.contentType, + expiresAt: signedUploadResult.expiresAt, + ...(signedUploadResult.url ? { url: signedUploadResult.url } : {}), + ...(signedUploadResult.multipartUploadId + ? { multipartUploadId: signedUploadResult.multipartUploadId } + : {}), + ...(signedUploadResult.multipartPartSize + ? { multipartPartSize: signedUploadResult.multipartPartSize } + : {}), + ...(signedUploadResult.multipartPartCount + ? { multipartPartCount: signedUploadResult.multipartPartCount } + : {}), + ...(signedUploadResult.multipartPartUrls + ? { multipartPartUrls: signedUploadResult.multipartPartUrls } + : {}), + }; + } + + #toDirectorySignedWriteResponse( + fsEntry: FSEntry, + directoryCreated: boolean, + ): SignedWriteResponse { + return { + sessionId: '', + uploadMode: 'single', + objectKey: fsEntry.uuid, + bucket: fsEntry.bucket ?? '', + bucketRegion: fsEntry.bucketRegion ?? '', + contentType: 'inode/directory', + expiresAt: Date.now(), + directoryCreated, + fsEntry, + }; + } + + async entryExistsByPath(path: string): Promise { + const entry = await this.stores.fsEntry.getEntryByPath(path); + return entry !== null; + } + + async getAncestorChain( + path: string, + ): Promise> { + const paths: string[] = []; + let cursor = this.#normalizePath(path); + while (cursor !== '/') { + paths.push(cursor); + cursor = pathPosix.dirname(cursor); + } + + const entriesByPath = + await this.stores.fsEntry.getEntriesByPaths(paths); + + const ancestors: Array<{ uid: string; path: string }> = []; + for (const p of paths) { + const entry = entriesByPath.get(p); + if (entry) { + ancestors.push({ uid: entry.uid, path: entry.path }); + } + } + return ancestors; + } + + async prepareBatchWrites( + userId: number, + writeRequests: BatchWritePrepareRequest[], + storageAllowanceMax?: number, + ): Promise { + if (writeRequests.length === 0) { + return { + userId, + items: [], + itemsByIndex: new Map(), + ...(storageAllowanceMax !== undefined + ? { storageAllowanceMax } + : {}), + }; + } + + const normalizedRequests = writeRequests.map((writeRequest, index) => { + const normalizedInput = this.#normalizeWriteInput( + userId, + writeRequest.fileMetadata, + ); + const requestedThumbnail = + writeRequest.thumbnailData ?? normalizedInput.thumbnail ?? null; + normalizedInput.thumbnail = null; + return { + index, + normalizedInput, + requestedThumbnail, + guiMetadata: writeRequest.guiMetadata, + }; + }); + + const resolvedTargets = await this.#resolveWriteTargets( + userId, + normalizedRequests.map((request) => ({ + index: request.index, + normalizedInput: request.normalizedInput, + })), + ); + const resolvedTargetMap = new Map( + resolvedTargets.map((resolvedTarget) => [ + resolvedTarget.index, + resolvedTarget, + ]), + ); + const resolvedRequests = normalizedRequests.map((request) => { + const resolvedTarget = resolvedTargetMap.get(request.index); + if (!resolvedTarget) { + throw new Error( + `Failed to resolve write target for index ${request.index}`, + ); + } + return { + ...request, + normalizedInput: resolvedTarget.normalizedInput, + existingEntry: resolvedTarget.existingEntry, + wasOverwrite: resolvedTarget.wasOverwrite, + }; + }); + + await this.stores.fsEntry.resolveParentDirectoriesBatch( + userId, + resolvedRequests.map((item) => ({ + parentPath: pathPosix.dirname(item.normalizedInput.path), + createPaths: item.normalizedInput.createMissingParents, + })), + ); + + const items = resolvedRequests.map((item) => ({ + index: item.index, + normalizedInput: item.normalizedInput, + existingEntry: item.existingEntry, + objectKey: item.existingEntry?.uuid ?? uuidv4(), + wasOverwrite: item.wasOverwrite, + requestedThumbnail: item.requestedThumbnail, + guiMetadata: item.guiMetadata, + })); + const itemsByIndex = new Map(); + for (const item of items) { + itemsByIndex.set(item.index, item); + } + + return { + userId, + items, + itemsByIndex, + ...(storageAllowanceMax !== undefined + ? { storageAllowanceMax } + : {}), + }; + } + + async assertStorageAllowanceForPreparedBatch( + preparedBatch: PreparedBatchWrite, + uploadedItems?: UploadedBatchWriteItem[], + storageAllowanceMaxOverride?: number, + ): Promise { + if (preparedBatch.items.length === 0) { + return; + } + + const uploadedItemMap = new Map(); + if (uploadedItems) { + for (const uploadedItem of uploadedItems) { + uploadedItemMap.set(uploadedItem.index, uploadedItem); + } + } + + const sizeChanges = preparedBatch.items.map((item) => { + const uploadedItem = uploadedItemMap.get(item.index); + return { + incomingSize: uploadedItem + ? uploadedItem.uploadedSize + : item.normalizedInput.size, + existingSize: item.existingEntry?.size ?? 0, + }; + }); + + const storageAllowanceMax = + storageAllowanceMaxOverride ?? preparedBatch.storageAllowanceMax; + await this.#assertStorageAllowanceForBatch( + preparedBatch.userId, + sizeChanges, + storageAllowanceMax, + ); + } + + async uploadPreparedBatchItem( + input: UploadPreparedBatchItemInput, + ): Promise { + const preparedItem = input.preparedBatch.itemsByIndex.get( + input.itemIndex, + ); + if (!preparedItem) { + throw new HttpError( + 400, + `Batch metadata was not found for index ${input.itemIndex}`, + ); + } + + const uploadBody = await this.#toUploadBody( + input.fileContent, + input.encoding, + input.uploadTracker, + ); + + await this.stores.s3Object.uploadFromServer( + { + bucket: preparedItem.normalizedInput.bucket, + objectKey: preparedItem.objectKey, + contentType: preparedItem.normalizedInput.contentType, + body: uploadBody.body, + ...(uploadBody.contentLength !== undefined + ? { contentLength: uploadBody.contentLength } + : {}), + ...(Number.isFinite(preparedItem.normalizedInput.size) + ? { sizeHint: preparedItem.normalizedInput.size } + : {}), + }, + preparedItem.normalizedInput.bucketRegion, + ); + + const uploadedSize = uploadBody.uploadedSize(); + if (input.uploadTracker) { + const currentTrackedSize = Number( + input.uploadTracker.progress ?? 0, + ); + if (uploadedSize > currentTrackedSize) { + input.uploadTracker.add(uploadedSize - currentTrackedSize); + } + } + + return { + index: preparedItem.index, + objectKey: preparedItem.objectKey, + uploadedSize, + contentHashSha256: uploadBody.finalizeContentHashSha256 + ? uploadBody.finalizeContentHashSha256() + : uploadBody.contentHashSha256, + }; + } + + async finalizePreparedBatchWrites( + preparedBatch: PreparedBatchWrite, + uploadedItems: UploadedBatchWriteItem[], + ): Promise { + try { + if (preparedBatch.items.length !== uploadedItems.length) { + throw new HttpError( + 400, + 'Some batch files were missing upload content', + ); + } + + await this.assertStorageAllowanceForPreparedBatch( + preparedBatch, + uploadedItems, + ); + + const uploadedItemMap = new Map(); + for (const uploadedItem of uploadedItems) { + uploadedItemMap.set(uploadedItem.index, uploadedItem); + } + + const createInputs = preparedBatch.items.map((item) => { + const uploadedItem = uploadedItemMap.get(item.index); + if (!uploadedItem) { + throw new HttpError( + 400, + `Missing uploaded file content for index ${item.index}`, + ); + } + item.normalizedInput.size = uploadedItem.uploadedSize; + return this.#toCreateInput( + item.normalizedInput, + uploadedItem.objectKey, + ); + }); + + const fsEntries = await this.stores.fsEntry.batchCreateEntries( + createInputs, + true, + ); + return preparedBatch.items.map((item, index) => { + const fsEntry = fsEntries[index]; + if (!fsEntry) { + throw new Error( + `Failed to resolve batch write result at index ${index}`, + ); + } + const uploadedItem = uploadedItemMap.get(item.index); + this.#emitFsEvent( + item.wasOverwrite ? 'fs.write.file' : 'fs.create.file', + fsEntry, + ); + return { + fsEntry, + wasOverwrite: item.wasOverwrite, + requestedThumbnail: item.requestedThumbnail, + contentHashSha256: uploadedItem?.contentHashSha256 ?? null, + }; + }); + } catch (error) { + await this.#cleanupPreparedBatchUploads( + preparedBatch, + uploadedItems, + ); + throw error; + } + } + + async startUrlWrite( + userId: number, + signedWriteRequest: SignedWriteRequest, + storageAllowanceMax?: number, + ): Promise { + const result = await this.startUrlWriteWithCreatedDirectories( + userId, + signedWriteRequest, + storageAllowanceMax, + ); + return result.response; + } + + async startUrlWriteWithCreatedDirectories( + userId: number, + signedWriteRequest: SignedWriteRequest, + storageAllowanceMax?: number, + ): Promise { + let normalizedInput = this.#normalizeWriteInput( + userId, + signedWriteRequest.fileMetadata, + ); + if (signedWriteRequest.directory) { + const { entries, createdDirectoryEntries } = + await this.stores.fsEntry.ensureDirectoriesForUserWithCreated( + userId, + [ + { + path: normalizedInput.path, + createPaths: normalizedInput.createMissingParents, + }, + ], + ); + const [directoryEntry] = entries; + if (!directoryEntry) { + throw new Error( + 'Failed to resolve directory entry after start write', + ); + } + const createdDirectoryPathSet = new Set( + createdDirectoryEntries.map((entry) => entry.path), + ); + return { + response: this.#toDirectorySignedWriteResponse( + directoryEntry, + createdDirectoryPathSet.has(normalizedInput.path), + ), + createdDirectoryEntries, + }; + } + + const [resolvedTarget] = await this.#resolveWriteTargets(userId, [ + { + index: 0, + normalizedInput, + }, + ]); + if (!resolvedTarget) { + throw new Error('Failed to resolve write target'); + } + normalizedInput = resolvedTarget.normalizedInput; + const existingEntry = resolvedTarget.existingEntry; + + const existingSize = existingEntry?.size ?? 0; + const parentPath = pathPosix.dirname(normalizedInput.path); + const [, { parentEntries, createdDirectoryEntries }] = + await Promise.all([ + this.#assertStorageAllowance( + userId, + normalizedInput.size, + existingSize, + storageAllowanceMax, + ), + this.stores.fsEntry.resolveParentDirectoriesBatchWithCreated( + userId, + [ + { + parentPath, + createPaths: normalizedInput.createMissingParents, + }, + ], + ), + ]); + const [parentEntry] = parentEntries; + if (!parentEntry) { + throw new Error( + 'Failed to resolve parent directory for signed write', + ); + } + + const objectKey = existingEntry?.uuid ?? uuidv4(); + const uploadMode = this.#determineUploadMode( + signedWriteRequest.uploadMode, + normalizedInput.size, + ); + const expiresInSeconds = + signedWriteRequest.expiresInSeconds ?? + DEFAULT_SIGNED_UPLOAD_EXPIRY_SECONDS; + const createInput = this.#toCreateInput(normalizedInput, objectKey); + + const signedUploadResult = + await this.stores.s3Object.createSignedUploadUrl( + { + bucket: normalizedInput.bucket, + objectKey, + size: normalizedInput.size, + contentType: normalizedInput.contentType, + uploadMode, + expiresInSeconds, + multipartPartSize: normalizedInput.multipartPartSize, + }, + normalizedInput.bucketRegion, + ); + + const sessionId = uuidv4(); + const pendingUploadInput: PendingUploadCreateInput = { + sessionId, + userId, + appId: normalizedInput.associatedAppId ?? null, + parentUid: parentEntry.uuid, + parentPath: parentEntry.path, + targetName: pathPosix.basename(normalizedInput.path), + targetPath: normalizedInput.path, + overwriteTargetUid: existingEntry?.uuid ?? null, + contentType: normalizedInput.contentType, + size: normalizedInput.size, + checksumSha256: normalizedInput.checksumSha256 ?? null, + uploadMode, + multipartUploadId: signedUploadResult.multipartUploadId ?? null, + multipartPartSize: signedUploadResult.multipartPartSize ?? null, + multipartPartCount: signedUploadResult.multipartPartCount ?? null, + storageProvider: 's3', + bucket: normalizedInput.bucket, + bucketRegion: normalizedInput.bucketRegion, + objectKey, + metadataJson: JSON.stringify(createInput), + expiresAt: signedUploadResult.expiresAt, + }; + + try { + await this.stores.fsEntry.createPendingEntry(pendingUploadInput); + } catch (error) { + await this.#cleanupSignedMultipartUploads([ + { + bucket: normalizedInput.bucket, + bucketRegion: normalizedInput.bucketRegion, + objectKey, + signedUploadResult, + }, + ]); + throw error; + } + + return { + response: this.#toSignedWriteResponse( + sessionId, + normalizedInput, + objectKey, + signedUploadResult, + ), + createdDirectoryEntries, + }; + } + + async batchStartUrlWrites( + userId: number, + signedWriteRequests: SignedWriteRequest[], + storageAllowanceMax?: number, + ): Promise { + const result = await this.batchStartUrlWritesWithCreatedDirectories( + userId, + signedWriteRequests, + storageAllowanceMax, + ); + return result.responses; + } + + async batchStartUrlWritesWithCreatedDirectories( + userId: number, + signedWriteRequests: SignedWriteRequest[], + storageAllowanceMax?: number, + ): Promise { + if (signedWriteRequests.length === 0) { + return { + responses: [], + createdDirectoryEntries: [], + }; + } + + const normalizedRequests = signedWriteRequests.map( + (signedWriteRequest, index) => ({ + index, + request: signedWriteRequest, + isDirectory: Boolean(signedWriteRequest.directory), + normalizedInput: this.#normalizeWriteInput( + userId, + signedWriteRequest.fileMetadata, + ), + }), + ); + const responsesByIndex = new Map(); + const createdDirectoryEntriesByPath = new Map(); + + const directoryItems = normalizedRequests.filter( + (item) => item.isDirectory, + ); + const directoryPathSet = new Set(); + for (const directoryItem of directoryItems) { + const targetPath = directoryItem.normalizedInput.path; + if (directoryPathSet.has(targetPath)) { + throw new HttpError( + 409, + `Batch contains duplicate target path: ${targetPath}`, + ); + } + directoryPathSet.add(targetPath); + } + if (directoryItems.length > 0) { + const { + entries: ensuredDirectoryEntries, + createdDirectoryEntries, + } = await this.stores.fsEntry.ensureDirectoriesForUserWithCreated( + userId, + directoryItems.map((item) => ({ + path: item.normalizedInput.path, + createPaths: item.normalizedInput.createMissingParents, + })), + ); + for (const createdDirectoryEntry of createdDirectoryEntries) { + createdDirectoryEntriesByPath.set( + createdDirectoryEntry.path, + createdDirectoryEntry, + ); + } + + for (let index = 0; index < directoryItems.length; index++) { + const item = directoryItems[index]; + const directoryEntry = ensuredDirectoryEntries[index]; + if (!item || !directoryEntry) { + throw new Error( + 'Failed to build directory response from batch start data', + ); + } + responsesByIndex.set( + item.index, + this.#toDirectorySignedWriteResponse( + directoryEntry, + createdDirectoryEntriesByPath.has( + item.normalizedInput.path, + ), + ), + ); + } + } + + const fileItems = normalizedRequests.filter( + (item) => !item.isDirectory, + ); + if (fileItems.length > 0) { + const resolvedTargets = await this.#resolveWriteTargets( + userId, + fileItems.map((item) => ({ + index: item.index, + normalizedInput: item.normalizedInput, + })), + ); + const resolvedTargetMap = new Map< + number, + WriteTargetResolutionResult + >( + resolvedTargets.map((resolvedTarget) => [ + resolvedTarget.index, + resolvedTarget, + ]), + ); + const resolvedFileItems = fileItems.map((item) => { + const resolvedTarget = resolvedTargetMap.get(item.index); + if (!resolvedTarget) { + throw new Error( + `Failed to resolve write target for batch index ${item.index}`, + ); + } + + return { + ...item, + normalizedInput: resolvedTarget.normalizedInput, + existingEntry: resolvedTarget.existingEntry, + }; + }); + + const allowanceChecks: Array<{ + incomingSize: number; + existingSize: number; + }> = []; + for (const item of resolvedFileItems) { + allowanceChecks.push({ + incomingSize: item.normalizedInput.size, + existingSize: item.existingEntry?.size ?? 0, + }); + } + const [ + , + { + parentEntries, + createdDirectoryEntries: createdParentDirectoryEntries, + }, + ] = await Promise.all([ + this.#assertStorageAllowanceForBatch( + userId, + allowanceChecks, + storageAllowanceMax, + ), + this.stores.fsEntry.resolveParentDirectoriesBatchWithCreated( + userId, + resolvedFileItems.map((item) => ({ + parentPath: pathPosix.dirname( + item.normalizedInput.path, + ), + createPaths: item.normalizedInput.createMissingParents, + })), + ), + ]); + for (const createdParentDirectoryEntry of createdParentDirectoryEntries) { + createdDirectoryEntriesByPath.set( + createdParentDirectoryEntry.path, + createdParentDirectoryEntry, + ); + } + + const objectKeys = resolvedFileItems.map((item) => { + return item.existingEntry?.uuid ?? uuidv4(); + }); + const uploadModes = resolvedFileItems.map((item) => { + return this.#determineUploadMode( + item.request.uploadMode, + item.normalizedInput.size, + ); + }); + const sessionIds = resolvedFileItems.map(() => uuidv4()); + + const signedResultsByIndex = new Map(); + const writesByRegion = new Map< + string, + Array<{ + requestIndex: number; + input: { + bucket: string; + objectKey: string; + size: number; + contentType: string; + uploadMode: UploadMode; + expiresInSeconds: number; + multipartPartSize?: number; + }; + }> + >(); + for (let index = 0; index < resolvedFileItems.length; index++) { + const item = resolvedFileItems[index]; + const objectKey = objectKeys[index]; + const uploadMode = uploadModes[index]; + if (!item || !objectKey || !uploadMode) { + throw new Error( + 'Failed to build batch signed upload request', + ); + } + const regionEntries = + writesByRegion.get(item.normalizedInput.bucketRegion) ?? []; + regionEntries.push({ + requestIndex: item.index, + input: { + bucket: item.normalizedInput.bucket, + objectKey, + size: item.normalizedInput.size, + contentType: item.normalizedInput.contentType, + uploadMode, + expiresInSeconds: + item.request.expiresInSeconds ?? + DEFAULT_SIGNED_UPLOAD_EXPIRY_SECONDS, + multipartPartSize: + item.normalizedInput.multipartPartSize, + }, + }); + writesByRegion.set( + item.normalizedInput.bucketRegion, + regionEntries, + ); + } + + const regionResults = await Promise.allSettled( + Array.from(writesByRegion.entries()).map( + async ([region, regionWrites]) => { + const signedResults = + await this.stores.s3Object.batchCreateSignedUploadUrls( + regionWrites.map((item) => item.input), + region, + ); + for ( + let index = 0; + index < regionWrites.length; + index++ + ) { + const regionWrite = regionWrites[index]; + const signedResult = signedResults[index]; + if (!regionWrite || !signedResult) { + throw new Error( + 'Failed to map signed upload result to request', + ); + } + signedResultsByIndex.set( + regionWrite.requestIndex, + signedResult, + ); + } + }, + ), + ); + const signedMultipartCleanupTargets = + this.#toSignedMultipartCleanupTargets( + resolvedFileItems, + objectKeys, + signedResultsByIndex, + ); + + const failedRegionResult = regionResults.find( + (result) => result.status === 'rejected', + ); + if (failedRegionResult?.status === 'rejected') { + await this.#cleanupSignedMultipartUploads( + signedMultipartCleanupTargets, + ); + + throw this.#toError( + failedRegionResult.reason, + 'Failed to create batch signed upload urls', + ); + } + + try { + const pendingInputs: PendingUploadCreateInput[] = []; + for (let index = 0; index < resolvedFileItems.length; index++) { + const item = resolvedFileItems[index]; + const parentEntry = parentEntries[index]; + const objectKey = objectKeys[index]; + const sessionId = sessionIds[index]; + const uploadMode = uploadModes[index]; + const existingEntry = item?.existingEntry; + if ( + !item || + !parentEntry || + !objectKey || + !sessionId || + !uploadMode + ) { + throw new Error( + 'Failed to build pending upload input from batch start data', + ); + } + const signedUploadResult = signedResultsByIndex.get( + item.index, + ); + if (!signedUploadResult) { + throw new Error( + 'Failed to resolve signed upload result for batch start data', + ); + } + + const createInput = this.#toCreateInput( + item.normalizedInput, + objectKey, + ); + pendingInputs.push({ + sessionId, + userId, + appId: item.normalizedInput.associatedAppId ?? null, + parentUid: parentEntry.uuid, + parentPath: parentEntry.path, + targetName: pathPosix.basename( + item.normalizedInput.path, + ), + targetPath: item.normalizedInput.path, + overwriteTargetUid: existingEntry?.uuid ?? null, + contentType: item.normalizedInput.contentType, + size: item.normalizedInput.size, + checksumSha256: + item.normalizedInput.checksumSha256 ?? null, + uploadMode, + multipartUploadId: + signedUploadResult.multipartUploadId ?? null, + multipartPartSize: + signedUploadResult.multipartPartSize ?? null, + multipartPartCount: + signedUploadResult.multipartPartCount ?? null, + storageProvider: 's3', + bucket: item.normalizedInput.bucket, + bucketRegion: item.normalizedInput.bucketRegion, + objectKey, + metadataJson: JSON.stringify(createInput), + expiresAt: signedUploadResult.expiresAt, + }); + } + + await this.stores.fsEntry.batchCreatePendingEntries( + pendingInputs, + ); + + for (let index = 0; index < resolvedFileItems.length; index++) { + const item = resolvedFileItems[index]; + const sessionId = sessionIds[index]; + const objectKey = objectKeys[index]; + if (!item || !sessionId || !objectKey) { + throw new Error( + 'Failed to build signed write response from batch start data', + ); + } + const signedUploadResult = signedResultsByIndex.get( + item.index, + ); + if (!signedUploadResult) { + throw new Error( + 'Failed to resolve signed upload result for batch response data', + ); + } + responsesByIndex.set( + item.index, + this.#toSignedWriteResponse( + sessionId, + item.normalizedInput, + objectKey, + signedUploadResult, + ), + ); + } + } catch (error) { + await this.#cleanupSignedMultipartUploads( + signedMultipartCleanupTargets, + ); + throw error; + } + } + + const responses = normalizedRequests.map((request) => { + const response = responsesByIndex.get(request.index); + if (!response) { + throw new Error( + `Failed to resolve signed batch response for index ${request.index}`, + ); + } + return response; + }); + return { + responses, + createdDirectoryEntries: Array.from( + createdDirectoryEntriesByPath.values(), + ), + }; + } + + async signMultipartParts( + userId: number, + request: SignMultipartPartsRequest, + ): Promise { + if (!request?.uploadId) { + throw new HttpError(400, 'Missing uploadId'); + } + if ( + !Array.isArray(request.partNumbers) || + request.partNumbers.length === 0 + ) { + throw new HttpError(400, 'Missing partNumbers'); + } + + const uniquePartNumbers = Array.from( + new Set(request.partNumbers.map((value) => Number(value))), + ); + if ( + uniquePartNumbers.some( + (partNumber) => + !Number.isInteger(partNumber) || partNumber <= 0, + ) + ) { + throw new HttpError(400, 'Invalid partNumbers'); + } + + const session = await this.stores.fsEntry.getPendingEntryBySessionId( + request.uploadId, + ); + if (!session) { + throw new HttpError(404, 'Upload session was not found'); + } + if (session.userId !== userId) { + throw new HttpError(403, 'Upload session access denied'); + } + if (session.status !== 'pending') { + throw new HttpError( + 409, + `Upload session is not pending (status=${session.status})`, + ); + } + if (session.expiresAt < Date.now()) { + await this.stores.fsEntry.markPendingEntryFailed( + session.sessionId, + 'Upload session expired', + ); + throw new HttpError(400, 'Upload session expired'); + } + if (session.uploadMode !== 'multipart') { + throw new HttpError(400, 'Upload session is not multipart'); + } + if (!session.multipartUploadId) { + throw new HttpError( + 400, + 'Multipart upload id missing from session', + ); + } + const multipartPartCount = session.multipartPartCount; + if ( + multipartPartCount !== null && + uniquePartNumbers.some( + (partNumber) => partNumber > multipartPartCount, + ) + ) { + throw new HttpError( + 400, + 'Part number exceeds multipart part count', + ); + } + if (!session.bucket || !session.bucketRegion) { + throw new HttpError( + 500, + 'Upload session storage metadata is missing', + ); + } + + const expiresInSeconds = + request.expiresInSeconds ?? DEFAULT_SIGNED_UPLOAD_EXPIRY_SECONDS; + const multipartPartUrls = + await this.stores.s3Object.createSignedMultipartPartUrls( + { + bucket: session.bucket, + objectKey: session.objectKey, + multipartUploadId: session.multipartUploadId, + partNumbers: uniquePartNumbers, + expiresInSeconds, + }, + session.bucketRegion, + ); + + const expiresAt = + Date.now() + + Math.max(60, Math.min(60 * 60, expiresInSeconds)) * 1000; + + return { + uploadId: session.sessionId, + multipartUploadId: session.multipartUploadId, + objectKey: session.objectKey, + bucket: session.bucket, + bucketRegion: session.bucketRegion, + expiresAt, + multipartPartUrls, + }; + } + + async completeUrlWrite( + userId: number, + completeWriteRequest: CompleteWriteRequest, + ): Promise { + const session = await this.stores.fsEntry.getPendingEntryBySessionId( + completeWriteRequest.uploadId, + ); + if (!session) { + throw new HttpError(404, 'Upload session was not found'); + } + if (session.userId !== userId) { + throw new HttpError(403, 'Upload session access denied'); + } + if (session.status !== 'pending') { + throw new HttpError( + 409, + `Upload session is not pending (status=${session.status})`, + ); + } + if (session.expiresAt < Date.now()) { + await this.stores.fsEntry.markPendingEntryFailed( + session.sessionId, + 'Upload session expired', + ); + throw new HttpError(400, 'Upload session expired'); + } + + const createInput = this.#parseSessionMetadata(session); + const requestedThumbnail = + completeWriteRequest.thumbnailData ?? createInput.thumbnail ?? null; + createInput.thumbnail = null; + + try { + if (session.uploadMode === 'multipart') { + if (!session.multipartUploadId) { + throw new HttpError( + 400, + 'Multipart upload id missing from session', + ); + } + + const completeParts = this.#toMultipartParts( + completeWriteRequest.parts, + ); + if (completeParts.length === 0) { + throw new HttpError( + 400, + 'Multipart upload completion requires parts', + ); + } + + await this.stores.s3Object.completeMultipartUpload( + { + bucket: + session.bucket ?? + createInput.bucket ?? + this.#resolveBucket(createInput), + objectKey: session.objectKey, + multipartUploadId: session.multipartUploadId, + parts: completeParts, + }, + session.bucketRegion ?? + createInput.bucketRegion ?? + this.#resolveBucketRegion(createInput), + ); + } + + const fsEntry = await this.stores.fsEntry.completePendingEntry( + session.sessionId, + createInput, + ); + this.#emitFsEvent( + session.overwriteTargetUid ? 'fs.write.file' : 'fs.create.file', + fsEntry, + ); + return { + sessionId: session.sessionId, + fsEntry, + wasOverwrite: Boolean(session.overwriteTargetUid), + requestedThumbnail, + }; + } catch (error) { + await this.stores.fsEntry.markPendingEntryFailed( + session.sessionId, + error instanceof Error + ? error.message + : 'Unknown error while completing upload', + ); + throw error; + } + } + + async batchCompleteUrlWrite( + userId: number, + completeWriteRequests: CompleteWriteRequest[], + ): Promise { + if (completeWriteRequests.length === 0) { + return []; + } + + const uploadIds = completeWriteRequests.map( + (request) => request.uploadId, + ); + const uniqueUploadIds = new Set(uploadIds); + if (uniqueUploadIds.size !== uploadIds.length) { + throw new HttpError( + 409, + 'Batch contains duplicate upload session ids', + ); + } + + const sessions = + await this.stores.fsEntry.getPendingEntriesBySessionIds(uploadIds); + const completionItems: Array<{ + index: number; + request: CompleteWriteRequest; + session: PendingUploadSession; + finalData: FSEntryCreateInput; + requestedThumbnail: string | null | undefined; + }> = []; + const expiredSessionIds: string[] = []; + + for (let index = 0; index < completeWriteRequests.length; index++) { + const request = completeWriteRequests[index]; + const session = sessions[index]; + if (!request || !session) { + throw new HttpError(404, 'Upload session was not found'); + } + if (session.userId !== userId) { + throw new HttpError(403, 'Upload session access denied'); + } + if (session.status !== 'pending') { + throw new HttpError( + 409, + `Upload session is not pending (status=${session.status})`, + ); + } + if (session.expiresAt < Date.now()) { + expiredSessionIds.push(session.sessionId); + continue; + } + + const finalData = this.#parseSessionMetadata(session); + const requestedThumbnail = + request.thumbnailData ?? finalData.thumbnail ?? null; + finalData.thumbnail = null; + completionItems.push({ + index, + request, + session, + finalData, + requestedThumbnail, + }); + } + + if (expiredSessionIds.length > 0) { + await this.stores.fsEntry.markPendingEntriesFailed( + expiredSessionIds, + 'Upload session expired', + ); + throw new HttpError(400, 'Upload session expired'); + } + + const multipartItems = completionItems.filter( + (item) => item.session.uploadMode === 'multipart', + ); + const multipartCompletions = await Promise.allSettled( + multipartItems.map(async (item) => { + if (!item.session.multipartUploadId) { + throw new HttpError( + 400, + 'Multipart upload id missing from session', + ); + } + + const completeParts = this.#toMultipartParts( + item.request.parts, + ); + if (completeParts.length === 0) { + throw new HttpError( + 400, + 'Multipart upload completion requires parts', + ); + } + + await this.stores.s3Object.completeMultipartUpload( + { + bucket: + item.session.bucket ?? + item.finalData.bucket ?? + this.#resolveBucket(item.finalData), + objectKey: item.session.objectKey, + multipartUploadId: item.session.multipartUploadId, + parts: completeParts, + }, + item.session.bucketRegion ?? + item.finalData.bucketRegion ?? + this.#resolveBucketRegion(item.finalData), + ); + }), + ); + + const failedMultipartItems: Array<{ + sessionId: string; + reason: unknown; + }> = []; + for (let index = 0; index < multipartCompletions.length; index++) { + const completion = multipartCompletions[index]; + const multipartItem = multipartItems[index]; + if (completion?.status === 'rejected' && multipartItem) { + failedMultipartItems.push({ + sessionId: multipartItem.session.sessionId, + reason: completion.reason, + }); + } + } + + if (failedMultipartItems.length > 0) { + await Promise.all( + failedMultipartItems.map((item) => { + return this.stores.fsEntry.markPendingEntryFailed( + item.sessionId, + this.#toErrorMessage(item.reason), + ); + }), + ); + + const firstReason = failedMultipartItems[0]?.reason; + if (firstReason instanceof HttpError) { + throw firstReason; + } + if (firstReason instanceof Error) { + throw firstReason; + } + throw new Error('Failed to complete multipart upload'); + } + + const completedEntries = + await this.stores.fsEntry.batchCompletePendingEntries( + completionItems.map((item) => ({ + sessionId: item.session.sessionId, + finalData: item.finalData, + })), + ); + + const responseByIndex = new Map(); + for (let index = 0; index < completionItems.length; index++) { + const completionItem = completionItems[index]; + const completedEntry = completedEntries[index]; + if (!completionItem || !completedEntry) { + throw new Error( + 'Failed to build completed batch write response', + ); + } + + this.#emitFsEvent( + completionItem.session.overwriteTargetUid + ? 'fs.write.file' + : 'fs.create.file', + completedEntry, + ); + responseByIndex.set(completionItem.index, { + sessionId: completionItem.session.sessionId, + fsEntry: completedEntry, + wasOverwrite: Boolean( + completionItem.session.overwriteTargetUid, + ), + requestedThumbnail: completionItem.requestedThumbnail, + }); + } + + const response: CompleteWriteResponse[] = []; + for (let index = 0; index < completeWriteRequests.length; index++) { + const result = responseByIndex.get(index); + if (!result) { + throw new Error( + `Failed to resolve completed batch response for index ${index}`, + ); + } + response.push(result); + } + return response; + } + + async abortUrlWrite(userId: number, uploadId: string): Promise { + const session = + await this.stores.fsEntry.getPendingEntryBySessionId(uploadId); + if (!session) { + return; + } + if (session.userId !== userId) { + throw new HttpError(403, 'Upload session access denied'); + } + + try { + const bucket = session.bucket; + const bucketRegion = session.bucketRegion; + if (bucket && bucketRegion) { + if ( + session.uploadMode === 'multipart' && + session.multipartUploadId + ) { + await this.stores.s3Object.abortMutipartUpload( + session.multipartUploadId, + bucketRegion, + bucket, + session.objectKey, + ); + } else { + await this.stores.s3Object.deleteObject( + bucket, + session.objectKey, + bucketRegion, + ); + } + } + } finally { + await this.stores.fsEntry.abortPendingEntry( + session.sessionId, + 'Upload aborted by caller', + ); + } + } + + async write( + userId: number, + writeRequest: WriteRequest, + uploadTracker?: UploadProgressTrackerLike, + storageAllowanceMax?: number, + ): Promise { + let normalizedInput = this.#normalizeWriteInput( + userId, + writeRequest.fileMetadata, + ); + const [resolvedTarget] = await this.#resolveWriteTargets(userId, [ + { + index: 0, + normalizedInput, + }, + ]); + if (!resolvedTarget) { + throw new Error('Failed to resolve write target'); + } + normalizedInput = resolvedTarget.normalizedInput; + const existingEntry = resolvedTarget.existingEntry; + const requestedThumbnail = + writeRequest.thumbnailData ?? normalizedInput.thumbnail ?? null; + normalizedInput.thumbnail = null; + + const existingSize = existingEntry?.size ?? 0; + await this.#assertStorageAllowance( + userId, + normalizedInput.size, + existingSize, + storageAllowanceMax, + ); + + const uploadBody = await this.#toUploadBody( + writeRequest.fileContent, + writeRequest.encoding, + uploadTracker, + ); + const objectKey = existingEntry?.uuid ?? uuidv4(); + await this.stores.s3Object.uploadFromServer( + { + bucket: normalizedInput.bucket, + objectKey, + contentType: normalizedInput.contentType, + body: uploadBody.body, + ...(uploadBody.contentLength !== undefined + ? { contentLength: uploadBody.contentLength } + : {}), + ...(Number.isFinite(normalizedInput.size) + ? { sizeHint: normalizedInput.size } + : {}), + }, + normalizedInput.bucketRegion, + ); + + const uploadedSize = uploadBody.uploadedSize(); + if (uploadTracker) { + const currentTrackedSize = Number(uploadTracker.progress ?? 0); + if (uploadedSize > currentTrackedSize) { + uploadTracker.add(uploadedSize - currentTrackedSize); + } + } + if (uploadedSize > normalizedInput.size) { + await this.#assertStorageAllowance( + userId, + uploadedSize, + existingSize, + storageAllowanceMax, + ); + } + normalizedInput.size = uploadedSize; + const contentHashSha256 = uploadBody.finalizeContentHashSha256 + ? uploadBody.finalizeContentHashSha256() + : uploadBody.contentHashSha256; + + const createInput = this.#toCreateInput(normalizedInput, objectKey); + const fsEntry = await this.stores.fsEntry.createEntry( + createInput, + normalizedInput.createMissingParents, + ); + + this.#emitFsEvent( + existingEntry ? 'fs.write.file' : 'fs.create.file', + fsEntry, + ); + + return { + fsEntry, + wasOverwrite: Boolean(existingEntry), + requestedThumbnail, + contentHashSha256, + }; + } + + async batchWrites( + userId: number, + writeRequests: WriteRequest[], + storageAllowanceMax?: number, + ): Promise { + if (writeRequests.length === 0) { + return []; + } + const preparedBatch = await this.prepareBatchWrites( + userId, + writeRequests.map((writeRequest) => ({ + fileMetadata: writeRequest.fileMetadata, + thumbnailData: writeRequest.thumbnailData, + guiMetadata: writeRequest.guiMetadata, + })), + storageAllowanceMax, + ); + await this.assertStorageAllowanceForPreparedBatch( + preparedBatch, + undefined, + storageAllowanceMax, + ); + + const uploadResults = await runWithConcurrencyLimitSettled( + writeRequests, + 8, + async (writeRequest, index) => { + return this.uploadPreparedBatchItem({ + preparedBatch, + itemIndex: index, + fileContent: writeRequest.fileContent, + encoding: writeRequest.encoding, + }); + }, + ); + const uploadedItems = uploadResults + .filter( + ( + result, + ): result is PromiseFulfilledResult => + result.status === 'fulfilled', + ) + .map((result) => result.value); + const failedUpload = uploadResults.find( + (result) => result.status === 'rejected', + ); + if (failedUpload?.status === 'rejected') { + await this.#cleanupPreparedBatchUploads( + preparedBatch, + uploadedItems, + ); + throw this.#toError( + failedUpload.reason, + 'Failed to upload batch write item', + ); + } + + return this.finalizePreparedBatchWrites(preparedBatch, uploadedItems); + } + + async cleanupPreparedBatchUploads( + preparedBatch: PreparedBatchWrite, + uploadedItems: UploadedBatchWriteItem[], + ): Promise { + await this.#cleanupPreparedBatchUploads(preparedBatch, uploadedItems); + } + + async updateEntryThumbnail( + userId: number, + entryUuid: string, + thumbnail: string | null, + ): Promise { + if (typeof entryUuid !== 'string' || entryUuid.length === 0) { + throw new HttpError( + 400, + 'Invalid file entry identifier for thumbnail update', + ); + } + + return this.stores.fsEntry.updateEntryThumbnailByUuidForUser( + userId, + entryUuid, + thumbnail, + ); + } + + async getUsersStorageAllowance( + userId: string | number, + ): Promise<{ curr: number; max: number }> { + const numericUserId = + typeof userId === 'string' ? Number(userId) : userId; + if (Number.isNaN(numericUserId)) { + throw new HttpError(400, 'Invalid user id'); + } + return this.stores.fsEntry.getUserStorageAllowance(numericUserId); + } + + // ── Reads ─────────────────────────────────────────────────────────── + + /** + * List direct children of a directory. Caller is responsible for any ACL + * check on the parent (usually 'list' mode). Returns entries in the + * requested sort order. + */ + async listDirectory( + parentUid: string, + options: { + limit?: number; + offset?: number; + sortBy?: 'name' | 'modified' | 'type' | 'size' | null; + sortOrder?: 'asc' | 'desc' | null; + } = {}, + ): Promise { + return this.stores.fsEntry.listChildren(parentUid, options); + } + + /** + * Search by file name for a user. Linear-scan with LIKE — cheap for + * typical library sizes, revisit if we need full-text. + */ + async searchByName( + userId: number, + query: string, + limit = 200, + ): Promise { + return this.stores.fsEntry.searchByNameForUser(userId, query, limit); + } + + /** + * Recursively compute total byte size under a directory. Called on demand + * from `stat` when the client asks for `size: true`. See the repository + * method for the perf caveat — this is O(descendants) and should get a + * materialized counter eventually. + */ + async getSubtreeSize(userId: number, path: string): Promise { + return this.stores.fsEntry.getSubtreeSize(userId, path); + } + + /** + * Stream bytes of a file entry from S3. The returned stream is a Node + * Readable; caller pipes it into the HTTP response and emits metering + * once the stream ends. Honours HTTP Range when provided. + * + * Throws 400 if the entry isn't a file, 500 if the entry has no backing + * bucket (should never happen for real files). + */ + async readContent( + entry: FSEntry, + options: { range?: string } = {}, + ): Promise<{ + body: Readable; + contentLength: number | null; + contentType: string | null; + contentRange: string | null; + etag: string | null; + lastModified: Date | null; + }> { + if (entry.isDir) { + throw new HttpError(400, 'Cannot read content of a directory'); + } + if (entry.isSymlink || entry.isShortcut) { + // Caller should resolve the link target before calling readContent. + throw new HttpError( + 400, + 'Cannot read content of a symlink or shortcut directly', + ); + } + // Derive the S3 object key from entry metadata if present, else fall + // back to the uuid convention used elsewhere (objectKey defaults to + // uuid during write when no metadata override is set). + const objectKey = this.#deriveObjectKeyFromEntry(entry); + try { + return await this.stores.s3Object.getObjectStream( + { + bucket: this.stores.s3Object.resolveBucket(entry.bucket), + objectKey, + range: options.range, + }, + this.stores.s3Object.resolveRegion(entry.bucketRegion), + ); + } catch (err) { + if (isNoSuchKeyError(err)) { + await this.#handleGhostFile(entry, objectKey); + throw new HttpError(404, 'File contents are missing', { + legacyCode: 'subject_does_not_exist', + cause: err, + fields: { + path: entry.path, + uid: entry.uuid, + }, + }); + } + throw err; + } + } + + // S3 returned NoSuchKey for an entry the DB still has — orphan. Delete + // the row (and emit fs.remove.node) so subsequent reads 404 cleanly via + // resolveNode instead of bubbling another S3 error. Best-effort: read + // path must not fail because cleanup failed. + async #handleGhostFile(entry: FSEntry, objectKey: string): Promise { + console.error('prodfsv2 ghost fsentry — backing S3 object missing', { + userId: entry.userId, + uuid: entry.uuid, + path: entry.path, + bucket: entry.bucket, + bucketRegion: entry.bucketRegion, + objectKey, + }); + try { + await this.remove(entry.userId, { entry }); + } catch (cleanupErr) { + console.error( + 'prodfsv2 ghost fsentry cleanup failed', + { uuid: entry.uuid }, + cleanupErr, + ); + } + } + + // Objects written by fsv2 use the pending-session's objectKey, which is + // persisted in FSEntry.metadata JSON under `objectKey`. Falls back to the + // entry uuid for entries that didn't record it (older data). + #deriveObjectKeyFromEntry(entry: FSEntry): string { + if (entry.metadata) { + try { + const parsed = JSON.parse(entry.metadata); + if ( + parsed && + typeof parsed.objectKey === 'string' && + parsed.objectKey.length > 0 + ) { + return parsed.objectKey; + } + } catch { + // Not JSON — fall through. + } + } + return entry.uuid; + } + + // ── Mutation: mkdir / touch / rename / mkshortcut ───────── + + /** + * Resolve a free child name under `parentEntry` by appending ` (N)` when + * `name` already exists. Mirrors the deduping convention used by + * `#findDedupedPath` but operates on the parent+name shape. + */ + async #findDedupedName( + parentEntry: FSEntry, + name: string, + ): Promise { + const repo = this.stores.fsEntry; + const parentPath = parentEntry.path; + const ext = pathPosix.extname(name); + const base = pathPosix.basename(name, ext); + for (let suffix = 1; suffix < 100_000; suffix++) { + const candidate = `${base} (${suffix})${ext}`; + const candidatePath = + parentPath === '/' + ? `/${candidate}` + : `${parentPath}/${candidate}`; + const existing = await repo.getEntryByPath(candidatePath); + if (!existing) return candidate; + } + throw new HttpError( + 500, + 'Could not dedupe name within 100000 attempts', + ); + } + + /** + * Resolve or create a parent directory for a given target path. Returns + * the parent entry. Throws 400 if the path has no parent (root) or 404 + * when parents are missing and create is disabled. + */ + async #resolveOrCreateParent( + userId: number, + targetPath: string, + createMissingParents: boolean, + ): Promise { + const normalized = targetPath.trim(); + if (normalized === '/') + throw new HttpError(400, 'Cannot operate on root'); + const parentPath = pathPosix.dirname(normalized); + if (parentPath === '/') + throw new HttpError(400, 'Cannot operate at root'); + return this.stores.fsEntry.resolveParentDirectory( + userId, + parentPath, + createMissingParents, + ); + } + + /** + * Create a directory at `path`. Options: + * - overwrite: if a non-directory exists, remove it and create dir + * - dedupeName: if conflict, append ` (N)` + * - createMissingParents: create intermediate dirs + * + * Returns the created (or existing-on-dedupe-false-no-conflict) entry. + */ + async mkdir( + userId: number, + input: { + path: string; + overwrite?: boolean; + dedupeName?: boolean; + createMissingParents?: boolean; + thumbnail?: string | null; + }, + ): Promise { + const targetPath = input.path.trim(); + const parent = await this.#resolveOrCreateParent( + userId, + targetPath, + !!input.createMissingParents, + ); + + let name = pathPosix.basename(targetPath); + const existing = await this.stores.fsEntry.getEntryByPath(targetPath); + if (existing) { + if (existing.isDir) { + // A directory already exists at path: idempotent success. + return existing; + } + if (input.overwrite) { + // Remove the non-directory occupant then create the dir. + await this.remove(userId, { + entry: existing, + recursive: false, + }); + } else if (input.dedupeName) { + name = await this.#findDedupedName(parent, name); + } else { + throw new HttpError( + 409, + `An entry already exists at ${targetPath}`, + ); + } + } + + const created = await this.stores.fsEntry.createNonFileEntry({ + userId, + parent, + name, + kind: 'directory', + thumbnail: input.thumbnail ?? null, + }); + this.#emitFsEvent('fs.create.directory', created); + return created; + } + + /** + * Touch: create an empty file at `path` if missing; otherwise bump + * timestamps. + */ + async touch( + userId: number, + input: { + path: string; + setAccessed?: boolean; + setModified?: boolean; + setCreated?: boolean; + createMissingParents?: boolean; + }, + ): Promise { + const targetPath = input.path.trim(); + const parent = await this.#resolveOrCreateParent( + userId, + targetPath, + !!input.createMissingParents, + ); + const name = pathPosix.basename(targetPath); + const existing = await this.stores.fsEntry.getEntryByPath(targetPath); + if (existing) { + return this.stores.fsEntry.touchEntryTimestamps(existing.uuid, { + setAccessed: input.setAccessed, + setModified: input.setModified, + setCreated: input.setCreated, + }); + } + const created = await this.stores.fsEntry.createNonFileEntry({ + userId, + parent, + name, + kind: 'empty-file', + }); + this.#emitFsEvent('fs.create.file', created); + return created; + } + + /** + * Rename an entry in place. The name changes and path rewrites; if the + * entry is a directory, descendant paths are rewritten too. + */ + async rename(entry: FSEntry, newName: string): Promise { + if (newName.includes('/')) + throw new HttpError(400, 'Name cannot contain a slash'); + if (newName.trim().length === 0) + throw new HttpError(400, 'Name cannot be empty'); + if (entry.name === newName) return entry; + + const parentPath = pathPosix.dirname(entry.path); + const newPath = + parentPath === '/' ? `/${newName}` : `${parentPath}/${newName}`; + + // Reject if another entry already owns the target path. + const collision = await this.stores.fsEntry.getEntryByPath(newPath); + if (collision && collision.uuid !== entry.uuid) { + throw new HttpError(409, `An entry already exists at ${newPath}`); + } + + const updated = await this.stores.fsEntry.updateEntry(entry.uuid, { + name: newName, + path: newPath, + }); + + if (entry.isDir) { + await this.stores.fsEntry.updatePathPrefixForUser( + entry.userId, + entry.path, + newPath, + ); + } + this.#emitFsEvent('fs.rename', updated, { + old_name: entry.name, + new_name: newName, + old_path: entry.path, + new_path: newPath, + }); + return updated; + } + + /** + * Create a shortcut pointing at `target`. Shortcuts are FS entries with + * `is_shortcut = 1` and `shortcut_to = target.id`. + */ + async mkshortcut( + userId: number, + input: { + parent: FSEntry; + name: string; + target: FSEntry; + dedupeName?: boolean; + }, + ): Promise { + let name = input.name; + const childPath = + input.parent.path === '/' + ? `/${name}` + : `${input.parent.path}/${name}`; + const collision = await this.stores.fsEntry.getEntryByPath(childPath); + if (collision) { + if (input.dedupeName) { + name = await this.#findDedupedName(input.parent, name); + } else { + throw new HttpError( + 409, + `An entry already exists at ${childPath}`, + ); + } + } + const created = await this.stores.fsEntry.createNonFileEntry({ + userId, + parent: input.parent, + name, + kind: 'shortcut', + shortcutTo: input.target.id, + }); + this.#emitFsEvent('fs.create.shortcut', created); + return created; + } + + // ── Mutation: remove / move / copy ───────────────────────────────── + + /** + * Remove an entry. For directories, descendants are walked and removed + * (both DB rows and S3 objects). Emits `fs.remove.node` per file so the + * thumbnail extension (and any other listener) can clean up side state. + * + * Does NOT enforce ACL — caller (controller) performs the `write` check. + */ + async remove( + userId: number, + input: { + entry: FSEntry; + recursive?: boolean; + descendantsOnly?: boolean; + }, + ): Promise { + const { entry } = input; + if (entry.userId !== userId) { + // Defensive — only the owner should be hitting this path; higher + // layers grant access via ACL, not raw ownership, but we still + // want to avoid a misrouted call taking out someone else's tree. + throw new HttpError( + 403, + 'Cannot remove an entry owned by another user', + ); + } + + if (entry.isDir) { + const descendants = await this.stores.fsEntry.listDescendantsByPath( + userId, + entry.path, + ); + if (descendants.length > 0 && !input.recursive) { + throw new HttpError(409, 'Directory is not empty'); + } + + // Delete descendants first (depth-descending). S3 objects are + // batched per bucket+region for efficiency. + await this.#removeDescendantsStorage(descendants); + if (descendants.length > 0) { + await this.stores.fsEntry.deleteEntries(descendants); + } + + if (!input.descendantsOnly) { + await this.stores.fsEntry.deleteEntry(entry); + this.#emitRemoveEvent(entry); + } + return; + } + + // File / shortcut / symlink: delete backing S3 object (if any) then the row. + if ( + entry.bucket && + entry.bucketRegion && + !entry.isShortcut && + !entry.isSymlink + ) { + try { + await this.stores.s3Object.deleteObject( + entry.bucket, + this.#deriveObjectKeyFromEntry(entry), + entry.bucketRegion, + ); + } catch { + // Best effort — DB row is the source of truth. Extensions + // will get the `fs.remove.node` event regardless. + } + } + await this.stores.fsEntry.deleteEntry(entry); + this.#emitRemoveEvent(entry); + } + + /** + * Hard-delete every FS entry owned by `userId`: S3 objects first, then + * every `fsentries` row. Used by account deletion. Paginates through + * files (5k at a time) so large users don't blow the heap, batches S3 + * deletes per bucket+region, and finishes with one bulk DELETE to + * sweep dirs/shortcuts/symlinks that don't have backing objects. + * + * Safe to call concurrently with other ops on the same user only in the + * sense that orphaned S3 objects may linger if a write races us; the DB + * state always converges to "user has no entries". + */ + async removeAllForUser(userId: number): Promise { + const pageSize = 5000; + // Files-first loop: delete backing S3 objects in batches, then DB rows. + for (;;) { + const files = (await this.clients.db.read( + `SELECT uuid, bucket, bucket_region FROM fsentries + WHERE user_id = ? AND is_dir = 0 AND (is_shortcut = 0 OR is_shortcut IS NULL) AND (is_symlink = 0 OR is_symlink IS NULL) + LIMIT ${pageSize}`, + [userId], + )) as Array<{ + uuid: string; + bucket: string | null; + bucket_region: string | null; + }>; + + if (files.length === 0) break; + + // Group by bucket+region so one S3 DeleteObjects call covers each. + const grouped = new Map< + string, + { bucket: string; region: string; keys: string[] } + >(); + for (const f of files) { + if (!f.bucket || !f.bucket_region) continue; + const groupKey = `${f.bucket_region}::${f.bucket}`; + const group = grouped.get(groupKey) ?? { + bucket: f.bucket, + region: f.bucket_region, + keys: [], + }; + group.keys.push(f.uuid); + grouped.set(groupKey, group); + } + await Promise.allSettled( + Array.from(grouped.values()).map((g) => + this.stores.s3Object.deleteObjects( + { bucket: g.bucket, objectKeys: g.keys }, + g.region, + ), + ), + ); + + const uuidPlaceholders = files.map(() => '?').join(', '); + await this.clients.db.write( + `DELETE FROM fsentries WHERE user_id = ? AND uuid IN (${uuidPlaceholders})`, + [userId, ...files.map((f) => f.uuid)], + ); + } + + // Sweep remaining non-file rows (dirs, shortcuts, symlinks). + await this.clients.db.write('DELETE FROM fsentries WHERE user_id = ?', [ + userId, + ]); + } + + async #removeDescendantsStorage(descendants: FSEntry[]): Promise { + // Group file descendants by bucket+region for batch delete. + const grouped = new Map< + string, + { bucket: string; region: string; keys: string[] } + >(); + for (const child of descendants) { + if (child.isDir || child.isShortcut || child.isSymlink) continue; + if (!child.bucket || !child.bucketRegion) continue; + const groupKey = `${child.bucketRegion}::${child.bucket}`; + const group = grouped.get(groupKey) ?? { + bucket: child.bucket, + region: child.bucketRegion, + keys: [], + }; + group.keys.push(this.#deriveObjectKeyFromEntry(child)); + grouped.set(groupKey, group); + // Fire individual removal events so thumbnail extension can clean up. + this.#emitRemoveEvent(child); + } + await Promise.allSettled( + Array.from(grouped.values()).map((group) => + this.stores.s3Object.deleteObjects( + { bucket: group.bucket, objectKeys: group.keys }, + group.region, + ), + ), + ); + } + + #emitRemoveEvent(entry: FSEntry): void { + // Ship the entry under every alias existing handlers use — `node`, + // `entry`, `target`. The thumbnails extension destructures + // `{ target }`, and the bare `{ node, entry }` shape landed + // `target: undefined` → crash on `target.thumbnail`. + try { + this.clients.event.emit( + 'fs.remove.node', + { node: entry, entry, target: entry }, + {}, + ); + } catch { + // Non-critical. + } + } + + /** + * Emit one of the lifecycle events that `extension.on('fs.…')` consumers + * expect (cf-file-cache, future thumbnails-style extensions). Payload + * carries multiple aliases (`node` / `entry` / `uid`) so handlers using + * any existing calling convention just work. + * + * Currently emitted: + * fs.create.{file,directory,shortcut,symlink} + * fs.write.file — overwrite of an existing file + * fs.rename — in-place name change (move emits fs.move.node separately) + * + * Skipped intentionally: `fs.pending.*` (no real entry yet at signed-URL + * issue time) and per-flavor `fs.move.file` (move already emits + * `fs.move.node`). + */ + #emitFsEvent( + name: string, + entry: FSEntry, + extras: Record = {}, + ): void { + try { + this.clients.event.emit( + name, + { + node: entry, + entry, + uid: entry.uuid, + ...extras, + }, + {}, + ); + } catch { + // Non-critical — the response is the source of truth. + } + } + + /** + * Move an entry to a new parent (and optionally rename in the same op). + * Works for files and directories. Updates descendant paths when moving + * a directory. + */ + async move( + userId: number, + input: { + source: FSEntry; + destinationParent: FSEntry; + newName?: string; + overwrite?: boolean; + dedupeName?: boolean; + /** + * Optional metadata to overwrite on the moved entry. Callers use this + * for trash/restore: when moving into Trash the GUI stores + * `{ original_name, original_path, trashed_ts }` here so the restore + * path and trash listing can recover the pre-trash name. + */ + newMetadata?: Record | null; + }, + ): Promise { + const { source, destinationParent } = input; + if (source.userId !== userId) { + throw new HttpError( + 403, + 'Cannot move an entry owned by another user', + ); + } + if (!destinationParent.isDir) { + throw new HttpError(400, 'Destination parent is not a directory'); + } + if ( + source.isDir && + destinationParent.path.startsWith(`${source.path}/`) + ) { + throw new HttpError(400, 'Cannot move a directory into itself'); + } + + let name = input.newName ?? source.name; + const targetPath = + destinationParent.path === '/' + ? `/${name}` + : `${destinationParent.path}/${name}`; + + const collision = await this.stores.fsEntry.getEntryByPath(targetPath); + if (collision && collision.uuid !== source.uuid) { + if (input.overwrite) { + await this.remove(userId, { + entry: collision, + recursive: true, + }); + } else if (input.dedupeName) { + name = await this.#findDedupedName(destinationParent, name); + } else { + throw new HttpError( + 409, + `An entry already exists at ${targetPath}`, + ); + } + } + + const finalPath = + destinationParent.path === '/' + ? `/${name}` + : `${destinationParent.path}/${name}`; + + // `metadata` column is a TEXT field; serialize when the caller sends + // an object, pass-through a bare string, and `null` clears it. + // `undefined` leaves the column untouched. + let metadataPatch: string | null | undefined; + if (input.newMetadata === null) metadataPatch = null; + else if (typeof input.newMetadata === 'object') + metadataPatch = JSON.stringify(input.newMetadata); + + const updated = await this.stores.fsEntry.updateEntry(source.uuid, { + name, + path: finalPath, + parentId: destinationParent.id, + parentUid: destinationParent.uuid, + ...(metadataPatch !== undefined ? { metadata: metadataPatch } : {}), + }); + + if (source.isDir && source.path !== finalPath) { + await this.stores.fsEntry.updatePathPrefixForUser( + userId, + source.path, + finalPath, + ); + } + + try { + this.clients.event.emit( + 'fs.move.node', + { + node: updated, + fromPath: source.path, + toPath: finalPath, + }, + {}, + ); + } catch { + // ignore — non-critical. + } + return updated; + } + + /** + * Copy an entry to a new parent. For directories, walks descendants and + * issues S3 CopyObject + DB inserts. Thumbnail URLs on entries ride + * along in the DB column — the thumbnail extension is notified via + * `fs.copy.node` so it can duplicate the backing S3 object (otherwise + * deleting one copy would nuke the other's thumbnail). + */ + async copy( + userId: number, + input: { + source: FSEntry; + destinationParent: FSEntry; + newName?: string; + overwrite?: boolean; + dedupeName?: boolean; + }, + ): Promise { + const { source, destinationParent } = input; + if (!destinationParent.isDir) { + throw new HttpError(400, 'Destination parent is not a directory'); + } + if ( + source.isDir && + (destinationParent.path === source.path || + destinationParent.path.startsWith(`${source.path}/`)) + ) { + throw new HttpError( + 400, + 'Cannot copy a directory into itself or a descendant', + ); + } + + let name = input.newName ?? source.name; + const targetPath = + destinationParent.path === '/' + ? `/${name}` + : `${destinationParent.path}/${name}`; + + const collision = await this.stores.fsEntry.getEntryByPath(targetPath); + if (collision) { + if (input.overwrite) { + await this.remove(userId, { + entry: collision, + recursive: true, + }); + } else if (input.dedupeName) { + name = await this.#findDedupedName(destinationParent, name); + } else { + throw new HttpError( + 409, + `An entry already exists at ${targetPath}`, + ); + } + } + + const finalPath = + destinationParent.path === '/' + ? `/${name}` + : `${destinationParent.path}/${name}`; + + if (!source.isDir) { + return this.#copyLeafEntry( + userId, + source, + destinationParent, + name, + finalPath, + ); + } + + // Recursive directory copy: + // 1) Create the new root directory at destination + // 2) Walk descendants; for each, compute new path by swapping prefix + // 3) Create a new row (files copy S3 object; dirs just insert) + const newRoot = await this.stores.fsEntry.createNonFileEntry({ + userId, + parent: destinationParent, + name, + kind: 'directory', + metadata: source.metadata, + thumbnail: source.thumbnail, + associatedAppId: source.associatedAppId, + isPublic: source.isPublic, + }); + + const descendants = await this.stores.fsEntry.listDescendantsByPath( + source.userId, + source.path, + ); + // Sort shallow-first so parents exist before children. + descendants.sort((a, b) => a.path.length - b.path.length); + + // Maintain a map from old-path → new parent entry so child inserts + // can reference the correct parent uuid/id. + const newByOldPath = new Map(); + newByOldPath.set(source.path, newRoot); + + for (const descendant of descendants) { + const oldParentPath = pathPosix.dirname(descendant.path); + const newParent = newByOldPath.get(oldParentPath); + if (!newParent) { + // Parent wasn't copied — skip (shouldn't happen with sort). + continue; + } + const copied = descendant.isDir + ? await this.stores.fsEntry.createNonFileEntry({ + userId, + parent: newParent, + name: descendant.name, + kind: 'directory', + metadata: descendant.metadata, + thumbnail: descendant.thumbnail, + associatedAppId: descendant.associatedAppId, + isPublic: descendant.isPublic, + }) + : await this.#copyLeafEntry( + userId, + descendant, + newParent, + descendant.name, + newParent.path === '/' + ? `/${descendant.name}` + : `${newParent.path}/${descendant.name}`, + ); + newByOldPath.set(descendant.path, copied); + } + + return newRoot; + } + + // Internal helper: copies a single non-directory entry. Handles files, + // shortcuts, and symlinks. Files trigger S3 CopyObject; shortcuts/symlinks + // are pure metadata clones. + async #copyLeafEntry( + userId: number, + source: FSEntry, + destinationParent: FSEntry, + newName: string, + _newPath: string, + ): Promise { + if (source.isSymlink) { + return this.stores.fsEntry.createNonFileEntry({ + userId, + parent: destinationParent, + name: newName, + kind: 'symlink', + symlinkPath: source.symlinkPath, + metadata: source.metadata, + associatedAppId: source.associatedAppId, + }); + } + if (source.isShortcut) { + return this.stores.fsEntry.createNonFileEntry({ + userId, + parent: destinationParent, + name: newName, + kind: 'shortcut', + shortcutTo: source.shortcutTo, + metadata: source.metadata, + associatedAppId: source.associatedAppId, + }); + } + + // Regular file: duplicate the S3 object under a new key (the new + // entry's uuid), then insert the DB row pointing at it. + const newUuid = uuidv4(); + const sourceObjectKey = this.#deriveObjectKeyFromEntry(source); + const resolvedBucket = this.stores.s3Object.resolveBucket( + source.bucket, + ); + await this.stores.s3Object.copyObject( + { + sourceBucket: resolvedBucket, + sourceKey: sourceObjectKey, + destinationBucket: resolvedBucket, + destinationKey: newUuid, + }, + this.stores.s3Object.resolveRegion(source.bucketRegion), + ); + + // Re-serialize metadata, swapping in the new objectKey. + const nextMetadata = this.#metadataWithObjectKey( + source.metadata, + newUuid, + ); + + // Insert as a file row. We reuse the files INSERT path (batchCreateEntries) + // since it handles bucket/metadata correctly. A single-row call is fine. + const [created] = await this.stores.fsEntry.batchCreateEntries( + [ + { + userId, + uuid: newUuid, + path: + destinationParent.path === '/' + ? `/${newName}` + : `${destinationParent.path}/${newName}`, + size: source.size ?? 0, + contentType: undefined, + metadata: nextMetadata, + thumbnail: source.thumbnail, + associatedAppId: source.associatedAppId, + immutable: source.immutable, + isPublic: source.isPublic, + bucket: source.bucket, + bucketRegion: source.bucketRegion, + } as FSEntryCreateInput, + ], + false, + ); + if (!created) { + throw new HttpError(500, 'Failed to copy file entry'); + } + + try { + this.clients.event.emit( + 'fs.copy.node', + { + source, + copy: created, + sourceObjectKey, + copyObjectKey: newUuid, + }, + {}, + ); + } catch { + // ignore — non-critical. + } + return created; + } + + // Preserves existing metadata JSON fields, overriding only objectKey. + #metadataWithObjectKey(metadata: string | null, objectKey: string): string { + let parsed: Record = {}; + if (metadata) { + try { + const tentative = JSON.parse(metadata); + if ( + tentative && + typeof tentative === 'object' && + !Array.isArray(tentative) + ) { + parsed = tentative as Record; + } + } catch { + // Non-JSON legacy metadata — drop and replace. + } + } + parsed.objectKey = objectKey; + return JSON.stringify(parsed); + } + + /** + * This method checks if the specified actor has permission to access the entry provided. It will throw an error if the actor is not permitted + */ + async checkFSAccess( + entry: FSEntry, + actor: Actor, + mode: AclMode = 'write', + ): Promise { + if (!entry) { + throw new HttpError(400, 'Invalid FS Entry provided'); + } + + let ancestorsCache: Promise< + Array<{ uid: string; path: string }> + > | null = null; + const descriptor = { + path: entry.path, + resolveAncestors: () => { + if (!ancestorsCache) { + ancestorsCache = this.getAncestorChain(entry.path); + } + return ancestorsCache; + }, + }; + const allowed = await this.services.acl.check(actor, descriptor, mode); + if (allowed) return; + + const safe = (await this.services.acl.getSafeAclError( + actor, + descriptor, + mode, + )) as { + status?: unknown; + message?: unknown; + fields?: { code?: unknown }; + }; + const status = Number(safe?.status); + const message = + typeof safe?.message === 'string' && safe.message.length > 0 + ? safe.message + : 'Access denied'; + const code = + typeof safe?.fields?.code === 'string' + ? safe.fields.code + : undefined; + const legacyCode = code === 'forbidden' ? 'access_denied' : code; + if (status === 404) { + throw new HttpError(404, message, { + ...(legacyCode ? { legacyCode } : {}), + }); + } + throw new HttpError(403, message, { + legacyCode: legacyCode ?? 'access_denied', + }); + } +} diff --git a/src/backend/services/fs/cacheInvalidation.ts b/src/backend/services/fs/cacheInvalidation.ts new file mode 100644 index 000000000..8109fb679 --- /dev/null +++ b/src/backend/services/fs/cacheInvalidation.ts @@ -0,0 +1,213 @@ +import type { EventClient } from '../../clients/EventClient.js'; +import type { FSEntryStore } from '../../stores/fs/FSEntryStore.js'; +import type { + FsRemoveNodeEventPayload, + FsRemoveNodeTarget, + OuterGuiItemEventPayload, +} from './eventTypes.js'; + +export class FSEntryCacheInvalidationEventHandler { + #fsEntryStore: FSEntryStore; + #eventClient: EventClient; + + constructor(fsEntryStore: FSEntryStore, eventClient: EventClient) { + this.#fsEntryStore = fsEntryStore; + this.#eventClient = eventClient; + this.#registerHandlers(); + } + + #registerHandlers(): void { + this.#eventClient.on( + 'outer.gui.item.added', + async (event: OuterGuiItemEventPayload) => { + await this.#runSafely( + () => this.#handleOuterGuiItemEvent(event), + 'outer.gui.item.added', + ); + }, + ); + this.#eventClient.on( + 'outer.gui.item.updated', + async (event: OuterGuiItemEventPayload) => { + await this.#runSafely( + () => this.#handleOuterGuiItemEvent(event), + 'outer.gui.item.updated', + ); + }, + ); + this.#eventClient.on( + 'outer.gui.item.moved', + async (event: OuterGuiItemEventPayload) => { + await this.#runSafely( + () => this.#handleOuterGuiItemEvent(event), + 'outer.gui.item.moved', + ); + }, + ); + this.#eventClient.on( + 'fs.remove.node', + async (event: FsRemoveNodeEventPayload) => { + await this.#runSafely( + () => this.#handleRemoveNodeEvent(event), + 'fs.remove.node', + ); + }, + ); + } + + async #runSafely( + handler: () => Promise, + eventName: string, + ): Promise { + try { + await handler(); + } catch (error) { + console.error( + `prodfsv2 cache invalidation failed for ${eventName}`, + error, + ); + } + } + + #toUserIds(value: unknown): number[] { + if (!Array.isArray(value)) { + return []; + } + + const userIds: number[] = []; + for (const item of value) { + const numeric = Number(item); + if (Number.isInteger(numeric) && numeric > 0) { + userIds.push(numeric); + } + } + return userIds; + } + + #toNonEmptyString(value: unknown): string | null { + if (typeof value !== 'string') { + return null; + } + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : null; + } + + #isUnrecognizedTargetKeyError(error: unknown): boolean { + if (!(error instanceof Error)) { + return false; + } + return error.message.includes('unrecognize key for FSNodeContext.get:'); + } + + async #readTargetValue( + target: FsRemoveNodeTarget, + keys: string[], + ): Promise { + if (typeof target.get !== 'function') { + return undefined; + } + + for (const key of keys) { + try { + return await target.get(key); + } catch (error) { + if (this.#isUnrecognizedTargetKeyError(error)) { + continue; + } + throw error; + } + } + + return undefined; + } + + #extractUidFromEntry(value: unknown): string | null { + if (!value || typeof value !== 'object') { + return null; + } + const entry = value as { uid?: unknown; uuid?: unknown }; + return ( + this.#toNonEmptyString(entry.uid) ?? + this.#toNonEmptyString(entry.uuid) + ); + } + + async #handleOuterGuiItemEvent( + event: OuterGuiItemEventPayload, + ): Promise { + const userIds = this.#toUserIds(event?.user_id_list); + const response = event?.response ?? {}; + + const path = this.#toNonEmptyString(response.path); + const oldPath = this.#toNonEmptyString(response.old_path); + const uid = + this.#toNonEmptyString(response.uid) ?? + this.#toNonEmptyString(response.uuid) ?? + this.#toNonEmptyString(response.id); + + const tasks: Promise[] = []; + for (const userId of userIds) { + if (path) { + tasks.push( + this.#fsEntryStore.invalidateEntryCacheByPathForUser( + userId, + path, + ), + ); + } + if (oldPath && oldPath !== path) { + tasks.push( + this.#fsEntryStore.invalidateEntryCacheByPathForUser( + userId, + oldPath, + ), + ); + } + } + if (uid) { + tasks.push(this.#fsEntryStore.invalidateEntryCacheByUuid(uid)); + } + + if (tasks.length > 0) { + await Promise.all(tasks); + } + } + + async #handleRemoveNodeEvent( + event: FsRemoveNodeEventPayload, + ): Promise { + const target = event?.target; + if (!target || typeof target.get !== 'function') { + return; + } + + const userIdValue = await this.#readTargetValue(target, ['user_id']); + const pathValue = await this.#readTargetValue(target, ['path']); + const uidValue = + (await this.#readTargetValue(target, ['uid', 'uuid'])) ?? + this.#extractUidFromEntry( + await this.#readTargetValue(target, ['entry']), + ); + + const userId = Number(userIdValue); + const path = this.#toNonEmptyString(pathValue); + const uuid = this.#toNonEmptyString(uidValue); + + const tasks: Promise[] = []; + if (Number.isInteger(userId) && userId > 0 && path) { + tasks.push( + this.#fsEntryStore.invalidateEntryCacheByPathForUser( + userId, + path, + ), + ); + } + if (uuid) { + tasks.push(this.#fsEntryStore.invalidateEntryCacheByUuid(uuid)); + } + + if (tasks.length > 0) { + await Promise.all(tasks); + } + } +} diff --git a/extensions/fsv2/src/eventHandlers/types.ts b/src/backend/services/fs/eventTypes.ts similarity index 100% rename from extensions/fsv2/src/eventHandlers/types.ts rename to src/backend/services/fs/eventTypes.ts diff --git a/src/backend/services/fs/resolveNode.ts b/src/backend/services/fs/resolveNode.ts new file mode 100644 index 000000000..043225792 --- /dev/null +++ b/src/backend/services/fs/resolveNode.ts @@ -0,0 +1,153 @@ +import { posix as pathPosix } from 'node:path'; +import { HttpError } from '../../core/http/HttpError.js'; +import type { FSEntry } from '../../stores/fs/FSEntry.js'; +import type { FSEntryStore } from '../../stores/fs/FSEntryStore.js'; + +/** + * Resolve an entry by one of several reference shapes (path, uid, id) to + * a plain FSEntry row. Everything else (size, descendants, subdomains, + * shares) is fetched by explicit service methods as needed. + * + * If a caller wants a batch resolve, do N individual calls — the repository + * caches each result in Redis on first read. + */ + +export interface NodeRef { + /** Absolute path, e.g. '/danielsalazar/Documents/foo.txt'. */ + path?: string; + /** UUID of the entry. Aliased as `uid` in request shapes. */ + uid?: string; + uuid?: string; + /** Numeric MySQL id. */ + id?: number | string; + /** Pre-fetched entry (no-op resolution — pass-through). */ + entry?: FSEntry; +} + +export interface ResolveNodeOptions { + /** Throw a 404 HttpError when nothing resolves; default `false` returns null. */ + required?: boolean; +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.trim().length > 0; +} + +export async function resolveNode( + fsEntryStore: FSEntryStore, + ref: NodeRef, + options: ResolveNodeOptions = {}, +): Promise { + if (ref.entry) return ref.entry; + + const uuid = ref.uid ?? ref.uuid; + if (isNonEmptyString(uuid)) { + const entry = await fsEntryStore.getEntryByUuid(uuid); + if (entry) return entry; + return notFoundOrNull( + options.required, + `Entry not found: uuid=${uuid}`, + ); + } + + if (ref.id !== undefined && ref.id !== null && String(ref.id).length > 0) { + const numericId = Number(ref.id); + if (!Number.isFinite(numericId)) { + throw new HttpError(400, 'Invalid id'); + } + const entry = await fsEntryStore.getEntryById(numericId); + if (entry) return entry; + return notFoundOrNull( + options.required, + `Entry not found: id=${numericId}`, + ); + } + + if (isNonEmptyString(ref.path)) { + const entry = await fsEntryStore.getEntryByPath(ref.path); + if (entry) return entry; + return notFoundOrNull( + options.required, + `Entry not found: path=${ref.path}`, + ); + } + + throw new HttpError( + 400, + 'Missing entry reference (expected one of: path, uid, id)', + ); +} + +function notFoundOrNull(required: boolean | undefined, message: string): null { + if (required) { + throw new HttpError(404, message, { + legacyCode: 'subject_does_not_exist', + }); + } + return null; +} + +/** + * Split an absolute path into `{ parentPath, name }`. Used for operations that + * accept "create child X of parent Y" shape (touch/mkdir/write), plus the + * `{ parent, name }` selector style (parent resolves first, then we append + * name to parent.path). + */ +export function splitParentAndName(absolutePath: string): { + parentPath: string; + name: string; +} { + const normalized = normalizeAbsolutePath(absolutePath); + if (normalized === '/') { + throw new HttpError(400, 'Cannot derive parent of root'); + } + const parentPath = pathPosix.dirname(normalized); + const name = pathPosix.basename(normalized); + return { parentPath: parentPath === '.' ? '/' : parentPath, name }; +} + +export function normalizeAbsolutePath(path: string): string { + const trimmed = typeof path === 'string' ? path.trim() : ''; + if (trimmed.length === 0) { + throw new HttpError(400, 'Path cannot be empty'); + } + let normalized = pathPosix.normalize(trimmed); + if (!normalized.startsWith('/')) { + normalized = `/${normalized}`; + } + if (normalized.length > 1 && normalized.endsWith('/')) { + normalized = normalized.slice(0, -1); + } + return normalized; +} + +/** + * Expand a leading `~` (home-dir shorthand) to `/`. Preserves + * non-tilde paths as-is. Throws 400 when the path needs expansion but no + * username was supplied. Used by legacy FS endpoints (stat/readdir/etc.) + * that accept user-authored paths verbatim. + */ +export function expandTildePath(path: string, username?: string): string { + if (typeof path !== 'string') return path; + const trimmed = path.trim(); + if (trimmed !== '~' && !trimmed.startsWith('~/')) return path; + if (!username) { + throw new HttpError(400, 'Unable to resolve home path'); + } + return `/${username}${trimmed.slice(1)}`; +} + +/** + * Build an absolute child path from a parent path + child name. Rejects names + * containing `/`. + */ +export function joinChildPath(parentPath: string, name: string): string { + if (typeof name !== 'string' || name.length === 0) { + throw new HttpError(400, 'Name cannot be empty'); + } + if (name.includes('/')) { + throw new HttpError(400, 'Name cannot contain a slash'); + } + const parent = normalizeAbsolutePath(parentPath); + return parent === '/' ? `/${name}` : `${parent}/${name}`; +} diff --git a/src/backend/services/fs/rootListing.ts b/src/backend/services/fs/rootListing.ts new file mode 100644 index 000000000..dd1f77870 --- /dev/null +++ b/src/backend/services/fs/rootListing.ts @@ -0,0 +1,64 @@ +import type { Actor } from '../../core/actor.js'; +import type { FSEntry } from '../../stores/fs/FSEntry.js'; +import type { FSEntryStore } from '../../stores/fs/FSEntryStore.js'; +import type { PermissionService } from '../permission/PermissionService.js'; + +/** + * Synthesize the listing for the virtual root `/`. There is no fsentry row + * at `/` — instead root is a virtual aggregate of user-directory entries + * the actor can see: the actor's own home plus any other users' homes + * granted via permission issuers (i.e. users that have shared something + * with this actor). Mirrors v1's `LLListUsers`. + */ +export async function listRootEntries( + actor: Actor, + fsEntryStore: FSEntryStore, + permissionService: PermissionService, +): Promise { + const entries: FSEntry[] = []; + const seenPaths = new Set(); + + const pushByUsername = async (username: string | undefined) => { + if (!username) return; + const path = `/${username}`; + if (seenPaths.has(path)) return; + seenPaths.add(path); + const entry = await fsEntryStore.getEntryByPath(path); + if (entry) entries.push(entry); + }; + + // For the actor's own home, heal first: a user whose home drifted + // (stale path after a rename that never cascaded, or legacy rows + // that were never path-populated) would otherwise be invisible to a + // `getEntryByPath('/{username}')` lookup. `renameUserHome` is a + // cheap no-op when the root already matches. + const userId = actor.user.id; + if (typeof userId === 'number' && actor.user.username) { + try { + const healed = await fsEntryStore.renameUserHome( + userId, + actor.user.username, + ); + if (healed) { + seenPaths.add(healed.path); + entries.push(healed); + } + } catch { + // Fall through to the path-based lookup below. + } + } + + await pushByUsername(actor.user.username); + + if (typeof userId === 'number') { + const issuers = await permissionService.listUserPermissionIssuers({ + id: userId, + }); + for (const issuer of issuers) { + if (!issuer) continue; + await pushByUsername(issuer.username); + } + } + + return entries; +} diff --git a/extensions/fsv2/src/services/types.ts b/src/backend/services/fs/types.ts similarity index 92% rename from extensions/fsv2/src/services/types.ts rename to src/backend/services/fs/types.ts index e3651959e..80006e156 100644 --- a/extensions/fsv2/src/services/types.ts +++ b/src/backend/services/fs/types.ts @@ -1,6 +1,9 @@ import type { Readable } from 'node:stream'; -import type { FSEntry, FSEntryWriteInput } from '../types/FSEntry.js'; -import type { WriteGuiMetadata, WriteRequest } from '../types/requests.js'; +import type { FSEntry, FSEntryWriteInput } from '../../stores/fs/FSEntry.js'; +import type { + WriteGuiMetadata, + WriteRequest, +} from '../../controllers/fs/requestTypes.js'; export interface NormalizedWriteInput { userId: number; diff --git a/src/backend/services/health/ServerHealthService.ts b/src/backend/services/health/ServerHealthService.ts new file mode 100644 index 000000000..a3a709608 --- /dev/null +++ b/src/backend/services/health/ServerHealthService.ts @@ -0,0 +1,300 @@ +import { PuterService } from '../types'; +import type { SocketService } from '../socket/SocketService'; + +/** + * Periodic liveness monitor for the backend. Other services register + * checks via `addCheck`; the internal loop runs them every + * `CHECK_INTERVAL_MS`, raises an alarm on first failure, fires `onFail` + * handlers (for self-heal hooks), and exposes `getStatus()` for the + * `/healthcheck` route. + * + * Default checks registered on server start: + * - `database-liveness` — `SELECT 1 AS ok` latency-gated against + * `config.server_health.db_liveness_latency_fail_ms` (default 1500ms). + * - `socket-initialized` — socket.io must be attached. Only registered + * when SocketService is present (skipped for API-only deployments). + * + * Draining mode: `onServerPrepareShutdown` flips the service into drain + * and clears failure state. `/healthcheck` returns 503 so load balancers + * route traffic away before the process exits. + */ + +const SECOND = 1000; +const CHECK_INTERVAL_MS = 5 * SECOND; +const CHECK_TIMEOUT_MS = 4 * SECOND; +const HEALTH_LOOP_STALE_MULTIPLIER = 3; +const DEFAULT_DB_LIVENESS_LATENCY_FAIL_MS = 1500; +const STATUS_CACHE_TTL_SECONDS = 5; +const STATUS_CACHE_KEY = 'server-health:status'; + +type CheckFn = () => Promise | unknown; +type FailHandler = (err: unknown) => Promise | void; + +interface Chainable { + onFail(handler: FailHandler): Chainable; +} + +interface RegisteredCheck { + name: string; + fn: CheckFn; + onFailHandlers: FailHandler[]; +} + +interface HealthStats { + last_check_cycle_completed_at: number; + check_durations_ms: Record; + failed_checks: string[]; + database_liveness_latency_ms?: number; +} + +export interface HealthStatus { + ok: boolean; + failed?: string[]; +} + +export class ServerHealthService extends PuterService { + #checks: RegisteredCheck[] = []; + #failures: { name: string }[] = []; + #healthStartedAt = Date.now(); + #lastCycleCompletedAt = 0; + #stats: HealthStats = { + last_check_cycle_completed_at: 0, + check_durations_ms: {}, + failed_checks: [], + }; + #loopRunning = false; + #intervalHandle: NodeJS.Timeout | null = null; + #draining = false; + + override onServerStart(): void { + this.#registerDefaultChecks(); + this.#startLoop(); + } + + override onServerPrepareShutdown(): void { + if (this.#draining) return; + this.#draining = true; + this.#failures = []; + this.#lastCycleCompletedAt = Date.now(); + this.#stats = { + last_check_cycle_completed_at: this.#lastCycleCompletedAt, + check_durations_ms: {}, + failed_checks: [], + }; + console.log('[server-health] entering drain mode'); + } + + override onServerShutdown(): void { + if (this.#intervalHandle) { + clearInterval(this.#intervalHandle); + this.#intervalHandle = null; + } + } + + /** + * Register a named health check. The returned chainable exposes + * `onFail(fn)` so callers can hook self-heal logic (e.g., recreating + * a pooled DB client after a liveness drop). + */ + addCheck(name: string, fn: CheckFn): Chainable { + const registered: RegisteredCheck = { name, fn, onFailHandlers: [] }; + this.#checks.push(registered); + const chainable: Chainable = { + onFail: (handler) => { + registered.onFailHandlers.push(handler); + return chainable; + }, + }; + return chainable; + } + + /** + * Current health status. Results are cached in Redis for 5 seconds + * so a busy /healthcheck endpoint doesn't hammer the DB on every hit. + */ + async getStatus(): Promise { + if (this.#draining) return { ok: false, failed: ['draining'] }; + + try { + const cached = await this.clients.redis.get(STATUS_CACHE_KEY); + if (cached) { + try { + return JSON.parse(cached) as HealthStatus; + } catch { + // Cache in invalid state — fall through and overwrite. + } + } + } catch (e) { + console.warn( + '[server-health] status cache read failed:', + (e as Error).message, + ); + } + + const failures = this.#collectFailures(); + const status: HealthStatus = + failures.length === 0 + ? { ok: true } + : { ok: false, failed: failures }; + + try { + await this.clients.redis.set( + STATUS_CACHE_KEY, + JSON.stringify(status), + 'EX', + STATUS_CACHE_TTL_SECONDS, + ); + } catch (e) { + console.warn( + '[server-health] status cache write failed:', + (e as Error).message, + ); + } + + return status; + } + + #registerDefaultChecks(): void { + const latencyFailMs = + Number(this.config.server_health?.db_liveness_latency_fail_ms) || + DEFAULT_DB_LIVENESS_LATENCY_FAIL_MS; + + const db = this.clients.db; + if (db && typeof db.read === 'function') { + this.addCheck('database-liveness', async () => { + const startedAt = Date.now(); + const rows = (await db.read('SELECT 1 AS ok')) as unknown[]; + const durationMs = Date.now() - startedAt; + this.#stats.database_liveness_latency_ms = durationMs; + + if (!Array.isArray(rows) || rows.length === 0) { + throw new Error('database liveness query returned no rows'); + } + if (durationMs > latencyFailMs) { + throw new Error( + `database liveness latency ${durationMs}ms > threshold ${latencyFailMs}ms`, + ); + } + }); + } + + const socket = this.services.socket as SocketService | undefined; + if (socket) { + this.addCheck('socket-initialized', () => { + // Attach happens in `attachHttpServer`, called by PuterServer + // after http is ready. If the internal io hasn't been set + // by the time checks start running, something is wrong. + const check = socket as unknown as { hasIO?: () => boolean }; + if (typeof check.hasIO === 'function' && !check.hasIO()) { + throw new Error('socket.io is not initialized'); + } + }); + } + } + + #startLoop(): void { + this.#intervalHandle = setInterval(() => { + if (this.#loopRunning) return; // reentrancy guard + this.#loopRunning = true; + this.#runCycle().finally(() => { + this.#loopRunning = false; + }); + }, CHECK_INTERVAL_MS); + // Don't keep the process alive just for health checks. + this.#intervalHandle.unref?.(); + } + + async #runCycle(): Promise { + if (this.#draining) { + this.#lastCycleCompletedAt = Date.now(); + this.#stats.last_check_cycle_completed_at = + this.#lastCycleCompletedAt; + this.#stats.check_durations_ms = {}; + this.#stats.failed_checks = []; + return; + } + + const newFailures: { name: string }[] = []; + const durations: Record = {}; + + for (const check of this.#checks) { + const startedAt = Date.now(); + let timeoutHandle: NodeJS.Timeout | null = null; + try { + await new Promise((resolve, reject) => { + timeoutHandle = setTimeout( + () => reject(new Error('Health check timed out')), + CHECK_TIMEOUT_MS, + ); + Promise.resolve(check.fn()).then(() => resolve(), reject); + }); + } catch (err) { + newFailures.push({ name: check.name }); + const alreadyFailing = this.#failures.some( + (f) => f.name === check.name, + ); + if (!alreadyFailing) { + this.clients.alarm?.create( + 'health-check-failure', + `Health check ${check.name} failed`, + { error: err as Error }, + ); + for (const handler of check.onFailHandlers) { + try { + await handler(err); + } catch (hErr) { + console.error( + `[server-health] onFail handler for ${check.name} threw:`, + hErr, + ); + } + } + } + console.error( + `[server-health] check "${check.name}" failed:`, + err, + ); + } finally { + if (timeoutHandle) clearTimeout(timeoutHandle); + durations[check.name] = Date.now() - startedAt; + } + } + + this.#failures = newFailures; + this.#lastCycleCompletedAt = Date.now(); + this.#stats.last_check_cycle_completed_at = this.#lastCycleCompletedAt; + this.#stats.check_durations_ms = durations; + this.#stats.failed_checks = newFailures.map((f) => f.name); + } + + #collectFailures(): string[] { + const names = this.#failures.map((f) => f.name); + const stale = this.#staleLoopFailure(); + if (stale) names.push(stale); + return names; + } + + #staleLoopFailure(): string | null { + const staleAfterMs = + Number(this.config.server_health?.stale_health_loop_fail_ms) || + CHECK_INTERVAL_MS * HEALTH_LOOP_STALE_MULTIPLIER; + const now = Date.now(); + + if (this.#lastCycleCompletedAt === 0) { + return now - this.#healthStartedAt > staleAfterMs + ? 'health-check-loop-not-running' + : null; + } + return now - this.#lastCycleCompletedAt > staleAfterMs + ? 'health-check-loop-stale' + : null; + } + + /** Snapshot of per-cycle timing + DB latency. */ + getStats(): HealthStats { + return { + ...this.#stats, + check_durations_ms: { ...this.#stats.check_durations_ms }, + }; + } +} diff --git a/src/backend/services/homepage/PuterHomepageService.ts b/src/backend/services/homepage/PuterHomepageService.ts new file mode 100644 index 000000000..bdf53d4d4 --- /dev/null +++ b/src/backend/services/homepage/PuterHomepageService.ts @@ -0,0 +1,370 @@ +import { encode } from 'html-entities'; +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import type { Request, Response } from 'express'; +import { PuterService } from '../types.js'; +import type { Actor } from '../../core/actor'; + +interface Manifest { + css_paths?: string[]; + js_paths?: string[]; + lib_paths?: string[]; + index?: string; + [k: string]: unknown; +} + +export interface PageMeta { + title: string; + description?: string; + short_description?: string; + company?: string; + canonical_url?: string; + social_media_image?: string; + icon?: string; + app?: { name?: string; [k: string]: unknown } | null; +} + +export interface LaunchOptions { + on_initialized?: Array>; +} + +interface PuterGuiAddonsEvent { + req: Request; + path: string; + logged_in_user: Actor['user'] | null; + guiParams: Record; + /** Extensions may append to these — rendered into the shell HTML. */ + bodyContent: string; + headContent: string; + prependHeadContent: string; + /** + * Scripts/markup that must run BEFORE the `gui(...)` bootstrap. Useful + * for loading jQuery or third-party SDKs (Stripe.js) that the GUI code + * expects to be present on window. + */ + prependBodyContent: string; +} + +/** + * Serves the root HTML shell that bootstraps the Puter GUI. + * + * Extensions contribute by: + * - `registerScript(url)` — adds a ``) + .join('\n'); + + const guiParamsJson = JSON.stringify(guiParams).replace( + / + + + ${e(title)} + ${event.prependHeadContent} + + + ${bundled ? `` : ''} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${bundled ? `` : ''} + + + + + + ${manifestCss} + + ${event.headContent} + + + ${event.prependBodyContent} + + ${bundled ? "" : ''} + + + + ${serviceScriptTags} + + + ${event.bodyContent} + +`; + } + + #renderError(message: string): string { + return ` + + + + + +

${encode(String(message), { mode: 'nonAsciiPrintable' })}

+ +`; + } + + #originFromRequest(req: Request): string { + // Prefer the pre-computed `config.origin` (protocol + domain + port). + // Without it, non-80/443 deployments end up with URLs missing the + // port, which breaks every self-referential fetch the GUI makes + // (`/get-gui-token`, `/login`, `/signup`, …). + if (this.config.origin) return this.config.origin; + const domain = this.config.domain ?? req.hostname; + return `${req.protocol}://${domain}`; + } + + #validSocialImage(raw: string | undefined, assetDir: string): string { + const fallback = `${assetDir}/images/screenshot.png`; + if (!raw) return fallback; + try { + const url = new URL(raw); + if (url.protocol !== 'http:' && url.protocol !== 'https:') + return fallback; + } catch { + return fallback; + } + if (!/\.(png|jpg|jpeg|gif|webp)$/i.test(raw)) return fallback; + return raw; + } +} diff --git a/src/backend/services/index.ts b/src/backend/services/index.ts new file mode 100644 index 000000000..fbebf1d6e --- /dev/null +++ b/src/backend/services/index.ts @@ -0,0 +1,54 @@ +import { ACLService } from './acl/ACLService'; +import { AppPermissionService } from './apps/AppPermissionService'; +import { RecommendedAppsService } from './apps/RecommendedAppsService'; +import { SuggestedAppsService } from './apps/SuggestedAppsService'; +import { AuthService } from './auth/AuthService'; +import { BroadcastService } from './broadcast/BroadcastService'; +import { NotificationService } from './notification/NotificationService'; +import { AppIconService } from './appIcon/AppIconService'; +import { DefaultUserService } from './selfhosted/DefaultUserService'; +import { PuterHomepageService } from './homepage/PuterHomepageService'; +import { OIDCService } from './auth/OIDCService'; +import { TokenService } from './auth/TokenService'; +import { FSService } from './fs/FSService'; +import { MeteringService } from './metering/MeteringService'; +import { PermissionService } from './permission/PermissionService'; +import { ServerHealthService } from './health/ServerHealthService'; +import { SocketService } from './socket/SocketService'; +import { SubdomainPermissionService } from './subdomain/SubdomainPermissionService'; +import type { IPuterServiceRegistry } from './types'; + +// Ordering matters: services declared later see earlier ones as peers. +// ACLService depends on PermissionService (for scan + grant/revoke), so +// PermissionService must be constructed first. +// AuthService depends on TokenService (JWT verify). +// FSService constructs its own internal repo + S3 provider in onServerStart. +// SocketService depends on AuthService (for handshake auth). +// NotificationService depends on notification store (for DB) + event client (for socket push). +// BroadcastService is independent — only needs the event client. +export const puterServices = { + metering: MeteringService, + permission: PermissionService, + acl: ACLService, + token: TokenService, + auth: AuthService, + fs: FSService, + // AppPermissionService + SubdomainPermissionService register permission + // rewriters/implicators only; no runtime state. Placed after fsEntry so + // the FS rewriter runs first for `fs:/path` → `fs:` before any + // downstream check that might chain app-root-dir → fs. + appPermission: AppPermissionService, + subdomainPermission: SubdomainPermissionService, + recommendedApps: RecommendedAppsService, + suggestedApps: SuggestedAppsService, + socket: SocketService, + notification: NotificationService, + broadcast: BroadcastService, + oidc: OIDCService, + appIcon: AppIconService, + defaultUser: DefaultUserService, + homepage: PuterHomepageService, + // Health comes after socket so its default `socket-initialized` + // check can reference the peer. + health: ServerHealthService, +} satisfies IPuterServiceRegistry; diff --git a/src/backend/services/metering/MeteringService.ts b/src/backend/services/metering/MeteringService.ts new file mode 100644 index 000000000..f34fa16f1 --- /dev/null +++ b/src/backend/services/metering/MeteringService.ts @@ -0,0 +1,899 @@ +import murmurhash from 'murmurhash'; +import { PuterService } from '../types'; +import type { Actor } from '../../core/actor'; +import { isSystemActor } from '../../core/actor'; +import { + DEFAULT_FREE_SUBSCRIPTION, + DEFAULT_TEMP_SUBSCRIPTION, + GLOBAL_APP_KEY, + METRICS_PREFIX, + PERIOD_ESCAPE, + POLICY_PREFIX, +} from './consts'; +import type { AppTotals, UsageAddons, UsageByType, UsageRecord } from './types'; +import { toMicroCents } from './utils'; + +import { SUB_POLICIES } from '../../data/subPolicies/index.js'; + +// ── Types ──────────────────────────────────────────────────────────── + +type SubscriptionPolicy = (typeof SUB_POLICIES)[number]; + +export type SubscriptionResolver = ( + actor: Actor, +) => Promise | string | null | undefined; + +interface UsageInput { + usageType: string; + usageAmount: number; + costOverride?: number; +} + +// ── MeteringService ────────────────────────────────────────────────── + +/** + * Tracks per-actor and global usage, and exposes subscription/addon lookup. + * All metering data is persisted under the system namespace via + * `stores.kv` (SystemKVStore) + * + * Callers (typically drivers or controllers) pass the user-scoped actor in; we fan that + * out into several aggregated KV records. + */ +export class MeteringService extends PuterService { + static GLOBAL_SHARD_COUNT = 10000; + static APP_SHARD_COUNT = 10000; + static MAX_GLOBAL_USAGE_PER_MINUTE = toMicroCents(0.2); + + private rateCheckTimer: ReturnType | null = null; + private extraPolicies: SubscriptionPolicy[] = []; + private subscriptionResolvers: SubscriptionResolver[] = []; + private defaultSubscriptionResolvers: SubscriptionResolver[] = []; + + // ── Lifecycle ──────────────────────────────────────────────────── + + override onServerStart(): void { + this.rateCheckTimer = setInterval( + () => { + this.checkRateOfChange().catch((e) => { + console.error('[metering] rate-of-change check failed', e); + }); + }, + 1000 * 60 * 25, + ); + this.rateCheckTimer.unref?.(); + } + + override onServerShutdown(): void { + if (this.rateCheckTimer) { + clearInterval(this.rateCheckTimer); + this.rateCheckTimer = null; + } + } + + // ── Extension hooks ────────────────────────────────────────────── + + /** Register a policy that should be available to actors. */ + registerPolicy(policy: SubscriptionPolicy): void { + this.extraPolicies.push(policy); + } + + /** + * Register a resolver that maps an actor to a subscription id. The first + * resolver that returns a non-empty id wins; later resolvers are skipped. + */ + registerSubscriptionResolver(fn: SubscriptionResolver): void { + this.subscriptionResolvers.push(fn); + } + + /** + * Register a resolver that maps an actor to a *default* subscription id, + * used when no explicit subscription is set. First non-empty wins. + */ + registerDefaultSubscriptionResolver(fn: SubscriptionResolver): void { + this.defaultSubscriptionResolvers.push(fn); + } + + // ── Public API: increment usage ────────────────────────────────── + + utilRecordUsageObject>( + trackedUsageObject: T, + actor: Actor, + modelPrefix: string, + costsOverrides?: Partial>, + ) { + return this.batchIncrementUsages( + actor, + Object.entries(trackedUsageObject).map(([usageKind, amount]) => { + const hasOverride = + !!costsOverrides && + Number.isFinite(costsOverrides[usageKind]); + return { + usageType: `${modelPrefix}:${usageKind}`, + usageAmount: amount, + costOverride: hasOverride + ? costsOverrides![usageKind as keyof T] + : undefined, + }; + }), + ); + } + + async incrementUsage( + actor: Actor, + usageType: string, + usageAmount: number, + costOverride?: number, + ): Promise { + usageAmount = usageAmount < 0 ? 1 : usageAmount; + + const costOverrideRaw = costOverride; + costOverride = !Number.isFinite(costOverride) + ? undefined + : (costOverride as number) < 0 + ? 1 + : costOverride; + + if (costOverrideRaw && costOverrideRaw < 0) { + this.clients.alarm.create( + `metering unexpected negative cost access to: ${usageType}`, + 'negative cost abuse vector!', + { + userId: actor.user?.uuid, + username: actor.user?.username, + appId: actor.app?.uid, + usageType, + usageAmount, + costOverride, + }, + ); + } + + try { + if (!usageAmount || !usageType || !actor) + return { total: 0 } as UsageByType; + if (isSystemActor(actor)) return { total: 0 } as UsageByType; + + const currentMonth = this.monthYearString(); + + const totalCost = costOverride ?? 0; + + const escapedUsageType = String(usageType).replace( + /\./g, + PERIOD_ESCAPE, + ); + const appId = actor.app?.uid || GLOBAL_APP_KEY; + const userId = actor.user.uuid; + const pathAndAmountMap = { + total: totalCost, + [`${escapedUsageType}.units`]: usageAmount, + [`${escapedUsageType}.cost`]: totalCost, + [`${escapedUsageType}.count`]: 1, + }; + + const actorUsageKey = `${METRICS_PREFIX}:actor:${userId}:${currentMonth}`; + const actorUsagesPromise = this.stores.kv + .incr({ + key: actorUsageKey, + pathAndAmountMap, + }) + .then((r) => r.res as unknown as UsageByType); + + // Aux writes — fire and forget + this.handleAuxPromise( + `puterConsumption ${userId}/${appId}`, + this.stores.kv.incr({ + key: this.globalUsageKey(userId, appId, currentMonth), + pathAndAmountMap, + }), + ); + + this.handleAuxPromise( + `actorAppUsage ${userId}/${appId}`, + this.stores.kv.incr({ + key: `${METRICS_PREFIX}:actor:${userId}:app:${appId}:${currentMonth}`, + pathAndAmountMap, + }), + ); + + if (appId !== GLOBAL_APP_KEY) { + this.handleAuxPromise( + `appUsage ${appId}/${userId}`, + this.stores.kv.incr({ + key: this.appUsageKey(appId, userId, currentMonth), + pathAndAmountMap, + }), + ); + } + + this.handleAuxPromise( + `actorAppTotals ${userId}`, + this.stores.kv.incr({ + key: `${METRICS_PREFIX}:actor:${userId}:apps:${currentMonth}`, + pathAndAmountMap: { + [`${appId}.total`]: totalCost, + [`${appId}.count`]: 1, + }, + }), + ); + + this.handleAuxPromise( + 'lastUpdated', + this.stores.kv.set({ + key: `${METRICS_PREFIX}:actor:${userId}:lastUpdated`, + value: Date.now(), + }), + ); + + const [actorUsages, actorSubscription, actorAddons] = + await Promise.all([ + actorUsagesPromise, + this.getActorSubscription(actor), + this.getActorAddons(actor), + ]); + + await this.maybeConsumeAddonCredits( + userId, + actorUsages.total, + actorSubscription.monthUsageAllowance, + actorAddons, + totalCost, + ); + + this.maybeAlertOveruse({ + actor, + userId, + actorUsages, + actorSubscription, + actorAddons, + incrementCost: totalCost, + usageType, + usageAmount, + costOverride, + }); + + return actorUsages; + } catch (e) { + console.error('[metering] incrementUsage failed', { + actor, + usageType, + usageAmount, + error: e, + }); + this.clients.alarm.create( + `metering service error for user: ${actor.user?.username} app: ${actor.app?.uid}`, + (e as Error).message, + { + userId: actor.user?.uuid, + username: actor.user?.username, + appId: actor.app?.uid, + error: e as Error, + usageType, + usageAmount, + costOverride, + }, + ); + return { total: 0 } as UsageByType; + } + } + + async batchIncrementUsages( + actor: Actor, + usages: UsageInput[], + ): Promise { + try { + if (!usages || usages.length === 0 || !actor) + return { total: 0 } as UsageByType; + if (isSystemActor(actor)) return { total: 0 } as UsageByType; + + const currentMonth = this.monthYearString(); + const aggregated: Record = {}; + let totalBatchCost = 0; + + for (const { + usageType, + usageAmount: usageAmountRaw, + costOverride: costOverrideRaw, + } of usages) { + const usageAmount = + !Number.isFinite(usageAmountRaw) || usageAmountRaw < 0 + ? 1 + : usageAmountRaw; + const costOverride = !Number.isFinite(costOverrideRaw) + ? undefined + : (costOverrideRaw as number) < 0 + ? 1 + : costOverrideRaw; + + if (!usageAmount || !usageType) continue; + + if (costOverrideRaw && costOverrideRaw < 0) { + this.clients.alarm.create( + `metering unexpected negative cost access to: ${usageType}`, + 'negative cost abuse vector!', + { + userId: actor.user?.uuid, + username: actor.user?.username, + appId: actor.app?.uid, + usageType, + usageAmount, + costOverride, + costOverrideRaw, + }, + ); + } + + const totalCost = costOverride ?? 0; + totalBatchCost += totalCost; + + const escaped = String(usageType).replace(/\./g, PERIOD_ESCAPE); + aggregated['total'] = (aggregated['total'] || 0) + totalCost; + aggregated[`${escaped}.units`] = + (aggregated[`${escaped}.units`] || 0) + usageAmount; + aggregated[`${escaped}.cost`] = + (aggregated[`${escaped}.cost`] || 0) + totalCost; + aggregated[`${escaped}.count`] = + (aggregated[`${escaped}.count`] || 0) + 1; + } + + const appId = actor.app?.uid || GLOBAL_APP_KEY; + const userId = actor.user.uuid; + + const actorUsageKey = `${METRICS_PREFIX}:actor:${userId}:${currentMonth}`; + const actorUsagesPromise = this.stores.kv + .incr({ + key: actorUsageKey, + pathAndAmountMap: aggregated, + }) + .then((r) => r.res as unknown as UsageByType); + + this.handleAuxPromise( + `puterConsumption ${userId}/${appId}`, + this.stores.kv.incr({ + key: this.globalUsageKey(userId, appId, currentMonth), + pathAndAmountMap: aggregated, + }), + ); + this.handleAuxPromise( + `actorAppUsage ${userId}/${appId}`, + this.stores.kv.incr({ + key: `${METRICS_PREFIX}:actor:${userId}:app:${appId}:${currentMonth}`, + pathAndAmountMap: aggregated, + }), + ); + this.handleAuxPromise( + `appUsage ${appId}/${userId}`, + this.stores.kv.incr({ + key: this.appUsageKey(appId, userId, currentMonth), + pathAndAmountMap: aggregated, + }), + ); + this.handleAuxPromise( + `actorAppTotals ${userId}`, + this.stores.kv.incr({ + key: `${METRICS_PREFIX}:actor:${userId}:apps:${currentMonth}`, + pathAndAmountMap: { + [`${appId}.total`]: totalBatchCost, + [`${appId}.count`]: usages.length, + }, + }), + ); + this.handleAuxPromise( + 'lastUpdated', + this.stores.kv.set({ + key: `${METRICS_PREFIX}:actor:${userId}:lastUpdated`, + value: Date.now(), + }), + ); + + const [actorUsages, actorSubscription, actorAddons] = + await Promise.all([ + actorUsagesPromise, + this.getActorSubscription(actor), + this.getActorAddons(actor), + ]); + + await this.maybeConsumeAddonCredits( + userId, + actorUsages.total, + actorSubscription.monthUsageAllowance, + actorAddons, + totalBatchCost, + ); + + this.maybeAlertOveruse({ + actor, + userId, + actorUsages, + actorSubscription, + actorAddons, + incrementCost: totalBatchCost, + batchUsages: usages, + }); + + return actorUsages; + } catch (e) { + console.error('[metering] batchIncrementUsages failed', { + actor, + usages, + error: e, + }); + this.clients.alarm.create( + `metering service error for user: ${actor.user?.username} app: ${actor.app?.uid}`, + (e as Error).message, + { + userId: actor.user?.uuid, + username: actor.user?.username, + appId: actor.app?.uid, + error: e as Error, + actor, + batchUsages: usages, + }, + ); + return { total: 0 } as UsageByType; + } + } + + // ── Public API: read usage ─────────────────────────────────────── + + async getActorCurrentMonthUsageDetails(actor: Actor): Promise<{ + usage: UsageByType; + appTotals: Record; + }> { + if (!actor.user?.uuid) + throw new Error('Actor must be a user to get usage details'); + + const currentMonth = this.monthYearString(); + const keys = [ + `${METRICS_PREFIX}:actor:${actor.user.uuid}:${currentMonth}`, + `${METRICS_PREFIX}:actor:${actor.user.uuid}:apps:${currentMonth}`, + ]; + + const { res } = await this.stores.kv.get({ key: keys }); + const [usage, appTotals] = (res ?? []) as [ + UsageByType | null, + Record | null, + ]; + + const appId = actor.app?.uid; + if (appTotals && appId) { + const filtered: Record = {}; + const others: AppTotals = {} as AppTotals; + Object.entries(appTotals).forEach(([appKey, appUsage]) => { + if (appKey === appId) { + filtered[appKey] = appUsage; + } else { + Object.entries(appUsage).forEach(([usageKind, amount]) => { + const key = usageKind as keyof AppTotals; + if (!others[key]) others[key] = 0; + others[key] += amount; + }); + } + }); + if (others) filtered['others'] = others; + return { + usage: usage || ({ total: 0 } as UsageByType), + appTotals: filtered, + }; + } + + return { + usage: usage || ({ total: 0 } as UsageByType), + appTotals: appTotals || {}, + }; + } + + async setActorCurrentMonthUsageTotal( + actor: Actor, + totalCost: number, + ): Promise { + if (!actor.user?.uuid) + throw new Error('Actor must be a user to set usage details'); + if (!Number.isFinite(totalCost) || totalCost < 0) { + throw new Error('Total cost must be a non-negative number'); + } + + const normalizedTotal = Math.round(totalCost); + const currentMonth = this.monthYearString(); + const userId = actor.user.uuid; + const appId = actor.app?.uid || GLOBAL_APP_KEY; + const actorUsageKey = `${METRICS_PREFIX}:actor:${userId}:${currentMonth}`; + + const { res: current } = await this.stores.kv.get({ + key: actorUsageKey, + }); + const currentTotal = (current as UsageByType | null)?.total ?? 0; + const delta = normalizedTotal - currentTotal; + + if (delta === 0) { + return (current as UsageByType) || ({ total: 0 } as UsageByType); + } + + const pathAndAmountMap = { + total: delta, + 'manual_adjustment.cost': delta, + 'manual_adjustment.units': delta, + 'manual_adjustment.count': 1, + }; + + const updated = ( + await this.stores.kv.incr({ key: actorUsageKey, pathAndAmountMap }) + ).res as unknown as UsageByType; + + this.handleAuxPromise( + `puterConsumption ${userId}/${appId}`, + this.stores.kv.incr({ + key: this.globalUsageKey(userId, appId, currentMonth), + pathAndAmountMap, + }), + ); + this.handleAuxPromise( + `actorAppUsage ${userId}/${appId}`, + this.stores.kv.incr({ + key: `${METRICS_PREFIX}:actor:${userId}:app:${appId}:${currentMonth}`, + pathAndAmountMap, + }), + ); + this.handleAuxPromise( + `actorAppTotals ${userId}`, + this.stores.kv.incr({ + key: `${METRICS_PREFIX}:actor:${userId}:apps:${currentMonth}`, + pathAndAmountMap: { + [`${appId}.total`]: delta, + [`${appId}.count`]: 1, + }, + }), + ); + this.handleAuxPromise( + 'lastUpdated', + this.stores.kv.set({ + key: `${METRICS_PREFIX}:actor:${userId}:lastUpdated`, + value: Date.now(), + }), + ); + + return updated; + } + + async getActorCurrentMonthAppUsageDetails( + actor: Actor, + appId?: string, + ): Promise { + if (!actor.user?.uuid) + throw new Error('Actor must be a user to get usage details'); + + const resolvedAppId = appId || actor.app?.uid || GLOBAL_APP_KEY; + + const actorAppId = actor.app?.uid; + if ( + actorAppId && + actorAppId !== resolvedAppId && + resolvedAppId !== GLOBAL_APP_KEY + ) { + throw new Error( + 'Actor can only get usage details for their own app or global app', + ); + } + + const currentMonth = this.monthYearString(); + const key = `${METRICS_PREFIX}:actor:${actor.user.uuid}:app:${resolvedAppId}:${currentMonth}`; + const { res } = await this.stores.kv.get({ key }); + return (res as UsageByType) || ({ total: 0 } as UsageByType); + } + + async getRemainingUsage(actor: Actor): Promise { + const { remaining } = await this.getAllowedUsage(actor); + return remaining || 0; + } + + async getAllowedUsage(actor: Actor): Promise<{ + remaining: number; + monthUsageAllowance: number; + addons: UsageAddons; + }> { + const [userSubscription, addons, currentMonthUsage] = await Promise.all( + [ + this.getActorSubscription(actor), + this.getActorAddons(actor), + this.getActorCurrentMonthUsageDetails(actor), + ], + ); + + const remaining = Math.max( + 0, + (userSubscription.monthUsageAllowance || 0) + + (addons?.purchasedCredits || 0) - + (currentMonthUsage.usage.total || 0) - + (addons?.consumedPurchaseCredits || 0), + ); + + return { + remaining, + monthUsageAllowance: userSubscription.monthUsageAllowance, + addons, + }; + } + + async hasAnyUsage(actor: Actor): Promise { + return (await this.getRemainingUsage(actor)) > 0; + } + + async hasEnoughCredits(actor: Actor, amount: number): Promise { + return (await this.getRemainingUsage(actor)) >= amount; + } + + async getActorSubscription(actor: Actor): Promise { + if (!actor.user?.uuid) + throw new Error('Actor must be a user to get policy'); + + const fallbackDefault = actor.user.email + ? DEFAULT_FREE_SUBSCRIPTION + : DEFAULT_TEMP_SUBSCRIPTION; + + const resolvedDefault = + (await this.firstResolver( + this.defaultSubscriptionResolvers, + actor, + )) || fallbackDefault; + const resolvedUser = + (await this.firstResolver(this.subscriptionResolvers, actor)) || + resolvedDefault; + + const availablePolicies: SubscriptionPolicy[] = [ + ...this.extraPolicies, + ...SUB_POLICIES, + ]; + return ( + availablePolicies.find((p) => p.id === resolvedUser) ?? + availablePolicies.find((p) => p.id === resolvedDefault)! + ); + } + + async getActorAddons(actor: Actor): Promise { + if (!actor.user?.uuid) + throw new Error('Actor must be a user to get policy addons'); + const key = `${POLICY_PREFIX}:actor:${actor.user.uuid}:addons`; + const { res } = await this.stores.kv.get({ key }); + return (res ?? {}) as UsageAddons; + } + + async getActorAppUsage(actor: Actor, appId: string): Promise { + if (!actor.user?.uuid) + throw new Error('Actor must be a user to get app usage'); + if (actor.app?.uid && actor.app.uid !== appId) { + throw new Error('Actor can only get usage for their own app'); + } + + const currentMonth = this.monthYearString(); + const key = `${METRICS_PREFIX}:actor:${actor.user.uuid}:app:${appId}:${currentMonth}`; + const { res } = await this.stores.kv.get({ key }); + return (res ?? { total: 0 }) as UsageByType; + } + + async getGlobalUsage(): Promise { + const currentMonth = this.monthYearString(); + const keyPrefix = `${METRICS_PREFIX}:puter:`; + const keys: string[] = []; + for ( + let shard = 0; + shard < MeteringService.GLOBAL_SHARD_COUNT; + shard++ + ) { + keys.push(`${keyPrefix}${shard}:${currentMonth}`); + } + keys.push(`${keyPrefix}${currentMonth}`); + + const { res } = await this.stores.kv.get({ key: keys }); + const usages = (res ?? []) as UsageByType[]; + const aggregated: UsageByType = { total: 0 } as UsageByType; + + usages.filter(Boolean).forEach((entry = {} as UsageByType) => { + const { total, ...rest } = entry; + aggregated.total += total || 0; + Object.entries(rest as Record).forEach( + ([usageKind, record]) => { + if (!aggregated[usageKind]) { + aggregated[usageKind] = { + cost: 0, + units: 0, + count: 0, + } as UsageRecord; + } + const agg = aggregated[usageKind] as UsageRecord; + agg.cost += record.cost; + agg.count += record.count; + agg.units += record.units; + }, + ); + }); + + return aggregated; + } + + async updateAddonCredit( + userId: string, + tokenAmount: number, + ): Promise { + if (!userId) throw new Error('User needed to update extra credits'); + await this.stores.kv.incr({ + key: `${POLICY_PREFIX}:actor:${userId}:addons`, + pathAndAmountMap: { purchasedCredits: tokenAmount }, + }); + } + + // ── Internals ──────────────────────────────────────────────────── + + private monthYearString(): string { + const now = new Date(); + return `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, '0')}`; + } + + /** + * Randomized shard key to spread writes across the global consumption bucket. + */ + private globalUsageKey( + userId: string, + appId: string, + currentMonth: string, + ): string { + const hash = + murmurhash.v3(`${userId}:${appId}`) % + MeteringService.GLOBAL_SHARD_COUNT; + return `${METRICS_PREFIX}:puter:${hash}:${currentMonth}`; + } + + private appUsageKey( + appId: string, + userId: string, + currentMonth: string, + ): string { + const hash = + murmurhash.v3(`${appId}${userId}`) % + MeteringService.APP_SHARD_COUNT; + return `${METRICS_PREFIX}:app:${appId}:${hash}:${currentMonth}`; + } + + private handleAuxPromise(label: string, promise: Promise): void { + promise.catch((e: Error) => { + console.warn( + `[metering] aux write failed (${label}): ${e.message}`, + ); + }); + } + + private async firstResolver( + resolvers: SubscriptionResolver[], + actor: Actor, + ): Promise { + for (const resolver of resolvers) { + try { + const result = await resolver(actor); + if (result) return result; + } catch (e) { + console.warn('[metering] subscription resolver failed', e); + } + } + return null; + } + + private async maybeConsumeAddonCredits( + userId: string, + totalUsage: number, + monthUsageAllowance: number, + addons: UsageAddons, + incrementCost: number, + ): Promise { + if (totalUsage <= monthUsageAllowance) return; + if (!addons.purchasedCredits) return; + if (addons.purchasedCredits <= (addons.consumedPurchaseCredits || 0)) + return; + + const withinBoundsUsage = Math.max( + 0, + monthUsageAllowance - totalUsage + incrementCost, + ); + const overageUsage = incrementCost - withinBoundsUsage; + if (overageUsage <= 0) return; + + const toConsume = Math.min( + overageUsage, + addons.purchasedCredits - (addons.consumedPurchaseCredits || 0), + ); + await this.stores.kv.incr({ + key: `${POLICY_PREFIX}:actor:${userId}:addons`, + pathAndAmountMap: { consumedPurchaseCredits: toConsume }, + }); + } + + private maybeAlertOveruse(ctx: { + actor: Actor; + userId: string; + actorUsages: UsageByType; + actorSubscription: SubscriptionPolicy; + actorAddons: UsageAddons; + incrementCost: number; + usageType?: string; + usageAmount?: number; + costOverride?: number; + batchUsages?: UsageInput[]; + }): void { + const { + actor, + userId, + actorUsages, + actorSubscription, + actorAddons, + incrementCost, + } = ctx; + + const allowedMultiple = Math.floor( + actorUsages.total / actorSubscription.monthUsageAllowance, + ); + const previousMultiple = Math.floor( + (actorUsages.total - incrementCost) / + actorSubscription.monthUsageAllowance, + ); + const isOver2x = allowedMultiple >= 2; + const crossedThreshold = previousMultiple < allowedMultiple; + const hasNoAddonCredit = + (actorAddons.purchasedCredits || 0) <= + (actorAddons.consumedPurchaseCredits || 0); + + if (!(isOver2x && crossedThreshold && hasNoAddonCredit)) return; + + this.clients.alarm.create( + `metering usage exceeded by user: ${actor.user?.username}`, + `Actor ${userId} has exceeded their usage allowance significantly`, + { + userId: actor.user?.uuid, + username: actor.user?.username, + appId: actor.app?.uid, + usageType: ctx.usageType, + usageAmount: ctx.usageAmount, + costOverride: ctx.costOverride, + batchUsages: ctx.batchUsages, + totalUsage: actorUsages.total, + monthUsageAllowance: actorSubscription.monthUsageAllowance, + }, + ); + } + + private async checkRateOfChange(): Promise { + const now = Date.now(); + const lastChangeKey = `${METRICS_PREFIX}:lastGlobalUsageCheck`; + const { res: lastChangeRaw } = await this.stores.kv.get({ + key: lastChangeKey, + }); + const lastChange = lastChangeRaw as { + total: number; + timestamp: number; + } | null; + + if (lastChange && now - lastChange.timestamp <= 14 * 60 * 1000) return; + + const globalUsage = await this.getGlobalUsage(); + const currTotal = globalUsage.total; + + if (lastChange) { + const timeDelta = now - lastChange.timestamp; + const usageDelta = currTotal - lastChange.total; + const usagePerMinute = usageDelta / (timeDelta / 60000); + + if (usagePerMinute > MeteringService.MAX_GLOBAL_USAGE_PER_MINUTE) { + this.clients.alarm.create( + 'metering:excessiveGlobalUsageRate', + `Global usage rate is excessive: ${usagePerMinute} micro-cents per minute`, + { + usagePerMinute, + maxAllowedPerMinute: + MeteringService.MAX_GLOBAL_USAGE_PER_MINUTE, + }, + ); + } + } + + await this.stores.kv.set({ + key: lastChangeKey, + value: { total: currTotal, timestamp: now }, + }); + } +} diff --git a/src/backend/services/metering/consts.ts b/src/backend/services/metering/consts.ts new file mode 100644 index 000000000..d40d4860e --- /dev/null +++ b/src/backend/services/metering/consts.ts @@ -0,0 +1,7 @@ +export const GLOBAL_APP_KEY = 'os-global'; +export const METRICS_PREFIX = 'metering'; +export const POLICY_PREFIX = 'policy'; +/** dots in usage types are escaped so they don't collide with kv nested paths */ +export const PERIOD_ESCAPE = '_dot_'; +export const DEFAULT_FREE_SUBSCRIPTION = 'user_free'; +export const DEFAULT_TEMP_SUBSCRIPTION = 'temp_free'; diff --git a/src/backend/services/metering/types.ts b/src/backend/services/metering/types.ts new file mode 100644 index 000000000..9d777c701 --- /dev/null +++ b/src/backend/services/metering/types.ts @@ -0,0 +1,23 @@ +export interface UsageAddons { + purchasedCredits: number; + consumedPurchaseCredits: number; + purchasedStorage: number; + rateDiscounts: { + [usageType: string]: number | string; + }; +} + +export interface UsageRecord { + cost: number; + count: number; + units: number; +} + +export type UsageByType = { total: number } & Partial< + Record, UsageRecord> +>; + +export interface AppTotals { + total: number; + count: number; +} diff --git a/src/backend/services/metering/utils.ts b/src/backend/services/metering/utils.ts new file mode 100644 index 000000000..ff70ef1e4 --- /dev/null +++ b/src/backend/services/metering/utils.ts @@ -0,0 +1,2 @@ +export const toMicroCents = (dollars: number): number => + dollars * 1_000_000 * 100; diff --git a/src/backend/services/notification/NotificationService.ts b/src/backend/services/notification/NotificationService.ts new file mode 100644 index 000000000..8a92b119b --- /dev/null +++ b/src/backend/services/notification/NotificationService.ts @@ -0,0 +1,196 @@ +import { v4 as uuidv4 } from 'uuid'; +import { PuterService } from '../types.js'; + +/** + * Notification orchestration — glues the NotificationStore (DB) to the + * event bus (socket push) and handles lifecycle events (user connects → + * send unreads, notification shown/acked → socket event). + * + * Other services push notifications via `notify(userIds, notification)`. + * The driver (`puter-notifications`) handles read/select/mark for API + * consumers; this service handles the write-and-push side. + */ +export class NotificationService extends PuterService { + #pendingWrites = new Map>(); + /** user.id → debounce timeout */ + #connectTimeouts = new Map>(); + + override onServerStart(): void { + // When a user opens the GUI, send their pending unreads. + this.clients.event.on( + 'web.socket.user-connected', + (_key: string, data: unknown) => { + const d = data as { user?: { id?: number } } | undefined; + const userId = d?.user?.id; + if (!userId) return; + + // Debounce: multiple tabs may fire user-connected in rapid succession. + const existing = this.#connectTimeouts.get(userId); + if (existing) clearTimeout(existing); + this.#connectTimeouts.set( + userId, + setTimeout(() => { + this.#connectTimeouts.delete(userId); + void this.#sendUnreads(userId).catch((err) => { + console.warn( + '[notification] sendUnreads failed', + err, + ); + }); + }, 2000), + ); + }, + ); + + // Track when a notification is actually delivered to a socket so + // we can mark it as shown. + this.clients.event.on( + 'sent-to-user.notif.message', + (_key: string, data: unknown) => { + const d = data as + | { user_id?: number; response?: { uid?: string } } + | undefined; + const uid = d?.response?.uid; + const userId = d?.user_id; + if (!uid || !userId) return; + void this.#markShownAfterWrite(uid, userId); + }, + ); + } + + // ── Public API ────────────────────────────────────────────────── + + /** + * Push a notification to one or more users. The notification is + * emitted to the socket bus immediately (real-time), then persisted + * to the DB asynchronously. + * + * @param userIds Target user ids + * @param notification Payload — { source, title, text?, icon?, template?, fields? } + */ + async notify( + userIds: number[], + notification: Record, + ): Promise { + const uid = uuidv4(); + + // Immediate socket push (before DB write completes) + this.clients.event.emit( + 'outer.gui.notif.message', + { + user_id_list: userIds, + response: { uid, notification }, + }, + {}, + ); + + // Async DB inserts — one row per user. + const writePromise = (async () => { + for (const userId of userIds) { + try { + await this.stores.notification.create({ + userId, + value: notification, + }); + } catch (err) { + console.warn( + `[notification] persist failed for user ${userId}`, + err, + ); + } + } + })(); + this.#pendingWrites.set(uid, writePromise); + writePromise.finally(() => this.#pendingWrites.delete(uid)); + + // Fire persisted event after all writes complete + writePromise + .then(() => { + this.clients.event.emit( + 'outer.gui.notif.persisted', + { + user_id_list: userIds, + response: { uid }, + }, + {}, + ); + }) + .catch(() => { + /* already logged per-user above */ + }); + + return uid; + } + + /** + * Mark a notification as acknowledged (user dismissed it) and push + * the ack event to sockets so other tabs update. + */ + async markAcknowledged(uid: string, userId: number): Promise { + await this.stores.notification.markAcknowledged(uid, userId); + this.clients.event.emit( + 'outer.gui.notif.ack', + { + user_id_list: [userId], + response: { uid }, + }, + {}, + ); + } + + /** + * Mark a notification as shown (user saw it) and push the ack event. + */ + async markShown(uid: string, userId: number): Promise { + await this.stores.notification.markShown(uid, userId); + this.clients.event.emit( + 'outer.gui.notif.ack', + { + user_id_list: [userId], + response: { uid }, + }, + {}, + ); + } + + // ── Internals ─────────────────────────────────────────────────── + + async #sendUnreads(userId: number): Promise { + // Fetch all unseen + unacknowledged notifications + const rows = await this.stores.notification.listByUserId(userId, { + filter: 'unseen', + limit: 200, + }); + if (rows.length === 0) return; + + // Mark them shown now that we're delivering them + for (const row of rows) { + if (row.uid) { + await this.stores.notification + .markShown(row.uid, userId) + .catch(() => {}); + } + } + + const unreads = rows.map((r: Record) => ({ + uid: r.uid, + notification: r.value, + })); + + this.clients.event.emit( + 'outer.gui.notif.unreads', + { + user_id_list: [userId], + response: { unreads }, + }, + {}, + ); + } + + async #markShownAfterWrite(uid: string, userId: number): Promise { + // Wait for the pending write to finish before trying to mark shown + const pending = this.#pendingWrites.get(uid); + if (pending) await pending.catch(() => {}); + await this.stores.notification.markShown(uid, userId).catch(() => {}); + } +} diff --git a/src/backend/services/permission/PermissionService.test.ts b/src/backend/services/permission/PermissionService.test.ts new file mode 100644 index 000000000..b260e5b43 --- /dev/null +++ b/src/backend/services/permission/PermissionService.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from 'vitest'; +import type { Actor } from '../../core/actor.js'; +import { PermissionService } from './PermissionService.js'; + +function createPermissionService(): PermissionService { + const permissionStore = { + getMultiCheckCache: async () => new Map(), + setMultiCheckCache: async () => undefined, + }; + const [config, clients, stores, services] = [ + {}, + {}, + { permission: permissionStore }, + {}, + ] as ConstructorParameters; + return new PermissionService(config, clients, stores, services); +} + +describe('PermissionService.checkMany', () => { + it('evaluates every uncached permission independently', async () => { + const service = createPermissionService(); + const actor: Actor = { + user: { + uuid: 'user-1', + id: 1, + username: 'user', + }, + }; + const checked: string[] = []; + service.check = async (_actor, permissionOptions) => { + const permission = String(permissionOptions); + checked.push(permission); + return permission === 'app:uid#a:access' || + permission === 'app:uid#b:access'; + }; + + const result = await service.checkMany(actor, [ + 'app:uid#a:access', + 'app:uid#b:access', + 'app:uid#c:access', + ]); + + expect(result).toEqual( + new Map([ + ['app:uid#a:access', true], + ['app:uid#b:access', true], + ['app:uid#c:access', false], + ]), + ); + expect(checked).toEqual([ + 'app:uid#a:access', + 'app:uid#b:access', + 'app:uid#c:access', + ]); + }); +}); diff --git a/src/backend/services/permission/PermissionService.ts b/src/backend/services/permission/PermissionService.ts new file mode 100644 index 000000000..d69ab6aa2 --- /dev/null +++ b/src/backend/services/permission/PermissionService.ts @@ -0,0 +1,1264 @@ +import { PuterService } from '../types'; +import type { Actor, ActorUser } from '../../core/actor'; +import { actorUid, isSystemActor, userRelatedActor } from '../../core/actor'; +import { Context } from '../../core/context'; +import { + PermissionUtil, + readingHasTerminal, + type ReadingNode, + type PermissionRewriter, + type PermissionImplicator, + type PermissionExploder, +} from './permissionUtil'; +import { + MANAGE_PERM_PREFIX, + PERMISSION_SCAN_CACHE_TTL_SECONDS, +} from './consts'; + +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore — hardcoded-permissions.js is plain JS +import { + default_implicit_user_app_permissions, + implicit_user_app_permissions, + hardcoded_user_group_permissions, +} from '../../data/hardcoded-permissions.js'; + +// ── Types ──────────────────────────────────────────────────────────── + +export interface ScanOptions { + noCache?: boolean; +} + +export interface ScanState { + antiCycleActors: Actor[]; +} + +export interface GrantMeta { + reason?: string; +} + +/** + * PermissionService owns the *semantics* side of permissions: + * - The rule registries (rewriters, implicators, exploders) + * - The `scan()` algorithm that traverses all the ways an actor might hold a permission + * - grant/revoke orchestration (rewrite → canManage → store writes → cache invalidation) + * + * All persistence is delegated to PermissionStore. + */ +export class PermissionService extends PuterService { + private readonly rewriters: PermissionRewriter[] = []; + private readonly implicators: PermissionImplicator[] = []; + private readonly exploders: PermissionExploder[] = []; + /** + * System-issued grants registered at runtime by other services. Keyed by + * group UID, then by permission string. Merged with the imported + * `hardcoded_user_group_permissions.system` map during the hc-user-group + * scan. + */ + private readonly systemGrantsByGroupUid: Record< + string, + Record + > = {}; + + // ── Extension hooks ────────────────────────────────────────────── + // + // Other services contribute domain semantics via these. A controller or + // driver *could* register too, but typically the owning service for a + // permission namespace (fs, app, site, ...) is the right place. + + registerRewriter(rewriter: PermissionRewriter): void { + this.rewriters.push(rewriter); + } + + registerImplicator(implicator: PermissionImplicator): void { + this.implicators.push(implicator); + } + + registerExploder(exploder: PermissionExploder): void { + this.exploders.push(exploder); + } + + /** + * Grant a permission (as issued by `system`) to everyone — members of + * both the default user group and the default temp group. + * + * Call from an owning service's `onServerStart` (or later). + */ + registerSystemGrantForEveryone( + permission: string, + data: unknown = {}, + ): void { + const userGroup = this.config.default_user_group; + const tempGroup = this.config.default_temp_group; + if (userGroup) this.#addSystemGrant(userGroup, permission, data); + if (tempGroup) this.#addSystemGrant(tempGroup, permission, data); + } + + /** + * Grant a permission (as issued by `system`) to non-temp users only — + * members of the default user group, but not the default temp group. + */ + registerSystemGrantForUsers(permission: string, data: unknown = {}): void { + const userGroup = this.config.default_user_group; + if (userGroup) this.#addSystemGrant(userGroup, permission, data); + } + + #addSystemGrant(groupUid: string, permission: string, data: unknown): void { + if (!this.systemGrantsByGroupUid[groupUid]) { + this.systemGrantsByGroupUid[groupUid] = {}; + } + this.systemGrantsByGroupUid[groupUid][permission] = data; + } + + // ── Rewrite / explode (pure-ish helpers) ──────────────────────── + + async rewritePermission(permission: string): Promise { + for (const rewriter of this.rewriters) { + if (!rewriter.matches(permission)) continue; + permission = await rewriter.rewrite(permission); + } + return permission; + } + + /** Return the given permission plus all parents and their exploder expansions. */ + async getHigherPermissions(permission: string): Promise { + const higher = new Set(); + higher.add(permission); + for (const parent of this.getParentPermissions(permission)) { + higher.add(parent); + for (const exploder of this.exploders) { + if (!exploder.matches(parent)) continue; + const more = await exploder.explode({ permission: parent }); + for (const p of more) higher.add(p); + } + } + return [...higher]; + } + + getParentPermissions(permission: string): string[] { + // Keep components escaped — we match against stored permission strings verbatim. + const parts = permission.split(':'); + const parents: string[] = []; + for (let i = 0; i < parts.length; i++) { + parents.push(parts.slice(0, i + 1).join(':')); + } + parents.reverse(); + return parents; + } + + // ── Public check / scan API ────────────────────────────────────── + + async check( + actor: Actor, + permissionOptions: string | string[], + scanOptions?: ScanOptions, + ): Promise { + const reading = await this.scan( + actor, + permissionOptions, + undefined, + scanOptions, + ); + const options = PermissionUtil.readingToOptions(reading); + return options.length > 0; + } + + /** + * Batch sibling of `check`. Returns a `Map` + * answering "does `actor` hold each of these permissions?" with one + * Redis MGET for cached decisions and per-permission evaluation for + * misses. + * + * `scan(actor, string[])` is intentionally an OR-style API: callers ask + * "does any option match?". That makes it unsafe to infer independent + * booleans for every requested permission from one combined scan, since + * some scanners are allowed to stop once one option is proven. Misses use + * the single-permission `check()` path to preserve exact semantics. + */ + async checkMany( + actor: Actor, + permissions: string[], + ): Promise> { + const out = new Map(); + if (!permissions || permissions.length === 0) return out; + + const dedup = Array.from(new Set(permissions.filter(Boolean))); + if (dedup.length === 0) return out; + + // System actors are universal — keep parity with the + // `grant_if_system` short-circuit inside `scan`. + if (isSystemActor(actor)) { + for (const p of dedup) out.set(p, true); + return out; + } + + // ── Cache pass: one pipelined MGET ── + const aUid = actorUid(actor); + const cached = await this.stores.permission.getMultiCheckCache( + aUid, + dedup, + ); + const missing: string[] = []; + for (const p of dedup) { + if (cached.has(p)) { + out.set(p, cached.get(p)!); + } else { + missing.push(p); + } + } + if (missing.length === 0) return out; + + const checked = await Promise.all( + missing.map(async (permission) => { + try { + return { + permission, + granted: await this.check(actor, permission, { + noCache: true, + }), + }; + } catch { + return { permission, granted: false }; + } + }), + ); + const writeBack: Array<{ permission: string; granted: boolean }> = []; + for (const { permission, granted } of checked) { + out.set(permission, granted); + writeBack.push({ permission, granted }); + } + + // Backfill cache (best-effort, fire-and-forget would also be + // fine — keeping it awaited so callers see deterministic state + // in tests). + await this.stores.permission.setMultiCheckCache(aUid, writeBack); + + return out; + } + + async canManagePermission( + actor: Actor, + permission: string, + ): Promise { + const managePerm = PermissionUtil.join( + MANAGE_PERM_PREFIX, + ...PermissionUtil.split(permission), + ); + return await this.check(actor, managePerm); + } + + /** + * Scan all paths by which `actor` might hold any of the given permission + * options. Returns a tree-shaped "reading". Use + * `PermissionUtil.readingToOptions()` to flatten to a yes/no answer. + */ + async scan( + actor: Actor, + permissionOptions: string | string[], + state?: ScanState, + scanOptions: ScanOptions = {}, + ): Promise { + let options = Array.isArray(permissionOptions) + ? [...permissionOptions] + : [permissionOptions]; + const reading: ReadingNode[] = []; + const workingState: ScanState = state ?? { antiCycleActors: [actor] }; + + // ── Redis scan cache ── + const cacheKey = this.stores.permission.buildScanCacheKey( + actorUid(actor), + options, + ); + if (!scanOptions.noCache) { + const cached = await this.stores.permission.getScanCache(cacheKey); + if (cached) return cached as ReadingNode[]; + } + + const startTs = Date.now(); + + // ── grant_if_system short-circuit ── + if (isSystemActor(actor)) { + reading.push({ + $: 'option', + key: 'sys', + permission: options[0], + source: 'implied', + by: 'system', + data: {}, + }); + reading.push({ $: 'time', value: Date.now() - startTs }); + await this.#maybeCacheScan(cacheKey, reading); + return reading; + } + + // ── rewrite ── + for (let i = 0; i < options.length; i++) { + const old = options[i]; + const rewritten = await this.rewritePermission(old); + if (rewritten === old) continue; + options[i] = rewritten; + reading.push({ $: 'rewrite', from: old, to: rewritten }); + } + + // ── explode (parents + exploders) ── + const exploded: string[][] = []; + for (let i = 0; i < options.length; i++) { + const perm = options[i]; + const higher = await this.getHigherPermissions(perm); + exploded[i] = higher; + if (higher.length > 1) { + reading.push({ $: 'explode', from: perm, to: higher }); + } + } + options = exploded.flat(); + + // ── shortcut implicators ── + let shortCircuit = false; + for (const permission of options) { + for (const implicator of this.implicators) { + if (!implicator.shortcut) continue; + if (!implicator.matches(permission)) continue; + const implied = await implicator.check({ actor, permission }); + if (!implied) continue; + reading.push({ + $: 'option', + permission, + source: 'implied', + by: implicator.id, + data: implied, + ...(actor.user?.username + ? { holder_username: actor.user.username } + : {}), + }); + shortCircuit = true; + break; + } + if (shortCircuit) break; + } + + if (!shortCircuit) { + // ── scanners (formerly PERMISSION_SCANNERS) ── + // Run in parallel — matches v1's `Promise.all(ps)` in the + // scan-permission Sequence. Each scanner has a cheap actor-shape + // guard at the top (e.g. `if (!actor.app) return` for app-only + // ones) so the ones that don't apply to this actor fall out + // immediately. Scanners only push into `reading`; they don't + // read each other's writes, so there are no ordering hazards. + await Promise.all([ + this.#scanNonShortcutImplicators(actor, options, reading), + this.#scanAccessToken(actor, options, reading), + this.#scanUserUser(actor, options, reading, workingState), + this.#scanHcUserGroupUser(actor, options, reading), + this.#scanUserGroup(actor, options, reading), + this.#scanUserAppImplied(actor, options, reading), + this.#scanUserApp(actor, options, reading), + this.#scanDevApp(actor, options, reading), + ]); + } + + reading.push({ $: 'time', value: Date.now() - startTs }); + await this.#maybeCacheScan(cacheKey, reading); + return reading; + } + + async #maybeCacheScan( + cacheKey: string, + reading: ReadingNode[], + ): Promise { + try { + await this.stores.permission.setScanCache( + cacheKey, + reading, + PERMISSION_SCAN_CACHE_TTL_SECONDS, + ); + } catch { + // cache write failures should never block a permission decision + } + } + + // ── Scanners (inlined, no Sequence) ────────────────────────────── + + async #scanNonShortcutImplicators( + actor: Actor, + options: string[], + reading: ReadingNode[], + ): Promise { + for (const permission of options) { + for (const implicator of this.implicators) { + if (implicator.shortcut) continue; + if (!implicator.matches(permission)) continue; + const implied = await implicator.check({ actor, permission }); + if (!implied) continue; + reading.push({ + $: 'option', + permission, + source: 'implied', + by: implicator.id, + data: implied, + ...(actor.user?.username + ? { holder_username: actor.user.username } + : {}), + }); + } + } + } + + async #scanAccessToken( + actor: Actor, + options: string[], + reading: ReadingNode[], + ): Promise { + if (!actor.accessToken) return; + const issuerActor = actor.accessToken.issuer; + for (const permission of options) { + const hasTokenPerm = + await this.stores.permission.hasAccessTokenPerm( + actor.accessToken.uid, + permission, + ); + if (!hasTokenPerm) continue; + const issuerReading = await this.scan(issuerActor, permission); + reading.push({ + $: 'path', + via: 'access-token', + has_terminal: readingHasTerminal(issuerReading), + permission, + reading: issuerReading, + }); + } + } + + async #scanUserUser( + actor: Actor, + options: string[], + reading: ReadingNode[], + state: ScanState, + ): Promise { + if (actor.app || actor.accessToken) return; + const subReadings = await this.validateUserPerms({ + actor, + permissions: options, + state, + }); + reading.push(...subReadings); + } + + /** + * Resolve permissions that a persistent group's members inherit from an + * issuer (typically `system`) via the hardcoded map in + * `hardcoded-permissions.js`, merged with any runtime grants registered + * through `registerSystemGrantForEveryone` / `registerSystemGrantForUsers`. + */ + async #scanHcUserGroupUser( + actor: Actor, + options: string[], + reading: ReadingNode[], + ): Promise { + if (actor.app || actor.accessToken) return; + if (!actor.user?.id) return; + + const memberGroups = await this.stores.group.listGroupsWithMember( + actor.user.id, + ); + if (memberGroups.length === 0) return; + + const groupByUid: Record = {}; + for (const g of memberGroups) { + groupByUid[g.uid] = { id: g.id, uid: g.uid }; + } + + // Compose the effective issuer → group → permission → data map by + // merging the imported hardcoded data with runtime-registered system + // grants. Runtime grants are always attributed to the `system` issuer. + const hcMap = hardcoded_user_group_permissions as Record< + string, + Record> + >; + const hasRuntimeGrants = + Object.keys(this.systemGrantsByGroupUid).length > 0; + const byIssuer: Record< + string, + Record> + > = hasRuntimeGrants + ? { ...hcMap, system: { ...(hcMap.system ?? {}) } } + : hcMap; + if (hasRuntimeGrants) { + for (const [gUid, perms] of Object.entries( + this.systemGrantsByGroupUid, + )) { + byIssuer.system[gUid] = { + ...(byIssuer.system[gUid] ?? {}), + ...perms, + }; + } + } + + for (const issuerUsername of Object.keys(byIssuer)) { + const issuerUser = + await this.stores.user.getByUsername(issuerUsername); + if (!issuerUser) continue; + const issuerActor = this.#userToActor(issuerUser); + const issuerGroups = byIssuer[issuerUsername]; + + for (const groupUid of Object.keys(issuerGroups)) { + if (!groupByUid[groupUid]) continue; + const issuerGroupPerms = issuerGroups[groupUid]; + + for (const permission of options) { + if ( + !Object.prototype.hasOwnProperty.call( + issuerGroupPerms, + permission, + ) + ) + continue; + const issuerReading = await this.scan( + issuerActor, + permission, + ); + reading.push({ + $: 'path', + via: 'hc-user-group', + has_terminal: readingHasTerminal(issuerReading), + permission, + data: issuerGroupPerms[permission], + holder_username: actor.user.username, + issuer_username: issuerUsername, + reading: issuerReading, + group_id: groupByUid[groupUid].id, + }); + } + } + } + } + + async #scanUserGroup( + actor: Actor, + options: string[], + reading: ReadingNode[], + ): Promise { + if (actor.app || actor.accessToken) return; + if (!actor.user?.id) return; + + const rows = await this.stores.permission.readUserGroupPerms( + actor.user.id, + options, + ); + for (const row of rows) { + const issuerUser = await this.stores.user.getById(row.user_id); + if (!issuerUser) continue; + const issuerActor = this.#userToActor(issuerUser); + const issuerReading = await this.scan(issuerActor, row.permission); + reading.push({ + $: 'path', + via: 'user-group', + has_terminal: readingHasTerminal(issuerReading), + permission: row.permission, + data: row.extra, + holder_username: actor.user?.username, + issuer_username: issuerUser.username, + reading: issuerReading, + group_id: row.group_id, + }); + } + } + + async #scanUserAppImplied( + actor: Actor, + options: string[], + reading: ReadingNode[], + ): Promise { + if (!actor.app) return; + const issuerActor = userRelatedActor(actor); + const issuerReading = await this.scan(issuerActor, options); + const hasTerminal = readingHasTerminal(issuerReading); + const appUid = actor.app.uid; + + for (const permission of options) { + const implied = ( + default_implicit_user_app_permissions as Record + )[permission]; + if (implied) { + reading.push({ + $: 'path', + permission, + has_terminal: hasTerminal, + source: 'user-app-implied', + by: 'user-app-hc-1', + data: implied, + issuer_username: actor.user?.username, + reading: issuerReading, + }); + } + + // per-app hardcoded overrides + const hits: Record = {}; + for (const bucket of implicit_user_app_permissions as Array<{ + apps: string[]; + permissions: Record; + }>) { + if (bucket.apps.includes(appUid)) { + hits[permission] = bucket.permissions[permission]; + } + } + if (hits[permission]) { + reading.push({ + $: 'path', + permission, + has_terminal: hasTerminal, + source: 'user-app-implied', + by: 'user-app-hc-2', + data: hits[permission], + issuer_username: actor.user?.username, + reading: issuerReading, + }); + } + } + } + + async #scanUserApp( + actor: Actor, + options: string[], + reading: ReadingNode[], + ): Promise { + if (!actor.app || !actor.user?.id || !actor.app.id) return; + const rows = await this.stores.permission.readUserAppPerms( + actor.user.id, + actor.app.id, + options, + ); + const row = rows[0]; + if (!row) return; + + const issuerActor = userRelatedActor(actor); + const issuerReading = await this.scan(issuerActor, row.permission); + reading.push({ + $: 'path', + via: 'user-app', + permission: row.permission, + has_terminal: readingHasTerminal(issuerReading), + data: row.extra, + issuer_username: actor.user?.username, + reading: issuerReading, + }); + } + + async #scanDevApp( + actor: Actor, + options: string[], + reading: ReadingNode[], + ): Promise { + if (!actor.app || !actor.app.id) return; + const rows = await this.stores.permission.readDevAppPerms( + actor.app.id, + options, + ); + const row = rows[0]; + if (!row) return; + + const issuerUser = await this.stores.user.getById(row.user_id); + if (!issuerUser) return; + const issuerActor = this.#userToActor(issuerUser); + const issuerReading = await this.scan(issuerActor, row.permission); + reading.push({ + $: 'path', + via: 'dev-app', + permission: row.permission, + has_terminal: readingHasTerminal(issuerReading), + data: row.extra, + issuer_username: actor.user?.username, + reading: issuerReading, + }); + } + + // ── validateUserPerms (flat + linked reads) ────────────────────── + + /** + * Resolves user-to-user permissions for an actor across the given + * permission strings. Prefers the "flat" KV view when present; otherwise + * falls back to a SQL traversal of `user_to_user_permissions` and + * warms the flat KV cache as a side-effect. + */ + async validateUserPerms({ + actor, + permissions, + state, + }: { + actor: Actor; + permissions: string[]; + state?: ScanState; + }): Promise { + if (!actor.user?.id) return []; + + const flatPromise = this.#flatValidateUserPerms(actor, permissions); + const linkedPromise = this.#linkedValidateUserPerms( + actor, + permissions, + state ?? { antiCycleActors: [actor] }, + ); + + const flatReading = await flatPromise; + if (flatReading.length > 0) { + return flatReading[0].deleted ? [] : flatReading; + } + + const linkedReading = await linkedPromise; + const flatOptions = PermissionUtil.readingToOptions(linkedReading); + + // Warm flat KV cache for future hits (fire-and-forget, don't block result) + for (const opt of flatOptions) { + if (!opt.permission) continue; + const data = Array.isArray(opt.data) ? opt.data : [opt.data]; + const issuerUserId = (data[0] as { issuer_user_id?: number }) + ?.issuer_user_id; + this.stores.permission + .setFlatUserPerm(actor.user.id, opt.permission, { + permission: opt.permission, + issuer_user_id: issuerUserId, + data, + }) + .catch(() => { + /* swallow — this is a cache warm */ + }); + } + + return flatReading; + } + + async #flatValidateUserPerms( + actor: Actor, + permissions: string[], + ): Promise { + if (!actor.user?.id) return []; + const values = await this.stores.permission.getFlatUserPerms( + actor.user.id, + permissions, + ); + + let anyDeleted = false; + for (const v of values) { + if (v.deleted) { + anyDeleted = true; + continue; + } + const { permission, issuer_user_id, ...extra } = v; + if (!permission) continue; + const issuer = issuer_user_id + ? await this.stores.user.getById(issuer_user_id) + : null; + return [ + { + $: 'option', + via: 'user', + has_terminal: true, + permission, + data: extra, + holder_username: actor.user.username, + issuer_username: issuer?.username, + issuer_user_id: issuer?.uuid, + reading: [], + }, + ]; + } + return anyDeleted ? [{ $: 'option', deleted: true }] : []; + } + + async #linkedValidateUserPerms( + actor: Actor, + permissions: string[], + state: ScanState, + ): Promise { + if (!actor.user?.id) return []; + const rows = await this.stores.permission.readLinkedUserUserPerms( + actor.user.id, + permissions, + ); + + const out: ReadingNode[] = []; + for (const row of rows) { + const issuerUser = await this.stores.user.getById( + row.issuer_user_id, + ); + if (!issuerUser) continue; + const issuerActor = this.#userToActor(issuerUser); + + // anti-cycle + let skip = false; + for (const seen of state.antiCycleActors) { + if (seen.user?.id === issuerActor.user.id) { + skip = true; + break; + } + } + if (skip) continue; + + const issuerReading = await this.scan(issuerActor, row.permission, { + antiCycleActors: [...state.antiCycleActors, issuerActor], + }); + + out.push({ + $: 'path', + via: 'user', + has_terminal: readingHasTerminal(issuerReading), + permission: row.permission, + data: row.extra, + holder_username: actor.user.username, + issuer_username: issuerUser.username, + issuer_user_id: issuerUser.uuid, + reading: issuerReading, + }); + } + return out; + } + + // ── Grant / revoke orchestration ───────────────────────────────── + + async grantUserUserPermission( + actor: Actor, + username: string, + permission: string, + extra: Record = {}, + meta: GrantMeta = {}, + ): Promise { + permission = await this.rewritePermission(permission); + const user = await this.stores.user.getByUsername(username); + if (!user) throw new Error(`user_does_not_exist: ${username}`); + if (user.id === actor.user?.id) + throw new Error('cannot grant permissions to yourself'); + + if (!(await this.canManagePermission(actor, permission))) { + throw new Error(`permission_denied: ${permission}`); + } + if (!actor.user?.id) + throw new Error('grantUserUserPermission: actor lacks user.id'); + const issuerId = actor.user.id; + + // Flat upsert (awaited so callers see immediate effect) + await this.stores.permission.setFlatUserPerm(user.id, permission, { + ...extra, + issuer_user_id: issuerId, + permission, + deleted: false, + }); + + // Linked upsert + audit fire-and-forget. + this.stores.permission + .upsertUserUserPerm(user.id, issuerId, permission, extra) + .catch(() => {}); + this.stores.permission + .auditUserUserPerm({ + holder_user_id: user.id, + issuer_user_id: issuerId, + permission, + action: 'grant', + reason: meta.reason ?? 'granted via PermissionService', + }) + .catch(() => {}); + } + + async revokeUserUserPermission( + actor: Actor, + username: string, + permission: string, + meta: GrantMeta = {}, + ): Promise { + permission = await this.rewritePermission(permission); + const user = await this.stores.user.getByUsername(username); + if (!user) throw new Error(`user_does_not_exist: ${username}`); + + if (!(await this.canManagePermission(actor, permission))) { + throw new Error(`permission_denied: ${permission}`); + } + if (!actor.user?.id) + throw new Error('revokeUserUserPermission: actor lacks user.id'); + const issuerId = actor.user.id; + + await this.stores.permission.delFlatUserPerm(user.id, permission); + this.stores.permission + .deleteUserUserPermByHolder(user.id, permission) + .catch(() => {}); + this.stores.permission + .auditUserUserPerm({ + holder_user_id: user.id, + issuer_user_id: issuerId, + permission, + action: 'revoke', + reason: meta.reason ?? 'revoked via PermissionService', + }) + .catch(() => {}); + } + + async grantUserAppPermission( + actor: Actor, + appIdentifier: string, + permission: string, + extra: Record = {}, + meta: GrantMeta = {}, + ): Promise { + // Flag the context so the app-root-dir rewriter knows it's safe to + // resolve the pseudo-permission to a real `fs::`. During + // scans (ACL.check) the rewriter returns PERMISSION_FOR_NOTHING_IN_PARTICULAR + // so `scan(actor, 'app-root-dir:…')` never accidentally matches + // through the fs path. + Context.set('is_grant_user_app_permission', true); + try { + permission = await this.rewritePermission(permission); + } finally { + Context.set('is_grant_user_app_permission', false); + } + const app = await this.stores.app.resolveApp(appIdentifier); + if (!app) throw new Error(`entity_not_found: app:${appIdentifier}`); + if (!actor.user?.id) + throw new Error('grantUserAppPermission: actor lacks user.id'); + + // Skip redundant upserts (saves db roundtrip + cache invalidation) + if ( + await this.stores.permission.hasUserAppPerm( + actor.user.id, + app.id, + permission, + ) + ) + return; + + await this.stores.permission.upsertUserAppPerm( + actor.user.id, + app.id, + permission, + extra, + ); + this.stores.permission + .auditUserAppPerm({ + user_id: actor.user.id, + app_id: app.id, + permission, + action: 'grant', + reason: meta.reason ?? 'granted via PermissionService', + }) + .catch(() => {}); + + // Invalidate app-under-user scan cache so the grant takes effect immediately + await this.invalidatePermissionScanCacheForAppUnderUser( + actor.user.uuid, + app.uid, + permission, + ); + } + + async revokeUserAppPermission( + actor: Actor, + appIdentifier: string, + permission: string, + meta: GrantMeta = {}, + ): Promise { + permission = await this.rewritePermission(permission); + if (actor.app) throw new Error('actor must be a user'); + const app = await this.stores.app.resolveApp(appIdentifier); + if (!app) throw new Error(`entity_not_found: app${appIdentifier}`); + if (!actor.user?.id) + throw new Error('revokeUserAppPermission: actor lacks user.id'); + + await this.stores.permission.deleteUserAppPerm( + actor.user.id, + app.id, + permission, + ); + this.stores.permission + .auditUserAppPerm({ + user_id: actor.user.id, + app_id: app.id, + permission, + action: 'revoke', + reason: meta.reason ?? 'revoked via PermissionService', + }) + .catch(() => {}); + } + + async revokeUserAppAll( + actor: Actor, + appIdentifier: string, + meta: GrantMeta = {}, + ): Promise { + if (actor.app) throw new Error('actor must be a user'); + const app = await this.stores.app.resolveApp(appIdentifier); + if (!app) throw new Error(`entity_not_found: app${appIdentifier}`); + if (!actor.user?.id) + throw new Error('revokeUserAppAll: actor lacks user.id'); + + await this.stores.permission.deleteUserAppAll(actor.user.id, app.id); + this.stores.permission + .auditUserAppPerm({ + user_id: actor.user.id, + app_id: app.id, + permission: '*', + action: 'revoke', + reason: meta.reason ?? 'revoked all via PermissionService', + }) + .catch(() => {}); + } + + async grantDevAppPermission( + actor: Actor, + appIdentifier: string, + permission: string, + extra: Record = {}, + meta: GrantMeta = {}, + ): Promise { + permission = await this.rewritePermission(permission); + const app = await this.stores.app.resolveApp(appIdentifier); + if (!app) throw new Error(`entity_not_found: app:${appIdentifier}`); + if (!(await this.canManagePermission(actor, permission))) + throw new Error(`permission_denied: ${permission}`); + if (!actor.user?.id) + throw new Error('grantDevAppPermission: actor lacks user.id'); + + await this.stores.permission.upsertDevAppPerm( + actor.user.id, + app.id, + permission, + extra, + ); + this.stores.permission + .auditDevAppPerm({ + user_id: actor.user.id, + app_id: app.id, + permission, + action: 'grant', + reason: meta.reason ?? 'granted via PermissionService', + }) + .catch(() => {}); + } + + async revokeDevAppPermission( + actor: Actor, + appIdentifier: string, + permission: string, + meta: GrantMeta = {}, + ): Promise { + permission = await this.rewritePermission(permission); + if (actor.app) throw new Error('actor must be a user'); + const app = await this.stores.app.resolveApp(appIdentifier); + if (!app) throw new Error(`entity_not_found: app${appIdentifier}`); + if (!actor.user?.id) + throw new Error('revokeDevAppPermission: actor lacks user.id'); + + await this.stores.permission.deleteDevAppPerm( + actor.user.id, + app.id, + permission, + ); + this.stores.permission + .auditDevAppPerm({ + user_id: actor.user.id, + app_id: app.id, + permission, + action: 'revoke', + reason: meta.reason ?? 'revoked via PermissionService', + }) + .catch(() => {}); + } + + async revokeDevAppAll( + actor: Actor, + appIdentifier: string, + meta: GrantMeta = {}, + ): Promise { + if (actor.app) throw new Error('actor must be a user'); + const app = await this.stores.app.resolveApp(appIdentifier); + if (!app) throw new Error(`entity_not_found: app${appIdentifier}`); + if (!actor.user?.id) + throw new Error('revokeDevAppAll: actor lacks user.id'); + + await this.stores.permission.deleteDevAppAll(actor.user.id, app.id); + this.stores.permission + .auditDevAppPerm({ + user_id: actor.user.id, + app_id: app.id, + permission: '*', + action: 'revoke', + reason: meta.reason ?? 'revoked all via PermissionService', + }) + .catch(() => {}); + } + + async grantUserGroupPermission( + actor: Actor, + group: { id: number; uid: string }, + permission: string, + extra: Record = {}, + meta: GrantMeta = {}, + ): Promise { + permission = await this.rewritePermission(permission); + if (!(await this.canManagePermission(actor, permission))) + throw new Error(`permission_denied: ${permission}`); + if (!actor.user?.id) + throw new Error('grantUserGroupPermission: actor lacks user.id'); + + await this.stores.permission.upsertUserGroupPerm( + actor.user.id, + group.id, + permission, + extra, + ); + this.stores.permission + .auditUserGroupPerm({ + user_id: actor.user.id, + group_id: group.id, + permission, + action: 'grant', + reason: meta.reason ?? 'granted via PermissionService', + }) + .catch(() => {}); + } + + async revokeUserGroupPermission( + actor: Actor, + group: { id: number; uid: string }, + permission: string, + meta: GrantMeta = {}, + ): Promise { + permission = await this.rewritePermission(permission); + if (!actor.user?.id) + throw new Error('revokeUserGroupPermission: actor lacks user.id'); + + await this.stores.permission.deleteUserGroupPerm( + actor.user.id, + group.id, + permission, + ); + this.stores.permission + .auditUserGroupPerm({ + user_id: actor.user.id, + group_id: group.id, + permission, + action: 'revoke', + reason: meta.reason ?? 'revoked via PermissionService', + }) + .catch(() => {}); + } + + // ── Issuer queries (share discovery et al) ─────────────────────── + + async listUserPermissionIssuers(user: { + id: number; + }): Promise> { + const ids = await this.stores.permission.listUserPermissionIssuerIds( + user.id, + ); + const usersById = await this.stores.user.getByIds(ids); + return ids.map((id) => { + const u = usersById.get(id); + return u + ? { + id: u.id, + uuid: u.uuid, + username: u.username, + email: u.email, + } + : null; + }); + } + + async queryIssuerPermissionsByPrefix( + issuer: { id: number }, + prefix: string, + ): Promise<{ + users: Array<{ user: UserRowSummary | null; permission: string }>; + apps: Array<{ + app: { id: number; uid: string; name?: string } | null; + permission: string; + }>; + }> { + const [userRows, appRows] = await Promise.all([ + this.stores.permission.queryIssuerUserPermsByPrefix( + issuer.id, + prefix, + ), + this.stores.permission.queryIssuerAppPermsByPrefix( + issuer.id, + prefix, + ), + ]); + const [usersById, appsById] = await Promise.all([ + this.stores.user.getByIds(userRows.map((r) => r.holder_user_id)), + this.stores.app.getByIds(appRows.map((r) => r.app_id)), + ]); + const users = userRows.map((r) => { + const u = usersById.get(r.holder_user_id); + return { + user: u + ? { + id: u.id, + uuid: u.uuid, + username: u.username, + email: u.email, + } + : null, + permission: r.permission, + }; + }); + const apps = appRows.map((r) => { + const a = appsById.get(r.app_id); + return { + app: a ? { id: a.id, uid: a.uid, name: a.name } : null, + permission: r.permission, + }; + }) as Array<{ + app: { id: number; uid: string; name?: string } | null; + permission: string; + }>; + return { users, apps }; + } + + async queryIssuerHolderPermissionsByPrefix( + issuer: Actor, + holder: Actor, + prefix: string, + ): Promise { + if (!issuer.user?.id || !holder.user?.id) return []; + return this.stores.permission.queryIssuerHolderPermsByPrefix( + issuer.user.id, + holder.user.id, + prefix, + ); + } + + // ── Cache invalidation ─────────────────────────────────────────── + + async invalidatePermissionScanCacheForAppUnderUser( + userUuid: string, + appUid: string, + permission: string, + ): Promise { + const actorUid = `app-under-user:${userUuid}:${appUid}`; + const cacheKey = this.stores.permission.buildScanCacheKey(actorUid, [ + permission, + ]); + await this.stores.permission.invalidateScanCache(cacheKey); + } + + // ── Internals ──────────────────────────────────────────────────── + + #userToActor(user: { + id: number; + uuid: string; + username: string; + email?: string | null; + }): Actor { + const actorUser: ActorUser = { + uuid: user.uuid, + id: user.id, + username: user.username, + email: user.email ?? null, + }; + return { user: actorUser }; + } +} + +// Minimal structural summary of a user row used in public return types. +interface UserRowSummary { + id: number; + uuid: string; + username: string; + email?: string | null; +} diff --git a/src/backend/services/permission/consts.ts b/src/backend/services/permission/consts.ts new file mode 100644 index 000000000..8d745095b --- /dev/null +++ b/src/backend/services/permission/consts.ts @@ -0,0 +1,12 @@ +export const MANAGE_PERM_PREFIX = 'manage'; +export const PERM_KEY_PREFIX = 'perm'; + +/** + * De-facto placeholder permission for permission rewrites that do not grant + * any access. + */ +export const PERMISSION_FOR_NOTHING_IN_PARTICULAR = + 'permission-for-nothing-in-particular'; + +/** TTL (seconds) for redis-cached permission scan readings. */ +export const PERMISSION_SCAN_CACHE_TTL_SECONDS = 20; diff --git a/src/backend/services/permission/permissionUtil.ts b/src/backend/services/permission/permissionUtil.ts new file mode 100644 index 000000000..13ba604c0 --- /dev/null +++ b/src/backend/services/permission/permissionUtil.ts @@ -0,0 +1,180 @@ +import type { Actor } from '../../core/actor'; +import { MANAGE_PERM_PREFIX } from './consts'; + +/** Shape of a single node in a permission scan "reading". */ +export interface ReadingNode { + $: 'option' | 'path' | 'rewrite' | 'explode' | 'time'; + permission?: string; + permissionOptions?: string[]; + via?: string; + source?: string; + by?: string; + key?: string; + has_terminal?: boolean; + data?: unknown; + holder_username?: string; + issuer_username?: string; + issuer_user_id?: string; + group_id?: number; + vgroup_id?: string; + reading?: ReadingNode[]; + from?: string; + to?: string | string[]; + value?: number; + deleted?: boolean; + [k: string]: unknown; +} + +/** Result of `readingToOptions`. */ +export interface ReadingOption extends ReadingNode { + path: Array<{ key?: string; holder?: string; data?: unknown }>; +} + +const unescape_permission_component = (component: string): string => { + let out = ''; + const ESCAPES: Record = { C: ':' }; + let escaping = false; + for (let i = 0; i < component.length; i++) { + const c = component[i]; + if (!escaping) { + if (c === '\\') escaping = true; + else out += c; + } else { + out += Object.prototype.hasOwnProperty.call(ESCAPES, c) + ? ESCAPES[c] + : c; + escaping = false; + } + } + return out; +}; + +const escape_permission_component = (component: string): string => { + let out = ''; + for (let i = 0; i < component.length; i++) { + const c = component[i]; + if (c === ':') { + out += '\\C'; + continue; + } + out += c; + } + return out; +}; + +/** + * Utility functions for handling permission strings: split/join/escape plus + * the `reading_to_options` tree flattener used by `check()` and consumers. + */ +export const PermissionUtil = { + unescape_permission_component, + escape_permission_component, + + split(permission: string): string[] { + return permission.split(':').map(unescape_permission_component); + }, + + join(...components: string[]): string { + return components.map(escape_permission_component).join(':'); + }, + + permission_scan_cache_prefix_for_app_under_user( + user_uuid: string, + app_uid: string, + ): string { + const actor_uid = `app-under-user:${user_uuid}:${app_uid}`; + return PermissionUtil.join( + 'permission-scan', + actor_uid, + 'options-list', + ); + }, + + readingToOptions( + reading: ReadingNode[], + _parameters: Record = {}, + options: ReadingOption[] = [], + extras: unknown[] = [], + path: Array<{ key?: string; holder?: string; data?: unknown }> = [], + ): ReadingOption[] { + const toPathItem = (finding: ReadingNode) => ({ + key: finding.key, + holder: finding.holder_username, + data: finding.data, + }); + for (const finding of reading) { + if (finding.$ === 'option') { + const nextPath = [toPathItem(finding), ...path]; + options.push({ + ...finding, + data: [...(finding.data ? [finding.data] : []), ...extras], + path: nextPath, + }); + } + if (finding.$ === 'path') { + if (finding.has_terminal === false) continue; + const newExtras = finding.data ? [finding.data, ...extras] : []; + const newPath = [toPathItem(finding), ...path]; + PermissionUtil.readingToOptions( + finding.reading ?? [], + _parameters, + options, + newExtras, + newPath, + ); + } + } + return options; + }, + + isManage(permission: string): boolean { + return permission.startsWith(`${MANAGE_PERM_PREFIX}:`); + }, +}; + +/** + * Check whether a reading includes any terminal node (an `option`, or a + * `path` that itself transitively terminates). + */ +export const readingHasTerminal = (reading: ReadingNode[]): boolean => { + for (const node of reading) { + if (node.has_terminal) return true; + if (node.$ === 'option') return true; + } + return false; +}; + +// ── Rules ──────────────────────────────────────────────────────────── +// +// Rewriters, Implicators, and Exploders are the extension points other +// services use to contribute domain semantics. These are plain objects — +// easy to construct from anywhere, easy to test. + +export interface PermissionRewriter { + id?: string; + matches: (permission: string) => boolean; + rewrite: (permission: string) => Promise | string; +} + +export interface ImplicatorCheckInput { + actor: Actor; + permission: string; + recurse?: (actor: Actor, permission: string) => Promise; +} + +export interface PermissionImplicator { + id?: string; + /** If true, the implicator's hit short-circuits the scan. */ + shortcut?: boolean; + matches: (permission: string) => boolean; + check: (input: ImplicatorCheckInput) => Promise | unknown; +} + +export interface PermissionExploder { + id?: string; + matches: (permission: string) => boolean; + explode: (input: { + actor?: Actor; + permission: string; + }) => Promise | string[]; +} diff --git a/src/backend/services/selfhosted/DefaultUserService.ts b/src/backend/services/selfhosted/DefaultUserService.ts new file mode 100644 index 000000000..974399555 --- /dev/null +++ b/src/backend/services/selfhosted/DefaultUserService.ts @@ -0,0 +1,111 @@ +import bcrypt from 'bcrypt'; +import crypto from 'node:crypto'; +import { v4 as uuidv4 } from 'uuid'; +import { PuterService } from '../types.js'; +import type { UserRow } from '../../stores/user/UserStore.js'; +import { generateDefaultFsentries } from '../../util/userProvisioning.js'; +import type { AppIconService } from '../appIcon/AppIconService.js'; + +const USERNAME = 'admin'; +const ADMIN_GROUP_UID = 'ca342a5e-b13d-4dee-9048-58b11a57cc55'; +const ADMIN_STORAGE_BYTES = 10 * 1024 * 1024 * 1024; + +/** + * Bootstraps the `admin` user on first boot for self-hosted deployments. + * + * If no admin exists, creates one with a random 8-char hex password, places + * them in the admin group, and stashes the plaintext under + * `metadata.tmp_password` so we can detect on later boots whether the + * operator has rotated it yet. + * + * Each boot where the current password hash still matches the stashed + * plaintext, the credentials are re-printed to stdout (CI scrapes this + * line to extract the default password). + */ +export class DefaultUserService extends PuterService { + override async onServerStart(): Promise { + let user = await this.stores.user.getByUsername(USERNAME); + let tmpPassword: string; + + if (!user) { + tmpPassword = crypto.randomBytes(4).toString('hex'); + user = await this.#createAdminUser(tmpPassword); + // AppIconService is registered before us, so its own onServerStart + // bailed on its first-boot bootstrap (admin didn't exist yet). + // Poke it here so the `/system/app_icons/` dir + subdomain exist + // by the time the first icon arrives. + await ( + this.services.appIcon as AppIconService + ).ensureIconsDirectory(); + } else { + const metadata = (user.metadata ?? {}) as Record; + const stashed = metadata.tmp_password; + if (typeof stashed !== 'string' || stashed === '') return; + tmpPassword = stashed; + } + + if (!user.password) return; + const isDefault = await bcrypt.compare( + tmpPassword, + String(user.password), + ); + if (!isDefault) return; + + this.#printCredentials(tmpPassword); + } + + async #createAdminUser(tmpPassword: string): Promise { + const passwordHash = await bcrypt.hash(tmpPassword, 8); + + const created = await this.stores.user.create({ + username: USERNAME, + uuid: uuidv4(), + password: passwordHash, + email: null, + free_storage: ADMIN_STORAGE_BYTES, + requires_email_confirmation: false, + }); + + await this.stores.user.updateMetadata(created.id, { + tmp_password: tmpPassword, + }); + + try { + await this.stores.group.addUsers(ADMIN_GROUP_UID, [USERNAME]); + } catch (e) { + console.warn( + '[default-user] failed to add admin to admin group', + e, + ); + } + + try { + await generateDefaultFsentries( + this.clients.db, + this.stores.user, + created, + ); + } catch (e) { + console.warn( + '[default-user] failed to provision admin home directory', + e, + ); + } + + return (await this.stores.user.getById(created.id)) ?? created; + } + + #printCredentials(tmpPassword: string): void { + console.log(`password for admin is: ${tmpPassword}`); + console.log( + '\n************************************************************', + ); + console.log('* Your default login credentials are:'); + console.log('* Username: admin'); + console.log(`* Password: ${tmpPassword}`); + console.log('* (change the password to remove this message)'); + console.log( + '************************************************************\n', + ); + } +} diff --git a/src/backend/services/socket/SocketService.ts b/src/backend/services/socket/SocketService.ts new file mode 100644 index 000000000..c450fab5b --- /dev/null +++ b/src/backend/services/socket/SocketService.ts @@ -0,0 +1,432 @@ +import type { Server as HttpServer } from 'node:http'; +import { createAdapter } from '@socket.io/redis-streams-adapter'; +import { Server as SocketIOServer, type Socket } from 'socket.io'; +import type { Actor } from '../../core/actor.js'; +import { isAppActor, isAccessTokenActor } from '../../core/actor.js'; +import { PuterService } from '../types.js'; +import type { AuthService } from '../auth/AuthService.js'; + +/** + * Socket push target. A `room` fans to every socket in that room; a + * `socket` targets one specific socket by id. Multiple specifiers may + * be passed as an array. + */ +export interface SocketSpecifier { + room?: string | number; + socket?: string; +} + +// ── Redis key format for cross-node FS-cache invalidation ────────── +// +// puter-js (browser) polls `GET /cache/last-change-timestamp` and purges +// its in-memory FS cache when the server's timestamp is ≥ ~2s ahead of +// the tab's local clock. We bump this key on every `outer.gui.item.*` +// mutation, so a write on node A invalidates puter-js caches in tabs +// connected to node B. +// +// 30-day TTL so dormant users' keys GC themselves. Active users keep +// rewriting the key, so the TTL never fires for them. +const LAST_CHANGE_KEY_PREFIX = 'fs:last-change:'; +const LAST_CHANGE_TTL_SECONDS = 60 * 60 * 24 * 30; + +// Bump the per-user `fs:last-change` Redis key only on item-mutation +// events — `cache.updated` and similar are themselves notifications +// ABOUT the timestamp, so re-bumping on them is wasted work. +const ITEM_MUTATION_PREFIX = 'outer.gui.item.'; + +interface OuterGuiPayload { + user_id_list?: Array; + response: unknown; +} + +interface UploadProgressPayload { + upload_tracker: { + total_: number; + progress_: number; + sub: (callback: (delta: number) => void) => void; + }; + meta?: Record; +} + +/** + * Extend the socket.io `Socket` with the actor attached by our auth + * middleware. Using the module-augmentation pattern keeps callers + * typed without casts. + */ +interface AuthenticatedSocket extends Socket { + actor?: Actor; +} + +/** + * socket.io wrapper with: + * + * 1. Auth middleware — reads `handshake.auth.auth_token`, validates it + * via `AuthService`, rejects anything other than plain user actors + * (no app-under-user, no access-token), and joins the socket to a + * per-user room keyed by `user.id`. + * + * 2. Event bus → socket fan-out — subscribes to the known set of + * `outer.gui.*` mutation events and pushes each to the affected + * users' rooms. Strips the `outer.gui.` prefix before emitting. + * + * 3. FS cache-invalidation timestamp — bumps a per-user Redis key on + * every mutation so puter-js running on a different node (or a + * different tab) can detect staleness on its next poll of + * `/cache/last-change-timestamp`. + * + * Cross-node fan-out comes free via `@socket.io/redis-streams-adapter`: + * `send()` on any node reaches every socket for that room cluster-wide. + */ +export class SocketService extends PuterService { + #io: SocketIOServer | null = null; + + // ── Lifecycle ─────────────────────────────────────────────────── + + /** + * Called by `PuterServer` after the http server is created but + * before it starts listening. Attaches socket.io, wires auth, + * subscribes to the event bus. Sync — no await on the caller side + * is required, but we accept a Promise return for symmetry. + */ + attachHttpServer(server: HttpServer): void { + // ioredis Cluster is compatible with the redis-streams adapter. + const adapter = createAdapter(this.clients.redis as unknown as never); + + // Restrict the upgrade-host to puter.com + api.puter.com (or + // whatever `config.domain` resolves to). Wildcard-DNS-served + // user sites at `*.puter.site` go to the same backend, but + // socket.io has no business answering there. CORS reflector + // stays wide — any *origin* may connect from those gated hosts. + const allowedHosts = this.#allowedSocketHosts(); + + this.#io = new SocketIOServer(server, { + cors: { + // Reflect whatever origin the client sent back. + // credentials:true means clients can send cookies. + origin: (origin, callback) => callback(null, origin ?? '*'), + credentials: true, + }, + allowRequest: (req, callback) => { + const rawHost = req.headers.host ?? ''; + const host = rawHost.split(':')[0].toLowerCase(); + if (allowedHosts.has(host)) { + callback(null, true); + return; + } + callback('socket.io: host not allowed', false); + }, + adapter, + }); + + this.#installAuthMiddleware(); + this.#installConnectionHandler(); + this.#subscribeEventBus(); + } + + /** + * Hostnames permitted to upgrade to a socket connection. Built from + * `config.domain` (e.g. `puter.com` → allows `puter.com` + + * `api.puter.com`). Subdomain user-sites and other wildcard-served + * hostnames are not in this set. + */ + #allowedSocketHosts(): Set { + const domain = (this.config.domain ?? '').toLowerCase().trim(); + if (!domain) return new Set(); + return new Set([domain, `api.${domain}`]); + } + + override onServerPrepareShutdown(): Promise { + // Close the io server so existing sockets disconnect cleanly + // before http's close() starts waiting for connections. + return new Promise((resolve) => { + if (!this.#io) return resolve(); + this.#io.close(() => resolve()); + }); + } + + // ── Public API (used by other services / controllers) ────────── + + /** + * Push an event to one or more specifiers. `room` targets every + * socket joined to that room (we use `user.id` as the room name), + * `socket` targets one specific socket by id. + */ + async send( + specifiers: SocketSpecifier | SocketSpecifier[], + key: string, + data: unknown, + ): Promise { + if (!this.#io) return; + const list = Array.isArray(specifiers) ? specifiers : [specifiers]; + for (const spec of list) { + if (spec.room !== undefined) { + this.#io.to(String(spec.room)).emit(key, data); + } else if (spec.socket) { + this.#io.to(spec.socket).emit(key, data); + } + } + } + + /** + * Check whether the specifier currently resolves to at least one + * live socket on *this* node. Note: doesn't check other cluster + * nodes — intended for best-effort local checks only. + */ + has(specifier: SocketSpecifier): boolean { + if (!this.#io) return false; + if (specifier.room !== undefined) { + const room = this.#io.sockets.adapter.rooms.get( + String(specifier.room), + ); + return !!room && room.size > 0; + } + if (specifier.socket) { + return this.#io.sockets.sockets.has(specifier.socket); + } + return false; + } + + /** True once `attachHttpServer` has wired up the io instance. */ + hasIO(): boolean { + return this.#io !== null; + } + + /** + * Read the last-change timestamp for a user from Redis. Returns 0 + * when unset. Called by `LegacyFSController`'s + * `/cache/last-change-timestamp` route. + */ + async getLastChangeTimestamp(userId: number | string): Promise { + try { + const raw = await this.clients.redis.get( + `${LAST_CHANGE_KEY_PREFIX}${userId}`, + ); + if (!raw) return 0; + const n = Number(raw); + return Number.isFinite(n) ? n : 0; + } catch { + return 0; + } + } + + // ── Auth + connection wiring ─────────────────────────────────── + + #installAuthMiddleware(): void { + if (!this.#io) return; + const authService = this.services.auth as AuthService | undefined; + if (!authService) { + console.warn( + '[socket] AuthService unavailable — sockets will reject all connections', + ); + } + + this.#io.use(async (socket: AuthenticatedSocket, next) => { + // socket.io's conventional location for handshake auth is + // `{ auth: { ... } }`, not the query string. puter-js uses + // `io(url, { auth: { auth_token } })`. + const handshakeAuth = socket.handshake.auth as + | Record + | undefined; + const tokenRaw = + typeof handshakeAuth?.auth_token === 'string' + ? handshakeAuth.auth_token + : undefined; + + if (!tokenRaw) { + next(new Error('socket auth token missing')); + return; + } + const token = tokenRaw.replace(/^Bearer\s+/i, '').trim(); + if (!token) { + next(new Error('socket auth token empty')); + return; + } + if (!authService) { + next(new Error('socket auth unavailable')); + return; + } + + try { + const actor = await authService.authenticateFromToken(token); + if (!actor || !actor.user) { + next(new Error('socket auth failed')); + return; + } + // Only user tokens accepted — no app-under-user, no access-token. + if (isAppActor(actor) || isAccessTokenActor(actor)) { + next(new Error('socket auth: only user tokens accepted')); + return; + } + + socket.actor = actor; + // user.id is numeric in the DB; stringify for room name + // so adapter lookups key on a stable type. + socket.join(String(actor.user.id)); + next(); + } catch (err) { + console.warn('[socket] auth error', err); + next( + err instanceof Error + ? err + : new Error('socket auth failed'), + ); + } + }); + } + + #installConnectionHandler(): void { + if (!this.#io) return; + + this.#io.on('connection', (socket: AuthenticatedSocket) => { + const actor = socket.actor; + if (!actor || !actor.user) return; + const userId = actor.user.id; + const userRoom = String(userId); + + // Peer-echo: one tab notifies others that trash is empty. + socket.on('trash.is_empty', (msg: unknown) => { + socket.broadcast.to(userRoom).emit('trash.is_empty', msg); + }); + + // Legacy probe some frontends use to signal "the UI is + // really up, not just a health-check connection". Extensions + // sometimes listen for the follow-up event. + socket.on('puter_is_actually_open', () => { + this.clients.event.emit( + 'web.socket.user-connected', + { + socket, + user: actor.user, + }, + {}, + ); + }); + + // Fire-and-forget connect event. + this.clients.event.emit( + 'web.socket.connected', + { + socket, + user: actor.user, + }, + {}, + ); + }); + } + + // ── Event bus → socket fan-out ────────────────────────────────── + + #subscribeEventBus(): void { + // One wildcard subscriber covers every `outer.gui.*` mutation + + // notification (item.added/updated/removed/moved/pending, + // cache.updated, submission.done, …). EventClient walks the + // dot-prefix tree at emit time so we get them all. + this.clients.event.on('outer.gui.*', (key: string, data: unknown) => { + this.#handleOuterGui(key, data as OuterGuiPayload).catch( + (err: unknown) => { + console.error('[socket] outer.gui handler error', err); + }, + ); + }); + + // Upload progress — each tracker fires `.sub()` callbacks as + // bytes flow. + this.clients.event.on( + 'fs.storage.upload-progress', + (_key: string, data: unknown) => { + this.#handleUploadProgress(data as UploadProgressPayload); + }, + ); + } + + async #handleOuterGui(key: string, data: OuterGuiPayload): Promise { + const userIds = data.user_id_list ?? []; + if (userIds.length === 0) return; + + // Event bus names are `outer.gui.item.removed` etc.; the wire + // name the client listens for is `item.removed` etc. + const wireName = key.startsWith('outer.gui.') + ? key.slice('outer.gui.'.length) + : key; + // Only item-mutation events should bump the cache-invalidation + // timestamp — `cache.updated` is itself a notification ABOUT the + // timestamp, re-bumping on it is wasted work. + const isMutation = key.startsWith(ITEM_MUTATION_PREFIX); + + const fanout = userIds.map(async (userId) => { + await this.send({ room: userId }, wireName, data.response); + // Post-send hook: listeners (e.g. NotificationService marking notif + // delivery) can react after each per-user fan-out. + this.clients.event.emit( + `sent-to-user.${wireName}`, + { + user_id: userId, + response: data.response, + }, + {}, + ); + if (isMutation) { + const timestamp = Date.now(); + await this.#bumpLastChange(userId, timestamp); + // Push `cache.updated` as a wire event so connected tabs + // invalidate their FS cache immediately (originator filters + // by `original_client_socket_id` to avoid self-refetch). + // Without this, other tabs only learn about the change on + // their next poll of /cache/last-change-timestamp. + const originalSocketId = ( + data.response as + | { original_client_socket_id?: string } + | undefined + )?.original_client_socket_id; + await this.send({ room: userId }, 'cache.updated', { + timestamp, + original_client_socket_id: originalSocketId, + }); + } + }); + await Promise.all(fanout); + } + + #handleUploadProgress(data: UploadProgressPayload): void { + const meta = data.meta ?? {}; + const userId = (meta.user_id ?? meta.userId) as + | number + | string + | undefined; + if (!userId) { + console.warn('[socket] upload-progress missing user_id', { meta }); + return; + } + const wireName = meta.call_it_download + ? 'download.progress' + : 'upload.progress'; + const tracker = data.upload_tracker; + + tracker.sub((delta) => { + void this.send({ room: userId }, wireName, { + ...meta, + total: tracker.total_, + loaded: tracker.progress_, + loaded_diff: delta, + }); + }); + } + + async #bumpLastChange( + userId: number | string, + timestamp: number, + ): Promise { + try { + await this.clients.redis.set( + `${LAST_CHANGE_KEY_PREFIX}${userId}`, + String(timestamp), + 'EX', + LAST_CHANGE_TTL_SECONDS, + ); + } catch (err) { + // Redis write failures shouldn't break the socket send — + // worst case is a stale puter-js cache on another tab. + console.warn('[socket] failed to bump last-change timestamp', err); + } + } +} diff --git a/src/backend/services/subdomain/SubdomainPermissionService.ts b/src/backend/services/subdomain/SubdomainPermissionService.ts new file mode 100644 index 000000000..791d025b7 --- /dev/null +++ b/src/backend/services/subdomain/SubdomainPermissionService.ts @@ -0,0 +1,44 @@ +import { PermissionUtil } from '../permission/permissionUtil.js'; +import type { LayerInstances } from '../../types.js'; +import type { puterStores } from '../../stores/index.js'; +import type { puterServices } from '../index.js'; +import { PuterService } from '../types.js'; + +/** + * Permission rewriter for the `site:*` namespace — maps `site::mode` + * to the uid form. Ported from v1 PuterSiteService. + * + * The v1 `in-site` implicator for SiteActorType is not ported — v2 serves + * hosted sites through the PuterSite middleware without a dedicated site + * actor type, so there's no callsite that could benefit from an implicit + * grant. + */ +export class SubdomainPermissionService extends PuterService { + declare protected stores: LayerInstances; + declare protected services: LayerInstances; + + override onServerStart(): void { + const permissions = this.services.permission; + const subdomainStore = this.stores.subdomain; + + // SubdomainStore.getBySubdomain caches (60m + 60s negative cache), + // and renames invalidate the old key via the store's update path. + permissions.registerRewriter({ + id: 'site-name-to-uid', + matches: (permission: string) => { + if (!permission.startsWith('site:')) return false; + const [, specifier] = PermissionUtil.split(permission); + return Boolean(specifier && !specifier.startsWith('uid#')); + }, + rewrite: async (permission: string): Promise => { + const [prefix, name, ...rest] = + PermissionUtil.split(permission); + const row = (await subdomainStore.getBySubdomain(name)) as { + uuid?: string; + } | null; + if (!row?.uuid) return permission; + return PermissionUtil.join(prefix, `uid#${row.uuid}`, ...rest); + }, + }); + } +} diff --git a/src/backend/services/types.ts b/src/backend/services/types.ts new file mode 100644 index 000000000..663842c43 --- /dev/null +++ b/src/backend/services/types.ts @@ -0,0 +1,39 @@ +import type { puterClients } from '../clients'; +import type { puterStores } from '../stores'; +import type { IConfig, LayerInstances, WithLifecycle } from '../types'; + +/** + * Services may depend on clients, stores, and *prior* services (those declared + * earlier in the registry). The `services` argument is the accumulating + * registry — it only contains peers constructed before this one. + */ +export type IPuterService = new ( + config: IConfig, + clients: LayerInstances, + stores: LayerInstances, + services: Partial>, +) => T; + +export const PuterService = class PuterService implements WithLifecycle { + constructor( + protected config: IConfig, + protected clients: LayerInstances, + protected stores: LayerInstances, + protected services: Partial> = {}, + ) {} + public onServerStart() { + return; + } + public onServerPrepareShutdown() { + return; + } + public onServerShutdown() { + return; + } +} satisfies IPuterService; + +export type IPuterServiceRegistry = Record< + string, + | IPuterService + | (InstanceType> & Record) +>; diff --git a/src/backend/src/CoreModule.js b/src/backend/src/CoreModule.js deleted file mode 100644 index f6b26a1c7..000000000 --- a/src/backend/src/CoreModule.js +++ /dev/null @@ -1,464 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('@heyputer/putility'); -const { NotificationES } = require('./om/entitystorage/NotificationES'); -const { ProtectedAppES } = require('./om/entitystorage/ProtectedAppES'); -const { Context } = require('./util/context'); -const { LLOWrite } = require('./deprecated/filesystem/ll_operations/ll_write'); -const { LLRead } = require('./deprecated/filesystem/ll_operations/ll_read'); -const { RuntimeModule } = require('./extension/RuntimeModule.js'); -const { TYPE_DIRECTORY, TYPE_FILE } = require('./deprecated/filesystem/FSNodeContext.js'); -const { TDetachable } = require('@heyputer/putility/src/traits/traits.js'); -const { MultiDetachable } = require('@heyputer/putility/src/libs/listener.js'); -const { OperationFrame } = require('./services/OperationTraceService'); -const opentelemetry = require('@opentelemetry/api'); -const query = require('./om/query/query'); -const { redisClient } = require('./clients/redis/redisSingleton'); -const { kv } = require('./util/kvSingleton'); -const { s3ClientProvider } = require('./clients/s3/s3ClientProvider'); -const { PuterS3Service } = require('./deprecated/filesystem/PuterS3Service'); - -/** - * @footgun - real install method is defined above - */ -const install = async ({ context, services, app, useapi, modapi }) => { - const config = require('./config'); - const registerServiceIfMissing = (name, service, options) => { - if ( services.has(name) ) return; - services.registerService(name, service, options); - }; - const { TelemetryService } = require('./modules/perfmon/TelemetryService'); - if ( ! services.has('telemetry') ) { - services.registerService('telemetry', TelemetryService); - } - - // === LIBRARIES === - - useapi.withuse(() => { - def('Service', require('./services/BaseService')); - def('Module', AdvancedBase); - - def('core.util.helpers', require('./helpers')); - def('core.util.permission', require('./services/auth/permissionUtils.mjs').PermissionUtil); - def('puter.middlewares.auth', require('./middleware/auth2')); - def('puter.middlewares.configurable_auth', require('./middleware/configurable_auth')); - def('puter.middlewares.anticsrf', require('./middleware/anticsrf')); - - def('core.APIError', require('./api/APIError')); - def('core.Context', Context); - - def('core', require('./services/auth/Actor'), { assign: true }); - def('core', { - TDetachable, - MultiDetachable, - }, { assign: true }); - def('core.config', config); - - // Note: this is an incomplete export; it was added for a proprietary - // extension. Contributors may wish to add definitions in the 'fs.' - // scope. Needing to add these individually is possibly a symptom of an - // anti-pattern; "export filesystem operations to extensions" is one - // statement in English, so maybe it should be one statement of code. - def('core.fs', { - LLOWrite, - LLRead, - TYPE_DIRECTORY, - TYPE_FILE, - OperationFrame, - }); - def('core.fs.selectors', require('./deprecated/filesystem/node/selectors')); - def('core.util.stream', require('./util/streamutil')); - def('web', require('./util/expressutil')); - def('core.validation', require('./validation')); - - def('core.database', require('./services/database/consts.js')); - - def('core.redisClient', redisClient); - def('core.s3ClientProvider', s3ClientProvider); - def('core.kvjs', kv); - - // Add otelutil functions to `core.` - def('core.spanify', require('./util/otelutil').spanify); - def('core.abtest', require('./util/otelutil').abtest); - - // Extension module: 'core' - { - const runtimeModule = new RuntimeModule({ name: 'core' }); - context.get('runtime-modules').register(runtimeModule); - runtimeModule.exports = useapi.use('core'); - } - { - const runtimeModule = new RuntimeModule({ name: 'query' }); - context.get('runtime-modules').register(runtimeModule); - runtimeModule.exports = query; - } - - // Extension module: 'tel' - { - const runtimeModule = new RuntimeModule({ name: 'tel' }); - runtimeModule.exports = { - trace: opentelemetry.trace, - }; - context.get('runtime-modules').register(runtimeModule); - } - }); - - modapi.libdir('core.util', './util'); - - // === SERVICES === - - // TODO: move these to top level imports or await imports and esm this file - - const { RateLimitService } = require('./services/sla/RateLimitService'); - const { AuthService } = require('./services/auth/AuthService'); - const { SLAService } = require('./services/sla/SLAService'); - const { PermissionService } = require('./services/auth/PermissionService'); - const { ACLService } = require('./services/auth/ACLService'); - const { CoercionService } = require('./services/drivers/CoercionService'); - const { PuterSiteService } = require('./services/PuterSiteService'); - const { ContextInitService } = require('./services/ContextInitService'); - const { IdentificationService } = require('./services/abuse-prevention/IdentificationService'); - const { AuthAuditService } = require('./services/abuse-prevention/AuthAuditService'); - const { RegistryService } = require('./services/RegistryService'); - const { RegistrantService } = require('./services/RegistrantService'); - const { SystemValidationService } = require('./services/SystemValidationService'); - const { EntityStoreService } = require('./services/EntityStoreService'); - const SQLES = require('./om/entitystorage/SQLES'); - const ValidationES = require('./om/entitystorage/ValidationES'); - const { SetOwnerES } = require('./om/entitystorage/SetOwnerES'); - const AppES = require('./om/entitystorage/AppES'); - const WriteByOwnerOnlyES = require('./om/entitystorage/WriteByOwnerOnlyES'); - const SubdomainES = require('./om/entitystorage/SubdomainES'); - const { MaxLimitES } = require('./om/entitystorage/MaxLimitES'); - const { AppLimitedES } = require('./om/entitystorage/AppLimitedES'); - const { ReadOnlyES } = require('./om/entitystorage/ReadOnlyES'); - const { OwnerLimitedES } = require('./om/entitystorage/OwnerLimitedES'); - const { ESBuilder } = require('./om/entitystorage/ESBuilder'); - const { Eq, Or } = require('./om/query/query'); - const { MakeProdDebuggingLessAwfulService } = require('./services/MakeProdDebuggingLessAwfulService'); - const { ConfigurableCountingService } = require('./services/ConfigurableCountingService'); - const { FSLockService } = require('./services/fs/FSLockService'); - const FilesystemAPIService = require('./services/FilesystemAPIService'); - const { ServeGUIService } = require('./services/ServeGUIService'); - const { PuterAPIService } = require('./services/PuterAPIService'); - const { RefreshAssociationsService } = require('./services/RefreshAssociationsService'); - // Service names beginning with '__' aren't called by other services; - // these provide data/functionality to other services or produce - // side-effects from the events of other services. - - // === Services which extend BaseService === - const { DDBClientWrapper } = require('./clients/dynamodb/DDBClientWrapper'); - services.registerService('dynamo', DDBClientWrapper); - - services.registerService('system-validation', SystemValidationService); - services.registerService('__api-filesystem', FilesystemAPIService); - services.registerService('__api', PuterAPIService); - services.registerService('__gui', ServeGUIService); - services.registerService('registry', RegistryService); - services.registerService('__registrant', RegistrantService); - services.registerService('fslock', FSLockService); - services.registerService('es:app', EntityStoreService, { - entity: 'app', - upstream: ESBuilder.create([ - SQLES, { table: 'app', debug: true }, - AppES, - AppLimitedES, { - permission_prefix: 'apps-of-user', - // When apps query es:apps, they're allowed to see apps which - // are approved for listing and they're allowed to see their - // own entry. - exception: async () => { - const actor = Context.get('actor'); - return new Or({ - children: [ - new Eq({ - key: 'approved_for_listing', - value: 1, - }), - new Eq({ - key: 'uid', - value: actor.type.app.uid, - }), - ], - }); - }, - }, - WriteByOwnerOnlyES, - ValidationES, - SetOwnerES, - ProtectedAppES, - MaxLimitES, { max: 5000 }, - ]), - }); - - const { EntriService } = require('./services/EntriService.js'); - services.registerService('entri-service', EntriService); - - const { FilesystemService } = require('./deprecated/filesystem/FilesystemService'); - services.registerService('filesystem', FilesystemService); - - services.registerService('es:subdomain', EntityStoreService, { - entity: 'subdomain', - upstream: ESBuilder.create([ - SQLES, { table: 'subdomains', debug: true }, - SubdomainES, - AppLimitedES, { permission_prefix: 'subdomains-of-user' }, - WriteByOwnerOnlyES, - ValidationES, - SetOwnerES, - MaxLimitES, { max: 5000 }, - ]), - }); - services.registerService('es:notification', EntityStoreService, { - entity: 'notification', - upstream: ESBuilder.create([ - SQLES, { table: 'notification', debug: true }, - NotificationES, - OwnerLimitedES, - ReadOnlyES, - SetOwnerES, - MaxLimitES, { max: 200 }, - ]), - }); - services.registerService('rate-limit', RateLimitService); - services.registerService('auth', AuthService); - // services.registerService('preauth', PreAuthService); - services.registerService('permission', PermissionService); - services.registerService('sla', SLAService); - services.registerService('acl', ACLService); - services.registerService('coercion', CoercionService); - services.registerService('puter-site', PuterSiteService); - services.registerService('context-init', ContextInitService); - services.registerService('identification', IdentificationService); - services.registerService('auth-audit', AuthAuditService); - services.registerService('counting', ConfigurableCountingService); - services.registerService('__refresh-assocs', RefreshAssociationsService); - services.registerService('__prod-debugging', MakeProdDebuggingLessAwfulService); - const { EventService } = require('./services/EventService'); - services.registerService('event', EventService); - - const { PuterVersionService } = require('./services/PuterVersionService'); - services.registerService('puter-version', PuterVersionService); - - const { SessionService } = require('./services/SessionService'); - services.registerService('session', SessionService); - - const { EdgeRateLimitService } = require('./services/abuse-prevention/EdgeRateLimitService'); - services.registerService('edge-rate-limit', EdgeRateLimitService); - - const { CleanEmailService } = require('./services/CleanEmailService'); - services.registerService('clean-email', CleanEmailService); - - const { Emailservice } = require('./services/EmailService'); - services.registerService('email', Emailservice); - - const { TokenService } = require('./services/auth/TokenService'); - services.registerService('token', TokenService); - - const { OTPService } = require('./services/auth/OTPService'); - services.registerService('otp', OTPService); - - const { OIDCService } = require('./services/auth/OIDCService'); - services.registerService('oidc', OIDCService); - - const { SignupService } = require('./services/auth/SignupService'); - services.registerService('signup', SignupService); - - const { UserProtectedEndpointsService } = require('./services/web/UserProtectedEndpointsService'); - services.registerService('__user-protected-endpoints', UserProtectedEndpointsService); - - const { AntiCSRFService } = require('./services/auth/AntiCSRFService'); - services.registerService('anti-csrf', AntiCSRFService); - - const { LockService } = require('./services/LockService'); - services.registerService('lock', LockService); - - const { PuterHomepageService } = require('./services/PuterHomepageService'); - services.registerService('puter-homepage', PuterHomepageService); - - const { GetUserService } = require('./services/GetUserService'); - services.registerService('get-user', GetUserService); - - const { DetailProviderService } = require('./services/DetailProviderService'); - services.registerService('whoami', DetailProviderService); - - const { DriverService } = require('./services/drivers/DriverService'); - services.registerService('driver', DriverService); - - const { ScriptService } = require('./services/ScriptService'); - services.registerService('script', ScriptService); - - const { NotificationService } = require('./services/NotificationService'); - services.registerService('notification', NotificationService); - - const { ShareService } = require('./services/ShareService'); - services.registerService('share', ShareService); - - const { GroupService } = require('./services/auth/GroupService'); - services.registerService('group', GroupService); - - const { VirtualGroupService } = require('./services/auth/VirtualGroupService'); - services.registerService('virtual-group', VirtualGroupService); - - const { PermissionAPIService } = require('./services/PermissionAPIService'); - services.registerService('__permission-api', PermissionAPIService); - - const { SystemDataService } = require('./services/SystemDataService'); - services.registerService('system-data', SystemDataService); - - const { SUService } = require('./services/SUService'); - services.registerService('su', SUService); - - const { BootScriptService } = require('./services/BootScriptService'); - services.registerService('boot-script', BootScriptService); - - const { FeatureFlagService } = require('./services/FeatureFlagService'); - services.registerService('feature-flag', FeatureFlagService); - - const { KernelInfoService } = require('./services/KernelInfoService'); - services.registerService('kernel-info', KernelInfoService); - - const { DriverUsagePolicyService } = require('./services/drivers/DriverUsagePolicyService'); - services.registerService('driver-usage-policy', DriverUsagePolicyService); - - const { ReferralCodeService } = require('./services/ReferralCodeService'); - services.registerService('referral-code', ReferralCodeService); - - const { VerifiedGroupService } = require('./services/VerifiedGroupService'); - services.registerService('__verified-group', VerifiedGroupService); - - const { UserService } = require('./services/UserService'); - services.registerService('user', UserService); - - const { WSPushService } = require('./services/WSPushService'); - services.registerService('__event-push-ws', WSPushService); - - const { WispService } = require('./services/WispService'); - services.registerService('wisp', WispService); - - const { WebDavFS } = require('./services/WebDAV/WebDAVService.js'); - services.registerService('dav', WebDavFS); - - const { RequestMeasureService } = require('./services/RequestMeasureService'); - services.registerService('request-measure', RequestMeasureService); - - const { ChatAPIService } = require('./services/ChatAPIService'); - services.registerService('__chat-api', ChatAPIService); - - const { WorkerService } = require('./services/worker/WorkerService'); - services.registerService('worker-service', WorkerService); - - const { MeteringServiceWrapper } = require('./services/MeteringService/MeteringServiceWrapper.mjs'); - services.registerService('meteringService', MeteringServiceWrapper); - - const { DynamoKVStoreWrapper } = require('./services/DynamoKVStore/DynamoKVStoreWrapper.js'); - services.registerService('puter-kvstore', DynamoKVStoreWrapper); - - const { PermissionShortcutService } = require('./services/auth/PermissionShortcutService'); - services.registerService('permission-shortcut', PermissionShortcutService); - - const { PeerService } = require('./services/PeerService'); - services.registerService('peer', PeerService); - - const { AIInterfaceService } = await import('./services/ai/AIInterfaceService.js'); - const { AIChatService } = await import('./services/ai/chat/AIChatService.js'); - const { AIImageGenerationService } = await import('./services/ai/image/AIImageGenerationService.js'); - const { AIVideoGenerationService } = await import('./services/ai/video/AIVideoGenerationService.js'); - registerServiceIfMissing('__ai-interfaces', AIInterfaceService); - registerServiceIfMissing('ai-chat', AIChatService); - registerServiceIfMissing('ai-image', AIImageGenerationService); - registerServiceIfMissing('ai-video', AIVideoGenerationService); - - if ( config?.services?.['aws-textract']?.aws ) { - const { AWSTextractService } = await import('./services/ai/ocr/AWSTextractService.js'); - registerServiceIfMissing('aws-textract', AWSTextractService); - } - - if ( config?.services?.['aws-polly']?.aws ) { - const { AWSPollyService } = await import('./services/ai/tts/AWSPollyService.js'); - registerServiceIfMissing('aws-polly', AWSPollyService); - } - - if ( config?.services?.['elevenlabs'] || config?.elevenlabs ) { - const { ElevenLabsTTSService } = await import('./services/ai/tts/ElevenLabsTTSService.js'); - const { ElevenLabsVoiceChangerService } = await import('./services/ai/sts/ElevenLabsVoiceChangerService.js'); - registerServiceIfMissing('elevenlabs-tts', ElevenLabsTTSService); - registerServiceIfMissing('elevenlabs-voice-changer', ElevenLabsVoiceChangerService); - } - - if ( config?.services?.openai || config?.openai ) { - const { OpenAITTSService } = await import('./services/ai/tts/OpenAITTSService.js'); - const { OpenAISpeechToTextService } = await import('./services/ai/stt/OpenAISpeechToTextService.js'); - registerServiceIfMissing('openai-tts', OpenAITTSService); - registerServiceIfMissing('openai-speech2txt', OpenAISpeechToTextService); - } - - // === Services which are deprecated and should at most be maintained for legacy support === - services.registerService('puter-s3', PuterS3Service); - -}; - -const install_legacy = async ({ services }) => { - const { OperationTraceService } = require('./services/OperationTraceService'); - const { ClientOperationService } = require('./services/ClientOperationService'); - - // === Services which do not yet extend BaseService === - // services.registerService('filesystem', FilesystemService); - services.registerService('operationTrace', OperationTraceService); - services.registerService('client-operation', ClientOperationService); -}; - -/** - * Core module for the Puter platform that includes essential services including - * authentication, filesystems, rate limiting, permissions, and various API endpoints. - * - * This is a monolithic module. Incrementally, services should be migrated to - * Core2Module and other modules instead. Core2Module has a smaller scope, and each - * new module will be a cohesive concern. Once CoreModule is empty, it will be removed - * and Core2Module will take on its name. - */ -class CoreModule extends AdvancedBase { - dirname () { - return __dirname; - } - async install (context) { - const services = context.get('services'); - const app = context.get('app'); - const useapi = context.get('useapi'); - const modapi = context.get('modapi'); - await install({ context, services, app, useapi, modapi }); - } - - /** - * Installs legacy services that don't extend BaseService and require special handling. - * These services were created before the BaseService class existed and don't listen - * to the init event. They need to be installed after the init event is dispatched - * due to initialization order dependencies. - * - * @param {Object} context - The context object containing service references - * @param {Object} context.services - Service registry for registering legacy services - * @returns {Promise} Resolves when legacy services are installed - */ - async install_legacy (context) { - const services = context.get('services'); - await install_legacy({ services }); - } -} - -module.exports = CoreModule; diff --git a/src/backend/src/DatabaseModule.js b/src/backend/src/DatabaseModule.js deleted file mode 100644 index ee785d935..000000000 --- a/src/backend/src/DatabaseModule.js +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { AdvancedBase } from '@heyputer/putility'; -import { StrategizedService } from './services/StrategizedService.js'; -import { SqliteDatabaseAccessService } from './services/database/SqliteDatabaseAccessService.js'; - -// import {BaseService} from './services/BaseService.js'; - -class DatabaseModule extends AdvancedBase { - async install (context) { - const services = context.get('services'); - - services.registerService('database', StrategizedService, { - strategy_key: 'engine', - strategies: { - sqlite: [SqliteDatabaseAccessService], - }, - }); - } -} - -export default DatabaseModule; diff --git a/src/backend/src/Extension.js b/src/backend/src/Extension.js deleted file mode 100644 index 7728388e4..000000000 --- a/src/backend/src/Extension.js +++ /dev/null @@ -1,428 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { AdvancedBase } = require('@heyputer/putility'); -const EmitterFeature = require('@heyputer/putility/src/features/EmitterFeature'); -const { Context } = require('./util/context'); -const { ExtensionServiceState } = require('./ExtensionService'); - -const module_epoch_d = new Date(); -const display_time = (now) => { - const pad2 = n => String(n).padStart(2, '0'); - - const yyyy = now.getFullYear(); - const mm = pad2(now.getMonth() + 1); - const dd = pad2(now.getDate()); - const HH = pad2(now.getHours()); - const MM = pad2(now.getMinutes()); - const SS = pad2(now.getSeconds()); - const time = `${HH}:${MM}:${SS}`; - - const needYear = yyyy !== module_epoch_d.getFullYear(); - const needMonth = needYear || (now.getMonth() !== module_epoch_d.getMonth()); - const needDay = needMonth || (now.getDate() !== module_epoch_d.getDate()); - - if ( needYear ) return `${yyyy}-${mm}-${dd} ${time}`; - if ( needMonth ) return `${mm}-${dd} ${time}`; - if ( needDay ) return `${dd} ${time}`; - return time; -}; - -let memoized_errors = null; - -/** - * This class creates the `extension` global that is seen by Puter backend - * extensions. - */ -class Extension extends AdvancedBase { - static FEATURES = [ - EmitterFeature({ - decorators: [ - fn => Context.get(undefined, { - allow_fallback: true, - }).abind(fn), - ], - }), - ]; - - constructor (...a) { - super(...a); - this.service = null; - this.log = null; - this.ensure_service_(); - - // this.terminal_color = this.randomBrightColor(); - this.terminal_color = 94; - - this.log = (...a) => { - this.log_context.info(a.join(' ')); - }; - this.LOG = (...a) => { - this.log_context.noticeme(a.join(' ')); - }; - ['info', 'warn', 'debug', 'error', 'tick', 'noticeme', 'system'].forEach(lvl => { - this.log[lvl] = (...a) => { - this.log_context[lvl](...a); - }; - }); - - this.only_one_preinit_fn = null; - this.only_one_init_fn = null; - - this.registry = { - register: this.register.bind(this), - of: (typeKey) => { - return { - named: name => { - if ( arguments.length === 0 ) { - return this.registry_[typeKey].named; - } - return this.registry_[typeKey].named[name]; - }, - all: () => [ - ...Object.values(this.registry_[typeKey].named), - ...this.registry_[typeKey].anonymous, - ], - }; - }, - }; - } - - randomBrightColor () { - // Bright colors in ANSI (foreground codes 90–97) - const brightColors = [ - // 91, // Bright Red - 92, // Bright Green - // 93, // Bright Yellow - 94, // Bright Blue - 95, // Bright Magenta - // 96, // Bright Cyan - ]; - - return brightColors[Math.floor(Math.random() * brightColors.length)]; - } - - example () { - console.log('Example method called by an extension.'); - } - - // === [START] RuntimeModule aliases === - set exports (value) { - this.runtime.exports = value; - } - get exports () { - return this.runtime.exports; - } - import (name) { - return this.runtime.import(name); - } - // === [END] RuntimeModule aliases === - - /** - * This will get a database instance from the default service. - */ - get db () { - const db = this.service.values.get('db'); - if ( ! db ) { - throw new Error('extension tried to access database before it was ' + - 'initialized'); - } - return db; - } - - get services () { - const services = this.service.values.get('services'); - if ( ! services ) { - throw new Error('extension tried to access "services" before it was ' + - 'initialized'); - } - return services; - } - - get log_context () { - const log_context = this.service.values.get('log_context'); - if ( ! log_context ) { - throw new Error('extension tried to access "log_context" before it was ' + - 'initialized'); - } - return log_context; - } - - get errors () { - return memoized_errors ?? (() => { - return this.services.get('error-service').create(this.log_context); - })(); - } - - /** - * Register anonymous or named data to a particular type/category. - * @param {string} typeKey Type of data being registered - * @param {string} [key] Key of data being registered - * @param {any} data The data to be registered - */ - register (typeKey, keyOrData, data) { - if ( ! this.registry_[typeKey] ) { - this.registry_[typeKey] = { - named: {}, - anonymous: [], - }; - } - - const typeRegistry = this.registry_[typeKey]; - - if ( arguments.length <= 1 ) { - throw new Error('you must specify what to register'); - } - - if ( arguments.length === 2 ) { - data = keyOrData; - if ( Array.isArray(data) ) { - for ( const datum of data ) { - typeRegistry.anonymous.push(datum); - } - return; - } - typeRegistry.anonymous.push(data); - return; - } - - const key = keyOrData; - typeRegistry.named[key] = data; - } - - /** - * Alias for .register() - * @param {string} typeKey Type of data being registered - * @param {string} [key] Key of data being registered - * @param {any} data The data to be registered - */ - reg (...a) { - this.register(...a); - } - - /** - * This will create a GET endpoint on the default service. - * @param {*} path - route for the endpoint - * @param {*} handler - function to handle the endpoint - * @param {*} options - options like noauth (bool) and mw (array) - */ - get (path, handler, options) { - // this extension will have a default service - this.ensure_service_(); - - // handler and options may be flipped - if ( typeof handler === 'object' ) { - [handler, options] = [options, handler]; - } - if ( ! options ) options = {}; - - this.service.register_route_handler_(path, handler, { - ...options, - methods: ['GET'], - }); - } - - /** - * This will create a POST endpoint on the default service. - * @param {*} path - route for the endpoint - * @param {*} handler - function to handle the endpoint - * @param {*} options - options like noauth (bool) and mw (array) - */ - post (path, handler, options) { - // this extension will have a default service - this.ensure_service_(); - - // handler and options may be flipped - if ( typeof handler === 'object' ) { - [handler, options] = [options, handler]; - } - if ( ! options ) options = {}; - - this.service.register_route_handler_(path, handler, { - ...options, - methods: ['POST'], - }); - } - - /** - * This will create a DELETE endpoint on the default service. - * @param {*} path - route for the endpoint - * @param {*} handler - function to handle the endpoint - * @param {*} options - options like noauth (bool) and mw (array) - */ - put (path, handler, options) { - // this extension will have a default service - this.ensure_service_(); - - // handler and options may be flipped - if ( typeof handler === 'object' ) { - [handler, options] = [options, handler]; - } - if ( ! options ) options = {}; - - this.service.register_route_handler_(path, handler, { - ...options, - methods: ['PUT'], - }); - } - /** - * This will create a DELETE endpoint on the default service. - * @param {*} path - route for the endpoint - * @param {*} handler - function to handle the endpoint - * @param {*} options - options like noauth (bool) and mw (array) - */ - - delete (path, handler, options) { - // this extension will have a default service - this.ensure_service_(); - - // handler and options may be flipped - if ( typeof handler === 'object' ) { - [handler, options] = [options, handler]; - } - if ( ! options ) options = {}; - - this.service.register_route_handler_(path, handler, { - ...options, - methods: ['DELETE'], - }); - } - - use (...args) { - this.ensure_service_(); - this.service.expressThings_.push({ - type: 'router', - value: args, - }); - } - - get preinit () { - return (function (callback) { - this.on('preinit', callback); - }).bind(this); - } - set preinit (callback) { - if ( this.only_one_preinit_fn === null ) { - this.on('preinit', (...a) => { - this.only_one_preinit_fn(...a); - }); - } - if ( callback === null ) { - this.only_one_preinit_fn = () => { - }; - } - this.only_one_preinit_fn = callback; - } - - get init () { - return (function (callback) { - this.on('init', callback); - }).bind(this); - } - set init (callback) { - if ( this.only_one_init_fn === null ) { - this.on('init', (...a) => { - this.only_one_init_fn(...a); - }); - } - if ( callback === null ) { - this.only_one_init_fn = () => { - }; - } - this.only_one_init_fn = callback; - } - - get console () { - const extensionConsole = Object.create(console); - const logfn = level => (...a) => { - let svc_log; - - try { - svc_log = this.services.get('log-service'); - } catch ( _e ) { - // NOOP - } - - if ( ! svc_log ) { - const realConsole = globalThis.original_console_object ?? console; - realConsole[(level => { - if ( ['error', 'warn', 'debug'].includes(level) ) return level; - return 'log'; - })(level)](`${display_time(new Date())} \x1B[${this.terminal_color};1m(extension/${this.name})\x1B[0m`, ...a); - return; - } - - const extensionLogger = svc_log.create(`extension/${this.name}`); - const util = require('node:util'); - const consoleStyle = a.map(arg => { - if ( typeof arg === 'string' ) return arg; - return util.inspect(arg, undefined, undefined, true); - }).join(' '); - extensionLogger[level](consoleStyle); - }; - extensionConsole.log = logfn('info'); - extensionConsole.error = logfn('error'); - extensionConsole.warn = logfn('warn'); - return extensionConsole; - } - - get tracer () { - const trace = this.import('tel').trace; - return trace.getTracer(`extension:${this.name}`); - } - - get span () { - const span = (label, fn) => { - const spanify = this.import('core').spanify; - return spanify(label, fn, this.tracer); - }; - - // Add `.run` for more readable immediate invocation - span.run = (label, fn) => { - if ( typeof label === 'function' ) { - fn = label; - label = fn.name || 'span.run'; - } - return span(label, fn)(); - }; - - return span; - } - - /** - * This method will create the "default service" for an extension. - * This is specifically for Puter extensions that do not define their - * own service classes. - * - * @returns {void} - */ - ensure_service_ () { - if ( this.service ) { - return; - } - - this.service = new ExtensionServiceState({ - extension: this, - }); - } -} - -module.exports = { - Extension, -}; diff --git a/src/backend/src/ExtensionModule.js b/src/backend/src/ExtensionModule.js deleted file mode 100644 index b19fd6336..000000000 --- a/src/backend/src/ExtensionModule.js +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { AdvancedBase } = require('@heyputer/putility'); -const uuid = require('uuid'); -const { ExtensionService } = require('./ExtensionService'); - -class ExtensionModule extends AdvancedBase { - async install (context) { - const services = context.get('services'); - - this.extension.name = this.extension.name ?? context.name; - this.extension.emit('install', { context, services }); - - if ( this.extension.service ) { - services.registerService(uuid.v4(), ExtensionService, { - state: this.extension.service, - }); // uuid for now - } - } -} - -module.exports = { - ExtensionModule, -}; diff --git a/src/backend/src/ExtensionService.js b/src/backend/src/ExtensionService.js deleted file mode 100644 index ae8f13257..000000000 --- a/src/backend/src/ExtensionService.js +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { AdvancedBase } = require('@heyputer/putility'); -const eggspress = require('./api/eggspress'); -const BaseService = require('./services/BaseService'); -const configurable_auth = require('./middleware/configurable_auth'); -const { Context } = require('./util/context'); -const { DB_WRITE } = require('./services/database/consts'); -const { Actor } = require('./services/auth/Actor'); - -/** - * State shared with the default service and the `extension` global so that - * methods on `extension` can register routes (and make other changes in the - * future) to the default service. - */ -class ExtensionServiceState extends AdvancedBase { - constructor (...a) { - super(...a); - - this.extension = a[0].extension; - - this.expressThings_ = []; - - // Values shared between the `extension` global and its service - this.values = new Context(); - } - register_route_handler_ (path, handler, options = {}) { - // handler and options may be flipped - if ( typeof handler === 'object' ) { - [handler, options] = [options, handler]; - } - - const mw = options.mw ?? []; - - // TODO: option for auth middleware is harcoded here, but eventually - // all exposed middlewares should be registered under the simpele names - // used in this options object (probably; still not 100% decided on that) - if ( ! options.noauth ) { - const auth_conf = typeof options.auth === 'object' ? - options.auth : {}; - mw.push(configurable_auth(auth_conf)); - } - - const router = eggspress(path, { - allowedMethods: options.methods ?? ['GET'], - mw, - ...(options.subdomain ? { subdomain: options.subdomain } : {}), - otherOpts: options.otherOpts || {}, - }, handler); - - this.expressThings_.push({ type: 'router', value: [router] }); - } -} - -/** - * A service that does absolutely nothing by default, but its behavior can be - * extended by adding route handlers and event listeners. This is used to - * provide a default service for extensions. - */ -class ExtensionService extends BaseService { - _construct () { - this.expressThings_ = []; - } - async _init (args) { - this.state = args.state; - - this.state.values.set('services', this.services); - this.state.values.set('log_context', this.services.get('log-service').create( - this.state.extension.name, - )); - - // Create database access object for extension - const db = this.services.get('database').get(DB_WRITE, 'extension'); - this.state.values.set('db', db); - - // Propagate all events from Puter's event bus to extensions - const svc_event = this.services.get('event'); - svc_event.on_all(async (key, data, meta = {}) => { - meta.from_outside_of_extension = true; - - await Context.sub({ - extension_name: this.state.extension.name, - }).arun(async () => { - const promises = [ - // push event to the extension's event bus - this.state.extension.emit(key, data, meta), - // legacy: older extensions prefix "core." to events from Puter - this.state.extension.emit(`core.${key}`, data, meta), - ]; - // await this.state.extension.emit(key, data, meta); - await Promise.all(promises); - }); - // await Promise.all(promises); - }); - - // Propagate all events from extension to Puter's event bus - this.state.extension.on_all(async (key, data, meta) => { - if ( meta.from_outside_of_extension ) return; - - await svc_event.emit(key, data, meta); - }); - - this.state.extension.kv = (() => { - const impls = this.services.get_implementors('puter-kvstore'); - const impl_kv = impls[0].impl; - - return new Proxy(impl_kv, { - get: (target, prop) => { - if ( typeof target[prop] !== 'function' ) { - return target[prop]; - } - - return (...args) => { - if ( typeof args[0] !== 'object' ) { - // Luckily named parameters don't have positional - // overlaps between the different kv methods, so - // we can just set them all. - args[0] = { - key: args[0], - as: args[0], - value: args[1], - amount: args[2], - timestamp: args[2], - ttl: args[2], - }; - } - return Context.sub({ - actor: Actor.get_system_actor(), - }).arun(() => target[prop](...args)); - }; - }, - }); - })(); - - this.state.extension.emit('preinit'); - } - - async '__on_boot.consolidation' () { - const svc_su = this.services.get('su'); - await svc_su.sudo(async () => { - await this.state.extension.emit('init', {}, { - from_outside_of_extension: true, - }); - }); - } - async '__on_boot.activation' () { - const svc_su = this.services.get('su'); - await svc_su.sudo(async () => { - await this.state.extension.emit('activate', {}, { - from_outside_of_extension: true, - }); - }); - } - async '__on_boot.ready' () { - const svc_su = this.services.get('su'); - await svc_su.sudo(async () => { - await this.state.extension.emit('ready', {}, { - from_outside_of_extension: true, - }); - }); - } - - '__on_install.routes' (_, { app }) { - for ( const thing of this.state.expressThings_ ) { - if ( thing.type === 'router' ) { - app.use(...thing.value); - continue; - } - } - } - -} - -module.exports = { - ExtensionService, - ExtensionServiceState, -}; diff --git a/src/backend/src/Kernel.js b/src/backend/src/Kernel.js deleted file mode 100644 index fbc6bdff8..000000000 --- a/src/backend/src/Kernel.js +++ /dev/null @@ -1,620 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase, libs } = require('@heyputer/putility'); -const { Context } = require('./util/context'); -const BaseService = require('./services/BaseService'); -const useapi = require('useapi'); -const yargs = require('yargs/yargs'); -const { hideBin } = require('yargs/helpers'); -const { Extension } = require('./Extension'); -const { ExtensionModule } = require('./ExtensionModule'); -const { spawn } = require('node:child_process'); -const fs = require('fs'); -const path_ = require('path'); -const { prependToJSFiles } = require('./util/modutil'); -const { tmp_provide_services } = require('./helpers'); -const uuid = require('uuid'); -const readline = require('node:readline/promises'); -const { RuntimeModuleRegistry } = require('./extension/RuntimeModuleRegistry'); -const { RuntimeModule } = require('./extension/RuntimeModule'); -const deep_proto_merge = require('./config/deep_proto_merge'); -const url = require('url'); -const { initializeS3Config } = require('./clients/s3/s3ClientProvider'); -const { quot } = libs.string; - -class Kernel extends AdvancedBase { - constructor ({ entry_path } = {}) { - super(); - - this.modules = []; - this.useapi = useapi(); - - this.useapi.withuse(() => { - def('Module', AdvancedBase); - def('Service', BaseService); - }); - - this.entry_path = entry_path; - this.extensionExports = {}; - this.extensionInfo = {}; - this.registry = {}; - - this.runtimeModuleRegistry = new RuntimeModuleRegistry(); - } - - add_module (module) { - this.modules.push(module); - } - - _runtime_init (boot_parameters) { - global.cl = console.log; - - const { RuntimeEnvironment } = require('./boot/RuntimeEnvironment'); - - // Determine config and runtime locations - const runtimeEnv = new RuntimeEnvironment({ - entry_path: this.entry_path, - logger: console, - boot_parameters, - }); - const environment = runtimeEnv.init(); - this.environment = environment; - - // polyfills - require('./polyfill/to-string-higher-radix'); - } - - boot () { - const args = yargs(hideBin(process.argv)).argv; - - this._runtime_init({ args }); - - const config = require('./config'); - - globalThis.ll = o => o; - globalThis.xtra_log = () => { - }; - if ( config.env === 'dev' ) { - globalThis.ll = o => { - console.log(`debug: ${ require('node:util').inspect(o)}`); - return o; - }; - globalThis.xtra_log = (...args) => { - // append to file in temp - const fs = require('fs'); - const path = require('path'); - const log_path = path.join('/tmp/xtra_log.txt'); - fs.appendFileSync(log_path, `${args.join(' ') }\n`); - }; - } - - const { consoleLogManager } = require('./util/consolelog'); - consoleLogManager.initialize_proxy_methods(); - - // === START: Initialize Service Registry === - const { Container } = require('./services/Container'); - - const services = new Container(); - this.services = services; - - const root_context = Context.create({ - environment: this.environment, - useapi: this.useapi, - services, - config, - logger: console, - extensionExports: this.extensionExports, - extensionInfo: this.extensionInfo, - registry: this.registry, - args, - 'runtime-modules': this.runtimeModuleRegistry, - }, 'app'); - globalThis.root_context = root_context; - - root_context.arun(async () => { - await initializeS3Config(); - await this._install_modules(); - await this._boot_services(); - }); - - Error.stackTraceLimit = 20; - } - - async _install_modules () { - const { services } = this; - - // Internal modules - for ( const module_ of this.modules ) { - services.registerModule(module_.constructor.name, module_); - const mod_context = this._create_mod_context(Context.get(), { - name: module_.constructor.name, - 'module': module_, - external: false, - }); - await module_.install(mod_context); - } - - for ( const k in services.instances_ ) { - const service_exports = new RuntimeModule({ name: `service:${k}` }); - this.runtimeModuleRegistry.register(service_exports); - service_exports.exports = services.instances_[k]; - } - - // External modules - await this.install_extern_mods_(); - - try { - await services.init(); - } catch (e) { - // First we'll try to mark the system as invalid via - // SystemValidationService. This might fail because this service - // may not be initialized yet. - - const svc_systemValidation = (() => { - try { - return services.get('system-validation'); - } catch (e) { - return null; - } - })(); - - if ( ! svc_systemValidation ) { - // If we can't mark the system as invalid, we'll just have to - // throw the error and let the server crash. - throw e; - } - - await svc_systemValidation.mark_invalid( - 'failed to initialize services', - e, - ); - } - - for ( const module of this.modules ) { - await module.install_legacy?.(Context.get()); - } - - services.ready.resolve(); - // provide services to helpers - - tmp_provide_services(services); - } - - async _boot_services () { - const { services } = this; - - await services.ready; - await services.emit('boot.consolidation'); - - // === END: Initialize Service Registry === - - // self check - (async () => { - await services.ready; - globalThis.services = services; - const log = services.get('log-service').create('init'); - log.system('server ready', { - deployment_type: globalThis.deployment_type, - }); - })(); - await services.emit('boot.activation'); - await services.emit('boot.ready'); - - // Notify process managers (e.g., PM2 wait_ready) that boot completed - if ( typeof process.send === 'function' ) { - try { - process.send('ready'); - } catch ( err ) { - console.error('failed to send ready signal', err); - } - } - } - - async install_extern_mods_ () { - - // In runtime directory, we'll create a `mod_packages` directory.` - if ( fs.existsSync('mod_packages') ) { - fs.rmSync('mod_packages', { recursive: true, force: true }); - } - fs.mkdirSync('mod_packages'); - - // Initialize some globals that external mods depend on - globalThis.__puter_extension_globals__ = { - extensionObjectRegistry: {}, - useapi: this.useapi, - global_config: require('./config'), - }; - - // Also expose global_config globally - globalThis.global_config = require('./config'); - - // Install the mods... - - const mod_install_root_context = Context.get(); - - const mod_directory_promises = []; - const mod_installation_promises = []; - - const mod_paths = this.environment.mod_paths; - for ( const mods_dirpath of mod_paths ) { - const p = (async () => { - if ( ! fs.existsSync(mods_dirpath) ) { - console.error(`mod directory not found: ${quot(mods_dirpath)}; skipping...`); - // intentional delay so error is seen - console.info('boot will continue in 4 seconds'); - await new Promise(rslv => setTimeout(rslv, 4000)); - return; - } - const mod_dirnames = await fs.promises.readdir(mods_dirpath); - - const ignoreList = new Set([ - '.git', - ]); - - for ( const mod_dirname of mod_dirnames ) { - if ( ignoreList.has(mod_dirname) ) continue; - mod_installation_promises.push(this.install_extern_mod_({ - mod_install_root_context, - mod_dirname, - mod_path: path_.join(mods_dirpath, mod_dirname), - })); - } - })(); - if ( process.env.SYNC_MOD_INSTALL ) await p; - mod_directory_promises.push(p); - } - - await Promise.all(mod_directory_promises); - - const mods_to_run = (await Promise.all(mod_installation_promises)) - .filter(v => v !== undefined); - mods_to_run.sort((a, b) => a.priority - b.priority); - let i = 0; - while ( i < mods_to_run.length ) { - const currentPriority = mods_to_run[i].priority; - const samePriorityMods = []; - - // Collect all mods with the same priority - while ( i < mods_to_run.length && mods_to_run[i].priority === currentPriority ) { - samePriorityMods.push(mods_to_run[i]); - i++; - } - - // Run all mods with the same priority concurrently - await Promise.all(samePriorityMods.map(mod_entry => { - return this._run_extern_mod(mod_entry); - })); - } - } - - async install_extern_mod_ ({ - mod_install_root_context, - mod_dirname, - mod_path, - }) { - let stat = fs.lstatSync(mod_path); - while ( stat.isSymbolicLink() ) { - mod_path = fs.readlinkSync(mod_path); - stat = fs.lstatSync(mod_path); - } - - // Mod must be a directory or javascript file - if ( !stat.isDirectory() && !(mod_path.endsWith('.js')) ) { - return; - } - - let mod_name = path_.parse(mod_path).name; - const mod_package_dir = `mod_packages/${mod_name}`; - fs.mkdirSync(mod_package_dir); - - const mod_entry = { - priority: 0, - jsons: {}, - }; - - if ( ! stat.isDirectory() ) { - const rl = readline.createInterface({ - input: fs.createReadStream(mod_path), - }); - for await ( const line of rl ) { - if ( line.trim() === '' ) continue; - if ( ! line.startsWith('//@extension') ) break; - const tokens = line.split(' '); - if ( tokens[1] === 'priority' ) { - mod_entry.priority = Number(tokens[2]); - } - if ( tokens[1] === 'name' ) { - mod_name = `${ tokens[2]}`; - } - } - mod_entry.jsons.package = await this.create_mod_package_json(mod_package_dir, { - name: mod_name, - entry: 'main.js', - }); - await fs.promises.copyFile(mod_path, path_.join(mod_package_dir, 'main.js')); - } else { - // If directory is empty, we'll just skip it - if ( fs.readdirSync(mod_path).length === 0 ) { - console.warn(`Empty mod directory ${quot(mod_path)}; skipping...`); - return; - } - - const promises = []; - - // Create package.json if it doesn't exist - promises.push((async () => { - if ( ! fs.existsSync(path_.join(mod_path, 'package.json')) ) { - mod_entry.jsons.package = await this.create_mod_package_json(mod_package_dir, { - name: mod_name, - }); - } else { - const bin = await fs.promises.readFile(path_.join(mod_path, 'package.json')); - const str = bin.toString(); - mod_entry.jsons.package = JSON.parse(str); - } - })()); - - const puter_json_path = path_.join(mod_path, 'puter.json'); - if ( fs.existsSync(puter_json_path) ) { - promises.push((async () => { - const buffer = await fs.promises.readFile(puter_json_path); - const json = buffer.toString(); - const obj = JSON.parse(json); - mod_entry.priority = obj.priority ?? mod_entry.priority; - mod_entry.jsons.puter = obj; - })()); - } - - const config_json_path = path_.join(mod_path, 'config.json'); - if ( fs.existsSync(config_json_path) ) { - promises.push((async () => { - const buffer = await fs.promises.readFile(config_json_path); - const json = buffer.toString(); - const obj = JSON.parse(json); - mod_entry.priority = obj.priority ?? mod_entry.priority; - mod_entry.jsons.config = obj; - })()); - } - - // Copy mod contents to `/mod_packages` - promises.push(fs.promises.cp(mod_path, mod_package_dir, { - recursive: true, - })); - - await Promise.all(promises); - } - - mod_entry.priority = mod_entry.jsons.puter?.priority ?? mod_entry.priority; - - const extension_id = uuid.v4(); - - await prependToJSFiles(mod_package_dir, `${[ - 'const { use, def } = globalThis.__puter_extension_globals__.useapi;', - 'const { use: puter } = globalThis.__puter_extension_globals__.useapi;', - 'const extension = globalThis.__puter_extension_globals__' + - `.extensionObjectRegistry[${JSON.stringify(extension_id)}];`, - 'const console = extension.console;', - 'const runtime = extension.runtime;', - 'const config = extension.config;', - 'const registry = extension.registry;', - 'const register = registry.register;', - 'const global_config = globalThis.__puter_extension_globals__.global_config', - ].join('\n') }\n`); - - mod_entry.require_dir = path_.join(process.cwd(), mod_package_dir); - - await this.run_npm_install(mod_entry.require_dir); - - const mod = new ExtensionModule(); - mod.extension = new Extension(); - - const runtimeModule = new RuntimeModule({ name: mod_name }); - this.runtimeModuleRegistry.register(runtimeModule); - mod.extension.runtime = runtimeModule; - - mod_entry.module = mod; - - globalThis.__puter_extension_globals__.extensionObjectRegistry[extension_id] - = mod.extension; - - const mod_context = this._create_mod_context(mod_install_root_context, { - name: mod_name, - 'module': mod, - external: true, - mod_path, - }); - - mod_entry.context = mod_context; - - return mod_entry; - }; - - async _run_extern_mod (mod_entry) { - let exportObject = null; - - let { - module: mod, - require_dir, - context, - } = mod_entry; - - const packageJSON = mod_entry.jsons.package; - - Object.defineProperty(mod.extension, 'config', { - get: () => { - const builtin_config = mod_entry.jsons.config ?? {}; - const user_config = require('./config').extensions?.[packageJSON.name] ?? {}; - return deep_proto_merge(user_config, builtin_config); - }, - }); - - mod.extension.name = packageJSON.name; - - // Platform normalization for if import is used in the place of require(); - let importPath = path_.join(require_dir, packageJSON.main ?? 'index.js'); - if ( process.platform === 'win32' ) { - importPath = (url.pathToFileURL(importPath)).href; - } - - const maybe_promise = (typ => typ.trim().toLowerCase())(packageJSON.type ?? '') === 'module' - ? await import(importPath) - : require(require_dir); - - if ( maybe_promise && maybe_promise instanceof Promise ) { - exportObject = await maybe_promise; - } else exportObject = maybe_promise; - - const extension_name = exportObject?.name ?? packageJSON.name; - this.extensionExports[extension_name] = exportObject; - this.extensionInfo[extension_name] = { - name: extension_name, - priority: mod_entry.priority, - type: packageJSON?.type ?? 'commonjs', - }; - mod.extension.registry = this.registry; - mod.extension.name = extension_name; - - if ( exportObject.construct ) { - mod.extension.on('construct', exportObject.construct); - } - if ( exportObject.preinit ) { - mod.extension.on('preinit', exportObject.preinit); - } - - if ( exportObject.init ) { - mod.extension.on('init', exportObject.init); - } - - // This is where the 'install' event gets triggered - await mod.install(context); - } - - _create_mod_context (parent, options) { - const modapi = {}; - - let mod_path = options.mod_path; - if ( !mod_path && options.module.dirname ) { - mod_path = options.module.dirname(); - } - - if ( mod_path ) { - modapi.libdir = (prefix, directory) => { - const fullpath = path_.join(mod_path, directory); - const fsitems = fs.readdirSync(fullpath); - for ( const item of fsitems ) { - if ( !item.endsWith('.js') && !item.endsWith('.cjs') && !item.endsWith('.mjs') ) { - continue; - } - if ( item.endsWith('.test.js') || item.endsWith('.bench.js') ) { - continue; - } - - const stat = fs.statSync(path_.join(fullpath, item)); - if ( ! stat.isFile() ) { - continue; - } - - const name = item.slice(0, -3); - const path = path_.join(fullpath, item); - let lib = require(path); - - // TODO: This context can be made dynamic by adding a - // getter-like behavior to useapi. - this.useapi.def(`${prefix}.${name}`, lib); - } - }; - } - const mod_context = parent.sub({ modapi }, `mod:${options.name}`); - return mod_context; - - } - - async create_mod_package_json (mod_path, { name, entry }) { - // Expect main.js or index.js to exist - const options = ['main.js', 'index.js']; - - // If no entry specified, find file with conventional name - if ( ! entry ) { - for ( const option of options ) { - if ( fs.existsSync(path_.join(mod_path, option)) ) { - entry = option; - break; - } - } - } - - // If no entry specified or found, skip or error - if ( ! entry ) { - console.error(`Expected main.js or index.js in ${quot(mod_path)}`); - if ( ! process.env.SKIP_INVALID_MODS ) { - console.error('Set SKIP_INVALID_MODS=1 (environment variable) to run anyway.'); - process.exit(1); - } else { - return; - } - } - - const data = { - name, - version: '1.0.0', - main: entry ?? 'main.js', - }; - const data_json = JSON.stringify(data); - - await fs.promises.writeFile(path_.join(mod_path, 'package.json'), data_json); - return data; - } - - async run_npm_install (path) { - const npmOptions = process.platform === 'win32' - ? ['npm.cmd', ['install'], { shell: true, cwd: path, stdio: 'pipe' }] - : ['npm', ['install'], { cwd: path, stdio: 'pipe' }]; - - const proc = spawn(...npmOptions); - - let buffer = ''; - - proc.stdout.on('data', (data) => { - buffer += data.toString(); - }); - - proc.stderr.on('data', (data) => { - buffer += data.toString(); - }); - - return new Promise((rslv, rjct) => { - proc.on('close', code => { - if ( code !== 0 ) { - // Print buffered output on error - if ( buffer ) process.stdout.write(buffer); - rjct(new Error(`exit code: ${code}`)); - return; - } - rslv(); - }); - proc.on('error', err => { - // Print buffered output on error - if ( buffer ) process.stdout.write(buffer); - rjct(err); - }); - }); - } -} - -module.exports = { Kernel }; diff --git a/src/backend/src/LocalDiskStorageModule.js b/src/backend/src/LocalDiskStorageModule.js deleted file mode 100644 index 0f01c910c..000000000 --- a/src/backend/src/LocalDiskStorageModule.js +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { AdvancedBase } from '@heyputer/putility'; -import { HostDiskUsageService } from './services/HostDiskUsageService.js'; - -class LocalDiskStorageModule extends AdvancedBase { - async install (context) { - const services = context.get('services'); - services.registerService('host-disk-usage', HostDiskUsageService); - } -} - -export default LocalDiskStorageModule; diff --git a/src/backend/src/api/APIError.js b/src/backend/src/api/APIError.js deleted file mode 100644 index bdbcbef13..000000000 --- a/src/backend/src/api/APIError.js +++ /dev/null @@ -1,710 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { URLSearchParams } = require('node:url'); -const { quot } = require('@heyputer/putility').libs.string; - -/** - * APIError represents an error that can be sent to the client. - * @class APIError - * @property {number} status the HTTP status code - * @property {string} message the error message - * @property {object} source the source of the error - */ -class APIError { - static codes = { - // General - 'unknown_error': { - status: 500, - message: () => 'An unknown error occurred', - }, - 'format_error': { - status: 400, - message: ({ message }) => `format error: ${message}`, - }, - 'temp_error': { - status: 400, - message: ({ message }) => `error: ${message}`, - }, - 'disallowed_value': { - status: 400, - message: ({ key, allowed }) => - `value of ${quot(key)} must be one of: ${ - allowed.map(v => quot(v)).join(', ')}`, - }, - 'invalid_token': { - status: 400, - message: () => 'Invalid token', - }, - 'unrecognized_offering': { - status: 400, - message: ({ name }) => { - return `offering ${quot(name)} was not recognized.`; - }, - }, - 'error_400_from_delegate': { - status: 400, - message: ({ delegate, message }) => `Error 400 from delegate ${quot(delegate)}: ${message}`, - }, - 'ai_chat_all_providers_failed': { - status: 502, - message: ({ attempts }) => - `All AI chat providers failed (${attempts.length} attempt${attempts.length === 1 ? '' : 's'})`, - }, - // Things - 'disallowed_thing': { - status: 400, - message: ({ thing_type, accepted }) => - `Request contained a ${quot(thing_type)} in a ` + - `place where ${quot(thing_type)} isn't accepted${ - - accepted - ? '; ' + - `accepted types are: ${ - accepted.map(v => quot(v)).join(', ')}` - : ''}.`, - }, - - // Unorganized - 'item_with_same_name_exists': { - status: 409, - message: ({ entry_name }) => entry_name - ? `An item with name ${quot(entry_name)} already exists.` - : 'An item with the same name already exists.' - , - }, - 'cannot_move_item_into_itself': { - status: 422, - message: 'Cannot move an item into itself.', - }, - 'cannot_copy_item_into_itself': { - status: 422, - message: 'Cannot copy an item into itself.', - }, - 'directory_depth_limit_exceeded': { - status: 422, - message: ({ limit, would_be }) => `Directory depth limit exceeded. Limit is ${limit}, would be ${would_be}.`, - }, - 'cannot_move_to_root': { - status: 422, - message: 'Cannot move an item to the root directory.', - }, - 'cannot_copy_to_root': { - status: 422, - message: 'Cannot copy an item to the root directory.', - }, - 'cannot_write_to_root': { - status: 422, - message: 'Cannot write an item to the root directory.', - }, - 'cannot_overwrite_a_directory': { - status: 422, - message: 'Cannot overwrite a directory.', - }, - 'cannot_read_a_directory': { - status: 422, - message: 'Cannot read a directory.', - }, - 'source_and_dest_are_the_same': { - status: 422, - message: 'Source and destination are the same.', - }, - 'dest_is_not_a_directory': { - status: 422, - message: 'Destination must be a directory.', - }, - 'dest_does_not_exist': { - status: 422, - message: ({ what_dest }) => { - if ( ! what_dest ) { - return 'Destination was not found.'; - } - - return `Destination of ${quot(what_dest)} was not found.`; - }, - }, - 'source_does_not_exist': { - status: 404, - message: 'Source was not found.', - }, - 'subject_does_not_exist': { - status: 404, - message: 'File or directory not found.', - }, - 'shortcut_target_not_found': { - status: 404, - message: 'Shortcut target not found.', - }, - 'shortcut_target_is_a_directory': { - status: 422, - message: 'Shortcut target is a directory; expected a file.', - }, - 'shortcut_target_is_a_file': { - status: 422, - message: 'Shortcut target is a file; expected a directory.', - }, - 'forbidden': { - status: 403, - message: ({ debug_reason }) => (process.env.DEBUG && debug_reason) - ? `Permission denied: ${debug_reason}` - : 'Permission denied.', - }, - 'immutable': { - status: 403, - message: 'File is immutable.', - }, - 'field_empty': { - status: 400, - message: ({ key }) => `Field ${quot(key)} is required.`, - }, - 'too_many_keys': { - status: 400, - message: ({ key }) => `Field ${quot(key)} cannot contain more than 100 elements.`, - }, - 'field_missing': { - status: 400, - message: ({ key }) => `Field ${quot(key)} is required.`, - }, - 'fields_missing': { - status: 400, - message: ({ keys }) => `The following fields are required but missing: ${keys.map(quot).join(', ')}.`, - }, - 'xor_field_missing': { - status: 400, - message: ({ names }) => { - let s = 'One of these mutually-exclusive fields is required: '; - s += names.map(quot).join(', '); - return s; - }, - }, - 'field_only_valid_with_other_field': { - status: 400, - message: ({ key, other_key }) => `Field ${quot(key)} is only valid when field ${quot(other_key)} is specified.`, - }, - 'invalid_id': { - status: 400, - message: ({ id }) => { - return `Invalid id ${id}`; - }, - }, - 'invalid_operation': { - status: 400, - message: ({ operation }) => `Invalid operation: ${quot(operation)}.`, - }, - 'field_invalid': { - status: 400, - message: ({ key, expected, got }) => { - return `Field ${quot(key)} is invalid.${ - expected ? ` Expected ${expected}.` : '' - }${got ? ` Got ${got}.` : ''}`; - }, - }, - 'fields_invalid': { - status: 400, - message: ({ errors }) => { - let s = 'The following validation errors occurred: '; - s += errors.map(error => `Field ${quot(error.key)} is invalid.${ - error.expected ? ` Expected ${error.expected}.` : '' - }${error.got ? ` Got ${error.got}.` : ''}`).join(', '); - return s; - }, - }, - 'field_immutable': { - status: 400, - message: ({ key }) => `Field ${quot(key)} is immutable.`, - }, - 'field_too_long': { - status: 400, - message: ({ key, max_length }) => `Field ${quot(key)} is too long. Max length is ${max_length}.`, - }, - 'field_too_short': { - status: 400, - message: ({ key, min_length }) => `Field ${quot(key)} is too short. Min length is ${min_length}.`, - }, - 'already_in_use': { - status: 409, - message: ({ what, value }) => `The ${what} ${quot(value)} is already in use.`, - }, - 'invalid_file_name': { - status: 400, - message: ({ name, reason }) => `Invalid file name: ${quot(name)}${reason ? `; ${reason}` : '.'}`, - }, - 'storage_limit_reached': { - status: 400, - message: 'Storage capacity limit reached.', - }, - 'internal_error': { - status: 500, - message: ({ message }) => message - ? `An internal error occurred: ${quot(message)}` - : 'An internal error occurred.', - }, - 'response_timeout': { - status: 504, - message: 'Response timed out.', - }, - 'file_too_large': { - status: 413, - message: ({ max_size }) => `File too large. Max size is ${max_size} bytes.`, - }, - 'thumbnail_too_large': { - status: 413, - message: ({ max_size }) => `Thumbnail too large. Max size is ${max_size} bytes.`, - }, - 'upload_failed': { - status: 500, - message: 'Upload failed.', - }, - 'missing_expected_metadata': { - status: 400, - message: ({ keys }) => `These fields must come first: ${(keys ?? []).map(quot).join(', ')}.`, - }, - 'overwrite_and_dedupe_exclusive': { - status: 400, - message: 'Cannot specify both overwrite and dedupe_name.', - }, - 'not_empty': { - status: 422, - message: 'Directory is not empty.', - }, - 'readdir_of_non_directory': { - status: 422, - message: 'Readdir target must be a directory.', - }, - - // Write - 'offset_without_existing_file': { - status: 404, - message: 'An offset was specified, but the file doesn\'t exist.', - }, - 'offset_requires_overwrite': { - status: 400, - message: 'An offset was specified, but overwrite conditions were not met.', - }, - 'offset_requires_stream': { - status: 400, - message: 'The offset option for write is not available for this upload.', - }, - - // Batch - 'batch_too_many_files': { - status: 400, - message: 'Received an extra file with no corresponding operation.', - }, - 'batch_missing_file': { - status: 400, - message: 'Missing fileinfo entry or BLOB for operation.', - }, - 'invalid_file_metadata': { - status: 400, - message: 'Invalid file metadata.', - }, - 'unresolved_relative_path': { - status: 400, - message: ({ path }) => `Unresolved relative path: ${quot(path)}. ` + - "You may need to specify a full path starting with '/'.", - }, - 'missing_filesystem_capability': { - status: 422, - message: ({ action, subjectName, providerName, capability }) => { - return `Cannot perform action ${quot(action)} on ` + - `${quot(subjectName)} because it is inside a filesystem ` + - `of type ${providerName}, which does not implement the ` + - `required capability called ${quot(capability)}.`; - }, - }, - - // Open - 'no_suitable_app': { - status: 422, - message: ({ entry_name }) => `No suitable app found for ${quot(entry_name)}.`, - }, - 'app_does_not_exist': { - status: 422, - message: ({ identifier }) => `App ${quot(identifier)} does not exist.`, - }, - - // Apps - 'app_name_already_in_use': { - status: 409, - message: ({ name }) => `App name ${quot(name)} is already in use.`, - }, - 'app_index_url_already_in_use': { - status: 409, - message: ({ index_url: indexUrl, app_uid: appUid }) => - `Index URL ${quot(indexUrl)} is already used by app ${quot(appUid)}.`, - }, - - // Subdomains - 'subdomain_limit_reached': { - status: 400, - message: ({ limit, isWorker }) => isWorker ? `You have exceeded the maximum number of workers for your plan! (${limit})` : `You have exceeded the number of subdomains under your current plan (${limit}).`, - }, - 'subdomain_reserved': { - status: 400, - message: ({ subdomain }) => `Subdomain ${quot(subdomain)} is not available.`, - }, - 'subdomain_not_owned': { - status: 403, - message: ({ subdomain }) => `You must own the ${quot(subdomain)} subdomain on Puter to use it for this app.`, - }, - - // Users - 'email_already_in_use': { - status: 409, - message: ({ email }) => `Email ${quot(email)} is already in use.`, - }, - 'email_not_allowed': { - status: 400, - message: ({ email }) => `The email ${quot(email)} is not allowed.`, - }, - 'username_already_in_use': { - status: 409, - - message: ({ username }) => `Username ${quot(username)} is already in use.`, - }, - 'too_many_username_changes': { - status: 429, - message: 'Too many username changes this month.', - }, - 'token_invalid': { - status: 400, - message: () => 'Invalid token.', - }, - - // SLA - 'rate_limit_exceeded': { - status: 429, - message: ({ method_name, rate_limit }) => - `Rate limit exceeded for method ${quot(method_name)}: ${rate_limit.max} requests per ${rate_limit.period}ms.`, - }, - 'server_rate_exceeded': { - status: 503, - message: 'System-wide rate limit exceeded. Please try again later.', - }, - - // New cost system - 'insufficient_funds': { - status: 402, - message: 'Available funding is insufficient for this request.', - }, - - // auth - 'token_missing': { - status: 401, - message: 'Missing authentication token.', - }, - 'unexpected_undefined': { - status: 401, - message: msg => msg ?? 'unexpected string undefined', - }, - 'token_auth_failed': { - status: 401, - message: 'Authentication failed.', - }, - 'user_not_found': { - status: 401, - message: 'User not found.', - }, - 'token_unsupported': { - status: 401, - message: 'This authentication token is not supported here.', - }, - 'token_expired': { - status: 401, - message: 'Authentication token has expired.', - }, - 'account_suspended': { - status: 403, - message: 'Account suspended.', - }, - 'permission_denied': { - status: 403, - message: 'Permission denied.', - }, - 'access_token_empty_permissions': { - status: 403, - message: 'Attempted to create an access token with no permissions.', - }, - 'invalid_action': { - status: 400, - message: ({ action }) => `Invalid action: ${quot(action)}.`, - }, - '2fa_already_enabled': { - status: 409, - message: '2FA is already enabled.', - }, - '2fa_not_configured': { - status: 409, - message: '2FA is not configured.', - }, - - // protected endpoints - 'too_many_requests': { - status: 429, - message: 'Too many requests.', - }, - 'user_tokens_only': { - status: 403, - message: 'This endpoint must be requested with a user session', - }, - 'session_required': { - status: 403, - message: 'This endpoint requires a full session (e.g. change password cannot be done with a GUI token).', - }, - 'temporary_accounts_not_allowed': { - status: 403, - message: 'Temporary accounts cannot perform this action', - }, - 'password_required': { - status: 400, - message: 'Password is required.', - }, - 'password_mismatch': { - status: 403, - message: 'Password does not match.', - }, - 'oidc_revalidation_required': { - status: 403, - message: 'Re-validate by signing in with your linked account (e.g. Google).', - }, - - // Object Mapping - 'field_not_allowed_for_create': { - status: 400, - message: ({ key }) => `Field ${quot(key)} is not allowed for create.`, - }, - 'field_required_for_update': { - status: 400, - message: ({ key }) => `Field ${quot(key)} is required for update.`, - }, - 'entity_not_found': { - status: 422, - message: ({ identifier }) => `Entity not found: ${quot(identifier)}`, - }, - - // Share - 'user_does_not_exist': { - status: 422, - message: ({ username }) => `The user ${quot(username)} does not exist.`, - }, - 'invalid_username_or_email': { - status: 400, - message: ({ value }) => - `The value ${quot(value)} is not a valid username or email.`, - }, - 'invalid_path': { - status: 400, - message: ({ value }) => - `The value ${quot(value)} is not a valid path.`, - }, - 'future': { - status: 400, - message: ({ what }) => `Not supported yet: ${what}`, - }, - // Temporary solution for lack of error composition - 'field_errors': { - status: 400, - message: ({ key, errors }) => - `The value for ${quot(key)} has the following errors: ${ - errors.join('; ')}`, - }, - 'share_expired': { - status: 422, - message: 'This share is expired.', - }, - 'email_must_be_confirmed': { - status: 422, - message: ({ action }) => - `Email must be confirmed to ${action ?? 'apply a share'}. Go to https://puter.com to confirm your email address.`, - }, - 'no_need_to_request': { - status: 422, - message: 'This share is already valid for this user; ' + - 'POST to /apply for access.', - }, - 'can_not_apply_to_this_user': { - status: 422, - message: 'This share can not be applied to this user.', - }, - 'no_origin_for_app': { - status: 400, - message: 'Puter apps must have a valid URL.', - }, - 'anti-csrf-incorrect': { - status: 400, - message: 'Incorrect or missing anti-CSRF token.', - }, - - 'not_yet_supported': { - status: 400, - message: ({ message }) => message, - }, - - // Captcha errors - 'captcha_required': { - status: 400, - message: ({ message }) => message || 'Captcha verification required', - }, - 'captcha_invalid': { - status: 400, - message: ({ message }) => message || 'Invalid captcha response', - }, - - // TTS Errors - 'invalid_engine': { - status: 400, - message: ({ engine, valid_engines }) => `Invalid engine: ${quot(engine)}. Valid engines are: ${valid_engines.map(quot).join(', ')}.`, - }, - - // Abuse prevention - 'moderation_failed': { - status: 422, - message: 'Content moderation failed', - }, - - // Requests - 'ip_not_allowed': { - status: 422, - message: () => 'Specifying host by IP address is not allowed here.', - }, - }; - - /** - * create() is a factory method for creating APIError instances. - * It accepts either a string or an Error object as the second - * argument. If a string is passed, it is used as the error message. - * If an Error object is passed, its message property is used as the - * error message. The Error object itself is stored in the source - * property. If no second argument is passed, the source property - * is set to null. The first argument is used as the status code. - * - * @static - * @param {number|string} status - * @param {Error | null} source - * @param {string|Error|object} fields one of the following: - * - a string to use as the error message - * - an Error object to use as the source of the error - * - an object with a message property to use as the error message - * @returns - */ - static create (status, source = {}, fields = {}) { - // Just the error code - if ( typeof status === 'string' ) { - const code = this.codes[status]; - if ( ! code ) { - return new APIError(500, 'Missing error message.', null, { - code: status, - }); - } - return new APIError(code.status, status, source, fields); - } - - // High-level errors like this: APIError.create(400, '...') - if ( typeof source === 'string' ) { - return new APIError(status, source, null, fields); - } - - // Errors from source like this: throw new Error('...') - if ( - typeof source === 'object' && - source instanceof Error - ) { - return new APIError(status, source?.message, source, fields); - } - - // Errors from sources like this: throw { message: '...', ... } - if ( - typeof source === 'object' && - source.constructor.name === 'Object' && - Object.prototype.hasOwnProperty.call(source, 'message') - ) { - const allfields = { ...source, ...fields }; - return new APIError(status, source.message, source, allfields); - } - - console.error('Invalid APIError source:', source); - return new APIError(500, 'Internal Server Error', null, {}); - } - static adapt (err) { - if ( err instanceof APIError ) return err; - - return APIError.create('internal_error'); - } - constructor (status, message, source, fields = {}) { - this.codes = this.constructor.codes; - this.status = status; - this._message = message; - this.source = source ?? new Error('error for trace'); - this.fields = fields; - - if ( Object.prototype.hasOwnProperty.call(this.codes, message) ) { - this.fields.code = message; - this._message = this.codes[message].message; - } - } - write (res) { - const message = typeof this.message === 'function' - ? this.message(this.fields) - : this.message; - return res.status(this.status).send({ - message, - ...this.fields, - }); - } - serialize () { - return { - ...this.fields, - $: 'heyputer:api/APIError', - message: this.message, - status: this.status, - }; - } - - querystringize (extra) { - return new URLSearchParams(this.querystringize_(extra)); - } - - querystringize_ (extra) { - const fields = {}; - for ( const k in this.fields ) { - fields[`field_${k}`] = this.fields[k]; - } - return { - ...extra, - error: true, - message: this.message, - status: this.status, - ...fields, - }; - } - - get message () { - const message = typeof this._message === 'function' - ? this._message(this.fields) - : this._message; - return message; - } - - toString () { - return `APIError(${this.status}, ${this.message})`; - } -}; - -module.exports = APIError; -module.exports.APIError = APIError; diff --git a/src/backend/src/api/PathOrUIDValidator.js b/src/backend/src/api/PathOrUIDValidator.js deleted file mode 100644 index 6ec78b96c..000000000 --- a/src/backend/src/api/PathOrUIDValidator.js +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('./APIError'); -const _path = require('path'); - -/** - * PathOrUIDValidator validates that either `path` or `uid` is present - * in the request and requires a valid value for the parameter that was - * used. Additionally, resolves the path if a path was provided. - * - * @class PathOrUIDValidator - * @static - * @throws {APIError} if `path` and `uid` are both missing - * @throws {APIError} if `path` and `uid` are both present - * @throws {APIError} if `path` is not a string - * @throws {APIError} if `path` is empty - * @throws {APIError} if `uid` is not a valid uuid - */ -module.exports = class PathOrUIDValidator { - static validate (req) { - const params = req.method === 'GET' - ? req.query : req.body ; - - if ( !params.path && !params.uid ) - { - throw new APIError(400, '`path` or `uid` must be provided.'); - } - // `path` must be a string - else if ( params.path && !params.uid && typeof params.path !== 'string' ) - { - throw new APIError(400, '`path` must be a string.'); - } - // `path` cannot be empty - else if ( params.path && !params.uid && params.path.trim() === '' ) - { - throw new APIError(400, '`path` cannot be empty'); - } - // `uid` must be a valid uuid - else if ( params.uid && !params.path && !require('uuid').validate(params.uid) ) - { - throw new APIError(400, '`uid` must be a valid uuid'); - } - - // resolve path if provided - if ( params.path ) - { - params.path = _path.resolve('/', params.path); - } - } -}; diff --git a/src/backend/src/api/api_error_handler.js b/src/backend/src/api/api_error_handler.js deleted file mode 100644 index 297715965..000000000 --- a/src/backend/src/api/api_error_handler.js +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('./APIError'); -const REDACTED_BODY_KEYS = new Set(['thumbnail', 'thumbnailData', 'base64']); -const MAX_LOG_STRING_LENGTH = 2048; - -const sanitizeAlarmBody = (value, key, seen = new WeakSet()) => { - if ( value === null || value === undefined ) { - return value; - } - - if ( typeof value === 'string' ) { - const isRedactedKey = typeof key === 'string' && REDACTED_BODY_KEYS.has(key); - const isDataUrl = value.startsWith('data:'); - if ( isRedactedKey || isDataUrl ) { - return `[redacted:${value.length}]`; - } - - if ( value.length > MAX_LOG_STRING_LENGTH ) { - return `${value.slice(0, MAX_LOG_STRING_LENGTH)}...[truncated:${value.length}]`; - } - return value; - } - - if ( typeof value !== 'object' ) { - return value; - } - - if ( seen.has(value) ) { - return '[circular]'; - } - seen.add(value); - - if ( Array.isArray(value) ) { - return value.map((item) => sanitizeAlarmBody(item, key, seen)); - } - - const output = {}; - for ( const [entryKey, entryValue] of Object.entries(value) ) { - output[entryKey] = sanitizeAlarmBody(entryValue, entryKey, seen); - } - return output; -}; - -/** - * api_error_handler() is an express error handler for API errors. - * It adheres to the express error handler signature and should be - * used as the last middleware in an express app. - * - * Since Express 5 is not yet released, this function is used by - * eggspress() to handle errors instead of as a middleware. - * - * @todo remove this function and use express error handling - * when Express 5 is released - * - * @param {*} err - * @param {*} req - * @param {*} res - * @param {*} next - * @returns - */ -module.exports = function (err, req, res, next) { - if ( res.headersSent ) { - console.error('error after headers were sent:', err); - return next(err); - } - - // API errors might have a response to help the - // developer resolve the issue. - if ( err instanceof APIError ) { - return err.write(res); - } - - if ( - typeof err === 'object' && - !(err instanceof Error) && - Object.prototype.hasOwnProperty.call(err, 'message') - ) { - const apiError = APIError.create(400, err); - return apiError.write(res); - } - - console.error('internal server error:', err); - - const services = globalThis.services; - if ( services && services.has('alarm') ) { - const alarm = services.get('alarm'); - alarm.create('api_error_handler', err.message, { - error: err, - url: req.url, - method: req.method, - body: sanitizeAlarmBody(req.body, undefined), - headers: req.headers, - }); - } - - req.__error_handled = true; - - // Other errors should provide as little information - // to the client as possible for security reasons. - return res.send(500, 'Internal Server Error'); -}; diff --git a/src/backend/src/api/eggspress.js b/src/backend/src/api/eggspress.js deleted file mode 100644 index 859035264..000000000 --- a/src/backend/src/api/eggspress.js +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -// This file is a legacy alias -module.exports = require('../modules/web/lib/eggspress.js'); diff --git a/src/backend/src/api/filesystem/FSNodeParam.js b/src/backend/src/api/filesystem/FSNodeParam.js deleted file mode 100644 index 60bb6b604..000000000 --- a/src/backend/src/api/filesystem/FSNodeParam.js +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { is_valid_path } = require('../../deprecated/filesystem/validation'); -const { is_valid_uuid4 } = require('../../helpers'); -const { Context } = require('../../util/context'); -const { PathBuilder } = require('../../util/pathutil'); -const APIError = require('../APIError'); - -class FSNodeParam { - constructor (srckey, options) { - this.srckey = srckey; - this.options = options ?? {}; - this.optional = this.options.optional ?? false; - } - - async consolidate ({ req, getParam }) { - const log = globalThis.services.get('log-service').create('fsnode-param'); - const fs = Context.get('services').get('filesystem'); - - let uidOrPath = getParam(this.srckey); - if ( uidOrPath === undefined ) { - if ( this.optional ) return undefined; - throw APIError.create('field_missing', null, { - key: this.srckey, - }); - } - - if ( uidOrPath.length === 0 ) { - if ( this.optional ) return undefined; - APIError.create('field_empty', null, { - key: this.srckey, - }); - } - - if ( ! ['/', '.', '~'].includes(uidOrPath[0]) ) { - if ( is_valid_uuid4(uidOrPath) ) { - return await fs.node({ uid: uidOrPath }); - } - - log.debug('tried uuid', { uidOrPath }); - throw APIError.create('field_invalid', null, { - key: this.srckey, - expected: 'unix-style path or uuid4', - }); - } - - if ( uidOrPath.startsWith('~') && req.user ) { - const homedir = `/${req.user.username}`; - uidOrPath = homedir + uidOrPath.slice(1); - } - - if ( ! is_valid_path(uidOrPath) ) { - log.debug('tried path', { uidOrPath }); - throw APIError.create('field_invalid', null, { - key: this.srckey, - expected: 'unix-style path or uuid4', - }); - } - - const resolved_path = PathBuilder.resolve(uidOrPath, { puterfs: true }); - return await fs.node({ path: resolved_path }); - } -}; - -module.exports = FSNodeParam; -module.exports.FSNodeParam = FSNodeParam; \ No newline at end of file diff --git a/src/backend/src/api/filesystem/FlagParam.js b/src/backend/src/api/filesystem/FlagParam.js deleted file mode 100644 index d1db3a6ba..000000000 --- a/src/backend/src/api/filesystem/FlagParam.js +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); - -module.exports = class FlagParam { - constructor (srckey, options) { - this.srckey = srckey; - this.options = options ?? {}; - this.optional = this.options.optional ?? false; - this.default = this.options.default ?? false; - } - - async consolidate ({ req, getParam }) { - const log = globalThis.services.get('log-service').create('flag-param'); - - const value = getParam(this.srckey); - if ( value === undefined || value === '' ) { - if ( this.optional ) return this.default; - throw APIError.create('field_missing', null, { - key: this.srckey, - }); - } - - if ( typeof value === 'string' ) { - if ( - value === 'true' || value === '1' || value === 'yes' - ) return true; - - if ( - value === 'false' || value === '0' || value === 'no' - ) return false; - - throw APIError.create('field_invalid', null, { - key: this.srckey, - expected: 'boolean', - }); - } - - if ( typeof value === 'boolean' ) { - return value; - } - - log.debug('tried boolean', { value }); - throw APIError.create('field_invalid', null, { - key: this.srckey, - expected: 'boolean', - }); - } -}; diff --git a/src/backend/src/api/filesystem/StringParam.js b/src/backend/src/api/filesystem/StringParam.js deleted file mode 100644 index 1d74aa0ac..000000000 --- a/src/backend/src/api/filesystem/StringParam.js +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); - -module.exports = class StringParam { - constructor (srckey, options) { - this.srckey = srckey; - this.options = options ?? {}; - this.optional = this.options.optional ?? false; - } - - async consolidate ({ req, getParam }) { - const log = globalThis.services.get('log-service').create('string-param'); - - const value = getParam(this.srckey); - if ( value === undefined ) { - if ( this.optional ) return undefined; - throw APIError.create('field_missing', null, { - key: this.srckey, - }); - } - - if ( value.length === 0 ) { - if ( this.optional ) return undefined; - APIError.create('field_empty', null, { - key: this.srckey, - }); - } - - if ( typeof value !== 'string' ) { - log.debug('tried string', { value }); - throw APIError.create('field_invalid', null, { - key: this.srckey, - expected: 'string', - }); - } - - return value; - } -}; diff --git a/src/backend/src/api/filesystem/UserParam.js b/src/backend/src/api/filesystem/UserParam.js deleted file mode 100644 index 9d8d04c00..000000000 --- a/src/backend/src/api/filesystem/UserParam.js +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -module.exports = class UserParam { - consolidate ({ req }) { - return req.user; - } -}; diff --git a/src/backend/src/boot/RuntimeEnvironment.js b/src/backend/src/boot/RuntimeEnvironment.js deleted file mode 100644 index 70830a613..000000000 --- a/src/backend/src/boot/RuntimeEnvironment.js +++ /dev/null @@ -1,372 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('@heyputer/putility'); -const { quot } = require('@heyputer/putility').libs.string; -const { print_error_help } = require('../errors/error_help_details'); -const default_config = require('./default_config'); -const config = require('../config'); -const { ConfigLoader } = require('../config/ConfigLoader'); - -// highlights a string -const hl = s => `\x1b[33;1m${s}\x1b[0m`; - -// Save the original working directory -const original_cwd = process.cwd(); - -// === [ Puter Runtime Environment ] === -// This file contains the RuntimeEnvironment class which is -// responsible for locating the configuration and runtime -// directories for the Puter Kernel. - -// Depending on which path we're checking for configuration -// or runtime from config_paths, there will be different -// requirements. These are all possible requirements. -// -// Each check may result in the following: -// - false: this is not the desired path; skip it -// - true: this is the desired path, and it's valid -// - throw: this is the desired path, but it's invalid -const path_checks = () => ({ fs, path_ }) => ({ - require_if_not_undefined: ({ path }) => { - if ( path == undefined ) return false; - - const exists = fs.existsSync(path); - if ( ! exists ) { - throw new Error(`Path does not exist: ${path}`); - } - - return true; - }, - skip_if_not_exists: ({ path }) => { - const exists = fs.existsSync(path); - return exists; - }, - skip_if_not_in_repo: ({ path }) => { - const exists = fs.existsSync(path_.join(path, '../../.is_puter_repository')); - return exists; - }, - require_read_permission: ({ path }) => { - try { - fs.readdirSync(path); - } catch (e) { - throw new Error(`Cannot readdir on path: ${path}`); - } - return true; - }, - require_write_permission: ({ path }) => { - try { - fs.writeFileSync(path_.join(path, '.tmp_test_write_permission'), 'test'); - fs.unlinkSync(path_.join(path, '.tmp_test_write_permission')); - } catch (e) { - throw new Error(`Cannot write to path: ${path}`); - } - return true; - }, - contains_config_file: ({ path }) => { - const valid_config_names = [ - 'config.json', - 'config.json5', - ]; - for ( const name of valid_config_names ) { - const exists = fs.existsSync(path_.join(path, name)); - if ( exists ) { - return true; - } - } - throw new Error(`No valid config file found in path: ${path}`); - }, - env_not_set: name => () => { - return !process.env[name]; - }, -}); - -// Configuration paths in order of precedence. -// We will load configuration from the first path that's suitable. -const config_paths = ({ path_checks }) => ({ path_ }) => [ - { - label: '$CONFIG_PATH', - get path () { - return process.env.CONFIG_PATH; - }, - checks: [ - path_checks.require_if_not_undefined, - ], - }, - { - path: '/etc/puter', - checks: [path_checks.skip_if_not_exists], - }, - { - get path () { - return path_.join(original_cwd, 'volatile/config'); - }, - checks: [path_checks.skip_if_not_in_repo], - }, - { - get path () { - return path_.join(original_cwd, 'config'); - }, - checks: [path_checks.skip_if_not_exists], - }, -]; - -const valid_config_names = [ - 'config.json', - 'config.json5', -]; - -// Suitable working directories in order of precedence. -// We will `process.chdir` to the first path that's suitable. -const runtime_paths = ({ path_checks }) => ({ path_ }) => [ - { - label: '$RUNTIME_PATH', - get path () { - return process.env.RUNTIME_PATH; - }, - checks: [ - path_checks.require_if_not_undefined, - ], - }, - { - path: '/var/puter', - checks: [ - path_checks.skip_if_not_exists, - path_checks.env_not_set('NO_VAR_RUNTIME'), - ], - }, - { - get path () { - return path_.join(original_cwd, 'volatile/runtime'); - }, - checks: [path_checks.skip_if_not_in_repo], - }, - { - get path () { - return path_.join(original_cwd, 'runtime'); - }, - checks: [path_checks.skip_if_not_exists], - }, -]; - -// Suitable mod paths in order of precedence. -const mod_paths = ({ path_checks, entry_path }) => ({ path_ }) => [ - { - label: '$MOD_PATH', - get path () { - return process.env.MOD_PATH; - }, - checks: [ - path_checks.require_if_not_undefined, - ], - }, - { - path: '/var/puter/mods', - checks: [ - path_checks.skip_if_not_exists, - path_checks.env_not_set('NO_VAR_MODS'), - ], - }, - { - get path () { - return path_.join(path_.dirname(entry_path || require.main.filename), '../mods'); - }, - checks: [path_checks.skip_if_not_exists], - }, -]; - -class RuntimeEnvironment extends AdvancedBase { - static MODULES = { - fs: require('node:fs'), - path_: require('node:path'), - crypto: require('node:crypto'), - format: require('string-template'), - }; - - constructor ({ entry_path, boot_parameters }) { - super(); - this.entry_path = entry_path; - this.boot_parameters = boot_parameters; - this.path_checks = path_checks(this)(this.modules); - this.config_paths = config_paths(this)(this.modules); - this.runtime_paths = runtime_paths(this)(this.modules); - this.mod_paths = mod_paths(this)(this.modules); - } - - init () { - try { - return this.init_(); - } catch (e) { - console.error(e); - print_error_help(e); - process.exit(1); - } - } - - init_ () { - // This variable, called "environment", will be passed back to Kernel - // with some helpful values. A partial-population of this object later - // in this function will be used when evaluating configured paths. - const environment = {}; - environment.source = this.modules.path_.dirname(this.entry_path || require.main.filename); - environment.repo = this.modules.path_.dirname(environment.source); - - const config_path_entry = this.get_first_suitable_path_( - { pathFor: 'configuration' }, - this.config_paths, - [ - this.path_checks.require_read_permission, - // this.path_checks.contains_config_file, - ], - ); - - // Note: there used to be a 'mods_path_entry' here too - // but it was never used - const pwd_path_entry = this.get_first_suitable_path_( - { pathFor: 'working directory' }, - this.runtime_paths, - [this.path_checks.require_write_permission], - ); - - process.chdir(pwd_path_entry.path); - - // Check for a valid config file in the config path - let using_config; - for ( const name of valid_config_names ) { - const exists = this.modules.fs.existsSync(this.modules.path_.join(config_path_entry.path, name)); - if ( exists ) { - using_config = name; - break; - } - } - - const owrite_config = this.boot_parameters.args.overwriteConfig; - - const { fs, path_, crypto } = this.modules; - if ( !using_config || owrite_config ) { - const generated_values = {}; - generated_values.cookie_name = crypto.randomUUID(); - generated_values.jwt_secret = crypto.randomUUID(); - generated_values.url_signature_secret = crypto.randomUUID(); - generated_values.private_uid_secret = crypto.randomBytes(24).toString('hex'); - generated_values.private_uid_namespace = crypto.randomUUID(); - if ( using_config ) { - // make backup - fs.copyFileSync( - path_.join(config_path_entry.path, using_config), - path_.join(config_path_entry.path, `${using_config }.bak`), - ); - // preserve generated values - { - const config_raw = fs.readFileSync( - path_.join(config_path_entry.path, using_config), - 'utf8', - ); - const config_values = JSON.parse(config_raw); - for ( const k in generated_values ) { - if ( ! config_values[k] ) continue; - generated_values[k] = config_values[k]; - } - } - } - const generated_config = { - ...default_config, - ...generated_values, - }; - generated_config[''] = null; // for trailing comma - fs.writeFileSync( - path_.join(config_path_entry.path, 'config.json'), - `${JSON.stringify(generated_config, null, 4) }\n`, - ); - using_config = 'config.json'; - } - - let config_to_load = 'config.json'; - if ( process.env.PUTER_CONFIG_PROFILE ) { - config_to_load = `${process.env.PUTER_CONFIG_PROFILE}.json`; - const exists = fs.existsSync(path_.join(config_path_entry.path, config_to_load)); - if ( ! exists ) { - fs.writeFileSync( - path_.join(config_path_entry.path, config_to_load), - `${JSON.stringify({ - config_name: process.env.PUTER_CONFIG_PROFILE, - $imports: ['config.json'], - }, null, 4) }\n`, - ); - } - } - - environment.config_path = path_.join(config_path_entry.path, config_to_load); - - const loader = new ConfigLoader(config_path_entry.path, config); - loader.enable(config_to_load); - - if ( ! config.config_name ) { - throw new Error('config_name is required'); - } - - const mod_paths = []; - environment.mod_paths = mod_paths; - - // Trying this as a default for now... - if ( ! config.mod_directories ) { - config.mod_directories = [ - '{source}/../mods/mods_enabled', - '{source}/../extensions', - ]; - } - - // If configured, add a user-specified mod path - if ( config.mod_directories ) { - for ( const dir of config.mod_directories ) { - const mods_directory = this.modules.format(dir, environment); - mod_paths.push(mods_directory); - } - } - - return environment; - } - - get_first_suitable_path_ (meta, paths, last_checks) { - for ( const entry of paths ) { - const checks = [...(entry.checks ?? []), ...last_checks]; - - let checks_pass = true; - for ( const check of checks ) { - const result = check(entry); - if ( result === false ) { - checks_pass = false; - break; - } - } - - if ( ! checks_pass ) continue; - - console.info(`${hl(meta.pathFor)} ${quot(entry.path)}`); - - return entry; - } - - if ( meta.optional ) return; - throw new Error(`No suitable path found for ${meta.pathFor}.`); - } -} - -module.exports = { - RuntimeEnvironment, -}; \ No newline at end of file diff --git a/src/backend/src/boot/default_config.js b/src/backend/src/boot/default_config.js deleted file mode 100644 index b2ad770ce..000000000 --- a/src/backend/src/boot/default_config.js +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -module.exports = { - config_name: 'generated default config', - env: 'dev', - nginx_mode: true, // really means "serve http instead of https" - server_id: 'localhost', - http_port: 'auto', - domain: 'puter.localhost', - protocol: 'http', - contact_email: 'hey@example.com', - - services: { - database: { - engine: 'sqlite', - path: 'puter-database.sqlite', - }, - dynamo: { - path: './puter-ddb', - }, - }, -}; diff --git a/src/backend/src/clients/dynamodb/.gitignore b/src/backend/src/clients/dynamodb/.gitignore deleted file mode 100644 index aa4a6da26..000000000 --- a/src/backend/src/clients/dynamodb/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.js -*.js.map \ No newline at end of file diff --git a/src/backend/src/clients/dynamodb/DDBClient.ts b/src/backend/src/clients/dynamodb/DDBClient.ts deleted file mode 100644 index d17017c72..000000000 --- a/src/backend/src/clients/dynamodb/DDBClient.ts +++ /dev/null @@ -1,404 +0,0 @@ -import { CreateTableCommand, CreateTableCommandInput, DynamoDBClient } from '@aws-sdk/client-dynamodb'; -import { BatchGetCommand, BatchGetCommandInput, BatchWriteCommand, BatchWriteCommandInput, DeleteCommand, DynamoDBDocumentClient, GetCommand, PutCommand, QueryCommand, ScanCommand, UpdateCommand } from '@aws-sdk/lib-dynamodb'; -import { NodeHttpHandler } from '@smithy/node-http-handler'; -import dynalite from 'dynalite'; -import { once } from 'node:events'; -import { Agent as httpsAgent } from 'node:https'; - -interface DBClientConfig { - aws?: { - access_key: string - secret_key: string - region: string - }, - path?: string, - endpoint?: string -} - -const LOCAL_DYNAMO_PATH_KEY = ':memory:'; -const localDynaliteEndpointPromises = new Map>(); -const MAX_BATCH_WRITE_ITEMS = 25; -const MAX_BATCH_WRITE_RETRIES = 8; -const BATCH_WRITE_RETRY_BASE_MS = 25; - -const getDynalitePathKey = (path?: string) => { - if ( path === ':memory:' ) return LOCAL_DYNAMO_PATH_KEY; - return path || './puter-ddb'; -}; - -const getOrCreateLocalDynaliteEndpoint = async (pathKey: string) => { - let endpointPromise = localDynaliteEndpointPromises.get(pathKey); - if ( endpointPromise ) return endpointPromise; - - endpointPromise = (async () => { - const dynaliteOptions = pathKey === LOCAL_DYNAMO_PATH_KEY - ? { createTableMs: 0 } - : { createTableMs: 0, path: pathKey }; - - const dynaliteInstance = dynalite(dynaliteOptions); - const dynaliteServer = dynaliteInstance.listen(0, '127.0.0.1'); - // Don't keep test workers alive just because dynalite is still open. - dynaliteServer.unref?.(); - await once(dynaliteServer, 'listening'); - - const address = dynaliteServer.address(); - const port = (typeof address === 'object' && address ? address.port : undefined) || 4567; - return `http://127.0.0.1:${port}`; - })(); - - localDynaliteEndpointPromises.set(pathKey, endpointPromise); - endpointPromise.catch(() => { - if ( localDynaliteEndpointPromises.get(pathKey) === endpointPromise ) { - localDynaliteEndpointPromises.delete(pathKey); - } - }); - return endpointPromise; -}; - -const chunkValues = (values: T[], size: number): T[][] => { - if ( values.length === 0 ) { - return []; - } - const chunks: T[][] = []; - for ( let index = 0; index < values.length; index += size ) { - chunks.push(values.slice(index, index + size)); - } - return chunks; -}; - -const sleep = async (ms: number) => { - await new Promise((resolve) => setTimeout(resolve, ms)); -}; - -export class DDBClient { - ddbClientPromise: Promise; - #documentClient!: DynamoDBDocumentClient; - config?: DBClientConfig; - - constructor (config?: DBClientConfig) { - this.config = config; - this.ddbClientPromise = this.#getClient(); - this.ddbClientPromise.then(client => { - this.#documentClient = DynamoDBDocumentClient.from(client, { - marshallOptions: { - removeUndefinedValues: true, - } }); - }); - } - - async recreateClient () { - const client = await this.#getClient(); - this.ddbClientPromise = Promise.resolve(client); - this.#documentClient = DynamoDBDocumentClient.from(client, { - marshallOptions: { - removeUndefinedValues: true, - } }); - } - - async #getClient () { - if ( ! this.config?.aws ) { - console.warn('No config for DynamoDB, will fall back on local dynalite'); - const pathKey = getDynalitePathKey(this.config?.path); - const dynamoEndpoint = await getOrCreateLocalDynaliteEndpoint(pathKey); - - const client = new DynamoDBClient({ - credentials: { - accessKeyId: 'fake', - secretAccessKey: 'fake', - }, - maxAttempts: 3, - requestHandler: new NodeHttpHandler({ - connectionTimeout: 5000, - requestTimeout: 5000, - httpsAgent: new httpsAgent({ keepAlive: true }), - }), - endpoint: dynamoEndpoint, - region: 'us-west-2', - }); - console.log(`Dynalite client created within instance for region: ${await client.config.region()}`); - return client; - } - - const client = new DynamoDBClient({ - credentials: { - accessKeyId: this.config.aws.access_key, - secretAccessKey: this.config.aws.secret_key, - }, - maxAttempts: 3, - requestHandler: new NodeHttpHandler({ - connectionTimeout: 5000, - requestTimeout: 5000, - httpsAgent: new httpsAgent({ keepAlive: true }), - }), - ...(this.config.endpoint ? { endpoint: this.config.endpoint } : {}), - region: this.config.aws.region || 'us-west-2', - }); - console.log(`DynamoDB client created with region ${await client.config.region()}`); - return client; - } - - async get >(table: string, key: T, consistentRead = false) { - const command = new GetCommand({ - TableName: table, - Key: key, - ConsistentRead: consistentRead, - ReturnConsumedCapacity: 'TOTAL', - }); - - const response = await this.#documentClient.send(command); - - return response; - } - - async put >(table: string, item: T) { - const command = new PutCommand({ - TableName: table, - Item: item, - ReturnConsumedCapacity: 'TOTAL', - }); - - const response = await this.#documentClient.send(command); - return response; - } - - async batchGet (params: { table: string, items: Record }[], consistentRead = false) { - // TODO DS: implement chunking for more than 100 items or more than allowed req size - const allRequestItemsPerTable = params.reduce((acc, curr) => { - if ( ! acc[curr.table] ) acc[curr.table] = []; - acc[curr.table].push(curr.items); - return acc; - }, {} as Record[]>); - - const RequestItems: BatchGetCommandInput['RequestItems'] = Object.entries(allRequestItemsPerTable).reduce( - (acc, [table, keyList]) => { - const Keys = keyList; - acc[table] = { - Keys, - ConsistentRead: consistentRead, - }; - return acc; - }, - {} as NonNullable, - ); - - const command = new BatchGetCommand({ - RequestItems, - ReturnConsumedCapacity: 'TOTAL', - }); - - return this.#documentClient.send(command); - } - - async batchPut (params: { table: string, item: Record }[]) { - const consumedCapacityByTable = new Map(); - if ( params.length === 0 ) { - return { ConsumedCapacity: [] }; - } - - const accumulateConsumedCapacity = ( - consumedCapacityEntries: Array<{ TableName?: string; CapacityUnits?: number }> | undefined, - ) => { - if ( ! consumedCapacityEntries ) { - return; - } - for ( const consumedCapacityEntry of consumedCapacityEntries ) { - const table = consumedCapacityEntry.TableName; - if ( ! table ) { - continue; - } - - const existingUsage = consumedCapacityByTable.get(table) ?? 0; - consumedCapacityByTable.set( - table, - existingUsage + Number(consumedCapacityEntry.CapacityUnits ?? 0), - ); - } - }; - - const chunks = chunkValues(params, MAX_BATCH_WRITE_ITEMS); - for ( const chunk of chunks ) { - let requestItems = chunk.reduce((acc, curr) => { - const tableRequests = acc[curr.table] ?? []; - tableRequests.push({ - PutRequest: { - Item: curr.item, - }, - }); - acc[curr.table] = tableRequests; - return acc; - }, {} as NonNullable); - - for ( let attempt = 0; attempt <= MAX_BATCH_WRITE_RETRIES; attempt++ ) { - if ( Object.keys(requestItems).length === 0 ) { - break; - } - - const response = await this.#documentClient.send(new BatchWriteCommand({ - RequestItems: requestItems, - ReturnConsumedCapacity: 'TOTAL', - })); - accumulateConsumedCapacity( - response.ConsumedCapacity as Array<{ TableName?: string; CapacityUnits?: number }> | undefined, - ); - - const unprocessedItems = response.UnprocessedItems ?? {}; - if ( Object.keys(unprocessedItems).length === 0 ) { - requestItems = {}; - break; - } - - requestItems = unprocessedItems as NonNullable; - if ( attempt < MAX_BATCH_WRITE_RETRIES ) { - const delayMs = Math.min(1000, BATCH_WRITE_RETRY_BASE_MS * (2 ** attempt)); - await sleep(delayMs); - } - } - - if ( Object.keys(requestItems).length > 0 ) { - throw new Error('Failed to batch write all items to DynamoDB'); - } - } - - return { - ConsumedCapacity: Array.from(consumedCapacityByTable.entries()).map(([TableName, CapacityUnits]) => ({ - TableName, - CapacityUnits, - })), - }; - } - - async del> (table: string, key: T) { - const command = new DeleteCommand({ - TableName: table, - Key: key, - ReturnConsumedCapacity: 'TOTAL', - }); - - return this.#documentClient.send(command); - } - - async query> ( - table: string, - keys: T, - limit = 0, - pageKey?: Record, - index = '', - consistentRead = false, - options?: { beginsWith?: { key: string; value: string } }, - ) { - - const keyExpressionParts = Object.keys(keys).map(key => `#${key} = :${key}`); - const expressionAttributeValues = Object.entries(keys).reduce((acc, [key, value]) => { - acc[`:${key}`] = value; - return acc; - }, {}); - const expressionAttributeNames = Object.keys(keys).reduce((acc, key) => { - acc[`#${key}`] = key; - return acc; - }, {}); - - if ( options?.beginsWith?.key && typeof options.beginsWith.value === 'string' && options.beginsWith.value !== '' ) { - const beginsKey = options.beginsWith.key; - const beginsValueToken = `:${beginsKey}_begins_with`; - keyExpressionParts.push(`begins_with(#${beginsKey}, ${beginsValueToken})`); - expressionAttributeValues[beginsValueToken] = options.beginsWith.value; - expressionAttributeNames[`#${beginsKey}`] = beginsKey; - } - - const keyExpression = keyExpressionParts.join(' AND '); - - const command = new QueryCommand({ - TableName: table, - ...(!index ? {} : { IndexName: index }), - KeyConditionExpression: keyExpression, - ExpressionAttributeValues: expressionAttributeValues, - ExpressionAttributeNames: expressionAttributeNames, - ConsistentRead: consistentRead, - ...(!pageKey ? {} : { ExclusiveStartKey: pageKey }), - ...(!limit ? {} : { Limit: limit }), - ReturnConsumedCapacity: 'TOTAL', - }); - - return await this.#documentClient.send(command); - } - - async update> ( - table: string, - key: T, - expression: string, - expressionValues?: Record, - expressionNames?: Record, - ) { - const hasValues = !!expressionValues && Object.keys(expressionValues).length > 0; - const hasNames = !!expressionNames && Object.keys(expressionNames).length > 0; - const command = new UpdateCommand({ - TableName: table, - Key: key, - UpdateExpression: expression, - ...(hasValues ? { ExpressionAttributeValues: expressionValues } : {}), - ...(hasNames ? { ExpressionAttributeNames: expressionNames } : {}), - ReturnValues: 'ALL_NEW', - ReturnConsumedCapacity: 'TOTAL', - }); - try { - return await this.#documentClient.send(command); - } catch ( e ) { - console.error('DDB Update Error', e); - throw e; - } - } - - async createTableIfNotExists (params: CreateTableCommandInput, ttlAttribute?: string) { - if ( this.config?.aws ) { - console.warn('Creating DynamoDB tables in AWS is disabled by default, but if you need to enable it, modify the DDBClient class'); - return; - } - try { - await this.#documentClient.send(new CreateTableCommand(params)); - } catch ( e ) { - if ( (e as Error)?.name !== 'ResourceInUseException' ) { - throw e; - } - } - if ( ttlAttribute ) { - await this.#deleteExpiredItems(params.TableName!, params.KeySchema!, ttlAttribute); - } - } - - async #deleteExpiredItems (table: string, keySchema: NonNullable, ttlAttribute: string) { - const now = Math.floor(Date.now() / 1000); - const keyNames = keySchema.map(k => k.AttributeName!); - - let lastEvaluatedKey: Record | undefined; - do { - const scan = await this.#documentClient.send(new ScanCommand({ - TableName: table, - FilterExpression: '#ttl < :now', - ExpressionAttributeNames: { - '#ttl': ttlAttribute, - ...Object.fromEntries(keyNames.map(k => [`#k_${k}`, k])), - }, - ExpressionAttributeValues: { ':now': now }, - ProjectionExpression: keyNames.map(k => `#k_${k}`).join(', '), - ...(lastEvaluatedKey ? { ExclusiveStartKey: lastEvaluatedKey } : {}), - })); - - lastEvaluatedKey = scan.LastEvaluatedKey as Record | undefined; - const items = scan.Items; - if ( !items || items.length === 0 ) continue; - - const chunks = chunkValues(items, MAX_BATCH_WRITE_ITEMS); - for ( const chunk of chunks ) { - await this.#documentClient.send(new BatchWriteCommand({ - RequestItems: { - [table]: chunk.map(item => ({ - DeleteRequest: { - Key: Object.fromEntries(keyNames.map(k => [k, item[k]])), - }, - })), - }, - })); - } - } while ( lastEvaluatedKey ); - } -} diff --git a/src/backend/src/clients/dynamodb/DDBClientWrapper.ts b/src/backend/src/clients/dynamodb/DDBClientWrapper.ts deleted file mode 100644 index ed8825783..000000000 --- a/src/backend/src/clients/dynamodb/DDBClientWrapper.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { BaseService } from '@heyputer/backend/src/services/BaseService.js'; -import { DDBClient } from './DDBClient.js'; - -/** Wrapping actual implementation to be usable through our core structure */ -class DDBClientServiceWrapper extends BaseService { - ddbClient!: DDBClient; - async _construct () { - this.ddbClient = new DDBClient(this.config as unknown as ConstructorParameters[0]); - - await this.ddbClient.ddbClientPromise; // ensure client is ready - - Object.getOwnPropertyNames(DDBClient.prototype).forEach(fn => { - if ( fn === 'constructor' ) return; - this[fn] = (...args: unknown[]) => this.ddbClient[fn](...args); - }); - } -} - -export const DDBClientWrapper = DDBClientServiceWrapper as unknown as DDBClient; diff --git a/src/backend/src/clients/redis/.gitignore b/src/backend/src/clients/redis/.gitignore deleted file mode 100644 index aa4a6da26..000000000 --- a/src/backend/src/clients/redis/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.js -*.js.map \ No newline at end of file diff --git a/src/backend/src/clients/redis/cacheUpdate.ts b/src/backend/src/clients/redis/cacheUpdate.ts deleted file mode 100644 index 4136cacb8..000000000 --- a/src/backend/src/clients/redis/cacheUpdate.ts +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import type { EventService } from '../../services/EventService.js'; -import { Context } from '../../util/context.js'; -import { redisClient } from './redisSingleton.js'; - -type CacheKeyInput = string | number | null | undefined | CacheKeyInput[]; -interface CacheUpdateOptions { - eventService?: EventService, - emitEvent?: boolean, -} - -const SERVICES_KEY = Symbol.for('puter.helpers.services'); - -const flattenCacheKeys = (inputs: CacheKeyInput[]): Array => { - const flattened: Array = []; - for ( const input of inputs ) { - if ( Array.isArray(input) ) { - flattened.push(...flattenCacheKeys(input)); - continue; - } - flattened.push(input); - } - return flattened; -}; - -export const normalizeCacheKeys = (cacheKey: CacheKeyInput | CacheKeyInput[]): string[] => { - const arr = Array.isArray(cacheKey) ? cacheKey : [cacheKey]; - return [...new Set(flattenCacheKeys(arr) - .map(key => key === null || key === undefined ? '' : String(key)) - .filter(Boolean))]; -}; - -const getEventService = (eventService?: CacheUpdateOptions['eventService']) => { - if ( eventService?.emit ) return eventService; - - const contextServices = Context.get('services', { allow_fallback: true }); - if ( contextServices?.get ) { - try { - return contextServices.get('event'); - } catch (e) { - // no-op - } - } - - const globalServices = (globalThis)[SERVICES_KEY]?.services as typeof contextServices; - if ( globalServices?.get ) { - try { - return globalServices.get('event'); - } catch (e) { - // no-op - } - } - - return null; -}; - -export const emitOuterCacheUpdate = ( - { - cacheKey, - data, - ttlSeconds, - }: { - cacheKey: CacheKeyInput | CacheKeyInput[], - data?: unknown, - ttlSeconds?: number, - }, - { - eventService, - emitEvent = true, - }: CacheUpdateOptions = {}, -) => { - if ( ! emitEvent ) return; - const keys = normalizeCacheKeys(cacheKey); - if ( ! keys.length ) return; - - const svc_event = getEventService(eventService); - if ( ! svc_event ) return; - - const payload: Record = { cacheKey: keys }; - if ( data !== undefined ) payload.data = data; - if ( ttlSeconds !== undefined && ttlSeconds !== null ) { - payload.ttlSeconds = ttlSeconds; - } - - svc_event.emit('outer.cacheUpdate', payload); -}; - -export const setRedisCacheValue = async ( - key: string, - value: string | number, - { - ttlSeconds, - }: { - ttlSeconds?: number, - eventData?: unknown, - eventService?: CacheUpdateOptions['eventService'], - emitEvent?: boolean, - } = {}, -) => { - if ( ttlSeconds ) { - await redisClient.set(key, value, 'EX', ttlSeconds); - } else { - await redisClient.set(key, value, 'EX', 60 * 10); // default to 10 min if no ttl provided - } -}; diff --git a/src/backend/src/clients/redis/deleteRedisKeys.ts b/src/backend/src/clients/redis/deleteRedisKeys.ts deleted file mode 100644 index fc33a2a8e..000000000 --- a/src/backend/src/clients/redis/deleteRedisKeys.ts +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { EventService } from '../../services/EventService.js'; -import { redisClient } from './redisSingleton.js'; - -type DeleteRedisKeysInput = string | number | null | undefined | DeleteRedisKeysInput[]; -interface DeleteRedisKeysOptions { - emitEvent?: boolean, - eventService?: EventService, -} - -const isDeleteOptions = (value: unknown): value is DeleteRedisKeysOptions => { - return !!value - && typeof value === 'object' - && !Array.isArray(value) - && ( - Object.prototype.hasOwnProperty.call(value, 'emitEvent') || - Object.prototype.hasOwnProperty.call(value, 'eventService') - ); -}; - -const flattenInputs = (inputs: DeleteRedisKeysInput[]): Array => { - const flattened: Array = []; - - for ( const input of inputs ) { - if ( Array.isArray(input) ) { - flattened.push(...flattenInputs(input)); - continue; - } - flattened.push(input); - } - - return flattened; -}; - -export const deleteRedisKeys = async (...inputs: (DeleteRedisKeysInput | DeleteRedisKeysOptions)[]) => { - const keysInput = [...inputs]; - if ( isDeleteOptions(keysInput[keysInput.length - 1]) ) { - keysInput.pop() as DeleteRedisKeysOptions; - } - - const keys = flattenInputs(keysInput as DeleteRedisKeysInput[]) - .map(key => key === null || key === undefined ? '' : String(key)) - .filter(Boolean); - - if ( keys.length === 0 ) { - return 0; - } - - const uniqueKeys = [...new Set(keys)]; - - const deleteResults = await Promise.allSettled(uniqueKeys.map(key => redisClient.del(key))); - const deleted = deleteResults.reduce((sum, promiseCount) => sum + (promiseCount.status === 'fulfilled' ? promiseCount.value : 0), 0); - - return deleted; -}; diff --git a/src/backend/src/clients/redis/redisSingleton.test.ts b/src/backend/src/clients/redis/redisSingleton.test.ts deleted file mode 100644 index 686afbaba..000000000 --- a/src/backend/src/clients/redis/redisSingleton.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; - -const redisMocks = vi.hoisted(() => { - const redisClusterInstances: Array<{ - on: ReturnType; - once: ReturnType; - }> = []; - - return { - redisClusterInstances, - redisClusterConstructorMock: vi.fn(), - mockRedisClusterConstructorMock: vi.fn(), - }; -}); - -vi.mock('ioredis', () => { - class RedisClusterMock { - on = vi.fn().mockReturnThis(); - once = vi.fn().mockReturnThis(); - - constructor (...args: unknown[]) { - redisMocks.redisClusterConstructorMock(...args); - redisMocks.redisClusterInstances.push(this); - } - } - - return { - default: { - Cluster: RedisClusterMock, - }, - }; -}); - -vi.mock('ioredis-mock', () => { - class MockRedisClusterMock { - constructor (...args: unknown[]) { - redisMocks.mockRedisClusterConstructorMock(...args); - } - } - - return { - default: { - Cluster: MockRedisClusterMock, - }, - }; -}); - -describe('redisSingleton', () => { - const initialRedisConfig = process.env.REDIS_CONFIG; - - beforeEach(() => { - vi.resetModules(); - redisMocks.redisClusterInstances.length = 0; - redisMocks.redisClusterConstructorMock.mockReset(); - redisMocks.mockRedisClusterConstructorMock.mockReset(); - process.env.REDIS_CONFIG = JSON.stringify([{ host: '127.0.0.1', port: 6379 }]); - vi.spyOn(console, 'log').mockImplementation(() => undefined); - vi.spyOn(console, 'warn').mockImplementation(() => undefined); - vi.spyOn(console, 'error').mockImplementation(() => undefined); - }); - - afterEach(() => { - if ( initialRedisConfig === undefined ) { - delete process.env.REDIS_CONFIG; - } else { - process.env.REDIS_CONFIG = initialRedisConfig; - } - vi.restoreAllMocks(); - }); - - it('uses resilient cluster options and registers startup-safe listeners', async () => { - const singletonModule = await import('./redisSingleton.ts'); - - expect(redisMocks.redisClusterConstructorMock).toHaveBeenCalledTimes(1); - const [startupNodes, clusterOptions] = redisMocks.redisClusterConstructorMock.mock.calls[0]; - - expect(startupNodes).toEqual([{ host: '127.0.0.1', port: 6379 }]); - expect(clusterOptions).toEqual(expect.objectContaining({ - enableOfflineQueue: true, - retryDelayOnFailover: 500, - retryDelayOnClusterDown: 1000, - retryDelayOnTryAgain: 300, - slotsRefreshTimeout: 5000, - clusterRetryStrategy: expect.any(Function), - dnsLookup: expect.any(Function), - redisOptions: expect.objectContaining({ - connectTimeout: 10000, - maxRetriesPerRequest: 2, - tls: {}, - }), - })); - expect(clusterOptions.clusterRetryStrategy(1)).toBe(200); - expect(clusterOptions.clusterRetryStrategy(100)).toBe(2000); - - const clusterInstance = redisMocks.redisClusterInstances[0]; - expect(singletonModule.redisClient).toBe(clusterInstance); - expect(clusterInstance.once).toHaveBeenCalledWith('connect', expect.any(Function)); - expect(clusterInstance.once).toHaveBeenCalledWith('ready', expect.any(Function)); - expect(clusterInstance.on).toHaveBeenCalledWith('error', expect.any(Function)); - expect(clusterInstance.on).toHaveBeenCalledWith('node error', expect.any(Function)); - }); -}); diff --git a/src/backend/src/clients/redis/redisSingleton.ts b/src/backend/src/clients/redis/redisSingleton.ts deleted file mode 100644 index 683dd00b6..000000000 --- a/src/backend/src/clients/redis/redisSingleton.ts +++ /dev/null @@ -1,70 +0,0 @@ -import Redis, { Cluster } from 'ioredis'; -import MockRedis from 'ioredis-mock'; - -const redisStartupRetryMaxDelayMs = 2000; -const redisSlotsRefreshTimeoutMs = 5000; -const redisConnectTimeoutMs = 10000; -const redisMaxRetriesPerRequest = 1; -const redisBootRetryRegex = /Cluster(All)?FailedError|None of startup nodes is available/i; - -const formatRedisError = (error: unknown): string => { - if ( error instanceof Error ) { - return `${error.name}: ${error.message}`; - } - return String(error); -}; - -const attachClusterEventHandlers = (clusterClient: Cluster): void => { - clusterClient.once('connect', () => { - console.log('[redis] cluster transport connected'); - }); - - clusterClient.once('ready', () => { - console.log('[redis] cluster ready'); - }); - - clusterClient.on('error', (error: unknown) => { - const errorText = formatRedisError(error); - if ( redisBootRetryRegex.test(errorText) ) { - console.warn(`[redis] startup issue while connecting to cluster; retrying automatically (${errorText})`); - return; - } - console.error('[redis] cluster error', error); - }); - - clusterClient.on('node error', (error: unknown, nodeKey: string) => { - const errorText = formatRedisError(error); - if ( redisBootRetryRegex.test(errorText) ) { - console.warn(`[redis] startup issue for cluster node ${nodeKey}; retrying automatically (${errorText})`); - return; - } - console.error(`[redis] cluster node error (${nodeKey})`, error); - }); -}; - -let redisOpt: Cluster; - -if ( process.env.REDIS_CONFIG ) { - const redisConfig = JSON.parse(process.env.REDIS_CONFIG); - redisOpt = new Redis.Cluster(redisConfig, { - dnsLookup: (address, callback) => callback(null, address), - clusterRetryStrategy: (attempts) => Math.min(100 + (attempts * 100), redisStartupRetryMaxDelayMs), - retryDelayOnFailover: 500, - retryDelayOnClusterDown: 1000, - retryDelayOnTryAgain: 300, - slotsRefreshTimeout: redisSlotsRefreshTimeoutMs, - enableOfflineQueue: true, - redisOptions: { - tls: {}, - connectTimeout: redisConnectTimeoutMs, - maxRetriesPerRequest: redisMaxRetriesPerRequest, - }, - }); - attachClusterEventHandlers(redisOpt); - console.log('connecting to redis from config'); -} else { - redisOpt = new MockRedis.Cluster(['redis://localhost:7001']); - console.log('connected to local redis mock'); -} - -export const redisClient = redisOpt; diff --git a/src/backend/src/clients/s3/.gitignore b/src/backend/src/clients/s3/.gitignore deleted file mode 100644 index aa4a6da26..000000000 --- a/src/backend/src/clients/s3/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.js -*.js.map \ No newline at end of file diff --git a/src/backend/src/clients/s3/s3ClientProvider.test.ts b/src/backend/src/clients/s3/s3ClientProvider.test.ts deleted file mode 100644 index a2e17e8f3..000000000 --- a/src/backend/src/clients/s3/s3ClientProvider.test.ts +++ /dev/null @@ -1,164 +0,0 @@ -import { - AbortMultipartUploadCommand, - CompleteMultipartUploadCommand, - CreateMultipartUploadCommand, - PutObjectCommand, - UploadPartCommand, -} from '@aws-sdk/client-s3'; -import { existsSync } from 'node:fs'; -import fs from 'node:fs/promises'; -import os from 'node:os'; -import path from 'node:path'; -import { afterEach, describe, expect, it, vi } from 'vitest'; - -import { migrateLegacyStorageToS3 } from './s3ClientProvider.ts'; - -describe('migrateLegacyStorageToS3', () => { - const cleanupPaths: string[] = []; - - afterEach(async () => { - await Promise.all(cleanupPaths.splice(0).map(async p => { - await fs.rm(p, { force: true, recursive: true }).catch(() => undefined); - })); - }); - - it('returns without sending requests when legacy storage is missing', async () => { - const send = vi.fn(); - const legacyPath = path.join(os.tmpdir(), `puter-s3-missing-${Date.now()}`); - - const result = await migrateLegacyStorageToS3({ - client: { send } as any, - legacyPath, - }); - - expect(result).toEqual({ - migratedFileCount: 0, - scannedEntryCount: 0, - }); - expect(send).not.toHaveBeenCalled(); - }); - - it('migrates simple files and ignores non-file entries', async () => { - const send = vi.fn().mockResolvedValue({}); - const legacyPath = await fs.mkdtemp(path.join(os.tmpdir(), 'puter-s3-simple-')); - cleanupPaths.push(legacyPath); - - await fs.writeFile(path.join(legacyPath, 'hello.txt'), 'hello world'); - await fs.writeFile(path.join(legacyPath, 'config.json'), '{"enabled":true}'); - await fs.mkdir(path.join(legacyPath, 'nested-dir')); - await fs.writeFile(path.join(legacyPath, 'nested-dir', 'ignored.txt'), 'ignored'); - - const result = await migrateLegacyStorageToS3({ - bucket: 'test-bucket', - client: { send } as any, - legacyPath, - }); - - expect(result).toEqual({ - migratedFileCount: 2, - scannedEntryCount: 3, - }); - expect(send).toHaveBeenCalledTimes(2); - - const migratedKeys = send.mock.calls - .map(([command]) => command.input.Key) - .sort(); - expect(migratedKeys).toEqual(['config.json', 'hello.txt']); - - send.mock.calls.forEach(([command]) => { - expect(command).toBeInstanceOf(PutObjectCommand); - expect(command.input.Bucket).toBe('test-bucket'); - }); - - expect(existsSync(legacyPath)).toBe(false); - }); - - it('uses multipart migration when file exceeds configured put-object limit', async () => { - const send = vi.fn(async command => { - if ( command instanceof CreateMultipartUploadCommand ) { - return { UploadId: 'upload-1' }; - } - if ( command instanceof UploadPartCommand ) { - return { ETag: `etag-${command.input.PartNumber}` }; - } - if ( command instanceof CompleteMultipartUploadCommand ) { - return {}; - } - throw new Error(`Unexpected command: ${command.constructor.name}`); - }); - - const legacyPath = await fs.mkdtemp(path.join(os.tmpdir(), 'puter-s3-multipart-')); - cleanupPaths.push(legacyPath); - - await fs.writeFile(path.join(legacyPath, 'big.bin'), Buffer.from('abcdefghijklmnopq')); - - const result = await migrateLegacyStorageToS3({ - bucket: 'test-bucket', - client: { send } as any, - legacyPath, - multipartPartSizeBytes: 6, - putObjectLimitBytes: 8, - }); - - expect(result).toEqual({ - migratedFileCount: 1, - scannedEntryCount: 1, - }); - - const createCalls = send.mock.calls.filter(([command]) => command instanceof CreateMultipartUploadCommand); - const uploadCalls = send.mock.calls.filter(([command]) => command instanceof UploadPartCommand); - const completeCalls = send.mock.calls.filter(([command]) => command instanceof CompleteMultipartUploadCommand); - const putObjectCalls = send.mock.calls.filter(([command]) => command instanceof PutObjectCommand); - - expect(createCalls).toHaveLength(1); - expect(uploadCalls).toHaveLength(3); - expect(completeCalls).toHaveLength(1); - expect(putObjectCalls).toHaveLength(0); - - const partNumbers = uploadCalls.map(([command]) => command.input.PartNumber); - expect(partNumbers).toEqual([1, 2, 3]); - - const partLengths = uploadCalls.map(([command]) => command.input.Body.length); - expect(partLengths).toEqual([6, 6, 5]); - - const completedParts = completeCalls[0][0].input.MultipartUpload.Parts; - expect(completedParts).toEqual([ - { ETag: 'etag-1', PartNumber: 1 }, - { ETag: 'etag-2', PartNumber: 2 }, - { ETag: 'etag-3', PartNumber: 3 }, - ]); - - expect(existsSync(legacyPath)).toBe(false); - }); - - it('aborts multipart upload when an upload part fails', async () => { - const send = vi.fn(async command => { - if ( command instanceof CreateMultipartUploadCommand ) { - return { UploadId: 'upload-2' }; - } - if ( command instanceof UploadPartCommand ) { - throw new Error('part failed'); - } - if ( command instanceof AbortMultipartUploadCommand ) { - return {}; - } - return {}; - }); - - const legacyPath = await fs.mkdtemp(path.join(os.tmpdir(), 'puter-s3-multipart-fail-')); - cleanupPaths.push(legacyPath); - - await fs.writeFile(path.join(legacyPath, 'will-fail.bin'), Buffer.from('123456789')); - - await expect(migrateLegacyStorageToS3({ - client: { send } as any, - legacyPath, - multipartPartSizeBytes: 5, - putObjectLimitBytes: 6, - })).rejects.toThrow('part failed'); - - const abortCalls = send.mock.calls.filter(([command]) => command instanceof AbortMultipartUploadCommand); - expect(abortCalls).toHaveLength(1); - expect(existsSync(legacyPath)).toBe(true); - }); -}); diff --git a/src/backend/src/clients/s3/s3ClientProvider.ts b/src/backend/src/clients/s3/s3ClientProvider.ts deleted file mode 100644 index 9948fbfd3..000000000 --- a/src/backend/src/clients/s3/s3ClientProvider.ts +++ /dev/null @@ -1,308 +0,0 @@ -import { - AbortMultipartUploadCommand, - CompleteMultipartUploadCommand, - CreateMultipartUploadCommand, - PutObjectCommand, - S3Client, - S3ClientConfig, - UploadPartCommand, -} from '@aws-sdk/client-s3'; -import { fromNodeProviderChain } from '@aws-sdk/credential-providers'; -import { NodeHttpHandler } from '@smithy/node-http-handler'; -import { FauxqsServer, startFauxqs } from 'fauxqs'; -import { existsSync } from 'node:fs'; -import fs from 'node:fs/promises'; -import { Agent as httpsAgent } from 'node:https'; -import path from 'node:path'; - -// Configuration -const s3Endpoint = process.env.S3_ENDPOINT; -const s3Credentials = process.env.S3_CREDENTIALS ? JSON.parse(process.env.S3_CREDENTIALS) : undefined; -const useProviderChain = process.env.S3_USE_PROVIDER_CHAIN === 'true'; -const LEGACY_STORAGE_BUCKET = 'puter-local'; -const FAUXQS_SAFE_PUT_OBJECT_LIMIT_BYTES = 10 * 1024 * 1024; -const DEFAULT_MULTIPART_PART_SIZE_BYTES = 5 * 1024 * 1024; - -let awsClientConfig: Partial; -let fauxqsServer: FauxqsServer; -let configInitialized = false; -let configInitializationPromise: Promise | null = null; - -const s3ClientMap = new Map(); - -type S3CommandSender = Pick; -interface MigrateLegacyStorageOptions { - bucket?: string; - client?: S3CommandSender; - existsSyncImpl?: typeof existsSync; - fsImpl?: typeof fs; - legacyPath?: string; - multipartPartSizeBytes?: number; - putObjectLimitBytes?: number; -} - -interface LegacyStorageMigrationResult { - migratedFileCount: number; - scannedEntryCount: number; -} - -const uploadFileMultipart = async ({ - bucket, - client, - filePath, - fileSize, - fsImpl, - key, - partSizeBytes, -}: { - bucket: string; - client: S3CommandSender; - filePath: string; - fileSize: number; - fsImpl: typeof fs; - key: string; - partSizeBytes: number; -}) => { - const createMultipartResult = await client.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - })); - const uploadId = createMultipartResult.UploadId; - if ( ! uploadId ) { - throw new Error(`Failed to start multipart upload for ${filePath}`); - } - - const uploadedParts: { ETag: string; PartNumber: number; }[] = []; - const fileHandle = await fsImpl.open(filePath, 'r'); - - try { - let offset = 0; - let partNumber = 1; - - while ( offset < fileSize ) { - const partLength = Math.min(partSizeBytes, fileSize - offset); - const partBuffer = Buffer.alloc(partLength); - const { bytesRead } = await fileHandle.read(partBuffer, 0, partLength, offset); - - if ( bytesRead <= 0 ) break; - - const uploadPartResult = await client.send(new UploadPartCommand({ - Bucket: bucket, - ContentLength: bytesRead, - Key: key, - PartNumber: partNumber, - UploadId: uploadId, - Body: bytesRead === partBuffer.length - ? partBuffer - : partBuffer.subarray(0, bytesRead), - })); - - if ( ! uploadPartResult.ETag ) { - throw new Error(`Multipart upload returned no ETag for ${filePath} part ${partNumber}`); - } - - uploadedParts.push({ - ETag: uploadPartResult.ETag, - PartNumber: partNumber, - }); - - offset += bytesRead; - partNumber++; - } - - await client.send(new CompleteMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - MultipartUpload: { - Parts: uploadedParts, - }, - })); - } catch ( error ) { - await client.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - })).catch(() => undefined); - throw error; - } finally { - await fileHandle.close(); - } -}; -export const s3ClientProvider = { - get: (region: string = 'us-west-2') => { - // Initialize config on first call - - if ( s3ClientMap.has(region) ) { - return s3ClientMap.get(region)!; - } - - try { - const s3Client = new S3Client({ - region, - requestStreamBufferSize: 32 * 1024, - requestHandler: new NodeHttpHandler({ - socketTimeout: 5000, - httpsAgent: new httpsAgent({ - maxSockets: 500, - keepAlive: true, - keepAliveMsecs: 1000, - }), - }), - ...awsClientConfig, - }); - - s3ClientMap.set(region, s3Client); - return s3Client; - } catch ( error ) { - console.error('Failed to create S3 client:', error); - throw new Error(`Failed to initialize S3 client for region ${region}: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - }, - partSize: useProviderChain ? 64 * 1024 * 1024 : DEFAULT_MULTIPART_PART_SIZE_BYTES, - maxSingleUploadSize: useProviderChain ? 128 * 1024 * 1024 : FAUXQS_SAFE_PUT_OBJECT_LIMIT_BYTES, -}; - -export const migrateLegacyStorageToS3 = async ({ - bucket = LEGACY_STORAGE_BUCKET, - client = s3ClientProvider.get(), - existsSyncImpl = existsSync, - fsImpl = fs, - legacyPath = path.join(process.cwd(), 'storage'), - multipartPartSizeBytes = s3ClientProvider.partSize, - putObjectLimitBytes = s3ClientProvider.maxSingleUploadSize, -}: MigrateLegacyStorageOptions = {}): Promise => { - if ( ! existsSyncImpl(legacyPath) ) { - return { - migratedFileCount: 0, - scannedEntryCount: 0, - }; - } - - const entries = await fsImpl.readdir(legacyPath); - let migratedFileCount = 0; - - for ( const entryName of entries ) { - const filePath = path.join(legacyPath, entryName); - const stat = await fsImpl.stat(filePath); - if ( ! stat.isFile() ) continue; - - if ( stat.size > putObjectLimitBytes ) { - await uploadFileMultipart({ - bucket, - client, - filePath, - fileSize: stat.size, - fsImpl, - key: entryName, - partSizeBytes: multipartPartSizeBytes, - }); - } else { - const body = await fsImpl.readFile(filePath); - await client.send(new PutObjectCommand({ - Bucket: bucket, - Key: entryName, - Body: body, - })); - } - - migratedFileCount++; - } - - await fsImpl.rm(legacyPath, { recursive: true }); - return { - migratedFileCount, - scannedEntryCount: entries.length, - }; -}; - -export const initializeS3Config = async (forceLocalInMem = false) => { - - if ( configInitialized ) return; - if ( configInitializationPromise ) return configInitializationPromise; - - configInitializationPromise = (async () => { - // Check if we should use fauxqs (no endpoint and no credentials configured) - const shouldUseS3Endpoint = s3Endpoint && s3Credentials; - - if ( !forceLocalInMem && useProviderChain ) { - awsClientConfig = { - credentials: fromNodeProviderChain(), - }; - } else if ( !forceLocalInMem && shouldUseS3Endpoint ) { - awsClientConfig = { - endpoint: s3Endpoint, - credentials: s3Credentials, - }; - } else { - console.log('No S3 endpoint or credentials configured, starting fauxqs for local S3 dev...'); - - const configuredFauxqsPort = Number.parseInt( - process.env.S3_FAUXQS_PORT ?? '', - 10, - ); - const fauxqsHost = forceLocalInMem ? '127.0.0.1' : process.env.S3_FAUXQS_HOST; - const fauxqsPort = forceLocalInMem - ? 0 - : Number.isFinite(configuredFauxqsPort) - ? configuredFauxqsPort - : 4566; - - fauxqsServer = await startFauxqs({ - host: fauxqsHost, - port: fauxqsPort, - logger: false, - dataDir: forceLocalInMem ? undefined : './fauxqs-data', - s3StorageDir: forceLocalInMem ? undefined : './fauxqs-s3-data', - init: { - region: 'us-west-2', - buckets: [ - 'puter-local', - ], - }, - }); - - let s3Endpoint = fauxqsServer.address; - // WSL Quirk! - if ( s3Endpoint.includes('10.255.255.254') ) { - s3Endpoint = s3Endpoint.replace('10.255.255.254', '127.0.0.1'); - } - - awsClientConfig = { - endpoint: s3Endpoint, - credentials: { - accessKeyId: 'fakeAccessKeyId', - secretAccessKey: 'fakeSecretAccessKey', - }, - }; - - // Gracefully stop server on SIGINT, SIGTERM, or SIGABRT - const shutdown = async () => { - if ( fauxqsServer ) { - await fauxqsServer.stop(); - } - }; - process.on('SIGINT', shutdown); - process.on('SIGTERM', shutdown); - process.on('SIGABRT', shutdown); - - // migrate to s3 if files exist in old local directory (legacy from before fauxqs) - if ( ! forceLocalInMem ) { - const client = s3ClientProvider.get(); - const migrationResult = await migrateLegacyStorageToS3({ client }); - if ( migrationResult.migratedFileCount > 0 ) { - console.log(`Migrated ${migrationResult.migratedFileCount} file(s) from legacy storage to S3.`); - } - } - - } - configInitialized = true; - })(); - - try { - await configInitializationPromise; - } catch ( error ) { - configInitializationPromise = null; - throw error; - } -}; diff --git a/src/backend/src/codex/CodeUtil.js b/src/backend/src/codex/CodeUtil.js deleted file mode 100644 index 131708550..000000000 --- a/src/backend/src/codex/CodeUtil.js +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -class CodeUtil { - /** - * Wrap a method*[1] with an implementation of a runnable class. - * The wrapper must be a class that implements `async run(values)`, - * and `run` should delegate to `this._run()` after setting this.values. - * The `BaseOperation` class is an example of such a class. - * - * [1]: since our runnable interface expects named parameters, this - * wrapping behavior is only useful for methods that accept a single - * object argument. - * @param {*} method - * @param {*} wrapper - */ - static mrwrap (method, wrapper, options = {}) { - const cls_name = options.name || method.name; - - const cls = class extends wrapper { - async _run () { - return await method.call(this.self, this.values); - } - }; - - Object.defineProperty(cls, 'name', { value: cls_name }); - - return async function (...a) { - const op = new cls(); - // eslint-disable-next-line no-invalid-this - op.self = this; // TODO: fix this odd structure, what is this even bound to ? - return await op.run(...a); - }; - } -} - -module.exports = { - CodeUtil, -}; diff --git a/src/backend/src/codex/README.md b/src/backend/src/codex/README.md deleted file mode 100644 index b0bed85d7..000000000 --- a/src/backend/src/codex/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# What is this? - -ChatGPT told me to call this codex and that sounds really cool so -I couldn't resist. - -This directory contains utilities for modelling code as data, so that -we can use static analysis techniques and prevent detectable errors -from reaching produciton. This is an attempt at making things more robust, -but it's not guarenteed to work or even be useful; we need to try it and -collect data about its effectiveness. diff --git a/src/backend/src/codex/Sequence.js b/src/backend/src/codex/Sequence.js deleted file mode 100644 index db02ab5cf..000000000 --- a/src/backend/src/codex/Sequence.js +++ /dev/null @@ -1,383 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -/** - * @typedef {Object} A - * @property {(key: string) => unknown} get - Get a value from the sequence scope. - * @property {function(string, any): void} set - Set a value in the sequence scope. - * @property {(valsToSet?: T) => T extends undefined ? unknown : T} values - Get or set multiple values in the sequence scope. - * @property {function(string=): any} iget - Get a value from the instance (thisArg). - * @property {(methodName: string, ...params: any[] ) => any} icall - Call a method on the instance (thisArg). - * @property {function(string, ...any): any} idcall - Call a method on the instance with the sequence state as the first argument. - * @property {Object} log - Logger, if available on the instance. - * @property {function(any): any} stop - Stop the sequence early and optionally return a value. - * @property {number} i - Current step index. - */ - -/** - * @typedef {(...args: any) => Promise} SequenceCallable - * A callable function returned by the Sequence constructor. - * @param {Object|Sequence.SequenceState} [opt_values] - Initial values for the sequence scope, or a SequenceState. - * @returns {Promise} The return value of the last step in the sequence. - */ -/** - * Sequence is a callable object that executes a series of functions in order. - * The functions are expected to be asynchronous; if they're not it might still - * work, but it's neither tested nor supported. - * - * Note: arrow functions are supported, but they are not recommended; - * using keyword functions allows each step to be named. - * - * Example usage: - * - * const seq = new Sequence([ - * async function set_foo (a) { - * a.set('foo', 'bar') - * }, - * async function print_foo (a) { - * console.log(a.get('foo')); - * }, - * async function third_step (a) { - * // do something - * }, - * ]); - * - * await seq(); - * - * Example with controlled conditional branches: - * - * const seq = new Sequence([ - * async function first_step (a) { - * // do something - * }, - * { - * condition: async a => a.get('foo') === 'bar', - * fn: async function second_step (a) { - * // do something - * } - * }, - * async function third_step (a) { - * // do something - * }, - * ]); - * - * If it is called with an argument, it must be an object containing values - * which will populate the "sequence scope". - * - * If it is called on an instance with a member called `values` - * (i.e. if `this.values` is defined), then these values will populate the - * sequence scope. This is to maintain compatibility for Sequence to be used - * as an implementation of a runnable class. (See CodeUtil.mrwrap or BaseOperation) - * - * The object returned by the constructor is a function, which is used to - * make the object callable. The callable object will execute the sequence - * when called. The return value of the sequence is the return value of the - * last function in the sequence. - * - * Each function in the sequence is passed a SequenceState object - * as its first argument. Conventionally, this argument is called `a`, - * which is short for either "API", "access", or "the `a` variable" - * depending on which you prefer. Sequence provides methods for accessing - * the sequence scope. - * - * By accessing the sequence scope through the `a` variable, changes to the - * sequence scope can be monitored and recorded. (TODO: implement observe methods) - */ -/** - * Sequence is a callable object that executes a series of asynchronous functions in order. - * Each function receives a SequenceState instance for accessing and mutating the sequence scope. - * Supports conditional steps, deferred steps, and can be used as a runnable implementation for classes. - * @class @extends Function - */ -class Sequence { - /** - * SequenceState represents the state of a Sequence execution. - * Provides access to the sequence scope, step control, and utility methods for step functions. - */ - static SequenceState = class SequenceState { - /** - * Create a new SequenceState. - * @param {Sequence|function} sequence - The Sequence instance or its callable function. - * @param {Object} [thisArg] - The instance to bind as `this` for step functions. - */ - constructor (sequence, thisArg) { - if ( typeof sequence === 'function' ) { - sequence = sequence.sequence; - } - - this.sequence_ = sequence; - this.thisArg = thisArg; - this.steps_ = null; - this.value_history_ = []; - this.scope_ = {}; - this.last_return_ = undefined; - this.i = 0; - this.stopped_ = false; - - this.defer_ptr_ = undefined; - this.defer = this.constructor.defer_0; - } - - /** - * Get the current steps array for this sequence execution. - * @returns {Array} The steps to execute. - */ - get steps () { - return this.steps_ ?? this.sequence_?.steps_; - } - - /** - * Run the sequence from the current step index. - * @param {Object} [values] - Initial values for the sequence scope. - * @returns {Promise} - */ - async run (values) { - // Initialize scope - values = values || this.thisArg?.values || {}; - Object.setPrototypeOf(this.scope_, values); - - // Run sequence - for ( ; this.i < this.steps.length ; this.i++ ) { - let step = this.steps[this.i]; - if ( typeof step !== 'object' ) { - step = { - name: step.name, - fn: step, - }; - } - - if ( step.condition && !await step.condition(this) ) { - continue; - } - - const parent_scope = this.scope_; - this.scope_ = {}; - // We could do Object.assign(this.scope_, parent_scope), but - // setting the prototype should be faster (in theory) - Object.setPrototypeOf(this.scope_, parent_scope); - - if ( this.sequence_.options_.record_history ) { - this.value_history_.push(this.scope_); - } - - if ( this.sequence_.options_.before_each ) { - await this.sequence_.options_.before_each(this, step); - } - - this.last_return_ = await step.fn.call(this.thisArg, this); - - if ( this.last_return_ instanceof Sequence.SequenceState ) { - this.scope_ = this.last_return_.scope_; - } - - if ( this.sequence_.options_.after_each ) { - await this.sequence_.options_.after_each(this, step); - } - - if ( this.stopped_ ) { - break; - } - } - } - - // Why check a condition every time code is called, - // when we can check it once and then replace the code? - - /** - * The first time defer is called, clones the steps and sets up for deferred insertion. - * @param {function(Sequence.SequenceState): Promise} fn - The function to defer. - */ - static defer_0 = function (fn) { - this.steps_ = [...this.sequence_.steps_]; - this.defer = this.constructor.defer_1; - this.defer_ptr_ = this.steps_.length; - this.defer(fn); - }; - /** - * Subsequent calls to defer insert the function before the deferred pointer. - * @param {function(Sequence.SequenceState): Promise} fn - The function to defer. - */ - static defer_1 = function (fn) { - // Deferred functions don't affect the return value - const real_fn = fn; - fn = async () => { - await real_fn(this); - return this.last_return_; - }; - - // Insert deferred step before the pointer - this.steps_.splice(this.defer_ptr_, 0, fn); - }; - - /** - * Get a value from the sequence scope. - * @param {string} k - The key to retrieve. - * @returns {any} The value associated with the key. - */ - get (k) { - // TODO: record read1 - return this.scope_[k]; - } - - /** - * Set a value in the sequence scope. - * @param {string} k - The key to set. - * @param {any} v - The value to assign. - */ - set (k, v) { - // TODO: record mutation - this.scope_[k] = v; - } - - /** - * Get or set multiple values in the sequence scope. - * @param {Object} [opt_itemsToSet] - Optional object of key-value pairs to set. - * @returns {Object} Proxy to the current scope for value access. - */ - values (opt_itemsToSet) { - if ( opt_itemsToSet ) { - for ( const k in opt_itemsToSet ) { - this.set(k, opt_itemsToSet[k]); - } - } - - return new Proxy(this.scope_, { - get: (target, property) => { - if ( property in target ) { - // TODO: record read - return target[property]; - } - return undefined; - }, - }); - } - - /** - * Get a value from the instance (`thisArg`). - * @param {string} [k] - The property name to retrieve. If omitted, returns the instance. - * @returns {any} The value from the instance or the instance itself. - */ - iget (k) { - if ( k === undefined ) return this.thisArg; - return this.thisArg?.[k]; - } - - // Instance call: call a method on the instance - /** - * Call a method on the instance (`thisArg`). - * @param {string} k - The method name. - * @param {...any} args - Arguments to pass to the method. - * @returns {any} The result of the method call. - */ - icall (k, ...args) { - return this.thisArg?.[k]?.call(this.thisArg, ...args); - } - - // Instance dynamic call: call a method on the instance, - // passing the sequence state as the first argument - /** - * Call a method on the instance, passing the sequence state as the first argument. - * @param {string} k - The method name. - * @param {...any} args - Arguments to pass after the sequence state. - * @returns {any} The result of the method call. - */ - idcall (k, ...args) { - return this.thisArg?.[k]?.call(this.thisArg, this, ...args); - } - - /** - * Get the logger from the instance, if available. - * @returns {Object|undefined} The logger object. - */ - get log () { - return this.iget('log'); - } - - /** - * Stop the sequence early and optionally return a value. - * @param {any} [return_value] - Value to return from the sequence. - * @returns {any} The provided return value. - */ - stop (return_value) { - this.stopped_ = true; - return return_value; - } - }; - - /** - * - * @param {Array | {condition: (a: A) => boolean | Promise, fn: function(A): Promise}> | function(A): Promise | Object} args - * @returns {Sequence} - */ - /** - * Create a new Sequence. - * @param {...(Array|Object>|function(Sequence.SequenceState): Promise|Object)} args - * - Arrays of step functions or step objects, individual step functions, or options objects. - * - Step objects may have a `condition` property (function) and a `fn` property (function). - * - Options object may include `name`, `record_history`, `before_each`, `after_each`. - * @returns {SequenceCallable} A callable function that runs the sequence. - */ - constructor (...args) { - const sequence = this; - - const steps = []; - const options = {}; - - for ( const arg of args ) { - if ( Array.isArray(arg) ) { - steps.push(...arg); - } else if ( typeof arg === 'object' ) { - Object.assign(options, arg); - } else if ( typeof arg === 'function' ) { - steps.push(arg); - } else { - throw new TypeError(`Invalid argument to Sequence constructor: ${arg}`); - } - } - - /** - * Callable function to execute the sequence. - * @param {Object|Sequence.SequenceState} [opt_values] - Initial values or a SequenceState. - * @returns {Promise} The return value of the last step. - */ - const fn = async function (opt_values) { - if ( opt_values && opt_values instanceof Sequence.SequenceState ) { - opt_values = opt_values.scope_; - } - // eslint-disable-next-line no-invalid-this - const state = new Sequence.SequenceState(sequence, this); // TODO: fix this odd structure, what is this even bound to ? - await state.run(opt_values ?? undefined); - return state.last_return_; - }; - - this.steps_ = steps; - this.options_ = options || {}; - - Object.defineProperty(fn, 'name', { - value: options.name || 'Sequence', - }); - Object.defineProperty(fn, 'sequence', { value: this }); - - return fn; - } -} - -module.exports = { - Sequence, -}; diff --git a/src/backend/src/config.d.ts b/src/backend/src/config.d.ts deleted file mode 100644 index d2ba4c6c5..000000000 --- a/src/backend/src/config.d.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { RecursiveRecord } from "./services/MeteringService/types"; - -type ConfigRecord = RecursiveRecord; - -export interface IConfig extends ConfigRecord { - load_config: (o: ConfigRecord) => void; - __set_config_object__: ( - object: ConfigRecord, - options?: { replacePrototype?: boolean; useInitialPrototype?: boolean } - ) => void; -} - -declare const config: IConfig; - -export = config; diff --git a/src/backend/src/config.js b/src/backend/src/config.js deleted file mode 100644 index d699b4fc4..000000000 --- a/src/backend/src/config.js +++ /dev/null @@ -1,292 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const deep_proto_merge = require('./config/deep_proto_merge'); -// const reserved_words = require('./config/reserved_words'); - -let config = {}; -config.__import_identity__ = require('uuid').v4(); - -// Static defaults -config.servers = []; - -config.disable_user_signup = false; -config.default_user_group = '78b1b1dd-c959-44d2-b02c-8735671f9997'; - -// Will disable the auto-generated temp users. If a user lands on the site, they will be required to sign up or log in. -config.disable_temp_users = false; -config.default_temp_group = 'b7220104-7905-4985-b996-649fdcdb3c8f'; - -config.max_file_size = 100_000_000_000; -config.max_thumb_size = 1_000; -config.max_fsentry_name_length = 767; - -config.username_regex = /^\w+$/; -config.username_max_length = 45; -config.subdomain_regex = /^[a-zA-Z0-9_-]+$/; -config.subdomain_max_length = 60; -config.app_name_regex = /^[a-zA-Z0-9_-]+$/; -config.app_name_max_length = 60; -config.app_title_max_length = 60; -config.min_pass_length = 6; - -config.strict_email_verification_required = false; -config.require_email_verification_to_publish_website = false; - -config.kv_max_key_size = 1024; -config.kv_max_value_size = 400 * 1024; - -// Captcha configuration -config.captcha = { - enabled: false, // Enable captcha by default - expirationTime: 10 * 60 * 1000, // 10 minutes default expiration time - difficulty: 'medium', // Default difficulty level -}; - -// OIDC/OAuth2 providers (e.g. Google). Keys in config only, not env vars. -// Example: config.oidc.providers.google = { client_id, client_secret } -config.oidc = { - providers: {}, -}; - -config.monitor = { - metricsInterval: 60000, - windowSize: 30, -}; - -config.max_subdomains_per_user = 2000; -config.storage_capacity = 1 * 1024 * 1024 * 1024; -config.static_hosting_base_domain_redirect = 'https://developer.puter.com/static-hosting/'; -config.enable_private_app_access_gate = true; - -// Storage limiting is set to false by default -// Storage available on the mountpoint/drive puter is running is the storage available -config.is_storage_limited = false; -config.available_device_storage = null; - -config.thumb_width = 80; -config.thumb_height = 80; -config.app_max_icon_size = 5 * 1024 * 1024; - -config.defaultjs_asset_path = '../../'; - -config.short_description = 'Puter is a privacy-first personal cloud that houses all your files, apps, and games in one private and secure place, accessible from anywhere at any time.'; -config.title = 'Puter'; -config.company = 'Puter Technologies Inc.'; - -config.puter_hosted_data = { - puter_versions: 'https://version.puter.site/puter_versions.json', -}; - -{ - const path_ = require('path'); - config.assets = { - gui: path_.join(__dirname, '../../gui'), - gui_profile: 'development', - }; -} - -// words that cannot be used by others as subdomains or app names -// config.reserved_words = reserved_words; -config.reserved_words = []; - -{ - config.reserved_words.push(...require('./config/reserved_words')); -} - -// set default S3 settings for this server, if any -if ( config.server_id ) { - // see if this server has a specific bucket - for ( const server of config.servers ) { - if ( server.id !== config.server_id ) continue; - if ( ! server.s3_bucket ) continue; - - config.s3_bucket = server.s3_bucket; - config.s3_region = server.region; - } -} - -config.contact_email = `hey@${ config.domain}`; - -// TODO: default value will be changed to false in a future release; -// details to follow in a future announcement. -config.legacy_token_migrate = true; - -// === OS Information === -const os = require('os'); -const fs = require('fs'); -const { Context, context_config } = require('./util/context'); -config.os = {}; -config.os.platform = os.platform(); - -if ( config.os.platform === 'linux' ) { - try { - const osRelease = fs.readFileSync('/etc/os-release').toString(); - // CONTRIBUTORS: If this is the behavior you expect, please add your - // Linux distro here. - if ( osRelease.includes('ID=arch') ) { - config.os.distro = 'arch'; - config.os.archbtw = true; - } - } catch (_) { - // We don't care if we can't read this file; - // we'll just assume it's not a Linux distro. - } -} - -// config.os.refined specifies if Puter is running within a host environment -// where a higher level of user configuration and control is expected. -config.os.refined = config.os.archbtw; - -if ( config.os.refined ) { - config.no_browser_launch = true; -} - -// NEW_CONFIG_LOADING -const maybe_port = config => - config.pub_port !== 80 && config.pub_port !== 443 ? `:${ config.pub_port}` : ''; - -const computed_defaults = { - pub_port: config => config.http_port, - origin: config => `${config.protocol }://${ config.domain }${maybe_port(config)}`, - api_base_url: config => config.experimental_no_subdomain - ? config.origin - : `${config.protocol }://api.${ config.domain }${maybe_port(config)}`, - social_card: config => `${config.origin}/assets/img/screenshot.png`, - static_hosting_domain: config => `site.${ config.domain }${ maybe_port(config)}`, - // Hostname-only fallback helps host matching code paths that compare against req.hostname. - static_hosting_domain_alt: (config) => `site.${ config.domain }`, - private_app_hosting_domain: config => `app.${ config.domain }${ maybe_port(config)}`, - private_app_hosting_domain_alt: () => `app.${ config.domain }`, // Hostname-only fallback helps host matching code paths that compare against req.hostname. - -}; - -// We're going to export a config object that's decorated -// with additional behavior -let config_to_export; - -// We have a pointer to some config object which -// load_config() may replace -const config_pointer = {}; -{ - Object.setPrototypeOf(config_pointer, config); - config_to_export = config_pointer; -} - -// We have some methods that can be called on `config` -{ - // Add configuration values with precedence over the current config - const load_config = o => { - let replacement_config = { - ...o, - }; - replacement_config = deep_proto_merge(replacement_config, Object.getPrototypeOf(config_pointer), { - preserve_flag: true, - }); - Object.setPrototypeOf(config_pointer, replacement_config); - }; - - const config_api = { load_config }; - Object.setPrototypeOf(config_api, config_to_export); - config_to_export = config_api; -} - -// We have some values with computed defaults -{ - const get_implied = (target, prop) => { - if ( prop in computed_defaults ) { - return computed_defaults[prop](target); - } - return undefined; - }; - config_to_export = new Proxy(config_to_export, { - get: (target, prop, _receiver) => { - if ( prop in target ) { - return target[prop]; - } else { - return get_implied(config_to_export, prop); - } - }, - }); -} - -// We'd like to store values changed at runtime separately -// for easier runtime debugging -{ - const config_runtime_values = { - $: 'runtime-values', - }; - let initialPrototype = config_to_export; - Object.setPrototypeOf(config_runtime_values, config_to_export); - config_to_export = config_runtime_values; - - config_to_export.__set_config_object__ = (object, options = {}) => { - // options for this method - const replacePrototype = options.replacePrototype ?? true; - const useInitialPrototype = options.useInitialPrototype ?? true; - - // maybe replace prototype - if ( replacePrototype ) { - const newProto = useInitialPrototype - ? initialPrototype - : Object.getPrototypeOf(config_runtime_values); - Object.setPrototypeOf(object, newProto); - } - - // use this object as the prototype - Object.setPrototypeOf(config_runtime_values, object); - }; - - // These can be difficult to find and cause painful - // confusing issues, so we log any time this happens - config_to_export = new Proxy(config_to_export, { - set: (target, prop, value, _receiver) => { - const logger = Context.get('logger', { allow_fallback: true }); - // If no logger, just give up - if ( logger ) { - logger.debug( - '\x1B[36;1mCONFIGURATION MUTATED AT RUNTIME\x1B[0m', - { prop, value }, - ); - } - target[prop] = value; - return true; - }, - }); -} - -// We configure the behavior in context.js from here to avoid a cyclic -// mutual dependency between it and this file. -// -// Previously we had this: -// context --(are we in "dev" environment?)--> config -// -// So we could not add this: -// config --(where is the logger?) --> context -// -// So instead we now have: -// config --(read this property to determine 'strict' mode)--> context -// config --(where is the logger?) --> context -// -Object.defineProperty(context_config, 'strict', { - get: () => config_to_export.env === 'dev', - configurable: true, -}); - -module.exports = config_to_export; diff --git a/src/backend/src/config/ConfigLoader.js b/src/backend/src/config/ConfigLoader.js deleted file mode 100644 index dea34c56b..000000000 --- a/src/backend/src/config/ConfigLoader.js +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('@heyputer/putility'); -const { quot } = require('@heyputer/putility').libs.string; - -class ConfigLoader extends AdvancedBase { - static MODULES = { - path_: require('path'), - fs: require('fs'), - }; - - constructor (path, config) { - super(); - this.path = path; - this.config = config; - } - - enable (name, meta = {}) { - const { path_, fs } = this.modules; - - const config_path = path_.join(this.path, name); - - if ( ! fs.existsSync(config_path) ) { - throw new Error(`Config file not found: ${config_path}`); - } - - const config_values = JSON.parse(fs.readFileSync(config_path, 'utf8')); - if ( config_values.$requires ) { - const config_list = config_values.$requires; - delete config_values.$requires; - this.apply_requires(this.path, config_list, { by: name }); - } - console.debug(`Applying config: ${path_.relative(this.path, config_path)}${ - meta.by ? ` (required by ${meta.by})` : ''}`); - this.config.load_config(config_values); - - } - - apply_requires (dir, config_list, { by } = {}) { - const { path_, fs } = this.modules; - - for ( const name of config_list ) { - const config_path = path_.join(dir, name); - if ( ! fs.existsSync(config_path) ) { - throw new Error(`could not find ${quot(config_path)} ` + - `required by ${quot(by)}`); - } - this.enable(name, { by }); - } - } -} - -module.exports = { ConfigLoader }; \ No newline at end of file diff --git a/src/backend/src/config/deep_proto_merge.js b/src/backend/src/config/deep_proto_merge.js deleted file mode 100644 index a2032112b..000000000 --- a/src/backend/src/config/deep_proto_merge.js +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -/** - * Sets replacement.__proto__ to `delegate` - * then iterates over members of `replacement` looking for - * objects that are not arrays. - * - * When an object is found, a recursive call is made to - * `deep_proto_merge` with the corresponding object in `delegate`. - * - * If `preserve_flag` is set to true, only objects containing - * a truthy property named `$preserve` will be merged. - * - * @param {*} replacement - * @param {*} delegate - */ -const deep_proto_merge = (replacement, delegate, options) => { - const is_object = (obj) => obj && - typeof obj === 'object' && !Array.isArray(obj); - - replacement.__proto__ = delegate; - - for ( const key in replacement ) { - if ( ! is_object(replacement[key]) ) continue; - - if ( options?.preserve_flag && !replacement[key].$preserve ) { - continue; - } - if ( ! is_object(delegate[key]) ) { - continue; - } - replacement[key] = deep_proto_merge(replacement[key], delegate[key], options); - } - - // use a Proxy object to ensure all keys are present - // when listing keys of `replacement` - replacement = new Proxy(replacement, { - // no get needed - // no set needed - ownKeys: (target) => { - const ownProps = Reflect.ownKeys(target); // Get own property names and symbols, including non-enumerable - const protoProps = Reflect.ownKeys(Object.getPrototypeOf(target)); // Get prototype's properties - - // Combine and deduplicate properties using a Set, then convert back to an array - const s = new Set([ - ...protoProps, - ...ownProps, - ]); - - if ( options?.preserve_flag ) { - // remove $preserve if it exists - s.delete('$preserve'); - } - - return Array.from(s); - }, - getOwnPropertyDescriptor: (target, prop) => { - // Real descriptor - let descriptor = Object.getOwnPropertyDescriptor(target, prop); - - if ( descriptor ) return descriptor; - - // Immediate prototype descriptor - const proto = Object.getPrototypeOf(target); - descriptor = Object.getOwnPropertyDescriptor(proto, prop); - - if ( descriptor ) return descriptor; - - return undefined; - }, - - }); - - return replacement; -}; - -module.exports = deep_proto_merge; diff --git a/src/backend/src/config/reserved_words.js b/src/backend/src/config/reserved_words.js deleted file mode 100644 index d32a00ecf..000000000 --- a/src/backend/src/config/reserved_words.js +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -module.exports = [ - // system and apps - 'about', - 'api', - 'camera', - 'changelog', - 'cloudjs', - 'cloud.js', - 'code', - 'dev-center', - 'draw', - 'editor', - 'markus', - 'pdf', - 'photopea', - 'player', - 'terminal', - 'viewer', - 'www', - - // UNIX directories - 'share', - 'usr', - 'dev', - 'var', - 'etc', - 'tmp', - 'lib', - 'mnt', - 'opt', - 'bin', - - // others - 'admin', - 'ads', - 'alt', - 'api', - 'app', - 'apps', - 'audio', - 'auth', - 'badge', - 'beta', - 'business', - 'buy', - 'cdn', - 'cli', - 'cloud', - 'cmd', - 'community', - 'careers', - 'config', - 'db', - 'demo', - 'dev', - 'developers', - 'dns1', - 'dns2', - 'dns3', - 'dns4', - 'dns5', - 'dns6', - 'dns7', - 'dns8', - 'dns9', - 'dns0', - 'doc', - 'docs', - 'email', - 'eng', - 'engineering', - 'exchange', - 'faq', - 'feeds', - 'files', - 'forum', - 'fs', - 'ftp', - 'gov', - 'groups', - 'help', - 'hq', - 'images', - 'img', - 'in', - 'inbound', - 'info', - 'jobs', - 'js', - 'lab', - 'learn', - 'live', - 'login', - 'mail', - 'media', - 'mobile', - 'mx', - 'mx1', - 'mx2', - 'mx3', - 'mx4', - 'mx5', - 'mx6', - 'mx7', - 'mx8', - 'mx9', - 'mx0', - 'my', - 'mysql', - 'news', - 'newsletter', - 'ns1', - 'ns2', - 'ns3', - 'ns4', - 'ns5', - 'ns6', - 'ns7', - 'ns8', - 'ns9', - 'ns0', - 'office', - 'out', - 'owa', - 'pop', - 'pop3', - 'portal', - 'private', - 'public', - 'puter', - 'remote', - 'sandbox', - 'sdk', - 'search', - 'secure', - 'service', - 'shell', - 'shop', - 'signin', - 'signup', - 'smtp', - 'smtpin', - 'socket', - 'ssl', - 'start', - 'static', - 'status', - 'store', - 'support', - 'test', - 'tutorials', - 'upload', - 'video', - 'videos', - 'vpn', - 'vps', - 'web', - 'wiki', - 'www', - - '1', - '2', - '3', - '4', - '5', - '6', - '7', - '8', - '9', - '0', - - 'a', - 'b', - 'c', - 'd', - 'e', - 'f', - 'g', - 'h', - 'i', - 'j', - 'k', - 'l', - 'm', - 'n', - 'o', - 'p', - 'q', - 'r', - 's', - 't', - 'u', - 'v', - 'w', - 'x', - 'y', - 'z', -]; diff --git a/src/backend/src/consts/app-icons.js b/src/backend/src/consts/app-icons.js deleted file mode 100644 index 9e05df952..000000000 --- a/src/backend/src/consts/app-icons.js +++ /dev/null @@ -1 +0,0 @@ -export const APP_ICONS_SUBDOMAIN = 'puter-app-icons'; diff --git a/src/backend/src/definitions/SimpleEntity.js b/src/backend/src/definitions/SimpleEntity.js deleted file mode 100644 index a53e32505..000000000 --- a/src/backend/src/definitions/SimpleEntity.js +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { Context } = require('../util/context'); - -module.exports = function SimpleEntity ({ name, methods, fetchers }) { - const create = function (values) { - const entity = { values }; - Object.assign(entity, methods); - for ( const fetcher_name in fetchers ) { - entity[`fetch_${ fetcher_name}`] = async function () { - if ( Object.prototype.hasOwnProperty.call(this.values, fetcher_name) ) { - return this.values[fetcher_name]; - } - const value = await fetchers[fetcher_name].call(this); - this.values[fetcher_name] = value; - return value; - }; - } - entity.context = values.context ?? Context.get(); - entity.services = entity.context.get('services'); - return entity; - }; - - create.name = name; - return create; -}; diff --git a/src/backend/src/deprecated/filesystem/ECMAP.js b/src/backend/src/deprecated/filesystem/ECMAP.js deleted file mode 100644 index 77a8d0537..000000000 --- a/src/backend/src/deprecated/filesystem/ECMAP.js +++ /dev/null @@ -1,125 +0,0 @@ -const { Context } = require('../../util/context'); -const { NodeUIDSelector, NodePathSelector, NodeInternalIDSelector } = require('./node/selectors'); - -const LOG_PREFIX = '\x1B[31;1m[[\x1B[33;1mEC\x1B[32;1mMAP\x1B[31;1m]]\x1B[0m'; - -/** - * The ECMAP class is a memoization structure used by FSNodeContext - * whenever it is present in the execution context (AsyncLocalStorage). - * It is assumed that this object is transient and invalidation of stale - * entries is not necessary. - * - * The name ECMAP simple means Execution Context Map, because the map - * exists in memory at a particular frame of the execution context. - */ -class ECMAP { - static SYMBOL = Symbol('ECMAP'); - - constructor () { - this.identifier = require('uuid').v4(); - - // entry caches - this.uuid_to_fsNodeContext = {}; - this.path_to_fsNodeContext = {}; - this.id_to_fsNodeContext = {}; - - // identifier association caches - this.path_to_uuid = {}; - this.uuid_to_path = {}; - - this.unlinked = false; - } - - /** - * unlink() clears all references from this ECMAP to ensure that it will be - * GC'd. This is called by ECMAP.arun() after the callback has resolved. - */ - unlink () { - this.unlinked = true; - this.uuid_to_fsNodeContext = null; - this.path_to_fsNodeContext = null; - this.id_to_fsNodeContext = null; - this.path_to_uuid = null; - this.uuid_to_path = null; - } - - get logPrefix () { - return `${LOG_PREFIX} \x1B[36[1m${this.identifier}\x1B[0m`; - } - - log (...a) { - if ( ! process.env.LOG_ECMAP ) return; - console.log(this.logPrefix, ...a); - } - - get_fsNodeContext_from_selector (selector) { - if ( this.unlinked ) return null; - - this.log('GET', selector.describe()); - const retvalue = (() => { - let value; - if ( selector instanceof NodeUIDSelector ) { - value = this.uuid_to_fsNodeContext[selector.value]; - if ( value ) return value; - - let maybe_path = this.uuid_to_path[value]; - if ( ! maybe_path ) return; - value = this.path_to_fsNodeContext[maybe_path]; - if ( value ) return value; - } - else - if ( selector instanceof NodePathSelector ) { - value = this.path_to_fsNodeContext[selector.value]; - if ( value ) return value; - - let maybe_uid = this.path_to_uuid[value]; - value = this.uuid_to_fsNodeContext[maybe_uid]; - if ( value ) return value; - } - })(); - if ( retvalue ) { - this.log('\x1B[32;1m <<<<< ECMAP HIT >>>>> \x1B[0m'); - } else { - this.log('\x1B[31;1m <<<<< ECMAP MISS >>>>> \x1B[0m'); - } - return retvalue; - } - - store_fsNodeContext_to_selector (selector, node) { - if ( this.unlinked ) return null; - - this.log('STORE', selector.describe()); - if ( selector instanceof NodeUIDSelector ) { - this.uuid_to_fsNodeContext[selector.value] = node; - } - if ( selector instanceof NodePathSelector ) { - this.path_to_fsNodeContext[selector.value] = node; - } - if ( selector instanceof NodeInternalIDSelector ) { - this.id_to_fsNodeContext[`${selector.service}:${selector.id}`] = node; - } - } - - store_fsNodeContext (node) { - if ( this.unlinked ) return; - - this.store_fsNodeContext_to_selector(node.selector, node); - } - - static async arun (cb) { - let context = Context.get(); - if ( ! context.get(this.SYMBOL) ) { - const ins = new this(); - context = context.sub({ - [this.SYMBOL]: ins, - }); - const result = await context.arun(cb); - ins.unlink(); - context.unlink(); - return result; - } - return await cb(); - } -} - -module.exports = { ECMAP }; diff --git a/src/backend/src/deprecated/filesystem/FSNodeContext.js b/src/backend/src/deprecated/filesystem/FSNodeContext.js deleted file mode 100644 index 787e18eef..000000000 --- a/src/backend/src/deprecated/filesystem/FSNodeContext.js +++ /dev/null @@ -1,996 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import putility from '@heyputer/putility'; -import { dirname, join } from 'path'; -import config from '../../config.js'; -import { get_app, get_user, id2path, id2uuid, is_empty, suggestedAppForFsEntry } from '../../helpers.js'; -import { Actor, AppUnderUserActorType, UserActorType } from '../../services/auth/Actor.js'; -import { MANAGE_PERM_PREFIX } from '../../services/auth/permissionConts.mjs'; -import { PermissionUtil } from '../../services/auth/permissionUtils.mjs'; -import { DB_READ } from '../../services/database/consts.js'; -import { Context } from '../../util/context.js'; -import { getTracer, span } from '../../util/otelutil.js'; -import { ECMAP } from './ECMAP.js'; -import { NodeChildSelector, NodeInternalIDSelector, NodePathSelector, NodeRawEntrySelector, NodeUIDSelector, RootNodeSelector } from './node/selectors.js'; - -/** - * Container for information collected about a node - * on the filesystem. - * - * Examples of such information include: - * - data collected by querying an fsentry - * - the location of a file's contents - * - * This is an implementation of the Facade design pattern, - * so information about a filesystem node should be collected - * via the methods on this class and not mutated directly. - * - * @class FSNodeContext - * @property {object} entry the filesystem entry - * @property {string} path the path to the filesystem entry - * @property {string} uid the UUID of the filesystem entry - */ - -export const TYPE_FILE = { label: 'File' }; -export const TYPE_DIRECTORY = { label: 'Directory' }; -export default class FSNodeContext { - static CONCERN = 'filesystem'; - - static TYPE_FILE = TYPE_FILE; - static TYPE_DIRECTORY = TYPE_DIRECTORY; - static TYPE_SYMLINK = {}; - static TYPE_SHORTCUT = {}; - static TYPE_UNDETERMINED = {}; - - static SELECTOR_PRIORITY_ORDER = [ - NodeRawEntrySelector, - RootNodeSelector, - NodeInternalIDSelector, - NodeUIDSelector, - NodeChildSelector, - NodePathSelector, - ]; - - #writable; - - /** - * Creates an instance of FSNodeContext. - * @param {*} opt_identifier - * @param {*} opt_identifier.path a path to the filesystem entry - * @param {*} opt_identifier.uid a UUID of the filesystem entry - * @param {*} opt_identifier.id please pass mysql_id instead - * @param {*} opt_identifier.mysql_id a MySQL ID of the filesystem entry - */ - constructor ({ - services, - selector, - provider, - fs, - }) { - const ecmap = Context.get(ECMAP.SYMBOL); - - if ( ecmap && !(selector instanceof NodeRawEntrySelector) ) { - // We might return an existing FSNodeContext - const maybe_node = ecmap - ?.get_fsNodeContext_from_selector?.(selector); - if ( maybe_node ) return maybe_node; - } else { - if ( process.env.LOG_ECMAP ) { - console.log('\x1B[31;1m !!! NO ECMAP !!! \x1B[0m'); - } - } - - // This will be used to avoid concurrent fetches. Whenever an entry is being fetched, - // a subsequent call to fetchEntry must await this promise. Usually this means the - // subsequent call will not perform any expensive operations. - this.fetching = null; - - this.log = services.get('log-service').create('fsnode-context', { - concern: this.constructor.CONCERN, - }); - this.selector_ = null; - this.selectors_ = []; - this.selector = selector; - this.provider = provider; - this.entry = {}; - this.found = undefined; - this.found_thumbnail = undefined; - - selector.setPropertiesKnownBySelector(this); - - this.services = services; - - this.fileContentsFetcher = null; - - this.fs = fs; - - // Decorate all fetch methods with otel span - // TODO: Apply method decorators using a putility class feature - const fetch_methods = [ - 'fetchEntry', - 'fetchPath', - 'fetchSubdomains', - 'fetchOwner', - 'fetchShares', - 'fetchVersions', - 'fetchSize', - 'fetchSuggestedApps', - 'fetchIsEmpty', - ]; - for ( const method of fetch_methods ) { - const original_method = this[method]; - this[method] = async (...args) => { - const tracer = getTracer(); - let result; - const opts = { attributes: { - selector: selector.describe(), - trace: (new Error()).stack, - } }; - await tracer.startActiveSpan(`fs:nodectx:fetch:${method}`, opts, async span => { - result = await original_method.call(this, ...args); - span.end(); - }); - return result; - }; - } - } - - set selector (new_selector) { - // Only add the selector if we don't already have it - for ( const selector of this.selectors_ ) { - if ( selector instanceof new_selector.constructor ) return; - } - - const ecmap = Context.get(ECMAP.SYMBOL); - if ( ecmap ) { - ecmap.store_fsNodeContext_to_selector(new_selector, this); - } - - this.selectors_.push(new_selector); - this.selector_ = new_selector; - } - - get selector () { - return this.get_optimal_selector(); - } - - get_selector_of_type (cls) { - // Reverse iterate over selectors - for ( let i = this.selectors_.length - 1; i >= 0; i-- ) { - const selector = this.selectors_[i]; - if ( selector instanceof cls ) { - return selector; - } - } - - if ( cls.implyFromFetchedData ) { - return cls.implyFromFetchedData(this); - } - - return null; - } - - get_optimal_selector () { - for ( const cls of FSNodeContext.SELECTOR_PRIORITY_ORDER ) { - const selector = this.get_selector_of_type(cls); - if ( selector ) return selector; - } - this.log.warn('Failed to get optimal selector'); - return this.selector_; - } - - get isRoot () { - return this.path === '/'; - } - - async isUserDirectory () { - if ( this.isRoot ) return false; - if ( this.found === undefined ) { - await this.fetchEntry(); - } - if ( this.isRoot ) return false; - if ( this.found === false ) return undefined; - return !this.entry.parent_uid; - } - - async isAppDataDirectory () { - if ( this.isRoot ) return false; - if ( this.found === undefined ) { - await this.fetchEntry(); - } - if ( this.isRoot ) return false; - - const components = await this.getPathComponents(); - if ( components.length < 2 ) return false; - return components[1] === 'AppData'; - } - - async isPublic () { - if ( this.isRoot ) return false; - const components = await this.getPathComponents(); - if ( await this.isUserDirectory() ) return false; - if ( components[1] === 'Public' ) return true; - return false; - } - - async getPathComponents () { - if ( this.isRoot ) return []; - - // We can get path components for non-existing nodes if they - // have a path selector - if ( ! await this.exists() ) { - if ( this.selector instanceof NodePathSelector ) { - let path = this.selector.value; - if ( path.startsWith('/') ) path = path.slice(1); - return path.split('/'); - } - - // TODO: add support for NodeChildSelector as well - } - - let path = await this.get('path'); - if ( path.startsWith('/') ) path = path.slice(1); - return path.split('/'); - } - - async getUserPart () { - if ( this.isRoot ) return; - const components = await this.getPathComponents(); - return components[0]; - } - - async getPathSize () { - if ( this.isRoot ) return; - const components = await this.getPathComponents(); - return components.length; - } - - async exists ({ fetch_options } = {}) { - if ( this.found !== undefined ) { - return this.found; - } - await this.fetchEntry(fetch_options); - if ( ! this.found ) { - this.log.debug(`here's why it doesn't exist: ${ - this.selector.describe() } -> ${ - this.uid } ${ - JSON.stringify(this.entry, null, ' ')}`); - } - return this.found; - } - - async fetchPath () { - if ( this.path ) return; - if ( this.entry?.path ) { - this.path = this.entry.path; - return; - } - const uid = this.entry?.uuid ?? this.uid; - if ( ! uid ) return; - this.path = await this.#resolvePathFromUuid(uid); - } - - async #resolvePathFromUuid (uuid) { - if ( ! uuid ) return undefined; - try { - return await id2path(uuid); - } catch (e) { - return `/-void/${ uuid }`; - } - } - - /** - * Fetches the filesystem entry associated with a - * filesystem node identified by a path or UID. - * - * If a UID exists, the path is ignored. - * If neither a UID nor a path is set, an error is thrown. - * - * @param {*} fsEntryFetcher fetches the filesystem entry - * @void - */ - async fetchEntry (fetch_entry_options = {}) { - if ( this.fetching !== null ) { - await span('fetching', async () => { - // ???: does this need to be double-checked? I'm not actually sure... - if ( this.fetching === null ) return; - await this.fetching; - }); - } - this.fetching = new putility.libs.promise.TeePromise(); - - if ( - this.found === true && - !fetch_entry_options.force && - ( - // thumbnail already fetched, or not asked for - !fetch_entry_options.thumbnail || this.entry?.thumbnail || - this.found_thumbnail !== undefined - ) - ) { - const promise = this.fetching; - this.fetching = null; - promise.resolve(); - return; - } - - const controls = { - log: this.log, - provide_selector: selector => { - this.selector = selector; - }, - }; - - this.log.debug(`fetching entry: ${ this.selector.describe()}`); - - const entry = await this.provider.stat({ - selector: this.selector, - options: fetch_entry_options, - node: this, - controls, - }); - - if ( ! entry ) { - this.found = false; - this.entry = false; - } else { - this.found = true; - - if ( !this.uid && entry.uuid ) { - this.uid = entry.uuid; - } - - if ( !this.mysql_id && entry.id ) { - this.mysql_id = entry.id; - } - - if ( !this.path && entry.path ) { - this.path = entry.path; - } - - if ( !this.name && entry.name ) { - this.name = entry.name; - } - - Object.assign(this.entry, entry); - } - - const promise = this.fetching; - this.fetching = null; - - promise.resolve(); - } - - /** - * Wait for an fsentry which might be enqueued for insertion - * into the database. - * - * This just calls ResourceService under the hood. - */ - async awaitStableEntry () { - const resourceService = Context.get('services').get('resourceService'); - await resourceService.waitForResource(this.selector); - } - - /** - * Fetches the subdomains associated with a directory or file - * and stores them on the `subdomains` property of the fsentry. - * @param {object} user the user is needed to query subdomains - * @param {bool} force fetch subdomains if they were already fetched - * - * @param fs:decouple-subdomains - */ - async fetchSubdomains (user, _force) { - const db = this.services.get('database').get(DB_READ, 'filesystem'); - - this.entry.subdomains = []; - this.entry.workers = []; - this.entry.has_website = false; - const subdomains = await db.read( - 'SELECT * FROM subdomains WHERE root_dir_id = ? AND user_id = ?', - [this.entry.id, user.id], - ); - if ( subdomains.length > 0 ) { - subdomains.forEach((sd) => { - this.applySingleSubdomain(sd); - }); - this.entry.has_website = true; - } - } - - applySingleSubdomain (sd) { - if ( this.entry.is_dir ) { - this.entry.subdomains.push({ - subdomain: sd.subdomain, - address: `${config.protocol }://${ sd.subdomain }.` + 'puter.site', - uuid: sd.uuid, - }); - } else { - const workerName = sd.subdomain.split('.').pop(); - this.entry.workers.push({ - subdomain: workerName, - address: `https://${ workerName }.` + 'puter.work', - uuid: sd.uuid, - }); - } - } - - /** - * Fetches the owner of a directory or file and stores it on the - * `owner` property of the fsentry. - * @param {bool} force fetch owner if it was already fetched - */ - async fetchOwner (_force) { - if ( this.isRoot ) return; - const owner = await get_user({ id: this.entry.user_id }); - this.entry.owner = { - username: owner.username, - email: owner.email, - }; - } - - /** - * Fetches shares, AKA "permissions", for a directory or file; - * then, stores them on the `permissions` property - * of the fsentry. - * @param {bool} force fetch shares if they were already fetched - */ - async fetchShares (force) { - if ( this.entry.shares && !force ) return; - - const actor = Context.get('actor'); - if ( ! actor ) { - this.entry.shares = { users: [], apps: [] }; - return; - } - - if ( ! (actor.type instanceof UserActorType) ) { - this.entry.shares = { users: [], apps: [] }; - return; - } - - const svc_permission = this.services.get('permission'); - - const fsPermPrefix = `fs:${await this.get('uid')}`; - const [readWritePerms, managePerms] = await Promise.all([ - svc_permission.query_issuer_permissions_by_prefix(actor.type.user, `${fsPermPrefix}:`), - svc_permission.query_issuer_permissions_by_prefix(actor.type.user, `${MANAGE_PERM_PREFIX}:${fsPermPrefix}`), - ]); - - this.entry.shares = { users: [], apps: [] }; - - for ( const readWriteUserPerms of readWritePerms.users ) { - const access = - PermissionUtil.split(readWriteUserPerms.permission).slice(-1)[0]; - this.entry.shares.users.push({ - user: { - uid: readWriteUserPerms.user.uuid, - username: readWriteUserPerms.user.username, - }, - access, - permission: readWriteUserPerms.permission, - }); - } - for ( const manageUserPerms of managePerms.users ) { - const access = MANAGE_PERM_PREFIX; - this.entry.shares.users.push({ - user: { - uid: manageUserPerms.user.uuid, - username: manageUserPerms.user.username, - }, - access, - permission: manageUserPerms.permission, - }); - } - - for ( const readWriteAppPerms of readWritePerms.apps ) { - const access = - PermissionUtil.split(readWriteAppPerms.permission).slice(-1)[0]; - this.entry.shares.apps.push({ - app: { - icon: readWriteAppPerms.app.icon, - uid: readWriteAppPerms.app.uid, - name: readWriteAppPerms.app.name, - }, - access, - permission: readWriteAppPerms.permission, - }); - } - - for ( const manageAppPerms of readWritePerms.apps ) { - const access = - MANAGE_PERM_PREFIX; - this.entry.shares.apps.push({ - app: { - icon: manageAppPerms.app.icon, - uid: manageAppPerms.app.uid, - name: manageAppPerms.app.name, - }, - access, - permission: manageAppPerms.permission, - }); - } - } - - /** - * Fetches versions associated with a filesystem entry, - * then stores them on the `versions` property of - * the fsentry. - * @param {bool} force fetch versions if they were already fetched - * - * @todo fs:decouple-versions - */ - async fetchVersions (force) { - if ( this.entry.versions && !force ) return; - - const db = this.services.get('database').get(DB_READ, 'filesystem'); - - let versions = await db.read( - 'SELECT * FROM fsentry_versions WHERE fsentry_id = ?', - [this.entry.id], - ); - const versions_tidy = []; - for ( const version of versions ) { - let username = version.user_id ? (await get_user({ id: version.user_id })).username : null; - versions_tidy.push({ - id: version.version_id, - message: version.message, - timestamp: version.ts_epoch, - user: { - username: username, - }, - }); - } - - this.entry.versions = versions_tidy; - } - - /** - * Fetches the size of a file or directory if it was not - * already fetched. - */ - async fetchSize () { - // we already have the size for files - if ( ! this.entry.is_dir ) { - await this.fetchEntry(); - return this.entry.size; - } - - this.entry.size = await this.provider.get_recursive_size({ node: this }); - - return this.entry.size; - } - - /** Avoid using if fetching directory items */ - async fetchSuggestedApps (user, force) { - if ( this.entry.suggested_apps && !force ) return; - - await this.fetchEntry(); - if ( ! this.entry ) return; - - this.entry.suggested_apps = - await suggestedAppForFsEntry(this.entry, { user }); - } - - async fetchIsEmpty () { - if ( !this.uid && !this.path ) return; - this.entry && (this.entry.is_empty = await is_empty({ - uid: this.uid, - path: this.path, - })); - } - - async fetchAll (_fsEntryFetcher, user, _force) { - await this.fetchEntry({ thumbnail: true }); - await this.fetchSubdomains(user); - await this.fetchOwner(); - await this.fetchShares(); - await this.fetchVersions(); - await this.fetchSize(user); - await this.fetchSuggestedApps(user); - await this.fetchIsEmpty(); - } - - async get (key, force) { - /* - This isn't supposed to stay like this! - - """ if ( key === something ) return this """ - - ^ we should use a map of getters instead - - Ideally I'd like to make a class trait for classes like - FSNodeContext that provide a key-value facade to access - information about some entity. - */ - - if ( this.found === false ) { - throw new Error(`Tried to get ${key} of non-existent fsentry: ${ - this.selector.describe(true)}`); - } - - if ( key === 'entry' ) { - await this.fetchEntry(); - if ( this.found === false ) { - throw new Error(`Tried to get entry of non-existent fsentry: ${ - this.selector.describe(true)}`); - } - return this.entry; - } - - if ( key === 'path' ) { - if ( ! this.path ) await this.fetchEntry(); - if ( this.found === false ) { - throw new Error(`Tried to get path of non-existent fsentry: ${ - this.selector.describe(true)}`); - } - if ( ! this.path ) { - await this.fetchPath(); - } - if ( ! this.path ) { - throw new Error('failed to get path'); - } - return this.path; - } - - if ( key === 'uid' || key === 'uuid' ) { - const uidSelector = this.get_selector_of_type(NodeUIDSelector); - if ( uidSelector ) { - return uidSelector.value; - } - await this.fetchEntry(); - return this.uid; - } - - if ( key === 'mysql-id' ) { - await this.fetchEntry(); - return this.mysql_id ?? this.entry.id; - } - - if ( key === 'owner' ) { - const user_id = await this.get('user_id'); - const actor = new Actor({ - type: new UserActorType({ - user: await get_user({ id: user_id }), - }), - }); - return actor; - } - - const values_from_entry = ['immutable', 'user_id', 'name', 'size', 'parent_uid', 'metadata']; - for ( const k of values_from_entry ) { - if ( key === k ) { - await this.fetchEntry(); - if ( this.found === false ) { - throw new Error(`Tried to get ${key} of non-existent fsentry: ${ - this.selector.describe(true)}`); - } - return this.entry[k]; - } - } - - if ( key === 'type' ) { - await this.fetchEntry(); - - // Longest ternary operator chain I've ever written? - return this.entry.is_shortcut - ? FSNodeContext.TYPE_SHORTCUT - : this.entry.is_symlink - ? FSNodeContext.TYPE_SYMLINK - : this.entry.is_dir - ? FSNodeContext.TYPE_DIRECTORY - : FSNodeContext.TYPE_FILE; - } - - if ( key === 'has-s3' ) { - await this.fetchEntry(); - if ( this.entry.is_dir ) return false; - if ( this.entry.is_shortcut ) return false; - return true; - } - - if ( key === 's3:location' ) { - await this.fetchEntry(); - if ( ! await this.exists() ) { - throw new Error('file does not exist'); - } - return { - bucket: this.entry.bucket ?? config.s3_bucket ?? 'puter-local', - bucket_region: this.entry.bucket_region ?? config.s3_region ?? config.region - ?? 'us-west-2', - key: this.entry.uuid, - }; - } - - if ( key === 'is-root' ) { - await this.fetchEntry(); - return this.isRoot; - } - - if ( key === 'writable' ) { - if ( this.#writable && !force ) return this.#writable; - const actor = Context.get('actor'); - if ( !actor || !actor.type.user ) return undefined; - const svc_acl = this.services.get('acl'); - return this.#writable = await svc_acl.check(actor, this, 'write'); - } - - throw new Error(`unrecognize key for FSNodeContext.get: ${key}`); - } - - async getParent () { - if ( this.isRoot ) { - throw new Error('tried to get parent of root'); - } - - if ( this.path ) { - const parent_fsNode = await this.fs.node({ - path: dirname(this.path), - }); - return parent_fsNode; - } - - if ( this.selector instanceof NodeChildSelector ) { - return this.fs.node(this.selector.parent); - } - - if ( ! await this.exists() ) { - throw new Error('unable to get parent'); - } - - const parent_uid = this.entry.parent_uid; - - if ( ! parent_uid ) { - return this.fs.node(new RootNodeSelector()); - } - - return this.fs.node(new NodeUIDSelector(parent_uid)); - } - - async getChild (name) { - // If we have a path, we can get an FSNodeContext for the child - // without fetching anything. - if ( this.path ) { - const child_fsNode = await this.fs.node({ - path: join(this.path, name), - }); - return child_fsNode; - } - - return await this.fs.node(new NodeChildSelector(this.selector, name)); - } - - async hasChild (name) { - return await this.provider.directory_has_name({ parent: this, name }); - } - - async getTarget () { - await this.fetchEntry(); - const type = await this.get('type'); - - if ( type === FSNodeContext.TYPE_SYMLINK ) { - const path = await this.entry.symlink_path; - return await this.fs.node({ path }); - } - - if ( type === FSNodeContext.TYPE_SHORTCUT ) { - const target_id = await this.entry.shortcut_to; - return await this.fs.node({ mysql_id: target_id }); - } - - return this; - } - - async is_above (child_fsNode) { - if ( this.isRoot ) return true; - - const path_this = await this.get('path'); - const path_child = await child_fsNode.get('path'); - - return path_child.startsWith(`${path_this }/`); - } - - async is (fsNode) { - if ( this.mysql_id && fsNode.mysql_id ) { - return this.mysql_id === fsNode.mysql_id; - } - - if ( this.uid && fsNode.uid ) { - return this.uid === fsNode.uid; - } - - if ( this.path && fsNode.path ) { - return await this.get('path') === await fsNode.get('path'); - } - - await this.fetchEntry(); - await fsNode.fetchEntry(); - return this.uid === fsNode.uid; - } - - async getSafeEntry (fetch_options = {}) { - const svc_event = this.services.get('event'); - - if ( this.found === false ) { - throw new Error(`Tried to get entry of non-existent fsentry: ${ - this.selector.describe(true)}`); - } - await this.fetchEntry(fetch_options); - - const res = this.entry; - const fsentry = {}; - if ( res.thumbnail ) { - await svc_event.emit('thumbnail.read', this.entry); - } - - // This property will not be serialized, but it can be checked - // by other code to verify that API calls do not send - // unsanitized filsystem entries. - Object.defineProperty(fsentry, '__is_safe__', { - enumerable: false, - value: true, - }); - - for ( const k in res ) { - fsentry[k] = res[k]; - } - - let actor; try { - actor = Context.get('actor'); - } catch ( _e ) { - // fail silently - } - if ( !actor?.type?.user || actor.type.user.id !== res.user_id ) { - if ( ! fsentry.owner ) await this.fetchOwner(); - fsentry.owner = { - username: res.owner?.username, - }; - } - if ( ! ( actor.type === AppUnderUserActorType ) ) { - if ( fsentry.owner ) delete fsentry.owner.email; - } - - if ( !this.uid && !this.entry.uuid ) { - console.warn(`Potential Error in getSafeEntry with no uid or entry.uuid ${ - this.selector.describe() } ${ - JSON.stringify(this.entry, null, ' ')}`); - } - - // If fsentry was found by a path but the entry doesn't - // have a path, use the path that was used to find it. - const entry_uid = this.uid ?? this.entry.uuid; - fsentry.path = res.path ?? this.path ?? await this.#resolvePathFromUuid(entry_uid); - - if ( fsentry.path && fsentry.path.startsWith('/-void/') ) { - fsentry.broken = true; - } - - fsentry.dirname = dirname(fsentry.path); - fsentry.dirpath = fsentry.dirname; - fsentry.writable = await this.get('writable'); - - // Do not send internal IDs to clients - fsentry.id = res.uuid; - fsentry.parent_id = res.parent_uid; - // The client calls it uid, not uuid. - fsentry.uid = res.uuid; - delete fsentry.uuid; - delete fsentry.user_id; - if ( fsentry.suggested_apps ) { - for ( const app of fsentry.suggested_apps ) { - if ( app === null ) { - this.log.warn('null app'); - continue; - } - delete app.owner_user_id; - } - } - - // Do not send S3 bucket information to clients - delete fsentry.bucket; - delete fsentry.bucket_region; - - // Use client-friendly IDs for shortcut_to - fsentry.shortcut_to = (res.shortcut_to - ? await id2uuid(res.shortcut_to) : undefined); - try { - fsentry.shortcut_to_path = (res.shortcut_to - ? await id2path(res.shortcut_to) : undefined); - } catch ( _e ) { - fsentry.shortcut_invalid = true; - fsentry.shortcut_uid = res.shortcut_to; - } - - // Add file_request_url - if ( res.file_request_token && res.file_request_token !== '' ) { - fsentry.file_request_url = `${config.origin - }/upload?token=${ res.file_request_token}`; - } - - if ( fsentry.associated_app_id ) { - if ( res.associated_app ) { - fsentry.associated_app = res.associated_app; - } else { - const app = await get_app({ id: fsentry.associated_app_id }); - fsentry.associated_app = app; - } - } - - // If this file is in an appdata directory, add `appdata_app` - const components = await this.getPathComponents(); - if ( components[1] === 'AppData' ) { - fsentry.appdata_app = components[2]; - } - - fsentry.is_dir = !!fsentry.is_dir; - - // Ensure `size` is numeric - if ( fsentry.size ) { - fsentry.size = parseInt(fsentry.size); - } - - if ( ! Array.isArray(fsentry.subdomains) ) { - fsentry.subdomains = []; - } - if ( ! Array.isArray(fsentry.workers) ) { - fsentry.workers = []; - } - if ( typeof fsentry.has_website !== 'boolean' ) { - fsentry.has_website = false; - } - - return fsentry; - } - - static sanitize_pending_entry_info (res) { - const fsentry = {}; - - // This property will not be serialized, but it can be checked - // by other code to verify that API calls do not send - // unsanitized filsystem entries. - Object.defineProperty(fsentry, '__is_safe__', { - enumerable: false, - value: true, - }); - - for ( const k in res ) { - fsentry[k] = res[k]; - } - - fsentry.dirname = dirname(fsentry.path); - - // Do not send internal IDs to clients - fsentry.id = res.uuid; - fsentry.parent_id = res.parent_uid; - // The client calls it uid, not uuid. - fsentry.uid = res.uuid; - - delete fsentry.uuid; - delete fsentry.user_id; - - // Do not send S3 bucket information to clients - delete fsentry.bucket; - delete fsentry.bucket_region; - - delete fsentry.shortcut_to; - delete fsentry.shortcut_to_path; - - return fsentry; - } -}; diff --git a/src/backend/src/deprecated/filesystem/FilesystemService.js b/src/backend/src/deprecated/filesystem/FilesystemService.js deleted file mode 100644 index 3eddf89d1..000000000 --- a/src/backend/src/deprecated/filesystem/FilesystemService.js +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -// TODO: database access can be a service -const { NodePathSelector, NodeUIDSelector, NodeInternalIDSelector, NodeSelector } = require('./node/selectors.js'); -const FSNodeContext = require('./FSNodeContext.js').default; -const { Context } = require('../../util/context.js'); -const APIError = require('../../api/APIError.js'); -const { PermissionUtil, PermissionRewriter, PermissionImplicator, PermissionExploder } = require('../../services/auth/permissionUtils.mjs'); -const { DB_WRITE } = require('../../services/database/consts.js'); -const { UserActorType } = require('../../services/auth/Actor.js'); -const { get_user } = require('../../helpers.js'); -const BaseService = require('../../services/BaseService.js'); -const { MANAGE_PERM_PREFIX } = require('../../services/auth/permissionConts.mjs'); -const { quot } = require('@heyputer/putility/src/libs/string.js'); -const fsCapabilities = require('./definitions/capabilities.js'); -const { RESOURCE_STATUS_PENDING_CREATE } = require('../../modules/puterfs/ResourceService.js'); - -/** @deprecated */ -class FilesystemService extends BaseService { - static MODULES = { - _path: require('path'), - uuidv4: require('uuid').v4, - config: require('../../config.js'), - }; - - old_constructor (args) { - const { services } = args; - - // The new fs entry service - this.log = services.get('log-service').create('filesystem-service'); - - // used by update_child_paths - this.db = services.get('database').get(DB_WRITE, 'filesystem'); - - } - - async _init () { - this.old_constructor({ services: this.services }); - const svc_permission = this.services.get('permission'); - svc_permission.register_rewriter(PermissionRewriter.create({ - matcher: permission => { - if ( !permission.startsWith('fs:') && !permission.startsWith('manage:fs:') ) return false; - const [_, specifier] = permission.split('fs:'); - if ( ! specifier.startsWith('/') ) return false; - return true; - }, - rewriter: async permission => { - const [manageOpt, pathPerm] = permission.split('fs:'); - const [path, ...rest] = PermissionUtil.split(pathPerm); - const node = await this.node(new NodePathSelector(path)); - if ( ! await node.exists() ) { - // TOOD: we need a general-purpose error that can have - // a user-safe message, instead of using APIError - // which is for API errors. - throw APIError.create('subject_does_not_exist'); - } - const uid = await node.get('uid'); - if ( uid === undefined || uid === 'undefined' ) { - throw new Error(`uid is undefined for path ${path}`); - } - return [manageOpt.replace(':', ''), 'fs', uid, ...rest].filter(Boolean).join(':'); - }, - })); - svc_permission.register_implicator(PermissionImplicator.create({ - id: 'is-owner', - shortcut: true, - matcher: permission => { - // TODO DS: for now users will only have manage access on files, that might change, and then this has to change too - return permission.startsWith('fs:') - || permission.startsWith(`${MANAGE_PERM_PREFIX}:fs:`) - || permission.startsWith(`${MANAGE_PERM_PREFIX}:${MANAGE_PERM_PREFIX}:fs:`); // owner has implicit rule to give others manage access; - }, - checker: async ({ actor, permission }) => { - if ( ! (actor.type instanceof UserActorType) ) { - return undefined; - } - - const [_, uid] = PermissionUtil.split(permission.replaceAll(`${MANAGE_PERM_PREFIX}:`, '')); - const node = await this.node(new NodeUIDSelector(uid)); - - if ( ! await node.exists() ) { - return undefined; - } - - const owner_id = await node.get('user_id'); - - // These conditions should never happen - if ( !owner_id || !actor.type.user.id ) { - throw new Error('something unexpected happened'); - } - - if ( owner_id === actor.type.user.id ) { - return {}; - } - - return undefined; - }, - })); - svc_permission.register_exploder(PermissionExploder.create({ - id: 'fs-access-levels', - matcher: permission => { - return permission.startsWith('fs:') && - PermissionUtil.split(permission).length >= 3; - }, - exploder: async ({ permission }) => { - const permissions = [permission]; - const [fsPrefix, fileId, specifiedMode, ...rest] = PermissionUtil.split(permission); - - const rules = { - see: ['list', 'read', 'write'], - list: ['read', 'write'], - read: ['write'], - }; - - if ( rules[specifiedMode] ) { - permissions.push(...rules[specifiedMode].map(mode => PermissionUtil.join(fsPrefix, fileId, mode, ...rest.slice(1)))); - // push manage permission as well - permissions.push(PermissionUtil.join(MANAGE_PERM_PREFIX, fsPrefix, fileId)); - } - - return permissions; - }, - })); - } - - async mkshortcut ({ parent, name, user, target }) { - - // Access Control - { - const svc_acl = this.services.get('acl'); - - if ( ! await svc_acl.check(user, target, 'read') ) { - throw await svc_acl.get_safe_acl_error(user, target, 'read'); - } - - if ( ! await svc_acl.check(user, parent, 'write') ) { - throw await svc_acl.get_safe_acl_error(user, parent, 'write'); - } - } - - if ( ! await target.exists() ) { - throw APIError.create('shortcut_to_does_not_exist'); - } - - if ( ! parent.provider.get_capabilities().has(fsCapabilities.PUTER_SHORTCUT) ) { - throw APIError.create('missing_filesystem_capability', null, { - action: 'make shortcut', - subjectName: parent.path ?? parent.uid, - providerName: parent.provider.name, - capability: 'PUTER_SHORTCUT', - }); - } - - return await parent.provider.puter_shortcut({ - parent, name, user, target, - }); - } - - async mklink ({ parent, name, user, target }) { - - // Access Control - { - const svc_acl = this.services.get('acl'); - - if ( ! await svc_acl.check(user, parent, 'write') ) { - throw await svc_acl.get_safe_acl_error(user, parent, 'write'); - } - } - - // We don't check if the target exists because broken links - // are allowed. - - const { _path, uuidv4 } = this.modules; - const resourceService = this.services.get('resourceService'); - const svc_fsEntry = this.services.get('fsEntryService'); - - const ts = Math.round(Date.now() / 1000); - const uid = uuidv4(); - - resourceService.register({ - uid, - status: RESOURCE_STATUS_PENDING_CREATE, - }); - - const raw_fsentry = { - is_symlink: 1, - symlink_path: target, - is_dir: 0, - uuid: uid, - parent_uid: await parent.get('uid'), - path: _path.join(await parent.get('path'), name), - user_id: user.id, - name, - created: ts, - updated: ts, - modified: ts, - immutable: false, - }; - - this.log.debug('creating symlink', { fsentry: raw_fsentry }); - - const entryOp = await svc_fsEntry.insert(raw_fsentry); - - (async () => { - await entryOp.awaitDone(); - this.log.debug('finished creating symlink', { uid }); - resourceService.free(uid); - })(); - - const node = await this.node(new NodeUIDSelector(uid)); - - const svc_event = this.services.get('event'); - svc_event.emit('fs.create.symlink', { - node, - context: Context.get(), - }); - - return node; - } - - async update_child_paths (old_path, new_path, user_id) { - - if ( ! old_path.endsWith('/') ) old_path += '/'; - if ( ! new_path.endsWith('/') ) new_path += '/'; - // TODO: fs:decouple-tree-storage - await this.db.write( - 'UPDATE fsentries SET path = CONCAT(?, SUBSTRING(path, ?)) WHERE path LIKE ? AND user_id = ?', - [new_path, old_path.length + 1, `${old_path}%`, user_id], - ); - - const log = this.services.get('log-service').create('update_child_paths'); - log.debug(`updated ${old_path} -> ${new_path}`); - - } - - /** - * node() returns a filesystem node using path, uid, - * or id associated with a filesystem node. Use this - * method when you need to get a filesystem node and - * need to collect information about the entry. - * - * @param {*} location - path, uid, or id associated with a filesystem node - * @returns - */ - async node (selector) { - if ( typeof selector === 'string' ) { - if ( selector.startsWith('/') ) { - selector = new NodePathSelector(selector); - } - } - - // COERCE: legacy selection objects to Node*Selector objects - if ( - typeof selector === 'object' && - selector.constructor.name === 'Object' - ) { - if ( selector.path ) { - selector = new NodePathSelector(selector.path); - } else if ( selector.uid ) { - selector = new NodeUIDSelector(selector.uid); - } else { - selector = new NodeInternalIDSelector('mysql', selector.mysql_id); - } - } - - if ( ! (selector instanceof NodeSelector) ) { - throw new Error(`FileSystemService could not resolve the specified node value ${ - quot(`${ selector}`) } (type: ${typeof selector}) ` + - 'to a filesystem node selector'); - } - - system_dir_check: { - if ( ! (selector instanceof NodePathSelector) ) break system_dir_check; - if ( ! selector.value.startsWith('/') ) break system_dir_check; - - // OPTIMIZATION: Check if the path matches a system directory pattern. - const systemDirRegex = /^\/([a-zA-Z0-9_]+)\/(Trash|AppData|Desktop|Documents|Pictures|Videos|Public)$/; - const match = selector.value.match(systemDirRegex); - if ( ! match ) break system_dir_check; - - const username = match[1]; - const dirName = match[2]; - - // Get the user object (this is likely cached). - const user = await get_user({ username }); - if ( ! user ) break system_dir_check; - - let uuidKey = ( selector.value === `/${user.username}` ) - ? 'home_uuid' - : `${dirName.toLowerCase()}_uuid`; // e.g., 'desktop_uuid' - - const cachedUUID = user[uuidKey]; - if ( ! cachedUUID ) break system_dir_check; - - // If we have a cached ID, use it for more direct lookup. - selector = new NodeUIDSelector(cachedUUID); - } - - const svc_mountpoint = this.services.get('mountpoint'); - const provider = await svc_mountpoint.get_provider(selector); - - let fsNode = new FSNodeContext({ - provider, - services: this.services, - selector, - fs: this, - }); - - return fsNode; - } - - /** - * get_entry() returns a filesystem entry using - * path, uid, or id associated with a filesystem - * node. Use this method when you need to get a - * filesystem entry but don't need to collect any - * other information about the entry. - * - * @warning The entry returned by this method is not - * client-safe. Use FSNodeContext to get a client-safe - * entry by calling it's fetchEntry() method. - * - * @param {*} param0 options for getting the entry - * @param {*} param0.path - * @param {*} param0.uid - * @param {*} param0.id please use mysql_id instead - * @param {*} param0.mysql_id - */ - async get_entry ({ path, uid, id, mysql_id, ...options }) { - let fsNode = await this.node({ path, uid, id, mysql_id }); - await fsNode.fetchEntry(options); - return fsNode.entry; - } -} - -module.exports = { - FilesystemService, -}; diff --git a/src/backend/src/deprecated/filesystem/PuterS3Service.js b/src/backend/src/deprecated/filesystem/PuterS3Service.js deleted file mode 100644 index acee663fd..000000000 --- a/src/backend/src/deprecated/filesystem/PuterS3Service.js +++ /dev/null @@ -1,499 +0,0 @@ -import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CopyObjectCommand, CreateMultipartUploadCommand, DeleteObjectCommand, GetObjectCommand, PutObjectCommand, UploadPartCommand, UploadPartCopyCommand } from '@aws-sdk/client-s3'; -import BaseService from '@heyputer/backend/src/services/BaseService.js'; -import { Context } from '@heyputer/backend/src/util/context.js'; -import { TeePromise } from '@heyputer/putility/src/libs/promise.js'; -import { Readable } from 'stream'; -import { s3ClientProvider } from '../../clients/s3/s3ClientProvider.js'; -import { EWMA } from '../../util/opmath.js'; -import { simple_retry } from '../../util/retryutil.js'; -import { chunk_stream, progress_stream } from '../../util/streamutil.js'; -import { PuterS3StorageStrategy } from '../filesystem/strategies/PuterS3StorageStrategy.js'; - -export class PuterS3Service extends BaseService { - - async _init () { - this.clients_ = {}; - this.config = this.global_config; - - this.global_average_S3_part_time = new EWMA({ - initial: 4000, // average from local testing - alpha: 0.1, - }); - } - - async '__on_install.context-initializers' () { - // async _init () { - const svc_contextInit = this.services.get('context-init'); - const storage = new PuterS3StorageStrategy({ services: this.services }); - svc_contextInit.register_value('storage', storage); - - const svc_mountpoint = this.services.get('mountpoint'); - svc_mountpoint.set_storage('PuterFSProvider', storage); - - // This alternative approach can be used if the arguments to - // the storage strategy become context-sensitive: - // svc_contextInit.register_async_factory('storage', async () => { - // return new PuterS3StorageStrategy({ services: this.services }); - // }); - } - - _get_client (region) { - return s3ClientProvider.get(region); - } - - async create_read_stream ({ bucket_region, bucket, key, version_id, range }) { - const client = this._get_client(bucket_region); - - let response; - try { - response = await client.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - ...(range ? { Range: range } : {}), - ...(version_id ? { VersionId: version_id } : {}), - })); - } catch ( e ) { - this.errors.report('s3:read', { - source: e, - message: 'Error reading from S3', - trace: true, - alarm: true, - extra: { - bucket_region, - bucket, - key, - version_id, - range, - }, - }); - - throw e; - } - - const stream = Readable.from(response.Body); - - return stream; - } - - async upload_buffer ({ bucket_region, bucket, key, buffer }) { - const client = this._get_client(bucket_region); - - let ret; - - try { - ret = await client.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: buffer, - })); - } catch ( e ) { - this.errors.report('s3:upload', { - source: e, - message: 'Error uploading to S3', - trace: true, - alarm: true, - extra: { - bucket, - key, - }, - }); - - throw e; - } - - return ret; - } - - async put_stream ({ size, bucket_region, bucket, key, stream, on_progress }) { - const verb_log = (() => { - const context = Context.get(); - const svc = context.get('services'); - const svc_operationTrace = svc.get('operationTrace'); - const svc_log = svc.get('log-service'); - const frame = context.get(svc_operationTrace.ckey('frame')); - const frame_id = frame.id; - const log = svc_log.create('s3-upload', { - operation: frame_id, - }); - return log.info.bind(log); - })(); - verb_log('put_stream', { bucket_region, bucket, key }); - - const client = this._get_client(bucket_region); - - let ret; - - // Intercept body stream for progress tracking - const body_stream = progress_stream(stream, { - total: size, - progress_callback: on_progress, - }); - - try { - ret = await client.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body_stream, - ContentLength: size, - })); - } catch ( e ) { - this.errors.report('s3:upload', { - source: e, - message: 'Error uploading to S3', - trace: true, - alarm: true, - extra: { - bucket, - key, - }, - }); - - throw e; - } - - return ret; - } - - async upload_stream ({ bucket_region, bucket, key, stream, on_progress }) { - const client = this._get_client(bucket_region); - - console.debug('upload_stream', { bucket_region, bucket, key }); - - const multipart_upload = await client.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - })); - - let ret; // return value - - try { - - const part_size = 1024 * 1024 * 5; // 5MB - // - - // get each part while streaming - const chunk_iterator = chunk_stream( - stream, - part_size, - this.global_average_S3_part_time, - ); - let i = 0; - let uploaded_bytes = 0; - let upload_promises = []; - const upload_results = []; - - let tp; - let count_parts_being_uploaded = 0; - - let check_queue; - - let queue_empty_promise = null; - - const upload_part = async part => { - - if ( count_parts_being_uploaded >= 4 ) { - console.warn('too many concurrent part uploads; halting'); - tp = new TeePromise(); - await tp; - } - - const part_number = ++i; - count_parts_being_uploaded++; - - const upload_promise = (async () => { - - const ts_start = Date.now(); - - const [err, success, result] = await simple_retry(async () => { - return await client.send(new UploadPartCommand({ - Bucket: bucket, - Key: key, - PartNumber: part_number, - UploadId: multipart_upload.UploadId, - Body: part, - })); - }, 3, 50); - - if ( err || !success ) { - this.errors.report('s3:upload', { - source: err || new Error('unknown'), - message: 'Error uploading to S3', - trace: true, - alarm: true, - extra: { - bucket, - key, - part_number, - }, - }); - throw err; - } - - const ts_end = Date.now(); - const elapsed = ts_end - ts_start; - const elapsed_per_part_size = elapsed * (part.length / part_size); - // this.global_average_S3_part_time.put(elapsed); - this.global_average_S3_part_time.put(elapsed_per_part_size); - - uploaded_bytes += part.length; - on_progress({ uploaded: uploaded_bytes }); - - count_parts_being_uploaded--; - if ( tp ) { - const p = tp; - tp = null; - p.resolve(); - } - - check_queue(); - - return result; - })(); - - upload_promises.push(upload_promise); - }; - - const part_queue = []; - - check_queue = () => { - if ( part_queue.length > 0 ) { - const part = part_queue.shift(); - upload_part(part); - if ( part_queue.length == 0 ) { - if ( queue_empty_promise ) { - const p = queue_empty_promise; - queue_empty_promise = null; - p.resolve(); - } - } - } - }; - - for await ( const chunk of chunk_iterator ) { - await upload_part(chunk); - } - - // If the file is empty we still need to upload a part - if ( i === 0 ) { - const upload_promise = (async () => { - const [err, success, result] = await simple_retry(async () => { - return await client.send(new UploadPartCommand({ - Bucket: bucket, - Key: key, - PartNumber: 1, - UploadId: multipart_upload.UploadId, - Body: Buffer.alloc(0), - })); - }, 3, 50); - - if ( err || !success ) { - this.errors.report('s3:upload', { - source: err || new Error('unknown'), - message: 'Error uploading to S3', - trace: true, - alarm: true, - extra: { - bucket, - key, - part_number: 1, - }, - }); - throw err; - } - - on_progress({ uploaded: uploaded_bytes }); - return result; - })(); - - upload_promises.push(upload_promise); - } - - if ( part_queue.length > 0 ) { - queue_empty_promise = new TeePromise(); - await queue_empty_promise; - } - - const some_results = await Promise.all(upload_promises); - upload_results.push(...some_results); - - try { - // complete the upload - ret = await client.send(new CompleteMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: multipart_upload.UploadId, - MultipartUpload: { - Parts: upload_results.map((_, i) => ({ - PartNumber: i + 1, - ETag: _.ETag, - })), - }, - })); - } catch ( e ) { - console.warn(`catch block: ${e.message}`); - } finally { - // no-op - } - } catch ( e ) { - console.error(`error: ${e.message}`); - // abort the upload - try { - await client.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: multipart_upload.UploadId, - })); - } catch ( e2 ) { - this.errors.report('s3:upload.abort', { - source: e2, - message: 'Error aborting multipart upload', - trace: true, - alarm: true, - extra: { - bucket, - key, - }, - }); - } - - this.errors.report('s3:upload', { - source: e, - message: 'Error uploading to S3', - trace: true, - alarm: true, - extra: { - bucket, - key, - }, - }); - - throw e; - } - - return ret; - } - - async copy_simple ({ - dst_bucket_region, - dst_bucket, - src_bucket, - src_key, - dst_key, - }) { - const client = this._get_client(dst_bucket_region); - - let ret; - - // const copy_source_urlencoded = encodeURIComponent(src_bucket + '/' + src_key); - - try { - ret = await client.send(new CopyObjectCommand({ - Bucket: dst_bucket, - Key: dst_key, - CopySource: `${src_bucket}/${src_key}`, - })); - } catch ( e ) { - this.errors.report('s3:copy', { - source: e, - message: 'Error copying to S3', - trace: true, - alarm: true, - extra: { - src_bucket, - src_key, - dst_bucket, - dst_key, - }, - }); - - throw e; - } - - return ret; - } - - async copy_multipart ({ - dst_bucket_region, - dst_bucket, - src_bucket, - src_key, - dst_key, - on_progress, - size, - }) { - const client = this._get_client(dst_bucket_region); - - const multipart_upload = await client.send(new CreateMultipartUploadCommand({ - Bucket: dst_bucket, - Key: dst_key, - })); - - const part_size = 4 * 1024 * 1024 * 1024; // 1GiB - - const results = []; - - let part_number_i = 0; - for ( let byte_start = 0 ; byte_start < size ; byte_start += part_size ) { - const part_number = ++part_number_i; - // byte range is inclusive... WTF? - const byte_end = Math.min(byte_start + part_size, size) - 1; - - const [err, success, result] = await simple_retry(async () => { - const params = { - Bucket: dst_bucket, - Key: dst_key, - PartNumber: part_number, - UploadId: multipart_upload.UploadId, - CopySource: `${src_bucket}/${src_key}`, - CopySourceRange: `bytes=${byte_start}-${byte_end}`, - }; - return await client.send(new UploadPartCopyCommand(params)); - }, 3, 50); - - if ( err || !success ) { - this.errors.report('s3:copy', { - source: err || new Error('unknown'), - message: 'Error copying to S3', - trace: true, - alarm: true, - extra: { - src_bucket, - src_key, - dst_bucket, - dst_key, - part_number, - }, - }); - throw err; - } - - results.push(result); - - on_progress({ uploaded: byte_end + 1 }); - } - - const ret = await client.send(new CompleteMultipartUploadCommand({ - Bucket: dst_bucket, - Key: dst_key, - UploadId: multipart_upload.UploadId, - MultipartUpload: { - Parts: results.map((_, i) => ({ - PartNumber: i + 1, - ETag: _.CopyPartResult.ETag, - })), - }, - })); - - return ret; - } - - async delete ({ bucket_region, bucket, key }) { - const client = this._get_client(bucket_region); - - return await client.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - })); - } - -} diff --git a/src/backend/src/deprecated/filesystem/batch/BatchExecutor.js b/src/backend/src/deprecated/filesystem/batch/BatchExecutor.js deleted file mode 100644 index e667305c9..000000000 --- a/src/backend/src/deprecated/filesystem/batch/BatchExecutor.js +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('@heyputer/putility'); -const commands = require('./commands').commands; -const APIError = require('../../../api/APIError'); -const { Context } = require('../../../util/context'); -const config = require('../../../config'); -const PathResolver = require('../../../routers/filesystem_api/batch/PathResolver'); -const { TeePromise } = require('@heyputer/putility').libs.promise; - -class BatchExecutor extends AdvancedBase { - static LOG_LEVEL = true; - - constructor (x, { actor, log, errors }) { - super(); - this.x = x; - this.actor = actor; - this.pathResolver = new PathResolver({ actor }); - this.log = log; - this.errors = errors; - this.responsePromises = []; - this.hasError = false; - - this.total_tbd = true; - this.total = 0; - this.counter = 0; - - this.concurrent_ops = 0; - this.max_concurrent_ops = 20; - this.ops_promise = null; - - this.log_batchCommands = (config.logging ?? []).includes('batch-commands'); - } - - async ready_for_more () { - if ( this.ops_promise === null ) { - this.ops_promise = new TeePromise(); - } - await this.ops_promise; - } - - async exec_op (req, op, file) { - while ( this.concurrent_ops >= this.max_concurrent_ops ) { - await this.ready_for_more(); - } - - this.concurrent_ops++; - - const command_cls = commands[op.op]; - if ( this.log_batchCommands ) { - console.log(command_cls, JSON.stringify(op, null, 2)); - } - delete op.op; - - // TEMP: event service will handle this - op.original_client_socket_id = req.body.original_client_socket_id; - op.socket_id = req.body.socket_id; - - // run the operation - let p = this.x.arun(async () => { - const x = Context.get(); - if ( ! x ) throw new Error('no context'); - - try { - if ( ! command_cls ) { - throw APIError.create('invalid_operation', null, { - operation: op.op, - }); - } - - const command_ins = await command_cls.run({ - getFile: () => file, - pathResolver: this.pathResolver, - actor: this.actor, - }, op); - - const res = await command_ins.awaitValue('result'); - return res; - } catch (e) { - this.hasError = true; - if ( ! ( e instanceof APIError ) ) { - // TODO: alarm condition - this.errors.report('batch-operation', { - source: e, - trace: true, - alarm: true, - }); - - e = APIError.adapt(e); // eslint-disable-line no-ex-assign - } - - // Consume stream if there's a file - if ( file ) { - try { - // read entire stream - await new Promise((resolve, reject) => { - file.stream.on('end', resolve); - file.stream.on('error', reject); - file.stream.resume(); - }); - } catch (e) { - this.errors.report('batch-operation-2', { - source: e, - trace: true, - alarm: true, - }); - } - } - - if ( config.env == 'dev' ) { - console.error(e); - // process.exit(1); - } - - const serialized_error = e.serialize(); - return serialized_error; - } finally { - this.concurrent_ops--; - if ( this.ops_promise && this.concurrent_ops < this.max_concurrent_ops ) { - this.ops_promise.resolve(); - this.ops_promise = null; - } - } - }); - - // decorate with logging - p = p.then(result => { - this.counter++; - const { log, total, total_tbd, counter } = this; - const total_str = total_tbd ? `TBD(>${total})` : `${total}`; - log.debug(`Batch Progress: ${counter} / ${total_str} operations`); - return result; - }); - - // this.responsePromises.push(p); - - // It doesn't really matter whether or not `await` is here - // (that's a design flaw in the Promise API; what if you - // want a promise that returns a promise?) - const result = await p; - return result; - - } -} - -module.exports = { - BatchExecutor, -}; diff --git a/src/backend/src/deprecated/filesystem/batch/commands.js b/src/backend/src/deprecated/filesystem/batch/commands.js deleted file mode 100644 index 18ad7e062..000000000 --- a/src/backend/src/deprecated/filesystem/batch/commands.js +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('@heyputer/putility'); -const { AsyncProviderFeature } = require('../../../traits/AsyncProviderFeature'); -const { HLMkdir, QuickMkdir } = require('../hl_operations/hl_mkdir'); -const { Context } = require('../../../util/context'); -const { HLWrite } = require('../hl_operations/hl_write'); -const { get_app } = require('../../../helpers'); -const { OperationFrame } = require('../../../services/OperationTraceService'); -const { HLMkShortcut } = require('../hl_operations/hl_mkshortcut'); -const { HLMkLink } = require('../hl_operations/hl_mklink'); -const { HLRemove } = require('../hl_operations/hl_remove'); -const { HLMove } = require('../hl_operations/hl_move'); -const { NodeUIDSelector } = require('../node/selectors'); -const { safeHasOwnProperty } = require('../../../util/safety'); - -class BatchCommand extends AdvancedBase { - static FEATURES = [ - new AsyncProviderFeature(), - ]; - static async run (executor, parameters) { - const instance = new this(); - let x = Context.get(); - const operationTraceSvc = x.get('services').get('operationTrace'); - const frame = await operationTraceSvc.add_frame(`batch:${ this.name}`); - if ( safeHasOwnProperty(parameters, 'item_upload_id') ) { - frame.attr('gui_metadata', { - ...(frame.get_attr('gui_metadata') || {}), - item_upload_id: parameters.item_upload_id, - }); - } - x = x.sub({ [operationTraceSvc.ckey('frame')]: frame }); - await x.arun(async () => { - await instance.run(executor, parameters); - }); - frame.status = OperationFrame.FRAME_STATUS_DONE; - return instance; - } -} - -class MkdirCommand extends BatchCommand { - async run (executor, parameters) { - const context = Context.get(); - const fs = context.get('services').get('filesystem'); - - const parent = parameters.parent - ? await fs.node(await executor.pathResolver.awaitSelector(parameters.parent)) - : undefined ; - - const meta = parameters.parent - ? executor.pathResolver.getMeta(parameters.parent) - : undefined ; - - if ( meta?.conflict_free ) { - // No potential conflict; just create the directory - const q_mkdir = new QuickMkdir(); - await q_mkdir.run({ - parent, - path: parameters.path, - }); - if ( parameters.as ) { - executor.pathResolver.putSelector( - parameters.as, - q_mkdir.created.selector, - { conflict_free: true }, - ); - } - this.setFactory('result', async () => { - await q_mkdir.created.awaitStableEntry(); - const response = await q_mkdir.created.getSafeEntry(); - return response; - }); - return; - } - - const hl_mkdir = new HLMkdir(); - const response = await hl_mkdir.run({ - parent, - path: parameters.path, - overwrite: parameters.overwrite, - dedupe_name: parameters.dedupe_name, - create_missing_parents: - parameters.create_missing_ancestors ?? - parameters.create_missing_parents ?? - false, - shortcut_to: parameters.shortcut_to, - actor: executor.actor, - }); - if ( parameters.as ) { - executor.pathResolver.putSelector( - parameters.as, - hl_mkdir.created.selector, - hl_mkdir.used_existing - ? undefined - : { conflict_free: true }, - ); - } - this.provideValue('result', response); - } -} - -class WriteCommand extends BatchCommand { - async run (executor, parameters) { - const context = Context.get(); - const fs = context.get('services').get('filesystem'); - - const uploaded_file = executor.getFile(); - - const destinationOrParent = - await fs.node(await executor.pathResolver.awaitSelector(parameters.path)); - - let app; - if ( parameters.app_uid ) { - app = await get_app({ uid: parameters.app_uid }); - } - - const hl_write = new HLWrite(); - if ( ! executor.actor ) { - throw new Error('Actor is missing here'); - } - const response = await hl_write.run({ - destination_or_parent: destinationOrParent, - specified_name: parameters.name, - fallback_name: uploaded_file.originalname, - - overwrite: parameters.overwrite, - dedupe_name: parameters.dedupe_name, - - create_missing_parents: - parameters.create_missing_ancestors ?? - parameters.create_missing_parents ?? - false, - actor: executor.actor, - - file: uploaded_file, - offset: parameters.offset, - - // TODO: handle these with event service instead - socket_id: parameters.socket_id, - operation_id: parameters.operation_id, - item_upload_id: parameters.item_upload_id, - app_id: app ? app.id : null, - - thumbnail: parameters.thumbnail, - }); - - this.provideValue('result', response); - - // const opctx = await fs.write(fs, { - // // --- per file --- - // name: parameters.name, - // fallbackName: uploaded_file.originalname, - // destinationOrParent, - // // app_id: app ? app.id : null, - // overwrite: parameters.overwrite, - // dedupe_name: parameters.dedupe_name, - // file: uploaded_file, - // thumbnail: parameters.thumbnail, - // target: parameters.target ? await req.fs.node(parameters.shortcut_to) : null, - // symlink_path: parameters.symlink_path, - // operation_id: parameters.operation_id, - // item_upload_id: parameters.item_upload_id, - // user: executor.user, - - // // --- per batch --- - // socket_id: parameters.socket_id, - // original_client_socket_id: parameters.original_client_socket_id, - // }); - - // opctx.onValue('response', v => this.provideValue('result', v)); - } -} - -class ShortcutCommand extends BatchCommand { - async run (executor, parameters) { - const context = Context.get(); - const fs = context.get('services').get('filesystem'); - - const destinationOrParent = - await fs.node(await executor.pathResolver.awaitSelector(parameters.path)); - - const shortcut_to = - await fs.node(await executor.pathResolver.awaitSelector(parameters.shortcut_to)); - - let app; - if ( parameters.app_uid ) { - app = await get_app({ uid: parameters.app_uid }); - } - - await destinationOrParent.fetchEntry({ thumbnail: true }); - await shortcut_to.fetchEntry({ thumbnail: true }); - - const hl_mkShortcut = new HLMkShortcut(); - const response = await hl_mkShortcut.run({ - parent: destinationOrParent, - name: parameters.name, - actor: executor.actor, - target: shortcut_to, - dedupe_name: parameters.dedupe_name, - - // TODO: handle these with event service instead - socket_id: parameters.socket_id, - operation_id: parameters.operation_id, - item_upload_id: parameters.item_upload_id, - app_id: app ? app.id : null, - }); - - this.provideValue('result', response); - } -} - -class SymlinkCommand extends BatchCommand { - async run (executor, parameters) { - const context = Context.get(); - const fs = context.get('services').get('filesystem'); - - const destinationOrParent = - await fs.node(await executor.pathResolver.awaitSelector(parameters.path)); - - let app; - if ( parameters.app_uid ) { - app = await get_app({ uid: parameters.app_uid }); - } - - await destinationOrParent.fetchEntry({ thumbnail: true }); - - const hl_mkLink = new HLMkLink(); - const response = await hl_mkLink.run({ - parent: destinationOrParent, - name: parameters.name, - actor: executor.actor, - target: parameters.target, - - // TODO: handle these with event service instead - socket_id: parameters.socket_id, - operation_id: parameters.operation_id, - item_upload_id: parameters.item_upload_id, - app_id: app ? app.id : null, - }); - - this.provideValue('result', response); - } -} - -class DeleteCommand extends BatchCommand { - async run (executor, parameters) { - const context = Context.get(); - const fs = context.get('services').get('filesystem'); - - const target = - await fs.node(await executor.pathResolver.awaitSelector(parameters.path)); - - const hl_remove = new HLRemove(); - const response = await hl_remove.run({ - target, - actor: executor.actor, - recursive: parameters.recursive ?? false, - descendants_only: parameters.descendants_only ?? false, - }); - this.provideValue('result', response); - } -} - -class MoveCommand extends BatchCommand { - async run (executor, parameters) { - const context = Context.get(); - const fs = context.get('services').get('filesystem'); - - console.log('what are the parameters???', parameters); - - const source = - await fs.node(await executor.pathResolver.awaitSelector(parameters.source)); - const destinationOrParent = - await fs.node(await executor.pathResolver.awaitSelector(parameters.destination)); - - const hl_move = new HLMove(); - const response = await hl_move.run({ - source, - destination_or_parent: destinationOrParent, - actor: executor.actor, - new_name: parameters.new_name, - overwrite: parameters.overwrite ?? false, - dedupe_name: parameters.dedupe_name ?? parameters.change_name ?? false, - create_missing_parents: - parameters.create_missing_ancestors ?? - parameters.create_missing_parents ?? - false, - new_metadata: parameters.new_metadata, - }); - - if ( parameters.as && response.moved?.uid ) { - executor.pathResolver.putSelector(parameters.as, new NodeUIDSelector(response.moved.uid)); - } - - this.provideValue('result', response); - } -} - -module.exports = { - commands: { - mkdir: MkdirCommand, - write: WriteCommand, - shortcut: ShortcutCommand, - symlink: SymlinkCommand, - delete: DeleteCommand, - move: MoveCommand, - }, -}; diff --git a/src/backend/src/deprecated/filesystem/definitions/capabilities.js b/src/backend/src/deprecated/filesystem/definitions/capabilities.js deleted file mode 100644 index 6ed45f30e..000000000 --- a/src/backend/src/deprecated/filesystem/definitions/capabilities.js +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const capabilityNames = [ - // PuterFS Capabilities - 'thumbnail', - 'uuid', - 'operation-trace', - 'readdir-uuid-mode', - 'update-thumbnail', - 'puter-shortcut', - - // Standard Capabilities - 'read', - 'write', - 'symlink', - 'trash', - - // Macro Capabilities - 'copy-tree', - 'move-tree', - 'remove-tree', - 'get-recursive-size', - 'readdirstat_uuid', - - // Behavior Capabilities - 'case-sensitive', - - // POSIX Capabilities - 'readdir-inode-numbers', - 'unix-perms', -]; - -const fsCapabilities = {}; -for ( const capabilityName of capabilityNames ) { - const key = capabilityName.toUpperCase().replace(/-/g, '_'); - fsCapabilities[key] = Symbol(capabilityName); -} - -module.exports = fsCapabilities; diff --git a/src/backend/src/deprecated/filesystem/hl_operations/definitions.js b/src/backend/src/deprecated/filesystem/hl_operations/definitions.js deleted file mode 100644 index 498169c46..000000000 --- a/src/backend/src/deprecated/filesystem/hl_operations/definitions.js +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { BaseOperation } = require('../../../services/OperationTraceService'); - -class HLFilesystemOperation extends BaseOperation { -} - -module.exports = { - HLFilesystemOperation, -}; diff --git a/src/backend/src/deprecated/filesystem/hl_operations/hl_copy.js b/src/backend/src/deprecated/filesystem/hl_operations/hl_copy.js deleted file mode 100644 index 1655df982..000000000 --- a/src/backend/src/deprecated/filesystem/hl_operations/hl_copy.js +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../../api/APIError'); -const { chkperm, validate_fsentry_name, get_user, is_ancestor_of } = require('../../../helpers'); -const { TYPE_DIRECTORY } = require('../FSNodeContext'); -const { NodePathSelector, RootNodeSelector } = require('../node/selectors'); -const { HLFilesystemOperation } = require('./definitions'); -const { MkTree } = require('./hl_mkdir'); -const { HLRemove } = require('./hl_remove'); -const { LLCopy } = require('../ll_operations/ll_copy'); -const { getTracer } = require('../../../util/otelutil'); - -class HLCopy extends HLFilesystemOperation { - static DESCRIPTION = ` - High-level copy operation. - - This operation is a wrapper around the low-level copy operation. - It provides the following features: - - create missing parent directories - - overwrite existing files or directories - - deduplicate files/directories with the same name - `; - - static MODULES = { - _path: require('path'), - }; - - static PARAMETERS = { - source: {}, - destionation_or_parent: {}, - new_name: {}, - - overwrite: {}, - dedupe_name: {}, - - create_missing_parents: {}, - - user: {}, - }; - - async _run () { - const { _path } = this.modules; - - const { values, context } = this; - const svc = context.get('services'); - const fs = svc.get('filesystem'); - - let parent = values.destination_or_parent; - let dest = null; - - const source = values.source; - - if ( values.overwrite && values.dedupe_name ) { - throw APIError.create('overwrite_and_dedupe_exclusive'); - } - - if ( ! await source.exists() ) { - throw APIError.create('source_does_not_exist'); - } - - if ( ! await chkperm(source.entry, values.user.id, 'cp') ) { - throw APIError.create('forbidden'); - } - - if ( await parent.get('is-root') ) { - throw APIError.create('cannot_copy_to_root'); - } - - // If parent exists and is a file, and a new name wasn't - // specified, the intention must be to overwrite the file. - if ( - !values.new_name && - await parent.exists() && - await parent.get('type') !== TYPE_DIRECTORY - ) { - dest = parent; - parent = await dest.getParent(); - await parent.fetchEntry(); - } - - // If parent is not found either throw an error or create - // the parent directory as specified by parameters. - if ( ! await parent.exists() ) { - if ( ! (parent.selector instanceof NodePathSelector) ) { - throw APIError.create('dest_does_not_exist', null, { - parent: parent.selector, - }); - } - const path = parent.selector.value; - const tree_op = new MkTree(); - await tree_op.run({ - parent: await fs.node(new RootNodeSelector()), - tree: [path], - }); - await parent.fetchEntry({ force: true }); - } - - if ( - await parent.get('type') !== TYPE_DIRECTORY - ) { - throw APIError.create('dest_is_not_a_directory'); - } - - if ( ! await chkperm(parent.entry, values.user.id, 'write') ) { - throw APIError.create('forbidden'); - } - - let target_name = values.new_name ?? await source.get('name'); - - try { - validate_fsentry_name(target_name); - } catch (e) { - throw APIError.create(400, e); - } - - // NEXT: implement _verify_room with profiling - const tracer = getTracer(); - await tracer.startActiveSpan('fs:cp:verify-size-constraints', async span => { - const source_file = source.entry; - const dest_fsentry = parent.entry; - - let source_user = await get_user({ id: source_file.user_id }); - let dest_user = source_user.id !== dest_fsentry.user_id - ? await get_user({ id: dest_fsentry.user_id }) - : source_user ; - const sizeService = svc.get('sizeService'); - let deset_usage = await sizeService.get_usage(dest_user.id); - - const size = await source.fetchSize(); - const capacity = await sizeService.get_storage_capacity(dest_user.id); - if ( capacity - deset_usage - size < 0 ) { - throw APIError.create('storage_limit_reached'); - } - span.end(); - }); - - if ( dest === null ) { - dest = await parent.getChild(target_name); - } - - // Ensure copy operation is legal - // TODO: maybe this is better in the low-level operation - if ( await source.get('uid') == await parent.get('uid') ) { - throw APIError.create('source_and_dest_are_the_same'); - } - - if ( await is_ancestor_of(source.uid, parent.uid) ) { - throw APIError.create('cannot_copy_item_into_itself'); - } - - let overwritten; - if ( await dest.exists() ) { - // condition: no overwrite behaviour specified - if ( !values.overwrite && !values.dedupe_name ) { - throw APIError.create('item_with_same_name_exists', null, { - entry_name: dest.entry.name, - }); - } - - if ( values.dedupe_name ) { - const target_ext = _path.extname(target_name); - const target_noext = _path.basename(target_name, target_ext); - for ( let i = 1 ;; i++ ) { - const try_new_name = `${target_noext} (${i})${target_ext}`; - const exists = await parent.hasChild(try_new_name); - if ( ! exists ) { - target_name = try_new_name; - break; - } - } - - dest = await parent.getChild(target_name); - } - else if ( values.overwrite ) { - if ( ! await chkperm(dest.entry, values.user.id, 'rm') ) { - throw APIError.create('forbidden'); - } - - // TODO: This will be LLRemove - // TODO: what to do with parent_operation? - overwritten = await dest.getSafeEntry(); - const hl_remove = new HLRemove(); - await hl_remove.run({ - target: dest, - user: values.user, - recursive: true, - }); - } - } - - const ll_copy = new LLCopy(); - this.copied = await ll_copy.run({ - source, - parent, - user: values.user, - target_name, - }); - - await this.copied.awaitStableEntry(); - const response = await this.copied.getSafeEntry({ thumbnail: true }); - return { - copied: response, - overwritten, - }; - } -} - -module.exports = { - HLCopy, -}; diff --git a/src/backend/src/deprecated/filesystem/hl_operations/hl_data_read.js b/src/backend/src/deprecated/filesystem/hl_operations/hl_data_read.js deleted file mode 100644 index 7d5c71c90..000000000 --- a/src/backend/src/deprecated/filesystem/hl_operations/hl_data_read.js +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { HLFilesystemOperation } = require('./definitions'); -const { chkperm } = require('../../../helpers'); -const { LLRead } = require('../ll_operations/ll_read'); -const APIError = require('../../../api/APIError'); - -/** - * HLDataRead reads a stream of objects from a file containing structured data. - * For .jsonl files, the stream will product multiple objects. - * For .json files, the stream will produce a single object. - */ -class HLDataRead extends HLFilesystemOperation { - static MODULES = { - 'stream': require('stream'), - }; - - async _run () { - const { context } = this; - - // We get the user from context so that an elevated system context - // can read files under the system user. - const user = await context.get('user'); - - const { - fsNode, - version_id, - } = this.values; - - if ( ! await fsNode.exists() ) { - throw APIError.create('subject_does_not_exist'); - } - - if ( ! await chkperm(fsNode.entry, user.id, 'read') ) { - throw APIError.create('forbidden'); - } - - const ll_read = new LLRead(); - let stream = await ll_read.run({ - fsNode, - user, - version_id, - }); - - stream = this._stream_bytes_to_lines(stream); - stream = this._stream_jsonl_lines_to_objects(stream); - - return stream; - } - - _stream_bytes_to_lines (stream) { - const readline = require('readline'); - const rl = readline.createInterface({ - input: stream, - terminal: false, - }); - - const { PassThrough } = this.modules.stream; - - const output_stream = new PassThrough(); - - rl.on('line', (line) => { - output_stream.write(line); - }); - rl.on('close', () => { - output_stream.end(); - }); - - return output_stream; - } - - _stream_jsonl_lines_to_objects (stream) { - const { PassThrough } = this.modules.stream; - const output_stream = new PassThrough(); - (async () => { - for await ( const line of stream ) { - output_stream.write(JSON.parse(line)); - } - output_stream.end(); - })(); - return output_stream; - } -} - -module.exports = { - HLDataRead, -}; diff --git a/src/backend/src/deprecated/filesystem/hl_operations/hl_mkdir.js b/src/backend/src/deprecated/filesystem/hl_operations/hl_mkdir.js deleted file mode 100644 index 83e37435f..000000000 --- a/src/backend/src/deprecated/filesystem/hl_operations/hl_mkdir.js +++ /dev/null @@ -1,561 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { chkperm } = require('../../../helpers'); -const { RootNodeSelector, NodeChildSelector, NodePathSelector } = require('../node/selectors'); -const APIError = require('../../../api/APIError'); -const FSNodeParam = require('../../../api/filesystem/FSNodeParam'); -const StringParam = require('../../../api/filesystem/StringParam'); -const FlagParam = require('../../../api/filesystem/FlagParam'); -const UserParam = require('../../../api/filesystem/UserParam'); -const FSNodeContext = require('../FSNodeContext').default; -const { OtelFeature } = require('../../../traits/OtelFeature'); -const { HLFilesystemOperation } = require('./definitions'); -const { is_valid_path } = require('../validation'); -const { HLRemove } = require('./hl_remove'); -const { LLMkdir } = require('../ll_operations/ll_mkdir'); - -/** - * Creates a directory, handling race conditions where another parallel request - * may have already created the same directory. - * - * @param {Object} params - * @param {FSNodeContext} params.parent - parent directory - * @param {NodeChildSelector} params.selector - selector for directory (contains name) - * @param {Actor} params.actor - actor to perform the operation on behalf of - * @param {Object} params.fs - filesystem service - * @returns {Promise} created or existing directory node - */ -async function createDirOrUseExisting ({ parent, selector, actor, fs }) { - try { - const ll_mkdir = new LLMkdir(); - return await ll_mkdir.run({ - parent, - name: selector.name, - actor, - }); - } catch ( error ) { - // This "error" can occur when multiple `hl_mkdir` operations are being - // run at the same time with the `createMissingParents` option enabled. - const errorCode = error.code || error.fields?.code; - if ( errorCode === 'item_with_same_name_exists' ) { - const existing_node = await fs.node(selector); - - // Wait for the entry to be stable (it might still be in the process - // of being created by another parallel request) - await existing_node.awaitStableEntry(); - await existing_node.fetchEntry(); - - // If this is a file we need to re-throw the error - if ( await existing_node.get('type') !== FSNodeContext.TYPE_DIRECTORY ) { - throw error; - } - - return existing_node; - } - - throw error; - } -} - -class MkTree extends HLFilesystemOperation { - static DESCRIPTION = ` - High-level operation for making directory trees - - The following input for 'tree': - ['a/b/c', ['i/j/k'], ['p', ['q'], ['r/s']]]] - - Would create a directory tree like this: - a - └── b - └── c - ├── i - │ └── j - │ └── k - └── p - ├── q - └── r - └── s - `; - - static PARAMETERS = { - parent: new FSNodeParam('parent', { optional: true }), - }; - - static PROPERTIES = { - leaves: () => [], - directories_created: () => [], - }; - - async _run () { - const { values, context } = this; - const fs = context.get('services').get('filesystem'); - - await this.create_branch_({ - parent_node: values.parent || await fs.node(new RootNodeSelector()), - tree: values.tree, - parent_exists: true, - }); - } - - async create_branch_ ({ parent_node, tree, parent_exists }) { - const { context } = this; - const fs = context.get('services').get('filesystem'); - const actor = context.get('actor'); - - const trunk = tree[0]; - const branches = tree.slice(1); - - let current = parent_node.selector; - - // trunk = a/b/c - - const dirs = trunk === '.' ? [] - : trunk.split('/').filter(Boolean); - - // dirs = [a, b, c] - - let parent_did_exist = parent_exists; - - // This is just a loop that goes through each part of the path - // until it finds the first directory that doesn't exist yet. - let i = 0; - if ( parent_exists ) { - for ( ; i < dirs.length ; i++ ) { - const dir = dirs[i]; - const currentParent = current; - current = new NodeChildSelector(current, dir); - - const maybe_dir = await fs.node(current); - - if ( maybe_dir.isRoot ) continue; - if ( await maybe_dir.isUserDirectory() ) continue; - - if ( await maybe_dir.exists() ) { - - if ( await maybe_dir.get('type') !== FSNodeContext.TYPE_DIRECTORY ) { - throw APIError.create('dest_is_not_a_directory'); - } - - continue; - } - - current = currentParent; - parent_exists = false; - break; - } - } - - if ( parent_did_exist && !parent_exists ) { - const node = await fs.node(current); - const has_perm = await chkperm(await node.get('entry'), actor.type.user.id, 'write'); - if ( ! has_perm ) throw APIError.create('permission_denied'); - } - - // This next loop creates the new directories - - // We break into a second loop because we know none of these directories - // exist yet. If we continued those checks each child operation would - // wait for the previous one to complete because FSNodeContext::fetchEntry - // will notice ResourceService has a lock on the previous operation - // we started. - - // In this way it goes nyyyoooom because all the database inserts - // happen concurrently (and probably end up in the same batch). - - for ( ; i < dirs.length ; i++ ) { - const dir = dirs[i]; - const currentParent = current; - current = new NodeChildSelector(current, dir); - - const node = await createDirOrUseExisting({ - parent: await fs.node(currentParent), - selector: current, - actor, - fs, - }); - - current = node.selector; - - this.directories_created.push(node); - } - - const bottom_parent = await fs.node(current); - - if ( branches.length === 0 ) { - this.leaves.push(bottom_parent); - } - - for ( const branch of branches ) { - await this.create_branch_({ - parent_node: bottom_parent, - tree: branch, - parent_exists, - }); - } - } -} - -class QuickMkdir extends HLFilesystemOperation { - async _run () { - const { context, values } = this; - let { parent, path } = values; - const { _path } = this.modules; - const fs = context.get('services').get('filesystem'); - const actor = context.get('actor'); - - parent = parent || await fs.node(new RootNodeSelector()); - - let current = parent.selector; - - const dirs = path === '.' ? [] - : path.split('/').filter(Boolean); - - const api = require('@opentelemetry/api'); - const currentSpan = api.trace.getSpan(api.context.active()); - if ( currentSpan ) { - currentSpan.setAttribute('path', path); - currentSpan.setAttribute('dirs', dirs.join('/')); - currentSpan.setAttribute('parent', parent.selector.describe()); - } - - for ( let i = 0 ; i < dirs.length ; i++ ) { - const dir = dirs[i]; - const currentParent = current; - current = new NodeChildSelector(current, dir); - - const node = await createDirOrUseExisting({ - parent: await fs.node(currentParent), - selector: current, - actor, - fs, - }); - - current = node.selector; - - // this.directories_created.push(node); - } - - this.created = await fs.node(current); - } -} - -class HLMkdir extends HLFilesystemOperation { - static DESCRIPTION = ` - High-level mkdir operation. - - This operation is a wrapper around the low-level mkdir operation. - It provides the following features: - - create missing parent directories - - overwrite existing files - - dedupe names - - create shortcuts - `; - - static PARAMETERS = { - parent: new FSNodeParam('parent', { optional: true }), - path: new StringParam('path'), - overwrite: new FlagParam('overwrite', { optional: true }), - create_missing_parents: new FlagParam('create_missing_parents', { optional: true }), - user: new UserParam(), - - shortcut_to: new FSNodeParam('shortcut_to', { optional: true }), - }; - - static MODULES = { - _path: require('path'), - }; - - static PROPERTIES = { - parent_directories_created: () => [], - }; - - static FEATURES = [ - new OtelFeature([ - '_get_existing_parent', - '_create_parents', - ]), - ]; - - async _run () { - const { context, values } = this; - const { _path } = this.modules; - const fs = context.get('services').get('filesystem'); - - if ( ! is_valid_path(values.path, { - no_relative_components: true, - allow_path_fragment: true, - }) ) { - throw APIError.create('field_invalid', null, { - key: 'path', - expected: 'valid path', - got: 'invalid path', - }); - } - - // Unify the following formats: - // - full path: {"path":"/foo/bar", args...}, used by apitest (./tools/api-tester/apitest.js) - // - parent + path: {"parent": "/foo", "path":"bar", args...}, used by puter-js (puter.fs.mkdir("/foo/bar")) - if ( !values.parent && values.path ) { - values.parent = await fs.node(new NodePathSelector(_path.dirname(values.path))); - values.path = _path.basename(values.path); - } - - let parent_node = values.parent || await fs.node(new RootNodeSelector()); - - let target_basename = _path.basename(values.path); - - // "top_parent" is the immediate parent of the target directory - // (e.g: /home/foo/bar -> /home/foo) - const top_parent = values.create_missing_parents - ? await this._create_dir(parent_node) - : await this._get_existing_top_parent({ top_parent: parent_node }) - ; - - // TODO: this can be removed upon completion of: https://github.com/HeyPuter/puter/issues/1352 - if ( top_parent.isRoot ) { - // root directory is read-only - throw APIError.create('forbidden', null, { - message: 'Cannot create directories in the root directory.', - }); - } - - // `parent_node` becomes the parent of the last directory name - // specified under `path`. - parent_node = await this._create_parents({ - parent_node: top_parent, - actor: values.actor, - }); - - const user_id = values.actor.type.user.id; - - const has_perm = await chkperm(await parent_node.get('entry'), user_id, 'write'); - if ( ! has_perm ) throw APIError.create('permission_denied'); - - const existing = await fs.node(new NodeChildSelector(parent_node.selector, target_basename)); - - await existing.fetchEntry(); - - if ( existing.found ) { - const { overwrite, dedupe_name, create_missing_parents } = values; - if ( overwrite ) { - // TODO: tag rm operation somehow - const has_perm = await chkperm(await existing.get('entry'), user_id, 'write'); - if ( ! has_perm ) throw APIError.create('permission_denied'); - const hl_remove = new HLRemove(); - await hl_remove.run({ - target: existing, - actor: values.actor, - recursive: true, - }); - } - else if ( dedupe_name ) { - const fs = context.get('services').get('filesystem'); - const parent_selector = parent_node.selector; - for ( let i = 1 ;; i++ ) { - let try_new_name = `${target_basename} (${i})`; - const selector = new NodeChildSelector(parent_selector, try_new_name); - const exists = await parent_node.provider.quick_check({ - selector, - }); - if ( ! exists ) { - target_basename = try_new_name; - break; - } - } - } - else if ( create_missing_parents ) { - if ( ! existing.entry.is_dir ) { - throw APIError.create('dest_is_not_a_directory'); - } - this.created = existing; - this.used_existing = true; - return await this.created.getSafeEntry(); - } else { - throw APIError.create('item_with_same_name_exists', null, { - entry_name: target_basename, - }); - } - } - - if ( values.shortcut_to ) { - const shortcut_to = values.shortcut_to; - if ( ! await shortcut_to.exists() ) { - throw APIError.create('shortcut_to_does_not_exist'); - } - if ( ! shortcut_to.entry.is_dir ) { - throw APIError.create('shortcut_target_is_a_directory'); - } - const has_perm = await chkperm(shortcut_to.entry, user_id, 'read'); - if ( ! has_perm ) throw APIError.create('forbidden'); - - this.created = await fs.mkshortcut({ - parent: parent_node, - name: target_basename, - actor: values.actor, - target: shortcut_to, - }); - - await this.created.awaitStableEntry(); - return await this.created.getSafeEntry(); - } - - let created_node; - try { - const ll_mkdir = new LLMkdir(); - created_node = await ll_mkdir.run({ - parent: parent_node, - name: target_basename, - actor: values.actor, - }); - } catch ( error ) { - // This "error" can occur when multiple `hl_mkdir` operations are being - // run at the same time with the `createMissingParents` option enabled. - const errorCode = error.code || error.fields?.code; - if ( errorCode === 'item_with_same_name_exists' ) { - const existing_node = await fs.node(new NodeChildSelector(parent_node.selector, target_basename)); - - // Wait for the entry to be stable (it might still be in the process - // of being created by another parallel request) - await existing_node.awaitStableEntry(); - await existing_node.fetchEntry(); - - // If this is a file we need to re-throw the error - if ( await existing_node.get('type') !== FSNodeContext.TYPE_DIRECTORY ) { - throw error; - } - - created_node = existing_node; - } else { - throw error; - } - } - - this.created = created_node; - - const all_nodes = [ - ...this.parent_directories_created, - this.created, - ]; - - await Promise.all(all_nodes.map(node => node.awaitStableEntry())); - - const response = await this.created.getSafeEntry(); - response.parent_dirs_created = []; - for ( const node of this.parent_directories_created ) { - response.parent_dirs_created.push(await node.getSafeEntry()); - } - response.requested_path = values.path; - - return response; - } - - async _create_parents ({ parent_node }) { - const { context, values } = this; - const { _path } = this.modules; - - const fs = context.get('services').get('filesystem'); - - // Determine the deepest existing node - let deepest_existing = parent_node; - let remaining_path = _path.dirname(values.path).split('/').filter(Boolean); - { - const parts = remaining_path.slice(); - for ( ;; ) { - if ( remaining_path.length === 0 ) { - return deepest_existing; - } - const component = remaining_path[0]; - const next_selector = new NodeChildSelector(deepest_existing.selector, component); - const next_node = await fs.node(next_selector); - if ( ! await next_node.exists() ) { - break; - } - deepest_existing = next_node; - remaining_path.shift(); - } - } - - const tree_op = new MkTree(); - await tree_op.run({ - parent: deepest_existing, - tree: [remaining_path.join('/')], - }); - - this.parent_directories_created = tree_op.directories_created; - - return tree_op.leaves[0]; - } - - /** - * Creates a directory and all its ancestors. - * - * @param {FSNodeContext} dir - The directory to create. - * @returns {Promise} The created directory. - */ - async _create_dir (dir) { - if ( await dir.exists() ) { - if ( ! dir.entry.is_dir ) { - throw APIError.create('dest_is_not_a_directory'); - } - return dir; - } - - const maybe_path_selector = - dir.get_selector_of_type(NodePathSelector); - - if ( ! maybe_path_selector ) { - throw APIError.create('dest_does_not_exist', null, { what_dest: 'path from selector' }); - } - - const path = maybe_path_selector.value; - - const fs = this.context.get('services').get('filesystem'); - - const tree_op = new MkTree(); - await tree_op.run({ - parent: await fs.node(new RootNodeSelector()), - tree: [path], - }); - - return tree_op.leaves[0]; - } - - async _get_existing_top_parent ({ top_parent }) { - if ( ! await top_parent.exists() ) { - throw APIError.create('dest_does_not_exist', null, { - // This seems verbose, but is necessary information when creating - // shortcuts, otherwise the developer doesn't know if we're talking - // about the shortcut's target directory or this parent directory. - what_dest: 'parent directory of the new directory being created', - }); - } - - if ( ! top_parent.entry.is_dir ) { - throw APIError.create('dest_is_not_a_directory'); - } - - return top_parent; - } -} - -module.exports = { - QuickMkdir, - HLMkdir, - MkTree, -}; diff --git a/src/backend/src/deprecated/filesystem/hl_operations/hl_mklink.js b/src/backend/src/deprecated/filesystem/hl_operations/hl_mklink.js deleted file mode 100644 index 9c82ce993..000000000 --- a/src/backend/src/deprecated/filesystem/hl_operations/hl_mklink.js +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const FSNodeParam = require('../../../api/filesystem/FSNodeParam'); -const StringParam = require('../../../api/filesystem/StringParam'); -const { HLFilesystemOperation } = require('./definitions'); -const APIError = require('../../../api/APIError'); -const { TYPE_DIRECTORY } = require('../FSNodeContext'); - -class HLMkLink extends HLFilesystemOperation { - static PARAMETERS = { - parent: new FSNodeParam('symlink'), - name: new StringParam('name'), - target: new StringParam('target'), - }; - - static MODULES = { - path: require('node:path'), - }; - - async _run () { - const { context, values } = this; - const fs = context.get('services').get('filesystem'); - - const { target, parent, user } = values; - let { name } = values; - - if ( ! name ) { - throw APIError.create('field_empty', null, { key: 'name' }); - } - - if ( ! await parent.exists() ) { - throw APIError.create('dest_does_not_exist'); - } - - if ( await parent.get('type') !== TYPE_DIRECTORY ) { - throw APIError.create('dest_is_not_a_directory'); - } - - { - const dest = await parent.getChild(name); - if ( await dest.exists() ) { - throw APIError.create('item_with_same_name_exists', null, { - entry_name: name, - }); - } - } - - const created = await fs.mklink({ - target, - parent, - name, - user, - }); - - await created.awaitStableEntry(); - return await created.getSafeEntry(); - } -} - -module.exports = { - HLMkLink, -}; diff --git a/src/backend/src/deprecated/filesystem/hl_operations/hl_mkshortcut.js b/src/backend/src/deprecated/filesystem/hl_operations/hl_mkshortcut.js deleted file mode 100644 index 5ed7d9892..000000000 --- a/src/backend/src/deprecated/filesystem/hl_operations/hl_mkshortcut.js +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../../api/APIError'); -const FSNodeParam = require('../../../api/filesystem/FSNodeParam'); -const FlagParam = require('../../../api/filesystem/FlagParam'); -const StringParam = require('../../../api/filesystem/StringParam'); -const { TYPE_DIRECTORY } = require('../FSNodeContext'); -const { HLFilesystemOperation } = require('./definitions'); - -class HLMkShortcut extends HLFilesystemOperation { - static PARAMETERS = { - parent: new FSNodeParam('shortcut'), - name: new StringParam('name'), - target: new FSNodeParam('target'), - - dedupe_name: new FlagParam('dedupe_name', { optional: true }), - }; - - static MODULES = { - path: require('node:path'), - }; - - async _run () { - const { context, values } = this; - const fs = context.get('services').get('filesystem'); - - const { target, parent, user, actor } = values; - let { name, dedupe_name } = values; - - if ( ! await target.exists() ) { - throw APIError.create('shortcut_to_does_not_exist'); - } - - if ( ! name ) { - dedupe_name = true; - name = `Shortcut to ${ await target.get('name')}`; - } - - { - const svc_acl = context.get('services').get('acl'); - if ( ! await svc_acl.check(actor, target, 'read') ) { - throw await svc_acl.get_safe_acl_error(actor, target, 'read'); - } - } - - if ( ! await parent.exists() ) { - throw APIError.create('dest_does_not_exist'); - } - - if ( await parent.get('type') !== TYPE_DIRECTORY ) { - throw APIError.create('dest_is_not_a_directory'); - } - - { - const dest = await parent.getChild(name); - if ( await dest.exists() ) { - if ( ! dedupe_name ) { - throw APIError.create('item_with_same_name_exists', null, { - entry_name: name, - }); - } - - const name_ext = this.modules.path.extname(name); - const name_noext = this.modules.path.basename(name, name_ext); - for ( let i = 1 ;; i++ ) { - const try_new_name = `${name_noext} (${i})${name_ext}`; - const try_dest = await parent.getChild(try_new_name); - if ( ! await try_dest.exists() ) { - name = try_new_name; - break; - } - } - } - } - - const created = await fs.mkshortcut({ - target, - parent, - name, - user, - }); - - await created.awaitStableEntry(); - return await created.getSafeEntry(); - } -} - -module.exports = { - HLMkShortcut, -}; diff --git a/src/backend/src/deprecated/filesystem/hl_operations/hl_move.js b/src/backend/src/deprecated/filesystem/hl_operations/hl_move.js deleted file mode 100644 index 9518b6e3b..000000000 --- a/src/backend/src/deprecated/filesystem/hl_operations/hl_move.js +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../../api/APIError'); -const { chkperm, validate_fsentry_name, is_ancestor_of, df, get_user } = require('../../../helpers'); -const { LLMove } = require('../ll_operations/ll_move'); -const { RootNodeSelector } = require('../node/selectors'); -const { HLFilesystemOperation } = require('./definitions'); -const { MkTree } = require('./hl_mkdir'); -const { HLRemove } = require('./hl_remove'); -const { TYPE_DIRECTORY } = require('../FSNodeContext'); - -class HLMove extends HLFilesystemOperation { - static MODULES = { - _path: require('path'), - }; - - static PROPERTIES = { - parent_directories_created: () => [], - }; - - async _run () { - const { _path } = this.modules; - - const { context, values } = this; - const svc = context.get('services'); - const fs = svc.get('filesystem'); - - const new_metadata = typeof values.new_metadata === 'string' - ? values.new_metadata : JSON.stringify(values.new_metadata); - - // !! new_name, create_missing_parents, overwrite, dedupe_name - - let parent = values.destination_or_parent; - let dest = null; - const source = values.source; - - if ( await source.get('is-root') ) { - throw APIError.create('immutable'); - } - if ( await parent.get('is-root') ) { - throw APIError.create('cannot_copy_to_root'); - } - - if ( ! await source.exists() ) { - throw APIError.create('source_does_not_exist'); - } - - if ( ! await chkperm(source.entry, values.user.id, 'cp') ) { - throw APIError.create('forbidden'); - } - - if ( source.entry.immutable ) { - throw APIError.create('immutable'); - } - - // If the "parent" is a file, then it's actually our destination; not the parent. - if ( !values.new_name && await parent.exists() && await parent.get('type') !== TYPE_DIRECTORY ) { - dest = parent; - parent = await dest.getParent(); - } - - if ( ! await parent.exists() ) { - if ( !parent.path || !values.create_missing_parents ) { - throw APIError.create('dest_does_not_exist'); - } - - const tree_op = new MkTree(); - await tree_op.run({ - parent: await fs.node(new RootNodeSelector()), - tree: [parent.path], - }); - - this.parent_directories_created = tree_op.directories_created; - - parent = tree_op.leaves[0]; - } - - await parent.fetchEntry(); - if ( ! await chkperm(parent.entry, values.user.id, 'write') ) { - throw APIError.create('forbidden'); - } - if ( await parent.get('type') !== TYPE_DIRECTORY ) { - throw APIError.create('dest_is_not_a_directory'); - } - - let source_user, dest_user; - - // 3. Verify cross-user size constraints - const src_user_id = await source.get('user_id'); - const parent_user_id = await parent.get('user_id'); - if ( src_user_id !== parent_user_id ) { - source_user = await get_user({ id: src_user_id }); - if ( source_user.id !== parent_user_id ) - { - dest_user = await get_user({ id: parent_user_id }); - } - else - { - dest_user = source_user; - } - await source.fetchSize(); - const item_size = source.entry.size; - const sizeService = svc.get('sizeService'); - const capacity = await sizeService.get_storage_capacity(dest_user.id); - if ( capacity - await df(dest_user.id) - item_size < 0 ) { - throw APIError.create('storage_limit_reached'); - } - } - - let target_name = values.new_name ?? await source.get('name'); - const metadata = new_metadata ?? await source.get('metadata'); - - try { - validate_fsentry_name(target_name); - } catch (e) { - throw APIError.create(400, e); - } - - if ( dest === null ) { - dest = await parent.getChild(target_name); - } - - const src_uid = await source.get('uid'); - // const dst_uid = await dest.get('uid'); - const par_uid = await parent.get('uid'); - - if ( src_uid === par_uid ) { - throw APIError.create('source_and_dest_are_the_same'); - } - if ( await is_ancestor_of(src_uid, par_uid) ) { - throw APIError('cannot_move_item_into_itself'); - } - - let overwritten; - if ( await dest.exists() ) { - if ( !values.overwrite && !values.dedupe_name ) { - throw APIError.create('item_with_same_name_exists', null, { - entry_name: await dest.get('name'), - }); - } - - if ( values.dedupe_name ) { - const target_ext = _path.extname(target_name); - const target_noext = _path.basename(target_name, target_ext); - for ( let i = 1 ;; i++ ) { - const try_new_name = `${target_noext} (${i})${target_ext}`; - const exists = await parent.hasChild(try_new_name); - if ( ! exists ) { - target_name = try_new_name; - break; - } - } - - dest = await parent.getChild(target_name); - } - else if ( values.overwrite ) { - overwritten = await dest.getSafeEntry(); - const hl_remove = new HLRemove(); - await hl_remove.run({ - target: dest, - user: values.user, - }); - } - else { - throw new Error('unreachable'); - } - } - - const old_path = await source.get('path'); - - const ll_move = new LLMove(); - const source_new = await ll_move.run({ - source, - parent, - target_name, - user: values.user, - metadata: metadata, - }); - - await source_new.awaitStableEntry(); - await source_new.fetchSuggestedApps(); - await source_new.fetchOwner(); - - const response = { - moved: await source_new.getSafeEntry({ thumbnail: true }), - overwritten, - old_path, - }; - - response.parent_dirs_created = []; - for ( const node of this.parent_directories_created ) { - response.parent_dirs_created.push(await node.getSafeEntry()); - } - - return response; - } -} - -module.exports = { - HLMove, -}; diff --git a/src/backend/src/deprecated/filesystem/hl_operations/hl_name_search.js b/src/backend/src/deprecated/filesystem/hl_operations/hl_name_search.js deleted file mode 100644 index 89a69b9fa..000000000 --- a/src/backend/src/deprecated/filesystem/hl_operations/hl_name_search.js +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { DB_READ } = require('../../../services/database/consts'); -const { Context } = require('../../../util/context'); -const { NodeUIDSelector } = require('../node/selectors'); -const { HLFilesystemOperation } = require('./definitions'); - -class HLNameSearch extends HLFilesystemOperation { - async _run () { - let { actor, term } = this.values; - const services = Context.get('services'); - const svc_fs = services.get('filesystem'); - const db = services.get('database') - .get(DB_READ, 'fs.namesearch'); - - term = term.replace(/%/g, ''); - term = `%${ term }%`; - - // Only user actors can do this, because the permission - // system would otherwise slow things down - if ( ! actor.type.user ) return []; - - const results = await db.read( - 'SELECT uuid FROM fsentries WHERE name LIKE ? AND ' + - 'user_id = ? LIMIT 50', - [term, actor.type.user.id], - ); - - const uuids = results.map(v => v.uuid); - - const fsnodes = await Promise.all(uuids.map(async uuid => { - return await svc_fs.node(new NodeUIDSelector(uuid)); - })); - - return Promise.all(fsnodes.map(async fsnode => { - return await fsnode.getSafeEntry(); - })); - } -} - -module.exports = { - HLNameSearch, -}; diff --git a/src/backend/src/deprecated/filesystem/hl_operations/hl_read.js b/src/backend/src/deprecated/filesystem/hl_operations/hl_read.js deleted file mode 100644 index f33253fac..000000000 --- a/src/backend/src/deprecated/filesystem/hl_operations/hl_read.js +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../../api/APIError'); -const { LLRead } = require('../ll_operations/ll_read'); -const { HLFilesystemOperation } = require('./definitions'); - -class HLRead extends HLFilesystemOperation { - static CONCERN = 'filesystem'; - static MODULES = { - 'stream': require('stream'), - }; - - async _run () { - const { - fsNode, actor, - line_count, byte_count, - offset, - version_id, range, - } = this.values; - - if ( ! await fsNode.exists() ) { - throw APIError.create('subject_does_not_exist'); - } - - const ll_read = new LLRead(); - let stream = await ll_read.run({ - fsNode, - actor, - version_id, - range, - ...(byte_count !== undefined ? { - offset: offset ?? 0, - length: byte_count, - } : {}), - }); - - if ( line_count !== undefined ) { - stream = this._wrap_stream_line_count(stream, line_count); - } - - return stream; - } - - /** - * returns a new stream that will only produce the first `line_count` lines - * @param {*} stream - input stream - * @param {*} line_count - number of lines to produce - */ - _wrap_stream_line_count (stream, line_count) { - const readline = require('readline'); - const rl = readline.createInterface({ - input: stream, - terminal: false, - }); - - const { PassThrough } = this.modules.stream; - - const output_stream = new PassThrough(); - - let lines_read = 0; - new Promise((resolve, reject) => { - rl.on('line', (line) => { - if ( lines_read++ >= line_count ) { - return rl.close(); - } - - output_stream.write(lines_read > 1 ? `\r\n${ line}` : line); - }); - rl.on('error', () => { - console.log('error'); - }); - rl.on('close', function () { - resolve(); - }); - }); - - return output_stream; - } -} - -module.exports = { - HLRead, -}; diff --git a/src/backend/src/deprecated/filesystem/hl_operations/hl_readdir.js b/src/backend/src/deprecated/filesystem/hl_operations/hl_readdir.js deleted file mode 100644 index 750dde780..000000000 --- a/src/backend/src/deprecated/filesystem/hl_operations/hl_readdir.js +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../../api/APIError'); -const { Context } = require('../../../util/context'); -const { get_apps, suggestedAppsForFsEntries } = require('../../../helpers'); -const { ECMAP } = require('../ECMAP'); -const { TYPE_DIRECTORY, TYPE_SYMLINK } = require('../FSNodeContext'); -const { LLListUsers } = require('../ll_operations/ll_listusers'); -const { LLReadDir } = require('../ll_operations/ll_readdir'); -const { LLReadShares } = require('../ll_operations/ll_readshares'); -const { HLFilesystemOperation } = require('./definitions'); -const { DB_READ } = require('../../../services/database/consts'); -const config = require('../../../config'); - -class HLReadDir extends HLFilesystemOperation { - static CONCERN = 'filesystem'; - async _run () { - return ECMAP.arun(async () => { - const ecmap = Context.get(ECMAP.SYMBOL); - ecmap.store_fsNodeContext(this.values.subject); - return await this.__run(); - }); - } - async __run () { - const { subject: subject_let, user, no_thumbs, no_assocs, no_subdomains, actor } = this.values; - let subject = subject_let; - - if ( ! await subject.exists() ) { - throw APIError.create('subject_does_not_exist'); - } - - if ( await subject.get('type') === TYPE_SYMLINK ) { - const { context } = this; - const svc_acl = context.get('services').get('acl'); - if ( ! await svc_acl.check(actor, subject, 'read') ) { - throw await svc_acl.get_safe_acl_error(actor, subject, 'read'); - } - const target = await subject.getTarget(); - subject = target; - } - - if ( await subject.get('type') !== TYPE_DIRECTORY ) { - const { context } = this; - const svc_acl = context.get('services').get('acl'); - if ( ! await svc_acl.check(actor, subject, 'see') ) { - throw await svc_acl.get_safe_acl_error(actor, subject, 'see'); - } - throw APIError.create('readdir_of_non_directory'); - } - - let children; - - this.log.debug( - 'READDIR', - { - userdir: await subject.isUserDirectory(), - namediff: await subject.get('name') !== user.username, - }, - ); - if ( subject.isRoot ) { - const ll_listusers = new LLListUsers(); - children = await ll_listusers.run(this.values); - } else if ( - await subject.getUserPart() !== user.username && - await subject.isUserDirectory() - ) { - const ll_readshares = new LLReadShares(); - children = await ll_readshares.run(this.values); - } else { - const ll_readdir = new LLReadDir(); - children = await ll_readdir.run(this.values); - } - - const associated_app_specifiers = []; - const children_with_assoc = []; - await Promise.all(children.map(async child => { - if ( ! no_thumbs ) { - await child.fetchEntry({ thumbnail: true }); - } else { - await child.fetchEntry(); - } - - const assoc_id = child.entry?.associated_app_id; - if ( assoc_id ) { - associated_app_specifiers.push({ id: assoc_id }); - children_with_assoc.push({ child, assoc_id }); - } - })); - - if ( associated_app_specifiers.length ) { - const assoc_apps = await get_apps(associated_app_specifiers); - const app_by_id = new Map(); - for ( let i = 0; i < associated_app_specifiers.length; i++ ) { - const app = assoc_apps[i]; - if ( app ) { - app_by_id.set(associated_app_specifiers[i].id, app); - } - } - for ( const { child, assoc_id } of children_with_assoc ) { - const app = app_by_id.get(assoc_id); - if ( app ) { - child.entry.associated_app = app; - } - } - } - - if ( ! no_assocs ) { - await this.#batchFetchSuggestedApps(children, user); - } - - if ( ! no_subdomains ) { - const usedPrefetchedSubdomains = await this.#applySubdomains(children); - if ( ! usedPrefetchedSubdomains ) { - await this.#batchFetchSubdomains(children, user); - } - } - - return Promise.all(children.map(async child => { - const entry = await child.getSafeEntry(); - if ( !no_thumbs && entry.associated_app ) { - const svc_appIcon = this.context.get('services').get('app-icon'); - const iconPath = svc_appIcon.getAppIconPath({ - appUid: entry.associated_app.uid ?? entry.associated_app.uuid, - size: 64, - }); - if ( iconPath ) { - entry.associated_app.icon = iconPath; - } - } - return entry; - })); - } - - async #applySubdomains (children) { - let usedPrefetchedSubdomains = false; - - for ( const child of children ) { - const entry = child.entry; - if ( ! entry ) continue; - this.#initializeSubdomainFields(entry); - - const prefetchedSubdomains = child.subdomains ?? entry.subdomains; - if ( prefetchedSubdomains === undefined ) return false; - - usedPrefetchedSubdomains = true; - if ( ! Array.isArray(prefetchedSubdomains) ) continue; - - for ( const subdomain of prefetchedSubdomains ) { - this.#appendSubdomainToEntry({ - entry, - subdomain: subdomain?.subdomain, - uuid: subdomain?.uuid, - }); - } - } - - return usedPrefetchedSubdomains; - } - - async #batchFetchSubdomains (children, user) { - const childIds = []; - const childById = new Map(); - - for ( const child of children ) { - const entry = child.entry; - if ( ! entry ) continue; - this.#initializeSubdomainFields(entry); - if ( entry.id == null ) continue; - childIds.push(entry.id); - childById.set(entry.id, child); - } - - if ( childIds.length === 0 ) return; - - const placeholders = childIds.map(() => '?').join(','); - const db = this.context.get('services').get('database').get(DB_READ, 'filesystem'); - const rows = await db.read( - `SELECT root_dir_id, subdomain, uuid - FROM subdomains - WHERE root_dir_id IN (${placeholders}) AND user_id = ?`, - [...childIds, user.id], - ); - - for ( const row of rows ) { - const child = childById.get(row.root_dir_id); - if ( ! child ) continue; - this.#appendSubdomainToEntry({ - entry: child.entry, - subdomain: row.subdomain, - uuid: row.uuid, - }); - } - } - - #initializeSubdomainFields (entry) { - entry.subdomains = []; - entry.workers = []; - entry.has_website = false; - } - - #appendSubdomainToEntry ({ entry, subdomain, uuid }) { - if ( ! subdomain ) return; - - if ( entry.is_dir ) { - entry.subdomains.push({ - subdomain, - address: `${config.protocol}://${subdomain}.puter.site`, - uuid, - }); - } else { - const workerName = subdomain.split('.').pop(); - entry.workers.push({ - subdomain: workerName, - address: `https://${workerName}.puter.work`, - uuid, - }); - } - - entry.has_website = true; - } - - async #batchFetchSuggestedApps (children, user) { - const entries = []; - const targets = []; - - for ( const child of children ) { - const entry = child.entry; - if ( !entry || entry.suggested_apps ) continue; - entries.push(entry); - targets.push(entry); - } - - if ( entries.length === 0 ) return; - - const suggestedLists = await suggestedAppsForFsEntries(entries, { user }); - for ( let index = 0; index < targets.length; index++ ) { - targets[index].suggested_apps = suggestedLists[index] ?? []; - } - } -} - -module.exports = { - HLReadDir, -}; diff --git a/src/backend/src/deprecated/filesystem/hl_operations/hl_remove.js b/src/backend/src/deprecated/filesystem/hl_operations/hl_remove.js deleted file mode 100644 index be88fd947..000000000 --- a/src/backend/src/deprecated/filesystem/hl_operations/hl_remove.js +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../../api/APIError'); -const { chkperm } = require('../../../helpers'); -const { TYPE_DIRECTORY } = require('../FSNodeContext'); -const { LLRmDir } = require('../ll_operations/ll_rmdir'); -const { LLRmNode } = require('../ll_operations/ll_rmnode'); -const { HLFilesystemOperation } = require('./definitions'); - -class HLRemove extends HLFilesystemOperation { - static PARAMETERS = { - target: {}, - user: {}, - recursive: {}, - descendants_only: {}, - }; - - async _run () { - const { target, user } = this.values; - - if ( ! await target.exists() ) { - throw APIError.create('subject_does_not_exist'); - } - - if ( ! chkperm(target.entry, user.id, 'rm') ) { - throw APIError.create('forbidden'); - } - - if ( await target.get('type') === TYPE_DIRECTORY ) { - const ll_rmdir = new LLRmDir(); - return await ll_rmdir.run(this.values); - } - - const ll_rmnode = new LLRmNode(); - return await ll_rmnode.run(this.values); - } -} - -module.exports = { - HLRemove, -}; diff --git a/src/backend/src/deprecated/filesystem/hl_operations/hl_stat.js b/src/backend/src/deprecated/filesystem/hl_operations/hl_stat.js deleted file mode 100644 index 232cd93be..000000000 --- a/src/backend/src/deprecated/filesystem/hl_operations/hl_stat.js +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { Context } = require('../../../util/context'); -const { HLFilesystemOperation } = require('./definitions'); -const APIError = require('../../../api/APIError'); -const { ECMAP } = require('../ECMAP'); -const { NodeUIDSelector } = require('../node/selectors'); - -class HLStat extends HLFilesystemOperation { - static MODULES = { - 'mime-types': require('mime-types'), - }; - - async _run () { - return await ECMAP.arun(async () => { - const ecmap = Context.get(ECMAP.SYMBOL); - ecmap.store_fsNodeContext(this.values.subject); - return await this.__run(); - }); - } - // async _run () { - // return await this.__run(); - // } - async __run () { - const { - subject, user, - return_subdomains, - return_permissions, // Deprecated: kept for backwards compatiable with `return_shares` - return_shares, - return_versions, - return_size, - } = this.values; - - const maybe_uid_selector = subject.get_selector_of_type(NodeUIDSelector); - - // users created before 2025-07-30 might have fsentries with NULL paths. - // we can remove this check once that is fixed. - const user_unix_ts = Number((`${Date.parse(Context.get('actor')?.type?.user?.timestamp)}`).slice(0, -3)); - const paths_are_fine = user_unix_ts >= 1722385593; - - const do_after_fetchEntry = []; - const do_alongside_fetchEntry = []; - - if ( return_size ) { - do_after_fetchEntry.push(async () => { - await subject.fetchSize(user); - }); - } - - if ( return_subdomains ) { - do_after_fetchEntry.push(async () => { - await subject.fetchSubdomains(user); - }); - } - - if ( return_shares || return_permissions ) { - do_after_fetchEntry.push(async () => { - await subject.fetchShares(); - }); - } - - if ( return_versions ) { - do_after_fetchEntry.push(async () => { - await subject.fetchVersions(); - }); - } - - do_after_fetchEntry.push(async () => { - await subject.fetchOwner(); - }, async () => { - await subject.get('writable'); - }); - - ((maybe_uid_selector || paths_are_fine) - ? do_alongside_fetchEntry - : do_after_fetchEntry).push(subject.fetchIsEmpty.bind(subject)); - - // if ( maybe_uid_selector || paths_are_fine ) { - // await Promise.all([ - // subject.fetchEntry(), - // subject.fetchIsEmpty(), - // ]); - // } else { - // // We need the entry first in order for is_empty to work correctly - // await subject.fetchEntry(); - // await subject.fetchIsEmpty(); - // } - - await Promise.all([ - (async () => { - await subject.fetchEntry(); - const context = Context.get(); - const svc_acl = context.get('services').get('acl'); - const actor = context.get('actor'); - if ( ! await svc_acl.check(actor, subject, 'read') ) { - throw await svc_acl.get_safe_acl_error(actor, subject, 'read'); - } - if ( ! subject.found ) { - throw APIError.create('subject_does_not_exist'); - } - await Promise.all(do_after_fetchEntry.map(f => f())); - })(), - ...(do_alongside_fetchEntry.map(f => f())), - ]); - - // file not found - - // await subject.fetchOwner(); - - // TODO: why is this specific to stat? - const mime = this.require('mime-types'); - const contentType = mime.contentType(subject.entry.name); - subject.entry.type = contentType ? contentType : null; - - // if ( return_size ) await subject.fetchSize(user); - // if ( return_subdomains ) await subject.fetchSubdomains(user); - // if ( return_shares || return_permissions ) { - // await subject.fetchShares(); - // } - // if ( return_versions ) await subject.fetchVersions(); - - return await subject.getSafeEntry(); - } -} - -module.exports = { - HLStat, -}; diff --git a/src/backend/src/deprecated/filesystem/hl_operations/hl_write.js b/src/backend/src/deprecated/filesystem/hl_operations/hl_write.js deleted file mode 100644 index 10abce10a..000000000 --- a/src/backend/src/deprecated/filesystem/hl_operations/hl_write.js +++ /dev/null @@ -1,413 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../../api/APIError'); -const FSNodeParam = require('../../../api/filesystem/FSNodeParam'); -const FlagParam = require('../../../api/filesystem/FlagParam'); -const StringParam = require('../../../api/filesystem/StringParam'); -const UserParam = require('../../../api/filesystem/UserParam'); -const config = require('../../../config'); -const { TeePromise } = require('@heyputer/putility').libs.promise; -const { offset_write_stream } = require('../../../util/streamutil'); -const { TYPE_DIRECTORY } = require('../FSNodeContext'); -const { LLRead } = require('../ll_operations/ll_read'); -const { RootNodeSelector, NodePathSelector } = require('../node/selectors'); -const { is_valid_node_name } = require('../validation'); -const { HLFilesystemOperation } = require('./definitions'); -const { MkTree } = require('./hl_mkdir'); -const { Actor } = require('../../../services/auth/Actor'); -const { LLCWrite, LLOWrite } = require('../ll_operations/ll_write'); -const { validate_fsentry_name, chkperm } = require('../../../helpers'); - -// 2 MiB limit for client-provided thumbnails -const MAX_THUMBNAIL_SIZE = 2 * 1024 * 1024; - -class WriteCommonFeature { - install_in_instance (instance) { - instance._verify_size = async function () { - if ( - this.values.file && - this.values.file.size > config.max_file_size - ) { - throw APIError.create('file_too_large', null, { - max_size: config.max_file_size, - }); - } - - if ( - this.values.thumbnail && - typeof this.values.thumbnail === 'string' - ) { - const RATIO = 4 / 3; // 4 bytes per 3 base64 characters - const decoded_size = Math.ceil(this.values.thumbnail.length * RATIO); - if ( decoded_size > MAX_THUMBNAIL_SIZE ) { - throw APIError.create('thumbnail_too_large', null, { - max_size: MAX_THUMBNAIL_SIZE, - }); - } - } - - // configured thumbnail size limit (can be lower than MAX_THUMBNAIL_SIZE) - if ( - this.values.thumbnail && - this.values.thumbnail.size > config.max_thumbnail_size - ) { - throw APIError.create('thumbnail_too_large', null, { - max_size: config.max_thumbnail_size, - }); - } - }; - - instance._verify_room = async function () { - if ( ! this.values.file ) return; - - const sizeService = this.context.get('services').get('sizeService'); - const { file, user: user_let } = this.values; - let user = user_let; - - if ( ! user ) user = this.values.actor.type.user; - - const usage = await sizeService.get_usage(user.id); - const capacity = await sizeService.get_storage_capacity(user.id); - if ( capacity - usage - file.size < 0 ) { - throw APIError.create('storage_limit_reached'); - } - }; - } -} - -class HLWrite extends HLFilesystemOperation { - static DESCRIPTION = ` - High-level write operation. - - This operation is a wrapper around the low-level write operation. - It provides the following features: - - create missing parent directories - - overwrite existing files - - deduplicate files with the same name - - accept client-provided thumbnails - - create shortcuts - `; - - static FEATURES = [ - new WriteCommonFeature(), - ]; - - static PARAMETERS = { - // the parent directory, or a filepath that doesn't exist yet - destination_or_parent: new FSNodeParam('path'), - - // if specified, destination_or_parent must be a directory - specified_name: new StringParam('specified_name', { optional: true }), - - // used if specified_name is undefined and destination_or_parent is a directory - // NB: if destination_or_parent does not exist and create_missing_parents - // is true then destination_or_parent will be a directory - fallback_name: new StringParam('fallback_name', { optional: true }), - - overwrite: new FlagParam('overwrite', { optional: true }), - dedupe_name: new FlagParam('dedupe_name', { optional: true }), - - // other options - shortcut_to: new FSNodeParam('shortcut_to', { optional: true }), - create_missing_parents: new FlagParam('create_missing_parents', { optional: true }), - user: new UserParam(), - - // client-provided thumbnail as a base64 string - thumbnail: new StringParam('thumbnail', { optional: true }), - - // file: multer.File - }; - - static MODULES = { - _path: require('path'), - }; - - async _run () { - const { context, values } = this; - const { _path } = this.modules; - - const fs = context.get('services').get('filesystem'); - const svc_event = context.get('services').get('event'); - - let parent = values.destination_or_parent; - let destination = null; - - await this._verify_size(); - await this._verify_room(); - - this.checkpoint('before parent exists check'); - - if ( !await parent.exists() && values.create_missing_parents ) { - if ( ! (parent.selector instanceof NodePathSelector) ) { - throw APIError.create('dest_does_not_exist', null, { - parent: parent.selector, - }); - } - const path = parent.selector.value; - const tree_op = new MkTree(); - await tree_op.run({ - parent: await fs.node(new RootNodeSelector()), - tree: [path], - }); - - parent = await fs.node(new NodePathSelector(path)); - const parent_exists_now = await parent.exists(); - if ( ! parent_exists_now ) { - this.log.error('FAILED TO CREATE DESTINATION'); - throw APIError.create('dest_does_not_exist', null, { - parent: parent.selector, - }); - } - } - - if ( parent.isRoot ) { - throw APIError.create('cannot_write_to_root'); - } - - let target_name = values.specified_name || values.fallback_name; - - // If a name is specified then the destination must be a directory - if ( values.specified_name ) { - this.checkpoint('specified name condition'); - if ( ! await parent.exists() ) { - throw APIError.create('dest_does_not_exist'); - } - if ( await parent.get('type') !== TYPE_DIRECTORY ) { - throw APIError.create('dest_is_not_a_directory'); - } - target_name = values.specified_name; - } - - this.checkpoint('check parent DNE or is not a directory'); - if ( - !await parent.exists() || - await parent.get('type') !== TYPE_DIRECTORY - ) { - destination = parent; - parent = await destination.getParent(); - target_name = destination.name; - } - - if ( parent.isRoot ) { - throw APIError.create('cannot_write_to_root'); - } - - try { - // old validator is kept here to avoid changing the - // error messages; eventually is_valid_node_name - // will support more detailed error reporting - validate_fsentry_name(target_name); - if ( ! is_valid_node_name(target_name) ) { - throw { message: 'invalid node name' }; - } - } catch (e) { - throw APIError.create('invalid_file_name', null, { - name: target_name, - reason: e.message, - }); - } - - if ( ! destination ) { - destination = await parent.getChild(target_name); - } - - let is_overwrite = false; - - // TODO: Gotta come up with a reasonable guideline for if/when we put - // object members in the scope; it feels too arbitrary right now. - const { overwrite, dedupe_name } = values; - - this.checkpoint('before overwrite behaviours'); - - const dest_exists = await destination.exists(); - - if ( values.offset !== undefined && !dest_exists ) { - throw APIError.create('offset_without_existing_file'); - } - - // The correct ACL check here depends on context. - // ll_write checks ACL, but we need to shortcut it here - // or else we might send the user too much information. - { - const node_to_check = - ( dest_exists && overwrite && !dedupe_name ) - ? destination : parent; - - const actor = values.actor ?? Actor.adapt(values.user); - const svc_acl = context.get('services').get('acl'); - if ( ! await svc_acl.check(actor, node_to_check, 'write') ) { - throw await svc_acl.get_safe_acl_error(actor, node_to_check, 'write'); - } - } - - if ( dest_exists ) { - if ( !overwrite && !dedupe_name ) { - throw APIError.create('item_with_same_name_exists', null, { - entry_name: target_name, - }); - } - - if ( dedupe_name ) { - const target_ext = _path.extname(target_name); - const target_noext = _path.basename(target_name, target_ext); - for ( let i = 1 ;; i++ ) { - const try_new_name = `${target_noext} (${i})${target_ext}`; - const exists = await parent.hasChild(try_new_name); - if ( ! exists ) { - target_name = try_new_name; - break; - } - } - - destination = await parent.getChild(target_name); - } - - else if ( overwrite ) { - if ( await destination.get('immutable') ) { - throw APIError.create('immutable'); - } - if ( await destination.get('type') === TYPE_DIRECTORY ) { - throw APIError.create('cannot_overwrite_a_directory'); - } - is_overwrite = true; - } - } - - if ( values.shortcut_to ) { - this.checkpoint('shortcut condition'); - const shortcut_to = values.shortcut_to; - if ( ! await shortcut_to.exists() ) { - throw APIError.create('shortcut_to_does_not_exist'); - } - if ( await shortcut_to.get('type') === TYPE_DIRECTORY ) { - throw APIError.create('shortcut_target_is_a_directory'); - } - // TODO: legacy check - likely not needed - const has_perm = await chkperm(shortcut_to.entry, values.actor.type.user.id, 'read'); - if ( ! has_perm ) throw APIError.create('permission_denied'); - - this.created = await fs.mkshortcut({ - parent, - name: target_name, - actor: values.actor, - target: shortcut_to, - }); - - await this.created.awaitStableEntry(); - await this.created.fetchEntry({ thumbnail: true }); - return await this.created.getSafeEntry(); - } - - this.checkpoint('before thumbnail'); - - let thumbnail_promise = new TeePromise(); - if ( await parent.isAppDataDirectory() || values.no_thumbnail || !values.thumbnail ) { - thumbnail_promise.resolve(undefined); - } else { - // Allow extensions to transform client-provided thumbnails before DB write. - const thumbnailData = { url: values.thumbnail }; - await svc_event.emit('thumbnail.created', thumbnailData); - thumbnail_promise.resolve(thumbnailData.url); - } - - this.checkpoint('before delegate'); - - if ( values.offset !== undefined ) { - if ( ! is_overwrite ) { - throw APIError.create('offset_requires_overwrite'); - } - - if ( ! values.file.stream ) { - throw APIError.create('offset_requires_stream'); - } - - const replace_length = values.file.size; - let dst_size = await destination.get('size'); - if ( values.offset > dst_size ) { - values.offset = dst_size; - } - - if ( values.offset + values.file.size > dst_size ) { - dst_size = values.offset + values.file.size; - } - - const ll_read = new LLRead(); - const read_stream = await ll_read.run({ - fsNode: destination, - }); - - values.file.stream = offset_write_stream({ - originalDataStream: read_stream, - newDataStream: values.file.stream, - offset: values.offset, - replace_length, - }); - values.file.size = dst_size; - } - - if ( is_overwrite ) { - const ll_owrite = new LLOWrite(); - this.written = await ll_owrite.run({ - node: destination, - actor: values.actor, - file: values.file, - tmp: { - socket_id: values.socket_id, - operation_id: values.operation_id, - item_upload_id: values.item_upload_id, - }, - fsentry_tmp: { - thumbnail_promise, - }, - message: values.message, - }); - } else { - const ll_cwrite = new LLCWrite(); - this.written = await ll_cwrite.run({ - parent, - name: target_name, - actor: values.actor, - file: values.file, - tmp: { - socket_id: values.socket_id, - operation_id: values.operation_id, - item_upload_id: values.item_upload_id, - }, - fsentry_tmp: { - thumbnail_promise, - }, - message: values.message, - app_id: values.app_id, - }); - } - - this.checkpoint('after delegate'); - - await this.written.awaitStableEntry(); - this.checkpoint('after await stable entry'); - const response = await this.written.getSafeEntry({ thumbnail: true }); - this.checkpoint('after get safe entry'); - - return response; - } -} - -module.exports = { - HLWrite, -}; diff --git a/src/backend/src/deprecated/filesystem/lib/PuterPath.js b/src/backend/src/deprecated/filesystem/lib/PuterPath.js deleted file mode 100644 index 2a2457ab1..000000000 --- a/src/backend/src/deprecated/filesystem/lib/PuterPath.js +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const _path = require('path'); - -/** - * Puter paths look like any of the following: - * - * Absolute path: /user/dir1/dir2/file - * From UID: AAAA-BBBB-CCCC-DDDD/../a/b/c - * - * The difference between an absolute path and a UID-relative path - * is the leading forward-slash character. - */ -class PuterPath { - static NULL_UUID = '00000000-0000-0000-0000-000000000000'; - - static adapt (value) { - if ( value instanceof PuterPath ) return value; - return new PuterPath(value); - } - - constructor (text) { - this.text = text; - } - - set text (text) { - this.text_ = text.trim(); - this.normUnix = _path.normalize(text); - this.normFlat = - (this.normUnix.endsWith('/') && this.normUnix.length > 1) - ? this.normUnix.slice(0, -1) : this.normUnix; - } - get text () { - return this.text_; - } - - isRoot () { - if ( this.normFlat === '/' ) return true; - if ( this.normFlat === this.constructor.NULL_UUID ) { - return true; - } - return false; - } - - isAbsolute () { - return this.text.startsWith('/'); - } - - isFromUID () { - return !this.isAbsolute(); - } - - get reference () { - if ( this.isAbsolute ) return this.constructor.NULL_UUID; - - return this.text.slice(0, this.text.indexOf('/')); - } - - get relativePortion () { - if ( this.isAbsolute() ) { - return this.text.slice(1); - } - - if ( ! this.text.includes('/') ) return ''; - return this.text.slice(this.text.indexOf('/') + 1); - } -} - -module.exports = { PuterPath }; diff --git a/src/backend/src/deprecated/filesystem/ll_operations/definitions.js b/src/backend/src/deprecated/filesystem/ll_operations/definitions.js deleted file mode 100644 index 5905fa4c1..000000000 --- a/src/backend/src/deprecated/filesystem/ll_operations/definitions.js +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { BaseOperation } = require('../../../services/OperationTraceService'); - -class LLFilesystemOperation extends BaseOperation { -} - -module.exports = { - LLFilesystemOperation, -}; diff --git a/src/backend/src/deprecated/filesystem/ll_operations/ll_copy.js b/src/backend/src/deprecated/filesystem/ll_operations/ll_copy.js deleted file mode 100644 index 885fc3f64..000000000 --- a/src/backend/src/deprecated/filesystem/ll_operations/ll_copy.js +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { LLFilesystemOperation } = require('./definitions'); -const fsCapabilities = require('../definitions/capabilities'); - -class LLCopy extends LLFilesystemOperation { - static MODULES = { - _path: require('path'), - uuidv4: require('uuid').v4, - }; - - async _run () { - const { _path, uuidv4 } = this.modules; - const { context } = this; - const { source, parent, user, actor, target_name } = this.values; - const svc = context.get('services'); - - const fs = svc.get('filesystem'); - const svc_event = svc.get('event'); - - const uuid = uuidv4(); - const ts = Math.round(Date.now() / 1000); - - this.field('target-uid', uuid); - this.field('source', source.selector.describe()); - - this.checkpoint('before fetch parent entry'); - await parent.fetchEntry(); - this.checkpoint('before fetch source entry'); - await source.fetchEntry({ thumbnail: true }); - this.checkpoint('fetched source and parent entries'); - - // Access Control - { - const svc_acl = context.get('services').get('acl'); - this.checkpoint('copy :: access control'); - - // Check read access to source - if ( ! await svc_acl.check(actor, source, 'read') ) { - throw await svc_acl.get_safe_acl_error(actor, source, 'read'); - } - - // Check write access to destination - if ( ! await svc_acl.check(actor, parent, 'write') ) { - throw await svc_acl.get_safe_acl_error(actor, source, 'write'); - } - } - - const capabilities = source.provider.get_capabilities(); - if ( capabilities.has(fsCapabilities.COPY_TREE) ) { - const result_node = await source.provider.copy_tree({ - context, - source, - parent, - target_name, - }); - return result_node; - } else { - throw new Error('only copy_tree is current supported by ll_copy'); - } - } -} - -module.exports = { - LLCopy, -}; diff --git a/src/backend/src/deprecated/filesystem/ll_operations/ll_listusers.js b/src/backend/src/deprecated/filesystem/ll_operations/ll_listusers.js deleted file mode 100644 index 6a8cdc21f..000000000 --- a/src/backend/src/deprecated/filesystem/ll_operations/ll_listusers.js +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { RootNodeSelector, NodeChildSelector } = require('../node/selectors'); -const { LLFilesystemOperation } = require('./definitions'); - -class LLListUsers extends LLFilesystemOperation { - static description = ` - List user directories which are relevant to the - current actor. - `; - - async _run () { - const { context } = this; - const svc = context.get('services'); - const svc_permission = svc.get('permission'); - const svc_fs = svc.get('filesystem'); - - const user = this.values.user; - const issuers = await svc_permission.list_user_permission_issuers(user); - - const nodes = []; - - nodes.push(await svc_fs.node(new NodeChildSelector( - new RootNodeSelector(), - user.username, - ))); - - for ( const issuer of issuers ) { - const node = await svc_fs.node(new NodeChildSelector( - new RootNodeSelector(), - issuer.username, - )); - nodes.push(node); - } - - return nodes; - } -} - -module.exports = { - LLListUsers, -}; diff --git a/src/backend/src/deprecated/filesystem/ll_operations/ll_mkdir.js b/src/backend/src/deprecated/filesystem/ll_operations/ll_mkdir.js deleted file mode 100644 index 6eef25e78..000000000 --- a/src/backend/src/deprecated/filesystem/ll_operations/ll_mkdir.js +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { MODE_WRITE } = require('../../../services/fs/FSLockService'); -const { LLFilesystemOperation } = require('./definitions'); - -class LLMkdir extends LLFilesystemOperation { - static CONCERN = 'filesystem'; - static MODULES = { - _path: require('path'), - uuidv4: require('uuid').v4, - }; - - async _run () { - const { parent, name, immutable } = this.values; - - const actor = this.values.actor ?? this.context.get('actor'); - - const services = this.context.get('services'); - - const svc_fsLock = services.get('fslock'); - const svc_acl = services.get('acl'); - - // -- Please fix this linter rule - const lock_handle = await svc_fsLock.lock_child( - await parent.get('path'), - name, - MODE_WRITE, - ); - - try { - if ( ! await svc_acl.check(actor, parent, 'write') ) { - throw await svc_acl.get_safe_acl_error(actor, parent, 'write'); - } - - return await parent.provider.mkdir({ - actor, - context: this.context, - parent, - name, - immutable, - }); - } finally { - lock_handle.unlock(); - } - } -} - -module.exports = { - LLMkdir, -}; diff --git a/src/backend/src/deprecated/filesystem/ll_operations/ll_move.js b/src/backend/src/deprecated/filesystem/ll_operations/ll_move.js deleted file mode 100644 index d1812e958..000000000 --- a/src/backend/src/deprecated/filesystem/ll_operations/ll_move.js +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { LLFilesystemOperation } = require('./definitions'); - -class LLMove extends LLFilesystemOperation { - static MODULES = { - _path: require('path'), - }; - - async _run () { - const { context } = this; - const { source, parent, actor, target_name, metadata } = this.values; - - // Access Control - { - const svc_acl = context.get('services').get('acl'); - this.checkpoint('move :: access control'); - - // Check write access to source - if ( ! await svc_acl.check(actor, source, 'write') ) { - throw await svc_acl.get_safe_acl_error(actor, source, 'write'); - } - - // Check write access to destination - if ( ! await svc_acl.check(actor, parent, 'write') ) { - throw await svc_acl.get_safe_acl_error(actor, parent, 'write'); - } - } - - await source.provider.move({ - context: this.context, - node: source, - new_parent: parent, - new_name: target_name, - metadata, - }); - return source; - } -} - -module.exports = { - LLMove, -}; diff --git a/src/backend/src/deprecated/filesystem/ll_operations/ll_read.js b/src/backend/src/deprecated/filesystem/ll_operations/ll_read.js deleted file mode 100644 index bd8aabf11..000000000 --- a/src/backend/src/deprecated/filesystem/ll_operations/ll_read.js +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import APIError from '../../../api/APIError.js'; -import { get_user } from '../../../helpers.js'; -import APIErrorService from '../../../modules/web/APIErrorService.js'; -import { Actor, UserActorType } from '../../../services/auth/Actor.js'; -import { BaseOperation } from '../../../services/OperationTraceService.js'; -import { Context } from '../../../util/context.js'; -import FSNodeContext from '../FSNodeContext.js'; - -const checkACLForRead = async (aclService, actor, fsNode, skip = false) => { - if ( skip ) { - return; - } - if ( ! await aclService.check(actor, fsNode, 'read') ) { - throw await aclService.get_safe_acl_error(actor, fsNode, 'read'); - } -}; -const typeCheckForRead = async (fsNode) => { - if ( await fsNode.get('type') === FSNodeContext.TYPE_DIRECTORY ) { - throw APIError.create('cannot_read_a_directory'); - } -}; - -export class LLRead extends BaseOperation { - static CONCERN = 'filesystem'; - async _run ({ fsNode, no_acl, actor, offset, length, range, version_id } = {}) { - // extract services from context - const aclService = Context.get('services').get('acl'); - const db = Context.get('services') - .get('database'); - - // validate input - if ( ! await fsNode.exists() ) { - throw APIErrorService.create('subject_does_not_exist'); - } - // validate initial node - await checkACLForRead(aclService, actor, fsNode, no_acl); - await typeCheckForRead(fsNode); - - let type = await fsNode.get('type'); - let traversedCount = 0; - while ( type === FSNodeContext.TYPE_SYMLINK ) { - fsNode = await fsNode.getTarget(); - type = await fsNode.get('type'); - traversedCount++; - } - - // validate symlink leaf node - if ( traversedCount > 0 ) { - await checkACLForRead(aclService, actor, fsNode, no_acl); - await typeCheckForRead(fsNode); - } - - // calculate range inputs - const has_range = ( - offset !== undefined && - offset !== 0 - ) || ( - length !== undefined && - length != await fsNode.get('size') - ) || range !== undefined; - - // timestamp access - db.write( - 'UPDATE `fsentries` SET `accessed` = ? WHERE `id` = ?', - [Date.now() / 1000, await fsNode.get('mysql-id')], - ); - - const ownerId = await fsNode.get('user_id'); - const chargedActor = actor ? actor : new Actor({ - type: new UserActorType({ - user: await get_user({ id: ownerId }), - }), - }); - - //define metering service - - /** @type {import("../../services/MeteringService/MeteringService").MeteringService} */ - const meteringService = Context.get('services').get('meteringService').meteringService; - const svc_mountpoint = Context.get('services').get('mountpoint'); - const provider = await svc_mountpoint.get_provider(fsNode.selector); - // const storage = svc_mountpoint.get_storage(provider.constructor.name); - - // Empty object here is in the case of local fiesystem, - // where s3:location will return null. - // TODO: storage interface shouldn't have S3-specific properties. - // const location = await fsNode.get('s3:location') ?? {}; - // const stream = (await storage.create_read_stream(await fsNode.get('uid'), { - // // TODO: fs:decouple-s3 - // bucket: location.bucket, - // bucket_region: location.bucket_region, - // version_id, - // key: location.key, - // memory_file: fsNode.entry, - // ...(range ? { range } : (has_range ? { - // range: `bytes=${offset}-${offset + length - 1}`, - // } : {})), - // })); - - const stream = await provider.read({ - context: this.context, - node: fsNode, - version_id: version_id, - ...(range ? { range } : (has_range ? { - range: `bytes=${offset}-${offset + length - 1}`, - } : {})), - }); - - // Meter ingress - const size = await (async () => { - if ( range ) { - const match = range.match(/bytes=(\d+)-(\d+)/); - if ( match ) { - const start = parseInt(match[1], 10); - const end = parseInt(match[2], 10); - return end - start + 1; - } - } - if ( has_range ) { - return length; - } - return await fsNode.get('size'); - })(); - meteringService.incrementUsage(chargedActor, 'filesystem:egress:bytes', size); - - return stream; - } -} \ No newline at end of file diff --git a/src/backend/src/deprecated/filesystem/ll_operations/ll_readdir.js b/src/backend/src/deprecated/filesystem/ll_operations/ll_readdir.js deleted file mode 100644 index 7743db397..000000000 --- a/src/backend/src/deprecated/filesystem/ll_operations/ll_readdir.js +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const fsCapabilities = require('../definitions/capabilities'); -const { ECMAP } = require('../ECMAP'); -const { TYPE_SYMLINK } = require('../FSNodeContext'); -const { RootNodeSelector } = require('../node/selectors'); -const { NodeUIDSelector, NodeChildSelector } = require('../node/selectors'); -const { LLFilesystemOperation } = require('./definitions'); - -class LLReadDir extends LLFilesystemOperation { - static CONCERN = 'filesystem'; - async _run () { - return ECMAP.arun(async () => { - return await this.__run(); - }); - } - async __run () { - const { context } = this; - const { subject: subject_let, actor, no_acl } = this.values; - let subject = subject_let; - - const svc_acl = context.get('services').get('acl'); - if ( ! no_acl ) { - if ( ! await svc_acl.check(actor, subject, 'list') ) { - throw await svc_acl.get_safe_acl_error(actor, subject, 'list'); - } - } - - // TODO: DRY ACL check here - const subject_type = await subject.get('type'); - if ( subject_type === TYPE_SYMLINK ) { - const target = await subject.getTarget(); - if ( ! no_acl ) { - if ( ! await svc_acl.check(actor, target, 'list') ) { - throw await svc_acl.get_safe_acl_error(actor, target, 'list'); - } - } - subject = target; - } - - const svc = context.get('services'); - const svc_fs = svc.get('filesystem'); - - if ( subject.isRoot ) { - if ( ! actor.type.user ) return []; - return [ - await svc_fs.node(new NodeChildSelector( - new RootNodeSelector(), - actor.type.user.username, - )), - ]; - } - - const capabilities = subject.provider.get_capabilities(); - - // Optimization for filesystems that implement it - { - const child_nodes = await this.#try_readdirstatUUID(); - if ( child_nodes !== null ) return child_nodes; - } - - if ( capabilities.has(fsCapabilities.READDIR_UUID_MODE) ) { - this.checkpoint('readdir uuid mode'); - const child_uuids = await subject.provider.readdir({ - context, - node: subject, - }); - this.checkpoint('after get direct descendants'); - const children = await Promise.all(child_uuids.map(async uuid => { - return await svc_fs.node(new NodeUIDSelector(uuid)); - })); - this.checkpoint('after get children'); - return children; - } - - // Conventional Mode - const child_entries = subject.provider.readdir({ - context, - node: subject, - }); - - return await Promise.all(child_entries.map(async entry => { - return await svc_fs.node(new NodeChildSelector(subject, entry.name)); - })); - } - async #try_readdirstatUUID () { - const subject = this.values.subject; - const capabilities = subject.provider.get_capabilities(); - const uuid_selector = subject.get_selector_of_type(NodeUIDSelector); - - // Skip this optimization if there is no UUID - if ( ! uuid_selector ) { - return null; - } - - // Skip this optimization if the filesystem doesn't implement - // the "readdirstat_uuid" macro operation. - if ( ! capabilities.has(fsCapabilities.READDIRSTAT_UUID) ) { - return null; - } - - const uuid = uuid_selector.value; - return await subject.provider.readdirstat_uuid({ - uuid, - options: { thumbnail: true }, - }); - } -} - -module.exports = { - LLReadDir, -}; diff --git a/src/backend/src/deprecated/filesystem/ll_operations/ll_readshares.js b/src/backend/src/deprecated/filesystem/ll_operations/ll_readshares.js deleted file mode 100644 index 4bd155585..000000000 --- a/src/backend/src/deprecated/filesystem/ll_operations/ll_readshares.js +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { get_user } = require('../../../helpers'); -const { MANAGE_PERM_PREFIX } = require('../../../services/auth/permissionConts.mjs'); -const { PermissionUtil } = require('../../../services/auth/permissionUtils.mjs'); -const { DB_WRITE } = require('../../../services/database/consts'); -const { NodeUIDSelector } = require('../node/selectors'); -const { LLFilesystemOperation } = require('./definitions'); -const { LLReadDir } = require('./ll_readdir'); - -class LLReadShares extends LLFilesystemOperation { - static description = ` - Obtain the highest-level entries under this directory - for which the current actor has at least "see" permission. - - This is a breadth-first search. When any node is - found with "see" permission is found, children of that node - will not be traversed. - `; - - async _run () { - const { subject, user, actor } = this.values; - - const svc = this.context.get('services'); - - const svc_fs = svc.get('filesystem'); - const svc_acl = svc.get('acl'); - const db = svc.get('database').get(DB_WRITE, 'll_readshares'); - - const issuer_username = await subject.getUserPart(); - const issuer_user = await get_user({ username: issuer_username }); - const rows = await db.read( - 'SELECT DISTINCT permission FROM `user_to_user_permissions` ' + - 'WHERE `holder_user_id` = ? AND `issuer_user_id` = ? ' + - 'AND (`permission` LIKE ? OR `permission` LIKE ?)', - [user.id, issuer_user.id, 'fs:%', 'manage:fs:%'], - ); - - const fsentry_uuids = []; - for ( const row of rows ) { - const parts = PermissionUtil.split(row.permission.replace(`${MANAGE_PERM_PREFIX}:`, '')); - fsentry_uuids.push(parts[1]); - } - - const results = []; - - const ll_readdir = new LLReadDir(); - let interm_results = await ll_readdir.run({ - subject, - actor, - user, - no_thumbs: true, - no_assocs: true, - no_acl: true, - }); - - // Clone interm_results in case ll_readdir ever implements caching - interm_results = interm_results.slice(); - - for ( const fsentry_uuid of fsentry_uuids ) { - const node = await svc_fs.node(new NodeUIDSelector(fsentry_uuid)); - if ( ! node ) continue; - interm_results.push(node); - } - - for ( const node of interm_results ) { - if ( ! await node.exists() ) continue; - if ( ! await svc_acl.check(actor, node, 'see') ) continue; - results.push(node); - } - - return results; - } -} - -module.exports = { - LLReadShares, -}; diff --git a/src/backend/src/deprecated/filesystem/ll_operations/ll_rmdir.js b/src/backend/src/deprecated/filesystem/ll_operations/ll_rmdir.js deleted file mode 100644 index 3a27e26d3..000000000 --- a/src/backend/src/deprecated/filesystem/ll_operations/ll_rmdir.js +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../../api/APIError'); -const { ParallelTasks, getTracer } = require('../../../util/otelutil'); -const FSNodeContext = require('../FSNodeContext').default; -const { NodeUIDSelector } = require('../node/selectors'); -const { LLFilesystemOperation } = require('./definitions'); -const { LLRmNode } = require('./ll_rmnode'); - -class LLRmDir extends LLFilesystemOperation { - async _run () { - const { - target, - user, - actor, - descendants_only, - recursive, - - // internal use only - not for clients - ignore_not_empty, - - max_tasks = 8, - } = this.values; - - const { context } = this; - - const svc = context.get('services'); - - // Access Control - { - const svc_acl = context.get('services').get('acl'); - this.checkpoint('remove :: access control'); - - // Check write access to target - if ( ! await svc_acl.check(actor, target, 'write') ) { - throw await svc_acl.get_safe_acl_error(actor, target, 'write'); - } - } - - if ( await target.get('immutable') && !descendants_only ) { - throw APIError.create('immutable'); - } - - const fs = svc.get('filesystem'); - - const children = await target.provider.readdir({ - node: target, - }); - - if ( children.length > 0 && !recursive && !ignore_not_empty ) { - throw APIError.create('not_empty'); - } - - const tracer = getTracer(); - const tasks = new ParallelTasks({ tracer, max: max_tasks }); - - for ( const child_uuid of children ) { - tasks.add('fs:rm:rm-child', async () => { - const child_node = await fs.node(new NodeUIDSelector(child_uuid)); - const type = await child_node.get('type'); - if ( type === FSNodeContext.TYPE_DIRECTORY ) { - const ll_rm = new LLRmDir(); - await ll_rm.run({ - target: await fs.node(new NodeUIDSelector(child_uuid)), - user, - recursive: true, - descendants_only: false, - - max_tasks: (v => v > 1 ? v : 1)(Math.floor(max_tasks / 2)), - }); - } else { - const ll_rm = new LLRmNode(); - await ll_rm.run({ - target: await fs.node(new NodeUIDSelector(child_uuid)), - user, - }); - } - }); - } - - await tasks.awaitAll(); - - if ( ! descendants_only ) { - await target.provider.rmdir({ - context, - node: target, - options: { - ignore_not_empty: true, - }, - }); - } - } -} - -module.exports = { - LLRmDir, -}; diff --git a/src/backend/src/deprecated/filesystem/ll_operations/ll_rmnode.js b/src/backend/src/deprecated/filesystem/ll_operations/ll_rmnode.js deleted file mode 100644 index c2c6417c2..000000000 --- a/src/backend/src/deprecated/filesystem/ll_operations/ll_rmnode.js +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { LLFilesystemOperation } = require('./definitions'); - -class LLRmNode extends LLFilesystemOperation { - async _run () { - const { target, actor } = this.values; - - const { context } = this; - - const svc_event = context.get('services').get('event'); - - // Access Control - { - const svc_acl = context.get('services').get('acl'); - this.checkpoint('remove :: access control'); - - // Check write access to target - if ( ! await svc_acl.check(actor, target, 'write') ) { - throw await svc_acl.get_safe_acl_error(actor, target, 'write'); - } - } - await svc_event.emit('fs.remove.node', this.values); - await target.provider.unlink({ context, node: target }); - } -} - -module.exports = { - LLRmNode, -}; diff --git a/src/backend/src/deprecated/filesystem/ll_operations/ll_write.js b/src/backend/src/deprecated/filesystem/ll_operations/ll_write.js deleted file mode 100644 index 1f80f470b..000000000 --- a/src/backend/src/deprecated/filesystem/ll_operations/ll_write.js +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { LLFilesystemOperation } = require('./definitions.js'); -const APIError = require('../../../api/APIError.js'); - -/** - * The "overwrite" write operation. - * - * This operation is used to write a file to an existing path. - * - * @extends LLFilesystemOperation - */ -class LLOWrite extends LLFilesystemOperation { - /** - * Executes the overwrite operation by writing to an existing file node. - * @returns {Promise} Result of the write operation - * @throws {APIError} When the target node does not exist - */ - async _run () { - const node = this.values.node; - - // Embed fields into this.context - this.context.set('immutable', this.values.immutable); - this.context.set('tmp', this.values.tmp); - this.context.set('fsentry_tmp', this.values.fsentry_tmp); - this.context.set('message', this.values.message); - this.context.set('actor', this.values.actor); - this.context.set('app_id', this.values.app_id); - - // TODO: Add symlink write - if ( ! await node.exists() ) { - // TODO: different class of errors for low-level operations - throw APIError.create('subject_does_not_exist'); - } - - return await node.provider.write_overwrite({ - context: this.context, - node: node, - file: this.values.file, - }); - } -} - -/** - * The "non-overwrite" write operation. - * - * This operation is used to write a file to a non-existent path. - * - * @extends LLFilesystemOperation - */ -class LLCWrite extends LLFilesystemOperation { - static MODULES = { - _path: require('path'), - uuidv4: require('uuid').v4, - config: require('../../../config.js'), - }; - - /** - * Executes the create operation by writing a new file to the parent directory. - * @returns {Promise} Result of the write operation - * @throws {APIError} When the parent directory does not exist - */ - async _run () { - const parent = this.values.parent; - - // Embed fields into this.context - this.context.set('immutable', this.context.get('immutable') ?? this.values.immutable); - this.context.set('tmp', this.context.get('tmp') ?? this.values.tmp); - this.context.set('fsentry_tmp', this.context.get('fsentry_tmp') ?? this.values.fsentry_tmp); - this.context.set('message', this.context.get('message') ?? this.values.message); - this.context.set('actor', this.context.get('actor') ?? this.values.actor); - this.context.set('app_id', this.context.get('app_id') ?? this.values.app_id); - - if ( ! await parent.exists() ) { - throw APIError.create('subject_does_not_exist'); - } - - return await parent.provider.write_new({ - context: this.context, - parent, - name: this.values.name, - file: this.values.file, - }); - } -} - -module.exports = { - LLCWrite, - LLOWrite, -}; diff --git a/src/backend/src/deprecated/filesystem/node/selectors.js b/src/backend/src/deprecated/filesystem/node/selectors.js deleted file mode 100644 index ce8ad529f..000000000 --- a/src/backend/src/deprecated/filesystem/node/selectors.js +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const _path = require('path'); -const { PuterPath } = require('../lib/PuterPath'); - -/** - * The base class doesn't add any functionality, but it's useful for - * `instanceof` checks. - */ -class NodeSelector { - constructor () { - if ( this.constructor === NodeSelector ) { - throw new Error('cannot instantiate NodeSelector directly; ' + - 'that would be like using this: https://devmeme.puter.site/plug.webp'); - } - } -} - -class NodePathSelector extends NodeSelector { - constructor (path) { - super(); - this.value = path; - } - - setPropertiesKnownBySelector (node) { - node.path = this.value; - node.name = _path.basename(this.value); - } - - describe () { - return this.value; - } -} - -class NodeUIDSelector extends NodeSelector { - constructor (uid) { - super(); - this.value = uid; - } - - setPropertiesKnownBySelector (node) { - node.uid = this.value; - } - - // Note: the selector could've been added by FSNodeContext - // during fetch, but this was more efficient because the - // object is created lazily, and it's somtimes not needed. - static implyFromFetchedData (node) { - if ( node.uid ) { - return new NodeUIDSelector(node.uid); - } - return null; - } - - describe () { - return `[uid:${this.value}]`; - } -} - -class NodeInternalIDSelector extends NodeSelector { - constructor (service, id, debugInfo) { - super(); - this.service = service; - this.id = id; - this.debugInfo = debugInfo; - } - - setPropertiesKnownBySelector (node) { - if ( this.service === 'mysql' ) { - node.mysql_id = this.id; - } - } - - describe (showDebug) { - if ( showDebug ) { - return `[db:${this.id}] (${ - JSON.stringify(this.debugInfo, null, 2) - })`; - } - return `[db:${this.id}]`; - } -} - -class NodeChildSelector extends NodeSelector { - constructor (parent, name) { - super(); - this.parent = parent; - this.name = name; - } - - setPropertiesKnownBySelector (node) { - node.name = this.name; - - try_infer_attributes(this); - if ( this.path ) { - node.path = this.path; - } - } - - describe () { - return `${this.parent.describe() }/${ this.name}`; - } -} - -class RootNodeSelector extends NodeSelector { - static entry = { - is_dir: true, - is_root: true, - uuid: PuterPath.NULL_UUID, - name: '/', - }; - setPropertiesKnownBySelector (node) { - node.path = '/'; - node.root = true; - node.uid = PuterPath.NULL_UUID; - } - constructor () { - super(); - this.entry = this.constructor.entry; - } - - describe () { - return '[root]'; - } -} - -class NodeRawEntrySelector extends NodeSelector { - constructor (entry, details_about_fetch = {}) { - super(); - - // The `details_about_fetch` object lets us simulate non-entry state - // that occurs after a node has been fetched - this.details_about_fetch = details_about_fetch; - - // Fix entries from get_descendants - if ( !entry.uuid && entry.uid ) { - entry.uuid = entry.uid; - if ( entry._id ) { - entry.id = entry._id; - delete entry._id; - } - } - - this.entry = entry; - } - - setPropertiesKnownBySelector (node) { - if ( this.details_about_fetch.found_thumbnail ) { - node.found_thumbnail = true; - } - node.found = true; - node.entry = this.entry; - node.uid = this.entry.uid ?? this.entry.uuid; - node.name = this.entry.name; - if ( this.entry.path ) node.path = this.entry.path; - - if ( this.entry.subdomains ) { - node.subdomains = this.entry.subdomains; - } - } - - describe () { - return '[raw entry]'; - } -} - -/** - * Try to infer following attributes for a selector: - * - path - * - uid - * - * @param {NodePathSelector | NodeUIDSelector | NodeChildSelector | RootNodeSelector | NodeRawEntrySelector} selector - */ -function try_infer_attributes (selector) { - if ( selector instanceof NodePathSelector ) { - selector.path = selector.value; - } else if ( selector instanceof NodeUIDSelector ) { - selector.uid = selector.value; - } else if ( selector instanceof NodeChildSelector ) { - try_infer_attributes(selector.parent); - if ( selector.parent.path ) { - selector.path = _path.join(selector.parent.path, selector.name); - } - } else if ( selector instanceof RootNodeSelector ) { - selector.path = '/'; - } else { - // give up - } -} - -const relativeSelector = (parent, path) => { - if ( path === '.' ) return parent; - if ( path.startsWith('..') ) { - throw new Error('currently unsupported'); - } - - let selector = parent; - - const parts = path.split('/').filter(Boolean); - for ( const part of parts ) { - selector = new NodeChildSelector(selector, part); - } - - return selector; -}; - -module.exports = { - NodeSelector, - NodePathSelector, - NodeUIDSelector, - NodeInternalIDSelector, - NodeChildSelector, - RootNodeSelector, - NodeRawEntrySelector, - relativeSelector, - try_infer_attributes, -}; diff --git a/src/backend/src/deprecated/filesystem/node/states.js b/src/backend/src/deprecated/filesystem/node/states.js deleted file mode 100644 index 10822c5e8..000000000 --- a/src/backend/src/deprecated/filesystem/node/states.js +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -class NodeFoundState { -} - -class NodeDoesNotExistState { -} - -class NodeInitialState { -} diff --git a/src/backend/src/deprecated/filesystem/storage/UploadProgressTracker.js b/src/backend/src/deprecated/filesystem/storage/UploadProgressTracker.js deleted file mode 100644 index 8ecbc0a44..000000000 --- a/src/backend/src/deprecated/filesystem/storage/UploadProgressTracker.js +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -class UploadProgressTracker { - constructor () { - this.progress_ = 0; - this.total_ = 0; - this.done_ = false; - - this.listeners_ = []; - } - - set_total (v) { - this.total_ = v; - } - - set (value) { - if ( value < this.progress_ ) { - // TODO: provide a logger for a warning - return; - } - const delta = value - this.progress_; - this.add(delta); - } - - add (amount) { - if ( this.done_ ) { - return; // TODO: warn - } - - this.progress_ += amount; - - for ( const lis of this.listeners_ ) { - lis(amount); - } - - this.check_if_done_(); - } - - sub (callback) { - if ( this.done_ ) { - return; - } - - const listeners = this.listeners_; - - listeners.push(callback); - - const det = { - detach: () => { - const idx = listeners.indexOf(callback); - if ( idx !== -1 ) { - listeners.splice(idx, 1); - } - }, - }; - - return det; - } - - check_if_done_ () { - if ( this.progress_ === this.total_ ) { - this.done_ = true; - // clear listeners so they get GC'd - this.listeners_ = []; - } - } -} - -module.exports = { - UploadProgressTracker, -}; \ No newline at end of file diff --git a/src/backend/src/deprecated/filesystem/strategies/PuterS3StorageStrategy.js b/src/backend/src/deprecated/filesystem/strategies/PuterS3StorageStrategy.js deleted file mode 100644 index 98a1927fc..000000000 --- a/src/backend/src/deprecated/filesystem/strategies/PuterS3StorageStrategy.js +++ /dev/null @@ -1,269 +0,0 @@ -import { BaseOperation } from '../../../services/OperationTraceService.js'; -import { Context } from '../../../util/context.js'; -import { simple_retry } from '../../../util/retryutil.js'; - -class PuterS3UploadStrategy extends BaseOperation { - constructor (parent) { - super(); - this.parent = parent; - this.s3_resp = null; - this.uid = null; - } - - async _run () { - const { uid, file, storage_meta, storage_api } = this.values; - this.uid = uid; - - // Deconstruct expected parameters - // TODO: parametize these to manage errors in backend usage - const { - bucket_region, - bucket, - } = storage_meta; - const { - progress_tracker, - } = storage_api; - - // Note: while it may seem redundant to deconstruct - // these arguments and then pass them to methods, - // this is done to assert that the concern of - // parameter validation is meant to happen - // within this upload() method. - - // Delegate to appropriate upload method - if ( file.buffer ) { - return await this._upload_buffer( - bucket_region, - bucket, - uid, - file, - progress_tracker, - ); - } - - return await this._upload_stream( - bucket_region, - bucket, - uid, - file, - progress_tracker, - ); - } - post_insert ({ db, user, node, uid, message, ts }) { - (async () => { - // This case happens in dev environments if a bucket doesn't - // have versioning enabled. - if ( ! this.s3_resp?.VersionId ) return; - - db.write( - 'INSERT INTO `fsentry_versions` (`user_id`, `fsentry_id`, `fsentry_uuid`, `version_id`, `message`, `ts_epoch`) VALUES (?, ?, ?, ?, ?, ?)', - [ - user.id, - node.mysql_id, - uid, - this.s3_resp.VersionId, - message ?? null, - ts, - ], - ); - })(); - } - - async _upload_buffer ( - bucket_region, - bucket, - uid, - file, - progress_tracker, - ) { - const svc_puterS3 = this.parent.svc_puterS3; - const [s3_error, s3_eventual_success, s3_resp] = await simple_retry(async () => { - const ret = await svc_puterS3.upload_buffer({ - bucket_region, - bucket, - key: uid, - buffer: file.buffer, - // TODO: progress tracker for buffers - }); - - progress_tracker.set_total(file.size); - progress_tracker.set(file.size); - - return ret; - }, 3, 200); - - if ( ! s3_eventual_success ) { - throw s3_error; - } - - this.s3_resp = s3_resp; - } - - async _upload_stream ( - bucket_region, - bucket, - uid, - file, - progress_tracker, - ) { - console.log('DOING STREAM UPLOAD'); - const svc_puterS3 = this.parent.svc_puterS3; - this.checkpoint('before upload stream'); - const [s3_error, s3_eventual_success, s3_resp] = await simple_retry(async () => { - try { - // if ( file.size < 5 * 1024 * 1024 ) { - // return await svc_puterS3.put_stream({ - // size: file.size, - // bucket_region, - // bucket, - // key: uid, - // stream: file.stream, - // on_progress: evt => { - // progress_tracker.set_total(file.size); - // progress_tracker.set(evt.uploaded); - // }, - // }); - // } - - return await svc_puterS3.upload_stream({ - bucket_region, - bucket, - key: uid, - stream: file.stream, - on_progress: evt => { - progress_tracker.set_total(file.size); - progress_tracker.set(evt.uploaded); - }, - }); - } catch ( e ) { - console.log('ERRORRRRRR', e); - } - }, 3, 200); - this.checkpoint('after upload stream'); - - if ( ! s3_eventual_success ) { - throw s3_error; - } - - this.s3_resp = s3_resp; - } -} - -class PuterS3CopyStrategy extends BaseOperation { - constructor (parent) { - super(); - this.parent = parent; - this.s3_resp = null; - } - - async _run () { - const { src_node, dst_storage, storage_api } = this.values; - - const { - progress_tracker, - } = storage_api; - - const src_storage = await src_node.get('s3:location'); - - const svc_puterS3 = this.parent.svc_puterS3; - - const size = await src_node.get('size'); - if ( size < 4 * 1000 ** 3 - 100 ) { - const ret = await svc_puterS3.copy_simple({ - src_key: src_storage.key, - src_bucket: src_storage.bucket, - - dst_key: dst_storage.key, - dst_bucket_region: dst_storage.bucket_region, - dst_bucket: dst_storage.bucket, - }); - progress_tracker.set_total(size); - progress_tracker.set(size); - return ret; - } - - return await svc_puterS3.copy_multipart({ - src_key: src_storage.key, - src_bucket: src_storage.bucket, - - dst_key: dst_storage.key, - dst_bucket_region: dst_storage.bucket_region, - dst_bucket: dst_storage.bucket, - - size, - - on_progress: evt => { - const x = Context.get(); - const log = x.get('services').get('log-service').create('PuterS3CopyStrategy'); - log.info('progress', { evt }); - progress_tracker.set_total(size); - progress_tracker.set(evt.uploaded); - }, - }); - } - - post_insert ({ db, user, node, uid, message, ts }) { - (async () => { - db.write( - 'INSERT INTO `fsentry_versions` (`user_id`, `fsentry_id`, `fsentry_uuid`, `version_id`, `message`, `ts_epoch`) VALUES (?, ?, ?, ?, ?, ?)', - [ - user.id, - node.mysql_id, - uid, - this.s3_resp.VersionId, - message ?? null, - ts, - ], - ); - })(); - } -} - -class PuterS3DeleteStrategy extends BaseOperation { - constructor (parent) { - super(); - this.parent = parent; - } - - async _run () { - const { node } = this.values; - - const node_storage = await node.get('s3:location'); - - const svc_puterS3 = this.parent.svc_puterS3; - - return await svc_puterS3.delete({ - bucket_region: node_storage.bucket_region, - bucket: node_storage.bucket, - key: node_storage.key, - }); - } -} - -export class PuterS3StorageStrategy { - constructor ({ services }) { - this.svc_puterS3 = services.get('puter-s3'); - } - - create_upload () { - const state_upload = new PuterS3UploadStrategy(this); - return state_upload; - } - - create_copy () { - const state_copy = new PuterS3CopyStrategy(this); - return state_copy; - } - - create_delete () { - const state_delete = new PuterS3DeleteStrategy(this); - return state_delete; - } - - async create_read_stream (uid, storage_meta) { - return await this.svc_puterS3.create_read_stream({ - ...storage_meta, - key: uid, - }); - } -} diff --git a/src/backend/src/deprecated/filesystem/validation.bench.js b/src/backend/src/deprecated/filesystem/validation.bench.js deleted file mode 100644 index 4790cca17..000000000 --- a/src/backend/src/deprecated/filesystem/validation.bench.js +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { bench, describe } from 'vitest'; -const { is_valid_path, is_valid_node_name } = require('./validation'); - -// Test data -const shortPath = '/home/user/file.txt'; -const mediumPath = '/home/user/documents/projects/puter/src/backend/file.js'; -const longPath = '/a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z/file.txt'; -const deeplyNestedPath = `${Array(50).fill('directory').join('/') }/file.txt`; - -const simpleFilename = 'document.pdf'; -const filenameWithSpaces = 'my document file.pdf'; -const filenameWithNumbers = 'report_2024_final_v2.xlsx'; -const maxLengthFilename = 'a'.repeat(255); - -// Invalid paths for testing rejection speed -const pathWithNull = '/home/user/\x00file.txt'; -const pathWithRTL = '/home/user/\u202Efile.txt'; -const pathWithLTR = '/home/user/\u200Efile.txt'; - -describe('is_valid_path - Valid paths', () => { - bench('short path (/home/user/file.txt)', () => { - is_valid_path(shortPath); - }); - - bench('medium path (~50 chars)', () => { - is_valid_path(mediumPath); - }); - - bench('long path (26 components)', () => { - is_valid_path(longPath); - }); - - bench('deeply nested path (50 components)', () => { - is_valid_path(`/${ deeplyNestedPath}`); - }); - - bench('relative path starting with dot', () => { - is_valid_path('./relative/path/to/file.txt'); - }); -}); - -describe('is_valid_path - With options', () => { - bench('with no_relative_components option', () => { - is_valid_path(mediumPath, { no_relative_components: true }); - }); - - bench('with allow_path_fragment option', () => { - is_valid_path('partial/path/fragment', { allow_path_fragment: true }); - }); - - bench('with both options', () => { - is_valid_path(shortPath, { no_relative_components: true, allow_path_fragment: true }); - }); -}); - -describe('is_valid_path - Invalid paths (rejection speed)', () => { - bench('path with null character', () => { - is_valid_path(pathWithNull); - }); - - bench('path with RTL override', () => { - is_valid_path(pathWithRTL); - }); - - bench('path with LTR mark', () => { - is_valid_path(pathWithLTR); - }); - - bench('empty string', () => { - is_valid_path(''); - }); - - bench('non-string input (number)', () => { - is_valid_path(12345); - }); - - bench('path not starting with / or .', () => { - is_valid_path('invalid/path/start'); - }); -}); - -describe('is_valid_node_name - Valid names', () => { - bench('simple filename', () => { - is_valid_node_name(simpleFilename); - }); - - bench('filename with spaces', () => { - is_valid_node_name(filenameWithSpaces); - }); - - bench('filename with numbers and underscores', () => { - is_valid_node_name(filenameWithNumbers); - }); - - bench('filename at max length (255 chars)', () => { - is_valid_node_name(maxLengthFilename); - }); - - bench('filename with multiple extensions', () => { - is_valid_node_name('archive.tar.gz'); - }); -}); - -describe('is_valid_node_name - Invalid names (rejection speed)', () => { - bench('name with forward slash', () => { - is_valid_node_name('invalid/name'); - }); - - bench('name with null character', () => { - is_valid_node_name('invalid\x00name'); - }); - - bench('single dot (.)', () => { - is_valid_node_name('.'); - }); - - bench('double dot (..)', () => { - is_valid_node_name('..'); - }); - - bench('only dots (...)', () => { - is_valid_node_name('...'); - }); - - bench('name exceeding max length', () => { - is_valid_node_name('a'.repeat(300)); - }); - - bench('non-string input', () => { - is_valid_node_name(null); - }); -}); - -describe('is_valid_path - Batch validation simulation', () => { - const paths = [ - '/home/user/file1.txt', - '/home/user/file2.txt', - '/home/user/documents/report.pdf', - '/var/log/system.log', - '/etc/config.json', - ]; - - bench('validate 5 paths sequentially', () => { - for ( const path of paths ) { - is_valid_path(path); - } - }); - - bench('validate 100 paths', () => { - for ( let i = 0; i < 100; i++ ) { - is_valid_path(paths[i % paths.length]); - } - }); -}); diff --git a/src/backend/src/deprecated/filesystem/validation.js b/src/backend/src/deprecated/filesystem/validation.js deleted file mode 100644 index 45f7e2b02..000000000 --- a/src/backend/src/deprecated/filesystem/validation.js +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -/* ~~~ Filesystem validation ~~~ - -This module contains functions that validate filesystem operations. - -*/ - -const config = require('../../config'); - -/* eslint-disable no-control-regex */ - -const path_excludes = () => /[\x00-\x1F]/g; -const node_excludes = () => /[/\x00-\x1F]/g; - -// this characters are not allowed in path names because -// they might be used to trick the user into thinking -// a filename is different from what it actually is. -const safety_excludes = [ - /[\u202A-\u202E]/, // RTL and LTR override - /[\u200E-\u200F]/, // RTL and LTR mark - /[\u2066-\u2069]/, // RTL and LTR isolate - /[\u2028-\u2029]/, // line and paragraph separator - /[\uFF01-\uFF5E]/, // fullwidth ASCII - /[\u2060]/, // word joiner - /[\uFEFF]/, // zero width no-break space - /[\uFFFE-\uFFFF]/, // non-characters -]; - -const is_valid_node_name = function is_valid_node_name (name) { - if ( typeof name !== 'string' ) return false; - if ( node_excludes().test(name) ) return false; - for ( const exclude of safety_excludes ) { - if ( exclude.test(name) ) return false; - } - if ( name.length > config.max_fsentry_name_length ) return false; - // Names are allowed to contain dots, but cannot - // contain only dots. (this covers '.' and '..') - const name_without_dots = name.replace(/\./g, ''); - if ( name_without_dots.length < 1 ) return false; - - return true; -}; - -const is_valid_path = function is_valid_path (path, { - no_relative_components, - allow_path_fragment, -} = {}) { - if ( typeof path !== 'string' ) return false; - if ( path.length < 1 ) false; - if ( path_excludes().test(path) ) return false; - for ( const exclude of safety_excludes ) { - if ( exclude.test(path) ) return false; - } - - if ( ! allow_path_fragment ) { - if ( path[0] !== '/' && path[0] !== '.' ) { - return false; - } - } - - if ( no_relative_components ) { - const components = path.split('/'); - for ( const component of components ) { - if ( component === '' ) continue; - const name_without_dots = component.replace(/\./g, ''); - if ( name_without_dots.length < 1 ) return false; - } - } - - return true; -}; - -module.exports = { - is_valid_node_name, - is_valid_path, -}; diff --git a/src/backend/src/env b/src/backend/src/env deleted file mode 100644 index 90012116c..000000000 --- a/src/backend/src/env +++ /dev/null @@ -1 +0,0 @@ -dev \ No newline at end of file diff --git a/src/backend/src/errors/error_help_details.js b/src/backend/src/errors/error_help_details.js deleted file mode 100644 index cae027516..000000000 --- a/src/backend/src/errors/error_help_details.js +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { quot } = require('@heyputer/putility').libs.string; - -const reused = { - runtime_env_references: [ - { - subject: 'ENVIRONMENT.md file', - location: 'root of the repository', - use: 'describes which paths are checked', - }, - { - subject: 'boot logger', - location: 'above this text', - use: 'shows what checks were performed', - }, - { - subject: 'RuntimeEnvironment.js', - location: 'src/boot/ in repository', - use: 'code that performs the checks', - }, - ], -}; - -const programmer_errors = [ - 'Assignment to constant variable.', -]; - -const error_help_details = [ - { - match: ({ message }) => ( - message.startsWith('No suitable path found for') - ), - apply (more) { - more.references = [ - ...reused.runtime_env_references, - ]; - }, - }, - { - match: ({ message }) => ( - message.match(/^No (read|write) permission for/) - ), - apply (more) { - more.solutions = [ - { - title: 'Change permissions with chmod', - }, - { - title: 'Remove the path to use working directory', - }, - { - title: 'Set CONFIG_PATH or RUNTIME_PATH environment variable', - }, - ]; - more.references = [ - ...reused.runtime_env_references, - ]; - }, - }, - { - match: ({ message }) => ( - message.startsWith('No valid config file found in path') - ), - apply (more) { - more.solutions = [ - { - title: 'Create a valid config file', - }, - ]; - }, - }, - { - match: ({ message }) => ( - message === 'config_name is required' - ), - apply (more) { - more.solutions = [ - 'ensure config_name is present in your config file', - 'Seek help on https://discord.gg/PQcx7Teh8u (our Discord server)', - ]; - }, - }, - { - match: ({ message }) => ( - message == 'Assignment to constant variable.' - ), - apply (more) { - more.references = [ - { - subject: 'MDN Reference for this error', - location: 'on the internet', - use: 'describes why this error occurs', - url: 'https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Invalid_const_assignment', - }, - ]; - }, - }, - { - match: ({ message }) => ( - programmer_errors.includes(message) - ), - apply (more) { - more.notes = [ - 'It looks like this might be our fault.', - ]; - more.solutions = [ - { title: 'Check for an issue on https://github.com/HeyPuter/puter/issues' }, - { title: 'If there is no issue, please create one: https://github.com/HeyPuter/puter/issues/new' }, - ]; - }, - }, - { - match: ({ message }) => ( - message.startsWith('Expected double-quoted property') - ), - apply (more) { - more.notes = [ - 'There might be a trailing-comma in your config', - ]; - }, - }, -]; - -/** - * Print error help information to a stream in a human-readable format. - * - * @param {Error} err - The error to print help for. - * @param {*} out - The stream to print to; defaults to process.stdout. - * @returns {undefined} - */ -const print_error_help = (err, out = process.stdout) => { - if ( ! err.more ) { - err.more = {}; - err.more.references = []; - err.more.solutions = []; - for ( const detail of error_help_details ) { - if ( detail.match(err) ) { - detail.apply(err.more); - } - } - } - - let write = out.write.bind(out); - - write('\n'); - - const wrap_msg = s => - `\x1B[31;1m┏━━ [ HELP:\x1B[0m ${quot(s)} \x1B[31;1m]\x1B[0m`; - const wrap_list_title = s => - `\x1B[36;1m${s}:\x1B[0m`; - - write(`${wrap_msg(err.message) }\n`); - - write = (s) => out.write(`\x1B[31;1m┃\x1B[0m ${ s}`); - - const vis = (stok, etok, str) => { - return `\x1B[36;1m${stok}\x1B[0m${str}\x1B[36;1m${etok}\x1B[0m`; - }; - - let lf_sep = false; - - write('Whoops! Looks like something isn\'t working!\n'); - let any_help = false; - - if ( err.more.notes ) { - write('\n'); - lf_sep = true; - any_help = true; - for ( const note of err.more.notes ) { - write(`\x1B[33;1m * ${note}\x1B[0m\n`); - } - } - - if ( err.more.solutions?.length > 0 ) { - if ( lf_sep ) write('\n'); - lf_sep = true; - any_help = true; - write('The suggestions below may help resolve this issue.\n'); - write('\n'); - write(`${wrap_list_title('Possible Solutions') }\n`); - for ( const sol of err.more.solutions ) { - write(` - ${sol.title}\n`); - } - } - - if ( err.more.references?.length > 0 ) { - if ( lf_sep ) write('\n'); - lf_sep = true; - any_help = true; - write('The references below may be related to this issue.\n'); - write('\n'); - write(`${wrap_list_title('References') }\n`); - for ( const ref of err.more.references ) { - write(` - ${vis('[', ']', ref.subject)} ` + - `${vis('(', ')', ref.location)};\n`); - write(` ${ref.use}\n`); - if ( ref.url ) { - write(` ${ref.url}\n`); - } - } - } - - if ( ! any_help ) { - write('No help is available for this error.\n'); - write('Help can be added in src/errors/error_help_details.\n'); - } - - out.write('\x1B[31;1m┗━━ [ END HELP ]\x1B[0m\n'); - out.write('\n'); -}; - -module.exports = { - error_help_details, - print_error_help, -}; diff --git a/src/backend/src/extension/RuntimeModule.js b/src/backend/src/extension/RuntimeModule.js deleted file mode 100644 index b86baa0f6..000000000 --- a/src/backend/src/extension/RuntimeModule.js +++ /dev/null @@ -1,30 +0,0 @@ -const { AdvancedBase } = require('@heyputer/putility'); - -class RuntimeModule extends AdvancedBase { - constructor (options = {}) { - super(); - this.exports_ = undefined; - this.exports_is_set_ = false; - this.remappings = options.remappings ?? {}; - - this.name = options.name ?? undefined; - } - set exports (value) { - this.exports_is_set_ = true; - this.exports_ = value; - } - get exports () { - if ( this.exports_is_set_ === false && this.defer ) { - this.exports = this.defer(); - } - return this.exports_; - } - import (name) { - if ( Object.prototype.hasOwnProperty.call(this.remappings, name) ) { - name = this.remappings[name]; - } - return this.runtimeModuleRegistry.exportsOf(name); - } -} - -module.exports = { RuntimeModule }; diff --git a/src/backend/src/extension/RuntimeModuleRegistry.js b/src/backend/src/extension/RuntimeModuleRegistry.js deleted file mode 100644 index 27968f7f7..000000000 --- a/src/backend/src/extension/RuntimeModuleRegistry.js +++ /dev/null @@ -1,33 +0,0 @@ -const { AdvancedBase } = require('@heyputer/putility'); -const { RuntimeModule } = require('./RuntimeModule'); - -class RuntimeModuleRegistry extends AdvancedBase { - constructor () { - super(); - this.modules_ = {}; - } - - register (extensionModule, options = {}) { - if ( ! (extensionModule instanceof RuntimeModule) ) { - throw new Error(`expected a RuntimeModule, but got: ${ - extensionModule?.constructor?.name ?? typeof extensionModule})`); - } - const uniqueName = options.as ?? extensionModule.name ?? require('uuid').v4(); - if ( this.modules_.hasOwnProperty(uniqueName) ) { - throw new Error(`duplicate runtime module: ${uniqueName}`); - } - this.modules_[uniqueName] = extensionModule; - extensionModule.runtimeModuleRegistry = this; - } - - exportsOf (name) { - if ( ! this.modules_[name] ) { - throw new Error(`could not find runtime module: ${name}`); - } - return this.modules_[name].exports; - } -} - -module.exports = { - RuntimeModuleRegistry, -}; diff --git a/src/backend/src/helpers.js b/src/backend/src/helpers.js deleted file mode 100644 index bc0e21295..000000000 --- a/src/backend/src/helpers.js +++ /dev/null @@ -1,2221 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { sha256 } from 'js-sha256'; -import { LRUCache } from 'lru-cache'; -import micromatch from 'micromatch'; -import { contentType as _contentType } from 'mime-types'; -import { resolve as _resolve, extname } from 'path'; -import { v4 } from 'uuid'; -import APIError from './api/APIError.js'; -import { setRedisCacheValue } from './clients/redis/cacheUpdate.js'; -import { redisClient } from './clients/redis/redisSingleton.js'; -import config from './config.js'; -import { APP_ICONS_SUBDOMAIN } from './consts/app-icons.js'; -import { NodeUIDSelector } from './deprecated/filesystem/node/selectors.js'; -import { AppRedisCacheSpace } from './modules/apps/AppRedisCacheSpace.js'; -import { DB_READ, DB_WRITE } from './services/database/consts.js'; -import { UserRedisCacheSpace } from './services/UserRedisCacheSpace.js'; -import { Context } from './util/context.js'; -import { ManagedError } from './util/errorutil.js'; -import { generate_identifier } from './util/identifier.js'; -import { kv } from './util/kvSingleton.js'; -import { spanify } from './util/otelutil.js'; - -export * from './validation.js'; - -// Use global singleton for services to handle ESM/CJS dual-loading in vitest -const SERVICES_KEY = Symbol.for('puter.helpers.services'); -globalThis[SERVICES_KEY] = globalThis[SERVICES_KEY] ?? { services: null }; -const servicesContainer = globalThis[SERVICES_KEY]; - -export async function tmp_provide_services (ss) { - servicesContainer.services = ss; - await servicesContainer.services.ready; -}; - -// TTL for pending get_app queries (request coalescing) -const PENDING_QUERY_TTL = 10; // seconds -const SUGGESTED_APPS_CACHE_MAX = 10000; -const suggestedAppsCache = new LRUCache({ max: SUGGESTED_APPS_CACHE_MAX }); -const DEFAULT_APP_ICON_SIZE = 256; -const RAW_BASE64_REGEX = /^[A-Za-z0-9+/]+={0,2}$/; - -const safe_json_parse = (value, fallback) => { - if ( value === null || value === undefined ) return fallback; - try { - return JSON.parse(value); - } catch ( error ) { - return fallback; - } -}; - -const redisGetJsonMany = async (keys) => { - if ( !Array.isArray(keys) || keys.length === 0 ) { - return new Map(); - } - - const uniqueKeys = [...new Set(keys)]; - let valuesByIndex = null; - - // MGET over Redis Cluster can fail for cross-slot keys; use pipelined GETs there. - if ( typeof redisClient.nodes === 'function' ) { - const pipeline = redisClient.pipeline(); - for ( const key of uniqueKeys ) { - pipeline.get(key); - } - const results = await pipeline.exec(); - if ( Array.isArray(results) ) { - valuesByIndex = results.map((item) => { - if ( !Array.isArray(item) || item.length < 2 ) return null; - const [error, value] = item; - return error ? null : value; - }); - } - } else if ( typeof redisClient.mget === 'function' ) { - valuesByIndex = await redisClient.mget(...uniqueKeys); - } - - if ( ! Array.isArray(valuesByIndex) ) { - valuesByIndex = await Promise.all(uniqueKeys.map(key => redisClient.get(key))); - } - - const valuesByKey = new Map(); - for ( let i = 0; i < uniqueKeys.length; i++ ) { - valuesByKey.set(uniqueKeys[i], safe_json_parse(valuesByIndex[i], null)); - } - return valuesByKey; -}; - -const normalizeAppUid = (app_uid) => { - if ( ! app_uid ) return null; - const uid_string = String(app_uid); - return uid_string.startsWith('app-') ? uid_string : `app-${uid_string}`; -}; - -const isRawBase64ImageString = value => { - if ( typeof value !== 'string' ) return false; - const trimmed = value.trim(); - if ( !trimmed || trimmed.length < 16 ) return false; - if ( ! RAW_BASE64_REGEX.test(trimmed) ) return false; - if ( trimmed.length % 4 !== 0 ) return false; - - try { - const decoded = Buffer.from(trimmed, 'base64'); - if ( decoded.length === 0 ) return false; - const normalizedInput = trimmed.replace(/=+$/, ''); - const reencoded = decoded.toString('base64').replace(/=+$/, ''); - return normalizedInput === reencoded; - } catch { - return false; - } -}; - -const isBase64AppIcon = (app) => { - if ( !app || typeof app !== 'object' ) return false; - - const flag = app.icon_is_base64; - if ( typeof flag === 'boolean' ) return flag; - if ( typeof flag === 'number' ) return flag !== 0; - if ( typeof flag === 'string' ) { - const lowered = flag.toLowerCase(); - if ( lowered === '1' || lowered === 'true' ) return true; - if ( lowered === '0' || lowered === 'false' ) return false; - } - - const icon = app.icon; - if ( typeof icon !== 'string' ) return false; - const trimmed = icon.trim(); - if ( trimmed.startsWith('data:image/') ) return true; - return isRawBase64ImageString(trimmed); -}; - -export async function is_empty (dir_uuid) { - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem'); - - let rows; - - if ( typeof dir_uuid === 'object' ) { - if ( typeof dir_uuid.path === 'string' && dir_uuid.path !== '' ) { - rows = await db.read( - `SELECT EXISTS(SELECT 1 FROM fsentries WHERE path LIKE ${db.case({ - sqlite: '? || \'%\'', - otherwise: 'CONCAT(?, \'%\')', - })} LIMIT 1) AS not_empty`, - [`${dir_uuid.path }/`], - ); - } else dir_uuid = dir_uuid.uid; - } - - if ( typeof dir_uuid === 'string' ) { - rows = await db.read( - 'SELECT EXISTS(SELECT 1 FROM fsentries WHERE parent_uid = ? LIMIT 1) AS not_empty', - [dir_uuid], - ); - } - - return !rows[0].not_empty; -} - -/** - * Checks to see if temp_users is disabled and return a boolean - * @returns {boolean} - */ -export async function is_temp_users_disabled () { - const svc_feature_flag = await servicesContainer.services.get('feature-flag'); - return await svc_feature_flag.check('temp-users-disabled'); -} - -/** - * Checks to see if user_signup is disabled and return a boolean - * @returns {boolean} - */ -export async function is_user_signup_disabled () { - const svc_feature_flag = await servicesContainer.services.get('feature-flag'); - return await svc_feature_flag.check('user-signup-disabled'); -} - -export const chkperm = spanify('chkperm', async (target_fsentry, requester_user_id, action) => { - // basic cases where false is the default response - if ( ! target_fsentry ) - { - return false; - } - - // pseudo-entry from FSNodeContext - if ( target_fsentry.is_root ) { - return action === 'read'; - } - - // requester is the owner of this entry - if ( target_fsentry.user_id === requester_user_id ) { - return true; - } - // special case: owner of entry has shared at least one entry with requester and requester is asking for the owner's root directory: /[owner_username] - else if ( target_fsentry.parent_uid === null && action !== 'write' ) - { - return true; - } - else - { - return false; - } -}); - -/** - * Checks if the string provided is a valid FileSystem Entry name. - * - * @param {string} name - * @returns - */ -export function validate_fsentry_name (name) { - if ( ! name ) - { - throw { message: 'Name can not be empty.' }; - } - else if ( ! isString(name) ) - { - throw { message: 'Name can only be a string.' }; - } - else if ( name.includes('/') ) - { - throw { message: "Name can not contain the '/' character." }; - } - else if ( name === '.' ) - { - throw { message: "Name can not be the '.' character." }; - } - else if ( name === '..' ) - { - throw { message: "Name can not be the '..' character." }; - } - else if ( name.length > config.max_fsentry_name_length ) - { - throw { message: `Name can not be longer than ${config.max_fsentry_name_length} characters` }; - } - else - { - return true; - } -} - -/** - * Convert a FSEntry ID to UUID - * - * @param {integer} id - `id` of FSEntry - * @returns {Promise} Promise object represents the UUID of the FileSystem Entry - */ -export async function id2uuid (id) { - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem'); - - let fsentry = await db.requireRead('SELECT `uuid`, immutable FROM `fsentries` WHERE `id` = ? LIMIT 1', [id]); - - if ( ! fsentry[0] ) - { - return null; - } - else - { - return fsentry[0].uuid; - } -} - -/** - * Get total data stored by a user - * - * @param {integer} user_id - `user_id` of user - * @returns {Promise} Promise object represents the UUID of the FileSystem Entry - */ -export async function df (user_id) { - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem'); - - const fsentry = await db.read('SELECT SUM(size) AS total FROM `fsentries` WHERE `user_id` = ? LIMIT 1', [user_id]); - if ( !fsentry[0] || !fsentry[0].total ) - { - return 0; - } - else - { - return fsentry[0].total; - } -} - -/** - * Get user by a variety of IDs - * - * Pass `cached: false` to options if a cached user entry would not be appropriate; - * for example: when performing authentication. - * - * @param {object} options - `options` - * @returns {Promise} - */ -export async function get_user (options) { - return await servicesContainer.services.get('get-user').get_user(options); -} - -/** - * Invalidate the cached entries for a user object - * - * @param {User} userID - the user entry to invalidate - */ -export const invalidate_cached_user = async (user) => { - await UserRedisCacheSpace.invalidateUser(user); -}; - -/** - * Invalidate the cached entries for the user specified by an id - * @param {number} id - the id of the user to invalidate - */ -export const invalidate_cached_user_by_id = async (id) => { - await UserRedisCacheSpace.invalidateById(id); -}; - -export async function refresh_associations_cache () { - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_READ, 'apps'); - console.debug('refresh file associations'); - const associations = await db.read('SELECT * FROM app_filetype_association'); - const lists = {}; - for ( const association of associations ) { - let ext = association.type; - if ( ext.startsWith('.') ) ext = ext.slice(1); - // Default file association entries were added with empty types; - // this prevents those from showing up. - if ( ext === '' ) continue; - if ( ! Object.prototype.hasOwnProperty.call(lists, ext) ) lists[ext] = []; - lists[ext].push(association.app_id); - } - - for ( const k in lists ) { - await setRedisCacheValue( - AppRedisCacheSpace.associationAppsKey(k), - JSON.stringify(lists[k]), - { eventData: lists[k] }, - ); - } -} - -/** - * Get App by a variety of IDs - * - * @param {{[key:'name'|'id'|'uid']?:string}} options - `options` - * @returns {Promise} - */ -export async function get_app (options) { - - const cacheApp = async (app) => { - if ( ! app ) return; - AppRedisCacheSpace.setCachedApp(app, { - ttlSeconds: 24 * 60 * 60, - }); - }; - const isDecoratedAppCacheEntry = (app) => ( - !!app && - typeof app === 'object' && - Object.prototype.hasOwnProperty.call(app, 'icon_is_base64') - ); - - // This condition should be updated if the code below is re-ordered. - if ( options.follow_old_names && !options.uid && options.name ) { - const svc_oldAppName = servicesContainer.services.get('old-app-name'); - const old_name = await svc_oldAppName.check_app_name(options.name); - if ( old_name ) { - options.uid = old_name.app_uid; - - // The following line is technically pointless, but may avoid a bug - // if the if...else chain below is re-ordered. - delete options.name; - } - } - - // Determine the query key for request coalescing - let queryKey; - let cacheKey; - if ( options.uid ) { - queryKey = `uid:${options.uid}`; - cacheKey = AppRedisCacheSpace.key({ - lookup: 'uid', - value: options.uid, - }); - } else if ( options.name ) { - queryKey = `name:${options.name}`; - cacheKey = AppRedisCacheSpace.key({ - lookup: 'name', - value: options.name, - }); - } else if ( options.id ) { - queryKey = `id:${options.id}`; - cacheKey = AppRedisCacheSpace.key({ - lookup: 'id', - value: options.id, - }); - } else { - // No valid lookup parameter - return null; - } - - // Check cache first - let app = safe_json_parse(await redisClient.get(cacheKey), null); - if ( isDecoratedAppCacheEntry(app) ) { - AppRedisCacheSpace.invalidateCachedApp(app); - app = null; - } - if ( app ) { - // shallow clone because we use the `delete` operator - // and it corrupts the cache otherwise - return { ...app }; - } - - // Check if there's already a pending query for this key (request coalescing) - const separatorIndex = queryKey.indexOf(':'); - const pendingLookup = queryKey.slice(0, separatorIndex); - const pendingValue = queryKey.slice(separatorIndex + 1); - const pendingKey = AppRedisCacheSpace.pendingKey({ - lookup: pendingLookup, - value: pendingValue, - }); - const pending = kv.get(pendingKey); - if ( pending ) { - // Reuse the existing pending query - const result = await pending; - // shallow clone the result - return result ? { ...result } : null; - } - - // Create a new pending query - let resolveQuery; - let rejectQuery; - const queryPromise = new Promise((resolve, reject) => { - resolveQuery = resolve; - rejectQuery = reject; - }); - - kv.set(pendingKey, queryPromise, { 'EX': PENDING_QUERY_TTL }); - - try { - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_READ, 'apps'); - - if ( options.uid ) { - app = (await db.read('SELECT * FROM `apps` WHERE `uid` = ? LIMIT 1', [options.uid]))[0]; - } else if ( options.name ) { - app = (await db.read('SELECT * FROM `apps` WHERE `name` = ? LIMIT 1', [options.name]))[0]; - } else if ( options.id ) { - app = (await db.read('SELECT * FROM `apps` WHERE `id` = ? LIMIT 1', [options.id]))[0]; - } - - cacheApp(app); - resolveQuery(app); - } catch ( err ) { - rejectQuery(err); - throw err; - } finally { - // Clean up the pending query after completion - kv.del(pendingKey); - } - - if ( ! app ) return null; - // shallow clone because we use the `delete` operator - // and it corrupts the cache otherwise - app = { ...app }; - - return app; -} - -export const get_app_icon_url = (app, size) => { - const iconIsBase64 = isBase64AppIcon(app); - const svc_appIcon = servicesContainer.services.get('app-icon'); - const app_uid = app.uid ?? app.uuid; - - // For base64 icons, or if `no_subdomain` was set in config, use the - // `/app-icon` endpoint on Puter's backend as the URL for this icon. - if ( !app.icon || iconIsBase64 || svc_appIcon.config.no_subdomain ) { - if ( ! app_uid ) return null; - const normalized_uid = normalizeAppUid(app_uid); - const iconSize = Number.isFinite(Number(size)) ? Number(size) : DEFAULT_APP_ICON_SIZE; - - try { - const iconPath = svc_appIcon?.getAppIconPath?.({ - appUid: normalized_uid, - size: iconSize, - }); - if ( iconPath ) return iconPath; - } catch { - // Fall back to direct URL generation below. - } - - const apiBaseUrl = String(config.api_base_url || '').replace(/\/+$/, ''); - if ( ! apiBaseUrl ) return null; - return `${apiBaseUrl}/app-icon/${normalized_uid}/${iconSize}`; - } - - // Otherwise, the icon has a URL under `puter-app-icons.puter.site` - // (or the `puter-app-icons` subdomain of this Puter instance's static hosting domain) - if ( ! app_uid ) return null; - const normalized_uid = normalizeAppUid(app_uid); - const iconSize = Number.isFinite(Number(size)) ? Number(size) : DEFAULT_APP_ICON_SIZE; - const static_hosting_domain = config.static_hosting_domain || config.static_hosting_domain_alt; - if ( ! static_hosting_domain ) return null; - const protocol = config.protocol || 'https'; - return `${protocol}://${APP_ICONS_SUBDOMAIN}.${static_hosting_domain}/${normalized_uid}-${iconSize}.png`; -}; - -/** - * Get multiple apps by uid/name/id, aligned to the input order. - * - * @param {Array<{uid?: string, name?: string, id?: string|number}>} specifiers - * @param {Object} [options] - * @returns {Promise>} - */ -export const get_apps = spanify('get_apps', async (specifiers, options = {}) => { - if ( ! Array.isArray(specifiers) ) { - specifiers = [specifiers]; - } - - const decorateApp = (app) => { - if ( ! app ) return app; - const icon_url = get_app_icon_url(app.uid ?? app.uuid); - if ( ! icon_url ) return { ...app }; - return { ...app, icon: icon_url }; - }; - const normalizeAppForCache = (app) => { - if ( ! app ) return app; - const normalized = { ...app }; - delete normalized.icon_is_base64; - return normalized; - }; - const isDecoratedAppCacheEntry = (app) => ( - !!app && - typeof app === 'object' && - Object.prototype.hasOwnProperty.call(app, 'icon_is_base64') - ); - const cacheApp = async (app) => { - if ( ! app ) return; - AppRedisCacheSpace.setCachedApp(app, { - ttlSeconds: 24 * 60 * 60, - }); - }; - - const normalized = specifiers.map(spec => spec ? { ...spec } : {}); - - if ( options.follow_old_names ) { - const svc_oldAppName = servicesContainer.services.get('old-app-name'); - for ( const spec of normalized ) { - if ( spec.uid || !spec.name ) continue; - const old_name = await svc_oldAppName.check_app_name(spec.name); - if ( old_name ) { - spec.uid = old_name.app_uid; - delete spec.name; - } - } - } - - const appByUid = new Map(); - const appByName = new Map(); - const appById = new Map(); - - const addApp = (app) => { - if ( ! app ) return; - appByUid.set(app.uid, app); - appByName.set(app.name, app); - appById.set(app.id, app); - }; - - const pendingLookups = new Map(); - const pendingToResolve = new Map(); - const queryUids = new Set(); - const queryNames = new Set(); - const queryIds = new Set(); - - const queueMissing = (type, value) => { - const queryKey = `${type}:${value}`; - if ( pendingToResolve.has(queryKey) || pendingLookups.has(queryKey) ) { - return; - } - - const separatorIndex = queryKey.indexOf(':'); - const lookup = queryKey.slice(0, separatorIndex); - value = queryKey.slice(separatorIndex + 1); - const pendingKey = AppRedisCacheSpace.pendingKey({ - lookup, - value, - }); - const pending = kv.get(pendingKey); - if ( pending ) { - pendingLookups.set(queryKey, pending); - return; - } - - let resolveQuery; - let rejectQuery; - const queryPromise = new Promise((resolve, reject) => { - resolveQuery = resolve; - rejectQuery = reject; - }); - kv.set(pendingKey, queryPromise, { 'EX': PENDING_QUERY_TTL }); - pendingToResolve.set(queryKey, { resolveQuery, rejectQuery, pendingKey }); - - if ( type === 'uid' ) { - queryUids.add(value); - } else if ( type === 'name' ) { - queryNames.add(value); - } else if ( type === 'id' ) { - queryIds.add(value); - } - }; - - const cacheLookupPlan = normalized.map((spec) => { - if ( spec.uid ) { - return { - lookup: 'uid', - value: spec.uid, - cacheKey: AppRedisCacheSpace.key({ - lookup: 'uid', - value: spec.uid, - }), - }; - } - if ( spec.name ) { - return { - lookup: 'name', - value: spec.name, - cacheKey: AppRedisCacheSpace.key({ - lookup: 'name', - value: spec.name, - }), - }; - } - if ( spec.id ) { - return { - lookup: 'id', - value: spec.id, - cacheKey: AppRedisCacheSpace.key({ - lookup: 'id', - value: spec.id, - }), - }; - } - return null; - }); - - const cachedAppsByKey = await redisGetJsonMany( - cacheLookupPlan.filter(Boolean).map(item => item.cacheKey), - ); - - for ( const plannedLookup of cacheLookupPlan ) { - if ( ! plannedLookup ) continue; - let cached = cachedAppsByKey.get(plannedLookup.cacheKey); - if ( isDecoratedAppCacheEntry(cached) ) { - AppRedisCacheSpace.invalidateCachedApp(cached); - cached = null; - } - if ( cached ) { - addApp(decorateApp(cached)); - } else { - queueMissing(plannedLookup.lookup, plannedLookup.value); - } - } - - const pendingResultsPromise = pendingLookups.size - ? Promise.all(Array.from(pendingLookups.values())) - : Promise.resolve([]); - - if ( queryUids.size || queryNames.size || queryIds.size ) { - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_READ, 'apps'); - - const clauses = []; - const params = []; - - if ( queryUids.size ) { - const uids = Array.from(queryUids); - clauses.push(`uid IN (${uids.map(() => '?').join(', ')})`); - params.push(...uids); - } - if ( queryNames.size ) { - const names = Array.from(queryNames); - clauses.push(`name IN (${names.map(() => '?').join(', ')})`); - params.push(...names); - } - if ( queryIds.size ) { - const ids = Array.from(queryIds); - clauses.push(`id IN (${ids.map(() => '?').join(', ')})`); - params.push(...ids); - } - - let rows = []; - const resolvedKeys = new Set(); - try { - rows = await db.read( - `SELECT *, CASE WHEN icon LIKE 'data:%' THEN 1 ELSE 0 END AS icon_is_base64 FROM \`apps\` WHERE ${clauses.join(' OR ')}`, - params, - ); - for ( const app of rows ) { - const appForCache = normalizeAppForCache(app); - cacheApp(appForCache); - const decorated_app = decorateApp(appForCache); - addApp(decorated_app); - - const uidKey = `uid:${appForCache.uid}`; - const nameKey = `name:${appForCache.name}`; - const idKey = `id:${appForCache.id}`; - - if ( pendingToResolve.has(uidKey) ) { - pendingToResolve.get(uidKey).resolveQuery(appForCache); - resolvedKeys.add(uidKey); - } - if ( pendingToResolve.has(nameKey) ) { - pendingToResolve.get(nameKey).resolveQuery(appForCache); - resolvedKeys.add(nameKey); - } - if ( pendingToResolve.has(idKey) ) { - pendingToResolve.get(idKey).resolveQuery(appForCache); - resolvedKeys.add(idKey); - } - } - - for ( const [key, { resolveQuery }] of pendingToResolve.entries() ) { - if ( ! resolvedKeys.has(key) ) { - resolveQuery(null); - } - } - } catch ( err ) { - for ( const { rejectQuery } of pendingToResolve.values() ) { - rejectQuery(err); - } - throw err; - } finally { - for ( const { pendingKey } of pendingToResolve.values() ) { - kv.del(pendingKey); - } - } - - } - - const pendingResults = await pendingResultsPromise; - for ( const app of pendingResults ) { - addApp(decorateApp(app)); - } - - return normalized.map(spec => { - let app; - if ( spec.uid ) { - app = appByUid.get(spec.uid); - } else if ( spec.name ) { - app = appByName.get(spec.name); - } else if ( spec.id ) { - app = appById.get(spec.id); - } - if ( ! app ) return null; - const result = { ...app }; - delete result.icon_is_base64; - return result; - }); - -}); - -/** - * Checks to see if an app exists - * - * @param {string} options - `options` - * @returns {Promise} - */ -export async function app_exists (options) { - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_READ, 'apps'); - - let app; - if ( options.uid ) - { - app = await db.read('SELECT `id` FROM `apps` WHERE `uid` = ? LIMIT 1', [options.uid]); - } - else if ( options.name ) - { - app = await db.read('SELECT `id` FROM `apps` WHERE `name` = ? LIMIT 1', [options.name]); - } - else if ( options.id ) - { - app = await db.read('SELECT `id` FROM `apps` WHERE `id` = ? LIMIT 1', [options.id]); - } - - return app[0]; -} - -/** - * change username - * - * @param {string} options - `options` - * @returns {Promise} - */ -export async function change_username (user_id, new_username) { - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_WRITE, 'auth'); - - const old_username = (await get_user({ id: user_id })).username; - - // update username - await db.write('UPDATE `user` SET username = ? WHERE `id` = ? LIMIT 1', [new_username, user_id]); - // update root directory name for this user - await db.write( - 'UPDATE `fsentries` SET `name` = ?, `path` = ? ' + - 'WHERE `user_id` = ? AND parent_uid IS NULL LIMIT 1', - [new_username, `/${ new_username}`, user_id], - ); - - console.log(`User ${old_username} changed username to ${new_username}`); - await servicesContainer.services.get('filesystem').update_child_paths(`/${old_username}`, `/${new_username}`, user_id); - - invalidate_cached_user_by_id(user_id); -} - -/** - * Find a FSEntry by its uuid - * - * @param {integer} id - `id` of FSEntry - * @returns {Promise} Promise object represents the UUID of the FileSystem Entry - * @deprecated Use fs middleware instead - */ -export async function uuid2fsentry (uuid, return_thumbnail) { - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem'); - - // todo optim, check if uuid is not exactly 36 characters long, if not it's invalid - // and we can avoid one unnecessary DB lookup - let fsentry = await db.requireRead( - `SELECT - id, - associated_app_id, - uuid, - public_token, - bucket, - bucket_region, - file_request_token, - user_id, - parent_uid, - is_dir, - is_public, - is_shortcut, - shortcut_to, - sort_by, - ${return_thumbnail ? 'thumbnail,' : ''} - immutable, - name, - metadata, - modified, - created, - accessed, - size - FROM fsentries WHERE uuid = ? LIMIT 1`, - [uuid], - ); - - if ( ! fsentry[0] ) - { - return false; - } - else - { - return fsentry[0]; - } -} - -/** - * Find a FSEntry by its id - * - * @param {integer} id - `id` of FSEntry - * @returns {Promise} Promise object represents the UUID of the FileSystem Entry - */ -export async function id2fsentry (id, return_thumbnail) { - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem'); - - // todo optim, check if uuid is not exactly 36 characters long, if not it's invalid - // and we can avoid one unnecessary DB lookup - let fsentry = await db.requireRead( - `SELECT - id, - uuid, - public_token, - file_request_token, - associated_app_id, - user_id, - parent_uid, - is_dir, - is_public, - is_shortcut, - shortcut_to, - sort_by, - ${return_thumbnail ? 'thumbnail,' : ''} - immutable, - name, - metadata, - modified, - created, - accessed, - size - FROM fsentries WHERE id = ? LIMIT 1`, - [id], - ); - - if ( ! fsentry[0] ) { - return false; - } else - { - return fsentry[0]; - } -} - -/** - * Takes a an absolute path and returns its corresponding FSEntry. - * - * @param {string} path - absolute path of the filesystem entry to be resolved - * @param {boolean} return_content - if FSEntry is a file, determines whether its content should be returned - * @returns {false|object} - `false` if path could not be resolved, otherwise an object representing the FSEntry - * @deprecated Use fs middleware instead - */ -export async function convert_path_to_fsentry (path) { - // todo optim, check if path is valid (e.g. contaisn valid characters) - // if syntactical errors are found we can potentially avoid some expensive db lookups - - // '/' means that parent_uid is null - // TODO: facade fsentry for root (devlog:2023-06-01) - if ( path === '/' ) - { - return null; - } - //first slash is redundant - path = path.substr(path.indexOf('/') + 1); - //last slash, if existing is redundant - if ( path[path.length - 1] === '/' ) - { - path = path.slice(0, -1); - } - //split path into parts - const fsentry_names = path.split('/'); - - // if no parts, return false - if ( fsentry_names.length === 0 ) - { - return false; - } - - let parent_uid = null; - let final_res = null; - let is_public = false; - let result; - - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem'); - - // Try stored path first - result = await db.read( - 'SELECT * FROM fsentries WHERE path=? LIMIT 1', - [`/${ path}`], - ); - - if ( result[0] ) { - return result[0]; - } - - for ( let i = 0; i < fsentry_names.length; i++ ) { - if ( parent_uid === null ) { - result = await db.read( - 'SELECT * FROM fsentries WHERE parent_uid IS NULL AND name=? LIMIT 1', - [fsentry_names[i]], - ); - } - else { - result = await db.read( - 'SELECT * FROM fsentries WHERE parent_uid = ? AND name=? LIMIT 1', - [parent_uid, fsentry_names[i]], - ); - } - - if ( result[0] ) { - parent_uid = result[0].uuid; - // is_public is either directly specified or inherited from parent dir - if ( result[0].is_public === null ) - { - result[0].is_public = is_public; - } - else - { - is_public = result[0].is_public; - } - - } else { - return false; - } - final_res = result; - } - return final_res[0]; -} - -/** - * - * @param {integer} bytes - size in bytes - * @returns {string} bytes in human-readable format - */ -export function byte_format (bytes) { - // calculate and return bytes in human-readable format - const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']; - if ( typeof bytes !== 'number' || bytes < 1 ) { - return '0 B'; - } - const i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024))); - return `${Math.round(bytes / Math.pow(1024, i), 2) } ${ sizes[i]}`; -}; - -export const get_descendants = spanify('get_descendants', async (...args) => { - return await getDescendantsHelper(...args); -}); - -/** - * - * @param {integer} entry_id - * @returns - */ -export const id2path = spanify('helpers:id2path', async (entry_uid) => { - if ( entry_uid == null ) { - throw new Error('got null or undefined entry id'); - } - - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem'); - - const log = servicesContainer.services.get('log-service').create('helpers.id2path'); - log.traceOn(); - const errors = servicesContainer.services.get('error-service').create(log); - log.called(); - - let result; - - log.debug(`entry id: ${entry_uid}`); - if ( typeof entry_uid === 'number' ) { - const old = entry_uid; - entry_uid = await id2uuid(entry_uid); - log.debug(`entry id resolved: resolved ${old} ${entry_uid}`); - } - - try { - result = await db.read(` - WITH RECURSIVE cte AS ( - SELECT uuid, parent_uid, name, name AS path - FROM fsentries - WHERE uuid = ? - - UNION ALL - - SELECT e.uuid, e.parent_uid, e.name, ${ - db.case({ - sqlite: 'e.name || \'/\' || cte.path', - otherwise: 'CONCAT(e.name, \'/\', cte.path)', - }) - } - FROM fsentries e - INNER JOIN cte ON cte.parent_uid = e.uuid - ) - SELECT * - FROM cte - WHERE parent_uid IS NULL - `, [entry_uid]); - } catch (e) { - errors.report('id2path.select', { - alarm: true, - source: e, - message: `error while resolving path for ${entry_uid}: ${e.message}`, - extra: { - entry_uid, - }, - }); - throw new ManagedError(`cannot create path for ${entry_uid}`); - } - - if ( !result || !result[0] ) { - errors.report('id2path.select', { - alarm: true, - message: `no result for ${entry_uid}`, - extra: { - entry_uid, - }, - }); - throw new ManagedError(`cannot create path for ${entry_uid}`); - } - - return `/${ result[0].path}`; -}); - -/** - * Recursively retrieve all files, directories, and subdirectories under `path`. - * Optionally the `depth` can be set. - * - * @param {string} path - * @param {object} user - * @param {integer} depth - * @returns - */ -async function getDescendantsHelper (path, user, depth, return_thumbnail = false) { - const log = servicesContainer.services.get('log-service').create('get_descendants'); - log.called(); - - // decrement depth if it's set - depth !== undefined && depth--; - // turn path into absolute form - path = _resolve('/', path); - // get parent dir - const parent = await convert_path_to_fsentry(path); - // holds array that will be returned - const ret = []; - // holds immediate children of this path - let children; - - // try to extract username from path - let username; - let split_path = path.split('/'); - if ( split_path.length === 2 && split_path[0] === '' ) - { - username = split_path[1]; - } - - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem'); - - // ------------------------------------- - // parent is root ('/') - // ------------------------------------- - if ( parent === null ) { - path = ''; - // direct children under root - children = await db.read( - `SELECT - id, uuid, parent_uid, name, metadata, is_dir, bucket, bucket_region, - modified, created, immutable, shortcut_to, is_shortcut, sort_by, associated_app_id, - ${return_thumbnail ? 'thumbnail, ' : ''} - accessed, size - FROM fsentries - WHERE user_id = ? AND parent_uid IS NULL`, - [user.id], - ); - // users that have shared files/dirs with this user - const sharing_users = await db.read( - `SELECT DISTINCT(owner_user_id), user.username - FROM share - INNER JOIN user ON user.id = share.owner_user_id - WHERE share.recipient_user_id = ?`, - [user.id], - ); - if ( sharing_users.length > 0 ) { - for ( let i = 0; i < sharing_users.length; i++ ) { - let dir = {}; - dir.id = null; - dir.uuid = null; - dir.parent_uid = null; - dir.name = sharing_users[i].username; - dir.is_dir = true; - dir.immutable = true; - children.push(dir); - } - } - } - // ------------------------------------- - // parent doesn't exist - // ------------------------------------- - else if ( parent === false ) { - return []; - } - // ------------------------------------- - // Parent is a shared-user directory: /[some_username](/) - // but make sure `[some_username]` is not the same as the requester's username - // ------------------------------------- - else if ( username && username !== user.username ) { - children = []; - let sharing_user; - sharing_user = await get_user({ username: username }); - if ( ! sharing_user ) - { - return []; - } - - // shared files/dirs with this user - const shared_fsentries = await db.read( - `SELECT - fsentries.id, fsentries.user_id, fsentries.uuid, fsentries.parent_uid, fsentries.bucket, fsentries.bucket_region, - fsentries.name, fsentries.shortcut_to, fsentries.is_shortcut, fsentries.metadata, fsentries.is_dir, fsentries.modified, - fsentries.created, fsentries.accessed, fsentries.size, fsentries.sort_by, fsentries.associated_app_id, - fsentries.is_symlink, fsentries.symlink_path, - fsentries.immutable ${return_thumbnail ? ', fsentries.thumbnail' : ''} - FROM share - INNER JOIN fsentries ON fsentries.id = share.fsentry_id - WHERE share.recipient_user_id = ? AND owner_user_id = ?`, - [user.id, sharing_user.id], - ); - // merge `children` and `shared_fsentries` - if ( shared_fsentries.length > 0 ) { - for ( let i = 0; i < shared_fsentries.length; i++ ) { - shared_fsentries[i].path = await id2path(shared_fsentries[i].id); - children.push(shared_fsentries[i]); - } - } - } - // ------------------------------------- - // All other cases - // ------------------------------------- - else { - children = []; - let temp_children = await db.read( - `SELECT - id, user_id, uuid, parent_uid, name, metadata, is_shortcut, - shortcut_to, is_dir, modified, created, accessed, size, sort_by, associated_app_id, - is_symlink, symlink_path, - immutable ${return_thumbnail ? ', thumbnail' : ''} - FROM fsentries - WHERE parent_uid = ?`, - [parent.uuid], - ); - // check if user has access to each file, if yes add it - if ( temp_children.length > 0 ) { - for ( let i = 0; i < temp_children.length; i++ ) { - const tchild = temp_children[i]; - if ( await chkperm(tchild, user.id) ) - { - children.push(tchild); - } - } - } - } - - // shortcut on empty result set - if ( children.length === 0 ) return []; - - const ids = children.map(child => child.id); - const qmarks = ids.map(() => '?').join(','); - - let rows = await db.read( - `SELECT root_dir_id FROM subdomains WHERE root_dir_id IN (${qmarks}) AND user_id=?`, - [...ids, user.id], - ); - - const websiteMap = {}; - for ( const row of rows ) websiteMap[row.root_dir_id] = true; - - for ( let i = 0; i < children.length; i++ ) { - const contentType = _contentType(children[i].name); - - // has_website - let has_website = false; - if ( children[i].is_dir ) { - has_website = websiteMap[children[i].id]; - } - - // object to return - // TODO: DRY creation of response fsentry from db fsentry - ret.push({ - path: children[i].path ?? (`${path }/${ children[i].name}`), - name: children[i].name, - metadata: children[i].metadata, - _id: children[i].id, - id: children[i].uuid, - uid: children[i].uuid, - is_shortcut: children[i].is_shortcut, - shortcut_to: (children[i].shortcut_to ? await id2uuid(children[i].shortcut_to) : undefined), - shortcut_to_path: (children[i].shortcut_to ? await id2path(children[i].shortcut_to) : undefined), - is_symlink: children[i].is_symlink, - symlink_path: children[i].symlink_path, - immutable: children[i].immutable, - is_dir: children[i].is_dir, - modified: children[i].modified, - created: children[i].created, - accessed: children[i].accessed, - size: children[i].size, - sort_by: children[i].sort_by, - thumbnail: children[i].thumbnail, - associated_app_id: children[i].associated_app_id, - type: contentType ? contentType : null, - has_website: has_website, - }); - if ( children[i].is_dir && - (depth === undefined || (depth !== undefined && depth > 0)) - ) { - ret.push(await get_descendants(`${path }/${ children[i].name}`, user, depth)); - } - } - return ret.flat(); -}; - -export const get_dir_size = async (path, user) => { - let size = 0; - const descendants = await get_descendants(path, user); - for ( let i = 0; i < descendants.length; i++ ) { - if ( ! descendants[i].is_dir ) { - size += descendants[i].size; - } - } - - return size; -}; - -/** - * - * @param {string} glob - * @param {object} user - * @returns - */ -export async function resolve_glob (glob, user) { - //turn glob into abs path - glob = _resolve('/', glob); - //get base of glob - const base = micromatch.scan(glob).base; - //estimate needed depth - let depth = 1; - const dirs = glob.split('/'); - for ( let i = 0; i < dirs.length; i++ ) { - if ( dirs[i].includes('**') ) { - depth = undefined; - break; - } else { - depth++; - } - } - - const descendants = await get_descendants(base, user, depth); - - return descendants.filter((fsentry) => { - return fsentry.path && micromatch.isMatch(fsentry.path, glob); - }); -} - -function isString (variable) { - return typeof variable === 'string' || variable instanceof String; -} - -export const body_parser_error_handler = (err, req, res, next) => { - if ( err instanceof SyntaxError && err.status === 400 && 'body' in err ) { - return res.status(400).send(err); // Bad request - } - next(); -}; - -/** - * Given a uid, returns a file node. - * - * TODO (xiaochen): It only works for MemoryFSProvider currently. - * - * @param {string} uid - The uid of the file to get. - * @returns {Promise} The file node, or null if the file does not exist. - */ -async function get_entry (uid) { - const svc_mountpoint = Context.get('services').get('mountpoint'); - const uid_selector = new NodeUIDSelector(uid); - const provider = await svc_mountpoint.get_provider(uid_selector); - - // NB: We cannot import MemoryFSProvider here because it will cause a circular dependency. - if ( provider.constructor.name !== 'MemoryFSProvider' ) { - return null; - } - - return provider.stat({ - selector: uid_selector, - }); -} - -export async function is_ancestor_of (ancestor_uid, descendant_uid) { - const ancestor = await get_entry(ancestor_uid); - const descendant = await get_entry(descendant_uid); - - if ( ancestor && descendant ) { - return descendant.path.startsWith(ancestor.path); - } - - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem'); - - // root is an ancestor to all FSEntries - if ( ancestor_uid === null ) - { - return true; - } - // root is never a descendant to any FSEntries - if ( descendant_uid === null ) - { - return false; - } - - if ( typeof ancestor_uid === 'number' ) { - ancestor_uid = await id2uuid(ancestor_uid); - } - if ( typeof descendant_uid === 'number' ) { - descendant_uid = await id2uuid(descendant_uid); - } - - let parent = await db.read('SELECT `uuid`, `parent_uid` FROM `fsentries` WHERE `uuid` = ? LIMIT 1', [descendant_uid]); - if ( parent[0] === undefined ) - { - parent = await db.pread('SELECT `uuid`, `parent_uid` FROM `fsentries` WHERE `uuid` = ? LIMIT 1', [descendant_uid]); - } - if ( parent[0].uuid === ancestor_uid || parent[0].parent_uid === ancestor_uid ) { - return true; - } - // keep checking as long as parent of parent is not root - while ( parent[0].parent_uid !== null ) { - parent = await db.read('SELECT `uuid`, `parent_uid` FROM `fsentries` WHERE `uuid` = ? LIMIT 1', [parent[0].parent_uid]); - if ( parent[0] === undefined ) { - parent = await db.pread('SELECT `uuid`, `parent_uid` FROM `fsentries` WHERE `uuid` = ? LIMIT 1', [descendant_uid]); - } - - if ( parent[0].uuid === ancestor_uid || parent[0].parent_uid === ancestor_uid ) { - return true; - } - } - - return false; -} - -export async function sign_file (fsentry, action) { - - // fsentry not found - if ( fsentry === false ) { - throw { message: 'No entry found with this uid' }; - } - - const uid = fsentry.uuid ?? (fsentry.uid ?? fsentry._id); - const ttl = 9999999999999; - const secret = config.url_signature_secret; - const expires = Math.ceil(Date.now() / 1000) + ttl; - const signature = sha256(`${uid}/${action}/${secret}/${expires}`); - const contentType = _contentType(fsentry.name); - - // return - return { - uid: uid, - expires: expires, - signature: signature, - url: `${config.api_base_url}/file?uid=${uid}&expires=${expires}&signature=${signature}`, - read_url: `${config.api_base_url}/file?uid=${uid}&expires=${expires}&signature=${signature}`, - write_url: `${config.api_base_url}/writeFile?uid=${uid}&expires=${expires}&signature=${signature}`, - metadata_url: `${config.api_base_url}/itemMetadata?uid=${uid}&expires=${expires}&signature=${signature}`, - fsentry_type: contentType, - fsentry_is_dir: !!fsentry.is_dir, - fsentry_name: fsentry.name, - fsentry_size: fsentry.size, - fsentry_accessed: fsentry.accessed, - fsentry_modified: fsentry.modified, - fsentry_created: fsentry.created, - }; -} - -export async function gen_public_token (file_uuid) { - - // get fsentry - let fsentry = await uuid2fsentry(file_uuid); - - // fsentry not found - if ( fsentry === false ) { - throw { message: 'No entry found with this uid' }; - } - - const uid = fsentry.uuid; - const token = v4(); - const contentType = _contentType(fsentry.name); - - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_WRITE, 'filesystem'); - - // insert into DB - try { - await db.write( - 'UPDATE fsentries SET public_token = ? WHERE id = ?', - [ - //token - token, - //fsentry_id - fsentry.id, - ], - ); - } catch (e) { - console.log(e); - return false; - } - - // return - return { - uid: uid, - token: token, - url: `${config.api_base_url}/pubfile?token=${token}`, - fsentry_type: contentType, - fsentry_is_dir: fsentry.is_dir, - fsentry_name: fsentry.name, - }; -} - -export async function deleteUser (user_id) { - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem'); - const svc_fs = servicesContainer.services.get('filesystem'); - - // get a list of up to 5000 files owned by this user - // eslint-disable-next-line no-constant-condition - for ( let offset = 0; true; offset += 5000 ) { - let files = await db.read( - `SELECT uuid, bucket, bucket_region FROM fsentries WHERE user_id = ? AND is_dir = 0 LIMIT 5000 OFFSET ${ offset}`, - [user_id], - ); - - if ( !files || files.length == 0 ) break; - - // delete all files from S3 - if ( files !== null && files.length > 0 ) { - for ( let i = 0; i < files.length; i++ ) { - const node = await svc_fs.node(new NodeUIDSelector(files[i].uuid)); - - await node.provider.unlink({ - context: Context.get(), - override_immutable: true, - node, - }); - } - } - } - - // delete all fsentries from DB - await db.write('DELETE FROM fsentries WHERE user_id = ?', [user_id]); - - // delete user - await db.write('DELETE FROM user WHERE id = ?', [user_id]); -} - -export function subdomain (req) { - if ( config.experimental_no_subdomain ) return 'api'; - return req.hostname.slice(0, -1 * (config.domain.length + 1)); -} - -export async function jwt_auth (req, authService) { - let token; - // HTTML Auth header - if ( req.header && req.header('Authorization') ) - { - token = req.header('Authorization'); - } - // Cookie - else if ( req.cookies && req.cookies[config.cookie_name] ) - { - token = req.cookies[config.cookie_name]; - } - // Auth token in URL - else if ( req.query && req.query.auth_token ) - { - token = req.query.auth_token; - } - // Socket - else if ( req.handshake && req.handshake.auth && req.handshake.auth.auth_token ) - { - token = req.handshake.auth.auth_token; - } - - if ( !token || token === 'null' ) - { - throw ('No auth token found'); - } - else if ( typeof token !== 'string' ) - { - throw ('token must be a string.'); - } - else - { - token = token.replace('Bearer ', ''); - } - - try { - if ( ! authService ) { - throw new Error('jwt_auth requires authService'); - } - - const actor = await authService.authenticate_from_token(token); - - if ( !actor.type?.constructor?.name === 'UserActorType' ) { - throw ({ - message: APIError.create('token_unsupported') - .serialize(), - }); - } - - return { - actor, - user: actor.type.user, - token: token, - }; - } catch (e) { - if ( ! (e instanceof APIError) ) { - console.log('ERROR', e); - } - throw (e.message); - } -} - -/** - * returns all ancestors of an fsentry - * - * @param {*} fsentry_id - */ -export async function ancestors (fsentry_id) { - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem'); - - const ancestors = []; - // first parent - let parent = await db.read('SELECT * FROM `fsentries` WHERE `id` = ? LIMIT 1', [fsentry_id]); - if ( parent.length === 0 ) { - return ancestors; - } - // get all subsequent parents - while ( parent[0].parent_uid !== null ) { - const parent_fsentry = await uuid2fsentry(parent[0].parent_uid); - parent = await db.read('SELECT * FROM `fsentries` WHERE `id` = ? LIMIT 1', [parent_fsentry.id]); - if ( parent[0].length !== 0 ) { - ancestors.push(parent[0]); - } - } - - return ancestors; -} - -export function hyphenize_confirm_code (email_confirm_code) { - email_confirm_code = email_confirm_code.toString(); - email_confirm_code = - `${email_confirm_code[0] + - email_confirm_code[1] + - email_confirm_code[2] - }-${ - email_confirm_code[3] - }${email_confirm_code[4] - }${email_confirm_code[5]}`; - return email_confirm_code; -} - -export async function username_exists (username) { - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem'); - - let rows = await db.read('SELECT EXISTS(SELECT 1 FROM user WHERE username=?) AS username_exists', [username]); - if ( rows[0].username_exists ) - { - return true; - } -} - -export async function generate_random_username () { - let username; - do { - username = generate_identifier(); - } while ( await username_exists(username) ); - return username; -} - -export async function app_name_exists (name) { - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem'); - - let rows = await db.read('SELECT EXISTS(SELECT 1 FROM apps WHERE apps.name=?) AS app_name_exists', [name]); - if ( rows[0].app_name_exists ) - { - return true; - } - - const svc_oldAppName = servicesContainer.services.get('old-app-name'); - const name_info = await svc_oldAppName.check_app_name(name); - if ( name_info ) return true; -} - -export function send_email_verification_code (email_confirm_code, email) { - const svc_email = Context.get('services').get('email'); - svc_email.send_email({ email }, 'email_verification_code', { - code: hyphenize_confirm_code(email_confirm_code), - }); -} - -export function send_email_verification_token (email_confirm_token, email, user_uuid) { - const svc_email = Context.get('services').get('email'); - const link = `${config.origin}/confirm-email-by-token?user_uuid=${user_uuid}&token=${email_confirm_token}`; - svc_email.send_email({ email }, 'email_verification_link', { link }); -} - -export function generate_random_str (length) { - let result = ''; - const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; - const charactersLength = characters.length; - for ( let i = 0; i < length; i++ ) { - result += characters.charAt(Math.floor(Math.random() * - charactersLength)); - } - return result; -} - -/** - * Converts a given number of seconds into a human-readable string format. - * - * @param {number} seconds - The number of seconds to be converted. - * @returns {string} The time represented in the format: 'X years Y days Z hours A minutes B seconds'. - * @throws {TypeError} If the `seconds` parameter is not a number. - */ -export function seconds_to_string (seconds) { - const numyears = Math.floor(seconds / 31536000); - const numdays = Math.floor((seconds % 31536000) / 86400); - const numhours = Math.floor(((seconds % 31536000) % 86400) / 3600); - const numminutes = Math.floor((((seconds % 31536000) % 86400) % 3600) / 60); - const numseconds = (((seconds % 31536000) % 86400) % 3600) % 60; - return `${numyears } years ${ numdays } days ${ numhours } hours ${ numminutes } minutes ${ numseconds } seconds`; -} - -/** - * returns a list of apps that could open the fsentry, ranked by relevance - * @param {*} fsentry - * @param {*} options - */ -const SUGGEST_APP_CODE_EXTS = [ - '.asm', - '.asp', - '.aspx', - '.bash', - '.c', - '.cpp', - '.css', - '.csv', - '.dhtml', - '.f', - '.go', - '.h', - '.htm', - '.html', - '.html5', - '.java', - '.jl', - '.js', - '.jsa', - '.json', - '.jsonld', - '.jsf', - '.jsp', - '.kt', - '.log', - '.lock', - '.lua', - '.md', - '.perl', - '.phar', - '.php', - '.pl', - '.py', - '.r', - '.rb', - '.rdata', - '.rda', - '.rdf', - '.rds', - '.rs', - '.rlib', - '.rpy', - '.scala', - '.sc', - '.scm', - '.sh', - '.sol', - '.sql', - '.ss', - '.svg', - '.swift', - '.toml', - '.ts', - '.wasm', - '.xhtml', - '.xml', - '.yaml', -]; - -const buildSuggestedAppSpecifiers = async (fsentry) => { - const name_specifiers = []; - - let content_type = _contentType(fsentry.name); - if ( ! content_type ) content_type = ''; - - // IIFE just so fsname can stay `const` - const fsname = (() => { - if ( ! fsentry.name ) { - return 'missing-fsentry-name'; - } - let fsname = fsentry.name.toLowerCase(); - // We add `.directory` so that this works as a file association - if ( fsentry.is_dir ) fsname += '.directory'; - return fsname; - })(); - const file_extension = extname(fsname).toLowerCase(); - - const any_of = (list, name) => list.some(v => name.endsWith(v)); - - //--------------------------------------------- - // Code - //--------------------------------------------- - if ( any_of(SUGGEST_APP_CODE_EXTS, fsname) || !fsname.includes('.') ) { - name_specifiers.push({ name: 'code' }); - name_specifiers.push({ name: 'editor' }); - } - - //--------------------------------------------- - // Editor - //--------------------------------------------- - if ( - fsname.endsWith('.txt') || - // files with no extension - !fsname.includes('.') - ) { - name_specifiers.push({ name: 'editor' }); - name_specifiers.push({ name: 'code' }); - } - //--------------------------------------------- - // Markus - //--------------------------------------------- - if ( fsname.endsWith('.md') ) { - name_specifiers.push({ name: 'markus' }); - } - //--------------------------------------------- - // Viewer - //--------------------------------------------- - if ( - fsname.endsWith('.jpg') || - fsname.endsWith('.png') || - fsname.endsWith('.webp') || - fsname.endsWith('.svg') || - fsname.endsWith('.bmp') || - fsname.endsWith('.jpeg') - ) { - name_specifiers.push({ name: 'viewer' }); - } - //--------------------------------------------- - // Draw - //--------------------------------------------- - if ( - fsname.endsWith('.bmp') || - content_type.startsWith('image/') - ) { - name_specifiers.push({ name: 'draw' }); - } - //--------------------------------------------- - // PDF - //--------------------------------------------- - if ( fsname.endsWith('.pdf') ) { - name_specifiers.push({ name: 'pdf' }); - } - //--------------------------------------------- - // Player - //--------------------------------------------- - if ( - fsname.endsWith('.mp4') || - fsname.endsWith('.webm') || - fsname.endsWith('.mpg') || - fsname.endsWith('.mpv') || - fsname.endsWith('.mp3') || - fsname.endsWith('.m4a') || - fsname.endsWith('.ogg') - ) { - name_specifiers.push({ name: 'player' }); - } - - //--------------------------------------------- - // 3rd-party apps - //--------------------------------------------- - const apps = safe_json_parse(await redisClient.get( - AppRedisCacheSpace.associationAppsKey(file_extension.slice(1)), - ), []); - /** @type {{id:string}[]} */ - const id_specifiers = apps.map(app_id => ({ id: app_id })); - - return { name_specifiers, id_specifiers }; -}; - -const buildSuggestedAppsFromResolved = (resolved, name_specifier_count, options) => { - const suggested_apps = []; - - const name_apps = resolved.slice(0, name_specifier_count); - suggested_apps.push(...name_apps); - - const third_party_apps = resolved.slice(name_specifier_count); - for ( const third_party_app of third_party_apps ) { - if ( ! third_party_app ) continue; - if ( third_party_app.approved_for_opening_items || - (options?.user && options.user.id === third_party_app.owner_user_id) ) - { - suggested_apps.push(third_party_app); - } - } - - const needs_codeapp = suggested_apps.some(app => app && app.name === 'editor'); - return { suggested_apps, needs_codeapp }; -}; - -const normalizeSuggestedApps = (suggested_apps) => ( - suggested_apps.filter((suggested_app, pos, self) => { - // Remove any null values caused by calling `get_app()` for apps that don't exist. - // This happens on self-host because we don't include `code`, among others. - if ( ! suggested_app ) { - return false; - } - - // Remove any duplicate entries - return self.indexOf(suggested_app) === pos; - }) -); - -const buildSuggestedAppsCacheKey = (fsentry, options) => { - const user_id = options?.user?.id ?? ''; - const entry_id = fsentry?.uuid ?? fsentry?.uid ?? fsentry?.id ?? fsentry?.path ?? ''; - const entry_name = fsentry?.name ?? ''; - const entry_type = fsentry?.is_dir ? 'd' : 'f'; - return `${user_id}:${entry_id}:${entry_type}:${entry_name}`; -}; - -const cloneSuggestedApps = (suggested_apps) => ( - Array.isArray(suggested_apps) - ? suggested_apps.map(app => (app ? { ...app } : app)) - : suggested_apps -); - -export async function suggestedAppsForFsEntries (fsentries, options) { - if ( ! Array.isArray(fsentries) ) { - fsentries = [fsentries]; - } - - const batches = []; - const specifiers = []; - const results = new Array(fsentries.length); - const cacheKeysByIndex = new Map(); - - for ( let index = 0; index < fsentries.length; index++ ) { - const fsentry = fsentries[index]; - if ( ! fsentry ) { - results[index] = []; - continue; - } - - const cache_key = buildSuggestedAppsCacheKey(fsentry, options); - const cached = suggestedAppsCache.get(cache_key); - if ( cached !== undefined ) { - results[index] = cloneSuggestedApps(cached); - continue; - } - - const { name_specifiers, id_specifiers } = await buildSuggestedAppSpecifiers(fsentry); - const entry_specifiers = [...name_specifiers, ...id_specifiers]; - - if ( entry_specifiers.length === 0 ) { - results[index] = []; - cacheKeysByIndex.set(index, cache_key); - continue; - } - - const offset = specifiers.length; - specifiers.push(...entry_specifiers); - batches.push({ - index, - offset, - count: entry_specifiers.length, - name_count: name_specifiers.length, - suggested_apps: [], - needs_codeapp: false, - }); - cacheKeysByIndex.set(index, cache_key); - } - - let resolved = []; - if ( specifiers.length > 0 ) { - resolved = await get_apps(specifiers); - } - - let any_needs_codeapp = false; - for ( const batch of batches ) { - const slice = resolved.slice(batch.offset, batch.offset + batch.count); - const { suggested_apps, needs_codeapp } = buildSuggestedAppsFromResolved( - slice, - batch.name_count, - options, - ); - batch.suggested_apps = suggested_apps; - batch.needs_codeapp = needs_codeapp; - if ( needs_codeapp ) any_needs_codeapp = true; - } - - let codeapp; - if ( any_needs_codeapp ) { - [codeapp] = await get_apps([{ name: 'codeapp' }]); - } - - for ( const batch of batches ) { - let suggested_apps = batch.suggested_apps; - if ( batch.needs_codeapp && codeapp ) { - suggested_apps = [...suggested_apps, codeapp]; - } - results[batch.index] = normalizeSuggestedApps(suggested_apps); - } - - // Deduplicate results by ID - const deduplicatedResults = results.map(apps => { - if ( ! Array.isArray(apps) ) return apps; - const seen = new Set(); - return apps.filter(app => { - if ( !app || !app.id ) return true; - if ( seen.has(app.id) ) return false; - seen.add(app.id); - return true; - }); - }); - - for ( const [index, cache_key] of cacheKeysByIndex ) { - const apps = deduplicatedResults[index]; - if ( apps !== undefined ) { - suggestedAppsCache.set(cache_key, cloneSuggestedApps(apps)); - } - } - - return deduplicatedResults; -} - -export async function suggestedAppForFsEntry (fsentry, options) { - const [result] = await suggestedAppsForFsEntries([fsentry], options); - return result; -} - -export async function get_taskbar_items (user, { - icon_size: iconSizeFromSnake, - iconSize: iconSizeFromCamel, - no_icons, -} = {}) { - const iconSize = iconSizeFromCamel ?? iconSizeFromSnake; - /** @type BaseDatabaseAccessService */ - const db = servicesContainer.services.get('database').get(DB_WRITE, 'filesystem'); - - let taskbar_items_from_db = []; - // If taskbar items don't exist (specifically NULL) - // add default apps. - if ( ! user.taskbar_items ) { - taskbar_items_from_db = [ - { name: 'app-center', type: 'app' }, - { name: 'dev-center', type: 'app' }, - { name: 'editor', type: 'app' }, - { name: 'code', type: 'app' }, - { name: 'camera', type: 'app' }, - { name: 'recorder', type: 'app' }, - ]; - await db.write( - 'UPDATE user SET taskbar_items = ? WHERE id = ?', - [ - JSON.stringify(taskbar_items_from_db), - user.id, - ], - ); - invalidate_cached_user(user); - } - // there are items from before - else { - try { - taskbar_items_from_db = JSON.parse(user.taskbar_items); - } catch (e) { - // ignore errors - } - } - - const app_specifiers = taskbar_items_from_db.map((taskbar_item_from_db) => { - if ( taskbar_item_from_db.type !== 'app' ) return {}; - if ( taskbar_item_from_db.name === 'explorer' ) return {}; - if ( taskbar_item_from_db.name ) { - return { name: taskbar_item_from_db.name }; - } - if ( taskbar_item_from_db.id ) { - return { id: taskbar_item_from_db.id }; - } - if ( taskbar_item_from_db.uid ) { - return { uid: taskbar_item_from_db.uid }; - } - return {}; - }); - - const taskbar_apps = await get_apps(app_specifiers); - - // get apps that these taskbar items represent - let taskbar_items = []; - for ( let index = 0; index < taskbar_items_from_db.length; index++ ) { - const taskbar_item_from_db = taskbar_items_from_db[index]; - if ( taskbar_item_from_db.type !== 'app' ) continue; - if ( taskbar_item_from_db.name === 'explorer' ) continue; - - const item = taskbar_apps[index]; - - // if item not found, skip it - if ( ! item ) continue; - - // delete sensitive attributes - delete item.id; - delete item.owner_user_id; - delete item.timestamp; - // delete item.godmode; - delete item.approved_for_listing; - delete item.approved_for_opening_items; - - if ( no_icons ) { - delete item.icon; - } else { - item.icon = get_app_icon_url(item, iconSize); - } - - // add to final object - taskbar_items.push(item); - } - - return taskbar_items; -} - -export function validate_signature_auth (url, action, options = {}) { - const query = new URL(url).searchParams; - - if ( ! query.get('uid') ) - { - throw { message: '`uid` is required for signature-based authentication.' }; - } - else if ( ! action ) - { - throw { message: '`action` is required for signature-based authentication.' }; - } - else if ( ! query.get('expires') ) - { - throw { message: '`expires` is required for signature-based authentication.' }; - } - else if ( ! query.get('signature') ) - { - throw { message: '`signature` is required for signature-based authentication.' }; - } - - if ( options.uid ) { - if ( query.get('uid') !== options.uid ) { - throw { message: 'Authentication failed. `uid` does not match.' }; - } - } - - const expired = query.get('expires') && (query.get('expires') < Date.now() / 1000); - - // expired? - if ( expired ) - { - throw { message: 'Authentication failed. Signature expired.' }; - } - - const uid = query.get('uid'); - const secret = config.url_signature_secret; - - // before doing anything, see if this signature is valid for 'write' action, if yes that means every action is allowed - if ( !expired && query.get('signature') === sha256(`${uid}/write/${secret}/${query.get('expires')}`) ) - { - return true; - } - // if not, check specific actions - else if ( !expired && query.get('signature') === sha256(`${uid}/${action}/${secret}/${query.get('expires')}`) ) - { - return true; - } - // auth failed - else - { - throw { message: 'Authentication failed' }; - } -} - -export function get_url_from_req (req) { - return `${req.protocol }://${ req.get('host') }${req.originalUrl}`; -} - -/** - * Formats a number with grouped thousands. - * - * @param {number|string} number - The number to be formatted. If a string is provided, it must only contain numerical characters, plus and minus signs, and the letter 'E' or 'e' (for scientific notation). - * @param {number} decimals - The number of decimal points. If a non-finite number is provided, it defaults to 0. - * @param {string} [dec_point='.'] - The character used for the decimal point. Defaults to '.' if not provided. - * @param {string} [thousands_sep=','] - The character used for the thousands separator. Defaults to ',' if not provided. - * @returns {string} The formatted number with grouped thousands, using the specified decimal point and thousands separator characters. - * @throws {TypeError} If the `number` parameter cannot be converted to a finite number, or if the `decimals` parameter is non-finite and cannot be converted to an absolute number. - */ -export function number_format (number, decimals, dec_point, thousands_sep) { - // Strip all characters but numerical ones. - number = (`${number }`).replace(/[^0-9+\-Ee.]/g, ''); - let n = !isFinite(+number) ? 0 : +number, - prec = !isFinite(+decimals) ? 0 : Math.abs(decimals), - sep = (typeof thousands_sep === 'undefined') ? ',' : thousands_sep, - dec = (typeof dec_point === 'undefined') ? '.' : dec_point, - s = '', - toFixedFix = function (n, prec) { - const k = Math.pow(10, prec); - return `${ Math.round(n * k) / k}`; - }; - // Fix for IE parseFloat(0.55).toFixed(0) = 0; - s = (prec ? toFixedFix(n, prec) : `${ Math.round(n)}`).split('.'); - if ( s[0].length > 3 ) { - s[0] = s[0].replace(/\B(?=(?:\d{3})+(?!\d))/g, sep); - } - if ( (s[1] || '').length < prec ) { - s[1] = s[1] || ''; - s[1] += new Array(prec - s[1].length + 1).join('0'); - } - return s.join(dec); -} diff --git a/src/backend/src/loadTestConfig.js b/src/backend/src/loadTestConfig.js deleted file mode 100644 index e683d8871..000000000 --- a/src/backend/src/loadTestConfig.js +++ /dev/null @@ -1,5 +0,0 @@ -const config = require('./config.js'); - -module.exports = { - config, -}; \ No newline at end of file diff --git a/src/backend/src/middleware/abuse.js b/src/backend/src/middleware/abuse.js deleted file mode 100644 index 81c2987fc..000000000 --- a/src/backend/src/middleware/abuse.js +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../api/APIError'); -const config = require('../config'); -const { Context } = require('../util/context'); - -const abuse = options => (req, res, next) => { - if ( config.disable_abuse_checks ) { - next(); return; - } - - const requester = Context.get('requester'); - - if ( options.no_bots ) { - if ( requester.is_bot ) { - if ( options.shadow_ban_responder ) { - return options.shadow_ban_responder(req, res); - } - throw APIError.create('forbidden'); - } - } - - if ( options.puter_origin ) { - if ( ! requester.is_puter_origin() ) { - throw APIError.create('forbidden'); - } - } - - next(); -}; - -module.exports = abuse; diff --git a/src/backend/src/middleware/anticsrf.js b/src/backend/src/middleware/anticsrf.js deleted file mode 100644 index aafd0eb4f..000000000 --- a/src/backend/src/middleware/anticsrf.js +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const APIError = require('../api/APIError'); - -/** - * Creates an anti-CSRF middleware that validates CSRF tokens in incoming requests. - * This middleware protects against Cross-Site Request Forgery attacks by verifying - * that requests contain a valid anti-CSRF token in the request body. - * - * @param {Object} options - Configuration options for the middleware - * @returns {Function} Express middleware function that validates CSRF tokens - * - * @example - * // Apply anti-CSRF protection to a route - * app.post('/api/secure-endpoint', anticsrf(), (req, res) => { - * // Route handler code - * }); - */ -const anticsrf = options => async (req, res, next) => { - const svc_antiCSRF = req.services.get('anti-csrf'); - if ( ! req.body.anti_csrf ) { - const err = APIError.create('anti-csrf-incorrect'); - err.write(res); - return; - } - const has = await svc_antiCSRF.consume_token(req.user.uuid, req.body.anti_csrf); - if ( ! has ) { - const err = APIError.create('anti-csrf-incorrect'); - err.write(res); - return; - } - - next(); -}; - -module.exports = anticsrf; diff --git a/src/backend/src/middleware/auth.js b/src/backend/src/middleware/auth.js deleted file mode 100644 index 977eb8a31..000000000 --- a/src/backend/src/middleware/auth.js +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const APIError = require('../api/APIError'); -const { UserActorType } = require('../services/auth/Actor'); -const auth2 = require('./auth2'); - -const auth = async (req, res, next) => { - let auth2_ok = false; - try { - // Delegate to new middleware - await auth2(req, res, () => { - auth2_ok = true; - }); - if ( ! auth2_ok ) return; - - // Everything using the old reference to the auth middleware - // should only allow session tokens - if ( ! (req.actor.type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - next(); - } - // auth failed - catch (e) { - return res.status(401).send(e); - } -}; - -module.exports = auth; \ No newline at end of file diff --git a/src/backend/src/middleware/auth2.js b/src/backend/src/middleware/auth2.js deleted file mode 100644 index bcd93591c..000000000 --- a/src/backend/src/middleware/auth2.js +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const configurable_auth = require('./configurable_auth'); - -const auth2 = configurable_auth({ optional: false }); - -module.exports = auth2; diff --git a/src/backend/src/middleware/configurable_auth.js b/src/backend/src/middleware/configurable_auth.js deleted file mode 100644 index 9dfa27c3a..000000000 --- a/src/backend/src/middleware/configurable_auth.js +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../api/APIError'); -const config = require('../config'); -const { LegacyTokenError } = require('../services/auth/AuthService'); -const { AccessTokenActorType } = require('../services/auth/Actor'); -const { Context } = require('../util/context'); - -// The "/whoami" endpoint is a special case where we want to allow -// a legacy token to be used for authentication. The "/whoami" -// endpoint will then return a new token for further requests. -// -const is_whoami = (req) => { - if ( ! config.legacy_token_migrate ) return; - - if ( req.path !== '/whoami' ) return; - - // const subdomain = req.subdomains[res.subdomains.length - 1]; - // if ( subdomain !== 'api' ) return; - return true; -}; - -// TODO: Allow auth middleware to be used without requiring -// authentication. This will allow us to use the auth middleware -// in endpoints that do not require authentication, but can -// provide additional functionality if the user is authenticated. -const configurable_auth = options => async (req, res, next) => { - if ( options?.no_options_auth && req.method === 'OPTIONS' ) { - return next(); - } - - const optional = options?.optional; - const allow_cached_user = options?.allow_cached_user; - - // Request might already have been authed (PreAuthService) - if ( req.actor ) return next(); - - // === Getting the Token === - // This step came from jwt_auth in src/helpers.js - // However, since request-response handling is a concern of the - // auth middleware, it makes more sense to put it here. - - let token; - let tokenSource; - // Auth token in body - if ( req.body && req.body.auth_token ) - { - token = req.body.auth_token; - tokenSource = 'body'; - } - // HTTML Auth header - else if ( req.header && req.header('Authorization') && !req.header('Authorization').startsWith('Basic ') && req.header('Authorization') !== 'Bearer' ) { // Bearer with no space is something office does - token = req.header('Authorization'); - token = token.replace('Bearer ', '').trim(); - tokenSource = 'header'; - if ( token === 'undefined' ) { - APIError.create('unexpected_undefined', null, { - msg: 'The Authorization token cannot be the string "undefined"', - }); - } - } - // Cookie - else if ( req.cookies && req.cookies[config.cookie_name] ) - { - token = req.cookies[config.cookie_name]; - tokenSource = 'cookie'; - } - // Auth token in URL - else if ( req.query && req.query.auth_token ) - { - token = req.query.auth_token; - tokenSource = 'query'; - } - // Socket - else if ( req.handshake && req.handshake.query && req.handshake.query.auth_token ) - { - token = req.handshake.query.auth_token; - tokenSource = 'socket'; - } - - if ( !token || token.startsWith('Basic ') ) { - if ( optional ) { - next(); - return; - } - APIError.create('token_missing').write(res); - return; - } else if ( typeof token !== 'string' ) { - APIError.create('token_auth_failed').write(res); - return; - } else { - token = token.replace('Bearer ', ''); - } - - // === Delegate to AuthService === - // AuthService will attempt to authenticate the token and return - // an Actor object, which is a high-level representation of the - // entity that is making the request; it could be a user, an app - // acting on behalf of a user, or an app acting on behalf of itself. - - const context = Context.get(); - const services = context.get('services'); - const svc_auth = services.get('auth'); - - let actor; - try { - actor = await svc_auth.authenticate_from_token(token); - } catch ( e ) { - if ( e instanceof APIError ) { - e.write(res); - return; - } - if ( e instanceof LegacyTokenError && is_whoami(req) ) { - const new_info = await svc_auth.check_session(token, { - req, - from_upgrade: true, - }); - context.set('actor', new_info.actor); - context.set('user', new_info.user); - req.new_token = new_info.token; - req.token = new_info.token; - req.user = new_info.user; - req.actor = new_info.actor; - - if ( req.user?.suspended ) { - throw APIError.create('forbidden'); - } - - // Use session token in cookie so cookie-based requests have hasHttpOnlyCookie; client gets GUI token in response - res.cookie(config.cookie_name, new_info.session_token ?? new_info.token, { - sameSite: 'none', - secure: true, - httpOnly: true, - }); - next(); - return; - } - const re = APIError.create('token_auth_failed'); - re.write(res); - return; - } - - // === Populate Context === - context.set('actor', actor); - if ( actor.type.user ) { - if ( allow_cached_user === false ) { - const svc_getUser = services.get('get-user'); - actor.type.user = await svc_getUser.get_user({ id: actor.type.user.id, force: true }); - } - if ( actor.type.user?.suspended ) { - throw APIError.create('forbidden'); - } - context.set('user', actor.type.user); - } - if ( actor.type instanceof AccessTokenActorType ) { - // AccessTokenActorType has no .user; the effective user is the authorizer's user - const authorizerUser = actor.type.authorizer?.type?.user; - if ( authorizerUser?.suspended ) { - throw APIError.create('forbidden'); - } - } - - // === Populate Request === - req.actor = actor; - req.user = actor.type.user ?? (actor.type instanceof AccessTokenActorType ? actor.type.authorizer?.type?.user : undefined); - req.token = token; - - next(); -}; - -module.exports = configurable_auth; \ No newline at end of file diff --git a/src/backend/src/middleware/measure.js b/src/backend/src/middleware/measure.js deleted file mode 100644 index ce43bca15..000000000 --- a/src/backend/src/middleware/measure.js +++ /dev/null @@ -1,94 +0,0 @@ -const { pausing_tee } = require('../util/streamutil'); -const putility = require('@heyputer/putility'); - -const _intercept_req = ({ data, req, next }) => { - if ( ! req.readable ) { - return next(); - } - - try { - const [req_monitor, req_pass] = pausing_tee(req, 2); - - req_monitor.on('data', (chunk) => { - data.sz_incoming += chunk.length; - }); - - const replaces = ['readable', 'pipe', 'on', 'once', 'removeListener']; - for ( const replace of replaces ) { - const replacement = req_pass[replace]; - Object.defineProperty(req, replace, { - get () { - if ( typeof replacement === 'function' ) { - return replacement.bind(req_pass); - } - return replacement; - }, - }); - } - } catch (e) { - console.error(e); - return next(); - } -}; - -const _intercept_res = ({ data, res, next }) => { - if ( ! res.writable ) { - return next(); - } - - try { - const org_write = res.write; - const org_end = res.end; - - // Override the `write` method - res.write = function (chunk, ...args) { - if ( Buffer.isBuffer(chunk) ) { - data.sz_outgoing += chunk.length; - } else if ( typeof chunk === 'string' ) { - data.sz_outgoing += Buffer.byteLength(chunk); - } - return org_write.apply(res, [chunk, ...args]); - }; - - // Override the `end` method - res.end = function (chunk, ...args) { - if ( chunk ) { - if ( Buffer.isBuffer(chunk) ) { - data.sz_outgoing += chunk.length; - } else if ( typeof chunk === 'string' ) { - data.sz_outgoing += Buffer.byteLength(chunk); - } - } - const result = org_end.apply(res, [chunk, ...args]); - return result; - }; - } catch (e) { - console.error(e); - return next(); - } -}; - -function measure () { - return async (req, res, next) => { - const data = { - sz_incoming: 0, - sz_outgoing: 0, - }; - - _intercept_req({ data, req }); - _intercept_res({ data, res }); - - req.measurements = new putility.libs.promise.TeePromise(); - - // Wait for the request to finish processing - res.on('finish', () => { - req.measurements.resolve(data); - // console.log(`Incoming Data: ${data.sz_incoming} bytes`); - // console.log(`Outgoing Data: ${data.sz_outgoing} bytes`); // future - }); - - next(); - }; -} - -module.exports = measure; diff --git a/src/backend/src/middleware/subdomain.js b/src/backend/src/middleware/subdomain.js deleted file mode 100644 index d578c4aa6..000000000 --- a/src/backend/src/middleware/subdomain.js +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -/** - * This middleware checks the subdomain, and if the subdomain doesn't - * match it calls `next('route')` to skip the current route. - * Be sure to use this before any middleware that might erroneously - * block the request. - * - * @param {string|string[]} allowedSubdomains - The subdomain to allow; - * if an array, any of the subdomains in the array will be allowed. - * - * @returns {function} - An express middleware function - */ -const subdomain = allowedSubdomains => { - if ( ! Array.isArray(allowedSubdomains) ) { - allowedSubdomains = [allowedSubdomains]; - } - return async (req, res, next) => { - // Note: at the time of implementing this, there is a config - // option called `experimental_no_subdomain` that is designed - // to lie and tell us the subdomain is `api` when it's not. - const actual_subdomain = require('../helpers').subdomain(req); - if ( ! allowedSubdomains.includes(actual_subdomain) ) { - next('route'); - return; - } - - next(); - }; -}; - -module.exports = subdomain; diff --git a/src/backend/src/middleware/verified.js b/src/backend/src/middleware/verified.js deleted file mode 100644 index da66f91d2..000000000 --- a/src/backend/src/middleware/verified.js +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const config = require('../config'); - -const verified = async (req, res, next) => { - if ( ! config.strict_email_verification_required ) { - next(); - return; - } - - if ( ! req.user.requires_email_confirmation ) { - next(); - return; - } - - if ( req.user.email_confirmed ) { - next(); - return; - } - - res.status(400).send({ - code: 'account_is_not_verified', - message: 'Account is not verified', - }); -}; - -module.exports = verified; diff --git a/src/backend/src/modules/apps/AppIconService.js b/src/backend/src/modules/apps/AppIconService.js deleted file mode 100644 index c12366c85..000000000 --- a/src/backend/src/modules/apps/AppIconService.js +++ /dev/null @@ -1,856 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { createRequire } from 'node:module'; -import config from '../../config.js'; -import { APP_ICONS_SUBDOMAIN } from '../../consts/app-icons.js'; -import { HLWrite } from '../../deprecated/filesystem/hl_operations/hl_write.js'; -import { LLMkdir } from '../../deprecated/filesystem/ll_operations/ll_mkdir.js'; -import { LLRead } from '../../deprecated/filesystem/ll_operations/ll_read.js'; -import { NodePathSelector } from '../../deprecated/filesystem/node/selectors.js'; -import { get_app } from '../../helpers.js'; -import BaseService from '../../services/BaseService.js'; -import { DB_READ, DB_WRITE } from '../../services/database/consts.js'; -import eggspress from '../../api/eggspress.js'; -import { buffer_to_stream, stream_to_buffer } from '../../util/streamutil.js'; -import { AppRedisCacheSpace } from './AppRedisCacheSpace.js'; -import DEFAULT_APP_ICON from './default-app-icon.js'; - -const require = createRequire(import.meta.url); - -const ICON_SIZES = [16, 32, 64, 128, 256, 512]; -const DEFAULT_ICON_SIZE = 128; -const RAW_BASE64_REGEX = /^[A-Za-z0-9+/]+={0,2}$/; -const LEGACY_ICON_FILENAME = ({ appUid, size }) => `${appUid}-${size}.png`; -const ORIGINAL_ICON_FILENAME = ({ appUid }) => `${appUid}.png`; -const REDIRECT_MAX_AGE_SIZE = 15 * 60; // 15 min -const REDIRECT_MAX_AGE_ORIGINAL = 60; // 1 min - -/** - * AppIconService handles icon generation and serving for apps. - * - * This is done by listening to the `app.new-icon` event which is - * dispatched by AppES. `sharp` is used to resize the images to - * pre-selected sizees in the `ICON_SIZES` constant defined above. - * - * Icons are stored in and served from the `/system/app_icons` - * directory. If the system user does not have this directory, - * it will be created in the consolidation boot phase after - * UserService emits the `user.system-user-ready` event on the - * service container event bus. - */ -export class AppIconService extends BaseService { - static MODULES = { - sharp: require('sharp'), - bmp: require('sharp-bmp'), - ico: require('sharp-ico'), - uuidv4: require('uuid').v4, - }; - - static ICON_SIZES = ICON_SIZES; - - /** - * AppIconService listens to this event to register the - * endpoints /app-icon/:app_uid and /app-icon/:app_uid/:size - * which serve the app icon at the requested size. - */ - async '__on_install.routes' (_, { app }) { - const handler = async (req, res) => { - // Validate parameters - let { app_uid: appUid, size } = req.params; - const resolvedSize = Number(size ?? DEFAULT_ICON_SIZE); - if ( ! ICON_SIZES.includes(resolvedSize) ) { - res.status(400).send('Invalid size'); - return; - } - if ( ! appUid.startsWith('app-') ) { - appUid = `app-${appUid}`; - } - - const { - stream, - mime, - redirectUrl, - redirectCacheControl, - } = await this.#getIconStream({ - appUid, - size: resolvedSize, - allowRedirect: !this.config.no_subdomain, - }); - - if ( redirectUrl ) { - if ( redirectCacheControl ) { - res.set('Cache-Control', redirectCacheControl); - } - return res.redirect(302, redirectUrl); - } - - res.set('Content-Type', mime); - res.set('Cache-Control', 'public, max-age=3600'); - stream.pipe(res); - }; - - app.use(eggspress('/app-icon/:app_uid', { - allowedMethods: ['GET'], - }, handler)); - app.use(eggspress('/app-icon/:app_uid/:size', { - allowedMethods: ['GET'], - }, handler)); - } - - getSizes () { - return this.constructor.ICON_SIZES; - } - - async iconifyApps ({ apps, size }) { - return apps.map(app => { - const iconPath = this.getAppIconPath({ - appUid: app.uid ?? app.uuid, - size, - }); - if ( iconPath ) { - app.icon = iconPath; - } - return app; - }); - } - - getAppIconPath ({ appUid, size }) { - const normalizedAppUid = this.normalizeAppUid(appUid); - if ( typeof normalizedAppUid !== 'string' || !normalizedAppUid ) { - return null; - } - - const apiBaseUrl = String(config.api_base_url || '').replace(/\/+$/, ''); - if ( ! apiBaseUrl ) { - return null; - } - - const resolvedSize = Number(size ?? DEFAULT_ICON_SIZE); - if ( ! ICON_SIZES.includes(resolvedSize) ) { - return null; - } - - return `${apiBaseUrl}/app-icon/${normalizedAppUid}/${resolvedSize}`; - } - - getAppIconEndpointUrl ({ appUid }) { - const normalizedAppUid = this.normalizeAppUid(appUid); - if ( typeof normalizedAppUid !== 'string' || !normalizedAppUid ) { - return null; - } - - const apiBaseUrl = String(config.api_base_url || '').replace(/\/+$/, ''); - if ( ! apiBaseUrl ) { - return null; - } - - return `${apiBaseUrl}/app-icon/${normalizedAppUid}`; - } - - normalizeAppUid (appUid) { - if ( typeof appUid !== 'string' ) return appUid; - return appUid.startsWith('app-') ? appUid : `app-${appUid}`; - } - - isDataUrl (value) { - return ( - typeof value === 'string' && - value.startsWith('data:') && - value.includes(',') - ); - } - - isRawBase64ImageString (value) { - if ( typeof value !== 'string' ) return false; - const trimmed = value.trim(); - if ( !trimmed || trimmed.length < 16 ) return false; - if ( ! RAW_BASE64_REGEX.test(trimmed) ) return false; - if ( trimmed.length % 4 !== 0 ) return false; - - try { - const decoded = Buffer.from(trimmed, 'base64'); - if ( decoded.length === 0 ) return false; - const normalizedInput = trimmed.replace(/=+$/, ''); - const reencoded = decoded.toString('base64').replace(/=+$/, ''); - return normalizedInput === reencoded; - } catch { - return false; - } - } - - normalizeRawBase64ImageString (value) { - if ( typeof value !== 'string' ) return value; - const trimmed = value.trim(); - if ( ! this.isRawBase64ImageString(trimmed) ) return value; - return `data:image/png;base64,${trimmed}`; - } - - parseAppIconEndpointUrl (iconUrl) { - if ( typeof iconUrl !== 'string' || iconUrl.startsWith('data:') ) { - return null; - } - - let pathname; - try { - pathname = new URL(iconUrl, 'http://localhost').pathname; - } catch { - return null; - } - - const match = pathname.match(/^\/app-icon\/([^/]+)(?:\/(\d+))?\/?$/); - if ( ! match ) return null; - - const size = Number(match[2] ?? DEFAULT_ICON_SIZE); - - return { - appUid: this.normalizeAppUid(match[1]), - size, - }; - } - - isAppIconEndpointUrl (iconUrl) { - return !!this.parseAppIconEndpointUrl(iconUrl); - } - - isSameAppIconEndpointUrl ({ iconUrl, appUid, size }) { - const parsed = this.parseAppIconEndpointUrl(iconUrl); - if ( ! parsed ) return false; - return ( - parsed.appUid === this.normalizeAppUid(appUid) && - Number(parsed.size) === Number(size) - ); - } - - extractPuterSubdomainFromUrl (url) { - if ( typeof url !== 'string' ) return null; - - let hostname; - try { - hostname = (new URL(url)).hostname.toLowerCase(); - } catch { - return null; - } - - const hostingDomains = [ - config.static_hosting_domain, - config.static_hosting_domain_alt, - ].filter(Boolean).map(v => v.toLowerCase()); - - for ( const domain of hostingDomains ) { - const suffix = `.${domain}`; - if ( hostname.endsWith(suffix) ) { - const subdomain = hostname.slice(0, hostname.length - suffix.length); - return subdomain || null; - } - } - - return null; - } - - isPuterSubdomainUrl (url) { - return !!this.extractPuterSubdomainFromUrl(url); - } - - getAppIconsBaseUrl () { - if ( this.appIconsBaseUrl !== undefined ) { - return this.appIconsBaseUrl; - } - - const host = config.static_hosting_domain || config.static_hosting_domain_alt; - if ( ! host ) { - this.appIconsBaseUrl = null; - return this.appIconsBaseUrl; - } - - const protocol = config.protocol || 'https'; - - this.appIconsBaseUrl = `${protocol}://${APP_ICONS_SUBDOMAIN}.${host}`; - return this.appIconsBaseUrl; - } - - getSizedIconUrl ({ appUid, size }) { - const baseUrl = this.getAppIconsBaseUrl(); - if ( ! baseUrl ) return null; - - const normalizedAppUid = this.normalizeAppUid(appUid); - return `${baseUrl}/${LEGACY_ICON_FILENAME({ - appUid: normalizedAppUid, - size, - })}`; - } - - getOriginalIconUrl ({ appUid }) { - const baseUrl = this.getAppIconsBaseUrl(); - if ( ! baseUrl ) return null; - - const normalizedAppUid = this.normalizeAppUid(appUid); - return `${baseUrl}/${ORIGINAL_ICON_FILENAME({ - appUid: normalizedAppUid, - })}`; - } - - async ensureAppIconsDirectory ({ dirSystem = null } = {}) { - const svcFs = this.services.get('filesystem'); - const svcSu = this.services.get('su'); - const svcUser = this.services.get('user'); - return await svcSu.sudo(async () => { - const dirAppIcons = await svcFs.node(new NodePathSelector('/system/app_icons')); - if ( await dirAppIcons.exists() ) { - this.dir_app_icons = dirAppIcons; - return dirAppIcons; - } - - dirSystem = dirSystem || await svcUser.get_system_dir(); - if ( ! dirSystem ) { - dirSystem = await svcFs.node(new NodePathSelector('/system')); - } - if ( ! await dirSystem.exists() ) { - return dirAppIcons; - } - - const llMkdir = new LLMkdir(); - await llMkdir.run({ - parent: dirSystem, - name: 'app_icons', - actor: await svcSu.get_system_actor(), - }); - - this.dir_app_icons = dirAppIcons; - return dirAppIcons; - }); - } - - async getOriginalIconLookup ({ dirAppIcons, appUid }) { - const normalizedAppUid = this.normalizeAppUid(appUid); - const originalFilename = ORIGINAL_ICON_FILENAME({ appUid: normalizedAppUid }); - const flatOriginalNode = await dirAppIcons.getChild(originalFilename); - if ( await flatOriginalNode.exists() ) { - return { - node: flatOriginalNode, - isFlatOriginal: true, - }; - } - return { - node: null, - isFlatOriginal: false, - }; - } - - async ensureAppIconsSubdomain ({ dirAppIcons }) { - const dbSites = this.services.get('database').get(DB_WRITE, 'sites'); - const existing = await dbSites.read( - 'SELECT * FROM subdomains WHERE subdomain = ? LIMIT 1', - [APP_ICONS_SUBDOMAIN], - ); - if ( existing[0] ) return existing[0]; - - const svcSu = this.services.get('su'); - const systemUser = await svcSu.get_system_user(); - if ( ! systemUser?.id ) return null; - - const rootDirId = await dirAppIcons.get('mysql-id'); - await dbSites.write(`INSERT ${dbSites.case({ - mysql: 'IGNORE', - sqlite: 'OR IGNORE', - })} INTO subdomains (subdomain, user_id, root_dir_id, uuid) VALUES (?, ?, ?, ?)`, [ - APP_ICONS_SUBDOMAIN, - systemUser.id, - rootDirId, - `sd-${this.modules.uuidv4()}`, - ]); - - const rows = await dbSites.read( - 'SELECT * FROM subdomains WHERE subdomain = ? LIMIT 1', - [APP_ICONS_SUBDOMAIN], - ); - return rows[0] ?? null; - } - - async readIconNodeBuffer ({ node }) { - const svcSu = this.services.get('su'); - const llRead = new LLRead(); - const stream = await llRead.run({ - fsNode: node, - actor: await svcSu.get_system_actor(), - }); - return await stream_to_buffer(stream); - } - - async writePngToDir ({ destination_or_parent, filename, output }) { - const svcSu = this.services.get('su'); - const sysActor = await svcSu.get_system_actor(); - const hlWrite = new HLWrite(); - await hlWrite.run({ - destination_or_parent, - specified_name: filename, - overwrite: true, - actor: sysActor, - user: sysActor.type.user, - no_thumbnail: true, - file: { - size: output.length, - name: filename, - mimetype: 'image/png', - type: 'image/png', - stream: buffer_to_stream(output), - }, - }); - } - - shouldRedirectIconUrl ({ iconUrl, appUid, size }) { - if ( !iconUrl || this.isDataUrl(iconUrl) ) return false; - - const canRedirect = - this.isPuterSubdomainUrl(iconUrl) || - this.isAppIconEndpointUrl(iconUrl); - if ( ! canRedirect ) return false; - - return !this.isSameAppIconEndpointUrl({ - iconUrl, - appUid, - size, - }); - } - - async generateMissingSizeFromOriginal ({ appUid, size }) { - const normalizedAppUid = this.normalizeAppUid(appUid); - const dirAppIcons = await this.ensureAppIconsDirectory(); - if ( ! await dirAppIcons.exists() ) return; - const { node: originalNode } = await this.getOriginalIconLookup({ - dirAppIcons, - appUid: normalizedAppUid, - }); - if ( ! originalNode ) return; - - const sizedFilename = LEGACY_ICON_FILENAME({ - appUid: normalizedAppUid, - size, - }); - const sizedNode = await dirAppIcons.getChild(sizedFilename); - if ( await sizedNode.exists() ) return; - - const originalBuffer = await this.readIconNodeBuffer({ node: originalNode }); - const output = await this.modules.sharp(originalBuffer) - .resize(size) - .png() - .toBuffer(); - - await this.writePngToDir({ - destination_or_parent: dirAppIcons, - filename: sizedFilename, - output, - }); - } - - queueMissingSizeFromOriginal ({ appUid, size }) { - if ( ! this.pendingIconSizeJobs ) { - this.pendingIconSizeJobs = new Set(); - } - - const key = `${this.normalizeAppUid(appUid)}:${size}`; - if ( this.pendingIconSizeJobs.has(key) ) return; - - this.pendingIconSizeJobs.add(key); - Promise.resolve() - .then(async () => { - await this.generateMissingSizeFromOriginal({ appUid, size }); - }) - .catch(error => { - this.errors.report('AppIconService.queueMissingSizeFromOriginal', { - source: error, - appUid, - size, - }); - }) - .finally(() => { - this.pendingIconSizeJobs.delete(key); - }); - } - - queueDataUrlIconWrite ({ appUid, dataUrl }) { - const normalizedAppUid = this.normalizeAppUid(appUid); - if ( typeof normalizedAppUid !== 'string' || !normalizedAppUid ) return; - if ( ! this.isDataUrl(dataUrl) ) return; - - if ( ! this.pendingDataUrlIconWrites ) { - this.pendingDataUrlIconWrites = new Set(); - } - - const key = normalizedAppUid; - if ( this.pendingDataUrlIconWrites.has(key) ) return; - - this.pendingDataUrlIconWrites.add(key); - Promise.resolve() - .then(async () => { - const data = { - app_uid: normalizedAppUid, - data_url: dataUrl, - }; - await this.createAppIcons({ - data, - }); - if ( typeof data.url === 'string' && data.url ) { - await this.persistConvertedIconUrl({ - appUid: normalizedAppUid, - iconUrl: data.url, - }); - } - }) - .catch(error => { - this.errors?.report('AppIconService.queueDataUrlIconWrite', { - source: error, - appUid: normalizedAppUid, - }); - }) - .finally(() => { - this.pendingDataUrlIconWrites.delete(key); - }); - } - - async persistConvertedIconUrl ({ appUid, iconUrl }) { - const normalizedAppUid = this.normalizeAppUid(appUid); - if ( typeof normalizedAppUid !== 'string' || !normalizedAppUid ) return; - if ( typeof iconUrl !== 'string' || !iconUrl ) return; - - const svcDb = this.services.get('database'); - const dbWrite = svcDb.get(DB_WRITE, 'apps'); - await dbWrite.write( - 'UPDATE apps SET icon = ? WHERE uid = ? AND icon LIKE \'data:%\' LIMIT 1', - [iconUrl, normalizedAppUid], - ); - - const dbRead = svcDb.get(DB_READ, 'apps'); - const rows = await dbRead.read( - 'SELECT id, uid, name FROM apps WHERE uid = ? LIMIT 1', - [normalizedAppUid], - ); - const app = rows[0]; - if ( app ) { - AppRedisCacheSpace.invalidateCachedApp(app); - } else { - AppRedisCacheSpace.invalidateCachedApp({ uid: normalizedAppUid }); - } - - const svcEvent = this.services.get('event'); - await svcEvent.emit('app.changed', { - app_uid: normalizedAppUid, - action: 'icon-migrated', - }); - } - - async #getIconStream ({ appIcon, appUid, size, tries = 0, allowRedirect = false }) { - appUid = this.normalizeAppUid(appUid); - const appIconOriginal = appIcon; - - if ( appIcon && !this.isDataUrl(appIcon) ) { - appIcon = null; - } - - // If there is an icon provided, and it's an SVG, we'll just return it - if ( appIcon ) { - const [metadata, data] = appIcon.split(','); - const inputMime = metadata.split(';')[0].split(':')[1]; - - // svg icons will be sent as-is - if ( inputMime === 'image/svg+xml' ) { - return { - mime: 'image/svg+xml', - get stream () { - return buffer_to_stream(Buffer.from(data, 'base64')); - }, - dataUrl: appIcon, - data_url: appIcon, - }; - } - } - - let app; - const getAppCached = async () => { - if ( app !== undefined ) return app; - app = await get_app({ uid: appUid }); - return app; - }; - - const getFallbackIcon = async () => { - const app = await getAppCached(); - const dbIcon = this.normalizeRawBase64ImageString(app?.icon); - - let fallbackIcon = appIcon || dbIcon || DEFAULT_APP_ICON; - if ( ! this.isDataUrl(fallbackIcon) ) { - fallbackIcon = DEFAULT_APP_ICON; - } - - if ( this.isDataUrl(dbIcon) && fallbackIcon === dbIcon ) { - this.queueDataUrlIconWrite({ - appUid, - dataUrl: dbIcon, - }); - } - - const [metadata, base64] = fallbackIcon.split(','); - const mime = metadata.split(';')[0].split(':')[1]; - const img = Buffer.from(base64, 'base64'); - return { - mime, - stream: buffer_to_stream(img), - }; - }; - - const getExternalRedirect = async () => { - if ( ! allowRedirect ) return null; - - const appIconUrl = this.shouldRedirectIconUrl({ - iconUrl: appIconOriginal, - appUid, - size, - }) ? appIconOriginal : null; - - let dbIcon; - if ( ! appIconUrl ) { - dbIcon = (await getAppCached())?.icon; - } - - const redirectUrl = [appIconUrl, dbIcon].find(url => this.shouldRedirectIconUrl({ - iconUrl: url, - appUid, - size, - })); - - if ( ! redirectUrl ) return null; - return { redirectUrl }; - }; - - const dirAppIcons = await this.getAppIcons(); - const legacyFilename = LEGACY_ICON_FILENAME({ appUid, size }); - const legacyNode = await dirAppIcons.getChild(legacyFilename); - - if ( await legacyNode.exists() ) { - if ( allowRedirect ) { - const redirectUrl = this.getSizedIconUrl({ appUid, size }); - if ( redirectUrl ) { - return { - redirectUrl, - redirectCacheControl: `public, max-age=${REDIRECT_MAX_AGE_SIZE}`, - }; - } - } - - try { - const output = await this.readIconNodeBuffer({ node: legacyNode }); - return { - mime: 'image/png', - stream: buffer_to_stream(output), - }; - } catch (e) { - this.errors.report('AppIconService.get_icon_stream', { - source: e, - }); - if ( tries < 1 ) { - // Choose the next size up, or 256 if we're already at 512. - const secondSize = size < 512 ? size * 2 : 256; - return await this.#getIconStream({ - appUid, - appIcon: appIconOriginal, - size: secondSize, - tries: tries + 1, - allowRedirect, - }); - } - } - } - - const { - node: originalNode, - isFlatOriginal, - } = await this.getOriginalIconLookup({ dirAppIcons, appUid }); - const hasOriginal = !!originalNode; - - if ( hasOriginal ) { - this.queueMissingSizeFromOriginal({ appUid, size }); - - if ( allowRedirect && isFlatOriginal ) { - const redirectUrl = this.getOriginalIconUrl({ appUid }); - if ( redirectUrl ) { - return { - redirectUrl, - redirectCacheControl: `public, max-age=${REDIRECT_MAX_AGE_ORIGINAL}`, - }; - } - } - - try { - const output = await this.readIconNodeBuffer({ node: originalNode }); - return { - mime: 'image/png', - stream: buffer_to_stream(output), - }; - } catch (e) { - this.errors.report('AppIconService.get_icon_stream:original-read', { - source: e, - }); - } - } - - return await getExternalRedirect() || await getFallbackIcon(); - } - - /** - * Returns an FSNodeContext instance for the app icons - * directory. - */ - async getAppIcons () { - if ( this.dir_app_icons ) { - return this.dir_app_icons; - } - - const svcFs = this.services.get('filesystem'); - const dirAppIcons = await svcFs.node(new NodePathSelector('/system/app_icons')); - - return this.dir_app_icons = dirAppIcons; - } - - getSharp ({ metadata, input }) { - const type = metadata.split(';')[0].split(':')[1]; - - if ( type === 'image/bmp' ) { - return this.modules.bmp.sharpFromBmp(input); - } - - const icotypes = ['image/x-icon', 'image/vnd.microsoft.icon']; - if ( icotypes.includes(type) ) { - const sharps = this.modules.ico.sharpsFromIco(input); - return sharps[0]; - } - - return this.modules.sharp(input); - } - - async loadIconSource ({ iconUrl }) { - if ( typeof iconUrl !== 'string' || !iconUrl ) { - return null; - } - - iconUrl = this.normalizeRawBase64ImageString(iconUrl); - - if ( iconUrl.startsWith('data:') ) { - const [metadata, base64] = iconUrl.split(','); - return { - metadata, - input: Buffer.from(base64, 'base64'), - }; - } - - try { - const response = await fetch(iconUrl); - if ( ! response.ok ) { - throw new Error(`HTTP error! status: ${response.status}`); - } - - return { - input: Buffer.from(await response.arrayBuffer()), - metadata: `data:${response.headers.get('content-type') || 'image/png'};base64`, - }; - } catch ( error ) { - this.errors.report('AppIconService.createAppIcons:fetchUrl', { - source: error, - iconUrl, - }); - return null; - } - } - - /** - * AppIconService listens to this event to create the - * `/system/app_icons` directory if it does not exist, - * and then to register the event listener for `app.new-icon`. - */ - async '__on_user.system-user-ready' () { - const svcSu = this.services.get('su'); - const svcUser = this.services.get('user'); - - const dirSystem = await svcUser.get_system_dir(); - - // Ensure app icons directory exists - await svcSu.sudo(async () => { - const dirAppIcons = await this.ensureAppIconsDirectory({ dirSystem }); - await this.ensureAppIconsSubdomain({ dirAppIcons }); - }); - - // Listen for new app icons - const svcEvent = this.services.get('event'); - svcEvent.on('app.new-icon', async (_, data) => { - await this.createAppIcons({ data }); - }); - } - - async createAppIcons ({ data }) { - const svcSu = this.services.get('su'); - const dataUrl = data.dataUrl ?? data.data_url; - const appUid = this.normalizeAppUid(data.appUid ?? data.app_uid); - if ( !dataUrl || !appUid ) return; - - const source = await this.loadIconSource({ iconUrl: dataUrl }); - if ( ! source ) return; - - const { input, metadata } = source; - const isInputDataUrl = this.isDataUrl(dataUrl); - - await svcSu.sudo(async () => { - const dirAppIcons = await this.ensureAppIconsDirectory(); - if ( ! await dirAppIcons.exists() ) { - throw new Error('app icons directory is missing'); - } - - const sharpInstance = this.getSharp({ metadata, input }); - - if ( isInputDataUrl ) { - const originalOutput = await sharpInstance.clone() - .png() - .toBuffer(); - await this.writePngToDir({ - destination_or_parent: dirAppIcons, - filename: ORIGINAL_ICON_FILENAME({ appUid }), - output: originalOutput, - }); - - const endpointUrl = this.getAppIconEndpointUrl({ appUid }); - if ( endpointUrl ) { - data.url = endpointUrl; - } - } - - const iconJobs = ICON_SIZES.map(async size => { - const output = await sharpInstance.clone() - .resize(size) - .png() - .toBuffer(); - await this.writePngToDir({ - destination_or_parent: dirAppIcons, - filename: LEGACY_ICON_FILENAME({ appUid, size }), - output, - }); - }); - await Promise.all(iconJobs); - }); - } - - async _init () { - } -} diff --git a/src/backend/src/modules/apps/AppIconService.test.js b/src/backend/src/modules/apps/AppIconService.test.js deleted file mode 100644 index 0e927cf36..000000000 --- a/src/backend/src/modules/apps/AppIconService.test.js +++ /dev/null @@ -1,185 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import config from '../../config.js'; -import { AppIconService } from './AppIconService.js'; - -describe('AppIconService', () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - afterEach(() => { - vi.restoreAllMocks(); - vi.unstubAllGlobals(); - }); - - describe('URL helpers', () => { - it('extracts a puter subdomain from a static hosting URL', () => { - const service = Object.create(AppIconService.prototype); - // TODO: We might need a better way to do this. A service with no - // initialization is difficult to test. - service.config = {}; - const domain = 'site.puter.localhost:4100'; - config.load_config({ - static_hosting_domain: domain, - static_hosting_domain_alt: 'site.puter.localhost', - }); - - const result = service.extractPuterSubdomainFromUrl(`https://dev-center-app-id.${domain}/icon.png`); - - expect(result).toBe('dev-center-app-id'); - }); - - it('does not redirect when URL is the same app-icon endpoint request', () => { - const service = Object.create(AppIconService.prototype); - - const shouldRedirect = service.shouldRedirectIconUrl({ - iconUrl: 'https://api.puter.localhost/app-icon/app-123/64', - appUid: 'app-123', - size: 64, - }); - - expect(shouldRedirect).toBe(false); - }); - - it('parses app-icon endpoint URLs without size as default size 128', () => { - const service = Object.create(AppIconService.prototype); - - const parsed = service.parseAppIconEndpointUrl('https://api.puter.localhost/app-icon/app-123'); - - expect(parsed).toEqual({ - appUid: 'app-123', - size: 128, - }); - }); - - it('normalizes raw base64 icon strings to png data URLs', () => { - const service = Object.create(AppIconService.prototype); - const rawBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ'; - - const result = service.normalizeRawBase64ImageString(rawBase64); - - expect(result).toBe(`data:image/png;base64,${rawBase64}`); - }); - }); - - describe('createAppIcons', () => { - it('stores original and resized icons in /system/app_icons for data URLs', async () => { - const sudo = vi.fn(async callback => await callback()); - const dirAppIcons = { - exists: vi.fn().mockResolvedValue(true), - }; - - const service = Object.create(AppIconService.prototype); - service.services = { - get: vi.fn(name => (name === 'su' ? { sudo } : null)), - }; - service.errors = { report: vi.fn() }; - service.ensureAppIconsDirectory = vi.fn().mockResolvedValue(dirAppIcons); - service.getAppIconEndpointUrl = vi.fn().mockReturnValue('https://api.puter.localhost/app-icon/app-abc'); - service.loadIconSource = vi.fn().mockResolvedValue({ - metadata: 'data:image/png;base64', - input: Buffer.from([1, 2, 3]), - }); - service.writePngToDir = vi.fn().mockResolvedValue(undefined); - service.getSharp = vi.fn(() => ({ - clone: vi.fn(() => ({ - resize: vi.fn().mockReturnThis(), - png: vi.fn().mockReturnThis(), - toBuffer: vi.fn().mockResolvedValue(Buffer.from([0x89, 0x50, 0x4e, 0x47])), - })), - })); - - const data = { - appUid: 'app-abc', - dataUrl: 'data:image/png;base64,AA==', - }; - - await service.createAppIcons({ data }); - - expect(service.writePngToDir).toHaveBeenCalledTimes(AppIconService.ICON_SIZES.length + 1); - expect(service.writePngToDir).toHaveBeenCalledWith(expect.objectContaining({ - destination_or_parent: dirAppIcons, - filename: 'app-abc.png', - })); - expect(service.writePngToDir).toHaveBeenCalledWith(expect.objectContaining({ - destination_or_parent: dirAppIcons, - filename: 'app-abc-64.png', - })); - expect(data.url).toBe('https://api.puter.localhost/app-icon/app-abc'); - }); - - it('queueDataUrlIconWrite persists migrated URL to DB when conversion succeeds', async () => { - const service = Object.create(AppIconService.prototype); - service.errors = { report: vi.fn() }; - service.createAppIcons = vi.fn(async ({ data }) => { - data.url = 'https://api.puter.localhost/app-icon/app-abc'; - }); - service.persistConvertedIconUrl = vi.fn().mockResolvedValue(undefined); - - service.queueDataUrlIconWrite({ - appUid: 'app-abc', - dataUrl: 'data:image/png;base64,AA==', - }); - - await Promise.resolve(); - await Promise.resolve(); - - expect(service.createAppIcons).toHaveBeenCalledTimes(1); - expect(service.persistConvertedIconUrl).toHaveBeenCalledWith({ - appUid: 'app-abc', - iconUrl: 'https://api.puter.localhost/app-icon/app-abc', - }); - }); - }); - - describe('icon URL mapping', () => { - it('builds a legacy app-icon path with normalized app uid', () => { - const service = Object.create(AppIconService.prototype); - - const result = service.getAppIconPath({ - appUid: 'abc', - size: 64, - }); - - expect(result).toBe(`${config.api_base_url}/app-icon/app-abc/64`); - }); - - it('defaults to size 128 when size is not provided', () => { - const service = Object.create(AppIconService.prototype); - - const result = service.getAppIconPath({ - appUid: 'abc', - }); - - expect(result).toBe(`${config.api_base_url}/app-icon/app-abc/128`); - }); - - it('iconifyApps rewrites icons to the legacy app-icon endpoint path', async () => { - const service = Object.create(AppIconService.prototype); - const apps = [ - { uid: 'app-abc', icon: 'data:image/png;base64,AA==' }, - { uuid: 'def', icon: 'https://example.com/icon.png' }, - ]; - - const result = await service.iconifyApps({ - apps, - size: 128, - }); - - expect(result[0].icon).toBe(`${config.api_base_url}/app-icon/app-abc/128`); - expect(result[1].icon).toBe(`${config.api_base_url}/app-icon/app-def/128`); - }); - - it('iconifyApps leaves icon unchanged when app uid is missing', async () => { - const service = Object.create(AppIconService.prototype); - const apps = [{ icon: 'existing-icon' }]; - - const result = await service.iconifyApps({ - apps, - size: 128, - }); - - expect(result[0].icon).toBe('existing-icon'); - }); - }); -}); diff --git a/src/backend/src/modules/apps/AppInformationService.js b/src/backend/src/modules/apps/AppInformationService.js deleted file mode 100644 index 7a18dcc77..000000000 --- a/src/backend/src/modules/apps/AppInformationService.js +++ /dev/null @@ -1,932 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { origin_from_url } = require('../../util/urlutil'); -const { DB_READ } = require('../../services/database/consts'); -const BaseService = require('../../services/BaseService'); -const { redisClient } = require('../../clients/redis/redisSingleton'); -const { deleteRedisKeys } = require('../../clients/redis/deleteRedisKeys.js'); -const { setRedisCacheValue } = require('../../clients/redis/cacheUpdate.js'); -const { AppRedisCacheSpace } = require('./AppRedisCacheSpace.js'); -const APP_UID_ALIAS_KEY_PREFIX = 'app:canonicalUidAlias'; -const APP_UID_ALIAS_REVERSE_KEY_PREFIX = 'app:canonicalUidAliasReverse'; - -/** -* @class AppInformationService -* @description -* The AppInformationService class manages application-related information, -* including caching, statistical data, and tags for applications within the Puter ecosystem. -* It provides methods for refreshing application data, managing app statistics, -* and handling tags associated with apps. This service is crucial for maintaining -* up-to-date information about applications, facilitating features like app listings -* and tag-based app discovery. -*/ -class AppInformationService extends BaseService { - static LOG_DEBUG = true; - - _construct () { - this.tags = {}; - - // MySQL date format mapping for different groupings - this.mysqlDateFormats = { - 'hour': '%Y-%m-%d %H:00:00', - 'day': '%Y-%m-%d', - 'week': '%Y-%U', - 'month': '%Y-%m', - 'year': '%Y', - }; - - // ClickHouse date format mapping for different groupings - this.clickhouseGroupByFormats = { - 'hour': 'toStartOfHour(fromUnixTimestamp(ts))', - 'day': 'toStartOfDay(fromUnixTimestamp(ts))', - 'week': 'toStartOfWeek(fromUnixTimestamp(ts))', - 'month': 'toStartOfMonth(fromUnixTimestamp(ts))', - 'year': 'toStartOfYear(fromUnixTimestamp(ts))', - }; - } - - '__on_boot.consolidation' () { - const svc_event = this.services.get('event'); - svc_event.on('app.rename', (_, { app_uid: appUid, old_name: oldName }) => { - this.invalidateAppCache({ appUid, oldName }).catch((e) => { - this.log.error('failed invalidating app cache after app.rename', { appUid, oldName, error: e }); - }); - }); - svc_event.on('app.changed', (_, { app_uid: appUid, app }) => { - this.invalidateAppCache({ appUid, app }).catch((e) => { - this.log.error('failed invalidating app cache after app.changed', { appUid, error: e }); - }); - }); - - (async () => { - try { - await this._refresh_app_stats(); - } catch (e) { - console.error('Some app cache portion failed to populate:', e); - } - setInterval(async () => { - try { - await this._refresh_app_stats(); - } catch (e) { - console.error('App stats cache failed to update:', e); - } - }, 13.314 * 60 * 1000); - })(); - } - - async invalidateAppCache ({ appUid, oldName, app }) { - let resolvedApp = app ?? null; - if ( !resolvedApp && appUid ) { - resolvedApp = await AppRedisCacheSpace.getCachedApp({ - lookup: 'uid', - value: appUid, - }); - } - if ( !resolvedApp && appUid ) { - const db = this.services.get('database').get(DB_READ, 'apps'); - resolvedApp = (await db.read( - 'SELECT id, uid, name FROM apps WHERE uid = ? LIMIT 1', - [appUid], - ))[0] ?? null; - } - - if ( resolvedApp ) { - await AppRedisCacheSpace.invalidateCachedApp(resolvedApp, { - includeStats: true, - }); - } else if ( appUid ) { - await Promise.all([ - deleteRedisKeys([ - AppRedisCacheSpace.key({ - lookup: 'uid', - value: appUid, - rawIcon: true, - }), - AppRedisCacheSpace.key({ - lookup: 'uid', - value: appUid, - rawIcon: false, - }), - AppRedisCacheSpace.objectKey({ - lookup: 'uid', - value: appUid, - }), - ]), - AppRedisCacheSpace.invalidateAppStats(appUid), - ]); - } - - if ( oldName ) { - await AppRedisCacheSpace.invalidateCachedAppName(oldName); - } - - const svc_event = this.services.get('event'); - await svc_event.emit('apps.invalidate', { - app: resolvedApp ?? app ?? { uid: appUid, name: oldName }, - }); - } - - /** - * Retrieves and returns statistical data for a specific application over different time periods. - * - * This method fetches various metrics such as the number of times the app has been opened, - * the count of unique users who have opened the app, and the number of referrals attributed to the app. - * It supports different time periods such as today, yesterday, past 7 days, past 30 days, and all time. - * - * @param {string} app_uid - The unique identifier for the application. - * @param {Object} [options] - Optional parameters to customize the query - * @param {string} [options.period='all'] - Time period for stats: 'today', 'yesterday', '7d', '30d', 'this_month', 'last_month', 'this_year', 'last_year', '12m', 'all' - * @param {string} [options.grouping=undefined] - Time grouping for stats: 'hour', 'day', 'week', 'month', 'year' - * @returns {Promise} An object containing: - * - {Object} open_count - Open counts for different time periods - * - {Object} user_count - Uniqu>e user counts for different time periods - * - {number|null} referral_count - The number of referrals (all-time only) - */ - async get_stats (app_uid, options = {}) { - let period = options.period ?? 'all'; - let stats_grouping = options.grouping; - let app_creation_ts = options.created_at; - const parse_cached_int = (value) => { - if ( value === null || value === undefined ) return null; - const parsed = parseInt(value, 10); - return Number.isNaN(parsed) ? null : parsed; - }; - - // Check cache first if period is 'all' and no grouping is requested - if ( period === 'all' && !stats_grouping ) { - const key_open_count = AppRedisCacheSpace.openCountKey(app_uid); - const key_user_count = AppRedisCacheSpace.userCountKey(app_uid); - const key_referral_count = AppRedisCacheSpace.referralCountKey(app_uid); - - const [cached_open_count, cached_user_count, cached_referral_count] = await Promise.all([ - redisClient.get(key_open_count), - redisClient.get(key_user_count), - redisClient.get(key_referral_count), - ]); - - const cached_open_count_parsed = parse_cached_int(cached_open_count); - const cached_user_count_parsed = parse_cached_int(cached_user_count); - if ( cached_open_count_parsed !== null && cached_user_count_parsed !== null ) { - return { - open_count: cached_open_count_parsed, - user_count: cached_user_count_parsed, - referral_count: parse_cached_int(cached_referral_count), - }; - } - } - - const db = this.services.get('database').get(DB_READ, 'apps'); - - const getTimeRange = (period) => { - const now = new Date(); - const today = new Date(now.getFullYear(), now.getMonth(), now.getDate()); - - switch ( period ) { - case 'today': - return { - start: today.getTime(), - end: now.getTime(), - }; - case 'yesterday': { - const yesterday = new Date(today); - yesterday.setDate(yesterday.getDate() - 1); - return { - start: yesterday.getTime(), - end: today.getTime() - 1, - }; - } - case '7d': { - const weekAgo = new Date(now); - weekAgo.setDate(weekAgo.getDate() - 7); - return { - start: weekAgo.getTime(), - end: now.getTime(), - }; - } - case '30d': { - const monthAgo = new Date(now); - monthAgo.setDate(monthAgo.getDate() - 30); - return { - start: monthAgo.getTime(), - end: now.getTime(), - }; - } - case 'this_week': { - const firstDayOfWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - now.getDay()); - return { - start: firstDayOfWeek.getTime(), - end: now.getTime(), - }; - } - case 'last_week': { - const firstDayOfLastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - now.getDay() - 7); - const firstDayOfThisWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - now.getDay()); - return { - start: firstDayOfLastWeek.getTime(), - end: firstDayOfThisWeek.getTime() - 1, - }; - } - case 'this_month': { - const firstDayOfMonth = new Date(now.getFullYear(), now.getMonth(), 1); - return { - start: firstDayOfMonth.getTime(), - end: now.getTime(), - }; - } - case 'last_month': { - const firstDayOfLastMonth = new Date(now.getFullYear(), now.getMonth() - 1, 1); - const firstDayOfThisMonth = new Date(now.getFullYear(), now.getMonth(), 1); - return { - start: firstDayOfLastMonth.getTime(), - end: firstDayOfThisMonth.getTime() - 1, - }; - } - case 'this_year': { - const firstDayOfYear = new Date(now.getFullYear(), 0, 1); - return { - start: firstDayOfYear.getTime(), - end: now.getTime(), - }; - } - case 'last_year': { - const firstDayOfLastYear = new Date(now.getFullYear() - 1, 0, 1); - const firstDayOfThisYear = new Date(now.getFullYear(), 0, 1); - return { - start: firstDayOfLastYear.getTime(), - end: firstDayOfThisYear.getTime() - 1, - }; - } - case '12m': { - const twelveMonthsAgo = new Date(now); - twelveMonthsAgo.setMonth(twelveMonthsAgo.getMonth() - 12); - return { - start: twelveMonthsAgo.getTime(), - end: now.getTime(), - }; - } - case 'all': { - const start = new Date(app_creation_ts); - return { - start: start.getTime(), - end: now.getTime(), - }; - } - default: - return null; - } - }; - - const timeRange = getTimeRange(period); - - // Handle time-based grouping if stats_grouping is specified - if ( stats_grouping ) { - const timeFormat = this.mysqlDateFormats[stats_grouping]; - if ( ! timeFormat ) { - throw new Error(`Invalid stats_grouping: ${stats_grouping}. Supported values are: hour, day, week, month, year`); - } - - // Generate all periods for the time range - const allPeriods = this.generateAllPeriods( - new Date(timeRange.start), - new Date(timeRange.end), - stats_grouping, - ); - - if ( global.clickhouseClient ) { - const groupByFormat = this.clickhouseGroupByFormats[stats_grouping]; - const timeCondition = timeRange ? - `AND ts >= ${Math.floor(timeRange.start / 1000)} AND ts < ${Math.floor(timeRange.end / 1000)}` : ''; - - const [openResult, userResult] = await Promise.all([ - global.clickhouseClient.query({ - query: ` - SELECT - ${groupByFormat} as period, - COUNT(_id) as count - FROM app_opens - WHERE app_uid = '${app_uid}' - ${timeCondition} - GROUP BY period - ORDER BY period - `, - format: 'JSONEachRow', - }), - global.clickhouseClient.query({ - query: ` - SELECT - ${groupByFormat} as period, - COUNT(DISTINCT user_id) as count - FROM app_opens - WHERE app_uid = '${app_uid}' - ${timeCondition} - GROUP BY period - ORDER BY period - `, - format: 'JSONEachRow', - }), - ]); - - const openRows = await openResult.json(); - const userRows = await userResult.json(); - - // Ensure counts are properly parsed as integers - const processedOpenRows = openRows.map(row => ({ - period: new Date(row.period), - count: parseInt(row.count), - })); - - const processedUserRows = userRows.map(row => ({ - period: new Date(row.period), - count: parseInt(row.count), - })); - - // Calculate totals from the processed rows - const totalOpenCount = processedOpenRows.reduce((sum, row) => sum + row.count, 0); - const totalUserCount = processedUserRows.reduce((sum, row) => sum + row.count, 0); - - // Generate all periods and merge with actual data - const allPeriods = this.generateAllPeriods( - new Date(timeRange.start), - new Date(timeRange.end), - stats_grouping, - ); - - const completeOpenStats = this.mergeWithGeneratedPeriods(processedOpenRows, allPeriods, stats_grouping); - const completeUserStats = this.mergeWithGeneratedPeriods(processedUserRows, allPeriods, stats_grouping); - - return { - open_count: totalOpenCount, - user_count: totalUserCount, - grouped_stats: { - open_count: completeOpenStats, - user_count: completeUserStats, - }, - referral_count: period === 'all' - ? parse_cached_int(await redisClient.get(AppRedisCacheSpace.referralCountKey(app_uid))) - : null, - }; - } - - else { - // MySQL queries for grouped stats - const queryParams = timeRange ? - [app_uid, timeRange.start / 1000, timeRange.end / 1000] : - [app_uid]; - - const [openResult, userResult] = await Promise.all([ - db.read(` - SELECT ${db.case({ - mysql: `DATE_FORMAT(FROM_UNIXTIME(ts/1000), '${timeFormat}') as period, `, - sqlite: `STRFTIME('%Y-%m-%d %H', datetime(ts/1000, 'unixepoch'), '${timeFormat}') as period, `, - }) - } - COUNT(_id) as count - FROM app_opens - WHERE app_uid = ? - ${timeRange ? 'AND ts >= ? AND ts < ?' : ''} - GROUP BY period - ORDER BY period - `, queryParams), - db.read(` - SELECT ${db.case({ - mysql: `DATE_FORMAT(FROM_UNIXTIME(ts/1000), '${timeFormat}') as period, `, - sqlite: `STRFTIME('%Y-%m-%d %H', datetime(ts/1000, 'unixepoch'), '${timeFormat}') as period, `, - }) - } - COUNT(DISTINCT user_id) as count - FROM app_opens - WHERE app_uid = ? - ${timeRange ? 'AND ts >= ? AND ts < ?' : ''} - GROUP BY period - ORDER BY period - `, queryParams), - ]); - - // Calculate totals - const totalOpenCount = openResult.reduce((sum, row) => sum + parseInt(row.count), 0); - const totalUserCount = userResult.reduce((sum, row) => sum + parseInt(row.count), 0); - - // Convert MySQL results to the same format as needed - const openRows = openResult.map(row => ({ - period: row.period, - count: parseInt(row.count), - })); - const userRows = userResult.map(row => ({ - period: row.period, - count: parseInt(row.count), - })); - - // Merge with generated periods to include zero-value periods - const completeOpenStats = this.mergeWithGeneratedPeriods(openRows, allPeriods, stats_grouping); - const completeUserStats = this.mergeWithGeneratedPeriods(userRows, allPeriods, stats_grouping); - - return { - open_count: totalOpenCount, - user_count: totalUserCount, - grouped_stats: { - open_count: completeOpenStats, - user_count: completeUserStats, - }, - referral_count: period === 'all' - ? parse_cached_int(await redisClient.get(AppRedisCacheSpace.referralCountKey(app_uid))) - : null, - }; - } - } - - // Handle non-grouped stats - if ( global.clickhouseClient ) { - const openCountQuery = timeRange - ? `SELECT COUNT(_id) AS open_count FROM app_opens - WHERE app_uid = '${app_uid}' - AND ts >= ${Math.floor(timeRange.start / 1000)} - AND ts < ${Math.floor(timeRange.end / 1000)}` - : `SELECT COUNT(_id) AS open_count FROM app_opens - WHERE app_uid = '${app_uid}'`; - - const userCountQuery = timeRange - ? `SELECT COUNT(DISTINCT user_id) AS uniqueUsers FROM app_opens - WHERE app_uid = '${app_uid}' - AND ts >= ${Math.floor(timeRange.start / 1000)} - AND ts < ${Math.floor(timeRange.end / 1000)}` - : `SELECT COUNT(DISTINCT user_id) AS uniqueUsers FROM app_opens - WHERE app_uid = '${app_uid}'`; - - const [openResult, userResult] = await Promise.all([ - global.clickhouseClient.query({ - query: openCountQuery, - format: 'JSONEachRow', - }), - global.clickhouseClient.query({ - query: userCountQuery, - format: 'JSONEachRow', - }), - ]); - - const openRows = await openResult.json(); - const userRows = await userResult.json(); - - const results = { - open_count: parseInt(openRows[0].open_count), - user_count: parseInt(userRows[0].uniqueUsers), - referral_count: period === 'all' - ? parse_cached_int(await redisClient.get(AppRedisCacheSpace.referralCountKey(app_uid))) - : null, - }; - - // Cache the results if period is 'all' - if ( period === 'all' ) { - const key_open_count = AppRedisCacheSpace.openCountKey(app_uid); - const key_user_count = AppRedisCacheSpace.userCountKey(app_uid); - void Promise.all([ - setRedisCacheValue(key_open_count, results.open_count), - setRedisCacheValue(key_user_count, results.user_count), - ]); - } - - return results; - } else { - // Regular MySQL queries for non-grouped stats - const baseOpenQuery = 'SELECT COUNT(_id) AS open_count FROM app_opens WHERE app_uid = ?'; - const baseUserQuery = 'SELECT COUNT(DISTINCT user_id) AS user_count FROM app_opens WHERE app_uid = ?'; - - const generateQuery = (baseQuery, timeRange) => { - if ( ! timeRange ) return baseQuery; - return `${baseQuery} AND ts >= ? AND ts < ?`; - }; - - const openQuery = generateQuery(baseOpenQuery, timeRange); - const userQuery = generateQuery(baseUserQuery, timeRange); - const queryParams = timeRange ? [app_uid, timeRange.start, timeRange.end] : [app_uid]; - - const [openResult, userResult] = await Promise.all([ - db.read(openQuery, queryParams), - db.read(userQuery, queryParams), - ]); - - const results = { - open_count: parseInt(openResult[0].open_count), - user_count: parseInt(userResult[0].user_count), - referral_count: period === 'all' - ? parse_cached_int(await redisClient.get(AppRedisCacheSpace.referralCountKey(app_uid))) - : null, - }; - - // Cache the results if period is 'all' - if ( period === 'all' ) { - const key_open_count = AppRedisCacheSpace.openCountKey(app_uid); - const key_user_count = AppRedisCacheSpace.userCountKey(app_uid); - void Promise.all([ - setRedisCacheValue(key_open_count, results.open_count), - setRedisCacheValue(key_user_count, results.user_count), - ]); - } - - return results; - } - } - - /** - * Refreshes the cache of app statistics including open and user counts. - * - * @notes - * - This method logs a tick event for performance monitoring. - * - * @async - * @returns {Promise} A promise that resolves when the cache refresh operation is complete. - */ - async _refresh_app_stats () { - this.log.tick('refresh app stats'); - - const db = this.services.get('database').get(DB_READ, 'apps'); - - let openCountMap; - let userCountMap; - - if ( global.clickhouseClient ) { - const [openResult, userResult] = await Promise.all([ - global.clickhouseClient.query({ - query: ` - SELECT app_uid, COUNT(_id) AS open_count - FROM app_opens - GROUP BY app_uid - `, - format: 'JSONEachRow', - }), - global.clickhouseClient.query({ - query: ` - SELECT app_uid, COUNT(DISTINCT user_id) AS user_count - FROM app_opens - GROUP BY app_uid - `, - format: 'JSONEachRow', - }), - ]); - const openRows = await openResult.json(); - const userRows = await userResult.json(); - openCountMap = new Map(openRows.map(row => [row.app_uid, parseInt(row.open_count, 10)])); - userCountMap = new Map(userRows.map(row => [row.app_uid, parseInt(row.user_count, 10)])); - } else { - const [openCounts, userCounts] = await Promise.all([ - db.read(` - SELECT app_uid, COUNT(_id) AS open_count - FROM app_opens - GROUP BY app_uid - `), - db.read(` - SELECT app_uid, COUNT(DISTINCT user_id) AS user_count - FROM app_opens - GROUP BY app_uid - `), - ]); - openCountMap = new Map(openCounts.map(row => [row.app_uid, row.open_count])); - userCountMap = new Map(userCounts.map(row => [row.app_uid, row.user_count])); - } - - // Get all app UIDs and update the cache (apps list lives in MySQL) - const apps = await db.read('SELECT uid FROM apps'); - - for ( const app of apps ) { - const key_open_count = AppRedisCacheSpace.openCountKey(app.uid); - const key_user_count = AppRedisCacheSpace.userCountKey(app.uid); - - // Background refresh writes should stay local to avoid broadcast churn. - void Promise.all([ - setRedisCacheValue(key_open_count, openCountMap.get(app.uid) ?? 0, { emitEvent: false }), - setRedisCacheValue(key_user_count, userCountMap.get(app.uid) ?? 0, { emitEvent: false }), - ]); - } - } - - /** - * Refreshes the cache of app referral statistics. - * - * This method queries the database for user counts referred by each app's origin URL - * and updates the cache with the referral counts for each app. - * - * @notes - * - This method logs a tick event for performance monitoring. - * - * @async - * @returns {Promise} A promise that resolves when the cache refresh operation is complete. - */ - async _refresh_app_stat_referrals () { - this.log.tick('refresh app stat referrals'); - - const db = this.services.get('database').get(DB_READ, 'apps'); - - const apps = await db.read('SELECT uid, index_url FROM apps'); - - // First, build a map of valid app origins to UIDs - const validApps = []; - const svc_auth = this.services.get('auth'); - - for ( const app of apps ) { - const origin = origin_from_url(app.index_url); - - // only count the referral if the origin hashes to the app's uid - let expected_uid; - try { - expected_uid = await svc_auth.app_uid_from_origin(origin); - } catch (e) { - // This happens if the app origin isn't valid - continue; - } - if ( expected_uid !== app.uid ) { - continue; - } - - validApps.push({ uid: app.uid, origin }); - } - - if ( validApps.length === 0 ) { - return; - } - - // Build a single query to get all referral counts - const likeConditions = validApps.map(() => 'referrer LIKE ?').join(' OR '); - const queryParams = validApps.map(app => `${app.origin}%`); - - const referralResults = await db.read(` - SELECT - referrer, - COUNT(id) as referral_count - FROM user - WHERE ${likeConditions} - GROUP BY referrer - `, queryParams); - - // Create a map to store referral counts by origin - const referralMap = new Map(); - - for ( const result of referralResults ) { - // Find which app this referrer belongs to - for ( const app of validApps ) { - if ( result.referrer.startsWith(app.origin) ) { - const currentCount = referralMap.get(app.uid) || 0; - referralMap.set(app.uid, currentCount + parseInt(result.referral_count)); - break; - } - } - } - - // Update cache with results - for ( const app of validApps ) { - const key_referral_count = AppRedisCacheSpace.referralCountKey(app.uid); - const count = referralMap.get(app.uid) || 0; - // Background refresh writes should stay local to avoid broadcast churn. - await setRedisCacheValue(key_referral_count, count, { emitEvent: false }); - } - - this.log.info('DONE refresh app stat referrals'); - } - - /** - * Deletes an application from the system. - * - * This method performs the following actions: - * - Retrieves the app data from cache or database if not provided. - * - Deletes the app record from the database. - * - Removes the app from all relevant caches (by name, id, and uid). - * - Removes the app from any associated tags. - * - * @param {string} app_uid - The unique identifier of the app to be deleted. - * @param {Object} [app] - The app object, if already fetched. If not provided, it will be retrieved. - * @param {Object} [options] - Optional delete behavior flags. - * @throws {Error} If the app is not found in either cache or database. - * @returns {Promise} A promise that resolves when the app has been successfully deleted. - */ - async delete_app (app_uid, app, options = {}) { - const db = this.services.get('database').get(DB_READ, 'apps'); - - if ( ! app ) { - app = await AppRedisCacheSpace.getCachedApp({ - lookup: 'uid', - value: app_uid, - }); - } - if ( ! app ) { - app = (await db.read( - 'SELECT * FROM apps WHERE uid = ?', - [app_uid], - ))[0]; - } - - if ( ! app ) { - throw new Error('app not found'); - } - - const associationRows = await db.read( - 'SELECT type FROM app_filetype_association WHERE app_id = ?', - [app.id], - ); - - await db.write( - 'DELETE FROM apps WHERE uid = ? LIMIT 1', - [app_uid], - ); - - if ( ! options.preserveCanonicalUidAlias ) { - await this.cleanupCanonicalAppUidAliases_(app_uid); - } - - // remove from caches - AppRedisCacheSpace.invalidateCachedApp(app, { - includeStats: true, - }); - const associationKeys = associationRows - .map(row => String(row.type ?? '').trim().toLowerCase().replace(/^\./, '')) - .filter(Boolean) - .map(ext => AppRedisCacheSpace.associationAppsKey(ext)); - if ( associationKeys.length ) { - await deleteRedisKeys(associationKeys); - } - - // remove from tags - const app_tags = (app.tags ?? '').split(',') - .map(tag => tag.trim()) - .filter(tag => tag.length > 0); - for ( const tag of app_tags ) { - if ( ! this.tags[tag] ) continue; - const index = this.tags[tag].indexOf(app_uid); - if ( index >= 0 ) { - this.tags[tag].splice(index, 1); - } - } - - const svc_event = this.services.get('event'); - await svc_event.emit('app.changed', { - app_uid: app.uid, - action: 'deleted', - app, - }); - } - - buildCanonicalAppUidAliasKey_ (appUid) { - return `${APP_UID_ALIAS_KEY_PREFIX}:${appUid}`; - } - - buildCanonicalAppUidAliasReverseKey_ (canonicalAppUid) { - return `${APP_UID_ALIAS_REVERSE_KEY_PREFIX}:${canonicalAppUid}`; - } - - normalizeCanonicalAliasUidList_ (value) { - if ( ! Array.isArray(value) ) return []; - const normalizedList = []; - const seen = new Set(); - for ( const item of value ) { - if ( typeof item !== 'string' || !item ) continue; - if ( seen.has(item) ) continue; - seen.add(item); - normalizedList.push(item); - } - return normalizedList; - } - - async cleanupCanonicalAppUidAliases_ (appUid) { - if ( typeof appUid !== 'string' || !appUid ) return; - - const kvStore = this.services.get('puter-kvstore'); - const suService = this.services.get('su'); - if ( !kvStore || typeof kvStore.get !== 'function' || typeof kvStore.del !== 'function' ) return; - if ( !suService || typeof suService.sudo !== 'function' ) return; - - const selfAliasKey = this.buildCanonicalAppUidAliasKey_(appUid); - const reverseKey = this.buildCanonicalAppUidAliasReverseKey_(appUid); - - try { - await suService.sudo(async () => { - const reverseValue = await kvStore.get({ key: reverseKey }); - const reverseAliases = this.normalizeCanonicalAliasUidList_(reverseValue); - - const deleteOps = [ - kvStore.del({ key: selfAliasKey }), - kvStore.del({ key: reverseKey }), - ]; - for ( const oldUid of reverseAliases ) { - deleteOps.push(kvStore.del({ - key: this.buildCanonicalAppUidAliasKey_(oldUid), - })); - } - await Promise.all(deleteOps); - }); - } catch { - // KV cleanup is best-effort. - } - } - - // Helper function to generate array of all periods between start and end dates - generateAllPeriods (startDate, endDate, grouping) { - const periods = []; - let currentDate = new Date(startDate); - - // ???: In local debugging, `currentDate` evaluates to `Invalid Date`. - // Does this work in prod? - - while ( currentDate <= endDate ) { - let period; - switch ( grouping ) { - case 'hour': - period = `${currentDate.toISOString().slice(0, 13)}:00:00`; - currentDate.setHours(currentDate.getHours() + 1); - break; - case 'day': - period = currentDate.toISOString().slice(0, 10); - currentDate.setDate(currentDate.getDate() + 1); - break; - case 'week': { - // Get the ISO week number - const weekNum = String(this.getWeekNumber(currentDate)).padStart(2, '0'); - period = `${currentDate.getFullYear()}-${weekNum}`; - currentDate.setDate(currentDate.getDate() + 7); - break; - } - case 'month': - period = currentDate.toISOString().slice(0, 7); - currentDate.setMonth(currentDate.getMonth() + 1); - break; - case 'year': - period = currentDate.getFullYear().toString(); - currentDate.setFullYear(currentDate.getFullYear() + 1); - break; - } - periods.push({ period, count: 0 }); - } - return periods; - } - - // Helper function to get ISO week number - getWeekNumber (date) { - const target = new Date(date.valueOf()); - const dayNumber = (date.getDay() + 6) % 7; - target.setDate(target.getDate() - dayNumber + 3); - const firstThursday = target.valueOf(); - target.setMonth(0, 1); - if ( target.getDay() !== 4 ) { - target.setMonth(0, 1 + ((4 - target.getDay()) + 7) % 7); - } - return 1 + Math.ceil((firstThursday - target) / 604800000); - } - - // Helper function to merge actual data with generated periods - mergeWithGeneratedPeriods (actualData, allPeriods, stats_grouping) { - // Create a map of period to count from actual data - // First normalize the period format from both MySQL and ClickHouse - const dataMap = new Map(actualData.map(item => { - let period = item.period; - // For ClickHouse results, convert the timestamp to match the expected format - if ( item.period instanceof Date ) { - switch ( stats_grouping ) { - case 'hour': - period = `${item.period.toISOString().slice(0, 13)}:00:00`; - break; - case 'day': - period = item.period.toISOString().slice(0, 10); - break; - case 'week': { - const weekNum = String(this.getWeekNumber(item.period)).padStart(2, '0'); - period = `${item.period.getFullYear()}-${weekNum}`; - break; - } - case 'month': - period = item.period.toISOString().slice(0, 7); - break; - case 'year': - period = item.period.getFullYear().toString(); - break; - } - } - return [period, parseInt(item.count)]; - })); - - // Map the generated periods to include actual counts where they exist - return allPeriods.map(periodObj => { - const count = dataMap.get(periodObj.period); - return { - period: periodObj.period, - count: count !== undefined ? count : 0, - }; - }); - } - -} - -module.exports = { - AppInformationService, -}; diff --git a/src/backend/src/modules/apps/AppPermissionService.js b/src/backend/src/modules/apps/AppPermissionService.js deleted file mode 100644 index 8170f0c50..000000000 --- a/src/backend/src/modules/apps/AppPermissionService.js +++ /dev/null @@ -1,30 +0,0 @@ -const { UserActorType } = require('../../services/auth/Actor'); -const { PermissionImplicator, PermissionUtil } = require('../../services/auth/permissionUtils.mjs'); -const BaseService = require('../../services/BaseService'); - -class AppPermissionService extends BaseService { - async _init () { - const svc_permission = this.services.get('permission'); - svc_permission.register_implicator(PermissionImplicator.create({ - id: 'user-can-grant-read-own-apps', - matcher: permission => { - return permission.startsWith('apps-of-user:') || - permission.startsWith('subdomains-of-user:'); - }, - checker: async ({ actor, permission }) => { - if ( ! (actor.type instanceof UserActorType) ) { - return undefined; - } - - const parts = PermissionUtil.split(permission); - if ( parts[1] === actor.type.user.uuid ) { - return {}; - } - }, - })); - } -} - -module.exports = { - AppPermissionService, -}; diff --git a/src/backend/src/modules/apps/AppRedisCacheSpace.js b/src/backend/src/modules/apps/AppRedisCacheSpace.js deleted file mode 100644 index 01802304e..000000000 --- a/src/backend/src/modules/apps/AppRedisCacheSpace.js +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { redisClient } from '../../clients/redis/redisSingleton.js'; -import { deleteRedisKeys } from '../../clients/redis/deleteRedisKeys.js'; - -const appFullNamespace = 'apps'; -const appLookupKeys = ['uid', 'name', 'id']; -const appObjectSuffix = 'object'; - -const safeParseJson = (value, fallback = null) => { - if ( value === null || value === undefined ) return fallback; - try { - return JSON.parse(value); - } catch (e) { - return fallback; - } -}; - -const setKey = async (key, value, { ttlSeconds } = {}) => { - if ( ttlSeconds ) { - await redisClient.set(key, value, 'EX', ttlSeconds); - return; - } - await redisClient.set(key, value); -}; - -const appNamespace = () => appFullNamespace; - -const appCacheKey = ({ lookup, value }) => ( - `${appNamespace()}:${lookup}:${value}` -); - -const appObjectNamespace = () => `${appNamespace()}:${appObjectSuffix}`; - -const appObjectCacheKey = ({ lookup, value }) => ( - `${appObjectNamespace()}:${lookup}:${value}` -); - -export const AppRedisCacheSpace = { - key: appCacheKey, - namespace: appNamespace, - objectNamespace: appObjectNamespace, - objectKey: appObjectCacheKey, - keysForApp: (app) => { - if ( ! app ) return []; - return appLookupKeys - .filter(lookup => app[lookup] !== undefined && app[lookup] !== null && app[lookup] !== '') - .map(lookup => appCacheKey({ lookup, value: app[lookup] })); - }, - objectKeysForApp: (app) => { - if ( ! app ) return []; - return appLookupKeys - .filter(lookup => app[lookup] !== undefined && app[lookup] !== null && app[lookup] !== '') - .map(lookup => appObjectCacheKey({ lookup, value: app[lookup] })); - }, - uidScanPattern: () => `${appNamespace()}:uid:*`, - pendingNamespace: () => 'pending_app', - pendingKey: ({ lookup, value }) => ( - `${AppRedisCacheSpace.pendingNamespace()}:${lookup}:${value}` - ), - openCountKey: uid => `apps:open_count:uid:${uid}`, - userCountKey: uid => `apps:user_count:uid:${uid}`, - referralCountKey: uid => `apps:referral_count:uid:${uid}`, - statsKeys: uid => [ - AppRedisCacheSpace.openCountKey(uid), - AppRedisCacheSpace.userCountKey(uid), - AppRedisCacheSpace.referralCountKey(uid), - ], - associationAppsKey: (fileExtension) => { - const ext = String(fileExtension ?? '') - .trim() - .replace(/^\./, '') - .toLowerCase(); - return `assocs:${ext}:apps`; - }, - getCachedApp: async ({ lookup, value }) => ( - safeParseJson(await redisClient.get(appCacheKey({ lookup, value }))) - ), - getCachedAppObject: async ({ lookup, value }) => ( - safeParseJson(await redisClient.get(appObjectCacheKey({ lookup, value }))) - ), - setCachedApp: async (app, { ttlSeconds } = {}) => { - if ( ! app ) return; - const serialized = JSON.stringify(app); - const writes = AppRedisCacheSpace.keysForApp(app) - .map(key => setKey(key, serialized, { ttlSeconds: ttlSeconds || 60 })); - if ( writes.length ) { - await Promise.all(writes); - } - }, - setCachedAppObject: async (app, { ttlSeconds } = {}) => { - if ( ! app ) return; - const serialized = JSON.stringify(app); - const writes = AppRedisCacheSpace.objectKeysForApp(app) - .map(key => setKey(key, serialized, { ttlSeconds: ttlSeconds || 60 })); - if ( writes.length ) { - await Promise.all(writes); - } - }, - invalidateCachedApp: (app, { includeStats = false } = {}) => { - if ( ! app ) return; - const keys = [ - ...AppRedisCacheSpace.keysForApp(app), - ...AppRedisCacheSpace.objectKeysForApp(app), - ]; - if ( includeStats && app.uid ) { - keys.push(...AppRedisCacheSpace.statsKeys(app.uid)); - } - if ( keys.length ) { - return deleteRedisKeys(keys); - } - }, - invalidateCachedAppName: async (name) => { - if ( ! name ) return; - const keys = [ - appCacheKey({ - lookup: 'name', - value: name, - }), - appObjectCacheKey({ - lookup: 'name', - value: name, - }), - ]; - return deleteRedisKeys(keys); - }, - invalidateAppStats: async (uid) => { - if ( ! uid ) return; - return deleteRedisKeys(AppRedisCacheSpace.statsKeys(uid)); - }, -}; diff --git a/src/backend/src/modules/apps/AppsModule.js b/src/backend/src/modules/apps/AppsModule.js deleted file mode 100644 index 6500752e4..000000000 --- a/src/backend/src/modules/apps/AppsModule.js +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { AdvancedBase } = require('@heyputer/putility'); - -class AppsModule extends AdvancedBase { - async install (context) { - const services = context.get('services'); - - const { AppInformationService } = require('./AppInformationService'); - services.registerService('app-information', AppInformationService); - - const { AppIconService } = require('./AppIconService'); - services.registerService('app-icon', AppIconService); - - const { OldAppNameService } = require('./OldAppNameService'); - services.registerService('old-app-name', OldAppNameService); - - const { ProtectedAppService } = require('./ProtectedAppService'); - services.registerService('__protected-app', ProtectedAppService); - - const RecommendedAppsService = require('./RecommendedAppsService').default; - services.registerService('recommended-apps', RecommendedAppsService); - - const { AppPermissionService } = require('./AppPermissionService'); - services.registerService('app-permission', AppPermissionService); - } -} - -module.exports = { - AppsModule, -}; diff --git a/src/backend/src/modules/apps/OldAppNameService.js b/src/backend/src/modules/apps/OldAppNameService.js deleted file mode 100644 index 8e00acec6..000000000 --- a/src/backend/src/modules/apps/OldAppNameService.js +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const BaseService = require('../../services/BaseService'); -const { DB_READ } = require('../../services/database/consts'); - -const N_MONTHS = 4; - -class OldAppNameService extends BaseService { - static LOG_DEBUG = true; - - _init () { - this.db = this.services.get('database').get(DB_READ, 'old-app-name'); - } - - async '__on_boot.consolidation' () { - const svc_event = this.services.get('event'); - svc_event.on('app.rename', async (_, { app_uid, old_name }) => { - this.log.info('GOT EVENT', { app_uid, old_name }); - await this.db.write('INSERT INTO `old_app_names` (`app_uid`, `name`) VALUES (?, ?)', - [app_uid, old_name]); - }); - } - - async check_app_name (name) { - const rows = await this.db.read('SELECT * FROM `old_app_names` WHERE `name` = ?', - [name]); - - if ( rows.length === 0 ) return; - - // Check if the app has been renamed in the last N months - const [row] = rows; - const timestamp = row.timestamp instanceof Date ? row.timestamp : new Date( - // Ensure timestamp ir processed as UTC - row.timestamp.endsWith('Z') ? row.timestamp : `${row.timestamp }Z`); - - const age = Date.now() - timestamp.getTime(); - - // const n_ms = 60 * 1000; - const n_ms = N_MONTHS * 30 * 24 * 60 * 60 * 1000; - this.log.info('AGE INFO', { - input_time: row.timestamp, - age, - n_ms, - }); - if ( age > n_ms ) { - // Remove record - await this.db.write('DELETE FROM `old_app_names` WHERE `id` = ?', - [row.id]); - // Return undefined - return; - } - - return { - id: row.id, - app_uid: row.app_uid, - }; - } - - async remove_name (id) { - await this.db.write('DELETE FROM `old_app_names` WHERE `id` = ?', - [id]); - } -} - -module.exports = { - OldAppNameService, -}; diff --git a/src/backend/src/modules/apps/ProtectedAppService.js b/src/backend/src/modules/apps/ProtectedAppService.js deleted file mode 100644 index 5adc302b1..000000000 --- a/src/backend/src/modules/apps/ProtectedAppService.js +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { get_app } = require('../../helpers'); -const { UserActorType } = require('../../services/auth/Actor'); -const { PermissionImplicator, PermissionUtil, PermissionRewriter } = - require('../../services/auth/permissionUtils.mjs'); -const BaseService = require('../../services/BaseService'); - -/** -* @class ProtectedAppService -* @extends BaseService -* @classdesc This class represents a service that handles protected applications. It extends the BaseService and includes -* methods for initializing permissions and registering rewriters and implicators for permission handling. The class -* ensures that the owner of a protected app has implicit permission to access it. -*/ -class ProtectedAppService extends BaseService { - /** - * Initializes the ProtectedAppService. - * Registers a permission rewriter and implicator to handle application-specific permissions. - * @async - * @method _init - * @memberof ProtectedAppService - * @returns {Promise} A promise that resolves when the initialization is complete. - */ - async _init () { - const svc_permission = this.services.get('permission'); - - svc_permission.register_rewriter(PermissionRewriter.create({ - matcher: permission => { - if ( ! permission.startsWith('app:') ) return false; - const [_, specifier] = PermissionUtil.split(permission); - if ( specifier.startsWith('uid#') ) return false; - return true; - }, - rewriter: async permission => { - const [_1, name, ...rest] = PermissionUtil.split(permission); - const app = await get_app({ name }); - return PermissionUtil.join(_1, `uid#${app.uid}`, ...rest); - }, - })); - - // track: object description in comment - // Owner of procted app has implicit permission to access it - svc_permission.register_implicator(PermissionImplicator.create({ - matcher: permission => { - return permission.startsWith('app:') || permission.startsWith('manage:app'); - }, - checker: async ({ actor, permission }) => { - if ( ! (actor.type instanceof UserActorType) ) { - return undefined; - } - - const parts = PermissionUtil.split(permission); - - if ( parts[0] === 'manage' ) parts.shift(); - - if ( parts.length < 2 ) return undefined; - - const [_, uid_part] = parts; - - // track: slice a prefix - const uid = uid_part.slice('uid#'.length); - - const app = await get_app({ uid }); - - if ( app.owner_user_id !== actor.type.user.id ) { - return undefined; - } - - return {}; - }, - })); - } -} - -module.exports = { - ProtectedAppService, -}; diff --git a/src/backend/src/modules/apps/RecommendedAppsService.js b/src/backend/src/modules/apps/RecommendedAppsService.js deleted file mode 100644 index 484de7fa3..000000000 --- a/src/backend/src/modules/apps/RecommendedAppsService.js +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { get_apps } from '../../helpers.js'; -import { BaseService } from '../../services/BaseService.js'; - -export default class RecommendedAppsService extends BaseService { - appNames = new Set([ - 'app-center', - 'dev-center', - 'editor', - 'code', - 'camera', - 'music-player', - 'recorder', - 'memos', - 'word-processor', - 'spreadsheet', - 'presentation', - 'pdf-editor', - 'basketball-tap', - 'blockup', - 'pretty-tiles', - 'galaxy-troops', - 'blend-fruits', - 'traffic-tap-puzzle', - ]); - - async get_recommended_apps ({ icon_size: iconSize }) { - - // Prepare each app for returning to user by only returning the necessary fields - // and adding them to the retobj array - let recommended = (await get_apps(Array.from(this.appNames).map(name => ({ name })))).filter(app => !!app).map(app => { - return { - uuid: app.uid, - name: app.name, - title: app.title, - icon: app.icon, - godmode: app.godmode, - maximize_on_start: app.maximize_on_start, - index_url: app.index_url, - }; - }); - - const svc_appIcon = this.services.get('app-icon'); - - // Iconify apps - if ( iconSize ) { - recommended = await svc_appIcon.iconifyApps({ - apps: recommended, - size: iconSize, - }); - } - - return recommended; - } -} diff --git a/src/backend/src/modules/apps/default-app-icon.js b/src/backend/src/modules/apps/default-app-icon.js deleted file mode 100644 index 43ede5d5e..000000000 --- a/src/backend/src/modules/apps/default-app-icon.js +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -module.exports = 'data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgdmVyc2lvbj0iMS4xIgogICB3aWR0aD0iNDgiCiAgIGhlaWdodD0iNDgiCiAgIGlkPSJzdmc2NjQ5IgogICB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayIKICAgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIgogICB4bWxuczpzdmc9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIgogICB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiCiAgIHhtbG5zOmNjPSJodHRwOi8vY3JlYXRpdmVjb21tb25zLm9yZy9ucyMiCiAgIHhtbG5zOmRjPSJodHRwOi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4xLyI+CiAgPGRlZnMKICAgICBpZD0iZGVmczY2NTEiPgogICAgPGxpbmVhckdyYWRpZW50CiAgICAgICB4bGluazpocmVmPSIjbGluZWFyR3JhZGllbnQxMjEzMDMiCiAgICAgICBpZD0ibGluZWFyR3JhZGllbnQxMjE3NjQiCiAgICAgICBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSIKICAgICAgIGdyYWRpZW50VHJhbnNmb3JtPSJtYXRyaXgoMS4wMDU5MTg0LDAsMCwwLjg1NzEwOTk5LC0wLjEyNzgyMjg3LDguMTA2NDc1MSkiCiAgICAgICB4MT0iMjUuMDg2MDM5IgogICAgICAgeTE9Ii0xLjM2MjM2OTEiCiAgICAgICB4Mj0iMjUuMDg2MDM5IgogICAgICAgeTI9IjE4LjI5OTMzNCIgLz4KICAgIDxsaW5lYXJHcmFkaWVudAogICAgICAgaWQ9ImxpbmVhckdyYWRpZW50MTIxMzAzIj4KICAgICAgPHN0b3AKICAgICAgICAgaWQ9InN0b3AxMjEyOTUiCiAgICAgICAgIHN0eWxlPSJzdG9wLWNvbG9yOiNmZmZmZmY7c3RvcC1vcGFjaXR5OjEiCiAgICAgICAgIG9mZnNldD0iMCIgLz4KICAgICAgPHN0b3AKICAgICAgICAgaWQ9InN0b3AxMjEyOTciCiAgICAgICAgIHN0eWxlPSJzdG9wLWNvbG9yOiNmZmZmZmY7c3RvcC1vcGFjaXR5OjAuMjM1Mjk0MTIiCiAgICAgICAgIG9mZnNldD0iMC4xMTQxOTQ2OCIgLz4KICAgICAgPHN0b3AKICAgICAgICAgaWQ9InN0b3AxMjEyOTkiCiAgICAgICAgIHN0eWxlPSJzdG9wLWNvbG9yOiNmZmZmZmY7c3RvcC1vcGFjaXR5OjAuMTU2ODYyNzUiCiAgICAgICAgIG9mZnNldD0iMC45Mzg5NjU5OCIgLz4KICAgICAgPHN0b3AKICAgICAgICAgaWQ9InN0b3AxMjEzMDEiCiAgICAgICAgIHN0eWxlPSJzdG9wLWNvbG9yOiNmZmZmZmY7c3RvcC1vcGFjaXR5OjAuMzkyMTU2ODciCiAgICAgICAgIG9mZnNldD0iMSIgLz4KICAgIDwvbGluZWFyR3JhZGllbnQ+CiAgICA8bGluZWFyR3JhZGllbnQKICAgICAgIHhsaW5rOmhyZWY9IiNsaW5lYXJHcmFkaWVudDM5MjQtMi0yLTUtOCIKICAgICAgIGlkPSJsaW5lYXJHcmFkaWVudDEyMTc2MCIKICAgICAgIGdyYWRpZW50VW5pdHM9InVzZXJTcGFjZU9uVXNlIgogICAgICAgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCgxLjAwMDAwMDMsMCwwLDAuODM3ODM4MTMsLTEuMjQ4MTQ2ZS01LDcuODkxODg1MykiCiAgICAgICB4MT0iMjMuOTk5OTkiCiAgICAgICB5MT0iNi4wNDQ1Mjc1IgogICAgICAgeDI9IjIzLjk5OTk5IgogICAgICAgeTI9IjQxLjc2MzIyMiIgLz4KICAgIDxsaW5lYXJHcmFkaWVudAogICAgICAgaWQ9ImxpbmVhckdyYWRpZW50MzkyNC0yLTItNS04Ij4KICAgICAgPHN0b3AKICAgICAgICAgaWQ9InN0b3AzOTI2LTktNC05LTYiCiAgICAgICAgIHN0eWxlPSJzdG9wLWNvbG9yOiNmZmZmZmY7c3RvcC1vcGFjaXR5OjEiCiAgICAgICAgIG9mZnNldD0iMCIgLz4KICAgICAgPHN0b3AKICAgICAgICAgaWQ9InN0b3AzOTI4LTktOC02LTUiCiAgICAgICAgIHN0eWxlPSJzdG9wLWNvbG9yOiNmZmZmZmY7c3RvcC1vcGFjaXR5OjAuMjM1Mjk0MTIiCiAgICAgICAgIG9mZnNldD0iMC4wOTMwMjMyNSIgLz4KICAgICAgPHN0b3AKICAgICAgICAgaWQ9InN0b3AzOTMwLTMtNS0xLTciCiAgICAgICAgIHN0eWxlPSJzdG9wLWNvbG9yOiNmZmZmZmY7c3RvcC1vcGFjaXR5OjAuMTU2ODYyNzUiCiAgICAgICAgIG9mZnNldD0iMC45MDY5NzY3IiAvPgogICAgICA8c3RvcAogICAgICAgICBpZD0ic3RvcDM5MzItOC0wLTQtOCIKICAgICAgICAgc3R5bGU9InN0b3AtY29sb3I6I2ZmZmZmZjtzdG9wLW9wYWNpdHk6MC4zOTIxNTY4NyIKICAgICAgICAgb2Zmc2V0PSIxIiAvPgogICAgPC9saW5lYXJHcmFkaWVudD4KICAgIDxsaW5lYXJHcmFkaWVudAogICAgICAgeGxpbms6aHJlZj0iI2QiCiAgICAgICBpZD0ibGluZWFyR3JhZGllbnQxMjE3NTgiCiAgICAgICBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSIKICAgICAgIGdyYWRpZW50VHJhbnNmb3JtPSJtYXRyaXgoMS4yMTIyOTAzLDAsMCwxLjExNDU1MTQsLTQuNDk5OTAzLC0yLjc2MTI1MzMpIgogICAgICAgeDE9IjIzLjQ1MiIKICAgICAgIHkxPSIzMC41NTUiCiAgICAgICB4Mj0iNDMuMDA3IgogICAgICAgeTI9IjQ1LjkzMzk5OCIgLz4KICAgIDxsaW5lYXJHcmFkaWVudAogICAgICAgaWQ9ImQiPgogICAgICA8c3RvcAogICAgICAgICBvZmZzZXQ9IjAiCiAgICAgICAgIHN0b3AtY29sb3I9IiNmZmYiCiAgICAgICAgIHN0b3Atb3BhY2l0eT0iMCIKICAgICAgICAgaWQ9InN0b3A2NSIgLz4KICAgICAgPHN0b3AKICAgICAgICAgb2Zmc2V0PSIxIgogICAgICAgICBzdG9wLWNvbG9yPSIjZmZmIgogICAgICAgICBzdG9wLW9wYWNpdHk9IjAiCiAgICAgICAgIGlkPSJzdG9wNjciIC8+CiAgICA8L2xpbmVhckdyYWRpZW50PgogICAgPGxpbmVhckdyYWRpZW50CiAgICAgICB4bGluazpocmVmPSIjbGluZWFyR3JhZGllbnQxMDYzMDUiCiAgICAgICBpZD0ibGluZWFyR3JhZGllbnQxMjE3NTYiCiAgICAgICBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSIKICAgICAgIGdyYWRpZW50VHJhbnNmb3JtPSJtYXRyaXgoMS4yMTk2MzY1LDAsMCwxLjMyMDM3MDgsNDAuNzg1OTE1LC0xMy4zMzg3NDQpIgogICAgICAgeDE9Ii01Ljg4NzAzMzUiCiAgICAgICB5MT0iMTkuMzQxOTE1IgogICAgICAgeDI9Ii01Ljg4NzAzMzUiCiAgICAgICB5Mj0iNDMuMzc1NzQ4IiAvPgogICAgPGxpbmVhckdyYWRpZW50CiAgICAgICBpZD0ibGluZWFyR3JhZGllbnQxMDYzMDUiPgogICAgICA8c3RvcAogICAgICAgICBvZmZzZXQ9IjAiCiAgICAgICAgIHN0b3AtY29sb3I9IiNkYWMxOTciCiAgICAgICAgIGlkPSJzdG9wMTA2MzAxIgogICAgICAgICBzdHlsZT0ic3RvcC1jb2xvcjojZTdjNTkxO3N0b3Atb3BhY2l0eToxIiAvPgogICAgICA8c3RvcAogICAgICAgICBvZmZzZXQ9IjEiCiAgICAgICAgIHN0b3AtY29sb3I9IiNiMTk5NzQiCiAgICAgICAgIGlkPSJzdG9wMTA2MzAzIgogICAgICAgICBzdHlsZT0ic3RvcC1jb2xvcjojY2ZhMjVlO3N0b3Atb3BhY2l0eToxIiAvPgogICAgPC9saW5lYXJHcmFkaWVudD4KICAgIDxsaW5lYXJHcmFkaWVudAogICAgICAgeGxpbms6aHJlZj0iI2xpbmVhckdyYWRpZW50MTA2MzA1IgogICAgICAgaWQ9ImxpbmVhckdyYWRpZW50MTcwMyIKICAgICAgIGdyYWRpZW50VW5pdHM9InVzZXJTcGFjZU9uVXNlIgogICAgICAgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCgxLjIxOTYzNjUsMCwwLDEuMzE1NDE2NSw0MC44MDAzMzgsLTEyLjk4MzQyMikiCiAgICAgICB4MT0iLTUuODg3MDMzNSIKICAgICAgIHkxPSIxMS40ODI5NzgiCiAgICAgICB4Mj0iLTUuODg3MDMzNSIKICAgICAgIHkyPSIyMi4xNDg4NjUiIC8+CiAgICA8cmFkaWFsR3JhZGllbnQKICAgICAgIGN4PSI1IgogICAgICAgY3k9IjQxLjUiCiAgICAgICBmeD0iNSIKICAgICAgIGZ5PSI0MS41IgogICAgICAgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCgxLjAwMjg4NzEsMCwwLDEuNiwtMTguMTY3MTM4LC0xMTEuOTgyODkpIgogICAgICAgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiCiAgICAgICB4bGluazpocmVmPSIjZyIKICAgICAgIGlkPSJrLTAtNy0zLTktMyIKICAgICAgIHI9IjUiIC8+CiAgICA8bGluZWFyR3JhZGllbnQKICAgICAgIGlkPSJnIj4KICAgICAgPHN0b3AKICAgICAgICAgb2Zmc2V0PSIwIgogICAgICAgICBpZD0ic3RvcDEzIiAvPgogICAgICA8c3RvcAogICAgICAgICBvZmZzZXQ9IjEiCiAgICAgICAgIHN0b3Atb3BhY2l0eT0iMCIKICAgICAgICAgaWQ9InN0b3AxNSIgLz4KICAgIDwvbGluZWFyR3JhZGllbnQ+CiAgICA8bGluZWFyR3JhZGllbnQKICAgICAgIHhsaW5rOmhyZWY9IiNoIgogICAgICAgaWQ9ImxpbmVhckdyYWRpZW50MTIxNzU0IgogICAgICAgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiCiAgICAgICBncmFkaWVudFRyYW5zZm9ybT0ibWF0cml4KDIuMTMwNDMzMiwwLDAsMS40NTQ1NSwtODcuNzE5MDE4LC0xMy4zMjcxMSkiCiAgICAgICB4MT0iMTcuNTU0MDAxIgogICAgICAgeTE9IjQ2IgogICAgICAgeDI9IjE3LjU1NDAwMSIKICAgICAgIHkyPSIzNSIgLz4KICAgIDxsaW5lYXJHcmFkaWVudAogICAgICAgaWQ9ImgiPgogICAgICA8c3RvcAogICAgICAgICBvZmZzZXQ9IjAiCiAgICAgICAgIHN0b3Atb3BhY2l0eT0iMCIKICAgICAgICAgaWQ9InN0b3A1NCIgLz4KICAgICAgPHN0b3AKICAgICAgICAgb2Zmc2V0PSIuNSIKICAgICAgICAgaWQ9InN0b3A1NiIgLz4KICAgICAgPHN0b3AKICAgICAgICAgb2Zmc2V0PSIxIgogICAgICAgICBzdG9wLW9wYWNpdHk9IjAiCiAgICAgICAgIGlkPSJzdG9wNTgiIC8+CiAgICA8L2xpbmVhckdyYWRpZW50PgogICAgPHJhZGlhbEdyYWRpZW50CiAgICAgICBjeD0iNSIKICAgICAgIGN5PSI0MS41IgogICAgICAgZng9IjUiCiAgICAgICBmeT0iNDEuNSIKICAgICAgIGdyYWRpZW50VHJhbnNmb3JtPSJtYXRyaXgoMS4wMDI4ODcxLDAsMCwxLjYsNTcuMTM5MDQ4LC0xMTEuOTgyODkpIgogICAgICAgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiCiAgICAgICB4bGluazpocmVmPSIjZyIKICAgICAgIGlkPSJpLTYtOS03LTgtOSIKICAgICAgIHI9IjUiIC8+CiAgICA8bGluZWFyR3JhZGllbnQKICAgICAgIGdyYWRpZW50VW5pdHM9InVzZXJTcGFjZU9uVXNlIgogICAgICAgeGxpbms6aHJlZj0iI2MtMyIKICAgICAgIGlkPSJuIgogICAgICAgeDE9IjI2IgogICAgICAgeDI9IjI2IgogICAgICAgeTE9IjIyIgogICAgICAgeTI9IjgiCiAgICAgICBncmFkaWVudFRyYW5zZm9ybT0idHJhbnNsYXRlKDAsLTMpIiAvPgogICAgPGxpbmVhckdyYWRpZW50CiAgICAgICBpZD0iYy0zIj4KICAgICAgPHN0b3AKICAgICAgICAgb2Zmc2V0PSIwIgogICAgICAgICBzdG9wLWNvbG9yPSIjZmZmIgogICAgICAgICBpZD0ic3RvcDM2LTYiIC8+CiAgICAgIDxzdG9wCiAgICAgICAgIG9mZnNldD0iMC40MjgxODMwNSIKICAgICAgICAgc3RvcC1jb2xvcj0iI2ZmZiIKICAgICAgICAgaWQ9InN0b3AzOC03IiAvPgogICAgICA8c3RvcAogICAgICAgICBvZmZzZXQ9IjAuNTAwOTMzMTciCiAgICAgICAgIHN0b3AtY29sb3I9IiNmZmYiCiAgICAgICAgIHN0b3Atb3BhY2l0eT0iLjY0MyIKICAgICAgICAgaWQ9InN0b3A0MC01IiAvPgogICAgICA8c3RvcAogICAgICAgICBvZmZzZXQ9IjEiCiAgICAgICAgIHN0b3AtY29sb3I9IiNmZmYiCiAgICAgICAgIHN0b3Atb3BhY2l0eT0iLjM5MSIKICAgICAgICAgaWQ9InN0b3A0Mi0zIiAvPgogICAgPC9saW5lYXJHcmFkaWVudD4KICA8L2RlZnM+CiAgPG1ldGFkYXRhCiAgICAgaWQ9Im1ldGFkYXRhNjY1NCI+CiAgICA8cmRmOlJERj4KICAgICAgPGNjOldvcmsKICAgICAgICAgcmRmOmFib3V0PSIiPgogICAgICAgIDxkYzpmb3JtYXQ+aW1hZ2Uvc3ZnK3htbDwvZGM6Zm9ybWF0PgogICAgICAgIDxkYzp0eXBlCiAgICAgICAgICAgcmRmOnJlc291cmNlPSJodHRwOi8vcHVybC5vcmcvZGMvZGNtaXR5cGUvU3RpbGxJbWFnZSIgLz4KICAgICAgPC9jYzpXb3JrPgogICAgPC9yZGY6UkRGPgogIDwvbWV0YWRhdGE+CiAgPGcKICAgICBpZD0iZzEyMTAiCiAgICAgdHJhbnNmb3JtPSJtYXRyaXgoMC43MTE4NjQzOCwwLDAsMC43NSw1MC44MDQ1NjIsNi44MTI4MzI4KSIKICAgICBzdHlsZT0ic3Ryb2tlLXdpZHRoOjEuMzY4NTgiPgogICAgPHJlY3QKICAgICAgIGZpbGw9InVybCgjaSkiCiAgICAgICBoZWlnaHQ9IjE2IgogICAgICAgb3BhY2l0eT0iMC40IgogICAgICAgdHJhbnNmb3JtPSJzY2FsZSgtMSkiCiAgICAgICB3aWR0aD0iNSIKICAgICAgIHg9IjYyLjE1NDAzIgogICAgICAgeT0iLTUzLjU4Mjg5IgogICAgICAgaWQ9InJlY3Q3Ny05LTkwLTItNy04IgogICAgICAgc3R5bGU9ImZpbGw6dXJsKCNpLTYtOS03LTgtOSk7c3Ryb2tlLXdpZHRoOjEuMzY4NTgiIC8+CiAgICA8cmVjdAogICAgICAgZmlsbD0idXJsKCNqKSIKICAgICAgIGhlaWdodD0iMTYiCiAgICAgICBvcGFjaXR5PSIwLjQiCiAgICAgICB3aWR0aD0iNDkiCiAgICAgICB4PSItNjIuMTU0MDMiCiAgICAgICB5PSIzNy41ODI4OSIKICAgICAgIGlkPSJyZWN0NzktNy0yLTAtMS00IgogICAgICAgc3R5bGU9ImZpbGw6dXJsKCNsaW5lYXJHcmFkaWVudDEyMTc1NCk7c3Ryb2tlLXdpZHRoOjEuMzY4NTgiIC8+CiAgICA8cmVjdAogICAgICAgZmlsbD0idXJsKCNrKSIKICAgICAgIGhlaWdodD0iMTYiCiAgICAgICBvcGFjaXR5PSIwLjQiCiAgICAgICB0cmFuc2Zvcm09InNjYWxlKDEsLTEpIgogICAgICAgd2lkdGg9IjUiCiAgICAgICB4PSItMTMuMTU0MDI4IgogICAgICAgeT0iLTUzLjU4Mjg5IgogICAgICAgaWQ9InJlY3Q4MS0zLTgtNi03LTgiCiAgICAgICBzdHlsZT0iZmlsbDp1cmwoI2stMC03LTMtOS0zKTtzdHJva2Utd2lkdGg6MS4zNjg1OCIgLz4KICA8L2c+CiAgPHBhdGgKICAgICBpZD0icmVjdDU1MDUtMjEtMS01LTAtNi01LTEtMi01LTEwIgogICAgIHN0eWxlPSJjb2xvcjojMDAwMDAwO2ZvbnQtdmFyaWF0aW9uLXNldHRpbmdzOm5vcm1hbDtkaXNwbGF5OmlubGluZTtvdmVyZmxvdzp2aXNpYmxlO3Zpc2liaWxpdHk6dmlzaWJsZTt2ZWN0b3ItZWZmZWN0Om5vbmU7ZmlsbDp1cmwoI2xpbmVhckdyYWRpZW50MTcwMyk7ZmlsbC1vcGFjaXR5OjE7ZmlsbC1ydWxlOm5vbnplcm87c3Ryb2tlOm5vbmU7c3Ryb2tlLXdpZHRoOjAuOTk5OTk5O3N0cm9rZS1saW5lY2FwOmJ1dHQ7c3Ryb2tlLWxpbmVqb2luOm1pdGVyO3N0cm9rZS1taXRlcmxpbWl0OjQ7c3Ryb2tlLWRhc2hhcnJheTpub25lO3N0cm9rZS1kYXNob2Zmc2V0OjA7c3Ryb2tlLW9wYWNpdHk6MC4zOy1pbmtzY2FwZS1zdHJva2U6bm9uZTttYXJrZXI6bm9uZTtlbmFibGUtYmFja2dyb3VuZDphY2N1bXVsYXRlO3N0b3AtY29sb3I6IzAwMDAwMCIKICAgICBkPSJNIDExLjU5MDkyMyw1LjUgQyA5LjIzMzkwNSw1LjUgOC4yOTM2NSw2Ljg5NjUxODMgNy4zMzYzNzgsOS4wNTgwMjUyIDYuNjAyNjI1LDEwLjcxMDQ1NyA1Ljc0ODksMTIuNDIwMTYyIDUuMDcwNjEzLDE0LjAzOTI2IDQuNzA5ODY5LDE0LjY2Njk5NCA0LjUwMDAxNCwxNS4zOTQ1MDYgNC41MDAwMTQsMTYuMTc0MDc1IGggMzkuMDAwMDAzIGMgMCwtMC43Nzk1NjkgLTAuMjA5ODU1LC0xLjUwNzA4MSAtMC41NzA1OTgsLTIuMTM0ODE1IEMgNDIuMjMyNzQ0LDEyLjQyODM2MSA0MS40MTc5MiwxMC43MDExOTIgNDAuNjYzNjUzLDkuMDU4MDI1MiAzOS42NzczNzksNi45MDk2ODc3IDM4Ljc2NjEyNiw1LjUgMzYuNDA5MTA4LDUuNSBaIiAvPgogIDxwYXRoCiAgICAgaWQ9InJlY3Q1NTA1LTIxLTEtNS0wLTYtNS0xLTItMyIKICAgICBzdHlsZT0iY29sb3I6IzAwMDAwMDtmb250LXZhcmlhdGlvbi1zZXR0aW5nczpub3JtYWw7ZGlzcGxheTppbmxpbmU7b3ZlcmZsb3c6dmlzaWJsZTt2aXNpYmlsaXR5OnZpc2libGU7dmVjdG9yLWVmZmVjdDpub25lO2ZpbGw6dXJsKCNsaW5lYXJHcmFkaWVudDEyMTc1Nik7ZmlsbC1vcGFjaXR5OjE7ZmlsbC1ydWxlOm5vbnplcm87c3Ryb2tlOm5vbmU7c3Ryb2tlLXdpZHRoOjAuOTk5OTk5O3N0cm9rZS1saW5lY2FwOmJ1dHQ7c3Ryb2tlLWxpbmVqb2luOm1pdGVyO3N0cm9rZS1taXRlcmxpbWl0OjQ7c3Ryb2tlLWRhc2hhcnJheTpub25lO3N0cm9rZS1kYXNob2Zmc2V0OjA7c3Ryb2tlLW9wYWNpdHk6MC4zOy1pbmtzY2FwZS1zdHJva2U6bm9uZTttYXJrZXI6bm9uZTtlbmFibGUtYmFja2dyb3VuZDphY2N1bXVsYXRlO3N0b3AtY29sb3I6IzAwMDAwMCIKICAgICBkPSJNIDguNzU0NTQ1LDEyIEMgNi45ODE4MTgsMTIgNC41LDEzLjU1NjQ1NyA0LjUsMTcuMzU3MTM5IHYgMjIuODU3MTI2IGMgMCwwLjE4MDAwMiAwLjAxNDU0LDAuMzU2MjQ0IDAuMDM2MDIsMC41MzAxMzQgMC4wMDUsMC4wNDAzMiAwLjAxMTk4LDAuMDgwMDEgMC4wMTgwMSwwLjExOTk3NiAwLjAyMTQyLDAuMTQwNDQzIDAuMDQ4NSwwLjI3ODg0MyAwLjA4MzEsMC40MTQzNDIgMC4wMDg5LDAuMDM0OTcgMC4wMTY2NywwLjA3MDAyIDAuMDI2MzEsMC4xMDQ2MzEgMC4wOTcxMywwLjM0MzgzNyAwLjIzMzc3MywwLjY3MDg5OCAwLjQwNzE3NCwwLjk3Mzc3MiA1LjFlLTQsOS4yOWUtNCA3LjA5ZS00LDAuMDAxOCAwLjAwMTQsMC4wMDI4IDAuNzM0MTUsMS4yODAyNTkgMi4xMDM0MTksMi4xNDAwNyAzLjY4MjUxNSwyLjE0MDA3IGggMzAuNDkwOTEyIGMgMS41NzkwOTYsMCAyLjk0ODM2NSwtMC44NTk4MTEgMy42ODI1NjUsLTIuMTQwMDY2IDMuOTZlLTQsLTkuMjllLTQgNy4wOWUtNCwtMC4wMDE5IDAuMDAxNCwtMC4wMDI4IDAuMTczNDAxLC0wLjMwMjg3NCAwLjMxMDA1LC0wLjYyOTkzNSAwLjQwNzE3NSwtMC45NzM3NzIgMC4wMDk2LC0wLjAzNDYxIDAuMDE3NTIsLTAuMDY5NjYgMC4wMjYzMSwtMC4xMDQ2MzEgMC4wMzQ2LC0wLjEzNTQ5OSAwLjA2MTY5LC0wLjI3Mzg5OCAwLjA4MzEsLTAuNDE0MzQxIDAuMDA1NywtMC4wMzk5NyAwLjAxMzEyLC0wLjA3OTY1IDAuMDE4MDEsLTAuMTE5OTc3IDAuMDIxNDksLTAuMTczODk0IDAuMDM1OTYsLTAuMzUwMTM2IDAuMDM1OTYsLTAuNTMwMTM4IFYgMTcuNzE0MjgyIGMgMCwtMi42NzU0NzUgLTEuMDYzNjM3LC01LjcxNDI4MSAtNC4yNTQ1NDYsLTUuNzE0MjgxIHoiIC8+CiAgPHBhdGgKICAgICBkPSJtIDEwLjY0NDg2MSwxMS4yOTY1MDUgaCAyNi4xNDQxODUgYyAxLjUyNjY3MywwIDIuNDcxMTgyLDAuNTI4MDExIDMuMTEwNzgyLDEuOTc5Njg1IGwgMi4yMDE3MjcsNi4wOTEzMzkgdiAyMS45NTk0MiBjIDAsMS4zODU0OTUgLTAuNzc0MzI3LDIuMDgzNTggLTIuMzAwMjkxLDIuMDgzNTggSCA3LjkwNzc3IGMgLTEuNTI1OTY0LDAgLTIuMTQ4NTQ2LC0wLjc2NzgyMiAtMi4xNDg1NDYsLTIuMTUzMzE3IFYgMTkuMzY2MTA1IGwgMi4xMzA4MTksLTYuMjIxNTYyIGMgMC40MjU0NTUsLTEuMTI0MzM2IDEuMjI4ODU1LC0xLjg0ODc1IDIuNzU0ODE4LC0xLjg0ODc1IHoiCiAgICAgZGlzcGxheT0iYmxvY2siCiAgICAgZmlsbD0ibm9uZSIKICAgICBvcGFjaXR5PSIwLjUwNSIKICAgICBvdmVyZmxvdz0idmlzaWJsZSIKICAgICBzdHJva2U9InVybCgjbSkiCiAgICAgc3Ryb2tlLXdpZHRoPSIwLjc0MTk5OCIKICAgICBzdHlsZT0ic3Ryb2tlOnVybCgjbGluZWFyR3JhZGllbnQxMjE3NTgpO21hcmtlcjpub25lIgogICAgIGlkPSJwYXRoODUtMS04LTUtNy0wIiAvPgogIDxyZWN0CiAgICAgc3R5bGU9Im9wYWNpdHk6MC4zO2ZpbGw6bm9uZTtzdHJva2U6dXJsKCNsaW5lYXJHcmFkaWVudDEyMTc2MCk7c3Ryb2tlLXdpZHRoOjAuOTk5OTg0O3N0cm9rZS1saW5lY2FwOnJvdW5kO3N0cm9rZS1saW5lam9pbjpyb3VuZDtzdHJva2UtbWl0ZXJsaW1pdDo0O3N0cm9rZS1kYXNoYXJyYXk6bm9uZTtzdHJva2UtZGFzaG9mZnNldDowO3N0cm9rZS1vcGFjaXR5OjEiCiAgICAgaWQ9InJlY3Q2NzQxLTUtMC0yLTMtNC0yLTQiCiAgICAgeT0iMTIuNDk5OTkyIgogICAgIHg9IjUuNDk5OTk0MyIKICAgICByeT0iMy41IgogICAgIGhlaWdodD0iMzEuMDAwMDE3IgogICAgIHdpZHRoPSIzNyIKICAgICByeD0iMy41IiAvPgogIDxwYXRoCiAgICAgaWQ9InJlY3Q1NTA1LTIxLTEtNS0wLTYtNS0xLTItNS0xLTQiCiAgICAgc3R5bGU9ImNvbG9yOiMwMDAwMDA7Zm9udC12YXJpYXRpb24tc2V0dGluZ3M6bm9ybWFsO2Rpc3BsYXk6aW5saW5lO292ZXJmbG93OnZpc2libGU7dmlzaWJpbGl0eTp2aXNpYmxlO3ZlY3Rvci1lZmZlY3Q6bm9uZTtmaWxsOm5vbmU7ZmlsbC1vcGFjaXR5OjE7ZmlsbC1ydWxlOm5vbnplcm87c3Ryb2tlOiM4MDRiMDA7c3Ryb2tlLXdpZHRoOjAuOTk5OTk5O3N0cm9rZS1saW5lY2FwOmJ1dHQ7c3Ryb2tlLWxpbmVqb2luOm1pdGVyO3N0cm9rZS1taXRlcmxpbWl0OjQ7c3Ryb2tlLWRhc2hhcnJheTpub25lO3N0cm9rZS1kYXNob2Zmc2V0OjA7c3Ryb2tlLW9wYWNpdHk6MC41Oy1pbmtzY2FwZS1zdHJva2U6bm9uZTttYXJrZXI6bm9uZTtlbmFibGUtYmFja2dyb3VuZDphY2N1bXVsYXRlO3N0b3AtY29sb3I6IzAwMDAwMCIKICAgICBkPSJtIDExLjU5MDkyMyw1LjQ5OTk5OTUgYyAtMi4zNTcwMTgsMCAtMy4yOTcyNzMsMS4zOTE1ODQ0IC00LjI1NDU0NSwzLjU0NTQ1NDYgQyA2LjYwMjYyNSwxMC42OTIwNDggNS43NDg5LDEyLjM5NTcxMyA1LjA3MDYxMywxNC4wMDkwOTEgNC43MDk4NjksMTQuNjM0NjA3IDQuNTAwMDE0LDE1LjM1OTU0OSA0LjUwMDAxNCwxNi4xMzYzNjMgdiAyNC4xMDkwOTIgYyAwLDIuMzU3MDE4IDEuODk3NTI3LDQuMjU0NTQ2IDQuMjU0NTQ1LDQuMjU0NTQ2IGggMzAuNDkwOTEzIGMgMi4zNTcwMTgsMCA0LjI1NDU0NSwtMS44OTc1MjggNC4yNTQ1NDUsLTQuMjU0NTQ2IFYgMTYuMTM2MzYzIGMgMCwtMC43NzY4MTQgLTAuMjA5ODU1LC0xLjUwMTc1NiAtMC41NzA1OTgsLTIuMTI3MjcyIEMgNDIuMjMyNzQ0LDEyLjQwMzg4MyA0MS40MTc5MiwxMC42ODI4MTYgNDAuNjYzNjUzLDkuMDQ1NDU0MSAzOS42NzczNzksNi45MDQ3MDY4IDM4Ljc2NjEyNiw1LjQ5OTk5OTUgMzYuNDA5MTA4LDUuNDk5OTk5NSBaIiAvPgogIDxwYXRoCiAgICAgaWQ9InJlY3Q1NTA1LTIxLTEtNS0wLTYtNS0xLTItNS0xLTctNyIKICAgICBzdHlsZT0iY29sb3I6IzAwMDAwMDtmb250LXZhcmlhdGlvbi1zZXR0aW5nczpub3JtYWw7ZGlzcGxheTppbmxpbmU7b3ZlcmZsb3c6dmlzaWJsZTt2aXNpYmlsaXR5OnZpc2libGU7b3BhY2l0eTowLjE1O3ZlY3Rvci1lZmZlY3Q6bm9uZTtmaWxsOm5vbmU7ZmlsbC1vcGFjaXR5OjE7ZmlsbC1ydWxlOm5vbnplcm87c3Ryb2tlOnVybCgjbGluZWFyR3JhZGllbnQxMjE3NjQpO3N0cm9rZS13aWR0aDowLjk5OTk5MTtzdHJva2UtbGluZWNhcDpyb3VuZDtzdHJva2UtbGluZWpvaW46cm91bmQ7c3Ryb2tlLW1pdGVybGltaXQ6NDtzdHJva2UtZGFzaGFycmF5Om5vbmU7c3Ryb2tlLWRhc2hvZmZzZXQ6MDtzdHJva2Utb3BhY2l0eToxOy1pbmtzY2FwZS1zdHJva2U6bm9uZTttYXJrZXI6bm9uZTtlbmFibGUtYmFja2dyb3VuZDphY2N1bXVsYXRlO3N0b3AtY29sb3I6IzAwMDAwMCIKICAgICBkPSJNIDQxLjU1OTA5NywxMy4xOCAzOS44NDYyNjEsOS42MDExMDA3IEMgMzkuMzY4MTczLDguNTU5Njc2MSAzOC45MjI4MjksNy43NTkzNzQ5IDM4LjQwNDc1NSw3LjI2MTE2MyAzNy44ODY2NzQsNi43NjI5NTEyIDM3LjMxMzE3Miw2LjQ5OTk5NDUgMzYuMjg5NzksNi40OTk5OTQ1IEggMTEuNzExMjE4IGMgLTEuMDI0NzMsMCAtMS42MDg4MjEsMC4yNjI2MDMyIC0yLjEyODY4MDQsMC43NTg0MTU4IEMgOS4wNjI2ODA1LDcuNzU0MjIyOCA4LjYyMDYzMSw4LjU0ODc0MjMgOC4xNTg4NDg4LDkuNTkxNDY3NyB2IDAuMDAxNDEgTCA2LjU5Nzg2MDMsMTMuMjU2NzI1IiAvPgogIDxwYXRoCiAgICAgZD0ibSAyMiw1IGggNCBWIDE5IEMgMjUuNjA2LDE5IDI1LjIxMywxOC4yMjkgMjQuODE5LDE4LjIyOSAyNC40MTYsMTguMjI5IDI0LjAxMywxOSAyMy42MDksMTkgMjMuMjg1LDE5IDIyLjk2LDE4LjMyNSAyMi42MzYsMTguMzI1IDIyLjQyNCwxOC4zMjUgMjIuMjEyLDE5IDIyLDE5IFoiCiAgICAgZmlsbD0idXJsKCNuKSIKICAgICBvcGFjaXR5PSIwLjMiCiAgICAgb3ZlcmZsb3c9InZpc2libGUiCiAgICAgc3R5bGU9ImZpbGw6dXJsKCNuKTttYXJrZXI6bm9uZSIKICAgICBpZD0icGF0aDg3IiAvPgo8L3N2Zz4K'; \ No newline at end of file diff --git a/src/backend/src/modules/apps/lib/IconResult.js b/src/backend/src/modules/apps/lib/IconResult.js deleted file mode 100644 index bb9ee48bc..000000000 --- a/src/backend/src/modules/apps/lib/IconResult.js +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { Context } = require('../../../util/context'); -const { stream_to_buffer } = require('../../../util/streamutil'); - -module.exports = class IconResult { - constructor (o) { - Object.assign(this, o); - } - - async get_data_url () { - if ( this.data_url ) { - return this.data_url; - } else { - try { - const buffer = await stream_to_buffer(this.stream); - return `data:${this.mime};base64,${buffer.toString('base64')}`; - } catch (e) { - const svc_error = Context.get(undefined, { - allow_fallback: true, - }).get('services').get('error'); - svc_error.report('IconResult:get_data_url', { - source: e, - }); - // TODO: broken image icon here - return `data:image/png;base64,${Buffer.from([]).toString('base64')}`; - } - } - } -}; diff --git a/src/backend/src/modules/apps/privateLaunchAccess.js b/src/backend/src/modules/apps/privateLaunchAccess.js deleted file mode 100644 index 951437833..000000000 --- a/src/backend/src/modules/apps/privateLaunchAccess.js +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright (C) 2026-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { UserActorType } from '../../services/auth/Actor.js'; - -const DEFAULT_FALLBACK_APP_NAME = 'app-center'; - -function isPrivateApp (app) { - return Number(app?.is_private ?? 0) > 0; -} - -function buildFallbackPath (appName) { - if ( typeof appName !== 'string' || !appName.trim() ) { - return '/app'; - } - return `/app/${encodeURIComponent(appName.trim())}`; -} - -function buildDefaultDeniedDecision (appName, reason) { - return { - hasAccess: false, - fallbackAppName: DEFAULT_FALLBACK_APP_NAME, - fallbackArgs: { - path: buildFallbackPath(appName), - }, - reason: reason ?? 'private-access-required', - checkedBy: 'core/private-launch-access', - }; -} - -function normalizeLaunchDecision (decision, appName) { - if ( !decision || typeof decision !== 'object' ) { - return buildDefaultDeniedDecision(appName, 'invalid-private-access-result'); - } - - const hasAccess = !!decision.hasAccess; - if ( hasAccess ) { - return { - hasAccess: true, - reason: typeof decision.reason === 'string' - ? decision.reason - : undefined, - checkedBy: typeof decision.checkedBy === 'string' - ? decision.checkedBy - : undefined, - }; - } - - const fallbackAppName = typeof decision.fallbackAppName === 'string' - && decision.fallbackAppName.trim() - ? decision.fallbackAppName.trim() - : DEFAULT_FALLBACK_APP_NAME; - const fallbackPath = decision.fallbackArgs?.path; - const fallbackArgs = typeof fallbackPath === 'string' && fallbackPath.trim() - ? { path: fallbackPath.trim() } - : { path: buildFallbackPath(appName) }; - - return { - hasAccess: false, - fallbackAppName, - fallbackArgs, - reason: typeof decision.reason === 'string' - ? decision.reason - : undefined, - checkedBy: typeof decision.checkedBy === 'string' - ? decision.checkedBy - : undefined, - }; -} - -function getActorUserUid (actor) { - if ( ! actor ) return null; - - if ( actor.type instanceof UserActorType ) { - const userUid = actor.type?.user?.uuid; - return typeof userUid === 'string' && userUid ? userUid : null; - } - - if ( typeof actor.get_related_actor === 'function' ) { - try { - const userActor = actor.get_related_actor(UserActorType); - const userUid = userActor?.type?.user?.uuid; - return typeof userUid === 'string' && userUid ? userUid : null; - } catch { - return null; - } - } - - return null; -} - -async function resolvePrivateLaunchAccess ({ - app, - services, - userUid, - source, - args, -}) { - if ( ! isPrivateApp(app) ) { - return { - hasAccess: true, - checkedBy: 'core/public-app', - }; - } - - const deniedDecision = buildDefaultDeniedDecision( - app?.name, - 'private-access-required', - ); - - const eventService = services?.get?.('event'); - if ( ! eventService ) { - return { - ...deniedDecision, - reason: 'private-access-event-service-unavailable', - }; - } - - const eventPayload = { - appUid: app?.uid, - appName: app?.name, - userUid: typeof userUid === 'string' && userUid ? userUid : null, - source: source ?? 'unknown', - args: args ?? {}, - result: { ...deniedDecision }, - }; - - try { - await eventService.emit('app.privateAccess.resolveLaunch', eventPayload); - } catch { - return { - ...deniedDecision, - reason: 'private-access-check-error', - }; - } - - return normalizeLaunchDecision(eventPayload.result, app?.name); -} - -export { - getActorUserUid, - isPrivateApp, - resolvePrivateLaunchAccess, -}; diff --git a/src/backend/src/modules/broadcast/BroadcastModule.js b/src/backend/src/modules/broadcast/BroadcastModule.js deleted file mode 100644 index ec2367aa2..000000000 --- a/src/backend/src/modules/broadcast/BroadcastModule.js +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { AdvancedBase } = require('@heyputer/putility'); - -class BroadcastModule extends AdvancedBase { - async install (context) { - const services = context.get('services'); - - const { BroadcastService } = require('./BroadcastService'); - services.registerService('broadcast', BroadcastService); - } -} - -module.exports = { - BroadcastModule, -}; diff --git a/src/backend/src/modules/broadcast/BroadcastService.js b/src/backend/src/modules/broadcast/BroadcastService.js deleted file mode 100644 index c6a1e7f0e..000000000 --- a/src/backend/src/modules/broadcast/BroadcastService.js +++ /dev/null @@ -1,518 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { createHmac, randomUUID, timingSafeEqual } from 'crypto'; -import { Agent as HttpsAgent } from 'https'; -import axios from 'axios'; -import { redisClient } from '../../clients/redis/redisSingleton.js'; -import eggspress from '../../api/eggspress.js'; -import { BaseService } from '../../services/BaseService.js'; -import { Context } from '../../util/context.js'; - -export class BroadcastService extends BaseService { - #peersByKey = {}; - #webhookPeers = []; - #incomingLastNonceByPeer = new Map(); - #outgoingNonceByPeer = new Map(); - #outboundEventsByDedupKey = new Map(); - #outboundFlushTimer = null; - #outboundIsFlushing = false; - #dedupFallbackCounter = 0; - #webhookReplayWindowSeconds = 300; - #outboundFlushMs = 5000; - #webhookHostHeader = null; - #webhookProtocol = 'https'; - #webhookHttpsAgent = new HttpsAgent({ rejectUnauthorized: false }); - #redisPubSubChannel = 'broadcast.webhook.events'; - #redisSubscriber = null; - #redisSourceId = randomUUID(); - - async _init () { - const peers = this.config.peers ?? []; - const replayWindowSeconds = this.config.webhook_replay_window_seconds ?? 300; - const outboundFlushMs = Number(this.config.outbound_flush_ms ?? 2000); - - for ( const peer_config of peers ) { - const peerId = this.#resolvePeerId(peer_config); - if ( ! peerId ) { - console.warn('ignoring broadcast peer config with missing key/peerId', { peer_config }); - continue; - } - - if ( this.#peersByKey[peerId] ) { - console.warn('duplicate broadcast peer id configured', { - peerId, - existing: this.#peersByKey[peerId]?.webhook_url, - duplicate: peer_config.webhook_url, - }); - } - - this.#peersByKey[peerId] = { - webhook_secret: peer_config.webhook_secret, - webhook_url: peer_config.webhook_url, - webhook: !!peer_config.webhook, - }; - - if ( peer_config.webhook ) { - this.#webhookPeers.push({ - ...peer_config, - peerId, - }); - } else { - console.warn('ignoring non-webhook broadcast peer; websocket transport is disabled', { - peerId, - }); - } - } - - this.#webhookReplayWindowSeconds = replayWindowSeconds; - this.#outboundFlushMs = Number.isFinite(outboundFlushMs) && outboundFlushMs >= 0 - ? outboundFlushMs - : 5000; - this.#webhookHostHeader = this.global_config.domain; - { - const protocol = String(this.global_config.protocol ?? '').trim().replace(/:$/, '').toLowerCase(); - this.#webhookProtocol = protocol === 'http' || protocol === 'https' ? protocol : 'https'; - } - this.#redisSourceId = `${String(this.global_config?.server_id ?? 'local')}:${randomUUID()}`; - - await this.#initRedisPubSub(); - - const svc_event = this.services.get('event'); - svc_event.on('outer.*', this.outBroadcastEventHandler.bind(this)); - } - - async outBroadcastEventHandler (key, data, meta) { - if ( meta?.from_outside ) return; - - const safeMeta = this.#normalizeMeta(meta); - const outboundEvent = { key, data, meta: safeMeta }; - - // Mirror local outer.pub events to Redis so same-cluster replicas - // receive them even when this instance is the originator. - this.#publishWebhookEventsToRedis([outboundEvent]).catch(error => { - console.warn('local redis pubsub publish failed', { error, key }); - }); - - this.#enqueueOutboundEvent(outboundEvent); - } - - #enqueueOutboundEvent (event) { - const dedupKey = this.#createDedupKey(event); - this.#outboundEventsByDedupKey.set(dedupKey, event); - this.#scheduleOutboundFlush(); - } - - #createDedupKey (event) { - try { - return JSON.stringify(event); - } catch { - const fallbackKey = `fallback-${this.#dedupFallbackCounter}`; - this.#dedupFallbackCounter += 1; - return fallbackKey; - } - } - - #scheduleOutboundFlush () { - if ( this.#outboundFlushTimer ) return; - - this.#outboundFlushTimer = setTimeout(async () => { - this.#outboundFlushTimer = null; - try { - await this.#flushOutboundEvents(); - } catch ( error ) { - console.warn('outbound broadcast flush failed', { error }); - } - }, this.#outboundFlushMs); - } - - async #flushOutboundEvents () { - if ( this.#outboundIsFlushing || this.#outboundEventsByDedupKey.size === 0 ) return; - - this.#outboundIsFlushing = true; - try { - const events = [...this.#outboundEventsByDedupKey.values()]; - this.#outboundEventsByDedupKey.clear(); - - for ( const peer_config of this.#webhookPeers ) { - try { - await this.#sendWebhookToPeer(peer_config, events); - } catch (e) { - console.warn(`webhook broadcast send error: ${ JSON.stringify({ peer: peer_config.peerId ?? peer_config.key, error: e.message })}`); - } - } - } finally { - this.#outboundIsFlushing = false; - if ( this.#outboundEventsByDedupKey.size > 0 ) { - this.#scheduleOutboundFlush(); - } - } - } - - #normalizeMeta (meta) { - if ( !meta || typeof meta !== 'object' || Array.isArray(meta) ) { - return {}; - } - return meta; - } - - #resolveLocalPeerId () { - const localPeerId = this.config?.webhook?.peerId ?? this.config?.webhook?.key; - if ( typeof localPeerId !== 'string' || localPeerId.trim() === '' ) return null; - return localPeerId.trim(); - } - - #resolvePeerId (peerConfig) { - if ( !peerConfig || typeof peerConfig !== 'object' ) return null; - const peerId = peerConfig.peerId ?? peerConfig.key; - if ( typeof peerId !== 'string' || peerId.trim() === '' ) return null; - return peerId.trim(); - } - - #isNonceReplayForPeer ({ timestamp, nonce, peerId }) { - const lastSeen = this.#incomingLastNonceByPeer.get(peerId); - if ( ! lastSeen ) return false; - - // A newer timestamp should reset nonce ordering for this peer. - if ( timestamp > lastSeen.timestamp ) return false; - if ( timestamp < lastSeen.timestamp ) return true; - return nonce <= lastSeen.nonce; - } - - async #initRedisPubSub () { - if ( typeof redisClient?.duplicate !== 'function' ) { - console.warn('redis pubsub unavailable; duplicate client is not supported'); - return; - } - - try { - this.#redisSubscriber = redisClient.duplicate(); - this.#redisSubscriber.on('error', error => { - console.warn('redis pubsub subscriber error', { error }); - }); - this.#redisSubscriber.on('message', (channel, message) => { - this.#handleRedisPubSubMessage(channel, message).catch(error => { - console.warn('redis pubsub message handling error', { error }); - }); - }); - await this.#redisSubscriber.subscribe(this.#redisPubSubChannel); - } catch ( error ) { - console.warn('failed to initialize redis pubsub subscriber', { error }); - this.#redisSubscriber = null; - } - } - - #isRedisWebhookEventKey (key) { - if ( typeof key !== 'string' ) return false; - return key === 'outer.pub' || - key.startsWith('outer.pub.'); - } - - #filterRedisWebhookEvents (events) { - return events.filter(event => this.#isRedisWebhookEventKey(event?.key)); - } - - async #publishWebhookEventsToRedis (events) { - if ( !Array.isArray(events) || events.length === 0 ) return; - - const eventsToPublish = this.#filterRedisWebhookEvents(events); - if ( eventsToPublish.length === 0 ) return; - - let payload; - try { - payload = JSON.stringify({ - sourceId: this.#redisSourceId, - events: eventsToPublish, - }); - } catch ( error ) { - console.warn('redis pubsub publish failed: payload not serializable', { error }); - return; - } - - try { - await redisClient.publish(this.#redisPubSubChannel, payload); - } catch ( error ) { - console.warn('redis pubsub publish failed', { error }); - } - } - - async #handleRedisPubSubMessage (channel, message) { - if ( channel !== this.#redisPubSubChannel ) return; - - let payload; - try { - payload = JSON.parse(message); - } catch { - console.warn('invalid redis pubsub payload: not json'); - return; - } - - if ( !payload || typeof payload !== 'object' || Array.isArray(payload) ) { - console.warn('invalid redis pubsub payload: expected object'); - return; - } - - if ( payload.sourceId && payload.sourceId === this.#redisSourceId ) { - return; - } - - const incomingEvents = this.#normalizeIncomingPayload(payload); - if ( ! incomingEvents ) { - console.warn('invalid redis pubsub payload: invalid events'); - return; - } - - const eventsToEmit = this.#filterRedisWebhookEvents(incomingEvents); - if ( eventsToEmit.length === 0 ) return; - - await this.#emitIncomingEventsSequentially(eventsToEmit); - } - - #normalizeIncomingPayload (payload) { - if ( !payload || typeof payload !== 'object' || Array.isArray(payload) ) { - return null; - } - - if ( Array.isArray(payload.events) ) { - const events = []; - for ( const event of payload.events ) { - const normalized = this.#normalizeIncomingEvent(event); - if ( ! normalized ) return null; - events.push(normalized); - } - return events; - } - - const normalized = this.#normalizeIncomingEvent(payload); - if ( ! normalized ) return null; - return [normalized]; - } - - #normalizeIncomingEvent (event) { - if ( !event || typeof event !== 'object' || Array.isArray(event) ) { - return null; - } - - const { key, data } = event; - if ( key === undefined || key === null ) { - return null; - } - if ( data === undefined ) { - return null; - } - - return { - key, - data, - meta: this.#normalizeMeta(event.meta), - }; - } - - async #emitIncomingEventsSequentially (events) { - const svcEvent = this.services.get('event'); - const context = Context.get(undefined, { allow_fallback: true }); - - for ( const event of events ) { - if ( event.meta?.from_outside ) { - console.warn('possible over-sending'); - continue; - } - - if ( event.key === 'test' ) { - console.debug(`test message: ${JSON.stringify(event.data)}`); - } - - const metaOut = { ...event.meta, from_outside: true }; - await context.arun(async () => { - await svcEvent.emit(event.key, event.data, metaOut); - }); - } - } - - async '__on_install.routes' (_, { app }) { - const svc_web = this.services.get('web-server'); - svc_web.allow_undefined_origin('/broadcast/webhook'); - - app.use(eggspress('/broadcast/webhook', { - allowedMethods: ['POST'], - }, this.#handleWebhookRequest.bind(this))); - } - - async #handleWebhookRequest (req, res) { - const rawBody = req.rawBody; - if ( rawBody === undefined || rawBody === null ) { - res.status(400).send({ error: { message: 'Missing or invalid body' } }); - return; - } - - const body = req.body; - if ( !body || typeof body !== 'object' ) { - res.status(400).send({ error: { message: 'Invalid JSON body' } }); - return; - } - - const incomingEvents = this.#normalizeIncomingPayload(body); - if ( ! incomingEvents ) { - res.status(400).send({ error: { message: 'Invalid broadcast payload' } }); - return; - } - - const peerIdHeader = req.headers['x-broadcast-peer-id']; - const peerId = Array.isArray(peerIdHeader) ? peerIdHeader[0] : peerIdHeader; - if ( ! peerId ) { - res.status(403).send({ error: { message: 'Missing X-Broadcast-Peer-Id' } }); - return; - } - const localPeerId = this.#resolveLocalPeerId(); - if ( localPeerId && peerId === localPeerId ) { - res.status(200).send({ ok: true, ignored: 'self-peer' }); - return; - } - - const peer = this.#peersByKey[peerId]; - if ( !peer || !peer.webhook_secret ) { - res.status(403).send({ error: { message: 'Unknown peer or webhook not configured' } }); - return; - } - - // Timestamp avoids nonce-reuse after a restart - const timestampHeader = req.headers['x-broadcast-timestamp']; - if ( ! timestampHeader ) { - res.status(400).send({ error: { message: 'Missing X-Broadcast-Timestamp' } }); - return; - } - const timestamp = Number(timestampHeader); - if ( Number.isNaN(timestamp) ) { - res.status(400).send({ error: { message: 'Invalid X-Broadcast-Timestamp' } }); - return; - } - const nowSeconds = Math.floor(Date.now() / 1000); - const window = this.#webhookReplayWindowSeconds; - if ( timestamp < nowSeconds - window || timestamp > nowSeconds + 60 ) { - res.status(400).send({ error: { message: 'Timestamp out of window' } }); - return; - } - - // Nonce avoids replay attacks - const nonceHeader = req.headers['x-broadcast-nonce']; - if ( nonceHeader === undefined || nonceHeader === null || nonceHeader === '' ) { - res.status(400).send({ error: { message: 'Missing X-Broadcast-Nonce' } }); - return; - } - const nonce = Number(nonceHeader); - if ( Number.isNaN(nonce) ) { - res.status(400).send({ error: { message: 'Invalid X-Broadcast-Nonce' } }); - return; - } - if ( this.#isNonceReplayForPeer({ timestamp, nonce, peerId }) ) { - res.status(403).send({ error: { message: 'Duplicate or stale nonce' } }); - return; - } - - // We verify a signature to ensure the message came from an authorized peer - const signatureHeader = req.headers['x-broadcast-signature']; - if ( ! signatureHeader ) { - res.status(403).send({ error: { message: 'Missing X-Broadcast-Signature' } }); - return; - } - - const payloadToSign = `${timestamp}.${nonce}.${rawBody}`; - const expectedHmac = createHmac('sha256', peer.webhook_secret).update(payloadToSign).digest('hex'); - const signatureBuffer = Buffer.from(signatureHeader, 'hex'); - const expectedBuffer = Buffer.from(expectedHmac, 'hex'); - if ( signatureBuffer.length !== expectedBuffer.length || !timingSafeEqual(signatureBuffer, expectedBuffer) ) { - res.status(403).send({ error: { message: 'Invalid signature' } }); - return; - } - - this.#incomingLastNonceByPeer.set(peerId, { timestamp, nonce }); - - await this.#publishWebhookEventsToRedis(incomingEvents); - await this.#emitIncomingEventsSequentially(incomingEvents); - - res.status(200).send({ ok: true }); - } - - async #sendWebhookToPeer (peer_config, events) { - const peerId = this.#resolvePeerId(peer_config); - if ( ! peerId ) return; - const url = peer_config.webhook_url; - const requestUrl = this.#normalizeWebhookUrl(url); - const mySecretKey = this.config.webhook?.secret ?? ''; - - if ( !requestUrl || !mySecretKey ) return; - - let nextNonce = this.#outgoingNonceByPeer.get(peerId) ?? 0; - this.#outgoingNonceByPeer.set(peerId, nextNonce + 1); - - const timestamp = Math.floor(Date.now() / 1000); - const body = { events }; - const rawBody = JSON.stringify(body); - const payloadToSign = `${timestamp}.${nextNonce}.${rawBody}`; - const signature = createHmac('sha256', mySecretKey).update(payloadToSign).digest('hex'); - - const myPublicKey = this.config.webhook?.peerId ?? this.config.webhook?.key ?? ''; - const headers = { - 'Content-Type': 'application/json', - 'Content-Length': String(Buffer.byteLength(rawBody)), - 'X-Broadcast-Peer-Id': myPublicKey, - 'X-Broadcast-Timestamp': String(timestamp), - 'X-Broadcast-Nonce': String(nextNonce), - 'X-Broadcast-Signature': signature, - ...(this.#webhookHostHeader ? { Host: this.#webhookHostHeader } : {}), - }; - - const response = await axios.request({ - method: 'POST', - url: requestUrl, - headers, - data: rawBody, - timeout: 15000, - validateStatus: () => true, - responseType: 'text', - transformResponse: value => value, - ...(requestUrl.startsWith('https:') - ? { httpsAgent: this.#webhookHttpsAgent } - : {}), - }); - - if ( response.status < 200 || response.status >= 300 ) { - console.warn(`error with body: ${response.data}`); - throw new Error(`Webhook POST failed: ${response.status} ${response.statusText}`); - } - } - - #normalizeWebhookUrl (url) { - if ( typeof url !== 'string' || url.trim() === '' ) { - return null; - } - - const urlValue = url.trim(); - let parsedUrl; - try { - parsedUrl = urlValue.includes('://') - ? new URL(urlValue) - : new URL(`${this.#webhookProtocol}://${urlValue}`); - } catch { - return null; - } - - parsedUrl.protocol = `${this.#webhookProtocol}:`; - return parsedUrl.toString(); - } -} diff --git a/src/backend/src/modules/broadcast/BroadcastService.redisPubSub.test.js b/src/backend/src/modules/broadcast/BroadcastService.redisPubSub.test.js deleted file mode 100644 index 5fb11d33b..000000000 --- a/src/backend/src/modules/broadcast/BroadcastService.redisPubSub.test.js +++ /dev/null @@ -1,157 +0,0 @@ -import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; -import { redisClient } from '../../clients/redis/redisSingleton.js'; -import { BroadcastService } from './BroadcastService.js'; - -const wait = (ms = 20) => new Promise(resolve => setTimeout(resolve, ms)); - -describe('BroadcastService redis pubsub', () => { - let eventService; - let service; - - beforeAll(async () => { - eventService = { - on: vi.fn(), - emit: vi.fn(async () => { - }), - }; - - service = new BroadcastService({ - services: { - get: (name) => { - if ( name === 'event' ) return eventService; - throw new Error(`unexpected service lookup: ${name}`); - }, - }, - config: { - domain: 'puter.com', - protocol: 'https', - server_id: 'test-broadcast-a', - services: { - broadcast: { - peers: [], - }, - }, - }, - name: 'broadcast', - args: {}, - context: { - get: () => ({ use: () => ({}) }), - }, - }); - - await service._init(); - }); - - afterAll(async () => { - }); - - beforeEach(() => { - eventService.emit.mockClear(); - }); - - it('re-emits only outer.pub events from redis pubsub payloads', async () => { - await redisClient.publish('broadcast.webhook.events', JSON.stringify({ - sourceId: 'other-instance', - events: [ - { key: 'outer.gui.notif.message', data: { id: 'gui-1' }, meta: {} }, - { key: 'outer.pub.notice', data: { id: 'pub-1' }, meta: {} }, - { key: 'outer.cacheUpdate', data: { cacheKey: 'skip-me' }, meta: {} }, - ], - })); - - await wait(); - - expect(eventService.emit).toHaveBeenCalledTimes(1); - expect(eventService.emit).toHaveBeenNthCalledWith( - 1, - 'outer.pub.notice', - { id: 'pub-1' }, - expect.objectContaining({ from_outside: true }), - ); - }); - - it('ignores malformed redis pubsub payloads', async () => { - await redisClient.publish('broadcast.webhook.events', 'not-json'); - await wait(); - - await redisClient.publish('broadcast.webhook.events', JSON.stringify({ - sourceId: 'other-instance', - events: [{ bad: 'shape' }], - })); - await wait(); - - expect(eventService.emit).not.toHaveBeenCalled(); - }); - - it('publishes local outer.pub events to redis pubsub for replicas', async () => { - const publishSpy = vi.spyOn(redisClient, 'publish'); - try { - await service.outBroadcastEventHandler('outer.pub.notice', { id: 'pub-local' }, {}); - await wait(); - - const publishCall = publishSpy.mock.calls.find(([channel]) => channel === 'broadcast.webhook.events'); - expect(publishCall).toBeDefined(); - const [channel, payload] = publishCall; - expect(channel).toBe('broadcast.webhook.events'); - - const parsedPayload = JSON.parse(payload); - expect(parsedPayload.sourceId).toBeDefined(); - expect(parsedPayload.events).toEqual([ - { - key: 'outer.pub.notice', - data: { id: 'pub-local' }, - meta: {}, - }, - ]); - } finally { - publishSpy.mockRestore(); - } - }); - - it('does not publish local outer.gui events to redis pubsub', async () => { - const publishSpy = vi.spyOn(redisClient, 'publish'); - try { - await service.outBroadcastEventHandler('outer.gui.notif.message', { id: 'gui-local' }, {}); - await wait(); - - const publishCall = publishSpy.mock.calls.find(([channel]) => channel === 'broadcast.webhook.events'); - expect(publishCall).toBeUndefined(); - } finally { - publishSpy.mockRestore(); - } - }); - - it('does not rebroadcast events marked from_outside', async () => { - const publishSpy = vi.spyOn(redisClient, 'publish'); - try { - await service.outBroadcastEventHandler('outer.gui.notif.message', { id: 'outside' }, { - from_outside: true, - }); - await wait(); - - expect(publishSpy).not.toHaveBeenCalled(); - } finally { - publishSpy.mockRestore(); - } - }); - - it('ignores redis pubsub payloads with this instance sourceId', async () => { - const publishSpy = vi.spyOn(redisClient, 'publish'); - try { - await service.outBroadcastEventHandler('outer.pub.notice', { id: 'self-source' }, {}); - await wait(); - - const publishCall = publishSpy.mock.calls.find(([channel]) => channel === 'broadcast.webhook.events'); - expect(publishCall).toBeDefined(); - const [_channel, payload] = publishCall; - - eventService.emit.mockClear(); - await redisClient.publish('broadcast.webhook.events', payload); - await wait(); - - expect(eventService.emit).not.toHaveBeenCalled(); - } finally { - publishSpy.mockRestore(); - } - }); -}); diff --git a/src/backend/src/modules/captcha/CaptchaModule.js b/src/backend/src/modules/captcha/CaptchaModule.js deleted file mode 100644 index 4a28027ad..000000000 --- a/src/backend/src/modules/captcha/CaptchaModule.js +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { AdvancedBase } = require('@heyputer/putility'); -const CaptchaService = require('./services/CaptchaService'); - -/** - * @class CaptchaModule - * @extends AdvancedBase - * @description Module that provides captcha verification functionality to protect - * against automated abuse, particularly for login and signup flows. Registers - * a CaptchaService for generating and verifying captchas as well as middlewares - * that can be used to protect routes and determine captcha requirements. - */ -class CaptchaModule extends AdvancedBase { - async install (context) { - - // Get services from context - const services = context.get('services'); - - // Register the captcha service - services.registerService('captcha', CaptchaService); - } -} - -module.exports = { CaptchaModule }; \ No newline at end of file diff --git a/src/backend/src/modules/captcha/README.md b/src/backend/src/modules/captcha/README.md deleted file mode 100644 index cf7796103..000000000 --- a/src/backend/src/modules/captcha/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# Captcha Module - -This module provides captcha verification functionality to protect against automated abuse, particularly for login and signup flows. - -## Components - -- **CaptchaModule.js**: Registers the service and middleware -- **CaptchaService.js**: Provides captcha generation and verification functionality -- **captcha-middleware.js**: Express middleware for protecting routes with captcha verification - -## Integration - -The CaptchaService is registered by the CaptchaModule and can be accessed by other services: - -```javascript -const captchaService = services.get('captcha'); -``` - -### Example Usage - -```javascript -// Generate a captcha -const captcha = captchaService.generateCaptcha(); -// captcha.token - The token to verify later -// captcha.image - SVG image data to display to the user - -// Verify a captcha -const isValid = captchaService.verifyCaptcha(token, userAnswer); -``` - -## Configuration - -The CaptchaService can be configured with the following options in the configuration file (`config.json`): - -- `captcha.enabled`: Whether the captcha service is enabled (default: false) -- `captcha.expirationTime`: How long captcha tokens are valid in milliseconds (default: 10 minutes) -- `captcha.difficulty`: The difficulty level of the captcha ('easy', 'medium', 'hard') (default: 'medium') - -These options are set in the main configuration file. For example: - -```json -{ - "services": { - "captcha": { - "enabled": false, - "expirationTime": 600000, - "difficulty": "medium" - } - } -} -``` - -### Development Configuration - -For local development, you can disable captcha by creating or modifying your local configuration file (e.g., in `volatile/config/config.json` or using a profile configuration): - -```json -{ - "$version": "v1.1.0", - "$requires": [ - "config.json" - ], - "config_name": "local", - - "services": { - "captcha": { - "enabled": false - } - } -} -``` - -These options are set when registering the service in CaptchaModule.js. \ No newline at end of file diff --git a/src/backend/src/modules/captcha/middleware/README.md b/src/backend/src/modules/captcha/middleware/README.md deleted file mode 100644 index 019df6391..000000000 --- a/src/backend/src/modules/captcha/middleware/README.md +++ /dev/null @@ -1,160 +0,0 @@ -# Captcha Middleware - -This middleware provides captcha verification for routes that need protection against automated abuse. - -## Middleware Components - -The captcha system is now split into two middleware components: - -1. **checkCaptcha**: Determines if captcha verification is required but doesn't perform verification. -2. **requireCaptcha**: Performs actual captcha verification based on the result from checkCaptcha. - -This split allows frontend applications to know in advance whether captcha verification will be needed for a particular action. - -## Usage Patterns - -### Using Both Middlewares (Recommended) - -For best user experience, use both middlewares together: - -```javascript -const express = require('express'); -const router = express.Router(); - -// Get both middleware components from the context -const { checkCaptcha, requireCaptcha } = context.get('captcha-middleware'); - -// Determine if captcha is required for this route -router.post('/login', checkCaptcha({ eventType: 'login' }), (req, res, next) => { - // Set a flag in the response so frontend knows if captcha is needed - res.locals.captchaRequired = req.captchaRequired; - next(); -}, requireCaptcha(), (req, res) => { - // Handle login logic - // If captcha was required, it has been verified at this point -}); -``` - -### Using Individual Middlewares - -You can also access each middleware separately: - -```javascript -const checkCaptcha = context.get('check-captcha-middleware'); -const requireCaptcha = context.get('require-captcha-middleware'); -``` - -### Using Only requireCaptcha (Legacy Mode) - -For backward compatibility, you can still use only the requireCaptcha middleware: - -```javascript -const requireCaptcha = context.get('require-captcha-middleware'); - -// Always require captcha for this route -router.post('/sensitive-route', requireCaptcha({ always: true }), (req, res) => { - // Route handler -}); - -// Conditionally require captcha based on extensions -router.post('/normal-route', requireCaptcha(), (req, res) => { - // Route handler -}); -``` - -## Configuration Options - -### checkCaptcha Options - -- `always` (boolean): Always require captcha regardless of other factors -- `strictMode` (boolean): If true, fails closed on errors (more secure) -- `eventType` (string): Type of event for extensions (e.g., 'login', 'signup') - -### requireCaptcha Options - -- `strictMode` (boolean): If true, fails closed on errors (more secure) - -## Frontend Integration - -There are two ways to integrate with the frontend: - -### 1. Using the checkCaptcha Result in API Responses - -You can include the captcha requirement in API responses: - -```javascript -router.get('/whoarewe', checkCaptcha({ eventType: 'login' }), (req, res) => { - res.json({ - // Other environment information - captchaRequired: { - login: req.captchaRequired - } - }); -}); -``` - -### 2. Setting GUI Parameters - -For PuterHomepageService, you can add captcha requirements to GUI parameters: - -```javascript -// In PuterHomepageService.js -gui_params: { - // Other parameters - captchaRequired: { - login: req.captchaRequired - } -} -``` - -## Client-Side Integration - -To integrate with the captcha middleware, the client needs to: - -1. Check if captcha is required for the action (using /whoarewe or GUI parameters) -2. If required, call the `/api/captcha/generate` endpoint to get a captcha token and image -3. Display the captcha image to the user and collect their answer -4. Include the captcha token and answer in the request body: - -```javascript -// Example client-side code -async function submitWithCaptcha(formData) { - // Check if captcha is required - const envInfo = await fetch('/api/whoarewe').then(r => r.json()); - - if (envInfo.captchaRequired?.login) { - // Get and display captcha to user - const captcha = await getCaptchaFromServer(); - showCaptchaToUser(captcha); - - // Add captcha token and answer to the form data - formData.captchaToken = captcha.token; - formData.captchaAnswer = await getUserCaptchaAnswer(); - } - - // Submit the form - const response = await fetch('/api/login', { - method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify(formData) - }); - - // Handle response - const data = await response.json(); - if (response.status === 400 && data.error === 'captcha_required') { - // Show captcha to the user if not already shown - showCaptcha(); - } -} -``` - -## Error Handling - -The middleware will throw the following errors: - -- `captcha_required`: When captcha verification is required but no token or answer was provided. -- `captcha_invalid`: When the provided captcha answer is incorrect. - -These errors can be caught by the API error handler and returned to the client. \ No newline at end of file diff --git a/src/backend/src/modules/captcha/middleware/captcha-middleware.js b/src/backend/src/modules/captcha/middleware/captcha-middleware.js deleted file mode 100644 index c75a83ec8..000000000 --- a/src/backend/src/modules/captcha/middleware/captcha-middleware.js +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const APIError = require('../../../api/APIError'); -const { Context } = require('../../../util/context'); - -/** - * Middleware that checks if captcha verification is required - * This is the "first half" of the captcha verification process - * It determines if verification is needed but doesn't perform verification - * - * @param {Object} options - Configuration options - * @param {boolean} [options.strictMode=true] - If true, fails closed on errors (more secure) - * @returns {Function} Express middleware function - */ -const checkCaptcha = ({ svc_captcha }) => async (req, res, next) => { - // Get services from the Context - const services = Context.get('services'); - - if ( ! svc_captcha.enabled ) { - req.captchaRequired = false; - return next(); - } - const ip = req.headers?.['x-forwarded-for'] || - req.connection?.remoteAddress; - - const svc_event = services.get('event'); - const event = { - ip, - // By default, captcha always appears if enabled - required: true, - }; - await svc_event.emit('captcha.check', event); - - // Set captcha requirement based on service status - req.captchaRequired = event.required; - next(); -}; - -/** - * Middleware that requires captcha verification - * This is the "second half" of the captcha verification process - * It uses the result from checkCaptcha to determine if verification is needed - * - * @param {Object} options - Configuration options - * @param {boolean} [options.strictMode=true] - If true, fails closed on errors (more secure) - * @returns {Function} Express middleware function - */ -const requireCaptcha = (options = {}) => async (req, res, next) => { - if ( ! req.captchaRequired ) { - return next(); - } - - const services = Context.get('services'); - - try { - let captchaService; - try { - captchaService = services.get('captcha'); - } catch ( error ) { - console.warn('Captcha verification: required service not available', error); - return next(APIError.create('internal_error', null, { - message: 'Captcha service unavailable', - status: 503, - })); - } - - // Fail closed if captcha service doesn't exist or isn't properly initialized - if ( !captchaService || typeof captchaService.verifyCaptcha !== 'function' ) { - return next(APIError.create('internal_error', null, { - message: 'Captcha service misconfigured', - status: 500, - })); - } - - // Check for captcha token and answer in request - const captchaToken = req.body.captchaToken; - const captchaAnswer = req.body.captchaAnswer; - - if ( !captchaToken || !captchaAnswer ) { - return next(APIError.create('captcha_required', null, { - message: 'Captcha verification required', - status: 400, - })); - } - - // Verify the captcha - let isValid; - try { - isValid = captchaService.verifyCaptcha(captchaToken, captchaAnswer); - } catch ( verifyError ) { - console.error('Captcha verification: threw an error', verifyError); - return next(APIError.create('captcha_invalid', null, { - message: 'Captcha verification failed', - status: 400, - })); - } - - // Check verification result - if ( ! isValid ) { - return next(APIError.create('captcha_invalid', null, { - message: 'Invalid captcha response', - status: 400, - })); - } - - // Captcha verified successfully, continue - next(); - } catch ( error ) { - console.error('Captcha verification: unexpected error', error); - return next(APIError.create('internal_error', null, { - message: 'Captcha verification failed', - status: 500, - })); - } -}; - -module.exports = { - checkCaptcha, - requireCaptcha, -}; \ No newline at end of file diff --git a/src/backend/src/modules/captcha/services/CaptchaService.js b/src/backend/src/modules/captcha/services/CaptchaService.js deleted file mode 100644 index d76eb1ec9..000000000 --- a/src/backend/src/modules/captcha/services/CaptchaService.js +++ /dev/null @@ -1,644 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const BaseService = require('../../../services/BaseService'); -const eggspress = require('../../../api/eggspress'); -const { checkCaptcha } = require('../middleware/captcha-middleware'); - -/** - * @class CaptchaService - * @extends BaseService - * @description Service that provides captcha generation and verification functionality - * to protect against automated abuse. Uses svg-captcha for generation and maintains - * a token-based verification system. - */ -class CaptchaService extends BaseService { - /** - * Initializes the captcha service with configuration and storage - */ - async _construct () { - // Load dependencies - this.crypto = require('crypto'); - this.svgCaptcha = require('svg-captcha'); - - // In-memory token storage with expiration - this.captchaTokens = new Map(); - - // Service instance diagnostic tracking - this.serviceId = Math.random().toString(36).substring(2, 10); - this.requestCounter = 0; - - // Get configuration from service config - this.enabled = this.config.enabled === true; - this.expirationTime = this.config.expirationTime || (10 * 60 * 1000); // 10 minutes default - this.difficulty = this.config.difficulty || 'medium'; - this.testMode = this.config.testMode === true; - - // Add a static test token for diagnostic purposes - this.captchaTokens.set('test-static-token', { - text: 'testanswer', - expiresAt: Date.now() + (365 * 24 * 60 * 60 * 1000), // 1 year - }); - - // Flag to track if endpoints are registered - this.endpointsRegistered = false; - } - - async '__on_install.middlewares.context-aware' (_, { app }) { - // Add express middleware - app.use(checkCaptcha({ svc_captcha: this })); - } - - /** - * Sets up API endpoints and cleanup tasks - */ - async _init () { - if ( ! this.enabled ) { - this.log.debug('Captcha service is disabled'); - return; - } - - // Set up periodic cleanup - this.cleanupInterval = setInterval(() => this.cleanupExpiredTokens(), 15 * 60 * 1000); - - // Register endpoints if not already done - if ( ! this.endpointsRegistered ) { - this.registerEndpoints(); - this.endpointsRegistered = true; - } - } - - /** - * Cleanup method called when service is being destroyed - */ - async _destroy () { - if ( this.cleanupInterval ) { - clearInterval(this.cleanupInterval); - } - this.captchaTokens.clear(); - } - - /** - * Registers the captcha API endpoints with the web service - * @private - */ - registerEndpoints () { - if ( this.endpointsRegistered ) { - return; - } - - try { - // Try to get the web service - let webService = null; - try { - webService = this.services.get('web-service'); - } catch ( error ) { - // Web service not available, try web-server - try { - webService = this.services.get('web-server'); - } catch ( innerError ) { - this.log.warn('Neither web-service nor web-server are available yet'); - return; - } - } - - if ( !webService || !webService.app ) { - this.log.warn('Web service found but app is not available'); - return; - } - - const app = webService.app; - - const api = this.require('express').Router(); - app.use('/api/captcha', api); - - // Generate captcha endpoint - api.use(eggspress('/generate', { - allowedMethods: ['GET'], - }, async (req, res) => { - const captcha = this.generateCaptcha(); - res.json({ - token: captcha.token, - image: captcha.data, - }); - })); - - // Verify captcha endpoint - api.use(eggspress('/verify', { - allowedMethods: ['POST'], - }, (req, res) => { - const { token, answer } = req.body; - - if ( !token || !answer ) { - return res.status(400).json({ - valid: false, - error: 'Missing token or answer', - }); - } - - const isValid = this.verifyCaptcha(token, answer); - res.json({ valid: isValid }); - })); - - // Special endpoint for automated testing - // This should be disabled in production - if ( this.testMode ) { - app.post('/api/captcha/create-test-token', (req, res) => { - try { - const { token, answer } = req.body; - - if ( !token || !answer ) { - return res.status(400).json({ - error: 'Missing token or answer', - }); - } - - // Store the test token with the provided answer - this.captchaTokens.set(token, { - text: answer.toLowerCase(), - expiresAt: Date.now() + this.expirationTime, - }); - - this.log.debug(`Created test token: ${token} with answer: ${answer}`); - res.json({ success: true }); - } catch ( error ) { - this.log.error(`Error creating test token: ${error.message}`); - res.status(500).json({ error: 'Failed to create test token' }); - } - }); - } - - // Diagnostic endpoint - should be used carefully and only during debugging - app.get('/api/captcha/diagnostic', (req, res) => { - try { - // Get information about the current state - const diagnosticInfo = { - serviceEnabled: this.enabled, - difficulty: this.difficulty, - expirationTime: this.expirationTime, - testMode: this.testMode, - activeTokenCount: this.captchaTokens.size, - serviceId: this.serviceId, - processId: process.pid, - requestCounter: this.requestCounter, - hasStaticTestToken: this.captchaTokens.has('test-static-token'), - tokensState: Array.from(this.captchaTokens).map(([token, data]) => ({ - tokenPrefix: `${token.substring(0, 8) }...`, - expiresAt: new Date(data.expiresAt).toISOString(), - expired: data.expiresAt < Date.now(), - expectedAnswer: data.text, - })), - }; - - res.json(diagnosticInfo); - } catch ( error ) { - this.log.error(`Error in diagnostic endpoint: ${error.message}`); - res.status(500).json({ error: 'Diagnostic error' }); - } - }); - - // Advanced token debugging endpoint - allows testing - app.get('/api/captcha/debug-tokens', (req, res) => { - try { - // Check if we're the same service instance - const currentTimestamp = Date.now(); - const currentTokens = Array.from(this.captchaTokens.keys()).map(t => t.substring(0, 8)); - - // Create a test token that won't expire soon - const debugToken = `debug-${ this.crypto.randomBytes(8).toString('hex')}`; - const debugAnswer = 'test123'; - - this.captchaTokens.set(debugToken, { - text: debugAnswer, - expiresAt: currentTimestamp + (60 * 60 * 1000), // 1 hour - }); - - // Information about the current service instance - const serviceInfo = { - message: 'Debug token created - use for testing captcha validation', - serviceId: this.serviceId, - debugToken: debugToken, - debugAnswer: debugAnswer, - tokensBefore: currentTokens, - tokensAfter: Array.from(this.captchaTokens.keys()).map(t => t.substring(0, 8)), - currentTokenCount: this.captchaTokens.size, - timestamp: currentTimestamp, - processId: process.pid, - }; - - res.json(serviceInfo); - } catch ( error ) { - this.log.error(`Error in debug-tokens endpoint: ${error.message}`); - res.status(500).json({ error: 'Debug token creation error' }); - } - }); - - // Configuration verification endpoint - app.get('/api/captcha/config-status', (req, res) => { - try { - // Information about configuration states - const configInfo = { - serviceEnabled: this.enabled, - serviceDifficulty: this.difficulty, - configSource: 'Service configuration', - centralConfig: { - enabled: this.enabled, - difficulty: this.difficulty, - expirationTime: this.expirationTime, - testMode: this.testMode, - }, - usingCentralizedConfig: true, - configConsistency: this.enabled === (this.enabled === true), - serviceId: this.serviceId, - processId: process.pid, - }; - - res.json(configInfo); - } catch ( error ) { - this.log.error(`Error in config-status endpoint: ${error.message}`); - res.status(500).json({ error: 'Configuration status error' }); - } - }); - - // Test endpoint to validate token lifecycle - app.get('/api/captcha/test-lifecycle', (req, res) => { - try { - // Create a test captcha - const testText = 'test123'; - const testToken = `lifecycle-${ this.crypto.randomBytes(16).toString('hex')}`; - - // Store the test token - this.captchaTokens.set(testToken, { - text: testText, - expiresAt: Date.now() + this.expirationTime, - }); - - // Verify the token exists - const tokenExists = this.captchaTokens.has(testToken); - // Try to verify with correct answer - const correctVerification = this.verifyCaptcha(testToken, testText); - // Check if token was deleted after verification - const tokenAfterVerification = this.captchaTokens.has(testToken); - - // Create another test token - const testToken2 = `lifecycle2-${ this.crypto.randomBytes(16).toString('hex')}`; - - // Store the test token - this.captchaTokens.set(testToken2, { - text: testText, - expiresAt: Date.now() + this.expirationTime, - }); - - res.json({ - message: 'Token lifecycle test completed', - serviceId: this.serviceId, - initialTokens: this.captchaTokens.size - 2, // minus the two we added - tokenCreated: true, - tokenExisted: tokenExists, - verificationResult: correctVerification, - tokenRemovedAfterVerification: !tokenAfterVerification, - secondTokenCreated: this.captchaTokens.has(testToken2), - processId: process.pid, - }); - } catch ( error ) { - console.error('TOKENS_TRACKING: Error in test-lifecycle endpoint:', error); - res.status(500).json({ error: 'Test lifecycle error' }); - } - }); - - this.endpointsRegistered = true; - this.log.debug('Captcha service endpoints registered successfully'); - - // Emit an event that captcha service is ready - try { - const eventService = this.services.get('event'); - if ( eventService ) { - eventService.emit('service-ready', 'captcha'); - } - } catch ( error ) { - // Ignore errors with event service - } - } catch ( error ) { - this.log.warn(`Could not register captcha endpoints: ${error.message}`); - } - } - - /** - * Generates a new captcha with a unique token - * @returns {Object} Object containing token and SVG image - */ - generateCaptcha () { - console.log('====== CAPTCHA GENERATION DIAGNOSTIC ======'); - console.log('TOKENS_TRACKING: generateCaptcha called. Service ID:', this.serviceId); - console.log('TOKENS_TRACKING: Token map size before generation:', this.captchaTokens.size); - console.log('TOKENS_TRACKING: Static test token exists:', this.captchaTokens.has('test-static-token')); - - // Increment request counter for diagnostics - this.requestCounter++; - console.log('TOKENS_TRACKING: Request counter value:', this.requestCounter); - - console.log('generateCaptcha called, service enabled:', this.enabled); - - if ( ! this.enabled ) { - console.log('Generation SKIPPED: Captcha service is disabled'); - throw new Error('Captcha service is disabled'); - } - - // Configure captcha options based on difficulty - const options = this._getCaptchaOptions(); - console.log('Using captcha options for difficulty:', this.difficulty); - - // Generate the captcha - const captcha = this.svgCaptcha.create(options); - console.log('Captcha created with text:', captcha.text); - - // Generate a unique token - const token = this.crypto.randomBytes(32).toString('hex'); - console.log('Generated token:', `${token.substring(0, 8) }...`); - - // Store token with captcha text and expiration - const expirationTime = Date.now() + this.expirationTime; - console.log('Token will expire at:', new Date(expirationTime)); - - console.log('TOKENS_TRACKING: Token map size before storing new token:', this.captchaTokens.size); - - this.captchaTokens.set(token, { - text: captcha.text.toLowerCase(), - expiresAt: expirationTime, - }); - - console.log('TOKENS_TRACKING: Token map size after storing new token:', this.captchaTokens.size); - console.log('Token stored in captchaTokens. Current token count:', this.captchaTokens.size); - this.log.debug(`Generated captcha with token: ${token}`); - - return { - token: token, - data: captcha.data, - }; - } - - /** - * Verifies a captcha answer against a stored token - * @param {string} token - The captcha token - * @param {string} userAnswer - The user's answer to verify - * @returns {boolean} Whether the answer is valid - */ - verifyCaptcha (token, userAnswer) { - console.debug('====== CAPTCHA SERVICE VERIFICATION DIAGNOSTIC ======'); - console.debug('TOKENS_TRACKING: verifyCaptcha called. Service ID:', this.serviceId); - console.debug('TOKENS_TRACKING: Request counter during verification:', this.requestCounter); - console.debug('TOKENS_TRACKING: Static test token exists:', this.captchaTokens.has('test-static-token')); - console.debug('TOKENS_TRACKING: Trying to verify token:', token ? `${token.substring(0, 8) }...` : 'undefined'); - console.debug('verifyCaptcha called with token:', token ? `${token.substring(0, 8) }...` : 'undefined'); - console.debug('userAnswer:', userAnswer); - console.debug('Service enabled:', this.enabled); - console.debug('Number of tokens in captchaTokens:', this.captchaTokens.size); - - // Service health check - this._checkServiceHealth(); - - if ( ! this.enabled ) { - console.log('Verification SKIPPED: Captcha service is disabled'); - this.log.warn('Captcha verification attempted while service is disabled'); - throw new Error('Captcha service is disabled'); - } - - // Get captcha data for token - const captchaData = this.captchaTokens.get(token); - console.log('Captcha data found for token:', !!captchaData); - - // Invalid token or expired - if ( ! captchaData ) { - console.log('Verification FAILED: No data found for this token'); - console.log( - 'TOKENS_TRACKING: Available tokens (first 8 chars):', - Array.from(this.captchaTokens.keys()).map(t => t.substring(0, 8)), - ); - this.log.debug(`Invalid captcha token: ${token}`); - return false; - } - - if ( captchaData.expiresAt < Date.now() ) { - console.log('Verification FAILED: Token expired at:', new Date(captchaData.expiresAt)); - this.log.debug(`Expired captcha token: ${token}`); - return false; - } - - // Normalize and compare answers - const normalizedUserAnswer = userAnswer.toLowerCase().trim(); - console.log('Expected answer:', captchaData.text); - console.log('User answer (normalized):', normalizedUserAnswer); - const isValid = captchaData.text === normalizedUserAnswer; - console.log('Answer comparison result:', isValid); - - // Remove token after verification (one-time use) - this.captchaTokens.delete(token); - console.log('Token removed after verification (one-time use)'); - console.log('TOKENS_TRACKING: Token map size after removing used token:', this.captchaTokens.size); - - this.log.debug(`Verified captcha token: ${token}, valid: ${isValid}`); - return isValid; - } - - /** - * Simple diagnostic method to check service health - * @private - */ - _checkServiceHealth () { - console.log('TOKENS_TRACKING: Service health check. ID:', this.serviceId, 'Token count:', this.captchaTokens.size); - return true; - } - - /** - * Removes expired captcha tokens from memory - */ - cleanupExpiredTokens () { - console.log('TOKENS_TRACKING: Running token cleanup. Service ID:', this.serviceId); - console.log('TOKENS_TRACKING: Token map size before cleanup:', this.captchaTokens.size); - - const now = Date.now(); - let expiredCount = 0; - let validCount = 0; - - // Log all tokens before cleanup - console.log('TOKENS_TRACKING: Current tokens before cleanup:'); - for ( const [token, data] of this.captchaTokens.entries() ) { - const isExpired = data.expiresAt < now; - console.log(`TOKENS_TRACKING: Token ${token.substring(0, 8)}... expires: ${new Date(data.expiresAt).toISOString()}, expired: ${isExpired}`); - - if ( isExpired ) { - expiredCount++; - } else { - validCount++; - } - } - - // Only do the actual cleanup if we found expired tokens - if ( expiredCount > 0 ) { - console.log(`TOKENS_TRACKING: Found ${expiredCount} expired tokens to remove and ${validCount} valid tokens to keep`); - - // Clean up expired tokens - for ( const [token, data] of this.captchaTokens.entries() ) { - if ( data.expiresAt < now ) { - this.captchaTokens.delete(token); - console.log(`TOKENS_TRACKING: Deleted expired token: ${token.substring(0, 8)}...`); - } - } - } else { - console.log('TOKENS_TRACKING: No expired tokens found, skipping cleanup'); - } - - // Skip cleanup for the static test token - if ( this.captchaTokens.has('test-static-token') ) { - console.log('TOKENS_TRACKING: Static test token still exists after cleanup'); - } else { - console.log('TOKENS_TRACKING: WARNING - Static test token was removed during cleanup'); - - // Restore the static test token for diagnostic purposes - this.captchaTokens.set('test-static-token', { - text: 'testanswer', - expiresAt: Date.now() + (365 * 24 * 60 * 60 * 1000), // 1 year - }); - console.log('TOKENS_TRACKING: Restored static test token'); - } - - console.log('TOKENS_TRACKING: Token map size after cleanup:', this.captchaTokens.size); - - if ( expiredCount > 0 ) { - this.log.debug(`Cleaned up ${expiredCount} expired captcha tokens`); - } - } - - /** - * Gets captcha options based on the configured difficulty - * @private - * @returns {Object} Captcha configuration options - */ - _getCaptchaOptions () { - const baseOptions = { - size: 6, // Default captcha length - ignoreChars: '0o1ilI', // Characters to avoid (confusing) - noise: 2, // Lines to add as noise - color: true, - background: '#f0f0f0', - }; - - switch ( this.difficulty ) { - case 'easy': - return { - ...baseOptions, - size: 4, - width: 150, - height: 50, - noise: 1, - }; - case 'hard': - return { - ...baseOptions, - size: 7, - width: 200, - height: 60, - noise: 3, - }; - case 'medium': - default: - return { - ...baseOptions, - width: 180, - height: 50, - }; - } - } - - /** - * Verifies that the captcha service is properly configured and working - * This is used during initialization and can be called to check system status - * @returns {boolean} Whether the service is properly configured and functioning - */ - verifySelfTest () { - try { - // Ensure required dependencies are available - if ( ! this.svgCaptcha ) { - this.log.error('Captcha service self-test failed: svg-captcha module not available'); - return false; - } - - if ( ! this.enabled ) { - this.log.warn('Captcha service self-test failed: service is disabled'); - return false; - } - - // Validate configuration - if ( !this.expirationTime || typeof this.expirationTime !== 'number' ) { - this.log.error('Captcha service self-test failed: invalid expiration time configuration'); - return false; - } - - // Basic functionality test - generate a test captcha and verify storage - const testToken = `test-${ this.crypto.randomBytes(8).toString('hex')}`; - const testText = 'testcaptcha'; - - // Store the test captcha - this.captchaTokens.set(testToken, { - text: testText, - expiresAt: Date.now() + this.expirationTime, - }); - - // Verify the test captcha - const correctVerification = this.verifyCaptcha(testToken, testText); - - // Check if verification worked and token was removed - if ( !correctVerification || this.captchaTokens.has(testToken) ) { - this.log.error('Captcha service self-test failed: verification test failed'); - return false; - } - - this.log.debug('Captcha service self-test passed'); - return true; - } catch ( error ) { - this.log.error(`Captcha service self-test failed with error: ${error.message}`); - return false; - } - } - - /** - * Returns the service's diagnostic information - * @returns {Object} Diagnostic information about the service - */ - getDiagnosticInfo () { - return { - serviceId: this.serviceId, - enabled: this.enabled, - tokenCount: this.captchaTokens.size, - requestCounter: this.requestCounter, - config: { - enabled: this.enabled, - difficulty: this.difficulty, - expirationTime: this.expirationTime, - testMode: this.testMode, - }, - processId: process.pid, - testTokenExists: this.captchaTokens.has('test-static-token'), - }; - } -} - -// Export both as a named export and as a default export for compatibility -module.exports = CaptchaService; -module.exports.CaptchaService = CaptchaService; diff --git a/src/backend/src/modules/core/AlarmService.d.ts b/src/backend/src/modules/core/AlarmService.d.ts deleted file mode 100644 index a838aaf08..000000000 --- a/src/backend/src/modules/core/AlarmService.d.ts +++ /dev/null @@ -1,6 +0,0 @@ -export class AlarmService { - create (id: string, message: string, fields?: object): void; - clear (id: string): void; - get_alarm (id: string): object | undefined; - // Add more methods/properties as needed for MeteringService usage -} \ No newline at end of file diff --git a/src/backend/src/modules/core/AlarmService.js b/src/backend/src/modules/core/AlarmService.js deleted file mode 100644 index 966497559..000000000 --- a/src/backend/src/modules/core/AlarmService.js +++ /dev/null @@ -1,357 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const seedrandom = require('seedrandom'); -const util = require('util'); -const fs = require('fs'); - -const BaseService = require('../../services/BaseService.js'); - -/** - * AlarmService class is responsible for managing alarms. - * It provides methods for creating, clearing, and handling alarms. - */ -class AlarmService extends BaseService { - static USE = { - logutil: 'core.util.logutil', - identutil: 'core.util.identutil', - stdioutil: 'core.util.stdioutil', - Context: 'core.context', - }; - /** - * This method initializes the AlarmService by setting up its internal data structures and initializing any required dependencies. - * - * It reads in the known errors from a JSON5 file and sets them as the known_errors property of the AlarmService instance. - */ - async _construct () { - this.alarms = {}; - this.alarm_aliases = {}; - - this.known_errors = []; - this.isDraining = false; - this.drainSuppressionLogged = false; - } - /** - * Method to initialize AlarmService. Sets the known errors and registers commands. - * @returns {Promise} - */ - async _init () { - const services = this.services; - this.pager = services.get('pager'); - - // TODO:[self-hosted] fix this properly - this.known_errors = []; - - } - - adapt_id_ (id) { - let shorten = true; - - if ( shorten ) { - const rng = seedrandom(id); - id = this.identutil.generate_identifier('-', rng); - } - - return id; - } - - beginDrain (reason = 'shutdown') { - if ( this.isDraining ) return; - this.isDraining = true; - this.log.info(`alarm service entering drain mode: ${reason}`); - } - - /** - * Method to create an alarm with the given ID, message, and fields. - * If the ID already exists, it will be updated with the new fields - * and the occurrence count will be incremented. - * - * @param {string} id - Unique identifier for the alarm. - * @param {string} message - Message associated with the alarm. - * @param {object} fields - Additional information about the alarm. - */ - create (id, message, fields) { - if ( this.isDraining ) { - if ( ! this.drainSuppressionLogged ) { - this.drainSuppressionLogged = true; - this.log.info('suppressing alarm create/pager dispatch while draining'); - } - return; - } - - if ( this.config.log_upcoming_alarms ) { - this.log.error(`upcoming alarm: ${id}: ${message}`); - } - let existing = false; - /** - * Method to create an alarm with the given ID, message, and fields. - * If the ID already exists, it will be updated with the new fields. - * @param {string} id - Unique identifier for the alarm. - * @param {string} message - Message associated with the alarm. - * @param {object} fields - Additional information about the alarm. - * @returns {void} - */ - const alarm = (() => { - const short_id = this.adapt_id_(id); - - if ( this.alarms[id] ) { - existing = true; - return this.alarms[id]; - } - - const alarm = this.alarms[id] = this.alarm_aliases[short_id] = { - id, - short_id, - started: Date.now(), - occurrences: [], - }; - - Object.defineProperty(alarm, 'count', { - /** - * Method to create a new alarm. - * - * This method takes an id, message, and optional fields as parameters. - * It creates a new alarm object with the provided id and message, - * and adds it to the alarms object. It also keeps track of the number of occurrences of the alarm. - * If the alarm already exists, it increments the occurrence count and calls the handle\_alarm\_repeat\_ method. - * If it's a new alarm, it calls the handle\_alarm\_on\_ method. - * - * @param {string} id - The unique identifier for the alarm. - * @param {string} message - The message associated with the alarm. - * @param {object} [fields] - Optional fields associated with the alarm. - * @returns {void} - */ - get () { - return alarm.timestamps?.length ?? 0; - }, - }); - - Object.defineProperty(alarm, 'id_string', { - /** - * Method to handle creating a new alarm with given parameters. - * This method adds the alarm to the `alarms` object, updates the occurrences count, - * and processes any known errors that may apply to the alarm. - * @param {string} id - The unique identifier for the alarm. - * @param {string} message - The message associated with the alarm. - * @param {Object} fields - Additional fields to associate with the alarm. - */ - get () { - if ( alarm.id.length < 20 ) { - return alarm.id; - } - - const truncatedLongId = `${alarm.id.slice(0, 20) }...`; - - return `${alarm.short_id} (${truncatedLongId})`; - }, - }); - - return alarm; - })(); - - const occurance = { - message, - fields, - timestamp: Date.now(), - }; - - // Keep logs from the previous occurrence if: - // - it's one of the first 3 occurrences - // - the 10th, 100th, 1000th...etc occurrence - if ( alarm.count > 3 && Math.log10(alarm.count) % 1 !== 0 ) { - delete alarm.occurrences[alarm.occurrences.length - 1].logs; - } - occurance.logs = this.log.get_log_buffer(); - - alarm.message = message; - alarm.fields = { ...alarm.fields, ...fields }; - alarm.timestamps = (alarm.timestamps ?? []).concat(Date.now()); - alarm.occurrences.push(occurance); - - if ( fields?.error ) { - alarm.error = fields.error; - } - - if ( alarm.source ) { - console.error(alarm.error); - } - - if ( existing ) { - this.handle_alarm_repeat_(alarm); - } else { - this.handle_alarm_on_(alarm); - } - } - - /** - * Method to clear an alarm with the given ID. - * @param {*} id - The ID of the alarm to clear. - * @returns {void} - */ - clear (id) { - const alarm = this.alarms[id]; - if ( ! alarm ) { - return; - } - delete this.alarms[id]; - this.handle_alarm_off_(alarm); - } - - apply_known_errors_ (alarm) { - const rule_matches = rule => { - const match = rule.match; - if ( match.id !== alarm.id ) return false; - if ( match.message && match.message !== alarm.message ) return false; - if ( match.fields ) { - for ( const [key, value] of Object.entries(match.fields) ) { - if ( alarm.fields[key] !== value ) return false; - } - } - return true; - }; - - const rule_actions = { - 'no-alert': () => alarm.no_alert = true, - 'severity': action => alarm.severity = action.value, - }; - - const apply_action = action => { - rule_actions[action.type](action); - }; - - for ( const rule of this.known_errors ) { - if ( rule_matches(rule) ) apply_action(rule.action); - } - } - - handle_alarm_repeat_ (alarm) { - this.log.warn( - `REPEAT ${alarm.id_string} :: ${alarm.message} (${alarm.count})`, - alarm.fields, - ); - - this.apply_known_errors_(alarm); - - if ( alarm.no_alert ) return; - - const severity = alarm.severity ?? 'critical'; - - const fields_clean = {}; - for ( const [key, value] of Object.entries(alarm.fields) ) { - fields_clean[key] = util.inspect(value); - } - - this.pager.alert({ - id: alarm.id ?? 'something-bad', - message: alarm.message ?? alarm.id ?? 'something bad happened', - source: 'alarm-service', - severity, - custom: { - fields: fields_clean, - trace: alarm.error?.stack, - repeat_count: alarm.count, - }, - }); - } - - handle_alarm_on_ (alarm) { - this.log.error( - `ACTIVE ${alarm.id_string} :: ${alarm.message} (${alarm.count})`, - alarm.fields, - ); - - this.apply_known_errors_(alarm); - - if ( this.global_config.env === 'dev' && !this.attached_dev ) { - this.attached_dev = true; - const realConsole = globalThis.original_console_object ?? console; - realConsole.error('\x1B[33;1m[alarm]\x1B[0m Active alarms detected; see logs for details.'); - } - - const args = this.Context.get('args') ?? {}; - if ( args['quit-on-alarm'] ) { - console.log('shutting down: --quit-on-alarm is set'); - process.exit(1); - } - - if ( alarm.no_alert ) return; - - const severity = alarm.severity ?? 'critical'; - - const fields_clean = {}; - for ( const [key, value] of Object.entries(alarm.fields) ) { - fields_clean[key] = util.inspect(value); - } - - this.pager.alert({ - id: alarm.id ?? 'something-bad', - message: alarm.message ?? alarm.id ?? 'something bad happened', - source: 'alarm-service', - severity, - custom: { - fields: fields_clean, - trace: alarm.error?.stack, - }, - }); - - // Write a .log file for the alert that happened - try { - const lines = []; - lines.push(`ALERT ${alarm.id_string} :: ${alarm.message} (${alarm.count})`); - lines.push(`started: ${new Date(alarm.started).toISOString()}`); - lines.push(`short id: ${alarm.short_id}`); - lines.push(`original id: ${alarm.id}`); - lines.push(`severity: ${severity}`); - lines.push(`message: ${alarm.message}`); - lines.push(`fields: ${JSON.stringify(fields_clean)}`); - - const alert_info = lines.join('\n'); - - (async () => { - try { - fs.appendFileSync(`alert_${alarm.id}.log`, `${alert_info }\n`); - } catch (e) { - this.log.error(`failed to write alert log: ${e.message}`); - } - })(); - } catch (e) { - this.log.error(`failed to write alert log: ${e.message}`); - } - } - - handle_alarm_off_ (alarm) { - this.log.info( - `CLEAR ${alarm.id} :: ${alarm.message} (${alarm.count})`, - alarm.fields, - ); - } - - /** - * Method to get an alarm by its ID. - * - * @param {*} id - The ID of the alarm to get. - * @returns - */ - get_alarm (id) { - return this.alarms[id] ?? this.alarm_aliases[id]; - } -} - -module.exports = { - AlarmService, -}; diff --git a/src/backend/src/modules/core/ContextService.js b/src/backend/src/modules/core/ContextService.js deleted file mode 100644 index 2494bbcb2..000000000 --- a/src/backend/src/modules/core/ContextService.js +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const BaseService = require('../../services/BaseService'); -const { Context } = require('../../util/context'); - -/** - * ContextService provides a way for other services to register a hook to be - * called when a context/subcontext is created. - * - * Contexts are used to provide contextual information in the execution - * context (dynamic scope). They can also be used to identify a "span"; - * a span is a labelled frame of execution that can be used to track - * performance, errors, and other metrics. - */ -class ContextService extends BaseService { - register_context_hook (event, hook) { - Context.context_hooks_[event].push(hook); - } -} - -module.exports = { - ContextService, -}; diff --git a/src/backend/src/modules/core/Core2Module.js b/src/backend/src/modules/core/Core2Module.js deleted file mode 100644 index 32efc2993..000000000 --- a/src/backend/src/modules/core/Core2Module.js +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { AdvancedBase } = require('@heyputer/putility'); - -/** - * A replacement for CoreModule with as few external relative requires as possible. - * This will eventually be the successor to CoreModule, the main module for Puter's backend. - * - * The scope of this module is: - * - logging and error handling - * - alarm handling - * - services that are tightly coupled with alarm handling are allowed - * - any essential information about server stats or health - * - any very generic service which other services can register - * behavior to. - */ -class Core2Module extends AdvancedBase { - async install (context) { - // === LIBS === // - const useapi = context.get('useapi'); - - const lib = require('./lib/__lib__.js'); - for ( const k in lib ) { - useapi.def(`core.${k}`, lib[k], { assign: true }); - } - - useapi.def('core.context', require('../../util/context.js').Context); - - // === SERVICES === // - const services = context.get('services'); - - const { LogService } = require('./LogService.js'); - services.registerService('log-service', LogService); - - const { AlarmService } = require('./AlarmService.js'); - services.registerService('alarm', AlarmService); - - const { ErrorService } = require('./ErrorService.js'); - services.registerService('error-service', ErrorService); - - const { PagerService } = require('./PagerService.js'); - services.registerService('pager', PagerService); - - const { ProcessEventService } = require('./ProcessEventService.js'); - services.registerService('process-event', ProcessEventService); - - const { ServerHealthService } = require('./ServerHealthService/ServerHealthService.js'); - services.registerService('server-health', ServerHealthService); - - const { ParameterService } = require('./ParameterService.js'); - services.registerService('params', ParameterService); - - const { ContextService } = require('./ContextService.js'); - services.registerService('context', ContextService); - } -} - -module.exports = { - Core2Module, -}; diff --git a/src/backend/src/modules/core/ErrorService.js b/src/backend/src/modules/core/ErrorService.js deleted file mode 100644 index 972d65c65..000000000 --- a/src/backend/src/modules/core/ErrorService.js +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const BaseService = require('../../services/BaseService'); - -/** -* **ErrorContext Class** -* -* The `ErrorContext` class is designed to encapsulate error reporting functionality within a specific logging context. -* It facilitates the reporting of errors by providing a method to log error details along with additional contextual information. -* -* @class -* @classdesc Provides a context for error reporting with specific logging details. -* @param {ErrorService} error_service - The error service instance to use for reporting errors. -* @param {object} log_context - The logging context to associate with the error reports. -*/ -class ErrorContext { - constructor (error_service, log_context) { - this.error_service = error_service; - this.log_context = log_context; - } - report (location, fields) { - fields = { - ...fields, - logger: this.log_context, - }; - this.error_service.report(location, fields); - } -} - -/** -* The ErrorService class is responsible for handling and reporting errors within the system. -* It provides methods to initialize the service, create error contexts, and report errors with detailed logging and alarm mechanisms. - -* @class ErrorService -* @extends BaseService -*/ -class ErrorService extends BaseService { - /** - * Initializes the ErrorService, setting up the alarm and backup logger services. - * - * @async - * @function init - * @memberof ErrorService - * @returns {Promise} A promise that resolves when the initialization is complete. - */ - async init () { - const services = this.services; - this.alarm = services.get('alarm'); - this.backupLogger = services.get('log-service').create('error-service'); - } - - /** - * Creates an ErrorContext instance with the provided logging context. - * - * @param {*} log_context The logging context to associate with the error reports. - * @returns {ErrorContext} An ErrorContext instance. - */ - create (log_context) { - return new ErrorContext(this, log_context); - } - - /** - * Reports an error with the specified location and details. - * The "location" is a string up to the callers discretion to identify - * the source of the error. - * - * @param {*} location The location where the error occurred. - * @param {*} fields The error details to report. - * @param {boolean} [alarm=true] Whether to raise an alarm for the error. - * @returns {void} - */ - report (location, { source, logger, trace, extra, message }, alarm = true) { - message = message ?? source?.message; - logger = logger ?? this.backupLogger; - logger.error(`Error @ ${location}: ${message}; ${ source?.stack}`); - - if ( alarm ) { - const alarm_id = `${location}:${message}`; - this.alarm.create(alarm_id, message, { - error: source, - ...extra, - }); - } - } -} - -module.exports = { ErrorService }; diff --git a/src/backend/src/modules/core/LogService.js b/src/backend/src/modules/core/LogService.js deleted file mode 100644 index e09cfd412..000000000 --- a/src/backend/src/modules/core/LogService.js +++ /dev/null @@ -1,671 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const logSeverity = (ordinal, label, esc, winst) => ({ ordinal, label, esc, winst }); -const LOG_LEVEL_ERRO = logSeverity(0, 'ERRO', '31;1', 'error'); -const LOG_LEVEL_WARN = logSeverity(1, 'WARN', '33;1', 'warn'); -const LOG_LEVEL_INFO = logSeverity(2, 'INFO', '36;1', 'info'); -const LOG_LEVEL_NOTICEME = logSeverity(3, 'NOTICE_ME', '33;1', 'error'); -const LOG_LEVEL_SYSTEM = logSeverity(3, 'SYSTEM', '36;1', 'system'); -const LOG_LEVEL_DEBU = logSeverity(4, 'DEBU', '37', 'debug'); -const LOG_LEVEL_TICK = logSeverity(10, 'TICK', '34;1', 'info'); - -const winston = require('winston'); -const { Context } = require('../../util/context'); -const BaseService = require('../../services/BaseService'); -const { stringify_log_entry } = require('./lib/log'); -require('winston-daily-rotate-file'); - -const WINSTON_LEVELS = { - system: 0, - error: 1, - warn: 10, - info: 20, - http: 30, - verbose: 40, - debug: 50, - silly: 60, -}; - -let display_log_level = process.env.DEBUG ? 100 : 3; -const display_log_level_label = { - 0: 'ERRO', - 1: 'WARN', - 2: 'INFO', - 3: 'SYSTEM', - 4: 'DEBUG', - 100: 'ALL', -}; - -/** -* Represents a logging context within the LogService. -* This class is used to manage logging operations with specific context information, -* allowing for hierarchical logging structures and dynamic field additions. -* @class LogContext -*/ -class LogContext { - constructor (logService, { crumbs, fields }) { - this.logService = logService; - this.crumbs = crumbs; - this.fields = fields; - } - - sub (name, fields = {}) { - return new LogContext( - this.logService, - { - crumbs: name ? [...this.crumbs, name] : [...this.crumbs], - fields: { ...this.fields, ...fields }, - }, - ); - } - - info (message, fields, objects) { - this.log(LOG_LEVEL_INFO, message, fields, objects); - } - warn (message, fields, objects) { - this.log(LOG_LEVEL_WARN, message, fields, objects); - } - debug (message, fields, objects) { - this.log(LOG_LEVEL_DEBU, message, fields, objects); - } - error (message, fields, objects) { - this.log(LOG_LEVEL_ERRO, message, fields, objects); - } - tick (message, fields, objects) { - this.log(LOG_LEVEL_TICK, message, fields, objects); - } - called (fields = {}) { - this.log(LOG_LEVEL_DEBU, 'called', fields); - } - noticeme (message, fields, objects) { - this.log(LOG_LEVEL_NOTICEME, message, fields, objects); - } - system (message, fields, objects) { - this.log(LOG_LEVEL_SYSTEM, message, fields, objects); - } - - cache (isCacheHit, identifier, fields = {}) { - this.log( - LOG_LEVEL_DEBU, - isCacheHit ? 'cache_hit' : 'cache_miss', - { identifier, ...fields }, - ); - } - - log (log_level, message, fields = {}, objects = {}) { - fields = { ...this.fields, ...fields }; - { - const x = Context.get(undefined, { allow_fallback: true }); - if ( x && x.get('trace_request') ) { - fields.trace_request = x.get('trace_request'); - } - if ( !fields.actor && x && x.get('actor') ) { - try { - fields.actor = x.get('actor'); - } catch (e) { - console.log('error logging actor (this is probably fine):', e); - } - } - } - for ( const k in fields ) { - if ( - fields[k] && - typeof fields[k].toLogFields === 'function' - ) fields[k] = fields[k].toLogFields(); - } - if ( Context.get('injected_logger', { allow_fallback: true }) ) { - Context.get('injected_logger').log( - message + (fields ? (`; fields: ${ JSON.stringify(fields)}`) : ''), - ); - } - this.logService.log_( - log_level, - this.crumbs, - message, - fields, - objects, - ); - } - - /** - * Generates a human-readable trace ID for logging purposes. - * - * @returns {string} A trace ID in the format 'xxxxxx-xxxxxx' where each segment is a - * random string of six lowercase letters and digits. - */ - mkid () { - // generate trace id - const trace_id = []; - for ( let i = 0; i < 2; i++ ) { - trace_id.push(Math.random().toString(36).slice(2, 8)); - } - return trace_id.join('-'); - } - - /** - * Adds a trace id to this logging context for tracking purposes. - * @returns {LogContext} The current logging context with the trace id added. - */ - traceOn () { - this.fields.trace_id = this.mkid(); - return this; - } - - /** - * Gets the log buffer maintained by the LogService. This shows the most - * recent log entries. - * @returns {Array} An array of log entries stored in the buffer. - */ - get_log_buffer () { - return this.logService.get_log_buffer(); - } -} - -/** -* Timestamp in milliseconds since the epoch, used for calculating log entry duration. -*/ - -/** -* @class DevLogger -* @classdesc -* A development logger class designed for logging messages during development. -* This logger can either log directly to console or delegate logging to another logger. -* It provides functionality to turn logging on/off, and can optionally write logs to a file. -* -* @param {function} log - The logging function, typically `console.log` or similar. -* @param {object} [opt_delegate] - An optional logger to which log messages can be delegated. -*/ -class DevLogger { - // TODO: this should eventually delegate to winston logger - constructor (log, opt_delegate) { - this.log = log; - this.off = false; - this.recto = null; - - if ( opt_delegate ) { - this.delegate = opt_delegate; - } - } - onLogMessage (log_lvl, crumbs, message, fields, objects) { - if ( this.delegate ) { - this.delegate.onLogMessage(log_lvl, crumbs, message, fields, objects); - } - - if ( this.off ) return; - - if ( !process.env.DEBUG && log_lvl.ordinal > display_log_level ) return; - - const ld = Context.get('logdent', { allow_fallback: true }); - const prefix = globalThis.dev_console_indent_on - ? Array(ld ?? 0).fill(' ').join('') - : ''; - this.log_(stringify_log_entry({ - prefix, - log_lvl, - crumbs, - message, - fields, - objects, - })); - } - - log_ (text) { - if ( this.recto ) { - const fs = require('node:fs'); - fs.appendFileSync(this.recto, `${text }\n`); - } - this.log(text); - } -} - -/** -* @class NullLogger -* @description A logger that does nothing, effectively disabling logging. -* This class is used when logging is not desired or during development -* to avoid performance overhead or for testing purposes. -*/ -class NullLogger { - // TODO: this should eventually delegate to winston logger - constructor (log, opt_delegate) { - this.log = log; - - if ( opt_delegate ) { - this.delegate = opt_delegate; - } - } - onLogMessage () { - } -} - -/** -* WinstonLogger Class -* -* A logger that delegates log messages to a Winston logger instance. -*/ -class WinstonLogger { - constructor (winst) { - this.winst = winst; - } - onLogMessage (log_lvl, crumbs, message, fields) { - this.winst.log({ - ...fields, - label: crumbs.join('.'), - level: log_lvl.winst, - message, - }); - } -} - -/** -* @class TimestampLogger -* @classdesc A logger that adds timestamps to log messages before delegating them to another logger. -* This class wraps another logger instance to ensure that all log messages include a timestamp, -* which can be useful for tracking the sequence of events in a system. -* -* @param {Object} delegate - The logger instance to which the timestamped log messages are forwarded. -*/ -class TimestampLogger { - constructor (delegate) { - this.delegate = delegate; - } - onLogMessage (log_lvl, crumbs, message, fields, ...a) { - fields = { ...fields, timestamp: new Date() }; - this.delegate.onLogMessage(log_lvl, crumbs, message, fields, ...a); - } -} - -/** -* The `BufferLogger` class extends the logging functionality by maintaining a buffer of log entries. -* This class is designed to: -* - Store a specified number of recent log messages. -* - Allow for retrieval of these logs for debugging or monitoring purposes. -* - Ensure that the log buffer does not exceed the defined size by removing older entries when necessary. -* - Delegate logging messages to another logger while managing its own buffer. -*/ -class BufferLogger { - constructor (size, delegate) { - this.size = size; - this.delegate = delegate; - this.buffer = []; - } - onLogMessage (log_lvl, crumbs, message, fields, ...a) { - this.buffer.push({ log_lvl, crumbs, message, fields, ...a }); - if ( this.buffer.length > this.size ) { - this.buffer.shift(); - } - this.delegate.onLogMessage(log_lvl, crumbs, message, fields, ...a); - } -} - -/** -* Represents a custom logger that can modify log messages before they are passed to another logger. -* @class CustomLogger -* @extends {Object} -* @param {Object} delegate - The delegate logger to which modified log messages will be passed. -* @param {Function} callback - A callback function that modifies log parameters before delegation. -*/ -class CustomLogger { - constructor (delegate, callback) { - this.delegate = delegate; - this.callback = callback; - } - async onLogMessage (log_lvl, crumbs, message, fields, ...a) { - // Logging is allowed to be performed without a context, but we - // don't want log functions to be asynchronous which rules out - // wrapping with Context.allow_fallback. Instead we provide a - // context as a parameter. - const context = Context.get(undefined, { allow_fallback: true }); - - let ret; - try { - ret = await this.callback({ - context, - log_lvl, - crumbs, - message, - fields, - args: a, - }); - } catch (e) { - console.error(e); - } - - if ( ret && ret.skip ) return; - - if ( ! ret ) { - this.delegate.onLogMessage( - log_lvl, - crumbs, - message, - fields, - ...a, - ); - return; - } - - const { - log_lvl: _log_lvl, - crumbs: _crumbs, - message: _message, - fields: _fields, - args, - } = ret; - - this.delegate.onLogMessage( - _log_lvl ?? log_lvl, - _crumbs ?? crumbs, - _message ?? message, - _fields ?? fields, - ...(args ?? a ?? []), - ); - } -} - -/** -* The `LogService` class extends `BaseService` and is responsible for managing and -* orchestrating various logging functionalities within the application. It handles -* log initialization, middleware registration, log directory management, and -* provides methods for creating log contexts and managing log output levels. -*/ -class LogService extends BaseService { - static MODULES = { - path: require('path'), - }; - /** - * Defines the modules required by the LogService class. - * This static property contains modules that are used for file path operations. - * @property {Object} MODULES - An object containing required modules. - * @property {Object} MODULES.path - The Node.js path module for handling and resolving file paths. - */ - async _construct () { - this.loggers = []; - this.bufferLogger = null; - } - - /** - * Registers a custom logging middleware with the LogService. - * @param {*} callback - The callback function that modifies log parameters before delegation. - */ - register_log_middleware (callback) { - this.loggers[0] = new CustomLogger(this.loggers[0], callback); - } - - /** - * Registers logging commands with the command service. - */ - '__on_boot.consolidation' () { - } - /** - * Registers logging commands with the command service. - * - * This method sets up various logging commands that can be used to - * interact with the log output, such as toggling log display, - * starting/stopping log recording, and toggling log indentation. - * - * @memberof LogService - */ - async _init () { - const config = this.global_config; - - this.ensure_log_directory_(); - - let logger; - - if ( ! config.no_winston ) { - const requested_level = config.logger?.level; - const winston_level = typeof requested_level === 'string' - ? requested_level.toLowerCase() - : undefined; - const transports = config.toConsole - ? [ - new winston.transports.Console({ - level: winston_level ?? 'info', - }), - ] - : [ - new winston.transports.DailyRotateFile({ - level: 'http', - filename: `${this.log_directory}/%DATE%.log`, - datePattern: 'YYYY-MM-DD', - maxSize: '20m', - maxFiles: '2d', - }), - new winston.transports.DailyRotateFile({ - level: 'error', - filename: `${this.log_directory}/error-%DATE%.log`, - datePattern: 'YYYY-MM-DD', - maxSize: '20m', - maxFiles: '2d', - }), - new winston.transports.DailyRotateFile({ - level: 'system', - filename: `${this.log_directory}/system-%DATE%.log`, - datePattern: 'YYYY-MM-DD', - maxSize: '20m', - maxFiles: '2d', - }), - ]; - - logger = new WinstonLogger(winston.createLogger({ - levels: WINSTON_LEVELS, - transports, - })); - } - - if ( config.env === 'dev' ) { - logger = config.flag_no_logs // useful for profiling - ? new NullLogger() - : new DevLogger(console.log.bind(console), logger); - - this.devlogger = logger; - } - - logger = new TimestampLogger(logger); - - logger = new BufferLogger(config.log_buffer_size ?? 20, logger); - this.bufferLogger = logger; - - this.loggers.push(logger); - - this.output_lvl = LOG_LEVEL_INFO; - if ( config.logger ) { - // config.logger.level is a string, e.g. 'debug' - - // first we find the appropriate log level - const output_lvl = Object.values({ - LOG_LEVEL_ERRO, - LOG_LEVEL_WARN, - LOG_LEVEL_INFO, - LOG_LEVEL_DEBU, - LOG_LEVEL_TICK, - }).find(lvl => { - return lvl.label === config.logger.level.toUpperCase() || - lvl.winst === config.logger.level.toLowerCase() || - lvl.ordinal === config.logger.level; - }); - - // then we set the output level to the ordinal of that level - this.output_lvl = output_lvl.ordinal; - } - - this.log = this.create('log-service'); - this.log.system('log service started'); - this.log.debug('log service configuration', { - output_lvl: this.output_lvl, - log_directory: this.log_directory, - }); - - this.services.logger = this.create('services-container'); - globalThis.root_context.set('logger', this.create('root-context')); - - { - const util = require('util'); - const logger = this.create('console'); - - if ( ! globalThis.original_console_object ) { - globalThis.original_console_object = console; - } - - // Keep console prototype - const logconsole = Object.create(console); - - // Override simple log functions - const logfn = level => (...a) => { - logger[level](a.map(arg => { - if ( typeof arg === 'string' ) return arg; - return util.inspect(arg, undefined, undefined, true); - }).join(' ')); - }; - - logconsole.log = logfn('info'); - logconsole.info = logfn('info'); - logconsole.warn = logfn('warn'); - logconsole.error = logfn('error'); - logconsole.debug = logfn('debug'); - - globalThis.console = logconsole; - } - } - - /** - * Create a new log context with the specified prefix - * - * @param {1} prefix - The prefix for the log context - * @param {*} fields - Optional fields to include in the log context - * @returns {LogContext} A new log context with the specified prefix and fields - */ - create (prefix, fields = {}) { - const logContext = new LogContext( - this, - { - crumbs: [prefix], - fields, - }, - ); - - return logContext; - } - - log_ (log_lvl, crumbs, message, fields, objects) { - try { - // skip messages that are above the output level - if ( log_lvl.ordinal > this.output_lvl ) return; - - if ( this.config.trace_logs ) { - fields.stack = (new Error('logstack')).stack; - } - - for ( const logger of this.loggers ) { - logger.onLogMessage(log_lvl, crumbs, message, fields, objects); - } - } catch (e) { - // If logging fails, we don't want anything to happen - // that might trigger a log message. This causes an - // infinite loop and I learned that the hard way. - console.error('Logging failed', e); - - // TODO: trigger an alarm either in a non-logging - // context (prereq: per-context service overrides) - // or with a cooldown window (prereq: cooldowns in AlarmService) - } - } - - /** - * Ensures that a log directory exists for logging purposes. - * This method attempts to create or locate a directory for log files, - * falling back through several predefined paths if the preferred - * directory does not exist or cannot be created. - * - * @throws {Error} If no suitable log directory can be found or created. - */ - ensure_log_directory_ () { - // STEP 1: Try /var/puter/logs/heyputer - { - const fs = require('fs'); - const path = '/var/puter/logs/heyputer'; - // Making this directory if it doesn't exist causes issues - // for users running with development instructions - if ( ! fs.existsSync('/var/puter') ) { - return; - } - try { - fs.mkdirSync(path, { recursive: true }); - this.log_directory = path; - return; - } catch (e) { - // ignore - } - } - - // STEP 2: Try /tmp/heyputer - { - const fs = require('fs'); - const path = '/tmp/heyputer'; - try { - fs.mkdirSync(path, { recursive: true }); - this.log_directory = path; - return; - } catch (e) { - // ignore - } - } - - // STEP 3: Try working directory - { - const fs = require('fs'); - const path = './heyputer'; - try { - fs.mkdirSync(path, { recursive: true }); - this.log_directory = path; - return; - } catch (e) { - // ignore - } - } - - // STEP 4: Give up - throw new Error('Unable to create or find log directory'); - } - - /** - * Generates a sanitized file path for log files. - * - * @param {string} name - The name of the log file, which will be sanitized to remove any path characters. - * @returns {string} A sanitized file path within the log directory. - */ - get_log_file (name) { - // sanitize name: cannot contain path characters - name = name.replace(/[^a-zA-Z0-9-_]/g, '_'); - return this.modules.path.join(this.log_directory, name); - } - - /** - * Get the most recent log entries from the buffer maintained by the LogService. - * By default, the buffer contains the last 20 log entries. - * @returns - */ - get_log_buffer () { - return this.bufferLogger.buffer; - } -} - -module.exports = { - LogService, - stringify_log_entry, -}; diff --git a/src/backend/src/modules/core/PagerService.js b/src/backend/src/modules/core/PagerService.js deleted file mode 100644 index 8d425e64c..000000000 --- a/src/backend/src/modules/core/PagerService.js +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const pdjs = require('@pagerduty/pdjs'); -const BaseService = require('../../services/BaseService'); -const util = require('util'); - -/** -* @class PagerService -* @extends BaseService -* @description The PagerService class is responsible for handling pager alerts. -* It extends the BaseService class and provides methods for constructing, -* initializing, and managing alert handlers. The class interacts with PagerDuty -* through the pdjs library to send alerts and integrates with other services via -* command registration. -*/ -class PagerService extends BaseService { - static USE = { - Context: 'core.context', - }; - - async _construct () { - this.config = this.global_config.pager; - this.alertHandlers_ = []; - - } - - /** - * Initializes the PagerService instance by setting the configuration and - * initializing an empty alert handler array. - * - * @async - * @memberOf PagerService - * @returns {Promise} - */ - async _init () { - this.alertHandlers_ = []; - - if ( ! this.config ) { - return; - } - - this.onInit(); - } - - /** - * Initializes PagerDuty configuration and registers alert handlers. - * If PagerDuty is enabled in the configuration, it sets up an alert handler - * to send alerts to PagerDuty. - * - * @method onInit - */ - onInit () { - if ( this.config.pagerduty && this.config.pagerduty.enabled ) { - this.alertHandlers_.push(async alert => { - const event = pdjs.event; - - const fields_clean = {}; - for ( const [key, value] of Object.entries(alert?.fields ?? {}) ) { - fields_clean[key] = util.inspect(value); - } - - const custom_details = { - ...(alert.custom || {}), - server_id: this.global_config.server_id, - }; - - const ctx = this.Context.get(undefined, { allow_fallback: true }); - - // Add request payload if any exists - const req = ctx.get('req'); - if ( req ) { - if ( req.body ) { - // Remove fields which may contain sensitive information - delete req.body.password; - delete req.body.email; - - // Add the request body to the custom details - custom_details.request_body = req.body; - } - } - - this.log.info('it is sending to PD'); - await event({ - data: { - routing_key: this.config.pagerduty.routing_key, - event_action: 'trigger', - dedup_key: alert.id, - payload: { - summary: alert.message, - source: alert.source, - severity: alert.severity, - custom_details, - }, - }, - }); - }); - } - } - - /** - * Sends an alert to all registered alert handlers. - * - * This method iterates through all alert handlers and attempts to send the alert. - * If any handler fails to send the alert, an error message is logged. - * - * @param {Object} alert - The alert object containing details about the alert. - */ - async alert (alert) { - for ( const handler of this.alertHandlers_ ) { - try { - await handler(alert); - } catch (e) { - this.log.error(`failed to send pager alert: ${e?.message}`); - } - } - } - -} - -module.exports = { - PagerService, -}; diff --git a/src/backend/src/modules/core/ParameterService.js b/src/backend/src/modules/core/ParameterService.js deleted file mode 100644 index d26599dce..000000000 --- a/src/backend/src/modules/core/ParameterService.js +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const BaseService = require('../../services/BaseService'); - -/** -* @class Parameter -* @description Represents a configurable parameter with value management, constraints, and change notification capabilities. -* Provides functionality for setting/getting values, binding to object instances, and subscribing to value changes. -* Supports validation through configurable constraints and maintains a list of value change listeners. -*/ -class Parameter { - constructor (spec) { - this.spec_ = spec; - this.valueListeners_ = []; - - if ( spec.default ) { - this.value_ = spec.default; - } - } - - /** - * Sets a new value for the parameter after validating against constraints - * @param {*} value - The new value to set for the parameter - * @throws {Error} If the value fails any constraint checks - * @fires valueListeners with new value and old value - * @async - */ - async set (value) { - for ( const constraint of (this.spec_.constraints ?? []) ) { - if ( ! await constraint.check(value) ) { - throw new Error(`value ${value} does not satisfy constraint ${constraint.id}`); - } - } - - const old = this.value_; - this.value_ = value; - for ( const listener of this.valueListeners_ ) { - listener(value, { old }); - } - } - - /** - * Gets the current value of this parameter - * @returns {Promise<*>} The parameter's current value - */ - async get () { - return this.value_; - } - - bindToInstance (instance, name) { - const value = this.value_; - instance[name] = value; - this.valueListeners_.push((value) => { - instance[name] = value; - }); - } - - subscribe (listener) { - this.valueListeners_.push(listener); - } -} - -/** -* @class ParameterService -* @extends BaseService -* @description Service class for managing system parameters and their values. -* Provides functionality for creating, getting, setting, and subscribing to parameters. -* Supports parameter binding to instances and includes command registration for parameter management. -* Parameters can have constraints, default values, and change listeners. -*/ -class ParameterService extends BaseService { - _construct () { - /** @type {Array} */ - this.parameters_ = []; - } - - /** - * Initializes the service by registering commands with the command service. - * This method is called during service startup to set up command handlers - * for parameter management. - * @private - */ - '__on_boot.consolidation' () { - } - - createParameters (serviceName, parameters, opt_instance) { - for ( const parameter of parameters ) { - this.log.debug(`registering parameter ${serviceName}:${parameter.id}`); - this.parameters_.push(new Parameter({ - ...parameter, - id: `${serviceName}:${parameter.id}`, - })); - if ( opt_instance ) { - this.bindToInstance( - `${serviceName}:${parameter.id}`, - opt_instance, - parameter.id, - ); - } - } - } - - /** - * Gets the value of a parameter by its ID - * @param {string} id - The unique identifier of the parameter to retrieve - * @returns {Promise<*>} The current value of the parameter - * @throws {Error} If parameter with given ID is not found - */ - async get (id) { - const parameter = this._get_param(id); - return await parameter.get(); - } - - bindToInstance (id, instance, name) { - const parameter = this._get_param(id); - return parameter.bindToInstance(instance, name); - } - - subscribe (id, listener) { - const parameter = this._get_param(id); - return parameter.subscribe(listener); - } - - _get_param (id) { - const parameter = this.parameters_.find(p => p.spec_.id === id); - if ( ! parameter ) { - throw new Error(`unknown parameter: ${id}`); - } - return parameter; - } -} - -module.exports = { - ParameterService, -}; diff --git a/src/backend/src/modules/core/ProcessEventService.js b/src/backend/src/modules/core/ProcessEventService.js deleted file mode 100644 index 8110431e0..000000000 --- a/src/backend/src/modules/core/ProcessEventService.js +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const BaseService = require('../../services/BaseService'); - -/** -* Service class that handles process-wide events and errors. -* Provides centralized error handling for uncaught exceptions and unhandled promise rejections. -* Sets up event listeners on the process object to capture and report critical errors -* through the logging and error reporting services. -* -* @class ProcessEventService -*/ -class ProcessEventService extends BaseService { - static USE = { - Context: 'core.context', - }; - - _init () { - const services = this.services; - const log = services.get('log-service').create('process-event-service'); - const errors = services.get('error-service').create(log); - - process.on('uncaughtException', async (err, origin) => { - /** - * Handles uncaught exceptions in the process - * Sets up an event listener that reports errors when uncaught exceptions occur - * @param {Error} err - The uncaught exception error object - * @param {string} origin - The origin of the uncaught exception - * @returns {Promise} - */ - await this.Context.allow_fallback(async () => { - errors.report('process:uncaughtException', { - source: err, - origin, - trace: true, - alarm: true, - }); - }); - - }); - - process.on('unhandledRejection', async (reason, promise) => { - /** - * Handles unhandled promise rejections by reporting them to the error service - * @param {*} reason - The rejection reason/error - * @param {Promise} promise - The rejected promise - * @returns {Promise} Resolves when error is reported - */ - await this.Context.allow_fallback(async () => { - errors.report('process:unhandledRejection', { - source: reason, - promise, - trace: true, - alarm: true, - }); - }); - }); - } -} - -module.exports = { - ProcessEventService, -}; diff --git a/src/backend/src/modules/core/README.md b/src/backend/src/modules/core/README.md deleted file mode 100644 index 026ce5d69..000000000 --- a/src/backend/src/modules/core/README.md +++ /dev/null @@ -1,229 +0,0 @@ -# Core2Module - -A replacement for CoreModule with as few external relative requires as possible. -This will eventually be the successor to CoreModule, the main module for Puter's backend. - -## Services - -### AlarmService - -AlarmService class is responsible for managing alarms. -It provides methods for creating, clearing, and handling alarms. - -#### Listeners - -##### `boot.consolidation` - -AlarmService registers its commands at the consolidation phase because -the '_init' method of CommandService may not have been called yet. - -#### Methods - -##### `create` - -Method to create an alarm with the given ID, message, and fields. -If the ID already exists, it will be updated with the new fields -and the occurrence count will be incremented. - -###### Parameters - -- **id:** Unique identifier for the alarm. -- **message:** Message associated with the alarm. -- **fields:** Additional information about the alarm. - -##### `clear` - -Method to clear an alarm with the given ID. - -###### Parameters - -- **id:** The ID of the alarm to clear. - -##### `get_alarm` - -Method to get an alarm by its ID. - -###### Parameters - -- **id:** The ID of the alarm to get. - -### ErrorService - -The ErrorService class is responsible for handling and reporting errors within the system. -It provides methods to initialize the service, create error contexts, and report errors with detailed logging and alarm mechanisms. - -#### Methods - -##### `init` - -Initializes the ErrorService, setting up the alarm and backup logger services. - -##### `create` - -Creates an ErrorContext instance with the provided logging context. - -###### Parameters - -- **log_context:** The logging context to associate with the error reports. - -##### `report` - -Reports an error with the specified location and details. -The "location" is a string up to the callers discretion to identify -the source of the error. - -###### Parameters - -- **location:** The location where the error occurred. -- **fields:** The error details to report. - -### LogService - -The `LogService` class extends `BaseService` and is responsible for managing and -orchestrating various logging functionalities within the application. It handles -log initialization, middleware registration, log directory management, and -provides methods for creating log contexts and managing log output levels. - -#### Listeners - -##### `boot.consolidation` - -Registers logging commands with the command service. - -#### Methods - -##### `register_log_middleware` - -Registers a custom logging middleware with the LogService. - -###### Parameters - -- **callback:** The callback function that modifies log parameters before delegation. - -##### `create` - -Create a new log context with the specified prefix - -###### Parameters - -- **prefix:** The prefix for the log context -- **fields:** Optional fields to include in the log context - -##### `get_log_file` - -Generates a sanitized file path for log files. - -###### Parameters - -- **name:** The name of the log file, which will be sanitized to remove any path characters. - -##### `get_log_buffer` - -Get the most recent log entries from the buffer maintained by the LogService. -By default, the buffer contains the last 20 log entries. - -### PagerService - - - -#### Listeners - -##### `boot.consolidation` - -PagerService registers its commands at the consolidation phase because -the '_init' method of CommandService may not have been called yet. - -#### Methods - -##### `onInit` - -Initializes PagerDuty configuration and registers alert handlers. -If PagerDuty is enabled in the configuration, it sets up an alert handler -to send alerts to PagerDuty. - -##### `alert` - -Sends an alert to all registered alert handlers. - -This method iterates through all alert handlers and attempts to send the alert. -If any handler fails to send the alert, an error message is logged. - -###### Parameters - -- **alert:** The alert object containing details about the alert. - -### ProcessEventService - -Service class that handles process-wide events and errors. -Provides centralized error handling for uncaught exceptions and unhandled promise rejections. -Sets up event listeners on the process object to capture and report critical errors -through the logging and error reporting services. - -## Libraries - -### core.util.identutil - -#### Functions - -##### `randomItem` - -Select a random item from an array using a random number generator function. - -###### Parameters - -- **arr:** The array to select an item from - -### core.util.logutil - -#### Functions - -##### `stringify_log_entry` - -Stringifies a log entry into a formatted string for console output. - -###### Parameters - -- **logEntry:** The log entry object containing: - -### stdio - -#### Functions - -##### `visible_length` - -METADATA // {"ai-commented":{"service":"claude"}} - -##### `split_lines` - -Split a string into lines according to the terminal width, -preserving ANSI escape sequences, and return an array of lines. - -###### Parameters - -- **str:** The string to split into lines - -### core.util.strutil - -#### Functions - -##### `quot` - -METADATA // {"def":"core.util.strutil","ai-params":{"service":"claude"},"ai-commented":{"service":"claude"}} - -## Notes - -### Outside Imports - -This module has external relative imports. When these are -removed it may become possible to move this module to an -extension. - -**Imports:** -- `../../services/BaseService.js` -- `../../util/context.js` -- `../../services/BaseService` (use.BaseService) -- `../../services/BaseService` (use.BaseService) -- `../../util/context` -- `../../services/BaseService` (use.BaseService) -- `../../services/BaseService` (use.BaseService) -- `../../services/BaseService` (use.BaseService) diff --git a/src/backend/src/modules/core/ServerHealthService/ServerHealthRedisCacheKeys.js b/src/backend/src/modules/core/ServerHealthService/ServerHealthRedisCacheKeys.js deleted file mode 100644 index 0828fcd18..000000000 --- a/src/backend/src/modules/core/ServerHealthService/ServerHealthRedisCacheKeys.js +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -export const ServerHealthRedisCacheKeys = { - status: 'server-health:status', -}; \ No newline at end of file diff --git a/src/backend/src/modules/core/ServerHealthService/ServerHealthService.js b/src/backend/src/modules/core/ServerHealthService/ServerHealthService.js deleted file mode 100644 index 1f09e1a84..000000000 --- a/src/backend/src/modules/core/ServerHealthService/ServerHealthService.js +++ /dev/null @@ -1,343 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { ServerHealthRedisCacheKeys } = require('./ServerHealthRedisCacheKeys.js'); -const BaseService = require('../../../services/BaseService'); -const { kv } = require('../../../util/kvSingleton'); -const { promise } = require('@heyputer/putility').libs; - -const SECOND = 1000; -const CHECK_INTERVAL_MS = 5 * SECOND; -const CHECK_TIMEOUT_MS = 4 * SECOND; -const HEALTH_LOOP_STALE_MULTIPLIER = 3; -const DEFAULT_DB_LIVENESS_LATENCY_FAIL_MS = 1500; - -/** -* The ServerHealthService class provides comprehensive health monitoring for the server. -* It extends the BaseService class to include functionality for: -* - Periodic system checks (e.g., RAM usage, service checks) -* - Managing health check results and failures -* - Triggering alarms for critical conditions -* - Logging and managing statistics for health metrics -* -* This service is designed to work primarily on Linux systems, reading system metrics -* from `/proc/meminfo` and handling alarms via an external 'alarm' service. -*/ -class ServerHealthService extends BaseService { - static USE = { - linuxutil: 'core.util.linuxutil', - }; - - static MODULES = { - fs: require('fs'), - }; - - _construct () { - this.checks_ = []; - this.failures_ = []; - this.health_started_at_ = Date.now(); - this.last_check_cycle_started_at_ = 0; - this.last_check_cycle_completed_at_ = 0; - this.web_checks_registered_ = false; - this.isDraining_ = false; - } - - async _init () { - this.stats_ = {}; - - this.#initDefaultChecks(); - this.#initServiceCheck(); - } - - async '__on_ready.webserver' () { - this.#registerWebChecks(); - } - - beginDrain (reason = 'shutdown') { - if ( this.isDraining_ ) return; - this.isDraining_ = true; - this.failures_ = []; - this.last_check_cycle_completed_at_ = Date.now(); - this.stats_ = this.stats_ ?? {}; - this.stats_.last_check_cycle_completed_at = this.last_check_cycle_completed_at_; - this.stats_.check_durations_ms = {}; - this.stats_.failed_checks = []; - this.log.info(`server health entering drain mode: ${reason}`); - } - - #initDefaultChecks () { - const dbService = this.#getServiceIfAvailable('database'); - if ( dbService && typeof dbService.read === 'function' ) { - const dbLivenessLatencyFailMs = Number( - this.global_config?.server_health?.db_liveness_latency_fail_ms, - ) || DEFAULT_DB_LIVENESS_LATENCY_FAIL_MS; - - this.add_check('database-liveness', async () => { - const startedAt = Date.now(); - const rows = await dbService.read('SELECT 1 AS ok'); - const durationMs = Date.now() - startedAt; - - this.stats_.database_liveness_latency_ms = durationMs; - - if ( !Array.isArray(rows) || rows.length === 0 ) { - throw new Error('database liveness check returned no rows'); - } - - if ( durationMs > dbLivenessLatencyFailMs ) { - throw new Error( - `database liveness query latency too high: ${durationMs}ms ` + - `(threshold ${dbLivenessLatencyFailMs}ms)`, - ); - } - }); - } - } - - #registerWebChecks () { - if ( this.web_checks_registered_ ) return; - - const webServerService = this.#getServiceIfAvailable('web-server'); - if ( ! webServerService ) return; - - this.add_check('web-server-listening', async () => { - const server = webServerService.get_server?.(); - if ( ! server ) { - throw new Error('web server is not initialized'); - } - - if ( server.listening !== true ) { - throw new Error('web server is not listening'); - } - }); - - const socketioService = this.#getServiceIfAvailable('socketio'); - if ( socketioService ) { - this.add_check('socketio-initialized', async () => { - if ( ! socketioService.io ) { - throw new Error('socket.io is not initialized'); - } - }); - } - - this.web_checks_registered_ = true; - } - - /** - * Initializes service health checks by setting up periodic checks. - * This method configures an interval-based execution of health checks, - * handles timeouts, and manages failure states. - * - * @param {none} - This method does not take any parameters. - * @returns {void} - This method does not return any value. - */ - #initServiceCheck () { - const svc_alarm = this.services.get('alarm'); - /** - * Initializes periodic health checks for the server. - * - * This method sets up an interval to run all registered health checks - * at a specified frequency. It manages the execution of checks, handles - * timeouts, and logs errors or triggers alarms when checks fail. - * - * @private - * @method init_service_checks_ - * @memberof ServerHealthService - * @param {none} - No parameters are passed to this method. - * @returns {void} - */ - promise.asyncSafeSetInterval(async () => { - if ( this.isDraining_ ) { - this.last_check_cycle_completed_at_ = Date.now(); - this.stats_.last_check_cycle_completed_at = this.last_check_cycle_completed_at_; - this.stats_.check_durations_ms = {}; - this.stats_.failed_checks = []; - return; - } - - const check_failures = []; - const check_durations_ms = {}; - for ( const { name, fn, chainable } of this.checks_ ) { - const p_timeout = new promise.TeePromise(); - /** - * Creates a TeePromise to handle potential timeouts during health checks. - * - * @returns {Promise} A promise that can be resolved or rejected from multiple places. - */ - const timeout = setTimeout(() => { - p_timeout.reject(new Error('Health check timed out')); - }, CHECK_TIMEOUT_MS); - const check_started_at = Date.now(); - try { - await Promise.race([ - fn(), - p_timeout, - ]); - } catch ( err ) { - check_failures.push({ name }); - const alreadyFailing = this.failures_.some(v => v.name === name); - - if ( ! alreadyFailing ) { - svc_alarm.create( - 'health-check-failure', - `Health check ${name} failed`, - { error: err }, - ); - - // Run the on_fail handlers only on new failures - for ( const fn of chainable.on_fail_ ) { - try { - await fn(err); - } catch ( e ) { - this.log.error(`Error in on_fail handler for ${name}`, e); - } - } - } - - this.log.error(`Error for healthcheck fail on ${name}: ${ err.stack}`); - } finally { - clearTimeout(timeout); - check_durations_ms[name] = Date.now() - check_started_at; - } - } - - this.failures_ = check_failures; - this.last_check_cycle_completed_at_ = Date.now(); - this.stats_.last_check_cycle_completed_at = this.last_check_cycle_completed_at_; - this.stats_.check_durations_ms = check_durations_ms; - this.stats_.failed_checks = this.failures_.map(v => v.name); - }, CHECK_INTERVAL_MS, null, { - onBehindSchedule: (drift) => { - svc_alarm.create( - 'health-checks-behind-schedule', - 'Health checks are behind schedule', - { drift }, - ); - }, - }); - } - - /** - * Retrieves the current server health statistics. - * - * @returns {Object} An object containing the current health statistics. - * This method returns a shallow copy of the internal `stats_` object to prevent - * direct manipulation of the service's data. - */ - async get_stats () { - return { ...this.stats_ }; - } - - add_check (name, fn) { - const chainable = { - on_fail_: [], - on_fail: (fn) => { - chainable.on_fail_.push(fn); - return chainable; - }, - }; - this.checks_.push({ name, fn, chainable }); - return chainable; - } - - /** - * Retrieves the current health status of the server. - * Results are cached for 30 seconds to reduce computation overhead. - * - * @returns {Object} An object containing: - * - `ok` {boolean}: Indicates if all health checks passed. - * - `failed` {Array}: An array of names of failed health checks, if any. - */ - async get_status () { - if ( this.isDraining_ ) { - return { - ok: false, - failed: ['draining'], - }; - } - - const cacheKey = ServerHealthRedisCacheKeys.status; - - // Check cache first - try { - const cached = await kv.get(cacheKey); - if ( cached ) { - try { - return JSON.parse(cached); - } catch (e) { - // no op cache is in an invalid state - } - } - } catch (e) { - this.log.warn(`Unable to read health status cache: ${e.message}`); - } - - // Compute status - const failures = this.#getStatusFailures(); - const status = { - ok: failures.length === 0, - ...(failures.length ? { failed: failures } : {}), - }; - - // Cache with 5 second TTL - try { - await kv.set(cacheKey, JSON.stringify(status), { - EX: 5, - }); - } catch (e) { - this.log.warn(`Unable to write health status cache: ${e.message}`); - } - - return status; - } - - #getStatusFailures () { - const failures = this.failures_.map(v => v.name); - const staleHealthRunnerFailure = this.#getStaleHealthRunnerFailure(); - if ( staleHealthRunnerFailure ) { - failures.push(staleHealthRunnerFailure); - } - return failures; - } - - #getStaleHealthRunnerFailure () { - const staleAfterMs = Number( - this.global_config?.server_health?.stale_health_loop_fail_ms, - ) || (CHECK_INTERVAL_MS * HEALTH_LOOP_STALE_MULTIPLIER); - const now = Date.now(); - - if ( this.last_check_cycle_completed_at_ === 0 ) { - return (now - this.health_started_at_) > staleAfterMs - ? 'health-check-loop-not-running' - : null; - } - - return (now - this.last_check_cycle_completed_at_) > staleAfterMs - ? 'health-check-loop-stale' - : null; - } - - #getServiceIfAvailable (serviceName) { - try { - return this.services.get(serviceName); - } catch { - return null; - } - } -} - -module.exports = { ServerHealthService }; diff --git a/src/backend/src/modules/core/lib/__lib__.js b/src/backend/src/modules/core/lib/__lib__.js deleted file mode 100644 index bb4fb2e54..000000000 --- a/src/backend/src/modules/core/lib/__lib__.js +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -module.exports = { - util: { - logutil: require('./log.js'), - identutil: require('./identifier.js'), - stdioutil: require('./stdio.js'), - linuxutil: require('./linux.js'), - }, -}; diff --git a/src/backend/src/modules/core/lib/identifier.js b/src/backend/src/modules/core/lib/identifier.js deleted file mode 100644 index f6244c163..000000000 --- a/src/backend/src/modules/core/lib/identifier.js +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const adjectives = [ - 'amazing', 'ambitious', 'articulate', 'cool', 'bubbly', 'mindful', 'noble', 'savvy', 'serene', - 'sincere', 'sleek', 'sparkling', 'spectacular', 'splendid', 'spotless', 'stunning', - 'awesome', 'beaming', 'bold', 'brilliant', 'cheerful', 'modest', 'motivated', - 'friendly', 'fun', 'funny', 'generous', 'gifted', 'graceful', 'grateful', - 'passionate', 'patient', 'peaceful', 'perceptive', 'persistent', - 'helpful', 'sensible', 'loyal', 'honest', 'clever', 'capable', - 'calm', 'smart', 'genius', 'bright', 'charming', 'creative', 'diligent', 'elegant', 'fancy', - 'colorful', 'avid', 'active', 'gentle', 'happy', 'intelligent', - 'jolly', 'kind', 'lively', 'merry', 'nice', 'optimistic', 'polite', - 'quiet', 'relaxed', 'silly', 'witty', 'young', - 'strong', 'brave', 'agile', 'bold', 'confident', 'daring', - 'fearless', 'heroic', 'mighty', 'powerful', 'valiant', 'wise', 'wonderful', 'zealous', - 'warm', 'swift', 'neat', 'tidy', 'nifty', 'lucky', 'keen', - 'blue', 'red', 'aqua', 'green', 'orange', 'pink', 'purple', 'cyan', 'magenta', 'lime', - 'teal', 'lavender', 'beige', 'maroon', 'navy', 'olive', 'silver', 'gold', 'ivory', -]; - -const nouns = [ - 'street', 'roof', 'floor', 'tv', 'idea', 'morning', 'game', 'wheel', 'bag', 'clock', 'pencil', 'pen', - 'magnet', 'chair', 'table', 'house', 'room', 'book', 'car', 'tree', 'candle', 'light', 'planet', - 'flower', 'bird', 'fish', 'sun', 'moon', 'star', 'cloud', 'rain', 'snow', 'wind', 'mountain', - 'river', 'lake', 'sea', 'ocean', 'island', 'bridge', 'road', 'train', 'plane', 'ship', 'bicycle', - 'circle', 'square', 'garden', 'harp', 'grass', 'forest', 'rock', 'cake', 'pie', 'cookie', 'candy', - 'butterfly', 'computer', 'phone', 'keyboard', 'mouse', 'cup', 'plate', 'glass', 'door', - 'window', 'key', 'wallet', 'pillow', 'bed', 'blanket', 'soap', 'towel', 'lamp', 'mirror', - 'camera', 'hat', 'shirt', 'pants', 'shoes', 'watch', 'ring', - 'necklace', 'ball', 'toy', 'doll', 'kite', 'balloon', 'guitar', 'violin', 'piano', 'drum', - 'trumpet', 'flute', 'viola', 'cello', 'harp', 'banjo', 'tuba', -]; - -const words = { - adjectives, - nouns, -}; - -/** - * Select a random item from an array using a random number generator function. - * - * @param {Array} arr - The array to select an item from - * @param {function} [random=Math.random] - Random number generator function - * @returns {T} A random item from the array - */ -const randomItem = (arr, random) => arr[Math.floor((random ?? Math.random)() * arr.length)]; - -/** - * A function that generates a unique identifier by combining a random adjective, a random noun, and a random number (between 0 and 9999). - * The result is returned as a string with components separated by the specified separator. - * It is useful when you need to create unique identifiers that are also human-friendly. - * - * @param {string} [separator='_'] - The character used to separate the adjective, noun, and number. Defaults to '_' if not provided. - * @param {function} [rng=Math.random] - Random number generator function - * @returns {string} A unique, human-friendly identifier. - * - * @example - * - * let identifier = window.generate_identifier(); - * // identifier would be something like 'clever-idea-123' - * - */ -function generate_identifier (separator = '_', rng = Math.random) { - // return a random combination of first_adj + noun + number (between 0 and 9999) - // e.g. clever-idea-123 - return [ - randomItem(adjectives, rng), - randomItem(nouns, rng), - Math.floor(rng() * 10000), - ].join(separator); -} - -// Character set used for generating human-readable, case-insensitive random codes -const HUMAN_READABLE_CASE_INSENSITIVE = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; - -function generate_random_code (n, { - rng = Math.random, - chars = HUMAN_READABLE_CASE_INSENSITIVE, -} = {}) { - let code = ''; - for ( let i = 0 ; i < n ; i++ ) { - code += randomItem(chars, rng); - } - return code; -} - -/** -* Composes a code by combining a mask string with a base-36 converted number -* @param {string} mask - Initial string template to use as base -* @param {number} value - Number to convert to base-36 and append to the right -* @returns {string} Combined uppercase code -*/ -function compose_code (mask, value) { - const right_str = value.toString(36); - let out_str = mask; - console.log('right_str', right_str); - console.log('out_str', out_str); - for ( let i = 0 ; i < right_str.length ; i++ ) { - out_str[out_str.length - 1 - i] = right_str[right_str.length - 1 - i]; - } - - out_str = out_str.toUpperCase(); - return out_str; -} - -module.exports = { - randomItem, - generate_identifier, - generate_random_code, -}; diff --git a/src/backend/src/modules/core/lib/linux.js b/src/backend/src/modules/core/lib/linux.js deleted file mode 100644 index 888566354..000000000 --- a/src/backend/src/modules/core/lib/linux.js +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -export const parse_meminfo = text => { - const lines = text.split('\n'); - - let meminfo = {}; - - for ( const line of lines ) { - if ( line.trim().length == 0 ) continue; - - const [keyPart, rest] = line.split(':'); - if ( rest === undefined ) continue; - - const key = keyPart.trim(); - // rest looks like " 123 kB"; parseInt ignores the unit. - const value = Number.parseInt(rest, 10); - meminfo[key] = value; - } - - return meminfo; -}; \ No newline at end of file diff --git a/src/backend/src/modules/core/lib/log.js b/src/backend/src/modules/core/lib/log.js deleted file mode 100644 index 53fd26ead..000000000 --- a/src/backend/src/modules/core/lib/log.js +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const config = require('../../../config.js'); - -const module_epoch = Date.now(); -const module_epoch_d = new Date(); -const display_time = (now) => { - const pad2 = n => String(n).padStart(2, '0'); - - const yyyy = now.getFullYear(); - const mm = pad2(now.getMonth() + 1); - const dd = pad2(now.getDate()); - const HH = pad2(now.getHours()); - const MM = pad2(now.getMinutes()); - const SS = pad2(now.getSeconds()); - const time = `${HH}:${MM}:${SS}`; - - const needYear = yyyy !== module_epoch_d.getFullYear(); - const needMonth = needYear || (now.getMonth() !== module_epoch_d.getMonth()); - const needDay = needMonth || (now.getDate() !== module_epoch_d.getDate()); - - if ( needYear ) return `${yyyy}-${mm}-${dd} ${time}`; - if ( needMonth ) return `${mm}-${dd} ${time}`; - if ( needDay ) return `${dd} ${time}`; - return time; -}; - -// Example: -// log("booting"); // → "14:07:12 booting" -// (next day) log("tick"); // → "16 00:00:01 tick" -// (next month) log("tick"); // → "11-01 00:00:01 tick" -// (next year) log("tick"); // → "2026-01-01 00:00:01 tick" - -/** -* Stringifies a log entry into a formatted string for console output. -* @param {Object} logEntry - The log entry object containing: -* @param {string} [prefix] - Optional prefix for the log message. -* @param {Object} log_lvl - Log level object with properties for label, escape code, etc. -* @param {string[]} crumbs - Array of context crumbs. -* @param {string} message - The log message. -* @param {Object} fields - Additional fields to be included in the log. -* @param {Object} objects - Objects to be logged. -* @returns {string} A formatted string representation of the log entry. -*/ -const stringify_log_entry = ({ prefix, log_lvl, crumbs, message, fields, objects, stack }) => { - const { colorize } = require('json-colorizer'); - - let lines = [], m; - - const lf = () => { - if ( ! m ) return; - lines.push(m); - m = ''; - }; - - m = ''; - - if ( ! config.show_relative_time ) { - m += `${display_time(fields.timestamp)} `; - } - - m += prefix ? `${prefix} ` : ''; - let levelLabelShown = false; - if ( log_lvl.label !== 'INFO' || !config.log_hide_info_label ) { - levelLabelShown = true; - m += `\x1B[${log_lvl.esc}m[${log_lvl.label}\x1B[0m`; - } else { - m += `\x1B[${log_lvl.esc}m[\x1B[0m`; - } - for ( let crumb of crumbs ) { - if ( crumb.startsWith('extension/') ) { - crumb = `\x1B[34;1m${crumb}\x1B[0m`; - } - if ( levelLabelShown ) { - m += '::'; - } else levelLabelShown = true; - m += crumb; - } - m += `\x1B[${log_lvl.esc}m]\x1B[0m`; - if ( fields.timestamp ) { - if ( config.show_relative_time ) { - // display seconds since logger epoch - const n = (fields.timestamp - module_epoch) / 1000; - m += ` (${n.toFixed(3)}s)`; - } - } - m += ` ${message} `; - lf(); - for ( const k in fields ) { - // Extensions always have the system actor in context which makes logs - // too verbose. To combat this, we disable logging the 'actor' field - // when the actor's username is 'system' and the `crumbs` include a - // string that starts with 'extension'. - if ( k === 'actor' && crumbs.some(crumb => crumb.startsWith('extension/')) ) { - if ( typeof fields[k] === 'object' && fields[k]?.username === 'system' ) { - continue; - } - } - - if ( k === 'timestamp' ) continue; - if ( k === 'stack' ) continue; - let v; try { - v = colorize(JSON.stringify(fields[k])); - } catch (e) { - v = `${ fields[k]}`; - } - m += ` \x1B[1m${k}:\x1B[0m ${v}`; - lf(); - } - if ( fields.stack ) { - lines.push(fields.stack); - } - return lines.join('\n'); -}; - -module.exports = { - stringify_log_entry, -}; diff --git a/src/backend/src/modules/core/lib/stdio.js b/src/backend/src/modules/core/lib/stdio.js deleted file mode 100644 index f61b1d383..000000000 --- a/src/backend/src/modules/core/lib/stdio.js +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -/** - * Strip ANSI escape sequences from a string (e.g. color codes) - * and then return the length of the resulting string. - * - * @param {string} str - The string to calculate visible length for - * @returns {number} The length of the string without ANSI escape sequences - */ -const visible_length = (str) => { - // eslint-disable-next-line no-control-regex - return str.replace(/\x1b\[[0-9;]*m/g, '').length; -}; - -/** - * Split a string into lines according to the terminal width, - * preserving ANSI escape sequences, and return an array of lines. - * - * @param {string} str The string to split into lines - * @returns {string[]} Array of lines split according to terminal width - */ -const split_lines = (str) => { - const lines = []; - let line = ''; - let line_length = 0; - for ( const c of str ) { - line += c; - if ( c === '\n' ) { - lines.push(line); - line = ''; - line_length = 0; - } else { - line_length++; - if ( line_length >= process.stdout.columns ) { - lines.push(line); - line = ''; - line_length = 0; - } - } - } - if ( line.length ) { - lines.push(line); - } - return lines; -}; - -module.exports = { - visible_length, - split_lines, -}; diff --git a/src/backend/src/modules/data-access/AppRepository.js b/src/backend/src/modules/data-access/AppRepository.js deleted file mode 100644 index 476b5effb..000000000 --- a/src/backend/src/modules/data-access/AppRepository.js +++ /dev/null @@ -1,3 +0,0 @@ -export default class AppRepository { - // -} diff --git a/src/backend/src/modules/data-access/AppService.comp.test.js b/src/backend/src/modules/data-access/AppService.comp.test.js deleted file mode 100644 index b8b37c710..000000000 --- a/src/backend/src/modules/data-access/AppService.comp.test.js +++ /dev/null @@ -1,1111 +0,0 @@ -import { createTestKernel } from '../../../tools/test.mjs'; -import { tmp_provide_services } from '../../helpers.js'; -import AppES from '../../om/entitystorage/AppES'; -import { AppLimitedES } from '../../om/entitystorage/AppLimitedES'; -import { ESBuilder } from '../../om/entitystorage/ESBuilder'; -import { MaxLimitES } from '../../om/entitystorage/MaxLimitES'; -import { ProtectedAppES } from '../../om/entitystorage/ProtectedAppES'; -import { SetOwnerES } from '../../om/entitystorage/SetOwnerES'; -import SQLES from '../../om/entitystorage/SQLES'; -import ValidationES from '../../om/entitystorage/ValidationES'; -import WriteByOwnerOnlyES from '../../om/entitystorage/WriteByOwnerOnlyES'; -import { Eq, Or } from '../../om/query/query'; -import { Actor, UserActorType } from '../../services/auth/Actor'; -import { VirtualGroupService } from '../../services/auth/VirtualGroupService'; -import { EntityStoreService } from '../../services/EntityStoreService'; -import { Context } from '../../util/esmcontext.js'; -import { AppIconService } from '../apps/AppIconService'; -import { AppInformationService } from '../apps/AppInformationService'; -import { OldAppNameService } from '../apps/OldAppNameService'; -import AppService from './AppService'; -import config from '../../config.js'; - -import { describe, expect, it } from 'vitest'; - -const getHostedIndexUrl = subdomain => { - const hostedDomainCandidate = [ - config.static_hosting_domain_alt, - config.static_hosting_domain, - config.private_app_hosting_domain_alt, - config.private_app_hosting_domain, - ].find(domainValue => typeof domainValue === 'string' && domainValue.trim()); - const hostedDomain = hostedDomainCandidate - ? hostedDomainCandidate.trim().toLowerCase().replace(/^\./, '').split(':')[0] - : 'site.puter.localhost'; - return `https://${subdomain}.${hostedDomain}`; -}; - -const ES_APP_ARGS = { - entity: 'app', - upstream: ESBuilder.create([ - SQLES, { table: 'app', debug: true }, - AppES, - AppLimitedES, { - permission_prefix: 'apps-of-user', - exception: async () => { - const actor = Context.get('actor'); - return new Or({ - children: [ - new Eq({ - key: 'approved_for_listing', - value: 1, - }), - new Eq({ - key: 'uid', - value: actor.type.app.uid, - }), - ], - }); - }, - }, - WriteByOwnerOnlyES, - ValidationES, - SetOwnerES, - ProtectedAppES, - MaxLimitES, { max: 5000 }, - ]), -}; - -// Fix: Manually initialize AsyncLocalStorage store for Vitest -// Under Vitest, AsyncLocalStorage may not have a store initialized, causing Context.get() to fail. -// This manually creates a store and sets the root context, ensuring Context operations work. -// This may be a side-effect of OpenTelemetry's own use of AsyncLocalStorage. -const fixContextInitialization = async (callback) => { - return await Context.contextAsyncLocalStorage.run(Context.root, async () => { - Context.contextAsyncLocalStorage.getStore().set('context', Context.root); - return await callback(); - }); -}; - -const testWithEachService = async (fnToRunOnBoth, { - fnToRunOnTheOther, -} = {}) => { - return await fixContextInitialization(async () => { - const setupUserAndRunWithContext = async (params, fn) => { - const { kernel } = params; - const db = kernel.services.get('database').get('write', 'test'); - const userId = 1; - const username = 'testuser'; - const uuid = `user-uuid-${userId}`; - - // Insert the user into the database if not exists - const existingUser = await kernel.services.get('database') - .get('read', 'test') - .read('SELECT * FROM user WHERE uuid = ?', [uuid]); - - if ( existingUser.length === 0 ) { - await db.write( - 'INSERT INTO user (uuid, username, free_storage) VALUES (?, ?, ?)', - [uuid, username, 1024 * 1024 * 1024], - ); - } - - // Read the user back to get the actual id - const users = await kernel.services.get('database') - .get('read', 'test') - .read('SELECT * FROM user WHERE uuid = ?', [uuid]); - - const user = users[0]; - if ( ! user ) { - throw new Error('Failed to create or retrieve test user'); - } - - const actor = await Actor.create(UserActorType, { user }); - if ( !actor || !actor.type ) { - throw new Error('Failed to create actor'); - } - - const userContext = kernel.root_context.sub({ - user, - actor, - }); - - await userContext.arun(async () => { - Context.set('actor', actor); - await fn({ ...params, user, actor }); - }); - }; - - const esAppTestKernel = await createTestKernel({ - testCore: true, - initLevelString: 'init', - serviceMap: { - 'app-information': AppInformationService, - 'app-icon': AppIconService, - 'old-app-name': OldAppNameService, - 'virtual-group': VirtualGroupService, - 'es:app': EntityStoreService, - }, - serviceMapArgs: { - 'es:app': ES_APP_ARGS, - }, - }); - await tmp_provide_services(esAppTestKernel.services); - - const appTestKernel = await createTestKernel({ - testCore: true, - initLevelString: 'init', - serviceMap: { - 'app-information': AppInformationService, - 'app-icon': AppIconService, - 'old-app-name': OldAppNameService, - 'virtual-group': VirtualGroupService, - 'app': AppService, - }, - }); - await tmp_provide_services(appTestKernel.services); - - tmp_provide_services(appTestKernel.services); - await setupUserAndRunWithContext({ kernel: appTestKernel, key: 'app' }, fnToRunOnBoth); - tmp_provide_services(esAppTestKernel.services); - if ( fnToRunOnTheOther ) { - await setupUserAndRunWithContext({ kernel: esAppTestKernel, key: 'es:app' }, fnToRunOnTheOther); - } else { - await setupUserAndRunWithContext({ kernel: esAppTestKernel, key: 'es:app' }, fnToRunOnBoth); - } - - // Expect these tables to have the same values: - const relevant_tables = ['apps', 'app_filetype_association']; - // Fields that are expected to differ (auto-generated UUIDs, timestamps) - const volatile_fields = ['uid', 'uuid', 'timestamp']; - const stripVolatile = (rows) => rows.map(row => { - const copy = { ...row }; - for ( const field of volatile_fields ) { - delete copy[field]; - } - return copy; - }); - - const db_esApp = esAppTestKernel.services.get('database').get('write', 'test'); - const db_app = appTestKernel.services.get('database').get('write', 'test'); - for ( const table_name of relevant_tables ) { - const rows_esApp = await db_esApp.read(`SELECT * FROM ${table_name}`); - const rows_app = await db_app.read(`SELECT * FROM ${table_name}`); - expect(stripVolatile(rows_app)).toEqual(stripVolatile(rows_esApp)); - } - }); -}; - -describe('AppService Regression Prevention Tests', () => { - it('should be testable with two test kernels', async () => { - await testWithEachService(() => { - }); - }); - it('test utility detects database deviations as expected', async () => { - // This should fail because we create apps with different names - let assertionErrorThrown = false; - try { - await testWithEachService( - async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - await crudQ.create.call(service, { - object: { - name: 'test-app', - title: 'Test App', - index_url: 'https://example.com', - }, - }); - }, - { - fnToRunOnTheOther: async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - // Create app with DIFFERENT name to cause deviation - await crudQ.create.call(service, { - object: { - name: 'different-app', // Different name! - title: 'Different Test App', - index_url: 'https://example.com', - }, - }); - }, - }, - ); - } catch ( error ) { - // Vitest assertion errors are thrown when expect() fails - // Check if it's an AssertionError or has assertion-related properties - if ( error.name === 'AssertionError' || - error.constructor.name === 'AssertionError' || - (error.message && error.message.includes('toEqual')) ) { - assertionErrorThrown = true; - } else { - // Re-throw if it's not an assertion error - throw error; - } - } - // Verify that the assertion error was thrown (meaning deviation was detected) - expect(assertionErrorThrown).toBe(true); - }); - - describe('create', () => { - it('should create the app', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - await crudQ.create.call(service, { - object: { - name: 'test-app', - title: 'Test App', - index_url: 'https://example.com', - }, - }); - }); - }); - }); - - describe('read', () => { - it('should read app by uid', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - // Create an app - const created = await crudQ.create.call(service, { - object: { - name: 'read-test-app', - title: 'Read Test App', - index_url: 'https://example.com', - }, - }); - - // Read it back by uid - const read = await crudQ.read.call(service, { uid: created.uid }); - expect(read).toBeDefined(); - expect(read.name).toBe('read-test-app'); - expect(read.title).toBe('Read Test App'); - }); - }); - - it('should read app by name', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - // Create an app - await crudQ.create.call(service, { - object: { - name: 'named-app', - title: 'Named App', - index_url: 'https://example.com', - }, - }); - - // Read it back by name - const read = await crudQ.read.call(service, { id: { name: 'named-app' } }); - expect(read).toBeDefined(); - expect(read.name).toBe('named-app'); - expect(read.title).toBe('Named App'); - }); - }); - - it('should throw error for non-existent app', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - // Try to read a non-existent app - should throw entity_not_found - let errorThrown = false; - try { - await crudQ.read.call(service, { uid: 'app-nonexistent-uid' }); - } catch ( error ) { - errorThrown = true; - const code = error.fields?.code || error.code; - expect(code).toBe('entity_not_found'); - } - expect(errorThrown).toBe(true); - }); - }); - }); - - describe('update', () => { - it('should update title and description', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - // Create an app - const created = await crudQ.create.call(service, { - object: { - name: 'update-test-app', - title: 'Original Title', - description: 'Original description', - index_url: 'https://example.com', - }, - }); - - // Update title and description - await crudQ.update.call(service, { - object: { - uid: created.uid, - title: 'Updated Title', - description: 'Updated description', - }, - id: { name: 'update-test-app' }, - }); - - const read = await crudQ.read.call(service, { uid: created.uid }); - expect(read.title).toBe('Updated Title'); - expect(read.description).toBe('Updated description'); - }); - }); - - it('should update index_url', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - // Create an app - const created = await crudQ.create.call(service, { - object: { - name: 'url-update-app', - title: 'URL Update App', - index_url: 'https://old-url.com', - }, - }); - - // Update index_url - await crudQ.update.call(service, { - object: { - uid: created.uid, - index_url: 'https://new-url.com', - }, - id: { name: 'url-update-app' }, - }); - - const read = await crudQ.read.call(service, { uid: created.uid }); - expect(read.index_url).toBe('https://new-url.com'); - }); - }); - - it('should update with filetype_associations', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - // Create an app - const created = await crudQ.create.call(service, { - object: { - name: 'filetype-app', - title: 'Filetype App', - index_url: 'https://example.com', - }, - }); - - // Update with filetype associations (include title to avoid empty SET clause) - await crudQ.update.call(service, { - object: { - uid: created.uid, - title: 'Filetype App Updated', - filetype_associations: ['txt', 'md', 'json'], - }, - id: { name: 'filetype-app' }, - }); - - const read = await crudQ.read.call(service, { uid: created.uid }); - expect(read.title).toBe('Filetype App Updated'); - expect(read.filetype_associations).toEqual( - expect.arrayContaining(['txt', 'md', 'json']), - ); - }); - }); - - it('should update name with dedupe_name option', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - // Create two apps - await crudQ.create.call(service, { - object: { - name: 'taken-name', - title: 'First App', - index_url: 'https://example.com/taken-name', - }, - }); - - const second = await crudQ.create.call(service, { - object: { - name: 'second-app', - title: 'Second App', - index_url: 'https://example.com/second-app', - }, - }); - - // Try to update second app to use first app's name with dedupe - await crudQ.update.call(service, { - object: { - uid: second.uid, - name: 'taken-name', - }, - id: { name: 'second-app' }, - options: { dedupe_name: true }, - }); - - const read = await crudQ.read.call(service, { uid: second.uid }); - // Should have been deduped to taken-name-1 - expect(read.name).toBe('taken-name-1'); - }); - }); - - it('should throw error when updating non-existent app', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - let errorThrown = false; - try { - await crudQ.update.call(service, { - object: { - uid: 'app-nonexistent', - title: 'New Title', - }, - id: { name: 'nonexistent-app' }, - }); - } catch ( error ) { - errorThrown = true; - // Error code is in fields.code for APIError - const code = error.fields?.code || error.code; - expect(code).toBe('entity_not_found'); - } - expect(errorThrown).toBe(true); - }); - }); - }); - - describe('upsert', () => { - it('should create when app does not exist', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - // Upsert a new app (should create) - const result = await crudQ.upsert.call(service, { - object: { - name: 'upsert-new-app', - title: 'Upsert New App', - index_url: 'https://example.com', - }, - }); - - expect(result).toBeDefined(); - expect(result.name).toBe('upsert-new-app'); - - // Verify it was created - const read = await crudQ.read.call(service, { id: { name: 'upsert-new-app' } }); - expect(read).toBeDefined(); - expect(read.title).toBe('Upsert New App'); - }); - }); - - it('should update when app exists', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - // Create an app first - const created = await crudQ.create.call(service, { - object: { - name: 'upsert-existing-app', - title: 'Original Title', - index_url: 'https://example.com', - }, - }); - - // Upsert with same uid (should update) - await crudQ.upsert.call(service, { - object: { - uid: created.uid, - title: 'Updated via Upsert', - }, - id: { name: 'upsert-existing-app' }, - }); - - // Verify it was updated - const read = await crudQ.read.call(service, { uid: created.uid }); - expect(read.title).toBe('Updated via Upsert'); - }); - }); - }); - - describe('select', () => { - it('should select all apps', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - // Create multiple apps - await crudQ.create.call(service, { - object: { - name: 'select-app-1', - title: 'Select App 1', - index_url: 'https://example.com/select-app-1', - }, - }); - await crudQ.create.call(service, { - object: { - name: 'select-app-2', - title: 'Select App 2', - index_url: 'https://example.com/select-app-2', - }, - }); - await crudQ.create.call(service, { - object: { - name: 'select-app-3', - title: 'Select App 3', - index_url: 'https://example.com/select-app-3', - }, - }); - - // Select all - const apps = await crudQ.select.call(service, {}); - expect(apps.length).toBeGreaterThanOrEqual(3); - - const names = apps.map(app => app.name); - expect(names).toContain('select-app-1'); - expect(names).toContain('select-app-2'); - expect(names).toContain('select-app-3'); - }); - }); - - it('should select with user-can-edit predicate', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - // Create an app - await crudQ.create.call(service, { - object: { - name: 'editable-app', - title: 'Editable App', - index_url: 'https://example.com', - }, - }); - - // Select with user-can-edit predicate - const apps = await crudQ.select.call(service, { - predicate: ['user-can-edit'], - }); - - // Should return the app since it's owned by the current user - const names = apps.map(app => app.name); - expect(names).toContain('editable-app'); - }); - }); - }); - - describe('delete', () => { - it('should delete app by uid', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - // Create an app - const created = await crudQ.create.call(service, { - object: { - name: 'delete-test-app', - title: 'Delete Test App', - index_url: 'https://example.com', - }, - }); - - // Delete it - await crudQ.delete.call(service, { uid: created.uid }); - - // Verify it's gone - should throw entity_not_found - let errorThrown = false; - try { - await crudQ.read.call(service, { uid: created.uid }); - } catch ( error ) { - errorThrown = true; - const code = error.fields?.code || error.code; - expect(code).toBe('entity_not_found'); - } - expect(errorThrown).toBe(true); - }); - }); - - it('should throw error when deleting non-existent app', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - let errorThrown = false; - try { - await crudQ.delete.call(service, { uid: 'app-nonexistent' }); - } catch ( error ) { - errorThrown = true; - // Error code is in fields.code for APIError - const code = error.fields?.code || error.code; - expect(code).toBe('entity_not_found'); - } - expect(errorThrown).toBe(true); - }); - }); - }); - - describe('edge cases', () => { - it('should throw validation error for invalid app name', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - let errorThrown = false; - try { - await crudQ.create.call(service, { - object: { - name: 'invalid name with spaces!', - title: 'Invalid App', - index_url: 'https://example.com', - }, - }); - } catch ( error ) { - errorThrown = true; - // Validation errors have specific codes in fields.code - const code = error.fields?.code || error.code; - expect(code).toBeDefined(); - } - expect(errorThrown).toBe(true); - }); - }); - - it('should throw error for missing required field', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - let errorThrown = false; - try { - await crudQ.create.call(service, { - object: { - name: 'missing-title-app', - // Missing title! - index_url: 'https://example.com', - }, - }); - } catch ( error ) { - errorThrown = true; - const code = error.fields?.code || error.code; - expect(code).toBe('field_missing'); - } - expect(errorThrown).toBe(true); - }); - }); - - it('should throw error for name conflict without dedupe', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - // Create first app - await crudQ.create.call(service, { - object: { - name: 'conflict-name', - title: 'First App', - index_url: 'https://example.com/conflict-name-1', - }, - }); - - // Try to create second app with same name - let errorThrown = false; - try { - await crudQ.create.call(service, { - object: { - name: 'conflict-name', - title: 'Second App', - index_url: 'https://example.com/conflict-name-2', - }, - }); - } catch ( error ) { - errorThrown = true; - const code = error.fields?.code || error.code; - expect(code).toBe('app_name_already_in_use'); - } - expect(errorThrown).toBe(true); - }); - }); - - it('should allow duplicate dev-center placeholder index_url', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - await crudQ.create.call(service, { - object: { - name: 'placeholder-app-1', - title: 'Placeholder App 1', - index_url: 'https://dev-center.puter.com/coming-soon.html', - }, - }); - - const second = await crudQ.create.call(service, { - object: { - name: 'placeholder-app-2', - title: 'Placeholder App 2', - index_url: 'https://dev-center.puter.com/coming-soon.html', - }, - }); - - expect(second.uid).toBeDefined(); - }); - }); - - it('should allow duplicate non-hosted index_url', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - const first = await crudQ.create.call(service, { - object: { - name: 'non-hosted-duplicate-1', - title: 'Non Hosted Duplicate 1', - index_url: 'https://example.com/shared-origin', - }, - }); - - const second = await crudQ.create.call(service, { - object: { - name: 'non-hosted-duplicate-2', - title: 'Non Hosted Duplicate 2', - index_url: 'https://example.com/shared-origin', - }, - }); - - expect(first.uid).toBeDefined(); - expect(second.uid).toBeDefined(); - expect(second.uid).not.toBe(first.uid); - }); - }); - - it('should join existing unowned hosted index_url app on create', async () => { - await testWithEachService(async ({ kernel, key, user }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - const db = kernel.services.get('database').get('write', 'test'); - const hostedIndexUrl = getHostedIndexUrl('joinable-site'); - const existingUid = 'app-11111111-1111-4111-8111-111111111111'; - - kernel.services.set('puter-site', { - get_subdomain: async (subdomain) => { - const rows = await db.read( - 'SELECT * FROM subdomains WHERE subdomain = ? LIMIT 1', - [subdomain], - ); - return rows[0] || null; - }, - }); - - await db.write( - 'INSERT INTO subdomains (uuid, subdomain, user_id, root_dir_id) VALUES (?, ?, ?, ?)', - ['sd-11111111-1111-4111-8111-111111111111', 'joinable-site', user.id, 111], - ); - await db.write( - 'INSERT INTO apps (uid, name, title, description, index_url, owner_user_id) VALUES (?, ?, ?, ?, ?, ?)', - [existingUid, 'joinable-existing-app', 'Joinable Existing App', 'Created from origin', hostedIndexUrl, null], - ); - - const joined = await crudQ.create.call(service, { - object: { - name: 'joinable-hosted-app', - title: 'Joinable Hosted App', - description: 'Claimed by owner', - index_url: hostedIndexUrl, - }, - }); - - expect(joined.uid).toBe(existingUid); - - const joinedRows = await db.read( - 'SELECT uid, name, owner_user_id FROM apps WHERE index_url = ?', - [hostedIndexUrl], - ); - expect(joinedRows).toHaveLength(1); - expect(joinedRows[0].uid).toBe(existingUid); - expect(joinedRows[0].name).toBe('joinable-hosted-app'); - expect(joinedRows[0].owner_user_id).toBe(user.id); - }); - }); - - it('should join existing unowned hosted index_url app on update', async () => { - await testWithEachService(async ({ kernel, key, user }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - const db = kernel.services.get('database').get('write', 'test'); - const hostedIndexUrl = getHostedIndexUrl('joinable-update-site'); - const existingUid = 'app-33333333-3333-4333-8333-333333333333'; - - kernel.services.set('puter-site', { - get_subdomain: async (subdomain) => { - const rows = await db.read( - 'SELECT * FROM subdomains WHERE subdomain = ? LIMIT 1', - [subdomain], - ); - return rows[0] || null; - }, - }); - - await db.write( - 'INSERT INTO subdomains (uuid, subdomain, user_id, root_dir_id) VALUES (?, ?, ?, ?)', - ['sd-33333333-3333-4333-8333-333333333333', 'joinable-update-site', user.id, 333], - ); - await db.write( - 'INSERT INTO apps (uid, name, title, description, index_url, owner_user_id) VALUES (?, ?, ?, ?, ?, ?)', - [existingUid, 'joinable-update-existing', 'Joinable Update Existing', 'Auto-created app', hostedIndexUrl, null], - ); - - const appToUpdate = await crudQ.create.call(service, { - object: { - name: 'joinable-update-source', - title: 'Joinable Update Source', - description: 'Source app to be merged', - index_url: 'https://example.com/update-source', - }, - }); - - const joined = await crudQ.update.call(service, { - object: { - uid: appToUpdate.uid, - name: 'joinable-update-merged', - title: 'Joinable Update Merged', - description: 'Merged by owner', - index_url: hostedIndexUrl, - }, - }); - - expect(joined.uid).toBe(existingUid); - - const joinedRows = await db.read( - 'SELECT uid, name, title, owner_user_id FROM apps WHERE index_url = ?', - [hostedIndexUrl], - ); - expect(joinedRows).toHaveLength(1); - expect(joinedRows[0].uid).toBe(existingUid); - expect(joinedRows[0].name).toBe('joinable-update-merged'); - expect(joinedRows[0].title).toBe('Joinable Update Merged'); - expect(joinedRows[0].owner_user_id).toBe(user.id); - - const sourceRows = await db.read( - 'SELECT uid FROM apps WHERE uid = ?', - [appToUpdate.uid], - ); - expect(sourceRows).toHaveLength(0); - - const aliasedRead = await crudQ.read.call(service, { - uid: appToUpdate.uid, - }); - expect(aliasedRead.uid).toBe(existingUid); - }); - }); - - it('should join on update when name matches source app name', async () => { - await testWithEachService(async ({ kernel, key, user }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - const db = kernel.services.get('database').get('write', 'test'); - const hostedIndexUrl = getHostedIndexUrl('joinable-update-self-name'); - const existingUid = 'app-44444444-4444-4444-8444-444444444444'; - - kernel.services.set('puter-site', { - get_subdomain: async (subdomain) => { - const rows = await db.read( - 'SELECT * FROM subdomains WHERE subdomain = ? LIMIT 1', - [subdomain], - ); - return rows[0] || null; - }, - }); - - await db.write( - 'INSERT INTO subdomains (uuid, subdomain, user_id, root_dir_id) VALUES (?, ?, ?, ?)', - ['sd-44444444-4444-4444-8444-444444444444', 'joinable-update-self-name', user.id, 444], - ); - await db.write( - 'INSERT INTO apps (uid, name, title, description, index_url, owner_user_id) VALUES (?, ?, ?, ?, ?, ?)', - [existingUid, 'existing-target-name', 'Existing Target', 'Auto-created app', hostedIndexUrl, null], - ); - - const source = await crudQ.create.call(service, { - object: { - name: 'staging-app-center', - title: 'Source App', - description: 'Source app before join', - index_url: 'https://example.com/staging-source', - }, - }); - - const joined = await crudQ.update.call(service, { - object: { - uid: source.uid, - name: 'staging-app-center', - title: 'Merged Title', - index_url: hostedIndexUrl, - }, - }); - - expect(joined.uid).toBe(existingUid); - - const targetRows = await db.read( - 'SELECT uid, name, title FROM apps WHERE uid = ?', - [existingUid], - ); - expect(targetRows).toHaveLength(1); - expect(targetRows[0].name).toBe('staging-app-center'); - expect(targetRows[0].title).toBe('Merged Title'); - - const sourceRows = await db.read( - 'SELECT uid FROM apps WHERE uid = ?', - [source.uid], - ); - expect(sourceRows).toHaveLength(0); - - const aliasedRead = await crudQ.read.call(service, { - uid: source.uid, - }); - expect(aliasedRead.uid).toBe(existingUid); - }); - }); - - it('should join owned bootstrap hosted app on update', async () => { - await testWithEachService(async ({ kernel, key, user }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - const db = kernel.services.get('database').get('write', 'test'); - const hostedIndexUrl = getHostedIndexUrl('joinable-owned-bootstrap'); - const existingUid = 'app-55555555-5555-4555-8555-555555555555'; - - kernel.services.set('puter-site', { - get_subdomain: async (subdomain) => { - const rows = await db.read( - 'SELECT * FROM subdomains WHERE subdomain = ? LIMIT 1', - [subdomain], - ); - return rows[0] || null; - }, - }); - - await db.write( - 'INSERT INTO subdomains (uuid, subdomain, user_id, root_dir_id) VALUES (?, ?, ?, ?)', - ['sd-55555555-5555-4555-8555-555555555555', 'joinable-owned-bootstrap', user.id, 555], - ); - await db.write( - 'INSERT INTO apps (uid, name, title, description, index_url, owner_user_id) VALUES (?, ?, ?, ?, ?, ?)', - [ - existingUid, - existingUid, - existingUid, - `App created from origin ${hostedIndexUrl}`, - hostedIndexUrl, - user.id, - ], - ); - - const source = await crudQ.create.call(service, { - object: { - name: 'owned-bootstrap-source', - title: 'Owned Bootstrap Source', - description: 'Source app to be merged', - index_url: 'https://example.com/owned-bootstrap-source', - }, - }); - - const joined = await crudQ.update.call(service, { - object: { - uid: source.uid, - title: 'Merged Bootstrap Title', - index_url: hostedIndexUrl, - }, - }); - - expect(joined.uid).toBe(existingUid); - - const targetRows = await db.read( - 'SELECT uid, title, owner_user_id FROM apps WHERE uid = ?', - [existingUid], - ); - expect(targetRows).toHaveLength(1); - expect(targetRows[0].title).toBe('Merged Bootstrap Title'); - expect(targetRows[0].owner_user_id).toBe(user.id); - }); - }); - - it('should reject hosted duplicate index_url owned by another user', async () => { - await testWithEachService(async ({ kernel, key, user }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - const db = kernel.services.get('database').get('write', 'test'); - const hostedIndexUrl = getHostedIndexUrl('foreign-owned'); - - kernel.services.set('puter-site', { - get_subdomain: async (subdomain) => { - const rows = await db.read( - 'SELECT * FROM subdomains WHERE subdomain = ? LIMIT 1', - [subdomain], - ); - return rows[0] || null; - }, - }); - - await db.write( - 'INSERT INTO user (uuid, username, free_storage) VALUES (?, ?, ?)', - ['user-uuid-2', 'otheruser', 1024 * 1024 * 1024], - ); - const otherUsers = await db.read('SELECT id FROM user WHERE uuid = ?', ['user-uuid-2']); - const otherUserId = otherUsers[0].id; - - await db.write( - 'INSERT INTO subdomains (uuid, subdomain, user_id, root_dir_id) VALUES (?, ?, ?, ?)', - ['sd-22222222-2222-4222-8222-222222222222', 'foreign-owned', user.id, 222], - ); - await db.write( - 'INSERT INTO apps (uid, name, title, description, index_url, owner_user_id) VALUES (?, ?, ?, ?, ?, ?)', - ['app-22222222-2222-4222-8222-222222222222', 'foreign-owned-existing', 'Foreign Owned Existing', 'Owned by another user', hostedIndexUrl, otherUserId], - ); - - let errorThrown = false; - try { - await crudQ.create.call(service, { - object: { - name: 'foreign-owned-new', - title: 'Foreign Owned New', - index_url: hostedIndexUrl, - }, - }); - } catch ( error ) { - errorThrown = true; - const code = error.fields?.code || error.code; - expect(code).toBe('app_index_url_already_in_use'); - } - expect(errorThrown).toBe(true); - }); - }); - - it('should dedupe name with dedupe_name option', async () => { - await testWithEachService(async ({ kernel, key }) => { - const service = kernel.services.get(key); - const crudQ = service.constructor.IMPLEMENTS['crud-q']; - - // Create first app - await crudQ.create.call(service, { - object: { - name: 'dedupe-name', - title: 'First App', - index_url: 'https://example.com/dedupe-name-1', - }, - }); - - // Create second app with same name but dedupe option - const second = await crudQ.create.call(service, { - object: { - name: 'dedupe-name', - title: 'Second App', - index_url: 'https://example.com/dedupe-name-2', - }, - options: { dedupe_name: true }, - }); - - // Should be deduped to dedupe-name-1 - expect(second.name).toBe('dedupe-name-1'); - }); - }); - }); -}); diff --git a/src/backend/src/modules/data-access/AppService.js b/src/backend/src/modules/data-access/AppService.js deleted file mode 100644 index 51637c3b0..000000000 --- a/src/backend/src/modules/data-access/AppService.js +++ /dev/null @@ -1,1757 +0,0 @@ -import { v4 as uuidv4 } from 'uuid'; -import APIError from '../../api/APIError.js'; -import { deleteRedisKeys } from '../../clients/redis/deleteRedisKeys.js'; -import config from '../../config.js'; -import { APP_ICONS_SUBDOMAIN } from '../../consts/app-icons.js'; -import { NodeInternalIDSelector } from '../../deprecated/filesystem/node/selectors.js'; -import { app_name_exists, get_app } from '../../helpers.js'; -import { AppUnderUserActorType, UserActorType } from '../../services/auth/Actor.js'; -import { PERMISSION_FOR_NOTHING_IN_PARTICULAR, PermissionRewriter, PermissionUtil } from '../../services/auth/permissionUtils.mjs'; -import BaseService from '../../services/BaseService.js'; -import { DB_READ, DB_WRITE } from '../../services/database/consts.js'; -import { Context } from '../../util/context.js'; -import { AppRedisCacheSpace } from '../apps/AppRedisCacheSpace.js'; -import AppRepository from './AppRepository.js'; -import { as_bool } from './lib/coercion.js'; -import { user_to_client } from './lib/filter.js'; -import { extract_from_prefix } from './lib/sqlutil.js'; -import { - validate_array_of_strings, - validate_image_base64, - validate_json, - validate_string, - validate_url, -} from './lib/validation.js'; - -const APP_ICON_ENDPOINT_PATH_REGEX = /^\/app-icon\/([^/?#]+)(?:\/(\d+))?\/?$/; -const LEGACY_APP_ICON_FILE_PATH_REGEX = /^\/(app-[^/?#]+?)(?:-(\d+))?\.png$/; -const ABSOLUTE_URL_REGEX = /^[a-zA-Z][a-zA-Z\d+\-.]*:/; -const RAW_BASE64_REGEX = /^[A-Za-z0-9+/]+={0,2}$/; -const APP_UID_ALIAS_KEY_PREFIX = 'app:canonicalUidAlias'; -const APP_UID_ALIAS_REVERSE_KEY_PREFIX = 'app:canonicalUidAliasReverse'; -const APP_UID_ALIAS_TTL_SECONDS = 60 * 60 * 24 * 90; -const indexUrlUniquenessExemptionCandidates = [ - 'https://dev-center.puter.com/coming-soon', -]; -const isAbsoluteUrl = value => ABSOLUTE_URL_REGEX.test(value) || value.startsWith('//'); -const hasIndexUrlUniquenessExemption = (candidates) => { - for ( const candidate of candidates ) { - if ( indexUrlUniquenessExemptionCandidates.find(exception => candidate.startsWith(exception)) ) { - return true; - } - } - return false; -}; - -const isRawBase64ImageString = value => { - if ( typeof value !== 'string' ) return false; - const trimmed = value.trim(); - if ( !trimmed || trimmed.length < 16 ) return false; - if ( ! RAW_BASE64_REGEX.test(trimmed) ) return false; - if ( trimmed.length % 4 !== 0 ) return false; - - try { - const decoded = Buffer.from(trimmed, 'base64'); - if ( decoded.length === 0 ) return false; - const normalizedInput = trimmed.replace(/=+$/, ''); - const reencoded = decoded.toString('base64').replace(/=+$/, ''); - return normalizedInput === reencoded; - } catch { - return false; - } -}; - -const normalizeRawBase64ImageString = value => { - if ( typeof value !== 'string' ) return value; - const trimmed = value.trim(); - if ( ! isRawBase64ImageString(trimmed) ) return value; - return `data:image/png;base64,${trimmed}`; -}; - -const isStoredBase64AppIcon = ({ icon, icon_is_base64: iconIsBase64 }) => { - if ( typeof iconIsBase64 === 'boolean' ) return iconIsBase64; - if ( typeof iconIsBase64 === 'number' ) return iconIsBase64 !== 0; - if ( typeof iconIsBase64 === 'string' ) { - const normalized = iconIsBase64.toLowerCase(); - if ( normalized === '1' || normalized === 'true' ) return true; - if ( normalized === '0' || normalized === 'false' ) return false; - } - - if ( typeof icon !== 'string' ) return false; - const trimmed = icon.trim(); - if ( trimmed.startsWith('data:image/') ) return true; - return isRawBase64ImageString(trimmed); -}; - -const getCanonicalAppIconBaseUrl = () => { - const candidate = [config.api_base_url, config.origin] - .find(value => typeof value === 'string' && value.trim()); - if ( ! candidate ) return null; - try { - return (new URL(candidate)).origin; - } catch { - return null; - } -}; - -const getAllowedAppIconOrigins = () => { - const origins = new Set(); - for ( const candidate of [config.api_base_url, config.origin] ) { - if ( typeof candidate !== 'string' || !candidate ) continue; - try { - origins.add((new URL(candidate)).origin); - } catch { - // Ignore invalid config values. - } - } - return origins; -}; - -const getAllowedLegacyAppIconHostnames = () => { - const hostnames = new Set(); - const domains = [config.static_hosting_domain, config.static_hosting_domain_alt]; - for ( const domain of domains ) { - if ( typeof domain !== 'string' || !domain.trim() ) continue; - hostnames.add(`${APP_ICONS_SUBDOMAIN}.${domain.trim().toLowerCase()}`); - } - return hostnames; -}; - -const normalizeAppUid = appUid => ( - typeof appUid === 'string' && appUid.startsWith('app-') - ? appUid - : `app-${appUid}` -); - -const parseAppIconEndpointPath = (value) => { - if ( typeof value !== 'string' ) return null; - const trimmed = value.trim(); - if ( ! trimmed ) return null; - - try { - const parsed = new URL(trimmed, 'http://localhost'); - const match = parsed.pathname.match(APP_ICON_ENDPOINT_PATH_REGEX); - if ( ! match ) return null; - - return { - appUid: normalizeAppUid(match[1]), - }; - } catch { - return null; - } -}; - -const isAppIconEndpointPath = value => !!parseAppIconEndpointPath(value); - -const isAllowedAppIconEndpointUrl = value => { - if ( ! isAppIconEndpointPath(value) ) return false; - - const trimmed = value.trim(); - if ( ! isAbsoluteUrl(trimmed) ) { - return true; - } - - try { - const parsed = new URL(trimmed, 'http://localhost'); - return getAllowedAppIconOrigins().has(parsed.origin); - } catch { - return false; - } -}; - -const parseLegacyHostedAppIconToEndpointPath = value => { - if ( typeof value !== 'string' ) return null; - const trimmed = value.trim(); - if ( !trimmed || trimmed.startsWith('data:') ) return null; - - let parsed; - try { - parsed = new URL(trimmed, 'http://localhost'); - } catch { - return null; - } - - if ( isAbsoluteUrl(trimmed) ) { - const allowedHostnames = getAllowedLegacyAppIconHostnames(); - const hostname = parsed.hostname.toLowerCase(); - if ( ! allowedHostnames.has(hostname) ) { - return null; - } - } - - const match = parsed.pathname.match(LEGACY_APP_ICON_FILE_PATH_REGEX); - if ( ! match ) return null; - - const appUid = normalizeAppUid(match[1]); - return `/app-icon/${appUid}`; -}; - -const migrateRelativeAppIconEndpointUrl = value => { - if ( typeof value !== 'string' ) return value; - const trimmed = value.trim(); - if ( ! trimmed ) return value; - - let canonicalEndpointPath = null; - const endpointPath = parseAppIconEndpointPath(trimmed); - if ( endpointPath ) { - if ( isAbsoluteUrl(trimmed) ) { - try { - const parsed = new URL(trimmed, 'http://localhost'); - if ( ! getAllowedAppIconOrigins().has(parsed.origin) ) { - return value; - } - } catch { - return value; - } - } - canonicalEndpointPath = `/app-icon/${endpointPath.appUid}`; - } else { - canonicalEndpointPath = parseLegacyHostedAppIconToEndpointPath(trimmed); - } - if ( ! canonicalEndpointPath ) return value; - - const baseUrl = getCanonicalAppIconBaseUrl(); - if ( ! baseUrl ) return canonicalEndpointPath; - - try { - return new URL(canonicalEndpointPath, `${baseUrl}/`).toString(); - } catch { - return canonicalEndpointPath; - } -}; - -/** - * AppService contains an instance using the repository pattern - */ -export default class AppService extends BaseService { - async _init () { - this.repository = new AppRepository(); - this.db = this.services.get('database').get(DB_READ, 'apps'); - this.db_write = this.services.get('database').get(DB_WRITE, 'apps'); - - const svc_permission = this.services.get('permission'); - const svc_app = this; - - // Rewrite app-root-dir:: to fs:: - svc_permission.register_rewriter(PermissionRewriter.create({ - matcher: permission => permission.startsWith('app-root-dir:'), - rewriter: async permission => { - const context = Context.get(); - - // Only "AppUnderUser" scope is allowed to have this permission rewritten to - // an actual filesystem permission - this is because apps will still be limited - // baesd on a user's own access. - const actor = context.get('actor'); - if ( ! Context.get('is_grant_user_app_permission') ) { - return PERMISSION_FOR_NOTHING_IN_PARTICULAR; - } - - const parts = PermissionUtil.split(permission); - if ( parts.length < 3 ) { - throw APIError.create('field_invalid', null, { key: 'permission', got: permission }); - } - - // <>:: - const target_app_uid = parts[1]; - const access = parts[2]; - if ( ! target_app_uid ) { - throw APIError.create('field_invalid', null, { key: 'target_app_uid', got: target_app_uid }); - } - - if ( ! (actor.type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - const target_app = await get_app({ uid: target_app_uid }); - if ( ! target_app ) { - throw APIError.create('entity_not_found', null, { identifier: `app:${target_app_uid}` }); - } - if ( target_app.owner_user_id !== actor.type.user.id ) { - throw APIError.create('forbidden'); - } - - const root_dir_id = await svc_app.getAppRootDirId(target_app); - const svc_fs = context.get('services').get('filesystem'); - const node = await svc_fs.node(new NodeInternalIDSelector('mysql', root_dir_id)); - await node.fetchEntry(); - if ( ! node.found ) throw APIError.create('subject_does_not_exist'); - - const node_uid = await node.get('uid'); - return PermissionUtil.join('fs', node_uid, access); - }, - })); - - } - - static PROTECTED_FIELDS = ['last_review']; - static READ_ONLY_FIELDS = [ - 'approved_for_listing', - 'approved_for_opening_items', - 'approved_for_incentive_program', - 'godmode', - 'is_private', - ]; - static WRITE_ALL_OWNER_PERMISSION = 'system:es:write-all-owners'; - - static IMPLEMENTS = { - 'crud-q': { - async create ({ object, options }) { - return await this.#create({ object, options }); - }, - async update ({ object, id, options }) { - return await this.#update({ object, id, options }); - }, - async upsert ({ object, id, options }) { - // Try to find an existing entity - let existing = null; - - if ( object.uid !== undefined || id !== undefined ) { - try { - existing = await this.#read({ - uid: object.uid, - id, - }); - } catch ( error ) { - // If entity not found, we'll create it - if ( error.fields?.code !== 'entity_not_found' ) { - throw error; - } - } - } - - if ( existing ) { - // Entity exists, call update - return await this.#update({ object, id, options }); - } else { - // Entity doesn't exist, call create - return await this.#create({ object, options }); - } - }, - async read ({ uid, id, params = {} }) { - return this.#read({ uid, id, params }); - }, - async select (options) { - return this.#select(options); - }, - async delete ({ uid, id }) { - return await this.#delete({ uid, id }); - }, - }, - }; - - // value of require('om/mappings/app.js').redundant_identifiers - static REDUNDANT_IDENTIFIERS = ['name']; - - async #select ({ predicate, params, ..._rest }) { - const db = this.db; - - if ( predicate === undefined ) predicate = []; - if ( params === undefined ) params = {}; - if ( ! Array.isArray(predicate) ) throw new Error('predicate must be an array'); - - const userCanEditOnly = Array.prototype.includes.call(predicate, 'user-can-edit'); - - const stmt = 'SELECT apps.*, ' + - 'CASE WHEN apps.icon LIKE \'data:%\' THEN 1 ELSE 0 END AS icon_is_base64, ' + - 'owner_user.username AS owner_user_username, ' + - 'owner_user.uuid AS owner_user_uuid, ' + - 'app_owner.uid AS app_owner_uid ' + - 'FROM apps ' + - 'LEFT JOIN user owner_user ON apps.owner_user_id = owner_user.id ' + - 'LEFT JOIN apps app_owner ON apps.app_owner = app_owner.id ' + - `${userCanEditOnly ? 'WHERE apps.owner_user_id=?' : ''} ` + - 'LIMIT 5000'; - const values = userCanEditOnly ? [Context.get('user').id] : []; - const rows = await db.read(stmt, values); - - const shouldFetchFiletypes = rows.some(row => typeof row.filetypes !== 'string'); - const filetypesByAppId = shouldFetchFiletypes - ? await this.#getFiletypeAssociationsByAppIds(rows.map(row => row.id)) - : new Map(); - - const iconSize = params.icon_size; - const shouldResolveIconPath = Boolean(iconSize) - || rows.some(row => isStoredBase64AppIcon(row)); - const svc_appIcon = shouldResolveIconPath - ? this.context.get('services').get('app-icon') - : null; - const svc_error = shouldResolveIconPath - ? this.context.get('services').get('error-service') - : null; - - const appAndOwnerIds = []; - for ( const row of rows ) { - const app = {}; - - // FROM ROW - app.approved_for_incentive_program = as_bool(row.approved_for_incentive_program); - app.approved_for_listing = as_bool(row.approved_for_listing); - app.approved_for_opening_items = as_bool(row.approved_for_opening_items); - app.background = as_bool(row.background); - app.created_at = row.created_at; - app.created_from_origin = row.created_from_origin; - app.description = row.description; - app.godmode = as_bool(row.godmode); - app.icon = row.icon; - app.is_private = as_bool(row.is_private); - app.index_url = row.index_url; - app.maximize_on_start = as_bool(row.maximize_on_start); - app.metadata = row.metadata; - app.name = row.name; - app.protected = as_bool(row.protected); - app.stats = row.stats; - app.title = row.title; - app.uid = row.uid; - - // REQURIES OTHER DATA - // app.app_owner; - // app.filetype_associations = row.filetype_associations; - // app.owner = row.owner; - - app.app_owner = { - uid: row.app_owner_uid, - }; - - { - const owner_user = extract_from_prefix(row, 'owner_user_'); - app.owner = user_to_client(owner_user); - } - - try { - if ( typeof row.filetypes === 'string' ) { - app.filetype_associations = this.#parseFiletypeAssociationsJson(row.filetypes); - } else { - app.filetype_associations = this.#normalizeFiletypeAssociations(filetypesByAppId.get(row.id) ?? []); - } - } catch (e) { - throw new Error(`failed to get app filetype associations: ${e.message}`, { cause: e }); - } - - // REFINED BY OTHER DATA - // app.icon; - if ( svc_appIcon && (iconSize || isStoredBase64AppIcon(row)) ) { - try { - const iconPath = svc_appIcon.getAppIconPath({ - appUid: row.uid, - size: iconSize, - }); - if ( iconPath ) { - app.icon = iconPath; - } - } catch (e) { - svc_error?.report('AppES:read_transform', { source: e }); - } - } - - appAndOwnerIds.push({ - app, - ownerUserId: row.owner_user_id, - }); - } - - // Check protected app access in parallel for faster large selections. - const allowed_apps = await Promise.all(appAndOwnerIds.map(async ({ app, ownerUserId }) => { - if ( await this.#check_protected_app_access(app, ownerUserId) ) { - return null; - } - return app; - })); - - return allowed_apps.filter(Boolean); - } - - async #read ({ uid, id, params = {}, backend_only_options = {} }) { - const db = this.db; - - if ( uid === undefined && id === undefined ) { - throw new Error('read requires either uid or id'); - } - - // Build WHERE clause based on identifier type - let whereClause; - let whereValues; - let canonicalUidAliasPromise = null; - - if ( uid !== undefined ) { - // Simple uid lookup - whereClause = 'apps.uid = ?'; - whereValues = [uid]; - canonicalUidAliasPromise = this.#readCanonicalAppUidAlias(uid); - } else if ( id !== null && typeof id === 'object' && !Array.isArray(id) ) { - // Complex id lookup (e.g., { name: 'editor' }) - const { clause, values } = this.#build_complex_id_where(id); - whereClause = clause; - whereValues = values; - } else { - throw APIError.create('invalid_id', null, { id }); - } - - const stmt = 'SELECT apps.*, ' + - 'CASE WHEN apps.icon LIKE \'data:%\' THEN 1 ELSE 0 END AS icon_is_base64, ' + - 'owner_user.username AS owner_user_username, ' + - 'owner_user.uuid AS owner_user_uuid, ' + - 'app_owner.uid AS app_owner_uid ' + - 'FROM apps ' + - 'LEFT JOIN user owner_user ON apps.owner_user_id = owner_user.id ' + - 'LEFT JOIN apps app_owner ON apps.app_owner = app_owner.id ' + - `WHERE ${whereClause} ` + - 'LIMIT 1'; - - let rows = await db.read(stmt, whereValues); - - if ( rows.length === 0 && canonicalUidAliasPromise ) { - const canonicalUid = await canonicalUidAliasPromise; - if ( - typeof canonicalUid === 'string' - && canonicalUid - && canonicalUid !== uid - ) { - rows = await db.read(stmt, [canonicalUid]); - } - } - - if ( rows.length === 0 ) { - throw APIError.create('entity_not_found', null, { - identifier: uid || JSON.stringify(id), - }); - } - - const row = rows[0]; - const app = {}; - - app.approved_for_incentive_program = as_bool(row.approved_for_incentive_program); - app.approved_for_listing = as_bool(row.approved_for_listing); - app.approved_for_opening_items = as_bool(row.approved_for_opening_items); - app.background = as_bool(row.background); - app.created_at = row.created_at; - app.created_from_origin = row.created_from_origin; - app.description = row.description; - app.godmode = as_bool(row.godmode); - app.icon = row.icon; - app.is_private = as_bool(row.is_private); - app.index_url = row.index_url; - app.maximize_on_start = as_bool(row.maximize_on_start); - app.metadata = row.metadata; - app.name = row.name; - app.protected = as_bool(row.protected); - app.stats = row.stats; - app.title = row.title; - app.uid = row.uid; - - app.app_owner = { - uid: row.app_owner_uid, - }; - - { - const owner_user = extract_from_prefix(row, 'owner_user_'); - if ( backend_only_options.no_filter_owner ) app.owner = owner_user; - else app.owner = user_to_client(owner_user); - } - - let protectedAccessPromise; - try { - if ( typeof row.filetypes === 'string' ) { - app.filetype_associations = this.#parseFiletypeAssociationsJson(row.filetypes); - } else { - protectedAccessPromise = this.#check_protected_app_access(app, row.owner_user_id); - const filetypeAssociations = await this.#getFiletypeAssociationsByAppId(row.id); - app.filetype_associations = this.#normalizeFiletypeAssociations(filetypeAssociations); - } - } catch (e) { - throw new Error(`failed to get app filetype associations: ${e.message}`, { cause: e }); - } - - // Check protected app access as soon as dependent fields are resolved. - if ( ! protectedAccessPromise ) { - protectedAccessPromise = this.#check_protected_app_access(app, row.owner_user_id); - } - if ( await protectedAccessPromise ) { - // App should not be accessible - throw APIError.create('entity_not_found', null, { - identifier: uid || JSON.stringify(id), - }); - } - - const iconSize = params.icon_size; - if ( iconSize || isStoredBase64AppIcon(row) ) { - const svc_appIcon = this.context.get('services').get('app-icon'); - if ( svc_appIcon ) { - try { - const iconPath = svc_appIcon.getAppIconPath({ - appUid: row.uid, - size: iconSize, - }); - if ( iconPath ) { - app.icon = iconPath; - } - } catch (e) { - const svc_error = this.context.get('services').get('error-service'); - svc_error.report('AppES:read_transform', { source: e }); - } - } - } - - return app; - } - - #parseFiletypeAssociationsJson (filetypes) { - return this.#normalizeFiletypeAssociations(JSON.parse(filetypes)); - } - - async #getFiletypeAssociationsByAppId (appId) { - if ( appId === undefined || appId === null ) return []; - - const rows = await this.db.read( - 'SELECT type FROM app_filetype_association WHERE app_id = ?', - [appId], - ); - return rows - .map(row => row.type) - .filter(type => typeof type === 'string' || type === null); - } - - #normalizeFiletypeAssociations (filetypesAsJSON) { - filetypesAsJSON = Array.isArray(filetypesAsJSON) - ? filetypesAsJSON - : []; - filetypesAsJSON = filetypesAsJSON.filter(ft => ft !== null); - for ( let i = 0 ; i < filetypesAsJSON.length ; i++ ) { - if ( typeof filetypesAsJSON[i] !== 'string' ) { - throw new Error(`expected filetypesAsJSON[${i}] to be a string, got: ${filetypesAsJSON[i]}`); - } - if ( String.prototype.startsWith.call(filetypesAsJSON[i], '.') ) { - filetypesAsJSON[i] = filetypesAsJSON[i].slice(1); - } - } - return filetypesAsJSON; - } - - async #getFiletypeAssociationsByAppIds (appIds) { - appIds = [...new Set(appIds.filter(appId => appId !== undefined && appId !== null))]; - if ( appIds.length === 0 ) return new Map(); - - const filetypesByAppId = new Map(); - for ( const appId of appIds ) { - filetypesByAppId.set(appId, []); - } - - // SQLite has a low bind-parameter limit; chunk to avoid oversized IN lists. - const chunkSize = 500; - for ( let i = 0 ; i < appIds.length ; i += chunkSize ) { - const chunk = appIds.slice(i, i + chunkSize); - const placeholders = chunk.map(() => '?').join(', '); - const rows = await this.db.read( - `SELECT app_id, type FROM app_filetype_association WHERE app_id IN (${placeholders})`, - chunk, - ); - for ( const row of rows ) { - if ( ! filetypesByAppId.has(row.app_id) ) { - filetypesByAppId.set(row.app_id, []); - } - filetypesByAppId.get(row.app_id).push(row.type); - } - } - - return filetypesByAppId; - } - - async #create ({ object, options }) { - // Only UserActorType and AppUnderUserActorType are allowed to do this - const actor = Context.get('actor'); - if ( ! (actor.type instanceof UserActorType || actor.type instanceof AppUnderUserActorType) ) { - throw APIError.create('forbidden'); - } - - const user = actor.type.user; - - // Remove protected/read_only fields from the input (ValidationES behavior) - { - object = { ...object }; - for ( const field of this.constructor.PROTECTED_FIELDS ) { - delete object[field]; - } - for ( const field of this.constructor.READ_ONLY_FIELDS ) { - delete object[field]; - } - } - - // Validate required fields - { - if ( object.name === undefined ) { - throw APIError.create('field_missing', null, { key: 'name' }); - } - if ( object.title === undefined ) { - throw APIError.create('field_missing', null, { key: 'title' }); - } - if ( object.index_url === undefined ) { - throw APIError.create('field_missing', null, { key: 'index_url' }); - } - } - - // Validate fields - { - validate_string(object.name, { - key: 'name', - maxlen: config.app_name_max_length, - regex: config.app_name_regex, - }); - - validate_string(object.title, { - key: 'title', - maxlen: config.app_title_max_length, - }); - - if ( object.description !== undefined && object.description !== null ) { - validate_string(object.description, { - key: 'description', - maxlen: 7000, - }); - } - - if ( object.icon !== undefined && object.icon !== null ) { - if ( typeof object.icon === 'string' ) { - object.icon = normalizeRawBase64ImageString(object.icon); - object.icon = migrateRelativeAppIconEndpointUrl(object.icon); - } - if ( typeof object.icon !== 'string' ) { - throw APIError.create('field_invalid', null, { key: 'icon' }); - } - object.icon = object.icon.trim(); - if ( ! object.icon ) { - // Empty icon is allowed to clear current icon. - } else if ( object.icon.startsWith('data:') ) { - validate_image_base64(object.icon, { key: 'icon' }); - } else if ( ! isAllowedAppIconEndpointUrl(object.icon) ) { - throw APIError.create('field_invalid', null, { key: 'icon' }); - } - } - - validate_url(object.index_url, { - key: 'index_url', - maxlen: 3000, - }); - - if ( object.maximize_on_start !== undefined ) { - object.maximize_on_start = as_bool(object.maximize_on_start); - } - if ( object.background !== undefined ) { - object.background = as_bool(object.background); - } - - if ( object.metadata !== undefined && object.metadata !== null ) { - validate_json(object.metadata, { key: 'metadata' }); - } - - if ( object.filetype_associations !== undefined ) { - validate_array_of_strings(object.filetype_associations, { - key: 'filetype_associations', - }); - } - } - - // Ensure puter.site subdomain is owned by user (if index_url uses it) - await this.#ensure_puter_site_subdomain_is_owned(object.index_url, user); - const joinedApp = await this.#maybeJoinOwnedHostedIndexUrlAppOnCreate({ - object, - options, - user, - }); - if ( joinedApp ) { - return joinedApp; - } - await this.#ensureIndexUrlNotAlreadyInUse({ - indexUrl: object.index_url, - }); - - // Handle app name conflicts (AppES behavior) - if ( await app_name_exists(object.name) ) { - if ( options?.dedupe_name ) { - const base = object.name; - let number = 1; - while ( await app_name_exists(`${base}-${number}`) ) { - number++; - } - object.name = `${base}-${number}`; - } else { - throw APIError.create('app_name_already_in_use', null, { - name: object.name, - }); - } - } - - // Generate UID for the new app (puter-uuid format: app-{uuid}) - const uid = `app-${uuidv4()}`; - - // Determine app_owner if actor is AppUnderUserActorType (SetOwnerES behavior) - let app_owner_id = null; - if ( actor.type instanceof AppUnderUserActorType ) { - app_owner_id = actor.type.app.id; - } - - // Execute SQL INSERT - const insert_id = await this.#execute_insert(object, uid, user.id, app_owner_id); - - // Handle file type associations - if ( object.filetype_associations ) { - await this.#update_filetype_associations(insert_id, object.filetype_associations); - } - - // Emit icon event if icon is set - if ( object.icon ) { - const svc_event = this.services.get('event'); - const event = { - app_uid: uid, - data_url: object.icon, - url: '', - }; - await svc_event.emit('app.new-icon', event); - if ( typeof event.url === 'string' && event.url ) { - this.db_write.write( - 'UPDATE apps SET icon = ? WHERE uid = ? LIMIT 1', - [event.url, uid], - ); - } - } - - // Return the created app - return await this.#read({ uid }); - } - - async #execute_insert (object, uid, owner_user_id, app_owner_id) { - const columns = ['uid', 'owner_user_id']; - const values = [uid, owner_user_id]; - - if ( app_owner_id !== null ) { - columns.push('app_owner'); - values.push(app_owner_id); - } - - const sql_column_map = { - name: 'name', - title: 'title', - description: 'description', - icon: 'icon', - index_url: 'index_url', - maximize_on_start: 'maximize_on_start', - background: 'background', - metadata: 'metadata', - }; - - for ( const [field, column] of Object.entries(sql_column_map) ) { - if ( object[field] === undefined ) continue; - - let value = object[field]; - - // Handle JSON fields - if ( field === 'metadata' && value !== null ) { - value = JSON.stringify(value); - } - - // Handle boolean fields - if ( field === 'maximize_on_start' || field === 'background' ) { - value = value ? 1 : 0; - } - - columns.push(column); - values.push(value); - } - - const placeholders = columns.map(() => '?').join(', '); - const stmt = `INSERT INTO apps (${columns.join(', ')}) VALUES (${placeholders})`; - const result = await this.db_write.write(stmt, values); - - return result.insertId; - } - - async #delete ({ uid, id }) { - // Only UserActorType and AppUnderUserActorType are allowed to do this - const actor = Context.get('actor'); - if ( ! (actor.type instanceof UserActorType || actor.type instanceof AppUnderUserActorType) ) { - throw APIError.create('forbidden'); - } - - // Read the existing app - const old_app = await this.#read({ - uid, - id, - backend_only_options: { no_filter_owner: true }, - }); - if ( ! old_app ) { - throw APIError.create('entity_not_found', null, { - identifier: uid || JSON.stringify(id), - }); - } - - // Check owner permission (WriteByOwnerOnlyES behavior) - await this.#check_owner_permission(old_app); - - // If actor is AppUnderUserActorType, check app_owner (AppLimitedES behavior) - if ( actor.type instanceof AppUnderUserActorType ) { - await this.#check_app_owner_permission(old_app, actor); - } - - // Call app-information service to perform the deletion (AppES behavior) - const svc_appInformation = this.services.get('app-information'); - await svc_appInformation.delete_app(old_app.uid); - - return { success: true, uid: old_app.uid }; - } - - async #check_app_owner_permission (old_app, actor) { - // Check if app has write permission to all user's apps - const svc_permission = this.services.get('permission'); - const user = actor.type.user; - const perm = `es:app:${user.uuid}:write`; - const can_write_any = await svc_permission.check(actor, perm); - if ( can_write_any ) { - return; - } - - // Otherwise verify the app owns this entity - const app = actor.type.app; - const app_owner = old_app.app_owner; - const app_owner_uid = app_owner?.uid; - - if ( !app_owner_uid || app_owner_uid !== app.uid ) { - throw APIError.create('forbidden'); - } - } - - async #update ({ object, id, options }) { - const old_app = await this.#read({ - uid: object.uid, - id, - backend_only_options: { no_filter_owner: true }, - }); - if ( ! old_app ) { - throw APIError.create('entity_not_found', null, { - identifier: object.uid || JSON.stringify(id), - }); - } - - // Only UserActorType and AppUnderUserActorType are allowed to do this - const actor = Context.get('actor'); - if ( ! (actor.type instanceof UserActorType || actor.type instanceof AppUnderUserActorType) ) { - throw APIError.create('forbidden'); - } - - // Check owner permission (WriteByOwnerOnlyES behavior) - await this.#check_owner_permission(old_app); - - // If actor is AppUnderUserActorType, check app_owner (AppLimitedES behavior) - if ( actor.type instanceof AppUnderUserActorType ) { - await this.#check_app_owner_permission(old_app, actor); - } - - // Remove protected/read_only fields from the update (ValidationES behavior) - { - object = { ...object }; - for ( const field of this.constructor.PROTECTED_FIELDS ) { - delete object[field]; - } - for ( const field of this.constructor.READ_ONLY_FIELDS ) { - delete object[field]; - } - } - - // Validate fields - { - if ( object.name !== undefined ) { - validate_string(object.name, { - key: 'name', - maxlen: config.app_name_max_length, - regex: config.app_name_regex, - }); - } - - if ( object.title !== undefined ) { - validate_string(object.title, { - key: 'title', - maxlen: config.app_title_max_length, - }); - } - - if ( object.description !== undefined && object.description !== null ) { - validate_string(object.description, { - key: 'description', - maxlen: 7000, - }); - } - - if ( object.icon !== undefined && object.icon !== null ) { - if ( typeof object.icon === 'string' ) { - object.icon = normalizeRawBase64ImageString(object.icon); - object.icon = migrateRelativeAppIconEndpointUrl(object.icon); - } - if ( typeof object.icon !== 'string' ) { - throw APIError.create('field_invalid', null, { key: 'icon' }); - } - object.icon = object.icon.trim(); - if ( ! object.icon ) { - // Empty icon is allowed to clear current icon. - } else if ( object.icon.startsWith('data:') ) { - validate_image_base64(object.icon, { key: 'icon' }); - } else if ( ! isAllowedAppIconEndpointUrl(object.icon) ) { - throw APIError.create('field_invalid', null, { key: 'icon' }); - } - } - - if ( object.index_url !== undefined ) { - validate_url(object.index_url, { - key: 'index_url', - maxlen: 3000, - }); - } - - // Flag type - adapt values using as_bool - if ( object.maximize_on_start !== undefined ) { - object.maximize_on_start = as_bool(object.maximize_on_start); - } - if ( object.background !== undefined ) { - object.background = as_bool(object.background); - } - - if ( object.metadata !== undefined && object.metadata !== null ) { - validate_json(object.metadata, { key: 'metadata' }); - } - - if ( object.filetype_associations !== undefined ) { - validate_array_of_strings(object.filetype_associations, { - key: 'filetype_associations', - }); - } - } - - // Handle app-specific logic (AppES behavior) - const user = actor.type.user; - const oldAppId = await this.#resolveAppId(old_app); - - // Ensure puter.site subdomain is owned by user (if index_url changed) - if ( object.index_url && object.index_url !== old_app.index_url ) { - await this.#ensure_puter_site_subdomain_is_owned(object.index_url, user); - const joinedApp = await this.#maybeJoinOwnedHostedIndexUrlAppOnCreate({ - object, - options, - user, - excludeAppId: oldAppId, - }); - if ( joinedApp ) { - return joinedApp; - } - await this.#ensureIndexUrlNotAlreadyInUse({ - indexUrl: object.index_url, - excludeAppId: oldAppId, - }); - } - - // Handle app name conflicts - if ( object.name !== undefined ) { - await this.#handle_name_conflict(object, old_app, options); - } - - // Build and execute SQL UPDATE - const { insert_id } = await this.#execute_update(object, old_app); - - // Handle file type associations - if ( object.filetype_associations !== undefined ) { - await this.#update_filetype_associations(insert_id, object.filetype_associations); - } - - // Emit events for icon/name or app changes - await this.#emit_change_events(object, old_app); - - // Return the updated app (re-fetch for client-safe output) - // TODO: optimize this - return await this.#read({ uid: old_app.uid }); - } - - async #resolveAppId (app) { - const appId = Number(app?.id); - if ( Number.isInteger(appId) && appId > 0 ) return appId; - if ( typeof app?.uid !== 'string' || !app.uid ) return undefined; - - const rows = await this.db.read( - 'SELECT id FROM apps WHERE uid = ? LIMIT 1', - [app.uid], - ); - const resolvedId = Number(rows?.[0]?.id); - if ( Number.isInteger(resolvedId) && resolvedId > 0 ) return resolvedId; - return undefined; - } - - async #check_owner_permission (old_app) { - const svc_permission = this.services.get('permission'); - const actor = Context.get('actor'); - - // Check if user has system-wide write permission - { - // We need to fix eslint rule for multi-line calls - const has_permission_to_write_all = await svc_permission.check( - actor, - this.constructor.WRITE_ALL_OWNER_PERMISSION, - ); - - if ( has_permission_to_write_all ) { - return; - } - } - - // Check if user owns the app - { - const user = Context.get('user'); - if ( ! old_app.owner ) { - throw APIError.create('forbidden'); - } - if ( user.id !== old_app.owner.id ) { - throw APIError.create('forbidden'); - } - } - } - - /** - * Resolves an app's subdomain to its puter.site root_dir_id. - * Tries associated_app_id first, then falls back to index_url-based lookup. - * @param {Object} app - App object with id, index_url, uid - * @returns {Promise} root_dir_id - * @throws {APIError} entity_not_found if the app has no subdomain / root directory - */ - async getAppRootDirId (app) { - const db_sites = this.services.get('database').get(DB_READ, 'sites'); - const rows = await db_sites.read( - 'SELECT root_dir_id FROM subdomains WHERE associated_app_id = ? AND root_dir_id IS NOT NULL LIMIT 1', - [app.id], - ); - if ( rows?.[0]?.root_dir_id != null ) { - return rows[0].root_dir_id; - } - - let hostname; - try { - hostname = (new URL(app.index_url)).hostname.toLowerCase(); - } catch { - throw APIError.create('entity_not_found', null, { identifier: `app ${app.uid} root directory` }); - } - const hosting_domain = config.static_hosting_domain?.toLowerCase(); - if ( !hosting_domain || !hostname.endsWith(`.${hosting_domain}`) ) { - throw APIError.create('entity_not_found', null, { identifier: `app ${app.uid} root directory` }); - } - const subdomain = hostname.slice(0, hostname.length - hosting_domain.length - 1); - const site = await this.services.get('puter-site').get_subdomain(subdomain, { is_custom_domain: false }); - if ( ! site?.root_dir_id ) { - throw APIError.create('entity_not_found', null, { identifier: `app ${app.uid} root directory` }); - } - return site.root_dir_id; - } - - async #ensure_puter_site_subdomain_is_owned (index_url, user) { - if ( ! user ) return; - const subdomain = this.#extractPuterHostedSubdomain(index_url); - if ( ! subdomain ) return; - - const svc_puterSite = this.services.get('puter-site'); - const site = await svc_puterSite.get_subdomain(subdomain, { is_custom_domain: false }); - - if ( !site || site.user_id !== user.id ) { - throw APIError.create('subdomain_not_owned', null, { subdomain }); - } - } - - #normalizeConfiguredHostedDomain (domainValue) { - if ( typeof domainValue !== 'string' ) return null; - const normalizedDomain = domainValue.trim().toLowerCase().replace(/^\./, ''); - if ( ! normalizedDomain ) return null; - return normalizedDomain.split(':')[0] || null; - } - - #getPuterHostedDomains () { - const domains = new Set(); - for ( const configuredDomain of [ - config.static_hosting_domain, - config.static_hosting_domain_alt, - config.private_app_hosting_domain, - config.private_app_hosting_domain_alt, - ] ) { - const normalizedConfiguredDomain = this.#normalizeConfiguredHostedDomain(configuredDomain); - if ( normalizedConfiguredDomain ) { - domains.add(normalizedConfiguredDomain); - } - } - return [...domains]; - } - - #extractPuterHostedSubdomain (indexUrl) { - if ( typeof indexUrl !== 'string' || !indexUrl ) return null; - - let hostname; - try { - hostname = (new URL(indexUrl)).hostname.toLowerCase(); - } catch { - return null; - } - - const hostedDomains = this.#getPuterHostedDomains(); - hostedDomains.sort((domainA, domainB) => domainB.length - domainA.length); - - for ( const hostedDomain of hostedDomains ) { - const suffix = `.${hostedDomain}`; - if ( hostname.endsWith(suffix) ) { - const subdomain = hostname.slice(0, hostname.length - suffix.length); - return subdomain || null; - } - } - - return null; - } - - #isPuterHostedIndexUrl (indexUrl) { - return !!this.#extractPuterHostedSubdomain(indexUrl); - } - - #buildEquivalentIndexUrlCandidates (indexUrl) { - if ( typeof indexUrl !== 'string' || !indexUrl.trim() ) { - return []; - } - - try { - const parsedIndexUrl = new URL(indexUrl); - const origin = `${parsedIndexUrl.protocol}//${parsedIndexUrl.host.toLowerCase()}`; - const pathname = parsedIndexUrl.pathname || '/'; - - const candidates = new Set(); - if ( pathname === '/' || pathname.toLowerCase() === '/index.html' ) { - candidates.add(origin); - candidates.add(`${origin}/`); - candidates.add(`${origin}/index.html`); - } else { - const normalizedPath = pathname.endsWith('/') - ? pathname.slice(0, -1) - : pathname; - candidates.add(`${origin}${normalizedPath}`); - candidates.add(`${origin}${normalizedPath}/`); - } - - return [...candidates]; - } catch { - return [indexUrl.trim()]; - } - } - - async #findIndexUrlConflictRow ({ indexUrl, excludeAppId } = {}) { - if ( ! this.#isPuterHostedIndexUrl(indexUrl) ) { - return null; - } - - const indexUrlCandidates = this.#buildEquivalentIndexUrlCandidates(indexUrl); - if ( indexUrlCandidates.length === 0 ) return null; - if ( hasIndexUrlUniquenessExemption(indexUrlCandidates) ) return null; - - const placeholders = indexUrlCandidates.map(() => '?').join(', '); - const parameters = [...indexUrlCandidates]; - let query = `SELECT id, uid, owner_user_id, index_url FROM apps WHERE index_url IN (${placeholders})`; - - if ( Number.isInteger(excludeAppId) && excludeAppId > 0 ) { - query += ' AND id != ?'; - parameters.push(excludeAppId); - } - - query += ' ORDER BY timestamp ASC, id ASC LIMIT 1'; - - const rows = await this.db.read(query, parameters); - const conflictRow = rows.find(row => { - if ( - Number.isInteger(excludeAppId) - && excludeAppId > 0 - && Number(row?.id) === excludeAppId - ) { - return false; - } - if ( typeof row?.index_url === 'string' ) { - return indexUrlCandidates.includes(row.index_url); - } - return true; - }); - return conflictRow || null; - } - - async #ensureIndexUrlNotAlreadyInUse ({ indexUrl, excludeAppId } = {}) { - const conflictRow = await this.#findIndexUrlConflictRow({ indexUrl, excludeAppId }); - if ( conflictRow ) { - throw APIError.create('app_index_url_already_in_use', null, { - index_url: indexUrl, - app_uid: conflictRow.uid, - }); - } - } - - async #claimAppOwnershipByIdForUser ({ appId, userId }) { - if ( !Number.isInteger(appId) || appId <= 0 ) return; - if ( !Number.isInteger(userId) || userId <= 0 ) return; - - await this.db_write.write( - 'UPDATE apps SET owner_user_id = ? WHERE id = ? AND owner_user_id IS NULL', - [userId, appId], - ); - } - - #buildCanonicalAppUidAliasKey (oldAppUid) { - return `${APP_UID_ALIAS_KEY_PREFIX}:${oldAppUid}`; - } - - #buildCanonicalAppUidAliasReverseKey (canonicalAppUid) { - return `${APP_UID_ALIAS_REVERSE_KEY_PREFIX}:${canonicalAppUid}`; - } - - #normalizeCanonicalAliasUidList (value) { - if ( ! Array.isArray(value) ) return []; - const normalizedList = []; - const seen = new Set(); - for ( const item of value ) { - if ( typeof item !== 'string' || !item ) continue; - if ( seen.has(item) ) continue; - seen.add(item); - normalizedList.push(item); - } - return normalizedList; - } - - async #readCanonicalAppUidAlias (oldAppUid) { - if ( typeof oldAppUid !== 'string' || !oldAppUid ) return null; - - const kvStore = this.services.get('puter-kvstore'); - const suService = this.services.get('su'); - if ( !kvStore || typeof kvStore.get !== 'function' ) return null; - if ( !suService || typeof suService.sudo !== 'function' ) return null; - - const key = this.#buildCanonicalAppUidAliasKey(oldAppUid); - try { - const canonicalAppUid = await suService.sudo(() => kvStore.get({ key })); - if ( typeof canonicalAppUid === 'string' && canonicalAppUid ) { - return canonicalAppUid; - } - } catch { - // Alias reads are best-effort. - } - return null; - } - - async #writeCanonicalAppUidAlias ({ oldAppUid, canonicalAppUid }) { - if ( typeof oldAppUid !== 'string' || !oldAppUid ) return; - if ( typeof canonicalAppUid !== 'string' || !canonicalAppUid ) return; - if ( oldAppUid === canonicalAppUid ) return; - - const kvStore = this.services.get('puter-kvstore'); - const suService = this.services.get('su'); - if ( !kvStore || typeof kvStore.set !== 'function' ) return; - if ( !suService || typeof suService.sudo !== 'function' ) return; - - const key = this.#buildCanonicalAppUidAliasKey(oldAppUid); - const reverseKey = this.#buildCanonicalAppUidAliasReverseKey(canonicalAppUid); - const expireAt = Math.floor(Date.now() / 1000) + APP_UID_ALIAS_TTL_SECONDS; - try { - await suService.sudo(async () => { - const reverseValue = await kvStore.get({ key: reverseKey }); - const reverseAliases = this.#normalizeCanonicalAliasUidList(reverseValue); - if ( ! reverseAliases.includes(oldAppUid) ) { - reverseAliases.push(oldAppUid); - } - - await kvStore.set({ - key, - value: canonicalAppUid, - expireAt, - }); - await kvStore.set({ - key: reverseKey, - value: reverseAliases, - expireAt, - }); - }); - } catch { - // Alias writes are best-effort. - } - } - - async #maybeJoinOwnedHostedIndexUrlAppOnCreate ({ - object, - options, - user, - excludeAppId, - } = {}) { - const indexUrl = object?.index_url; - const sourceAppUid = object?.uid; - if ( ! this.#isPuterHostedIndexUrl(indexUrl) ) { - return null; - } - - const conflictRow = await this.#findIndexUrlConflictRow({ - indexUrl, - excludeAppId, - }); - if ( ! conflictRow ) { - return null; - } - - const conflictOwnerUserId = Number(conflictRow.owner_user_id); - if ( - Number.isInteger(conflictOwnerUserId) - && conflictOwnerUserId > 0 - && conflictOwnerUserId !== user.id - ) { - throw APIError.create('app_index_url_already_in_use', null, { - index_url: indexUrl, - app_uid: conflictRow.uid, - }); - } - - if ( !Number.isInteger(conflictOwnerUserId) || conflictOwnerUserId <= 0 ) { - await this.#claimAppOwnershipByIdForUser({ - appId: conflictRow.id, - userId: user.id, - }); - } - - const appToJoin = await this.#read({ - uid: conflictRow.uid, - backend_only_options: { - no_filter_owner: true, - }, - }); - if ( !appToJoin || appToJoin.uid !== conflictRow.uid ) { - throw APIError.create('app_index_url_already_in_use', null, { - index_url: indexUrl, - app_uid: conflictRow.uid, - }); - } - const appToJoinOwnerId = Number(appToJoin.owner?.id); - if ( !Number.isInteger(appToJoinOwnerId) || appToJoinOwnerId !== user.id ) { - throw APIError.create('app_index_url_already_in_use', null, { - index_url: indexUrl, - app_uid: conflictRow.uid, - }); - } - if ( - Number.isInteger(conflictOwnerUserId) - && conflictOwnerUserId === user.id - && !this.#isOriginBootstrapApp(appToJoin) - ) { - // Prevent merging arbitrary same-owner apps; only allow the - // auto-created origin bootstrap app to be absorbed. - throw APIError.create('app_index_url_already_in_use', null, { - index_url: indexUrl, - app_uid: conflictRow.uid, - }); - } - - const joinedObject = { - ...object, - uid: appToJoin.uid, - }; - const requestedJoinedName = ( - typeof joinedObject.name === 'string' - ? joinedObject.name.trim() - : '' - ) || null; - const shouldReapplyRequestedNameAfterMerge = ( - !!object?.uid - && !!requestedJoinedName - ); - if ( object?.uid && joinedObject.name !== undefined ) { - delete joinedObject.name; - } - - let joinedApp = await this.#update({ - object: joinedObject, - options, - }); - - if ( sourceAppUid && sourceAppUid !== appToJoin.uid ) { - await this.#writeCanonicalAppUidAlias({ - oldAppUid: sourceAppUid, - canonicalAppUid: appToJoin.uid, - }); - const svc_appInformation = this.services.get('app-information'); - if ( svc_appInformation?.delete_app ) { - await svc_appInformation.delete_app(sourceAppUid, undefined, { - preserveCanonicalUidAlias: true, - }); - } - } - - if ( shouldReapplyRequestedNameAfterMerge ) { - joinedApp = await this.#update({ - object: { - uid: appToJoin.uid, - name: requestedJoinedName, - }, - options, - }); - } - - return joinedApp; - } - - #isOriginBootstrapApp (app) { - if ( !app || typeof app !== 'object' ) return false; - if ( typeof app.uid !== 'string' || !app.uid ) return false; - if ( app.name !== app.uid ) return false; - if ( app.title !== app.uid ) return false; - if ( typeof app.description !== 'string' ) return false; - return app.description.startsWith('App created from origin '); - } - - async #handle_name_conflict (object, old_app, options) { - const new_name = object.name; - const old_name = old_app.name; - - // If the name hasn't changed, nothing to do - if ( new_name === old_name ) { - delete object.name; - return; - } - - // Check if the name is taken - if ( await app_name_exists(new_name) ) { - if ( options?.dedupe_name ) { - // Auto-deduplicate the name - let number = 1; - while ( await app_name_exists(`${new_name}-${number}`) ) { - number++; - } - object.name = `${new_name}-${number}`; - } else { - // Check if this is an old name of the same app - const svc_oldAppName = this.services.get('old-app-name'); - const name_info = await svc_oldAppName.check_app_name(new_name); - if ( !name_info || name_info.app_uid !== old_app.uid ) { - throw APIError.create('app_name_already_in_use', null, { - name: new_name, - }); - } - // Remove the old name from the old-app-name service - await svc_oldAppName.remove_name(name_info.id); - } - } - } - - async #execute_update (object, old_app) { - // Map object fields to SQL columns - const sql_column_map = { - name: 'name', - title: 'title', - description: 'description', - icon: 'icon', - index_url: 'index_url', - maximize_on_start: 'maximize_on_start', - background: 'background', - metadata: 'metadata', - }; - - const set_clauses = []; - const values = []; - - for ( const [field, column] of Object.entries(sql_column_map) ) { - if ( object[field] === undefined ) continue; - - let value = object[field]; - - // Handle JSON fields - if ( field === 'metadata' && value !== null ) { - value = JSON.stringify(value); - } - - // Handle boolean fields - if ( field === 'maximize_on_start' || field === 'background' ) { - value = value ? 1 : 0; - } - - set_clauses.push(`${column} = ?`); - values.push(value); - } - - if ( set_clauses.length > 0 ) { - values.push(old_app.uid); - const stmt = `UPDATE apps SET ${set_clauses.join(', ')} WHERE uid = ? LIMIT 1`; - await this.db_write.write(stmt, values); - } - - // Fetch the internal ID - const rows = await this.db.read( - 'SELECT id FROM apps WHERE uid = ?', - [old_app.uid], - ); - return { insert_id: rows[0]?.id }; - } - - async #update_filetype_associations (app_id, filetype_associations) { - const oldAssociations = await this.db.read( - 'SELECT type FROM app_filetype_association WHERE app_id = ?', - [app_id], - ); - const normalizedOld = oldAssociations - .map(row => String(row.type ?? '').trim().toLowerCase().replace(/^\./, '')) - .filter(Boolean); - const normalizedNew = (filetype_associations ?? []) - .map(ft => String(ft).trim().toLowerCase().replace(/^\./, '')) - .filter(Boolean); - - // Remove old file associations - await this.db_write.write( - 'DELETE FROM app_filetype_association WHERE app_id = ?', - [app_id], - ); - - // Add new file associations - if ( ! normalizedNew.length ) { - const affectedExtensions = new Set(normalizedOld); - if ( affectedExtensions.size ) { - await deleteRedisKeys(Array.from(affectedExtensions) - .map(ext => AppRedisCacheSpace.associationAppsKey(ext))); - } - return; - } - - const stmt = - `INSERT INTO app_filetype_association (app_id, type) VALUES ${ - normalizedNew.map(() => '(?, ?)').join(', ')}`; - const values = normalizedNew.flatMap(ft => [app_id, ft]); - await this.db_write.write(stmt, values); - - const affectedExtensions = new Set([...normalizedOld, ...normalizedNew]); - if ( affectedExtensions.size ) { - await deleteRedisKeys(Array.from(affectedExtensions) - .map(ext => AppRedisCacheSpace.associationAppsKey(ext))); - } - } - - async #emit_change_events (object, old_app) { - const svc_event = this.services.get('event'); - const app = { - ...old_app, - ...object, - uid: old_app.uid, - }; - - await svc_event.emit('app.changed', { - app_uid: old_app.uid, - action: 'updated', - app, - old_app, - }); - - // Emit icon change event - if ( object.icon !== undefined && object.icon !== old_app.icon ) { - const event = { - app_uid: old_app.uid, - data_url: object.icon, - }; - await svc_event.emit('app.new-icon', event); - if ( typeof event.url === 'string' && event.url ) { - await this.db_write.write( - 'UPDATE apps SET icon = ? WHERE uid = ? LIMIT 1', - [event.url, old_app.uid], - ); - } - } - - // Emit name change event - if ( object.name !== undefined && object.name !== old_app.name ) { - const event = { - app_uid: old_app.uid, - new_name: object.name, - old_name: old_app.name, - }; - await svc_event.emit('app.rename', event); - } - } - - #build_complex_id_where (id) { - const id_keys = Object.keys(id); - id_keys.sort(); - - // 1. Validate the identifier key from `id` - - const redundant_identifiers = this.constructor.REDUNDANT_IDENTIFIERS; - let match_found = false; - - for ( let key_set of redundant_identifiers ) { - key_set = Array.isArray(key_set) ? key_set : [key_set]; - const sorted_key_set = [...key_set].sort(); - - // Check if id_keys matches this key_set exactly - if ( id_keys.length === sorted_key_set.length && - id_keys.every((k, i) => k === sorted_key_set[i]) ) { - match_found = true; - break; - } - } - - if ( ! match_found ) { - throw new Error(`Invalid complex id keys: ${id_keys.join(', ')}. ` + - `Allowed: ${redundant_identifiers.join(', ')}`); - } - - // 2. Build the SQL string for the predicate - - const conditions = []; - const values = []; - - for ( const key of id_keys ) { - conditions.push(`apps.${key} = ?`); - values.push(id[key]); - } - - return { - clause: conditions.join(' AND '), - values, - }; - } - - /** - * Checks if a protected app should be filtered out (not accessible to the current actor). - * Returns true if the app should be filtered out, false if it's accessible. - * - * @param {Object} app - The app object with protected, uid, and owner fields - * @param {number} owner_user_id - The database ID of the app owner (for accurate comparison) - * @returns {Promise} true if app should be filtered out, false if accessible - */ - async #check_protected_app_access (app, owner_user_id) { - // If it's not a protected app, no worries - allow it - if ( ! app.protected ) { - return false; - } - - const actor = Context.get('actor'); - const services = this.services; - - // If actor is this app itself, allow it - if ( - actor.type instanceof AppUnderUserActorType && - app.uid === actor.type.app.uid - ) { - return false; - } - - // If actor is owner of this app, allow it - // Compare using owner_user_id from database for accuracy - if ( - actor.type instanceof UserActorType && - owner_user_id && - owner_user_id === actor.type.user.id - ) { - return false; - } - - // Now we need to check for permission - const app_uid = app.uid; - const svc_permission = services.get('permission'); - const permission_to_check = `app:uid#${app_uid}:access`; - - // If they have permission, allow it - if ( await svc_permission.check(actor, permission_to_check) ) { - return false; - } - - // No access - filter it out - return true; - } -} diff --git a/src/backend/src/modules/data-access/AppService.test.js b/src/backend/src/modules/data-access/AppService.test.js deleted file mode 100644 index 7e0dc6816..000000000 --- a/src/backend/src/modules/data-access/AppService.test.js +++ /dev/null @@ -1,2212 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import AppService from './AppService.js'; - -// Mock the Context module -vi.mock('../../util/context.js', () => ({ - Context: { - get: vi.fn(), - }, -})); - -// Mock the helpers module -vi.mock('../../helpers.js', () => ({ - app_name_exists: vi.fn(), -})); - -// Mock the Actor module -vi.mock('../../services/auth/Actor.js', () => ({ - UserActorType: class UserActorType { - }, - AppUnderUserActorType: class AppUnderUserActorType { - }, -})); - -// Mock the validation module -vi.mock('./lib/validation.js', () => ({ - validate_string: vi.fn(), - validate_url: vi.fn(), - validate_image_base64: vi.fn(), - validate_json: vi.fn(), - validate_array_of_strings: vi.fn(), -})); - -// Mock config -vi.mock('../../config.js', () => ({ - default: { - app_name_max_length: 100, - app_name_regex: /^[a-z0-9-]+$/, - app_title_max_length: 200, - static_hosting_domain: 'puter.site', - static_hosting_domain_alt: 'puter.host', - private_app_hosting_domain: 'puter.app', - private_app_hosting_domain_alt: 'puter.dev', - origin: 'https://puter.localhost', - api_base_url: 'https://api.puter.localhost', - }, -})); - -import { app_name_exists } from '../../helpers.js'; -import { AppUnderUserActorType, UserActorType } from '../../services/auth/Actor.js'; -import { Context } from '../../util/context.js'; -import { - validate_string, - validate_url, -} from './lib/validation.js'; - -describe('AppService', () => { - let appService; - let mockDb; - let mockDbWrite; - let mockServices; - let mockEventService; - let mockPermissionService; - let mockPuterSiteService; - let mockOldAppNameService; - let mockAppInformationService; - let mockKvStoreService; - let mockSuService; - - // Helper to create a mock database row - const createMockAppRow = (overrides = {}) => ({ - id: 1, - uid: 'app-uid-123', - name: 'test-app', - title: 'Test App', - description: 'A test application', - icon: 'icon.png', - index_url: 'https://example.com/app', - created_at: '2024-01-01T00:00:00Z', - created_from_origin: 'localhost', - metadata: '{}', - stats: '{}', - approved_for_incentive_program: 0, - approved_for_listing: 1, - approved_for_opening_items: 1, - background: 0, - godmode: 0, - is_private: 0, - maximize_on_start: 0, - protected: 0, - owner_user_id: 1, - owner_user_username: 'testuser', - owner_user_uuid: 'user-uuid-456', - app_owner_uid: 'owner-app-uid-789', - filetypes: '["txt", "doc"]', - ...overrides, - }); - - // Helper to create a mock actor - const createMockUserActor = (userId = 1) => ({ - type: Object.assign(new UserActorType(), { user: { id: userId } }), - }); - - const createMockAppUnderUserActor = (userId = 1, appId = 100) => ({ - type: Object.assign(new AppUnderUserActorType(), { - user: { id: userId }, - app: { id: appId, uid: 'creator-app-uid' }, - }), - }); - - // Helper to setup Context.get mock for create/update tests - const setupContextForWrite = (actor, user = { id: 1 }) => { - Context.get.mockImplementation((key) => { - if ( key === 'actor' ) return actor; - if ( key === 'user' ) return user; - return null; - }); - }; - - beforeEach(() => { - // Reset mocks - vi.clearAllMocks(); - - // Reset helper mocks - app_name_exists.mockResolvedValue(false); - - // Mock database (read) - mockDb = { - read: vi.fn(), - case: vi.fn().mockImplementation(({ sqlite }) => sqlite), - }; - - // Mock database (write) - mockDbWrite = { - write: vi.fn().mockResolvedValue({ insertId: 1 }), - }; - - // Mock event service - mockEventService = { - emit: vi.fn().mockResolvedValue(undefined), - }; - - // Mock permission service - mockPermissionService = { - check: vi.fn().mockResolvedValue(false), - scan: vi.fn().mockResolvedValue([]), - }; - - // Mock puter-site service - mockPuterSiteService = { - get_subdomain: vi.fn().mockResolvedValue(null), - }; - - // Mock old-app-name service - mockOldAppNameService = { - check_app_name: vi.fn().mockResolvedValue(null), - remove_name: vi.fn().mockResolvedValue(undefined), - }; - - // Mock app-information service - mockAppInformationService = { - delete_app: vi.fn().mockResolvedValue(undefined), - }; - - mockKvStoreService = { - get: vi.fn().mockResolvedValue(null), - set: vi.fn().mockResolvedValue(true), - }; - - mockSuService = { - sudo: vi.fn(async (actorOrCallback, maybeCallback) => { - const callback = maybeCallback || actorOrCallback; - return await callback(); - }), - }; - - // Mock services - mockServices = { - get: vi.fn().mockImplementation((serviceName) => { - if ( serviceName === 'database' ) { - return { - get: vi.fn().mockImplementation((mode) => { - if ( mode === 'write' ) return mockDbWrite; - return mockDb; - }), - }; - } - if ( serviceName === 'event' ) return mockEventService; - if ( serviceName === 'permission' ) return mockPermissionService; - if ( serviceName === 'puter-site' ) return mockPuterSiteService; - if ( serviceName === 'old-app-name' ) return mockOldAppNameService; - if ( serviceName === 'app-information' ) return mockAppInformationService; - if ( serviceName === 'puter-kvstore' ) return mockKvStoreService; - if ( serviceName === 'su' ) return mockSuService; - return null; - }), - }; - - // Create AppService instance - appService = new AppService({ - services: mockServices, - config: {}, - name: 'app-service', - args: {}, - context: { - get: vi.fn().mockReturnValue(mockServices), - }, - }); - - // Manually call _init to set up the service - appService.repository = {}; - appService.db = mockDb; - appService.db_write = mockDbWrite; - }); - - describe('#read', () => { - it('should read an app by uid', async () => { - const mockRow = createMockAppRow(); - mockDb.read.mockResolvedValueOnce([mockRow]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.read.call(appService, { uid: 'app-uid-123' }); - - expect(mockDb.read).toHaveBeenCalledTimes(1); - expect(mockDb.read).toHaveBeenNthCalledWith( - 1, - expect.stringContaining('WHERE apps.uid = ?'), - ['app-uid-123'], - ); - expect(result).toBeDefined(); - expect(result.uid).toBe('app-uid-123'); - expect(result.name).toBe('test-app'); - expect(result.title).toBe('Test App'); - }); - - it('should read an app by complex id (name)', async () => { - const mockRow = createMockAppRow(); - mockDb.read.mockResolvedValueOnce([mockRow]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.read.call(appService, { id: { name: 'test-app' } }); - - expect(mockDb.read).toHaveBeenCalledTimes(1); - expect(mockDb.read).toHaveBeenNthCalledWith( - 1, - expect.stringContaining('WHERE apps.name = ?'), - ['test-app'], - ); - expect(result).toBeDefined(); - expect(result.name).toBe('test-app'); - }); - - it('should throw entity_not_found when no app is found', async () => { - mockDb.read.mockResolvedValue([]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.read.call(appService, { uid: 'nonexistent-uid' })).rejects.toMatchObject({ - fields: { code: 'entity_not_found' }, - }); - }); - - it('should resolve app by canonical uid alias when old uid is missing', async () => { - const canonicalRow = createMockAppRow({ - uid: 'app-canonical-uid-123', - }); - mockDb.read - .mockResolvedValueOnce([]) - .mockResolvedValueOnce([canonicalRow]); - mockKvStoreService.get.mockResolvedValue('app-canonical-uid-123'); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.read.call(appService, { uid: 'app-old-uid-123' }); - - expect(result.uid).toBe('app-canonical-uid-123'); - expect(mockSuService.sudo).toHaveBeenCalled(); - expect(mockKvStoreService.get).toHaveBeenCalledWith({ - key: 'app:canonicalUidAlias:app-old-uid-123', - }); - expect(mockDb.read).toHaveBeenNthCalledWith( - 2, - expect.stringContaining('WHERE apps.uid = ?'), - ['app-canonical-uid-123'], - ); - }); - - it('should throw an error when neither uid nor id is provided', async () => { - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.read.call(appService, {})).rejects.toThrow( - 'read requires either uid or id', - ); - }); - - it('should throw an error for invalid complex id keys', async () => { - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.read.call(appService, { id: { invalidKey: 'value' } })).rejects.toThrow('Invalid complex id keys'); - }); - - it('should correctly coerce boolean fields from database', async () => { - const mockRow = createMockAppRow({ - approved_for_incentive_program: 1, - approved_for_listing: '1', - approved_for_opening_items: 0, - background: '0', - godmode: 1, - is_private: '1', - maximize_on_start: '1', - protected: 0, - }); - mockDb.read.mockResolvedValue([mockRow]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.read.call(appService, { uid: 'app-uid-123' }); - - expect(result.approved_for_incentive_program).toBe(true); - expect(result.approved_for_listing).toBe(true); - expect(result.approved_for_opening_items).toBe(false); - expect(result.background).toBe(false); - expect(result.godmode).toBe(true); - expect(result.is_private).toBe(true); - expect(result.maximize_on_start).toBe(true); - expect(result.protected).toBe(false); - }); - - it('should parse filetypes JSON and strip leading dots', async () => { - const mockRow = createMockAppRow({ - filetypes: '[".txt", ".doc", "pdf"]', - }); - mockDb.read.mockResolvedValueOnce([mockRow]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.read.call(appService, { uid: 'app-uid-123' }); - - expect(result.filetype_associations).toEqual(['txt', 'doc', 'pdf']); - expect(mockDb.read).toHaveBeenCalledTimes(1); - }); - - it('should filter out null values in filetypes array', async () => { - const mockRow = createMockAppRow({ - filetypes: '[".txt", null, "pdf"]', - }); - mockDb.read.mockResolvedValueOnce([mockRow]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.read.call(appService, { uid: 'app-uid-123' }); - - expect(result.filetype_associations).toEqual(['txt', 'pdf']); - expect(mockDb.read).toHaveBeenCalledTimes(1); - }); - - it('should query filetype associations table when filetypes JSON is missing', async () => { - const mockRow = createMockAppRow({ filetypes: null }); - mockDb.read - .mockResolvedValueOnce([mockRow]) - .mockResolvedValueOnce([ - { type: '.txt' }, - { type: null }, - { type: 'pdf' }, - ]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.read.call(appService, { uid: 'app-uid-123' }); - - expect(result.filetype_associations).toEqual(['txt', 'pdf']); - expect(mockDb.read).toHaveBeenCalledTimes(2); - expect(mockDb.read).toHaveBeenNthCalledWith( - 2, - 'SELECT type FROM app_filetype_association WHERE app_id = ?', - [mockRow.id], - ); - }); - - it('should have owner parameter', async () => { - const mockRow = createMockAppRow(); - mockDb.read.mockResolvedValue([mockRow]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.read.call(appService, { uid: 'app-uid-123' }); - - expect(result.owner).toEqual({ - username: 'testuser', - uuid: 'user-uuid-456', - }); - }); - - it('should include app_owner in the result', async () => { - const mockRow = createMockAppRow(); - mockDb.read.mockResolvedValue([mockRow]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.read.call(appService, { uid: 'app-uid-123' }); - - expect(result.app_owner).toEqual({ - uid: 'owner-app-uid-789', - }); - }); - - it('should fetch icon with size when icon_size param is provided', async () => { - const mockRow = createMockAppRow(); - mockDb.read.mockResolvedValue([mockRow]); - - const mockIconService = { - getAppIconPath: vi.fn().mockReturnValue('/app-icon/app-uid-123/64'), - }; - - appService.context = { - get: vi.fn().mockImplementation((key) => { - if ( key === 'services' ) { - return { - get: vi.fn().mockImplementation((name) => { - if ( name === 'app-icon' ) return mockIconService; - return null; - }), - }; - } - return null; - }), - }; - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.read.call(appService, { - uid: 'app-uid-123', - params: { icon_size: 64 }, - }); - - expect(mockIconService.getAppIconPath).toHaveBeenCalledWith({ - appUid: 'app-uid-123', - size: 64, - }); - expect(result.icon).toBe('/app-icon/app-uid-123/64'); - }); - - it('should route base64 icons through app-icon endpoint even without icon_size', async () => { - const mockRow = createMockAppRow({ - icon: 'data:image/png;base64,abc123', - icon_is_base64: 1, - }); - mockDb.read.mockResolvedValue([mockRow]); - - const mockIconService = { - getAppIconPath: vi.fn().mockReturnValue('/app-icon/app-uid-123/128'), - }; - - appService.context = { - get: vi.fn().mockImplementation((key) => { - if ( key === 'services' ) { - return { - get: vi.fn().mockImplementation((name) => { - if ( name === 'app-icon' ) return mockIconService; - return null; - }), - }; - } - return null; - }), - }; - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.read.call(appService, { uid: 'app-uid-123' }); - - expect(mockIconService.getAppIconPath).toHaveBeenCalledWith({ - appUid: 'app-uid-123', - size: undefined, - }); - expect(result.icon).toBe('/app-icon/app-uid-123/128'); - }); - - it('should keep original icon when icon service throws', async () => { - const mockRow = createMockAppRow(); - mockDb.read.mockResolvedValue([mockRow]); - - const mockErrorService = { - report: vi.fn(), - }; - - const mockIconService = { - getAppIconPath: vi.fn().mockImplementation(() => { - throw new Error('Icon fetch failed'); - }), - }; - - appService.context = { - get: vi.fn().mockImplementation((key) => { - if ( key === 'services' ) { - return { - get: vi.fn().mockImplementation((name) => { - if ( name === 'app-icon' ) return mockIconService; - if ( name === 'error-service' ) return mockErrorService; - return null; - }), - }; - } - return null; - }), - }; - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.read.call(appService, { - uid: 'app-uid-123', - params: { icon_size: 64 }, - }); - - expect(mockErrorService.report).toHaveBeenCalledWith( - 'AppES:read_transform', - expect.objectContaining({ source: expect.any(Error) }), - ); - expect(result.icon).toBe('icon.png'); - }); - - }); - - describe('#select', () => { - it('should select all apps with default parameters', async () => { - const mockRows = [ - createMockAppRow({ id: 1, uid: 'app-1', name: 'app-one' }), - createMockAppRow({ id: 2, uid: 'app-2', name: 'app-two' }), - ]; - mockDb.read.mockResolvedValue(mockRows); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.select.call(appService, {}); - - expect(mockDb.read).toHaveBeenCalledTimes(1); - expect(mockDb.read).toHaveBeenCalledWith( - expect.not.stringContaining('WHERE'), - [], - ); - expect(result).toHaveLength(2); - expect(result[0].uid).toBe('app-1'); - expect(result[1].uid).toBe('app-2'); - }); - - it('should filter by user-can-edit predicate', async () => { - const mockUser = { id: 42 }; - Context.get.mockReturnValue(mockUser); - - const mockRows = [createMockAppRow()]; - mockDb.read.mockResolvedValue(mockRows); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.select.call(appService, { - predicate: ['user-can-edit'], - }); - - expect(mockDb.read).toHaveBeenCalledWith( - expect.stringContaining('WHERE apps.owner_user_id=?'), - [42], - ); - expect(result).toHaveLength(1); - }); - - it('should throw error when predicate is not an array', async () => { - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.select.call(appService, { predicate: 'invalid' })).rejects.toThrow('predicate must be an array'); - }); - - it('should correctly coerce boolean fields for all selected apps', async () => { - const mockRows = [ - createMockAppRow({ - id: 1, - approved_for_listing: 1, - godmode: 0, - }), - createMockAppRow({ - id: 2, - approved_for_listing: '0', - godmode: '1', - }), - ]; - mockDb.read.mockResolvedValue(mockRows); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.select.call(appService, {}); - - expect(result[0].approved_for_listing).toBe(true); - expect(result[0].godmode).toBe(false); - expect(result[1].approved_for_listing).toBe(false); - expect(result[1].godmode).toBe(true); - }); - - it('should parse filetypes for all selected apps', async () => { - const mockRows = [ - createMockAppRow({ id: 1, filetypes: '[".txt"]' }), - createMockAppRow({ id: 2, filetypes: '[".pdf", ".doc"]' }), - ]; - mockDb.read.mockResolvedValue(mockRows); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.select.call(appService, {}); - - expect(result[0].filetype_associations).toEqual(['txt']); - expect(result[1].filetype_associations).toEqual(['pdf', 'doc']); - }); - - it('should fetch icons with size for all apps when icon_size is provided', async () => { - const mockRows = [ - createMockAppRow({ id: 1, uid: 'app-1', icon: 'icon1.png' }), - createMockAppRow({ id: 2, uid: 'app-2', icon: 'icon2.png' }), - ]; - mockDb.read.mockResolvedValue(mockRows); - - const mockIconService = { - getAppIconPath: vi.fn().mockImplementation(({ appUid, size }) => `/app-icon/${appUid}/${size}`), - }; - - appService.context = { - get: vi.fn().mockImplementation((key) => { - if ( key === 'services' ) { - return { - get: vi.fn().mockImplementation((name) => { - if ( name === 'app-icon' ) return mockIconService; - return null; - }), - }; - } - return null; - }), - }; - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.select.call(appService, { - params: { icon_size: 32 }, - }); - - expect(mockIconService.getAppIconPath).toHaveBeenCalledTimes(2); - expect(result[0].icon).toBe('/app-icon/app-1/32'); - expect(result[1].icon).toBe('/app-icon/app-2/32'); - }); - - it('should only route base64 icons through app-icon endpoint when icon_size is not provided', async () => { - const mockRows = [ - createMockAppRow({ - id: 1, - uid: 'app-1', - icon: 'data:image/png;base64,abc123', - icon_is_base64: 1, - }), - createMockAppRow({ - id: 2, - uid: 'app-2', - icon: 'https://puter-app-icons.puter.site/app-2-128.png', - icon_is_base64: 0, - }), - ]; - mockDb.read.mockResolvedValue(mockRows); - - const mockIconService = { - getAppIconPath: vi.fn().mockImplementation(({ appUid }) => `/app-icon/${appUid}/128`), - }; - - appService.context = { - get: vi.fn().mockImplementation((key) => { - if ( key === 'services' ) { - return { - get: vi.fn().mockImplementation((name) => { - if ( name === 'app-icon' ) return mockIconService; - return null; - }), - }; - } - return null; - }), - }; - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.select.call(appService, {}); - - expect(mockIconService.getAppIconPath).toHaveBeenCalledTimes(1); - expect(result[0].icon).toBe('/app-icon/app-1/128'); - expect(result[1].icon).toBe('https://puter-app-icons.puter.site/app-2-128.png'); - }); - - it('should return empty array when no apps exist', async () => { - mockDb.read.mockResolvedValue([]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.select.call(appService, {}); - - expect(result).toEqual([]); - }); - - it('should have owner parameter for all selected apps', async () => { - const mockRows = [ - createMockAppRow({ - id: 1, - owner_user_username: 'user1', - owner_user_uuid: 'uuid-1', - }), - createMockAppRow({ - id: 2, - owner_user_username: 'user2', - owner_user_uuid: 'uuid-2', - }), - ]; - mockDb.read.mockResolvedValue(mockRows); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.select.call(appService, {}); - - expect(result[0].owner).toEqual({ - username: 'user1', - uuid: 'uuid-1', - }); - expect(result[1].owner).toEqual({ - username: 'user2', - uuid: 'uuid-2', - }); - }); - - it('should handle filetypes that are not strings', async () => { - const mockRows = [ - createMockAppRow({ id: 1, filetypes: '[".txt", 123]' }), - ]; - mockDb.read.mockResolvedValue(mockRows); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.select.call(appService, {})).rejects.toThrow( - 'expected filetypesAsJSON[1] to be a string', - ); - }); - - it('should handle malformed filetypes JSON', async () => { - const mockRows = [ - createMockAppRow({ id: 1, filetypes: 'not valid json' }), - ]; - mockDb.read.mockResolvedValue(mockRows); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.select.call(appService, {})).rejects.toThrow( - 'failed to get app filetype associations', - ); - }); - - it('should not require dialect-specific JSON aggregation for app selection', async () => { - mockDb.read.mockResolvedValue([]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.select.call(appService, {}); - - expect(mockDb.case).not.toHaveBeenCalled(); - }); - }); - - describe('#build_complex_id_where (via #read)', () => { - it('should accept "name" as a valid redundant identifier', async () => { - mockDb.read.mockResolvedValue([createMockAppRow()]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.read.call(appService, { id: { name: 'test' } }); - - expect(mockDb.read).toHaveBeenCalledWith( - expect.stringContaining('apps.name = ?'), - ['test'], - ); - }); - - it('should reject identifiers not in REDUNDANT_IDENTIFIERS', async () => { - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.read.call(appService, { id: { title: 'test' } })).rejects.toThrow('Invalid complex id keys: title'); - }); - }); - - describe('#create', () => { - it('should create an app with valid input', async () => { - setupContextForWrite(createMockUserActor(1)); - - // Mock the read after insert - mockDb.read.mockImplementation(async (query) => { - if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) { - return []; - } - return [createMockAppRow({ - uid: expect.stringContaining('app-'), - name: 'new-app', - title: 'New App', - index_url: 'https://example.com/new', - })]; - }); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.create.call(appService, { - object: { - name: 'new-app', - title: 'New App', - index_url: 'https://example.com/new', - }, - }); - - expect(mockDbWrite.write).toHaveBeenCalledWith( - expect.stringContaining('INSERT INTO apps'), - expect.arrayContaining(['new-app', 'New App', 'https://example.com/new']), - ); - }); - - it('should throw forbidden for non-user actors', async () => { - // Mock an invalid actor type - Context.get.mockImplementation((key) => { - if ( key === 'actor' ) return { type: {} }; - return null; - }); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.create.call(appService, { - object: { - name: 'test-app', - title: 'Test', - index_url: 'https://example.com', - }, - })).rejects.toThrow(); - }); - - it('should throw field_missing when name is not provided', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.create.call(appService, { - object: { - title: 'Test', - index_url: 'https://example.com', - }, - })).rejects.toThrow(); - }); - - it('should throw field_missing when title is not provided', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.create.call(appService, { - object: { - name: 'test-app', - index_url: 'https://example.com', - }, - })).rejects.toThrow(); - }); - - it('should throw field_missing when index_url is not provided', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.create.call(appService, { - object: { - name: 'test-app', - title: 'Test', - }, - })).rejects.toThrow(); - }); - - it('should remove protected fields from input', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockResolvedValue([createMockAppRow()]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.create.call(appService, { - object: { - name: 'test-app', - title: 'Test', - index_url: 'https://example.com', - last_review: '2024-01-01', // protected field - }, - }); - - // The INSERT should not include last_review - expect(mockDbWrite.write).toHaveBeenCalledWith( - expect.stringContaining('INSERT INTO apps'), - expect.not.arrayContaining(['2024-01-01']), - ); - }); - - it('should remove read_only fields from input', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockResolvedValue([createMockAppRow()]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.create.call(appService, { - object: { - name: 'test-app', - title: 'Test', - index_url: 'https://example.com', - approved_for_listing: true, // read_only field - godmode: true, // read_only field - is_private: true, // read_only field - }, - }); - - // These fields should not appear in the INSERT - const writeCall = mockDbWrite.write.mock.calls[0]; - expect(writeCall[0]).not.toContain('approved_for_listing'); - expect(writeCall[0]).not.toContain('godmode'); - expect(writeCall[0]).not.toContain('is_private'); - }); - - it('should handle name conflict with dedupe_name option', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockResolvedValue([createMockAppRow()]); - - // First check returns true (name exists), second returns false - app_name_exists - .mockResolvedValueOnce(true) // 'new-app' exists - .mockResolvedValueOnce(false); // 'new-app-1' doesn't exist - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.create.call(appService, { - object: { - name: 'new-app', - title: 'New App', - index_url: 'https://example.com', - }, - options: { dedupe_name: true }, - }); - - // Should have inserted with deduped name - expect(mockDbWrite.write).toHaveBeenCalledWith( - expect.stringContaining('INSERT INTO apps'), - expect.arrayContaining(['new-app-1']), - ); - }); - - it('should throw error when name conflict without dedupe_name', async () => { - setupContextForWrite(createMockUserActor(1)); - app_name_exists.mockResolvedValue(true); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.create.call(appService, { - object: { - name: 'existing-app', - title: 'Test', - index_url: 'https://example.com', - }, - })).rejects.toThrow(); - }); - - it('should allow equivalent index_url already in use on create for non-hosted origins', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockImplementation(async (query) => { - if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) { - return [{ - id: 999, - uid: 'app-existing-uid', - owner_user_id: 1, - index_url: 'https://example.com/', - }]; - } - return [createMockAppRow()]; - }); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.create.call(appService, { - object: { - name: 'new-app', - title: 'New App', - index_url: 'https://example.com/index.html', - }, - })).resolves.toBeDefined(); - }); - - it('should allow duplicate dev-center placeholder index_url on create', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockImplementation(async (query) => { - if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) { - return [{ - id: 999, - uid: 'app-existing-placeholder', - owner_user_id: 1, - index_url: 'https://dev-center.puter.com/coming-soon.html', - }]; - } - return [createMockAppRow()]; - }); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.create.call(appService, { - object: { - name: 'new-app', - title: 'New App', - index_url: 'https://dev-center.puter.com/coming-soon.html', - }, - })).resolves.toBeDefined(); - }); - - it('should join existing hosted app when index_url is owned and already used', async () => { - setupContextForWrite(createMockUserActor(1)); - mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 }); - mockDb.read.mockImplementation(async (query) => { - if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) { - return [{ - id: 999, - uid: 'app-existing-hosted', - owner_user_id: null, - index_url: 'https://mysite.puter.site', - }]; - } - return [createMockAppRow({ - id: 999, - uid: 'app-existing-hosted', - name: 'existing-hosted-app', - title: 'Existing Hosted App', - index_url: 'https://mysite.puter.site', - owner_user_id: 1, - })]; - }); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const joined = await crudQ.create.call(appService, { - object: { - name: 'joined-hosted-app', - title: 'Joined Hosted App', - index_url: 'https://mysite.puter.site', - }, - }); - - expect(joined.uid).toBe('app-existing-hosted'); - expect(mockDbWrite.write).not.toHaveBeenCalledWith( - expect.stringContaining('INSERT INTO apps'), - expect.any(Array), - ); - expect(mockKvStoreService.set).not.toHaveBeenCalled(); - }); - - it('should throw when hosted index_url is already in use by another owner on create', async () => { - setupContextForWrite(createMockUserActor(1)); - mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 }); - mockDb.read.mockImplementation(async (query) => { - if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) { - return [{ - id: 999, - uid: 'app-existing-hosted', - owner_user_id: 2, - index_url: 'https://mysite.puter.site', - }]; - } - return [createMockAppRow()]; - }); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.create.call(appService, { - object: { - name: 'new-app', - title: 'New App', - index_url: 'https://mysite.puter.site', - }, - })).rejects.toMatchObject({ - fields: { - code: 'app_index_url_already_in_use', - }, - }); - }); - - it('should set app_owner when actor is AppUnderUserActorType', async () => { - setupContextForWrite(createMockAppUnderUserActor(1, 100)); - mockDb.read.mockResolvedValue([createMockAppRow()]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.create.call(appService, { - object: { - name: 'test-app', - title: 'Test', - index_url: 'https://example.com', - }, - }); - - // Should include app_owner in the INSERT - expect(mockDbWrite.write).toHaveBeenCalledWith( - expect.stringContaining('app_owner'), - expect.arrayContaining([100]), - ); - }); - - it('should emit app.new-icon event when icon is provided', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockResolvedValue([createMockAppRow()]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.create.call(appService, { - object: { - name: 'test-app', - title: 'Test', - index_url: 'https://example.com', - icon: 'data:image/png;base64,abc123', - }, - }); - - expect(mockEventService.emit).toHaveBeenCalledWith( - 'app.new-icon', - expect.objectContaining({ - data_url: 'data:image/png;base64,abc123', - }), - ); - }); - - it('should accept raw base64 icon and normalize to data URL on create', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockResolvedValue([createMockAppRow()]); - - const rawBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ'; - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.create.call(appService, { - object: { - name: 'test-app', - title: 'Test', - index_url: 'https://example.com', - icon: rawBase64, - }, - }); - - expect(mockEventService.emit).toHaveBeenCalledWith( - 'app.new-icon', - expect.objectContaining({ - data_url: `data:image/png;base64,${rawBase64}`, - }), - ); - }); - - it('should migrate relative app-icon endpoint path to absolute URL on create', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockResolvedValue([createMockAppRow()]); - validate_url.mockImplementation((_value, { key }) => { - if ( key === 'icon' ) { - throw new Error('icon should not be validated as a URL'); - } - }); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.create.call(appService, { - object: { - name: 'test-app', - title: 'Test', - index_url: 'https://example.com', - icon: '/app-icon/app-uid-123/64', - }, - }); - - expect(mockEventService.emit).toHaveBeenCalledWith( - 'app.new-icon', - expect.objectContaining({ - data_url: 'https://api.puter.localhost/app-icon/app-uid-123', - }), - ); - expect(validate_url).toHaveBeenCalledWith('https://example.com', expect.objectContaining({ key: 'index_url' })); - }); - - it('should reject object icon payloads on create', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockResolvedValue([createMockAppRow()]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await expect(crudQ.create.call(appService, { - object: { - name: 'test-app', - title: 'Test', - index_url: 'https://example.com', - icon: { url: '/app-icon/app-uid-123/64' }, - }, - })).rejects.toMatchObject({ - fields: { code: 'field_invalid', key: 'icon' }, - }); - }); - - it('should allow empty icon string on create', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockResolvedValue([createMockAppRow()]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.create.call(appService, { - object: { - name: 'test-app', - title: 'Test', - index_url: 'https://example.com', - icon: '', - }, - }); - - expect(mockEventService.emit).not.toHaveBeenCalledWith('app.new-icon', expect.anything()); - }); - - it('should migrate legacy app-icons host URL to app-icon endpoint URL on create', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockResolvedValue([createMockAppRow()]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.create.call(appService, { - object: { - name: 'test-app', - title: 'Test', - index_url: 'https://example.com', - icon: 'https://puter-app-icons.puter.site/app-uid-123-64.png', - }, - }); - - expect(mockEventService.emit).toHaveBeenCalledWith( - 'app.new-icon', - expect.objectContaining({ - data_url: 'https://api.puter.localhost/app-icon/app-uid-123', - }), - ); - }); - - it('should allow absolute app-icon endpoint URL on API origin', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockResolvedValue([createMockAppRow()]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.create.call(appService, { - object: { - name: 'test-app', - title: 'Test', - index_url: 'https://example.com', - icon: 'https://api.puter.localhost/app-icon/app-uid-123/64', - }, - }); - - expect(mockEventService.emit).toHaveBeenCalledWith( - 'app.new-icon', - expect.objectContaining({ - data_url: 'https://api.puter.localhost/app-icon/app-uid-123', - }), - ); - }); - - it('should reject foreign absolute app-icon endpoint URL on create', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await expect(crudQ.create.call(appService, { - object: { - name: 'test-app', - title: 'Test', - index_url: 'https://example.com', - icon: 'https://evil.example/app-icon/app-uid-123/64', - }, - })).rejects.toMatchObject({ - fields: { code: 'field_invalid', key: 'icon' }, - }); - }); - - it('should reject non app-icon URL icon on create', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await expect(crudQ.create.call(appService, { - object: { - name: 'test-app', - title: 'Test', - index_url: 'https://example.com', - icon: 'https://example.com/webhook', - }, - })).rejects.toMatchObject({ - fields: { code: 'field_invalid', key: 'icon' }, - }); - }); - - it('should handle filetype_associations', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockResolvedValue([createMockAppRow()]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.create.call(appService, { - object: { - name: 'test-app', - title: 'Test', - index_url: 'https://example.com', - filetype_associations: ['txt', 'pdf'], - }, - }); - - // Should have three write calls: INSERT app, DELETE old associations, INSERT new associations - // (DELETE is called even for create since #update_filetype_associations always clears first) - expect(mockDbWrite.write).toHaveBeenCalledTimes(3); - expect(mockDbWrite.write).toHaveBeenCalledWith( - expect.stringContaining('DELETE FROM app_filetype_association'), - [1], - ); - expect(mockDbWrite.write).toHaveBeenCalledWith( - expect.stringContaining('INSERT INTO app_filetype_association'), - expect.arrayContaining([1, 'txt', 1, 'pdf']), - ); - }); - - it('should call validate_string for name and title', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockResolvedValue([createMockAppRow()]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.create.call(appService, { - object: { - name: 'test-app', - title: 'Test Title', - index_url: 'https://example.com', - }, - }); - - expect(validate_string).toHaveBeenCalledWith('test-app', expect.objectContaining({ key: 'name' })); - expect(validate_string).toHaveBeenCalledWith('Test Title', expect.objectContaining({ key: 'title' })); - }); - - it('should call validate_url for index_url', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockImplementation(async (query) => { - if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) { - return []; - } - return [createMockAppRow()]; - }); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.create.call(appService, { - object: { - name: 'test-app', - title: 'Test', - index_url: 'https://example.com/app', - }, - }); - - expect(validate_url).toHaveBeenCalledWith('https://example.com/app', expect.objectContaining({ key: 'index_url' })); - }); - - it('should generate a UID with app- prefix', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockResolvedValue([createMockAppRow()]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.create.call(appService, { - object: { - name: 'test-app', - title: 'Test', - index_url: 'https://example.com', - }, - }); - - const writeCall = mockDbWrite.write.mock.calls[0]; - const values = writeCall[1]; - const uidValue = values[0]; // uid is first value - expect(uidValue).toMatch(/^app-[0-9a-f-]{36}$/); - }); - }); - - describe('#update', () => { - beforeEach(() => { - // Default: return an existing app for updates - mockDb.read.mockResolvedValue([createMockAppRow({ - owner_user_id: 1, - })]); - }); - - it('should update an app with valid input', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { uid: 'app-uid-123', title: 'Updated Title' }, - }); - - expect(mockDbWrite.write).toHaveBeenCalledWith( - expect.stringContaining('UPDATE apps SET'), - expect.arrayContaining(['Updated Title', 'app-uid-123']), - ); - }); - - it('should throw entity_not_found when app does not exist', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockResolvedValue([]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.update.call(appService, { - object: { uid: 'nonexistent-uid', title: 'Test' }, - })).rejects.toThrow(); - }); - - it('should throw forbidden when user does not own the app', async () => { - // User 2 trying to update app owned by user 1 - setupContextForWrite(createMockUserActor(2), { id: 2 }); - mockDb.read.mockResolvedValue([createMockAppRow({ - owner_user_id: 1, - })]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.update.call(appService, { - object: { uid: 'app-uid-123', title: 'Hacked Title' }, - })).rejects.toThrow(); - }); - - it('should allow update when user has write-all-owners permission', async () => { - setupContextForWrite(createMockUserActor(2), { id: 2 }); - mockDb.read.mockResolvedValue([createMockAppRow({ - owner_user_id: 1, - })]); - mockPermissionService.check.mockResolvedValue(true); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { uid: 'app-uid-123', title: 'Admin Update' }, - }); - - expect(mockDbWrite.write).toHaveBeenCalledWith( - expect.stringContaining('UPDATE apps SET'), - expect.arrayContaining(['Admin Update']), - ); - }); - - it('should remove protected fields from update', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - title: 'Updated', - last_review: '2024-12-01', // protected field - }, - }); - - const writeCall = mockDbWrite.write.mock.calls[0]; - expect(writeCall[0]).not.toContain('last_review'); - }); - - it('should remove read_only fields from update', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - title: 'Updated', - approved_for_listing: true, - godmode: true, - is_private: true, - }, - }); - - const writeCall = mockDbWrite.write.mock.calls[0]; - expect(writeCall[0]).not.toContain('approved_for_listing'); - expect(writeCall[0]).not.toContain('godmode'); - expect(writeCall[0]).not.toContain('is_private'); - }); - - it('should handle name change with conflict', async () => { - setupContextForWrite(createMockUserActor(1)); - app_name_exists.mockResolvedValue(true); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.update.call(appService, { - object: { uid: 'app-uid-123', name: 'taken-name' }, - })).rejects.toThrow(); - }); - - it('should allow name change with dedupe_name option', async () => { - setupContextForWrite(createMockUserActor(1)); - app_name_exists - .mockResolvedValueOnce(true) // 'new-name' exists - .mockResolvedValueOnce(false); // 'new-name-1' doesn't exist - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { uid: 'app-uid-123', name: 'new-name' }, - options: { dedupe_name: true }, - }); - - expect(mockDbWrite.write).toHaveBeenCalledWith( - expect.stringContaining('UPDATE apps SET'), - expect.arrayContaining(['new-name-1']), - ); - }); - - it('should allow reclaiming old app name', async () => { - setupContextForWrite(createMockUserActor(1)); - app_name_exists.mockResolvedValue(true); - mockOldAppNameService.check_app_name.mockResolvedValue({ - id: 99, - app_uid: 'app-uid-123', // Same app - }); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { uid: 'app-uid-123', name: 'old-name' }, - }); - - expect(mockOldAppNameService.remove_name).toHaveBeenCalledWith(99); - expect(mockDbWrite.write).toHaveBeenCalled(); - }); - - it('should not update name if unchanged', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { uid: 'app-uid-123', name: 'test-app' }, // Same as existing - }); - - // Should only have the read for ID, no name in update - const writeCall = mockDbWrite.write.mock.calls.find(call => call[0].includes('UPDATE')); - if ( writeCall ) { - expect(writeCall[1]).not.toContain('test-app'); - } - }); - - it('should emit app.new-icon event when icon changes', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - icon: 'data:image/png;base64,newicon', - }, - }); - - expect(mockEventService.emit).toHaveBeenCalledWith( - 'app.new-icon', - expect.objectContaining({ - app_uid: 'app-uid-123', - data_url: 'data:image/png;base64,newicon', - }), - ); - }); - - it('should accept raw base64 icon and normalize to data URL on update', async () => { - setupContextForWrite(createMockUserActor(1)); - - const rawBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ'; - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - icon: rawBase64, - }, - }); - - expect(mockEventService.emit).toHaveBeenCalledWith( - 'app.new-icon', - expect.objectContaining({ - app_uid: 'app-uid-123', - data_url: `data:image/png;base64,${rawBase64}`, - }), - ); - }); - - it('should migrate relative app-icon endpoint path to absolute URL on update', async () => { - setupContextForWrite(createMockUserActor(1)); - validate_url.mockImplementation((_value, { key }) => { - if ( key === 'icon' ) { - throw new Error('icon should not be validated as a URL'); - } - }); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - icon: '/app-icon/app-uid-123/64', - }, - }); - - expect(mockEventService.emit).toHaveBeenCalledWith( - 'app.new-icon', - expect.objectContaining({ - app_uid: 'app-uid-123', - data_url: 'https://api.puter.localhost/app-icon/app-uid-123', - }), - ); - }); - - it('should reject object icon payloads on update', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await expect(crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - icon: { url: '/app-icon/app-uid-123/64' }, - }, - })).rejects.toMatchObject({ - fields: { code: 'field_invalid', key: 'icon' }, - }); - }); - - it('should allow empty icon string on update', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - icon: '', - }, - }); - - expect(mockEventService.emit).toHaveBeenCalledWith( - 'app.new-icon', - expect.objectContaining({ - app_uid: 'app-uid-123', - data_url: '', - }), - ); - }); - - it('should migrate legacy app-icons host URL to app-icon endpoint URL on update', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - icon: 'https://puter-app-icons.puter.site/app-uid-123-64.png', - }, - }); - - expect(mockEventService.emit).toHaveBeenCalledWith( - 'app.new-icon', - expect.objectContaining({ - app_uid: 'app-uid-123', - data_url: 'https://api.puter.localhost/app-icon/app-uid-123', - }), - ); - }); - - it('should allow absolute app-icon endpoint URL on API origin when updating icon', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - icon: 'https://api.puter.localhost/app-icon/app-uid-123/64', - }, - }); - - expect(mockEventService.emit).toHaveBeenCalledWith( - 'app.new-icon', - expect.objectContaining({ - app_uid: 'app-uid-123', - data_url: 'https://api.puter.localhost/app-icon/app-uid-123', - }), - ); - }); - - it('should reject foreign absolute app-icon endpoint URL on update', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await expect(crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - icon: 'https://evil.example/app-icon/app-uid-123/64', - }, - })).rejects.toMatchObject({ - fields: { code: 'field_invalid', key: 'icon' }, - }); - }); - - it('should reject non app-icon URL icon on update', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await expect(crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - icon: 'https://example.com/webhook', - }, - })).rejects.toMatchObject({ - fields: { code: 'field_invalid', key: 'icon' }, - }); - }); - - it('should emit app.rename event when name changes', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { uid: 'app-uid-123', name: 'renamed-app' }, - }); - - expect(mockEventService.emit).toHaveBeenCalledWith( - 'app.rename', - expect.objectContaining({ - app_uid: 'app-uid-123', - new_name: 'renamed-app', - old_name: 'test-app', - }), - ); - }); - - it('should update filetype_associations', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - filetype_associations: ['doc', 'xls'], - }, - }); - - // Should delete old associations - expect(mockDbWrite.write).toHaveBeenCalledWith( - expect.stringContaining('DELETE FROM app_filetype_association'), - [1], - ); - - // Should insert new associations - expect(mockDbWrite.write).toHaveBeenCalledWith( - expect.stringContaining('INSERT INTO app_filetype_association'), - expect.arrayContaining([1, 'doc', 1, 'xls']), - ); - }); - - it('should validate fields when provided', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - name: 'updated-name', - title: 'Updated Title', - description: 'Updated description', - index_url: 'https://updated.com', - }, - }); - - expect(validate_string).toHaveBeenCalledWith('updated-name', expect.objectContaining({ key: 'name' })); - expect(validate_string).toHaveBeenCalledWith('Updated Title', expect.objectContaining({ key: 'title' })); - expect(validate_string).toHaveBeenCalledWith('Updated description', expect.objectContaining({ key: 'description' })); - expect(validate_url).toHaveBeenCalledWith('https://updated.com', expect.objectContaining({ key: 'index_url' })); - }); - - it('should check subdomain ownership when index_url changes to puter.site', async () => { - setupContextForWrite(createMockUserActor(1)); - mockPuterSiteService.get_subdomain.mockResolvedValue(null); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - index_url: 'https://mysite.puter.site', - }, - })).rejects.toThrow(); - }); - - it('should allow index_url change when subdomain is owned', async () => { - setupContextForWrite(createMockUserActor(1)); - mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 }); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - index_url: 'https://mysite.puter.site', - }, - }); - - expect(mockDbWrite.write).toHaveBeenCalledWith( - expect.stringContaining('UPDATE apps SET'), - expect.arrayContaining(['https://mysite.puter.site']), - ); - }); - - it('should allow index_url change when private hosted subdomain is owned', async () => { - setupContextForWrite(createMockUserActor(1)); - mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 }); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - index_url: 'https://mysite.puter.dev', - }, - }); - - expect(mockDbWrite.write).toHaveBeenCalledWith( - expect.stringContaining('UPDATE apps SET'), - expect.arrayContaining(['https://mysite.puter.dev']), - ); - }); - - it('should allow equivalent index_url already in use on update for non-hosted origins', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockImplementation(async (query) => { - if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) { - return [{ - id: 777, - uid: 'app-conflict-uid', - owner_user_id: 2, - index_url: 'https://updated.com/', - }]; - } - return [createMockAppRow()]; - }); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await expect(crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - index_url: 'https://updated.com/index.html', - }, - })).resolves.toBeDefined(); - }); - - it('should allow duplicate dev-center placeholder index_url on update', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockImplementation(async (query) => { - if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) { - return [{ - id: 777, - uid: 'app-existing-placeholder', - owner_user_id: 1, - index_url: 'https://dev-center.puter.com/coming-soon.html', - }]; - } - return [createMockAppRow()]; - }); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await expect(crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - index_url: 'https://dev-center.puter.com/coming-soon.html', - }, - })).resolves.toBeDefined(); - }); - - it('should join existing unowned hosted app when index_url is already in use on update', async () => { - setupContextForWrite(createMockUserActor(1)); - mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 }); - let readCallCount = 0; - mockDb.read.mockImplementation(async (query, params) => { - readCallCount++; - if ( readCallCount > 100 ) { - throw new Error(`excessive mockDb.read calls in join test: ${String(query)} :: ${JSON.stringify(params)}`); - } - if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) { - if ( Array.isArray(params) && params[params.length - 1] === 777 ) { - // Mirrors SQL `AND id != ?` behavior during join follow-up updates. - return []; - } - return [{ - id: 777, - uid: 'app-conflict-uid', - owner_user_id: null, - index_url: 'https://mysite.puter.site/', - }]; - } - if ( Array.isArray(params) && params[0] === 'app-conflict-uid' ) { - return [createMockAppRow({ - id: 777, - uid: 'app-conflict-uid', - name: 'existing-hosted-app', - title: 'Existing Hosted App', - index_url: 'https://mysite.puter.site/', - owner_user_id: 1, - })]; - } - return [createMockAppRow({ - id: 1, - uid: 'app-uid-123', - name: 'updating-app', - title: 'Updating App', - index_url: 'https://other.puter.site', - owner_user_id: 1, - })]; - }); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - title: 'Joined Update Title', - index_url: 'https://mysite.puter.site/index.html', - }, - }); - - expect(result.uid).toBe('app-conflict-uid'); - expect(mockDbWrite.write).toHaveBeenCalledWith( - expect.stringContaining('UPDATE apps SET'), - expect.arrayContaining(['Joined Update Title', 'app-conflict-uid']), - ); - expect(mockAppInformationService.delete_app).toHaveBeenCalledWith( - 'app-uid-123', - undefined, - { preserveCanonicalUidAlias: true }, - ); - expect(mockKvStoreService.set).toHaveBeenCalledWith(expect.objectContaining({ - key: 'app:canonicalUidAlias:app-uid-123', - value: 'app-conflict-uid', - })); - }); - - it('should throw when owned hosted index_url is already in use on update', async () => { - setupContextForWrite(createMockUserActor(1)); - mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 }); - mockDb.read.mockImplementation(async (query) => { - if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) { - return [{ - id: 777, - uid: 'app-conflict-uid', - owner_user_id: 1, - index_url: 'https://mysite.puter.site/', - }]; - } - return [createMockAppRow()]; - }); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await expect(crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - index_url: 'https://mysite.puter.site/index.html', - }, - })).rejects.toMatchObject({ - fields: { - code: 'app_index_url_already_in_use', - }, - }); - }); - - it('should throw when equivalent hosted index_url is already in use on update', async () => { - setupContextForWrite(createMockUserActor(1)); - mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 }); - mockDb.read.mockImplementation(async (query) => { - if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) { - return [{ - id: 777, - uid: 'app-conflict-uid', - owner_user_id: 2, - index_url: 'https://mysite.puter.site/', - }]; - } - return [createMockAppRow()]; - }); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await expect(crudQ.update.call(appService, { - object: { - uid: 'app-uid-123', - index_url: 'https://mysite.puter.site/index.html', - }, - })).rejects.toMatchObject({ - fields: { - code: 'app_index_url_already_in_use', - }, - }); - }); - - it('should throw forbidden when app actor does not own the entity (AppLimitedES behavior)', async () => { - // App actor trying to update an app it didn't create - setupContextForWrite(createMockAppUnderUserActor(1, 999)); - mockDb.read.mockResolvedValue([createMockAppRow({ - owner_user_id: 1, - app_owner_uid: 'different-app-uid', - })]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.update.call(appService, { - object: { uid: 'app-uid-123', title: 'Hacked Title' }, - })).rejects.toThrow(); - }); - - it('should allow app actor to update entity it owns (AppLimitedES behavior)', async () => { - // App actor updating an app it created - const actor = createMockAppUnderUserActor(1, 100); - actor.type.app.uid = 'creator-app-uid'; - setupContextForWrite(actor); - mockDb.read.mockResolvedValue([createMockAppRow({ - owner_user_id: 1, - app_owner_uid: 'creator-app-uid', - })]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { uid: 'app-uid-123', title: 'Updated by App' }, - }); - - expect(mockDbWrite.write).toHaveBeenCalledWith( - expect.stringContaining('UPDATE apps SET'), - expect.arrayContaining(['Updated by App']), - ); - }); - - it('should allow app actor with write permission to update any entity (AppLimitedES behavior)', async () => { - setupContextForWrite(createMockAppUnderUserActor(1, 999)); - mockDb.read.mockResolvedValue([createMockAppRow({ - owner_user_id: 1, - app_owner_uid: 'different-app-uid', - })]); - // Grant write permission - mockPermissionService.check.mockResolvedValue(true); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.update.call(appService, { - object: { uid: 'app-uid-123', title: 'Admin Update' }, - }); - - expect(mockDbWrite.write).toHaveBeenCalledWith( - expect.stringContaining('UPDATE apps SET'), - expect.arrayContaining(['Admin Update']), - ); - }); - }); - - describe('#upsert', () => { - it('should call create when entity does not exist', async () => { - setupContextForWrite(createMockUserActor(1)); - - mockDb.read.mockResolvedValue([createMockAppRow()]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.upsert.call(appService, { - object: { - name: 'new-app', - title: 'New App', - index_url: 'https://example.com', - }, - }); - - expect(mockDbWrite.write).toHaveBeenCalledWith( - expect.stringContaining('INSERT INTO apps'), - expect.any(Array), - ); - }); - - it('should call update when entity exists', async () => { - setupContextForWrite(createMockUserActor(1)); - - // Read returns existing entity - mockDb.read.mockResolvedValue([createMockAppRow()]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.upsert.call(appService, { - object: { uid: 'app-uid-123', title: 'Updated Title' }, - }); - - expect(mockDbWrite.write).toHaveBeenCalledWith( - expect.stringContaining('UPDATE apps SET'), - expect.any(Array), - ); - }); - }); - - describe('#delete', () => { - beforeEach(() => { - // Mock app-information service - mockAppInformationService = { - delete_app: vi.fn().mockResolvedValue(undefined), - }; - - // Update mockServices to include app-information - mockServices.get.mockImplementation((serviceName) => { - if ( serviceName === 'database' ) { - return { - get: vi.fn().mockImplementation((mode) => { - if ( mode === 'write' ) return mockDbWrite; - return mockDb; - }), - }; - } - if ( serviceName === 'event' ) return mockEventService; - if ( serviceName === 'permission' ) return mockPermissionService; - if ( serviceName === 'puter-site' ) return mockPuterSiteService; - if ( serviceName === 'old-app-name' ) return mockOldAppNameService; - if ( serviceName === 'app-information' ) return mockAppInformationService; - if ( serviceName === 'puter-kvstore' ) return mockKvStoreService; - if ( serviceName === 'su' ) return mockSuService; - return null; - }); - - // Default: return an existing app for deletes - mockDb.read.mockResolvedValue([createMockAppRow({ - owner_user_id: 1, - })]); - }); - - it('should delete an app by uid', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.delete.call(appService, { uid: 'app-uid-123' }); - - expect(mockAppInformationService.delete_app).toHaveBeenCalledWith('app-uid-123'); - expect(result.success).toBe(true); - expect(result.uid).toBe('app-uid-123'); - }); - - it('should delete an app by complex id (name)', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.delete.call(appService, { id: { name: 'test-app' } }); - - expect(mockAppInformationService.delete_app).toHaveBeenCalledWith('app-uid-123'); - expect(result.success).toBe(true); - }); - - it('should throw entity_not_found when app does not exist', async () => { - setupContextForWrite(createMockUserActor(1)); - mockDb.read.mockResolvedValue([]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.delete.call(appService, { uid: 'nonexistent-uid' })) - .rejects.toThrow(); - }); - - it('should throw forbidden for non-user actors', async () => { - Context.get.mockImplementation((key) => { - if ( key === 'actor' ) return { type: {} }; - return null; - }); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.delete.call(appService, { uid: 'app-uid-123' })) - .rejects.toThrow(); - }); - - it('should throw forbidden when user does not own the app', async () => { - // User 2 trying to delete app owned by user 1 - setupContextForWrite(createMockUserActor(2), { id: 2 }); - mockDb.read.mockResolvedValue([createMockAppRow({ - owner_user_id: 1, - })]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.delete.call(appService, { uid: 'app-uid-123' })) - .rejects.toThrow(); - }); - - it('should allow delete when user has write-all-owners permission', async () => { - setupContextForWrite(createMockUserActor(2), { id: 2 }); - mockDb.read.mockResolvedValue([createMockAppRow({ - owner_user_id: 1, - })]); - mockPermissionService.check.mockResolvedValue(true); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.delete.call(appService, { uid: 'app-uid-123' }); - - expect(mockAppInformationService.delete_app).toHaveBeenCalled(); - expect(result.success).toBe(true); - }); - - it('should invalidate app cache after delete', async () => { - setupContextForWrite(createMockUserActor(1)); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - await crudQ.delete.call(appService, { uid: 'app-uid-123' }); - }); - - it('should throw forbidden when app actor does not own the entity', async () => { - // App actor trying to delete an app it didn't create - setupContextForWrite(createMockAppUnderUserActor(1, 999)); - mockDb.read.mockResolvedValue([createMockAppRow({ - owner_user_id: 1, - app_owner_uid: 'different-app-uid', - })]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - - await expect(crudQ.delete.call(appService, { uid: 'app-uid-123' })) - .rejects.toThrow(); - }); - - it('should allow app actor to delete entity it owns', async () => { - // App actor deleting an app it created - const actor = createMockAppUnderUserActor(1, 100); - actor.type.app.uid = 'creator-app-uid'; - setupContextForWrite(actor); - mockDb.read.mockResolvedValue([createMockAppRow({ - owner_user_id: 1, - app_owner_uid: 'creator-app-uid', - })]); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.delete.call(appService, { uid: 'app-uid-123' }); - - expect(mockAppInformationService.delete_app).toHaveBeenCalled(); - expect(result.success).toBe(true); - }); - - it('should allow app actor with write permission to delete any entity', async () => { - setupContextForWrite(createMockAppUnderUserActor(1, 999)); - mockDb.read.mockResolvedValue([createMockAppRow({ - owner_user_id: 1, - app_owner_uid: 'different-app-uid', - })]); - // Grant write permission - mockPermissionService.check.mockResolvedValue(true); - - const crudQ = AppService.IMPLEMENTS['crud-q']; - const result = await crudQ.delete.call(appService, { uid: 'app-uid-123' }); - - expect(mockAppInformationService.delete_app).toHaveBeenCalled(); - expect(result.success).toBe(true); - }); - }); -}); diff --git a/src/backend/src/modules/data-access/DEV.md b/src/backend/src/modules/data-access/DEV.md deleted file mode 100644 index 02d48d61d..000000000 --- a/src/backend/src/modules/data-access/DEV.md +++ /dev/null @@ -1,362 +0,0 @@ -## Development for `data-access` module - -This document will contain notes, documentation, and snippets written -while developing the `data-access` module replacements for what was -formerly handled by EntityStoreService and OM (Object Mapping). - -### App List Test Code - -This code is used to test listing apps with one of the available -CRUD-implementing drivers. - -```javascript -await (async () => { - const resp = await fetch('http://api.puter.localhost:4100/drivers/call', { - method: 'POST', - headers: { - Authorization: `Bearer ${puter.authToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - args: { predicate: ['user-can-edit'] }, - driver: 'es:app', - interface: 'puter-apps', - method: 'select', - }), - }) - return (await resp.json()).result; -})(); -``` - -### AI-Generated Compare Function - -I asked an LLM to find me a javascript object compare function -that I can paste in developer tools and it started generating -one from scratch. To my surprise it worked just fine, so I'm pasting -this here for the time being for convenience: - -```javascript -(() => { - // Deep compare + diff reporter for DevTools (no deps) - // Usage: - // const r = deepCompare(a, b); - // console.log(r.pass, r.message); - // r.print(); // pretty console output - // Options: - // deepCompare(a,b,{ showSame:false, maxDiffs:200, sortKeys:true }) - - function deepCompare(a, b, opts = {}) { - const options = { - showSame: false, // include "same" entries in the diff list - maxDiffs: 200, // cap diffs so you don't nuke your console - sortKeys: true, // stable key ordering when iterating plain objects - ...opts, - }; - - const diffs = []; - const seenPairs = new WeakMap(); // a -> WeakMap(b -> true) - - const isObjectLike = (v) => v !== null && (typeof v === "object" || typeof v === "function"); - const tagOf = (v) => Object.prototype.toString.call(v); // "[object X]" - const isPlainObject = (v) => { - if (tagOf(v) !== "[object Object]") return false; - const proto = Object.getPrototypeOf(v); - return proto === Object.prototype || proto === null; - }; - - const typeLabel = (v) => { - if (v === null) return "null"; - const t = typeof v; - if (t !== "object") return t; - return tagOf(v).slice(8, -1); - }; - - const formatVal = (v) => { - // Safe-ish inline formatter for messages (keeps things short) - try { - if (typeof v === "string") return JSON.stringify(v.length > 120 ? v.slice(0, 117) + "…" : v); - if (typeof v === "number" && Object.is(v, -0)) return "-0"; - if (typeof v === "bigint") return `${v}n`; - if (typeof v === "symbol") return v.toString(); - if (typeof v === "function") return `[Function ${v.name || "anonymous"}]`; - if (v instanceof Date) return isNaN(v.getTime()) ? "Invalid Date" : `Date(${v.toISOString()})`; - if (v instanceof RegExp) return v.toString(); - if (v instanceof Map) return `Map(${v.size})`; - if (v instanceof Set) return `Set(${v.size})`; - if (ArrayBuffer.isView(v) && !(v instanceof DataView)) return `${v.constructor.name}(${v.length})`; - if (v instanceof ArrayBuffer) return `ArrayBuffer(${v.byteLength})`; - if (v && v.constructor && v.constructor !== Object) return `${v.constructor.name}{…}`; - if (Array.isArray(v)) return `Array(${v.length})`; - if (isPlainObject(v)) return "Object{…}"; - return `${typeLabel(v)}{…}`; - } catch { - return "[Unformattable]"; - } - }; - - const pathToString = (path) => { - if (!path.length) return "(root)"; - let s = ""; - for (const p of path) { - if (typeof p === "number") s += `[${p}]`; - else if (typeof p === "string") { - if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(p)) s += (s ? "." : "") + p; - else s += `[${JSON.stringify(p)}]`; - } else if (typeof p === "symbol") s += `[${p.toString()}]`; - else s += `[${String(p)}]`; - } - return s; - }; - - const pushDiff = (kind, path, left, right, extra) => { - if (diffs.length >= options.maxDiffs) return; - diffs.push({ - kind, // "type" | "value" | "missing-left" | "missing-right" | "prototype" | "keys" | ... - path: [...path], - left, - right, - extra, - }); - }; - - const markSeen = (x, y) => { - if (!isObjectLike(x) || !isObjectLike(y)) return false; - let inner = seenPairs.get(x); - if (!inner) { - inner = new WeakMap(); - seenPairs.set(x, inner); - } - if (inner.get(y)) return true; - inner.set(y, true); - return false; - }; - - const sameValueZero = (x, y) => Object.is(x, y); // handles NaN, -0 - - const compareArrays = (x, y, path) => { - if (x.length !== y.length) pushDiff("value", [...path, "length"], x.length, y.length, "array length mismatch"); - const n = Math.max(x.length, y.length); - for (let i = 0; i < n; i++) { - if (i >= x.length) pushDiff("missing-left", [...path, i], undefined, y[i], "missing index in left"); - else if (i >= y.length) pushDiff("missing-right", [...path, i], x[i], undefined, "missing index in right"); - else walk(x[i], y[i], [...path, i]); - if (diffs.length >= options.maxDiffs) return; - } - }; - - const compareTypedArrays = (x, y, path) => { - if (x.constructor !== y.constructor) { - pushDiff("type", path, x.constructor?.name, y.constructor?.name, "typed array class mismatch"); - return; - } - if (x.length !== y.length) pushDiff("value", [...path, "length"], x.length, y.length, "typed array length mismatch"); - const n = Math.min(x.length, y.length); - for (let i = 0; i < n; i++) { - if (!sameValueZero(x[i], y[i])) pushDiff("value", [...path, i], x[i], y[i], "typed array element mismatch"); - if (diffs.length >= options.maxDiffs) return; - } - }; - - const compareArrayBuffer = (x, y, path) => { - if (x.byteLength !== y.byteLength) { - pushDiff("value", [...path, "byteLength"], x.byteLength, y.byteLength, "ArrayBuffer byteLength mismatch"); - return; - } - const a8 = new Uint8Array(x); - const b8 = new Uint8Array(y); - for (let i = 0; i < a8.length; i++) { - if (a8[i] !== b8[i]) { - pushDiff("value", [...path, i], a8[i], b8[i], "ArrayBuffer byte mismatch"); - if (diffs.length >= options.maxDiffs) return; - } - } - }; - - const compareDates = (x, y, path) => { - const tx = x.getTime(); - const ty = y.getTime(); - if (!sameValueZero(tx, ty)) pushDiff("value", path, x, y, "Date mismatch"); - }; - - const compareRegex = (x, y, path) => { - if (x.source !== y.source || x.flags !== y.flags) pushDiff("value", path, x, y, "RegExp mismatch"); - }; - - const compareMaps = (x, y, path) => { - if (x.size !== y.size) pushDiff("value", [...path, "size"], x.size, y.size, "Map size mismatch"); - - // Map key equality is identity-based; here we: - // 1) try direct key lookup for primitive keys - // 2) for object keys, we require the *same object reference* exists as key in the other map - // (test frameworks do similar unless they do expensive key deep-matching) - for (const [k, xv] of x.entries()) { - if (!y.has(k)) { - pushDiff("missing-right", [...path, `MapKey(${formatVal(k)})`], xv, undefined, "Map missing key on right"); - continue; - } - walk(xv, y.get(k), [...path, `MapKey(${formatVal(k)})`]); - if (diffs.length >= options.maxDiffs) return; - } - - for (const [k, yv] of y.entries()) { - if (!x.has(k)) { - pushDiff("missing-left", [...path, `MapKey(${formatVal(k)})`], undefined, yv, "Map missing key on left"); - if (diffs.length >= options.maxDiffs) return; - } - } - }; - - const compareSets = (x, y, path) => { - if (x.size !== y.size) pushDiff("value", [...path, "size"], x.size, y.size, "Set size mismatch"); - - // Same logic: membership is identity for object values. - for (const v of x.values()) { - if (!y.has(v)) pushDiff("missing-right", [...path, `SetVal(${formatVal(v)})`], v, undefined, "Set missing value on right"); - if (diffs.length >= options.maxDiffs) return; - } - for (const v of y.values()) { - if (!x.has(v)) pushDiff("missing-left", [...path, `SetVal(${formatVal(v)})`], undefined, v, "Set missing value on left"); - if (diffs.length >= options.maxDiffs) return; - } - }; - - const comparePlainObjects = (x, y, path) => { - // Compare prototypes (handy when something is class instance vs plain object) - const px = Object.getPrototypeOf(x); - const py = Object.getPrototypeOf(y); - if (px !== py) pushDiff("prototype", path, px?.constructor?.name || px, py?.constructor?.name || py, "Prototype mismatch"); - - const keysX = Reflect.ownKeys(x); - const keysY = Reflect.ownKeys(y); - - const norm = (ks) => { - // Sort only string keys for stability; keep symbols in original order - if (!options.sortKeys) return ks; - const str = ks.filter(k => typeof k === "string").sort(); - const sym = ks.filter(k => typeof k === "symbol"); - const numLike = []; // keep numeric-looking strings in numeric order if you want; leaving out to stay simple - // We'll just do lexical sort for strings; okay for devtools output. - return [...str, ...sym]; - }; - - const kx = norm(keysX); - const ky = norm(keysY); - - const setY = new Set(keysY); - const setX = new Set(keysX); - - for (const k of kx) { - if (!setY.has(k)) { - pushDiff("missing-right", [...path, k], x[k], undefined, "Missing property on right"); - } else { - walk(x[k], y[k], [...path, k]); - } - if (diffs.length >= options.maxDiffs) return; - } - for (const k of ky) { - if (!setX.has(k)) { - pushDiff("missing-left", [...path, k], undefined, y[k], "Missing property on left"); - if (diffs.length >= options.maxDiffs) return; - } - } - }; - - function walk(x, y, path) { - if (diffs.length >= options.maxDiffs) return; - - if (sameValueZero(x, y)) { - if (options.showSame) pushDiff("same", path, x, y); - return; - } - - const tx = typeLabel(x); - const ty = typeLabel(y); - if (tx !== ty) { - pushDiff("type", path, tx, ty, "Type mismatch"); - return; - } - - // Circular / repeated references - if (markSeen(x, y)) return; - - // Per-type comparisons - if (Array.isArray(x)) return compareArrays(x, y, path); - - if (ArrayBuffer.isView(x) && !(x instanceof DataView)) return compareTypedArrays(x, y, path); - if (x instanceof ArrayBuffer) return compareArrayBuffer(x, y, path); - - if (x instanceof Date) return compareDates(x, y, path); - if (x instanceof RegExp) return compareRegex(x, y, path); - if (x instanceof Map) return compareMaps(x, y, path); - if (x instanceof Set) return compareSets(x, y, path); - - // Functions: compare by reference already failed; treat as value mismatch - if (typeof x === "function") { - pushDiff("value", path, x, y, "Function reference mismatch"); - return; - } - - // Objects (including class instances): compare own keys + nested values. - if (isObjectLike(x)) return comparePlainObjects(x, y, path); - - // Primitives (should have been caught by Object.is earlier) - pushDiff("value", path, x, y, "Value mismatch"); - } - - walk(a, b, []); - - const pass = diffs.length === 0; - - const message = pass - ? "✅ Values are deeply equal." - : buildMessage(diffs, options); - - function buildMessage(diffs, options) { - const lines = []; - lines.push(`❌ Values differ (${diffs.length}${diffs.length >= options.maxDiffs ? "+" : ""} diff${diffs.length === 1 ? "" : "s"}):`); - for (let i = 0; i < diffs.length; i++) { - const d = diffs[i]; - const p = pathToString(d.path); - const left = formatVal(d.left); - const right = formatVal(d.right); - const label = d.kind.padEnd(14, " "); - const extra = d.extra ? ` — ${d.extra}` : ""; - lines.push(`${String(i + 1).padStart(3, " ")}. ${label} ${p}${extra}`); - lines.push(` left : ${left}`); - lines.push(` right: ${right}`); - } - if (diffs.length >= options.maxDiffs) { - lines.push(`… (diffs capped at maxDiffs=${options.maxDiffs})`); - } - return lines.join("\n"); - } - - function print() { - if (pass) { - console.log("%c✅ deepCompare: PASS", "font-weight:bold"); - return; - } - console.groupCollapsed(`%c❌ deepCompare: FAIL (${diffs.length}${diffs.length >= options.maxDiffs ? "+" : ""})`, "font-weight:bold"); - console.log(message); - - // Also log a structured table for quick scanning - const table = diffs.map((d) => ({ - kind: d.kind, - path: pathToString(d.path), - left: formatVal(d.left), - right: formatVal(d.right), - note: d.extra || "", - })); - try { console.table(table); } catch {} - console.groupEnd(); - } - - return { pass, diffs, message, print }; - } - - // Expose globally for DevTools convenience - window.deepCompare = deepCompare; - console.log("deepCompare installed. Usage: deepCompare(a,b).print()"); -})(); - -``` \ No newline at end of file diff --git a/src/backend/src/modules/data-access/DataAccessModule.js b/src/backend/src/modules/data-access/DataAccessModule.js deleted file mode 100644 index 78f34a5b3..000000000 --- a/src/backend/src/modules/data-access/DataAccessModule.js +++ /dev/null @@ -1,10 +0,0 @@ -import { AdvancedBase } from '@heyputer/putility'; -import AppService from './AppService.js'; - -export class DataAccessModule extends AdvancedBase { - async install (context) { - const services = context.get('services'); - - services.registerService('app', AppService); - } -} diff --git a/src/backend/src/modules/data-access/lib/coercion.js b/src/backend/src/modules/data-access/lib/coercion.js deleted file mode 100644 index af49601bb..000000000 --- a/src/backend/src/modules/data-access/lib/coercion.js +++ /dev/null @@ -1,28 +0,0 @@ -// These utility functions describe how values stored in the database -// are to be understood as their higher-level counterparts. - -import { CoercionTypeError } from './error.js'; - -/** - * MySQL lets us store `1` (an integer) or `0` (also an integer) as - * the closest parallel to a boolean "true or false" value. - * Sqlite lets us store `"1"` (a string) or `0` (also a string) as - * the closest parallel to a boolean "true of false" value. - * - * So we define a function here called `as_bool` that will make - * `"0"` or `0` become `false`, and `"1"` or `1` become `true`. - * - * @param {any} value - The value to coerce to a boolean. - * @returns {boolean} The coerced boolean value. - */ -export const as_bool = value => { - if ( value === undefined ) return false; - if ( value === 0 ) value = false; - if ( value === 1 ) value = true; - if ( value === '0' ) value = false; - if ( value === '1' ) value = true; - if ( typeof value !== 'boolean' ) { - throw new CoercionTypeError({ expected: 'boolean', got: typeof value }); - } - return value; -}; diff --git a/src/backend/src/modules/data-access/lib/error.js b/src/backend/src/modules/data-access/lib/error.js deleted file mode 100644 index a14d1b850..000000000 --- a/src/backend/src/modules/data-access/lib/error.js +++ /dev/null @@ -1,11 +0,0 @@ -/** - * Replaces `OMTypeError` from ES/OM implementation. - * This might be removed or replaced in the future. - */ -export class CoercionTypeError extends Error { - constructor ({ expected, got }) { - const message = `expected ${expected}, got ${got}`; - super(message); - this.name = 'CoercionTypeError'; - } -} diff --git a/src/backend/src/modules/data-access/lib/filter.js b/src/backend/src/modules/data-access/lib/filter.js deleted file mode 100644 index 8adddaf2f..000000000 --- a/src/backend/src/modules/data-access/lib/filter.js +++ /dev/null @@ -1,10 +0,0 @@ -// These utility functions describe how to produce an object safe -// for transfer that came from a "raw" object. - -export const user_to_client = raw_user => { - return { - username: raw_user.username, - // This `uuid` is not an internal-only ID. - uuid: raw_user.uuid, - }; -}; diff --git a/src/backend/src/modules/data-access/lib/sqlutil.js b/src/backend/src/modules/data-access/lib/sqlutil.js deleted file mode 100644 index 6c146b6db..000000000 --- a/src/backend/src/modules/data-access/lib/sqlutil.js +++ /dev/null @@ -1,21 +0,0 @@ -/** - * When columns are selected from a joined table and prefixed: - * - * SELECT joined_table.* AS joined_table_ - * - * This function is able to extract the object from the result: - * - * extract_from_prefix(row, 'joined_table_') // columns of joined_table - * - * @param {*} row - * @param {*} prefix - */ -export const extract_from_prefix = (row, prefix) => { - const result = {}; - for ( const [key, value] of Object.entries(row) ) { - if ( key.startsWith(prefix) ) { - result[key.replace(prefix, '')] = value; - } - } - return result; -}; diff --git a/src/backend/src/modules/data-access/lib/validation.js b/src/backend/src/modules/data-access/lib/validation.js deleted file mode 100644 index 225cef00f..000000000 --- a/src/backend/src/modules/data-access/lib/validation.js +++ /dev/null @@ -1,97 +0,0 @@ -import validator from 'validator'; -import APIError from '../../../api/APIError.js'; - -/** - * Validates a string value with optional maxlen and regex constraints. - * @param {string} value - The value to validate - * @param {object} meta - Metadata for the validation - * @param {string} meta.key - The field name (for error messages) - * @param {number} [meta.maxlen] - Maximum length allowed - * @param {RegExp} [meta.regex] - Regex pattern the string must match - */ -export const validate_string = (value, { key, maxlen, regex }) => { - if ( typeof value !== 'string' ) { - throw APIError.create('field_invalid', null, { key }); - } - if ( maxlen !== undefined && value.length > maxlen ) { - throw APIError.create('field_too_long', null, { key, max_length: maxlen }); - } - if ( regex !== undefined && !regex.test(value) ) { - throw APIError.create('field_invalid', null, { key }); - } -}; - -/** - * Validates an image-base64 value (data URL for images). - * Checks for proper prefix and XSS characters. - * @param {string} value - The value to validate - * @param {object} meta - Metadata for the validation - * @param {string} meta.key - The field name (for error messages) - */ -export const validate_image_base64 = (value, { key }) => { - if ( typeof value !== 'string' ) { - throw APIError.create('field_invalid', null, { key }); - } - if ( ! value.startsWith('data:image/') ) { - throw APIError.create('field_invalid', null, { key }); - } - // XSS character check from image-base64 prop type - const xss_chars = ['<', '>', '&', '"', "'", '`']; - if ( xss_chars.some(char => value.includes(char)) ) { - throw APIError.create('field_invalid', null, { key }); - } -}; - -/** - * Validates a URL value with optional maxlen constraint. - * Uses the validator library, allowing localhost. - * @param {string} value - The value to validate - * @param {object} meta - Metadata for the validation - * @param {string} meta.key - The field name (for error messages) - * @param {number} [meta.maxlen] - Maximum length allowed - */ -export const validate_url = (value, { key, maxlen }) => { - if ( typeof value !== 'string' ) { - throw APIError.create('field_invalid', null, { key }); - } - if ( maxlen !== undefined && value.length > maxlen ) { - throw APIError.create('field_too_long', null, { key, max_length: maxlen }); - } - // URL validation using validator library (same as url prop type) - let valid = validator.isURL(value); - if ( ! valid ) { - valid = validator.isURL(value, { host_whitelist: ['localhost'] }); - } - if ( ! valid ) { - throw APIError.create('field_invalid', null, { key }); - } -}; - -/** - * Validates a JSON value (must be an object or array). - * @param {*} value - The value to validate - * @param {object} meta - Metadata for the validation - * @param {string} meta.key - The field name (for error messages) - */ -export const validate_json = (value, { key }) => { - if ( typeof value !== 'object' ) { - throw APIError.create('field_invalid', null, { key }); - } -}; - -/** - * Validates an array where each element is a string. - * @param {*} value - The value to validate - * @param {object} meta - Metadata for the validation - * @param {string} meta.key - The field name (for error messages) - */ -export const validate_array_of_strings = (value, { key }) => { - if ( ! Array.isArray(value) ) { - throw APIError.create('field_invalid', null, { key }); - } - for ( const item of value ) { - if ( typeof item !== 'string' ) { - throw APIError.create('field_invalid', null, { key }); - } - } -}; diff --git a/src/backend/src/modules/entitystore/EntityStoreInterfaceService.js b/src/backend/src/modules/entitystore/EntityStoreInterfaceService.js deleted file mode 100644 index 3b0f3ce0b..000000000 --- a/src/backend/src/modules/entitystore/EntityStoreInterfaceService.js +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (C) 2025-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const BaseService = require('../../services/BaseService'); - -/** -* Service class that manages Entity Store interface registrations. -* Handles registration of the crud-q interface which is used by various -* entity storage services. -* @extends BaseService -*/ -class EntityStoreInterfaceService extends BaseService { - /** - * Service class for managing Entity Store interface registrations. - * Extends the base service to provide entity storage interface management. - */ - async '__on_driver.register.interfaces' () { - const svc_registry = this.services.get('registry'); - const col_interfaces = svc_registry.get('interfaces'); - - // Define the standard CRUD interface methods that will be reused - const crudMethods = { - create: { - parameters: { - object: { - type: 'json', - subtype: 'object', - required: true, - }, - options: { type: 'json' }, - }, - }, - read: { - parameters: { - uid: { type: 'string' }, - id: { type: 'json' }, - params: { type: 'json' }, - }, - }, - select: { - parameters: { - predicate: { type: 'json' }, - offset: { type: 'number' }, - limit: { type: 'number' }, - params: { type: 'json' }, - }, - }, - update: { - parameters: { - id: { type: 'json' }, - object: { - type: 'json', - subtype: 'object', - required: true, - }, - options: { type: 'json' }, - }, - }, - upsert: { - parameters: { - id: { type: 'json' }, - object: { - type: 'json', - subtype: 'object', - required: true, - }, - options: { type: 'json' }, - }, - }, - delete: { - parameters: { - uid: { type: 'string' }, - id: { type: 'json' }, - }, - }, - }; - - // Register the crud-q interface - col_interfaces.set('crud-q', { - methods: { ...crudMethods }, - }); - - // Register entity-specific interfaces that use crud-q - const entityInterfaces = [ - { - name: 'puter-apps', - description: 'Manage a developer\'s apps on Puter.', - }, - { - name: 'puter-subdomains', - description: 'Manage subdomains on Puter.', - }, - { - name: 'puter-notifications', - description: 'Read notifications on Puter.', - }, - ]; - - // Register each entity interface with the same CRUD methods - for ( const entity of entityInterfaces ) { - col_interfaces.set(entity.name, { - description: entity.description, - methods: { ...crudMethods }, - }); - } - } -} - -module.exports = { - EntityStoreInterfaceService, -}; \ No newline at end of file diff --git a/src/backend/src/modules/entitystore/EntityStoreModule.js b/src/backend/src/modules/entitystore/EntityStoreModule.js deleted file mode 100644 index afad23600..000000000 --- a/src/backend/src/modules/entitystore/EntityStoreModule.js +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2025-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { AdvancedBase } = require('@heyputer/putility'); -const { EntityStoreInterfaceService } = require('./EntityStoreInterfaceService'); - -/** - * A module for registering entity store interfaces. - */ -class EntityStoreModule extends AdvancedBase { - async install (context) { - const services = context.get('services'); - - // Register interface services - services.registerService('entitystore-interface', EntityStoreInterfaceService); - } -} - -module.exports = { - EntityStoreModule, -}; \ No newline at end of file diff --git a/src/backend/src/modules/filesystem/roadmap.md b/src/backend/src/modules/filesystem/roadmap.md deleted file mode 100644 index 8080f81bd..000000000 --- a/src/backend/src/modules/filesystem/roadmap.md +++ /dev/null @@ -1,21 +0,0 @@ -## Mountpounts hurdles - -- [ ] subdomains use integer IDs to to reference files, which - only works with PuterFS. This means other filesystem - providers will not be usable for subdomains. - - Possible solutions: - - GUI logic to disable subdomains feature for other providers - - Add a new column to associate subdomains with paths - - Map non-puterfs nodes to (1B + path_id), where path_id is - a numeric identifier that is associated with the path, and - the association is stored in the database or system runtime - directory. - -- [ ] permissions are associated with UUIDs, but will need to - be able to be associated with paths instead for non-puterfs - mountpoints. - - - Make path-to-uuid re-writer act on puter-fs only. - - ACL needs to be able to check path-based permissions - on non-puterfs mountpoints. diff --git a/src/backend/src/modules/hostos/HostOSModule.js b/src/backend/src/modules/hostos/HostOSModule.js deleted file mode 100644 index 0ee2e1136..000000000 --- a/src/backend/src/modules/hostos/HostOSModule.js +++ /dev/null @@ -1,14 +0,0 @@ -const { AdvancedBase } = require('@heyputer/putility'); - -class HostOSModule extends AdvancedBase { - async install (context) { - const services = context.get('services'); - - const ProcessService = require('./ProcessService'); - services.registerService('process', ProcessService); - } -} - -module.exports = { - HostOSModule, -}; diff --git a/src/backend/src/modules/hostos/ProcessService.js b/src/backend/src/modules/hostos/ProcessService.js deleted file mode 100644 index 1a4deb70e..000000000 --- a/src/backend/src/modules/hostos/ProcessService.js +++ /dev/null @@ -1,96 +0,0 @@ -const BaseService = require('../../services/BaseService'); - -class ProxyLogger { - constructor (log) { - this.log = log; - } - attach (stream) { - let buffer = ''; - stream.on('data', (chunk) => { - buffer += chunk.toString(); - let lineEndIndex = buffer.indexOf('\n'); - while ( lineEndIndex !== -1 ) { - const line = buffer.substring(0, lineEndIndex); - this.log(line); - buffer = buffer.substring(lineEndIndex + 1); - lineEndIndex = buffer.indexOf('\n'); - } - }); - - stream.on('end', () => { - if ( buffer.length ) { - this.log(buffer); - } - }); - } -} - -class ProcessService extends BaseService { - static CONCERN = 'workers'; - - static MODULES = { - path: require('path'), - spawn: require('child_process').spawn, - }; - - _construct () { - this.instances = []; - } - - async _init (args) { - this.args = args; - - process.on('exit', () => { - this.exit_all_(); - }); - } - - log_ (name, isErr, line) { - let txt = `[${name}:`; - txt += isErr - ? '\x1B[34;1m2\x1B[0m' - : '\x1B[32;1m1\x1B[0m'; - txt += `] ${ line}`; - this.log.info(txt); - } - - async exit_all_ () { - for ( const { proc } of this.instances ) { - proc.kill(); - } - } - - async start ({ name, fullpath, command, args, env }) { - this.log.info(`Starting ${name} in ${fullpath}`); - const env_processed = { ...(env ?? {}) }; - for ( const k in env_processed ) { - if ( typeof env_processed[k] !== 'function' ) continue; - env_processed[k] = env_processed[k]({ - global_config: this.global_config, - }); - } - this.log.debug('command', - { command, args }); - const proc = this.modules.spawn(command, args, { - shell: true, - env: { - ...process.env, - ...env_processed, - }, - cwd: fullpath, - }); - this.instances.push({ - name, proc, - }); - const out = new ProxyLogger((line) => this.log_(name, false, line)); - out.attach(proc.stdout); - const err = new ProxyLogger((line) => this.log_(name, true, line)); - err.attach(proc.stderr); - proc.on('exit', () => { - this.log.info(`[${name}:exit] Process exited (${proc.exitCode})`); - this.instances = this.instances.filter((inst) => inst.proc !== proc); - }); - } -} - -module.exports = ProcessService; diff --git a/src/backend/src/modules/internet/InternetModule.js b/src/backend/src/modules/internet/InternetModule.js deleted file mode 100644 index 84a13af55..000000000 --- a/src/backend/src/modules/internet/InternetModule.js +++ /dev/null @@ -1,16 +0,0 @@ -const { AdvancedBase } = require('@heyputer/putility'); -const config = require('../../config.js'); - -class InternetModule extends AdvancedBase { - async install (context) { - const services = context.get('services'); - - if ( config?.services?.['wisp-relay'] ) { - const WispRelayService = require('./WispRelayService.js'); - services.registerService('wisp-relay', WispRelayService); - } - - } -} - -module.exports = { InternetModule }; diff --git a/src/backend/src/modules/internet/WispRelayService.js b/src/backend/src/modules/internet/WispRelayService.js deleted file mode 100644 index 95f012f7a..000000000 --- a/src/backend/src/modules/internet/WispRelayService.js +++ /dev/null @@ -1,20 +0,0 @@ -const BaseService = require('../../services/BaseService'); - -class WispRelayService extends BaseService { - _init () { - const path_ = require('path'); - const svc_process = this.services.get('process'); - svc_process.start({ - name: 'internet.js', - command: this.config.node_path, - fullpath: this.config.wisp_relay_path, - args: ['index.js'], - env: { - PORT: this.config.wisp_relay_port, - WISP_AUTH_SERVER: this.config.origin, - }, - }); - } -} - -module.exports = WispRelayService; diff --git a/src/backend/src/modules/kvstore/KVStoreInterfaceService.js b/src/backend/src/modules/kvstore/KVStoreInterfaceService.js deleted file mode 100644 index f0d9dc38e..000000000 --- a/src/backend/src/modules/kvstore/KVStoreInterfaceService.js +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Copyright (C) 2025-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const BaseService = require('../../services/BaseService'); - -/** - * @typedef {Object} KVStoreInterface - * @property {function(KVStoreGetParams): Promise} get - Retrieve the value(s) for the given key(s). - * @property {function(KVStoreSetParams): Promise} set - Set a value for a key, with optional expiration. - * @property {function(KVStoreBatchPutParams): Promise} batchPut - Set many key-value entries in one call. - * @property {function(KVStoreDelParams): Promise} del - Delete a value by key. - * @property {function(KVStoreListParams): Promise} list - List key-value pairs, optionally with pagination. - * @property {function(): Promise} flush - Delete all key-value pairs in the store. - * @property {(params: KVStoreUpdateParams) => Promise} update - Update nested values by key. - * @property {(params: KVStoreAddParams) => Promise} add - Append values into list paths by key. - * @property {(params: KVStoreRemoveParams) => Promise} remove - Remove nested values by key. - * @property {(params: {key:string, pathAndAmountMap: Record}) => Promise} incr - Increment a numeric value by key. - * @property {(params: {key:string, pathAndAmountMap: Record}) => Promise} decr - Decrement a numeric value by key. - * @property {function(KVStoreExpireAtParams): Promise} expireAt - Set a key to expire at a specific UNIX timestamp (seconds). - * @property {function(KVStoreExpireParams): Promise} expire - Set a key to expire after a given TTL (seconds). - * - * @typedef {Object} KVStoreGetParams - * @property {string|string[]} key - The key or array of keys to retrieve. - * - * @typedef {Object} KVStoreSetParams - * @property {string} key - The key to set. - * @property {*} value - The value to store. - * @property {number} [expireAt] - Optional UNIX timestamp (seconds) when the key should expire. - * - * @typedef {Object} KVStoreBatchPutParams - * @property {{key: string, value: *, expireAt?: number}[]} items - Key/value pairs to store. - * - * @typedef {Object} KVStoreDelParams - * @property {string} key - The key to delete. - * - * @typedef {Object} KVStoreListParams - * @property {string} [as] - Optional type to list as ("keys", "values", or "entries"). - * @property {string} [pattern] - Optional key prefix to match. - * @property {number} [limit] - Optional max number of items to return. - * @property {string} [cursor] - Optional cursor to continue listing from. - * - * @typedef {Object} KVStoreListResult - * @property {Array} items - Items in the current page. - * @property {string} [cursor] - Cursor for the next page, if available. - * - * @typedef {Object} KVStoreUpdateParams - * @property {string} key - The key to update. - * @property {Object.} pathAndValueMap - Map of period-joined paths to values. - * @property {number} [ttl] - Optional TTL in seconds for the whole object. - * - * @typedef {Object} KVStoreAddParams - * @property {string} key - The key to update. - * @property {Object.} pathAndValueMap - Map of period-joined paths to values to append. - * - * @typedef {Object} KVStoreRemoveParams - * @property {string} key - The key to update. - * @property {string[]} paths - List of period-joined paths to remove. - * - * @typedef {Object} KVStoreExpireAtParams - * @property {string} key - The key to set expiration for. - * @property {number} timestamp - UNIX timestamp (seconds) when the key should expire. - * - * @typedef {Object} KVStoreExpireParams - * @property {string} key - The key to set expiration for. - * @property {number} ttl - Time-to-live in seconds. - */ - -/** - * Service for registering the puter-kvstore interface, exposing a simple key-value store API - * with support for get, set, delete, list, flush, increment, decrement, and key expiration. - * @extends BaseService - */ -class KVStoreInterfaceService extends BaseService { - /** - * Service class for managing KVStore interface registrations. - * Extends the base service to provide key-value store interface management. - */ - async '__on_driver.register.interfaces' () { - const svc_registry = this.services.get('registry'); - const col_interfaces = svc_registry.get('interfaces'); - - // Register the puter-kvstore interface - col_interfaces.set('puter-kvstore', { - description: 'A simple key-value store.', - methods: { - get: { - description: 'Get a value by key.', - parameters: { - key: { type: 'json', required: true }, - optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, - }, - result: { type: 'json' }, - }, - set: { - description: 'Set a value by key.', - parameters: { - key: { type: 'string', required: true }, - value: { type: 'json' }, - expireAt: { type: 'number' }, - optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, - - }, - result: { type: 'void' }, - }, - batchPut: { - description: 'Set many values by key in a single call.', - parameters: { - items: { type: 'json', required: true }, - optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, - }, - result: { type: 'void' }, - }, - del: { - description: 'Delete a value by key.', - parameters: { - key: { type: 'string' }, - optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, - }, - result: { type: 'void' }, - }, - list: { - description: 'List key-value pairs with optional pagination.', - parameters: { - as: { - type: 'string', - }, - pattern: { - type: 'string', - }, - limit: { - type: 'number', - }, - cursor: { - type: 'string', - }, - optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, - }, - result: { type: 'json' }, - }, - flush: { - description: 'Delete all key-value pairs.', - parameters: { optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' } }, - result: { type: 'void' }, - }, - update: { - description: 'Update nested values by key.', - parameters: { - key: { type: 'string', required: true }, - pathAndValueMap: { type: 'json', required: true, description: 'map of period-joined path to value' }, - ttl: { type: 'number', description: 'optional TTL in seconds for the whole object' }, - optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, - }, - result: { type: 'json', description: 'The updated value' }, - }, - add: { - description: 'Append values into list paths by key.', - parameters: { - key: { type: 'string', required: true }, - pathAndValueMap: { type: 'json', required: true, description: 'map of period-joined path to value to append' }, - optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, - }, - result: { type: 'json', description: 'The updated value' }, - }, - remove: { - description: 'Remove nested values by key.', - parameters: { - key: { type: 'string', required: true }, - paths: { type: 'json', required: true, description: 'list of period-joined paths to remove' }, - optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, - }, - result: { type: 'json', description: 'The updated value' }, - }, - incr: { - description: 'Increment a value by key.', - parameters: { - key: { type: 'string', required: true }, - pathAndAmountMap: { type: 'json', required: true, description: 'map of period-joined path to amount to increment by' }, - optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, - }, - result: { type: 'json', description: 'The updated value' }, - }, - decr: { - description: 'Decrement a value by key.', - parameters: { - key: { type: 'string', required: true }, - pathAndAmountMap: { type: 'json', required: true, description: 'map of period-joined path to amount to increment by' }, - optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, - - }, - result: { type: 'json', description: 'The updated value' }, - }, - expireAt: { - description: 'Set a key to expire at a given timestamp in sec.', - parameters: { - key: { type: 'string', required: true }, - timestamp: { type: 'number', required: true }, - optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, - - }, - result: { type: 'number' }, - }, - expire: { - description: 'Set a key to expire in ttl many seconds.', - parameters: { - key: { type: 'string', required: true }, - ttl: { type: 'number', required: true }, - optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, - - }, - result: { type: 'number' }, - }, - }, - }); - } -} - -module.exports = { - KVStoreInterfaceService, -}; diff --git a/src/backend/src/modules/kvstore/KVStoreModule.js b/src/backend/src/modules/kvstore/KVStoreModule.js deleted file mode 100644 index 101e7c000..000000000 --- a/src/backend/src/modules/kvstore/KVStoreModule.js +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2025-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { AdvancedBase } = require('@heyputer/putility'); -const { KVStoreInterfaceService } = require('./KVStoreInterfaceService'); - -/** - * A module for registering key-value store interfaces. - */ -class KVStoreModule extends AdvancedBase { - async install (context) { - const services = context.get('services'); - - // Register interface services - services.registerService('kvstore-interface', KVStoreInterfaceService); - } -} - -module.exports = { - KVStoreModule, -}; \ No newline at end of file diff --git a/src/backend/src/modules/perfmon/TelemetryService.js b/src/backend/src/modules/perfmon/TelemetryService.js deleted file mode 100644 index 249fb8d95..000000000 --- a/src/backend/src/modules/perfmon/TelemetryService.js +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { SpanStatusCode, trace } from '@opentelemetry/api'; -import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node'; -import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'; -import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'; -import { Resource } from '@opentelemetry/resources'; -import { ConsoleMetricExporter, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; -import { NodeSDK } from '@opentelemetry/sdk-node'; -import { ConsoleSpanExporter } from '@opentelemetry/sdk-trace-base'; -import { SemanticAttributes, SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; -import config from '../../config.js'; -import BaseService from '../../services/BaseService.js'; - -export class TelemetryService extends BaseService { - static TRACER_NAME = 'puter-tracer'; - static #sharedSdk = null; - static #sharedTracer = null; - static #telemetryStarted = false; - - /** @type {import('@opentelemetry/api').Tracer} */ - #tracer = null; - - constructor (service_resources, ...args) { - super(service_resources, ...args); - const { sdk, tracer } = TelemetryService.#startTelemetry({ - serviceConfig: this.config, - }); - this.sdk = sdk; - this.#tracer = tracer; - } - - _init () { - if ( ! this.#tracer ) { - return; - } - const svc_context = this.services.get('context', { optional: true }); - if ( ! svc_context ) { - return; - } - svc_context.register_context_hook('pre_arun', ({ hints, trace_name, callback, replace_callback }) => { - if ( ! trace_name ) return; - if ( ! hints.trace ) return; - replace_callback(async () => { - return await this.#tracer.startActiveSpan(trace_name, async span => { - try { - return await callback(); - } catch ( error ) { - span.setStatus({ code: SpanStatusCode.ERROR, message: error.message }); - throw error; - } finally { - span.end(); - } - }); - }); - }); - } - - static #normalizeRoute (route) { - if ( Array.isArray(route) ) { - for ( const entry of route ) { - if ( typeof entry === 'string' ) { - return entry; - } - } - return undefined; - } - if ( typeof route === 'string' ) { - return route; - } - if ( route instanceof RegExp ) { - return route.toString(); - } - } - - static #buildRoute (req, route) { - const normalized = TelemetryService.#normalizeRoute(route); - if ( ! normalized ) { - return undefined; - } - const baseUrl = typeof req?.baseUrl === 'string' ? req.baseUrl : ''; - const combined = `${baseUrl}${normalized}`; - return combined || normalized; - } - - static #applyRouteToSpan (span, req, route) { - if ( ! route ) { - return; - } - span.setAttribute(SemanticAttributes.HTTP_ROUTE, route); - if ( typeof span.updateName === 'function' && req?.method ) { - span.updateName(`HTTP ${req.method} ${route}`); - } - } - - static #buildInstrumentationConfig () { - return { - '@opentelemetry/instrumentation-http': { - responseHook: (span, response) => { - const req = response?.req; - const route = TelemetryService.#buildRoute(req, req?.route?.path); - TelemetryService.#applyRouteToSpan(span, req, route); - }, - }, - '@opentelemetry/instrumentation-express': { - spanNameHook: (info, defaultName) => { - if ( info.layerType !== 'request_handler' ) { - return defaultName; - } - const route = TelemetryService.#buildRoute(info.request, info.route); - if ( !route || !info.request?.method ) { - return defaultName; - } - return `HTTP ${info.request.method} ${route}`; - }, - requestHook: (span, info) => { - const route = TelemetryService.#buildRoute(info.request, info.route); - if ( route ) { - span.setAttribute(SemanticAttributes.HTTP_ROUTE, route); - } - }, - }, - }; - } - - static #resolveExporterConfig (_serviceConfig) { - // return config.jaeger ?? serviceConfig?.jaeger; - // TODO DS: reenable if needed - return false; - } - - static #getConfiguredExporter (serviceConfig) { - const exporterConfig = TelemetryService.#resolveExporterConfig(serviceConfig); - if ( exporterConfig ) { - return new OTLPTraceExporter(exporterConfig); - } - if ( serviceConfig?.console ) { - return new ConsoleSpanExporter(); - } - } - - static #getMetricExporter (serviceConfig) { - const exporterConfig = TelemetryService.#resolveExporterConfig(serviceConfig); - if ( exporterConfig ) { - return new OTLPMetricExporter(exporterConfig); - } - if ( serviceConfig?.console ) { - return new ConsoleMetricExporter(); - } - } - - static #startTelemetry ({ serviceConfig } = {}) { - if ( TelemetryService.#telemetryStarted ) { - return { sdk: TelemetryService.#sharedSdk, tracer: TelemetryService.#sharedTracer }; - } - TelemetryService.#telemetryStarted = true; - - const effectiveConfig = serviceConfig ?? config.services?.telemetry ?? {}; - const traceExporter = TelemetryService.#getConfiguredExporter(effectiveConfig); - const metricExporter = TelemetryService.#getMetricExporter(effectiveConfig); - - if ( !traceExporter && !metricExporter ) { - console.log('TelemetryService not configured, skipping initialization.'); - return { sdk: null, tracer: null }; - } - - const resource = Resource.default().merge( - new Resource({ - [SemanticResourceAttributes.SERVICE_NAME]: 'puter-backend', - [SemanticResourceAttributes.SERVICE_VERSION]: '0.1.0', - }), - ); - - const sdkConfig = { - resource, - instrumentations: [ - getNodeAutoInstrumentations(TelemetryService.#buildInstrumentationConfig()), - ], - }; - - if ( traceExporter ) { - sdkConfig.traceExporter = traceExporter; - } - if ( metricExporter ) { - sdkConfig.metricReader = new PeriodicExportingMetricReader({ - exporter: metricExporter, - }); - } - - TelemetryService.#sharedSdk = new NodeSDK(sdkConfig); - TelemetryService.#sharedSdk.start(); - TelemetryService.#sharedTracer = trace.getTracer(TelemetryService.TRACER_NAME); - - return { sdk: TelemetryService.#sharedSdk, tracer: TelemetryService.#sharedTracer }; - } -} diff --git a/src/backend/src/modules/puterfs/MountpointService.js b/src/backend/src/modules/puterfs/MountpointService.js deleted file mode 100644 index 7eec12f40..000000000 --- a/src/backend/src/modules/puterfs/MountpointService.js +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { RootNodeSelector, NodeUIDSelector, NodeChildSelector, NodePathSelector, try_infer_attributes } = require('../../deprecated/filesystem/node/selectors'); -const BaseService = require('../../services/BaseService'); - -/** - * This will eventually be a service which manages the storage - * backends for mountpoints. - * - * For the moment, this is a way to access the storage backend - * in situations where ContextInitService isn't able to - * initialize a context. - */ - -/** -* @class MountpointService -* @extends BaseService -* @description Service class responsible for managing storage backends for mountpoints. -* Currently provides a temporary solution for accessing storage backend when context -* initialization is not possible. Will be expanded to handle multiple mountpoints -* and their associated storage backends in future implementations. -*/ -class MountpointService extends BaseService { - - #storage = {}; - #mounters = {}; - #mountpoints = {}; - - register_mounter (name, mounter) { - this.#mounters[name] = mounter; - } - - async '__on_boot.consolidation' () { - // Emit event for registering filesystem types - const svc_event = this.services.get('event'); - const event = {}; - event.createFilesystemType = (name, filesystemType) => { - this.#mounters[name] = filesystemType; - }; - await svc_event.emit('create.filesystem-types', event); - - // Determine mountpoints configuration - const mountpoints = this.config.mountpoints ?? { - '/': { - mounter: 'puterfs', - }, - }; - - // Mount filesystems - for ( const path of Object.keys(mountpoints) ) { - const { mounter: mounter_name, options } = - mountpoints[path]; - const mounter = this.#mounters[mounter_name]; - if ( ! mounter ) { - throw new Error(`unrecognized filesystem type: ${mounter_name}`); - } - const provider = await mounter.mount({ - path, - options, - }); - this.#mountpoints[path] = { - provider, - }; - } - - this.services.emit('filesystem.ready', { - mountpoints: Object.keys(this.#mountpoints), - }); - } - - async get_provider (selector) { - // If there is only one provider, we don't need to do any of this, - // and that's a big deal because the current implementation requires - // fetching a filesystem entry before we even have operation-level - // transient memoization instantiated. - if ( Object.keys(this.#mountpoints).length === 1 ) { - return Object.values(this.#mountpoints)[0].provider; - } - - try_infer_attributes(selector); - - if ( selector instanceof RootNodeSelector ) { - return this.#mountpoints['/'].provider; - } - - if ( selector instanceof NodeUIDSelector ) { - for ( const { provider } of Object.values(this.#mountpoints) ) { - const result = await provider.quick_check({ - selector, - }); - if ( result ) { - return provider; - } - } - - // No provider found, but we shouldn't throw an error here - // because it's a valid case for a node that doesn't exist. - } - - if ( selector instanceof NodeChildSelector ) { - if ( selector.path ) { - return this.get_provider(new NodePathSelector(selector.path)); - } else { - return this.get_provider(selector.parent); - } - } - - const probe = {}; - selector.setPropertiesKnownBySelector(probe); - if ( probe.path ) { - let longest_mount_path = ''; - for ( const path of Object.keys(this.#mountpoints) ) { - if ( ! probe.path.startsWith(path) ) { - continue; - } - if ( path.length > longest_mount_path.length ) { - longest_mount_path = path; - } - } - - if ( longest_mount_path ) { - return this.#mountpoints[longest_mount_path].provider; - } - } - - // Use root mountpoint as fallback - return this.#mountpoints['/'].provider; - } - - // Temporary solution - we'll develop this incrementally - set_storage (provider, storage) { - this.#storage[provider] = storage; - } - - /** - * Gets the current storage backend instance - * @returns {Object} The storage backend instance - */ - get_storage (provider) { - const storage = this.#storage[provider]; - if ( ! storage ) { - throw new Error(`MountpointService.get_storage: storage for provider "${provider}" not found`); - } - return storage; - } -} - -module.exports = { - MountpointService, -}; diff --git a/src/backend/src/modules/puterfs/PuterFSModule.js b/src/backend/src/modules/puterfs/PuterFSModule.js deleted file mode 100644 index 83b118707..000000000 --- a/src/backend/src/modules/puterfs/PuterFSModule.js +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { AdvancedBase } = require('@heyputer/putility'); -const FSNodeContext = require('../../deprecated/filesystem/FSNodeContext').default; -const capabilities = require('../../deprecated/filesystem/definitions/capabilities'); -const selectors = require('../../deprecated/filesystem/node/selectors'); -const { RuntimeModule } = require('../../extension/RuntimeModule'); -const { MODE_READ, MODE_WRITE } = require('../../services/fs/FSLockService'); -const { UploadProgressTracker } = require('../../deprecated/filesystem/storage/UploadProgressTracker'); -const { PuterPath } = require('../../deprecated/filesystem/lib/PuterPath'); - -class PuterFSModule extends AdvancedBase { - async install (context) { - const services = context.get('services'); - - const { RESOURCE_STATUS_PENDING_CREATE } = require('./ResourceService'); - - // Expose filesystem declarations to extensions - const runtimeModule = new RuntimeModule({ name: 'fs' }); - runtimeModule.exports = { - capabilities, - selectors, - FSNodeContext, - PuterPath, - lock: { - MODE_READ, - MODE_WRITE, - }, - resource: { - RESOURCE_STATUS_PENDING_CREATE, - }, - util: { - UploadProgressTracker, - }, - }; - context.get('runtime-modules').register(runtimeModule); - - const { ResourceService } = require('./ResourceService'); - services.registerService('resourceService', ResourceService); - - const { SizeService } = require('./SizeService'); - services.registerService('sizeService', SizeService); - - const { MountpointService } = require('./MountpointService'); - services.registerService('mountpoint', MountpointService); - } -} - -module.exports = { PuterFSModule }; diff --git a/src/backend/src/modules/puterfs/ResourceService.js b/src/backend/src/modules/puterfs/ResourceService.js deleted file mode 100644 index b287b7ec9..000000000 --- a/src/backend/src/modules/puterfs/ResourceService.js +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const BaseService = require('../../services/BaseService'); -const { - NodePathSelector, - NodeUIDSelector, - NodeInternalIDSelector, - NodeChildSelector, -} = require('../../deprecated/filesystem/node/selectors'); - -const RESOURCE_STATUS_PENDING_CREATE = {}; -const RESOURCE_STATUS_PENDING_UPDATE = {}; -const RS_DIRECTORY_PENDING_CHILD_INSERT = {}; - -/** - * ResourceService is a very simple locking mechanism meant - * only to ensure consistency between requests being sent - * to the same server. - * - * For example, if you send an HTTP request to `/write`, and - * then a subsequent HTTP request to `/read`, you would expect - * the newly written file to be available. Therefore, the call - * to `/read` should wait until the write is complete. - * - * At least for now; I'm sure we'll think of a smarter way to - * handle this in the future. - */ -class ResourceService extends BaseService { - _construct () { - this.uidToEntry = {}; - this.uidToPath = {}; - this.pathToEntry = {}; - } - - register (entry) { - entry = { ...entry }; - - if ( ! entry.uid ) { - // TODO: resource service needs logger access - return; - } - - entry.freePromise = new Promise((resolve, reject) => { - entry.free = () => { - resolve(); - }; - }); - entry.onFree = entry.freePromise.then.bind(entry.freePromise); - this.log.debug('registering resource', { uid: entry.uid }); - this.uidToEntry[entry.uid] = entry; - if ( entry.path ) { - this.uidToPath[entry.uid] = entry.path; - this.pathToEntry[entry.path] = entry; - } - return entry; - } - - free (uid) { - this.log.debug('freeing', { uid }); - const entry = this.uidToEntry[uid]; - if ( ! entry ) return; - delete this.uidToEntry[uid]; - if ( this.uidToPath.hasOwnProperty(uid) ) { - const path = this.uidToPath[uid]; - delete this.pathToEntry[path]; - delete this.uidToPath[uid]; - } - entry.free(); - } - - async waitForResourceByPath (path) { - const entry = this.pathToEntry[path]; - if ( ! entry ) { - return; - } - await entry.freePromise; - } - - async waitForResourceByUID (uid) { - const entry = this.uidToEntry[uid]; - if ( ! entry ) { - return; - } - await entry.freePromise; - } - - async waitForResource (selector) { - if ( selector instanceof NodePathSelector ) { - await this.waitForResourceByPath(selector.value); - } - else - if ( selector instanceof NodeUIDSelector ) { - await this.waitForResourceByUID(selector.value); - } - else - if ( selector instanceof NodeInternalIDSelector ) { - // Can't wait intelligently for this - } - if ( selector instanceof NodeChildSelector ) { - await this.waitForResource(selector.parent); - } - } - - getResourceInfo (uid) { - if ( ! uid ) return; - return this.uidToEntry[uid]; - } -} - -module.exports = { - ResourceService, - RESOURCE_STATUS_PENDING_CREATE, - RESOURCE_STATUS_PENDING_UPDATE, - RS_DIRECTORY_PENDING_CHILD_INSERT, -}; diff --git a/src/backend/src/modules/puterfs/SizeService.js b/src/backend/src/modules/puterfs/SizeService.js deleted file mode 100644 index 67a5afa59..000000000 --- a/src/backend/src/modules/puterfs/SizeService.js +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { get_dir_size, id2path, get_user, invalidate_cached_user_by_id } = require('../../helpers'); -const BaseService = require('../../services/BaseService'); -const { DB_WRITE } = require('../../services/database/consts'); - -// TODO: expose to a utility library -class UserParameter { - static async adapt (value) { - if ( typeof value == 'object' ) return value; - const query_object = typeof value === 'number' - ? { id: value } - : { username: value }; - return await get_user(query_object); - } -} - -class SizeService extends BaseService { - _construct () { - this.usages = {}; - } - - _init () { - this.db = this.services.get('database').get(DB_WRITE, 'filesystem'); - - } - - '__on_boot.consolidate' () { - } - - async get_usage (user_id) { - // if ( this.usages.hasOwnProperty(user_id) ) { - // return this.usages[user_id]; - // } - - const fsentry = await this.db.read( - 'SELECT SUM(size) AS total FROM `fsentries` WHERE `user_id` = ? LIMIT 1', - [user_id], - ); - if ( !fsentry[0] || !fsentry[0].total ) { - this.usages[user_id] = 0; - } else { - this.usages[user_id] = parseInt(fsentry[0].total); - } - - return this.usages[user_id]; - } - - async change_usage (user_id, delta) { - const usage = await this.get_usage(user_id); - this.usages[user_id] = usage + delta; - } - - // TODO: remove fs arg and update all calls - async add_node_size (fs, node, user, factor = 1) { - - let sz; - if ( node.entry.is_dir ) { - if ( node.entry.uuid ) { - sz = await node.fetchSize(); - } else { - // very unlikely, but a warning is better than a throw right now - // TODO: remove this once we're sure this is never hit - this.log.warn('add_node_size: node has no uuid :(', node); - sz = await get_dir_size(await id2path(node.mysql_id), user); - } - } else { - sz = node.entry.size; - } - await this.change_usage(user.id, sz * factor); - } - - /** - * - * @param {*} user_or_id - * @param {*} param1.exclude_transient - set to `true` to exclude - * paid storage, and other temporary storage grants which are - * not persisted in the `user.free_storage` column. - * @returns - */ - async get_storage_capacity (user_or_id, { exclude_transient } = {}) { - const user = await UserParameter.adapt(user_or_id); - if ( ! this.global_config.is_storage_limited ) { - return this.global_config.available_device_storage; - } - - if ( !user.free_storage && user.free_storage !== 0 ) { - return this.global_config.storage_capacity; - } - - return exclude_transient - ? user.actual_free_storage ?? user.free_storage - : user.free_storage; - } - - /** - * Attempt to add storage for a user. - - * In the case of an error, this method will fail silently to the caller and - * produce an alarm for further investigation. - * - * @param {*} user_or_id - user id, username, or user object - * @param {*} amount_in_bytes - amount of bytes to add - * @param {*} reason - please specify a reason for the storage increase - * @param {*} param3 - optional fields to add to the audit log - */ - async add_storage (user_or_id, amount_in_bytes, reason, { field_a, field_b } = {}) { - const user = await UserParameter.adapt(user_or_id); - const capacity = await this.get_storage_capacity(user, { exclude_transient: true }); - - // Audit log - { - const entry = { - user_id: user.id, - user_id_keep: user.id, - amount: amount_in_bytes, - reason, - ...(field_a ? { field_a } : {}), - ...(field_b ? { field_b } : {}), - }; - - const fields_ = Object.keys(entry); - const fields = fields_.join(', '); - const placeholders = fields_.map(_ => '?').join(', '); - const values = fields_.map(f => entry[f]); - - try { - await this.db.write( - `INSERT INTO storage_audit (${fields}) VALUES (${placeholders})`, - values, - ); - } catch (e) { - this.errors.report('size-service.audit-add-storage', { - source: e, - trace: true, - alarm: true, - }); - } - } - - // Storage increase - { - try { - const res = await this.db.write( - 'UPDATE `user` SET `free_storage` = ? WHERE `id` = ? LIMIT 1', - [capacity + amount_in_bytes, user.id], - ); - if ( ! res.anyRowsAffected ) { - throw new Error(`add_storage: failed to update user ${user.id}`); - } - } catch (e) { - this.errors.report('size-service.add-storage', { - source: e, - trace: true, - alarm: true, - }); - } - invalidate_cached_user_by_id(user.id); - } - } -} - -module.exports = { - SizeService, -}; diff --git a/src/backend/src/modules/selfhosted/DefaultUserService.js b/src/backend/src/modules/selfhosted/DefaultUserService.js deleted file mode 100644 index daa79d620..000000000 --- a/src/backend/src/modules/selfhosted/DefaultUserService.js +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { QuickMkdir } = require('../../deprecated/filesystem/hl_operations/hl_mkdir'); -const { HLWrite } = require('../../deprecated/filesystem/hl_operations/hl_write'); -const { NodePathSelector } = require('../../deprecated/filesystem/node/selectors'); -const { get_user, invalidate_cached_user } = require('../../helpers'); -const { Context } = require('../../util/context'); -const { buffer_to_stream } = require('../../util/streamutil'); -const BaseService = require('../../services/BaseService'); -const { Actor, UserActorType } = require('../../services/auth/Actor'); -const { DB_WRITE } = require('../../services/database/consts'); -const { quot } = require('@heyputer/putility').libs.string; -const bcrypt = require('bcrypt'); -const uuidv4 = require('uuid').v4; -const crypto = require('crypto'); - -const USERNAME = 'admin'; - -const DEFAULT_FILES = {}; - -class DefaultUserService extends BaseService { - async _init () { - } - async '__on_ready.webserver' () { - // check if a user named `admin` exists - let user = await get_user({ username: USERNAME, cached: false }); - if ( ! user ) { - user = await this.create_default_user_(); - } else { - await this.#createDefaultUserFiles(Actor.adapt(user)); - } - - // check if user named `admin` is using default password - const tmp_password = await this.get_tmp_password_(user); - const is_default_password = await bcrypt.compare( - tmp_password, - user.password, - ); - if ( ! is_default_password ) return; - - // console.log(`password for admin is: ${tmp_password}`); - // NB: this is needed for the CI to extract the password - console.log(`password for admin is: ${tmp_password}`); - - const realConsole = globalThis.original_console_object ?? console; - realConsole.log('\n************************************************************'); - realConsole.log('* Your default login credentials are:'); - realConsole.log('* Username: admin'); - realConsole.log(`* Password: ${tmp_password}`); - realConsole.log('* (change the password to remove this message)'); - realConsole.log('************************************************************\n'); - } - async create_default_user_ () { - const db = this.services.get('database').get(DB_WRITE, USERNAME); - await db.write( - ` - INSERT INTO user (uuid, username, free_storage) - VALUES (?, ?, ?) - `, - [ - uuidv4(), - USERNAME, - 1024 * 1024 * 1024 * 10, // 10 GB - ], - ); - const svc_group = this.services.get('group'); - await svc_group.add_users({ - uid: 'ca342a5e-b13d-4dee-9048-58b11a57cc55', // admin - users: [USERNAME], - }); - const user = await get_user({ username: USERNAME, cached: false }); - const actor = Actor.adapt(user); - const tmp_password = await this.get_tmp_password_(user); - const password_hashed = await bcrypt.hash(tmp_password, 8); - await db.write( - 'UPDATE user SET password = ? WHERE id = ?', - [ - password_hashed, - user.id, - ], - ); - user.password = password_hashed; - const svc_user = this.services.get('user'); - await svc_user.generate_default_fsentries({ user }); - // generate default files for admin user - - await this.#createDefaultUserFiles(actor); - - invalidate_cached_user(user); - await new Promise(rslv => setTimeout(rslv, 2000)); - return user; - } - - async #recursiveCreateDefaultFilesIfMissing ({ components, tree, actor }) { - const svc_fs = this.services.get('filesystem'); - - const parent = await svc_fs.node(new NodePathSelector(`/${components.join('/')}`)); - for ( const k in tree ) { - - if ( typeof tree[k] === 'string' ) { - try { - const buffer = Buffer.from(tree[k], 'utf-8'); - const hl_write = new HLWrite(); - await hl_write.run({ - destination_or_parent: parent, - specified_name: k, - file: { - size: buffer.length, - stream: buffer_to_stream(buffer), - }, - actor, - }); - } catch (e) { - if ( e.message.includes('already exists.') ) { - // ignore - } else { - // throw if it actually fails to create the files - throw e; - } - } - } else { - try { - const hl_qmkdir = new QuickMkdir(); - await hl_qmkdir.run({ - parent, - path: k, - actor, - }); - } catch (e) { - if ( e.message.includes('already exists.') ) { - // ignore - } else { - // throw if it actually fails to create the files - throw e; - } - } - const components_ = [...components, k]; - await this.#recursiveCreateDefaultFilesIfMissing({ - components: components_, - tree: tree[k], - actor, - }); - } - - } - }; - async #createDefaultUserFiles (actor) { - await this.services.get('su').sudo(actor, async () => { - await this.#recursiveCreateDefaultFilesIfMissing({ - components: ['admin'], - tree: DEFAULT_FILES, - actor, - }); - }); - - } - async get_tmp_password_ (user) { - const actor = await Actor.create(UserActorType, { user }); - return await Context.get().sub({ actor }).arun(async () => { - const svc_driver = this.services.get('driver'); - const driver_response = await svc_driver.call({ - iface: 'puter-kvstore', - method: 'get', - args: { key: 'tmp_password' }, - }); - - if ( driver_response.result ) return driver_response.result; - - const tmp_password = crypto.randomBytes(4).toString('hex'); - await svc_driver.call({ - iface: 'puter-kvstore', - method: 'set', - args: { - key: 'tmp_password', - value: tmp_password, - }, - }); - return tmp_password; - }); - } - async force_tmp_password_ (user) { - const db = this.services.get('database') - .get(DB_WRITE, 'terminal-password-reset'); - const actor = await Actor.create(UserActorType, { user }); - return await Context.get().sub({ actor }).arun(async () => { - const svc_driver = this.services.get('driver'); - const tmp_password = crypto.randomBytes(4).toString('hex'); - const password_hashed = await bcrypt.hash(tmp_password, 8); - await svc_driver.call({ - iface: 'puter-kvstore', - method: 'set', - args: { - key: 'tmp_password', - value: tmp_password, - }, - }); - await db.write( - 'UPDATE user SET password = ? WHERE id = ?', - [ - password_hashed, - user.id, - ], - ); - return tmp_password; - }); - } -} - -module.exports = DefaultUserService; diff --git a/src/backend/src/modules/selfhosted/DevWatcherService.js b/src/backend/src/modules/selfhosted/DevWatcherService.js deleted file mode 100644 index 193c76cbd..000000000 --- a/src/backend/src/modules/selfhosted/DevWatcherService.js +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { webpack, web } = require('webpack'); -const BaseService = require('../../services/BaseService'); - -const path_ = require('node:path'); -const fs = require('node:fs'); -const url = require('node:url'); - -class ProxyLogger { - constructor (log) { - this.log = log; - } - attach (stream) { - let buffer = ''; - stream.on('data', (chunk) => { - buffer += chunk.toString(); - let lineEndIndex = buffer.indexOf('\n'); - while ( lineEndIndex !== -1 ) { - const line = buffer.substring(0, lineEndIndex); - this.log(line); - buffer = buffer.substring(lineEndIndex + 1); - lineEndIndex = buffer.indexOf('\n'); - } - }); - - stream.on('end', () => { - if ( buffer.length ) { - this.log(buffer); - } - }); - } -} - -/** - * @description - * This service is used to run webpack watchers. - */ -class DevWatcherService extends BaseService { - static MODULES = { - path: require('path'), - spawn: require('child_process').spawn, - }; - - async _init (args) { - this.args = args; - } - - // Oh geez we need to wait for the web server to initialize - // so that `config.origin` has the actual port in it if the - // port is set to `auto` - you have no idea how confusing - // this was to debug the first time, like Ahhhhhh!! - // but hey at least we have this convenient event listener. - async '__on_ready.webserver' () { - const svc_process = this.services.get('process'); - - let { root, commands, webpack } = this.args; - if ( ! webpack ) webpack = []; - - let promises = []; - for ( const entry of commands ) { - const { directory } = entry; - const fullpath = this.modules.path.join(root, directory); - // promises.push(this.start_({ ...entry, fullpath })); - promises.push(svc_process.start({ ...entry, fullpath })); - } - for ( const entry of webpack ) { - const p = this.start_a_webpack_watcher_(entry); - promises.push(p); - } - await Promise.all(promises); - - // It's difficult to tell when webpack is "done" its first - // run so we just wait a bit before we say we're ready. - await new Promise((resolve) => setTimeout(resolve, 5000)); - } - - async get_configjs ({ directory, configIsFor, possibleConfigNames }) { - let configjsPath, moduleType; - - for ( const [configName, supposedModuleType] of possibleConfigNames ) { - // There isn't really an async fs.exists() funciton. I assume this - // is because 'exists' is already a very fast operation. - const supposedPath = path_.join(this.args.root, directory, configName); - if ( fs.existsSync(supposedPath) ) { - configjsPath = supposedPath; - moduleType = supposedModuleType; - break; - } - } - - if ( ! configjsPath ) { - throw new Error(`could not find ${configIsFor} config for: ${directory}`); - } - - // If the webpack config ends with .js it could be an ES6 module or a - // CJS module, so the absolute safest thing to do so as not to completely - // break in specific patch version of supported versions of node.js is - // to read the package.json and see what it says is the import mechanism. - if ( moduleType === 'package.json' ) { - const packageJSONPath = path_.join(this.args.root, directory, 'package.json'); - const packageJSONObject = JSON.parse(fs.readFileSync(packageJSONPath)); - moduleType = packageJSONObject?.type ?? 'module'; - } - - return { - configjsPath, - moduleType, - }; - } - - async start_a_webpack_watcher_ (entry) { - const possibleConfigNames = [ - ['webpack.config.js', 'package.json'], - ['webpack.config.cjs', 'commonjs'], - ['webpack.config.mjs', 'module'], - ]; - - let { - configjsPath: webpackConfigPath, - moduleType, - } = await this.get_configjs({ - directory: entry.directory, - configIsFor: 'webpack', // for error message - possibleConfigNames, - }); - - let oldEnv; - - if ( entry.env ) { - oldEnv = process.env; - const newEnv = Object.create(process.env); - let global_config = null; - try { - const svc_config = this.services.get('config'); - global_config = svc_config ? svc_config.get('global_config') : null; - } catch (e) { - // Config service not available yet, will use null - } - - for ( const k in entry.env ) { - const envValue = entry.env[k]; - // If it's a function, call it with the config, otherwise use the value directly - if ( typeof envValue === 'function' ) { - try { - const result = envValue({ global_config: global_config }); - // Only set the env var if we got a non-empty result - // This allows the webpack config to use its fallback values - if ( result ) { - newEnv[k] = result; - } - } catch (e) { - // If config is not available yet, don't set the env var - // This allows the webpack config to use its fallback values from config files - // Only log if it's not a null/undefined access error (which is expected) - if ( !e.message.includes('Cannot read properties of null') && - !e.message.includes('Cannot read properties of undefined') ) { - this.log.warn(`Could not evaluate env function for ${k}: ${e.message}`); - } - } - } else { - newEnv[k] = envValue; - } - } - process.env = newEnv; // Yep, it totally lets us do this - } - - if ( moduleType === 'module' && process.platform === 'win32' ) { - webpackConfigPath = url.pathToFileURL(webpackConfigPath).href; - } - - let webpackConfig = moduleType === 'module' - ? (await import(webpackConfigPath)).default - : require(webpackConfigPath); - - // The webpack config can sometimes be a function - if ( typeof webpackConfig === 'function' ) { - webpackConfig = await webpackConfig(); - } - - if ( oldEnv ) process.env = oldEnv; - - webpackConfig.context = webpackConfig.context - ? path_.resolve(path_.join(this.args.root, entry.directory), webpackConfig.context) - : path_.join(this.args.root, entry.directory); - - if ( entry.onConfig ) entry.onConfig(webpackConfig); - - const webpacker = webpack(webpackConfig); - - let errorAfterLastEnd = false; - let firstEvent = true; - webpacker.watch({}, (err, stats) => { - let hideSuccess = false; - if ( firstEvent ) { - firstEvent = false; - hideSuccess = true; - } - if ( err || stats.hasErrors() ) { - // Extract error information without serializing the entire stats object - const errorInfo = { - err: err ? err.message : null, - errors: stats.compilation?.errors?.map(e => e.message) || [], - warnings: stats.compilation?.warnings?.map(w => w.message) || [], - }; - this.log.error(`error information: ${entry.directory} using Webpack`, errorInfo); - this.log.error(`❌ failed to update ${entry.directory} using Webpack`); - } else { - // Normally success messages aren't important, but sometimes it takes - // a little bit for the bundle to update so a developer probably would - // like to have a visual indication in the console when it happens. - if ( ! hideSuccess ) { - this.log.info(`✅ updated ${entry.directory} using Webpack`); - } - } - }); - } -}; - -module.exports = DevWatcherService; diff --git a/src/backend/src/modules/selfhosted/SelfHostedModule.js b/src/backend/src/modules/selfhosted/SelfHostedModule.js deleted file mode 100644 index 9317858b7..000000000 --- a/src/backend/src/modules/selfhosted/SelfHostedModule.js +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('@heyputer/putility'); -const config = require('../../config'); - -class SelfHostedModule extends AdvancedBase { - async install (context) { - const services = context.get('services'); - - const { SelfhostedService } = require('./SelfhostedService'); - services.registerService('__selfhosted', SelfhostedService); - - const DefaultUserService = require('./DefaultUserService'); - services.registerService('__default-user', DefaultUserService); - - const DevWatcherService = require('./DevWatcherService'); - const path_ = require('path'); - // TODO: sucks - const RELATIVE_PATH = '../../../../../'; - - if ( ! config.no_devwatch ) - { - services.registerService('__dev-watcher', DevWatcherService, { - root: path_.resolve(__dirname, RELATIVE_PATH), - webpack: [ - { - name: 'puter.js', - directory: 'src/puter-js', - onConfig: config => { - config.output.filename = 'puter.dev.js'; - config.devtool = 'source-map'; - }, - env: { - PUTER_ORIGIN: ({ global_config: config }) => config?.origin || '', - PUTER_API_ORIGIN: ({ global_config: config }) => config?.api_base_url || '', - }, - }, - { - name: 'gui', - directory: 'src/gui', - }, - ], - commands: [ - ], - }); - } - - const { ServeStaticFilesService } = require('./ServeStaticFilesService'); - services.registerService('__serve-puterjs', ServeStaticFilesService, { - directories: [ - { - prefix: '/sdk', - path: path_.resolve(__dirname, RELATIVE_PATH, 'src/puter-js/dist'), - }, - { - prefix: '/builtin/git', - path: path_.resolve(__dirname, RELATIVE_PATH, 'src/git/dist'), - }, - { - prefix: '/builtin/dev-center', - path: path_.resolve(__dirname, RELATIVE_PATH, 'src/dev-center'), - }, - { - prefix: '/builtin/dev-center', - path: path_.resolve(__dirname, RELATIVE_PATH, 'src/dev-center'), - }, - { - prefix: '/vendor/v86/bios', - path: path_.resolve(__dirname, RELATIVE_PATH, 'submodules/v86/bios'), - }, - { - prefix: '/vendor/v86', - path: path_.resolve(__dirname, RELATIVE_PATH, 'submodules/v86/build'), - }, - ], - }); - - const { ServeSingleFileService } = require('./ServeSingeFileService'); - services.registerService('__serve-puterjs-new', ServeSingleFileService, { - path: path_.resolve( - __dirname, - RELATIVE_PATH, - 'src/puter-js/dist/puter.dev.js', - ), - route: '/puter.js/v2', - }); - services.registerService('__serve-putilityjs-new', ServeSingleFileService, { - path: path_.resolve( - __dirname, - RELATIVE_PATH, - 'src/putility/dist/putility.dev.js', - ), - route: '/putility.js/v1', - }); - services.registerService('__serve-gui-js', ServeSingleFileService, { - path: path_.resolve( - __dirname, - RELATIVE_PATH, - 'src/gui/dist/gui.dev.js', - ), - route: '/putility.js/v1', - }); - } -} - -module.exports = SelfHostedModule; diff --git a/src/backend/src/modules/selfhosted/SelfhostedService.js b/src/backend/src/modules/selfhosted/SelfhostedService.js deleted file mode 100644 index fbdde8752..000000000 --- a/src/backend/src/modules/selfhosted/SelfhostedService.js +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const BaseService = require('../../services/BaseService'); -const { DB_WRITE } = require('../../services/database/consts'); - -class SelfhostedService extends BaseService { - static description = ` - Registers drivers for self-hosted Puter instances. - `; - - async _init () { - } -} - -module.exports = { SelfhostedService }; diff --git a/src/backend/src/modules/selfhosted/ServeSingeFileService.js b/src/backend/src/modules/selfhosted/ServeSingeFileService.js deleted file mode 100644 index 9723c0a04..000000000 --- a/src/backend/src/modules/selfhosted/ServeSingeFileService.js +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const BaseService = require('../../services/BaseService'); - -class ServeSingleFileService extends BaseService { - async _init (args) { - this.route = args.route; - this.path = args.path; - } - async '__on_install.routes' () { - const { app } = this.services.get('web-server'); - - app.get(this.route, (req, res) => { - return res.sendFile(this.path); - }); - } -} - -module.exports = { - ServeSingleFileService, -}; diff --git a/src/backend/src/modules/selfhosted/ServeStaticFilesService.js b/src/backend/src/modules/selfhosted/ServeStaticFilesService.js deleted file mode 100644 index 8408bf957..000000000 --- a/src/backend/src/modules/selfhosted/ServeStaticFilesService.js +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const BaseService = require('../../services/BaseService'); - -class ServeStaticFilesService extends BaseService { - async _init (args) { - this.directories = args.directories; - } - - async '__on_install.routes' () { - const { app } = this.services.get('web-server'); - - for ( const { prefix, path } of this.directories ) { - app.use(prefix, require('express').static(path)); - } - } -} - -module.exports = { ServeStaticFilesService }; diff --git a/src/backend/src/modules/template/README.md b/src/backend/src/modules/template/README.md deleted file mode 100644 index e272a18fe..000000000 --- a/src/backend/src/modules/template/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# TemplateModule - -This is a template module that you can copy and paste to create new modules. - -This module is also included in `EssentialModules`, which means it will load -when Puter boots. If you're just testing something, you can add it here -temporarily. - -## Services - -### TemplateService - -This is a template service that you can copy and paste to create new services. -You can also add to this service temporarily to test something. - -#### Listeners - -##### `install.routes` - -TemplateService listens to this event to provide an example endpoint - -##### `boot.consolidation` - -TemplateService listens to this event to provide an example event - -##### `boot.activation` - -TemplateService listens to this event to show you that it's here - -##### `start.webserver` - -TemplateService listens to this event to show you that it's here - -## Libraries - -### hello_world - -#### Functions - -##### `hello_world` - -This is a simple function that returns a string. -You can probably guess what string it returns. - -## Notes - -### Outside Imports - -This module has external relative imports. When these are -removed it may become possible to move this module to an -extension. - -**Imports:** -- `../../util/context.js` -- `../../services/BaseService` (use.BaseService) -- `../../util/expressutil` diff --git a/src/backend/src/modules/template/TemplateModule.js b/src/backend/src/modules/template/TemplateModule.js deleted file mode 100644 index 971cafddd..000000000 --- a/src/backend/src/modules/template/TemplateModule.js +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { AdvancedBase } = require('@heyputer/putility'); - -/** - * This is a template module that you can copy and paste to create new modules. - * - * This module is also included in `EssentialModules`, which means it will load - * when Puter boots. If you're just testing something, you can add it here - * temporarily. - */ -class TemplateModule extends AdvancedBase { - async install (context) { - // === LIBS === // - const useapi = context.get('useapi'); - - const lib = require('./lib/__lib__.js'); - - // In extensions: use('workinprogress').hello_world(); - // In services classes: see TemplateService.js - useapi.def('workinprogress', lib, { assign: true }); - - useapi.def('core.context', require('../../util/context.js').Context); - - // === SERVICES === // - const services = context.get('services'); - - const { TemplateService } = require('./TemplateService.js'); - services.registerService('template-service', TemplateService); - } - -} - -module.exports = { - TemplateModule, -}; diff --git a/src/backend/src/modules/template/TemplateService.js b/src/backend/src/modules/template/TemplateService.js deleted file mode 100644 index cc51b86d9..000000000 --- a/src/backend/src/modules/template/TemplateService.js +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -// TODO: import via `USE` static member -const BaseService = require('../../services/BaseService'); -const eggspress = require('../../api/eggspress'); - -/** - * This is a template service that you can copy and paste to create new services. - * You can also add to this service temporarily to test something. - */ -class TemplateService extends BaseService { - static USE = { - // - Defined by lib/__lib__.js, - // - Exposed to `useapi` by TemplateModule.js - workinprogress: 'workinprogress', - }; - - _construct () { - // Use this override to initialize instance variables. - } - - async _init () { - // This is where you initialize the service and prepare - // for the consolidation phase. - this.log.info('I am the template service.'); - } - - /** - * TemplateService listens to this event to provide an example endpoint - */ - '__on_install.routes' (_, { app }) { - this.log.info('TemplateService get the event for installing endpoint.'); - app.use(eggspress('/example-endpoint', { - allowedMethods: ['GET'], - }, async (req, res) => { - res.send(this.workinprogress.hello_world()); - })); - } - - /** - * TemplateService listens to this event to provide an example event - */ - '__on_boot.consolidation' () { - // At this stage, all services have been initialized and it is - // safe to start emitting events. - this.log.info('TemplateService sees consolidation boot phase.'); - - const svc_event = this.services.get('event'); - - svc_event.on('template-service.hello', (_eventid, event_data) => { - this.log.info('template-service said hello to itself; this is expected', { - event_data, - }); - }); - - svc_event.emit('template-service.hello', { - message: 'Hello all you other services! I am the template service.', - }); - } - /** - * TemplateService listens to this event to show you that it's here - */ - '__on_boot.activation' () { - this.log.info('TemplateService sees activation boot phase.'); - } - - /** - * TemplateService listens to this event to show you that it's here - */ - '__on_start.webserver' () { - this.log.info("TemplateService sees it's time to start web servers."); - } -} - -module.exports = { - TemplateService, -}; diff --git a/src/backend/src/modules/template/lib/__lib__.js b/src/backend/src/modules/template/lib/__lib__.js deleted file mode 100644 index f37ca72d5..000000000 --- a/src/backend/src/modules/template/lib/__lib__.js +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -module.exports = { - hello_world: require('./hello_world.js'), -}; diff --git a/src/backend/src/modules/template/lib/hello_world.js b/src/backend/src/modules/template/lib/hello_world.js deleted file mode 100644 index 4842c0830..000000000 --- a/src/backend/src/modules/template/lib/hello_world.js +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -/** - * This is a simple function that returns a string. - * You can probably guess what string it returns. - */ -const hello_world = () => { - return 'Hello, world!'; -}; - -module.exports = hello_world; diff --git a/src/backend/src/modules/test-config/TestConfigModule.js b/src/backend/src/modules/test-config/TestConfigModule.js deleted file mode 100644 index 700000727..000000000 --- a/src/backend/src/modules/test-config/TestConfigModule.js +++ /dev/null @@ -1,15 +0,0 @@ -const { AdvancedBase } = require('@heyputer/putility'); - -class TestConfigModule extends AdvancedBase { - async install (context) { - const services = context.get('services'); - const TestConfigUpdateService = require('./TestConfigUpdateService'); - services.registerService('__test-config-update', TestConfigUpdateService); - const TestConfigReadService = require('./TestConfigReadService'); - services.registerService('__test-config-read', TestConfigReadService); - } -} - -module.exports = { - TestConfigModule, -}; diff --git a/src/backend/src/modules/test-config/TestConfigReadService.js b/src/backend/src/modules/test-config/TestConfigReadService.js deleted file mode 100644 index b5c489354..000000000 --- a/src/backend/src/modules/test-config/TestConfigReadService.js +++ /dev/null @@ -1,10 +0,0 @@ -const BaseService = require('../../services/BaseService'); - -class TestConfigReadService extends BaseService { - async _init () { - this.log.debug(`test config value (should be abcdefg) is: ${ - this.global_config.testConfigValue}`); - } -} - -module.exports = TestConfigReadService; diff --git a/src/backend/src/modules/test-config/TestConfigUpdateService.js b/src/backend/src/modules/test-config/TestConfigUpdateService.js deleted file mode 100644 index 566131cfb..000000000 --- a/src/backend/src/modules/test-config/TestConfigUpdateService.js +++ /dev/null @@ -1,12 +0,0 @@ -const BaseService = require('../../services/BaseService'); - -class TestConfigUpdateService extends BaseService { - async _run_as_early_as_possible () { - const config = this.global_config; - config.__set_config_object__({ - testConfigValue: 'abcdefg', - }); - } -} - -module.exports = TestConfigUpdateService; diff --git a/src/backend/src/modules/test-core/TestCoreModule.js b/src/backend/src/modules/test-core/TestCoreModule.js deleted file mode 100644 index 18bf6902f..000000000 --- a/src/backend/src/modules/test-core/TestCoreModule.js +++ /dev/null @@ -1,50 +0,0 @@ -import { DDBClientWrapper } from '../../clients/dynamodb/DDBClientWrapper.js'; -import { FilesystemService } from '../../deprecated/filesystem/FilesystemService.js'; -import { AuthService } from '../../services/auth/AuthService.js'; -import { GroupService } from '../../services/auth/GroupService.js'; -import { PermissionService } from '../../services/auth/PermissionService.js'; -import { TokenService } from '../../services/auth/TokenService.js'; -import { SqliteDatabaseAccessService } from '../../services/database/SqliteDatabaseAccessService.js'; -import { DetailProviderService } from '../../services/DetailProviderService.js'; -import { DynamoKVStoreWrapper } from '../../services/DynamoKVStore/DynamoKVStoreWrapper.js'; -import { EventService } from '../../services/EventService.js'; -import { FeatureFlagService } from '../../services/FeatureFlagService.js'; -import { GetUserService } from '../../services/GetUserService.js'; -import { MeteringServiceWrapper } from '../../services/MeteringService/MeteringServiceWrapper.mjs'; -import { NotificationService } from '../../services/NotificationService'; -import { RegistrantService } from '../../services/RegistrantService'; -import { RegistryService } from '../../services/RegistryService'; -import { ScriptService } from '../../services/ScriptService'; -import { SessionService } from '../../services/SessionService'; -import { SUService } from '../../services/SUService'; -import { SystemValidationService } from '../../services/SystemValidationService'; -import { AlarmService } from '../core/AlarmService'; -import APIErrorService from '../web/APIErrorService'; - -export class TestCoreModule { - async install (context) { - const services = context.get('services'); - services.registerService('dynamo', DDBClientWrapper); - services.registerService('whoami', DetailProviderService); - services.registerService('get-user', GetUserService); - services.registerService('database', SqliteDatabaseAccessService); - services.registerService('su', SUService); - services.registerService('alarm', AlarmService); - services.registerService('event', EventService); - services.registerService('meteringService', MeteringServiceWrapper); - services.registerService('puter-kvstore', DynamoKVStoreWrapper); - services.registerService('permission', PermissionService); - services.registerService('group', GroupService); - services.registerService('api-error', APIErrorService); - services.registerService('system-validation', SystemValidationService); - services.registerService('registry', RegistryService); - services.registerService('__registrant', RegistrantService); - services.registerService('feature-flag', FeatureFlagService); - services.registerService('token', TokenService); - services.registerService('auth', AuthService); - services.registerService('session', SessionService); - services.registerService('notification', NotificationService); - services.registerService('script', ScriptService); - services.registerService('filesystem', FilesystemService); - } -} diff --git a/src/backend/src/modules/test-drivers/TestAssetHostService.js b/src/backend/src/modules/test-drivers/TestAssetHostService.js deleted file mode 100644 index f367db232..000000000 --- a/src/backend/src/modules/test-drivers/TestAssetHostService.js +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const BaseService = require('../../services/BaseService'); - -class TestAssetHostService extends BaseService { - async '__on_install.routes' () { - const { app } = this.services.get('web-server'); - const path_ = require('node:path'); - - app.use('/test-assets', require('express').static( - path_.join(__dirname, 'assets'))); - } -} - -module.exports = { - TestAssetHostService, -}; diff --git a/src/backend/src/modules/test-drivers/TestDriversModule.js b/src/backend/src/modules/test-drivers/TestDriversModule.js deleted file mode 100644 index 86d56e403..000000000 --- a/src/backend/src/modules/test-drivers/TestDriversModule.js +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { AdvancedBase } = require('@heyputer/putility'); - -class TestDriversModule extends AdvancedBase { - async install (context) { - const services = context.get('services'); - - const { TestAssetHostService } = require('./TestAssetHostService'); - services.registerService('__test-assets', TestAssetHostService); - - const { TestImageService } = require('./TestImageService'); - services.registerService('test-image', TestImageService); - } -} - -module.exports = { - TestDriversModule, -}; diff --git a/src/backend/src/modules/test-drivers/TestImageService.js b/src/backend/src/modules/test-drivers/TestImageService.js deleted file mode 100644 index 36034207c..000000000 --- a/src/backend/src/modules/test-drivers/TestImageService.js +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const config = require('../../config'); -const BaseService = require('../../services/BaseService'); -const { TypedValue } = require('../../services/drivers/meta/Runtime'); -const { buffer_to_stream } = require('../../util/streamutil'); - -const PUBLIC_DOMAIN_IMAGES = [ - { - name: 'starry-night', - url: 'https://upload.wikimedia.org/wikipedia/commons/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg', - file: 'starry.jpg', - }, -]; - -class TestImageService extends BaseService { - async '__on_driver.register.interfaces' () { - const svc_registry = this.services.get('registry'); - const col_interfaces = svc_registry.get('interfaces'); - - col_interfaces.set('test-image', { - methods: { - echo_image: { - parameters: { - source: { - type: 'file', - }, - }, - result: { - type: { - $: 'stream', - content_type: 'image', - }, - }, - }, - get_image: { - parameters: { - source_type: { - type: 'string', - }, - }, - result: { - type: { - $: 'stream', - content_type: 'image', - }, - }, - }, - }, - }); - } - - static IMPLEMENTS = { - 'version': { - get_version () { - return 'v1.0.0'; - }, - }, - 'test-image': { - async echo_image ({ - source, - }) { - const stream = await source.get('stream'); - return new TypedValue({ - $: 'stream', - content_type: 'image/jpeg', - }, stream); - }, - async get_image ({ - source_type, - }) { - const image = PUBLIC_DOMAIN_IMAGES[0]; - if ( source_type === 'string:url:web' ) { - return new TypedValue({ - $: 'string:url:web', - content_type: 'image', - }, `${config.origin}/test-assets/${image.file}`); - } - throw new Error('not implemented yet'); - }, - }, - }; -} - -module.exports = { - TestImageService, -}; diff --git a/src/backend/src/modules/test-drivers/assets/starry.jpg b/src/backend/src/modules/test-drivers/assets/starry.jpg deleted file mode 100644 index 9a24b899d..000000000 Binary files a/src/backend/src/modules/test-drivers/assets/starry.jpg and /dev/null differ diff --git a/src/backend/src/modules/test-drivers/assets/wave.jpg b/src/backend/src/modules/test-drivers/assets/wave.jpg deleted file mode 100644 index cf6a3940d..000000000 Binary files a/src/backend/src/modules/test-drivers/assets/wave.jpg and /dev/null differ diff --git a/src/backend/src/modules/test-drivers/doc/requests.md b/src/backend/src/modules/test-drivers/doc/requests.md deleted file mode 100644 index 39275fb9b..000000000 --- a/src/backend/src/modules/test-drivers/doc/requests.md +++ /dev/null @@ -1,98 +0,0 @@ -```javascript -blob = await (await fetch("http://api.puter.localhost:4100/drivers/call", { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - interface: 'test-image', - method: 'get_image', - args: { - source_type: 'string:url:web' - } - }), - "method": "POST", -})).blob(); -dataurl = await new Promise((y, n) => { - a = new FileReader(); - a.onload = _ => y(a.result); - a.onerror = _ => n(a.error); - a.readAsDataURL(blob) -}); -URL.createObjectURL(await (await fetch("http://api.puter.localhost:4100/drivers/call", { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - interface: 'test-image', - method: 'echo_image', - args: { - source: dataurl, - } - }), - "method": "POST", -})).blob()); -``` - -```javascript -await(async () => { - - blob = await (await fetch("http://api.puter.localhost:4100/drivers/call", { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - interface: 'test-image', - method: 'get_image', - args: { - source_type: 'string:url:web' - } - }), - "method": "POST", - })).blob(); - - const endpoint = 'http://api.puter.localhost:4100/drivers/call'; - - const body = { - object: { - interface: 'test-image', - method: 'echo_image', - ['args.source']: { - $: 'file', - size: blob.size, - type: blob.type, - }, - }, - file: [ - blob, - ] - }; - - const formData = new FormData(); - for ( const k in body ) { - console.log('k', k); - const append = v => { - if ( v instanceof Blob ) { - formData.append(k, v, 'filename'); - } else { - formData.append(k, JSON.stringify(v)); - } - }; - if ( Array.isArray(body[k]) ) { - for ( const v of body[k] ) append(v); - } else { - append(body[k]); - } - } - const response = await fetch(endpoint, { - method: 'POST', - headers: { 'Authorization': `Bearer ${puter.authToken}` }, - body: formData - }); - const echo_blob = await response.blob(); - const echo_url = URL.createObjectURL(echo_blob); - return echo_url; -})(); -``` \ No newline at end of file diff --git a/src/backend/src/modules/web/APIErrorService.js b/src/backend/src/modules/web/APIErrorService.js deleted file mode 100644 index ecdee4cbf..000000000 --- a/src/backend/src/modules/web/APIErrorService.js +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const APIError = require('../../api/APIError'); -const BaseService = require('../../services/BaseService'); - -/** - * @typedef {Object} ErrorSpec - * @property {string} code - The error code - * @property {string} status - HTTP status code - * @property {function} message - A function that generates an error message - */ - -/** - * The APIErrorService class provides a mechanism for registering and managing - * error codes and messages which may be sent to clients. - * - * This allows for a single source-of-truth for error codes and messages that - * are used by multiple services. - */ -class APIErrorService extends BaseService { - _construct () { - this.codes = { - ...this.constructor.codes, - }; - } - - // Hardcoded error codes from before this service was created - static codes = APIError.codes; - - /** - * Registers API error codes. - * - * @param {Object.} codes - A map of error codes to error specifications - */ - register (codes) { - for ( const code in codes ) { - this.codes[code] = codes[code]; - } - } - - create (code, fields) { - const error_spec = this.codes[code]; - if ( ! error_spec ) { - return new APIError(500, 'Missing error message.', null, { - code, - }); - } - - return new APIError(error_spec.status, error_spec.message, null, { - ...fields, - code, - }); - } -} - -module.exports = APIErrorService; diff --git a/src/backend/src/modules/web/README.md b/src/backend/src/modules/web/README.md deleted file mode 100644 index a10f5b06f..000000000 --- a/src/backend/src/modules/web/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# WebModule - -This module initializes a pre-configured web server and socket.io server. -The main service, WebServerService, emits 'install.routes' and provides -the server instance to the callback. - -## Services - -### SocketioService - -SocketioService provides a service for sending messages to clients. -socket.io is used behind the scenes. This service provides a simpler -interface for sending messages to rooms or socket ids. - -#### Listeners - -##### `install.socketio` - -Initializes socket.io - -###### Parameters - -- **server:** The server to attach socket.io to. - -### WebServerService - -This class, WebServerService, is responsible for starting and managing the Puter web server. -It initializes the Express app, sets up middlewares, routes, and handles authentication and web sockets. -It also validates the host header and IP addresses to prevent security vulnerabilities. - -#### Listeners - -##### `boot.consolidation` - -This method initializes the backend web server for Puter. It sets up the Express app, configures middleware, and starts the HTTP server. - -##### `boot.activation` - -Starts the web server and listens for incoming connections. -This method sets up the Express app, sets up middleware, and starts the server on the specified port. -It also sets up the Socket.io server for real-time communication. - -##### `start.webserver` - -This method starts the web server by listening on the specified port. It tries multiple ports if the first one is in use. -If the `config.http_port` is set to 'auto', it will try to find an available port in a range of 4100 to 4299. -Once the server is up and running, it emits the 'start.webserver' and 'ready.webserver' events. -If the `config.env` is set to 'dev' and `config.no_browser_launch` is false, it will open the Puter URL in the default browser. - -## Notes - -### Outside Imports - -This module has external relative imports. When these are -removed it may become possible to move this module to an -extension. - -**Imports:** -- `../../services/BaseService` (use.BaseService) -- `../../util/context.js` -- `../../services/BaseService.js` -- `../../config.js` -- `../../middleware/auth.js` -- `../../util/strutil.js` -- `../../helpers.js` diff --git a/src/backend/src/modules/web/SocketioService.js b/src/backend/src/modules/web/SocketioService.js deleted file mode 100644 index 667f995af..000000000 --- a/src/backend/src/modules/web/SocketioService.js +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const BaseService = require('../../services/BaseService'); -const socketio = require('socket.io'); -const { createAdapter } = require('@socket.io/redis-streams-adapter'); -const { redisClient } = require('../../clients/redis/redisSingleton'); - -/** - * SocketioService provides a service for sending messages to clients. - * socket.io is used behind the scenes. This service provides a simpler - * interface for sending messages to rooms or socket ids. - */ -class SocketioService extends BaseService { - /** - * Initializes socket.io - * - * @evtparam server The server to attach socket.io to. - */ - '__on_install.socketio' (_, { server }) { - /** - * @type {import('socket.io').Server} - */ - const socketioOptions = { - cors: { - origin: (origin, callback) => { - callback(null, origin); - }, - credentials: true, - }, - adapter: createAdapter(redisClient), - }; - this.io = socketio(server, socketioOptions); - } - - /** - * Sends a message to specified socket(s) or room(s) - * - * @param {Array|Object} socket_specifiers - Single or array of objects specifying target sockets/rooms - * @param {string} key - The event key/name to emit - * @param {*} data - The data payload to send - * @returns {Promise} - */ - async send (socket_specifiers, key, data) { - if ( ! Array.isArray(socket_specifiers) ) { - socket_specifiers = [socket_specifiers]; - } - - for ( const socket_specifier of socket_specifiers ) { - if ( socket_specifier.room ) { - this.io.to(socket_specifier.room).emit(key, data); - } else if ( socket_specifier.socket ) { - this.io.to(socket_specifier.socket).emit(key, data); - } - } - } - - /** - * Checks if the specified socket or room exists - * - * @param {Object} socket_specifier - The socket specifier object - * @returns {boolean} True if the socket exists, false otherwise - */ - has (socket_specifier) { - if ( socket_specifier.room ) { - const room = this.io?.sockets.adapter.rooms.get(socket_specifier.room); - return (!!room) && room.size > 0; - } - if ( socket_specifier.socket ) { - return this.io?.sockets.sockets.has(socket_specifier.socket); - } - } -} - -module.exports = SocketioService; diff --git a/src/backend/src/modules/web/WebModule.js b/src/backend/src/modules/web/WebModule.js deleted file mode 100644 index 56d3f388f..000000000 --- a/src/backend/src/modules/web/WebModule.js +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { AdvancedBase } = require('@heyputer/putility'); -const { RuntimeModule } = require('../../extension/RuntimeModule.js'); - -/** - * This module initializes a pre-configured web server and socket.io server. - * The main service, WebServerService, emits 'install.routes' and provides - * the server instance to the callback. - */ -class WebModule extends AdvancedBase { - async install (context) { - // === LIBS === // - const useapi = context.get('useapi'); - useapi.def('web', require('./lib/__lib__.js'), { assign: true }); - - // Prevent extensions from loading incompatible versions of express - useapi.def('web.express', require('express')); - - // Extension compatibility - const runtimeModule = new RuntimeModule({ name: 'web' }); - context.get('runtime-modules').register(runtimeModule); - runtimeModule.exports = useapi.use('web'); - - // === SERVICES === // - const services = context.get('services'); - - const SocketioService = require('./SocketioService'); - services.registerService('socketio', SocketioService); - - const WebServerService = require('./WebServerService'); - services.registerService('web-server', WebServerService); - - const APIErrorService = require('./APIErrorService'); - services.registerService('api-error', APIErrorService); - } -} - -module.exports = { - WebModule, -}; diff --git a/src/backend/src/modules/web/WebServerService.d.ts b/src/backend/src/modules/web/WebServerService.d.ts deleted file mode 100644 index 77a4c10dc..000000000 --- a/src/backend/src/modules/web/WebServerService.d.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Server } from 'http'; -import BaseService from '../../services/BaseService'; - -/** - * WebServerService is responsible for starting and managing the Puter web server. - */ -export class WebServerService extends BaseService { - /** - * Allow requests with undefined Origin header for a specific route. - * @param route The route (string or RegExp) to allow. - */ - allow_undefined_origin (route: string | RegExp): void; - - /** - * Returns the underlying HTTP server instance. - */ - get_server (): Server; -} - -export = WebServerService; \ No newline at end of file diff --git a/src/backend/src/modules/web/WebServerService.js b/src/backend/src/modules/web/WebServerService.js deleted file mode 100644 index be74ef6f7..000000000 --- a/src/backend/src/modules/web/WebServerService.js +++ /dev/null @@ -1,859 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const express = require('express'); -const eggspress = require('./lib/eggspress.js'); -const { Context, ContextExpressMiddleware } = require('../../util/context.js'); -const BaseService = require('../../services/BaseService.js'); - -const config = require('../../config.js'); -var http = require('http'); -const auth = require('../../middleware/auth.js'); -const measure = require('../../middleware/measure.js'); -const yargs = require('yargs/yargs'); -const { hideBin } = require('yargs/helpers'); - -const relative_require = require; - -const normalizeHostDomain = (domain) => { - if ( typeof domain !== 'string' ) return null; - const normalizedDomain = domain.trim().toLowerCase().replace(/^\./, ''); - if ( ! normalizedDomain ) return null; - return normalizedDomain.split(':')[0]; -}; - -const hostMatchesDomain = (hostname, domain) => { - const normalizedHost = normalizeHostDomain(hostname); - const normalizedDomain = normalizeHostDomain(domain); - if ( !normalizedHost || !normalizedDomain ) return false; - return normalizedHost === normalizedDomain || - normalizedHost.endsWith(`.${normalizedDomain}`); -}; - -/** -* This class, WebServerService, is responsible for starting and managing the Puter web server. -* It initializes the Express app, sets up middlewares, routes, and handles authentication and web sockets. -* It also validates the host header and IP addresses to prevent security vulnerabilities. -*/ -class WebServerService extends BaseService { - static CONCERN = 'web'; - - static MODULES = { - https: require('https'), - http: require('http'), - fs: require('fs'), - express: require('express'), - helmet: require('helmet'), - cookieParser: require('cookie-parser'), - compression: require('compression'), - 'on-finished': require('on-finished'), - morgan: require('morgan'), - }; - - allowedRoutesWithUndefinedOrigins = []; - isDraining = false; - shutdownStarted = false; - shutdownForceExitTimer = null; - shutdownCloseTimer = null; - gracefulShutdownHandlersInstalled = false; - - allow_undefined_origin (route) { - this.allowedRoutesWithUndefinedOrigins.push(route); - } - - /** - * This method initializes the backend web server for Puter. It sets up the Express app, configures middleware, and starts the HTTP server. - * - * @param {Express} app - The Express app instance to configure. - * @returns {void} - * @private - */ - // comment above line 44 in WebServerService.js - async '__on_boot.consolidation' () { - const app = this.app; - const services = this.services; - await services.emit('install.middlewares.early', { app }); - await services.emit('install.middlewares.context-aware', { app }); - this.install_post_middlewares_({ app }); - await services.emit('install.routes', { - app, - router_webhooks: this.router_webhooks, - }); - await services.emit('install.routes-gui', { app }); - - // Register after other services registers theirs: Options for all requests (for CORS) - app.options('/*', (_req, res) => { - return res.sendStatus(200); - }); - - // Catch-all 404 for unmatched routes (e.g. api subdomain with unknown path) - // There seem to be some cases (ex: other subdomains) where this doesn't work - // as intended still, but this is an improvement over the previous behavior. - app.use((req, res) => { - res.status(404).send('Not Found'); - }); - - this.log.debug('web server setup done'); - } - - install_post_middlewares_ ({ app }) { - app.use(async (req, res, next) => { - const svc_event = this.services.get('event'); - - const event = { - req, - res, - end_: false, - end () { - this.end_ = true; - }, - }; - await svc_event.emit('request.will-be-handled', event); - if ( ! event.end_ ) next(); - }); - } - - /** - * Starts the web server and listens for incoming connections. - * This method sets up the Express app, sets up middleware, and starts the server on the specified port. - * It also sets up the Socket.io server for real-time communication. - * - * @returns {Promise} A promise that resolves once the server is started. - */ - async '__on_boot.activation' () { - console.log('starting webser'); - const services = this.services; - await services.emit('start.webserver'); - await services.emit('ready.webserver'); - console.log('in case you care, ready.webserver hooks are done'); - } - - /** - * This method starts the web server by listening on the specified port. It tries multiple ports if the first one is in use. - * If the `config.http_port` is set to 'auto', it will try to find an available port in a range of 4100 to 4299. - * Once the server is up and running, it emits the 'start.webserver' and 'ready.webserver' events. - * If the `config.env` is set to 'dev' and `config.no_browser_launch` is false, it will open the Puter URL in the default browser. - * - * @return {Promise} A promise that resolves when the server is up and running. - */ - async '__on_start.webserver' () { - // error handling middleware goes last, as per the - // expressjs documentation: - // https://expressjs.com/en/guide/error-handling.html - this.app.use(require('./lib/api_error_handler.js')); - - const { jwt_auth } = require('../../helpers.js'); - - config.http_port = process.env.PORT ?? config.http_port; - - globalThis.deployment_type = - config.http_port === 5101 ? 'green' : - config.http_port === 5102 ? 'blue' : - 'not production'; - - let server; - - const auto_port = config.http_port === 'auto'; - let ports_to_try = auto_port ? (() => { - const ports = []; - for ( let i = 0 ; i < 20 ; i++ ) { - ports.push(4100 + i); - } - return ports; - })() : [Number.parseInt(config.http_port)]; - - for ( let i = 0 ; i < ports_to_try.length ; i++ ) { - const port = ports_to_try[i]; - const is_last_port = i === ports_to_try.length - 1; - if ( auto_port ) this.log.debug(`trying port: ${ port}`); - try { - server = http.createServer(this.app).listen(port); - server.timeout = 1000 * 60 * 60 * 2; // 2 hours - let should_continue = false; - await new Promise((rslv, rjct) => { - server.on('error', e => { - if ( e.code === 'EADDRINUSE' ) { - if ( !is_last_port && e.code === 'EADDRINUSE' ) { - this.log.info(`port in use: ${ port}`); - should_continue = true; - } - rslv(); - } else { - rjct(e); - } - }); - /** - * Starts the web server. - * - * This method is responsible for creating the HTTP server, setting up middleware, and starting the server on the specified port. If the specified port is "auto", it will attempt to find an available port within a range. - * - * @returns {Promise} - */ - // Add this comment above line 110 - // (line 110 of the provided code) - server.on('listening', () => { - rslv(); - }); - }); - if ( should_continue ) continue; - } catch (e) { - if ( !is_last_port && e.code === 'EADDRINUSE' ) { - this.log.info(`port in use:${ port}`); - continue; - } - throw e; - } - config.http_port = port; - break; - } - ports_to_try = null; // GC - - const url = config.origin; - - const args = yargs(hideBin(process.argv)).argv; - if ( args['server'] ) { - (async () => { - (await import('./../../../../../tools/auth_gui.js')).default(args['puter-backend']); - })(); - config.no_browser_launch = true; - } - // Open the browser to the URL of Puter - // (if we are in development mode only) - if ( config.env === 'dev' && !config.no_browser_launch ) { - try { - const openModule = await import('open'); - openModule.default(url); - } catch (e) { - console.log('Error opening browser', e); - } - } - - const link = `\x1B[34;1m${url}\x1B[0m`; - const lines = [ - `Puter is now live at: ${link}`, - `listening on port: ${config.http_port}`, - ]; - const realConsole = globalThis.original_console_object ?? console; - lines.forEach(line => realConsole.log(line)); - - realConsole.log('\n************************************************************'); - realConsole.log(`* Puter is now live at: ${url}`); - realConsole.log('************************************************************'); - - server.timeout = 1000 * 60 * 60 * 2; // 2 hours - server.requestTimeout = 1000 * 60 * 60 * 2; // 2 hours - server.headersTimeout = 1000 * 60 * 60 * 2; // 2 hours - const albIdleTimeoutMs = 1000 * 60 * 5; - server.keepAliveTimeout = albIdleTimeoutMs + (1000 * 15); - - // Socket.io server instance - // const socketio = require('../../socketio.js').init(server); - - // TODO: ^ Replace above line with the following code: - await this.services.emit('install.socketio', { server }); - const socketio = this.services.get('socketio').io; - const authService = this.services.get('auth'); - - // Socket.io middleware for authentication - socketio.use(async (socket, next) => { - const authToken = socket.handshake?.auth?.auth_token; - if ( ! authToken ) { - next(new Error('socket auth token missing')); - return; - } - - try { - const authRes = await jwt_auth(socket, authService); - // successful auth - socket.actor = authRes.actor; - socket.user = authRes.user; - socket.token = authRes.token; - // join user room - socket.join(socket.user.id); - - // setTimeout 0 is needed because we need to send - // the notifications after this handler is done - // setTimeout(() => { - // }, 1000); - next(); - } catch ( error ) { - console.warn('socket auth err', error); - const authError = error instanceof Error - ? error - : new Error('socket auth failed'); - next(authError); - } - }); - - const context = Context.get(); - socketio.on('connection', (socket) => { - socket.on('disconnect', () => { - }); - socket.on('trash.is_empty', (msg) => { - socket.broadcast.to(socket.user.id).emit('trash.is_empty', msg); - }); - const svc_event = this.services.get('event'); - context.arun(async () => { - await svc_event.emit('web.socket.connected', { - socket, - user: socket.user, - }); - }); - socket.on('puter_is_actually_open', async (_msg) => { - await context.sub({ - actor: socket.actor, - }).arun(async () => { - await svc_event.emit('web.socket.user-connected', { - socket, - user: socket.user, - }); - }); - }); - }); - - this.server_ = server; - this.registerGracefulShutdownHandlers(); - await this.services.emit('install.websockets'); - } - - /** - * Starts the Puter web server and sets up routes, middleware, and error handling. - * - * @param {object} services - An object containing all services available to the web server. - * @returns {Promise} A promise that resolves when the web server is fully started. - */ - get_server () { - return this.server_; - } - - registerGracefulShutdownHandlers () { - if ( this.gracefulShutdownHandlersInstalled ) return; - this.gracefulShutdownHandlersInstalled = true; - - process.on('SIGTERM', () => { - this.beginGracefulShutdown('SIGTERM'); - }); - process.on('SIGINT', () => { - this.beginGracefulShutdown('SIGINT'); - }); - } - - beginGracefulShutdown (signal) { - if ( ! process.env.PUTER_SERVER_ID ) { - // if not set not running in production, so we can skip setting up graceful shutdown handlers - console.warn('PUTER_SERVER_ID is not set; not waiting for graceful shutdown handlers to complete'); - process.exit(0); - return; - } - if ( this.shutdownStarted ) return; - this.shutdownStarted = true; - this.isDraining = true; - this.app?.set('isDraining', true); - this.drainCoreServicesForShutdown(signal); - const albFailoverDelayMs = 15 * 1000; - - this.log.info( - `received ${signal}; beginning graceful shutdown with ${albFailoverDelayMs}ms ALB failover delay`, - ); - const server = this.server_; - if ( ! server ) { - process.exit(0); - return; - } - - this.shutdownForceExitTimer = setTimeout(() => { - this.log.error('graceful shutdown timed out; forcing process exit'); - process.exit(1); - }, 110 * 1000); - if ( typeof this.shutdownForceExitTimer.unref === 'function' ) { - this.shutdownForceExitTimer.unref(); - } - - this.shutdownCloseTimer = setTimeout(() => { - this.shutdownCloseTimer = null; - if ( typeof server.closeIdleConnections === 'function' ) { - server.closeIdleConnections(); - } - - server.close((error) => { - if ( this.shutdownForceExitTimer ) { - clearTimeout(this.shutdownForceExitTimer); - this.shutdownForceExitTimer = null; - } - - if ( error ) { - this.log.error('error while closing HTTP server during shutdown', error); - process.exit(1); - return; - } - - this.log.info('graceful shutdown completed'); - process.exit(0); - }); - }, albFailoverDelayMs); - if ( typeof this.shutdownCloseTimer.unref === 'function' ) { - this.shutdownCloseTimer.unref(); - } - } - - drainCoreServicesForShutdown (signal) { - const reason = `signal:${signal}`; - for ( const serviceName of ['alarm', 'server-health'] ) { - try { - const service = this.services.get(serviceName); - if ( typeof service?.beginDrain === 'function' ) { - service.beginDrain(reason); - } - } catch ( error ) { - this.log.error( - `failed to drain ${serviceName} during shutdown`, - error, - ); - } - } - } - - /** - * Handles starting and managing the Puter web server. - * - * @param {Object} services - An object containing all services. - */ - async _init () { - const app = express(); - this.app = app; - - app.set('services', this.services); - app.set('isDraining', false); - - this.middlewares = { auth }; - - const require = this.require; - - const config = this.global_config; - new ContextExpressMiddleware({ - parent: globalThis.root_context.sub({ - puter_environment: Context.create({ - env: config.env, - version: relative_require('../../../package.json').version, - }), - }, 'mw'), - }).install(app); - - app.use(async (req, res, next) => { - req.services = this.services; - next(); - }); - - // When the user visits the main origin (not api/dav subdomain) with ?auth_token= - // (e.g. QR login), set the HTTP-only session cookie so user-protected endpoints work. - app.use(async (req, res, next) => { - const has_subdomain = req.hostname.slice(0, -1 * (config.domain.length + 1)) !== ''; - if ( has_subdomain ) return next(); - - const token = req.query?.auth_token; - if ( !token || typeof token !== 'string' ) return next(); - - try { - const svc_auth = req.services.get('auth'); - const cleanToken = token.replace('Bearer ', '').trim(); - const actor = await svc_auth.authenticate_from_token(cleanToken); - const session_token = svc_auth.create_session_token_for_session( - actor.type.user, - actor.type.session, - ); - res.cookie(config.cookie_name, session_token, { - sameSite: 'none', - secure: true, - httpOnly: true, - }); - } catch ( e ) { - console.log('query auth token (QR Code login probably) failed'); - console.error(e); - } - next(); - }); - - // Measure data transfer amounts - app.use(measure()); - - // Instrument logging to use our log service - { - // Switch log function at config time; info log is configurable - const logfn = (config.logging ?? []).includes('http') - ? (log, { message, fields }) => { - log.info(message); - log.debug(message, fields); - } - : (log, { message, fields }) => { - log.debug(message, fields); - }; - - const morgan = require('morgan'); - const stream = { - write: (message) => { - const [method, url, status, responseTime] = message.split(' '); - const fields = { - method, - url, - status: parseInt(status, 10), - responseTime: parseFloat(responseTime), - }; - if ( url.includes('android-icon') ) return; - - // remove `puter.auth.*` query params - const safe_url = (u => { - // We need to prepend an arbitrary domain to the URL - const url = new URL(`https://example.com${ u}`); - const search = url.searchParams; - for ( const key of search.keys() ) { - if ( key.startsWith('puter.auth.') ) search.delete(key); - } - return `${url.pathname }?${ search.toString()}`; - })(fields.url); - fields.url = safe_url; - // re-write message - message = [ - fields.method, fields.url, - fields.status, fields.responseTime, - ].join(' '); - - const log = this.services.get('log-service').create('morgan'); - try { - this.context.arun(() => { - logfn(log, { message, fields }); - }); - } catch (e) { - console.log('failed to log this message properly:', message, fields); - console.error(e); - } - }, - }; - - app.use(morgan(':method :url :status :response-time', { stream })); - } - - /** - * Initialize the web server, start it, and handle any related logic. - * - * This method is responsible for creating the server and listening on the - * appropriate port. It also sets up middleware, routes, and other necessary - * configurations. - * - * @returns {Promise} A promise that resolves once the server is up and running. - */ - app.use((() => { - // const router = express.Router(); - // router.get('/wut', express.json(), (req, res, next) => { - // return res.status(500).send('Internal Error'); - // }); - // return router; - - return eggspress('/wut', { - allowedMethods: ['GET'], - }, async (req, res, _next) => { - // throw new Error('throwy error'); - return res.status(200).send('test endpoint'); - }); - })()); - - (() => { - const onFinished = require('on-finished'); - app.use((req, res, next) => { - onFinished(res, () => { - if ( res.statusCode !== 500 ) return; - if ( req.__error_handled ) return; - if ( req.path === '/healthcheck' ) return; - const alarm = this.services.get('alarm'); - alarm.create('responded-500', 'server sent a 500 response', { - error: req.__error_source, - url: req.url, - method: req.method, - body: req.body, - headers: req.headers, - }); - }); - next(); - }); - })(); - - app.use(async function (req, res, next) { - // Express does not document that this can be undefined. - // The browser likely doesn't follow the HTTP/1.1 spec - // (bot client?) and express is handling this badly by - // not setting the header at all. (that's my theory) - if ( req.hostname === undefined ) { - res.status(400).send( - 'Please verify your browser is up-to-date.', - ); - return; - } - - return next(); - }); - - // Validate host header against allowed domains to prevent host header injection - // https://www.owasp.org/index.php/Host_Header_Injection - app.use((req, res, next) => { - const allowedDomains = new Set(); - const pushAllowedDomain = (domain) => { - const normalizedDomain = normalizeHostDomain(domain); - if ( normalizedDomain ) { - allowedDomains.add(normalizedDomain); - } - }; - - const staticHostingDomain = normalizeHostDomain(config.static_hosting_domain); - pushAllowedDomain(config.domain); - pushAllowedDomain(staticHostingDomain); - pushAllowedDomain(config.static_hosting_domain_alt); - pushAllowedDomain(config.private_app_hosting_domain); - pushAllowedDomain(config.private_app_hosting_domain_alt); - if ( staticHostingDomain ) { - pushAllowedDomain(`at.${staticHostingDomain}`); - } - - if ( config.allow_nipio_domains ) { - pushAllowedDomain('nip.io'); - } - - // Retrieve the Host header and ensure it's in a valid format - const hostHeader = req.headers.host; - - if ( !config.allow_no_host_header && !hostHeader ) { - return res.status(400).send('Missing Host header.'); - } - - if ( config.allow_all_host_values ) { - next(); - return; - } - - // Parse the Host header to isolate the hostname (strip out port if present) - const hostName = hostHeader.split(':')[0].trim().toLowerCase(); - // Check if the hostname matches any of the allowed domains or is a subdomain of an allowed domain - // Exception: allow /healthcheck endpoint on the root domain - if ( - req.path === '/healthcheck' - ) { - next(); - return; - } - if ( [...allowedDomains].some(allowedDomain => hostMatchesDomain(hostName, allowedDomain)) ) { - next(); // Proceed if the host is valid - return; - } else { - if ( ! config.custom_domains_enabled ) { - res.status(400).send('Invalid Host header.'); - return; - } - req.is_custom_domain = true; - next(); - return; - } - }); - - // Validate IP with any IP checkers - app.use(async (req, res, next) => { - const svc_event = this.services.get('event'); - const event = { - allow: true, - ip: req.headers?.['x-forwarded-for'] || - req.connection?.remoteAddress, - }; - - if ( ! this.config.disable_ip_validate_event ) { - await svc_event.emit('ip.validate', event); - } - - // rules that don't apply to notification endpoints - const undefined_origin_allowed = config.undefined_origin_allowed || this.allowedRoutesWithUndefinedOrigins.some(rule => { - if ( typeof rule === 'string' ) return rule === req.path; - return rule.test(req.path); - }); - if ( ! undefined_origin_allowed ) { - // check if no origin - if ( req.method === 'POST' && req.headers.origin === undefined ) { - event.allow = false; - } - } - if ( ! event.allow ) { - return res.status(403).send('Forbidden'); - } - next(); - }); - - // Web hooks need a router that occurs before JSON parse middleware - // so that signatures of the raw JSON can be verified - this.router_webhooks = express.Router(); - app.use(this.router_webhooks); - - app.use((req, res, next) => { - if ( req.get('x-amz-sns-message-type') ) { - req.headers['content-type'] = 'application/json'; - } - next(); - }); - - const rawBodyBuffer = (req, res, buf, encoding) => { - req.rawBody = buf.toString(encoding || 'utf8'); - }; - - app.use(express.json({ limit: '50mb', verify: rawBodyBuffer })); - app.use((req, res, next) => { - if ( req.headers['content-type']?.startsWith('application/json') - && req.body - && Buffer.isBuffer(req.body) - ) { - try { - req.rawBody = req.body; - req.body = JSON.parse(req.body.toString('utf8')); - } catch { - return res.status(400).send({ - error: { - message: 'Invalid JSON body', - }, - }); - } - } - next(); - }); - - const cookieParser = require('cookie-parser'); - app.use(cookieParser({ limit: '50mb' })); - - // gzip compression for all requests - const compression = require('compression'); - app.use(compression()); - - // Helmet and other security - const helmet = require('helmet'); - app.use(helmet.noSniff()); - app.use(helmet.hsts()); - app.use(helmet.ieNoOpen()); - app.use(helmet.permittedCrossDomainPolicies()); - app.use(helmet.xssFilter()); - // app.use(helmet.referrerPolicy()); - app.disable('x-powered-by'); - - // remove object and array query parameters - app.use(function (req, res, next) { - for ( let k in req.query ) { - if ( req.query[k] === undefined || req.query[k] === null ) { - continue; - } - - const allowed_types = ['string', 'number', 'boolean']; - if ( ! allowed_types.includes(typeof req.query[k]) ) { - req.query[k] = undefined; - } - } - next(); - }); - - const uaParser = require('ua-parser-js'); - app.use(function (req, res, next) { - const ua_header = req.headers['user-agent']; - const ua = uaParser(ua_header); - req.ua = ua; - next(); - }); - - app.use(function (req, res, next) { - req.co_isolation_enabled = - ['Chrome', 'Edge'].includes(req.ua.browser.name) - && (Number(req.ua.browser.major) >= 110); - next(); - }); - - app.use(function (req, res, next) { - const origin = req.headers.origin; - const subdomain = req.subdomains[req.subdomains.length - 1]; - const isApiOrDavRequest = - config.experimental_no_subdomain || - subdomain === 'api' || - subdomain === 'dav'; - const isCrossOriginAuthRoute = - req.path === '/signup' || - req.path === '/login' || - req.path.startsWith('/extensions/') || - req.path.startsWith('/auth/oidc'); - - const is_site = - hostMatchesDomain(req.hostname, config.static_hosting_domain) || - hostMatchesDomain(req.hostname, config.static_hosting_domain_alt) || - hostMatchesDomain(req.hostname, config.private_app_hosting_domain) || - hostMatchesDomain(req.hostname, config.private_app_hosting_domain_alt); - req.hostname === 'docs.puter.com' - ; - const is_popup = !!req.query.embedded_in_popup; - const is_parent_co = !!req.query.cross_origin_isolated; - const is_app = !!req.query['puter.app_instance_id']; - - const co_isolation_okay = - (!is_popup || is_parent_co) && - (is_app || !is_site) && - req.co_isolation_enabled - ; - - if ( isCrossOriginAuthRoute || isApiOrDavRequest ) { - res.setHeader('Access-Control-Allow-Origin', origin ?? '*'); - if ( origin ) { - res.vary('Origin'); - } - } - - // Allow browser credentials on API/DAV cross-origin requests. - if ( isApiOrDavRequest && origin ) { - res.setHeader('Access-Control-Allow-Credentials', 'true'); - } - - // Request methods to allow - res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS, PUT, PATCH, DELETE, PROPFIND, PROPPATCH, MKCOL, COPY, MOVE, LOCK, UNLOCK'); - - const allowed_headers = [ - 'Origin', 'X-Requested-With', 'Content-Type', 'Accept', 'Authorization', 'sentry-trace', 'baggage', - 'Depth', 'Destination', 'Overwrite', 'If', 'Lock-Token', 'DAV', 'stripe-signature', - ]; - - // Request headers to allow - res.header('Access-Control-Allow-Headers', allowed_headers.join(', ')); - - // Needed for SharedArrayBuffer - // NOTE: This is put behind a configuration flag because we - // need some experimentation to ensure the interface - // between apps and Puter doesn't break. - if ( config.cross_origin_isolation && co_isolation_okay ) { - res.setHeader('Cross-Origin-Opener-Policy', 'same-origin'); - res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp'); - } - res.setHeader('Cross-Origin-Resource-Policy', 'cross-origin'); - - // Pass to next layer of middleware - - // disable iframes on the main domain - if ( req.hostname === config.domain ) { - // disable iframes - res.setHeader('X-Frame-Options', 'SAMEORIGIN'); - } - - next(); - }); - } -} - -module.exports = WebServerService; diff --git a/src/backend/src/modules/web/lib/__lib__.js b/src/backend/src/modules/web/lib/__lib__.js deleted file mode 100644 index bd9e95dd6..000000000 --- a/src/backend/src/modules/web/lib/__lib__.js +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -module.exports = { - eggspress: require('./eggspress'), - api_error_handler: require('./api_error_handler'), -}; diff --git a/src/backend/src/modules/web/lib/api_error_handler.js b/src/backend/src/modules/web/lib/api_error_handler.js deleted file mode 100644 index 8b8576f5d..000000000 --- a/src/backend/src/modules/web/lib/api_error_handler.js +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../../api/APIError.js'); - -/** - * api_error_handler() is an express error handler for API errors. - * It adheres to the express error handler signature and should be - * used as the last middleware in an express app. - * - * Since Express 5 is not yet released, this function is used by - * eggspress() to handle errors instead of as a middleware. - * - * @param {*} err - * @param {*} req - * @param {*} res - * @param {*} next - * @returns - */ -module.exports = function api_error_handler (err, req, res, next) { - if ( res.headersSent ) { - console.error('error after headers were sent:', err); - return next(err); - } - - // API errors might have a response to help the - // developer resolve the issue. - if ( err instanceof APIError ) { - return err.write(res); - } - - if ( - typeof err === 'object' && - !(err instanceof Error) && - err.hasOwnProperty('message') - ) { - const apiError = APIError.create(400, err); - return apiError.write(res); - } - - console.error('internal server error:', err); - - const services = globalThis.services; - if ( services && services.has('alarm') ) { - const alarm = services.get('alarm'); - alarm.create('api_error_handler', err.message, { - error: err, - url: req.url, - method: req.method, - body: req.body, - headers: req.headers, - }); - } - - req.__error_handled = true; - - // Other errors should provide as little information - // to the client as possible for security reasons. - return res.send(500, 'Internal Server Error'); -}; diff --git a/src/backend/src/modules/web/lib/eggspress.js b/src/backend/src/modules/web/lib/eggspress.js deleted file mode 100644 index dadd0df81..000000000 --- a/src/backend/src/modules/web/lib/eggspress.js +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const express = require('express'); -const multer = require('multer'); -const multest = require('@heyputer/multest'); -const api_error_handler = require('./api_error_handler.js'); - -const APIError = require('../../../api/APIError.js'); -const { Context } = require('../../../util/context.js'); -const { subdomain } = require('../../../helpers.js'); -const config = require('../../../config.js'); - -// Oneday, this will be a typescript typedef. -// Doesn't seem like that day is today. - -/** - * @typedef {"GET"|"HEAD"|"POST"|"PUT"|"DELETE"|"PROPFIND"|"PROPPATCH"|"MKCOL"|"COPY"|"MOVE"|"LOCK"|"UNLOCK"|"OPTIONS"} EggspressMethod - */ - -/** - * @typedef {{ - * consolidate(args: { - * req: import('express').Request, - * getParam: (key: string) => unknown, - * }): Promise|unknown, - * }} EggspressParamDefinition - */ - -/** - * @typedef {object} EggspressSettings - * @property {boolean} [auth] - * @property {boolean} [auth2] - * @property {unknown} [abuse] - * @property {boolean} [verified] - * @property {boolean} [json] - * @property {boolean} [jsonCanBeLarge] - * @property {boolean} [noReallyItsJson] - * @property {string[]} [files] - * @property {boolean} [multest] - * @property {string[]} [multipart_jsons] - * @property {Record} [alias] - * @property {Record} [parameters] - * @property {import('express').RequestHandler} [customArgs] - * @property {number} [alarm_timeout] - * @property {number} [response_timeout] - * @property {import('express').RequestHandler[]} [mw] - * @property {EggspressMethod[]} allowedMethods - * @property {string} [subdomain] - */ - -/** - * eggspress() is a factory function for creating express routers. - * - * @param {string|RegExp|(string|RegExp)[]} route the route to the router - * @param {EggspressSettings} settings the settings for the router - * @param {import('express').RequestHandler} handler the handler for the router - * @returns {express.Router} the router - */ -module.exports = function eggspress (route, settings, handler) { - const router = express.Router(); - const mw = []; - const afterMW = []; - - const _defaultJsonOptions = {}; - if ( settings.jsonCanBeLarge ) { - _defaultJsonOptions.limit = '10mb'; - } - - // Subdomain should be checked before any other middleware to prevent - // unnecessary processing and re-sending headers. - if ( settings.subdomain ) { - mw.push((req, res, next) => { - if ( subdomain(req) !== settings.subdomain ) { - next('route'); - return; - } - next(); - }); - } - - // These flags enable specific middleware. - if ( settings.abuse ) mw.push(require('../../../middleware/abuse')(settings.abuse)); - if ( settings.verified ) mw.push(require('../../../middleware/verified')); - - // if json explicitly set false, don't use it - if ( settings.json !== false ) { - if ( settings.json ) mw.push(express.json(_defaultJsonOptions)); - // A hack so plain text is parsed as JSON in methods which need to be lower latency/avoid the cors roundtrip - if ( settings.noReallyItsJson ) mw.push(express.json({ ..._defaultJsonOptions, type: '*/*' })); - - mw.push(express.json({ - ..._defaultJsonOptions, - type: (req) => req.headers['content-type'] === 'text/plain;actually=json', - })); - } - - if ( settings.auth ) mw.push(require('../../../middleware/auth')); - if ( settings.auth2 ) mw.push(require('../../../middleware/auth2')); - - // The `files` setting is an array of strings. Each string is the name - // of a multipart field that contains files. `multer` is used to parse - // the multipart request and store the files in `req.files`. - if ( settings.files ) { - for ( const key of settings.files ) { - mw.push(multer().array(key)); - } - } - - if ( settings.multest ) { - mw.push(multest()); - } - - // The `multipart_jsons` setting is an array of strings. Each string - // is the name of a multipart field that contains JSON. This middleware - // parses the JSON in each field and stores the result in `req.body`. - if ( settings.multipart_jsons ) { - for ( const key of settings.multipart_jsons ) { - mw.push((req, res, next) => { - try { - if ( ! Array.isArray(req.body[key]) ) { - req.body[key] = [JSON.parse(req.body[key])]; - } else { - req.body[key] = req.body[key].map(JSON.parse); - } - } catch ( _e ) { - return res.status(400).send({ - error: { - message: `Invalid JSON in multipart field ${key}`, - }, - }); - } - next(); - }); - } - } - - // The `alias` setting is an object. Each key is the name of a - // parameter. Each value is the name of a parameter that should - // be aliased to the key. - if ( settings.alias ) { - for ( const alias in settings.alias ) { - const target = settings.alias[alias]; - mw.push((req, res, next) => { - const values = req.method === 'GET' ? req.query : req.body; - if ( values[alias] ) { - values[target] = values[alias]; - } - next(); - }); - } - } - - // The `parameters` setting is an object. Each key is the name of a - // parameter. Each value is a `Param` object. The `Param` object - // specifies how to validate the parameter. - if ( settings.parameters ) { - for ( const key in settings.parameters ) { - const param = settings.parameters[key]; - mw.push(async (req, res, next) => { - if ( ! req.values ) req.values = {}; - - const values = req.method === 'GET' ? req.query : req.body; - const getParam = (key) => values[key]; - try { - const result = await param.consolidate({ req, getParam }); - req.values[key] = result; - } catch (e) { - api_error_handler(e, req, res, next); - return; - } - next(); - }); - } - } - - // what if I wanted to pass arguments to, for example, `json`? - if ( settings.customArgs ) mw.push(settings.customArgs); - - if ( settings.alarm_timeout ) { - mw.push((req, res, next) => { - setTimeout(() => { - if ( ! res.headersSent ) { - const log = req.services.get('log-service').create('eggspress:timeout'); - const errors = req.services.get('error-service').create(log); - let id = Array.isArray(route) ? route[0] : route; - id = id.replace(/\//g, '_'); - errors.report(id, { - source: new Error('Response timed out.'), - message: 'Response timed out.', - trace: true, - alarm: true, - }); - } - }, settings.alarm_timeout); - next(); - }); - } - - if ( settings.response_timeout ) { - mw.push((req, res, next) => { - setTimeout(() => { - if ( ! res.headersSent ) { - api_error_handler(APIError.create('response_timeout'), req, res, next); - } - }, settings.response_timeout); - next(); - }); - } - - if ( settings.mw ) { - mw.push(...settings.mw); - } - - const errorHandledHandler = async function (req, res, next) { - if ( settings.subdomain ) { - if ( subdomain(req) !== settings.subdomain ) { - return next(); - } - } - if ( config.env === 'dev' && process.env.DEBUG ) { - console.log(`request url: ${req.url}, body: ${JSON.stringify(req.body)}`); - } - try { - const expected_ctx = res.locals.ctx; - const received_ctx = Context.get(undefined, { allow_fallback: true }); - - if ( expected_ctx != received_ctx ) { - await expected_ctx.arun(async () => { - await handler(req, res, next); - }); - } else await handler(req, res, next); - } catch (e) { - if ( config.env === 'dev' ) { - if ( ! (e instanceof APIError) ) { - // Any non-APIError indicates an unhandled error (i.e. a bug) from the backend. - // We add a dedicated branch to facilitate debugging. - console.error(e); - } - } - api_error_handler(e, req, res, next); - } - }; - if ( settings.allowedMethods.includes('GET') ) { - router.get(route, ...mw, errorHandledHandler, ...afterMW); - } - - if ( settings.allowedMethods.includes('HEAD') ) { - router.head(route, ...mw, errorHandledHandler, ...afterMW); - } - - if ( settings.allowedMethods.includes('POST') ) { - router.post(route, ...mw, errorHandledHandler, ...afterMW); - } - - if ( settings.allowedMethods.includes('PUT') ) { - router.put(route, ...mw, errorHandledHandler, ...afterMW); - } - - if ( settings.allowedMethods.includes('DELETE') ) { - router.delete(route, ...mw, errorHandledHandler, ...afterMW); - } - - if ( settings.allowedMethods.includes('PROPFIND') ) { - router.propfind(route, ...mw, errorHandledHandler, ...afterMW); - } - - if ( settings.allowedMethods.includes('PROPPATCH') ) { - router.proppatch(route, ...mw, errorHandledHandler, ...afterMW); - } - - if ( settings.allowedMethods.includes('MKCOL') ) { - router.mkcol(route, ...mw, errorHandledHandler, ...afterMW); - } - - if ( settings.allowedMethods.includes('COPY') ) { - router.copy(route, ...mw, errorHandledHandler, ...afterMW); - } - - if ( settings.allowedMethods.includes('MOVE') ) { - router.move(route, ...mw, errorHandledHandler, ...afterMW); - } - - if ( settings.allowedMethods.includes('LOCK') ) { - router.lock(route, ...mw, errorHandledHandler, ...afterMW); - } - - if ( settings.allowedMethods.includes('UNLOCK') ) { - router.unlock(route, ...mw, errorHandledHandler, ...afterMW); - } - - if ( settings.allowedMethods.includes('OPTIONS') ) { - router.options(route, ...mw, errorHandledHandler, ...afterMW); - } - - return router; -}; diff --git a/src/backend/src/om/IdentifierUtil.js b/src/backend/src/om/IdentifierUtil.js deleted file mode 100644 index a2540f282..000000000 --- a/src/backend/src/om/IdentifierUtil.js +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('@heyputer/putility'); -const { WeakConstructorFeature } = require('../traits/WeakConstructorFeature'); -const { Eq, And } = require('./query/query'); -const { Entity } = require('./entitystorage/Entity'); - -class IdentifierUtil extends AdvancedBase { - static FEATURES = [ - new WeakConstructorFeature(), - ]; - - async detect_identifier (object, allow_mutation = false) { - const redundant_identifiers = this.om.redundant_identifiers ?? []; - - let match_found = null; - for ( let key_set of redundant_identifiers ) { - key_set = Array.isArray(key_set) ? key_set : [key_set]; - key_set.sort(); - - for ( let i = 0 ; i < key_set.length ; i++ ) { - const key = key_set[i]; - const has_key = object instanceof Entity ? - await object.has(key) : object[key] !== undefined; - if ( ! has_key ) { - break; - } - if ( i === key_set.length - 1 ) { - match_found = key_set; - break; - } - } - } - - if ( ! match_found ) return; - - // Construct a query predicate based on the keys - const key_eqs = []; - for ( const key of match_found ) { - key_eqs.push(new Eq({ - key, - value: object instanceof Entity ? - await object.get(key) : object[key], - })); - if ( object instanceof Entity ) { - if ( allow_mutation ) await object.del(key); - } else { - if ( allow_mutation ) delete object[key]; - } - } - let predicate = new And({ children: key_eqs }); - - return predicate; - } -} - -module.exports = { - IdentifierUtil, -}; diff --git a/src/backend/src/om/definitions/Mapping.js b/src/backend/src/om/definitions/Mapping.js deleted file mode 100644 index 54f075c2a..000000000 --- a/src/backend/src/om/definitions/Mapping.js +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('@heyputer/putility'); -const { WeakConstructorFeature } = require('../../traits/WeakConstructorFeature'); -const { Property } = require('./Property'); -const { Entity } = require('../entitystorage/Entity'); -const FSNodeContext = require('../../deprecated/filesystem/FSNodeContext').default; - -/** - * An instance of Mapping wraps every definition in ../mappings before - * it is registered in the 'om' collection in RegistryService. - * Both wrapping and registering are done by RegistrantService. - */ -class Mapping extends AdvancedBase { - static FEATURES = [ - // Whenever you can override something, it's reasonable to want - // to pull the desired implementation from somewhere else to - // avoid repeating yourself. Class constructors are one of a few - // examples where this is typically not possible. - // However, javascript is magic, and we do what we want. - new WeakConstructorFeature(), - ]; - - static create (context, data) { - const properties = {}; - - // NEXT - for ( const k in data.properties ) { - properties[k] = Property.create(context, k, data.properties[k]); - } - - return new Mapping({ - ...data, - properties, - sql: data.sql, - }); - } - - async get_client_safe (data) { - const client_safe = {}; - - for ( const k in this.properties ) { - const prop = this.properties[k]; - let value = data[k]; - - if ( prop.descriptor.protected ) { - continue; - } - - if ( value === undefined ) { - continue; - } - - let sanitized = false; - - if ( value instanceof Entity ) { - value = await value.get_client_safe(); - sanitized = true; - } - - if ( value instanceof FSNodeContext ) { - if ( ! await value.exists() ) { - value = undefined; - continue; - } - value = await value.getSafeEntry(); - sanitized = true; - } - - // This is for reference properties to remove sensitive - // information in case a decorator added the real object. - if ( - ( !sanitized ) && - typeof value === 'object' && value !== null && - prop.descriptor.permissible_subproperties - ) { - const old_value = value; - value = {}; - for ( const subprop_name of prop.descriptor.permissible_subproperties ) { - if ( ! old_value.hasOwnProperty(subprop_name) ) { - continue; - } - value[subprop_name] = old_value[subprop_name]; - } - } - - // client_safe[k] = await prop.typ.get_client_safe(value); - client_safe[k] = value; - } - - return client_safe; - } -} - -module.exports = { - Mapping, -}; diff --git a/src/backend/src/om/definitions/PropType.js b/src/backend/src/om/definitions/PropType.js deleted file mode 100644 index 9520b7ca5..000000000 --- a/src/backend/src/om/definitions/PropType.js +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('@heyputer/putility'); -const { WeakConstructorFeature } = require('../../traits/WeakConstructorFeature'); - -class PropType extends AdvancedBase { - static FEATURES = [ - new WeakConstructorFeature(), - ]; - - static create (context, data, k) { - const chains = {}; - const super_type = data.from && (() => { - const registry = context.get('registry'); - const types = registry.get('om:proptype'); - const super_type = types.get(data.from); - if ( ! super_type ) { - throw new Error(`Failed to find super type "${data.from}"`); - } - return super_type; - })(); - - data = { ...data }; - delete data.from; - - if ( super_type ) { - super_type.populate_subtype_(chains); - } - - for ( const k in data ) { - if ( ! Object.prototype.hasOwnProperty.call(chains, k) ) { - chains[k] = []; - } - chains[k].push(data[k]); - } - - return new PropType({ - chains, name: k, - }); - } - - populate_subtype_ (chains) { - for ( const k in this.chains ) { - if ( ! Object.prototype.hasOwnProperty.call(chains, k) ) { - chains[k] = []; - } - chains[k].push(...this.chains[k]); - } - } - - async adapt (value, extra) { - const adapters = this.chains.adapt - ? [...this.chains.adapt].reverse() - : []; - - for ( const adapter of adapters ) { - value = await adapter(value, extra); - } - - return value; - } - - async sql_dereference (value, extra) { - const sql_dereferences = this.chains.sql_dereference || []; - - for ( const sql_dereference of sql_dereferences ) { - value = await sql_dereference(value, extra); - } - - return value; - } - - async sql_reference (value, extra) { - const sql_references = this.chains.sql_reference || []; - - for ( const sql_reference of sql_references ) { - value = await sql_reference(value, extra); - } - - return value; - } - - async validate (value, extra) { - const validators = this.chains.validate || []; - - for ( const validator of validators ) { - const result = await validator(value, extra); - if ( result !== true && result !== undefined ) { - return result; - } - } - - return true; - } - - async factory (extra) { - const factories = ( - this.chains.factory && [...this.chains.factory].reverse() - ) || []; - - if ( process.env.DEBUG ) { - console.log('FACTORIES', factories); - } - - for ( const factory of factories ) { - const result = await factory(extra); - if ( result !== undefined ) { - return result; - } - } - - return undefined; - } - - async is_set (value) { - const is_setters = this.chains.is_set || []; - - for ( const is_setter of is_setters ) { - const result = await is_setter(value); - if ( ! result ) { - return false; - } - } - - return true; - } -} - -module.exports = { - PropType, -}; diff --git a/src/backend/src/om/definitions/PropType.test.js b/src/backend/src/om/definitions/PropType.test.js deleted file mode 100644 index fd712ee7a..000000000 --- a/src/backend/src/om/definitions/PropType.test.js +++ /dev/null @@ -1,37 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -const { PropType } = require('./PropType'); - -describe('PropType adapt chain ordering', () => { - it('runs subtype adapters before supertype adapters on every call', async () => { - const callOrder = []; - const typ = new PropType({ - name: 'test', - chains: { - adapt: [ - value => { - callOrder.push('super'); - if ( typeof value !== 'string' ) { - throw new Error('expected string'); - } - return value; - }, - value => { - callOrder.push('sub'); - if ( value && typeof value === 'object' && typeof value.url === 'string' ) { - return value.url; - } - return value; - }, - ], - }, - }); - - await expect(typ.adapt({ url: 'https://example.com/icon-a.png' })) - .resolves.toBe('https://example.com/icon-a.png'); - await expect(typ.adapt({ url: 'https://example.com/icon-b.png' })) - .resolves.toBe('https://example.com/icon-b.png'); - - expect(callOrder).toEqual(['sub', 'super', 'sub', 'super']); - }); -}); diff --git a/src/backend/src/om/definitions/Property.js b/src/backend/src/om/definitions/Property.js deleted file mode 100644 index 741a2b83f..000000000 --- a/src/backend/src/om/definitions/Property.js +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('@heyputer/putility'); -const { WeakConstructorFeature } = require('../../traits/WeakConstructorFeature'); - -class Property extends AdvancedBase { - static FEATURES = [ - new WeakConstructorFeature(), - ]; - - static create (context, name, descriptor) { - // Adapt descriptor - if ( typeof descriptor === 'string' ) { - descriptor = { type: descriptor }; - } - - const registry = context.get('registry'); - const types = registry.get('om:proptype'); - const typ = types.get(descriptor['type']); - - if ( ! typ ) { - throw new Error(`Failed to find type "${descriptor['type']}"`); - } - - // NEXT - - return new Property({ name, descriptor, typ }); - } - - constructor (...a) { - super(...a); - } - - async adapt (value) { - const { name, descriptor } = this; - try { - value = await this.typ.adapt(value, { name, descriptor }); - if ( descriptor.adapt && typeof descriptor.adapt === 'function' ) { - value = await descriptor.adapt(value, { name, descriptor }); - } - } catch ( e ) { - throw new Error(`Failed to adapt ${name} to ${descriptor.type}: ${e.message}`); - } - return value; - } - - async sql_dereference (value) { - const { name, descriptor } = this; - return await this.typ.sql_dereference(value, { name, descriptor }); - } - - async sql_reference (value) { - const { name, descriptor } = this; - return await this.typ.sql_reference(value, { name, descriptor }); - } - - async validate (value) { - const { name, descriptor } = this; - if ( this.descriptor.validate ) { - let result = await this.descriptor.validate(value); - if ( result && result !== true ) return result; - } - return await this.typ.validate(value, { name, descriptor }); - } - - async factory () { - const { name, descriptor } = this; - if ( this.descriptor.factory ) { - let value = await this.descriptor.factory(); - if ( value ) return value; - } - return await this.typ.factory({ name, descriptor }); - } - - async is_set (value) { - return await this.typ.is_set(value); - } -} - -module.exports = { - Property, -}; diff --git a/src/backend/src/om/docs/DESIGN.md b/src/backend/src/om/docs/DESIGN.md deleted file mode 100644 index 291417fea..000000000 --- a/src/backend/src/om/docs/DESIGN.md +++ /dev/null @@ -1,19 +0,0 @@ -## Entity Storage - -### Chain of events - -When `create` is called on an OM/ES driver: -1. The request is handled by `src/routers/drivers/call.js` -2. DriverService's `call` method is called -3. An instance of `EntityStoreImplementation` is called -4. `EntityStoreImplementation` calls the corresponding service, - such as `es:app`, which is an instance of `EntityStoreService` -5. `EntityStoreService` calls the upstream implementation of `BaseES` -6. `BaseES` has a public method which calls the implementor method -7. The implementor method (ex: `SQLES`) handles the operation - -``` -/call -> DriverService - -> EntityStoreImplementation -> EntityStoreService -> BaseES - -> ...(storage decorators) -> SQLES -``` diff --git a/src/backend/src/om/entitystorage/AppES.js b/src/backend/src/om/entitystorage/AppES.js deleted file mode 100644 index 4287d32af..000000000 --- a/src/backend/src/om/entitystorage/AppES.js +++ /dev/null @@ -1,1008 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const { AppRedisCacheSpace } = require('../../modules/apps/AppRedisCacheSpace.js'); -const { deleteRedisKeys } = require('../../clients/redis/deleteRedisKeys.js'); -const config = require('../../config'); -const { app_name_exists } = require('../../helpers'); -const { AppUnderUserActorType } = require('../../services/auth/Actor'); -const { DB_WRITE } = require('../../services/database/consts'); -const { Context } = require('../../util/context'); -const { origin_from_url } = require('../../util/urlutil'); -const { Eq, Like, Or, And } = require('../query/query'); -const { BaseES } = require('./BaseES'); -const { Entity } = require('./Entity'); - -const uuidv4 = require('uuid').v4; -const APP_UID_ALIAS_KEY_PREFIX = 'app:canonicalUidAlias'; -const APP_UID_ALIAS_REVERSE_KEY_PREFIX = 'app:canonicalUidAliasReverse'; -const APP_UID_ALIAS_TTL_SECONDS = 60 * 60 * 24 * 90; -const APP_OBJECT_CACHE_TTL_SECONDS = 24 * 60 * 60; -const indexUrlUniquenessExemptionCandidates = [ - 'https://dev-center.puter.com/coming-soon', -]; -const hasIndexUrlUniquenessExemption = (candidates) => { - for ( const candidate of candidates ) { - if ( indexUrlUniquenessExemptionCandidates.find(exception => candidate.startsWith(exception)) ) { - return true; - } - } - return false; -}; - -const normalizeConfiguredHostedDomain = (domainValue) => { - if ( typeof domainValue !== 'string' ) return null; - const normalizedDomainValue = domainValue.trim().toLowerCase().replace(/^\./, ''); - if ( ! normalizedDomainValue ) return null; - return normalizedDomainValue.split(':')[0] || null; -}; - -const getConfiguredHostedDomains = () => { - const hostedDomains = new Set(); - for ( const configuredDomain of [ - config.static_hosting_domain, - config.static_hosting_domain_alt, - config.private_app_hosting_domain, - config.private_app_hosting_domain_alt, - ] ) { - const normalizedDomain = normalizeConfiguredHostedDomain(configuredDomain); - if ( normalizedDomain ) { - hostedDomains.add(normalizedDomain); - } - } - return [...hostedDomains]; -}; - -const extractPuterHostedSubdomainFromIndexUrl = (indexUrl) => { - if ( typeof indexUrl !== 'string' || !indexUrl ) return null; - - let hostname; - try { - hostname = (new URL(indexUrl)).hostname.toLowerCase(); - } catch { - return null; - } - - const hostedDomains = getConfiguredHostedDomains() - .sort((domainA, domainB) => domainB.length - domainA.length); - - for ( const hostedDomain of hostedDomains ) { - const suffix = `.${hostedDomain}`; - if ( hostname.endsWith(suffix) ) { - const subdomain = hostname.slice(0, hostname.length - suffix.length); - return subdomain || null; - } - } - - return null; -}; - -let privateLaunchAccessModulePromise; -const getPrivateLaunchAccessModule = async () => { - if ( ! privateLaunchAccessModulePromise ) { - privateLaunchAccessModulePromise = import('../../modules/apps/privateLaunchAccess.js'); - } - return privateLaunchAccessModulePromise; -}; - -class AppES extends BaseES { - static METHODS = { - async _on_context_provided () { - const services = this.context.get('services'); - this.db = services.get('database').get(DB_WRITE, 'apps'); - }, - - /** - * Creates query predicates for filtering apps - * @param {string} id - Predicate identifier - * @param {...any} args - Additional arguments for predicate creation - * @returns {Promise} Query predicate object - */ - async create_predicate (id, ...args) { - if ( id === 'user-can-edit' ) { - return new Eq({ - key: 'owner', - value: Context.get('user').id, - }); - } - if ( id === 'name-like' ) { - return new Like({ - key: 'name', - value: args[0], - }); - } - }, - async delete (uid, _extra) { - const svc_appInformation = this.context.get('services').get('app-information'); - await svc_appInformation.delete_app(uid); - }, - - async read (uid) { - if ( typeof uid !== 'string' || !uid ) { - return await this.upstream.read(uid); - } - - const canonicalUidAliasPromise = this.read_canonical_app_uid_alias_(uid); - const entity = await this.upstream.read(uid); - if ( entity ) { - return entity; - } - - const canonicalUid = await canonicalUidAliasPromise; - if ( !canonicalUid || canonicalUid === uid ) { - return null; - } - - return await this.upstream.read(canonicalUid); - }, - - /** - * Filters app selection based on user permissions and visibility settings - * @param {Object} options - Selection options including predicates - * @returns {Promise} Filtered selection results - */ - async select (options) { - const actor = Context.get('actor'); - const user = actor.type.user; - - const additional = []; - - // An app is also allowed to read itself - if ( actor.type instanceof AppUnderUserActorType ) { - additional.push(new Eq({ - key: 'uid', - value: actor.type.app.uid, - })); - } - - options.predicate = options.predicate.and(new Or({ - children: [ - new Eq({ - key: 'approved_for_listing', - value: 1, - }), - new Eq({ - key: 'owner', - value: user.id, - }), - ...additional, - ], - })); - - return await this.upstream.select(options); - }, - - /** - * Creates or updates an application with proper name handling and associations - * @param {Object} entity - Application entity to upsert - * @param {Object} extra - Additional upsert parameters - * @returns {Promise} Upsert operation results - */ - async upsert (entity, extra) { - extra = extra || {}; - const actor = Context.get('actor'); - const user = actor?.type?.user; - - const preJoinFullEntity = extra.old_entity - ? await (await extra.old_entity.clone()).apply(entity) - : entity - ; - await this.ensurePuterSiteSubdomainIsOwned(preJoinFullEntity, extra, user); - - await this.maybe_join_owned_hosted_index_url_app_on_create_(entity, extra, user); - - const full_entity = extra.old_entity - ? await (await extra.old_entity.clone()).apply(entity) - : entity - ; - - await this.ensureIndexUrlUnique(full_entity, extra); - - if ( await app_name_exists(await entity.get('name')) ) { - const { old_entity } = extra; - const is_name_change = ( !old_entity ) || - ( await old_entity.get('name') !== await entity.get('name') ); - if ( is_name_change && extra?.options?.dedupe_name ) { - const base = await entity.get('name'); - let number = 1; - while ( await app_name_exists(`${base}-${number}`) ) { - number++; - } - await entity.set('name', `${base}-${number}`); - } - else if ( is_name_change ) { - // The name might be taken because it's the old name - // of this same app. If it is, the app takes it back. - const svc_oldAppName = this.context.get('services').get('old-app-name'); - const name_info = await svc_oldAppName.check_app_name(await entity.get('name')); - if ( !name_info || name_info.app_uid !== await entity.get('uid') ) { - // Throw error because the name really is taken - throw APIError.create('app_name_already_in_use', null, { - name: await entity.get('name'), - }); - } - - // Remove the old name from the old-app-name service - await svc_oldAppName.remove_name(name_info.id); - } else { - entity.del('name'); - } - } - - const subdomain_id = await this.maybe_insert_subdomain_(entity); - const result = await this.upstream.upsert(entity, extra); - const { insert_id } = result; - const oldAssociations = await this.db.read( - 'SELECT type FROM app_filetype_association WHERE app_id = ?', - [insert_id], - ); - const normalizedOldAssociations = oldAssociations - .map(row => String(row.type ?? '').trim().toLowerCase().replace(/^\./, '')) - .filter(Boolean); - - // Remove old file associations (if applicable) - if ( extra.old_entity ) { - await this.db.write( - 'DELETE FROM app_filetype_association WHERE app_id = ?', - [insert_id], - ); - } - - // Add file associations (if applicable) - const filetype_associations = await entity.get('filetype_associations'); - const normalizedNewAssociations = (filetype_associations ?? []) - .map(association => String(association).trim().toLowerCase().replace(/^\./, '')) - .filter(Boolean); - if ( (a => a && a.length > 0)(filetype_associations) ) { - const stmt = - 'INSERT INTO app_filetype_association ' + - `(app_id, type) VALUES ${ - normalizedNewAssociations.map(() => '(?, ?)').join(', ')}`; - const rows = normalizedNewAssociations.map(a => [insert_id, a]); - await this.db.write(stmt, rows.flat()); - } - const affectedAssociationExtensions = new Set([ - ...normalizedOldAssociations, - ...normalizedNewAssociations, - ]); - if ( affectedAssociationExtensions.size ) { - await deleteRedisKeys(Array.from(affectedAssociationExtensions) - .map(ext => AppRedisCacheSpace.associationAppsKey(ext))); - } - - const has_new_icon = - ( !extra.old_entity ) || ( - await entity.get('icon') !== await extra.old_entity.get('icon') - ); - - if ( has_new_icon ) { - const svc_event = this.context.get('services').get('event'); - const event = { - app_uid: await entity.get('uid'), - data_url: await entity.get('icon'), - url: '', - }; - await svc_event.emit('app.new-icon', event); - if ( typeof event.url === 'string' && event.url ) { - await this.db.write( - 'UPDATE apps SET icon = ? WHERE id = ? LIMIT 1', - [event.url, insert_id], - ); - await entity.set('icon', event.url); - } - } - - const has_new_name = - extra.old_entity && ( - await entity.get('name') !== await extra.old_entity.get('name') - ); - - if ( has_new_name ) { - const svc_event = this.context.get('services').get('event'); - const event = { - app_uid: await entity.get('uid'), - new_name: await entity.get('name'), - old_name: await extra.old_entity.get('name'), - }; - await svc_event.emit('app.rename', event); - } - - // Associate app with subdomain (if applicable) - if ( subdomain_id ) { - await this.db.write( - 'UPDATE subdomains SET associated_app_id = ? WHERE id = ?', - [insert_id, subdomain_id], - ); - } - if ( extra.old_entity ) { - const svc_event = this.context.get('services').get('event'); - const [app] = await this.db.read( - 'SELECT * FROM apps WHERE uid = ? LIMIT 1', - [await full_entity.get('uid')], - ); - const old_app = { - uid: await extra.old_entity.get('uid'), - index_url: await extra.old_entity.get('index_url'), - }; - await svc_event.emit('app.changed', { - app_uid: await full_entity.get('uid'), - action: 'updated', - app, - old_app, - }); - } - - if ( extra.joined_source_app_uid ) { - await this.write_canonical_app_uid_alias_({ - oldAppUid: extra.joined_source_app_uid, - canonicalAppUid: await full_entity.get('uid'), - }); - const svc_appInformation = this.context.get('services').get('app-information'); - if ( svc_appInformation?.delete_app ) { - await svc_appInformation.delete_app(extra.joined_source_app_uid, undefined, { - preserveCanonicalUidAlias: true, - }); - } - } - - if ( typeof extra.joined_requested_name === 'string' && extra.joined_requested_name.trim() ) { - const renameResult = await this.apply_joined_requested_name_({ - canonicalUid: await full_entity.get('uid'), - requestedName: extra.joined_requested_name, - }); - if ( renameResult ) { - const svc_event = this.context.get('services').get('event'); - await svc_event.emit('app.rename', { - app_uid: await full_entity.get('uid'), - old_name: renameResult.oldName, - new_name: renameResult.newName, - }); - await full_entity.set('name', renameResult.newName); - } - } - - return result; - }, - async retry_predicate_rewrite ({ predicate }) { - const recurse = async (predicate) => { - if ( predicate instanceof Or ) { - return new Or({ - children: await Promise.all(predicate.children.map(recurse)), - }); - } - if ( predicate instanceof And ) { - return new And({ - children: await Promise.all(predicate.children.map(recurse)), - }); - } - if ( predicate instanceof Eq ) { - if ( predicate.key === 'name' ) { - const svc_oldAppName = this.context.get('services').get('old-app-name'); - const name_info = await svc_oldAppName.check_app_name(predicate.value); - return new Eq({ - key: 'uid', - value: name_info?.app_uid, - }); - } - } - }; - return await recurse(predicate); - }, - - async queueIconMigration (entity) { - if ( ! this.pending_icon_migrations_ ) { - this.pending_icon_migrations_ = new Set(); - } - - const migration_key = entity.private_meta?.mysql_id ?? Symbol('app-icon-migration'); - if ( this.pending_icon_migrations_.has(migration_key) ) { - return; - } - this.pending_icon_migrations_.add(migration_key); - - Promise.resolve().then(async () => { - const icon = await entity.get('icon'); - if ( typeof icon !== 'string' || !icon.startsWith('data:') ) { - return; - } - - const app_uid = await entity.get('uid'); - if ( ! app_uid ) { - return; - } - - const svc_event = this.context.get('services').get('event'); - const event = { - app_uid, - data_url: icon, - }; - await svc_event.emit('app.new-icon', event); - if ( typeof event.url !== 'string' || !event.url ) return; - - await this.db.write( - 'UPDATE apps SET icon = ? WHERE uid = ? LIMIT 1', - [event.url, app_uid], - ); - }).catch(e => { - const svc_error = this.context.get('services').get('error-service'); - svc_error.report('AppES:queue_icon_migration', { source: e }); - }).finally(() => { - this.pending_icon_migrations_.delete(migration_key); - }); - }, - - async get_cached_app_object_ (appUid) { - if ( typeof appUid !== 'string' || !appUid ) return null; - return await AppRedisCacheSpace.getCachedAppObject({ - lookup: 'uid', - value: appUid, - }); - }, - - async set_cached_app_object_ (entity) { - if ( ! entity ) return; - - const cacheable = await entity.get_client_safe(); - delete cacheable.stats; - delete cacheable.privateAccess; - - await AppRedisCacheSpace.setCachedAppObject(cacheable, { - ttlSeconds: APP_OBJECT_CACHE_TTL_SECONDS, - }); - }, - - /** - * Transforms app data before reading by adding associations and handling permissions - * @param {Object} entity - App entity to transform - */ - async read_transform (entity) { - const { - getActorUserUid, - resolvePrivateLaunchAccess, - } = await getPrivateLaunchAccessModule(); - const services = this.context.get('services'); - const actor = Context.get('actor'); - const esParams = Context.get('es_params') ?? {}; - const appUid = await entity.get('uid'); - const appName = await entity.get('name'); - const appIndexUrl = await entity.get('index_url'); - const appCreatedAt = await entity.get('created_at'); - const appIsPrivate = await entity.get('is_private'); - const cachedAppObject = await this.get_cached_app_object_(appUid); - - const appInformationService = services.get('app-information'); - const authService = services.get('auth'); - const statsPromise = appInformationService - ? appInformationService.get_stats(appUid, { - period: esParams.stats_period, - grouping: esParams.stats_grouping, - created_at: appCreatedAt, - }) - : Promise.resolve(undefined); - const cachedFiletypeAssociations = Array.isArray(cachedAppObject?.filetype_associations) - ? cachedAppObject.filetype_associations - : null; - const hasCachedCreatedFromOrigin = !!( - cachedAppObject && - Object.prototype.hasOwnProperty.call(cachedAppObject, 'created_from_origin') - ); - const shouldRefreshCachedAppObject = - !cachedAppObject || - !cachedFiletypeAssociations || - !hasCachedCreatedFromOrigin; - const fileAssociationsPromise = cachedFiletypeAssociations - ? Promise.resolve(cachedFiletypeAssociations) - : this.db.read( - 'SELECT type FROM app_filetype_association WHERE app_id = ?', - [entity.private_meta.mysql_id], - ).then(rows => rows.map(row => row.type)); - const createdFromOriginPromise = hasCachedCreatedFromOrigin - ? Promise.resolve(cachedAppObject.created_from_origin ?? null) - : (async () => { - if ( ! authService ) return null; - try { - const origin = origin_from_url(appIndexUrl); - const expectedUid = await authService.app_uid_from_origin(origin); - return expectedUid === appUid ? origin : null; - } catch { - // This happens when index_url is not a valid URL. - return null; - } - })(); - const privateAccessPromise = resolvePrivateLaunchAccess({ - app: { - uid: appUid, - name: appName, - is_private: appIsPrivate, - }, - services, - userUid: getActorUserUid(actor), - source: 'driverRead', - args: esParams, - }); - - const [ - filetypeAssociations, - stats, - createdFromOrigin, - privateAccess, - ] = await Promise.all([ - fileAssociationsPromise, - statsPromise, - createdFromOriginPromise, - privateAccessPromise, - ]); - await entity.set('filetype_associations', filetypeAssociations); - await entity.set('stats', stats); - await entity.set('created_from_origin', createdFromOrigin); - await entity.set('privateAccess', privateAccess); - if ( shouldRefreshCachedAppObject ) { - await this.set_cached_app_object_(entity); - } - - // Migrate b64 icons to the filesystem-backed icon flow without blocking reads. - this.queueIconMigration(entity); - - // Check if the user is the owner - const is_owner = await (async () => { - let owner = await entity.get('owner'); - - // TODO: why does this happen? - if ( typeof owner === 'number' ) { - owner = { id: owner }; - } - - if ( ! owner ) return false; - const actor = Context.get('actor'); - return actor.type.user.id === owner.id; - })(); - - // Remove fields that are not allowed for non-owners - if ( ! is_owner ) { - entity.del('approved_for_listing'); - entity.del('approved_for_opening_items'); - entity.del('approved_for_incentive_program'); - } - - // Replace icon if an icon size is specified - const iconSize = Context.get('es_params')?.icon_size; - if ( iconSize ) { - const svc_appIcon = this.context.get('services').get('app-icon'); - try { - const iconPath = svc_appIcon.getAppIconPath({ - appUid: await entity.get('uid'), - size: iconSize, - }); - if ( iconPath ) { - await entity.set('icon', iconPath); - } - } catch (e) { - const svc_error = this.context.get('services').get('error-service'); - svc_error.report('AppES:read_transform', { source: e }); - } - } - }, - - /** - * Creates a subdomain entry for the app if required - * @param {Object} entity - App entity - * @returns {Promise} Subdomain ID if created - * @private - */ - async maybe_insert_subdomain_ (entity) { - // Create and update is a situation where we might create a subdomain - - let subdomain_id; - if ( await entity.get('source_directory') ) { - await (await entity.get('source_directory') - ).fetchEntry(); - const subdomain = await entity.get('subdomain'); - const user = Context.get('user'); - let subdomain_res = await this.db.write( - `INSERT ${this.db.case({ - mysql: 'IGNORE', - sqlite: 'OR IGNORE', - })} INTO subdomains - (subdomain, user_id, root_dir_id, uuid) VALUES - ( ?, ?, ?, ?)`, - [ - //subdomain - subdomain, - //user_id - user.id, - //root_dir_id - (await entity.get('source_directory')).mysql_id, - //uuid, `sd` stands for subdomain - `sd-${ uuidv4()}`, - ], - ); - subdomain_id = subdomain_res.insertId; - } - - return subdomain_id; - }, - - /** - * Ensures that when an app uses a puter.site subdomain as its index_url, - * the subdomain belongs to the user creating/updating the app. - */ - async ensurePuterSiteSubdomainIsOwned (entity, extra, user) { - if ( ! user ) return; - - // Only enforce when the index_url is being set or changed - const new_index_url = await entity.get('index_url'); - if ( ! new_index_url ) return; - if ( extra.old_entity ) { - const old_index_url = await extra.old_entity.get('index_url'); - if ( old_index_url === new_index_url ) { - return; - } - } - - const subdomain = extractPuterHostedSubdomainFromIndexUrl(new_index_url); - if ( ! subdomain ) return; - - const svc_puterSite = this.context.get('services').get('puter-site'); - const site = await svc_puterSite.get_subdomain(subdomain, { is_custom_domain: false }); - - if ( !site || site.user_id !== user.id ) { - throw APIError.create('subdomain_not_owned', null, { subdomain }); - } - }, - - is_puter_hosted_index_url_ (index_url) { - return !!extractPuterHostedSubdomainFromIndexUrl(index_url); - }, - - build_equivalent_index_url_candidates_ (index_url) { - if ( typeof index_url !== 'string' || !index_url.trim() ) { - return []; - } - - try { - const parsedUrl = new URL(index_url); - const origin = `${parsedUrl.protocol}//${parsedUrl.host.toLowerCase()}`; - const pathname = parsedUrl.pathname || '/'; - const values = new Set(); - if ( pathname === '/' || pathname.toLowerCase() === '/index.html' ) { - values.add(origin); - values.add(`${origin}/`); - values.add(`${origin}/index.html`); - } else { - const normalizedPath = pathname.endsWith('/') - ? pathname.slice(0, -1) - : pathname; - values.add(`${origin}${normalizedPath}`); - values.add(`${origin}${normalizedPath}/`); - } - return [...values]; - } catch { - return [index_url.trim()]; - } - }, - - async find_index_url_conflict_ ({ indexUrl, excludeMysqlId }) { - if ( ! this.is_puter_hosted_index_url_(indexUrl) ) { - return null; - } - - const candidates = this.build_equivalent_index_url_candidates_(indexUrl); - if ( candidates.length === 0 ) return null; - if ( hasIndexUrlUniquenessExemption(candidates) ) return null; - - const placeholders = candidates.map(() => '?').join(', '); - const parameters = [...candidates]; - let query = `SELECT id, uid, owner_user_id, index_url FROM apps WHERE index_url IN (${placeholders})`; - if ( Number.isInteger(excludeMysqlId) && excludeMysqlId > 0 ) { - query += ' AND id != ?'; - parameters.push(excludeMysqlId); - } - query += ' ORDER BY timestamp ASC, id ASC LIMIT 1'; - - const rows = await this.db.read(query, parameters); - const normalizedExcludeMysqlId = Number(excludeMysqlId); - const conflictRow = rows.find(row => { - if ( - Number.isInteger(normalizedExcludeMysqlId) - && normalizedExcludeMysqlId > 0 - && Number(row?.id) === normalizedExcludeMysqlId - ) { - return false; - } - if ( typeof row?.index_url === 'string' ) { - return candidates.includes(row.index_url); - } - return true; - }); - return conflictRow || null; - }, - - async resolve_entity_mysql_id_ (entity) { - const directMysqlId = Number(entity?.private_meta?.mysql_id); - if ( Number.isInteger(directMysqlId) && directMysqlId > 0 ) { - return directMysqlId; - } - - if ( !entity || typeof entity.get !== 'function' ) { - return undefined; - } - - const uid = await entity.get('uid'); - if ( typeof uid !== 'string' || !uid ) { - return undefined; - } - - const rows = await this.db.read( - 'SELECT id FROM apps WHERE uid = ? LIMIT 1', - [uid], - ); - const mysqlId = Number(rows?.[0]?.id); - if ( Number.isInteger(mysqlId) && mysqlId > 0 ) { - return mysqlId; - } - - return undefined; - }, - - async claim_app_ownership_by_id_for_user_ ({ appId, userId }) { - if ( !Number.isInteger(appId) || appId <= 0 ) return; - if ( !Number.isInteger(userId) || userId <= 0 ) return; - - await this.db.write( - 'UPDATE apps SET owner_user_id = ? WHERE id = ? AND owner_user_id IS NULL', - [userId, appId], - ); - }, - - build_canonical_app_uid_alias_key_ (oldAppUid) { - return `${APP_UID_ALIAS_KEY_PREFIX}:${oldAppUid}`; - }, - - build_canonical_app_uid_alias_reverse_key_ (canonicalAppUid) { - return `${APP_UID_ALIAS_REVERSE_KEY_PREFIX}:${canonicalAppUid}`; - }, - - normalize_canonical_alias_uid_list_ (value) { - if ( ! Array.isArray(value) ) return []; - const normalizedList = []; - const seen = new Set(); - for ( const item of value ) { - if ( typeof item !== 'string' || !item ) continue; - if ( seen.has(item) ) continue; - seen.add(item); - normalizedList.push(item); - } - return normalizedList; - }, - - async read_canonical_app_uid_alias_ (oldAppUid) { - if ( typeof oldAppUid !== 'string' || !oldAppUid ) return null; - - const services = this.context.get('services'); - const kvStore = services.get('puter-kvstore'); - const suService = services.get('su'); - if ( !kvStore || typeof kvStore.get !== 'function' ) return null; - if ( !suService || typeof suService.sudo !== 'function' ) return null; - - const key = this.build_canonical_app_uid_alias_key_(oldAppUid); - try { - const canonicalAppUid = await suService.sudo(() => kvStore.get({ key })); - if ( typeof canonicalAppUid === 'string' && canonicalAppUid ) { - return canonicalAppUid; - } - } catch { - // Alias reads are best-effort. - } - return null; - }, - - async write_canonical_app_uid_alias_ ({ oldAppUid, canonicalAppUid }) { - if ( typeof oldAppUid !== 'string' || !oldAppUid ) return; - if ( typeof canonicalAppUid !== 'string' || !canonicalAppUid ) return; - if ( oldAppUid === canonicalAppUid ) return; - - const services = this.context.get('services'); - const kvStore = services.get('puter-kvstore'); - const suService = services.get('su'); - if ( !kvStore || typeof kvStore.set !== 'function' ) return; - if ( !suService || typeof suService.sudo !== 'function' ) return; - - const key = this.build_canonical_app_uid_alias_key_(oldAppUid); - const reverseKey = this.build_canonical_app_uid_alias_reverse_key_(canonicalAppUid); - const expireAt = Math.floor(Date.now() / 1000) + APP_UID_ALIAS_TTL_SECONDS; - try { - await suService.sudo(async () => { - const reverseValue = await kvStore.get({ key: reverseKey }); - const reverseAliases = this.normalize_canonical_alias_uid_list_(reverseValue); - if ( ! reverseAliases.includes(oldAppUid) ) { - reverseAliases.push(oldAppUid); - } - - await kvStore.set({ - key, - value: canonicalAppUid, - expireAt, - }); - await kvStore.set({ - key: reverseKey, - value: reverseAliases, - expireAt, - }); - }); - } catch { - // Alias writes are best-effort. - } - }, - - async maybe_join_owned_hosted_index_url_app_on_create_ (entity, extra, user) { - if ( ! user ) return; - - const new_index_url = await entity.get('index_url'); - const source_entity = extra.old_entity; - const currentMysqlId = await this.resolve_entity_mysql_id_(extra.old_entity); - const conflictRow = await this.find_index_url_conflict_({ - indexUrl: new_index_url, - excludeMysqlId: currentMysqlId, - }); - if ( ! conflictRow ) return; - - const conflictOwnerUserId = Number(conflictRow.owner_user_id); - if ( - Number.isInteger(conflictOwnerUserId) - && conflictOwnerUserId > 0 - && conflictOwnerUserId !== user.id - ) { - throw APIError.create('app_index_url_already_in_use', null, { - index_url: new_index_url, - app_uid: conflictRow.uid, - }); - } - - if ( !Number.isInteger(conflictOwnerUserId) || conflictOwnerUserId <= 0 ) { - await this.claim_app_ownership_by_id_for_user_({ - appId: conflictRow.id, - userId: user.id, - }); - } - - const old_entity = await this.upstream.read(conflictRow.uid); - const owner = await old_entity?.get('owner'); - let ownerUserId = owner?.id ?? owner; - if ( owner instanceof Entity ) { - ownerUserId = owner.private_meta.mysql_id; - } - ownerUserId = Number(ownerUserId); - if ( !old_entity || !Number.isInteger(ownerUserId) || ownerUserId !== user.id ) { - throw APIError.create('app_index_url_already_in_use', null, { - index_url: new_index_url, - app_uid: conflictRow.uid, - }); - } - if ( - Number.isInteger(conflictOwnerUserId) - && conflictOwnerUserId === user.id - && !await this.is_origin_bootstrap_app_entity_(old_entity) - ) { - // Prevent merging arbitrary same-owner apps; only allow the - // auto-created origin bootstrap app to be absorbed. - throw APIError.create('app_index_url_already_in_use', null, { - index_url: new_index_url, - app_uid: conflictRow.uid, - }); - } - - if ( source_entity ) { - const sourceUid = await source_entity.get('uid'); - const targetUid = await old_entity.get('uid'); - const requestedName = await entity.get('name'); - - if ( - sourceUid - && targetUid - && sourceUid !== targetUid - && requestedName !== undefined - ) { - entity.del('name'); - if ( typeof requestedName === 'string' && requestedName.trim() ) { - extra.joined_requested_name = requestedName.trim(); - } - } - - if ( sourceUid && targetUid && sourceUid !== targetUid ) { - extra.joined_source_app_uid = sourceUid; - } - } - - await entity.set('uid', await old_entity.get('uid')); - extra.old_entity = old_entity; - }, - - async apply_joined_requested_name_ ({ canonicalUid, requestedName }) { - if ( typeof canonicalUid !== 'string' || !canonicalUid ) return null; - if ( typeof requestedName !== 'string' || !requestedName.trim() ) return null; - const normalizedName = requestedName.trim(); - - const currentRows = await this.db.read( - 'SELECT name FROM apps WHERE uid = ? LIMIT 1', - [canonicalUid], - ); - const currentName = currentRows?.[0]?.name; - if ( typeof currentName !== 'string' ) return null; - if ( currentName === normalizedName ) return null; - - const conflictRows = await this.db.read( - 'SELECT uid FROM apps WHERE name = ? AND uid != ? LIMIT 1', - [normalizedName, canonicalUid], - ); - if ( conflictRows.length > 0 ) { - throw APIError.create('app_name_already_in_use', null, { - name: normalizedName, - }); - } - - await this.db.write( - 'UPDATE apps SET name = ? WHERE uid = ? LIMIT 1', - [normalizedName, canonicalUid], - ); - - return { - oldName: currentName, - newName: normalizedName, - }; - }, - - async is_origin_bootstrap_app_entity_ (entity) { - if ( ! entity ) return false; - const uid = await entity.get('uid'); - if ( typeof uid !== 'string' || !uid ) return false; - if ( await entity.get('name') !== uid ) return false; - if ( await entity.get('title') !== uid ) return false; - const description = await entity.get('description'); - if ( typeof description !== 'string' ) return false; - return description.startsWith('App created from origin '); - }, - - async ensureIndexUrlUnique (entity, extra) { - const new_index_url = await entity.get('index_url'); - if ( ! new_index_url ) return; - if ( ! this.is_puter_hosted_index_url_(new_index_url) ) return; - - if ( extra.old_entity ) { - const old_index_url = await extra.old_entity.get('index_url'); - if ( old_index_url === new_index_url ) { - return; - } - } - - const currentMysqlId = await this.resolve_entity_mysql_id_(extra.old_entity); - const conflictRow = await this.find_index_url_conflict_({ - indexUrl: new_index_url, - excludeMysqlId: currentMysqlId, - }); - if ( conflictRow ) { - throw APIError.create('app_index_url_already_in_use', null, { - index_url: new_index_url, - app_uid: conflictRow.uid, - }); - } - }, - }; -} - -module.exports = AppES; diff --git a/src/backend/src/om/entitystorage/AppLimitedES.js b/src/backend/src/om/entitystorage/AppLimitedES.js deleted file mode 100644 index ff18507b2..000000000 --- a/src/backend/src/om/entitystorage/AppLimitedES.js +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const { AppUnderUserActorType } = require('../../services/auth/Actor'); -const { PermissionUtil } = require('../../services/auth/permissionUtils.mjs'); -const { Context } = require('../../util/context'); -const { Eq, Or } = require('../query/query'); -const { BaseES } = require('./BaseES'); -const { Entity } = require('./Entity'); - -class AppLimitedES extends BaseES { - - // #region read operations - - // Limit selection to entities owned by the app of the current actor. - async select (options) { - const actor = Context.get('actor'); - - app_under_user_check: - if ( actor.type instanceof AppUnderUserActorType ) { - const svc_permission = Context.get('services').get('permission'); - const perm = PermissionUtil.join(this.permission_prefix, actor.type.user.uuid, 'read'); - const can_read_any = await svc_permission.check(actor, perm); - - if ( can_read_any ) break app_under_user_check; - - if ( this.exception && typeof this.exception === 'function' ) { - this.exception = await this.exception(); - } - - let condition = new Eq({ - key: 'app_owner', - value: actor.type.app, - }); - if ( this.exception ) { - condition = new Or({ - children: [ - condition, - this.exception, - ], - }); - } - options.predicate = options.predicate.and(condition); - } - - return await this.upstream.select(options); - } - - // Limit read to entities owned by the app of the current actor. - async read (uid) { - const entity = await this.upstream.read(uid); - if ( ! entity ) return null; - - const actor = Context.get('actor'); - - if ( actor.type instanceof AppUnderUserActorType ) { - if ( this.exception && typeof this.exception === 'function' ) { - this.exception = await this.exception(); - } - - // On the exception, we don't have to check app_owner - // (for `es:apps` this is `approved_for_listing == 1`) - if ( this.exception && await entity.check(this.exception) ) { - return entity; - } - - const app = actor.type.app; - const app_owner = await entity.get('app_owner'); - let app_owner_id = app_owner?.id; - if ( app_owner instanceof Entity ) { - app_owner_id = app_owner.private_meta.mysql_id; - } - if ( ( !app_owner ) || app_owner_id !== app.id ) { - return null; - } - } - - return entity; - } - - // #endregion - - // #region write operations - - // Limit edit to entities owned by the app of the current actor - async upsert (entity, extra) { - const actor = Context.get('actor'); - if ( actor.type instanceof AppUnderUserActorType ) { - const { old_entity } = extra; - if ( old_entity ) { - await this._check_edit_allowed({ old_entity }); - } - } - return await this.upstream.upsert(entity, extra); - } - async delete (uid, extra) { - const actor = Context.get('actor'); - if ( actor.type instanceof AppUnderUserActorType ) { - const { old_entity } = extra; - await this._check_edit_allowed({ old_entity }); - } - return await this.upstream.delete(uid, extra); - } - async _check_edit_allowed ({ old_entity }) { - const actor = Context.get('actor'); - - // Maybe the app has been granted write access to all the user's apps - // (in which case we return early) - { - const svc_permission = Context.get('services').get('permission'); - const perm = PermissionUtil.join(this.permission_prefix, actor.type.user.uuid, 'write'); - const can_write_any = await svc_permission.check(actor, perm); - if ( can_write_any ) return; - } - - // Otherwise, verify the app owner - // (or we throw an APIError) - { - const app = actor.type.app; - const app_owner = await old_entity.get('app_owner'); - let app_owner_id = app_owner?.id; - if ( app_owner instanceof Entity ) { - app_owner_id = app_owner.private_meta.mysql_id; - } - if ( ( !app_owner ) || app_owner_id !== app.id ) { - throw APIError.create('forbidden'); - } - } - } - // #endregion -} - -module.exports = { - AppLimitedES, -}; diff --git a/src/backend/src/om/entitystorage/BaseES.js b/src/backend/src/om/entitystorage/BaseES.js deleted file mode 100644 index d6e5270c1..000000000 --- a/src/backend/src/om/entitystorage/BaseES.js +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('@heyputer/putility'); -const { WeakConstructorFeature } = require('../../traits/WeakConstructorFeature'); - -/** - * BaseES is a base class for Entity Store classes. - */ -class BaseES extends AdvancedBase { - static FEATURES = [ - new WeakConstructorFeature(), - ]; - - // Default implementations - static METHODS = { - async upsert (entity, extra) { - if ( ! this.upstream ) { - throw Error('Missing terminal operation'); - } - return await this.upstream.upsert(entity, extra); - }, - async read (uid) { - if ( ! this.upstream ) { - throw Error('Missing terminal operation'); - } - return await this.upstream.read(uid); - }, - async delete (uid, extra) { - if ( ! this.upstream ) { - throw Error('Missing terminal operation'); - } - return await this.upstream.delete(uid, extra); - }, - async select (options) { - if ( ! this.upstream ) { - throw Error('Missing terminal operation'); - } - return await this.upstream.select(options); - }, - async create_predicate (id, ...args) { - if ( ! this.upstream ) { - throw Error('Missing terminal operation'); - } - return await this.upstream.create_predicate(id, ...args); - }, - }; - - constructor (...a) { - super(...a); - - const public_wrappers = [ - 'upsert', 'read', 'delete', 'select', - 'read_transform', - 'retry_predicate_rewrite', - ]; - - this.impl_methods = this._get_merged_static_object('METHODS'); - - for ( const k in this.impl_methods ) { - // Some methods are part of the implicit EntityStorage interface. - // We won't let the implementor override these; instead we - // provide a delegating implementation where they override a - // lower-level method of the same name. - if ( public_wrappers.includes(k) ) continue; - - this[k] = this.impl_methods[k]; - } - } - - async provide_context ( args ) { - for ( const k in args ) this[k] = args[k]; - if ( this.upstream ) { - await this.upstream.provide_context(args); - } - if ( this._on_context_provided ) { - await this._on_context_provided(args); - } - } - async read (uid) { - let entity = await this.call_on_impl_('read', uid); - if ( ! entity ) { - const retry_predicate = await this.retry_predicate_rewrite(uid); - if ( retry_predicate ) { - entity = await this.call_on_impl_('read', - { predicate: retry_predicate }); - } - } - if ( ! this.impl_methods.read_transform ) return entity; - return await this.read_transform(entity); - } - async upsert (entity, extra) { - return await this.call_on_impl_('upsert', entity, extra ?? {}); - } - async delete (uid, extra) { - return await this.call_on_impl_('delete', uid, extra ?? {}); - } - - async select (options) { - - const results = await this.call_on_impl_('select', options); - if ( ! this.impl_methods.read_transform ) return results; - - // Promises "solved callback hell" but like... - return await Promise.all(results.map(async entity => { - return await this.read_transform(entity); - })); - } - - async retry_predicate_rewrite ({ predicate }) { - if ( ! this.impl_methods.retry_predicate_rewrite ) return; - return await this.call_on_impl_('retry_predicate_rewrite', { predicate }); - } - - async read_transform (entity) { - if ( ! entity ) return entity; - if ( ! this.impl_methods.read_transform ) return entity; - const maybe_entity = await this.call_on_impl_('read_transform', entity); - if ( ! maybe_entity ) return entity; - return maybe_entity; - } - - call_on_impl_ (method_name, ...args) { - // const pseudo_this = { ...this }; - // pseudo_this.next = this.upstream?.call_on_impl?.bind(this.upstream, method_name); - return this.impl_methods[method_name].call(this, ...args); - } -} - -module.exports = { - BaseES, -}; diff --git a/src/backend/src/om/entitystorage/ESBuilder.js b/src/backend/src/om/entitystorage/ESBuilder.js deleted file mode 100644 index 3179b0148..000000000 --- a/src/backend/src/om/entitystorage/ESBuilder.js +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -class ESBuilder { - static create (list) { - let stack = []; - let head = null; - const apply_next = () => { - const args = []; - let last_was_cons = false; - while ( !last_was_cons ) { - const item = stack.pop(); - if ( typeof item === 'function' ) { - last_was_cons = true; - } - args.unshift(item); - } - - const cls = args.shift(); - head = new cls({ - ...(args[0] ?? {}), - ...(head ? { upstream: head } : {}), - }); - }; - for ( const item of list ) { - const is_cons = typeof item === 'function'; - - if ( is_cons ) { - if ( stack.length > 0 ) apply_next(); - } - - stack.push(item); - } - - if ( stack.length > 0 ) apply_next(); - - // Print the classes in order - let current = head; - while ( current ) { - current = current.upstream; - } - - return head; - } -} - -module.exports = { - ESBuilder, -}; diff --git a/src/backend/src/om/entitystorage/Entity.js b/src/backend/src/om/entitystorage/Entity.js deleted file mode 100644 index e79000ef8..000000000 --- a/src/backend/src/om/entitystorage/Entity.js +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('@heyputer/putility'); -const { WeakConstructorFeature } = require('../../traits/WeakConstructorFeature'); - -class Entity extends AdvancedBase { - static FEATURES = [ - new WeakConstructorFeature(), - ]; - - constructor (args) { - super(args); - this.init_arg_keys_ = Object.keys(args); - - this.found = undefined; - this.private_meta = {}; - - this.values_ = {}; - } - - static async create (args, data) { - const entity = new Entity(args); - - for ( const prop of Object.values(args.om.properties) ) { - if ( ! data.hasOwnProperty(prop.name) ) continue; - - await entity.set(prop.name, data[prop.name]); - } - - return entity; - } - - async clone () { - const args = {}; - for ( const k of this.init_arg_keys_ ) { - args[k] = this[k]; - } - const entity = new Entity(args); - - const BEHAVIOUR = 'A'; - - if ( BEHAVIOUR === 'A' ) { - entity.found = this.found; - entity.private_meta = { ...this.private_meta }; - entity.values_ = { ...this.values_ }; - } - if ( BEHAVIOUR === 'B' ) { - for ( const prop of Object.values(this.om.properties) ) { - if ( ! this.has(prop.name) ) continue; - - await entity.set(prop.name, await this.get(prop.name)); - } - } - - return entity; - } - - async apply (other) { - for ( const prop of Object.values(this.om.properties) ) { - if ( ! await other.has(prop.name) ) continue; - await this.set(prop.name, await other.get(prop.name)); - } - - return this; - } - - async set (key, value) { - const prop = this.om.properties[key]; - if ( ! prop ) { - throw Error(`property ${key} unrecognized`); - } - this.values_[key] = await prop.adapt(value); - } - - async get (key) { - const prop = this.om.properties[key]; - if ( ! prop ) { - throw Error(`property ${key} unrecognized`); - } - let value = this.values_[key]; - let is_set = await prop.is_set(value); - - // If value is not set but we have a factory, use it. - if ( ! is_set ) { - value = await prop.factory(); - value = await prop.adapt(value); - is_set = await prop.is_set(value); - if ( is_set ) this.values_[key] = value; - } - - // If value is not set but we have an implicator, use it. - if ( !is_set && prop.descriptor.imply ) { - const { given, make } = prop.descriptor.imply; - let imply_available = true; - for ( const g of given ) { - if ( ! await this.has(g) ) { - imply_available = false; - break; - } - } - if ( imply_available ) { - value = await make(this.values_); - value = await prop.adapt(value); - is_set = await prop.is_set(value); - } - if ( is_set ) this.values_[key] = value; - } - - return value; - } - - async del (key) { - const prop = this.om.properties[key]; - if ( ! prop ) { - throw Error(`property ${key} unrecognized`); - } - delete this.values_[key]; - } - - async has (key) { - const prop = this.om.properties[key]; - if ( ! prop ) { - throw Error(`property ${key} unrecognized`); - } - return await prop.is_set(await this.get(key)); - } - - async check (condition) { - return await condition.check(this); - } - - om_has_property (key) { - return this.om.properties.hasOwnProperty(key); - } - - // alias for `has` - async is_set (key) { - return await this.has(key); - } - - async get_client_safe () { - return await this.om.get_client_safe(this.values_); - } -} - -module.exports = { - Entity, -}; diff --git a/src/backend/src/om/entitystorage/MaxLimitES.js b/src/backend/src/om/entitystorage/MaxLimitES.js deleted file mode 100644 index 4e71f5ef0..000000000 --- a/src/backend/src/om/entitystorage/MaxLimitES.js +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { BaseES } = require('./BaseES'); - -class MaxLimitES extends BaseES { - static METHODS = { - async select (options) { - let limit = options.limit; - - // `limit` is numeric but a value of 0 doesn't make sense, - // so we can treat 0 and undefined as the same case. - if ( ! limit ) { - limit = this.max; - } - - if ( limit > this.max ) { - limit = this.max; - } - - options.limit = limit; - - return await this.upstream.select(options); - }, - }; -} - -module.exports = { - MaxLimitES, -}; diff --git a/src/backend/src/om/entitystorage/NotificationES.js b/src/backend/src/om/entitystorage/NotificationES.js deleted file mode 100644 index 9b9bc639f..000000000 --- a/src/backend/src/om/entitystorage/NotificationES.js +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { Eq, IsNotNull } = require('../query/query'); -const { BaseES } = require('./BaseES'); - -class NotificationES extends BaseES { - static METHODS = { - async create_predicate (id) { - if ( id === 'unseen' ) { - return new Eq({ - key: 'shown', - value: null, - }).and(new Eq({ - key: 'acknowledge', - value: null, - })); - } - if ( id === 'unacknowledge' ) { - return new Eq({ - key: 'acknowledge', - value: null, - }); - } - if ( id === 'acknowledge' ) { - return new IsNotNull({ - key: 'acknowledge', - }); - } - }, - async read_transform (entity) { - let value = await entity.get('value'); - if ( typeof value === 'string' ) { - value = JSON.parse(value); - } - if ( ! value ) { - value = {}; - } - await entity.set('value', value); - }, - }; -} - -module.exports = { NotificationES }; \ No newline at end of file diff --git a/src/backend/src/om/entitystorage/OwnerLimitedES.js b/src/backend/src/om/entitystorage/OwnerLimitedES.js deleted file mode 100644 index a1125f316..000000000 --- a/src/backend/src/om/entitystorage/OwnerLimitedES.js +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { UserActorType } = require('../../services/auth/Actor'); -const { Context } = require('../../util/context'); -const { Eq } = require('../query/query'); -const { BaseES } = require('./BaseES'); - -class OwnerLimitedES extends BaseES { - // Limit selection to entities owned by the app of the current actor. - async select (options) { - const actor = Context.get('actor'); - - if ( ! (actor.type instanceof UserActorType) ) { - return []; - } - - let condition = new Eq({ - key: 'owner', - value: actor.type.user.id, - }); - - options.predicate = options.predicate?.and - ? options.predicate.and(condition) - : condition; - - return await this.upstream.select(options); - } - - // Limit read to entities owned by the app of the current actor. - async read (uid) { - const actor = Context.get('actor'); - if ( ! (actor.type instanceof UserActorType) ) { - return null; - } - - const entity = await this.upstream.read(uid); - if ( ! entity ) return null; - - const entity_owner = await entity.get('owner'); - let owner_id = entity_owner?.id; - if ( entity_owner.id !== actor.type.user.id ) { - return null; - } - - return entity; - } -} - -module.exports = { - OwnerLimitedES, -}; diff --git a/src/backend/src/om/entitystorage/ProtectedAppES.js b/src/backend/src/om/entitystorage/ProtectedAppES.js deleted file mode 100644 index 03206ad33..000000000 --- a/src/backend/src/om/entitystorage/ProtectedAppES.js +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AppUnderUserActorType, UserActorType } = require('../../services/auth/Actor'); -const { PermissionUtil } = require('../../services/auth/permissionUtils.mjs'); -const { Context } = require('../../util/context'); -const { BaseES } = require('./BaseES'); - -class ProtectedAppES extends BaseES { - async select (options) { - const results = await this.upstream.select(options); - - const actor = Context.get('actor'); - const services = Context.get('services'); - - for ( let i = 0 ; i < results.length ; i++ ) { - const entity = results[i]; - - if ( ! await this.check_({ actor, services }, entity) ) { - continue; - } - results[i] = undefined; - } - - return results.filter(e => e !== undefined); - } - - async read (uid) { - const entity = await this.upstream.read(uid); - if ( ! entity ) return null; - - const actor = Context.get('actor'); - const services = Context.get('services'); - - if ( await this.check_({ actor, services }, entity) ) { - return null; - } - - return entity; - } - - /** - * returns true if the entity should not be sent downstream - */ - async check_ ({ actor, services }, entity) { - // track: ruleset - { - // if it's not a protected app, no worries - if ( ! await entity.get('protected') ) return; - - // if actor is this app, no worries - if ( - actor.type instanceof AppUnderUserActorType && - await entity.get('uid') === actor.type.app.uid - ) return; - - // if actor is owner of this app, no worries - if ( - actor.type instanceof UserActorType && - (await entity.get('owner')).id === actor.type.user.id - ) return; - } - - // now we need to check for permission - const app_uid = await entity.get('uid'); - const svc_permission = services.get('permission'); - const permission_to_check = `app:uid#${app_uid}:access`; - const reading = await svc_permission.scan(actor, permission_to_check); - const options = PermissionUtil.reading_to_options(reading); - - if ( options.length > 0 ) return; - - // `true` here means "do not send downstream" - return true; - } -}; - -module.exports = { - ProtectedAppES, -}; diff --git a/src/backend/src/om/entitystorage/ReadOnlyES.js b/src/backend/src/om/entitystorage/ReadOnlyES.js deleted file mode 100644 index 952bde665..000000000 --- a/src/backend/src/om/entitystorage/ReadOnlyES.js +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const { BaseES } = require('./BaseES'); - -class ReadOnlyES extends BaseES { - async upsert () { - throw APIError.create('forbidden'); - } - async delete () { - throw APIError.create('forbidden'); - } -} - -module.exports = ReadOnlyES; diff --git a/src/backend/src/om/entitystorage/SQLES.js b/src/backend/src/om/entitystorage/SQLES.js deleted file mode 100644 index de9c14a2f..000000000 --- a/src/backend/src/om/entitystorage/SQLES.js +++ /dev/null @@ -1,427 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('@heyputer/putility'); -const { BaseES } = require('./BaseES'); -const APIError = require('../../api/APIError'); -const { Entity } = require('./Entity'); -const { WeakConstructorFeature } = require('../../traits/WeakConstructorFeature'); -const { And, Or, Eq, Like, Null, Predicate, PredicateUtil, IsNotNull, StartsWith } = require('../query/query'); -const { DB_WRITE } = require('../../services/database/consts'); -const { safeHasOwnProperty } = require('../../util/safety'); -const { ParallelTasks } = require('../../util/otelutil'); -const opentelemetry = require('@opentelemetry/api'); - -class RawCondition extends AdvancedBase { - // properties: sql:string, values:any[] - static FEATURES = [ - new WeakConstructorFeature(), - ]; -} - -class SQLES extends BaseES { - async _on_context_provided () { - const services = this.context.get('services'); - this.db = services.get('database').get(DB_WRITE, 'entity-storage'); - } - static METHODS = { - async create_predicate (id, args) { - if ( id === 'raw-sql-condition' ) { - return new RawCondition(args); - } - }, - async read (uid) { - - const [stmt_where, where_vals] = await (async () => { - if ( typeof uid !== 'object' ) { - const id_prop = - this.om.properties[this.om.primary_identifier]; - let id_col = - id_prop.descriptor.sql?.column_name ?? id_prop.name; - // Temporary hack until multiple identifiers are supported - // (allows us to query using an internal ID; users can't do this) - if ( typeof uid === 'number' ) { - id_col = 'id'; - } - return [` WHERE ${id_col} = ?`, [uid]]; - } - - if ( ! Object.prototype.hasOwnProperty.call(uid, 'predicate') ) { - throw new Error('SQLES.read does not understand this input: ' + - 'object with no predicate property'); - } - let predicate = uid.predicate; // uid is actually a predicate - if ( predicate instanceof Predicate ) { - predicate = await this.om_to_sql_condition_(predicate); - } - const stmt_where = ` WHERE ${predicate.sql} LIMIT 1` ; - const where_vals = predicate.values; - return [stmt_where, where_vals]; - })(); - - const stmt = - `SELECT * FROM ${this.om.sql.table_name}${stmt_where}`; - - const rows = await this.db.read(stmt, where_vals); - - if ( rows.length === 0 ) { - return null; - } - - const data = rows[0]; - const entity = await this.sql_row_to_entity_(data); - - return entity; - }, - - async select ({ predicate, limit, offset }) { - if ( predicate instanceof Predicate ) { - predicate = await this.om_to_sql_condition_(predicate); - } - - const stmt_where = predicate ? ` WHERE ${predicate.sql}` : ''; - - let stmt = - `SELECT * FROM ${this.om.sql.table_name}${stmt_where}`; - - if ( offset !== undefined && limit === undefined ) { - throw new Error('Cannot use offset without limit'); - } - - if ( limit ) { - stmt += ` LIMIT ${limit}`; - } - if ( offset ) { - stmt += ` OFFSET ${offset}`; - } - - const values = []; - if ( predicate ) values.push(...(predicate.values || [])); - - const rows = await this.db.read(stmt, values); - - const entities = await Promise.all(rows.map(async (data) => { - return await this.sql_row_to_entity_(data); - })); - return entities; - }, - - async upsert (entity, extra) { - const { old_entity } = extra; - - // Check unique constraints - for ( const prop of Object.values(this.om.properties) ) { - const options = prop.descriptor.sql ?? {}; - if ( ! prop.descriptor.unique ) continue; - - const col_name = options.column_name ?? prop.name; - const value = await entity.get(prop.name); - - const values = []; - let stmt = - `SELECT COUNT(*) FROM ${this.om.sql.table_name} WHERE ${col_name} = ?`; - values.push(value); - - if ( old_entity ) { - stmt += ' AND id != ?'; - values.push(old_entity.private_meta.mysql_id); - } - - const rows = await this.db.read(stmt, values); - const count = rows[0]['COUNT(*)']; - - if ( count > 0 ) { - throw APIError.create('already_in_use', null, { - what: prop.name, - value, - }); - } - } - - // Update or create - if ( old_entity ) { - const result = await this.update_(entity, old_entity); - result.insert_id = old_entity.private_meta.mysql_id; - return result; - } else { - return await this.create_(entity); - } - }, - - async delete (uid) { - const id_prop = this.om.properties[this.om.primary_identifier]; - let id_col = - id_prop.descriptor.sql?.column_name ?? id_prop.name; - - const stmt = - `DELETE FROM ${this.om.sql.table_name} WHERE ${id_col} = ?`; - - const res = await this.db.write(stmt, [uid]); - - if ( ! res.anyRowsAffected ) { - throw APIError.create('entity_not_found', null, { - 'identifier': uid, - }); - } - - return { - data: {}, - }; - }, - - async sql_row_to_entity_ (data) { - const entity_data = {}; - const tasks = new ParallelTasks({ tracer: opentelemetry.trace.getTracer('sqles') }); - for ( const prop of Object.values(this.om.properties) ) { - const options = prop.descriptor.sql ?? {}; - - if ( options.ignore ) { - continue; - } - - const col_name = options.column_name ?? prop.name; - - if ( ! safeHasOwnProperty(data, col_name) ) { - continue; - } - - let value = data[col_name]; - tasks.add(`sql_row_to_entity_::${prop.name}`, async () => { - value = await prop.sql_dereference(value); - if ( prop.typ.name === 'json' ) { - if ( !value || typeof (value) === 'string' ) { - value = JSON.parse(value || '{}'); - } - } - entity_data[prop.name] = value; - }); - } - await tasks.awaitAll(); - const entity = await Entity.create({ om: this.om }, entity_data); - entity.private_meta.mysql_id = data.id; - return entity; - }, - - async create_ (entity) { - const sql_data = await this.get_sql_data_(entity); - - const sql_cols = Object.keys(sql_data).join(', '); - const sql_placeholders = Object.keys(sql_data).map(() => '?').join(', '); - const execute_vals = Object.values(sql_data); - - const stmt = - `INSERT INTO ${this.om.sql.table_name} (${sql_cols}) VALUES (${sql_placeholders})`; - - // Very useful when debugging! Keep these here but commented out. - // console.log('SQL STMT', stmt); - // console.log('SQL VALS', execute_vals); - - const res = await this.db.write(stmt, execute_vals); - - return { - data: sql_data, - entity, - insert_id: res.insertId, - }; - }, - async update_ (entity, old_entity) { - const sql_data = await this.get_sql_data_(entity); - const id_value = await entity.get(this.om.primary_identifier); - delete sql_data[this.om.primary_identifier]; - - const sql_assignments = Object.keys(sql_data).map((col_name) => { - return `${col_name} = ?`; - }).join(', '); - const execute_vals = Object.values(sql_data); - - const id_prop = this.om.properties[this.om.primary_identifier]; - const id_col = - id_prop.descriptor.sql?.column_name ?? id_prop.name; - - const stmt = - `UPDATE ${this.om.sql.table_name} SET ${sql_assignments} WHERE ${id_col} = ?`; - - execute_vals.push(id_value); - - // Very useful when debugging! Keep these here but commented out. - // console.log('SQL STMT', stmt); - // console.log('SQL VALS', execute_vals); - - await this.db.write(stmt, execute_vals); - - const full_entity = await (await old_entity.clone()).apply(entity); - - return { - data: sql_data, - entity: full_entity, - }; - }, - - async get_sql_data_ (entity) { - const sql_data = {}; - - for ( const prop of Object.values(this.om.properties) ) { - const options = prop.descriptor.sql ?? {}; - - if ( ! await entity.has(prop.name) ) { - continue; - } - - if ( options.ignore ) { - continue; - } - - const col_name = options.column_name ?? prop.name; - let value = await entity.get(prop.name); - if ( value === undefined ) { - continue; - } - - value = await prop.sql_reference(value); - - // TODO: This is done here for consistency; - // see the larger comment in sql_row_to_entity_ - // which does the reverse operation. - if ( prop.typ.name === 'json' ) { - value = JSON.stringify(value); - } - - if ( value && options.use_id ) { - if ( Object.prototype.hasOwnProperty.call(value, 'id') ) { - value = value.id; - } - } - - sql_data[col_name] = value; - } - - return sql_data; - }, - - async om_to_sql_condition_ (om_query) { - om_query = PredicateUtil.simplify(om_query); - - if ( om_query instanceof Null ) { - return undefined; - } - - if ( om_query instanceof And ) { - const child_raw_conditions = []; - const values = []; - for ( const child of om_query.children ) { - // if ( child instanceof Null ) continue; - const sql_condition = await this.om_to_sql_condition_(child); - child_raw_conditions.push(sql_condition.sql); - values.push(...(sql_condition.values || [])); - } - - const sql = child_raw_conditions.map((sql) => { - return `(${sql})`; - }).join(' AND '); - - return new RawCondition({ sql, values }); - } - - if ( om_query instanceof Or ) { - const child_raw_conditions = []; - const values = []; - for ( const child of om_query.children ) { - // if ( child instanceof Null ) continue; - const sql_condition = await this.om_to_sql_condition_(child); - child_raw_conditions.push(sql_condition.sql); - values.push(...(sql_condition.values || [])); - } - - const sql = child_raw_conditions.map((sql) => { - return `(${sql})`; - }).join(' OR '); - - return new RawCondition({ sql, values }); - } - - if ( om_query instanceof Eq ) { - const key = om_query.key; - let value = om_query.value; - const prop = this.om.properties[key]; - - value = await prop.sql_reference(value); - - const options = prop.descriptor.sql ?? {}; - const col_name = options.column_name ?? prop.name; - - const sql = value === null ? `${col_name} IS NULL` : `${col_name} = ?`; - const values = value === null ? [] : [value]; - - return new RawCondition({ sql, values }); - } - - if ( om_query instanceof StartsWith ) { - const key = om_query.key; - let value = om_query.value; - const prop = this.om.properties[key]; - - value = await prop.sql_reference(value); - - const options = prop.descriptor.sql ?? {}; - const col_name = options.column_name ?? prop.name; - - const sql = `${col_name} LIKE ${this.db.case({ - sqlite: '? || \'%\'', - otherwise: 'CONCAT(?, \'%\')', - })}`; - const values = value === null ? [] : [value]; - - return new RawCondition({ sql, values }); - } - - if ( om_query instanceof IsNotNull ) { - const key = om_query.key; - let value = om_query.value; - const prop = this.om.properties[key]; - - value = await prop.sql_reference(value); - - const options = prop.descriptor.sql ?? {}; - const col_name = options.column_name ?? prop.name; - - const sql = `${col_name} IS NOT NULL`; - const values = [value]; - - return new RawCondition({ sql, values }); - } - - if ( om_query instanceof Like ) { - const key = om_query.key; - let value = om_query.value; - const prop = this.om.properties[key]; - - value = await prop.sql_reference(value); - - const options = prop.descriptor.sql ?? {}; - const col_name = options.column_name ?? prop.name; - - const sql = `${col_name} LIKE ?`; - const values = [value]; - - return new RawCondition({ sql, values }); - } - }, - }; -} - -module.exports = SQLES; diff --git a/src/backend/src/om/entitystorage/SetOwnerES.js b/src/backend/src/om/entitystorage/SetOwnerES.js deleted file mode 100644 index c1e8edc18..000000000 --- a/src/backend/src/om/entitystorage/SetOwnerES.js +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { get_user } = require('../../helpers'); -const { AppUnderUserActorType, UserActorType } = require('../../services/auth/Actor'); -const { Context } = require('../../util/context'); -const { BaseES } = require('./BaseES'); - -class SetOwnerES extends BaseES { - static METHODS = { - async upsert (entity, extra) { - const { old_entity } = extra; - if ( ! old_entity ) { - await entity.set('owner', Context.get('user')); - - if ( entity.om_has_property('app_owner') ) { - const actor = Context.get('actor'); - if ( actor.type instanceof AppUnderUserActorType ) { - const app = actor.type.app; - - // We need to escalate privileges to set the app owner - // because the app may not have permission to read - // its own entry from es:app. - const upgraded_actor = actor.get_related_actor(UserActorType); - await Context.get().sub({ - actor: upgraded_actor, - }).arun(async () => { - await entity.set('app_owner', app.uid); - }); - } - } - } - return await this.upstream.upsert(entity, extra); - }, - async read (uid) { - const entity = await this.upstream.read(uid); - if ( ! entity ) return null; - - await this._sanitize_owner(entity); - - return entity; - }, - async select (...args) { - const entities = await this.upstream.select(...args); - for ( const entity of entities ) { - await this._sanitize_owner(entity); - } - return entities; - }, - async _sanitize_owner (entity) { - let owner = await entity.get('owner'); - if ( ! owner ) return null; - owner = get_user({ id: owner }); - await entity.set('owner', owner); - }, - }; -} - -module.exports = { - SetOwnerES, -}; diff --git a/src/backend/src/om/entitystorage/SubdomainES.js b/src/backend/src/om/entitystorage/SubdomainES.js deleted file mode 100644 index 93d1ebfb3..000000000 --- a/src/backend/src/om/entitystorage/SubdomainES.js +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const config = require('../../config'); - -const { DB_READ } = require('../../services/database/consts'); -const { Context } = require('../../util/context'); -const { Eq } = require('../query/query'); -const { BaseES } = require('./BaseES'); - -const PERM_READ_ALL_SUBDOMAINS = 'read-all-subdomains'; - -class SubdomainES extends BaseES { - async _on_context_provided () { - const services = this.context.get('services'); - this.db = services.get('database').get(DB_READ, 'subdomains'); - } - async create_predicate (id) { - if ( id === 'user-can-edit' ) { - return new Eq({ - key: 'owner', - value: Context.get('user').id, - }); - } - } - async upsert (entity, extra) { - if ( ! extra.old_entity ) { - await this._check_max_subdomains(); - } - - return await this.upstream.upsert(entity, extra); - } - async select (options) { - const actor = Context.get('actor'); - const user = actor.type.user; - - // Note: we don't need to worry about read; - // non-owner users don't have permission to list - // but they still have permission to read. - const svc_permission = this.context.get('services').get('permission'); - const has_permission_to_read_all = await svc_permission.check(Context.get('actor'), PERM_READ_ALL_SUBDOMAINS); - - if ( ! has_permission_to_read_all ) { - options.predicate = options.predicate.and(new Eq({ - key: 'owner', - value: user.id, - })); - } - - return await this.upstream.select(options); - } - async _check_max_subdomains () { - const user = Context.get('user'); - - let cnt = await this.db.read('SELECT COUNT(id) AS subdomain_count FROM subdomains WHERE user_id = ?', - [user.id]); - - const max_subdomains = user.max_subdomains ?? config.max_subdomains_per_user; - - if ( max_subdomains && cnt[0].subdomain_count >= max_subdomains ) { - throw APIError.create('subdomain_limit_reached', null, { - limit: max_subdomains, - }); - } - }; -} - -module.exports = SubdomainES; \ No newline at end of file diff --git a/src/backend/src/om/entitystorage/ValidationES.js b/src/backend/src/om/entitystorage/ValidationES.js deleted file mode 100644 index 8e89b5858..000000000 --- a/src/backend/src/om/entitystorage/ValidationES.js +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { BaseES } = require('./BaseES'); - -const APIError = require('../../api/APIError'); -const { Context } = require('../../util/context'); -const { SKIP_ES_VALIDATION } = require('./consts'); - -class ValidationES extends BaseES { - async _on_context_provided () { - // const services = this.context.get('services'); - // const svc_mysql = services.get('mysql'); - // this.dbrw = svc_mysql.get(DB_MODE_WRITE, `es:${this.entity_name}:rw`); - // this.dbrr = svc_mysql.get(DB_MODE_WRITE, `es:${this.entity_name}:rr`); - } - static METHODS = { - // async create (entity) { - // await this.validate_(entity); - // return await this.om.get_client_safe((await this.upstream.create(entity)).data); - // }, - // async update (entity) { - // await this.validate_(entity); - // return await this.om.get_client_safe((await this.upstream.update(entity)).data); - // }, - async upsert (entity, extra) { - for ( const prop of Object.values(this.om.properties) ) { - if ( - prop.descriptor.protected || - prop.descriptor.read_only - ) { - await entity.del(prop.name); - } - } - - const valid_entity = extra.old_entity - ? await (await extra.old_entity.clone()).apply(entity) - : entity - ; - await this.validate_(valid_entity, - extra.old_entity ? entity : undefined); - const { entity: out_entity } = await this.upstream.upsert(entity, extra); - return await out_entity.get_client_safe(); - }, - async validate_ (entity, diff) { - if ( Context.get(SKIP_ES_VALIDATION) ) return; - - for ( const prop of Object.values(this.om.properties) ) { - let value = await entity.get(prop.name); - - if ( prop.descriptor.required ) { - if ( ! await entity.is_set(prop.name) ) { - throw APIError.create('field_missing', null, { key: prop.name }); - } - } - - if ( ! await entity.is_set(prop.name) ) continue; - - if ( prop.descriptor.immutable && diff && await diff.has(prop.name) ) { - throw APIError.create('field_immutable', null, { key: prop.name }); - } - - try { - const validation_result = await prop.validate(value); - if ( validation_result !== true ) { - throw validation_result || APIError.create('field_invalid', null, { key: prop.name }); - } - } catch ( e ) { - if ( ! (e instanceof APIError) ) { - // eslint-disable-next-line no-ex-assign - e = APIError.create('field_invalid', null, { - key: prop.name, - converted_from_another_error: true, - }); - } - throw e; - } - } - - }, - }; -} - -module.exports = ValidationES; diff --git a/src/backend/src/om/entitystorage/WriteByOwnerOnlyES.js b/src/backend/src/om/entitystorage/WriteByOwnerOnlyES.js deleted file mode 100644 index 6fa434ea9..000000000 --- a/src/backend/src/om/entitystorage/WriteByOwnerOnlyES.js +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const { Context } = require('../../util/context'); -const { BaseES } = require('./BaseES'); - -const WRITE_ALL_OWNER_ES = 'system:es:write-all-owners'; - -/** - * Entity storage layer that restricts write operations to entity owners only. - * Extends BaseES to add ownership-based access control for upsert and delete operations. - */ -class WriteByOwnerOnlyES extends BaseES { - /** - * Static methods object containing the access-controlled entity storage operations. - */ - static METHODS = { - /** - * Updates or inserts an entity after verifying ownership permissions. - * @param {Object} entity - The entity to upsert - * @param {Object} extra - Additional parameters including old_entity - * @returns {Promise} Result of the upstream upsert operation - */ - async upsert (entity, extra) { - const { old_entity } = extra; - - if ( old_entity ) { - await this._check_allowed({ old_entity }); - } - - return await this.upstream.upsert(entity, extra); - }, - - /** - * Deletes an entity after verifying the current user owns it. - * @param {string} uid - The unique identifier of the entity to delete - * @param {Object} extra - Additional parameters including old_entity - * @returns {Promise} Result of the upstream delete operation - */ - async delete (uid, extra) { - const { old_entity } = extra; - - // Owner check is required first - await this._check_allowed({ old_entity: extra.old_entity }); - return await this.upstream.delete(uid, extra); - }, - - /** - * Verifies that the current user has permission to modify the entity. - * Allows access if user has system-wide write permission or owns the entity. - * @param {Object} params - Parameters object - * @param {Object} params.old_entity - The existing entity to check ownership for - * @throws {APIError} Throws forbidden error if user lacks permission - */ - async _check_allowed ({ old_entity }) { - const svc_permission = this.context.get('services').get('permission'); - const has_permission_to_write_all = await svc_permission.check(Context.get('actor'), WRITE_ALL_OWNER_ES); - if ( has_permission_to_write_all ) { - return; - } - - const owner = await old_entity.get('owner'); - if ( ! owner ) { - throw APIError.create('forbidden'); - } - const user = Context.get('user'); - - if ( user.id !== owner.id ) { - throw APIError.create('forbidden'); - } - }, - - }; -} - -module.exports = WriteByOwnerOnlyES; diff --git a/src/backend/src/om/entitystorage/consts.js b/src/backend/src/om/entitystorage/consts.js deleted file mode 100644 index 1dade2a72..000000000 --- a/src/backend/src/om/entitystorage/consts.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports = { - SKIP_ES_VALIDATION: Symbol('SKIP_ES_VALIDATION'), -}; diff --git a/src/backend/src/om/mappings/__all__.js b/src/backend/src/om/mappings/__all__.js deleted file mode 100644 index bf4b37d2e..000000000 --- a/src/backend/src/om/mappings/__all__.js +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -module.exports = { - app: require('./app'), - subdomain: require('./subdomain'), - notification: require('./notification'), -}; diff --git a/src/backend/src/om/mappings/access-token.js b/src/backend/src/om/mappings/access-token.js deleted file mode 100644 index 28a9637ae..000000000 --- a/src/backend/src/om/mappings/access-token.js +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -module.exports = { - sql: { - table_name: 'access_token_permissions', - }, - primary_identifier: 'token', -}; \ No newline at end of file diff --git a/src/backend/src/om/mappings/app.js b/src/backend/src/om/mappings/app.js deleted file mode 100644 index bb5643988..000000000 --- a/src/backend/src/om/mappings/app.js +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const config = require('../../config'); - -module.exports = { - sql: { - table_name: 'apps', - }, - primary_identifier: 'uid', - redundant_identifiers: ['name'], - properties: { - // INHERENT - uid: { - type: 'puter-uuid', - prefix: 'app', - }, - - // DOMAIN - icon: 'image-base64', - name: { - type: 'string', - required: true, - maxlen: config.app_name_max_length, - regex: config.app_name_regex, - }, - title: { - type: 'string', - required: true, - maxlen: config.app_title_max_length, - }, - description: { - type: 'string', - // longest description in prod is currently 3444, - // so I've doubled that and rounded up - maxlen: 7000, - }, - metadata: { - type: 'json', - }, - maximize_on_start: 'flag', - background: 'flag', - subdomain: { - type: 'string', - transient: true, - factory: () => `app-${ require('uuid').v4()}`, - sql: { ignore: true }, - }, - index_url: { - type: 'url', - required: true, - maxlen: 3000, - imply: { - given: ['subdomain', 'source_directory'], - make: async ({ subdomain }) => { - return `${config.protocol }://${ subdomain }.puter.site`; - }, - }, - }, - source_directory: { - type: 'puter-node', - node_type: 'directory', - sql: { ignore: true }, - }, - created_at: { - type: 'datetime', - aliases: ['timestamp'], - sql: { - column_name: 'timestamp', - }, - }, - - filetype_associations: { - type: 'array', - of: 'string', - sql: { ignore: true }, - }, - - // DOMAIN :: CALCULATED - stats: { - type: 'json', - sql: { ignore: true }, - }, - privateAccess: { - type: 'json', - sql: { ignore: true }, - }, - created_from_origin: { - type: 'string', - sql: { ignore: true }, - }, - - // ACCESS - owner: { - type: 'reference', - to: 'user', - permissions: ['write'], // write = update,delete,create - permissible_subproperties: ['username', 'uuid'], - sql: { - use_id: true, - column_name: 'owner_user_id', - }, - }, - app_owner: { - type: 'reference', - service: 'es:app', - to: 'app', - sql: { use_id: true }, - }, - protected: { - type: 'flag', - }, - is_private: { - type: 'flag', - read_only: true, - }, - - // OPERATIONS - last_review: { - type: 'datetime', - protected: true, - }, - approved_for_listing: { - type: 'flag', - read_only: true, - }, - approved_for_opening_items: { - type: 'flag', - read_only: true, - }, - approved_for_incentive_program: { - type: 'flag', - read_only: true, - }, - - // SYSTEM - godmode: { - type: 'flag', - read_only: true, - }, - }, -}; diff --git a/src/backend/src/om/mappings/notification.js b/src/backend/src/om/mappings/notification.js deleted file mode 100644 index c204d6ed9..000000000 --- a/src/backend/src/om/mappings/notification.js +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -module.exports = { - sql: { - table_name: 'notification', - }, - primary_identifier: 'uid', - properties: { - uid: { type: 'uuid' }, - value: { type: 'json' }, - read: { type: 'flag' }, - owner: { - type: 'reference', - to: 'user', - permissions: ['read'], - permissible_subproperties: ['username', 'uuid'], - sql: { - use_id: true, - column_name: 'user_id', - }, - }, - }, -}; diff --git a/src/backend/src/om/mappings/subdomain.js b/src/backend/src/om/mappings/subdomain.js deleted file mode 100644 index 07bd34ce4..000000000 --- a/src/backend/src/om/mappings/subdomain.js +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const config = require('../../config'); - -module.exports = { - sql: { - table_name: 'subdomains', - }, - primary_identifier: 'uid', - redundant_identifiers: ['subdomain'], - properties: { - // INHERENT - uid: { - type: 'puter-uuid', - prefix: 'sd', - sql: { column_name: 'uuid' }, - }, - - // DOMAIN - subdomain: { - type: 'string', - required: true, - immutable: true, - unique: true, - maxlen: config.subdomain_max_length, - regex: config.subdomain_regex, - // TODO: can this 'adapt' be data instead? - async adapt (value) { - return value.toLowerCase(); - }, - async validate (value) { - if ( config.reserved_words.includes(value) ) { - return APIError.create('subdomain_reserved', null, { - subdomain: value, - }); - } - }, - }, - domain: { - type: 'string', - maxlen: 253, - - // It turns out validating domain names kind of sucks - // source: https://stackoverflow.com/questions/10306690 - regex: '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})$', - - // TODO: can this 'adapt' be data instead? - async adapt (value) { - if ( value !== null ) - { - return value.toLowerCase(); - } - return null; - }, - }, - root_dir: { - type: 'puter-node', - fs_permission: 'read', - sql: { - column_name: 'root_dir_id', - }, - }, - associated_app: { - type: 'reference', - service: 'es:app', - to: 'app', - sql: { - use_id: true, - column_name: 'associated_app_id', - }, - }, - created_at: { - type: 'datetime', - aliases: ['timestamp'], - sql: { - column_name: 'ts', - }, - }, - - // ACCESS - owner: { - type: 'reference', - to: 'user', - permissions: ['write'], - permissible_subproperties: ['username', 'uuid'], - sql: { - use_id: true, - column_name: 'user_id', - }, - }, - app_owner: { - type: 'reference', - service: 'es:app', - to: 'app', - sql: { use_id: true }, - }, - protected: { - type: 'flag', - }, - }, -}; diff --git a/src/backend/src/om/proptypes/__all__.js b/src/backend/src/om/proptypes/__all__.js deleted file mode 100644 index d30c74455..000000000 --- a/src/backend/src/om/proptypes/__all__.js +++ /dev/null @@ -1,472 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const config = require('../../config'); -const { NodeUIDSelector, NodeInternalIDSelector, NodePathSelector } = require('../../deprecated/filesystem/node/selectors'); -const { is_valid_uuid4, is_valid_uuid } = require('../../helpers'); -const validator = require('validator'); -const { Context } = require('../../util/context'); -const { is_valid_path } = require('../../deprecated/filesystem/validation'); -const FSNodeContext = require('../../deprecated/filesystem/FSNodeContext').default; -const { Entity } = require('../entitystorage/Entity'); -const { APP_ICONS_SUBDOMAIN } = require('../../consts/app-icons'); -const NULL = Symbol('NULL'); -const APP_ICON_ENDPOINT_PATH_REGEX = /^\/app-icon\/([^/?#]+)(?:\/(\d+))?\/?$/; -const LEGACY_APP_ICON_FILE_PATH_REGEX = /^\/(app-[^/?#]+?)(?:-(\d+))?\.png$/; -const ABSOLUTE_URL_REGEX = /^[a-zA-Z][a-zA-Z\d+\-.]*:/; -const RAW_BASE64_REGEX = /^[A-Za-z0-9+/]+={0,2}$/; -const isAbsoluteUrl = value => ABSOLUTE_URL_REGEX.test(value) || value.startsWith('//'); - -const isRawBase64ImageString = value => { - if ( typeof value !== 'string' ) return false; - const trimmed = value.trim(); - if ( !trimmed || trimmed.length < 16 ) return false; - if ( ! RAW_BASE64_REGEX.test(trimmed) ) return false; - if ( trimmed.length % 4 !== 0 ) return false; - - try { - const decoded = Buffer.from(trimmed, 'base64'); - if ( decoded.length === 0 ) return false; - const normalizedInput = trimmed.replace(/=+$/, ''); - const reencoded = decoded.toString('base64').replace(/=+$/, ''); - return normalizedInput === reencoded; - } catch { - return false; - } -}; - -const normalizeRawBase64ImageString = value => { - if ( typeof value !== 'string' ) return value; - const trimmed = value.trim(); - if ( ! isRawBase64ImageString(trimmed) ) return value; - return `data:image/png;base64,${trimmed}`; -}; - -const isStoredBase64AppIcon = ({ icon, icon_is_base64: iconIsBase64 }) => { - if ( typeof iconIsBase64 === 'boolean' ) return iconIsBase64; - if ( typeof iconIsBase64 === 'number' ) return iconIsBase64 !== 0; - if ( typeof iconIsBase64 === 'string' ) { - const normalized = iconIsBase64.toLowerCase(); - if ( normalized === '1' || normalized === 'true' ) return true; - if ( normalized === '0' || normalized === 'false' ) return false; - } - - if ( typeof icon !== 'string' ) return false; - const trimmed = icon.trim(); - if ( trimmed.startsWith('data:image/') ) return true; - return isRawBase64ImageString(trimmed); -}; - -const getCanonicalAppIconBaseUrl = () => { - const candidate = [config.api_base_url, config.origin] - .find(value => typeof value === 'string' && value.trim()); - if ( ! candidate ) return null; - try { - return (new URL(candidate)).origin; - } catch { - return null; - } -}; - -const normalizeAppUid = appUid => ( - typeof appUid === 'string' && appUid.startsWith('app-') - ? appUid - : `app-${appUid}` -); - -const parseAppIconEndpointPath = value => { - if ( typeof value !== 'string' ) return null; - const trimmed = value.trim(); - if ( ! trimmed ) return null; - try { - const match = new URL(trimmed, 'http://localhost').pathname.match(APP_ICON_ENDPOINT_PATH_REGEX); - if ( ! match ) return null; - return { - appUid: normalizeAppUid(match[1]), - }; - } catch { - return null; - } -}; - -const isAppIconEndpointPath = value => !!parseAppIconEndpointPath(value); - -const getAllowedAppIconOrigins = () => { - const origins = new Set(); - for ( const candidate of [config.api_base_url, config.origin] ) { - if ( typeof candidate !== 'string' || !candidate ) continue; - try { - origins.add((new URL(candidate)).origin); - } catch { - // Ignore invalid config values. - } - } - return origins; -}; - -const getAllowedLegacyAppIconHostnames = () => { - const hostnames = new Set(); - const domains = [config.static_hosting_domain, config.static_hosting_domain_alt]; - for ( const domain of domains ) { - if ( typeof domain !== 'string' || !domain.trim() ) continue; - hostnames.add(`${APP_ICONS_SUBDOMAIN}.${domain.trim().toLowerCase()}`); - } - return hostnames; -}; - -const isAllowedAppIconEndpointUrl = value => { - if ( ! isAppIconEndpointPath(value) ) return false; - - const trimmed = value.trim(); - if ( ! isAbsoluteUrl(trimmed) ) { - return true; - } - - try { - const parsed = new URL(trimmed, 'http://localhost'); - return getAllowedAppIconOrigins().has(parsed.origin); - } catch { - return false; - } -}; - -const parseLegacyHostedAppIconToEndpointPath = value => { - if ( typeof value !== 'string' ) return null; - const trimmed = value.trim(); - if ( !trimmed || trimmed.startsWith('data:') ) return null; - - let parsed; - try { - parsed = new URL(trimmed, 'http://localhost'); - } catch { - return null; - } - - if ( isAbsoluteUrl(trimmed) ) { - const allowedHostnames = getAllowedLegacyAppIconHostnames(); - const hostname = parsed.hostname.toLowerCase(); - if ( ! allowedHostnames.has(hostname) ) { - return null; - } - } - - const match = parsed.pathname.match(LEGACY_APP_ICON_FILE_PATH_REGEX); - if ( ! match ) return null; - - const appUid = normalizeAppUid(match[1]); - return `/app-icon/${appUid}`; -}; - -const migrateRelativeAppIconEndpointUrl = value => { - if ( typeof value !== 'string' ) return value; - const trimmed = value.trim(); - if ( ! trimmed ) return value; - - let canonicalEndpointPath = null; - const endpointPath = parseAppIconEndpointPath(trimmed); - if ( endpointPath ) { - if ( isAbsoluteUrl(trimmed) ) { - try { - const parsed = new URL(trimmed, 'http://localhost'); - if ( ! getAllowedAppIconOrigins().has(parsed.origin) ) { - return value; - } - } catch { - return value; - } - } - canonicalEndpointPath = `/app-icon/${endpointPath.appUid}`; - } else { - canonicalEndpointPath = parseLegacyHostedAppIconToEndpointPath(trimmed); - } - if ( ! canonicalEndpointPath ) return value; - - const baseUrl = getCanonicalAppIconBaseUrl(); - if ( ! baseUrl ) return canonicalEndpointPath; - - try { - return new URL(canonicalEndpointPath, `${baseUrl}/`).toString(); - } catch { - return canonicalEndpointPath; - } -}; - -class OMTypeError extends Error { - constructor ({ expected, got }) { - const message = `expected ${expected}, got ${got}`; - super(message); - this.name = 'OMTypeError'; - } -} - -module.exports = { - base: { - is_set (value) { - return !!value; - }, - }, - json: { - from: 'base', - }, - string: { - is_set (value) { - return (!!value) || value === null; - }, - async adapt (value) { - if ( value === undefined ) return ''; - - // SQL stores strings as null. If one-way adapt from db is supported - // then this should become an sql-to-entity adapt only. - if ( value === null ) return ''; - - if ( value === NULL ) { - return null; - } - - if ( typeof value !== 'string' ) { - throw new OMTypeError({ expected: 'string', got: typeof value }); - } - return value; - }, - validate (value, { name, descriptor }) { - if ( typeof value !== 'string' ) { - return new OMTypeError({ expected: 'string', got: typeof value }); - } - if ( Object.prototype.hasOwnProperty.call(descriptor, 'maxlen') && value.length > descriptor.maxlen ) { - throw APIError.create('field_too_long', null, { key: name, max_length: descriptor.maxlen }); - } - if ( Object.prototype.hasOwnProperty.call(descriptor, 'minlen') && value.length > descriptor.minlen ) { - throw APIError.create('field_too_short', null, { key: name, min_length: descriptor.maxlen }); - } - if ( Object.prototype.hasOwnProperty.call(descriptor, 'regex') && !value.match(descriptor.regex) ) { - return new Error(`string does not match regex ${descriptor.regex}`); - } - return true; - }, - }, - array: { - from: 'base', - validate (value, { name, descriptor }) { - if ( ! Array.isArray(value) ) { - return new OMTypeError({ expected: 'array', got: typeof value }); - } - if ( Object.prototype.hasOwnProperty.call(descriptor, 'maxlen') && value.length > descriptor.maxlen ) { - throw APIError.create('field_too_long', null, { key: name, max_length: descriptor.maxlen }); - } - if ( Object.prototype.hasOwnProperty.call(descriptor, 'minlen') && value.length > descriptor.minlen ) { - throw APIError.create('field_too_short', null, { key: name, min_length: descriptor.maxlen }); - } - if ( Object.prototype.hasOwnProperty.call(descriptor, 'mod') && value.length % descriptor.mod !== 0 ) { - throw APIError.create('field_invalid', null, { key: name, mod: descriptor.mod }); - } - return true; - }, - }, - flag: { - adapt: value => { - if ( value === undefined ) return false; - if ( value === 0 ) value = false; - if ( value === 1 ) value = true; - if ( value === '0' ) value = false; - if ( value === '1' ) value = true; - if ( typeof value !== 'boolean' ) { - throw new OMTypeError({ expected: 'boolean', got: typeof value }); - } - return value; - }, - }, - uuid: { - from: 'string', - validate (value) { - return is_valid_uuid4(value); - }, - }, - 'puter-uuid': { - from: 'string', - validate (value, { descriptor }) { - const prefix = `${descriptor.prefix }-`; - if ( ! value.startsWith(prefix) ) { - return new Error(`UUID does not start with prefix ${prefix}`); - } - return is_valid_uuid(value.slice(prefix.length)); - }, - factory ({ descriptor }) { - const prefix = `${descriptor.prefix }-`; - const uuid = require('uuid').v4(); - return prefix + uuid; - }, - }, - 'image-base64': { - from: 'string', - is_set (value) { - return typeof value === 'string' && value.trim().length > 0; - }, - adapt (value) { - if ( value === NULL ) return null; - if ( value === undefined || value === null ) return ''; - if ( typeof value !== 'string' ) { - throw new OMTypeError({ expected: 'string', got: typeof value }); - } - value = normalizeRawBase64ImageString(value); - if ( isStoredBase64AppIcon({ icon: value }) ) { - return value; - } - return migrateRelativeAppIconEndpointUrl(value); - }, - validate (value) { - if ( typeof value !== 'string' ) { - return new OMTypeError({ expected: 'string', got: typeof value }); - } - - const trimmed = value.trim(); - if ( ! trimmed ) { - return true; - } - - if ( isStoredBase64AppIcon({ icon: trimmed }) ) { - // XSS characters - const chars = ['<', '>', '&', '"', "'", '`']; - if ( chars.some(char => trimmed.includes(char)) ) { - return new Error('icon is not an image'); - } - return true; - } - - if ( isAllowedAppIconEndpointUrl(trimmed) ) { - return true; - } - - return new Error('icon must be base64 encoded or an app-icon endpoint URL'); - }, - }, - url: { - from: 'string', - validate (value) { - let valid = validator.isURL(value); - if ( ! valid ) { - valid = validator.isURL(value, { host_whitelist: ['localhost'] }); - } - return valid; - }, - }, - reference: { - from: 'base', - async sql_reference (value, { descriptor }) { - if ( ! descriptor.service ) return value; - if ( ! value ) return null; - if ( value instanceof Entity ) { - return value.private_meta.mysql_id; - } - return value.id; - }, - async sql_dereference (value, { descriptor }) { - if ( ! descriptor.service ) return value; - if ( ! value ) return null; - const svc = Context.get().get('services').get(descriptor.service); - const entity = await svc.read(value); - return entity; - }, - async adapt (value, { descriptor }) { - if ( ! descriptor.service ) return value; - if ( ! value ) return null; - if ( value instanceof Entity ) return value; - const svc = Context.get().get('services').get(descriptor.service); - const entity = await svc.read(value); - return entity; - }, - }, - datetime: { - from: 'base', - }, - 'puter-node': { - // from: 'base', - async sql_reference (value) { - if ( value === null ) return null; - if ( ! (value instanceof FSNodeContext) ) { - throw new Error('Cannot reference non-FSNodeContext'); - } - await value.fetchEntry(); - return value.mysql_id ?? null; - }, - async is_set (value) { - return ( !!value ) || value === null; - }, - async sql_dereference (value) { - if ( value === null ) return null; - if ( typeof value !== 'number' ) { - throw new Error(`Cannot dereference non-number: ${value}`); - } - const svc_fs = Context.get().get('services').get('filesystem'); - return svc_fs.node(new NodeInternalIDSelector('mysql', value)); - }, - async adapt (value, { name }) { - if ( value === null ) return null; - - if ( value instanceof FSNodeContext ) { - return value; - } - const ctx = Context.get(); - - if ( typeof value !== 'string' ) return; - - let selector; - if ( ! ['/', '.', '~'].includes(value[0]) ) { - if ( is_valid_uuid4(value) ) { - selector = new NodeUIDSelector(value); - } - } else { - if ( value.startsWith('~') ) { - const user = ctx.get('user'); - if ( ! user ) { - throw new Error('Cannot use ~ without a user'); - } - const homedir = `/${user.username}`; - value = homedir + value.slice(1); - } - - if ( ! is_valid_path(value) ) { - throw APIError.create('field_invalid', null, { - key: name, - expected: 'unix-style path or UUID', - }); - } - - selector = new NodePathSelector(value); - } - - const svc_fs = ctx.get('services').get('filesystem'); - const node = await svc_fs.node(selector); - return node; - }, - async validate (value, { descriptor }) { - if ( value === null ) return; - const actor = Context.get('actor'); - const permission = descriptor.fs_permission ?? 'see'; - - const svc_acl = Context.get('services').get('acl'); - if ( await value.get('path') === '/' ) { - return APIError.create('forbidden'); - } - if ( ! await svc_acl.check(actor, value, permission) ) { - return await svc_acl.get_safe_acl_error(actor, value, permission); - } - }, - }, - NULL, -}; diff --git a/src/backend/src/om/proptypes/__all__.test.js b/src/backend/src/om/proptypes/__all__.test.js deleted file mode 100644 index f86557350..000000000 --- a/src/backend/src/om/proptypes/__all__.test.js +++ /dev/null @@ -1,77 +0,0 @@ -import { beforeAll, describe, expect, it } from 'vitest'; - -const proptypes = require('./__all__'); -const config = require('../../config'); - -describe('OM image-base64 proptype', () => { - const validateIcon = proptypes['image-base64'].validate; - const adaptIcon = proptypes['image-base64'].adapt; - - beforeAll(() => { - config.origin = 'https://puter.localhost'; - config.api_base_url = 'https://api.puter.localhost'; - config.static_hosting_domain = 'puter.site'; - }); - - it('accepts data URL icons', () => { - expect(validateIcon('data:image/png;base64,abc123')).toBe(true); - }); - - it('accepts raw base64 icon strings', () => { - expect(validateIcon('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ')).toBe(true); - }); - - it('accepts absolute app-icon endpoint URLs', () => { - expect(validateIcon('https://api.puter.localhost/app-icon/app-uid-123/64')).toBe(true); - }); - - it('accepts absolute app-icon endpoint URLs without size', () => { - expect(validateIcon('https://api.puter.localhost/app-icon/app-uid-123')).toBe(true); - }); - - it('accepts relative app-icon endpoint paths', () => { - expect(validateIcon('/app-icon/app-uid-123/64')).toBe(true); - }); - - it('accepts relative app-icon endpoint paths without size', () => { - expect(validateIcon('/app-icon/app-uid-123')).toBe(true); - }); - - it('migrates relative app-icon endpoint paths to absolute URLs', () => { - expect(adaptIcon('/app-icon/app-uid-123/64')).toBe('https://api.puter.localhost/app-icon/app-uid-123'); - }); - - it('normalizes raw base64 icon strings to png data URLs', () => { - expect(adaptIcon('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ')) - .toBe('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ'); - }); - - it('migrates legacy app-icons host URLs to absolute app-icon endpoint URLs', () => { - expect(adaptIcon('https://puter-app-icons.puter.site/app-uid-123-64.png')) - .toBe('https://api.puter.localhost/app-icon/app-uid-123'); - }); - - it('treats empty icon as valid', () => { - expect(validateIcon('')).toBe(true); - }); - - it('adapts null icon to empty string', () => { - expect(adaptIcon(null)).toBe(''); - }); - - it('accepts relative app-icon endpoint paths with query params', () => { - expect(validateIcon('/app-icon/app-uid-123/64?v=123')).toBe(true); - }); - - it('rejects invalid icon values', () => { - expect(validateIcon('not-an-icon')).toBeInstanceOf(Error); - }); - - it('rejects object icon values', () => { - expect(validateIcon({ url: '/app-icon/app-uid-123/64' })).toBeInstanceOf(Error); - }); - - it('rejects foreign absolute app-icon endpoint URLs', () => { - expect(validateIcon('https://evil.example/app-icon/app-uid-123/64')).toBeInstanceOf(Error); - }); -}); diff --git a/src/backend/src/om/query/query.js b/src/backend/src/om/query/query.js deleted file mode 100644 index 82a515560..000000000 --- a/src/backend/src/om/query/query.js +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('@heyputer/putility'); -const { WeakConstructorFeature } = require('../../traits/WeakConstructorFeature'); - -class Predicate extends AdvancedBase { - static FEATURES = [ - new WeakConstructorFeature(), - ]; -} - -class Null extends Predicate { - // -} - -class And extends Predicate { - // -} - -class Or extends Predicate { - async check (entity) { - for ( const child of this.children ) { - if ( await entity.check(child) ) { - return true; - } - } - return false; - } -} - -class Eq extends Predicate { - async check (entity) { - return (await entity.get(this.key)) == this.value; - } -} - -class StartsWith extends Predicate { - async check (entity) { - return (await entity.get(this.key)).startsWith(this.value); - } -} - -class IsNotNull extends Predicate { - async check (entity) { - return (await entity.get(this.key)) !== null; - } -} - -class Like extends Predicate { - async check (entity) { - // Convert SQL LIKE pattern to RegExp - // TODO: Support escaping the pattern characters - const regex = new RegExp(this.value.replaceAll('%', '.*').replaceAll('_', '.'), 'i'); - return regex.test(await entity.get(this.key)); - } -} - -Predicate.prototype.and = function (other) { - return new And({ children: [this, other] }); -}; - -class PredicateUtil { - static simplify (predicate) { - if ( predicate instanceof And ) { - const simplified = []; - for ( const p of predicate.children ) { - const s = PredicateUtil.simplify(p); - if ( s instanceof And ) { - simplified.push(...s.children); - } else if ( ! (s instanceof Null) ) { - simplified.push(s); - } - } - if ( simplified.length === 0 ) { - return new Null(); - } - if ( simplified.length === 1 ) { - return simplified[0]; - } - return new And({ children: simplified }); - } - - if ( predicate instanceof Or ) { - const simplified = []; - for ( const p of predicate.children ) { - const s = PredicateUtil.simplify(p); - if ( s instanceof Or ) { - simplified.push(...s.children); - } else if ( ! (s instanceof Null) ) { - simplified.push(s); - } - } - if ( simplified.length === 0 ) { - return new Null(); - } - if ( simplified.length === 1 ) { - return simplified[0]; - } - return new Or({ children: simplified }); - } - - return predicate; - } - - static write_human_readable (predicate) { - if ( predicate instanceof Eq ) { - return `${predicate.key}=${predicate.value}`; - } - - if ( predicate instanceof And ) { - const parts = predicate.children.map(child => - PredicateUtil.write_human_readable(child)); - return parts.join(' and '); - } - - if ( predicate instanceof Or ) { - const parts = predicate.children.map(child => - PredicateUtil.write_human_readable(child)); - return parts.join(' or '); - } - - if ( predicate instanceof StartsWith ) { - return `${predicate.key} starts with "${predicate.value}"`; - } - - if ( predicate instanceof IsNotNull ) { - return `${predicate.key} is not null`; - } - - if ( predicate instanceof Like ) { - return `${predicate.key} like "${predicate.value}"`; - } - - if ( predicate instanceof Null ) { - return ''; - } - - return String(predicate); - } -} - -module.exports = { - Predicate, - PredicateUtil, - Null, - And, - Or, - Eq, - IsNotNull, - Like, - StartsWith, -}; diff --git a/src/backend/src/om/query/query.test.js b/src/backend/src/om/query/query.test.js deleted file mode 100644 index 9e0c6cd19..000000000 --- a/src/backend/src/om/query/query.test.js +++ /dev/null @@ -1,309 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -const { - Eq, - And, - Or, - Null, - IsNotNull, - Like, - StartsWith, - PredicateUtil, -} = require('./query'); - -describe('PredicateUtil', () => { - describe('write_human_readable', () => { - it('writes Eq predicate as key=value', () => { - const predicate = new Eq({ key: 'name', value: 'John' }); - const result = PredicateUtil.write_human_readable(predicate); - expect(result).toBe('name=John'); - }); - - it('writes And predicate with "and" separator', () => { - const predicate = new And({ - children: [ - new Eq({ key: 'name', value: 'John' }), - new Eq({ key: 'age', value: 25 }), - ], - }); - const result = PredicateUtil.write_human_readable(predicate); - expect(result).toBe('name=John and age=25'); - }); - - it('writes nested And predicates', () => { - const predicate = new And({ - children: [ - new Eq({ key: 'name', value: 'John' }), - new Eq({ key: 'age', value: 25 }), - new Eq({ key: 'city', value: 'NYC' }), - ], - }); - const result = PredicateUtil.write_human_readable(predicate); - expect(result).toBe('name=John and age=25 and city=NYC'); - }); - - it('writes Or predicate with "or" separator', () => { - const predicate = new Or({ - children: [ - new Eq({ key: 'status', value: 'active' }), - new Eq({ key: 'status', value: 'pending' }), - ], - }); - const result = PredicateUtil.write_human_readable(predicate); - expect(result).toBe('status=active or status=pending'); - }); - - it('writes StartsWith predicate', () => { - const predicate = new StartsWith({ key: 'email', value: 'admin' }); - const result = PredicateUtil.write_human_readable(predicate); - expect(result).toBe('email starts with "admin"'); - }); - - it('writes IsNotNull predicate', () => { - const predicate = new IsNotNull({ key: 'verified_at' }); - const result = PredicateUtil.write_human_readable(predicate); - expect(result).toBe('verified_at is not null'); - }); - - it('writes Like predicate', () => { - const predicate = new Like({ key: 'name', value: '%John%' }); - const result = PredicateUtil.write_human_readable(predicate); - expect(result).toBe('name like "%John%"'); - }); - - it('writes Null predicate as empty string', () => { - const predicate = new Null(); - const result = PredicateUtil.write_human_readable(predicate); - expect(result).toBe(''); - }); - - it('writes complex nested predicates', () => { - const predicate = new And({ - children: [ - new Eq({ key: 'status', value: 'active' }), - new Or({ - children: [ - new Eq({ key: 'role', value: 'admin' }), - new Eq({ key: 'role', value: 'moderator' }), - ], - }), - ], - }); - const result = PredicateUtil.write_human_readable(predicate); - expect(result).toBe('status=active and role=admin or role=moderator'); - }); - }); - - describe('simplify', () => { - it('simplifies nested And predicates', () => { - const predicate = new And({ - children: [ - new And({ - children: [ - new Eq({ key: 'a', value: 1 }), - new Eq({ key: 'b', value: 2 }), - ], - }), - new Eq({ key: 'c', value: 3 }), - ], - }); - const result = PredicateUtil.simplify(predicate); - expect(result).toBeInstanceOf(And); - expect(result.children.length).toBe(3); - expect(result.children[0]).toBeInstanceOf(Eq); - expect(result.children[1]).toBeInstanceOf(Eq); - expect(result.children[2]).toBeInstanceOf(Eq); - }); - - it('simplifies And with single child', () => { - const predicate = new And({ - children: [ - new Eq({ key: 'a', value: 1 }), - ], - }); - const result = PredicateUtil.simplify(predicate); - expect(result).toBeInstanceOf(Eq); - expect(result.key).toBe('a'); - }); - - it('simplifies And with Null children', () => { - const predicate = new And({ - children: [ - new Eq({ key: 'a', value: 1 }), - new Null(), - new Eq({ key: 'b', value: 2 }), - ], - }); - const result = PredicateUtil.simplify(predicate); - expect(result).toBeInstanceOf(And); - expect(result.children.length).toBe(2); - }); - - it('simplifies And with all Null children to Null', () => { - const predicate = new And({ - children: [ - new Null(), - new Null(), - ], - }); - const result = PredicateUtil.simplify(predicate); - expect(result).toBeInstanceOf(Null); - }); - - it('simplifies nested Or predicates', () => { - const predicate = new Or({ - children: [ - new Or({ - children: [ - new Eq({ key: 'a', value: 1 }), - new Eq({ key: 'b', value: 2 }), - ], - }), - new Eq({ key: 'c', value: 3 }), - ], - }); - const result = PredicateUtil.simplify(predicate); - expect(result).toBeInstanceOf(Or); - expect(result.children.length).toBe(3); - }); - - it('returns non-composite predicates unchanged', () => { - const predicate = new Eq({ key: 'a', value: 1 }); - const result = PredicateUtil.simplify(predicate); - expect(result).toBe(predicate); - }); - }); -}); - -describe('Predicate classes', () => { - describe('Eq', () => { - it('checks equality', async () => { - const predicate = new Eq({ key: 'status', value: 'active' }); - const entity = { - get: async (key) => key === 'status' ? 'active' : null, - }; - const result = await predicate.check(entity); - expect(result).toBe(true); - }); - - it('fails when not equal', async () => { - const predicate = new Eq({ key: 'status', value: 'active' }); - const entity = { - get: async (key) => key === 'status' ? 'inactive' : null, - }; - const result = await predicate.check(entity); - expect(result).toBe(false); - }); - }); - - describe('StartsWith', () => { - it('checks if string starts with value', async () => { - const predicate = new StartsWith({ key: 'email', value: 'admin' }); - const entity = { - get: async (key) => key === 'email' ? 'admin@example.com' : null, - }; - const result = await predicate.check(entity); - expect(result).toBe(true); - }); - - it('fails when string does not start with value', async () => { - const predicate = new StartsWith({ key: 'email', value: 'admin' }); - const entity = { - get: async (key) => key === 'email' ? 'user@example.com' : null, - }; - const result = await predicate.check(entity); - expect(result).toBe(false); - }); - }); - - describe('IsNotNull', () => { - it('checks if value is not null', async () => { - const predicate = new IsNotNull({ key: 'verified_at' }); - const entity = { - get: async (key) => key === 'verified_at' ? '2025-01-01' : null, - }; - const result = await predicate.check(entity); - expect(result).toBe(true); - }); - - it('fails when value is null', async () => { - const predicate = new IsNotNull({ key: 'verified_at' }); - const entity = { - get: async (key) => null, - }; - const result = await predicate.check(entity); - expect(result).toBe(false); - }); - }); - - describe('Like', () => { - it('matches pattern with wildcards', async () => { - const predicate = new Like({ key: 'name', value: '%John%' }); - const entity = { - get: async (key) => key === 'name' ? 'John Doe' : null, - }; - const result = await predicate.check(entity); - expect(result).toBe(true); - }); - - it('fails when pattern does not match', async () => { - const predicate = new Like({ key: 'name', value: '%Jane%' }); - const entity = { - get: async (key) => key === 'name' ? 'John Doe' : null, - }; - const result = await predicate.check(entity); - expect(result).toBe(false); - }); - - it('is case insensitive', async () => { - const predicate = new Like({ key: 'name', value: '%john%' }); - const entity = { - get: async (key) => key === 'name' ? 'JOHN DOE' : null, - }; - const result = await predicate.check(entity); - expect(result).toBe(true); - }); - }); - - describe('Or', () => { - it('returns true if any child matches', async () => { - const predicate = new Or({ - children: [ - new Eq({ key: 'status', value: 'active' }), - new Eq({ key: 'status', value: 'pending' }), - ], - }); - const entity = { - get: async (key) => key === 'status' ? 'pending' : null, - check: async (pred) => await pred.check(entity), - }; - const result = await predicate.check(entity); - expect(result).toBe(true); - }); - - it('returns false if no children match', async () => { - const predicate = new Or({ - children: [ - new Eq({ key: 'status', value: 'active' }), - new Eq({ key: 'status', value: 'pending' }), - ], - }); - const entity = { - get: async (key) => key === 'status' ? 'inactive' : null, - check: async (pred) => await pred.check(entity), - }; - const result = await predicate.check(entity); - expect(result).toBe(false); - }); - }); - - describe('Predicate.and', () => { - it('creates an And predicate', () => { - const pred1 = new Eq({ key: 'a', value: 1 }); - const pred2 = new Eq({ key: 'b', value: 2 }); - const result = pred1.and(pred2); - expect(result).toBeInstanceOf(And); - expect(result.children).toEqual([pred1, pred2]); - }); - }); -}); diff --git a/src/backend/src/polyfill/to-string-higher-radix.js b/src/backend/src/polyfill/to-string-higher-radix.js deleted file mode 100644 index 6144c1fdc..000000000 --- a/src/backend/src/polyfill/to-string-higher-radix.js +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -/** - * Polyfill written by Chat GPT that increases the highest suppored - * radix on Number.prototype.toString from 36 to 62. - */ -(function () { - const originalToString = Number.prototype.toString; - - const characters = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'; - const base = characters.length; // 62 - - Number.prototype.toString = function (radix) { - // Use the original toString for bases 36 or lower - if ( !radix || radix <= 36 ) { - return originalToString.call(this, radix); - } - - // Custom implementation for base 62 - let value = this; - let result = ''; - while ( value > 0 ) { - result = characters[value % base] + result; - value = Math.floor(value / base); - } - return result || '0'; - }; -})(); diff --git a/src/backend/src/routers/_default.js b/src/backend/src/routers/_default.js deleted file mode 100644 index 1f09d01b2..000000000 --- a/src/backend/src/routers/_default.js +++ /dev/null @@ -1,493 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const config = require('../config'); -const router = express.Router(); -const _path = require('path'); -const _fs = require('fs'); -const { Context } = require('../util/context'); -const { DB_READ } = require('../services/database/consts'); -const { PathBuilder } = require('../util/pathutil.js'); -const { jwt_auth, get_app, invalidate_cached_user } = require('../helpers'); - -// Helper function to safely handle metadata parsing -const parseMetadata = (metadata) => { - try { - // If metadata is null or undefined, return empty object - if ( ! metadata ) { - return {}; - } - - // If metadata is already an object, return it - if ( typeof metadata === 'object' && !Array.isArray(metadata) ) { - return metadata; - } - - // If metadata is a string, try to parse it - if ( typeof metadata === 'string' ) { - return JSON.parse(metadata); - } - - // If we get here, metadata is of an unexpected type - console.warn('Unexpected metadata type:', typeof metadata); - return {}; - } catch ( error ) { - console.error('Error parsing metadata:', error); - return {}; - } -}; - -// -----------------------------------------------------------------------// -// All other requests -// -----------------------------------------------------------------------// -router.all('*', async function (req, res, next) { - const authService = Context.get('services').get('auth'); - - const subdomain = req.hostname.slice(0, -1 * (config.domain.length + 1)); - let path = req.params[0] ? req.params[0] : 'index.html'; - let auth_user; - // TODO DS: we should just do this as a middleware for every request, and check all possible types of auth - try { - auth_user = (await jwt_auth(req, authService))?.user; - } - catch (e) { - // no-op - } - - // -------------------------------------- - // API - // -------------------------------------- - if ( subdomain === 'api' ) { - return next(); - } - // -------------------------------------- - // /puter.js/v1 must be accessible globally regardless of subdomain - // -------------------------------------- - else if ( path === '/puter.js/v1' || path === '/puter.js/v1/' ) { - return res.sendFile(_path.join(__dirname, config.defaultjs_asset_path, 'puter.js/v1.js'), function (err) { - if ( err && err.statusCode ) { - return res.status(err.statusCode).send('Error /puter.js'); - } - }); - } - else if ( path === '/puter.js/v2' || path === '/puter.js/v2/' ) { - return res.sendFile(_path.join(__dirname, config.defaultjs_asset_path, 'puter.js/v2.js'), function (err) { - if ( err && err.statusCode ) { - return res.status(err.statusCode).send('Error /puter.js'); - } - }); - } - // -------------------------------------- - // https://js.[domain]/v1/ - // -------------------------------------- - else if ( subdomain === 'js' ) { - if ( path === '/v1' || path === '/v1/' ) { - return res.sendFile(_path.join(__dirname, config.defaultjs_asset_path, 'puter.js/v1.js'), function (err) { - if ( err && err.statusCode ) { - return res.status(err.statusCode).send('Error /puter.js'); - } - }); - } - if ( path === '/v2' || path === '/v2/' ) { - return res.sendFile(_path.join(__dirname, config.defaultjs_asset_path, 'puter.js/v2.js'), function (err) { - if ( err && err.statusCode ) { - return res.status(err.statusCode).send('Error /puter.js'); - } - }); - } - if ( path === '/putility/v1' ) { - return res.sendFile(_path.join(__dirname, config.defaultjs_asset_path, 'putility.js/v1.js'), function (err) { - if ( err && err.statusCode ) { - return res.status(err.statusCode).send('Error /putility.js'); - } - }); - } - } - - const db = Context.get('services').get('database').get(DB_READ, 'default'); - - // -------------------------------------- - // POST to login/signup/logout - // -------------------------------------- - if ( subdomain === '' && req.method === 'POST' && - ( - path === '/login' || - path === '/signup' || - path === '/logout' || - path === '/send-pass-recovery-email' || - path === '/set-pass-using-token' - ) - ) { - return next(); - } - // -------------------------------------- - // No subdomain: either GUI or landing pages - // -------------------------------------- - else if ( subdomain === '' ) { - - if ( path === '/robots.txt' ) { - res.set('Content-Type', 'text/plain'); - let r = ''; - r += 'User-agent: AhrefsBot\nDisallow:/\n\n'; - r += 'User-agent: BLEXBot\nDisallow: /\n\n'; - r += 'User-agent: DotBot\nDisallow: /\n\n'; - r += 'User-agent: ia_archiver\nDisallow: /\n\n'; - r += 'User-agent: MJ12bot\nDisallow: /\n\n'; - r += 'User-agent: SearchmetricsBot\nDisallow: /\n\n'; - r += 'User-agent: SemrushBot\nDisallow: /\n\n'; - // sitemap - r += `\nSitemap: ${config.protocol}://${config.domain}/sitemap.xml\n`; - return res.send(r); - } - else if ( path === '/sitemap.xml' ) { - let h = ''; - h += ''; - h += ''; - - // docs - h += ''; - h += `${config.protocol}://docs.${config.domain}/`; - h += ''; - - // apps - // TODO: use service for app discovery - let apps = await db.read('SELECT * FROM apps WHERE approved_for_listing = 1'); - if ( apps.length > 0 ) { - for ( let i = 0; i < apps.length; i++ ) { - const app = apps[i]; - h += ''; - h += `${config.protocol}://${config.domain}/app/${app.name}`; - h += ''; - } - } - h += ''; - res.set('Content-Type', 'application/xml'); - return res.send(h); - } - else if ( path === '/unsubscribe' ) { - let h = ''; - if ( req.query.user_uuid === undefined ) - { - h += '

user_uuid is required

'; - } - else { - // modules - const { get_user } = require('../helpers'); - - // get user - const user = await get_user({ uuid: req.query.user_uuid }); - - // more validation - if ( ! user ) - { - h += '

User not found.

'; - } - else if ( user.unsubscribed === 1 ) - { - h += '

You are already unsubscribed.

'; - } - // mark user as confirmed - else { - await db.write( - 'UPDATE `user` SET `unsubscribed` = 1 WHERE id = ?', - [user.id], - ); - - invalidate_cached_user(user); - - // return results - h += '

Your have successfully unsubscribed from all emails.

'; - } - } - - h += ''; - res.send(h); - } - else if ( path === '/confirm-email-by-token' ) { - let h = ''; - if ( req.query.user_uuid === undefined ) - { - h += '

user_uuid is required

'; - } - else if ( req.query.token === undefined ) - { - h += '

token is required

'; - } - else { - // modules - const { get_user } = require('../helpers'); - - // get user - const user = await get_user({ uuid: req.query.user_uuid, force: true }); - - // more validation - if ( user === undefined || user === null || user === false ) - { - h += '

user not found.

'; - } - else if ( user.email_confirmed === 1 ) - { - h += '

Email already confirmed.

'; - } - else if ( user.email_confirm_token !== req.query.token ) - { - h += '

invalid token.

'; - } - // mark user as confirmed - else { - // This IIFE is here to return early on conditions, and - // avoid further nested branching. This is a temporary - // solution; next time this code should be refactored. - await (async () => { - const svc_cleanEmail = req.services.get('clean-email'); - const clean_email = svc_cleanEmail.clean(user.email); - // If other users have the same CONFIRMED email, display an error - const maybe_rows = await db.read( - `SELECT EXISTS( - SELECT 1 FROM user WHERE (email=? OR clean_email=?) - AND email_confirmed=1 - AND password IS NOT NULL - ) AS email_exists`, - [user.email, clean_email], - ); - if ( maybe_rows[0]?.email_exists ) { - // TODO: maybe display the username of that account - h += '

' + - 'This email was confirmed on a different account.

'; - return; - } - - // If other users have the same unconfirmed email, revoke it - await db.write( - 'UPDATE `user` SET `unconfirmed_change_email` = NULL, `change_email_confirm_token` = NULL WHERE `unconfirmed_change_email` = ?', - [user.email], - ); - - // update user - await db.write( - 'UPDATE `user` SET `email_confirmed` = 1, `requires_email_confirmation` = 0 WHERE id = ?', - [user.id], - ); - invalidate_cached_user(user); - - // send realtime success msg to client - const svc_socketio = req.services.get('socketio'); - svc_socketio.send({ room: user.id }, 'user.email_confirmed', {}); - - // return results - h += '

Your email has been successfully confirmed.

'; - - const svc_event = req.services.get('event'); - svc_event.emit('user.email-confirmed', { - user_uid: user.uuid, - email: user.email, - }); - })(); - } - } - - h += ''; - res.send(h); - } - // ------------------------ - // /assets/ - // ------------------------ - else if ( path.startsWith('/assets/') ) { - path = PathBuilder.resolve(path); - return res.sendFile(path, { root: `${__dirname }../../public` }, function (err) { - if ( err && err.statusCode ) { - return res.status(err.statusCode).send('Error /public/'); - } - }); - } - // ------------------------ - // GUI - // ------------------------ - else { - let app; - let canonical_url = config.origin + path; - let app_name, app_title, app_description, app_icon, app_social_media_image; - let launch_options = { - on_initialized: [], - }; - - // default title - app_title = config.title; - - // /action/ - if ( path.startsWith('/action/') || path.startsWith('/@') ) { - path = '/'; - } - // /settings - else if ( path.startsWith('/settings') ) { - path = '/'; - } - // /dashboard - else if ( path === '/dashboard' || path === '/dashboard/' ) { - path = '/'; - } - // /app/ - else if ( path.startsWith('/app/') ) { - app_name = path.replace('/app/', ''); - app = await get_app({ - follow_old_names: true, - name: app_name, - }); - - if ( app ) { - // parse app metadata if available - app.metadata = parseMetadata(app.metadata); - // set app attributes to be passed to the homepage service - app_title = app.title; - app_description = app.description; - app_icon = app.icon; - app_social_media_image = app.metadata?.social_image; - } - // 404 - Not found! - else if ( app_name ) { - app_title = app_name.charAt(0).toUpperCase() + app_name.slice(1); - res.status(404); - } - - path = '/'; - } - else if ( path.startsWith('/show/') ) { - const filepath = path.slice('/show'.length); - launch_options.on_initialized.push({ - $: 'window-call', - fn_name: 'launch_app', - args: [{ - name: 'explorer', - path: filepath, - }], - }); - path = '/'; - } - - // index.js - if ( path === '/' ) { - const svc_puterHomepage = Context.get('services').get('puter-homepage'); - return svc_puterHomepage.send({ req, res, auth_user }, { - title: app_title, - description: app_description || config.short_description, - short_description: app_description || config.short_description, - social_media_image: app_social_media_image || config.social_media_image, - company: 'Puter Technologies Inc.', - canonical_url: canonical_url, - icon: app_icon, - app: app, - }, launch_options); - } - - // /dist/... - else if ( path.startsWith('/dist/') || path.startsWith('/src/') ) { - path = PathBuilder.resolve(path); - return res.sendFile(path, { root: config.assets.gui }, function (err) { - if ( err && err.statusCode ) { - return res.status(err.statusCode).send('Error /gui/dist/'); - } - }); - } - - // All other paths - else { - path = PathBuilder.resolve(path); - return res.sendFile(path, { root: _path.join(config.assets.gui, 'src') }, function (err) { - if ( err && err.statusCode ) { - return res.status(err.statusCode).send('Error /gui/'); - } - }); - } - } - } - // -------------------------------------- - // Native Apps - // -------------------------------------- - else if ( subdomain === 'viewer' || subdomain === 'editor' || subdomain === 'about' || subdomain === 'docs' || - subdomain === 'player' || subdomain === 'pdf' || subdomain === 'code' || subdomain === 'markus' || - subdomain === 'draw' || subdomain === 'camera' || subdomain === 'recorder' || - subdomain === 'dev-center' || subdomain === 'developer' ) { - - let root = PathBuilder - .add(__dirname) - .add(config.defaultjs_asset_path, { allow_traversal: true }) - .add('apps').add(subdomain) - .build(); - const has_dist = ['docs', 'developer']; - if ( has_dist.includes(subdomain) ) { - root += '/dist'; - } - root = _path.normalize(root); - - path = _path.normalize(path); - const real_path = _path.normalize(_path.join(root, path)); - - // Determine if the path is a directory - // (necessary because otherwise res.sendFile() will HANG!) - try { - const is_dir = (await _fs.promises.stat(real_path)).isDirectory(); - if ( is_dir && !path.endsWith('/') ) { - // Redirect to directory (use 307 to avoid browser caching) - path += '/'; - let redirect_url = `${req.protocol }://${ req.get('host') }${path}`; - - // We need to add the query string to the redirect URL - if ( req.query ) { - const old_url = `${req.protocol }://${ req.get('host') }${req.originalUrl}`; - redirect_url += new URL(old_url).search; - } - - return res.redirect(307, redirect_url); - } - } catch (e) { - console.error(e); - return res.status(404).send('Not found'); - } - - try { - return res.sendFile(path, { root }, function (err) { - if ( err && err.statusCode ) { - return res.status(err.statusCode).send('Error /apps/'); - } - }); - } catch (e) { - console.error('error from sendFile', e); - return res.status(e.statusCode).send('Error /apps/'); - } - } - // -------------------------------------- - // WWW, redirect to root domain - // -------------------------------------- - else if ( subdomain === 'www' ) { - return res.redirect(config.origin); - } - //------------------------------------------ - // User-defined subdomains: *.puter.com - // redirect to static hosting domain *.puter.site - //------------------------------------------ - else { - if ( req.get('host').toLowerCase().endsWith(config.domain) ) { - return res.redirect(302, `${req.protocol }://${ req.get('host').replace(config.domain, config.static_hosting_domain) }${req.originalUrl}`); - // replace hostname with static hosting domain and redirect to the same path - } - } -}); - -module.exports.catchAllRouter = router; diff --git a/src/backend/src/routers/apps.js b/src/backend/src/routers/apps.js deleted file mode 100644 index af5fbcaf6..000000000 --- a/src/backend/src/routers/apps.js +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = new express.Router(); -const auth = require('../middleware/auth.js'); -const config = require('../config'); -const { get_apps, app_name_exists } = require('../helpers'); -const { DB_READ } = require('../services/database/consts.js'); -const subdomain = require('../middleware/subdomain.js'); -let privateLaunchAccessModulePromise; -const getPrivateLaunchAccessModule = async () => { - if ( ! privateLaunchAccessModulePromise ) { - privateLaunchAccessModulePromise = import('../modules/apps/privateLaunchAccess.js'); - } - return privateLaunchAccessModulePromise; -}; - -// -----------------------------------------------------------------------// -// GET /apps -// -----------------------------------------------------------------------// -router.get( - '/apps', - subdomain('api'), - auth, - express.json({ limit: '50mb' }), - async (req, res) => { - // /!\ open brace on end of previous line - - // check if user is verified - if ( (config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed ) - { - return res.status(400).send({ code: 'account_is_not_verified', message: 'Account is not verified' }); - } - - const db = req.services.get('database').get(DB_READ, 'apps'); - - let apps_res = await db.read( - 'SELECT * FROM apps WHERE owner_user_id = ? ORDER BY timestamp DESC', - [req.user.id], - ); - - const svc_appInformation = req.services.get('app-information'); - - let apps = []; - - if ( apps_res.length > 0 ) { - for ( let i = 0; i < apps_res.length; i++ ) { - // filetype associations - let ftassocs = await db.read( - 'SELECT * FROM app_filetype_association WHERE app_id = ?', - [apps_res[i].id], - ); - - let filetype_associations = []; - if ( ftassocs.length > 0 ) { - ftassocs.forEach(ftassoc => { - filetype_associations.push(ftassoc.type); - }); - } - - const stats = await svc_appInformation.get_stats(apps_res[i].uid); - - apps.push({ - uid: apps_res[i].uid, - name: apps_res[i].name, - description: apps_res[i].description, - title: apps_res[i].title, - icon: apps_res[i].icon, - index_url: apps_res[i].index_url, - godmode: apps_res[i].godmode, - background: apps_res[i].background, - maximize_on_start: apps_res[i].maximize_on_start, - filetype_associations: filetype_associations, - ...stats, - approved_for_incentive_program: apps_res[i].approved_for_incentive_program, - created_at: apps_res[i].timestamp, - }); - } - } - - return res.send(apps); - }, -); - -// -----------------------------------------------------------------------// -// GET /apps/nameAvailable?name= -// -----------------------------------------------------------------------// -router.get( - '/apps/nameAvailable', - subdomain('api'), - auth, - express.json({ limit: '50mb' }), - async (req, res) => { - const name = req.query.name; - - // check if user is verified - if ( (config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed ) - { - return res.status(400).send({ code: 'account_is_not_verified', message: 'Account is not verified' }); - } - - if ( typeof name !== 'string' ) { - return res.status(400).send({ - code: 'invalid_request', - message: 'name query parameter must be a string', - }); - } - - if ( name.length === 0 ) { - return res.status(400).send({ - code: 'invalid_request', - message: 'name query parameter is required', - }); - } - - if ( name.length > config.app_name_max_length || !config.app_name_regex.test(name) ) { - return res.status(400).send({ - code: 'invalid_request', - message: `name must match app naming rules (max length: ${config.app_name_max_length})`, - }); - } - - const exists = !!(await app_name_exists(name)); - return res.send({ - name, - available: !exists, - }); - }, -); - -// -----------------------------------------------------------------------// -// GET /apps/:name(s) -// -----------------------------------------------------------------------// -router.get( - '/apps/:name', - subdomain('api'), - auth, - express.json({ limit: '50mb' }), - async (req, res, next) => { - // /!\ open brace on end of previous line - - // check subdomain - if ( require('../helpers').subdomain(req) !== 'api' ) - { - next(); - } - - // check if user is verified - if ( (config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed ) - { - return res.status(400).send({ code: 'account_is_not_verified', message: 'Account is not verified' }); - } - - const { - getActorUserUid, - resolvePrivateLaunchAccess, - } = await getPrivateLaunchAccessModule(); - let app_names = req.params.name.split('|'); - const apps = await get_apps(app_names.map(name => ({ name }))); - const actorUserUid = getActorUserUid(req.actor) || req.user?.uuid || null; - const privateAccessDecisions = await Promise.all(apps.map(app => { - if ( ! app ) return Promise.resolve(null); - return resolvePrivateLaunchAccess({ - app, - services: req.services, - userUid: actorUserUid, - source: 'appsRoute', - args: req.query ?? {}, - }); - })); - - const final_obj = apps.map((app, index) => { - if ( ! app ) return null; - return { - uuid: app.uid, - name: app.name, - title: app.title, - icon: app.icon, - godmode: app.godmode, - background: app.background, - maximize_on_start: app.maximize_on_start, - index_url: app.index_url, - privateAccess: privateAccessDecisions[index] ?? { - hasAccess: true, - checkedBy: 'core/apps-route-default', - }, - }; - }).filter(Boolean); - - return res.send(final_obj); - }, -); - -module.exports = router; diff --git a/src/backend/src/routers/auth/app-uid-from-origin.js b/src/backend/src/routers/auth/app-uid-from-origin.js deleted file mode 100644 index 73bc8b287..000000000 --- a/src/backend/src/routers/auth/app-uid-from-origin.js +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const eggspress = require('../../api/eggspress'); -const { Context } = require('../../util/context'); - -module.exports = eggspress('/auth/app-uid-from-origin', { - subdomain: 'api', - auth2: true, - allowedMethods: ['POST', 'GET'], -}, async (req, res, next) => { - const x = Context.get(); - const svc_auth = x.get('services').get('auth'); - - const origin = req.body.origin || req.query.origin; - - if ( ! origin ) { - throw APIError.create('field_missing', null, { key: 'origin' }); - } - - res.json({ - uid: await svc_auth.app_uid_from_origin(origin), - }); -}); diff --git a/src/backend/src/routers/auth/check-app-acl.endpoint.js b/src/backend/src/routers/auth/check-app-acl.endpoint.js deleted file mode 100644 index e6b4de240..000000000 --- a/src/backend/src/routers/auth/check-app-acl.endpoint.js +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const APIError = require('../../api/APIError'); -const FSNodeParam = require('../../api/filesystem/FSNodeParam'); -const StringParam = require('../../api/filesystem/StringParam'); -const { get_app } = require('../../helpers'); -const configurable_auth = require('../../middleware/configurable_auth'); -const { Eq, Or } = require('../../om/query/query'); -const { UserActorType, Actor, AppUnderUserActorType } = require('../../services/auth/Actor'); -const { Context } = require('../../util/context'); - -module.exports = { - route: '/check-app-acl', - methods: ['POST'], - - // TODO: "alias" should be part of parameters somehow - alias: { - uid: 'subject', - path: 'subject', - }, - parameters: { - subject: new FSNodeParam('subject'), - mode: new StringParam('mode', { optional: true }), - - // TODO: There should be an "AppParam", but it feels wrong to include - // so many concerns into `src/api/filesystem` like that. This needs to - // be de-coupled somehow first. - app: new StringParam('app'), - }, - mw: [configurable_auth()], - handler: async (req, res) => { - const context = Context.get(); - const actor = req.actor; - - if ( ! (actor.type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - const subject = req.values.subject; - - const svc_acl = context.get('services').get('acl'); - if ( ! await svc_acl.check(actor, subject, 'see') ) { - throw APIError.create('subject_does_not_exist'); - } - - const es_app = context.get('services').get('es:app'); - const app = await es_app.read({ - predicate: new Or({ - children: [ - new Eq({ key: 'uid', value: req.values.app }), - new Eq({ key: 'name', value: req.values.app }), - ], - }), - }); - if ( ! app ) { - throw APIError.create('app_does_not_exist', null, { - identifier: req.values.app, - }); - } - - const app_actor = new Actor({ - type: new AppUnderUserActorType({ - user: actor.type.user, - // TODO: get legacy app object from entity instead of fetching again - app: await get_app({ uid: await app.get('uid') }), - }), - }); - - res.json({ - allowed: await svc_acl.check(app_actor, subject, - // If mode is not specified, check the HIGHEST mode, because this - // will grant the LEAST cases - req.values.mode ?? svc_acl.get_highest_mode()), - }); - }, -}; diff --git a/src/backend/src/routers/auth/check-app.js b/src/backend/src/routers/auth/check-app.js deleted file mode 100644 index 52104590b..000000000 --- a/src/backend/src/routers/auth/check-app.js +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const eggspress = require('../../api/eggspress'); -const { get_app } = require('../../helpers'); -const { UserActorType, Actor, AppUnderUserActorType } = require('../../services/auth/Actor'); -const { PermissionUtil } = require('../../services/auth/permissionUtils.mjs'); -const { Context } = require('../../util/context'); - -module.exports = eggspress('/auth/check-app', { - subdomain: 'api', - auth2: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - const x = Context.get(); - const svc_auth = x.get('services').get('auth'); - const svc_permission = x.get('services').get('permission'); - - // Only users can get user-app tokens - const actor = Context.get('actor'); - if ( ! (actor.type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - if ( req.body.app_uid === undefined && req.body.origin === undefined ) { - throw APIError.create('field_missing', null, { - // TODO: standardize a way to provide multiple options - key: 'app_uid or origin', - }); - } - - const app_uid = req.body.app_uid ?? - await svc_auth.app_uid_from_origin(req.body.origin); - - const app = await get_app({ uid: app_uid }); - if ( ! app ) { - throw APIError.create('app_does_not_exist', null, { - identifier: app_uid, - }); - } - - const user = actor.type.user; - - const app_actor = new Actor({ - user_uid: user.uuid, - app_uid, - type: new AppUnderUserActorType({ - user, - app, - }), - }); - - const reading = await svc_permission.scan(app_actor, 'flag:app-is-authenticated'); - const options = PermissionUtil.reading_to_options(reading); - const authenticated = options.length > 0; - - let token; - if ( authenticated ) token = await svc_auth.get_user_app_token(app_uid); - - res.json({ - ...(token ? { token } : {}), - app_uid: app_uid || - await svc_auth.app_uid_from_origin(req.body.origin), - authenticated, - }); -}); diff --git a/src/backend/src/routers/auth/check-permissions.js b/src/backend/src/routers/auth/check-permissions.js deleted file mode 100644 index 74b6937eb..000000000 --- a/src/backend/src/routers/auth/check-permissions.js +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const eggspress = require('../../api/eggspress'); -const { UserActorType } = require('../../services/auth/Actor'); -const { Context } = require('../../util/context'); -const APIError = require('../../api/APIError'); - -module.exports = eggspress('/auth/check-permissions', { - subdomain: 'api', - auth2: true, - allowedMethods: ['POST'], -}, async (req, res, _next) => { - const context = Context.get(); - /** @type {import('../../services/auth/PermissionService').PermissionService} */ - const permissionService = context.get('services').get('permission'); - - const permsToCheck = req.body.permissions; - - const actor = context.get('actor'); - - const permEntryPromises = [...new Set(permsToCheck)].map(async (perm) => { - try { - return [perm, permissionService.check(actor, perm)]; - } catch { - return [perm, false]; - } - }); - - const permEntries = Promise.all(permEntryPromises); - - res.json({ permissions: Object.fromEntries(await permEntries) }); -}); diff --git a/src/backend/src/routers/auth/configure-2fa.js b/src/backend/src/routers/auth/configure-2fa.js deleted file mode 100644 index f9c2d930e..000000000 --- a/src/backend/src/routers/auth/configure-2fa.js +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const eggspress = require('../../api/eggspress'); -const { get_user, invalidate_cached_user_by_id } = require('../../helpers'); -const { UserActorType } = require('../../services/auth/Actor'); -const { DB_WRITE } = require('../../services/database/consts'); -const { Context } = require('../../util/context'); - -module.exports = eggspress('/auth/configure-2fa/:action', { - subdomain: 'api', - auth2: true, - allowedMethods: ['POST'], -}, async (req, res) => { - const action = req.params.action; - const x = Context.get(); - - // Only users can configure 2FA - const actor = Context.get('actor'); - if ( ! (actor.type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - const actions = {}; - - const db = await x.get('services').get('database').get(DB_WRITE, '2fa'); - - actions.setup = async () => { - const user = await get_user({ id: req.user.id, force: true }); - - if ( user.otp_enabled ) { - throw APIError.create('2fa_already_enabled'); - } - - const svc_otp = x.get('services').get('otp'); - - // generate secret - const result = svc_otp.create_secret(user.username); - - // generate recovery codes - result.codes = []; - for ( let i = 0; i < 10; i++ ) { - result.codes.push(svc_otp.create_recovery_code()); - } - - const hashed_recovery_codes = result.codes.map(code => { - const crypto = require('crypto'); - const hash = crypto - .createHash('sha256') - .update(code) - .digest('base64') - // We're truncating the hash for easier storage, so we have 128 - // bits of entropy instead of 256. This is plenty for recovery - // codes, which have only 48 bits of entropy to begin with. - .slice(0, 22); - return hash; - }); - - // update user - await db.write( - 'UPDATE user SET otp_secret = ?, otp_recovery_codes = ? WHERE uuid = ?', - [result.secret, hashed_recovery_codes.join(','), user.uuid], - ); - req.user.otp_secret = result.secret; - req.user.otp_recovery_codes = hashed_recovery_codes.join(','); - user.otp_secret = result.secret; - user.otp_recovery_codes = hashed_recovery_codes.join(','); - invalidate_cached_user_by_id(req.user.id); - - return result; - }; - - // IMPORTANT: only use to verify the user's 2FA setup; - // this should never be used to verify the user's 2FA code - // for authentication purposes. - actions.test = async () => { - const user = await get_user({ id: req.user.id, force: true }); - const svc_otp = x.get('services').get('otp'); - const code = req.body.code; - const ok = svc_otp.verify(user.username, user.otp_secret, code); - return { ok }; - }; - - actions.enable = async () => { - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('enable-2fa') ) { - return res.status(429).send('Too many requests.'); - } - - const user = await get_user({ id: req.user.id, force: true }); - - if ( ! user.email_confirmed ) { - throw APIError.create('email_must_be_confirmed', null, { - action: 'enable 2FA', - }); - } - - // Verify that 2FA isn't already enabled - if ( user.otp_enabled ) { - throw APIError.create('2fa_already_enabled'); - } - - // Verify that TOTP secret was set (configuration step not skipped) - if ( ! user.otp_secret ) { - throw APIError.create('2fa_not_configured'); - } - - await db.write( - 'UPDATE user SET otp_enabled = 1 WHERE uuid = ?', - [user.uuid], - ); - invalidate_cached_user_by_id(req.user.id); - // update cached user - req.user.otp_enabled = 1; - - const svc_email = req.services.get('email'); - await svc_email.send_email({ email: user.email }, 'enabled_2fa', { - username: user.username, - }); - - return {}; - }; - - if ( ! actions[action] ) { - throw APIError.create('invalid_action', null, { action }); - } - - const result = await actions[action](); - - res.json(result); -}); diff --git a/src/backend/src/routers/auth/create-access-token.js b/src/backend/src/routers/auth/create-access-token.js deleted file mode 100644 index 9df6f3561..000000000 --- a/src/backend/src/routers/auth/create-access-token.js +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const eggspress = require('../../api/eggspress'); -const { Context } = require('../../util/context'); - -module.exports = eggspress('/auth/create-access-token', { - subdomain: 'api', - auth2: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - const x = Context.get(); - const svc_auth = x.get('services').get('auth'); - - const permissions = req.body.permissions || []; - - if ( permissions.length === 0 ) { - throw APIError.create('field_missing', null, { key: 'permissions' }); - } - - for ( let i = 0 ; i < permissions.length ; i++ ) { - let perm = permissions[i]; - if ( typeof perm === 'string' ) { - perm = permissions[i] = [perm]; - } - if ( ! Array.isArray(perm) ) { - throw APIError.create('field_invalid', null, { key: 'permissions' }); - } - if ( perm.length === 0 || perm.length > 2 ) { - throw APIError.create('field_invalid', null, { key: 'permissions' }); - } - if ( typeof perm[0] !== 'string' ) { - throw APIError.create('field_invalid', null, { key: 'permissions' }); - } - if ( perm.length === 2 && typeof perm[1] !== 'object' ) { - throw APIError.create('field_invalid', null, { key: 'permissions' }); - } - } - - const actor = Context.get('actor'); - - const options = { - ...(req.body.expiresIn ? { expiresIn: `${ req.body.expiresIn}` } : {}), - }; - - const token = await svc_auth.create_access_token(actor, permissions, options); - - res.json({ token }); -}); diff --git a/src/backend/src/routers/auth/get-user-app-token.js b/src/backend/src/routers/auth/get-user-app-token.js deleted file mode 100644 index d3bfb1fbe..000000000 --- a/src/backend/src/routers/auth/get-user-app-token.js +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const eggspress = require('../../api/eggspress'); -const { LLMkdir } = require('../../deprecated/filesystem/ll_operations/ll_mkdir'); -const { NodeUIDSelector, NodePathSelector } = require('../../deprecated/filesystem/node/selectors'); -const { NodeChildSelector } = require('../../deprecated/filesystem/node/selectors'); -const { get_app } = require('../../helpers'); -const { UserActorType } = require('../../services/auth/Actor'); -const { Context } = require('../../util/context'); - -module.exports = eggspress('/auth/get-user-app-token', { - subdomain: 'api', - auth2: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - const x = Context.get(); - const svc_auth = x.get('services').get('auth'); - - // Only users can get user-app tokens - const actor = Context.get('actor'); - if ( ! (actor.type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - if ( req.body.app_uid === undefined && req.body.origin === undefined ) { - throw APIError.create('field_missing', null, { - // TODO: standardize a way to provide multiple options - key: 'app_uid or origin', - }); - } - - const token = ( req.body.app_uid !== undefined ) - ? await svc_auth.get_user_app_token(req.body.app_uid) - : await svc_auth.get_user_app_token_from_origin(req.body.origin) - ; - - const app_uid = req.body.app_uid ?? - await svc_auth.app_uid_from_origin(req.body.origin); - - const app = await get_app({ uid: app_uid }); - if ( ! app ) { - throw APIError.create('app_does_not_exist', null, { - identifier: app_uid, - }); - } - - const svc_fs = x.get('services').get('filesystem'); - const appdata_dir_sel = actor.type.user.appdata_uuid - ? new NodeUIDSelector(actor.type.user.appdata_uuid) - : new NodePathSelector(`/${actor.type.user.username}/AppData`); - const appdata_app_dir_node = await svc_fs.node(new NodeChildSelector( - appdata_dir_sel, - app_uid, - )); - - if ( ! await appdata_app_dir_node.exists() ) { - const ll_mkdir = new LLMkdir(); - await ll_mkdir.run({ - thumbnail: app.icon, - parent: await svc_fs.node(appdata_dir_sel), - name: app_uid, - actor: actor, - }); - } - - const svc_permission = x.get('services').get('permission'); - svc_permission.grant_user_app_permission(actor, app_uid, 'flag:app-is-authenticated'); - - res.json({ - token, - app_uid: app_uid || - await svc_auth.app_uid_from_origin(req.body.origin), - }); -}); diff --git a/src/backend/src/routers/auth/grant-dev-app.js b/src/backend/src/routers/auth/grant-dev-app.js deleted file mode 100644 index df4aef776..000000000 --- a/src/backend/src/routers/auth/grant-dev-app.js +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const eggspress = require('../../api/eggspress'); -const { UserActorType } = require('../../services/auth/Actor'); -const { Context } = require('../../util/context'); -const { validate_fields } = require('../../util/validutil'); - -module.exports = eggspress('/auth/grant-dev-app', { - subdomain: 'api', - auth2: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - const x = Context.get(); - const svc_permission = x.get('services').get('permission'); - - // Only users can grant user-app permissions - const actor = Context.get('actor'); - if ( ! (actor.type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - if ( req.body.origin ) { - const svc_auth = x.get('services').get('auth'); - req.body.app_uid = await svc_auth.app_uid_from_origin(req.body.origin); - } - - validate_fields({ - app_uid: { type: 'string', optional: false }, - permission: { type: 'string', optional: false }, - extra: { type: 'object', optional: true }, - meta: { type: 'object', optional: true }, - }, req.body); - - await svc_permission.grant_dev_app_permission(actor, req.body.app_uid, req.body.permission, req.body.extra || {}, req.body.meta || {}); - - res.json({}); -}); diff --git a/src/backend/src/routers/auth/grant-user-app.js b/src/backend/src/routers/auth/grant-user-app.js deleted file mode 100644 index dd78561f7..000000000 --- a/src/backend/src/routers/auth/grant-user-app.js +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const eggspress = require('../../api/eggspress'); -const { UserActorType } = require('../../services/auth/Actor'); -const { Context } = require('../../util/context'); -const { validate_fields } = require('../../util/validutil'); - -module.exports = eggspress('/auth/grant-user-app', { - subdomain: 'api', - auth2: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - const x = Context.get(); - const svc_permission = x.get('services').get('permission'); - - // Only users can grant user-app permissions - const actor = Context.get('actor'); - if ( ! (actor.type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - if ( req.body.origin ) { - const svc_auth = x.get('services').get('auth'); - req.body.app_uid = await svc_auth.app_uid_from_origin(req.body.origin); - } - - validate_fields({ - app_uid: { type: 'string', optional: false }, - permission: { type: 'string', optional: false }, - extra: { type: 'object', optional: true }, - meta: { type: 'object', optional: true }, - }, req.body); - - await svc_permission.grant_user_app_permission(actor, req.body.app_uid, req.body.permission, req.body.extra || {}, req.body.meta || {}); - - res.json({}); -}); diff --git a/src/backend/src/routers/auth/grant-user-group.js b/src/backend/src/routers/auth/grant-user-group.js deleted file mode 100644 index 66218bb48..000000000 --- a/src/backend/src/routers/auth/grant-user-group.js +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const eggspress = require('../../api/eggspress'); -const { UserActorType } = require('../../services/auth/Actor'); -const { Context } = require('../../util/context'); -const { validate_fields } = require('../../util/validutil'); - -module.exports = eggspress('/auth/grant-user-group', { - subdomain: 'api', - auth2: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - const x = Context.get(); - const svc_permission = x.get('services').get('permission'); - - // Only users can grant user-group permissions - const actor = Context.get('actor'); - if ( ! (actor.type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - validate_fields({ - group_uid: { type: 'string', optional: false }, - permission: { type: 'string', optional: false }, - extra: { type: 'object', optional: true }, - meta: { type: 'object', optional: true }, - }, req.body); - - await svc_permission.grant_user_group_permission(actor, req.body.group_uid, req.body.permission, req.body.extra || {}, req.body.meta || {}); - - res.json({}); -}); diff --git a/src/backend/src/routers/auth/grant-user-user.js b/src/backend/src/routers/auth/grant-user-user.js deleted file mode 100644 index 90a9151c4..000000000 --- a/src/backend/src/routers/auth/grant-user-user.js +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const eggspress = require('../../api/eggspress'); -const { UserActorType } = require('../../services/auth/Actor'); -const { Context } = require('../../util/context'); -const { validate_fields } = require('../../util/validutil'); - -module.exports = eggspress('/auth/grant-user-user', { - subdomain: 'api', - auth2: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - const x = Context.get(); - const svc_permission = x.get('services').get('permission'); - - // Only users can grant user-user permissions - const actor = Context.get('actor'); - if ( ! (actor.type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - validate_fields({ - target_username: { type: 'string', optional: false }, - permission: { type: 'string', optional: false }, - extra: { type: 'object', optional: true }, - meta: { type: 'object', optional: true }, - }, req.body); - - await svc_permission.grant_user_user_permission(actor, req.body.target_username, req.body.permission, req.body.extra || {}, req.body.meta || {}); - - res.json({}); -}); diff --git a/src/backend/src/routers/auth/list-permissions.js b/src/backend/src/routers/auth/list-permissions.js deleted file mode 100644 index 87082fbc6..000000000 --- a/src/backend/src/routers/auth/list-permissions.js +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import eggspress from '../../api/eggspress.js'; -import { get_apps, get_user } from '../../helpers.js'; -import { UserActorType } from '../../services/auth/Actor.js'; -import { DB_READ } from '../../services/database/consts.js'; -import { Context } from '../../util/context.js'; -import { APIError } from '../../api/APIError.js'; - -export default eggspress('/auth/list-permissions', { - subdomain: 'api', - auth2: true, - allowedMethods: ['GET'], -}, async (_req, res, _next) => { - const x = Context.get(); - - const actor = x.get('actor'); - - // Apps cannot (currently) check permissions on behalf of users - if ( ! ( actor.type instanceof UserActorType ) ) { - throw APIError.create('forbidden'); - } - - const db = x.get('services').get('database').get(DB_READ, 'permissions'); - - const permissions = {}; - - { - permissions.myself_to_app = []; - - const rows = await db.read('SELECT * FROM `user_to_app_permissions` WHERE user_id=?', - [ actor.type.user.id ]); - const apps = await get_apps(rows.map(row => ({ id: row.app_id }))); - - for ( let i = 0; i < rows.length; i++ ) { - const row = rows[i]; - const app = apps[i]; - if ( ! app ) continue; - - delete app.id; - delete app.approved_for_listing; - delete app.approved_for_opening_items; - delete app.godmode; - delete app.owner_user_id; - - const permission = { - app, - permission: row.permission, - extra: row.extra, - }; - - permissions.myself_to_app.push(permission); - } - } - { - permissions.myself_to_user = []; - - const rows = await db.read('SELECT * FROM `user_to_user_permissions` WHERE issuer_user_id=?', - [ actor.type.user.id ]); - - for ( const row of rows ) { - const user = await get_user({ id: row.holder_user_id }); - - const permission = { - user: user.username, - permission: row.permission, - extra: row.extra, - }; - - permissions.myself_to_user.push(permission); - } - } - { - permissions.user_to_myself = []; - - const rows = await db.read('SELECT * FROM `user_to_user_permissions` WHERE holder_user_id=?', - [ actor.type.user.id ]); - - for ( const row of rows ) { - const user = await get_user({ id: row.issuer_user_id }); - - const permission = { - user: user.username, - permission: row.permission, - extra: row.extra, - }; - - permissions.user_to_myself.push(permission); - } - } - - res.json(permissions); -}); diff --git a/src/backend/src/routers/auth/list-sessions.js b/src/backend/src/routers/auth/list-sessions.js deleted file mode 100644 index 0c73d8678..000000000 --- a/src/backend/src/routers/auth/list-sessions.js +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const eggspress = require('../../api/eggspress'); -const { UserActorType } = require('../../services/auth/Actor'); -const { Context } = require('../../util/context'); -const APIError = require('../../api/APIError'); - -module.exports = eggspress('/auth/list-sessions', { - subdomain: 'api', - auth2: true, - allowedMethods: ['GET'], -}, async (req, res, next) => { - const x = Context.get(); - const svc_auth = x.get('services').get('auth'); - - // Only users can list their own sessions - // apps, access tokens, etc should NEVER access this - const actor = x.get('actor'); - if ( ! (actor.type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - const sessions = await svc_auth.list_sessions(actor); - - res.json(sessions); -}); diff --git a/src/backend/src/routers/auth/oidc.js b/src/backend/src/routers/auth/oidc.js deleted file mode 100644 index 88cc767b5..000000000 --- a/src/backend/src/routers/auth/oidc.js +++ /dev/null @@ -1,344 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import express from 'express'; -import jwt from 'jsonwebtoken'; -import config from '../../config.js'; -import { get_user, subdomain } from '../../helpers.js'; - -export const router = express.Router(); - -const REVALIDATION_COOKIE_NAME = 'puter_revalidation'; -const REVALIDATION_EXPIRY_SEC = 300; // 5 minutes - -const MISSING_CODE_OR_STATE = Symbol('MISSING_CODE_OR_STATE'); -const INVALID_OR_EXPIRED_STATE = Symbol('INVALID_OR_EXPIRED_STATE'); -const TOKEN_EXCHANGE_FAILED = Symbol('TOKEN_EXCHANGE_FAILED'); -const COULD_NOT_GET_USER_INFO = Symbol('COULD_NOT_GET_USER_INFO'); - -const OIDC_CALLBACK_ERROR_RESPONSES = { - [MISSING_CODE_OR_STATE]: { status: 400, message: 'Missing code or state.' }, - [INVALID_OR_EXPIRED_STATE]: { status: 400, message: 'Invalid or expired state.' }, - [TOKEN_EXCHANGE_FAILED]: { status: 401, message: 'Token exchange failed.' }, - [COULD_NOT_GET_USER_INFO]: { status: 401, message: 'Could not get user info.' }, -}; - -const OIDC_ERROR_REDIRECT_MAP = { - login: { - account_not_found: 'signup', - other: 'login', - }, - signup: { - account_already_exists: 'login', - other: 'signup', - }, -}; - -/** - * The error redirect URL is the origin with a query parameter included to - * display an error message on the login or signup page. - * - * In a popup context, `stateDecoded` should contain the query parameters - * that reflect the popup state. `stateDecoded` is obtained from a JWT - * sent in the querystring of an OIDC callback page, and will decode to - * an object representing the query parameters that should go in the popup - * page invoked by puter.js - * - * @param {string} sourceFlow - 'login' or 'signup' - * @param {string} errorCondition - string that identifies the error message - * @param {string} message - default error message (before i18n) - * @param {object} [stateDecoded] - decoded OIDC state (may contain embedded_in_popup, msg_id for popup flow) - * @returns {string} URL to redirect to - */ -function buildOIDCErrorRedirectUrl (sourceFlow, errorCondition, message, stateDecoded) { - const targetFlow = OIDC_ERROR_REDIRECT_MAP[sourceFlow]?.[errorCondition] ?? sourceFlow; - const origin = (config.origin || '').replace(/\/$/, '') || '/'; - const params = new URLSearchParams({ action: targetFlow, auth_error: '1', message: message === 'This account is suspended.' ? 'account_suspended' : 'unauthorized' }); - if ( stateDecoded?.embedded_in_popup && stateDecoded?.msg_id != null ) { - const popupParams = new URLSearchParams({ - embedded_in_popup: 'true', - msg_id: String(stateDecoded.msg_id), - auth_error: '1', - message: message === 'This account is suspended.' ? 'account_suspended' : 'unauthorized', - action: targetFlow, - }); - if ( stateDecoded?.opener_origin ) { - popupParams.set('opener_origin', stateDecoded.opener_origin); - } - return `${origin}/?${popupParams.toString()}`; - } - return `${origin}/?${params.toString()}`; -} - -/** Applies a query parameter to a URL */ -function appendQueryParam (url, key, value) { - if ( !url || key == null ) return url; - const sep = url.includes('?') ? '&' : '?'; - const encoded = `${encodeURIComponent(key)}=${encodeURIComponent(value)}`; - return `${url}${sep}${encoded}`; -} - -/** Returns { session_token, target } for the caller to set cookie and redirect. */ -const finishOidcSuccess_ = async (req, res, user, stateDecoded, extraQueryParams = null) => { - const svc_auth = req.services.get('auth'); - const { token: session_token } = await svc_auth.create_session_token(user, { req }); - let target = stateDecoded.redirect_uri || config.origin || '/'; - const origin = config.origin || ''; - if ( target && origin && !target.startsWith(origin) ) { - target = origin; - } - if ( extraQueryParams && typeof extraQueryParams === 'object' ) { - for ( const [k, v] of Object.entries(extraQueryParams) ) { - if ( v != null ) target = appendQueryParam(target, k, String(v)); - } - } - return { session_token, target }; -}; - -/** Exchange code for tokens, get userinfo. Returns { provider, userinfo, stateDecoded } or { error } (symbol). */ -const processOIDCCallbackRequest_ = async (req, callbackRedirectUri) => { - const svc_oidc = req.services.get('oidc'); - const code = req.query.code; - const state = req.query.state; - if ( !code || !state ) { - return { error: MISSING_CODE_OR_STATE }; - } - const stateDecoded = svc_oidc.verifyState(state); - if ( !stateDecoded || !stateDecoded.provider ) { - return { error: INVALID_OR_EXPIRED_STATE }; - } - const provider = stateDecoded.provider; - const tokens = await svc_oidc.exchangeCodeForTokens(provider, code, callbackRedirectUri); - if ( !tokens || !tokens.access_token ) { - return { error: TOKEN_EXCHANGE_FAILED }; - } - const userinfo = await svc_oidc.getUserInfo(provider, tokens.access_token); - if ( !userinfo || !userinfo.sub ) { - return { error: COULD_NOT_GET_USER_INFO }; - } - return { provider, userinfo, stateDecoded }; -}; - -// GET /auth/oidc/providers - list enabled provider ids for frontend -router.get('/auth/oidc/providers', async (req, res) => { - if ( subdomain(req) !== 'api' ) { - return res.status(404).end(); - } - const svc_oidc = req.services.get('oidc'); - const providers = await svc_oidc.getEnabledProviderIds(); - return res.json({ providers }); -}); - -// GET /auth/oidc/:provider/start - redirect to IdP authorization -router.get('/auth/oidc/:provider/start', async (req, res) => { - if ( subdomain(req) !== '' ) { - return res.status(404).end(); - } - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('oidc-general') ) { - return res.status(429).send('Too many requests.'); - } - const provider = req.params.provider; - const svc_oidc = req.services.get('oidc'); - const cfg = await svc_oidc.getProviderConfig(provider); - if ( ! cfg ) { - return res.status(404).send('Provider not configured.'); - } - const flow = req.query.flow ? String(req.query.flow) : undefined; - const flowRedirects = { - login: config.origin || '/', - signup: config.origin || '/', - revalidate: `${(config.origin || '').replace(/\/$/, '')}/auth/revalidate-done`, - }; - let appRedirectUri = (flow && flowRedirects[flow]) ? flowRedirects[flow] : (config.origin || '/'); - const embeddedInPopup = req.query.embedded_in_popup === 'true' || req.query.embedded_in_popup === '1'; - const msgId = req.query.msg_id != null && req.query.msg_id !== '' ? String(req.query.msg_id) : null; - const openerOrigin = req.query.opener_origin != null && req.query.opener_origin !== '' ? String(req.query.opener_origin) : null; - if ( embeddedInPopup && msgId ) { - const origin = (config.origin || '').replace(/\/$/, ''); - appRedirectUri = `${origin}/action/sign-in?embedded_in_popup=true&msg_id=${encodeURIComponent(msgId)}`; - if ( openerOrigin ) { - appRedirectUri += `&opener_origin=${encodeURIComponent(openerOrigin)}`; - } - } - const statePayload = { provider, redirect_uri: appRedirectUri }; - if ( embeddedInPopup && msgId ) { - statePayload.embedded_in_popup = true; - statePayload.msg_id = msgId; - if ( openerOrigin ) { - statePayload.opener_origin = openerOrigin; - } - } - if ( flow === 'revalidate' ) { - const user_id = req.query.user_id; - if ( ! user_id ) { - return res.status(400).send('user_id required for revalidate flow.'); - } - statePayload.user_id = Number(user_id); - statePayload.flow = 'revalidate'; - } - const state = svc_oidc.signState(statePayload); - const url = await svc_oidc.getAuthorizationUrl(provider, state, flow); - if ( ! url ) { - return res.status(502).send('Could not build authorization URL.'); - } - return res.redirect(302, url); -}); - -// GET /auth/oidc/callback/login - login: existing account or create one if none exists. -router.get('/auth/oidc/callback/login', async (req, res) => { - if ( subdomain(req) !== '' ) { - return res.status(404).end(); - } - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('oidc-general') ) { - return res.status(429).send('Too many requests.'); - } - const svc_oidc = req.services.get('oidc'); - const callbackRedirectUri = svc_oidc.getCallbackUrlForFlow('login'); - const result = await processOIDCCallbackRequest_(req, callbackRedirectUri); - if ( result.error ) { - const { message } = OIDC_CALLBACK_ERROR_RESPONSES[result.error]; - return res.redirect(302, buildOIDCErrorRedirectUrl('login', 'other', message)); - } - const { provider, userinfo, stateDecoded } = result; - let user = await svc_oidc.findUserByProviderSub(provider, userinfo.sub); - if ( ! user ) { - // No account found: create one instead (login flow switches to signup). - const outcome = await svc_oidc.createUserFromOIDC(provider, userinfo); - if ( outcome.failed ) { - return res.redirect(302, buildOIDCErrorRedirectUrl('login', 'other', outcome.userMessage, stateDecoded)); - } - user = await get_user({ id: outcome.infoObject.user_id }); - } - if ( user.suspended ) { - return res.redirect(302, buildOIDCErrorRedirectUrl('login', 'other', 'This account is suspended.', stateDecoded)); - } - const { session_token, target } = await finishOidcSuccess_(req, res, user, stateDecoded); - res.cookie(config.cookie_name, session_token, { - sameSite: 'none', - secure: true, - httpOnly: true, - }); - return res.redirect(302, target); -}); - -// GET /auth/oidc/callback/signup - signup: create new account or log in to existing if already registered. -router.get('/auth/oidc/callback/signup', async (req, res) => { - if ( subdomain(req) !== '' ) { - return res.status(404).end(); - } - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('oidc-general') ) { - return res.status(429).send('Too many requests.'); - } - const svc_oidc = req.services.get('oidc'); - const callbackRedirectUri = svc_oidc.getCallbackUrlForFlow('signup'); - const result = await processOIDCCallbackRequest_(req, callbackRedirectUri); - if ( result.error ) { - const { message } = OIDC_CALLBACK_ERROR_RESPONSES[result.error]; - return res.redirect(302, buildOIDCErrorRedirectUrl('signup', 'other', message)); - } - const { provider, userinfo, stateDecoded } = result; - const existingUser = await svc_oidc.findUserByProviderSub(provider, userinfo.sub); - if ( existingUser ) { - // Account already exists: log in instead and inform the user (signup flow switches to login). - const { session_token, target } = await finishOidcSuccess_(req, res, existingUser, stateDecoded, { oidc_switched: 'login' }); - res.cookie(config.cookie_name, session_token, { - sameSite: 'none', - secure: true, - httpOnly: true, - }); - return res.redirect(302, target); - } - const outcome = await svc_oidc.createUserFromOIDC(provider, userinfo); - if ( outcome.failed ) { - return res.redirect(302, buildOIDCErrorRedirectUrl('signup', 'other', outcome.userMessage, stateDecoded)); - } - const user = await get_user({ id: outcome.infoObject.user_id }); - const { session_token, target } = await finishOidcSuccess_(req, res, user, stateDecoded); - res.cookie(config.cookie_name, session_token, { - sameSite: 'none', - secure: true, - httpOnly: true, - }); - return res.redirect(302, target); -}); - -// GET /auth/oidc/callback/revalidate - re-validate identity for protected actions (e.g. change username). Sets short-lived cookie and redirects. -router.get('/auth/oidc/callback/revalidate', async (req, res) => { - if ( subdomain(req) !== '' ) { - return res.status(404).end(); - } - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('oidc-general') ) { - return res.status(429).send('Too many requests.'); - } - const svc_oidc = req.services.get('oidc'); - const callbackRedirectUri = svc_oidc.getCallbackUrlForFlow('revalidate'); - const result = await processOIDCCallbackRequest_(req, callbackRedirectUri); - if ( result.error ) { - const { status, message } = OIDC_CALLBACK_ERROR_RESPONSES[result.error]; - return res.status(status).send(message); - } - const { provider, userinfo, stateDecoded } = result; - if ( stateDecoded.flow !== 'revalidate' || stateDecoded.user_id == null ) { - return res.status(400).send('Invalid revalidate state.'); - } - const user = await svc_oidc.findUserByProviderSub(provider, userinfo.sub); - if ( ! user ) { - return res.status(400).send('No account found.'); - } - if ( user.id !== stateDecoded.user_id ) { - return res.status(403).send('Wrong account. Sign in with the account linked to this session.'); - } - const token = jwt.sign( - { user_id: user.id, purpose: 'revalidate' }, - config.jwt_secret, - { expiresIn: REVALIDATION_EXPIRY_SEC }, - ); - res.cookie(REVALIDATION_COOKIE_NAME, token, { - sameSite: 'lax', - secure: true, - httpOnly: true, - maxAge: REVALIDATION_EXPIRY_SEC * 1000, - path: '/', - }); - const target = stateDecoded.redirect_uri || `${(config.origin || '').replace(/\/$/, '')}/auth/revalidate-done`; - return res.redirect(302, target); -}); - -// GET /auth/revalidate-done - landing page after OIDC revalidate; posts to opener and closes (for popup flow). -router.get('/auth/revalidate-done', (req, res) => { - if ( subdomain(req) !== '' ) { - return res.status(404).end(); - } - const origin = config.origin || ''; - res.set('Content-Type', 'text/html; charset=utf-8'); - res.send(`Re-validated

Re-validated. Closing…

`); -}); diff --git a/src/backend/src/routers/auth/request-app-root-dir.js b/src/backend/src/routers/auth/request-app-root-dir.js deleted file mode 100644 index 659477211..000000000 --- a/src/backend/src/routers/auth/request-app-root-dir.js +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const eggspress = require('../../api/eggspress'); -const APIError = require('../../api/APIError'); -const { AppUnderUserActorType } = require('../../services/auth/Actor'); -const { Context } = require('../../util/context'); -const { validate_fields } = require('../../util/validutil'); -const { get_app } = require('../../helpers'); -const { NodeInternalIDSelector } = require('../../deprecated/filesystem/node/selectors'); -const { HLStat } = require('../../deprecated/filesystem/hl_operations/hl_stat'); -const { PermissionUtil } = require('../../services/auth/permissionUtils.mjs'); -const { quot } = require('@heyputer/putility').libs.string; - -module.exports = eggspress('/auth/request-app-root-dir', { - subdomain: 'api', - auth2: true, - allowedMethods: ['POST'], -}, async (req, res) => { - const context = Context.get(); - const actor = context.get('actor'); - - if ( ! (actor.type instanceof AppUnderUserActorType) ) { - throw APIError.create('forbidden', null, { debug_reason: 'not app actor' }); - } - - validate_fields({ - app_uid: { type: 'string', optional: false }, - access: { type: 'string', optional: false }, - }, req.body); - - const { app_uid: target_app_uid, access } = req.body; - if ( access !== 'read' && access !== 'write' ) { - throw APIError.create('field_invalid', null, { - key: 'access', - expected: "'read' or 'write'", - got: access, - }); - } - - if ( ! target_app_uid ) { - throw APIError.create('field_invalid', null, { - key: 'resource_request_code', - expected: 'app_uid', - got: target_app_uid, - }); - } - - const target_app = await get_app({ uid: target_app_uid }); - if ( ! target_app ) { - throw APIError.create('entity_not_found', null, { identifier: `app:${target_app_uid}` }); - } - - if ( target_app.owner_user_id !== actor.type.user.id ) { - throw APIError.create('forbidden', null, { - debug_reason: 'Expected to match: ' + - `${quot(target_app.owner_user_id)} and ${quot(actor.type.user.id)}`, - }); - } - - const svc_app = context.get('services').get('app'); - const root_dir_id = await svc_app.getAppRootDirId(target_app); - const svc_fs = context.get('services').get('filesystem'); - const node = await svc_fs.node(new NodeInternalIDSelector('mysql', root_dir_id)); - await node.fetchEntry(); - if ( ! node.found ) { - throw APIError.create('subject_does_not_exist'); - } - - const node_uid = await node.get('uid'); - const fs_perm = PermissionUtil.join('fs', node_uid, access); - const svc_permission = context.get('services').get('permission'); - const has_perm = await svc_permission.check(actor, fs_perm); - if ( ! has_perm ) { - throw APIError.create('permission_denied', null, { permission: fs_perm }); - } - - const hl_stat = new HLStat(); - const stat_result = await hl_stat.run({ - subject: node, - user: actor.type.user, - return_subdomains: false, - return_permissions: false, - return_shares: false, - return_versions: false, - return_size: true, - }); - - res.json(stat_result); -}); diff --git a/src/backend/src/routers/auth/revoke-access-token.js b/src/backend/src/routers/auth/revoke-access-token.js deleted file mode 100644 index 4081cc8af..000000000 --- a/src/backend/src/routers/auth/revoke-access-token.js +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const eggspress = require('../../api/eggspress'); -const { Context } = require('../../util/context'); - -/** - * Coerces a read-URL string to the token (JWT) from its query. - * Works for absolute or relative URLs (e.g. .../token-read?uid=...&token=...). - * Returns the given value unchanged if it does not look like a read URL. - */ -function tokenOrUuidFromInput (value) { - if ( typeof value !== 'string' || !value.trim() ) { - return value; - } - const s = value.trim(); - console.log('s?', s); - if ( s.includes('/token-read') ) { - try { - const url = new URL(s); - const token = url.searchParams.get('token'); - console.log('token?', token); - return token ?? s; - } catch (_) { - return s; - } - } - return s; -} - -module.exports = eggspress('/auth/revoke-access-token', { - subdomain: 'api', - auth2: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - const x = Context.get(); - const svc_auth = x.get('services').get('auth'); - - const raw = req.body.tokenOrUuid; - if ( raw === undefined || raw === null ) { - throw APIError.create('field_missing', null, { key: 'tokenOrUuid' }); - } - const tokenOrUuid = tokenOrUuidFromInput(raw); - - await svc_auth.revoke_access_token(tokenOrUuid); - - res.json({ ok: true }); -}); diff --git a/src/backend/src/routers/auth/revoke-dev-app.js b/src/backend/src/routers/auth/revoke-dev-app.js deleted file mode 100644 index 4ee11bc63..000000000 --- a/src/backend/src/routers/auth/revoke-dev-app.js +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const eggspress = require('../../api/eggspress'); -const { UserActorType } = require('../../services/auth/Actor'); -const { Context } = require('../../util/context'); -const APIError = require('../../api/APIError'); - -module.exports = eggspress('/auth/revoke-dev-app', { - subdomain: 'api', - auth2: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - const x = Context.get(); - const svc_permission = x.get('services').get('permission'); - - // Only users can grant user-app permissions - const actor = Context.get('actor'); - if ( ! (actor.type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - if ( req.body.origin ) { - const svc_auth = x.get('services').get('auth'); - req.body.app_uid = await svc_auth.app_uid_from_origin(req.body.origin); - } - - if ( ! req.body.app_uid ) { - throw APIError.create('field_missing', null, { key: 'app_uid' }); - } - - if ( req.body.permission === '*' ) { - await svc_permission.revoke_dev_app_all(actor, req.body.app_uid, req.body.meta || {}); - } - - await svc_permission.revoke_dev_app_permission(actor, req.body.app_uid, req.body.permission, req.body.meta || {}); - - res.json({}); -}); diff --git a/src/backend/src/routers/auth/revoke-session.js b/src/backend/src/routers/auth/revoke-session.js deleted file mode 100644 index f8c51fe38..000000000 --- a/src/backend/src/routers/auth/revoke-session.js +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const eggspress = require('../../api/eggspress'); -const { UserActorType } = require('../../services/auth/Actor'); -const { Context } = require('../../util/context'); - -module.exports = eggspress('/auth/revoke-session', { - subdomain: 'api', - auth2: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - const x = Context.get(); - const svc_auth = x.get('services').get('auth'); - - // Only users can list their own sessions - // apps, access tokens, etc should NEVER access this - const actor = x.get('actor'); - if ( ! (actor.type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - const svc_antiCSRF = req.services.get('anti-csrf'); - if ( ! await svc_antiCSRF.consume_token(actor.type.user.uuid, req.body.anti_csrf) ) { - return res.status(400).json({ message: 'incorrect anti-CSRF token' }); - } - - // Ensure valid UUID - if ( !req.body.uuid || typeof req.body.uuid !== 'string' ) { - throw APIError.create('field_invalid', null, { - key: 'uuid', - expected: 'string', - }); - } - - const sessions = await svc_auth.revoke_session(actor, req.body.uuid); - - res.json({ sessions }); -}); diff --git a/src/backend/src/routers/auth/revoke-user-app.js b/src/backend/src/routers/auth/revoke-user-app.js deleted file mode 100644 index 7a4b58880..000000000 --- a/src/backend/src/routers/auth/revoke-user-app.js +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const eggspress = require('../../api/eggspress'); -const { UserActorType } = require('../../services/auth/Actor'); -const { Context } = require('../../util/context'); -const APIError = require('../../api/APIError'); - -module.exports = eggspress('/auth/revoke-user-app', { - subdomain: 'api', - auth2: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - const x = Context.get(); - const svc_permission = x.get('services').get('permission'); - - // Only users can grant user-app permissions - const actor = Context.get('actor'); - if ( ! (actor.type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - if ( req.body.origin ) { - const svc_auth = x.get('services').get('auth'); - req.body.app_uid = await svc_auth.app_uid_from_origin(req.body.origin); - } - - if ( ! req.body.app_uid ) { - throw APIError.create('field_missing', null, { key: 'app_uid' }); - } - - if ( req.body.permission === '*' ) { - await svc_permission.revoke_user_app_all(actor, req.body.app_uid, req.body.meta || {}); - } - - await svc_permission.revoke_user_app_permission(actor, req.body.app_uid, req.body.permission, req.body.meta || {}); - - res.json({}); -}); diff --git a/src/backend/src/routers/auth/revoke-user-group.js b/src/backend/src/routers/auth/revoke-user-group.js deleted file mode 100644 index a6c461ddd..000000000 --- a/src/backend/src/routers/auth/revoke-user-group.js +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const eggspress = require('../../api/eggspress'); -const { UserActorType } = require('../../services/auth/Actor'); -const { Context } = require('../../util/context'); - -module.exports = eggspress('/auth/revoke-user-group', { - subdomain: 'api', - auth2: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - const x = Context.get(); - const svc_permission = x.get('services').get('permission'); - - // Only users can grant user-user permissions - const actor = Context.get('actor'); - if ( ! (actor.type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - if ( ! req.body.group_uid ) { - throw APIError.create('field_missing', null, { - key: 'group_uid', - }); - } - - if ( ! req.body.permission ) { - throw APIError.create('field_missing', null, { - key: 'permission', - }); - } - - await svc_permission.revoke_user_group_permission(actor, req.body.group_uid, req.body.permission, req.body.meta || {}); - - res.json({}); -}); diff --git a/src/backend/src/routers/auth/revoke-user-user.js b/src/backend/src/routers/auth/revoke-user-user.js deleted file mode 100644 index 736bd1d3c..000000000 --- a/src/backend/src/routers/auth/revoke-user-user.js +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const eggspress = require('../../api/eggspress'); -const { UserActorType } = require('../../services/auth/Actor'); -const { Context } = require('../../util/context'); - -module.exports = eggspress('/auth/revoke-user-user', { - subdomain: 'api', - auth2: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - const x = Context.get(); - const svc_permission = x.get('services').get('permission'); - - // Only users can grant user-user permissions - const actor = Context.get('actor'); - if ( ! (actor.type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - if ( ! req.body.target_username ) { - throw APIError.create('field_missing', null, { key: 'target_username' }); - } - - await svc_permission.revoke_user_user_permission(actor, req.body.target_username, req.body.permission, req.body.meta || {}); - - res.json({}); -}); diff --git a/src/backend/src/routers/change_email.js b/src/backend/src/routers/change_email.js deleted file mode 100644 index 3e86b64fa..000000000 --- a/src/backend/src/routers/change_email.js +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const eggspress = require('../api/eggspress.js'); -const APIError = require('../api/APIError.js'); -const { DB_WRITE } = require('../services/database/consts.js'); - -const config = require('../config.js'); - -const jwt = require('jsonwebtoken'); -const { invalidate_cached_user_by_id } = require('../helpers.js'); - -const CHANGE_EMAIL_CONFIRM = eggspress('/change_email/confirm', { - allowedMethods: ['GET'], -}, async (req, res ) => { - const jwt_token = req.query.token; - - if ( ! jwt_token ) { - throw APIError.create('field_missing', null, { key: 'token' }); - } - - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('change-email-confirm') ) { - return res.status(429).send('Too many requests.'); - } - - const { token, user_id } = jwt.verify(jwt_token, config.jwt_secret); - - const db = req.services.get('database').get(DB_WRITE, 'auth'); - const rows = await db.read( - 'SELECT `unconfirmed_change_email`, `suspended` FROM `user` WHERE `id` = ? AND `change_email_confirm_token` = ?', - [user_id, token], - ); - if ( rows.length === 0 ) { - throw APIError.create('token_invalid'); - } - - if ( rows[0].suspended ) { - throw APIError.create('forbidden'); - } - - const svc_cleanEmail = req.services.get('clean-email'); - const clean_email = svc_cleanEmail.clean(rows[0].unconfirmed_change_email); - - // Scenario: email was confirmed on another account already - const rows2 = await db.read( - 'SELECT `id` FROM `user` WHERE `email` = ? OR `clean_email` = ?', - [rows[0].unconfirmed_change_email, clean_email], - ); - if ( rows2.length > 0 ) { - throw APIError.create('email_already_in_use'); - } - - // If other users have the same unconfirmed email, revoke it - await db.write( - 'UPDATE `user` SET `unconfirmed_change_email` = NULL, `email_confirmed`=1, `change_email_confirm_token` = NULL WHERE `id` = ?', - [user_id], - ); - - const new_email = rows[0].unconfirmed_change_email; - - await db.write( - 'UPDATE `user` SET `email` = ?, `clean_email` = ?, `unconfirmed_change_email` = NULL, `change_email_confirm_token` = NULL, `pass_recovery_token` = NULL WHERE `id` = ?', - [new_email, clean_email, user_id], - ); - - const svc_event = req.services.get('event'); - svc_event.emit('user.email-changed', { - user_id: user_id, - new_email, - }); - - invalidate_cached_user_by_id(user_id); - const svc_socketio = req.services.get('socketio'); - svc_socketio.send({ room: user_id }, 'user.email_changed', {}); - - const h = '

Your email has been successfully confirmed.

'; - return res.send(h); -}); - -module.exports = app => { - app.use(CHANGE_EMAIL_CONFIRM); -}; diff --git a/src/backend/src/routers/change_username.js b/src/backend/src/routers/change_username.js deleted file mode 100644 index 5955d0950..000000000 --- a/src/backend/src/routers/change_username.js +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const config = require('../config'); -const eggspress = require('../api/eggspress.js'); -const { Context } = require('../util/context.js'); -const { UserActorType } = require('../services/auth/Actor.js'); -const APIError = require('../api/APIError.js'); -const { DB_WRITE } = require('../services/database/consts'); - -module.exports = eggspress('/change_username', { - subdomain: 'api', - auth2: true, - verified: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - - const { username_exists, change_username } = require('../helpers'); - - const actor = Context.get('actor'); - - // Only users can change their username (apps can't do this) - if ( ! ( actor.type instanceof UserActorType ) ) { - throw APIError.create('forbidden'); - } - - // validation - if ( ! req.body.new_username ) - { - throw APIError.create('field_missing', null, { key: 'new_username' }); - } - // new_username must be a string - else if ( typeof req.body.new_username !== 'string' ) - { - throw APIError.create('field_invalid', null, { key: 'new_username', expected: 'a string' }); - } - else if ( ! req.body.new_username.match(config.username_regex) ) - { - throw APIError.create('field_invalid', null, { key: 'new_username', expected: 'letters, numbers, underscore (_)' }); - } - else if ( req.body.new_username.length > config.username_max_length ) - { - throw APIError.create('field_too_long', null, { key: 'new_username', max_length: config.username_max_length }); - } - // duplicate username check - if ( await username_exists(req.body.new_username) ) - { - throw APIError.create('username_already_in_use', null, { username: req.body.new_username }); - } - - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('change-email-start') ) { - return res.status(429).send('Too many requests.'); - } - - const db = Context.get('services').get('database').get(DB_WRITE, 'auth'); - - // Has the user already changed their username twice this month? - const rows = await db.read('SELECT COUNT(*) AS `count` FROM `user_update_audit` ' + - `WHERE \`user_id\`=? AND \`reason\`=? AND ${ - db.case({ - mysql: '`created_at` > DATE_SUB(NOW(), INTERVAL 1 MONTH)', - sqlite: "`created_at` > datetime('now', '-1 month')", - })}`, - [req.user.id, 'change_username']); - - if ( rows[0].count >= (config.max_username_changes ?? 2) ) { - throw APIError.create('too_many_username_changes'); - } - - // Update username change audit table - await db.write('INSERT INTO `user_update_audit` ' + - '(`user_id`, `user_id_keep`, `old_username`, `new_username`, `reason`) ' + - 'VALUES (?, ?, ?, ?, ?)', - [ - req.user.id, req.user.id, - req.user.username, req.body.new_username, - 'change_username', - ]); - - await change_username(req.user.id, req.body.new_username); - - res.json({}); -}); diff --git a/src/backend/src/routers/confirmEmail/ConfirmEmailRedisCacheSpace.js b/src/backend/src/routers/confirmEmail/ConfirmEmailRedisCacheSpace.js deleted file mode 100644 index c1d20a9e5..000000000 --- a/src/backend/src/routers/confirmEmail/ConfirmEmailRedisCacheSpace.js +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const ConfirmEmailRedisCacheSpace = { - key: ({ ipAddress, emailOrUsername }) => `confirm-email|${ipAddress}|${emailOrUsername}`, -}; - -export { ConfirmEmailRedisCacheSpace }; diff --git a/src/backend/src/routers/confirmEmail/confirm-email.js b/src/backend/src/routers/confirmEmail/confirm-email.js deleted file mode 100644 index 6b3a2bef5..000000000 --- a/src/backend/src/routers/confirmEmail/confirm-email.js +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = new express.Router(); -const auth = require('../../middleware/auth.js'); -const { DB_WRITE } = require('../../services/database/consts.js'); -const APIError = require('../../api/APIError.js'); -const { redisClient } = require('../../clients/redis/redisSingleton.js'); -const { ConfirmEmailRedisCacheSpace } = require('./ConfirmEmailRedisCacheSpace.js'); -const { invalidate_cached_user_by_id } = require('../../helpers.js'); - -// -----------------------------------------------------------------------// -// POST /confirm-email -// -----------------------------------------------------------------------// -router.post('/confirm-email', auth, express.json(), async (req, res, next) => { - // Either api. subdomain or no subdomain - if ( require('../../helpers.js').subdomain(req) !== 'api' && require('../../helpers.js').subdomain(req) !== '' ) - { - next(); - } - - if ( ! req.body.code ) - { - return res.status(400).send('code is required'); - } - - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('confirm-email') ) { - return res.status(429).send('Too many requests.'); - } - - // Modules - const db = req.services.get('database').get(DB_WRITE, 'auth'); - - // Increment & check rate limit - const rateLimitKey = ConfirmEmailRedisCacheSpace.key({ - ipAddress: req.ip, - emailOrUsername: req.user.email ?? req.user.username, - }); - if ( await redisClient.incr(rateLimitKey) > 10 ) - { - return res.status(429).send({ error: 'Too many requests.' }); - } - // Set expiry for rate limit - redisClient.expire(rateLimitKey, 60 * 10, 'NX'); - - // Force a primary read so confirmation checks do not rely on possibly stale cache entries. - const svc_getUser = req.services.get('get-user'); - const user = await svc_getUser.get_user({ id: req.user.id, force: true }); - if ( ! user ) { - APIError.create('user_not_found').write(res); - return; - } - - if ( String(req.body.code) !== String(user.email_confirm_code) ) { - res.send({ email_confirmed: false }); - return; - } - - // Scenario: email was confirmed on another account already - { - const svc_cleanEmail = req.services.get('clean-email'); - const clean_email = svc_cleanEmail.clean(user.email); - - if ( ! await svc_cleanEmail.validate(clean_email) ) { - APIError.create('field_invalid', null, { - key: 'email', - expected: 'valid email', - got: req.body.email, - }); - } - const rows = await db.read(`SELECT EXISTS( - SELECT 1 FROM user WHERE (email=? OR clean_email=?) AND email_confirmed=1 AND password IS NOT NULL - ) AS email_exists`, [user.email, clean_email]); - if ( rows[0].email_exists ) { - APIError.create('email_already_in_use').write(res); - return; - } - } - - // If other users have the same unconfirmed email, revoke it - await db.write( - 'UPDATE `user` SET `unconfirmed_change_email` = NULL, `change_email_confirm_token` = NULL WHERE `unconfirmed_change_email` = ?', - [user.email], - ); - - // Update user record to say email is confirmed - await db.write( - 'UPDATE `user` SET `email_confirmed` = 1, `requires_email_confirmation` = 0 WHERE id = ? LIMIT 1', - [user.id], - ); - - // Invalidate user cache - await invalidate_cached_user_by_id(req.user.id); - - // Emit internal event - const svc_event = req.services.get('event'); - svc_event.emit('user.email-confirmed', { - user_uid: user.uuid, - email: user.email, - }); - - // Emit websocket event (TODO: should come from internal event above) - const svc_socketio = req.services.get('socketio'); - svc_socketio.send({ room: user.id }, 'user.email_confirmed', { - original_client_socket_id: req.body.original_client_socket_id, - }); - - // return results - return res.send({ - email_confirmed: true, - original_client_socket_id: req.body.original_client_socket_id, - }); -}); - -module.exports = router; diff --git a/src/backend/src/routers/contactUs.js b/src/backend/src/routers/contactUs.js deleted file mode 100644 index 361f85b7c..000000000 --- a/src/backend/src/routers/contactUs.js +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = express.Router(); -const auth = require('../middleware/auth.js'); -const { get_user, generate_random_str } = require('../helpers'); -const { DB_WRITE } = require('../services/database/consts.js'); - -// -----------------------------------------------------------------------// -// POST /contactUs -// -----------------------------------------------------------------------// -router.post('/contactUs', auth, express.json(), async (req, res, next) => { - // check subdomain - if ( require('../helpers').subdomain(req) !== 'api' ) - { - next(); - } - - // message is required - if ( ! req.body.message ) - { - return res.status(400).send({ message: 'message is required' }); - } - // message must be a string - if ( typeof req.body.message !== 'string' ) - { - return res.status(400).send('message must be a string.'); - } - // message is too long - else if ( req.body.message.length > 100000 ) - { - return res.status(400).send({ message: 'message is too long' }); - } - - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('contact-us') ) { - return res.status(429).send('Too many requests.'); - } - - // modules - const db = req.services.get('database').get(DB_WRITE, 'feedback'); - - try { - db.write(`INSERT INTO feedback - (user_id, message) VALUES - ( ?, ?)`, - [ - //user_id - req.user.id, - //message - req.body.message, - ]); - - // get user - let user = await get_user({ id: req.user.id }); - - // send email to support - const svc_email = req.services.get('email'); - svc_email.sendMail({ - from: '"Puter" no-reply@puter.com', // sender address - to: 'support@puter.com', // list of receivers - replyTo: user.email === null ? undefined : user.email, - subject: `Your Feedback/Support Request (#${generate_random_str(4)})`, // Subject line - text: req.body.message, - }); - - return res.send({}); - } catch (e) { - return res.status(400).send(e); - } -}); - -module.exports = router; \ No newline at end of file diff --git a/src/backend/src/routers/delete-site.js b/src/backend/src/routers/delete-site.js deleted file mode 100644 index 46909d0b6..000000000 --- a/src/backend/src/routers/delete-site.js +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = express.Router(); -const auth = require('../middleware/auth.js'); -const config = require('../config'); -const { DB_WRITE } = require('../services/database/consts.js'); - -// -----------------------------------------------------------------------// -// POST /delete-site -// -----------------------------------------------------------------------// -router.post('/delete-site', auth, express.json(), async (req, res, next) => { - // check subdomain - if ( require('../helpers').subdomain(req) !== 'api' ) - { - next(); - } - - // check if user is verified - if ( (config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed ) - { - return res.status(400).send({ code: 'account_is_not_verified', message: 'Account is not verified' }); - } - - // validation - if ( req.body.site_uuid === undefined ) - { - return res.status(400).send('site_uuid is required'); - } - - // modules - const db = req.services.get('database').get(DB_WRITE, 'subdomains:legacy'); - - await db.write('DELETE FROM subdomains WHERE user_id = ? AND uuid = ?', - [req.user.id, req.body.site_uuid]); - res.send({}); -}); - -module.exports = router; \ No newline at end of file diff --git a/src/backend/src/routers/df.js b/src/backend/src/routers/df.js deleted file mode 100644 index 114194e11..000000000 --- a/src/backend/src/routers/df.js +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const config = require('../config.js'); -const router = new express.Router(); -const auth = require('../middleware/auth.js'); - -// TODO: Why is this both a POST and a GET? - -// -----------------------------------------------------------------------// -// POST /df -// -----------------------------------------------------------------------// -router.post('/df', auth, express.json(), async (req, response, next) => { - // check subdomain - if ( require('../helpers').subdomain(req) !== 'api' ) - { - next(); - } - - // check if user is verified - if ( (config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed ) - { - return response.status(400).send({ code: 'account_is_not_verified', message: 'Account is not verified' }); - } - - const { df } = require('../helpers'); - const svc_hostDiskUsage = req.services.get('host-disk-usage', { optional: true }); - try { - // auth - response.send({ - used: parseInt(await df(req.user.id)), - capacity: config.is_storage_limited ? (req.user.free_storage === undefined || req.user.free_storage === null) ? config.storage_capacity : req.user.free_storage : config.available_device_storage, - ...(svc_hostDiskUsage ? svc_hostDiskUsage.get_extra() : {}), - }); - } catch (e) { - console.log(e); - response.status(400).send(); - } -}); - -// -----------------------------------------------------------------------// -// GET /df -// -----------------------------------------------------------------------// -router.get('/df', auth, express.json(), async (req, response, next) => { - // check subdomain - if ( require('../helpers').subdomain(req) !== 'api' ) - { - next(); - } - - // check if user is verified - if ( (config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed ) - { - return response.status(400).send({ code: 'account_is_not_verified', message: 'Account is not verified' }); - } - - const { df } = require('../helpers'); - const svc_hostDiskUsage = req.services.get('host-disk-usage', { optional: true }); - try { - // auth - response.send({ - used: parseInt(await df(req.user.id)), - capacity: config.is_storage_limited ? (req.user.free_storage === undefined || req.user.free_storage === null) ? config.storage_capacity : req.user.free_storage : config.available_device_storage, - ...(svc_hostDiskUsage ? svc_hostDiskUsage.get_extra() : {}), - }); - } catch (e) { - console.log(e); - response.status(400).send(); - } -}); - -module.exports = router; \ No newline at end of file diff --git a/src/backend/src/routers/down.js b/src/backend/src/routers/down.js deleted file mode 100644 index b55c8c685..000000000 --- a/src/backend/src/routers/down.js +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = express.Router(); -const config = require('../config.js'); -const { NodePathSelector } = require('../deprecated/filesystem/node/selectors.js'); -const { HLRead } = require('../deprecated/filesystem/hl_operations/hl_read.js'); -const { UserActorType } = require('../services/auth/Actor.js'); -const configurable_auth = require('../middleware/configurable_auth.js'); -const { subdomain } = require('../helpers'); -const _path = require('path'); - -// -----------------------------------------------------------------------// -// GET /down -// -----------------------------------------------------------------------// -router.post('/down', express.json(), express.urlencoded({ extended: true }), configurable_auth(), async (req, res, next) => { - // check subdomain - const actor = req.actor; - - if ( !actor || !(actor.type instanceof UserActorType) ) { - if ( subdomain(req) !== 'api' ) - { - next(); - } - } - - // check if user is verified - if ( (config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed ) - { - return res.status(400).send({ code: 'account_is_not_verified', message: 'Account is not verified' }); - } - - // check anti-csrf token - const svc_antiCSRF = req.services.get('anti-csrf'); - if ( ! await svc_antiCSRF.consume_token(req.user.uuid, req.body.anti_csrf) ) { - return res.status(400).json({ message: 'incorrect anti-CSRF token' }); - } - - // validation - if ( ! req.query.path ) - { - return res.status(400).send('path is required'); - } - // path must be a string - else if ( typeof req.query.path !== 'string' ) - { - return res.status(400).send('path must be a string.'); - } - else if ( req.query.path.trim() === '' ) - { - return res.status(400).send('path cannot be empty'); - } - - // modules - const path = _path.resolve('/', req.query.path); - - // cannot download the root, because it's a directory! - if ( path === '/' ) - { - return res.status(400).send('Cannot download a directory.'); - } - - // resolve path to its FSEntry - const svc_fs = req.services.get('filesystem'); - const fsnode = await svc_fs.node(new NodePathSelector(path)); - - // not found - if ( ! fsnode.exists() ) { - return res.status(404).send('File not found'); - } - - // stream data from S3 - try { - res.setHeader('Content-Type', 'application/octet-stream'); - res.attachment(await fsnode.get('name')); - - const hl_read = new HLRead(); - const stream = await hl_read.run({ - fsNode: fsnode, - user: req.user, - }); - return stream.pipe(res); - } catch (e) { - console.log(e); - return res.type('application/json').status(500).send({ message: 'There was an internal problem reading the file.' }); - } -}); - -module.exports = router; diff --git a/src/backend/src/routers/drivers/call.js b/src/backend/src/routers/drivers/call.js deleted file mode 100644 index 100afb9a2..000000000 --- a/src/backend/src/routers/drivers/call.js +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const eggspress = require('../../api/eggspress'); -const { FileFacade } = require('../../services/drivers/FileFacade'); -const { TypeSpec } = require('../../services/drivers/meta/Construct'); -const { TypedValue } = require('../../services/drivers/meta/Runtime'); -const { Context } = require('../../util/context'); -const { TeePromise } = require('@heyputer/putility').libs.promise; -const { valid_file_size } = require('../../util/validutil'); - -let _handle_multipart; -const responseHelper = (res, result) => { - if ( result.result instanceof TypedValue ) { - const tv = result.result; - if ( TypeSpec.adapt({ $: 'stream' }).equals(tv.type) ) { - res.set('Content-Type', tv.type.raw.content_type); - if ( tv.type.raw.chunked ) { - res.set('Transfer-Encoding', 'chunked'); - } - tv.value.pipe(res); - return; - } - - // This is the - if ( typeof tv.value === 'object' ) { - tv.value.type_fallback = true; - } - res.json(tv.value); - return; - } - res.json(result); -}; - -/** - * POST /drivers/call - * - * This endpoint is used to call methods offered by driver interfaces. - * The implementation used by each interface depends on the user's - * configuration. - * - * The request body can be a JSON object or multipart/form-data. - * For multipart/form-data, the caller must be aware that all fields - * are required to be sent before files so that the request handler - * and underlying driver implementation can decide what to do with - * file streams as they come. - * - * Example request body: - * { - * "interface": "puter-ocr", - * "method": "recognize", - * "args": { - * "file": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAB... - * } - * } - */ -module.exports = eggspress('/drivers/call', { - subdomain: 'api', - auth2: true, - // noReallyItsJson: true, - jsonCanBeLarge: true, - allowedMethods: ['POST'], -}, async (req, res) => { - const x = Context.get(); - const svc_driver = x.get('services').get('driver'); - - let p_request = null; - let body; - if ( req.headers['content-type'].includes('multipart/form-data') ) { - ({ params: body, p_data_end: p_request } = await _handle_multipart(req)); - } else body = req.body; - - const interface_name = body.interface; - const test_mode = body.test_mode; - - let context = Context.get(); - if ( test_mode ) context = context.sub({ test_mode: true }); - - const result = await context.arun(async () => { - return await svc_driver.call({ - iface: interface_name, - driver: body.driver ?? body.service, - method: body.method, - format: body.format, - args: body.args, - }); - }); - - // We can't wait for the request to finish before responding; - // consider the case where a driver method implements a - // stream transformation, thus the stream from the request isn't - // consumed until the response is being sent. - - responseHelper(res, result); - - // What we _can_ do is await the request promise while responding - // to ensure errors are caught here. - await p_request; -}); - -_handle_multipart = async (req) => { - const Busboy = require('busboy'); - const { PassThrough } = require('stream'); - - const params = Object.create(null); - const files = []; - let file_index = 0; - - const bb = Busboy({ - headers: req.headers, - }); - - const p_data_end = new TeePromise(); - const p_nonfile_data_end = new TeePromise(); - bb.on('file', (fieldname, stream, _details) => { - p_nonfile_data_end.resolve(); - const fileinfo = files[file_index++]; - stream.pipe(fileinfo.stream); - }); - - const on_field = (fieldname, value) => { - const key_parts = fieldname.split('.'); - const last_key = key_parts.pop(); - let dst = params; - for ( let i = 0; i < key_parts.length; i++ ) { - if ( ! Object.prototype.hasOwnProperty.call(dst, key_parts[i]) ) { - dst[key_parts[i]] = Object.create(null); - } - if ( !dst[key_parts[i]] || typeof dst[key_parts[i]] !== 'object' || Array.isArray(dst[key_parts[i]]) ) { - throw new Error(`Tried to set member of non-object: ${key_parts[i]} in ${fieldname}`); - } - dst = dst[key_parts[i]]; - } - if ( value && value.$ === 'file' ) { - const fileinfo = value; - const { v: size, ok: size_ok } = - valid_file_size(fileinfo.size); - if ( ! size_ok ) { - throw APIError.create('invalid_file_metadata'); - } - fileinfo.size = size; - fileinfo.stream = new PassThrough(); - const file_facade = new FileFacade(); - file_facade.values.set('stream', fileinfo.stream); - fileinfo.facade = file_facade, - files.push(fileinfo); - value = file_facade; - } - if ( Object.prototype.hasOwnProperty.call(dst, last_key) ) { - if ( ! Array.isArray(dst[last_key]) ) { - dst[last_key] = [dst[last_key]]; - } - dst[last_key].push(value); - } else { - dst[last_key] = value; - } - }; - - bb.on('field', (fieldname, value, _details) => { - const o = JSON.parse(value, (key, val) => { - if ( val !== null && typeof val === 'object' && !Array.isArray(val) ) { - return Object.assign(Object.create(null), val); - } - return val; - }); - for ( const k in o ) { - on_field(k, o[k]); - } - }); - bb.on('error', (err) => { - p_data_end.reject(err); - }); - bb.on('close', () => { - p_data_end.resolve(); - }); - - req.pipe(bb); - - (async () => { - await p_data_end; - p_nonfile_data_end.resolve(); - })(); - - await p_nonfile_data_end; - - return { params, p_data_end }; -}; diff --git a/src/backend/src/routers/drivers/list-interfaces.js b/src/backend/src/routers/drivers/list-interfaces.js deleted file mode 100644 index a0f135d9c..000000000 --- a/src/backend/src/routers/drivers/list-interfaces.js +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const eggspress = require('../../api/eggspress'); -const { Interface } = require('../../services/drivers/meta/Construct'); -const { Context } = require('../../util/context'); - -module.exports = eggspress('/drivers/list-interfaces', { - subdomain: 'api', - auth2: true, - allowedMethods: ['GET'], -}, async (req, res, next) => { - const x = Context.get(); - const svc_driver = x.get('services').get('driver'); - - const interfaces_raw = await svc_driver.list_interfaces(); - - const interfaces = {}; - for ( const interface_name in interfaces_raw ) { - if ( interfaces_raw[interface_name].no_sdk ) continue; - interfaces[interface_name] = (new Interface(interfaces_raw[interface_name], - { name: interface_name })).serialize(); - } - - res.json(interfaces); -}); diff --git a/src/backend/src/routers/drivers/usage.js b/src/backend/src/routers/drivers/usage.js deleted file mode 100644 index 9400f1ed4..000000000 --- a/src/backend/src/routers/drivers/usage.js +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const eggspress = require('../../api/eggspress'); -const { UserActorType } = require('../../services/auth/Actor'); -const { DB_READ } = require('../../services/database/consts'); -const { Context } = require('../../util/context'); - -module.exports = eggspress('/drivers/usage', { - subdomain: 'api', - auth2: true, - allowedMethods: ['GET'], -}, async (req, res, next) => { - const x = Context.get(); - - const actor = x.get('actor'); - - // Apps cannot (currently) check usage on behalf of users - if ( ! ( actor.type instanceof UserActorType ) ) { - throw APIError.create('forbidden'); - } - - const db = x.get('services').get('database').get(DB_READ, 'drivers'); - - const usages = { - user: {}, // map[str(iface:method)]{date,count,max} - apps: {}, // []{app,map[str(iface:method)]{date,count,max}} - app_objects: {}, - usages: [], - }; - - const event = { - actor, - usages: [], - }; - const svc_event = x.get('services').get('event'); - await svc_event.emit('usages.query', event); - usages.usages = event.usages; - - const user_is_verified = actor.type.user.email_confirmed; - - for ( const k in usages.apps ) { - usages.apps[k] = Object.values(usages.apps[k]); - } - - res.json({ - user: Object.values(usages.user), - apps: usages.apps, - app_objects: usages.app_objects, - usages: usages.usages, - }); -}); diff --git a/src/backend/src/routers/drivers/xd.js b/src/backend/src/routers/drivers/xd.js deleted file mode 100644 index 1cda8c7d2..000000000 --- a/src/backend/src/routers/drivers/xd.js +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const eggspress = require('../../api/eggspress'); - -const init_client_js = code => { - return ` - document.addEventListener('DOMContentLoaded', function() { - (${code})(); - }); - `; -}; - -const script = async function script () { - const call = async ({ - interface_name, - method_name, - params, - }) => { - const response = await fetch('/drivers/call', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - interface: interface_name, - method: method_name, - params, - }), - }); - return await response.json(); - }; - - const fcall = async ({ - interface_name, - method_name, - params, - }) => { - // multipart request - const form = new FormData(); - form.append('interface', interface_name); - form.append('method', method_name); - for ( const k in params ) { - form.append(k, params[k]); - } - const response = await fetch('/drivers/call', { - method: 'POST', - body: form, - }); - return await response.json(); - }; - - /* global window */ - window.addEventListener('message', async event => { - const { id, interface: interface_, method, params } = event.data; - let has_file = false; - for ( const k in params ) { - if ( params[k] instanceof File ) { - has_file = true; - break; - } - } - const result = has_file ? await fcall({ - interface_name: interface_, - method_name: method, - params, - }) : await call({ - interface_name: interface_, - method_name: method, - params, - }); - const response = { - id, - result, - }; - event.source.postMessage(response, event.origin); - }); -}; - -/** - * POST /drivers/xd - * - * This endpoint services the document which receives - * cross-document messages from the SDK and forwards - * them to the Puter Driver API. - */ -module.exports = eggspress('/drivers/xd', { - auth: true, - allowedMethods: ['GET'], -}, async (req, res, next) => { - res.type('text/html'); - res.send(` - - - - Puter Driver API - - - - - `); -}); diff --git a/src/backend/src/routers/file.js b/src/backend/src/routers/file.js deleted file mode 100644 index 0fb9ab1a4..000000000 --- a/src/backend/src/routers/file.js +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = new express.Router(); -const { subdomain, validate_signature_auth, get_url_from_req, get_descendants, id2path, get_user, sign_file } = require('../helpers'); -const { DB_WRITE } = require('../services/database/consts'); -const { UserActorType } = require('../services/auth/Actor'); -const { Actor } = require('../services/auth/Actor'); -const { LLRead } = require('../deprecated/filesystem/ll_operations/ll_read'); -const { NodeRawEntrySelector } = require('../deprecated/filesystem/node/selectors'); - -// -----------------------------------------------------------------------// -// GET /file -// -----------------------------------------------------------------------// -router.get('/file', async (req, res, next) => { - // services and "services" - /** @type {import('../services/MeteringService/MeteringService').MeteringService} */ - const meteringService = req.services.get('meteringService').meteringService; - const log = req.services.get('log-service').create('/file'); - const errors = req.services.get('error-service').create(log); - const db = req.services.get('database').get(DB_WRITE, 'filesystem'); - - // check subdomain - if ( subdomain(req) !== 'api' ) { - next(); - } - - // validate URL signature - try { - validate_signature_auth(get_url_from_req(req), 'read'); - } catch (e) { - console.log(e); - return res.status(403).send(e); - } - - let can_write = false; - try { - validate_signature_auth(get_url_from_req(req), 'write'); - can_write = true; - } catch ( _e ) { - // slent fail - } - - // modules - const uid = req.query.uid; - let download = req.query.download ?? false; - if ( download === 'true' || download === '1' || download === true ) { - download = true; - } - - // retrieve FSEntry from db - const fsentry = await db.read('SELECT * FROM fsentries WHERE uuid = ? LIMIT 1', [uid]); - - // FSEntry not found - if ( ! fsentry[0] ) - { - return res.status(400).send({ message: 'No entry found with this uid' }); - } - - // check if item owner is suspended - const user = await get_user({ id: fsentry[0].user_id }); - if ( user.suspended ) - { - return res.status(401).send({ error: 'Account suspended' }); - } - - // ---------------------------------------------------------------// - // FSEntry is dir - // ---------------------------------------------------------------// - if ( fsentry[0].is_dir ) { - // convert to path - const dirpath = await id2path(fsentry[0].id); - // get all children of this dir - const children = await get_descendants(dirpath, await get_user({ id: fsentry[0].user_id }), 1); - const signed_children = []; - if ( children.length > 0 ) { - for ( const child of children ) { - // sign file - const signed_child = await sign_file( - child, - can_write ? 'write' : 'read', - ); - signed_children.push(signed_child); - } - } - // send to client - return res.send(signed_children); - } - - // force download? - if ( download ) { - res.attachment(fsentry[0].name); - } - - // record fsentry owner - res.resource_owner = fsentry[0].user_id; - - // try to deduce content-type - const contentType = 'application/octet-stream'; - - // update `accessed` - db.write( - 'UPDATE fsentries SET accessed = ? WHERE `id` = ?', - [Date.now() / 1000, fsentry[0].id], - ); - - const range = req.headers.range; - const ownerActor = new Actor({ - type: new UserActorType({ - user: user, - }), - }); - const fileSize = fsentry[0].size; - - res.setHeader('Accept-Ranges', 'bytes'); - - const parseRangeHeader = (rangeHeader) => { - // Check if this is a multipart range request - if ( rangeHeader.includes(',') ) { - // For now, we'll only serve the first range in multipart requests - // as the underlying storage layer doesn't support multipart responses - const firstRange = rangeHeader.split(',')[0].trim(); - const matches = firstRange.match(/bytes=(\d+)-(\d*)/); - if ( ! matches ) return null; - - const start = parseInt(matches[1], 10); - const end = matches[2] ? parseInt(matches[2], 10) : null; - - return { start, end, isMultipart: true }; - } - - // Single range request - const matches = rangeHeader.match(/bytes=(\d+)-(\d*)/); - if ( ! matches ) return null; - - const start = parseInt(matches[1], 10); - const end = matches[2] ? parseInt(matches[2], 10) : null; - - return { start, end, isMultipart: false }; - }; - - //-------------------------------------------------- - // Range - //-------------------------------------------------- - if ( range ) { - res.status(206); - const rangeInfo = parseRangeHeader(req.headers['range']); - if ( rangeInfo ) { - const { start, end, isMultipart } = rangeInfo; - - // For open-ended ranges, we need to calculate the actual end byte - let actualEnd = end; - let fileSize = null; - - try { - fileSize = fsentry[0].size; - if ( end === null ) { - actualEnd = fileSize - 1; // File size is 1-based, end byte is 0-based - } - } catch (e) { - // If we can't get file size, we'll let the storage layer handle it - // and not set Content-Range header - actualEnd = null; - fileSize = null; - } - - if ( actualEnd !== null ) { - const totalSize = fileSize !== null ? fileSize : '*'; - const contentRange = `bytes ${start}-${actualEnd}/${totalSize}`; - res.set('Content-Range', contentRange); - } - - // If this was a multipart request, modify the range header to only include the first range - if ( isMultipart ) { - req.headers['range'] = end !== null - ? `bytes=${start}-${end}` - : `bytes=${start}-`; - } - } - } - - //-------------------------------------------------- - // No range - //-------------------------------------------------- - // set content-type, if available - if ( contentType !== null ) { - res.setHeader('Content-Type', contentType); - } - - const svc_filesystem = req.services.get('filesystem'); - - // stream data from S3 - try { - - const fsNode = await svc_filesystem.node( - new NodeRawEntrySelector(fsentry[0]), - ); - - const ll_read = new LLRead(); - const stream = await ll_read.run({ - range, - no_acl: true, - actor: req.actor ?? ownerActor, - fsNode, - }); - - return stream.pipe(res); - } catch (e) { - errors.report('read from storage', { - source: e, - trace: true, - alarm: true, - }); - return res.type('application/json').status(500).send({ message: 'There was an internal problem reading the file.' }); - } -}); - -module.exports = router; diff --git a/src/backend/src/routers/filesystem_api/batch/PathResolver.js b/src/backend/src/routers/filesystem_api/batch/PathResolver.js deleted file mode 100644 index 81b21b538..000000000 --- a/src/backend/src/routers/filesystem_api/batch/PathResolver.js +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../../api/APIError.js'); -const { relativeSelector } = require('../../../deprecated/filesystem/node/selectors.js'); -const ERR_INVALID_PATHREF = 'Invalid path reference in path: '; -const ERR_UNKNOWN_PATHREF = 'Unknown path reference in path: '; - -/** - * Resolves path references in batch requests. - * - * A path reference is a path that starts with a dollar sign ($). - * It will resolve to the path that was returned by the operation - * with the same name in its `as` field. - * - * For example, if the operation `mkdir` has an `as` field with the - * value `newdir`, then the path `$newdir` will resolve to the path - * that was returned by the `mkdir` operation. - */ -module.exports = class PathResolver { - constructor ({ actor }) { - this.references = {}; - this.selectors = {}; - this.meta = {}; - this.actor = actor; - - this.listeners = {}; - - this.log = globalThis.services.get('log-service').create('path-resolver'); - } - - /** - * putPath - Add a path reference. - * - * The path reference will be resolved to the given path. - * - * @param {string} refName - The name of the path reference. - * @param {string} path - The path to resolve to. - */ - putPath (refName, path) { - this.references[refName] = { path }; - } - - putSelector (refName, selector, meta) { - this.log.debug(`putSelector called for: ${refName}`); - this.selectors[refName] = selector; - this.meta[refName] = meta; - if ( ! this.listeners.hasOwnProperty(refName) ) return; - - for ( const lis of this.listeners[refName] ) lis(); - } - - /** - * resolve - Resolve a path reference. - * - * If the given path does not start with a dollar sign ($), - * it will be returned as-is. Otherwise, the path reference - * will be resolved to the path that was given to `putPath`. - * - * @param {string} inputPath - * @returns {string} The resolved path. - */ - - resolve (inputPath) { - const refName = this.getReferenceUsed(inputPath); - if ( refName === null ) return inputPath; - if ( ! this.references.hasOwnProperty(refName) ) { - throw APIError.create(400, ERR_UNKNOWN_PATHREF + refName); - } - - return this.references[refName].path + - inputPath.substring(refName.length + 1); - } - - async awaitSelector (inputPath) { - // TODO: I feel like there's a better way to get username - const username = this.actor.type.user.username; - if ( inputPath.startsWith('~/') ) { - return `/${username}/${inputPath.substring(2)}`; - } - if ( inputPath === '~' ) { - return `/${username}`; - } - if ( inputPath.startsWith('.') ) { - throw APIError.create('unresolved_relative_path', null, { path: inputPath }); - } - const refName = this.getReferenceUsed(inputPath); - if ( refName === null ) return inputPath; - - this.log.debug(`-- awaitSelector -- input path is ${inputPath}`); - this.log.debug(`-- awaitSelector -- refName is ${refName}`); - if ( ! this.selectors.hasOwnProperty(refName) ) { - this.log.debug('-- awaitSelector -- doing the await'); - if ( ! this.listeners[refName] ) { - this.listeners[refName] = []; - } - await new Promise (rslv => { - this.listeners[refName].push(rslv); - }); - } - - const subpath = inputPath.substring(refName.length + 1); - const selector = this.selectors[refName]; - - return relativeSelector(selector, subpath); - } - - getMeta (inputPath) { - const refName = this.getReferenceUsed(inputPath); - if ( refName === null ) return null; - - return this.meta[refName]; - } - - getReferenceUsed (inputPath) { - if ( ! inputPath.startsWith('$') ) return null; - - const endOfRefName = inputPath.includes('/') - ? inputPath.indexOf('/', 1) : inputPath.length; - const refName = inputPath.substring(1, endOfRefName); - - if ( refName === '' ) { - throw APIError.create(400, ERR_INVALID_PATHREF + inputPath); - } - - return refName; - } -}; diff --git a/src/backend/src/routers/filesystem_api/batch/all.js b/src/backend/src/routers/filesystem_api/batch/all.js deleted file mode 100644 index bae83260e..000000000 --- a/src/backend/src/routers/filesystem_api/batch/all.js +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../../api/APIError'); -const eggspress = require('../../../api/eggspress'); -const { Context } = require('../../../util/context'); -const Busboy = require('busboy'); -const { BatchExecutor } = require('../../../deprecated/filesystem/batch/BatchExecutor'); -const { TeePromise } = require('@heyputer/putility').libs.promise; -const { MovingMode } = require('../../../util/opmath'); -const { get_app } = require('../../../helpers'); -const { valid_file_size } = require('../../../util/validutil'); -const { OnlyOnceFn } = require('../../../util/fnutil.js'); - -module.exports = eggspress('/batch', { - subdomain: 'api', - verified: true, - auth2: true, - // json: true, - // files: ['file'], - // multest: true, - // multipart_jsons: ['operation'], - allowedMethods: ['POST'], -}, async (req, res, _next) => { - const log = req.services.get('log-service').create('batch'); - const errors = req.services.get('error-service').create(log); - - const x = Context.get(); - - let app; - if ( req.body.app_uid ) { - // eslint-disable-next-line no-unused-vars - app = await get_app({ uid: req.body.app_uid }); - } - - const expected_metadata = { - original_client_socket_id: undefined, - socket_id: undefined, - operation_id: undefined, - }; - - // Errors not within operations that can only be detected - // while the request is streaming will be assigned to this - // value. - let request_errors_ = []; - - let frame; - const create_frame = () => { - const operationTraceSvc = x.get('services').get('operationTrace'); - frame = operationTraceSvc.add_frame_sync('api:/batch', x) - .attr('gui_metadata', { - ...expected_metadata, - user_id: req.user.id, - }) - ; - x.set(operationTraceSvc.ckey('frame'), frame); - - const svc_clientOperation = x.get('services').get('client-operation'); - const tracker = svc_clientOperation.add_operation({ - name: 'batch', - tags: ['fs'], - frame, - metadata: { - user_id: req.user.id, - }, - }); - x.set(svc_clientOperation.ckey('tracker'), tracker); - }; - - // Make sure usage is cached - const sizeService = x.get('services').get('sizeService'); - await sizeService.get_usage(req.user.id); - - globalThis.average_chunk_size = new MovingMode({ - alpha: 0.7, - initial: 1, - }); - - //------------------------------------------------------------- - // Variables used by busboy callbacks - //------------------------------------------------------------- - // --- library - const operation_requires_file = op_spec => { - if ( op_spec.op === 'write' ) return true; - return false; - }; - if ( ! req.actor ) { - throw new Error('Actor is missing here'); - } - const batch_exe = new BatchExecutor(x, { - log, - errors, - actor: req.actor, - }); - // --- state - const pending_operations = []; - const response_promises = []; - const fileinfos = []; - let request_error = null; - - const on_nonfile_data_end = OnlyOnceFn(() => { - if ( request_error ) { - return; - } - - const indexes_to_remove = []; - - for ( let i = 0 ; i < pending_operations.length ; i++ ) { - const op_spec = pending_operations[i]; - if ( ! operation_requires_file(op_spec) ) { - indexes_to_remove.push(i); - log.debug(`executing ${op_spec.op}`); - response_promises[i] = batch_exe.exec_op(req, op_spec); - } else { - // no handler - } - } - - for ( let i = indexes_to_remove.length - 1 ; i >= 0 ; i-- ) { - const index = indexes_to_remove[i]; - pending_operations.splice(index, 1)[0]; - } - }); - - //------------------------------------------------------------- - // Multipart processing (using busboy) - //------------------------------------------------------------- - const busboy = Busboy({ - headers: req.headers, - }); - - const still_reading = new TeePromise(); - - busboy.on('field', (fieldname, value, details) => { - try { - if ( details.fieldnameTruncated ) { - throw new Error('fieldnameTruncated'); - } - if ( details.valueTruncated ) { - throw new Error('valueTruncated'); - } - - if ( Object.prototype.hasOwnProperty.call(expected_metadata, fieldname) ) { - expected_metadata[fieldname] = value; - req.body[fieldname] = value; - return; - } - - if ( fieldname === 'fileinfo' ) { - const fileinfo = JSON.parse(value); - const { v: size, ok: size_ok } = valid_file_size(fileinfo.size); - if ( ! size_ok ) { - throw APIError.create('invalid_file_metadata'); - } - fileinfo.size = size; - fileinfos.push(fileinfo); - return; - } - - if ( ! frame ) { - create_frame(); - } - - if ( fieldname === 'operation' ) { - const op_spec = JSON.parse(value); - batch_exe.total++; - pending_operations.push(op_spec); - response_promises.push(null); - return; - } - - req.body[fieldname] = value; - } catch (e) { - request_error = e; - req.unpipe(busboy); - res.set('Connection', 'close'); - res.sendStatus(400); - } - }); - - busboy.on('file', async (fieldname, stream ) => { - if ( batch_exe.total_tbd ) { - batch_exe.total_tbd = false; - on_nonfile_data_end(); - } - - if ( fileinfos.length == 0 ) { - request_errors_.push(new APIError('batch_too_many_files')); - stream.on('data', () => { - }); - stream.on('end', () => { - stream.destroy(); - }); - return; - } - - const file = fileinfos.shift(); - file.stream = stream; - - if ( pending_operations.length == 0 ) { - request_errors_.push(new APIError('batch_too_many_files')); - // Elimiate the stream - stream.on('data', () => { - }); - stream.on('end', () => { - stream.destroy(); - }); - return; - } - - const op_spec = pending_operations.shift(); - - // Copy thumbnail from fileinfo to the file object if provided - if ( file.thumbnail ) { - op_spec.thumbnail = file.thumbnail; - } - - // index in response_promises is first null value - const index = response_promises.findIndex(p => p === null); - response_promises[index] = batch_exe.exec_op(req, op_spec, file); - // response_promises[index] = Promise.resolve(out); - }); - - busboy.on('close', () => { - log.debug('busboy close'); - still_reading.resolve(); - }); - - req.pipe(busboy); - - //------------------------------------------------------------- - // Awaiting responses - //------------------------------------------------------------- - await still_reading; - on_nonfile_data_end(); - - if ( request_error ) { - return; - } - - log.debug('waiting for operations'); - let responsePromises = response_promises; - // let responsePromises = batch_exe.responsePromises; - const results = await Promise.all(responsePromises); - log.debug('sending response'); - - frame.done(); - - if ( pending_operations.length ) { - - // eslint-disable-next-line no-unused-vars - for ( const _op_spec of pending_operations ) { - const err = new APIError('batch_missing_file'); - request_errors_.push(err); - } - } - - if ( request_errors_ ) { - results.push(...request_errors_.map(e => { - return e.serialize(); - })); - } - - res.status(batch_exe.hasError ? 218 : 200).send({ results }); -}); diff --git a/src/backend/src/routers/filesystem_api/cache.js b/src/backend/src/routers/filesystem_api/cache.js deleted file mode 100644 index bf13c9d47..000000000 --- a/src/backend/src/routers/filesystem_api/cache.js +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const eggspress = require('../../api/eggspress.js'); -const { Context } = require('../../util/context.js'); - -module.exports = eggspress('/cache/last-change-timestamp', { - subdomain: 'api', - auth2: true, - verified: true, - fs: true, - json: true, - allowedMethods: ['GET'], -}, async (req, res) => { - /** @type {import('../../clients/dynamodb/DynamoKVStore/DynamoKVStore.js').DynamoKVStore} */ - const kvStore = Context.get('services').get('puter-kvstore'); - const timestamp = await kvStore.get({ key: `last_change_timestamp:${req.user?.id}` }); - res.json({ timestamp }); -}); diff --git a/src/backend/src/routers/filesystem_api/copy.js b/src/backend/src/routers/filesystem_api/copy.js deleted file mode 100644 index c56e0a7db..000000000 --- a/src/backend/src/routers/filesystem_api/copy.js +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const eggspress = require('../../api/eggspress.js'); -const FSNodeParam = require('../../api/filesystem/FSNodeParam.js'); -const { HLCopy } = require('../../deprecated/filesystem/hl_operations/hl_copy.js'); -const { Context } = require('../../util/context.js'); -const { getTracer } = require('../../util/otelutil.js'); - -// -----------------------------------------------------------------------// -// POST /copy -// -----------------------------------------------------------------------// -module.exports = eggspress('/copy', { - subdomain: 'api', - auth2: true, - verified: true, - fs: true, - json: true, - allowedMethods: ['POST'], - parameters: { - source: new FSNodeParam('source'), - destination: new FSNodeParam('destination'), - }, -}, async (req, res) => { - const user = req.user; - const dedupe_name = - req.body.dedupe_name ?? - req.body.change_name ?? false; - - let frame; - { - const x = Context.get(); - const operationTraceSvc = x.get('services').get('operationTrace'); - frame = (await operationTraceSvc.add_frame('api:/copy')) - .attr('gui_metadata', { - original_client_socket_id: req.body.original_client_socket_id, - socket_id: req.body.socket_id, - operation_id: req.body.operation_id, - user_id: req.user.id, - item_upload_id: req.body.item_upload_id, - }) - ; - x.set(operationTraceSvc.ckey('frame'), frame); - } - - const tracer = getTracer(); - await tracer.startActiveSpan('filesystem_api.copy', async span => { - - // === upcoming copy behaviour === - const hl_copy = new HLCopy(); - const response = await hl_copy.run({ - destination_or_parent: req.values.destination, - source: req.values.source, - new_name: req.body.new_name, - - overwrite: req.body.overwrite ?? false, - dedupe_name, - - user: user, - }); - - span.end(); - frame.done(); - return res.send([response]); - }); - - // res.send(new_fsentries) -}); diff --git a/src/backend/src/routers/filesystem_api/delete.js b/src/backend/src/routers/filesystem_api/delete.js deleted file mode 100644 index 36da7b64a..000000000 --- a/src/backend/src/routers/filesystem_api/delete.js +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const config = require('../../config.js'); -const eggspress = require('../../api/eggspress.js'); -const { HLRemove } = require('../../deprecated/filesystem/hl_operations/hl_remove.js'); -const FSNodeParam = require('../../api/filesystem/FSNodeParam.js'); - -// -----------------------------------------------------------------------// -// POST /delete -// -----------------------------------------------------------------------// -module.exports = eggspress('/delete', { - subdomain: 'api', - auth2: true, - json: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - // check if user is verified - if ( (config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed ) - { - return res.status(400).send({ code: 'account_is_not_verified', message: 'Account is not verified' }); - } - - const user = req.user; - const paths = req.body.paths; - const recursive = req.body.recursive ?? false; - const descendants_only = req.body.descendants_only ?? false; - - if ( paths === undefined ) - { - return res.status(400).send('paths is required'); - } - else if ( ! Array.isArray(paths) ) - { - return res.status(400).send('paths must be an array'); - } - else if ( paths.length === 0 ) - { - return res.status(400).send('paths cannot be empty'); - } - - // try to delete each path in the array one by one (if glob, resolve first) - // TODO: remove this pseudo-batch - for ( const item_path of paths ) { - const target = await (new FSNodeParam('path')).consolidate({ - req: { user }, - getParam: () => item_path, - }); - const hl_remove = new HLRemove(); - await hl_remove.run({ - target, - user, - recursive, - descendants_only, - }); - - // send realtime success msg to client - const svc_socketio = req.services.get('socketio'); - svc_socketio.send({ room: req.user.id }, 'item.removed', { - path: item_path, - descendants_only: descendants_only, - }); - } - - res.send({}); -}); diff --git a/src/backend/src/routers/filesystem_api/mkdir.js b/src/backend/src/routers/filesystem_api/mkdir.js deleted file mode 100644 index e764f8845..000000000 --- a/src/backend/src/routers/filesystem_api/mkdir.js +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const eggspress = require('../../api/eggspress'); -const FSNodeParam = require('../../api/filesystem/FSNodeParam'); -const { HLMkdir } = require('../../deprecated/filesystem/hl_operations/hl_mkdir'); -const { Context } = require('../../util/context'); -const { boolify } = require('../../util/hl_types'); - -// -----------------------------------------------------------------------// -// POST /mkdir -// -----------------------------------------------------------------------// -module.exports = eggspress('/mkdir', { - subdomain: 'api', - verified: true, - auth2: true, - fs: true, - json: true, - allowedMethods: ['POST'], - parameters: { - parent: new FSNodeParam('parent', { optional: true }), - shortcut_to: new FSNodeParam('shortcut_to', { optional: true }), - }, -}, async (req, res, next) => { - // validation - if ( req.body.path === undefined ) - { - return res.status(400).send({ message: 'path is required' }); - } - else if ( req.body.path === '' ) - { - return res.status(400).send({ message: 'path cannot be empty' }); - } - else if ( req.body.path === null ) - { - return res.status(400).send({ message: 'path cannot be null' }); - } - else if ( typeof req.body.path !== 'string' ) - { - return res.status(400).send({ message: 'path must be a string' }); - } - - const overwrite = req.body.overwrite ?? false; - - // modules - let frame; - { - const x = Context.get(); - const operationTraceSvc = x.get('services').get('operationTrace'); - frame = (await operationTraceSvc.add_frame('api:/mkdir')) - .attr('gui_metadata', { - original_client_socket_id: req.body.original_client_socket_id, - operation_id: req.body.operation_id, - user_id: req.user.id, - }) - ; - x.set(operationTraceSvc.ckey('frame'), frame); - } - - // PEDANTRY: in theory there's no difference between creating an object just to call - // a method on it and calling a utility function. HLMkdir is a class because - // it uses traits and supports dependency injection, but those features are - // not concerns of this endpoint handler. - const hl_mkdir = new HLMkdir(); - const response = await hl_mkdir.run({ - parent: req.values.parent, - path: req.body.path, - overwrite: overwrite, - dedupe_name: req.body.dedupe_name ?? false, - create_missing_parents: boolify(req.body.create_missing_ancestors ?? - req.body.create_missing_parents), - actor: req.actor, - shortcut_to: req.values.shortcut_to, - }); - - // TODO: maybe endpoint handlers are operations too. It would be much - // nicer to not have to explicitly call frame.done() here. - frame.done(); - - return res.send(response); -}); diff --git a/src/backend/src/routers/filesystem_api/move.js b/src/backend/src/routers/filesystem_api/move.js deleted file mode 100644 index 6222a312e..000000000 --- a/src/backend/src/routers/filesystem_api/move.js +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const eggspress = require('../../api/eggspress.js'); -const FSNodeParam = require('../../api/filesystem/FSNodeParam.js'); -const { HLMove } = require('../../deprecated/filesystem/hl_operations/hl_move.js'); -const { Context } = require('../../util/context.js'); -const { getTracer } = require('../../util/otelutil.js'); - -// -----------------------------------------------------------------------// -// POST /move -// -----------------------------------------------------------------------// -module.exports = eggspress('/move', { - subdomain: 'api', - auth2: true, - verified: true, - fs: true, - json: true, - allowedMethods: ['POST'], - parameters: { - source: new FSNodeParam('source'), - destination: new FSNodeParam('destination'), - }, -}, async (req, res, next) => { - const dedupe_name = - req.body.dedupe_name ?? - req.body.change_name ?? false; - - let frame; - { - const x = Context.get(); - const operationTraceSvc = x.get('services').get('operationTrace'); - frame = (await operationTraceSvc.add_frame('api:/move')) - .attr('gui_metadata', { - original_client_socket_id: req.body.original_client_socket_id, - socket_id: req.body.socket_id, - operation_id: req.body.operation_id, - user_id: req.user.id, - item_upload_id: req.body.item_upload_id, - }) - ; - x.set(operationTraceSvc.ckey('frame'), frame); - } - - const tracer = getTracer(); - await tracer.startActiveSpan('filesystem_api.move', async span => { - const hl_move = new HLMove(); - const response = await hl_move.run({ - destination_or_parent: req.values.destination, - source: req.values.source, - user: req.user, - new_name: req.body.new_name, - overwrite: req.body.overwrite ?? false, - dedupe_name, - new_metadata: req.body.new_metadata, - create_missing_parents: req.body.create_missing_parents ?? false, - }); - - span.end(); - frame.done(); - res.send(response); - }); -}); diff --git a/src/backend/src/routers/filesystem_api/read.js b/src/backend/src/routers/filesystem_api/read.js deleted file mode 100644 index 46f1cd1c1..000000000 --- a/src/backend/src/routers/filesystem_api/read.js +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const APIError = require('../../api/APIError.js'); -const eggspress = require('../../api/eggspress'); -const FSNodeParam = require('../../api/filesystem/FSNodeParam'); -const { HLRead } = require('../../deprecated/filesystem/hl_operations/hl_read'); - -module.exports = eggspress('/read', { - subdomain: 'api', - auth2: true, - verified: true, - fs: true, - json: true, - allowedMethods: ['GET'], - alias: { - path: 'file', - uid: 'file', - }, - parameters: { - fsNode: new FSNodeParam('file'), - }, -}, async (req, res, next) => { - const line_count = !req.query.line_count ? undefined : parseInt(req.query.line_count); - const byte_count = !req.query.byte_count ? undefined : parseInt(req.query.byte_count); - const offset = !req.query.offset ? undefined : parseInt(req.query.offset); - - if ( line_count && (!Number.isInteger(line_count) || line_count < 1) ) { - throw new APIError(400, '`line_count` must be a positive integer'); - } - if ( byte_count && (!Number.isInteger(byte_count) || byte_count < 1) ) { - throw new APIError(400, '`byte_count` must be a positive integer'); - } - if ( offset && (!Number.isInteger(offset) || offset < 0) ) { - throw new APIError(400, '`offset` must be a positive integer'); - } - if ( byte_count && line_count ) { - throw new APIError(400, 'cannot use both line_count and byte_count'); - } - - if ( offset && !byte_count ) { - throw APIError.create('field_only_valid_with_other_field', null, { - key: 'offset', - other_key: 'byte_count', - }); - } - - // Helper function to parse Range header - const parseRangeHeader = (rangeHeader) => { - // Check if this is a multipart range request - if ( rangeHeader.includes(',') ) { - // For now, we'll only serve the first range in multipart requests - // as the underlying storage layer doesn't support multipart responses - const firstRange = rangeHeader.split(',')[0].trim(); - const matches = firstRange.match(/bytes=(\d+)-(\d*)/); - if ( ! matches ) return null; - - const start = parseInt(matches[1], 10); - const end = matches[2] ? parseInt(matches[2], 10) : null; - - return { start, end, isMultipart: true }; - } - - // Single range request - const matches = rangeHeader.match(/bytes=(\d+)-(\d*)/); - if ( ! matches ) return null; - - const start = parseInt(matches[1], 10); - const end = matches[2] ? parseInt(matches[2], 10) : null; - - return { start, end, isMultipart: false }; - }; - - if ( req.headers['range'] ) { - res.status(206); - - // Parse the Range header and set Content-Range - const rangeInfo = parseRangeHeader(req.headers['range']); - if ( rangeInfo ) { - const { start, end, isMultipart } = rangeInfo; - - // For open-ended ranges, we need to calculate the actual end byte - let actualEnd = end; - let fileSize = null; - - try { - fileSize = await req.values.fsNode.get('size'); - if ( end === null ) { - actualEnd = fileSize - 1; // File size is 1-based, end byte is 0-based - } - } catch (e) { - // If we can't get file size, we'll let the storage layer handle it - // and not set Content-Range header - actualEnd = null; - fileSize = null; - } - - if ( actualEnd !== null ) { - const totalSize = fileSize !== null ? fileSize : '*'; - const contentRange = `bytes ${start}-${actualEnd}/${totalSize}`; - res.set('Content-Range', contentRange); - } - - // If this was a multipart request, modify the range header to only include the first range - if ( isMultipart ) { - req.headers['range'] = end !== null - ? `bytes=${start}-${end}` - : `bytes=${start}-`; - } - } - } - res.set({ 'Accept-Ranges': 'bytes' }); - - const hl_read = new HLRead(); - const stream = await hl_read.run({ - ...(req.headers['range'] ? { range: req.headers['range'] } : { - line_count, - byte_count, - offset, - }), - fsNode: req.values.fsNode, - user: req.user, - - version_id: req.query.version_id, - }); - - res.set('Content-Type', 'application/octet-stream'); - - stream.pipe(res); -}); diff --git a/src/backend/src/routers/filesystem_api/readdir-subdomains.mjs b/src/backend/src/routers/filesystem_api/readdir-subdomains.mjs deleted file mode 100644 index 09ea0b580..000000000 --- a/src/backend/src/routers/filesystem_api/readdir-subdomains.mjs +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (C) 2026-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { Context } from '../../util/context.js'; -import eggspress from '../../api/eggspress.js'; -import { DB_READ } from '../../services/database/consts.js'; -import config from '../../config.js'; - -// -----------------------------------------------------------------------// -// POST /readdir-subdomains -// -----------------------------------------------------------------------// -export default eggspress('/readdir-subdomains', { - subdomain: 'api', - auth2: true, - verified: true, - json: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - const log = (() => { - return Context.get('services').get('log-service').create('readdir-subdomains', { - concern: 'filesystem', - }); - })(); - log.debug('readdir-subdomains: batch fetch subdomains'); - - const { directory_ids } = req.body; - - if ( !Array.isArray(directory_ids) || directory_ids.length === 0 ) { - return res.status(400).send({ - code: 'invalid_request', - message: 'directory_ids must be a non-empty array', - }); - } - - const user = req.user; - const db = Context.get().get('services').get('database').get(DB_READ, 'filesystem'); - - // Note: directory_ids are actually UUIDs (not database IDs) because fsentry.id is set to uuid in getSafeEntry() - // We need to convert UUIDs to database IDs first - // Convert UUIDs to database IDs - const uuidPlaceholders = directory_ids.map(() => '?').join(','); - const fsentries = await db.read(`SELECT id, uuid FROM fsentries WHERE uuid IN (${uuidPlaceholders})`, - directory_ids); - - // Create maps: uuid -> db_id and db_id -> uuid - const uuidToDbId = new Map(); - const dbIdToUuid = new Map(); - for ( const fsentry of fsentries ) { - uuidToDbId.set(fsentry.uuid, fsentry.id); - dbIdToUuid.set(fsentry.id, fsentry.uuid); - } - - const dbIds = Array.from(uuidToDbId.values()); - - if ( dbIds.length === 0 ) { - return res.send(directory_ids.map(dirUuid => ({ - directory_id: dirUuid, - subdomains: [], - has_website: false, - }))); - } - - // Build the query with placeholders using database IDs - const placeholders = dbIds.map(() => '?').join(','); - const rows = await db.read(`SELECT root_dir_id, subdomain, uuid - FROM subdomains - WHERE root_dir_id IN (${placeholders}) AND user_id = ?`, - [...dbIds, user.id]); - - // Group subdomains by database ID - const subdomainsByDbId = {}; - - for ( const row of rows ) { - if ( ! subdomainsByDbId[row.root_dir_id] ) { - subdomainsByDbId[row.root_dir_id] = []; - } - subdomainsByDbId[row.root_dir_id].push({ - subdomain: row.subdomain, - address: `${config.protocol}://${row.subdomain}.puter.site`, - uuid: row.uuid, - }); - } - - // Build response: array of { directory_id, subdomains, has_website } - // Map back to original UUIDs (directory_ids) - const result = directory_ids.map(dirUuid => { - const dbId = uuidToDbId.get(dirUuid); - const subdomains = dbId ? (subdomainsByDbId[dbId] || []) : []; - const has_website = subdomains.length > 0; - - return { - directory_id: dirUuid, - subdomains: subdomains, - has_website: has_website, - }; - }); - - res.send(result); - return; -}); diff --git a/src/backend/src/routers/filesystem_api/readdir.js b/src/backend/src/routers/filesystem_api/readdir.js deleted file mode 100644 index de4fe2faf..000000000 --- a/src/backend/src/routers/filesystem_api/readdir.js +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const { Context } = require('../../util/context.js'); -const eggspress = require('../../api/eggspress.js'); -const FSNodeParam = require('../../api/filesystem/FSNodeParam.js'); -const FlagParam = require('../../api/filesystem/FlagParam.js'); -const { HLReadDir } = require('../../deprecated/filesystem/hl_operations/hl_readdir.js'); - -// -----------------------------------------------------------------------// -// POST /readdir -// -----------------------------------------------------------------------// -module.exports = eggspress('/readdir', { - subdomain: 'api', - auth2: true, - verified: true, - fs: true, - json: true, - allowedMethods: ['POST'], - alias: { - path: 'subject', - uid: 'subject', - }, - parameters: { - subject: new FSNodeParam('subject'), - recursive: new FlagParam('recursive', { optional: true }), - no_thumbs: new FlagParam('no_thumbs', { optional: true }), - no_assocs: new FlagParam('no_assocs', { optional: true }), - no_subdomains: new FlagParam('no_subdomains', { optional: true }), - }, -}, async (req, res, next) => { - let log; { - const x = Context.get(); - log = x.get('services').get('log-service').create('readdir', { - concern: 'filesystem', - }); - log.debug(`readdir: ${req.body.subject || req.body.path || req.body.uid}`); - } - - const subject = req.values.subject; - const recursive = req.values.recursive; - const no_thumbs = req.values.no_thumbs; - const no_assocs = req.values.no_assocs; - const no_subdomains = req.values.no_subdomains; - - const hl_readdir = new HLReadDir(); - const result = await hl_readdir.run({ - subject, - recursive, - no_thumbs, - no_assocs, - no_subdomains, - user: req.user, - actor: req.actor, - }); - - // check for duplicate names - if ( ! recursive ) { - const names = new Set(); - for ( const entry of result ) { - if ( names.has(entry.name) ) { - log.error(`Duplicate name: ${entry.name}`); - // throw new Error(`Duplicate name: ${entry.name}`); - } - names.add(entry.name); - } - } - - res.send(result); - return; -}); diff --git a/src/backend/src/routers/filesystem_api/rename.js b/src/backend/src/routers/filesystem_api/rename.js deleted file mode 100644 index 09cd48b20..000000000 --- a/src/backend/src/routers/filesystem_api/rename.js +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const eggspress = require('../../api/eggspress.js'); -const APIError = require('../../api/APIError.js'); -const { Context } = require('../../util/context.js'); -const FSNodeParam = require('../../api/filesystem/FSNodeParam.js'); -const { DB_WRITE } = require('../../services/database/consts.js'); - -// -----------------------------------------------------------------------// -// POST /rename -// -----------------------------------------------------------------------// -module.exports = eggspress('/rename', { - subdomain: 'api', - auth2: true, - verified: true, - fs: true, - json: true, - allowedMethods: ['POST'], - alias: { uid: 'path' }, - parameters: { - subject: new FSNodeParam('path'), - }, -}, async (req, res, next) => { - if ( ! req.body.new_name ) { - throw APIError.create('field_missing', null, { - key: 'new_name', - }); - } - if ( typeof req.body.new_name !== 'string' ) { - throw APIError.create('field_invalid', null, { - key: 'new_name', - expected: 'string', - got: typeof req.body.new_name, - }); - } - - // modules - const db = req.services.get('database').get(DB_WRITE, 'filesystem'); - const mime = require('mime-types'); - const { get_app, validate_fsentry_name, id2path } = require('../../helpers.js'); - const _path = require('path'); - - // new_name validation - try { - validate_fsentry_name(req.body.new_name); - } catch (e) { - return res.status(400).send({ - error: { - message: e.message, - }, - }); - } - - const { subject } = req.values; - - //get fsentry - if ( ! await subject.exists() ) { - throw APIError.create('subject_does_not_exist'); - } - - // Access control - { - const actor = Context.get('actor'); - const svc_acl = Context.get('services').get('acl'); - if ( ! await svc_acl.check(actor, subject, 'write') ) { - throw await svc_acl.get_safe_acl_error(actor, subject, 'write'); - } - } - - await subject.fetchEntry(); - let fsentry = subject.entry; - - // immutable - if ( fsentry.immutable ) { - return res.status(400).send({ - error: { - message: 'Immutable: cannot rename.', - }, - }); - } - - let res1; - - // parent is root - if ( fsentry.parent_uid === null ) { - try { - res1 = await db.read('SELECT uuid FROM fsentries WHERE parent_uid IS NULL AND name = ? AND id != ? LIMIT 1', - [ - //name - req.body.new_name, - await subject.get('mysql-id'), - ]); - } catch (e) { - console.log(e); - } - } - // parent is regular dir - else { - res1 = await db.read('SELECT uuid FROM fsentries WHERE parent_uid = ? AND name = ? AND id != ? LIMIT 1', - [ - //parent_uid - fsentry.parent_uid, - //name - req.body.new_name, - await subject.get('mysql-id'), - ]); - } - if ( res1[0] ) { - throw APIError.create('item_with_same_name_exists', null, { - entry_name: req.body.new_name, - }); - } - - const old_path = await id2path(await subject.get('mysql-id')); - const new_path = _path.join(_path.dirname(old_path), req.body.new_name); - - // update `name` - await db.write('UPDATE fsentries SET name = ?, path = ? WHERE id = ?', - [req.body.new_name, new_path, await subject.get('mysql-id')]); - - const filesystem = req.services.get('filesystem'); - await filesystem.update_child_paths(old_path, new_path, req.user.id); - - // associated_app - let associated_app; - if ( fsentry.associated_app_id ) { - const app = await get_app({ id: fsentry.associated_app_id }); - // remove some privileged information - delete app.id; - delete app.approved_for_listing; - delete app.approved_for_opening_items; - delete app.godmode; - delete app.owner_user_id; - // add to array - associated_app = app; - } else { - associated_app = {}; - } - - // send the fsentry of the new object created - const contentType = mime.contentType(req.body.new_name); - const return_obj = { - uid: req.body.uid, - name: req.body.new_name, - is_dir: fsentry.is_dir, - path: new_path, - old_path: old_path, - type: contentType || null, - associated_app: associated_app, - original_client_socket_id: req.body.original_client_socket_id, - }; - - // send realtime success msg to client - const svc_socketio = req.services.get('socketio'); - svc_socketio.send({ room: req.user.id }, 'item.renamed', return_obj); - - (async () => { - try { - const svc_event = req.services.get('event'); - await svc_event.emit('fs.rename', { - uid: fsentry.uuid, - new_name: req.body.new_name, - }); - } catch (e) { - const log = req.services.get('log-service').create('rename-endpoint'); - const errors = req.services.get('error-service').create(log); - errors.report('emit.rename', { - alarm: true, - source: e, - }); - } - })(); - - return res.send(return_obj); -}); diff --git a/src/backend/src/routers/filesystem_api/search.js b/src/backend/src/routers/filesystem_api/search.js deleted file mode 100644 index d2884aab4..000000000 --- a/src/backend/src/routers/filesystem_api/search.js +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const eggspress = require('../../api/eggspress'); -const { HLNameSearch } = require('../../deprecated/filesystem/hl_operations/hl_name_search'); - -module.exports = eggspress('/search', { - subdomain: 'api', - auth2: true, - verified: true, - fs: true, - json: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - const hl_name_search = new HLNameSearch(); - const result = await hl_name_search.run({ - actor: req.actor, - term: req.body.text, - }); - res.send(result); -}); diff --git a/src/backend/src/routers/filesystem_api/stat.js b/src/backend/src/routers/filesystem_api/stat.js deleted file mode 100644 index 6ff546cb6..000000000 --- a/src/backend/src/routers/filesystem_api/stat.js +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const eggspress = require('../../api/eggspress.js'); -const FSNodeParam = require('../../api/filesystem/FSNodeParam'); -const { HLStat } = require('../../deprecated/filesystem/hl_operations/hl_stat.js'); - -module.exports = eggspress('/stat', { - subdomain: 'api', - auth2: true, - verified: true, - fs: true, - json: true, - allowedMethods: ['GET', 'POST'], - alias: { - path: 'subject', - uid: 'subject', - }, - parameters: { - subject: new FSNodeParam('subject'), - }, -}, async (req, res, next) => { - // modules - const hl_stat = new HLStat(); - const result = await hl_stat.run({ - subject: req.values.subject, - user: req.user, - return_subdomains: req.body.return_subdomains, - return_permissions: req.body.return_permissions, - return_shares: req.body.return_shares, - return_versions: req.body.return_versions, - return_size: req.body.return_size, - }); - res.send(result); -}); diff --git a/src/backend/src/routers/filesystem_api/token-read.js b/src/backend/src/routers/filesystem_api/token-read.js deleted file mode 100644 index 32dcd620c..000000000 --- a/src/backend/src/routers/filesystem_api/token-read.js +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const APIError = require('../../api/APIError.js'); -const eggspress = require('../../api/eggspress'); -const FSNodeParam = require('../../api/filesystem/FSNodeParam'); -const { HLRead } = require('../../deprecated/filesystem/hl_operations/hl_read'); -const { Context } = require('../../util/context'); -const { AccessTokenActorType } = require('../../services/auth/Actor'); -const mime = require('mime-types'); - -module.exports = eggspress('/token-read', { - subdomain: 'api', - verified: true, - fs: true, - json: true, - allowedMethods: ['GET'], - alias: { - path: 'file', - uid: 'file', - }, - parameters: { - fsNode: new FSNodeParam('file'), - }, -}, async (req, res, next) => { - const line_count = !req.query.line_count ? undefined : parseInt(req.query.line_count); - const byte_count = !req.query.byte_count ? undefined : parseInt(req.query.byte_count); - const offset = !req.query.offset ? undefined : parseInt(req.query.offset); - - const access_jwt = req.query.token; - - const svc_auth = Context.get('services').get('auth'); - const actor = await svc_auth.authenticate_from_token(access_jwt); - - if ( ! actor ) { - throw APIError.create('token_auth_failed'); - } - - if ( ! (actor.type instanceof AccessTokenActorType) ) { - throw APIError.create('token_auth_failed'); - } - - const context = Context.get(); - context.set('actor', actor); - - if ( line_count && (!Number.isInteger(line_count) || line_count < 1) ) { - throw new APIError(400, '`line_count` must be a positive integer'); - } - if ( byte_count && (!Number.isInteger(byte_count) || byte_count < 1) ) { - throw new APIError(400, '`byte_count` must be a positive integer'); - } - if ( offset && (!Number.isInteger(offset) || offset < 0) ) { - throw new APIError(400, '`offset` must be a positive integer'); - } - if ( byte_count && line_count ) { - throw new APIError(400, 'cannot use both line_count and byte_count'); - } - - if ( offset && !byte_count ) { - throw APIError.create('field_only_valid_with_other_field', null, { - key: 'offset', - other_key: 'byte_count', - }); - } - - // Helper function to parse Range header - const parseRangeHeader = (rangeHeader) => { - // Check if this is a multipart range request - if ( rangeHeader.includes(',') ) { - // For now, we'll only serve the first range in multipart requests - // as the underlying storage layer doesn't support multipart responses - const firstRange = rangeHeader.split(',')[0].trim(); - const matches = firstRange.match(/bytes=(\d+)-(\d*)/); - if ( ! matches ) return null; - - const start = parseInt(matches[1], 10); - const end = matches[2] ? parseInt(matches[2], 10) : null; - - return { start, end, isMultipart: true }; - } - - // Single range request - const matches = rangeHeader.match(/bytes=(\d+)-(\d*)/); - if ( ! matches ) return null; - - const start = parseInt(matches[1], 10); - const end = matches[2] ? parseInt(matches[2], 10) : null; - - return { start, end, isMultipart: false }; - }; - - if ( req.headers['range'] ) { - res.status(206); - - // Parse the Range header and set Content-Range - const rangeInfo = parseRangeHeader(req.headers['range']); - if ( rangeInfo ) { - const { start, end, isMultipart } = rangeInfo; - - // For open-ended ranges, we need to calculate the actual end byte - let actualEnd = end; - let fileSize = null; - - try { - fileSize = await req.values.fsNode.get('size'); - if ( end === null ) { - actualEnd = fileSize - 1; // File size is 1-based, end byte is 0-based - } - } catch (e) { - // If we can't get file size, we'll let the storage layer handle it - // and not set Content-Range header - actualEnd = null; - fileSize = null; - } - - if ( actualEnd !== null ) { - const totalSize = fileSize !== null ? fileSize : '*'; - const contentRange = `bytes ${start}-${actualEnd}/${totalSize}`; - res.set('Content-Range', contentRange); - } - - // If this was a multipart request, modify the range header to only include the first range - if ( isMultipart ) { - req.headers['range'] = end !== null - ? `bytes=${start}-${end}` - : `bytes=${start}-`; - } - } - } - res.set({ 'Accept-Ranges': 'bytes' }); - - const hl_read = new HLRead(); - const stream = await context.arun(async () => await hl_read.run({ - ...(req.headers['range'] ? { range: req.headers['range'] } : { - line_count, - byte_count, - offset, - }), - fsNode: req.values.fsNode, - user: req.user, - actor, - version_id: req.query.version_id, - })); - - const name = await req.values.fsNode.get('name'); - const mime_type = mime.contentType(name); - res.setHeader('Content-Type', mime_type); - - stream.pipe(res); -}); diff --git a/src/backend/src/routers/filesystem_api/touch.js b/src/backend/src/routers/filesystem_api/touch.js deleted file mode 100644 index 0cf24e398..000000000 --- a/src/backend/src/routers/filesystem_api/touch.js +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = express.Router(); -const auth = require('../../middleware/auth.js'); -const config = require('../../config.js'); -const { DB_WRITE } = require('../../services/database/consts.js'); - -// -----------------------------------------------------------------------// -// POST /touch -// -----------------------------------------------------------------------// -router.post('/touch', auth, express.json(), async (req, res, next) => { - // check subdomain - if ( require('../../helpers.js').subdomain(req) !== 'api' ) - { - next(); - } - - // check if user is verified - if ( (config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed ) - { - return res.status(400).send({ code: 'account_is_not_verified', message: 'Account is not verified' }); - } - - const db = req.services.get('database').get(DB_WRITE, 'filesystem'); - const { v4: uuidv4 } = require('uuid'); - const _path = require('path'); - const { convert_path_to_fsentry, validate_fsentry_name, chkperm } = require('../../helpers.js'); - - // validation - if ( req.body.path === undefined ) - { - return res.status(400).send('path is required'); - } - // path must be a string - else if ( typeof req.body.path !== 'string' ) - { - return res.status(400).send('path must be a string.'); - } - else if ( req.body.path.trim() === '' ) - { - return res.status(400).send('path cannot be empty'); - } - - const dirpath = _path.dirname(_path.resolve('/', req.body.path)); - const target_name = _path.basename(_path.resolve('/', req.body.path)); - const set_accessed_to_now = req.body.set_accessed_to_now; - const set_modified_to_now = req.body.set_modified_to_now; - - // cannot touch in root - if ( dirpath === '/' ) - { - return res.status(400).send('Can not touch in root.'); - } - - // name validation - try { - validate_fsentry_name(target_name); - } catch (e) { - return res.status(400).send(e); - } - - // convert dirpath to its fsentry - const parent = await convert_path_to_fsentry(dirpath); - - // dirpath not found - if ( parent === false ) - { - return res.status(400).send('Target path not found'); - } - - // check permission - if ( ! await chkperm(parent, req.user.id, 'write') ) - { - return res.status(403).send({ code: 'forbidden', message: 'permission denied.' }); - } - - // check if a FSEntry with the same name exists under this path - const existing_fsentry = await convert_path_to_fsentry(_path.resolve('/', `${dirpath }/${ target_name}`)); - - // current epoch - const ts = Date.now() / 1000; - - // set_accessed_to_now - if ( set_accessed_to_now ) { - await db.write(`INSERT INTO fsentries - (uuid, parent_uid, user_id, name, is_dir, created, modified, size) VALUES - ( ?, ?, ?, ?, false, ?, ?, 0) - ON DUPLICATE KEY UPDATE accessed=?`, - [ - //uuid - (existing_fsentry !== false) ? existing_fsentry.uuid : uuidv4(), - //parent_uid - (parent === null) ? null : parent.uuid, - //user_id - parent === null ? req.user.id : parent.user_id, - //name - target_name, - //created - ts, - //modified - ts, - //accessed - ts, - ]); - } - // set_modified_to_now - else if ( set_modified_to_now ) { - await db.write(`INSERT INTO fsentries - (uuid, parent_uid, user_id, name, is_dir, created, modified, size) VALUES - ( ?, ?, ?, ?, false, ?, ?, 0) - ON DUPLICATE KEY UPDATE modified=?`, - [ - //uuid - (existing_fsentry !== false) ? existing_fsentry.uuid : uuidv4(), - //parent_uid - (parent === null) ? null : parent.uuid, - //user_id - parent === null ? req.user.id : parent.user_id, - //name - target_name, - //created - ts, - //modified - ts, - //modified - ts, - ]); - } else { - await db.write(`INSERT INTO fsentries - (uuid, parent_uid, user_id, name, is_dir, created, modified, size) VALUES - ( ?, ?, ?, ?, false, ?, ?, 0) - ON DUPLICATE KEY UPDATE accessed=?, modified=?, created=?`, - [ - //uuid - (existing_fsentry !== false) ? existing_fsentry.uuid : uuidv4(), - //parent_uid - (parent === null) ? null : parent.uuid, - //user_id - parent === null ? req.user.id : parent.user_id, - //name - target_name, - //created - ts, - //modified - ts, - //accessed - ts, - //modified - ts, - //created - ts, - ]); - } - return res.send(''); -}); - -module.exports = router; \ No newline at end of file diff --git a/src/backend/src/routers/filesystem_api/update.js b/src/backend/src/routers/filesystem_api/update.js deleted file mode 100644 index 59d698b7e..000000000 --- a/src/backend/src/routers/filesystem_api/update.js +++ /dev/null @@ -1,45 +0,0 @@ -const APIError = require('../../api/APIError'); -const eggspress = require('../../api/eggspress'); -const FSNodeParam = require('../../api/filesystem/FSNodeParam'); -const StringParam = require('../../api/filesystem/StringParam'); -const { is_valid_url } = require('../../helpers'); -const { Context } = require('../../util/context'); - -module.exports = eggspress('/update-fsentry-thumbnail', { - subdomain: 'api', - verified: true, - auth2: true, - fs: true, - json: true, - allowedMethods: ['POST'], - parameters: { - fsNode: new FSNodeParam('path'), - thumbnail: new StringParam('thumbnail'), - }, -}, async (req, res, next) => { - if ( ! is_valid_url(req.values.thumbnail) ) { - throw new APIError.create('field_invalid', null, { - key: 'thumbnail', - expected: 'a valid URL', - got: typeof req.values.thumbnail, - }); - } - - if ( ! await req.values.fsNode.exists() ) { - throw new APIError.create('subject_does_not_exist'); - } - - const svc = Context.get('services'); - - const svc_mountpoint = svc.get('mountpoint'); - const provider = - await svc_mountpoint.get_provider(req.values.fsNode.selector); - - provider.update_thumbnail({ - context: Context.get(), - node: req.values.fsNode, - thumbnail: req.body.thumbnail, - }); - - res.json({}); -}); diff --git a/src/backend/src/routers/filesystem_api/write.js b/src/backend/src/routers/filesystem_api/write.js deleted file mode 100644 index 70177650a..000000000 --- a/src/backend/src/routers/filesystem_api/write.js +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const eggspress = require('../../api/eggspress.js'); -const FSNodeParam = require('../../api/filesystem/FSNodeParam.js'); -const { HLWrite } = require('../../deprecated/filesystem/hl_operations/hl_write.js'); -const { boolify } = require('../../util/hl_types.js'); -const { Context } = require('../../util/context.js'); -const Busboy = require('busboy'); -const { TeePromise } = require('@heyputer/putility').libs.promise; -const APIError = require('../../api/APIError.js'); -const { valid_file_size } = require('../../util/validutil.js'); - -// -----------------------------------------------------------------------// -// POST /up | /write -// -----------------------------------------------------------------------// -module.exports = eggspress(['/up', '/write'], { - subdomain: 'api', - verified: true, - auth2: true, - fs: true, - json: true, - allowedMethods: ['POST'], - // files: ['file'], - // multest: true, - alias: { uid: 'path' }, - // parameters: { - // fsNode: new FSNodeParam('path'), - // target: new FSNodeParam('shortcut_to', { optional: true }), - // } -}, async (req, res, _next) => { - // Note: parameters moved here because the parameter - // middleware won't work while using busboy - const parameters = { - fsNode: new FSNodeParam('path'), - target: new FSNodeParam('shortcut_to', { optional: true }), - }; - - // modules - const { get_app } = require('../../helpers.js'); - - // Is this an entry for an app? - let app; - if ( req.body.app_uid ) { - app = await get_app({ uid: req.body.app_uid }); - } - - const x = Context.get(); - let frame; - async () => { - const operationTraceSvc = x.get('services').get('operationTrace'); - frame = (await operationTraceSvc.add_frame('api:/write')) - .attr('gui_metadata', { - original_client_socket_id: req.body.original_client_socket_id, - socket_id: req.body.socket_id, - operation_id: req.body.operation_id, - user_id: req.user.id, - item_upload_id: req.body.item_upload_id, - }) - ; - x.set(operationTraceSvc.ckey('frame'), frame); - - const svc_clientOperation = x.get('services').get('client-operation'); - const tracker = svc_clientOperation.add_operation({ - frame, - metadata: { - user_id: req.user.id, - }, - }); - x.set(svc_clientOperation.ckey('tracker'), tracker); - }; - - //------------------------------------------------------------- - // Multipart processing (using busboy) - //------------------------------------------------------------- - const busboy = Busboy({ headers: req.headers }); - - let uploaded_file = null; - const p_ready = new TeePromise(); - - busboy.on('field', (fieldname, value, details) => { - if ( details.fieldnameTruncated ) { - throw new Error('fieldnameTruncated'); - } - if ( details.valueTruncated ) { - throw new Error('valueTruncated'); - } - - req.body[fieldname] = value; - }); - - busboy.on('file', (fieldname, stream, details) => { - const { - filename, mimetype, - } = details; - - const { v: size, ok: size_ok } = - valid_file_size(req.body.size); - - if ( ! size_ok ) { - p_ready.reject(APIError.create('invalid_file_metadata')); - return; - } - - uploaded_file = { - size: size, - name: filename, - mimetype, - stream, - - // TODO: Standardize the fileinfo object - - // thumbnailer expects `mimetype` to be `type` - type: mimetype, - - // alias for name, used only in here it seems - originalname: filename, - }; - - p_ready.resolve(); - }); - - busboy.on('error', err => { - console.log('GOT ERROR READING', err); - p_ready.reject(err); - }); - - busboy.on('close', () => { - p_ready.resolve(); - }); - - req.pipe(busboy); - - await p_ready; - - // Copied from eggspress; needed here because we're using busboy - for ( const key in parameters ) { - const param = parameters[key]; - if ( ! req.values ) req.values = {}; - - const values = req.method === 'GET' ? req.query : req.body; - const getParam = (key) => values[key]; - const result = await param.consolidate({ req, getParam }); - req.values[key] = result; - } - - if ( req.body.size === undefined ) { - throw APIError.create('missing_expected_metadata', null, { - keys: ['size'], - }); - } - - const hl_write = new HLWrite(); - const response = await hl_write.run({ - destination_or_parent: req.values.fsNode, - specified_name: req.body.name, - fallback_name: uploaded_file.originalname, - overwrite: await boolify(req.body.overwrite), - dedupe_name: await boolify(req.body.dedupe_name), - shortcut_to: req.values.target, - - create_missing_parents: boolify(req.body.create_missing_ancestors ?? - req.body.create_missing_parents), - - actor: req.actor, - user: req.user, - file: uploaded_file, - - app_id: app ? app.id : null, - - thumbnail: req.body.thumbnail, - }); - - if ( frame ) frame.done(); - return res.send(response); -}); diff --git a/src/backend/src/routers/get-dev-profile.js b/src/backend/src/routers/get-dev-profile.js deleted file mode 100644 index 4e8860a5e..000000000 --- a/src/backend/src/routers/get-dev-profile.js +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const config = require('../config.js'); -const router = new express.Router(); -const auth = require('../middleware/auth.js'); - -// -----------------------------------------------------------------------// -// GET /get-dev-profile -// -----------------------------------------------------------------------// -router.get('/get-dev-profile', auth, express.json(), async (req, response, next) => { - // check subdomain - if ( require('../helpers').subdomain(req) !== 'api' ) - { - next(); - } - - // check if user is verified - if ( (config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed ) - { - return response.status(400).send({ code: 'account_is_not_verified', message: 'Account is not verified' }); - } - - // TODO: we currently invalidate the cache on every request, this is because a developer may - // have been approved for the incentive program from one server, but the cache on another server - // may not have been updated yet. This is a temporary solution until we implement a better way to - // handle this. The better way would be for different servers to communicate with each other - // when a developer is approved for the incentive program (or any other change that affects the - // cache) and update the cache on all servers. - require('../helpers').invalidate_cached_user(req.user); - const { get_user } = require('../helpers'); - - let dev = await get_user(req.user); - dev = dev ?? {}; - - try { - // auth - response.send({ - first_name: dev.dev_first_name, - last_name: dev.dev_last_name, - approved_for_incentive_program: dev.dev_approved_for_incentive_program, - joined_incentive_program: dev.dev_joined_incentive_program, - paypal: dev.dev_paypal, - }); - } catch (e) { - console.log(e); - response.status(400).send(); - } -}); -module.exports = router; diff --git a/src/backend/src/routers/get-launch-apps.js b/src/backend/src/routers/get-launch-apps.js deleted file mode 100644 index 0dc63e07b..000000000 --- a/src/backend/src/routers/get-launch-apps.js +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -import { redisClient } from '../clients/redis/redisSingleton.js'; -import { setRedisCacheValue } from '../clients/redis/cacheUpdate.js'; -import { get_apps } from '../helpers.js'; -import { RecentAppOpensRedisCacheSpace } from './recentAppOpens/RecentAppOpensRedisCacheSpace.js'; -import { DB_READ } from '../services/database/consts.js'; - -const iconify_apps = async (context, { apps, size }) => { - const svc_appIcon = context.services.get('app-icon'); - return await svc_appIcon.iconifyApps({ apps, size }); -}; - -// -----------------------------------------------------------------------// -// GET /get-launch-apps -// -----------------------------------------------------------------------// -export default async (req, res) => { - let result = {}; - const iconSize = req.query.icon_size; - - // Verify query params - if ( iconSize ) { - const ALLOWED_SIZES = ['16', '32', '64', '128', '256', '512']; - - if ( ! ALLOWED_SIZES.includes(iconSize) ) { - res.status(400).send({ error: 'Invalid icon_size' }); - } - } - - // -----------------------------------------------------------------------// - // Recommended apps - // -----------------------------------------------------------------------// - const svc_recommendedApps = req.services.get('recommended-apps'); - result.recommended = await svc_recommendedApps.get_recommended_apps({ - icon_size: iconSize, - }); - - // -----------------------------------------------------------------------// - // Recent apps - // -----------------------------------------------------------------------// - let apps = []; - - const db = req.services.get('database').get(DB_READ, 'apps'); - - // First try the cache to see if we have recent apps - const cached_apps = await redisClient.get(RecentAppOpensRedisCacheSpace.key(req.user.id)); - if ( cached_apps ) { - try { - apps = JSON.parse(cached_apps); - } catch (e) { - apps = []; - } - } - - // If cache is empty, query the db and update the cache - if ( !apps || !Array.isArray(apps) || apps.length === 0 ) { - apps = await db.read( - 'SELECT DISTINCT app_uid FROM app_opens WHERE user_id = ? GROUP BY app_uid ORDER BY MAX(_id) DESC LIMIT 10', - [req.user.id], - ); - // Update cache with the results from the db (if any results were returned) - if ( apps && Array.isArray(apps) && apps.length > 0 ) { - await setRedisCacheValue( - RecentAppOpensRedisCacheSpace.key(req.user.id), - JSON.stringify(apps), - { eventData: apps }, - ); - } - } - - // prepare each app for returning to user by only returning the necessary fields - // and adding them to the retobj array - const recent_apps = await get_apps(apps.map(({ app_uid: uid }) => ({ uid }))); - - result.recent = recent_apps.map((app) => { - if ( ! app ) return null; - return { - uuid: app.uid, - name: app.name, - title: app.title, - icon: app.icon, - godmode: app.godmode, - maximize_on_start: app.maximize_on_start, - index_url: app.index_url, - }; - }).filter(Boolean); - - // Iconify apps - if ( iconSize ) { - result.recent = await iconify_apps({ services: req.services }, { - apps: result.recent, - size: iconSize, - }); - } - - return res.send(result); -}; diff --git a/src/backend/src/routers/get-launch-apps.test.js b/src/backend/src/routers/get-launch-apps.test.js deleted file mode 100644 index 773efe34f..000000000 --- a/src/backend/src/routers/get-launch-apps.test.js +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { describe, it, expect, beforeEach, vi } from 'vitest'; -import * as uuid from 'uuid'; - -vi.mock('../helpers.js', () => ({ - get_apps: vi.fn(), -})); - -import { get_apps } from '../helpers.js'; -import get_launch_apps from './get-launch-apps'; - -const TEST_UUID_NAMESPACE = '5568ab95-229d-4d87-b98c-0b12680a9524'; - -const apps_names_expected_to_exist = [ - 'app-center', - 'dev-center', - 'editor', -]; - -const data_mockapps = (() => { - const data_mockapps = []; - // List of app names that get-launch-apps expects to exist - for ( const name of apps_names_expected_to_exist ) { - data_mockapps.push({ - uid: `app-${ uuid.v5(name, TEST_UUID_NAMESPACE)}`, - name, - title: 'App Name', - icon: 'icon-goes-here', - godmode: false, - maximize_on_start: false, - index_url: 'index-url', - }); - } - - // An additional app that won't show up in taskbar - data_mockapps.push({ - uid: `app-${ uuid.v5('hidden-app', TEST_UUID_NAMESPACE)}`, - name: 'hidden-app', - title: 'Hidden App', - icon: 'icon-goes-here', - godmode: false, - maximize_on_start: false, - index_url: 'index-url', - }); - - // An additional app tha only shows up in recents - data_mockapps.push({ - uid: `app-${ uuid.v5('recent-app', TEST_UUID_NAMESPACE)}`, - name: 'recent-app', - title: 'Recent App', - icon: 'icon-goes-here', - godmode: false, - maximize_on_start: false, - index_url: 'index-url', - }); - - return data_mockapps; -})(); - -const data_appopens = [ - { - app_uid: `app-${ uuid.v5('app-center', TEST_UUID_NAMESPACE)}`, - }, - { - app_uid: `app-${ uuid.v5('editor', TEST_UUID_NAMESPACE)}`, - }, - { - app_uid: `app-${ uuid.v5('recent-app', TEST_UUID_NAMESPACE)}`, - }, -]; - -const get_mock_context = () => { - get_apps.mockImplementation(async (specifiers) => { - return specifiers.map(({ uid, name, id }) => { - if ( uid ) { - return data_mockapps.find(app => app.uid === uid); - } - if ( name ) { - return data_mockapps.find(app => app.name === name); - } - if ( id ) { - return data_mockapps.find(app => app.id === id); - } - return null; - }); - }); - - const database_mock = { - read: async (query) => { - if ( query.includes('FROM app_opens') ) { - return data_appopens; - } - }, - }; - const recommendedApps_mock = { - get_recommended_apps: async () => { - return data_mockapps - .filter(app => apps_names_expected_to_exist.includes(app.name)) - .map(app => ({ - uuid: app.uid, - name: app.name, - title: app.title, - icon: app.icon, - godmode: app.godmode, - maximize_on_start: app.maximize_on_start, - index_url: app.index_url, - })); - }, - }; - const services_mock = { - get: (key) => { - if ( key === 'database' ) { - return { - get: () => database_mock, - }; - } - if ( key === 'recommended-apps' ) { - return recommendedApps_mock; - } - }, - }; - - const req_mock = { - user: { - id: 1 + Math.floor(Math.random() * 1000 ** 3), - }, - services: services_mock, - send: vi.fn(), - }; - - const res_mock = { - send: vi.fn(), - }; - - return { - get_launch_apps, - req_mock, - res_mock, - spies: { - get_apps, - }, - }; -}; - -describe('GET /launch-apps', () => { - - beforeEach(() => { - vi.clearAllMocks(); - }); - - it('should return expected format', async () => { - // First call - { - const { get_launch_apps, req_mock, res_mock } = get_mock_context(); - req_mock.query = {}; - await get_launch_apps(req_mock, res_mock); - - // << HOW TO FIX >> - // If you updated the list of recommended apps, - // you can simply update this number to match the new length - // expect(spies.get_apps).toHaveBeenCalledTimes(1); - } - - // Second call - { - const { get_launch_apps, req_mock, res_mock, spies } = get_mock_context(); - req_mock.query = {}; - await get_launch_apps(req_mock, res_mock); - - expect(res_mock.send).toHaveBeenCalledOnce(); - - const call = res_mock.send.mock.calls[0]; - const response = call[0]; - - expect(response).toBeTypeOf('object'); - - expect(response).toHaveProperty('recommended'); - expect(response.recommended).toBeInstanceOf(Array); - expect(response.recommended).toHaveLength(apps_names_expected_to_exist.length); - expect(response.recommended).toEqual( - data_mockapps - .filter(app => apps_names_expected_to_exist.includes(app.name)) - .map(app => ({ - uuid: app.uid, - name: app.name, - title: app.title, - icon: app.icon, - godmode: app.godmode, - maximize_on_start: app.maximize_on_start, - index_url: app.index_url, - }))); - - expect(response).toHaveProperty('recent'); - expect(response.recent).toBeInstanceOf(Array); - expect(response.recent).toHaveLength(data_appopens.length); - expect(response.recent).toEqual( - data_mockapps - .filter(app => data_appopens.map(app_open => app_open.app_uid).includes(app.uid)) - .map(app => ({ - uuid: app.uid, - name: app.name, - title: app.title, - icon: app.icon, - godmode: app.godmode, - maximize_on_start: app.maximize_on_start, - index_url: app.index_url, - }))); - - expect(spies.get_apps).toHaveBeenCalledTimes(2); - expect(spies.get_apps).toHaveBeenCalledWith( - data_appopens.map(({ app_uid: uid }) => ({ uid }))); - } - }); -}); diff --git a/src/backend/src/routers/healthcheck.js b/src/backend/src/routers/healthcheck.js deleted file mode 100644 index 515e54b26..000000000 --- a/src/backend/src/routers/healthcheck.js +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const config = require('../config'); -const router = new express.Router(); - -const normalizeHostDomain = (domain) => { - if ( typeof domain !== 'string' ) return null; - const normalizedDomain = domain.trim().toLowerCase().replace(/^\./, ''); - if ( ! normalizedDomain ) return null; - - try { - return new URL(`http://${normalizedDomain}`).hostname.toLowerCase(); - } catch { - return normalizedDomain.split(':')[0] || null; - } -}; - -const hostMatchesDomain = (hostname, domain) => { - const normalizedHost = normalizeHostDomain(hostname); - const normalizedDomain = normalizeHostDomain(domain); - if ( !normalizedHost || !normalizedDomain ) return false; - return normalizedHost === normalizedDomain || - normalizedHost.endsWith(`.${normalizedDomain}`); -}; - -const isHostedDomainRequest = (req) => { - const requestHost = normalizeHostDomain(req.hostname ?? req.headers?.host); - if ( ! requestHost ) return false; - - const hostedDomains = new Set(); - for ( const domain of [ - config.static_hosting_domain, - config.static_hosting_domain_alt, - config.private_app_hosting_domain, - config.private_app_hosting_domain_alt, - ] ) { - const normalizedDomain = normalizeHostDomain(domain); - if ( normalizedDomain ) { - hostedDomains.add(normalizedDomain); - } - } - - return [...hostedDomains].some(hostedDomain => - hostMatchesDomain(requestHost, hostedDomain)); -}; - -const get_status = async (req) => { - const svc_serverHealth = req.services.get('server-health'); - return await svc_serverHealth.get_status(); -}; - -const send_health_status = async (req, res, { fail_with_http_error = false }) => { - const status = await get_status(req); - const shouldFailWithHttpError = - fail_with_http_error || !!req.query['return-http-error']; - const httpStatus = shouldFailWithHttpError && !status.ok ? 500 : 200; - res.status(httpStatus).json(status); -}; - -// -----------------------------------------------------------------------// -// GET /healthcheck -// -----------------------------------------------------------------------// -router.get('/healthcheck', async (req, res, next) => { - if ( req.app?.get('isDraining') ) { - res.status(503).json({ - ok: false, - failed: ['draining'], - }); - return; - } - - if ( isHostedDomainRequest(req) ) { - next(); - return; - } - - await send_health_status(req, res, { fail_with_http_error: false }); -}); - -module.exports = router; diff --git a/src/backend/src/routers/hosting/puter-site-config.js b/src/backend/src/routers/hosting/puter-site-config.js deleted file mode 100644 index 4c39176fc..000000000 --- a/src/backend/src/routers/hosting/puter-site-config.js +++ /dev/null @@ -1,311 +0,0 @@ -/* - * Copyright (C) 2026-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const path = require('path'); - -const ERROR_CLASS_REGEX = /^([45])xx$/i; -const STATUS_CODE_REGEX = /^[1-5][0-9][0-9]$/; - -const createEmptyConfig = () => ({ - exactRules: Object.create(null), - classRules: Object.create(null), - defaultRule: null, -}); - -const normalizeStatusCode = value => { - if ( value === undefined || value === null ) return null; - - const status = Number.parseInt(String(value), 10); - if ( ! Number.isInteger(status) ) return null; - if ( status < 100 || status > 599 ) return null; - return status; -}; - -const normalizeFilePath = value => { - if ( typeof value !== 'string' ) return null; - - let v = value.trim(); - if ( v === '' ) return null; - if ( v.startsWith('@') ) return null; - if ( /^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(v) ) return null; - - v = v.replaceAll('\\', '/'); - v = v.split('?')[0].split('#')[0]; - if ( ! v.startsWith('/') ) { - v = `/${v}`; - } - - const resolved = path.posix.resolve('/', v); - if ( resolved === '/' ) return null; - return resolved; -}; - -const normalizeRule = rawRule => { - if ( rawRule === undefined || rawRule === null ) return null; - - if ( typeof rawRule === 'string' ) { - const file = normalizeFilePath(rawRule); - return file ? { file, status: null } : null; - } - - if ( typeof rawRule === 'number' ) { - const status = normalizeStatusCode(rawRule); - return status ? { file: null, status } : null; - } - - if ( typeof rawRule !== 'object' ) return null; - - const file = normalizeFilePath( - rawRule.file ?? - rawRule.path ?? - rawRule.page ?? - rawRule.responsePagePath ?? - rawRule.response_page_path ?? - rawRule.destination ?? - rawRule.dest, - ); - - const status = normalizeStatusCode( - rawRule.status ?? - rawRule.code ?? - rawRule.statusCode ?? - rawRule.responseCode ?? - rawRule.response_code ?? - rawRule.responseStatus ?? - rawRule.response_status, - ); - - if ( !file && !status ) return null; - return { file: file ?? null, status: status ?? null }; -}; - -const setRule = (config, key, rule) => { - if ( ! rule ) return false; - - if ( key === 'default' ) { - config.defaultRule = rule; - return true; - } - - if ( STATUS_CODE_REGEX.test(key) ) { - config.exactRules[key] = rule; - return true; - } - - const classMatch = key.match(ERROR_CLASS_REGEX); - if ( classMatch ) { - config.classRules[`${classMatch[1]}xx`] = rule; - return true; - } - - return false; -}; - -const parseKeyedRules = (config, object) => { - if ( !object || typeof object !== 'object' || Array.isArray(object) ) { - return false; - } - - let matched = false; - for ( const [key, value] of Object.entries(object) ) { - if ( - key !== 'default' && - !STATUS_CODE_REGEX.test(key) && - !ERROR_CLASS_REGEX.test(key) - ) { - continue; - } - matched = setRule(config, key.toLowerCase(), normalizeRule(value)) || matched; - } - return matched; -}; - -const parseCloudfrontRules = (config, value) => { - if ( ! Array.isArray(value) ) return false; - - let matched = false; - for ( const entry of value ) { - if ( !entry || typeof entry !== 'object' ) continue; - - const errorCode = normalizeStatusCode(entry.ErrorCode ?? entry.errorCode ?? entry.error_code); - if ( ! errorCode ) continue; - - const rule = normalizeRule({ - responsePagePath: entry.ResponsePagePath ?? entry.responsePagePath ?? entry.response_page_path, - responseCode: entry.ResponseCode ?? entry.responseCode ?? entry.response_code, - }); - if ( ! rule ) continue; - - config.exactRules[String(errorCode)] = rule; - matched = true; - } - - return matched; -}; - -const isCatchAllSource = source => { - if ( typeof source !== 'string' ) return false; - const s = source.trim(); - if ( s === '' ) return false; - - if ( [ - '/:path*', - '/:match*', - '/(.*)', - '/(.*)?', - '/.*', - '^/(.*)$', - ].includes(s) ) { - return true; - } - - if ( /^\/:\w+\*$/.test(s) ) return true; - if ( /^\^?\/\(\.\*\)\$?$/.test(s) ) return true; - return false; -}; - -const parseVercelRules = (config, value) => { - if ( ! Array.isArray(value) ) return false; - - let matched = false; - for ( const entry of value ) { - if ( !entry || typeof entry !== 'object' ) continue; - const source = entry.source ?? entry.src; - if ( ! isCatchAllSource(source) ) continue; - - const rule = normalizeRule({ - destination: entry.destination ?? entry.dest, - status: entry.status ?? 200, - }); - if ( ! rule ) continue; - - config.exactRules['404'] = rule; - matched = true; - } - - return matched; -}; - -const parseJsonConfig = text => { - let parsed; - try { - parsed = JSON.parse(text); - } catch { - return null; - } - - const config = createEmptyConfig(); - let matched = false; - - matched = parseCloudfrontRules(config, parsed?.CustomErrorResponses ?? parsed?.customErrorResponses) || matched; - - matched = parseKeyedRules(config, parsed?.errors) || matched; - matched = parseKeyedRules(config, parsed?.errorPages) || matched; - matched = parseKeyedRules(config, parsed?.error_pages) || matched; - - matched = parseKeyedRules(config, parsed) || matched; - - const topLevelRule = normalizeRule(parsed); - if ( topLevelRule ) { - config.defaultRule = topLevelRule; - matched = true; - } - - matched = parseVercelRules(config, parsed?.rewrites) || matched; - matched = parseVercelRules(config, parsed?.routes) || matched; - - return matched ? config : null; -}; - -const parseNginxStyleConfig = text => { - const config = createEmptyConfig(); - let matched = false; - - const cleaned = text - .replace(/\r\n/g, '\n') - .replace(/#.*$/gm, ''); - - const directives = cleaned.matchAll(/\berror_page\s+([^;]+);/gi); - for ( const directive of directives ) { - const args = directive[1]; - const tokens = args.trim().split(/\s+/).filter(Boolean); - if ( tokens.length < 2 ) continue; - - const uriToken = tokens.pop(); - const file = normalizeFilePath(uriToken); - if ( ! file ) continue; - - let statusOverride = null; - if ( tokens.length > 0 && tokens[tokens.length - 1].startsWith('=') ) { - const overrideToken = tokens.pop(); - if ( overrideToken !== '=' ) { - statusOverride = normalizeStatusCode(overrideToken.slice(1)); - } - } - - const statusCodes = tokens - .map(token => normalizeStatusCode(token)) - .filter(Boolean); - - if ( statusCodes.length === 0 ) continue; - - const rule = { - file, - status: statusOverride, - }; - - for ( const statusCode of statusCodes ) { - config.exactRules[String(statusCode)] = rule; - matched = true; - } - } - - return matched ? config : null; -}; - -const parseSiteErrorConfig = rawText => { - if ( typeof rawText !== 'string' ) return null; - const text = rawText.trim(); - if ( text === '' ) return null; - - const jsonConfig = parseJsonConfig(text); - if ( jsonConfig ) return jsonConfig; - - return parseNginxStyleConfig(text); -}; - -const getSiteErrorRule = (config, statusCode) => { - if ( !config || typeof config !== 'object' ) return null; - - const status = normalizeStatusCode(statusCode); - if ( ! status ) return null; - - const exactRule = config.exactRules?.[String(status)]; - if ( exactRule ) return { ...exactRule }; - - const classRule = config.classRules?.[`${Math.floor(status / 100)}xx`]; - if ( classRule ) return { ...classRule }; - - if ( config.defaultRule ) return { ...config.defaultRule }; - return null; -}; - -module.exports = { - parseSiteErrorConfig, - getSiteErrorRule, -}; diff --git a/src/backend/src/routers/hosting/puter-site-config.test.js b/src/backend/src/routers/hosting/puter-site-config.test.js deleted file mode 100644 index ca0576f85..000000000 --- a/src/backend/src/routers/hosting/puter-site-config.test.js +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (C) 2026-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { describe, expect, it } from 'vitest'; - -const { - parseSiteErrorConfig, - getSiteErrorRule, -} = require('./puter-site-config'); - -describe('puter-site-config parser', () => { - it('parses nginx error_page syntax', () => { - const config = parseSiteErrorConfig(` - error_page 404 /404.html; - error_page 500 502 503 504 =200 /index.html; - `); - - expect(getSiteErrorRule(config, 404)).toEqual({ - file: '/404.html', - status: null, - }); - expect(getSiteErrorRule(config, 500)).toEqual({ - file: '/index.html', - status: 200, - }); - expect(getSiteErrorRule(config, 503)).toEqual({ - file: '/index.html', - status: 200, - }); - }); - - it('parses cloudfront custom error responses', () => { - const config = parseSiteErrorConfig(JSON.stringify({ - CustomErrorResponses: [ - { - ErrorCode: 404, - ResponsePagePath: '/404.html', - ResponseCode: '200', - }, - { - ErrorCode: 500, - ResponseCode: '404', - }, - ], - })); - - expect(getSiteErrorRule(config, 404)).toEqual({ - file: '/404.html', - status: 200, - }); - expect(getSiteErrorRule(config, 500)).toEqual({ - file: null, - status: 404, - }); - }); - - it('parses puter-native json with exact, wildcard, and default rules', () => { - const config = parseSiteErrorConfig(JSON.stringify({ - errors: { - 404: { - file: 'not-found.html', - }, - '5xx': { - file: '/error.html', - status: 404, - }, - default: { - status: 404, - }, - }, - })); - - expect(getSiteErrorRule(config, 404)).toEqual({ - file: '/not-found.html', - status: null, - }); - expect(getSiteErrorRule(config, 502)).toEqual({ - file: '/error.html', - status: 404, - }); - expect(getSiteErrorRule(config, 418)).toEqual({ - file: null, - status: 404, - }); - }); - - it('parses vercel-style catch-all rewrite as 404 fallback', () => { - const config = parseSiteErrorConfig(JSON.stringify({ - rewrites: [ - { - source: '/:path*', - destination: '/index.html', - }, - ], - })); - - expect(getSiteErrorRule(config, 404)).toEqual({ - file: '/index.html', - status: 200, - }); - }); - - it('returns null for unsupported config text', () => { - const config = parseSiteErrorConfig('this is not a supported config format'); - expect(config).toBeNull(); - }); -}); diff --git a/src/backend/src/routers/hosting/puterSiteMiddleware.js b/src/backend/src/routers/hosting/puterSiteMiddleware.js deleted file mode 100644 index e786b1b7b..000000000 --- a/src/backend/src/routers/hosting/puterSiteMiddleware.js +++ /dev/null @@ -1,1853 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import dedent from 'dedent'; -import { contentType as contentTypeFromMime } from 'mime-types'; -import { resolve } from 'path'; -import { v5 as uuidv5 } from 'uuid'; -import APIError from '../../api/APIError.js'; -import config from '../../config.js'; -import fsNodeContext from '../../deprecated/filesystem/FSNodeContext.js'; -import { LLRead } from '../../deprecated/filesystem/ll_operations/ll_read.js'; -import selectors from '../../deprecated/filesystem/node/selectors.js'; -import { get_app, get_user } from '../../helpers.js'; -import api_error_handler from '../../modules/web/lib/api_error_handler.js'; -import { Actor, SiteActorType, UserActorType } from '../../services/auth/Actor.js'; -import { PermissionUtil } from '../../services/auth/permissionUtils.mjs'; -import { DB_READ } from '../../services/database/consts.js'; -import { Context } from '../../util/context.js'; -import { stream_to_buffer as streamToBuffer } from '../../util/streamutil.js'; -import { - getSiteErrorRule, - parseSiteErrorConfig, -} from './puter-site-config.js'; - -const { - origin: originUrl, - cookie_name: cookieName, - private_app_hosting_domain: privateAppHostingDomain, - private_app_hosting_domain_alt: privateAppHostingDomainAlt, - static_hosting_base_domain_redirect: staticHostingBaseDomainRedirect, - static_hosting_domain: staticHostingDomain, - static_hosting_domain_alt: staticHostingDomainAlt, - username_regex: usernameRegex, -} = config; -const { TYPE_DIRECTORY } = fsNodeContext; -const { - NodeInternalIDSelector, - NodePathSelector, -} = selectors; - -const AT_DIRECTORY_NAMESPACE = '4aa6dc52-34c1-4b8a-b63c-a62b27f727cf'; -const puterSiteConfigFilename = '.puter_site_config'; -const puterSiteConfigMaxSize = 256 * 1024; -const defaultPublicHostedActorCookieName = 'puter.public.hosted.actor.token'; - -function isPrivateApp (app) { - return Number(app?.is_private ?? 0) > 0; -} - -function normalizeConfiguredHostname (hostValue) { - if ( typeof hostValue !== 'string' ) return null; - const normalizedHost = hostValue.trim().toLowerCase().replace(/^\./, ''); - if ( ! normalizedHost ) return null; - try { - return new URL(`http://${normalizedHost}`).hostname.toLowerCase(); - } catch { - return normalizedHost.split(':')[0] || null; - } -} - -function getPrivateHostingDomainsForMatch () { - const domains = new Set(); - for ( const candidate of [ - privateAppHostingDomain, - privateAppHostingDomainAlt, - ] ) { - const normalizedCandidate = normalizeConfiguredHostname(candidate); - if ( normalizedCandidate ) { - domains.add(normalizedCandidate); - } - } - return [...domains]; -} - -function getPrivateHostingDomainForRedirect () { - const primaryDomainCandidate = normalizeConfiguredHost(privateAppHostingDomain); - if ( primaryDomainCandidate ) return primaryDomainCandidate; - - const altDomainCandidate = normalizeConfiguredHost(privateAppHostingDomainAlt); - if ( altDomainCandidate ) return altDomainCandidate; - - return 'puter.app'; -} - -function hostMatchesPrivateDomain (hostname) { - const host = normalizeConfiguredHostname(hostname); - if ( ! host ) return false; - - const privateHostingDomains = getPrivateHostingDomainsForMatch(); - return privateHostingDomains.some(privateHostingDomain => - host === privateHostingDomain || host.endsWith(`.${privateHostingDomain}`)); -} - -function getSubdomainFromHostedRequest (req) { - const host = normalizeConfiguredHostname(req.hostname); - if ( ! host ) return ''; - - const privateHostingDomains = getPrivateHostingDomainsForMatch() - .sort((a, b) => b.length - a.length); - for ( const privateHostingDomain of privateHostingDomains ) { - const privateDomainSuffix = `.${privateHostingDomain}`; - if ( host === privateHostingDomain ) { - return ''; - } - if ( host.endsWith(privateDomainSuffix) ) { - const privateSubdomain = host.slice(0, host.length - privateDomainSuffix.length); - return privateSubdomain.split('.')[0] || ''; - } - } - - return host.split('.')[0] || ''; -} - -function getRequestedPrivateHost (req) { - const normalizedRequestHost = normalizeConfiguredHostname(req.hostname); - if ( ! normalizedRequestHost ) return undefined; - if ( ! hostMatchesPrivateDomain(normalizedRequestHost) ) return undefined; - return normalizedRequestHost; -} - -function buildPrivateHostRedirectUrl (req, app) { - if ( ! app ) { - return null; - } - - try { - const privateHostingDomain = getPrivateHostingDomainForRedirect(); - if ( ! privateHostingDomain ) { - return null; - } - - const subdomain = req.subdomains?.[0] || getSubdomainFromHostedRequest(req); - if ( ! subdomain ) { - return null; - } - - const protocol = `${config.protocol ?? 'https'}` - .trim() - .replace(/:$/, '') || 'https'; - const requestUrl = `${req.originalUrl || '/'}`.startsWith('/') - ? req.originalUrl || '/' - : `/${req.originalUrl}`; - const privateHostOrigin = `${protocol}://${subdomain}.${privateHostingDomain}`; - const redirectUrl = new URL(requestUrl, privateHostOrigin); - return redirectUrl.toString(); - } catch { - return null; - } -} - -function normalizeHostFromHeader (hostValue) { - if ( typeof hostValue !== 'string' ) return null; - const normalizedHost = hostValue.trim().toLowerCase(); - if ( ! normalizedHost ) return null; - try { - return new URL(`http://${normalizedHost}`).host; - } catch { - return normalizedHost; - } -} - -function normalizeConfiguredHost (hostValue) { - if ( typeof hostValue !== 'string' ) return null; - const normalizedHost = hostValue.trim().toLowerCase().replace(/^\./, ''); - if ( ! normalizedHost ) return null; - return normalizedHost; -} - -function buildPrivateAppIndexUrlCandidates (req) { - const protocol = `${config.protocol ?? 'https'}`.trim().replace(/:$/, '') || 'https'; - const hostCandidates = new Set(); - - const hostnameCandidate = normalizeHostFromHeader(req.hostname); - if ( hostnameCandidate ) { - hostCandidates.add(hostnameCandidate); - } - - const headerHostCandidate = normalizeHostFromHeader(req.headers?.host); - if ( headerHostCandidate ) { - hostCandidates.add(headerHostCandidate); - } - - const hostedSubdomain = getSubdomainFromHostedRequest(req); - if ( hostedSubdomain ) { - const staticHostingDomainCandidate = normalizeConfiguredHost(staticHostingDomain); - const staticHostingDomainAltCandidate = normalizeConfiguredHost(staticHostingDomainAlt); - const privateHostingDomainCandidate = normalizeConfiguredHost(privateAppHostingDomain); - const privateHostingDomainAltCandidate = normalizeConfiguredHost(privateAppHostingDomainAlt); - - if ( staticHostingDomainCandidate ) { - hostCandidates.add(`${hostedSubdomain}.${staticHostingDomainCandidate}`); - } - if ( staticHostingDomainAltCandidate ) { - hostCandidates.add(`${hostedSubdomain}.${staticHostingDomainAltCandidate}`); - } - if ( privateHostingDomainCandidate ) { - hostCandidates.add(`${hostedSubdomain}.${privateHostingDomainCandidate}`); - } - if ( privateHostingDomainAltCandidate ) { - hostCandidates.add(`${hostedSubdomain}.${privateHostingDomainAltCandidate}`); - } - } - - const candidates = []; - for ( const host of hostCandidates ) { - const base = `${protocol}://${host}`; - candidates.push(base); - candidates.push(`${base}/`); - candidates.push(`${base}/index.html`); - } - - return [...new Set(candidates)]; -} - -async function resolvePrivateAppForHostedSite ({ req, site, services, associatedApp }) { - if ( associatedApp ) return associatedApp; - if ( ! site?.user_id ) return null; - - const indexUrlCandidates = buildPrivateAppIndexUrlCandidates(req); - if ( indexUrlCandidates.length === 0 ) return null; - - const databaseService = services.get('database'); - const dbService = databaseService.get(DB_READ, 'apps'); - const placeholders = indexUrlCandidates.map(() => '?').join(', '); - - const apps = await dbService.read( - `SELECT * FROM apps WHERE owner_user_id = ? AND is_private = 1 AND index_url IN (${placeholders}) LIMIT 2`, - [site.user_id, ...indexUrlCandidates], - ); - - if ( apps.length > 1 ) { - logPrivateAccessEvent('private_access.host_match_ambiguous', { - requestHost: req.hostname, - siteOwnerUserId: site.user_id, - matchCount: apps.length, - }); - } - - return apps[0] || null; -} - -function getPrivateDeniedRedirectUrl (app, denyRedirectUrl) { - if ( typeof denyRedirectUrl === 'string' && denyRedirectUrl.trim() ) { - return denyRedirectUrl.trim(); - } - - const origin = `${originUrl ?? ''}`.trim().replace(/\/$/, ''); - if ( origin ) { - return `${origin}/app/app-center/?item=${encodeURIComponent(app?.uid ?? '')}`; - } - - return '/'; -} - -function getMarketplaceAppUrl (app) { - const appName = typeof app?.name === 'string' - ? app.name.trim() - : ''; - if ( ! appName ) return null; - - const origin = `${originUrl ?? ''}`.trim().replace(/\/$/, ''); - if ( ! origin ) return null; - - return `${origin}/app/${encodeURIComponent(appName)}/`; -} - -function appendLinkHeader (res, linkValue) { - if ( ! linkValue ) return; - const existingValue = typeof res.get === 'function' - ? res.get('Link') - : ( - typeof res.getHeader === 'function' - ? res.getHeader('Link') - : undefined - ); - const setHeader = typeof res.set === 'function' - ? (value) => res.set('Link', value) - : ( - typeof res.setHeader === 'function' - ? (value) => res.setHeader('Link', value) - : null - ); - if ( ! setHeader ) return; - if ( ! existingValue ) { - setHeader(linkValue); - return; - } - setHeader(`${existingValue}, ${linkValue}`); -} - -function setReferrerPolicyHeader (res, policyValue = 'no-referrer') { - const setHeader = typeof res.set === 'function' - ? () => res.set('Referrer-Policy', policyValue) - : ( - typeof res.setHeader === 'function' - ? () => res.setHeader('Referrer-Policy', policyValue) - : null - ); - if ( ! setHeader ) return; - setHeader(); -} - -function isPrivateAccessGateEnabled () { - return config.enable_private_app_access_gate !== false; -} - -function logPrivateAccessEvent (eventName, fields = {}) { - console.info('private_access', { - eventName, - ...fields, - }); -} - -function getPrivateAccessRejectionReason (error) { - return error?.code || error?.message || 'unknown'; -} - -function getTokenFromAuthorizationHeader (req) { - const authorizationHeader = req.headers?.authorization; - if ( typeof authorizationHeader !== 'string' ) return null; - const match = authorizationHeader.match(/^Bearer\s+(.+)$/i); - return match?.[1]?.trim() || null; -} - -function getBootstrapTokenFromReferrer (req) { - const referrerHeader = req.headers?.referer ?? req.headers?.referrer; - if ( typeof referrerHeader !== 'string' || !referrerHeader.trim() ) { - return null; - } - - try { - const referrerUrl = new URL(referrerHeader); - return referrerUrl.searchParams.get('puter.auth.token') - || referrerUrl.searchParams.get('auth_token'); - } catch { - return null; - } -} - -function getBootstrapPrivateToken (req) { - const authorizationToken = getTokenFromAuthorizationHeader(req); - if ( authorizationToken ) return authorizationToken; - - const queryTokenCandidates = [ - req.query?.['puter.auth.token'], - req.query?.puter?.auth?.token, - req.query?.auth_token, - ]; - for ( const queryTokenCandidate of queryTokenCandidates ) { - if ( typeof queryTokenCandidate === 'string' && queryTokenCandidate.trim() ) { - return queryTokenCandidate.trim(); - } - } - - const headerToken = req.headers?.['x-puter-auth-token']; - if ( typeof headerToken === 'string' && headerToken.trim() ) { - return headerToken.trim(); - } - - return getBootstrapTokenFromReferrer(req); -} - -function getBootstrapPrivateTokenSource (req) { - if ( getTokenFromAuthorizationHeader(req) ) { - return 'authorization'; - } - if ( - (typeof req.query?.['puter.auth.token'] === 'string' && req.query['puter.auth.token'].trim()) - || (typeof req.query?.puter?.auth?.token === 'string' && req.query.puter.auth.token.trim()) - || (typeof req.query?.auth_token === 'string' && req.query.auth_token.trim()) - ) { - return 'query'; - } - if ( - typeof req.headers?.['x-puter-auth-token'] === 'string' - && req.headers['x-puter-auth-token'].trim() - ) { - return 'x-puter-auth-token'; - } - if ( getBootstrapTokenFromReferrer(req) ) { - return 'referrer'; - } - return 'none'; -} - -function actorToPrivateIdentity (actor) { - if ( ! actor ) return null; - - let userActor = null; - if ( actor.type instanceof UserActorType ) { - userActor = actor; - } else { - try { - userActor = actor.get_related_actor(UserActorType); - } catch { - userActor = null; - } - } - - const userUid = userActor?.type?.user?.uuid; - if ( typeof userUid !== 'string' || !userUid ) { - return null; - } - - const sessionCandidate = actor.type?.session ?? userActor.type?.session; - const sessionUuid = typeof sessionCandidate === 'string' - ? sessionCandidate - : sessionCandidate?.uuid; - - return { - userUid, - sessionUuid: typeof sessionUuid === 'string' && sessionUuid ? sessionUuid : undefined, - }; -} - -async function resolvePrivateIdentity ({ req, services, appUid }) { - const authService = services.get('auth'); - const privateCookieName = authService.getPrivateAssetCookieName(); - const privateCookieToken = req.cookies?.[privateCookieName]; - const privateAppSubdomain = getSubdomainFromHostedRequest(req) || undefined; - const requestedPrivateHost = getRequestedPrivateHost(req); - const hasPrivateCookie = typeof privateCookieToken === 'string' && !!privateCookieToken; - let hasInvalidPrivateCookie = false; - let hostedOriginAppUid; - if ( typeof authService.app_uid_from_origin === 'function' ) { - try { - const protocol = `${config.protocol ?? 'https'}` - .trim() - .replace(/:$/, '') || 'https'; - const requestedHostedOrigin = `${protocol}://${req.hostname}`; - const hostedOriginUid = await authService.app_uid_from_origin(requestedHostedOrigin); - if ( typeof hostedOriginUid === 'string' && hostedOriginUid ) { - hostedOriginAppUid = hostedOriginUid; - } - } catch { - // best effort only - } - } - const tokenAppUid = hostedOriginAppUid || appUid; - const expectedBootstrapAppUids = [tokenAppUid]; - if ( appUid && appUid !== tokenAppUid ) { - expectedBootstrapAppUids.push(appUid); - } - - if ( typeof privateCookieToken === 'string' && privateCookieToken ) { - try { - const claims = authService.verifyPrivateAssetToken(privateCookieToken, { - expectedAppUid: tokenAppUid, - expectedSubdomain: privateAppSubdomain, - expectedPrivateHost: requestedPrivateHost, - }); - return { - source: 'private-cookie', - userUid: claims.userUid, - sessionUuid: claims.sessionUuid, - tokenAppUid, - subdomain: claims.subdomain || privateAppSubdomain, - privateHost: claims.privateHost || requestedPrivateHost, - hasValidPrivateCookie: true, - hasPrivateCookie, - hasInvalidPrivateCookie, - }; - } catch (e) { - hasInvalidPrivateCookie = true; - logPrivateAccessEvent('private_access.identity_private_cookie_rejected', { - appUid, - requestHost: req.hostname, - reason: getPrivateAccessRejectionReason(e), - expectedAppUid: tokenAppUid ?? null, - expectedSubdomain: privateAppSubdomain ?? null, - expectedPrivateHost: requestedPrivateHost ?? null, - }); - // fallback to next token source - } - } - - const sessionToken = req.cookies?.[cookieName]; - if ( typeof sessionToken === 'string' && sessionToken ) { - try { - const actor = await authService.authenticate_from_token(sessionToken); - const identity = actorToPrivateIdentity(actor); - if ( identity ) { - return { - source: 'session-cookie', - ...identity, - tokenAppUid, - subdomain: privateAppSubdomain, - privateHost: requestedPrivateHost, - hasValidPrivateCookie: false, - hasPrivateCookie, - hasInvalidPrivateCookie, - }; - } - } catch (e) { - logPrivateAccessEvent('private_access.identity_session_cookie_rejected', { - appUid, - requestHost: req.hostname, - reason: getPrivateAccessRejectionReason(e), - }); - // fallback to next token source - } - } - - const bootstrapToken = getBootstrapPrivateToken(req); - const bootstrapTokenSource = getBootstrapPrivateTokenSource(req); - if ( typeof bootstrapToken === 'string' && bootstrapToken ) { - let strictAuthError; - try { - const actor = await authService.authenticate_from_token(bootstrapToken); - const identity = actorToPrivateIdentity(actor); - if ( identity ) { - if ( typeof authService.resolvePrivateBootstrapIdentityFromToken === 'function' ) { - await authService.resolvePrivateBootstrapIdentityFromToken(bootstrapToken, { - expectedAppUids: expectedBootstrapAppUids, - }); - } - return { - source: 'bootstrap-token', - ...identity, - tokenAppUid, - subdomain: privateAppSubdomain, - privateHost: requestedPrivateHost, - hasValidPrivateCookie: false, - hasPrivateCookie, - hasInvalidPrivateCookie, - }; - } - logPrivateAccessEvent('private_access.bootstrap_strict_missing_identity', { - appUid, - requestHost: req.hostname, - source: bootstrapTokenSource, - }); - } catch (e) { - strictAuthError = e; - logPrivateAccessEvent('private_access.bootstrap_strict_rejected', { - appUid, - requestHost: req.hostname, - source: bootstrapTokenSource, - reason: getPrivateAccessRejectionReason(e), - }); - } - - if ( typeof authService.resolvePrivateBootstrapIdentityFromToken === 'function' ) { - try { - const identity = await authService.resolvePrivateBootstrapIdentityFromToken(bootstrapToken, { - expectedAppUids: expectedBootstrapAppUids, - }); - if ( identity ) { - logPrivateAccessEvent('private_access.bootstrap_fallback_allowed', { - appUid, - userUid: identity.userUid ?? null, - requestHost: req.hostname, - source: 'bootstrap-token', - }); - return { - source: 'bootstrap-token', - ...identity, - tokenAppUid, - subdomain: privateAppSubdomain, - privateHost: requestedPrivateHost, - hasValidPrivateCookie: false, - hasPrivateCookie, - hasInvalidPrivateCookie, - }; - } - logPrivateAccessEvent('private_access.bootstrap_fallback_missing_identity', { - appUid, - requestHost: req.hostname, - source: bootstrapTokenSource, - strictReason: strictAuthError?.code || strictAuthError?.message || null, - }); - } catch (e) { - logPrivateAccessEvent('private_access.bootstrap_fallback_rejected', { - appUid, - requestHost: req.hostname, - source: bootstrapTokenSource, - reason: e?.code || e?.message || 'unknown', - strictReason: strictAuthError?.code || strictAuthError?.message || null, - }); - } - } else if ( strictAuthError ) { - logPrivateAccessEvent('private_access.bootstrap_rejected_no_fallback', { - appUid, - requestHost: req.hostname, - source: bootstrapTokenSource, - reason: getPrivateAccessRejectionReason(strictAuthError), - }); - } - } - - return { - source: 'none', - userUid: undefined, - sessionUuid: undefined, - tokenAppUid, - subdomain: privateAppSubdomain, - privateHost: requestedPrivateHost, - hasValidPrivateCookie: false, - hasPrivateCookie, - hasInvalidPrivateCookie, - }; -} - -function getPublicHostedActorCookieName (authService) { - if ( typeof authService?.getPublicHostedActorCookieName === 'function' ) { - return authService.getPublicHostedActorCookieName(); - } - return defaultPublicHostedActorCookieName; -} - -function getRequestedHostedHost (req) { - const normalizedHost = normalizeConfiguredHostname(req.hostname); - return normalizedHost || undefined; -} - -function buildLightweightHostedActor ({ userUid, sessionUuid }) { - if ( typeof userUid !== 'string' || !userUid ) { - return null; - } - - return new Actor({ - user_uid: userUid, - type: new UserActorType({ - user: { uuid: userUid }, - ...(sessionUuid ? { session: sessionUuid } : {}), - hasHttpOnlyCookie: false, - }), - }); -} - -function setHostedActorOnRequestContext ({ req, actor }) { - if ( ! actor ) return; - req.actor = actor; - Context.set('actor', actor); -} - -async function resolvePublicHostedIdentity ({ req, services, appUid }) { - const authService = services.get('auth'); - const publicHostedCookieName = getPublicHostedActorCookieName(authService); - const publicHostedCookieToken = req.cookies?.[publicHostedCookieName]; - const hostedSubdomain = getSubdomainFromHostedRequest(req) || undefined; - const requestedHost = getRequestedHostedHost(req); - const hasPublicCookie = typeof publicHostedCookieToken === 'string' && !!publicHostedCookieToken; - let hasInvalidPublicCookie = false; - - if ( - typeof publicHostedCookieToken === 'string' - && publicHostedCookieToken - && typeof authService.verifyPublicHostedActorToken === 'function' - ) { - try { - const claims = authService.verifyPublicHostedActorToken(publicHostedCookieToken, { - ...(appUid ? { expectedAppUid: appUid } : {}), - expectedSubdomain: hostedSubdomain, - expectedHost: requestedHost, - }); - return { - source: 'public-cookie', - userUid: claims.userUid, - sessionUuid: claims.sessionUuid, - tokenAppUid: claims.appUid || appUid, - subdomain: claims.subdomain || hostedSubdomain, - host: claims.host || requestedHost, - hasValidPublicCookie: true, - hasPublicCookie, - hasInvalidPublicCookie, - actor: null, - }; - } catch (e) { - hasInvalidPublicCookie = true; - logPrivateAccessEvent('public_actor.identity_public_cookie_rejected', { - appUid: appUid ?? null, - requestHost: req.hostname, - reason: getPrivateAccessRejectionReason(e), - }); - } - } - - const sessionToken = req.cookies?.[cookieName]; - if ( typeof sessionToken === 'string' && sessionToken ) { - try { - const actor = await authService.authenticate_from_token(sessionToken); - const identity = actorToPrivateIdentity(actor); - if ( identity ) { - return { - source: 'session-cookie', - ...identity, - tokenAppUid: appUid, - subdomain: hostedSubdomain, - host: requestedHost, - hasValidPublicCookie: false, - hasPublicCookie, - hasInvalidPublicCookie, - actor, - }; - } - } catch (e) { - logPrivateAccessEvent('public_actor.identity_session_cookie_rejected', { - appUid: appUid ?? null, - requestHost: req.hostname, - reason: getPrivateAccessRejectionReason(e), - }); - } - } - - const bootstrapToken = getBootstrapPrivateToken(req); - if ( typeof bootstrapToken === 'string' && bootstrapToken ) { - if ( typeof authService.resolvePrivateBootstrapIdentityFromToken === 'function' ) { - try { - const identity = await authService.resolvePrivateBootstrapIdentityFromToken( - bootstrapToken, - { - ...(appUid ? { expectedAppUid: appUid } : {}), - }, - ); - if ( identity?.userUid ) { - return { - source: 'bootstrap-token', - ...identity, - tokenAppUid: appUid, - subdomain: hostedSubdomain, - host: requestedHost, - hasValidPublicCookie: false, - hasPublicCookie, - hasInvalidPublicCookie, - actor: null, - }; - } - } catch (e) { - logPrivateAccessEvent('public_actor.identity_bootstrap_rejected', { - appUid: appUid ?? null, - requestHost: req.hostname, - reason: getPrivateAccessRejectionReason(e), - }); - } - } else { - try { - const actor = await authService.authenticate_from_token(bootstrapToken); - const identity = actorToPrivateIdentity(actor); - if ( identity ) { - return { - source: 'bootstrap-token', - ...identity, - tokenAppUid: appUid, - subdomain: hostedSubdomain, - host: requestedHost, - hasValidPublicCookie: false, - hasPublicCookie, - hasInvalidPublicCookie, - actor, - }; - } - } catch (e) { - logPrivateAccessEvent('public_actor.identity_bootstrap_rejected', { - appUid: appUid ?? null, - requestHost: req.hostname, - reason: getPrivateAccessRejectionReason(e), - }); - } - } - } - - return { - source: 'none', - userUid: undefined, - sessionUuid: undefined, - tokenAppUid: appUid, - subdomain: hostedSubdomain, - host: requestedHost, - hasValidPublicCookie: false, - hasPublicCookie, - hasInvalidPublicCookie, - actor: null, - }; -} - -async function evaluatePublicHostedActorContext ({ - req, - res, - services, - appUid, -}) { - const existingActor = req.actor || Context.get('actor'); - if ( existingActor ) { - const existingIdentity = actorToPrivateIdentity(existingActor); - if ( existingIdentity?.userUid ) { - return true; - } - } - - const authService = services.get('auth'); - const identity = await resolvePublicHostedIdentity({ - req, - services, - appUid, - }); - - if ( identity.actor ) { - setHostedActorOnRequestContext({ - req, - actor: identity.actor, - }); - } else if ( identity.userUid ) { - const lightweightActor = buildLightweightHostedActor({ - userUid: identity.userUid, - sessionUuid: identity.sessionUuid, - }); - setHostedActorOnRequestContext({ - req, - actor: lightweightActor, - }); - } - - if ( !identity.userUid || identity.hasValidPublicCookie ) { - return true; - } - - let tokenAppUid = identity.tokenAppUid; - if ( !tokenAppUid && typeof authService.app_uid_from_origin === 'function' ) { - try { - const protocol = `${config.protocol ?? 'https'}` - .trim() - .replace(/:$/, '') || 'https'; - tokenAppUid = await authService.app_uid_from_origin(`${protocol}://${req.hostname}`); - } catch { - tokenAppUid = undefined; - } - } - if ( !tokenAppUid || typeof authService.createPublicHostedActorToken !== 'function' ) { - return true; - } - - try { - const publicHostedActorToken = authService.createPublicHostedActorToken({ - appUid: tokenAppUid, - userUid: identity.userUid, - sessionUuid: identity.sessionUuid, - subdomain: identity.subdomain, - host: identity.host, - }); - res.cookie( - getPublicHostedActorCookieName(authService), - publicHostedActorToken, - typeof authService.getPublicHostedActorCookieOptions === 'function' - ? authService.getPublicHostedActorCookieOptions({ - requestHostname: req.hostname, - }) - : undefined, - ); - } catch (e) { - logPrivateAccessEvent('public_actor.cookie_set_failed', { - appUid: tokenAppUid ?? null, - userUid: identity.userUid ?? null, - requestHost: req.hostname, - reason: getPrivateAccessRejectionReason(e), - }); - return true; - } - - return true; -} - -function escapeHtml (value) { - const raw = `${value ?? ''}`; - return raw - .replaceAll('&', '&') - .replaceAll('<', '<') - .replaceAll('>', '>') - .replaceAll('"', '"') - .replaceAll('\'', '''); -} - -function respondPrivateLoginBootstrap ({ res, app }) { - const appName = - typeof app?.name === 'string' && app.name.trim() - ? app.name.trim() - : 'this app'; - const appTitle = typeof app?.title === 'string' && app.title.trim() - ? app.title.trim() - : appName; - const appDescription = typeof app?.description === 'string' && app.description.trim() - ? app.description.trim() - : `${appTitle} requires Puter authentication before private files can load.`; - const appIcon = typeof app?.icon === 'string' && app.icon.trim() - ? app.icon.trim() - : null; - const marketplaceAppUrl = getMarketplaceAppUrl(app); - const safeAppName = escapeHtml(appName); - const safeAppTitle = escapeHtml(appTitle); - const safeAppDescription = escapeHtml(appDescription); - const safeMarketplaceAppUrl = escapeHtml(marketplaceAppUrl ?? ''); - const safeAppIcon = escapeHtml(appIcon ?? ''); - - const loginHtml = dedent(` - - - - - - Sign In Required | ${safeAppTitle} - - - - - - ${safeMarketplaceAppUrl ? `` : ''} - ${safeAppIcon ? `` : ''} - - - - ${safeAppIcon ? `` : ''} - ${safeMarketplaceAppUrl ? `` : ''} - - - -
-

Sign in required

-

${safeAppName} requires Puter authentication before private files can load.

-

Click “Sign In with Puter” to continue.

-
- - -
-
- - - - - `); - - res.status(200); - res.set('Cache-Control', 'no-store'); - res.set('X-Robots-Tag', 'noindex, nofollow'); - setReferrerPolicyHeader(res); - appendLinkHeader( - res, - marketplaceAppUrl ? `<${marketplaceAppUrl}>; rel="canonical"` : null, - ); - res.set('Content-Type', 'text/html; charset=UTF-8'); - return res.send(loginHtml); -} - -async function evaluatePrivateAppAccess ({ req, res, services, app, requestPath }) { - const identity = await resolvePrivateIdentity({ - req, - services, - appUid: app.uid, - }); - - if ( ! identity.userUid ) { - logPrivateAccessEvent('private_access.auth_required', { - appUid: app.uid, - userUid: null, - requestHost: req.hostname, - requestPath, - source: identity.source, - hasPrivateCookie: identity.hasPrivateCookie, - hasInvalidPrivateCookie: identity.hasInvalidPrivateCookie, - }); - respondPrivateLoginBootstrap({ res, app }); - return false; - } - - const eventService = services.get('event'); - const accessCheckEvent = { - appUid: app.uid, - userUid: identity.userUid ?? null, - requestHost: req.hostname, - requestPath, - result: { - allowed: false, - }, - }; - - try { - await eventService.emit('app.privateAccess.check', accessCheckEvent); - } catch (e) { - logPrivateAccessEvent('private_access.entitlement_check_error', { - appUid: app.uid, - userUid: identity.userUid ?? null, - requestHost: req.hostname, - requestPath, - source: identity.source, - error: e?.message || String(e), - }); - console.error('private app access check failed', e); - } - - if ( ! accessCheckEvent.result.allowed ) { - const redirectUrl = getPrivateDeniedRedirectUrl( - app, - accessCheckEvent.result.redirectUrl, - ); - logPrivateAccessEvent('private_access.denied', { - appUid: app.uid, - userUid: identity.userUid ?? null, - requestHost: req.hostname, - requestPath, - source: identity.source, - reason: accessCheckEvent.result.reason ?? null, - redirectUrl, - hasPrivateCookie: identity.hasPrivateCookie, - hasInvalidPrivateCookie: identity.hasInvalidPrivateCookie, - }); - const marketplaceAppUrl = getMarketplaceAppUrl(app); - appendLinkHeader( - res, - marketplaceAppUrl ? `<${marketplaceAppUrl}>; rel="alternate"` : null, - ); - res.redirect(redirectUrl); - return false; - } - - const shouldRefreshPrivateCookie = identity.userUid && !identity.hasValidPrivateCookie; - if ( identity.userUid && !identity.hasValidPrivateCookie ) { - const authService = services.get('auth'); - const privateToken = authService.createPrivateAssetToken({ - appUid: identity.tokenAppUid || app.uid, - userUid: identity.userUid, - sessionUuid: identity.sessionUuid, - subdomain: identity.subdomain, - privateHost: identity.privateHost, - }); - res.cookie( - authService.getPrivateAssetCookieName(), - privateToken, - authService.getPrivateAssetCookieOptions({ - requestHostname: req.hostname, - }), - ); - - } - - logPrivateAccessEvent('private_access.allowed', { - appUid: app.uid, - userUid: identity.userUid ?? null, - requestHost: req.hostname, - requestPath, - source: identity.source, - cookieRefreshed: !!shouldRefreshPrivateCookie, - hasPrivateCookie: identity.hasPrivateCookie, - hasInvalidPrivateCookie: identity.hasInvalidPrivateCookie, - }); - return true; -} - -async function runInternal (req, res, next) { - const isPrivateHostedRequest = hostMatchesPrivateDomain(req.hostname); - const subdomain = - req.is_custom_domain && !isPrivateHostedRequest ? req.hostname : - req.subdomains[0] === 'devtest' ? 'devtest' : - getSubdomainFromHostedRequest(req); - - let path = (req.baseUrl + req.path) || 'index.html'; - - const context = Context.get(); - const services = context.get('services'); - - const getUsernameSite = (async () => { - if ( ! subdomain.endsWith('.at') ) return; - const parts = subdomain.split('.'); - if ( parts.length !== 2 ) return; - const username = parts[0]; - if ( ! username.match(usernameRegex) ) { - return; - } - const filesystemService = services.get('filesystem'); - const indexNode = await filesystemService.node(new NodePathSelector(`/${username}/Public/index.html`)); - const node = await filesystemService.node(new NodePathSelector(`/${username}/Public`)); - if ( ! await indexNode.exists() ) return; - - return { - name: `${username }.at`, - uuid: uuidv5(username, AT_DIRECTORY_NAMESPACE), - root_dir_id: await node.get('mysql-id'), - }; - }); - - if ( req.hostname === staticHostingDomain || req.hostname === staticHostingDomainAlt || subdomain === 'www' ) { - - // redirect to information page about static hosting - return res.redirect(staticHostingBaseDomainRedirect); - } - - const site = - await getUsernameSite() || - await (async () => { - const puterSiteService = services.get('puter-site'); - const site = await puterSiteService.get_subdomain(subdomain, { - is_custom_domain: req.is_custom_domain && !isPrivateHostedRequest, - }); - return site; - })(); - - if ( site === null ) { - return res.status(404).send('Subdomain not found'); - } - - const subdomainOwner = await get_user({ id: site.user_id }); - if ( subdomainOwner?.suspended ) { - // This used to be "401 Account suspended", but this implies - // the client user is suspended, which is not the case. - // Instead we simply return 404, indicating that this page - // doesn't exist without further specifying that the owner's - // account is suspended. (the client user doesn't need to know) - return res.status(404).send('Subdomain not found'); - } - - const associatedApp = site.associated_app_id - ? await get_app({ id: site.associated_app_id }) - : null; - const privateApp = await resolvePrivateAppForHostedSite({ - req, - site, - services, - associatedApp, - }); - const privateAppEnabled = isPrivateApp(privateApp); - const privateAccessGateEnabled = isPrivateAccessGateEnabled(); - - if ( privateAppEnabled ) { - setReferrerPolicyHeader(res); - } - - if ( - privateAccessGateEnabled - && privateAppEnabled - && !hostMatchesPrivateDomain(req.hostname) - ) { - const privateHostRedirect = buildPrivateHostRedirectUrl(req, privateApp); - if ( privateHostRedirect ) { - logPrivateAccessEvent('private_access.host_redirect', { - appUid: privateApp?.uid ?? null, - requestHost: req.hostname, - requestPath: req.path, - redirectUrl: privateHostRedirect, - }); - const marketplaceAppUrl = getMarketplaceAppUrl(privateApp); - appendLinkHeader( - res, - marketplaceAppUrl - ? `<${marketplaceAppUrl}>; rel="alternate"` - : null, - ); - return res.redirect(privateHostRedirect); - } - logPrivateAccessEvent('private_access.host_mismatch_denied', { - appUid: privateApp?.uid ?? null, - requestHost: req.hostname, - requestPath: req.path, - }); - return res.status(403).send('Private app host mismatch'); - } - - if ( - site.associated_app_id && - !privateAppEnabled && - !req.query['puter.app_instance_id'] && - ( path === '' || path.endsWith('/') ) - ) { - const app = associatedApp || await get_app({ id: site.associated_app_id }); - return res.redirect(`${originUrl}/app/${app.name}/`); - } - - if ( path === '' ) path += '/index.html'; - else if ( path.endsWith('/') ) path += 'index.html'; - - const resolvedUrlPath = - resolve('/', path); - - const filesystemService = services.get('filesystem'); - - let subdomainRootPath = ''; - if ( site.root_dir_id !== null && site.root_dir_id !== undefined ) { - const node = await filesystemService.node(new NodeInternalIDSelector('mysql', site.root_dir_id)); - if ( ! await node.exists() ) { - return res.status(502).send('subdomain is pointing to deleted directory'); - } - if ( await node.get('type') !== TYPE_DIRECTORY ) { - return res.status(502).send('subdomain is pointing to non-directory'); - } - - // Verify subdomain owner permission - const subdomainActor = Actor.adapt(subdomainOwner); - const aclService = services.get('acl'); - if ( ! await aclService.check(subdomainActor, node, 'read') ) { - res.status(502).send('subdomain owner does not have access to directory'); - return; - } - - subdomainRootPath = await node.get('path'); - } - - if ( ! subdomainRootPath ) { - return respondHtmlError({ - html: dedent(` - Subdomain or site is not pointing to a directory. - `), - }, req, res, next); - } - - if ( !subdomainRootPath || subdomainRootPath === '/' ) { - throw APIError.create('forbidden'); - } - - req.__puterSiteRootPath = subdomainRootPath; - - if ( ! privateAppEnabled ) { - try { - const actorContextReady = await evaluatePublicHostedActorContext({ - req, - res, - services, - appUid: privateApp?.uid || associatedApp?.uid, - }); - if ( ! actorContextReady ) return; - } catch (e) { - logPrivateAccessEvent('public_actor.evaluate_failed', { - appUid: privateApp?.uid || associatedApp?.uid || null, - requestHost: req.hostname, - reason: getPrivateAccessRejectionReason(e), - }); - } - } - - if ( privateAccessGateEnabled && privateAppEnabled ) { - const accessAllowed = await evaluatePrivateAppAccess({ - req, - res, - services, - app: privateApp, - requestPath: req.path, - }); - if ( ! accessAllowed ) return; - } - - const filepath = subdomainRootPath + decodeURIComponent(resolvedUrlPath); - - const targetNode = await filesystemService.node(new NodePathSelector(filepath)); - await targetNode.fetchEntry(); - - if ( ! await targetNode.exists() ) { - return await respond404({ path }, req, res, next, subdomainRootPath); - } - - const targetIsDir = await targetNode.get('type') === TYPE_DIRECTORY; - - if ( targetIsDir && !resolvedUrlPath.endsWith('/') ) { - return res.redirect(`${resolvedUrlPath }/`); - } - - if ( targetIsDir ) { - return await respond404({ path }, req, res, next, subdomainRootPath); - } - - const contentType = contentTypeFromMime(await targetNode.get('name')); - res.set('Content-Type', contentType); - - const aclConfig = { - no_acl: true, - actor: null, - }; - - if ( site.protected ) { - const authService = req.services.get('auth'); - - const getSiteActorFromToken = async () => { - const siteToken = req.cookies['puter.site.token']; - if ( ! siteToken ) return; - - let failed = false; - let siteActor; - try { - siteActor = - await authService.authenticate_from_token(siteToken); - } catch (e) { - failed = true; - } - - if ( failed ) return; - - if ( ! siteActor ) return; - - // security measure: if 'puter.site.token' is set - // to a different actor type, someone is likely - // trying to exploit the system. - if ( ! (siteActor.type instanceof SiteActorType) ) { - return; - } - - aclConfig.actor = siteActor; - - // Refresh the token if it's been 30 seconds since - // the last request - if ( - (Date.now() - siteActor.type.iat * 1000) - > - 1000 * 30 - ) { - const siteToken = authService.get_site_app_token({ - site_uid: site.uuid, - }); - res.cookie('puter.site.token', siteToken); - } - - return true; - }; - - const makeSiteActorFromAppToken = async () => { - const token = req.query['puter.auth.token']; - - aclConfig.no_acl = false; - - if ( ! token ) { - const e = APIError.create('token_missing'); - return respondError({ req, res, e }); - } - - const appActor = - await authService.authenticate_from_token(token); - - const userActor = - appActor.get_related_actor(UserActorType); - - const permissionService = req.services.get('permission'); - const perm = await (async () => { - if ( userActor.type.user.id === site.user_id ) { - return {}; - } - - const reading = await permissionService.scan(userActor, `site:uid#${site.uuid}:access`); - const options = PermissionUtil.reading_to_options(reading); - return options.length > 0; - })(); - - if ( ! perm ) { - const e = APIError.create('forbidden'); - respondError({ req, res, e }); - return false; - } - - const siteActor = await Actor.create(SiteActorType, { site }); - aclConfig.actor = siteActor; - - // This subdomain is allowed to keep the site actor token, - // so we send it here as a cookie so other html files can - // also load. - const siteToken = authService.get_site_app_token({ - site_uid: site.uuid, - }); - res.cookie('puter.site.token', siteToken); - return true; - }; - - let ok = await getSiteActorFromToken(); - if ( ! ok ) { - ok = await makeSiteActorFromAppToken(); - } - if ( ! ok ) return; - - Object.freeze(aclConfig); - } - - // Helper function to parse Range header - const parseRangeHeader = (rangeHeader) => { - // Check if this is a multipart range request - if ( rangeHeader.includes(',') ) { - // For now, we'll only serve the first range in multipart requests - // as the underlying storage layer doesn't support multipart responses - const firstRange = rangeHeader.split(',')[0].trim(); - const matches = firstRange.match(/bytes=(\d+)-(\d*)/); - if ( ! matches ) return null; - - const start = parseInt(matches[1], 10); - const end = matches[2] ? parseInt(matches[2], 10) : null; - - return { start, end, isMultipart: true }; - } - - // Single range request - const matches = rangeHeader.match(/bytes=(\d+)-(\d*)/); - if ( ! matches ) return null; - - const start = parseInt(matches[1], 10); - const end = matches[2] ? parseInt(matches[2], 10) : null; - - return { start, end, isMultipart: false }; - }; - if ( req.headers['range'] ) { - res.status(206); - - // Parse the Range header and set Content-Range - const rangeInfo = parseRangeHeader(req.headers['range']); - if ( rangeInfo ) { - const { start, end, isMultipart } = rangeInfo; - - // For open-ended ranges, we need to calculate the actual end byte - let actualEnd = end; - let fileSize = null; - - try { - fileSize = await targetNode.get('size'); - if ( end === null ) { - actualEnd = fileSize - 1; // File size is 1-based, end byte is 0-based - } - } catch (e) { - // If we can't get file size, we'll let the storage layer handle it - // and not set Content-Range header - actualEnd = null; - fileSize = null; - } - - if ( actualEnd !== null ) { - const totalSize = fileSize !== null ? fileSize : '*'; - const contentRange = `bytes ${start}-${actualEnd}/${totalSize}`; - res.set('Content-Range', contentRange); - } - - // If this was a multipart request, modify the range header to only include the first range - if ( isMultipart ) { - req.headers['range'] = end !== null - ? `bytes=${start}-${end}` - : `bytes=${start}-`; - } - } - } else { - if ( targetNode.entry.size ) { - res.set('x-expected-entity-length', targetNode.entry.size); - } - } - res.set({ 'Accept-Ranges': 'bytes' }); - - const llRead = new LLRead(); - // const actor = Actor.adapt(req.user); - const stream = await llRead.run({ - no_acl: aclConfig.no_acl, - actor: aclConfig.actor, - fsNode: targetNode, - ...(req.headers['range'] ? { range: req.headers['range'] } : { }), - }); - - // Destroy the stream if the client disconnects - req.on('close', () => { - stream.destroy(); - }); - - try { - return stream.pipe(res); - } catch (e) { - const handled = await respondSiteError({ - path, - req, - res, - next, - subdomainRootPath, - }); - if ( handled ) return; - return res.status(500).send(`Error reading file: ${ e.message}`); - } -} - -async function respondSiteError ({ path, html, req, res, next, subdomainRootPath }) { - const handled = await maybeRespondWithSiteConfig({ - path, - html, - req, - res, - next, - subdomainRootPath, - errorStatus: 500, - }); - return handled; -} - -async function getSiteErrorConfig (req, subdomainRootPath) { - if ( ! subdomainRootPath ) return null; - req.__puterSiteErrorConfigCache ??= Object.create(null); - - if ( req.__puterSiteErrorConfigCache[subdomainRootPath] !== undefined ) { - return req.__puterSiteErrorConfigCache[subdomainRootPath]; - } - - try { - const context = Context.get(); - const services = context.get('services'); - const filesystemService = services.get('filesystem'); - - const configPath = `${subdomainRootPath}/${puterSiteConfigFilename}`; - const configNode = await filesystemService.node(new NodePathSelector(configPath)); - await configNode.fetchEntry(); - - if ( ! await configNode.exists() ) { - req.__puterSiteErrorConfigCache[subdomainRootPath] = null; - return null; - } - if ( await configNode.get('type') === TYPE_DIRECTORY ) { - req.__puterSiteErrorConfigCache[subdomainRootPath] = null; - return null; - } - - const size = Number(await configNode.get('size') ?? 0); - if ( Number.isFinite(size) && size > puterSiteConfigMaxSize ) { - req.__puterSiteErrorConfigCache[subdomainRootPath] = null; - return null; - } - - const llRead = new LLRead(); - const stream = await llRead.run({ - no_acl: true, - actor: null, - fsNode: configNode, - }); - const buffer = await streamToBuffer(stream); - const text = buffer.toString('utf8'); - const parsed = parseSiteErrorConfig(text); - - req.__puterSiteErrorConfigCache[subdomainRootPath] = parsed; - return parsed; - } catch { - req.__puterSiteErrorConfigCache[subdomainRootPath] = null; - return null; - } -} - -async function getSiteFileNode (subdomainRootPath, sitePath) { - const context = Context.get(); - const services = context.get('services'); - const filesystemService = services.get('filesystem'); - - const fullPath = `${subdomainRootPath}${sitePath}`; - const node = await filesystemService.node(new NodePathSelector(fullPath)); - await node.fetchEntry(); - if ( ! await node.exists() ) return null; - if ( await node.get('type') === TYPE_DIRECTORY ) return null; - return node; -} - -async function maybeRespondWithSiteConfig ({ - path, - html, - req, - res, - next, - subdomainRootPath, - errorStatus, -}) { - if ( ! subdomainRootPath ) return false; - - const parsedConfig = await getSiteErrorConfig(req, subdomainRootPath); - if ( ! parsedConfig ) return false; - - const rule = getSiteErrorRule(parsedConfig, errorStatus); - if ( ! rule ) return false; - - const responseStatus = rule.status ?? errorStatus; - if ( rule.file ) { - const node = await getSiteFileNode(subdomainRootPath, rule.file); - if ( node ) { - await streamSiteFile({ - req, - res, - fsNode: node, - status: responseStatus, - }); - return true; - } - } - - if ( rule.status !== null && rule.status !== undefined ) { - respondHtmlError({ path, html, status: responseStatus }, req, res, next); - return true; - } - - return false; -} - -async function streamSiteFile ({ req, res, fsNode, status }) { - res.status(status); - const contentType = - contentTypeFromMime(await fsNode.get('name')) || - 'application/octet-stream'; - res.set('Content-Type', contentType); - - const llRead = new LLRead(); - const stream = await llRead.run({ - no_acl: true, - actor: null, - fsNode, - }); - - req.on('close', () => { - stream.destroy(); - }); - - return stream.pipe(res); -} - -async function respond404 ({ path, html }, req, res, next, subdomainRootPath) { - const handled = await maybeRespondWithSiteConfig({ - path, - html, - req, - res, - next, - subdomainRootPath, - errorStatus: 404, - }); - if ( handled ) return; - - if ( subdomainRootPath ) { - const custom404Node = await getSiteFileNode(subdomainRootPath, '/404.html'); - if ( custom404Node ) { - return streamSiteFile({ - req, - res, - fsNode: custom404Node, - status: 404, - }); - } - } - - return respondHtmlError({ path, html, status: 404 }, req, res, next); -} - -function respondHtmlError ({ path, html, status = 404 }, req, res, _next) { - res.status(status); - res.set('Content-Type', 'text/html; charset=UTF-8'); - res.write(`
`); - res.write(`

${status}

`); - res.write('

'); - if ( status === 404 && path ) { - if ( path === '/index.html' ) { - res.write('index.html Not Found'); - } else { - res.write('Not Found'); - } - } else { - res.write(html || 'Request failed'); - } - res.write('

'); - - res.write('
'); - - return res.end(); -} - -function respondError ({ req, res, e }) { - if ( ! (e instanceof APIError) ) { - // TODO: alarm here - e = APIError.create('unknown_error'); - } - - res.redirect(`${originUrl}?${e.querystringize({ - ...(req.query['puter.app_instance_id'] ? { - 'error_from_within_iframe': true, - } : {}), - })}`); -} - -export async function puterSiteMiddleware (req, res, next) { - const isSubdomain = - req.hostname.endsWith(staticHostingDomain) - || (staticHostingDomainAlt && req.hostname.endsWith(staticHostingDomainAlt)) - || hostMatchesPrivateDomain(req.hostname) - || req.subdomains[0] === 'devtest' - ; - - if ( !isSubdomain && !req.is_custom_domain ) return next(); - - res.setHeader('Access-Control-Allow-Origin', '*'); - - try { - const expectedCtx = req.ctx; - const receivedCtx = Context.get(); - - if ( expectedCtx && !receivedCtx ) { - await expectedCtx.arun(async () => { - await runInternal(req, res, next); - }); - } else await runInternal(req, res, next); - } catch ( e ) { - console.error('puter-site middleware error', e); - if ( !res.headersSent && req.__puterSiteRootPath ) { - try { - const handled = await respondSiteError({ - path: req.path, - req, - res, - next, - subdomainRootPath: req.__puterSiteRootPath, - }); - if ( handled ) return; - } catch ( siteError ) { - console.error('failed handling site error response', siteError); - } - } - api_error_handler(e, req, res, next); - } -} diff --git a/src/backend/src/routers/hosting/puterSiteMiddleware.test.js b/src/backend/src/routers/hosting/puterSiteMiddleware.test.js deleted file mode 100644 index 30c31ab25..000000000 --- a/src/backend/src/routers/hosting/puterSiteMiddleware.test.js +++ /dev/null @@ -1,2039 +0,0 @@ -/* - * Copyright (C) 2026-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { puterSiteMiddleware } from './puterSiteMiddleware'; -import config from '../../config.js'; -import { Context } from '../../util/context.js'; - -// Mocks to test middleware logic with minimal integration complexity -// (I added region markers, so this can be collapsed for readability) - -// #region: mocks -let getUserMockImpl = async () => null; -let getAppMockImpl = async () => null; - -vi.mock('../../config.js', () => ({ - default: { - static_hosting_domain: 'site.puter.localhost', - static_hosting_base_domain_redirect: 'https://developer.puter.com/static-hosting/', - private_app_hosting_domain: 'puter.dev', - private_app_hosting_domain_alt: 'puter.dev', - enable_private_app_access_gate: true, - origin: 'https://puter.com', - cookie_name: 'puter.session.token', - username_regex: /^[a-z0-9_]+$/, - }, - static_hosting_domain: 'site.puter.localhost', - static_hosting_base_domain_redirect: 'https://developer.puter.com/static-hosting/', - private_app_hosting_domain: 'puter.dev', - private_app_hosting_domain_alt: 'puter.dev', - enable_private_app_access_gate: true, - origin: 'https://puter.com', - cookie_name: 'puter.session.token', - username_regex: /^[a-z0-9_]+$/, -})); - -vi.mock('../../modules/web/lib/api_error_handler.js', () => ({ - default: vi.fn(), -})); - -vi.mock('../../helpers.js', () => ({ - get_user: vi.fn((...args) => getUserMockImpl(...args)), - get_app: vi.fn((...args) => getAppMockImpl(...args)), -})); - -vi.mock('../../util/context.js', () => ({ - Context: { - get: vi.fn(), - set: vi.fn(), - }, -})); - -// Mock Context to allow arun passthrough -const mockContextInstance = { - get: vi.fn(), - arun: vi.fn().mockImplementation(async (fn) => await fn()), -}; - -vi.mock('../../deprecated/filesystem/node/selectors.js', () => ({ - default: { - NodeInternalIDSelector: class { - }, - NodePathSelector: class { - }, - }, - NodeInternalIDSelector: class { - }, - NodePathSelector: class { - }, -})); - -vi.mock('../../deprecated/filesystem/FSNodeContext.js', () => ({ - default: { - TYPE_DIRECTORY: 'directory', - }, - TYPE_DIRECTORY: 'directory', -})); - -vi.mock('../../deprecated/filesystem/ll_operations/ll_read.js', () => ({ - default: { - LLRead: class { - }, - }, - LLRead: class { - }, -})); - -vi.mock('../../services/auth/Actor.js', () => { - const adapt = vi.fn(); - const create = vi.fn(); - class UserActorType { - constructor ({ user, session, hasHttpOnlyCookie } = {}) { - this.user = user; - this.session = session; - this.hasHttpOnlyCookie = hasHttpOnlyCookie; - } - } - class SiteActorType { - } - class Actor { - constructor ({ user_uid, app_uid, type } = {}) { - this.user_uid = user_uid; - this.app_uid = app_uid; - this.type = type; - } - - get_related_actor (actorType) { - if ( this.type instanceof actorType ) { - return this; - } - throw new Error('related_actor_not_found'); - } - } - Actor.adapt = adapt; - Actor.create = create; - return { - Actor, - UserActorType, - SiteActorType, - }; -}); - -vi.mock('../../api/APIError.js', () => ({ - default: class APIError { - static create () { - return new this(); - } - }, -})); - -vi.mock('../../services/auth/permissionUtils.mjs', () => ({ - PermissionUtil: { - reading_to_options: vi.fn().mockReturnValue([]), - }, -})); - -vi.mock('dedent', () => ({ - default: (str) => str, -})); -// #endregion - -// Now import the module under test - this will use our mocks -describe('PuterSiteMiddleware', () => { - describe('base domain redirect', () => { - let capturedMiddleware; - - beforeEach(() => { - vi.clearAllMocks(); - config.enable_private_app_access_gate = true; - config.private_app_hosting_domain = 'puter.dev'; - config.private_app_hosting_domain_alt = 'puter.dev'; - Context.get = vi.fn().mockImplementation((key) => { - if ( key === 'actor' ) return undefined; - return mockContextInstance; - }); - Context.set = vi.fn(); - getUserMockImpl = async () => null; - getAppMockImpl = async () => null; - capturedMiddleware = puterSiteMiddleware; - }); - - /** - * Creates a mock request for static hosting domain - */ - const createMockRequest = (subdomain) => { - const hostname = subdomain - ? `${subdomain}.${config.static_hosting_domain}` - : config.static_hosting_domain; - - return { - hostname, - subdomains: subdomain ? [subdomain] : [], - is_custom_domain: false, - baseUrl: '', - path: '/', - ctx: mockContextInstance, - }; - }; - - it('should redirect to info page when subdomain is empty (bare domain)', async () => { - const mockReq = createMockRequest(''); - const mockRes = { - redirect: vi.fn(), - setHeader: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - expect(mockRes.redirect).toHaveBeenCalledWith('https://developer.puter.com/static-hosting/'); - expect(mockNext).not.toHaveBeenCalled(); - }); - - it('should redirect to info page when subdomain is www', async () => { - const mockReq = createMockRequest('www'); - const mockRes = { - redirect: vi.fn(), - setHeader: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - expect(mockRes.redirect).toHaveBeenCalledWith('https://developer.puter.com/static-hosting/'); - expect(mockNext).not.toHaveBeenCalled(); - }); - - it('should NOT redirect when subdomain is a valid site name', async () => { - // Setup mock services for the "site not found" path - const mockServices = { - get: vi.fn().mockImplementation((svc) => { - if ( svc === 'puter-site' ) { - return { - get_subdomain: vi.fn().mockResolvedValue(null), - }; - } - if ( svc === 'filesystem' ) { - return { - node: vi.fn().mockResolvedValue({ - exists: vi.fn().mockResolvedValue(false), - }), - }; - } - return {}; - }), - }; - - mockContextInstance.get.mockImplementation((key) => { - if ( key === 'services' ) return mockServices; - return null; - }); - - const mockReq = createMockRequest('mysite'); - const mockRes = { - redirect: vi.fn(), - setHeader: vi.fn(), - status: vi.fn().mockReturnThis(), - send: vi.fn(), - }; - const mockNext = vi.fn(); - - // The middleware will error out further down (due to incomplete mocks) - // but the important thing is: did it try to redirect to the info page? - try { - await capturedMiddleware(mockReq, mockRes, mockNext); - } catch (e) { - // Expected - incomplete mocks cause errors after the redirect check - } - - // The key assertion: it should NOT have redirected to the info page - // because 'mysite' is a valid subdomain, not '' or 'www' - expect(mockRes.redirect).not.toHaveBeenCalledWith('https://developer.puter.com/static-hosting/'); - }); - - it('should use exactly the URL from config (not hardcoded)', async () => { - // This test verifies the middleware reads from config.static_hosting_base_domain_redirect - // If someone hardcodes a different URL, this assertion will catch that the - // redirect URL matches what is in the mocked config. - const mockReq = createMockRequest(''); - const mockRes = { - redirect: vi.fn(), - setHeader: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - // Verify it uses the exact URL from the mocked config - expect(mockRes.redirect).toHaveBeenCalledWith(config.static_hosting_base_domain_redirect); - }); - }); - - describe('private app access gate', () => { - let capturedMiddleware; - - beforeEach(() => { - vi.clearAllMocks(); - config.enable_private_app_access_gate = true; - Context.get = vi.fn().mockImplementation((key) => { - if ( key === 'actor' ) return undefined; - return mockContextInstance; - }); - Context.set = vi.fn(); - getUserMockImpl = async () => null; - getAppMockImpl = async () => null; - capturedMiddleware = puterSiteMiddleware; - }); - - it('redirects private app assets to puter.dev host even before index_url migration', async () => { - const mockServices = { - get: vi.fn().mockImplementation((serviceName) => { - if ( serviceName === 'puter-site' ) { - return { - get_subdomain: vi.fn().mockResolvedValue({ - user_id: 101, - associated_app_id: 202, - root_dir_id: null, - }), - }; - } - return {}; - }), - }; - mockContextInstance.get.mockImplementation((key) => { - if ( key === 'services' ) return mockServices; - return null; - }); - getUserMockImpl = async () => ({ id: 101, suspended: false }); - getAppMockImpl = async () => ({ - uid: 'app-11111111-1111-1111-1111-111111111111', - name: 'paid-app', - is_private: 1, - index_url: 'https://paid.site.puter.localhost/', - }); - - const mockReq = { - hostname: 'paid.site.puter.localhost', - subdomains: ['paid'], - is_custom_domain: false, - baseUrl: '', - path: '/asset.js', - originalUrl: '/asset.js?foo=1', - query: {}, - cookies: {}, - headers: {}, - ctx: mockContextInstance, - }; - const mockRes = { - redirect: vi.fn(), - setHeader: vi.fn(), - status: vi.fn().mockReturnThis(), - send: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - expect(mockRes.redirect).toHaveBeenCalledWith('https://paid.puter.dev/asset.js?foo=1'); - expect(mockNext).not.toHaveBeenCalled(); - }); - - it('accepts private app host matching the configured alt private domain', async () => { - config.private_app_hosting_domain = 'app.puter.localhost:4100'; - config.private_app_hosting_domain_alt = 'puter.dev'; - - const authService = { - getPrivateAssetCookieName: vi.fn().mockReturnValue('puter.private.asset.token'), - verifyPrivateAssetToken: vi.fn().mockImplementation(() => { - throw new Error('invalid'); - }), - authenticate_from_token: vi.fn().mockImplementation(() => { - throw new Error('invalid'); - }), - createPrivateAssetToken: vi.fn().mockReturnValue('private-token'), - getPrivateAssetCookieOptions: vi.fn().mockReturnValue({}), - }; - const mockServices = { - get: vi.fn().mockImplementation((serviceName) => { - if ( serviceName === 'puter-site' ) { - return { - get_subdomain: vi.fn().mockResolvedValue({ - user_id: 101, - associated_app_id: 202, - root_dir_id: 303, - }), - }; - } - if ( serviceName === 'filesystem' ) { - return { - node: vi.fn().mockResolvedValue({ - exists: vi.fn().mockResolvedValue(true), - get: vi.fn().mockImplementation(async (fieldName) => { - if ( fieldName === 'type' ) return 'directory'; - if ( fieldName === 'path' ) return '/alice/Public'; - return null; - }), - }), - }; - } - if ( serviceName === 'acl' ) { - return { - check: vi.fn().mockResolvedValue(true), - }; - } - if ( serviceName === 'auth' ) return authService; - return {}; - }), - }; - mockContextInstance.get.mockImplementation((key) => { - if ( key === 'services' ) return mockServices; - return null; - }); - getUserMockImpl = async () => ({ id: 101, suspended: false }); - getAppMockImpl = async () => ({ - uid: 'app-11111111-1111-1111-1111-111111111111', - name: 'paid-app', - is_private: 1, - index_url: 'https://paid.puter.dev/', - }); - - const mockReq = { - hostname: 'paid.puter.dev', - subdomains: ['paid'], - is_custom_domain: false, - baseUrl: '', - path: '/index.html', - originalUrl: '/index.html', - query: {}, - cookies: {}, - headers: {}, - ctx: mockContextInstance, - }; - const mockRes = { - redirect: vi.fn(), - set: vi.fn().mockReturnThis(), - setHeader: vi.fn(), - status: vi.fn().mockReturnThis(), - send: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - expect(mockRes.redirect).not.toHaveBeenCalledWith( - expect.stringContaining('app.puter.localhost:4100'), - ); - expect(mockRes.status).toHaveBeenCalledWith(200); - expect(mockRes.send).toHaveBeenCalledWith(expect.stringContaining('Sign in required')); - expect(mockNext).not.toHaveBeenCalled(); - }); - - it('serves login bootstrap html when private app identity is missing', async () => { - const eventEmit = vi.fn().mockImplementation(async (_eventName, event) => { - event.result.allowed = false; - event.result.redirectUrl = 'https://puter.com/app/app-center/?item=app-11111111-1111-1111-1111-111111111111'; - }); - const dbRead = vi.fn().mockResolvedValue([ - { - uid: 'app-11111111-1111-1111-1111-111111111111', - name: 'paid-app', - is_private: 1, - index_url: 'https://paid.puter.dev/', - owner_user_id: 101, - }, - ]); - const authService = { - getPrivateAssetCookieName: vi.fn().mockReturnValue('puter.private.asset.token'), - verifyPrivateAssetToken: vi.fn().mockImplementation(() => { - throw new Error('invalid'); - }), - authenticate_from_token: vi.fn().mockImplementation(() => { - throw new Error('invalid'); - }), - createPrivateAssetToken: vi.fn().mockReturnValue('private-token'), - getPrivateAssetCookieOptions: vi.fn().mockReturnValue({}), - }; - const mockServices = { - get: vi.fn().mockImplementation((serviceName) => { - if ( serviceName === 'puter-site' ) { - return { - get_subdomain: vi.fn().mockResolvedValue({ - user_id: 101, - associated_app_id: null, - root_dir_id: 303, - }), - }; - } - if ( serviceName === 'filesystem' ) { - return { - node: vi.fn().mockResolvedValue({ - exists: vi.fn().mockResolvedValue(true), - get: vi.fn().mockImplementation(async (fieldName) => { - if ( fieldName === 'type' ) return 'directory'; - if ( fieldName === 'path' ) return '/alice/Public'; - return null; - }), - }), - }; - } - if ( serviceName === 'acl' ) { - return { - check: vi.fn().mockResolvedValue(true), - }; - } - if ( serviceName === 'database' ) { - return { - get: vi.fn().mockReturnValue({ - read: dbRead, - }), - }; - } - if ( serviceName === 'event' ) return { emit: eventEmit }; - if ( serviceName === 'auth' ) return authService; - return {}; - }), - }; - mockContextInstance.get.mockImplementation((key) => { - if ( key === 'services' ) return mockServices; - return null; - }); - getUserMockImpl = async () => ({ id: 101, suspended: false }); - getAppMockImpl = async () => ({ - uid: 'app-11111111-1111-1111-1111-111111111111', - name: 'paid-app', - is_private: 1, - index_url: 'https://paid.puter.dev/', - }); - - const mockReq = { - hostname: 'paid.puter.dev', - subdomains: [], - is_custom_domain: false, - baseUrl: '', - path: '/index.html', - originalUrl: '/index.html', - cookies: {}, - headers: {}, - query: {}, - ctx: mockContextInstance, - }; - const mockRes = { - redirect: vi.fn(), - cookie: vi.fn(), - set: vi.fn().mockReturnThis(), - setHeader: vi.fn(), - status: vi.fn().mockReturnThis(), - send: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - expect(eventEmit).not.toHaveBeenCalled(); - expect(dbRead).toHaveBeenCalledWith( - expect.stringContaining('index_url IN'), - expect.arrayContaining([ - 101, - 'https://paid.puter.dev', - 'https://paid.puter.dev/', - 'https://paid.puter.dev/index.html', - 'https://paid.site.puter.localhost', - 'https://paid.site.puter.localhost/', - 'https://paid.site.puter.localhost/index.html', - ]), - ); - expect(mockRes.status).toHaveBeenCalledWith(200); - expect(mockRes.send).toHaveBeenCalledWith(expect.stringContaining('https://js.puter.com/v2/')); - expect(mockRes.send).toHaveBeenCalledWith(expect.stringContaining('puter.auth.signIn()')); - expect(mockRes.send).toHaveBeenCalledWith(expect.stringContaining('localStorage.getItem(\'auth_token\')')); - expect(mockRes.send).toHaveBeenCalledWith(expect.stringContaining('tryStoredTokenBootstrap')); - expect(mockRes.set).toHaveBeenCalledWith('Referrer-Policy', 'no-referrer'); - expect(mockRes.redirect).not.toHaveBeenCalled(); - expect(mockRes.cookie).not.toHaveBeenCalled(); - expect(mockNext).not.toHaveBeenCalled(); - }); - - it('does not redirect private root requests to puter.com app route before access bootstrap', async () => { - const eventEmit = vi.fn().mockImplementation(async (_eventName, event) => { - event.result.allowed = false; - event.result.redirectUrl = 'https://puter.com/app/app-center/?item=app-11111111-1111-1111-1111-111111111111'; - }); - const authService = { - getPrivateAssetCookieName: vi.fn().mockReturnValue('puter.private.asset.token'), - verifyPrivateAssetToken: vi.fn().mockImplementation(() => { - throw new Error('invalid'); - }), - authenticate_from_token: vi.fn().mockImplementation(() => { - throw new Error('invalid'); - }), - createPrivateAssetToken: vi.fn().mockReturnValue('private-token'), - getPrivateAssetCookieOptions: vi.fn().mockReturnValue({}), - }; - const mockServices = { - get: vi.fn().mockImplementation((serviceName) => { - if ( serviceName === 'puter-site' ) { - return { - get_subdomain: vi.fn().mockResolvedValue({ - user_id: 101, - associated_app_id: 202, - root_dir_id: 303, - }), - }; - } - if ( serviceName === 'filesystem' ) { - return { - node: vi.fn().mockResolvedValue({ - exists: vi.fn().mockResolvedValue(true), - get: vi.fn().mockImplementation(async (fieldName) => { - if ( fieldName === 'type' ) return 'directory'; - if ( fieldName === 'path' ) return '/alice/Public'; - return null; - }), - }), - }; - } - if ( serviceName === 'acl' ) { - return { - check: vi.fn().mockResolvedValue(true), - }; - } - if ( serviceName === 'event' ) return { emit: eventEmit }; - if ( serviceName === 'auth' ) return authService; - return {}; - }), - }; - mockContextInstance.get.mockImplementation((key) => { - if ( key === 'services' ) return mockServices; - return null; - }); - getUserMockImpl = async () => ({ id: 101, suspended: false }); - getAppMockImpl = async () => ({ - uid: 'app-11111111-1111-1111-1111-111111111111', - name: 'paid-app', - is_private: 1, - index_url: 'https://paid.site.puter.localhost/', - }); - - const mockReq = { - hostname: 'paid.puter.dev', - subdomains: [], - is_custom_domain: false, - baseUrl: '', - path: '/', - originalUrl: '/?puter.auth.token=abc', - cookies: {}, - headers: {}, - query: { - 'puter.auth.token': 'abc', - }, - ctx: mockContextInstance, - }; - const mockRes = { - redirect: vi.fn(), - cookie: vi.fn(), - set: vi.fn().mockReturnThis(), - setHeader: vi.fn(), - status: vi.fn().mockReturnThis(), - send: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - expect(mockRes.redirect).not.toHaveBeenCalledWith('https://puter.com/app/paid-app/'); - expect(mockRes.status).toHaveBeenCalledWith(200); - expect(mockRes.send).toHaveBeenCalledWith(expect.stringContaining('https://js.puter.com/v2/')); - expect(mockRes.send).toHaveBeenCalledWith(expect.stringContaining('puter.auth.signIn()')); - expect(mockRes.send).toHaveBeenCalledWith(expect.stringContaining('meta property="og:title"')); - expect(mockRes.send).toHaveBeenCalledWith(expect.stringContaining('/app/paid-app/')); - expect(mockNext).not.toHaveBeenCalled(); - }); - - it('denies private app access and redirects using entitlement response', async () => { - const eventEmit = vi.fn().mockImplementation(async (_eventName, event) => { - event.result.allowed = false; - event.result.redirectUrl = 'https://puter.com/app/app-center/?item=app-11111111-1111-1111-1111-111111111111'; - }); - const authService = { - getPrivateAssetCookieName: vi.fn().mockReturnValue('puter.private.asset.token'), - app_uid_from_origin: vi.fn().mockResolvedValue('app-origin-111'), - verifyPrivateAssetToken: vi.fn().mockImplementation(() => { - throw new Error('invalid'); - }), - authenticate_from_token: vi.fn().mockResolvedValue({ - type: {}, - get_related_actor: vi.fn().mockReturnValue({ - type: { - user: { uuid: 'user-111' }, - session: 'session-111', - }, - }), - }), - createPrivateAssetToken: vi.fn().mockReturnValue('private-token'), - getPrivateAssetCookieOptions: vi.fn().mockReturnValue({}), - }; - const mockServices = { - get: vi.fn().mockImplementation((serviceName) => { - if ( serviceName === 'puter-site' ) { - return { - get_subdomain: vi.fn().mockResolvedValue({ - user_id: 101, - associated_app_id: 202, - root_dir_id: 303, - }), - }; - } - if ( serviceName === 'filesystem' ) { - return { - node: vi.fn().mockResolvedValue({ - exists: vi.fn().mockResolvedValue(true), - get: vi.fn().mockImplementation(async (fieldName) => { - if ( fieldName === 'type' ) return 'directory'; - if ( fieldName === 'path' ) return '/alice/Public'; - return null; - }), - }), - }; - } - if ( serviceName === 'acl' ) { - return { - check: vi.fn().mockResolvedValue(true), - }; - } - if ( serviceName === 'event' ) return { emit: eventEmit }; - if ( serviceName === 'auth' ) return authService; - return {}; - }), - }; - mockContextInstance.get.mockImplementation((key) => { - if ( key === 'services' ) return mockServices; - return null; - }); - getUserMockImpl = async () => ({ id: 101, suspended: false }); - getAppMockImpl = async () => ({ - uid: 'app-11111111-1111-1111-1111-111111111111', - name: 'paid-app', - is_private: 1, - index_url: 'https://paid.puter.dev/', - }); - - const mockReq = { - hostname: 'paid.puter.dev', - subdomains: [], - is_custom_domain: false, - baseUrl: '', - path: '/index.html', - originalUrl: '/index.html', - cookies: { - 'puter.session.token': 'session-token', - }, - headers: {}, - query: {}, - ctx: mockContextInstance, - }; - const mockRes = { - redirect: vi.fn(), - cookie: vi.fn(), - setHeader: vi.fn(), - status: vi.fn().mockReturnThis(), - send: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - expect(eventEmit).toHaveBeenCalledWith( - 'app.privateAccess.check', - expect.objectContaining({ - appUid: 'app-11111111-1111-1111-1111-111111111111', - userUid: 'user-111', - }), - ); - expect(mockRes.redirect).toHaveBeenCalledWith('https://puter.com/app/app-center/?item=app-11111111-1111-1111-1111-111111111111'); - expect(mockRes.cookie).not.toHaveBeenCalled(); - expect(mockNext).not.toHaveBeenCalled(); - }); - - it('uses bootstrap fallback identity when strict bootstrap auth fails', async () => { - const eventEmit = vi.fn().mockImplementation(async (_eventName, event) => { - event.result.allowed = false; - event.result.redirectUrl = 'https://apps.puter.com/app/paid-app'; - }); - const authService = { - getPrivateAssetCookieName: vi.fn().mockReturnValue('puter.private.asset.token'), - verifyPrivateAssetToken: vi.fn().mockImplementation(() => { - throw new Error('invalid'); - }), - authenticate_from_token: vi.fn().mockImplementation(() => { - throw new Error('token_auth_failed'); - }), - resolvePrivateBootstrapIdentityFromToken: vi.fn().mockResolvedValue({ - userUid: 'user-111', - sessionUuid: 'session-111', - }), - createPrivateAssetToken: vi.fn().mockReturnValue('private-token'), - getPrivateAssetCookieOptions: vi.fn().mockReturnValue({}), - }; - const mockServices = { - get: vi.fn().mockImplementation((serviceName) => { - if ( serviceName === 'puter-site' ) { - return { - get_subdomain: vi.fn().mockResolvedValue({ - user_id: 101, - associated_app_id: 202, - root_dir_id: 303, - }), - }; - } - if ( serviceName === 'filesystem' ) { - return { - node: vi.fn().mockResolvedValue({ - exists: vi.fn().mockResolvedValue(true), - get: vi.fn().mockImplementation(async (fieldName) => { - if ( fieldName === 'type' ) return 'directory'; - if ( fieldName === 'path' ) return '/alice/Public'; - return null; - }), - }), - }; - } - if ( serviceName === 'acl' ) { - return { - check: vi.fn().mockResolvedValue(true), - }; - } - if ( serviceName === 'event' ) return { emit: eventEmit }; - if ( serviceName === 'auth' ) return authService; - return {}; - }), - }; - mockContextInstance.get.mockImplementation((key) => { - if ( key === 'services' ) return mockServices; - return null; - }); - getUserMockImpl = async () => ({ id: 101, suspended: false }); - getAppMockImpl = async () => ({ - uid: 'app-11111111-1111-1111-1111-111111111111', - name: 'paid-app', - is_private: 1, - index_url: 'https://paid.puter.dev/', - }); - - const mockReq = { - hostname: 'paid.puter.dev', - subdomains: [], - is_custom_domain: false, - baseUrl: '', - path: '/index.html', - originalUrl: '/index.html?puter.auth.token=bootstrap-token', - cookies: {}, - headers: {}, - query: { - 'puter.auth.token': 'bootstrap-token', - }, - ctx: mockContextInstance, - }; - const mockRes = { - redirect: vi.fn(), - cookie: vi.fn(), - setHeader: vi.fn(), - status: vi.fn().mockReturnThis(), - send: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - expect(authService.authenticate_from_token).toHaveBeenCalledWith('bootstrap-token'); - expect(authService.resolvePrivateBootstrapIdentityFromToken) - .toHaveBeenCalledWith('bootstrap-token', { - expectedAppUids: ['app-11111111-1111-1111-1111-111111111111'], - }); - expect(eventEmit).toHaveBeenCalledWith( - 'app.privateAccess.check', - expect.objectContaining({ - appUid: 'app-11111111-1111-1111-1111-111111111111', - userUid: 'user-111', - }), - ); - expect(mockRes.redirect).toHaveBeenCalledWith('https://apps.puter.com/app/paid-app'); - expect(mockRes.send).not.toHaveBeenCalled(); - expect(mockRes.cookie).not.toHaveBeenCalled(); - expect(mockNext).not.toHaveBeenCalled(); - }); - - it('passes request hostname to private asset cookie options on allow', async () => { - const eventEmit = vi.fn().mockImplementation(async (_eventName, event) => { - event.result.allowed = true; - }); - const rootDirectoryNode = { - fetchEntry: vi.fn().mockResolvedValue(undefined), - exists: vi.fn().mockResolvedValue(true), - get: vi.fn().mockImplementation(async (fieldName) => { - if ( fieldName === 'type' ) return 'directory'; - if ( fieldName === 'path' ) return '/alice/Public'; - return null; - }), - }; - const missingFileNode = { - fetchEntry: vi.fn().mockResolvedValue(undefined), - exists: vi.fn().mockResolvedValue(false), - get: vi.fn().mockResolvedValue(null), - }; - let filesystemNodeCallCount = 0; - const authService = { - getPrivateAssetCookieName: vi.fn().mockReturnValue('puter.private.asset.token'), - app_uid_from_origin: vi.fn().mockResolvedValue('app-origin-111'), - verifyPrivateAssetToken: vi.fn().mockImplementation(() => { - throw new Error('invalid'); - }), - authenticate_from_token: vi.fn().mockResolvedValue({ - type: {}, - get_related_actor: vi.fn().mockReturnValue({ - type: { - user: { uuid: 'user-allow-111' }, - session: 'session-allow-111', - }, - }), - }), - createPrivateAssetToken: vi.fn().mockReturnValue('private-token'), - getPrivateAssetCookieOptions: vi.fn().mockReturnValue({ sameSite: 'none' }), - }; - const mockServices = { - get: vi.fn().mockImplementation((serviceName) => { - if ( serviceName === 'puter-site' ) { - return { - get_subdomain: vi.fn().mockResolvedValue({ - user_id: 101, - associated_app_id: 202, - root_dir_id: 303, - }), - }; - } - if ( serviceName === 'filesystem' ) { - return { - node: vi.fn().mockImplementation(async () => { - filesystemNodeCallCount += 1; - return filesystemNodeCallCount === 1 - ? rootDirectoryNode - : missingFileNode; - }), - }; - } - if ( serviceName === 'acl' ) { - return { - check: vi.fn().mockResolvedValue(true), - }; - } - if ( serviceName === 'event' ) return { emit: eventEmit }; - if ( serviceName === 'auth' ) return authService; - return {}; - }), - }; - mockContextInstance.get.mockImplementation((key) => { - if ( key === 'services' ) return mockServices; - return null; - }); - getUserMockImpl = async () => ({ id: 101, suspended: false }); - getAppMockImpl = async () => ({ - uid: 'app-11111111-1111-1111-1111-111111111111', - name: 'paid-app', - is_private: 1, - index_url: 'https://paid.puter.dev/', - }); - - const mockReq = { - hostname: 'paid.puter.dev', - subdomains: [], - is_custom_domain: false, - baseUrl: '', - path: '/asset.js', - originalUrl: '/asset.js', - cookies: { - 'puter.session.token': 'session-token', - }, - headers: {}, - query: {}, - ctx: mockContextInstance, - }; - const mockRes = { - redirect: vi.fn(), - cookie: vi.fn(), - setHeader: vi.fn(), - set: vi.fn().mockReturnThis(), - status: vi.fn().mockReturnThis(), - send: vi.fn(), - write: vi.fn(), - end: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - expect(authService.getPrivateAssetCookieOptions).toHaveBeenCalledWith({ - requestHostname: 'paid.puter.dev', - }); - expect(authService.createPrivateAssetToken).toHaveBeenCalledWith({ - appUid: 'app-origin-111', - userUid: 'user-allow-111', - sessionUuid: 'session-allow-111', - subdomain: 'paid', - privateHost: 'paid.puter.dev', - }); - expect(mockRes.cookie).toHaveBeenCalledWith( - 'puter.private.asset.token', - 'private-token', - { sameSite: 'none' }, - ); - expect(mockNext).not.toHaveBeenCalled(); - }); - - it('includes subdomain and private host when strict bootstrap token auth succeeds', async () => { - const eventEmit = vi.fn().mockImplementation(async (_eventName, event) => { - event.result.allowed = true; - }); - const rootDirectoryNode = { - fetchEntry: vi.fn().mockResolvedValue(undefined), - exists: vi.fn().mockResolvedValue(true), - get: vi.fn().mockImplementation(async (fieldName) => { - if ( fieldName === 'type' ) return 'directory'; - if ( fieldName === 'path' ) return '/alice/Public'; - return null; - }), - }; - const missingFileNode = { - fetchEntry: vi.fn().mockResolvedValue(undefined), - exists: vi.fn().mockResolvedValue(false), - get: vi.fn().mockResolvedValue(null), - }; - let filesystemNodeCallCount = 0; - const authService = { - getPrivateAssetCookieName: vi.fn().mockReturnValue('puter.private.asset.token'), - verifyPrivateAssetToken: vi.fn().mockImplementation(() => { - throw new Error('invalid'); - }), - authenticate_from_token: vi.fn().mockResolvedValue({ - type: {}, - get_related_actor: vi.fn().mockReturnValue({ - type: { - user: { uuid: 'user-bootstrap-111' }, - session: 'session-bootstrap-111', - }, - }), - }), - resolvePrivateBootstrapIdentityFromToken: vi.fn().mockResolvedValue({ - userUid: 'user-bootstrap-111', - sessionUuid: 'session-bootstrap-111', - }), - createPrivateAssetToken: vi.fn().mockReturnValue('private-token'), - getPrivateAssetCookieOptions: vi.fn().mockReturnValue({ sameSite: 'none' }), - }; - const mockServices = { - get: vi.fn().mockImplementation((serviceName) => { - if ( serviceName === 'puter-site' ) { - return { - get_subdomain: vi.fn().mockResolvedValue({ - user_id: 101, - associated_app_id: 202, - root_dir_id: 303, - }), - }; - } - if ( serviceName === 'filesystem' ) { - return { - node: vi.fn().mockImplementation(async () => { - filesystemNodeCallCount += 1; - return filesystemNodeCallCount === 1 - ? rootDirectoryNode - : missingFileNode; - }), - }; - } - if ( serviceName === 'acl' ) { - return { - check: vi.fn().mockResolvedValue(true), - }; - } - if ( serviceName === 'event' ) return { emit: eventEmit }; - if ( serviceName === 'auth' ) return authService; - return {}; - }), - }; - mockContextInstance.get.mockImplementation((key) => { - if ( key === 'services' ) return mockServices; - return null; - }); - getUserMockImpl = async () => ({ id: 101, suspended: false }); - getAppMockImpl = async () => ({ - uid: 'app-11111111-1111-1111-1111-111111111111', - name: 'paid-app', - is_private: 1, - index_url: 'https://paid.puter.dev/', - }); - - const mockReq = { - hostname: 'paid.puter.dev', - subdomains: [], - is_custom_domain: false, - baseUrl: '', - path: '/asset.js', - originalUrl: '/asset.js?puter.auth.token=bootstrap-token&foo=bar', - cookies: {}, - headers: {}, - query: { - 'puter.auth.token': 'bootstrap-token', - foo: 'bar', - }, - ctx: mockContextInstance, - }; - const mockRes = { - redirect: vi.fn(), - cookie: vi.fn(), - setHeader: vi.fn(), - set: vi.fn().mockReturnThis(), - status: vi.fn().mockReturnThis(), - send: vi.fn(), - write: vi.fn(), - end: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - expect(authService.authenticate_from_token).toHaveBeenCalledWith('bootstrap-token'); - expect(authService.resolvePrivateBootstrapIdentityFromToken).toHaveBeenCalledWith('bootstrap-token', { - expectedAppUids: ['app-11111111-1111-1111-1111-111111111111'], - }); - expect(authService.createPrivateAssetToken).toHaveBeenCalledWith({ - appUid: 'app-11111111-1111-1111-1111-111111111111', - userUid: 'user-bootstrap-111', - sessionUuid: 'session-bootstrap-111', - subdomain: 'paid', - privateHost: 'paid.puter.dev', - }); - expect(authService.getPrivateAssetCookieOptions).toHaveBeenCalledWith({ - requestHostname: 'paid.puter.dev', - }); - expect(mockRes.cookie).toHaveBeenCalledWith( - 'puter.private.asset.token', - 'private-token', - { sameSite: 'none' }, - ); - expect(mockRes.redirect).not.toHaveBeenCalled(); - expect(mockNext).not.toHaveBeenCalled(); - }); - - it('does not server-redirect bootstrap token for iframe app instance requests', async () => { - const eventEmit = vi.fn().mockImplementation(async (_eventName, event) => { - event.result.allowed = true; - }); - const rootDirectoryNode = { - fetchEntry: vi.fn().mockResolvedValue(undefined), - exists: vi.fn().mockResolvedValue(true), - get: vi.fn().mockImplementation(async (fieldName) => { - if ( fieldName === 'type' ) return 'directory'; - if ( fieldName === 'path' ) return '/alice/Public'; - return null; - }), - }; - const missingFileNode = { - fetchEntry: vi.fn().mockResolvedValue(undefined), - exists: vi.fn().mockResolvedValue(false), - get: vi.fn().mockResolvedValue(null), - }; - let filesystemNodeCallCount = 0; - const authService = { - getPrivateAssetCookieName: vi.fn().mockReturnValue('puter.private.asset.token'), - verifyPrivateAssetToken: vi.fn().mockImplementation(() => { - throw new Error('invalid'); - }), - authenticate_from_token: vi.fn().mockResolvedValue({ - type: {}, - get_related_actor: vi.fn().mockReturnValue({ - type: { - user: { uuid: 'user-bootstrap-111' }, - session: 'session-bootstrap-111', - }, - }), - }), - resolvePrivateBootstrapIdentityFromToken: vi.fn().mockResolvedValue({ - userUid: 'user-bootstrap-111', - sessionUuid: 'session-bootstrap-111', - }), - createPrivateAssetToken: vi.fn().mockReturnValue('private-token'), - getPrivateAssetCookieOptions: vi.fn().mockReturnValue({ sameSite: 'none' }), - }; - const mockServices = { - get: vi.fn().mockImplementation((serviceName) => { - if ( serviceName === 'puter-site' ) { - return { - get_subdomain: vi.fn().mockResolvedValue({ - user_id: 101, - associated_app_id: 202, - root_dir_id: 303, - }), - }; - } - if ( serviceName === 'filesystem' ) { - return { - node: vi.fn().mockImplementation(async () => { - filesystemNodeCallCount += 1; - return filesystemNodeCallCount === 1 - ? rootDirectoryNode - : missingFileNode; - }), - }; - } - if ( serviceName === 'acl' ) { - return { - check: vi.fn().mockResolvedValue(true), - }; - } - if ( serviceName === 'event' ) return { emit: eventEmit }; - if ( serviceName === 'auth' ) return authService; - return {}; - }), - }; - mockContextInstance.get.mockImplementation((key) => { - if ( key === 'services' ) return mockServices; - return null; - }); - getUserMockImpl = async () => ({ id: 101, suspended: false }); - getAppMockImpl = async () => ({ - uid: 'app-11111111-1111-1111-1111-111111111111', - name: 'paid-app', - is_private: 1, - index_url: 'https://paid.puter.dev/', - }); - - const mockReq = { - hostname: 'paid.puter.dev', - subdomains: [], - is_custom_domain: false, - baseUrl: '', - path: '/asset.js', - originalUrl: '/asset.js?puter.auth.token=bootstrap-token&puter.app_instance_id=instance-111&foo=bar', - cookies: {}, - headers: {}, - query: { - 'puter.auth.token': 'bootstrap-token', - 'puter.app_instance_id': 'instance-111', - foo: 'bar', - }, - ctx: mockContextInstance, - }; - const mockRes = { - redirect: vi.fn(), - cookie: vi.fn(), - setHeader: vi.fn(), - set: vi.fn().mockReturnThis(), - status: vi.fn().mockReturnThis(), - send: vi.fn(), - write: vi.fn(), - end: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - expect(authService.authenticate_from_token).toHaveBeenCalledWith('bootstrap-token'); - expect(authService.createPrivateAssetToken).toHaveBeenCalledWith({ - appUid: 'app-11111111-1111-1111-1111-111111111111', - userUid: 'user-bootstrap-111', - sessionUuid: 'session-bootstrap-111', - subdomain: 'paid', - privateHost: 'paid.puter.dev', - }); - expect(mockRes.cookie).toHaveBeenCalledWith( - 'puter.private.asset.token', - 'private-token', - { sameSite: 'none' }, - ); - expect(mockRes.redirect).not.toHaveBeenCalled(); - expect(filesystemNodeCallCount).toBeGreaterThanOrEqual(2); - expect(mockNext).not.toHaveBeenCalled(); - }); - - it('accepts nested query token key for bootstrap auth', async () => { - const eventEmit = vi.fn().mockImplementation(async (_eventName, event) => { - event.result.allowed = false; - event.result.redirectUrl = 'https://apps.puter.com/app/paid-app'; - }); - const authService = { - getPrivateAssetCookieName: vi.fn().mockReturnValue('puter.private.asset.token'), - verifyPrivateAssetToken: vi.fn().mockImplementation(() => { - throw new Error('invalid'); - }), - authenticate_from_token: vi.fn().mockImplementation(() => { - throw new Error('token_auth_failed'); - }), - resolvePrivateBootstrapIdentityFromToken: vi.fn().mockResolvedValue({ - userUid: 'user-111', - sessionUuid: 'session-111', - }), - createPrivateAssetToken: vi.fn().mockReturnValue('private-token'), - getPrivateAssetCookieOptions: vi.fn().mockReturnValue({}), - }; - const mockServices = { - get: vi.fn().mockImplementation((serviceName) => { - if ( serviceName === 'puter-site' ) { - return { - get_subdomain: vi.fn().mockResolvedValue({ - user_id: 101, - associated_app_id: 202, - root_dir_id: 303, - }), - }; - } - if ( serviceName === 'filesystem' ) { - return { - node: vi.fn().mockResolvedValue({ - exists: vi.fn().mockResolvedValue(true), - get: vi.fn().mockImplementation(async (fieldName) => { - if ( fieldName === 'type' ) return 'directory'; - if ( fieldName === 'path' ) return '/alice/Public'; - return null; - }), - }), - }; - } - if ( serviceName === 'acl' ) { - return { - check: vi.fn().mockResolvedValue(true), - }; - } - if ( serviceName === 'event' ) return { emit: eventEmit }; - if ( serviceName === 'auth' ) return authService; - return {}; - }), - }; - mockContextInstance.get.mockImplementation((key) => { - if ( key === 'services' ) return mockServices; - return null; - }); - getUserMockImpl = async () => ({ id: 101, suspended: false }); - getAppMockImpl = async () => ({ - uid: 'app-11111111-1111-1111-1111-111111111111', - name: 'paid-app', - is_private: 1, - index_url: 'https://paid.puter.dev/', - }); - - const mockReq = { - hostname: 'paid.puter.dev', - subdomains: [], - is_custom_domain: false, - baseUrl: '', - path: '/index.html', - originalUrl: '/index.html?puter.auth.token=bootstrap-token', - cookies: {}, - headers: {}, - query: { - puter: { - auth: { - token: 'bootstrap-token', - }, - }, - }, - ctx: mockContextInstance, - }; - const mockRes = { - redirect: vi.fn(), - cookie: vi.fn(), - setHeader: vi.fn(), - status: vi.fn().mockReturnThis(), - send: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - expect(authService.authenticate_from_token).toHaveBeenCalledWith('bootstrap-token'); - expect(authService.resolvePrivateBootstrapIdentityFromToken) - .toHaveBeenCalledWith('bootstrap-token', { - expectedAppUids: ['app-11111111-1111-1111-1111-111111111111'], - }); - expect(mockRes.redirect).toHaveBeenCalledWith('https://apps.puter.com/app/paid-app'); - expect(mockRes.send).not.toHaveBeenCalled(); - expect(mockRes.cookie).not.toHaveBeenCalled(); - expect(mockNext).not.toHaveBeenCalled(); - }); - - it('skips private app gate when feature flag is disabled', async () => { - config.enable_private_app_access_gate = false; - - const eventEmit = vi.fn(); - const rootDirectoryNode = { - fetchEntry: vi.fn().mockResolvedValue(undefined), - exists: vi.fn().mockResolvedValue(true), - get: vi.fn().mockImplementation(async (fieldName) => { - if ( fieldName === 'type' ) return 'directory'; - if ( fieldName === 'path' ) return '/alice/Public'; - return null; - }), - }; - const missingFileNode = { - fetchEntry: vi.fn().mockResolvedValue(undefined), - exists: vi.fn().mockResolvedValue(false), - get: vi.fn().mockResolvedValue(null), - }; - - let filesystemNodeCallCount = 0; - const mockServices = { - get: vi.fn().mockImplementation((serviceName) => { - if ( serviceName === 'puter-site' ) { - return { - get_subdomain: vi.fn().mockResolvedValue({ - user_id: 101, - associated_app_id: 202, - root_dir_id: 303, - }), - }; - } - if ( serviceName === 'filesystem' ) { - return { - node: vi.fn().mockImplementation(async () => { - filesystemNodeCallCount += 1; - return filesystemNodeCallCount === 1 - ? rootDirectoryNode - : missingFileNode; - }), - }; - } - if ( serviceName === 'acl' ) { - return { - check: vi.fn().mockResolvedValue(true), - }; - } - if ( serviceName === 'event' ) return { emit: eventEmit }; - return {}; - }), - }; - mockContextInstance.get.mockImplementation((key) => { - if ( key === 'services' ) return mockServices; - return null; - }); - getUserMockImpl = async () => ({ id: 101, suspended: false }); - getAppMockImpl = async () => ({ - uid: 'app-11111111-1111-1111-1111-111111111111', - name: 'paid-app', - is_private: 1, - index_url: 'https://paid.puter.dev/', - }); - - const mockReq = { - hostname: 'paid.site.puter.localhost', - subdomains: ['paid'], - is_custom_domain: false, - baseUrl: '', - path: '/asset.js', - originalUrl: '/asset.js', - query: {}, - cookies: {}, - headers: {}, - on: vi.fn(), - ctx: mockContextInstance, - }; - const mockRes = { - redirect: vi.fn(), - cookie: vi.fn(), - setHeader: vi.fn(), - set: vi.fn().mockReturnThis(), - status: vi.fn().mockReturnThis(), - send: vi.fn(), - write: vi.fn(), - end: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - expect(mockRes.redirect).not.toHaveBeenCalled(); - expect(eventEmit).not.toHaveBeenCalled(); - expect(mockRes.status).toHaveBeenCalledWith(404); - expect(mockNext).not.toHaveBeenCalled(); - }); - }); - - describe('public hosted actor bootstrap', () => { - let capturedMiddleware; - - const createRootAndMissingNodes = () => { - const rootDirectoryNode = { - fetchEntry: vi.fn().mockResolvedValue(undefined), - exists: vi.fn().mockResolvedValue(true), - get: vi.fn().mockImplementation(async (fieldName) => { - if ( fieldName === 'type' ) return 'directory'; - if ( fieldName === 'path' ) return '/alice/Public'; - return null; - }), - }; - const missingFileNode = { - fetchEntry: vi.fn().mockResolvedValue(undefined), - exists: vi.fn().mockResolvedValue(false), - get: vi.fn().mockResolvedValue(null), - }; - return { rootDirectoryNode, missingFileNode }; - }; - - beforeEach(() => { - vi.clearAllMocks(); - config.enable_private_app_access_gate = true; - Context.get = vi.fn().mockImplementation((key) => { - if ( key === 'actor' ) return undefined; - return mockContextInstance; - }); - Context.set = vi.fn(); - getUserMockImpl = async () => null; - getAppMockImpl = async () => null; - capturedMiddleware = puterSiteMiddleware; - }); - - it('mints public hosted actor cookie from session identity on non-private app', async () => { - const { rootDirectoryNode, missingFileNode } = createRootAndMissingNodes(); - let filesystemNodeCallCount = 0; - const authService = { - getPublicHostedActorCookieName: vi.fn().mockReturnValue('puter.public.hosted.actor.token'), - verifyPublicHostedActorToken: vi.fn().mockImplementation(() => { - throw new Error('invalid'); - }), - authenticate_from_token: vi.fn().mockResolvedValue({ - type: {}, - get_related_actor: vi.fn().mockReturnValue({ - type: { - user: { uuid: 'user-public-111' }, - session: 'session-public-111', - }, - }), - }), - createPublicHostedActorToken: vi.fn().mockReturnValue('public-hosted-token'), - getPublicHostedActorCookieOptions: vi.fn().mockReturnValue({ sameSite: 'none' }), - app_uid_from_origin: vi.fn().mockResolvedValue('app-origin-fallback-111'), - }; - const mockServices = { - get: vi.fn().mockImplementation((serviceName) => { - if ( serviceName === 'puter-site' ) { - return { - get_subdomain: vi.fn().mockResolvedValue({ - user_id: 101, - associated_app_id: 202, - root_dir_id: 303, - }), - }; - } - if ( serviceName === 'filesystem' ) { - return { - node: vi.fn().mockImplementation(async () => { - filesystemNodeCallCount += 1; - return filesystemNodeCallCount === 1 - ? rootDirectoryNode - : missingFileNode; - }), - }; - } - if ( serviceName === 'acl' ) { - return { - check: vi.fn().mockResolvedValue(true), - }; - } - if ( serviceName === 'auth' ) return authService; - return {}; - }), - }; - mockContextInstance.get.mockImplementation((key) => { - if ( key === 'services' ) return mockServices; - return null; - }); - getUserMockImpl = async () => ({ id: 101, suspended: false }); - getAppMockImpl = async () => ({ - uid: 'app-public-11111111-1111-1111-1111-111111111111', - name: 'public-app', - is_private: 0, - index_url: 'https://paid.site.puter.localhost/', - }); - - const mockReq = { - hostname: 'paid.site.puter.localhost', - subdomains: ['paid'], - is_custom_domain: false, - baseUrl: '', - path: '/asset.js', - originalUrl: '/asset.js', - query: {}, - cookies: { - 'puter.session.token': 'session-token', - }, - headers: {}, - on: vi.fn(), - ctx: mockContextInstance, - }; - const mockRes = { - redirect: vi.fn(), - cookie: vi.fn(), - setHeader: vi.fn(), - set: vi.fn().mockReturnThis(), - status: vi.fn().mockReturnThis(), - send: vi.fn(), - write: vi.fn(), - end: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - expect(authService.verifyPublicHostedActorToken).not.toHaveBeenCalled(); - expect(authService.authenticate_from_token).toHaveBeenCalledWith('session-token'); - expect(authService.createPublicHostedActorToken).toHaveBeenCalledWith({ - appUid: 'app-public-11111111-1111-1111-1111-111111111111', - userUid: 'user-public-111', - sessionUuid: 'session-public-111', - subdomain: 'paid', - host: 'paid.site.puter.localhost', - }); - expect(authService.app_uid_from_origin).not.toHaveBeenCalled(); - expect(authService.getPublicHostedActorCookieOptions).toHaveBeenCalledWith({ - requestHostname: 'paid.site.puter.localhost', - }); - expect(mockRes.cookie).toHaveBeenCalledWith( - 'puter.public.hosted.actor.token', - 'public-hosted-token', - { sameSite: 'none' }, - ); - expect(Context.set).toHaveBeenCalledWith('actor', expect.any(Object)); - expect(mockRes.redirect).not.toHaveBeenCalled(); - expect(mockRes.status).toHaveBeenCalledWith(404); - expect(mockNext).not.toHaveBeenCalled(); - }); - - it('uses valid public hosted actor cookie without re-authenticating', async () => { - const { rootDirectoryNode, missingFileNode } = createRootAndMissingNodes(); - let filesystemNodeCallCount = 0; - const authService = { - getPublicHostedActorCookieName: vi.fn().mockReturnValue('puter.public.hosted.actor.token'), - verifyPublicHostedActorToken: vi.fn().mockReturnValue({ - appUid: 'app-public-22222222-2222-2222-2222-222222222222', - userUid: 'user-public-222', - sessionUuid: 'session-public-222', - subdomain: 'paid', - host: 'paid.site.puter.localhost', - }), - authenticate_from_token: vi.fn(), - createPublicHostedActorToken: vi.fn(), - getPublicHostedActorCookieOptions: vi.fn(), - app_uid_from_origin: vi.fn(), - }; - const mockServices = { - get: vi.fn().mockImplementation((serviceName) => { - if ( serviceName === 'puter-site' ) { - return { - get_subdomain: vi.fn().mockResolvedValue({ - user_id: 101, - associated_app_id: 202, - root_dir_id: 303, - }), - }; - } - if ( serviceName === 'filesystem' ) { - return { - node: vi.fn().mockImplementation(async () => { - filesystemNodeCallCount += 1; - return filesystemNodeCallCount === 1 - ? rootDirectoryNode - : missingFileNode; - }), - }; - } - if ( serviceName === 'acl' ) { - return { - check: vi.fn().mockResolvedValue(true), - }; - } - if ( serviceName === 'auth' ) return authService; - return {}; - }), - }; - mockContextInstance.get.mockImplementation((key) => { - if ( key === 'services' ) return mockServices; - return null; - }); - getUserMockImpl = async () => ({ id: 101, suspended: false }); - getAppMockImpl = async () => ({ - uid: 'app-public-22222222-2222-2222-2222-222222222222', - name: 'public-app', - is_private: 0, - index_url: 'https://paid.site.puter.localhost/', - }); - - const mockReq = { - hostname: 'paid.site.puter.localhost', - subdomains: ['paid'], - is_custom_domain: false, - baseUrl: '', - path: '/asset.js', - originalUrl: '/asset.js', - query: {}, - cookies: { - 'puter.public.hosted.actor.token': 'public-cookie-token', - }, - headers: {}, - on: vi.fn(), - ctx: mockContextInstance, - }; - const mockRes = { - redirect: vi.fn(), - cookie: vi.fn(), - setHeader: vi.fn(), - set: vi.fn().mockReturnThis(), - status: vi.fn().mockReturnThis(), - send: vi.fn(), - write: vi.fn(), - end: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - expect(authService.verifyPublicHostedActorToken).toHaveBeenCalledWith( - 'public-cookie-token', - { - expectedAppUid: 'app-public-22222222-2222-2222-2222-222222222222', - expectedSubdomain: 'paid', - expectedHost: 'paid.site.puter.localhost', - }, - ); - expect(authService.authenticate_from_token).not.toHaveBeenCalled(); - expect(authService.createPublicHostedActorToken).not.toHaveBeenCalled(); - expect(authService.app_uid_from_origin).not.toHaveBeenCalled(); - expect(mockRes.cookie).not.toHaveBeenCalled(); - expect(Context.set).toHaveBeenCalledWith('actor', expect.any(Object)); - const [, actor] = Context.set.mock.calls[0]; - expect(actor?.type?.user?.uuid).toBe('user-public-222'); - expect(mockRes.redirect).not.toHaveBeenCalled(); - expect(mockRes.status).toHaveBeenCalledWith(404); - expect(mockNext).not.toHaveBeenCalled(); - }); - - it('sets public hosted cookie for bootstrap tokens without server-side token stripping redirect', async () => { - const { rootDirectoryNode, missingFileNode } = createRootAndMissingNodes(); - let filesystemNodeCallCount = 0; - const authService = { - getPublicHostedActorCookieName: vi.fn().mockReturnValue('puter.public.hosted.actor.token'), - verifyPublicHostedActorToken: vi.fn().mockImplementation(() => { - throw new Error('invalid'); - }), - authenticate_from_token: vi.fn().mockResolvedValue({ - type: {}, - get_related_actor: vi.fn().mockReturnValue({ - type: { - user: { uuid: 'user-public-333' }, - session: 'session-public-333', - }, - }), - }), - createPublicHostedActorToken: vi.fn().mockReturnValue('public-hosted-token-333'), - getPublicHostedActorCookieOptions: vi.fn().mockReturnValue({ sameSite: 'none' }), - app_uid_from_origin: vi.fn(), - }; - const mockServices = { - get: vi.fn().mockImplementation((serviceName) => { - if ( serviceName === 'puter-site' ) { - return { - get_subdomain: vi.fn().mockResolvedValue({ - user_id: 101, - associated_app_id: 202, - root_dir_id: 303, - }), - }; - } - if ( serviceName === 'filesystem' ) { - return { - node: vi.fn().mockImplementation(async () => { - filesystemNodeCallCount += 1; - return filesystemNodeCallCount === 1 - ? rootDirectoryNode - : missingFileNode; - }), - }; - } - if ( serviceName === 'acl' ) { - return { - check: vi.fn().mockResolvedValue(true), - }; - } - if ( serviceName === 'auth' ) return authService; - return {}; - }), - }; - mockContextInstance.get.mockImplementation((key) => { - if ( key === 'services' ) return mockServices; - return null; - }); - getUserMockImpl = async () => ({ id: 101, suspended: false }); - getAppMockImpl = async () => ({ - uid: 'app-public-33333333-3333-3333-3333-333333333333', - name: 'public-app', - is_private: 0, - index_url: 'https://paid.site.puter.localhost/', - }); - - const mockReq = { - hostname: 'paid.site.puter.localhost', - subdomains: ['paid'], - is_custom_domain: false, - baseUrl: '', - path: '/asset.js', - originalUrl: '/asset.js?puter.auth.token=bootstrap-token&foo=bar', - query: { - 'puter.auth.token': 'bootstrap-token', - foo: 'bar', - }, - cookies: {}, - headers: {}, - on: vi.fn(), - ctx: mockContextInstance, - }; - const mockRes = { - redirect: vi.fn(), - cookie: vi.fn(), - setHeader: vi.fn(), - set: vi.fn().mockReturnThis(), - status: vi.fn().mockReturnThis(), - send: vi.fn(), - write: vi.fn(), - end: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - expect(authService.authenticate_from_token).toHaveBeenCalledWith('bootstrap-token'); - expect(authService.createPublicHostedActorToken).toHaveBeenCalledWith({ - appUid: 'app-public-33333333-3333-3333-3333-333333333333', - userUid: 'user-public-333', - sessionUuid: 'session-public-333', - subdomain: 'paid', - host: 'paid.site.puter.localhost', - }); - expect(mockRes.cookie).toHaveBeenCalledWith( - 'puter.public.hosted.actor.token', - 'public-hosted-token-333', - { sameSite: 'none' }, - ); - expect(mockRes.redirect).not.toHaveBeenCalled(); - expect(mockNext).not.toHaveBeenCalled(); - }); - - it('uses strict bootstrap identity verification when available', async () => { - const { rootDirectoryNode, missingFileNode } = createRootAndMissingNodes(); - let filesystemNodeCallCount = 0; - const authService = { - getPublicHostedActorCookieName: vi.fn().mockReturnValue('puter.public.hosted.actor.token'), - verifyPublicHostedActorToken: vi.fn().mockImplementation(() => { - throw new Error('invalid'); - }), - resolvePrivateBootstrapIdentityFromToken: vi.fn().mockResolvedValue({ - userUid: 'user-public-555', - sessionUuid: 'session-public-555', - }), - authenticate_from_token: vi.fn(), - createPublicHostedActorToken: vi.fn().mockReturnValue('public-hosted-token-555'), - getPublicHostedActorCookieOptions: vi.fn().mockReturnValue({ sameSite: 'none' }), - app_uid_from_origin: vi.fn(), - }; - const mockServices = { - get: vi.fn().mockImplementation((serviceName) => { - if ( serviceName === 'puter-site' ) { - return { - get_subdomain: vi.fn().mockResolvedValue({ - user_id: 101, - associated_app_id: 202, - root_dir_id: 303, - }), - }; - } - if ( serviceName === 'filesystem' ) { - return { - node: vi.fn().mockImplementation(async () => { - filesystemNodeCallCount += 1; - return filesystemNodeCallCount === 1 - ? rootDirectoryNode - : missingFileNode; - }), - }; - } - if ( serviceName === 'acl' ) { - return { - check: vi.fn().mockResolvedValue(true), - }; - } - if ( serviceName === 'auth' ) return authService; - return {}; - }), - }; - mockContextInstance.get.mockImplementation((key) => { - if ( key === 'services' ) return mockServices; - return null; - }); - getUserMockImpl = async () => ({ id: 101, suspended: false }); - getAppMockImpl = async () => ({ - uid: 'app-public-55555555-5555-5555-5555-555555555555', - name: 'public-app', - is_private: 0, - index_url: 'https://paid.site.puter.localhost/', - }); - - const mockReq = { - hostname: 'paid.site.puter.localhost', - subdomains: ['paid'], - is_custom_domain: false, - baseUrl: '', - path: '/asset.js', - originalUrl: '/asset.js?puter.auth.token=bootstrap-token&foo=bar', - query: { - 'puter.auth.token': 'bootstrap-token', - foo: 'bar', - }, - cookies: {}, - headers: {}, - on: vi.fn(), - ctx: mockContextInstance, - }; - const mockRes = { - redirect: vi.fn(), - cookie: vi.fn(), - setHeader: vi.fn(), - set: vi.fn().mockReturnThis(), - status: vi.fn().mockReturnThis(), - send: vi.fn(), - write: vi.fn(), - end: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - expect(authService.resolvePrivateBootstrapIdentityFromToken).toHaveBeenCalledWith( - 'bootstrap-token', - { - expectedAppUid: 'app-public-55555555-5555-5555-5555-555555555555', - }, - ); - expect(authService.authenticate_from_token).not.toHaveBeenCalled(); - expect(authService.createPublicHostedActorToken).toHaveBeenCalledWith({ - appUid: 'app-public-55555555-5555-5555-5555-555555555555', - userUid: 'user-public-555', - sessionUuid: 'session-public-555', - subdomain: 'paid', - host: 'paid.site.puter.localhost', - }); - expect(mockRes.redirect).not.toHaveBeenCalled(); - expect(mockNext).not.toHaveBeenCalled(); - }); - - it('short-circuits without auth calls when no identity tokens exist', async () => { - const { rootDirectoryNode, missingFileNode } = createRootAndMissingNodes(); - let filesystemNodeCallCount = 0; - const authService = { - getPublicHostedActorCookieName: vi.fn().mockReturnValue('puter.public.hosted.actor.token'), - verifyPublicHostedActorToken: vi.fn(), - authenticate_from_token: vi.fn(), - createPublicHostedActorToken: vi.fn(), - getPublicHostedActorCookieOptions: vi.fn(), - app_uid_from_origin: vi.fn(), - }; - const mockServices = { - get: vi.fn().mockImplementation((serviceName) => { - if ( serviceName === 'puter-site' ) { - return { - get_subdomain: vi.fn().mockResolvedValue({ - user_id: 101, - associated_app_id: 202, - root_dir_id: 303, - }), - }; - } - if ( serviceName === 'filesystem' ) { - return { - node: vi.fn().mockImplementation(async () => { - filesystemNodeCallCount += 1; - return filesystemNodeCallCount === 1 - ? rootDirectoryNode - : missingFileNode; - }), - }; - } - if ( serviceName === 'acl' ) { - return { - check: vi.fn().mockResolvedValue(true), - }; - } - if ( serviceName === 'auth' ) return authService; - return {}; - }), - }; - mockContextInstance.get.mockImplementation((key) => { - if ( key === 'services' ) return mockServices; - return null; - }); - getUserMockImpl = async () => ({ id: 101, suspended: false }); - getAppMockImpl = async () => ({ - uid: 'app-public-44444444-4444-4444-4444-444444444444', - name: 'public-app', - is_private: 0, - index_url: 'https://paid.site.puter.localhost/', - }); - - const mockReq = { - hostname: 'paid.site.puter.localhost', - subdomains: ['paid'], - is_custom_domain: false, - baseUrl: '', - path: '/asset.js', - originalUrl: '/asset.js', - query: {}, - cookies: {}, - headers: {}, - on: vi.fn(), - ctx: mockContextInstance, - }; - const mockRes = { - redirect: vi.fn(), - cookie: vi.fn(), - setHeader: vi.fn(), - set: vi.fn().mockReturnThis(), - status: vi.fn().mockReturnThis(), - send: vi.fn(), - write: vi.fn(), - end: vi.fn(), - }; - const mockNext = vi.fn(); - - await capturedMiddleware(mockReq, mockRes, mockNext); - - expect(authService.verifyPublicHostedActorToken).not.toHaveBeenCalled(); - expect(authService.authenticate_from_token).not.toHaveBeenCalled(); - expect(authService.createPublicHostedActorToken).not.toHaveBeenCalled(); - expect(authService.app_uid_from_origin).not.toHaveBeenCalled(); - expect(mockRes.cookie).not.toHaveBeenCalled(); - expect(mockRes.redirect).not.toHaveBeenCalled(); - expect(mockRes.status).toHaveBeenCalledWith(404); - expect(mockNext).not.toHaveBeenCalled(); - }); - }); -}); diff --git a/src/backend/src/routers/itemMetadata.js b/src/backend/src/routers/itemMetadata.js deleted file mode 100644 index f6b2b95f8..000000000 --- a/src/backend/src/routers/itemMetadata.js +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = express.Router(); -const { validate_signature_auth, get_url_from_req, is_valid_uuid4, get_dir_size, id2path } = require('../helpers'); -const { DB_READ } = require('../services/database/consts'); - -// -----------------------------------------------------------------------// -// GET /itemMetadata -// -----------------------------------------------------------------------// -router.get('/itemMetadata', async (req, res, next) => { - // Check subdomain - if ( require('../helpers').subdomain(req) !== 'api' ) - { - next(); - } - - // Validate URL signature - try { - validate_signature_auth(get_url_from_req(req), 'read'); - } - catch (e) { - console.log(e); - return res.status(403).send(e); - } - - // Validation - if ( ! req.query.uid ) - { - return res.status(400).send('`uid` is required'); - } - // uid must be a string - else if ( req.query.uid && typeof req.query.uid !== 'string' ) - { - return res.status(400).send('uid must be a string.'); - } - // uid cannot be empty - else if ( req.query.uid && req.query.uid.trim() === '' ) - { - return res.status(400).send('uid cannot be empty'); - } - // uid must be a valid uuid - else if ( ! is_valid_uuid4(req.query.uid) ) - { - return res.status(400).send('uid must be a valid uuid'); - } - - // modules - const { uuid2fsentry } = require('../helpers'); - - const uid = req.query.uid; - - const item = await uuid2fsentry(uid); - - // check if item owner is suspended - const user = await require('../helpers').get_user({ id: item.user_id }); - - if ( ! user ) { - return res.status(400).send('User not found'); - } - - if ( user.suspended ) - { - return res.status(401).send({ error: 'Account suspended' }); - } - - if ( ! item ) - { - return res.status(400).send('Item not found'); - } - - const mime = require('mime-types'); - const contentType = mime.contentType(res.name); - - const itemMetadata = { - uid: item.uuid, - name: item.name, - is_dir: item.is_dir, - type: contentType, - size: item.is_dir ? await get_dir_size(await id2path(item.id), user) : item.size, - created: item.created, - modified: item.modified, - }; - - // ---------------------------------------------------------------// - // return_path - // ---------------------------------------------------------------// - if ( req.query.return_path === 'true' || req.query.return_path === '1' ) { - const { id2path } = require('../helpers'); - itemMetadata.path = await id2path(item.id); - } - // ---------------------------------------------------------------// - // Versions - // ---------------------------------------------------------------// - if ( req.query.return_versions ) { - const db = req.services.get('database').get(DB_READ, 'itemMetadata.js'); - itemMetadata.versions = []; - - let versions = await db.read('SELECT * FROM fsentry_versions WHERE fsentry_id = ?', - [item.id]); - if ( versions.length > 0 ) { - for ( let index = 0; index < versions.length; index++ ) { - const version = versions[index]; - itemMetadata.versions.push({ - id: version.version_id, - message: version.message, - timestamp: version.ts_epoch, - }); - } - } - } - - return res.send(itemMetadata); -}); - -module.exports = router; \ No newline at end of file diff --git a/src/backend/src/routers/kvstore/clearItems.js b/src/backend/src/routers/kvstore/clearItems.js deleted file mode 100644 index f4ec5ad21..000000000 --- a/src/backend/src/routers/kvstore/clearItems.js +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const eggspress = require('../../api/eggspress'); - -module.exports = eggspress('/clearItems', { - subdomain: 'api', - auth: true, - verified: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - - // TODO: model these parameters; validation is contained in brackets - // so that it can be easily move. - let { app } = req.body; - - // Validation for `app` - if ( ! app ) { - throw APIError.create('field_missing', null, { key: 'app' }); - } - - const svc_mysql = req.services.get('mysql'); - // TODO: Check if used anywhere, maybe remove - // eslint-disable-next-line no-undef - const dbrw = svc_mysql.get(DB_MODE_WRITE, 'kvstore-clearItems'); - await dbrw.execute('DELETE FROM kv WHERE user_id=? AND app=?', - [ - req.user.id, - app, - ]); - - return res.send({}); -}); diff --git a/src/backend/src/routers/kvstore/getItem.js b/src/backend/src/routers/kvstore/getItem.js deleted file mode 100644 index 663b030b1..000000000 --- a/src/backend/src/routers/kvstore/getItem.js +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = express.Router(); -const auth = require('../../middleware/auth.js'); -const config = require('../../config.js'); -const { Context } = require('../../util/context.js'); -const { Actor, AppUnderUserActorType, UserActorType } = require('../../services/auth/Actor.js'); -const { DB_READ } = require('../../services/database/consts.js'); - -// -----------------------------------------------------------------------// -// POST /getItem -// -----------------------------------------------------------------------// -router.post('/getItem', auth, express.json(), async (req, res, next) => { - // check subdomain - if ( require('../../helpers.js').subdomain(req) !== 'api' ) - { - next(); - } - - // check if user is verified - if ( (config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed ) - { - return res.status(400).send({ code: 'account_is_not_verified', message: 'Account is not verified' }); - } - - // validation - if ( ! req.body.key ) - { - return res.status(400).send('`key` is required.'); - } - // check size of key, if it's too big then it's an invalid key and we don't want to waste time on it - else if ( Buffer.byteLength(req.body.key, 'utf8') > config.kv_max_key_size ) - { - return res.status(400).send('`key` is too long.'); - } - - const actor = req.body.app - ? await Actor.create(AppUnderUserActorType, { - user: req.user, - app_uid: req.body.app, - }) - : await Actor.create(UserActorType, { - user: req.user, - }) - ; - - Context.set('actor', actor); - - // Try KV 1 first - const svc_driver = Context.get('services').get('driver'); - let driver_result; - try { - const driver_response = await svc_driver.call({ - iface: 'puter-kvstore', - method: 'get', - args: { key: req.body.key }, - }); - if ( ! driver_response.success ) { - throw new Error(driver_response.error?.message ?? 'Unknown error'); - } - driver_result = driver_response.result; - } catch ( e ) { - return res.status(400).send(`puter-kvstore driver error: ${ e.message}`); - } - - if ( driver_result ) { - return res.send({ key: req.body.key, value: driver_result }); - } - - // modules - const db = req.services.get('database').get(DB_READ, 'getItem-fallback'); - // get murmurhash module - const murmurhash = require('murmurhash'); - // hash key for faster search in DB - const key_hash = murmurhash.v3(req.body.key); - - let kv; - // Get value from DB - // If app is specified, then get value for that app - if ( req.body.app ) { - kv = await db.read('SELECT * FROM kv WHERE user_id=? AND app=? AND kkey_hash=? LIMIT 1', - [ - req.user.id, - req.body.app, - key_hash, - ]); - // If app is not specified, then get value for global (i.e. system) variables which is app='global' - } else { - kv = await db.read('SELECT * FROM kv WHERE user_id=? AND (app IS NULL OR app = \'global\') AND kkey_hash=? LIMIT 1', - [ - req.user.id, - key_hash, - ]); - } - - // send results to client - if ( kv[0] ) - { - return res.send({ - key: kv[0].kkey, - value: kv[0].value, - }); - } - else - { - return res.send(null); - } -}); -module.exports = router; \ No newline at end of file diff --git a/src/backend/src/routers/kvstore/listItems.js b/src/backend/src/routers/kvstore/listItems.js deleted file mode 100644 index eea6dedc4..000000000 --- a/src/backend/src/routers/kvstore/listItems.js +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const eggspress = require('../../api/eggspress'); -const { DB_READ } = require('../../services/database/consts'); - -module.exports = eggspress('/listItems', { - subdomain: 'api', - auth: true, - verified: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - - let { app } = req.body; - - // Validation for `app` - if ( ! app ) { - throw APIError.create('field_missing', null, { key: 'app' }); - } - - const db = req.services.get('database').get(DB_READ, 'kv'); - let rows = await db.read('SELECT kkey, value FROM kv WHERE user_id=? AND app=?', - [ - req.user.id, - app, - ]); - - rows = rows.map(row => ({ - key: row.kkey, - value: row.value, - })); - - return res.send(rows); -}); diff --git a/src/backend/src/routers/kvstore/setItem.js b/src/backend/src/routers/kvstore/setItem.js deleted file mode 100644 index c54d7e896..000000000 --- a/src/backend/src/routers/kvstore/setItem.js +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = express.Router(); -const auth = require('../../middleware/auth.js'); -const config = require('../../config.js'); -const { app_exists, byte_format } = require('../../helpers.js'); -const { Actor, AppUnderUserActorType, UserActorType } = require('../../services/auth/Actor.js'); -const { Context } = require('../../util/context.js'); - -// -----------------------------------------------------------------------// -// POST /setItem -// -----------------------------------------------------------------------// -router.post('/setItem', auth, express.json(), async (req, res, next) => { - // check subdomain - if ( require('../../helpers.js').subdomain(req) !== 'api' ) - { - next(); - } - - // check if user is verified - if ( (config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed ) - { - return res.status(400).send({ code: 'account_is_not_verified', message: 'Account is not verified' }); - } - - // validation - if ( ! req.body.key ) - { - return res.status(400).send('`key` is required'); - } - else if ( typeof req.body.key !== 'string' ) - { - return res.status(400).send('`key` must be a string'); - } - else if ( ! req.body.value ) - { - return res.status(400).send('`value` is required'); - } - - req.body.key = String(req.body.key); - req.body.value = String(req.body.value); - - if ( Buffer.byteLength(req.body.key, 'utf8') > config.kv_max_key_size ) - { - return res.status(400).send(`\`key\` is too large. Max size is ${byte_format(config.kv_max_key_size)}.`); - } - else if ( Buffer.byteLength(req.body.value, 'utf8') > config.kv_max_value_size ) - { - return res.status(400).send(`\`value\` is too large. Max size is ${byte_format(config.kv_max_value_size)}.`); - } - else if ( req.body.app && !await app_exists({ uid: req.body.app }) ) - { - return res.status(400).send('`app` does not exist'); - } - - // insert into KV 1 - const actor = req.body.app - ? await Actor.create(AppUnderUserActorType, { - user: req.user, - app_uid: req.body.app, - }) - : await Actor.create(UserActorType, { - user: req.user, - }) - ; - - Context.set('actor', actor); - - const svc_driver = Context.get('services').get('driver'); - let driver_result; - try { - const driver_response = await svc_driver.call({ - iface: 'puter-kvstore', - method: 'set', - args: { - key: req.body.key, - value: req.body.value, - }, - }); - if ( ! driver_response.success ) { - throw new Error(driver_response.error?.message ?? 'Unknown error'); - } - driver_result = driver_response.result; - } catch (e) { - return res.status(400).send(`puter-kvstore driver error: ${ e.message}`); - } - - // send results to client - return res.send({}); -}); -module.exports = router; \ No newline at end of file diff --git a/src/backend/src/routers/login.js b/src/backend/src/routers/login.js deleted file mode 100644 index 5f1f2bbca..000000000 --- a/src/backend/src/routers/login.js +++ /dev/null @@ -1,322 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = new express.Router(); -const { get_user, body_parser_error_handler, invalidate_cached_user } = require('../helpers'); -const config = require('../config'); -const { DB_WRITE } = require('../services/database/consts'); -const { requireCaptcha } = require('../modules/captcha/middleware/captcha-middleware'); - -const complete_ = async ({ req, res, user }) => { - const svc_auth = req.services.get('auth'); - const { session, token: session_token } = await svc_auth.create_session_token(user, { req }); - const gui_token = svc_auth.create_gui_token(user, session); - - // HTTP-only cookie gets session token (cookie-based requests have hasHttpOnlyCookie) - res.cookie(config.cookie_name, session_token, { - sameSite: 'none', - secure: true, - httpOnly: true, - }); - - // response body: GUI token only (client never gets session token) - return res.send({ - proceed: true, - next_step: 'complete', - token: gui_token, - user: { - username: user.username, - uuid: user.uuid, - email: user.email, - email_confirmed: user.email_confirmed, - is_temp: (user.password === null && user.email === null), - }, - }); -}; - -// -----------------------------------------------------------------------// -// POST /login -// -----------------------------------------------------------------------// -router.post('/login', express.json(), body_parser_error_handler, (req, res, next) => { - // Add diagnostic middleware to log captcha data - if ( process.env.DEBUG ) { - console.log('====== LOGIN CAPTCHA DIAGNOSTIC ======'); - console.log('LOGIN REQUEST RECEIVED with captcha data:', { - hasCaptchaToken: !!req.body.captchaToken, - hasCaptchaAnswer: !!req.body.captchaAnswer, - captchaToken: req.body.captchaToken ? `${req.body.captchaToken.substring(0, 8) }...` : undefined, - captchaAnswer: req.body.captchaAnswer, - }); - } - next(); -}, requireCaptcha({ strictMode: true, eventType: 'login' }), async (req, res, next) => { - // either api. subdomain or no subdomain - if ( require('../helpers').subdomain(req) !== 'api' && require('../helpers').subdomain(req) !== '' ) { - next(); - } - - // modules - const bcrypt = require('bcrypt'); - const validator = require('validator'); - - // either username or email must be provided - if ( !req.body.username && !req.body.email ) { - return res.status(400).send('Username or email is required.'); - } - // password is required - else if ( ! req.body.password ) - { - return res.status(400).send('Password is required.'); - } - // password must be a string - else if ( typeof req.body.password !== 'string' && !(req.body.password instanceof String) ) - { - return res.status(400).send('Password must be a string.'); - } - // if password is too short it's invalid, no need to do a db lookup - else if ( req.body.password.length < config.min_pass_length ) - { - return res.status(400).send('Invalid password.'); - } - // username, if present, must be a string - else if ( req.body.username && typeof req.body.username !== 'string' && !(req.body.username instanceof String) ) - { - return res.status(400).send('username must be a string.'); - } - // if username doesn't pass regex test it's invalid anyway, no need to do DB lookup - else if ( req.body.username && !req.body.username.match(config.username_regex) ) - { - return res.status(400).send('Invalid username.'); - } - // email, if present, must be a string - else if ( req.body.email && typeof req.body.email !== 'string' && !(req.body.email instanceof String) ) - { - return res.status(400).send('email must be a string.'); - } - // if email is invalid, no need to do DB lookup anyway - else if ( req.body.email && !validator.isEmail(req.body.email) ) - { - return res.status(400).send('Invalid email.'); - } - - /** @type {import('../services/abuse-prevention/EdgeRateLimitService').EdgeRateLimitService} */ - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('login', true) ) { - return res.status(429).send('Too many requests.'); - } - - try { - let user; - // log in using username - if ( req.body.username ) { - user = await get_user({ username: req.body.username, cached: false }); - if ( ! user ) { - svc_edgeRateLimit.incr('login'); - return res.status(400).send('Username not found.'); - } - } - // log in using email - else if ( validator.isEmail(req.body.email) ) { - user = await get_user({ email: req.body.email, cached: false }); - if ( ! user ) { - svc_edgeRateLimit.incr('login'); - return res.status(400).send('Email not found.'); - } - } - if ( user.username === 'system' && config.allow_system_login !== true ) { - svc_edgeRateLimit.incr('login'); - return res.status(400).send( - req.body.username - ? 'Username not found.' - : 'Email not found.', - ); - } - // is user suspended? - if ( user.suspended ) { - svc_edgeRateLimit.incr('login'); - return res.status(401).send('This account is suspended.'); - } - // pseudo user? - // todo make this better, maybe ask them to create an account or send them an activation link - if ( user.password === null ) { - svc_edgeRateLimit.incr('login'); - return res.status(400).send('Incorrect password.'); - } - // check password - if ( await bcrypt.compare(req.body.password, user.password) ) { - // We create a JWT that can ONLY be used on the endpoint that - // accepts the OTP code. - if ( user.otp_enabled ) { - const svc_token = req.services.get('token'); - const otp_jwt_token = svc_token.sign('otp', { - user_uid: user.uuid, - }, { expiresIn: '5m' }); - - return res.status(202).send({ - proceed: true, - next_step: 'otp', - otp_jwt_token: otp_jwt_token, - }); - } - - return await complete_({ req, res, user }); - } else { - svc_edgeRateLimit.incr('login'); - return res.status(400).send('Incorrect password.'); - } - } catch (e) { - console.error(e); - svc_edgeRateLimit.incr('login'); - return res.status(400).send(e); - } - -}); - -router.post('/login/otp', express.json(), body_parser_error_handler, requireCaptcha({ strictMode: true, eventType: 'login_otp' }), async (req, res, next) => { - // either api. subdomain or no subdomain - if ( require('../helpers').subdomain(req) !== 'api' && require('../helpers').subdomain(req) !== '' ) - { - next(); - } - - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('login-otp') ) { - return res.status(429).send('Too many requests.'); - } - - if ( ! req.body.token ) { - return res.status(400).send('token is required.'); - } - - if ( ! req.body.code ) { - return res.status(400).send('code is required.'); - } - - const svc_token = req.services.get('token'); - let decoded; try { - decoded = svc_token.verify('otp', req.body.token); - } catch ( e ) { - return res.status(400).send('Invalid token.'); - } - - if ( ! decoded.user_uid ) { - return res.status(400).send('Invalid token.'); - } - - const user = await get_user({ uuid: decoded.user_uid, cached: false }); - if ( ! user ) { - return res.status(400).send('User not found.'); - } - - const svc_otp = req.services.get('otp'); - if ( ! svc_otp.verify(user.username, user.otp_secret, req.body.code) ) { - - // THIS MAY BE COUNTER-INTUITIVE - // - // A successfully handled request, with the correct format, - // but incorrect credentials when NOT using the HTTP - // authentication framework provided by RFC 7235, SHOULD - // return status 200. - // - // Source: I asked Julian Reschke in an email, and then he - // contributed to this discussion: - // https://stackoverflow.com/questions/32752578 - - return res.status(200).send({ - proceed: false, - }); - } - - return await complete_({ req, res, user }); -}); - -router.post('/login/recovery-code', express.json(), body_parser_error_handler, requireCaptcha({ strictMode: true, eventType: 'login_recovery' }), async (req, res, next) => { - // either api. subdomain or no subdomain - if ( require('../helpers').subdomain(req) !== 'api' && require('../helpers').subdomain(req) !== '' ) - { - next(); - } - - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('login-recovery') ) { - return res.status(429).send('Too many requests.'); - } - - if ( ! req.body.token ) { - return res.status(400).send('token is required.'); - } - - if ( ! req.body.code ) { - return res.status(400).send('code is required.'); - } - - const svc_token = req.services.get('token'); - let decoded; try { - decoded = svc_token.verify('otp', req.body.token); - } catch ( e ) { - return res.status(400).send('Invalid token.'); - } - - if ( ! decoded.user_uid ) { - return res.status(400).send('Invalid token.'); - } - - const user = await get_user({ uuid: decoded.user_uid, cached: false }); - if ( ! user ) { - return res.status(400).send('User not found.'); - } - - const code = req.body.code; - - const crypto = require('crypto'); - - const codes = user.otp_recovery_codes.split(','); - const hashed_code = crypto - .createHash('sha256') - .update(code) - .digest('base64') - // We're truncating the hash for easier storage, so we have 128 - // bits of entropy instead of 256. This is plenty for recovery - // codes, which have only 48 bits of entropy to begin with. - .slice(0, 22); - - if ( ! codes.includes(hashed_code) ) { - return res.status(200).send({ - proceed: false, - }); - } - - // Remove the code from the list - const index = codes.indexOf(hashed_code); - codes.splice(index, 1); - - // update user - const db = req.services.get('database').get(DB_WRITE, '2fa'); - await db.write( - 'UPDATE user SET otp_recovery_codes = ? WHERE uuid = ?', - [codes.join(','), user.uuid], - ); - user.otp_recovery_codes = codes.join(','); - invalidate_cached_user(user); - - return await complete_({ req, res, user }); -}); - -module.exports = router; diff --git a/src/backend/src/routers/logout.js b/src/backend/src/routers/logout.js deleted file mode 100644 index 0b840131b..000000000 --- a/src/backend/src/routers/logout.js +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = new express.Router(); -const auth = require('../middleware/auth.js'); -const config = require('../config'); - -// -----------------------------------------------------------------------// -// POST /logout -// -----------------------------------------------------------------------// -router.post('/logout', auth, express.json(), async (req, res, next) => { - // check subdomain - if ( require('../helpers').subdomain(req) !== 'api' && require('../helpers').subdomain(req) !== '' ) - { - next(); - } - // check anti-csrf token - const svc_antiCSRF = req.services.get('anti-csrf'); - if ( ! await svc_antiCSRF.consume_token(req.user.uuid, req.body.anti_csrf) ) { - return res.status(400).json({ message: 'incorrect anti-CSRF token' }); - } - // delete cookie - res.clearCookie(config.cookie_name); - // delete session - (async () => { - if ( ! req.token ) return; - try { - const svc_auth = req.services.get('auth'); - await svc_auth.remove_session_by_token(req.token); - } catch (e) { - console.log(e); - } - })(); - //--------------------------------------------------------- - // DANGER ZONE: delete temp user and all its data - //--------------------------------------------------------- - if ( req.user.password === null && req.user.email === null ) { - const { deleteUser } = require('../helpers'); - deleteUser(req.user.id); - } - // send response - res.send('logged out'); -}); - -module.exports = router; \ No newline at end of file diff --git a/src/backend/src/routers/open_item.js b/src/backend/src/routers/open_item.js deleted file mode 100644 index aa44807ba..000000000 --- a/src/backend/src/routers/open_item.js +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const eggspress = require('../api/eggspress.js'); -const FSNodeParam = require('../api/filesystem/FSNodeParam.js'); -const { Context } = require('../util/context.js'); -const { UserActorType } = require('../services/auth/Actor.js'); -const APIError = require('../api/APIError.js'); -const { sign_file, suggestedAppForFsEntry, get_app } = require('../helpers.js'); - -// -----------------------------------------------------------------------// -// POST /open_item -// -----------------------------------------------------------------------// -module.exports = eggspress('/open_item', { - subdomain: 'api', - auth2: true, - verified: true, - json: true, - allowedMethods: ['POST'], - alias: { uid: 'path' }, - parameters: { - subject: new FSNodeParam('path'), - }, -}, async (req, res) => { - const subject = req.values.subject; - - const actor = Context.get('actor'); - if ( ! (actor.type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - if ( ! await subject.exists() ) { - throw APIError.create('subject_does_not_exist'); - } - - const svc_acl = Context.get('services').get('acl'); - if ( ! await svc_acl.check(actor, subject, 'read') ) { - throw await svc_acl.get_safe_acl_error(actor, subject, 'read'); - } - - let action = 'write'; - if ( ! await svc_acl.check(actor, subject, 'write') ) { - action = 'read'; - } - - const signature = await sign_file(subject.entry, action); - const suggested_apps = await suggestedAppForFsEntry(subject.entry); - const apps_only_one = suggested_apps.slice(0, 1); - const _app = apps_only_one[0]; - if ( ! _app ) { - throw APIError.create('no_suitable_app', null, { entry_name: subject.entry.name }); - } - const app = await get_app(Object.prototype.hasOwnProperty.call(_app, 'id') - ? { id: _app.id } - : { uid: _app.uid }) ?? apps_only_one[0]; - - if ( ! app ) { - throw APIError.create('no_suitable_app', null, { entry_name: subject.entry.name }); - } - - // Grant permission to open the file - // Note: We always grant write permission here. If the user only - // has read permission this is still safe; user permissions - // are always checked during an app access. - const perm = action === 'write' ? 'write' : 'read'; - const permission = `fs:${subject.uid}:${perm}`; - const svc_permission = Context.get('services').get('permission'); - await svc_permission.grant_user_app_permission(actor, app.uid, permission, {}, { reason: 'open_item' }); - - // Generate user-app token - const svc_auth = Context.get('services').get('auth'); - const token = await svc_auth.get_user_app_token(app.uid); - - // TODO: DRY - // remove some privileged information - delete app.id; - delete app.approved_for_listing; - delete app.approved_for_opening_items; - delete app.godmode; - delete app.owner_user_id; - - return res.send({ - signature: signature, - token, - suggested_apps: [app], - }); -}); diff --git a/src/backend/src/routers/passwd.js b/src/backend/src/routers/passwd.js deleted file mode 100644 index 5acc55d29..000000000 --- a/src/backend/src/routers/passwd.js +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const { invalidate_cached_user, get_user } = require('../helpers'); -const router = new express.Router(); -const auth = require('../middleware/auth.js'); -const { DB_WRITE } = require('../services/database/consts'); - -// -----------------------------------------------------------------------// -// POST /passwd -// -----------------------------------------------------------------------// -router.post('/passwd', auth, express.json(), async (req, res, next) => { - // check subdomain - if ( require('../helpers').subdomain(req) !== 'api' ) - { - next(); - } - - const db = req.services.get('database').get(DB_WRITE, 'auth'); - const bcrypt = require('bcrypt'); - - if ( ! req.body.old_pass ) - { - return res.status(401).send('old_pass is required'); - } - // old_pass must be a string - else if ( typeof req.body.old_pass !== 'string' ) - { - return res.status(400).send('old_pass must be a string.'); - } - else if ( ! req.body.new_pass ) - { - return res.status(401).send('new_pass is required'); - } - // new_pass must be a string - else if ( typeof req.body.new_pass !== 'string' ) - { - return res.status(400).send('new_pass must be a string.'); - } - - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('passwd') ) { - return res.status(429).send('Too many requests.'); - } - - try { - const user = await get_user({ id: req.user.id, force: true }); - // check old_pass - const isMatch = await bcrypt.compare(req.body.old_pass, user.password); - if ( ! isMatch ) - { - return res.status(400).send('old_pass does not match your current password.'); - } - // check new_pass length - // todo use config, 6 is hard-coded and wrong - else if ( req.body.new_pass.length < 6 ) - { - return res.status(400).send('new_pass must be at least 6 characters long.'); - } - else { - await db.write( - 'UPDATE user SET password=?, `pass_recovery_token` = NULL, `change_email_confirm_token` = NULL WHERE `id` = ?', - [await bcrypt.hash(req.body.new_pass, 8), req.user.id], - ); - invalidate_cached_user(req.user); - - const svc_email = req.services.get('email'); - svc_email.send_email({ email: user.email }, 'password_change_notification'); - - return res.send('Password successfully updated.'); - } - } catch (e) { - return res.status(401).send('an error occured'); - } -}); - -module.exports = router; diff --git a/src/backend/src/routers/puterai/anthropic/messages.js b/src/backend/src/routers/puterai/anthropic/messages.js deleted file mode 100644 index 7c94dc565..000000000 --- a/src/backend/src/routers/puterai/anthropic/messages.js +++ /dev/null @@ -1,442 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; - -const crypto = require('node:crypto'); -const APIError = require('../../../api/APIError.js'); -const eggspress = require('../../../api/eggspress.js'); -const { TypedValue } = require('../../../services/drivers/meta/Runtime.js'); -const { Context } = require('../../../util/context.js'); -const auth2 = require('../../../middleware/auth2.js'); - -const DEFAULT_PROVIDER = 'claude'; - -/** - * Translate Anthropic-style tool definitions to the OpenAI/Puter internal - * format so that `svcAiChat.complete()` handles them uniformly. - */ -const normalizeTools = (tools) => { - if ( !Array.isArray(tools) || tools.length === 0 ) return undefined; - return tools.map((t) => { - // Already in OpenAI format (e.g. from passthrough) - if ( t.type === 'function' && t.function ) return t; - // Anthropic format: { name, description, input_schema } - return { - type: 'function', - function: { - name: t.name, - description: t.description || '', - parameters: t.input_schema || { type: 'object', properties: {} }, - }, - }; - }); -}; - -/** - * Extract plain text from a Puter/OpenAI-style message content field. - */ -const extractTextContent = (content) => { - if ( content === undefined || content === null ) return ''; - if ( typeof content === 'string' ) return content; - if ( Array.isArray(content) ) { - return content.map((part) => { - if ( typeof part === 'string' ) return part; - if ( part && typeof part.text === 'string' ) return part.text; - if ( part && typeof part.content === 'string' ) return part.content; - return ''; - }).join(''); - } - if ( typeof content === 'object' ) { - if ( typeof content.text === 'string' ) return content.text; - if ( typeof content.content === 'string' ) return content.content; - } - return ''; -}; - -/** - * Build an Anthropic-style usage object from internal usage data. - */ -const buildUsage = (usage) => { - return { - input_tokens: usage?.input_tokens ?? usage?.prompt_tokens ?? 0, - output_tokens: usage?.output_tokens ?? usage?.completion_tokens ?? 0, - }; -}; - -/** - * Extract tool_use blocks from an internal message result and return them - * as Anthropic content blocks. - */ -const extractToolUseBlocks = (message) => { - const blocks = []; - - // Check for OpenAI-style tool_calls on the message object - if ( message.tool_calls && Array.isArray(message.tool_calls) ) { - for ( const tc of message.tool_calls ) { - blocks.push({ - type: 'tool_use', - id: tc.id, - name: tc.function?.name ?? '', - input: typeof tc.function?.arguments === 'string' - ? (() => { - try { - return JSON.parse(tc.function.arguments); - } catch { - return {}; - } - })() - : (tc.function?.arguments ?? {}), - }); - } - } - - // Check for tool_use blocks inside array-style content - if ( Array.isArray(message.content) ) { - for ( const part of message.content ) { - if ( !part || typeof part !== 'object' ) continue; - if ( part.type === 'tool_use' ) { - blocks.push({ - type: 'tool_use', - id: part.id, - name: part.name, - input: typeof part.input === 'string' - ? (() => { - try { - return JSON.parse(part.input); - } catch { - return {}; - } - })() - : (part.input ?? {}), - }); - } - } - } - - return blocks; -}; - -/** - * Translate Anthropic-format messages into Puter/OpenAI-format messages. - * Specifically, this converts `tool_result` content blocks into `tool` role - * messages that Puter's internal pipeline expects. - */ -const normalizeMessages = (messages, system) => { - const result = []; - - // Inject system message at the start if supplied - if ( system ) { - if ( typeof system === 'string' ) { - result.push({ role: 'system', content: system }); - } else if ( Array.isArray(system) ) { - const text = system.map((s) => { - if ( typeof s === 'string' ) return s; - if ( s && typeof s.text === 'string' ) return s.text; - return ''; - }).join('\n'); - if ( text ) result.push({ role: 'system', content: text }); - } - } - - for ( const msg of messages ) { - // Anthropic places tool_result blocks inside user messages. - // Convert each to a separate `role: 'tool'` message. - if ( msg.role === 'user' && Array.isArray(msg.content) ) { - const toolResults = []; - const otherParts = []; - for ( const part of msg.content ) { - if ( part && part.type === 'tool_result' ) { - toolResults.push(part); - } else { - otherParts.push(part); - } - } - - // Push non-tool content first (if any) - if ( otherParts.length > 0 ) { - result.push({ role: 'user', content: otherParts }); - } - - // Convert each tool_result to a `tool` message - for ( const tr of toolResults ) { - let contentStr = ''; - if ( typeof tr.content === 'string' ) { - contentStr = tr.content; - } else if ( Array.isArray(tr.content) ) { - contentStr = tr.content.map((p) => { - if ( typeof p === 'string' ) return p; - if ( p && typeof p.text === 'string' ) return p.text; - return ''; - }).join(''); - } - result.push({ - role: 'tool', - tool_call_id: tr.tool_use_id, - content: contentStr, - }); - } - - // If the message was entirely tool_results, we already handled it - if ( otherParts.length === 0 && toolResults.length > 0 ) continue; - if ( toolResults.length > 0 ) continue; // already pushed otherParts above - } - - result.push(msg); - } - - return result; -}; - -const svc_web = Context.get('services').get('web-server'); -svc_web.allow_undefined_origin(/^\/puterai\/anthropic\/v1\/messages(\/.*)?$/); - -module.exports = eggspress('/anthropic/v1/messages', { - json: true, - jsonCanBeLarge: true, - allowedMethods: ['POST'], - mw: [(req, _res, next) => { - if ( !req.headers.authorization && req.headers['x-api-key'] ) { - req.headers.authorization = `Bearer ${req.headers['x-api-key']}`; - } - next(); - }, auth2], -}, async (req, res) => { - // We don't allow apps - if ( Context.get('actor').type.app ) { - throw APIError.create('permission_denied'); - } - - const body = req.body || {}; - const stream = !!body.stream; - - if ( ! Array.isArray(body.messages) ) { - throw APIError.create('field_invalid', { - key: 'messages', - expected: 'an array of chat messages', - got: typeof body.messages, - }); - } - - const ctx = Context.get(); - const services = ctx.get('services'); - const svcAiChat = services.get('ai-chat'); - - let model = body.model; - if ( ! model ) { - const providerName = body.provider || DEFAULT_PROVIDER; - const provider = svcAiChat.getProvider(providerName); - if ( ! provider ) { - throw APIError.create('field_missing', { key: 'model' }); - } - model = provider.getDefaultModel(); - } - - // Translate messages from Anthropic format to Puter internal format - const normalizedMessages = normalizeMessages(body.messages, body.system); - const tools = normalizeTools(body.tools); - - const completeArgs = { - messages: normalizedMessages, - model, - stream, - ...(tools ? { tools } : {}), - ...(body.temperature !== undefined ? { temperature: body.temperature } : {}), - ...(body.max_tokens !== undefined ? { max_tokens: body.max_tokens } : {}), - ...(body.provider ? { provider: body.provider } : {}), - }; - - const messageId = `msg_${crypto.randomUUID().replace(/-/g, '')}`; - - const result = await svcAiChat.complete(completeArgs); - - // ================================================================ - // STREAMING RESPONSE — Anthropic SSE format - // ================================================================ - if ( stream ) { - if ( ! (result instanceof TypedValue) ) { - throw APIError.create('internal_error', { message: 'expected streaming response' }); - } - - res.setHeader('Content-Type', 'text/event-stream; charset=utf-8'); - res.setHeader('Cache-Control', 'no-cache, no-transform'); - res.setHeader('Connection', 'keep-alive'); - - const sendEvent = (eventType, data) => { - res.write(`event: ${eventType}\ndata: ${JSON.stringify(data)}\n\n`); - }; - - // message_start - sendEvent('message_start', { - type: 'message_start', - message: { - id: messageId, - type: 'message', - role: 'assistant', - content: [], - model, - stop_reason: null, - stop_sequence: null, - usage: { input_tokens: 0, output_tokens: 0 }, - }, - }); - - let buffer = ''; - let usage = null; - let contentIndex = 0; - let blockOpen = false; - let sawToolCalls = false; - - const openTextBlock = () => { - if ( blockOpen ) return; - sendEvent('content_block_start', { - type: 'content_block_start', - index: contentIndex, - content_block: { type: 'text', text: '' }, - }); - blockOpen = true; - }; - - const closeBlock = () => { - if ( ! blockOpen ) return; - sendEvent('content_block_stop', { - type: 'content_block_stop', - index: contentIndex, - }); - blockOpen = false; - contentIndex++; - }; - - const streamValue = result.value; - streamValue.on('data', (chunk) => { - buffer += chunk.toString('utf8'); - let newlineIndex; - while ( (newlineIndex = buffer.indexOf('\n')) >= 0 ) { - const line = buffer.slice(0, newlineIndex).trim(); - buffer = buffer.slice(newlineIndex + 1); - if ( ! line ) continue; - let event; - try { - event = JSON.parse(line); - } catch { - continue; - } - - if ( event.type === 'text' && typeof event.text === 'string' ) { - openTextBlock(); - sendEvent('content_block_delta', { - type: 'content_block_delta', - index: contentIndex, - delta: { type: 'text_delta', text: event.text }, - }); - } - - if ( event.type === 'tool_use' ) { - sawToolCalls = true; - closeBlock(); // close any open text block first - sendEvent('content_block_start', { - type: 'content_block_start', - index: contentIndex, - content_block: { - type: 'tool_use', - id: event.id, - name: event.name, - input: {}, - }, - }); - blockOpen = true; - - // Emit the input as a single JSON delta - const inputStr = typeof event.input === 'string' - ? event.input - : JSON.stringify(event.input ?? {}); - sendEvent('content_block_delta', { - type: 'content_block_delta', - index: contentIndex, - delta: { type: 'input_json_delta', partial_json: inputStr }, - }); - closeBlock(); - } - - if ( event.type === 'usage' ) { - usage = event.usage; - } - } - }); - - streamValue.on('end', () => { - closeBlock(); - - const stopReason = sawToolCalls ? 'tool_use' : 'end_turn'; - const resolvedUsage = buildUsage(usage || {}); - - sendEvent('message_delta', { - type: 'message_delta', - delta: { stop_reason: stopReason, stop_sequence: null }, - usage: { output_tokens: resolvedUsage.output_tokens }, - }); - - sendEvent('message_stop', { type: 'message_stop' }); - res.end(); - }); - - streamValue.on('error', (err) => { - sendEvent('error', { - type: 'error', - error: { - type: 'api_error', - message: err?.message || 'stream error', - }, - }); - res.end(); - }); - - return; - } - - // ================================================================ - // NON-STREAMING RESPONSE — Anthropic message object - // ================================================================ - const message = result.message || {}; - const toolUseBlocks = extractToolUseBlocks(message); - const textContent = extractTextContent(message.content); - - const contentBlocks = []; - if ( textContent ) { - contentBlocks.push({ type: 'text', text: textContent }); - } - contentBlocks.push(...toolUseBlocks); - - // If there's no content at all, include an empty text block - if ( contentBlocks.length === 0 ) { - contentBlocks.push({ type: 'text', text: '' }); - } - - const stopReason = toolUseBlocks.length > 0 ? 'tool_use' : 'end_turn'; - - res.json({ - id: messageId, - type: 'message', - role: 'assistant', - content: contentBlocks, - model, - stop_reason: stopReason, - stop_sequence: null, - usage: buildUsage(result.usage), - }); -}); diff --git a/src/backend/src/routers/puterai/openai/chat_completions.js b/src/backend/src/routers/puterai/openai/chat_completions.js deleted file mode 100644 index d3f576d89..000000000 --- a/src/backend/src/routers/puterai/openai/chat_completions.js +++ /dev/null @@ -1,250 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; - -const crypto = require('node:crypto'); -const APIError = require('../../../api/APIError.js'); -const eggspress = require('../../../api/eggspress.js'); -const { TypedValue } = require('../../../services/drivers/meta/Runtime.js'); -const { Context } = require('../../../util/context.js'); - -const DEFAULT_PROVIDER = 'openai-completion'; - -const extractTextContent = (content) => { - if ( content === undefined || content === null ) return ''; - if ( typeof content === 'string' ) return content; - if ( Array.isArray(content) ) { - return content.map((part) => { - if ( typeof part === 'string' ) return part; - if ( part && typeof part.text === 'string' ) return part.text; - if ( part && typeof part.content === 'string' ) return part.content; - return ''; - }).join(''); - } - if ( typeof content === 'object' ) { - if ( typeof content.text === 'string' ) return content.text; - if ( typeof content.content === 'string' ) return content.content; - } - return ''; -}; - -const normalizeToolCallsFromContent = (content) => { - if ( ! Array.isArray(content) ) return undefined; - const toolCalls = []; - for ( const part of content ) { - if ( !part || typeof part !== 'object' ) continue; - if ( part.type !== 'tool_use' ) continue; - toolCalls.push({ - id: part.id, - type: 'function', - function: { - name: part.name, - arguments: typeof part.input === 'string' ? part.input : JSON.stringify(part.input ?? {}), - }, - }); - } - return toolCalls.length ? toolCalls : undefined; -}; - -const buildUsage = (usage) => { - const promptTokens = usage?.prompt_tokens ?? usage?.input_tokens ?? 0; - const completionTokens = usage?.completion_tokens ?? usage?.output_tokens ?? 0; - return { - prompt_tokens: promptTokens, - completion_tokens: completionTokens, - total_tokens: promptTokens + completionTokens, - }; -}; - -const svc_web = Context.get('services').get('web-server'); -svc_web.allow_undefined_origin(/^\/puterai\/openai\/v1\/chat\/completions(\/.*)?$/); - -module.exports = eggspress('/openai/v1/chat/completions', { - auth2: true, - json: true, - jsonCanBeLarge: true, - allowedMethods: ['POST'], -}, async (req, res) => { - // We don't allow apps - if ( Context.get('actor').type.app ) { - throw APIError.create('permission_denied'); - } - - const body = req.body || {}; - const stream = !!body.stream; - - if ( ! Array.isArray(body.messages) ) { - throw APIError.create('field_invalid', { - key: 'messages', - expected: 'an array of chat messages', - got: typeof body.messages, - }); - } - - const ctx = Context.get(); - const services = ctx.get('services'); - const svcAiChat = services.get('ai-chat'); - - let model = body.model; - if ( ! model ) { - const providerName = body.provider || DEFAULT_PROVIDER; - const provider = svcAiChat.getProvider(providerName); - if ( ! provider ) { - throw APIError.create('field_missing', { key: 'model' }); - } - model = provider.getDefaultModel(); - } - - const completeArgs = { - messages: body.messages, - model, - stream, - ...(body.tools ? { tools: body.tools } : {}), - ...(body.temperature !== undefined ? { temperature: body.temperature } : {}), - ...(body.max_tokens !== undefined ? { max_tokens: body.max_tokens } : {}), - ...(body.provider ? { provider: body.provider } : {}), - ...(body.image_config ? { image_config: body.image_config } : {}), - }; - - const completionId = `chatcmpl-${crypto.randomUUID().replace(/-/g, '')}`; - const created = Math.floor(Date.now() / 1000); - - const result = await svcAiChat.complete(completeArgs); - - if ( stream ) { - if ( ! (result instanceof TypedValue) ) { - throw APIError.create('internal_error', { message: 'expected streaming response' }); - } - - res.setHeader('Content-Type', 'text/event-stream; charset=utf-8'); - res.setHeader('Cache-Control', 'no-cache, no-transform'); - res.setHeader('Connection', 'keep-alive'); - - let buffer = ''; - let usage = null; - let toolCallIndex = 0; - let sawToolCalls = false; - - const sendChunk = (delta, finishReason = null, extra = {}) => { - const payload = { - id: completionId, - object: 'chat.completion.chunk', - created, - model, - choices: [ - { - index: 0, - delta, - logprobs: null, - finish_reason: finishReason, - }, - ], - ...extra, - }; - res.write(`data: ${JSON.stringify(payload)}\n\n`); - }; - - const streamValue = result.value; - streamValue.on('data', (chunk) => { - buffer += chunk.toString('utf8'); - let newlineIndex; - while ( (newlineIndex = buffer.indexOf('\n')) >= 0 ) { - const line = buffer.slice(0, newlineIndex).trim(); - buffer = buffer.slice(newlineIndex + 1); - if ( ! line ) continue; - let event; - try { - event = JSON.parse(line); - } catch { - continue; - } - if ( event.type === 'text' && typeof event.text === 'string' ) { - sendChunk({ content: event.text }); - } - if ( event.type === 'image' && event.image ) { - sendChunk({ images: [event.image] }); - } - if ( event.type === 'tool_use' ) { - sawToolCalls = true; - sendChunk({ - tool_calls: [ - { - index: toolCallIndex++, - id: event.id, - type: 'function', - function: { - name: event.name, - arguments: typeof event.input === 'string' ? event.input : JSON.stringify(event.input ?? {}), - }, - }, - ], - }); - } - if ( event.type === 'usage' ) { - usage = event.usage; - } - } - }); - - streamValue.on('end', () => { - const finishReason = sawToolCalls ? 'tool_calls' : 'stop'; - sendChunk({}, finishReason, usage ? { usage: buildUsage(usage) } : {}); - res.write('data: [DONE]\n\n'); - res.end(); - }); - - streamValue.on('error', (err) => { - res.write(`data: ${JSON.stringify({ - error: { - message: err?.message || 'stream error', - type: 'stream_error', - }, - })}\n\n`); - res.write('data: [DONE]\n\n'); - res.end(); - }); - - return; - } - - const message = result.message || {}; - const toolCalls = message.tool_calls || normalizeToolCallsFromContent(message.content); - const contentText = extractTextContent(message.content); - - res.json({ - id: completionId, - object: 'chat.completion', - created, - model, - choices: [ - { - index: 0, - message: { - role: message.role || 'assistant', - content: contentText, - ...(toolCalls ? { tool_calls: toolCalls } : {}), - ...(message.images ? { images: message.images } : {}), - }, - logprobs: null, - finish_reason: result.finish_reason ?? 'stop', - }, - ], - usage: buildUsage(result.usage), - }); -}); diff --git a/src/backend/src/routers/puterai/openai/completions.js b/src/backend/src/routers/puterai/openai/completions.js deleted file mode 100644 index 4226e52eb..000000000 --- a/src/backend/src/routers/puterai/openai/completions.js +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; - -const crypto = require('node:crypto'); -const APIError = require('../../../api/APIError.js'); -const eggspress = require('../../../api/eggspress.js'); -const { TypedValue } = require('../../../services/drivers/meta/Runtime.js'); -const { Context } = require('../../../util/context.js'); - -const DEFAULT_PROVIDER = 'openai-completion'; - -const getPromptText = (prompt) => { - if ( prompt === undefined || prompt === null ) { - return ''; - } - if ( Array.isArray(prompt) ) { - if ( prompt.length === 0 ) return ''; - if ( prompt.length === 1 ) { - if ( typeof prompt[0] !== 'string' ) { - throw APIError.create('field_invalid', { - key: 'prompt', - expected: 'a string', - got: typeof prompt[0], - }); - } - return prompt[0]; - } - throw APIError.create('field_invalid', { - key: 'prompt', - expected: 'a string or single-item array', - got: `array length ${prompt.length}`, - }); - } - if ( typeof prompt !== 'string' ) { - throw APIError.create('field_invalid', { - key: 'prompt', - expected: 'a string', - got: typeof prompt, - }); - } - return prompt; -}; - -const extractMessageText = (message) => { - if ( message === undefined || message === null ) return ''; - if ( typeof message === 'string' ) return message; - if ( typeof message !== 'object' ) return ''; - - if ( Array.isArray(message.content) ) { - return message.content.map((part) => { - if ( typeof part === 'string' ) return part; - if ( part && typeof part.text === 'string' ) return part.text; - if ( part && typeof part.content === 'string' ) return part.content; - return ''; - }).join(''); - } - - if ( typeof message.content === 'string' ) return message.content; - if ( message.content && typeof message.content.text === 'string' ) return message.content.text; - return ''; -}; - -const buildUsage = (usage) => { - const promptTokens = usage?.prompt_tokens ?? usage?.input_tokens ?? 0; - const completionTokens = usage?.completion_tokens ?? usage?.output_tokens ?? 0; - return { - prompt_tokens: promptTokens, - completion_tokens: completionTokens, - total_tokens: promptTokens + completionTokens, - }; -}; - -const svc_web = Context.get('services').get('web-server'); -svc_web.allow_undefined_origin(/^\/puterai\/openai\/v1\/completions(\/.*)?$/); - -module.exports = eggspress('/openai/v1/completions', { - auth2: true, - json: true, - jsonCanBeLarge: true, - allowedMethods: ['POST'], -}, async (req, res) => { - // We don't allow apps - if ( Context.get('actor').type.app ) { - throw APIError.create('permission_denied'); - } - - const body = req.body || {}; - const stream = !!body.stream; - - const ctx = Context.get(); - const services = ctx.get('services'); - const svcAiChat = services.get('ai-chat'); - - let messages = body.messages; - if ( ! messages ) { - const prompt = getPromptText(body.prompt); - messages = [{ role: 'user', content: prompt }]; - } - - let model = body.model; - if ( ! model ) { - const providerName = body.provider || DEFAULT_PROVIDER; - const provider = svcAiChat.getProvider(providerName); - if ( ! provider ) { - throw APIError.create('field_missing', { key: 'model' }); - } - model = provider.getDefaultModel(); - } - - const completeArgs = { - messages, - model, - stream, - ...(body.temperature !== undefined ? { temperature: body.temperature } : {}), - ...(body.max_tokens !== undefined ? { max_tokens: body.max_tokens } : {}), - ...(body.provider ? { provider: body.provider } : {}), - }; - - const completionId = `cmpl-${crypto.randomUUID().replace(/-/g, '')}`; - const created = Math.floor(Date.now() / 1000); - - const result = await svcAiChat.complete(completeArgs); - - if ( stream ) { - if ( ! (result instanceof TypedValue) ) { - throw APIError.create('internal_error', { message: 'expected streaming response' }); - } - - res.setHeader('Content-Type', 'text/event-stream; charset=utf-8'); - res.setHeader('Cache-Control', 'no-cache, no-transform'); - res.setHeader('Connection', 'keep-alive'); - - let buffer = ''; - let usage = null; - - const sendChunk = (text, finishReason = null, extra = {}) => { - const payload = { - id: completionId, - object: 'text_completion', - created, - model, - choices: [ - { - text, - index: 0, - logprobs: null, - finish_reason: finishReason, - }, - ], - ...extra, - }; - res.write(`data: ${JSON.stringify(payload)}\n\n`); - }; - - const streamValue = result.value; - streamValue.on('data', (chunk) => { - buffer += chunk.toString('utf8'); - let newlineIndex; - while ( (newlineIndex = buffer.indexOf('\n')) >= 0 ) { - const line = buffer.slice(0, newlineIndex).trim(); - buffer = buffer.slice(newlineIndex + 1); - if ( ! line ) continue; - let event; - try { - event = JSON.parse(line); - } catch { - continue; - } - if ( event.type === 'text' && typeof event.text === 'string' ) { - sendChunk(event.text); - } - if ( event.type === 'usage' ) { - usage = event.usage; - } - } - }); - - streamValue.on('end', () => { - sendChunk('', 'stop', usage ? { usage: buildUsage(usage) } : {}); - res.write('data: [DONE]\n\n'); - res.end(); - }); - - streamValue.on('error', (err) => { - res.write(`data: ${JSON.stringify({ - error: { - message: err?.message || 'stream error', - type: 'stream_error', - }, - })}\n\n`); - res.write('data: [DONE]\n\n'); - res.end(); - }); - - return; - } - - const messageText = extractMessageText(result.message); - const usage = buildUsage(result.usage); - - res.json({ - id: completionId, - object: 'text_completion', - created, - model, - choices: [ - { - text: messageText, - index: 0, - logprobs: null, - finish_reason: result.finish_reason ?? 'stop', - }, - ], - usage, - }); -}); diff --git a/src/backend/src/routers/puterai/openai/responses.js b/src/backend/src/routers/puterai/openai/responses.js deleted file mode 100644 index 6f044b050..000000000 --- a/src/backend/src/routers/puterai/openai/responses.js +++ /dev/null @@ -1,535 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; - -const crypto = require('node:crypto'); -const APIError = require('../../../api/APIError.js'); -const eggspress = require('../../../api/eggspress.js'); -const { TypedValue } = require('../../../services/drivers/meta/Runtime.js'); -const { Context } = require('../../../util/context.js'); - -const DEFAULT_PROVIDER = 'openai-responses'; - -const generateId = (prefix) => `${prefix}_${crypto.randomUUID().replace(/-/g, '')}`; - -const parseJsonMaybe = (value) => { - if ( typeof value !== 'string' ) return value ?? {}; - try { - return JSON.parse(value); - } catch { - return value; - } -}; - -const normalizeToolToResponsesTool = (tool) => { - if ( !tool || typeof tool !== 'object' ) return tool; - if ( tool.type !== 'function' ) return tool; - return { - ...tool.function, - type: 'function', - }; -}; - -const normalizeContentPart = (part) => { - if ( typeof part === 'string' ) { - return { type: 'text', text: part }; - } - if ( !part || typeof part !== 'object' ) { - return { type: 'text', text: '' }; - } - if ( part.type === 'input_text' ) { - return { type: 'text', text: part.text ?? '' }; - } - if ( part.type === 'output_text' ) { - return { type: 'text', text: part.text ?? '' }; - } - if ( part.type === 'input_image' ) { - return { - type: 'image_url', - ...(part.detail ? { detail: part.detail } : {}), - ...(part.image_url ? { image_url: { url: part.image_url } } : {}), - ...(part.file_id ? { file_id: part.file_id } : {}), - }; - } - if ( part.type === 'input_audio' ) { - return { - type: 'input_audio', - input_audio: part.input_audio, - }; - } - if ( part.type === 'input_file' ) { - return { - type: 'input_file', - ...(part.file_data ? { file_data: part.file_data } : {}), - ...(part.file_id ? { file_id: part.file_id } : {}), - ...(part.file_url ? { file_url: part.file_url } : {}), - ...(part.filename ? { filename: part.filename } : {}), - }; - } - return part; -}; - -const normalizeMessageContent = (content) => { - if ( content === undefined || content === null ) return ''; - if ( typeof content === 'string' ) return content; - if ( Array.isArray(content) ) { - return content.map(normalizeContentPart); - } - return [normalizeContentPart(content)]; -}; - -const responseInputToMessages = (input) => { - if ( input === undefined || input === null ) return []; - if ( typeof input === 'string' ) { - return [{ role: 'user', content: input }]; - } - if ( ! Array.isArray(input) ) { - throw APIError.create('field_invalid', { - key: 'input', - expected: 'a string or array', - got: typeof input, - }); - } - - const messages = []; - for ( const item of input ) { - if ( typeof item === 'string' ) { - messages.push({ role: 'user', content: item }); - continue; - } - if ( !item || typeof item !== 'object' ) continue; - - if ( item.type === 'function_call_output' ) { - messages.push({ - role: 'tool', - tool_call_id: item.call_id, - content: typeof item.output === 'string' - ? item.output - : JSON.stringify(item.output ?? {}), - }); - continue; - } - - if ( item.type === 'function_call' ) { - messages.push({ - role: 'assistant', - content: [ - { - type: 'tool_use', - id: item.call_id || item.id || generateId('call'), - canonical_id: item.id, - name: item.name, - input: parseJsonMaybe(item.arguments), - }, - ], - }); - continue; - } - - if ( item.type === 'message' || item.role ) { - messages.push({ - role: item.role === 'developer' ? 'system' : (item.role || 'user'), - content: normalizeMessageContent(item.content), - }); - continue; - } - - messages.push({ - role: 'user', - content: normalizeMessageContent(item), - }); - } - - return messages; -}; - -const buildUsage = (usage) => { - const inputTokens = usage?.prompt_tokens ?? usage?.input_tokens ?? 0; - const outputTokens = usage?.completion_tokens ?? usage?.output_tokens ?? 0; - return { - input_tokens: inputTokens, - input_tokens_details: { - cached_tokens: usage?.cached_tokens ?? usage?.input_tokens_details?.cached_tokens ?? 0, - }, - output_tokens: outputTokens, - output_tokens_details: { - reasoning_tokens: usage?.output_tokens_details?.reasoning_tokens ?? 0, - }, - total_tokens: inputTokens + outputTokens, - }; -}; - -const createBaseResponse = ({ responseId, createdAt, model, body, output = [], usage, status }) => ({ - id: responseId, - object: 'response', - created_at: createdAt, - status, - error: null, - incomplete_details: null, - instructions: body.instructions ?? null, - metadata: body.metadata ?? null, - model, - output, - output_text: output - .filter(item => item?.type === 'message') - .flatMap(item => item.content || []) - .filter(part => part?.type === 'output_text') - .map(part => part.text || '') - .join(''), - parallel_tool_calls: body.parallel_tool_calls ?? false, - temperature: body.temperature ?? null, - tool_choice: body.tool_choice ?? 'auto', - tools: Array.isArray(body.tools) ? body.tools.map(normalizeToolToResponsesTool) : [], - top_p: body.top_p ?? null, - ...(body.max_output_tokens !== undefined ? { max_output_tokens: body.max_output_tokens } : {}), - ...(body.previous_response_id ? { previous_response_id: body.previous_response_id } : {}), - ...(body.store !== undefined ? { store: body.store } : {}), - ...(body.text ? { text: body.text } : {}), - ...(body.truncation ? { truncation: body.truncation } : {}), - ...(usage ? { usage } : {}), -}); - -const responseOutputFromResult = (result) => { - const output = []; - const message = result?.message || {}; - const content = typeof message.content === 'string' - ? message.content - : Array.isArray(message.content) - ? message.content - .filter(part => part?.type === 'text') - .map(part => part.text || '') - .join('') - : ''; - - if ( content ) { - output.push({ - id: generateId('msg'), - type: 'message', - role: 'assistant', - status: 'completed', - content: [ - { - type: 'output_text', - text: content, - annotations: [], - }, - ], - }); - } - - for ( const toolCall of message.tool_calls || [] ) { - output.push({ - id: toolCall.canonical_id || generateId('fc'), - type: 'function_call', - call_id: toolCall.id, - name: toolCall.function?.name, - arguments: toolCall.function?.arguments ?? '{}', - status: 'completed', - }); - } - - return output; -}; - -const svc_web = Context.get('services').get('web-server'); -svc_web.allow_undefined_origin(/^\/puterai\/openai\/v1\/responses(\/.*)?$/); - -module.exports = eggspress('/openai/v1/responses', { - auth2: true, - json: true, - jsonCanBeLarge: true, - allowedMethods: ['POST'], -}, async (req, res) => { - if ( Context.get('actor').type.app ) { - throw APIError.create('permission_denied'); - } - - const body = req.body || {}; - const stream = !!body.stream; - - const ctx = Context.get(); - const services = ctx.get('services'); - const svcAiChat = services.get('ai-chat'); - const providerName = body.provider || DEFAULT_PROVIDER; - - if ( providerName !== DEFAULT_PROVIDER ) { - throw APIError.create('field_invalid', { - key: 'provider', - expected: DEFAULT_PROVIDER, - got: providerName, - }); - } - - let model = body.model; - if ( ! model ) { - const provider = svcAiChat.getProvider(providerName); - if ( ! provider ) { - throw APIError.create('field_missing', { key: 'model' }); - } - model = provider.getDefaultModel(); - } - - const messages = [ - ...(body.instructions ? [{ role: 'system', content: body.instructions }] : []), - ...responseInputToMessages(body.input), - ]; - - const completeArgs = { - messages, - model, - stream, - ...(body.tools ? { tools: body.tools } : {}), - ...(body.tool_choice ? { tool_choice: body.tool_choice } : {}), - ...(body.parallel_tool_calls !== undefined ? { parallel_tool_calls: body.parallel_tool_calls } : {}), - ...(body.temperature !== undefined ? { temperature: body.temperature } : {}), - ...(body.max_output_tokens !== undefined ? { max_tokens: body.max_output_tokens } : {}), - ...(body.top_p !== undefined ? { top_p: body.top_p } : {}), - ...(body.reasoning ? { reasoning: body.reasoning } : {}), - ...(body.text ? { text: body.text } : {}), - ...(body.include ? { include: body.include } : {}), - ...(body.instructions ? { instructions: body.instructions } : {}), - ...(body.metadata ? { metadata: body.metadata } : {}), - ...(body.conversation ? { conversation: body.conversation } : {}), - ...(body.previous_response_id ? { previous_response_id: body.previous_response_id } : {}), - ...(body.prompt ? { prompt: body.prompt } : {}), - ...(body.prompt_cache_key ? { prompt_cache_key: body.prompt_cache_key } : {}), - ...(body.prompt_cache_retention ? { prompt_cache_retention: body.prompt_cache_retention } : {}), - ...(body.store !== undefined ? { store: body.store } : {}), - ...(body.truncation ? { truncation: body.truncation } : {}), - ...(body.background !== undefined ? { background: body.background } : {}), - ...(body.service_tier ? { service_tier: body.service_tier } : {}), - provider: providerName, - }; - - const responseId = generateId('resp'); - const createdAt = Math.floor(Date.now() / 1000); - const result = await svcAiChat.complete(completeArgs); - - if ( stream ) { - if ( ! (result instanceof TypedValue) ) { - throw APIError.create('internal_error', { message: 'expected streaming response' }); - } - - res.setHeader('Content-Type', 'text/event-stream; charset=utf-8'); - res.setHeader('Cache-Control', 'no-cache, no-transform'); - res.setHeader('Connection', 'keep-alive'); - - let buffer = ''; - let sequenceNumber = 0; - let usage = null; - let messageItem = null; - let messageOutputIndex = null; - const output = []; - let textContent = ''; - - const sendEvent = (event) => { - res.write(`event: ${event.type}\n`); - res.write(`data: ${JSON.stringify({ - ...event, - sequence_number: ++sequenceNumber, - })}\n\n`); - }; - - sendEvent({ - type: 'response.created', - response: createBaseResponse({ - responseId, - createdAt, - model, - body, - output: [], - status: 'in_progress', - }), - }); - - const streamValue = result.value; - streamValue.on('data', (chunk) => { - buffer += chunk.toString('utf8'); - let newlineIndex; - while ( (newlineIndex = buffer.indexOf('\n')) >= 0 ) { - const line = buffer.slice(0, newlineIndex).trim(); - buffer = buffer.slice(newlineIndex + 1); - if ( ! line ) continue; - - let event; - try { - event = JSON.parse(line); - } catch { - continue; - } - - if ( event.type === 'text' && typeof event.text === 'string' ) { - if ( ! messageItem ) { - messageItem = { - id: generateId('msg'), - type: 'message', - role: 'assistant', - status: 'in_progress', - content: [], - }; - output.push(messageItem); - messageOutputIndex = output.length - 1; - sendEvent({ - type: 'response.output_item.added', - output_index: messageOutputIndex, - item: messageItem, - }); - const part = { - type: 'output_text', - text: '', - annotations: [], - }; - messageItem.content.push(part); - sendEvent({ - type: 'response.content_part.added', - output_index: messageOutputIndex, - item_id: messageItem.id, - content_index: 0, - part, - }); - } - - textContent += event.text; - messageItem.content[0].text = textContent; - sendEvent({ - type: 'response.output_text.delta', - output_index: messageOutputIndex, - item_id: messageItem.id, - content_index: 0, - delta: event.text, - }); - } - - if ( event.type === 'tool_use' ) { - const item = { - id: event.canonical_id || generateId('fc'), - type: 'function_call', - call_id: event.id, - name: event.name, - arguments: typeof event.input === 'string' - ? event.input - : JSON.stringify(event.input ?? {}), - status: 'completed', - }; - output.push(item); - const outputIndex = output.length - 1; - sendEvent({ - type: 'response.output_item.added', - output_index: outputIndex, - item: { - ...item, - status: 'in_progress', - arguments: '', - }, - }); - sendEvent({ - type: 'response.function_call_arguments.delta', - output_index: outputIndex, - item_id: item.id, - delta: item.arguments, - }); - sendEvent({ - type: 'response.function_call_arguments.done', - output_index: outputIndex, - item_id: item.id, - name: item.name, - arguments: item.arguments, - }); - sendEvent({ - type: 'response.output_item.done', - output_index: outputIndex, - item, - }); - } - - if ( event.type === 'usage' ) { - usage = buildUsage(event.usage); - } - } - }); - - streamValue.on('end', () => { - if ( messageItem ) { - messageItem.status = 'completed'; - sendEvent({ - type: 'response.output_text.done', - output_index: messageOutputIndex, - item_id: messageItem.id, - content_index: 0, - text: textContent, - logprobs: [], - }); - sendEvent({ - type: 'response.content_part.done', - output_index: messageOutputIndex, - item_id: messageItem.id, - content_index: 0, - part: messageItem.content[0], - }); - sendEvent({ - type: 'response.output_item.done', - output_index: messageOutputIndex, - item: messageItem, - }); - } - - sendEvent({ - type: 'response.completed', - response: createBaseResponse({ - responseId, - createdAt, - model, - body, - output, - usage, - status: 'completed', - }), - }); - res.write('data: [DONE]\n\n'); - res.end(); - }); - - streamValue.on('error', (err) => { - sendEvent({ - type: 'error', - error: { - message: err?.message || 'stream error', - type: 'stream_error', - }, - }); - res.write('data: [DONE]\n\n'); - res.end(); - }); - - return; - } - - const usage = buildUsage(result.usage); - const output = responseOutputFromResult(result); - - res.json(createBaseResponse({ - responseId, - createdAt, - model, - body, - output, - usage, - status: 'completed', - })); -}); diff --git a/src/backend/src/routers/puterai/video/proxy.js b/src/backend/src/routers/puterai/video/proxy.js deleted file mode 100644 index 2e0c6254d..000000000 --- a/src/backend/src/routers/puterai/video/proxy.js +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; - -const { Readable } = require('stream'); -const { sha256 } = require('js-sha256'); -const config = require('../../../config.js'); -const eggspress = require('../../../api/eggspress.js'); - -const GEMINI_DOWNLOAD_BASE = 'https://generativelanguage.googleapis.com/download/v1beta/files'; - -module.exports = eggspress('/video/proxy', { - allowedMethods: ['GET'], -}, async (req, res) => { - const fileId = req.query.fileId; - const provider = req.query.provider; - const expires = req.query.expires; - const signature = req.query.signature; - - if ( !fileId || typeof fileId !== 'string' || !/^[a-zA-Z0-9_-]+$/.test(fileId) ) { - return res.status(400).send('Invalid or missing fileId parameter'); - } - - if ( !expires || !signature ) { - return res.status(403).send('Missing signature'); - } - - if ( Number(expires) < Date.now() / 1000 ) { - return res.status(403).send('Signature expired'); - } - - const secret = config.url_signature_secret; - const expected = sha256(`${fileId}/video-proxy/${secret}/${expires}`); - if ( signature !== expected ) { - return res.status(403).send('Invalid signature'); - } - - if ( provider === 'gemini' ) { - const geminiConfig = config.services?.gemini; - const apiKey = geminiConfig?.apiKey || geminiConfig?.secret_key; - - if ( ! apiKey ) { - return res.status(500).send('Gemini API key not configured'); - } - - const url = `${GEMINI_DOWNLOAD_BASE}/${fileId}:download?alt=media&key=${apiKey}`; - const videoUriResponse = await fetch(url); - - if ( ! videoUriResponse.ok ) { - return res.status(videoUriResponse.status).send('Failed to fetch video'); - } - - const contentType = videoUriResponse.headers.get('content-type'); - if ( contentType ) { - res.setHeader('Content-Type', contentType); - } - - if ( videoUriResponse.body ) { - Readable.fromWeb(videoUriResponse.body).pipe(res); - } else { - res.status(500).send('Empty response body'); - } - } else { - return res.status(400).send('Unsupported provider'); - } -}); diff --git a/src/backend/src/routers/query/app.js b/src/backend/src/routers/query/app.js deleted file mode 100644 index addfddd16..000000000 --- a/src/backend/src/routers/query/app.js +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const eggspress = require('../../api/eggspress'); -const { is_valid_uuid4, get_app } = require('../../helpers'); -const express = require('express'); -const { fuzz_number } = require('../../util/fuzz'); -const { DB_READ } = require('../../services/database/consts'); - -const PREFIX_APP_UID = 'app-'; - -module.exports = eggspress('/query/app', { - subdomain: 'api', - auth: true, - verified: true, - fs: true, - mw: [express.json({ extended: true })], - allowedMethods: ['POST'], -}, async (req, res, _next) => { - const results = []; - - const db = req.services.get('database').get(DB_READ, 'apps'); - - const svc_appInformation = req.services.get('app-information'); - - const app_list = [...req.body]; - - for ( let i = 0 ; i < app_list.length ; i++ ) { - const P = 'collection:'; - if ( app_list[i].startsWith(P) ) { - let [col_name, amount] = app_list[i].slice(P.length).split(':'); - if ( amount === undefined ) amount = 20; - let uids = svc_appInformation.collections?.[col_name] ?? []; - uids = uids.slice(0, Math.min(uids.length, amount)); - app_list.splice(i, 1, ...uids); - } - } - - for ( let i = 0 ; i < app_list.length ; i++ ) { - const P = 'tag:'; - if ( app_list[i].startsWith(P) ) { - let [tag_name, amount] = app_list[i].slice(P.length).split(':'); - if ( amount === undefined ) amount = 20; - let uids = svc_appInformation.tags[tag_name] ?? []; - uids = uids.slice(0, Math.min(uids.length, amount)); - app_list.splice(i, 1, ...uids); - } - } - - for ( const app_selector_raw of app_list ) { - const app_selector = - app_selector_raw.startsWith(PREFIX_APP_UID) && - is_valid_uuid4(app_selector_raw.slice(PREFIX_APP_UID.length)) - ? { uid: app_selector_raw } - : { name: app_selector_raw } - ; - - const app = await get_app(app_selector); - if ( ! app ) continue; - - // uuid, name, title, description, icon, created, filetype_associations, number of users - - // emit event for extra data gathering - const extraDataEventObject = Object.fromEntries(app_list.map((appId) => [appId, {}])); - await req.services.get('event').emit('apps.queried.extra', extraDataEventObject); - - // TODO: cache - const associations = []; { - const res_associations = await db.read( - 'SELECT * FROM app_filetype_association WHERE app_id = ?', - [app.id], - ); - for ( const row of res_associations ) { - associations.push(row.type); - } - } - - const stats = await svc_appInformation.get_stats(app.uid); - for ( const k in stats ) stats[k] = fuzz_number(stats[k]); - - delete stats.open_count; - - // TODO: imply from app model - results.push({ - uuid: app.uid, - name: app.name, - title: app.title, - // icon: app.icon, - description: app.description, - metadata: app.metadata, - tags: app.tags ? app.tags.split(',') : [], - created: app.timestamp, - associations, - ...stats, - ...extraDataEventObject[app.uid], - }); - } - - res.send(results); -}); diff --git a/src/backend/src/routers/recentAppOpens/RecentAppOpensRedisCacheSpace.js b/src/backend/src/routers/recentAppOpens/RecentAppOpensRedisCacheSpace.js deleted file mode 100644 index 9415ca955..000000000 --- a/src/backend/src/routers/recentAppOpens/RecentAppOpensRedisCacheSpace.js +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const RecentAppOpensRedisCacheSpace = { - key: userId => `app_opens:user:${userId}`, -}; - -export { RecentAppOpensRedisCacheSpace }; diff --git a/src/backend/src/routers/recentAppOpens/rao.js b/src/backend/src/routers/recentAppOpens/rao.js deleted file mode 100644 index 9a55ce139..000000000 --- a/src/backend/src/routers/recentAppOpens/rao.js +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -// records app opens - -'use strict'; -const express = require('express'); -const router = express.Router(); -const config = require('../../config'); -const { is_valid_uuid4, get_app } = require('../../helpers'); -const { DB_WRITE } = require('../../services/database/consts.js'); -const configurable_auth = require('../../middleware/configurable_auth.js'); -const { UserActorType, AppUnderUserActorType } = require('../../services/auth/Actor.js'); -const APIError = require('../../api/APIError.js'); -const { redisClient } = require('../../clients/redis/redisSingleton'); -const { setRedisCacheValue } = require('../../clients/redis/cacheUpdate.js'); -const { RecentAppOpensRedisCacheSpace } = require('./RecentAppOpensRedisCacheSpace.js'); - -// -----------------------------------------------------------------------// -// POST /rao -// -----------------------------------------------------------------------// -router.post('/rao', configurable_auth(), express.json(), async (req, res, next) => { - const { actor } = req; - // check subdomain - if ( require('../../helpers').subdomain(req) !== 'api' ) - { - next(); - } - - // check if user is verified - if ( (config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed ) - { - return res.status(400).send({ code: 'account_is_not_verified', message: 'Account is not verified' }); - } - - let app_uid; - if ( actor.type instanceof UserActorType ) { - // validation - if ( !req.body.app_uid || typeof req.body.app_uid !== 'string' && !(req.body.app_uid instanceof String) ) - { - return res.status(400).send({ code: 'invalid_app_uid', message: 'Invalid app uid' }); - } - // must be a valid uuid - // app uuids start with 'app-', so in order to validate them we remove the prefix first - else if ( ! is_valid_uuid4(req.body.app_uid.replace('app-', '')) ) - { - return res.status(400).send({ code: 'invalid_app_uid', message: 'Invalid app uid' }); - } - - app_uid = req.body.app_uid; - } else if ( actor.type instanceof AppUnderUserActorType ) { - app_uid = actor.type.app.uid; - } else { - throw APIError.create('forbidden'); - } - - // get db connection - const db = req.services.get('database').get(DB_WRITE, 'apps'); - - // insert into db - db.write( - 'INSERT INTO app_opens (app_uid, user_id, ts) VALUES (?, ?, ?)', - [app_uid, req.user.id, Math.floor(new Date().getTime() / 1000)], - ); - - // get app - const opened_app = await get_app({ uid: app_uid }); - - // send process event `puter.app_open` - process.emit('puter.app_open', { - app_uid: app_uid, - user_id: req.user.id, - app_owner_user_id: opened_app.owner_user_id, - ts: Math.floor(new Date().getTime() / 1000), - }); - - // -----------------------------------------------------------------------// - // Update the 'app opens' cache - // -----------------------------------------------------------------------// - // First try the cache to see if we have recent apps - let recent_apps; - const recent_apps_raw = await redisClient.get(RecentAppOpensRedisCacheSpace.key(req.user.id)); - if ( recent_apps_raw ) { - try { - recent_apps = JSON.parse(recent_apps_raw); - } catch ( e ) { - recent_apps = null; - } - } - - // If cache is not empty, prepend it with the new app - if ( recent_apps && Array.isArray(recent_apps) && recent_apps.length > 0 ) { - // add the app to the beginning of the array - recent_apps.unshift({ app_uid: app_uid }); - - // dedupe the array - recent_apps = recent_apps.filter((v, i, a) => a.findIndex(t => (t.app_uid === v.app_uid)) === i); - - // limit to 10 - recent_apps = recent_apps.slice(0, 10); - - // update cache - await setRedisCacheValue( - RecentAppOpensRedisCacheSpace.key(req.user.id), - JSON.stringify(recent_apps), - { eventData: recent_apps }, - ); - } - // Cache is empty, query the db and update the cache - else { - db.read( - 'SELECT DISTINCT app_uid FROM app_opens WHERE user_id = ? GROUP BY app_uid ORDER BY MAX(_id) DESC LIMIT 10', - [req.user.id], - ).then(async ([apps]) => { - // Update cache with the results from the db (if any results were returned) - if ( apps && Array.isArray(apps) && apps.length > 0 ) { - await setRedisCacheValue( - RecentAppOpensRedisCacheSpace.key(req.user.id), - JSON.stringify(apps), - { eventData: apps }, - ); - } - }); - } - - // Update clients - const svc_socketio = req.services.get('socketio'); - svc_socketio.send({ room: req.user.id }, 'app.opened', { - uuid: opened_app.uid, - uid: opened_app.uid, - name: opened_app.name, - title: opened_app.title, - icon: opened_app.icon, - godmode: opened_app.godmode, - maximize_on_start: opened_app.maximize_on_start, - index_url: opened_app.index_url, - original_client_socket_id: req.body.original_client_socket_id, - }); - - // return - return res.status(200).send({ code: 'ok', message: 'ok' }); -}); - -module.exports = router; diff --git a/src/backend/src/routers/save_account.js b/src/backend/src/routers/save_account.js deleted file mode 100644 index 9e2e86994..000000000 --- a/src/backend/src/routers/save_account.js +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = new express.Router(); -const { - get_taskbar_items, username_exists, send_email_verification_code, send_email_verification_token, invalidate_cached_user, get_user, - is_user_signup_disabled: lazy_user_signup, -} = require('../helpers'); -const auth = require('../middleware/auth.js'); -const config = require('../config'); -const { DB_WRITE } = require('../services/database/consts'); -const SECOND = 1000; - -// -----------------------------------------------------------------------// -// POST /save_account -// -----------------------------------------------------------------------// -router.post('/save_account', auth, express.json(), async (req, res, next) => { - // either api. subdomain or no subdomain - if ( require('../helpers').subdomain(req) !== 'api' && require('../helpers').subdomain(req) !== '' ) - { - next(); - } - - const is_user_signup_disabled = await lazy_user_signup(); - if ( is_user_signup_disabled ) { - return res.status(403).send('User signup is disabled.'); - } - - // modules - const db = req.services.get('database').get(DB_WRITE, 'auth'); - const validator = require('validator'); - const bcrypt = require('bcrypt'); - const { v4: uuidv4 } = require('uuid'); - - // validation - if ( req.user.password !== null ) - { - return res.status(400).send('User account already saved.'); - } - else if ( ! req.body.username ) - { - return res.status(400).send('Username is required'); - } - // username must be a string - else if ( typeof req.body.username !== 'string' ) - { - return res.status(400).send('username must be a string.'); - } - else if ( ! req.body.username.match(config.username_regex) ) - { - return res.status(400).send('Username can only contain letters, numbers and underscore (_).'); - } - else if ( req.body.username.length > config.username_max_length ) - { - return res.status(400).send(`Username cannot have more than ${config.username_max_length} characters.`); - } - // check if username matches any reserved words - else if ( config.reserved_words.includes(req.body.username) ) - { - return res.status(400).send({ message: 'This username is not available.' }); - } - else if ( ! req.body.email ) - { - return res.status(400).send('Email is required'); - } - // email must be a string - else if ( typeof req.body.email !== 'string' ) - { - return res.status(400).send('email must be a string.'); - } - else if ( ! validator.isEmail(req.body.email) ) - { - return res.status(400).send('Please enter a valid email address.'); - } - else if ( ! req.body.password ) - { - return res.status(400).send('Password is required'); - } - // password must be a string - else if ( typeof req.body.password !== 'string' ) - { - return res.status(400).send('password must be a string.'); - } - else if ( req.body.password.length < config.min_pass_length ) - { - return res.status(400).send(`Password must be at least ${config.min_pass_length} characters long.`); - } - - const svc_cleanEmail = req.services.get('clean-email'); - const clean_email = svc_cleanEmail.clean(req.body.email); - - if ( ! await svc_cleanEmail.validate(clean_email) ) { - return res.status(400).send('This email cannot be used. Please try a different email address.'); - } - - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('save-account') ) { - return res.status(429).send('Too many requests.'); - } - - const svc_lock = req.services.get('lock'); - return svc_lock.lock([ - `save-account:username:${req.body.username}`, - `save-account:email:${req.body.email}`, - ], { timeout: 5 * SECOND }, async () => { - // duplicate username check, do this only if user has supplied a new username - if ( req.body.username !== req.user.username && await username_exists(req.body.username) ) - { - return res.status(400).send('Username already taken. Try another one.'); - } - // duplicate email check (pseudo-users don't count) - let rows2 = await db.read('SELECT EXISTS(SELECT 1 FROM user WHERE email=? AND password IS NOT NULL) AS email_exists', [req.body.email]); - if ( rows2[0].email_exists ) - { - return res.status(400).send('An account with this email already exists. Please use another email.'); - } - // get pseudo user, if exists - let pseudo_user = await db.read('SELECT * FROM user WHERE email = ? AND password IS NULL', [req.body.email]); - pseudo_user = pseudo_user[0]; - - // send_confirmation_code - req.body.send_confirmation_code = req.body.send_confirmation_code ?? true; - - // todo email confirmation is required by default unless: - // Pseudo user converting and matching uuid is provided - let email_confirmation_required = 0; - - // ----------------------------------- - // Get referral user - // ----------------------------------- - let referred_by_user = undefined; - if ( req.body.referral_code ) { - referred_by_user = await get_user({ referral_code: req.body.referral_code }); - if ( ! referred_by_user ) { - return res.status(400).send('Referral code not found'); - } - } - - // ----------------------------------- - // New User - // ----------------------------------- - const user_uuid = req.user.uuid; - let email_confirm_code = Math.floor(100000 + Math.random() * 900000); - const email_confirm_token = uuidv4(); - - if ( pseudo_user === undefined ) { - await db.write( - `UPDATE user - SET - username = ?, email = ?, password = ?, email_confirm_code = ?, email_confirm_token = ?${ - referred_by_user ? ', referred_by = ?' : '' } - WHERE - id = ?`, - [ - // username - req.body.username, - // email - req.body.email, - // password - await bcrypt.hash(req.body.password, 8), - // email_confirm_code - `${ email_confirm_code}`, - //email_confirm_token - email_confirm_token, - // referred_by - ...(referred_by_user ? [referred_by_user.id] : []), - // id - req.user.id, - ], - ); - invalidate_cached_user(req.user); - - // Update root directory name - await db.write( - 'UPDATE fsentries SET name = ?, path = ? WHERE user_id = ? and parent_uid IS NULL', - [ - // name - req.body.username, - `/${ req.body.username}`, - // id - req.user.id, - ], - ); - const filesystem = req.services.get('filesystem'); - await filesystem.update_child_paths(`/${req.user.username}`, `/${req.body.username}`, req.user.id); - - if ( req.body.send_confirmation_code ) - { - send_email_verification_code(email_confirm_code, req.body.email); - } - else - { - send_email_verification_token(email_confirm_token, req.body.email, user_uuid); - } - } - - // create token for login: session token for cookie, GUI token for client - const svc_auth = req.services.get('auth'); - const { session, token: session_token } = await svc_auth.create_session_token(req.user, { req }); - const gui_token = svc_auth.create_gui_token(req.user, session); - - // user id - // todo if pseudo user, assign directly no need to do another DB lookup - const user_id = req.user.id; - const user_res = await db.read('SELECT * FROM `user` WHERE `id` = ? LIMIT 1', [user_id]); - const user = user_res[0]; - - // todo send LINK-based verification email - - // HTTP-only cookie gets session token (cookie-based requests have hasHttpOnlyCookie) - res.cookie(config.cookie_name, session_token); - - { - const svc_event = req.services.get('event'); - svc_event.emit('user.save_account', { user }); - } - - // return results - return res.send({ - token: gui_token, - user: { - username: user.username, - uuid: user.uuid, - email: user.email, - is_temp: false, - requires_email_confirmation: user.requires_email_confirmation, - email_confirmed: user.email_confirmed, - email_confirmation_required: email_confirmation_required, - taskbar_items: await get_taskbar_items(user), - referral_code: user.referral_code, - }, - }); - }); -}); - -module.exports = router; diff --git a/src/backend/src/routers/send-confirm-email.js b/src/backend/src/routers/send-confirm-email.js deleted file mode 100644 index f62187f98..000000000 --- a/src/backend/src/routers/send-confirm-email.js +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = new express.Router(); -const auth = require('../middleware/auth.js'); -const { send_email_verification_code, invalidate_cached_user } = require('../helpers'); -const { DB_WRITE } = require('../services/database/consts.js'); - -// -----------------------------------------------------------------------// -// POST /send-confirm-email -// -----------------------------------------------------------------------// -router.post('/send-confirm-email', auth, express.json(), async (req, res, next) => { - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('send-confirm-email') ) { - return res.status(429).send('Too many requests.'); - } - - // check subdomain - if ( require('../helpers').subdomain(req) !== 'api' ) - { - next(); - } - - const db = req.services.get('database').get(DB_WRITE, 'auth'); - let email_confirm_code = Math.floor(100000 + Math.random() * 900000); - - if ( req.user.suspended ) - { - return res.status(401).send({ error: 'Account suspended' }); - } - - await db.write( - 'UPDATE user SET email_confirm_code = ? WHERE id = ?', - [ - // email_confirm_code - `${email_confirm_code}`, - // id - req.user.id, - ], - ); - await invalidate_cached_user(req.user); - - // send email verification - send_email_verification_code(email_confirm_code, req.user.email); - - res.send(); -}); - -module.exports = router; diff --git a/src/backend/src/routers/send-pass-recovery-email.js b/src/backend/src/routers/send-pass-recovery-email.js deleted file mode 100644 index 7903c2419..000000000 --- a/src/backend/src/routers/send-pass-recovery-email.js +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = new express.Router(); -const { body_parser_error_handler, get_user, invalidate_cached_user } = require('../helpers'); -const config = require('../config'); -const { DB_WRITE } = require('../services/database/consts'); - -const jwt = require('jsonwebtoken'); - -// -----------------------------------------------------------------------// -// POST /send-pass-recovery-email -// -----------------------------------------------------------------------// -router.post('/send-pass-recovery-email', express.json(), body_parser_error_handler, async (req, res, next) => { - // either api. subdomain or no subdomain - if ( require('../helpers').subdomain(req) !== 'api' && require('../helpers').subdomain(req) !== '' ) - { - next(); - } - - // modules - const db = req.services.get('database').get(DB_WRITE, 'auth'); - const validator = require('validator'); - - // validation - if ( !req.body.username && !req.body.email ) - { - return res.status(400).send('Username or email is required.'); - } - // username, if provided, must be a string - else if ( req.body.username && typeof req.body.username !== 'string' ) - { - return res.status(400).send('username must be a string.'); - } - // if username doesn't pass regex test it's invalid anyway, no need to do DB lookup - else if ( req.body.username && !req.body.username.match(config.username_regex) ) - { - return res.status(400).send('Invalid username.'); - } - // email, if provided, must be a string - else if ( req.body.email && typeof req.body.email !== 'string' ) - { - return res.status(400).send('email must be a string.'); - } - // if email is invalid, no need to do DB lookup anyway - else if ( req.body.email && !validator.isEmail(req.body.email) ) - { - return res.status(400).send('Invalid email.'); - } - - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('send-pass-recovery-email') ) { - return res.status(429).send('Too many requests.'); - } - - try { - let user; - // see if username exists - if ( req.body.username ) { - user = await get_user({ username: req.body.username }); - if ( ! user ) - { - return res.status(400).send('Username not found.'); - } - } - // see if email exists - else if ( req.body.email ) { - user = await get_user({ email: req.body.email }); - if ( ! user ) - { - return res.status(400).send('Email not found.'); - } - } - - if ( user.username === 'system' && config.allow_system_login !== true ) { - return res.status(400).send( - req.body.username - ? 'Username not found.' - : 'Email not found.', - ); - } - - // check if user is suspended - if ( user.suspended ) { - return res.status(401).send('Account suspended'); - } - - // check if user even has an email for recovery - if ( ! user.email ) { - return res.status(422).send('No email associated with this account.'); - } - - // set pass_recovery_token - const { v4: uuidv4 } = require('uuid'); - const token = uuidv4(); - await db.write( - 'UPDATE user SET pass_recovery_token=? WHERE `id` = ?', - [token, user.id], - ); - invalidate_cached_user(user); - - // create jwt - const jwt_token = jwt.sign({ - user_uid: user.uuid, - token, - // email change invalidates password recovery - email: user.email, - }, config.jwt_secret, { expiresIn: '1h' }); - - // create link - const rec_link = `${config.origin }/action/set-new-password?token=${ jwt_token}`; - - const svc_email = req.services.get('email'); - await svc_email.send_email({ email: user.email }, 'email_password_recovery', { - link: rec_link, - }); - - // Send response - if ( req.body.username ) - { - return res.send({ message: `Password recovery sent to the email associated with ${user.username}. Please check your email for instructions on how to reset your password.` }); - } - else - { - return res.send({ message: `Password recovery email sent to ${user.email}. Please check your email for instructions on how to reset your password.` }); - } - - } catch (e) { - console.log(e); - return res.status(400).send(e); - } - -}); - -module.exports = router; diff --git a/src/backend/src/routers/set-desktop-bg.js b/src/backend/src/routers/set-desktop-bg.js deleted file mode 100644 index 15a2c9d80..000000000 --- a/src/backend/src/routers/set-desktop-bg.js +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const config = require('../config.js'); -const { invalidate_cached_user } = require('../helpers'); -const router = new express.Router(); -const auth = require('../middleware/auth.js'); -const { DB_WRITE } = require('../services/database/consts.js'); - -// -----------------------------------------------------------------------// -// POST /set-desktop-bg -// -----------------------------------------------------------------------// -router.post('/set-desktop-bg', auth, express.json(), async (req, res, next) => { - // check subdomain - if ( require('../helpers').subdomain(req) !== 'api' ) - { - next(); - } - - // check if user is verified - if ( (config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed ) - { - return res.status(400).send({ code: 'account_is_not_verified', message: 'Account is not verified' }); - } - - // modules - const db = req.services.get('database').get(DB_WRITE, 'ui'); - - // insert into DB - await db.write( - 'UPDATE user SET desktop_bg_url = ?, desktop_bg_color = ?, desktop_bg_fit = ? WHERE user.id = ?', - [ - req.body.url ?? null, - req.body.color ?? null, - req.body.fit ?? null, - req.user.id, - ], - ); - invalidate_cached_user(req.user); - - // send results to client - return res.send({}); -}); -module.exports = router; diff --git a/src/backend/src/routers/set-pass-using-token.js b/src/backend/src/routers/set-pass-using-token.js deleted file mode 100644 index 394077180..000000000 --- a/src/backend/src/routers/set-pass-using-token.js +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = new express.Router(); -const config = require('../config'); -const { invalidate_cached_user_by_id, get_user } = require('../helpers'); -const { DB_WRITE } = require('../services/database/consts'); - -const jwt = require('jsonwebtoken'); - -// Ensure we don't expose branches with differing messages. -const SAFE_NEGATIVE_RESPONSE = 'This password recovery token is no longer valid.'; - -// -----------------------------------------------------------------------// -// POST /set-pass-using-token -// -----------------------------------------------------------------------// -router.post('/set-pass-using-token', express.json(), async (req, res, next) => { - // check subdomain - if ( require('../helpers').subdomain(req) !== 'api' && require('../helpers').subdomain(req) !== '' ) - { - next(); - } - - // modules - const bcrypt = require('bcrypt'); - - const db = req.services.get('database').get(DB_WRITE, 'auth'); - - // password is required - if ( ! req.body.password ) - { - return res.status(401).send('password is required'); - } - // token is required - else if ( ! req.body.token ) - { - return res.status(401).send('token is required'); - } - // password must be a string - else if ( typeof req.body.password !== 'string' ) - { - return res.status(400).send('password must be a string.'); - } - // check password length - else if ( req.body.password.length < config.min_pass_length ) - { - return res.status(400).send(`Password must be at least ${config.min_pass_length} characters long.`); - } - - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('set-pass-using-token') ) { - return res.status(429).send('Too many requests.'); - } - - const { token, user_uid, email } = jwt.verify(req.body.token, config.jwt_secret); - - const user = await get_user({ uuid: user_uid, force: true }); - if ( user.email !== email ) { - return res.status(400).send(SAFE_NEGATIVE_RESPONSE); - } - - try { - const info = await db.write( - 'UPDATE user SET password=?, pass_recovery_token=NULL, change_email_confirm_token=NULL WHERE `uuid` = ? AND pass_recovery_token = ?', - [await bcrypt.hash(req.body.password, 8), user_uid, token], - ); - - if ( ! info?.anyRowsAffected ) { - return res.status(400).send(SAFE_NEGATIVE_RESPONSE); - } - - invalidate_cached_user_by_id(user.id); - - return res.send('Password successfully updated.'); - } catch (e) { - return res.status(500).send('An internal error occured.'); - } -}); - -module.exports = router; diff --git a/src/backend/src/routers/set_layout.js b/src/backend/src/routers/set_layout.js deleted file mode 100644 index 4b7cc33c0..000000000 --- a/src/backend/src/routers/set_layout.js +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = express.Router(); -const auth = require('../middleware/auth.js'); -const config = require('../config'); -const { DB_WRITE } = require('../services/database/consts.js'); - -// -----------------------------------------------------------------------// -// POST /set_layout -// -----------------------------------------------------------------------// -router.post('/set_layout', auth, express.json(), async (req, res, next) => { - // check subdomain - if ( require('../helpers').subdomain(req) !== 'api' ) - { - next(); - } - - // check if user is verified - if ( (config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed ) - { - return res.status(400).send({ code: 'account_is_not_verified', message: 'Account is not verified' }); - } - - // validation - if ( req.body.item_uid === undefined && req.body.item_path === undefined ) - { - return res.status(400).send('`item_uid` or `item_path` is required'); - } - else if ( req.body.layout === undefined ) - { - return res.status(400).send('`layout` is required'); - } - else if ( req.body.layout !== 'icons' && req.body.layout !== 'details' && req.body.layout !== 'list' ) - { - return res.status(400).send('invalid `layout`'); - } - // modules - const db = req.services.get('database').get(DB_WRITE, 'ui'); - const { uuid2fsentry, convert_path_to_fsentry, chkperm } = require('../helpers'); - - //get dir - let item; - if ( req.body.item_uid ) - { - item = await uuid2fsentry(req.body.item_uid); - } - else if ( req.body.item_path ) - { - item = await convert_path_to_fsentry(req.body.item_path); - } - - // item not found - if ( item === false ) { - return res.status(400).send({ - error: { - message: 'No entry found with this uid', - }, - }); - } - - // must be dir - if ( ! item.is_dir ) - { - return res.status(400).send('must be a directory'); - } - - // check permission - if ( ! await chkperm(item, req.user.id, 'write') ) - { - return res.status(403).send({ code: 'forbidden', message: 'permission denied.' }); - } - - // insert into DB - await db.write('UPDATE fsentries SET layout = ? WHERE id = ?', - [req.body.layout, item.id]); - - // send results to client - return res.send({}); -}); -module.exports = router; \ No newline at end of file diff --git a/src/backend/src/routers/set_sort_by.js b/src/backend/src/routers/set_sort_by.js deleted file mode 100644 index 8a4d5b381..000000000 --- a/src/backend/src/routers/set_sort_by.js +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = express.Router(); -const auth = require('../middleware/auth.js'); -const config = require('../config'); -const { DB_WRITE } = require('../services/database/consts.js'); - -// -----------------------------------------------------------------------// -// POST /set_sort_by -// -----------------------------------------------------------------------// -router.post('/set_sort_by', auth, express.json(), async (req, res, next) => { - // check subdomain - if ( require('../helpers').subdomain(req) !== 'api' ) - { - next(); - } - - // check if user is verified - if ( (config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed ) - { - return res.status(400).send({ code: 'account_is_not_verified', message: 'Account is not verified' }); - } - - // validation - if ( req.body.item_uid === undefined && req.body.item_path === undefined ) - { - return res.status(400).send('`item_uid` or `item_path` is required'); - } - else if ( req.body.sort_by === undefined ) - { - return res.status(400).send('`sort_by` is required'); - } - else if ( req.body.sort_by !== 'name' && req.body.sort_by !== 'size' && req.body.sort_by !== 'modified' && req.body.sort_by !== 'type' ) - { - return res.status(400).send('invalid `sort_by`'); - } - else if ( req.body.sort_order !== 'asc' && req.body.sort_order !== 'desc' ) - { - return res.status(400).send('invalid `sort_order`'); - } - - // modules - const db = req.services.get('database').get(DB_WRITE, 'ui'); - const { uuid2fsentry, convert_path_to_fsentry, chkperm } = require('../helpers'); - - //get dir - let item; - if ( req.body.item_uid ) - { - item = await uuid2fsentry(req.body.item_uid); - } - else if ( req.body.item_path ) - { - item = await convert_path_to_fsentry(req.body.item_path); - } - - // item not found - if ( item === false ) { - return res.status(400).send({ - error: { - message: 'No entry found with this uid', - }, - }); - } - - // must be dir - if ( ! item.is_dir ) - { - return res.status(400).send('must be a directory'); - } - - // check permission - if ( ! await chkperm(item, req.user.id, 'write') ) - { - return res.status(403).send({ code: 'forbidden', message: 'permission denied.' }); - } - - // set sort_by - await db.write('UPDATE fsentries SET sort_by = ? WHERE id = ?', - [req.body.sort_by, item.id]); - - // set sort_order - await db.write('UPDATE fsentries SET sort_order = ? WHERE id = ?', - [req.body.sort_order, item.id]); - - // send results to client - return res.send({}); -}); -module.exports = router; \ No newline at end of file diff --git a/src/backend/src/routers/sign.js b/src/backend/src/routers/sign.js deleted file mode 100644 index af1b76919..000000000 --- a/src/backend/src/routers/sign.js +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const { sign_file, get_app } = require('../helpers'); -const eggspress = require('../api/eggspress.js'); -const APIError = require('../api/APIError.js'); -const { Context } = require('../util/context.js'); -const { UserActorType, AppUnderUserActorType } = require('../services/auth/Actor.js'); -const { NodePathSelector } = require('../deprecated/filesystem/node/selectors.js'); - -// -----------------------------------------------------------------------// -// POST /sign -// -----------------------------------------------------------------------// -module.exports = eggspress('/sign', { - subdomain: 'api', - auth2: true, - verified: true, - json: true, - allowedMethods: ['POST'], -}, async (req, res, next) => { - const actor = Context.get('actor'); - const svc_fs = Context.get('services').get('filesystem'); - - if ( ! req.body.items ) { - throw APIError.create('field_missing', null, { key: 'items' }); - } - - let items = Array.isArray(req.body.items) ? req.body.items : [res]; - let signatures = []; - - // Static request validation happens first - for ( const item of items ) { - if ( ! item ) { - throw APIError.create('field_invalid', null, { - key: 'items', - expected: 'each item to have: (uid OR path) AND action', - }).serialize(); - } - - if ( typeof item !== 'object' || Array.isArray(item) ) { - throw APIError.create('field_invalid', null, { - key: 'items', - expected: 'each item to be an object', - }).serialize(); - } - - // validation - if ( (!item.uid && !item.path) || !item.action ) { - throw APIError.create('field_invalid', null, { - key: 'items', - expected: 'each item to have: (uid OR path) AND action', - }).serialize(); - } - - if ( typeof item.uid !== 'string' && typeof item.path !== 'string' ) { - throw APIError.create('field_invalid', null, { - key: 'items', - expected: 'each item to have only string values for uid and path', - }).serialize(); - } - } - - // Usually, only users can sign - if ( ! (actor.type instanceof UserActorType) ) { - - if ( ! (actor.type instanceof AppUnderUserActorType) ) { - throw APIError.create('forbidden'); - } - - // But, apps can sign files in their own AppData directory - for ( const item of req.body.items ) { - const node = await svc_fs.node(item); - const appdata_path = `/${actor.type.user.username}/AppData/${actor.type.app.uid}`; - const appdata_node = await svc_fs.node(new NodePathSelector(appdata_path)); - if ( ! appdata_node.is_above(node) ) { - throw APIError.create('forbidden'); - } - } - } - - const result = { - signatures, - }; - - let app = null; - if ( req.body.app_uid ) { - if ( typeof req.body.app_uid !== 'string' ) { - throw APIError.create('field_invalid', null, { - key: 'app_uid', - expected: 'string', - }); - } - - app = await get_app({ uid: req.body.app_uid }); - if ( ! app ) { - // FIXME: subject.entry.name isn't available here - throw APIError.create('no_suitable_app', null); //, { entry_name: subject.entry.name }); - } - // Generate user-app token - const svc_auth = Context.get('services').get('auth'); - const token = await svc_auth.get_user_app_token(app.uid); - result.token = token; - } - - for ( const item of items ) { - const node = await svc_fs.node(item); - - if ( ! await node.exists() ) { - // throw APIError.create('subject_does_not_exist').serialize() - signatures.push({}); - continue; - } - - const svc_acl = Context.get('services').get('acl'); - if ( ! await svc_acl.check(actor, node, 'read') ) { - throw await svc_acl.get_safe_acl_error(actor, node, 'read'); - } - - if ( item.action === 'write' ) { - if ( ! await svc_acl.check(actor, node, 'write') ) { - item.action = 'read'; - } - } - - if ( app !== null ) { - // Grant write permission to app - const svc_permission = Context.get('services').get('permission'); - const permission = `fs:${await node.get('uid')}:write`; - await svc_permission.grant_user_app_permission(actor, app.uid, permission, {}, { reason: 'endpoint:sign' }); - } - - // sign - try { - let signature = await sign_file(node.entry, item.action); - signature.path = signature.path ?? item.path ?? await node.get('path'); - signatures.push(signature); - } - catch (e) { - signatures.push({}); - } - } - - res.send(result); -}); diff --git a/src/backend/src/routers/signup.js b/src/backend/src/routers/signup.js deleted file mode 100644 index 8cc0b7d4c..000000000 --- a/src/backend/src/routers/signup.js +++ /dev/null @@ -1,495 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const { get_taskbar_items, send_email_verification_code, send_email_verification_token, username_exists, invalidate_cached_user_by_id, get_user } = require('../helpers'); -const config = require('../config'); -const eggspress = require('../api/eggspress'); -const { Context } = require('../util/context'); -const { DB_WRITE } = require('../services/database/consts'); -const { generate_identifier } = require('../util/identifier'); -const { is_temp_users_disabled: lazy_temp_users, - is_user_signup_disabled: lazy_user_signup } = require('../helpers'); -const { requireCaptcha } = require('../modules/captcha/middleware/captcha-middleware'); - -async function generate_random_username () { - let username; - do { - username = generate_identifier(); - } while ( await username_exists(username) ); - return username; -} - -// -----------------------------------------------------------------------// -// POST /signup -// -----------------------------------------------------------------------// -module.exports = eggspress(['/signup'], { - allowedMethods: ['POST'], - alarm_timeout: 7000, // when it calls us - response_timeout: 20000, // when it gives up - abuse: { - no_bots: true, - // puter_origin: false, - shadow_ban_responder: (req, res) => { - res.status(400).send('email username mismatch; please provide a password'); - }, - }, - mw: [requireCaptcha({ strictMode: true, eventType: 'signup' })], // Conditionally require captcha for signup -}, async (req, res, next) => { - // either api. subdomain or no subdomain - if ( require('../helpers').subdomain(req) !== 'api' && require('../helpers').subdomain(req) !== '' ) - { - next(); - } - - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('signup') ) { - return res.status(429).send('Too many requests.'); - } - - // modules - const db = req.services.get('database').get(DB_WRITE, 'auth'); - const bcrypt = require('bcrypt'); - const { v4: uuidv4 } = require('uuid'); - const validator = require('validator'); - let uuid_user; - - const svc_auth = Context.get('services').get('auth'); - const svc_authAudit = Context.get('services').get('auth-audit'); - svc_authAudit.record({ - requester: Context.get('requester'), - action: req.body.is_temp ? 'signup:temp' : 'signup:real', - body: req.body, - }); - - // check bot trap, if `p102xyzname` is anything but an empty string it means - // that a bot has filled the form - // doesn't apply to temp users - if ( !req.body.is_temp && req.body.p102xyzname !== '' ) - { - return res.send(); - } - - // cloudflare turnstile validation - // - // ref: https://developers.cloudflare.com/turnstile/get-started/server-side-validation/ - if ( config.services?.['cloudflare-turnstile']?.enabled ) { - const formData = new FormData(); - formData.append('secret', config.services?.['cloudflare-turnstile']?.secret_key); - formData.append('response', req.body['cf-turnstile-response']); - formData.append('remoteip', req.headers['x-forwarded-for'] || req.connection.remoteAddress); - - const response = await fetch('https://challenges.cloudflare.com/turnstile/v0/siteverify', { - method: 'POST', - body: formData, - }); - - const result = await response.json(); - if ( ! result.success ) - { - return res.status(400).send('captcha verification failed'); - } - } - - // send event - let event = { - allow: true, - ip: req.headers?.['x-forwarded-for'] || - req.connection?.remoteAddress, - user_agent: req.headers?.['user-agent'], - body: req.body, - }; - - const svc_event = Context.get('services').get('event'); - await svc_event.emit('puter.signup', event); - - if ( ! event.allow ) { - return res.status(400).send({ message: event.error ?? 'You are not allowed to sign up.', code: 'not_allowed_to_signup' }); - } - - // check if user is already logged in - if ( req.body.is_temp && req.cookies[config.cookie_name] ) { - const { user, token } = await svc_auth.check_session(req.cookies[config.cookie_name]); - res.cookie(config.cookie_name, token, { - sameSite: 'none', - secure: true, - httpOnly: true, - }); - // const decoded = await jwt.verify(token, config.jwt_secret); - // const user = await get_user({ uuid: decoded.uuid }); - if ( user ) { - return res.send({ - token: token, - user: { - username: user.username, - uuid: user.uuid, - email: user.email, - email_confirmed: user.email_confirmed, - requires_email_confirmation: user.requires_email_confirmation, - is_temp: (user.password === null && user.email === null), - taskbar_items: await get_taskbar_items(user), - }, - }); - } - } - - const is_temp_users_disabled = await lazy_temp_users(); - const is_user_signup_disabled = await lazy_user_signup(); - - if ( is_temp_users_disabled && is_user_signup_disabled ) { - return res.status(403).send({ message: 'User signup and Temporary users are disabled.', code: 'user_signup_and_temp_users_disabled' }); - } - - if ( !req.body.is_temp && is_user_signup_disabled ) { - return res.status(403).send({ message: 'User signup is disabled.', code: 'user_signup_disabled' }); - } - - if ( req.body.is_temp && is_temp_users_disabled ) { - return res.status(403).send({ message: 'Temporary users are disabled.', code: 'temp_users_disabled' }); - } - - if ( req.body.is_temp && event.no_temp_user ) { - return res.status(403).send({ message: 'You must login or signup.', code: 'must_login_or_signup' }); - } - - // Create temp user data - req.body.username = req.body.username ?? await generate_random_username(); - req.body.email = req.body.email ?? `${req.body.username }@gmail.com`; - req.body.password = req.body.password ?? 'sadasdfasdfsadfsa'; - - // send_confirmation_code - req.body.send_confirmation_code = req.body.send_confirmation_code ?? true; - - // username is required - if ( ! req.body.username ) - { - return res.status(400).send('Username is required'); - } - // username must be a string - else if ( typeof req.body.username !== 'string' ) - { - return res.status(400).send('username must be a string.'); - } - // check if username is valid - else if ( ! req.body.username.match(config.username_regex) ) - { - return res.status(400).send('Username can only contain letters, numbers and underscore (_).'); - } - // check if username is of proper length - else if ( req.body.username.length > config.username_max_length ) - { - return res.status(400).send(`Username cannot be longer than ${config.username_max_length} characters.`); - } - // check if username matches any reserved words - else if ( config.reserved_words.includes(req.body.username) ) - { - return res.status(400).send({ message: 'This username is not available.' }); - } - // TODO: DRY: change_email.js - else if ( !req.body.is_temp && !req.body.email ) - { - return res.status(400).send('Email is required'); - } - // email, if present, must be a string - else if ( req.body.email && typeof req.body.email !== 'string' ) - { - return res.status(400).send('email must be a string.'); - } - // if email is present, validate it - else if ( !req.body.is_temp && !validator.isEmail(req.body.email) ) - { - return res.status(400).send('Please enter a valid email address.'); - } - else if ( !req.body.is_temp && !req.body.password ) - { - return res.status(400).send('Password is required'); - } - // password, if present, must be a string - else if ( req.body.password && typeof req.body.password !== 'string' ) - { - return res.status(400).send('password must be a string.'); - } - else if ( !req.body.is_temp && req.body.password.length < config.min_pass_length ) - { - return res.status(400).send(`Password must be at least ${config.min_pass_length} characters long.`); - } - - const svc_cleanEmail = req.services.get('clean-email'); - const clean_email = svc_cleanEmail.clean(req.body.email); - - if ( !req.body.is_temp && !await svc_cleanEmail.validate(clean_email) ) { - return res.status(400).send('This email cannot be used. Please try a different email address.'); - } - - // duplicate username check - if ( await username_exists(req.body.username) ) - { - return res.status(400).send('Username already taken. Try another one.'); - } - // Email check is here :: Add condition for email_confirmed=1 - // duplicate email check (pseudo-users don't count) - let rows2 = await db.read(`SELECT EXISTS( - SELECT 1 FROM user WHERE (email=? OR clean_email=?) AND email_confirmed=1 AND password IS NOT NULL - ) AS email_exists`, [req.body.email, clean_email]); - if ( rows2[0].email_exists ) - { - return res.status(400).send('This email already exists in our database. Please use another one.'); - } - // get pseudo user, if exists - let pseudo_user = await db.read('SELECT * FROM user WHERE email = ? AND password IS NULL', [req.body.email]); - pseudo_user = pseudo_user[0]; - // get uuid user, if exists - if ( req.body.uuid ) { - uuid_user = await db.read('SELECT * FROM user WHERE uuid = ? LIMIT 1', [req.body.uuid]); - uuid_user = uuid_user[0]; - } - - // email confirmation is not required by default - let email_confirmation_required = 0; - - // Pseudo user converting and matching uuid is provided - if ( pseudo_user && uuid_user && pseudo_user.id === uuid_user.id ) - { - email_confirmation_required = 0; - } - - // if an extension requires email confirmation, set it to required - if ( event.requires_email_confirmation ) { - email_confirmation_required = 1; - } - - // ----------------------------------- - // Get referral user - // ----------------------------------- - let referred_by_user = undefined; - if ( req.body.referral_code ) { - referred_by_user = await get_user({ referral_code: req.body.referral_code }); - if ( ! referred_by_user ) { - return res.status(400).send('Referral code not found'); - } - } - - // ----------------------------------- - // New User - // ----------------------------------- - const user_uuid = uuidv4(); - const email_confirm_token = uuidv4(); - let insert_res; - let email_confirm_code = Math.floor(100000 + Math.random() * 900000); - - const audit_metadata = { - ip: req.connection.remoteAddress, - ip_fwd: req.headers['x-forwarded-for'], - user_agent: req.headers['user-agent'], - origin: req.headers['origin'], - server: config.server_id, - }; - - if ( pseudo_user === undefined ) { - insert_res = await db.write( - `INSERT INTO user - ( - username, email, clean_email, password, uuid, referrer, - email_confirm_code, email_confirm_token, free_storage, - referred_by, audit_metadata, signup_ip, signup_ip_forwarded, - signup_user_agent, signup_origin, signup_server, requires_email_confirmation - ) - VALUES - (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - [ - // username - req.body.username, - // email - req.body.is_temp ? null : req.body.email, - // normalized email - req.body.is_temp ? null : clean_email, - // password - req.body.is_temp ? null : await bcrypt.hash(req.body.password, 8), - // uuid - user_uuid, - // referrer - req.body.referrer ?? null, - // email_confirm_code - `${ email_confirm_code}`, - // email_confirm_token - email_confirm_token, - // free_storage - config.storage_capacity, - // referred_by - referred_by_user ? referred_by_user.id : null, - // audit_metadata - JSON.stringify(audit_metadata), - // signup_ip - req.connection.remoteAddress ?? null, - // signup_ip_fwd - req.headers['x-forwarded-for'] ?? null, - // signup_user_agent - req.headers['user-agent'] ?? null, - // signup_origin - req.headers['origin'] ?? null, - // signup_server - config.server_id ?? null, - // requires_email_confirmation - email_confirmation_required, - ], - ); - - // record activity - db.write( - 'UPDATE `user` SET `last_activity_ts` = now() WHERE id=? LIMIT 1', - [insert_res.insertId], - ); - - // TODO: cache group id - const svc_group = req.services.get('group'); - await svc_group.add_users({ - uid: req.body.is_temp ? - config.default_temp_group : config.default_user_group, - users: [req.body.username], - }); - - // send an event for successful signup - const svc_event = req.services.get('event'); - svc_event.emit('puter.signup.success', { - user_id: insert_res.insertId, - user_uuid: user_uuid, - email: req.body.email, - username: req.body.username, - password: req.body.password, - ip: req.headers['x-forwarded-for'] || req.connection.remoteAddress, - }); - } - // ----------------------------------- - // Pseudo User converting - // ----------------------------------- - else { - insert_res = await db.write( - `UPDATE user SET - username = ?, password = ?, uuid = ?, email_confirm_code = ?, email_confirm_token = ?, email_confirmed = ?, requires_email_confirmation = 1, - referred_by = ? - WHERE id = ?`, - [ - // username - req.body.username, - // password - await bcrypt.hash(req.body.password, 8), - // uuid - user_uuid, - // email_confirm_code - `${ email_confirm_code}`, - // email_confirm_token - email_confirm_token, - // email_confirmed - !email_confirmation_required, - // id - pseudo_user.id, - // referred_by - referred_by_user ? referred_by_user.id : null, - ], - ); - - // TODO: cache group ids - const svc_group = req.services.get('group'); - await svc_group.remove_users({ - uid: config.default_temp_group, - users: [req.body.username], - }); - await svc_group.add_users({ - uid: config.default_user_group, - users: [req.body.username], - }); - - // record activity - db.write('UPDATE `user` SET `last_activity_ts` = now() WHERE id=? LIMIT 1', [pseudo_user.id]); - invalidate_cached_user_by_id(pseudo_user.id); - } - - // user id - // todo if pseudo user, assign directly no need to do another DB lookup - const user_id = (pseudo_user === undefined) ? insert_res.insertId : pseudo_user.id; - - const [user] = await db.pread( - 'SELECT * FROM `user` WHERE `id` = ? LIMIT 1', - [user_id], - ); - - // create token for login: session token for cookie, GUI token for client - const { session, token: session_token } = await svc_auth.create_session_token(user, { - req, - }); - const gui_token = svc_auth.create_gui_token(user, session); - // jwt.sign({uuid: user_uuid}, config.jwt_secret); - - //------------------------------------------------------------- - // email confirmation - //------------------------------------------------------------- - // Email confirmation from signup is sent here - if ( (!req.body.is_temp && email_confirmation_required) || user.requires_email_confirmation ) { - if ( req.body.send_confirmation_code || user.requires_email_confirmation ) - { - send_email_verification_code(email_confirm_code, user.email); - } - else - { - send_email_verification_token(user.email_confirm_token, user.email, user.uuid); - } - } - - //------------------------------------------------------------- - // referral code - //------------------------------------------------------------- - let referral_code; - if ( pseudo_user === undefined ) { - const svc_referralCode = Context.get('services') - .get('referral-code', { optional: true }); - if ( svc_referralCode ) { - referral_code = await svc_referralCode.gen_referral_code(user); - } - } - - const svc_user = Context.get('services').get('user'); - await svc_user.generate_default_fsentries({ user }); - - // HTTP-only cookie gets session token (cookie-based requests have hasHttpOnlyCookie) - res.cookie(config.cookie_name, session_token, { - sameSite: 'none', - secure: true, - httpOnly: true, - }); - - // add to mailchimp - if ( ! req.body.is_temp ) { - const svc_event = Context.get('services').get('event'); - svc_event.emit('user.save_account', { user }); - } - - // return results - return res.send({ - token: gui_token, - user: { - username: user.username, - uuid: user.uuid, - email: user.email, - email_confirmed: user.email_confirmed, - requires_email_confirmation: user.requires_email_confirmation, - is_temp: (user.password === null && user.email === null), - taskbar_items: await get_taskbar_items(user), - referral_code, - }, - }); -}); diff --git a/src/backend/src/routers/signup_create_new_user.js b/src/backend/src/routers/signup_create_new_user.js deleted file mode 100644 index 902b82fbe..000000000 --- a/src/backend/src/routers/signup_create_new_user.js +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import config from '../config.js'; -import { DB_WRITE } from '../services/database/consts.js'; -import { generate_identifier } from '../util/identifier.js'; -import { v4 as uuidv4 } from 'uuid'; - -/** - * Create a new user for signup. Common behavior shared by POST /signup and OIDC signup. - * Form-signup path is still handled in signup.js; this handles OIDC and will support form signup after refactor. - * - * @param {object} services - Backend services (from req.services) - * @param {object} options - Creation options. For OIDC: { providerId, userinfo }. For form signup: TBD (to be refactored from signup.js). - * @returns {Promise} The created user, or null on failure (e.g. email already registered). - */ -async function signup_create_new_user (services, options) { - const { providerId, userinfo } = options; - if ( !providerId || !userinfo ) { - // Form signup: to be refactored from signup.js; not implemented here yet. - return null; - } - - const db = await services.get('database').get(DB_WRITE, 'auth'); - const svc_group = services.get('group'); - const svc_user = services.get('user'); - const svc_oidc = services.get('oidc'); - if ( ! svc_oidc ) return null; - - const claims = userinfo; - let username = (claims.name || claims.email || '').toString().trim(); - if ( username ) { - username = username.replace(/\s+/g, '_').replace(/[^a-zA-Z0-9_-]/g, ''); - if ( username.length > 45 ) username = username.slice(0, 45); - } - if ( !username || !/^\w+$/.test(username) ) { - let candidate; - do { - candidate = generate_identifier(); - const [r] = await db.pread('SELECT 1 FROM user WHERE username = ? LIMIT 1', [candidate]); - if ( ! r ) username = candidate; - } while ( !username ); - } else { - const [existing] = await db.pread('SELECT 1 FROM user WHERE username = ? LIMIT 1', [username]); - if ( existing ) { - let suffix = 1; - while ( true ) { - const candidate = `${username}${suffix}`; - const [r] = await db.pread('SELECT 1 FROM user WHERE username = ? LIMIT 1', [candidate]); - if ( ! r ) { - username = candidate; break; - } - suffix++; - } - } - } - - const email = (claims.email || '').toString().trim() || null; - const clean_email = email ? email.toLowerCase().trim() : null; - if ( clean_email ) { - const [existingEmail] = await db.pread('SELECT 1 FROM user WHERE clean_email = ? LIMIT 1', [clean_email]); - if ( existingEmail ) { - return null; // email already registered; caller should return error - } - } - - const user_uuid = uuidv4(); - const email_confirm_code = String(Math.floor(100000 + Math.random() * 900000)); - const email_confirm_token = uuidv4(); - - await db.write(`INSERT INTO user ( - username, email, clean_email, password, uuid, referrer, - email_confirm_code, email_confirm_token, free_storage, - referred_by, email_confirmed, requires_email_confirmation - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - [ - username, - email, - clean_email, - null, - user_uuid, - null, - email_confirm_code, - email_confirm_token, - config.storage_capacity, - null, - 1, - 0, - ]); - const [inserted] = await db.pread('SELECT id FROM user WHERE uuid = ? LIMIT 1', [user_uuid]); - const user_id = inserted.id; - - await svc_oidc.linkProviderToUser(user_id, providerId, claims.sub, null); - - await svc_group.add_users({ - uid: config.default_user_group, - users: [username], - }); - - const [user] = await db.pread('SELECT * FROM user WHERE id = ? LIMIT 1', [user_id]); - if ( user && user.metadata && typeof user.metadata === 'string' ) { - user.metadata = JSON.parse(user.metadata); - } else if ( user && !user.metadata ) { - user.metadata = {}; - } - await svc_user.generate_default_fsentries({ user }); - - return user; -} - -export default signup_create_new_user; diff --git a/src/backend/src/routers/suggest_apps.js b/src/backend/src/routers/suggest_apps.js deleted file mode 100644 index 32baf2d0a..000000000 --- a/src/backend/src/routers/suggest_apps.js +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = new express.Router(); -const auth = require('../middleware/auth.js'); -const config = require('../config'); -const { Context } = require('../util/context.js'); -const { NodeInternalIDSelector } = require('../deprecated/filesystem/node/selectors.js'); -const { convert_path_to_fsentry, uuid2fsentry, suggestedAppForFsEntry } = require('../helpers'); - -// -----------------------------------------------------------------------// -// POST /suggest_apps -// -----------------------------------------------------------------------// -router.post('/suggest_apps', auth, express.json(), async (req, res, next) => { - // check subdomain - if ( require('../helpers').subdomain(req) !== 'api' ) - { - next(); - } - - // check if user is verified - if ( (config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed ) - { - return res.status(400).send({ code: 'account_is_not_verified', message: 'Account is not verified' }); - } - - // validation - if ( req.body.uid === undefined && req.body.path === undefined ) - { - return res.status(400).send({ message: '`uid` or `path` required' }); - } - - let fsentry; - - // by uid - if ( req.body.uid ) - { - fsentry = await uuid2fsentry(req.body.uid); - } - // by path - else { - fsentry = await convert_path_to_fsentry(req.body.path); - if ( fsentry === false ) - { - return res.status(400).send('Path not found.'); - } - } - - const services = Context.get('services'); - const fs = services.get('filesystem'); - const node = await fs.node(new NodeInternalIDSelector('mysql', fsentry.id, { - source: 'suggest_apps', - })); - - // check permission - const actor = req.actor ?? Context.get('actor'); - if ( ! actor ) { - return res.status(500).send('failed to get Actor object'); - } - const svc_acl = services.get('acl'); - if ( ! await svc_acl.check(actor, node, 'read') ) { - (await svc_acl.get_safe_acl_error(actor, node, 'read')) - .write(res); - return; - } - - // get suggestions - try { - return res.send(await suggestedAppForFsEntry(fsentry)); - } - catch (e) { - return res.status(400).send(e); - } -}); - -module.exports = router; \ No newline at end of file diff --git a/src/backend/src/routers/test.js b/src/backend/src/routers/test.js deleted file mode 100644 index 67b8c35fd..000000000 --- a/src/backend/src/routers/test.js +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = new express.Router(); - -// -----------------------------------------------------------------------// -// GET /test -// -----------------------------------------------------------------------// -router.get('/test', async (req, res, next) => { - res.send('It\'s working!'); -}); -module.exports = router; diff --git a/src/backend/src/routers/update-taskbar-items.js b/src/backend/src/routers/update-taskbar-items.js deleted file mode 100644 index 4f9736b06..000000000 --- a/src/backend/src/routers/update-taskbar-items.js +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const config = require('../config.js'); -const { invalidate_cached_user } = require('../helpers'); -const router = new express.Router(); -const auth = require('../middleware/auth.js'); -const { DB_WRITE } = require('../services/database/consts.js'); - -// -----------------------------------------------------------------------// -// POST /update-taskbar-items -// -----------------------------------------------------------------------// -router.post('/update-taskbar-items', auth, express.json(), async (req, res, next) => { - // check subdomain - if ( require('../helpers').subdomain(req) !== 'api' ) - { - next(); - } - - // check if user is verified - if ( (config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed ) - { - return res.status(400).send({ code: 'account_is_not_verified', message: 'Account is not verified' }); - } - - // modules - const db = req.services.get('database').get(DB_WRITE, 'ui'); - - // Check if req.body.items is set - if ( ! req.body.items ) - { - return res.status(400).send({ code: 'invalid_request', message: 'items is required.' }); - } - // Check if req.body.items is an array - else if ( ! Array.isArray(req.body.items) ) - { - return res.status(400).send({ code: 'invalid_request', message: 'items must be an array.' }); - } - - // insert into DB - await db.write( - 'UPDATE user SET taskbar_items = ? WHERE user.id = ?', - [ - req.body.items ?? null, - req.user.id, - ], - ); - - invalidate_cached_user(req.user); - - // send results to client - return res.send({}); -}); -module.exports = router; diff --git a/src/backend/src/routers/user-protected/change-email.js b/src/backend/src/routers/user-protected/change-email.js deleted file mode 100644 index f04856cd7..000000000 --- a/src/backend/src/routers/user-protected/change-email.js +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -const eggspress = require('../../api/eggspress'); -const APIError = require('../../api/APIError'); -const { DB_WRITE } = require('../../services/database/consts'); -const jwt = require('jsonwebtoken'); -const validator = require('validator'); -const crypto = require('crypto'); -const config = require('../../config'); -const { Context } = require('../../util/context'); -const { v4: uuidv4 } = require('uuid'); -const { invalidate_cached_user_by_id } = require('../../helpers'); - -module.exports = eggspress('/change-email', { - allowedMethods: ['POST'], -}, async (req, res) => { - const user = req.user; - const new_email = req.body.new_email; - - // TODO: DRY: signup.js - // validation - if ( ! new_email ) { - throw APIError.create('field_missing', null, { key: 'new_email' }); - } - if ( typeof new_email !== 'string' ) { - throw APIError.create('field_invalid', null, { - key: 'new_email', expected: 'a valid email address' }); - } - if ( ! validator.isEmail(new_email) ) { - throw APIError.create('field_invalid', null, { - key: 'new_email', expected: 'a valid email address' }); - } - - const svc_cleanEmail = req.services.get('clean-email'); - const clean_email = svc_cleanEmail.clean(new_email); - - if ( ! await svc_cleanEmail.validate(clean_email) ) { - throw APIError.create('email_not_allowed', undefined, { - email: clean_email, - }); - } - - // check if email is already in use - const db = req.services.get('database').get(DB_WRITE, 'auth'); - const rows = await db.read( - 'SELECT COUNT(*) AS `count` FROM `user` WHERE (`email` = ? OR `clean_email` = ?) AND `email_confirmed` = 1', - [new_email, clean_email], - ); - - // TODO: DRY: signup.js, save_account.js - if ( rows[0].count > 0 ) { - throw APIError.create('email_already_in_use', null, { email: new_email }); - } - - // If user does not have a confirmed email, then update `email` directly - // and send a new confirmation email for their account instead. - if ( ! user.email_confirmed ) { - const email_confirm_token = uuidv4(); - await db.write( - 'UPDATE `user` SET `email` = ?, `email_confirm_token` = ? WHERE `id` = ?', - [new_email, email_confirm_token, user.id], - ); - invalidate_cached_user_by_id(user.id); - - const svc_email = Context.get('services').get('email'); - const link = `${config.origin}/confirm-email-by-token?user_uuid=${user.uuid}&token=${email_confirm_token}`; - svc_email.send_email({ email: new_email }, 'email_verification_link', { link }); - - res.send({ success: true }); - return; - } - - // generate confirmation token - const token = crypto.randomBytes(4).toString('hex'); - const jwt_token = jwt.sign({ - user_id: user.id, - token, - }, config.jwt_secret, { expiresIn: '24h' }); - - // send confirmation email - const svc_email = req.services.get('email'); - await svc_email.send_email({ email: new_email }, 'email_change_request', { - confirm_url: `${config.origin}/change_email/confirm?token=${jwt_token}`, - username: user.username, - }); - const old_email = user.email; - // TODO: NotificationService - await svc_email.send_email({ email: old_email }, 'email_change_notification', { - new_email: new_email, - }); - - // update user - await db.write( - 'UPDATE `user` SET `unconfirmed_change_email` = ?, `change_email_confirm_token` = ? WHERE `id` = ?', - [new_email, token, user.id], - ); - invalidate_cached_user_by_id(user.id); - - // Update email change audit table - await db.write( - 'INSERT INTO `user_update_audit` ' + - '(`user_id`, `user_id_keep`, `old_email`, `new_email`, `reason`) ' + - 'VALUES (?, ?, ?, ?, ?)', - [ - req.user.id, req.user.id, - old_email, new_email, - 'change_username', - ], - ); - - res.send({ success: true }); -}); diff --git a/src/backend/src/routers/user-protected/change-password.js b/src/backend/src/routers/user-protected/change-password.js deleted file mode 100644 index 25ebc1290..000000000 --- a/src/backend/src/routers/user-protected/change-password.js +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -// TODO: DRY: This is the same function used by UIWindowChangePassword! - -const eggspress = require('../../api/eggspress'); -const { invalidate_cached_user } = require('../../helpers'); -const { DB_WRITE } = require('../../services/database/consts'); - -// duplicate definition is in src/helpers.js (puter GUI) -const check_password_strength = (password) => { - // Define criteria for password strength - const criteria = { - minLength: 8, - hasUpperCase: /[A-Z]/.test(password), - hasLowerCase: /[a-z]/.test(password), - hasNumber: /\d/.test(password), - hasSpecialChar: /[!@#$%^&*()_+\-=[\]{};':"\\|,.<>/?]/.test(password), - }; - - let overallPass = true; - - // Initialize report object - let criteria_report = { - minLength: { - message: `Password must be at least ${criteria.minLength} characters long`, - pass: password.length >= criteria.minLength, - }, - hasUpperCase: { - message: 'Password must contain at least one uppercase letter', - pass: criteria.hasUpperCase, - }, - hasLowerCase: { - message: 'Password must contain at least one lowercase letter', - pass: criteria.hasLowerCase, - }, - hasNumber: { - message: 'Password must contain at least one number', - pass: criteria.hasNumber, - }, - hasSpecialChar: { - message: 'Password must contain at least one special character', - pass: criteria.hasSpecialChar, - }, - }; - - // Check overall pass status and add messages - for ( let criterion in criteria ) { - if ( ! criteria_report[criterion].pass ) { - overallPass = false; - break; - } - } - - return { - overallPass: overallPass, - report: criteria_report, - }; -}; - -module.exports = eggspress('/change-password', { - allowedMethods: ['POST'], -}, async (req, res) => { - // Validate new password - const { new_pass } = req.body; - const { overallPass: strong } = check_password_strength(new_pass); - if ( ! strong ) { - req.status(400).send('Password does not meet requirements.'); - } - - // Update user - // TODO: DI for endpoint definitions like this one - const bcrypt = require('bcrypt'); - const db = req.services.get('database').get(DB_WRITE, 'auth'); - await db.write( - 'UPDATE user SET password=?, `pass_recovery_token` = NULL, `change_email_confirm_token` = NULL WHERE `id` = ?', - [await bcrypt.hash(req.body.new_pass, 8), req.user.id], - ); - invalidate_cached_user(req.user); - - // Notify user about password change - // TODO: audit log for user in security tab - const svc_email = req.services.get('email'); - svc_email.send_email({ email: req.user.email }, 'password_change_notification'); - - // Kick out all other sessions - const svc_auth = req.services.get('auth'); - const sessions = await svc_auth.list_sessions(req.actor); - for ( const session of sessions ) { - if ( session.current ) continue; - await svc_auth.revoke_session(req.actor, session.uuid); - } - - return res.send('Password successfully updated.'); -}); diff --git a/src/backend/src/routers/user-protected/change-username.js b/src/backend/src/routers/user-protected/change-username.js deleted file mode 100644 index 55ab57a3f..000000000 --- a/src/backend/src/routers/user-protected/change-username.js +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -const eggspress = require('../../api/eggspress'); -const config = require('../../config'); -const APIError = require('../../api/APIError.js'); -const { DB_WRITE } = require('../../services/database/consts'); -const { username_exists, change_username } = require('../../helpers'); -const { Context } = require('../../util/context'); - -module.exports = eggspress('/change-username', { - allowedMethods: ['POST'], -}, async (req, res, _next) => { - const user = req.user; - const new_username = req.body.new_username; - - if ( ! new_username ) { - throw APIError.create('field_missing', null, { key: 'new_username' }); - } - if ( typeof new_username !== 'string' ) { - throw APIError.create('field_invalid', null, { key: 'new_username', expected: 'a string' }); - } - if ( ! new_username.match(config.username_regex) ) { - throw APIError.create('field_invalid', null, { key: 'new_username', expected: 'letters, numbers, underscore (_)' }); - } - if ( new_username.length > config.username_max_length ) { - throw APIError.create('field_too_long', null, { key: 'new_username', max_length: config.username_max_length }); - } - if ( await username_exists(new_username) ) { - throw APIError.create('username_already_in_use', null, { username: new_username }); - } - - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('/user-protected/change-username') ) { - return res.status(429).send('Too many requests.'); - } - - const db = Context.get('services').get('database').get(DB_WRITE, 'auth'); - const rows = await db.read( - 'SELECT COUNT(*) AS `count` FROM `user_update_audit` ' + - `WHERE \`user_id\`=? AND \`reason\`=? AND ${ - db.case({ - mysql: '`created_at` > DATE_SUB(NOW(), INTERVAL 1 MONTH)', - sqlite: "`created_at` > datetime('now', '-1 month')", - })}`, - [user.id, 'change_username'], - ); - - if ( rows[0].count >= (config.max_username_changes ?? 2) ) { - throw APIError.create('too_many_username_changes'); - } - - await db.write( - 'INSERT INTO `user_update_audit` ' + - '(`user_id`, `user_id_keep`, `old_username`, `new_username`, `reason`) ' + - 'VALUES (?, ?, ?, ?, ?)', - [user.id, user.id, user.username, new_username, 'change_username'], - ); - - await change_username(user.id, new_username); - - res.json({}); -}); diff --git a/src/backend/src/routers/user-protected/delete-own-user.js b/src/backend/src/routers/user-protected/delete-own-user.js deleted file mode 100644 index 9b3e483d4..000000000 --- a/src/backend/src/routers/user-protected/delete-own-user.js +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2026-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -const eggspress = require('../../api/eggspress'); -const config = require('../../config'); -const { deleteUser, invalidate_cached_user } = require('../../helpers'); - -const REVALIDATION_COOKIE_NAME = 'puter_revalidation'; - -module.exports = eggspress('/delete-own-user', { - allowedMethods: ['POST'], -}, async (req, res) => { - res.clearCookie(config.cookie_name); - res.clearCookie(REVALIDATION_COOKIE_NAME); - - await deleteUser(req.user.id); - invalidate_cached_user(req.user); - - return res.send({ success: true }); -}); diff --git a/src/backend/src/routers/user-protected/disable-2fa.js b/src/backend/src/routers/user-protected/disable-2fa.js deleted file mode 100644 index dbcd83319..000000000 --- a/src/backend/src/routers/user-protected/disable-2fa.js +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ -const eggspress = require('../../api/eggspress'); -const { DB_WRITE } = require('../../services/database/consts'); -const { invalidate_cached_user_by_id } = require('../../helpers'); - -module.exports = eggspress('/disable-2fa', { - allowedMethods: ['POST'], -}, async (req, res) => { - const db = req.services.get('database').get(DB_WRITE, '2fa.disable'); - await db.write( - 'UPDATE user SET otp_enabled = 0, otp_recovery_codes = NULL, otp_secret = NULL WHERE uuid = ?', - [req.user.uuid], - ); - // update cached user - req.user.otp_enabled = 0; - invalidate_cached_user_by_id(req.user.id); - - const svc_email = req.services.get('email'); - await svc_email.send_email({ email: req.user.email }, 'disabled_2fa', { - username: req.user.username, - }); - - res.send({ success: true }); -}); diff --git a/src/backend/src/routers/verify-pass-recovery-token.js b/src/backend/src/routers/verify-pass-recovery-token.js deleted file mode 100644 index 1ad19d363..000000000 --- a/src/backend/src/routers/verify-pass-recovery-token.js +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const express = require('express'); -const router = new express.Router(); -const config = require('../config'); -const { get_user } = require('../helpers'); - -const jwt = require('jsonwebtoken'); - -// Ensure we don't expose branches with differing messages. -const SAFE_NEGATIVE_RESPONSE = 'This password recovery token is no longer valid.'; - -// -----------------------------------------------------------------------// -// POST /verify-pass-recovery-token -// -----------------------------------------------------------------------// -router.post('/verify-pass-recovery-token', express.json(), async (req, res, next) => { - // check subdomain - if ( require('../helpers').subdomain(req) !== 'api' && require('../helpers').subdomain(req) !== '' ) - { - next(); - } - - if ( ! req.body.token ) { - return res.status(401).send('token is required'); - } - - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('verify-pass-recovery-token') ) { - return res.status(429).send('Too many requests.'); - } - - const { exp, user_uid, email } = jwt.verify(req.body.token, config.jwt_secret); - - const user = await get_user({ uuid: user_uid, force: true }); - if ( user.email !== email ) { - return res.status(400).send(SAFE_NEGATIVE_RESPONSE); - } - - const current_time = Math.floor(Date.now() / 1000); - const time_remaining = exp - current_time; - - return res.status(200).send({ time_remaining }); -}); - -module.exports = router; diff --git a/src/backend/src/routers/version.js b/src/backend/src/routers/version.js deleted file mode 100644 index 4f69764f3..000000000 --- a/src/backend/src/routers/version.js +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const eggspress = require('../api/eggspress'); - -module.exports = eggspress(['/version'], { - allowedMethods: ['GET'], - subdomain: 'api', - json: true, -}, async (req, res, next) => { - const svc_puterVersion = req.services.get('puter-version'); - - const response = svc_puterVersion.get_version(); - - // Add user-friendly version information - { - response.version_text = response.version; - const components = response.version.split('-'); - if ( components.length > 1 ) { - response.release_type = components[1]; - if ( components[1] === 'rc' ) { - response.version_text = - `${components[0]} (Release Candidate ${components[2]})`; - } - else if ( components[1] === 'dev' ) { - response.version_text = - `${components[0]} (Development Build)`; - } - else if ( components[1] === 'beta' ) { - response.version_text = - `${components[0]} (Beta Release)`; - } - else if ( ! isNaN(components[1]) ) { - response.version_text = `${components[0]} (Build ${components[1]})`; - response.sub_version = components[1]; - response.hash = components[2]; - response.release_type = 'build'; - } - if ( isNaN(components[1]) && components.length > 2 ) { - response.sub_version = components[2]; - } - } - } - - res.send(response); -}); diff --git a/src/backend/src/routers/writeFile.js b/src/backend/src/routers/writeFile.js deleted file mode 100644 index 7cad45d5a..000000000 --- a/src/backend/src/routers/writeFile.js +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -const { uuid2fsentry, validate_signature_auth, get_url_from_req, get_user } = require('../helpers'); -const eggspress = require('../api/eggspress'); -const { Context } = require('../util/context'); -const { Actor } = require('../services/auth/Actor'); -const FSNodeParam = require('../api/filesystem/FSNodeParam'); - -// TODO: eggspressify - -// -----------------------------------------------------------------------// -// POST /writeFile -// -----------------------------------------------------------------------// -module.exports = eggspress('/writeFile', { - files: ['file'], - allowedMethods: ['POST'], -}, async (req, res, next) => { - // check subdomain - if ( require('../helpers').subdomain(req) !== 'api' ) - { - next(); - } - - const log = req.services.get('log-service').create('writeFile'); - const errors = req.services.get('error-service').create(log); - - // validate URL signature - try { - validate_signature_auth(get_url_from_req(req), 'write'); - } - catch (e) { - return res.status(403).send(e); - } - - // Get fsentry - // todo this is done again in the following section, super inefficient - let requested_item = await uuid2fsentry(req.query.uid); - - if ( ! requested_item ) { - return res.status(404).send({ error: 'Item not found' }); - } - - // check if requested_item owner is suspended - const owner_user = await require('../helpers').get_user({ id: requested_item.user_id }); - - if ( ! owner_user ) { - errors.report('writeFile_no_owner', { - message: `User not found: ${requested_item.user_id}`, - trace: true, - alarm: true, - extra: { - requested_item, - body: req.body, - query: req.query, - }, - }); - - return res.status(500).send({ error: 'User not found' }); - } - - if ( owner_user.suspended ) - { - return res.status(401).send({ error: 'Account suspended' }); - } - - const writeFile_handler_api = { - async get_dest_node () { - if ( ! req.body.destination_write_url ) { - res.status(400).send({ - error: { - message: 'No destination specified.', - }, - }); - return; - } - try { - validate_signature_auth(req.body.destination_write_url, 'write', { - uid: req.body.destination_uid, - }); - } catch (e) { - res.status(403).send(e); - return; - } - try { - return await (new FSNodeParam('dest_path')).consolidate({ - req, getParam: () => req.body.dest_path ?? req.body.destination_uid, - }); - } catch (e) { - res.status(500).send('Internal Server Error'); - } - }, - }; - - const writeFile_handlers = require('./writeFile/writeFile_handlers.js'); - - let operation = req.query.operation ?? 'write'; - // Responding with an error here would typically be better, - // but it would cause a regression for apps. - if ( ! writeFile_handlers.hasOwnProperty(operation) ) { - operation = 'write'; - } - - console.log(`\x1B[36;1mwriteFile: ${ req.query.operation }\x1B[0m`); - const node = await (new FSNodeParam('uid')).consolidate({ - req, getParam: () => req.query.uid, - }); - const user = await get_user({ id: await node.get('user_id') }); - const actor = Actor.adapt(user); - - return await Context.get().sub({ - actor: Actor.adapt(user), user, - }).arun(async () => { - return await writeFile_handlers[operation]({ - api: writeFile_handler_api, - req, - res, - actor, - node, - }); - }); -}); diff --git a/src/backend/src/routers/writeFile/copy.js b/src/backend/src/routers/writeFile/copy.js deleted file mode 100644 index 564813f07..000000000 --- a/src/backend/src/routers/writeFile/copy.js +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { HLCopy } = require('../../deprecated/filesystem/hl_operations/hl_copy'); - -module.exports = async function writeFile_handle_copy ({ - api, - req, res, actor, node, -}) { - - // check if destination_write_url provided - - // check if destination_write_url is valid - const dest_node = await api.get_dest_node(); - if ( ! dest_node ) return; - - const overwrite = req.body.overwrite ?? false; - const change_name = req.body.auto_rename ?? false; - - const opts = { - source: node, - destination_or_parent: dest_node, - dedupe_name: change_name, - overwrite, - user: actor.type.user, - }; - - const hl_copy = new HLCopy(); - - const r = await hl_copy.run({ - ...opts, - actor, - }); - return res.send([r]); -}; diff --git a/src/backend/src/routers/writeFile/delete.js b/src/backend/src/routers/writeFile/delete.js deleted file mode 100644 index 8874204f5..000000000 --- a/src/backend/src/routers/writeFile/delete.js +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { HLRemove } = require('../../deprecated/filesystem/hl_operations/hl_remove'); - -module.exports = async function writeFile_handle_delete ({ - req, res, actor, node, -}) { - // Delete - const hl_remove = new HLRemove(); - await hl_remove.run({ - target: node, - user: actor.type.user, - actor, - }); - - // Send success msg - return res.send(); -}; diff --git a/src/backend/src/routers/writeFile/mkdir.js b/src/backend/src/routers/writeFile/mkdir.js deleted file mode 100644 index fbac2d9ba..000000000 --- a/src/backend/src/routers/writeFile/mkdir.js +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { HLMkdir } = require('../../deprecated/filesystem/hl_operations/hl_mkdir'); -const { NodeUIDSelector } = require('../../deprecated/filesystem/node/selectors'); -const { sign_file } = require('../../helpers'); - -module.exports = async function writeFile_handle_mkdir ({ - req, res, actor, node, -}) { - if ( ! req.body.name ) { - return res.status(400).send({ - error: { - message: 'Name is required.', - }, - }); - } - - const hl_mkdir = new HLMkdir(); - const r = await hl_mkdir.run({ - parent: node, - path: req.body.name, - overwrite: false, - dedupe_name: req.body.dedupe_name ?? false, - user: actor.type.user, - actor, - }); - - const svc_fs = req.services.get('filesystem'); - - const newdir_node = await svc_fs.node(new NodeUIDSelector(r.uid)); - return res.send(await sign_file(await newdir_node.get('entry'), 'write')); -}; diff --git a/src/backend/src/routers/writeFile/move.js b/src/backend/src/routers/writeFile/move.js deleted file mode 100644 index 5bee66e60..000000000 --- a/src/backend/src/routers/writeFile/move.js +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { HLMove } = require('../../deprecated/filesystem/hl_operations/hl_move'); - -module.exports = async function writeFile_handle_move ({ - api, - req, res, actor, node, -}) { - // check if destination_write_url provided - if ( ! req.body.destination_write_url ) { - return res.status(400).send({ - error: { - message: 'No destination specified.', - }, - }); - } - - const dest_node = await api.get_dest_node(); - if ( ! dest_node ) return; - - const hl_move = new HLMove(); - - const opts = { - user: actor.type.user, - source: node, - destination_or_parent: dest_node, - overwrite: req.body.overwrite ?? false, - new_name: req.body.new_name, - new_metadata: req.body.new_metadata, - create_missing_parents: req.body.create_missing_parents, - }; - - const r = await hl_move.run({ - ...opts, - actor, - }); - - return res.send({ - ...r.moved, - old_path: r.old_path, - new_path: r.moved.path, - }); -}; diff --git a/src/backend/src/routers/writeFile/rename.js b/src/backend/src/routers/writeFile/rename.js deleted file mode 100644 index babc1f9bd..000000000 --- a/src/backend/src/routers/writeFile/rename.js +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const mime = require('mime-types'); -const { validate_fsentry_name } = require('../../helpers'); -const { DB_WRITE } = require('../../services/database/consts'); - -module.exports = async function writeFile_handle_rename ({ - req, res, node, -}) { - const new_name = req.body.new_name; - - try { - validate_fsentry_name(new_name); - } catch (e) { - return res.status(400).send({ - error: { - message: e.message, - }, - }); - } - - if ( await node.get('immutable') ) { - return res.status(400).send({ - error: { - message: 'Immutable: cannot rename.', - }, - }); - } - - if ( await node.isUserDirectory() || await node.isRoot ) { - return res.status(403).send({ - error: { - message: 'Not allowed to rename this item via writeFile.', - }, - }); - } - - const old_path = await node.get('path'); - - const db = req.services.get('database').get(DB_WRITE, 'writeFile:rename'); - const mysql_id = await node.get('mysql-id'); - await db.write('UPDATE fsentries SET name = ? WHERE id = ?', - [new_name, mysql_id]); - - const contentType = mime.contentType(req.body.new_name); - const return_obj = { - ...await node.getSafeEntry(), - old_path, - type: contentType ? contentType : null, - original_client_socket_id: req.body.original_client_socket_id, - }; - - return res.send(return_obj); -}; diff --git a/src/backend/src/routers/writeFile/trash.js b/src/backend/src/routers/writeFile/trash.js deleted file mode 100644 index d64d8d9cd..000000000 --- a/src/backend/src/routers/writeFile/trash.js +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { HLMove } = require('../../deprecated/filesystem/hl_operations/hl_move'); -const { NodePathSelector } = require('../../deprecated/filesystem/node/selectors'); - -module.exports = async function writeFile_handle_trash ({ - req, res, actor, node, -}) { - // metadata for trashed file - const new_name = await node.get('uid'); - const metadata = { - original_name: await node.get('name'), - original_path: await node.get('path'), - trashed_ts: Math.round(Date.now() / 1000), - }; - - // Get Trash fsentry - const fs = req.services.get('filesystem'); - const trash = await fs.node(new NodePathSelector(`/${ actor.type.user.username }/Trash`)); - - // No Trash? - if ( ! trash ) { - return res.status(400).send({ - error: { - message: 'No Trash directory found.', - }, - }); - } - - const hl_move = new HLMove(); - await hl_move.run({ - source: node, - destination_or_parent: trash, - user: actor.type.user, - actor, - new_name: new_name, - new_metadata: metadata, - }); - - return res.status(200).send({ - message: 'Item trashed', - }); -}; diff --git a/src/backend/src/routers/writeFile/write.js b/src/backend/src/routers/writeFile/write.js deleted file mode 100644 index a5713c779..000000000 --- a/src/backend/src/routers/writeFile/write.js +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { TYPE_DIRECTORY } = require('../../deprecated/filesystem/FSNodeContext'); -const { HLWrite } = require('../../deprecated/filesystem/hl_operations/hl_write'); -const { NodePathSelector } = require('../../deprecated/filesystem/node/selectors'); -const _path = require('path'); -const { sign_file } = require('../../helpers'); - -module.exports = async function writeFile_handle_write ({ - req, res, actor, node, -}) { - - // Check if files were uploaded - if ( ! req.files ) { - return res.status(400).send('No files uploaded'); - } - - // Get fsentry - let dirname; - - try { - dirname = (await node.get('type') !== TYPE_DIRECTORY - ? _path.dirname.bind(_path) : a => a)(await node.get('path')); - } catch (e) { - console.log(e); - req.__error_source = e; - return res.status(500).send(e); - } - - const svc_fs = req.services.get('filesystem'); - const dirNode = await svc_fs.node(new NodePathSelector(dirname)); - - // Upload files one by one - const returns = []; - for ( const uploaded_file of req.files ) { - try { - const normalized_file = { ...uploaded_file }; - - if ( normalized_file.mimetype && !normalized_file.type ) { - normalized_file.type = normalized_file.mimetype; - } - - if ( normalized_file.buffer ) { - normalized_file.size = normalized_file.buffer.length; - } - - const hl_write = new HLWrite(); - const ret_obj = await hl_write.run({ - destination_or_parent: dirNode, - specified_name: await node.get('type') === TYPE_DIRECTORY - ? req.body.name : await node.get('name'), - fallback_name: normalized_file.originalname, - overwrite: true, - user: actor.type.user, - actor, - - file: normalized_file, - }); - - // add signature to object - ret_obj.signature = await sign_file(ret_obj, 'write'); - - // send results back to app - returns.push(ret_obj); - } catch ( error ) { - req.__error_source = error; - console.log(error); - return res.contentType('application/json').status(500).send(error); - } - } - - if ( returns.length === 1 ) { - return res.send(returns[0]); - } - - return res.send(returns); -}; diff --git a/src/backend/src/routers/writeFile/writeFile_handlers.js b/src/backend/src/routers/writeFile/writeFile_handlers.js deleted file mode 100644 index 24f9483aa..000000000 --- a/src/backend/src/routers/writeFile/writeFile_handlers.js +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -module.exports = { - move: require('./move'), - copy: require('./copy'), - mkdir: require('./mkdir'), - trash: require('./trash'), - delete: require('./delete'), - rename: require('./rename'), - write: require('./write'), -}; diff --git a/src/backend/src/server b/src/backend/src/server deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/backend/src/services/BaseService.d.ts b/src/backend/src/services/BaseService.d.ts deleted file mode 100644 index e1f0a6490..000000000 --- a/src/backend/src/services/BaseService.d.ts +++ /dev/null @@ -1,104 +0,0 @@ -import type { ErrorService } from '@heyputer/backend/src/modules/core/ErrorService'; -import type { DriverService } from '@heyputer/backend/src/services/drivers/DriverService'; -import type { DynamoKVStore } from '@heyputer/backend/src/services/DynamoKVStore/DynamoKVStore'; -import type { DDBClient } from '../clients/dynamodb/DDBClient'; -import type { ServerHealthService } from '../modules/core/ServerHealthService/ServerHealthService'; -import type { WebServerService } from '../modules/web/WebServerService'; -import type { GroupService } from './auth/GroupService'; -import type { SignupService } from './auth/SignupService'; -import type { CleanEmailService } from './CleanEmailService'; -import type { SqliteDatabaseAccessService } from './database/SqliteDatabaseAccessService'; -import type { IDynamoKVStoreWrapper } from './DynamoKVStore/DynamoKVStoreWrapper'; -import type { Emailservice } from './EmailService'; -import type { EntityStoreService } from './EntityStoreService'; -import type { EventService } from './EventService'; -import type { FeatureFlagService } from './FeatureFlagService'; -import type { GetUserService } from './GetUserService'; -import type { MeteringService } from './MeteringService/MeteringService'; -import type { MeteringServiceWrapper } from './MeteringService/MeteringServiceWrapper.mjs'; -import type { SUService } from './SUService'; -import type { UserService } from './UserService'; -import type { TokenService } from './auth/TokenService'; -import type { SessionService } from './SessionService'; -import type { PermissionService } from './auth/PermissionService'; -import { MountpointService } from '../modules/puterfs/MountpointService'; - -export interface ServicesMap { - su: SUService; - user: UserService; - 'get-user': GetUserService; - 'web-server': WebServerService; - email: Emailservice; - 'es:app': EntityStoreService; - meteringService: MeteringService & MeteringServiceWrapper; - 'puter-kvstore': DynamoKVStore & IDynamoKVStoreWrapper; - database: SqliteDatabaseAccessService; - 'server-health': ServerHealthService; - su: SUService; - dynamo: DDBClient; - user: UserService; - event: EventService; - signup: SignupService; - group: GroupService; - 'feature-flag': FeatureFlagService; - 'clean-email': CleanEmailService; - 'error-service': ErrorService; - driver: DriverService; - 'token': TokenService; - 'session': SessionService; - 'permission': PermissionService; - 'mountpoint': MountpointService; -} - -export interface ServiceResources { - services: { - get( - name: T - ): T extends `${infer R extends keyof ServicesMap}` - ? ServicesMap[R] - : unknown; - }; - config: Record & { services?: Record; server_id?: string }; - name?: string; - args?: any; - context: { get (key: string): any }; -} - -export type EventHandler = (id: string, ...args: any[]) => any; - -export interface Logger { - debug: (...args: any[]) => any; - info: (...args: any[]) => any; - [key: string]: any; -} - -export class BaseService { - constructor (service_resources: ServiceResources, ...a: any[]); - - args: any; - service_name: string; - services: ServiceResources['services']; - config: Record; - global_config: ServiceResources['config']; - context: ServiceResources['context']; - log: Logger; - errors: any; - - as (interfaceName: string): Record; - - run_as_early_as_possible (): Promise; - construct (): Promise; - init (): Promise; - __on (id: string, args: any[]): Promise; - protected __get_event_handler (id: string): EventHandler; - - protected _run_as_early_as_possible? (args?: any): any; - protected _construct? (args?: any): any; - protected _init? (args?: any): any; - protected _get_merged_static_object? (key: string): Record; - - static LOG_DEBUG?: boolean; - static CONCERN?: string; -} - -export default BaseService; diff --git a/src/backend/src/services/BaseService.js b/src/backend/src/services/BaseService.js deleted file mode 100644 index 2fda6ee7e..000000000 --- a/src/backend/src/services/BaseService.js +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { concepts } = require('@heyputer/putility'); - -// This is a no-op function that AI is incapable of writing a comment for. -// That said, I suppose it didn't need one anyway. -const NOOP = async () => { -}; - -/** -* @class BaseService -* @extends concepts.Service -* @description -* BaseService is the foundational class for all services in the Puter backend. -* It provides lifecycle methods like `construct` and `init` that are invoked during -* different phases of the boot sequence. This class ensures that services can be -* instantiated, initialized, and activated in a coordinated manner through -* events emitted by the Kernel. It also manages common service resources like -* logging and error handling, and supports legacy services by allowing -* instantiation after initialization but before consolidation. -*/ -class BaseService extends concepts.Service { - constructor (service_resources, ...a) { - const { services, config, name, args, context } = service_resources; - super(service_resources, ...a); - - this.args = args; - this.service_name = name || this.constructor.name; - this.services = services; - let configOverride = undefined; - Object.defineProperty(this, 'config', { - get: () => configOverride ?? config.services?.[name] ?? {}, - set: why => { - // TODO: uncomment and fix these in legacy services - // (not very important; low priority) - // console.warn('replacing config like this is probably a bad idea'); - configOverride = why; - }, - }); - this.global_config = config; - this.context = context; - - if ( this.global_config.server_id === '' ) { - this.global_config.server_id = 'local'; - } - } - - async run_as_early_as_possible () { - await (this._run_as_early_as_possible || NOOP).call(this, this.args); - } - - /** - * Creates the service's data structures and initial values. - * This method sets up logging and error handling, and calls a custom `_construct` method if defined. - * - * @returns {Promise} A promise that resolves when construction is complete. - */ - async construct () { - const useapi = this.context.get('useapi'); - const use = this._get_merged_static_object('USE'); - for ( const [key, value] of Object.entries(use) ) { - this[key] = useapi.use(value); - } - await (this._construct || NOOP).call(this, this.args); - } - - /** - * Performs the initialization phase of the service lifecycle. - * This method sets up logging and error handling for the service, - * then calls the service-specific initialization logic if defined. - * - * @async - * @memberof BaseService - * @instance - * @returns {Promise} A promise that resolves when initialization is complete. - */ - async init () { - const services = this.services; - const log_fields = {}; - if ( this.constructor.CONCERN ) { - log_fields.concern = this.constructor.CONCERN; - } - this.log = services.get('log-service').create(this.service_name, log_fields); - - // INFO logs are treated as DEBUG logs instead if... - if ( - // The configuration file explicitly says to do so - this.config.log_debug || - // The class has `static LOG_DEBUG = true`; AND, - // the configuration file does NOT explicitly say NOT to do this - (!this.config.log_info && this.constructor.LOG_DEBUG) - ) { - this.log.info = this.log.debug; - } - this.errors = services.get('error-service').create(this.log); - - await (this._init || NOOP).call(this, this.args); - } - - /** - * Handles an event by retrieving the appropriate event handler - * and executing it with the provided arguments. - * - * @param {string} id - The identifier of the event to handle. - * @param {Array} args - The arguments to pass to the event handler. - * @returns {Promise} The result of the event handler execution. - */ - async __on (id, args) { - const handler = this.__get_event_handler(id); - - return await handler(id, ...args); - } - - __get_event_handler (id) { - return this[`__on_${id}`]?.bind?.(this) - || this.constructor[`__on_${id}`]?.bind?.(this.constructor) - || NOOP; - } -} - -module.exports = BaseService; -module.exports.BaseService = BaseService; diff --git a/src/backend/src/services/BootScriptService.js b/src/backend/src/services/BootScriptService.js deleted file mode 100644 index 77cd9a96b..000000000 --- a/src/backend/src/services/BootScriptService.js +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { Context } = require('../util/context'); -const BaseService = require('./BaseService'); - -/** -* @class BootScriptService -* @extends BaseService -* @description The BootScriptService class extends BaseService and is responsible for -* managing and executing boot scripts. It provides methods to handle boot scripts when -* the system is ready and to run individual script commands. -*/ -class BootScriptService extends BaseService { - static MODULES = { - fs: require('fs'), - }; - /** - * Loads and executes a boot script if specified in the arguments. - * - * This method reads the provided boot script file, parses it, and runs the script using the `run_script` method. - * If no boot script is specified in the arguments, the method returns immediately. - * - * @async - * @function - * @returns {Promise} - */ - async '__on_boot.ready' () { - const args = Context.get('args'); - if ( ! args['boot-script'] ) return; - const script_name = args['boot-script']; - - const require = this.require; - const fs = require('fs'); - const boot_json_raw = fs.readFileSync(script_name, 'utf8'); - const boot_json = JSON.parse(boot_json_raw); - await this.run_script(boot_json); - } - - /** - * Executes a series of commands defined in a JSON boot script. - * - * This method processes each command in the boot_json array. - * If the command is recognized within the predefined scope, it will be executed. - * If not, an error is thrown. - * - * @param {Array} boot_json - An array of commands to execute. - * @throws {Error} Thrown if an unknown command is encountered. - */ - async run_script (boot_json) { - const scope = { - runner: 'boot-script', - 'end-puter-process': ({ args }) => { - console.log('shutting down puter: BootScriptService'); - process.exit(0); - }, - }; - - for ( const statement of boot_json ) { - const [cmd, ...args] = statement; - if ( ! scope[cmd] ) { - throw new Error(`Unknown command: ${cmd}`); - } - await scope[cmd]({ scope, args }); - } - } -} - -module.exports = { - BootScriptService, -}; diff --git a/src/backend/src/services/ChatAPIService.js b/src/backend/src/services/ChatAPIService.js deleted file mode 100644 index 5f3172066..000000000 --- a/src/backend/src/services/ChatAPIService.js +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const eggspress = require('../api/eggspress'); -const BaseService = require('./BaseService'); -const APIError = require('../api/APIError'); - -/** -* @class ChatAPIService -* @extends BaseService -* @description Service class that handles public (unauthenticated) API endpoints for AI chat functionality. -* This service provides endpoints for retrieving available AI chat models without requiring authentication. -*/ -class ChatAPIService extends BaseService { - static MODULES = { - express: require('express'), - }; - - /** - * Installs routes for chat API endpoints into the Express app - * @param {Object} _ Unused parameter - * @param {Object} options Installation options - * @param {Express} options.app Express application instance to install routes on - * @returns {Promise} - */ - async '__on_install.routes' (_, { app }) { - // Create a router for chat API endpoints - const router = (() => { - const require = this.require; - const express = require('express'); - return express.Router(); - })(); - - // Register the router with the Express app - app.use('/puterai', router); - - // Install endpoints - this.install_chat_endpoints_({ router }); - } - - /** - * Installs chat API endpoints on the provided router - * @param {Object} options Options object - * @param {express.Router} options.router Express router to install endpoints on - * @private - */ - install_chat_endpoints_ ({ router }) { - router.use(require('../routers/puterai/openai/completions')); - router.use(require('../routers/puterai/openai/chat_completions')); - router.use(require('../routers/puterai/openai/responses')); - router.use(require('../routers/puterai/anthropic/messages')); - router.use(require('../routers/puterai/video/proxy')); - // Endpoint to list available AI chat models - router.use(eggspress('/chat/models', { - allowedMethods: ['GET'], - }, async (req, res) => { - try { - // Use SUService to access AIChatService as system user - const svc_su = this.services.get('su'); - const models = await svc_su.sudo(async () => { - const svc_aiChat = this.services.get('ai-chat'); - // Return the simple model list which contains basic model information - return svc_aiChat.list(); - }); - - // Return the list of models - res.json({ models: models.filter(e => !['costly', 'fake', 'abuse', 'model-fallback-test-1'].includes(e)) }); - } catch ( error ) { - this.log.error('Error fetching models:', error); - throw APIError.create('internal_server_error'); - } - })); - - // Endpoint to get detailed information about available AI chat models - router.use(eggspress('/chat/models/details', { - allowedMethods: ['GET'], - }, async (req, res) => { - try { - // Use SUService to access AIChatService as system user - const svc_su = this.services.get('su'); - const models = await svc_su.sudo(async () => { - const svc_aiChat = this.services.get('ai-chat'); - // Return the detailed model list which includes cost and capability information - return svc_aiChat.models(); - }); - - // Return the detailed list of models - res.json({ models: models.filter(e => !['costly', 'fake', 'abuse', 'model-fallback-test-1'].includes(e.id)) }); - } catch ( error ) { - this.log.error('Error fetching model details:', error); - throw APIError.create('internal_server_error'); - } - })); - - router.use(eggspress('/image/models', { - allowedMethods: ['GET'], - }, async (req, res) => { - try { - // Use SUService to access AIImageGenerationService as system user - const svc_su = this.services.get('su'); - const models = await svc_su.sudo(async () => { - const svc_imageGen = this.services.get('ai-image'); - // Return the simple model list which contains basic model information - return svc_imageGen.list(); - }); - // Return the list of models - res.json({ models }); - } catch ( error ) { - this.log.error('Error fetching image models:', error); - throw APIError.create('internal_server_error'); - } - })); - - router.use(eggspress('/image/models/details', { - allowedMethods: ['GET'], - }, async (req, res) => { - try { - // Use SUService to access AIImageGenerationService as system user - const svc_su = this.services.get('su'); - const models = await svc_su.sudo(async () => { - const svc_imageGen = this.services.get('ai-image'); - // Return the detailed model list which includes cost and capability information - return svc_imageGen.models(); - }); - // Return the detailed list of models - res.json({ models }); - } catch ( error ) { - this.log.error('Error fetching image model details:', error); - throw APIError.create('internal_server_error'); - } - })); - - router.use(eggspress('/video/models/details', { - allowedMethods: ['GET'], - }, async (req, res) => { - try { - const svc_su = this.services.get('su'); - const models = await svc_su.sudo(async () => { - const svc_video = this.services.get('ai-video'); - return svc_video.models(); - }); - res.json({ models }); - } catch ( error ) { - this.log.error('Error fetching video model details:', error); - throw APIError.create('internal_server_error'); - } - })); - - router.use(eggspress('/video/models', { - allowedMethods: ['GET'], - }, async (req, res) => { - try { - const svc_su = this.services.get('su'); - const models = await svc_su.sudo(async () => { - const svc_video = this.services.get('ai-video'); - return svc_video.list(); - }); - res.json({ models }); - } catch ( error ) { - this.log.error('Error fetching video models:', error); - throw APIError.create('internal_server_error'); - } - })); - } -} - -module.exports = { - ChatAPIService, -}; diff --git a/src/backend/src/services/ChatAPIService.test.js b/src/backend/src/services/ChatAPIService.test.js deleted file mode 100644 index 2c71c2de7..000000000 --- a/src/backend/src/services/ChatAPIService.test.js +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -/* - IMPORTANT NOTE ABOUT THIS UNIT TEST IN PARTICULAR - - This was generated by AI, and I just wanted to see if I could get this - test working properly. It took me about a half hour, and then I got it - working using the DI mechanism provided by NodeModuleDIFeature.js. - - So this DI mechanism works, and the test written by AI would have worked - perfectly on the first try if the AI knew about this DI mechanism. - - That said, DO NOT REFERENCE THIS FILE FOR TEST CONVENTIONS. - - Also, DO NOT SPEND MORE THAN AN HOUR MAINTAINING THIS. If you are - approaching an hour of maintanence effort, JUST DELETE THIS TEST; - it was written by AI, and fixed up as an experiment - it's not important. -*/ - -import { describe, it, expect, beforeEach, vi } from 'vitest'; -import { Context } from '../util/context.js'; -const { ChatAPIService } = require('./ChatAPIService'); - -describe('ChatAPIService', () => { - let chatApiService; - let mockServices; - let mockRouter; - let mockSUService; - let mockAIChatService; - let mockWebServer; - let mockReq; - let mockRes; - let currentContext; - - beforeEach(() => { - // Mock AIChatService - mockAIChatService = { - list: () => ['model1', 'model2'], - models: () => [ - { id: 'model1', name: 'Model 1', cost: { input: 1, output: 2 } }, - { id: 'model2', name: 'Model 2', cost: { input: 3, output: 4 } }, - ], - }; - - // Mock SUService - mockSUService = { - sudo: vi.fn().mockImplementation(async (callback) => { - if ( typeof callback === 'function' ) { - return await callback(); - } - return await mockSUService.sudo.mockImplementation(async (cb) => await cb()); - }), - }; - - // Mock web server - mockWebServer = { - allow_undefined_origin: vi.fn(), - }; - - // Mock services - mockServices = { - get: vi.fn().mockImplementation((serviceName) => { - if ( serviceName === 'su' ) return mockSUService; - if ( serviceName === 'ai-chat' ) return mockAIChatService; - if ( serviceName === 'web-server' ) return mockWebServer; - return null; - }), - }; - - // Mock router and app - mockRouter = { - use: vi.fn(), - get: vi.fn(), - post: vi.fn(), - }; - // Mock request and response - mockReq = {}; - mockRes = { - json: vi.fn(), - locals: {}, - }; - - // Setup ChatAPIService - chatApiService = new ChatAPIService({ - global_config: {}, - config: {}, - }); - chatApiService.services = mockServices; - chatApiService.log = { - error: vi.fn(), - }; - - Context.root.set('services', mockServices); - currentContext = Context.get(undefined, { allow_fallback: true }); - mockRes.locals.ctx = currentContext; - - // Mock the require function - const oldInstanceRequire_ = chatApiService.require; - chatApiService.require = vi.fn().mockImplementation((module) => { - if ( module === 'express' ) return { Router: () => mockRouter }; - return oldInstanceRequire_.call(chatApiService, module); - }); - }); - - const getMountedRouteLayer = (router, path) => { - const mountedRouters = router.use.mock.calls.map(([mounted]) => mounted); - const mountedRouter = mountedRouters.find(candidate => - candidate?.stack?.some(layer => layer.route?.path === path)); - expect(mountedRouter).toBeTruthy(); - return mountedRouter.stack.find(layer => layer.route?.path === path); - }; - - describe('install_chat_endpoints_', () => { - it('should attach models endpoint to router', () => { - chatApiService.install_chat_endpoints_({ router: mockRouter }); - - expect(getMountedRouteLayer(mockRouter, '/chat/models').route.methods.get).toBe(true); - expect(getMountedRouteLayer(mockRouter, '/image/models').route.methods.get).toBe(true); - }); - - it('should attach models/details endpoint to router', () => { - chatApiService.install_chat_endpoints_({ router: mockRouter }); - - expect(getMountedRouteLayer(mockRouter, '/chat/models/details').route.methods.get).toBe(true); - expect(getMountedRouteLayer(mockRouter, '/image/models/details').route.methods.get).toBe(true); - }); - }); - - describe('/models endpoint', () => { - it('should return list of models', async () => { - chatApiService.install_chat_endpoints_({ router: mockRouter }); - - const layer = getMountedRouteLayer(mockRouter, '/chat/models'); - const handler = layer.route.stack.at(-1).handle; - - await currentContext.arun(async () => { - await handler(mockReq, mockRes, vi.fn()); - }); - - expect(mockSUService.sudo).toHaveBeenCalled(); - expect(mockRes.json).toHaveBeenCalledWith({ - models: mockAIChatService.list(), - }); - }); - }); - - describe('/models/details endpoint', () => { - it('should return detailed list of models', async () => { - chatApiService.install_chat_endpoints_({ router: mockRouter }); - - const layer = getMountedRouteLayer(mockRouter, '/chat/models/details'); - const handler = layer.route.stack.at(-1).handle; - - await currentContext.arun(async () => { - await handler(mockReq, mockRes, vi.fn()); - }); - - expect(mockSUService.sudo).toHaveBeenCalled(); - expect(mockRes.json).toHaveBeenCalledWith({ - models: mockAIChatService.models(), - }); - }); - }); -}); diff --git a/src/backend/src/services/CleanEmailService.js b/src/backend/src/services/CleanEmailService.js deleted file mode 100644 index ebd2f2b09..000000000 --- a/src/backend/src/services/CleanEmailService.js +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const BaseService = require('./BaseService'); - -/** -* CleanEmailService - A service class for cleaning and validating email addresses -* Handles email normalization by applying provider-specific rules (e.g. Gmail's dot-insensitivity), -* manages subaddressing (plus addressing), and validates against blocked domains. -* Extends BaseService to integrate with the application's service infrastructure. -* @extends BaseService -*/ -class CleanEmailService extends BaseService { - static NAMED_RULES = { - // For some providers, dots don't matter - dots_dont_matter: { - name: 'dots_dont_matter', - description: 'Dots don\'t matter', - rule: ({ eml }) => { - eml.local = eml.local.replace(/\./g, ''); - }, - }, - remove_subaddressing: { - name: 'remove_subaddressing', - description: 'Remove subaddressing', - rule: ({ eml }) => { - eml.local = eml.local.split('+')[0]; - }, - }, - }; - static PROVIDERS = { - gmail: { - name: 'gmail', - description: 'Gmail', - rules: ['dots_dont_matter'], - }, - icloud: { - name: 'icloud', - description: 'iCloud', - rules: ['dots_dont_matter'], - }, - yahoo: { - name: 'yahoo', - description: 'Yahoo', - // Yahoo doesn't allow subaddressing, which would be a non-issue, - // except Yahoo allows '+' symbols in the primary email address. - rmrules: ['remove_subaddressing'], - }, - }; - // Service providers may have multiple subdomains a user can choose - static DOMAIN_TO_PROVIDER = { - 'gmail.com': 'gmail', - 'googlemail.com': 'gmail', - 'yahoo.com': 'yahoo', - 'yahoo.co.uk': 'yahoo', - 'yahoo.ca': 'yahoo', - 'yahoo.com.au': 'yahoo', - 'icloud.com': 'icloud', - 'me.com': 'icloud', - 'mac.com': 'icloud', - }; - // Service providers may allow the same primary email address to be - // used with different domains - static DOMAIN_NONDISTINCT = { - 'googlemail.com': 'gmail.com', - }; - /** - * Maps non-distinct email domains to their canonical equivalents. - * For example, 'googlemail.com' is mapped to 'gmail.com' since they - * represent the same email service. - * @type {Object.} - */ - _construct () { - this.named_rules = this.constructor.NAMED_RULES; - this.providers = this.constructor.PROVIDERS; - this.domain_to_provider = this.constructor.DOMAIN_TO_PROVIDER; - this.domain_nondistinct = this.constructor.DOMAIN_NONDISTINCT; - } - - /** - * Cleans an email address by applying provider-specific rules and standardizations - * @param {string} email - The email address to clean - * @returns {string} The cleaned email address with applied rules and standardizations - * - * Splits email into local and domain parts, applies provider-specific rules like: - * - Removing dots for certain providers (Gmail, iCloud) - * - Handling subaddressing (removing +suffix) - * - Normalizing domains (e.g. googlemail.com -> gmail.com) - */ - clean (email) { - const eml = (() => { - const [local, domain] = email.split('@'); - return { local, domain }; - })(); - - if ( this.domain_nondistinct[eml.domain] ) { - eml.domain = this.domain_nondistinct[eml.domain]; - } - - const rules = [ - 'remove_subaddressing', - ]; - - const provider = this.domain_to_provider[eml.domain] || eml.domain; - const provider_info = this.providers[provider]; - if ( provider_info ) { - provider_info.rules = provider_info.rules || []; - provider_info.rmrules = provider_info.rmrules || []; - - for ( const rule_name of provider_info.rules ) { - rules.push(rule_name); - } - - for ( const rule_name of provider_info.rmrules ) { - const idx = rules.indexOf(rule_name); - if ( idx !== -1 ) { - rules.splice(idx, 1); - } - } - } - - for ( const rule_name of rules ) { - const rule = this.named_rules[rule_name]; - rule.rule({ eml }); - } - - return `${eml.local }@${ eml.domain}`; - } - - /** - * Validates an email address against blocked domains and custom validation rules - * @param {string} email - The email address to validate - * @returns {Promise} True if email is valid, false if blocked or invalid - * @description First cleans the email, then checks against blocked domains from config. - * Emits 'email.validate' event to allow custom validation rules. Event handlers can - * set event.allow=false to reject the email. - */ - async validate (email) { - if ( this?.global_config?.env === 'dev' ) return true; - - email = this.clean(email); - const config = this.global_config; - - if ( Array.isArray(config.blocked_email_domains) ) { - for ( const suffix of config.blocked_email_domains ) { - if ( email.endsWith(suffix) ) { - return false; - } - } - } - - const svc_event = this.services.get('event'); - const event = { allow: true, email }; - await svc_event.emit('email.validate', event); - - if ( ! event.allow ) return false; - - return true; - } - -} - -module.exports = { CleanEmailService }; diff --git a/src/backend/src/services/CleanEmailService.test.ts b/src/backend/src/services/CleanEmailService.test.ts deleted file mode 100644 index 78d62e25e..000000000 --- a/src/backend/src/services/CleanEmailService.test.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { createTestKernel } from '../../tools/test.mjs'; -import { CleanEmailService } from './CleanEmailService.js'; - -describe('CleanEmailService', () => { - it('should clean email addresses correctly', async () => { - const testKernel = await createTestKernel({ - serviceMap: { - 'clean-email': CleanEmailService, - }, - }); - - const cleanEmailService = testKernel.services!.get('clean-email') as CleanEmailService; - - const cases = [ - { - email: 'bob.ross+happy-clouds@googlemail.com', - expected: 'bobross@gmail.com', - }, - { - email: 'under.rated+email-service@yahoo.com', - expected: 'under.rated+email-service@yahoo.com', - }, - { - email: 'the-absolute+best@protonmail.com', - expected: 'the-absolute@protonmail.com', - }, - ]; - - for ( const { email, expected } of cases ) { - const cleaned = cleanEmailService.clean(email); - expect(cleaned).toBe(expected); - } - }); -}); diff --git a/src/backend/src/services/ClientOperationService.js b/src/backend/src/services/ClientOperationService.js deleted file mode 100644 index bf3fe8b6c..000000000 --- a/src/backend/src/services/ClientOperationService.js +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { Context } = require('../util/context'); - -// Key for tracing operations in the context, used for logging and tracking. -const CONTEXT_KEY = Context.make_context_key('operation-trace'); -/** -* Class representing a tracker for individual client operations. -* The ClientOperationTracker class is designed to handle the metadata -* and attributes associated with each operation, allowing for better -* management and organization of client data during processing. -*/ -class ClientOperationTracker { - constructor (parameters) { - this.name = parameters.name || 'untitled'; - this.tags = parameters.tags || []; - this.frame = parameters.frame || null; - this.metadata = parameters.metadata || {}; - this.objects = parameters.objects || []; - } -} - -/** -* Class representing the ClientOperationService, which manages the -* operations related to client interactions. It provides methods to -* add new operations and handle their associated client operation -* trackers, ensuring efficient management and tracking of client-side -* operations during their lifecycle. -*/ -class ClientOperationService { - constructor ({ services }) { - this.operations_ = []; - } - - /** - * Adds a new operation to the service by creating a ClientOperationTracker instance. - * - * @param {Object} parameters - The parameters for the new operation. - * @returns {Promise} A promise that resolves to the created ClientOperationTracker instance. - */ - async add_operation (parameters) { - const tracker = new ClientOperationTracker(parameters); - - return tracker; - } - - ckey (key) { - return `${CONTEXT_KEY }:${ key}`; - } -} - -module.exports = { - ClientOperationService, -}; diff --git a/src/backend/src/services/ClientOperationService.test.ts b/src/backend/src/services/ClientOperationService.test.ts deleted file mode 100644 index 0c7e7be09..000000000 --- a/src/backend/src/services/ClientOperationService.test.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { ClientOperationService } from './ClientOperationService'; - -describe('ClientOperationService', async () => { - // ClientOperationService doesn't extend BaseService, so we can't use init - // We need to create it directly - const services = { _instances: {} }; - const clientOperationService = new ClientOperationService({ services }); - - it('should be instantiated', () => { - expect(clientOperationService).toBeDefined(); - expect(clientOperationService.operations_).toBeDefined(); - }); - - it('should have operations array', () => { - expect(clientOperationService.operations_).toBeDefined(); - expect(Array.isArray(clientOperationService.operations_)).toBe(true); - }); - - it('should create operation with default parameters', async () => { - const tracker = await clientOperationService.add_operation({}); - - expect(tracker).toBeDefined(); - expect(tracker.name).toBe('untitled'); - expect(Array.isArray(tracker.tags)).toBe(true); - expect(tracker.tags.length).toBe(0); - expect(tracker.frame).toBe(null); - expect(tracker.metadata).toBeDefined(); - expect(typeof tracker.metadata).toBe('object'); - expect(Array.isArray(tracker.objects)).toBe(true); - }); - - it('should create operation with name', async () => { - const tracker = await clientOperationService.add_operation({ - name: 'test-operation', - }); - - expect(tracker.name).toBe('test-operation'); - }); - - it('should create operation with tags', async () => { - const tags = ['tag1', 'tag2', 'tag3']; - const tracker = await clientOperationService.add_operation({ - tags, - }); - - expect(tracker.tags).toEqual(tags); - }); - - it('should create operation with frame', async () => { - const frame = { type: 'test-frame' }; - const tracker = await clientOperationService.add_operation({ - frame, - }); - - expect(tracker.frame).toBe(frame); - }); - - it('should create operation with metadata', async () => { - const metadata = { key1: 'value1', key2: 'value2' }; - const tracker = await clientOperationService.add_operation({ - metadata, - }); - - expect(tracker.metadata).toEqual(metadata); - }); - - it('should create operation with objects', async () => { - const objects = [{ id: 1 }, { id: 2 }]; - const tracker = await clientOperationService.add_operation({ - objects, - }); - - expect(tracker.objects).toEqual(objects); - }); - - it('should create operation with all parameters', async () => { - const params = { - name: 'full-operation', - tags: ['full', 'test'], - frame: { type: 'frame' }, - metadata: { meta: 'data' }, - objects: [{ obj: 1 }], - }; - - const tracker = await clientOperationService.add_operation(params); - - expect(tracker.name).toBe(params.name); - expect(tracker.tags).toEqual(params.tags); - expect(tracker.frame).toBe(params.frame); - expect(tracker.metadata).toEqual(params.metadata); - expect(tracker.objects).toEqual(params.objects); - }); - - it('should create multiple operations', async () => { - const tracker1 = await clientOperationService.add_operation({ name: 'op1' }); - const tracker2 = await clientOperationService.add_operation({ name: 'op2' }); - const tracker3 = await clientOperationService.add_operation({ name: 'op3' }); - - expect(tracker1.name).toBe('op1'); - expect(tracker2.name).toBe('op2'); - expect(tracker3.name).toBe('op3'); - }); - - it('should have ckey method', () => { - expect(clientOperationService.ckey).toBeDefined(); - expect(typeof clientOperationService.ckey).toBe('function'); - }); - - it('should generate context key with ckey', () => { - const key = clientOperationService.ckey('test-key'); - - expect(key).toBeDefined(); - expect(typeof key).toBe('string'); - expect(key).toContain('test-key'); - }); - - it('should generate different keys for different inputs', () => { - const key1 = clientOperationService.ckey('key1'); - const key2 = clientOperationService.ckey('key2'); - - expect(key1).not.toBe(key2); - }); -}); - diff --git a/src/backend/src/services/ConfigurableCountingService.js b/src/backend/src/services/ConfigurableCountingService.js deleted file mode 100644 index 6f7420229..000000000 --- a/src/backend/src/services/ConfigurableCountingService.js +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -var crypto = require('crypto'); -const BaseService = require('./BaseService'); -const { Context } = require('../util/context'); -const { DB_WRITE } = require('./database/consts'); - -const hash = v => { - const sum = crypto.createHash('sha1'); - sum.update(v); - return sum.digest(); -}; - -/** -* @class ConfigurableCountingService -* @extends BaseService -* @description The ConfigurableCountingService class extends BaseService and is responsible for managing and incrementing -* configurable counting types for different services. -* It defines counting types and SQL columns, and provides a method to increment counts based on specific service -* types and values. This class is used to manage usage counts for various services, ensuring accurate tracking -* and updating of counts in the database. -*/ -class ConfigurableCountingService extends BaseService { - static counting_types = { - gpt: { - category: [ - { - name: 'model', - type: 'string', - }, - ], - values: [ - { - name: 'input_tokens', - type: 'uint', - }, - { - name: 'output_tokens', - type: 'uint', - }, - ], - }, - dalle: { - category: [ - { - name: 'model', - type: 'string', - }, - { - name: 'quality', - type: 'string', - }, - { - name: 'resolution', - type: 'string', - }, - ], - }, - }; - - static sql_columns = { - uint: [ - 'value_uint_1', - 'value_uint_2', - 'value_uint_3', - ], - }; - - /** - * Initializes the database accessor for the ConfigurableCountingService. - * This method sets up the database service for writing counting data. - * - * @async - * @function _init - * @returns {Promise} A promise that resolves when the database connection is established. - * @memberof ConfigurableCountingService - */ - async _init () { - this.db = this.services.get('database').get(DB_WRITE, 'counting'); - } - - /** - * Increments the count for a given service based on the provided parameters. - * This method builds an SQL query to update the count and other custom values - * in the database. It handles different SQL dialects (MySQL and SQLite) and - * ensures that the pricing category is correctly hashed and stored. - * - * @param {Object} params - The parameters for incrementing the count. - * @param {string} params.service_name - The name of the service. - * @param {string} params.service_type - The type of the service. - * @param {Object} params.values - The values to be incremented. - * @throws {Error} If the service type is unknown or if there are no more available columns. - * @returns {Promise} A promise that resolves when the count is successfully incremented. - */ - async increment ({ service_name, service_type, values }) { - values = values ? { ...values } : {}; - - const now = new Date(); - const year = now.getUTCFullYear(); - const month = now.getUTCMonth() + 1; - - const counting_type = this.constructor.counting_types[service_type]; - if ( ! counting_type ) { - throw new Error(`unknown counting type ${service_type}`); - } - - const available_columns = {}; - for ( const k in this.constructor.sql_columns ) { - available_columns[k] = [...this.constructor.sql_columns[k]]; - } - - const custom_col_names = counting_type.values.map((value, index) => { - const column = available_columns[value.type].shift(); - if ( ! column ) { - // TODO: this could be an init check on all the available service types - throw new Error(`no more available columns for type ${value.type}`); - } - return column; - }); - - const custom_col_values = counting_type.values.map((value, index) => { - return values[value.name]; - }); - - // `pricing_category` is a JSON field. Keys from `values` used for - // the pricing category will be removed from ths `values` object - const pricing_category = {}; - for ( const category of counting_type.category ) { - pricing_category[category.name] = values[category.name]; - delete values[category.name]; - } - - // `JSON.stringify` cannot be used here because it does not sort - // the keys. - const pricing_category_str = counting_type.category.map((category) => { - return `${category.name}:${pricing_category[category.name]}`; - }).join(','); - - const pricing_category_hash = hash(pricing_category_str); - - const actor = Context.get('actor'); - const actor_key = actor.uid; - - const required_data = { - year, - month, - service_name, - service_type, - actor_key, - pricing_category_hash, - pricing_category: JSON.stringify(pricing_category), - }; - - const duplicate_update_part = - `count = count + 1${ - custom_col_names.length > 0 ? ', ' : '' - } ${ - custom_col_names.map((name) => `${name} = ${name} + ?`).join(', ') - }`; - - const identifying_keys = [ - 'year', 'month', - 'service_type', 'service_name', - 'actor_key', - 'pricing_category_hash', - ]; - - const sql = - `INSERT INTO monthly_usage_counts (${ - Object.keys(required_data).join(', ') - }, count, ${ - custom_col_names.join(', ') - }) ` + - `VALUES (${ - Object.keys(required_data).map(() => '?').join(', ') - }, 1, ${custom_col_values.map(() => '?').join(', ')}) ${ - this.db.case({ - mysql: `ON DUPLICATE KEY UPDATE ${ duplicate_update_part}`, - sqlite: `ON CONFLICT(${ - identifying_keys.map(v => `\`${v}\``).join(', ') - }) DO UPDATE SET ${duplicate_update_part}`, - })}` - ; - - const value_array = [ - ...Object.values(required_data), - ...custom_col_values, - ...custom_col_values, - ]; - - await this.db.write(sql, value_array); - } -} - -module.exports = { - ConfigurableCountingService, -}; diff --git a/src/backend/src/services/ConfigurableCountingService.test.ts b/src/backend/src/services/ConfigurableCountingService.test.ts deleted file mode 100644 index 9926b9011..000000000 --- a/src/backend/src/services/ConfigurableCountingService.test.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { createTestKernel } from '../../tools/test.mjs'; -import * as config from '../config'; -import { ConfigurableCountingService } from './ConfigurableCountingService'; - -describe('ConfigurableCountingService', async () => { - config.load_config({ - 'services': { - 'database': { - path: ':memory:', - }, - }, - }); - - const testKernel = await createTestKernel({ - serviceMap: { - 'counting': ConfigurableCountingService, - }, - initLevelString: 'init', - testCore: true, - }); - - const countingService = testKernel.services!.get('counting') as ConfigurableCountingService; - - it('should be instantiated', () => { - expect(countingService).toBeInstanceOf(ConfigurableCountingService); - }); - - it('should have counting types defined', () => { - expect(ConfigurableCountingService.counting_types).toBeDefined(); - expect(ConfigurableCountingService.counting_types.gpt).toBeDefined(); - expect(ConfigurableCountingService.counting_types.dalle).toBeDefined(); - }); - - it('should have sql columns defined', () => { - expect(ConfigurableCountingService.sql_columns).toBeDefined(); - expect(ConfigurableCountingService.sql_columns.uint).toBeDefined(); - expect(ConfigurableCountingService.sql_columns.uint.length).toBe(3); - }); - - it('should validate GPT counting type structure', () => { - const gptType = ConfigurableCountingService.counting_types.gpt; - expect(gptType.category).toBeDefined(); - expect(gptType.values).toBeDefined(); - expect(gptType.category.length).toBeGreaterThan(0); - expect(gptType.values.length).toBeGreaterThan(0); - }); - - it('should validate DALL-E counting type structure', () => { - const dalleType = ConfigurableCountingService.counting_types.dalle; - expect(dalleType.category).toBeDefined(); - expect(dalleType.category.length).toBeGreaterThan(0); - expect(dalleType.category.some(c => c.name === 'model')).toBe(true); - expect(dalleType.category.some(c => c.name === 'quality')).toBe(true); - expect(dalleType.category.some(c => c.name === 'resolution')).toBe(true); - }); - - it('should have gpt token value definitions', () => { - const gptType = ConfigurableCountingService.counting_types.gpt; - expect(gptType.values.some(v => v.name === 'input_tokens')).toBe(true); - expect(gptType.values.some(v => v.name === 'output_tokens')).toBe(true); - expect(gptType.values.every(v => v.type === 'uint')).toBe(true); - }); - - it('should have available sql columns for uint type', () => { - const columns = ConfigurableCountingService.sql_columns.uint; - expect(columns).toBeDefined(); - expect(Array.isArray(columns)).toBe(true); - expect(columns.length).toBe(3); - expect(columns.every(col => typeof col === 'string')).toBe(true); - }); - - it('should have model category for gpt', () => { - const gptType = ConfigurableCountingService.counting_types.gpt; - const modelCategory = gptType.category.find(c => c.name === 'model'); - expect(modelCategory).toBeDefined(); - expect(modelCategory!.type).toBe('string'); - }); -}); - diff --git a/src/backend/src/services/Container.js b/src/backend/src/services/Container.js deleted file mode 100644 index 117803b20..000000000 --- a/src/backend/src/services/Container.js +++ /dev/null @@ -1,318 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('@heyputer/putility'); -const config = require('../config'); -const { Context } = require('../util/context'); -const { CompositeError } = require('../util/errorutil'); -const { TeePromise } = require('@heyputer/putility').libs.promise; - -// 17 lines of code instead of an entire dependency-injection framework -/** -* The `Container` class is a lightweight dependency-injection container designed to manage -* service instances within the application. It provides functionality for registering, -* retrieving, and managing the lifecycle of services, including initialization and event -* handling. This class is intended to simplify dependency management and ensure that services -* are properly initialized and available throughout the application. -* -* @class -*/ -class Container { - constructor () { - this.instances_ = {}; - this.implementors_ = {}; - this.ready = new TeePromise(); - - this.modname_ = null; - this.modules_ = {}; - this.enforcers = []; - } - /** - * - * @param {(object: {name: string, options: any, meta: {disallow: boolean|undefined}})=>void} func - */ - registerEnforcer (func) { - this.enforcers.push(func); - } - - registerModule (name, module) { - this.modules_[name] = { - services_l: [], - services_m: {}, - module, - }; - this.setModuleName(name); - } - - /** - * Sets the name of the current module registering services. - * - * Note: this is an antipattern; it would be a bit better to - * provide the module name while registering a service, but - * this requires making an implementor of Container's interface - * with this as a hidden variable so as not to break existing - * modules. - */ - setModuleName (name) { - this.modname_ = name; - } - - /** - * registerService registers a service with the services container. - * - * @param {String} name - the name of the service - * @param {BaseService.constructor} cls - an implementation of BaseService - * @param {Array} args - arguments to pass to the service constructor - */ - registerService (name, cls, args) { - const my_config = config.services?.[name] || {}; - const instance = cls.getInstance - ? cls.getInstance({ services: this, config, my_config, name, args }) - : new cls({ - context: Context.get(), - services: this, - config, - my_config, - name, - args, - }) ; - this.instances_[name] = instance; - - if ( this.modname_ ) { - const mod_entry = this.modules_[this.modname_]; - mod_entry.services_l.push(name); - mod_entry.services_m[name] = true; - } - - if ( ! (instance instanceof AdvancedBase) ) return; - - const traits = instance.list_traits(); - for ( const trait of traits ) { - if ( ! this.implementors_[trait] ) { - this.implementors_[trait] = []; - } - this.implementors_[trait].push({ - name, - instance, - impl: instance.as(trait), - }); - } - } - /** - * patchService allows overriding methods on a service that is already - * constructed and initialized. - * - * @param {String} name - the name of the service to patch - * @param {ServicePatch.constructor} patch - the patch - * @param {Array} args - arguments to pass to the patch - */ - patchService (name, patch, args) { - const original_service = this.instances_[name]; - const patch_instance = new patch(); - patch_instance.patch({ original_service, args }); - } - - // get_implementors returns a list of implementors for the specified - // interface name. - get_implementors (interface_name) { - const internal_list = this.implementors_[interface_name]; - const clone = [...internal_list]; - return clone; - } - - set (name, instance) { - this.instances_[name] = instance; - } - _get (name, opts) { - if ( this.instances_[name] ) { - return this.instances_[name]; - } - if ( ! opts?.optional ) { - throw new Error(`missing service: ${name}`); - } - } - - get (name, opts) { - let meta = {}; - // Extensions should be allowed to (synchronously) guard extensions and access to them - this.enforcers.forEach(func => { - func({ name, opts, meta }); - }); - - if ( ! meta.disallow ) { - return this._get(name, opts); - } - } - /** - * Checks if a service is registered in the container. - * - * @param {String} name - The name of the service to check. - * @returns {Boolean} - Returns true if the service is registered, false otherwise. - */ - has (name) { - return !!this.instances_[name]; - } - get values () { - const values = {}; - for ( const k in this.instances_ ) { - let k2 = k; - - // Replace lowerCamelCase with underscores - // (just an idea; more effort than it's worth right now) - // let k2 = k.replace(/([a-z])([A-Z])/g, '$1_$2') - - // Replace dashes with underscores - k2 = k2.replace(/-/g, '_'); - // Convert to lower case - k2 = k2.toLowerCase(); - - values[k2] = this.instances_[k]; - } - return this.instances_; - } - - /** - * Initializes all registered services in the container. - * - * This method first constructs each service by calling its `construct` method, - * and then initializes each service by calling its `init` method. If any service - * initialization fails, it logs the failures and throws a `CompositeError` - * containing details of all failed initializations. - * - * @returns {Promise} A promise that resolves when all services are - * initialized or rejects if any service initialization fails. - */ - async init () { - for ( const k in this.instances_ ) { - if ( ! this.instances_[k]._run_as_early_as_possible ) continue; - await this.instances_[k].run_as_early_as_possible(); - } - for ( const k in this.instances_ ) { - await this.instances_[k].construct(); - } - const init_failures = []; - const promises = []; - const PARALLEL = config.experimental_parallel_init; - for ( const k in this.instances_ ) { - try { - if ( PARALLEL ) promises.push(this.instances_[k].init()); - else { - // Logic to get name of a service, unused but - // if you ever need to log the name - // this will be accurate - let name = this.instances_[k].constructor.name; - if ( name === 'ExtensionService' ) { - name = this.instances_[k].args.state.extension.runtime.name; - } - - await this.instances_[k].init(); - } - } catch (e) { - init_failures.push({ k, e }); - } - } - if ( PARALLEL ) await Promise.all(promises); - - if ( init_failures.length ) { - console.error('init failures', init_failures); - throw new CompositeError( - `failed to initialize these services: ${ - init_failures.map(({ k }) => k).join(', ')}`, - init_failures.map(({ k, e }) => e), - ); - } - } - - /** - * Emits an event to all registered services. - * - * This method sends an event identified by `id` along with any additional arguments to all - * services registered in the container. If a logger is available, it logs the event. - * - * @param {string} id - The identifier of the event. - * @param {...*} args - Additional arguments to pass to the event handler. - * @returns {Promise} A promise that resolves when all event handlers have completed. - */ - async emit (id, ...args) { - console.debug(`services:event ${id}`, { args }); - - const promises = []; - for ( const k in this.instances_ ) { - if ( this.instances_[k].__on ) { - promises.push(Context.arun(() => this.instances_[k].__on(id, args))); - } - } - await Promise.all(promises); - } -} - -/** -* @class ProxyContainer -* @classdesc The ProxyContainer class is a proxy for the Container class, allowing for delegation of service management tasks. -* It extends the functionality of the Container class by providing a delegation mechanism. -* This class is useful for scenarios where you need to manage services through a proxy, -* enabling additional flexibility and control over service instances. -*/ -class ProxyContainer { - constructor (delegate) { - this.delegate = delegate; - this.instances_ = {}; - } - set (name, instance) { - this.instances_[name] = instance; - } - get (name) { - if ( this.instances_.hasOwnProperty(name) ) { - return this.instances_[name]; - } - return this.delegate.get(name); - } - /** - * Checks if the container has a service with the specified name. - * - * @param {string} name - The name of the service to check. - * @returns {boolean} - Returns true if the service exists, false otherwise. - */ - has (name) { - if ( this.instances_.hasOwnProperty(name) ) { - return true; - } - return this.delegate.has(name); - } - get values () { - const values = {}; - Object.assign(values, this.delegate.values); - for ( const k in this.instances_ ) { - let k2 = k; - - // Replace lowerCamelCase with underscores - // (just an idea; more effort than it's worth right now) - // let k2 = k.replace(/([a-z])([A-Z])/g, '$1_$2') - - // Replace dashes with underscores - k2 = k2.replace(/-/g, '_'); - // Convert to lower case - k2 = k2.toLowerCase(); - - values[k2] = this.instances_[k]; - } - return values; - } -} - -module.exports = { Container, ProxyContainer }; diff --git a/src/backend/src/services/ContextInitService.js b/src/backend/src/services/ContextInitService.js deleted file mode 100644 index 1f5b441c4..000000000 --- a/src/backend/src/services/ContextInitService.js +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { Context } = require('../util/context'); -const BaseService = require('./BaseService'); - -// DRY: (2/3) - src/util/context.js; move install() to base class -/** -* @class ContextInitExpressMiddleware -* @description Express middleware that initializes context values for requests. -* Manages a collection of value initializers that can be synchronous values -* or asynchronous factory functions. Each initializer sets a key-value pair -* in the request context. Part of a DRY implementation shared with context.js. -* TODO: Consider moving install() method to base class. -*/ -class ContextInitExpressMiddleware { - /** - * Express middleware class that initializes context values for requests - * - * Manages a list of value initializers that populate the Context with - * either static values or async-generated values when handling requests. - * Part of DRY pattern with src/util/context.js. - */ - constructor () { - this.value_initializers_ = []; - } - register_initializer (initializer) { - this.value_initializers_.push(initializer); - } - install (app) { - app.use(this.run.bind(this)); - } - /** - * Installs the middleware into the Express application - * @param {Express} app - The Express application instance - * @returns {void} - */ - async run (req, res, next) { - const x = Context.get(); - for ( const initializer of this.value_initializers_ ) { - if ( initializer.value ) { - x.set(initializer.key, initializer.value); - } else if ( initializer.async_factory ) { - x.set(initializer.key, await initializer.async_factory()); - } - } - next(); - } -} - -/** -* @class ContextInitService -* @extends BaseService -* @description Service responsible for initializing and managing context values in the application. -* Provides methods to register both synchronous values and asynchronous factories for context -* initialization. Works in conjunction with Express middleware to ensure proper context setup -* for each request. Extends BaseService to integrate with the application's service architecture. -*/ -class ContextInitService extends BaseService { - /** - * Service for initializing request context with values and async factories. - * Extends BaseService to provide middleware for Express that populates the Context - * with registered values and async-generated values at the start of each request. - * - * @extends BaseService - */ - _construct () { - this.mw = new ContextInitExpressMiddleware(); - } - register_value (key, value) { - this.mw.register_initializer({ - key, value, - }); - } - /** - * Registers an asynchronous factory function to initialize a context value - * @param {string} key - The key to store the value under in the context - * @param {Function} async_factory - Async function that returns the value to store - */ - register_async_factory (key, async_factory) { - this.mw.register_initializer({ - key, async_factory, - }); - } - async '__on_install.middlewares.context-aware' (_, { app }) { - this.mw.install(app); - await this.services.emit('install.context-initializers'); - } -} - -module.exports = { - ContextInitService, -}; \ No newline at end of file diff --git a/src/backend/src/services/ContextInitService.test.ts b/src/backend/src/services/ContextInitService.test.ts deleted file mode 100644 index fb3acf836..000000000 --- a/src/backend/src/services/ContextInitService.test.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { createTestKernel } from '../../tools/test.mjs'; -import { ContextInitService } from './ContextInitService'; - -describe('ContextInitService', async () => { - const testKernel = await createTestKernel({ - serviceMap: { - 'context-init': ContextInitService, - }, - initLevelString: 'init', - }); - - const contextInitService = testKernel.services!.get('context-init') as any; - - it('should be instantiated', () => { - expect(contextInitService).toBeInstanceOf(ContextInitService); - }); - - it('should have middleware instance', () => { - expect(contextInitService.mw).toBeDefined(); - expect(contextInitService.mw.value_initializers_).toBeDefined(); - expect(Array.isArray(contextInitService.mw.value_initializers_)).toBe(true); - }); - - it('should register a value initializer', () => { - const initialLength = contextInitService.mw.value_initializers_.length; - - contextInitService.register_value('test-key', 'test-value'); - - expect(contextInitService.mw.value_initializers_.length).toBe(initialLength + 1); - }); - - it('should store key-value pair in initializer', () => { - const service = testKernel.services!.get('context-init') as any; - - service.register_value('stored-key', 'stored-value'); - - const lastInitializer = service.mw.value_initializers_[service.mw.value_initializers_.length - 1]; - expect(lastInitializer.key).toBe('stored-key'); - expect(lastInitializer.value).toBe('stored-value'); - }); - - it('should register async factory', () => { - const service = testKernel.services!.get('context-init') as any; - const initialLength = service.mw.value_initializers_.length; - - const factory = async () => 'async-value'; - service.register_async_factory('async-key', factory); - - expect(service.mw.value_initializers_.length).toBe(initialLength + 1); - }); - - it('should store async factory in initializer', () => { - const service = testKernel.services!.get('context-init') as any; - - const factory = async () => 'factory-result'; - service.register_async_factory('factory-key', factory); - - const lastInitializer = service.mw.value_initializers_[service.mw.value_initializers_.length - 1]; - expect(lastInitializer.key).toBe('factory-key'); - expect(lastInitializer.async_factory).toBe(factory); - }); - - it('should handle multiple value registrations', () => { - const service = testKernel.services!.get('context-init') as any; - - service.register_value('key1', 'value1'); - service.register_value('key2', 'value2'); - service.register_value('key3', 'value3'); - - const keys = service.mw.value_initializers_.map((init: any) => init.key); - expect(keys).toContain('key1'); - expect(keys).toContain('key2'); - expect(keys).toContain('key3'); - }); - - it('should have install method on middleware', () => { - expect(contextInitService.mw.install).toBeDefined(); - expect(typeof contextInitService.mw.install).toBe('function'); - }); - - it('should have run method on middleware', () => { - expect(contextInitService.mw.run).toBeDefined(); - expect(typeof contextInitService.mw.run).toBe('function'); - }); -}); - diff --git a/src/backend/src/services/DetailProviderService.js b/src/backend/src/services/DetailProviderService.js deleted file mode 100644 index bb8e34eab..000000000 --- a/src/backend/src/services/DetailProviderService.js +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const BaseService = require('./BaseService'); - -/** - * A generic service class for any service that enables registering - * detail providers. A detail provider is a function that takes an - * input object and uses its values to populate another object. - */ -class DetailProviderService extends BaseService { - _construct () { - this.providers_ = []; - } - - register_provider (fn) { - this.providers_.push(fn); - } - - /** - * Asynchronously retrieves details by invoking registered detail providers - * in list. Populates the provided output object with the results of - * each provider. If no output object is provided, a new one is created - * by default. - * - * @param {Object} context - The context object containing input data for - * the providers. - * @param {Object} [out={}] - An optional output object to populate with - * the details. - * @returns {Promise} The populated output object after all - * providers have been processed. - */ - async get_details (context, out) { - out = out || {}; - - for ( const provider of this.providers_ ) { - await provider(context, out); - } - - return out; - } -} - -module.exports = { DetailProviderService }; diff --git a/src/backend/src/services/DetailProviderService.test.ts b/src/backend/src/services/DetailProviderService.test.ts deleted file mode 100644 index 32ae245db..000000000 --- a/src/backend/src/services/DetailProviderService.test.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { createTestKernel } from '../../tools/test.mjs'; -import { DetailProviderService } from './DetailProviderService'; - -describe('DetailProviderService', async () => { - const testKernel = await createTestKernel({ - serviceMap: { - 'detail-provider': DetailProviderService, - }, - initLevelString: 'init', - }); - - const detailProviderService = testKernel.services!.get('detail-provider') as any; - - it('should be instantiated', () => { - expect(detailProviderService).toBeInstanceOf(DetailProviderService); - }); - - it('should have empty providers array initially', () => { - expect(detailProviderService.providers_).toBeDefined(); - expect(Array.isArray(detailProviderService.providers_)).toBe(true); - }); - - it('should register a provider', () => { - const initialLength = detailProviderService.providers_.length; - const provider = async (context: any, out: any) => { - out.test = 'value'; - }; - - detailProviderService.register_provider(provider); - - expect(detailProviderService.providers_.length).toBe(initialLength + 1); - }); - - it('should get details with single provider', async () => { - const service = testKernel.services!.get('detail-provider') as any; - - service.register_provider(async (context: any, out: any) => { - out.name = context.input; - }); - - const result = await service.get_details({ input: 'test-name' }); - - expect(result.name).toBe('test-name'); - }); - - it('should get details with multiple providers', async () => { - const service = testKernel.services!.get('detail-provider') as any; - - service.register_provider(async (context: any, out: any) => { - out.field1 = 'value1'; - }); - - service.register_provider(async (context: any, out: any) => { - out.field2 = 'value2'; - }); - - const result = await service.get_details({}); - - expect(result.field1).toBe('value1'); - expect(result.field2).toBe('value2'); - }); - - it('should allow providers to modify existing output', async () => { - const service = testKernel.services!.get('detail-provider') as any; - - service.register_provider(async (context: any, out: any) => { - out.counter = 1; - }); - - service.register_provider(async (context: any, out: any) => { - out.counter = out.counter + 1; - }); - - const result = await service.get_details({}); - - expect(result.counter).toBe(2); - }); - - it('should use provided output object', async () => { - const service = testKernel.services!.get('detail-provider') as any; - - service.register_provider(async (context: any, out: any) => { - out.added = true; - }); - - const existingOut = { existing: 'value' }; - const result = await service.get_details({}, existingOut); - - expect(result.existing).toBe('value'); - expect(result.added).toBe(true); - }); - - it('should handle async providers', async () => { - const service = testKernel.services!.get('detail-provider') as any; - - service.register_provider(async (context: any, out: any) => { - await new Promise(resolve => setTimeout(resolve, 10)); - out.async = true; - }); - - const result = await service.get_details({}); - - expect(result.async).toBe(true); - }); -}); - diff --git a/src/backend/src/services/DynamoKVStore/.gitignore b/src/backend/src/services/DynamoKVStore/.gitignore deleted file mode 100644 index aa4a6da26..000000000 --- a/src/backend/src/services/DynamoKVStore/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.js -*.js.map \ No newline at end of file diff --git a/src/backend/src/services/DynamoKVStore/DynamoKVStore.test.ts b/src/backend/src/services/DynamoKVStore/DynamoKVStore.test.ts deleted file mode 100644 index 443e182f5..000000000 --- a/src/backend/src/services/DynamoKVStore/DynamoKVStore.test.ts +++ /dev/null @@ -1,574 +0,0 @@ -import { Actor } from '@heyputer/backend/src/services/auth/Actor.js'; -import { SUService } from '@heyputer/backend/src/services/SUService'; -import { createTestKernel } from '@heyputer/backend/tools/test.mjs'; -import { describe, expect, it } from 'vitest'; -import { config } from '../../loadTestConfig.js'; -import { DynamoKVStore } from './DynamoKVStore.js'; -import { DynamoKVStoreWrapper, IDynamoKVStoreWrapper } from './DynamoKVStoreWrapper.js'; - -describe('DynamoKVStore', async () => { - const TABLE_NAME = 'store-kv-v1'; - - const makeActor = (userId: number | string, appUid?: string) => ({ - type: { - user: { id: userId, uuid: String(userId) }, - ...(appUid ? { app: { uid: appUid } } : {}), - }, - }) as Actor; - - const testKernel = await createTestKernel({ - serviceMap: { - 'puter-kvstore': DynamoKVStoreWrapper, - }, - initLevelString: 'init', - testCore: true, - serviceConfigOverrideMap: { - 'services': { - 'puter-kvstore': { tableName: TABLE_NAME }, - }, - }, - }); - - const testSubject = testKernel.services!.get('puter-kvstore') as IDynamoKVStoreWrapper; - const kvStore = testSubject.kvStore!; - const su = testKernel.services!.get('su') as SUService; - - it('should be instantiated', () => { - expect(testSubject).toBeInstanceOf(DynamoKVStoreWrapper); - }); - - it('should contain a copy of the public methods of DynamoKVStore too', () => { - const meteringMethods = Object.getOwnPropertyNames(DynamoKVStore.prototype) - .filter((name) => name !== 'constructor'); - const wrapperMethods = testSubject as unknown as Record; - const missing = meteringMethods.filter((name) => typeof wrapperMethods[name] !== 'function'); - - expect(missing).toEqual([]); - }); - - it('should have DynamoKVStore instantiated', async () => { - expect(testSubject.kvStore).toBeInstanceOf(DynamoKVStore); - }); - it('sets and retrieves values for the current actor context', async () => { - const actor = makeActor(1); - const key = 'greeting'; - const value = { hello: 'world' }; - - await su.sudo(actor, () => kvStore.set({ key, value })); - const stored = await su.sudo(actor, () => kvStore.get({ key })); - - expect(stored).toEqual(value); - }); - - it('batchPut writes multiple values and honors expiration timestamps', async () => { - const actor = makeActor(101); - const nowInSeconds = Math.floor(Date.now() / 1000); - - await su.sudo(actor, () => kvStore.batchPut({ - items: [ - { key: 'batch-a', value: 'first' }, - { key: 'batch-b', value: 'expired', expireAt: nowInSeconds - 1 }, - { key: 'batch-a', value: 'overridden' }, - ], - })); - - const values = await su.sudo(actor, () => kvStore.get({ key: ['batch-a', 'batch-b'] })); - expect(values).toEqual(['overridden', null]); - }); - - it('scopes data to the app when provided', async () => { - const userId = 2; - const actorAppOne = makeActor(userId, 'app-one'); - const actorAppTwo = makeActor(userId, 'app-two'); - const key = 'scoped-key'; - - await su.sudo(actorAppOne, () => kvStore.set({ key, value: 'one' })); - await su.sudo(actorAppTwo, () => kvStore.set({ key, value: 'two' })); - - const fromOne = await su.sudo(actorAppOne, () => kvStore.get({ key })); - const fromTwo = await su.sudo(actorAppTwo, () => kvStore.get({ key })); - - expect(fromOne).toBe('one'); - expect(fromTwo).toBe('two'); - }); - - it('increments nested numeric paths and persists the aggregated totals', async () => { - const actor = makeActor(3); - const key = 'counter-key'; - - const first = await su.sudo(actor, () => kvStore.incr({ - key, - pathAndAmountMap: { 'total': 5, 'nested.count': 2 }, - })); - const second = await su.sudo(actor, () => kvStore.incr({ - key, - pathAndAmountMap: { 'total': 1, 'nested.count': 3 }, - })); - - expect(first).toMatchObject({ total: 5, nested: { count: 2 } }); - expect(second).toMatchObject({ total: 6, nested: { count: 5 } }); - - const persisted = await su.sudo(actor, () => kvStore.get({ key })); - expect(persisted).toMatchObject({ total: 6, nested: { count: 5 } }); - }); - - it('decrements numeric paths via decr and keeps values in sync', async () => { - const actor = makeActor(4); - const key = 'decr-key'; - - await su.sudo(actor, () => kvStore.incr({ - key, - pathAndAmountMap: { total: 5, 'nested.count': 4 }, - })); - const afterDecr = await su.sudo(actor, () => kvStore.decr({ - key, - pathAndAmountMap: { total: 2, 'nested.count': 1 }, - })); - - expect(afterDecr).toMatchObject({ total: 3, nested: { count: 3 } }); - - const persisted = await su.sudo(actor, () => kvStore.get({ key })); - expect(persisted).toMatchObject({ total: 3, nested: { count: 3 } }); - }); - - it('deletes keys with del', async () => { - const actor = makeActor(5); - const key = 'delete-me'; - await su.sudo(actor, () => { - return kvStore.set({ key, value: 'bye' }); - }); - - const res = await su.sudo(actor, () => kvStore.del({ key })); - const value = await su.sudo(actor, () => kvStore.get({ key })); - - expect(res).toBe(true); - expect(value).toBeNull(); - }); - - it('lists entries, keys, and values while omitting expired rows', async () => { - const actor = makeActor(6); - await su.sudo(actor, () => kvStore.set({ key: 'k1', value: 'v1' })); - await su.sudo(actor, () => kvStore.set({ key: 'expired', value: 'gone', expireAt: Math.floor(Date.now() / 1000) - 10 })); - - const entries = await su.sudo(actor, () => kvStore.list({ as: 'entries' })); - const keys = await su.sudo(actor, () => kvStore.list({ as: 'keys' })); - const values = await su.sudo(actor, () => kvStore.list({ as: 'values' })); - - expect(entries).toEqual([{ key: 'k1', value: 'v1' }]); - expect(keys).toEqual(['k1']); - expect(values).toEqual(['v1']); - }); - - it('rejects invalid list selector', async () => { - const actor = makeActor(7); - expect(su.sudo(actor, () => kvStore.list({ as: 'bad' as never }))) - .rejects; - }); - - it('supports paginated list results with cursors', async () => { - const actor = makeActor(71); - await su.sudo(actor, () => kvStore.set({ key: 'a', value: 1 })); - await su.sudo(actor, () => kvStore.set({ key: 'b', value: 2 })); - await su.sudo(actor, () => kvStore.set({ key: 'c', value: 3 })); - - const firstPage = await su.sudo(actor, () => kvStore.list({ as: 'keys', limit: 2 })) as { items: string[]; cursor?: string }; - expect(firstPage.items).toHaveLength(2); - expect(firstPage.cursor).toBeTypeOf('string'); - - const secondPage = await su.sudo(actor, () => kvStore.list({ as: 'keys', limit: 2, cursor: firstPage.cursor })) as { items: string[]; cursor?: string }; - expect(secondPage.items).toHaveLength(1); - expect(secondPage.cursor).toBeUndefined(); - - const allKeys = [...firstPage.items, ...secondPage.items].sort(); - expect(allKeys).toEqual(['a', 'b', 'c']); - }); - - it('supports prefix pattern semantics', async () => { - const actor = makeActor(72); - const allKeys = [ - 'abc', - 'abc123', - 'abc123xyz', - 'ab', - 'key*literal', - 'key*literal-2', - 'k*y', - 'k*y-extra', - 'other', - ]; - - await Promise.all(allKeys.map((key, idx) => su.sudo(actor, () => kvStore.set({ key, value: idx })))); - - const expectedAbc = ['abc', 'abc123', 'abc123xyz']; - const expectedKeyStar = ['key*literal', 'key*literal-2']; - const expectedMiddleStar = ['k*y', 'k*y-extra']; - - const abcKeys = await su.sudo(actor, () => kvStore.list({ as: 'keys', pattern: 'abc' })) as string[]; - expect([...abcKeys].sort()).toEqual([...expectedAbc].sort()); - - const abcWildcardKeys = await su.sudo(actor, () => kvStore.list({ as: 'keys', pattern: 'abc*' })) as string[]; - expect([...abcWildcardKeys].sort()).toEqual([...expectedAbc].sort()); - - const keyStarKeys = await su.sudo(actor, () => kvStore.list({ as: 'keys', pattern: 'key**' })) as string[]; - expect([...keyStarKeys].sort()).toEqual([...expectedKeyStar].sort()); - - const middleStarKeys = await su.sudo(actor, () => kvStore.list({ as: 'keys', pattern: 'k*y*' })) as string[]; - expect([...middleStarKeys].sort()).toEqual([...expectedMiddleStar].sort()); - - const allList = await su.sudo(actor, () => kvStore.list({ as: 'keys', pattern: '*' })) as string[]; - expect([...allList].sort()).toEqual([...allKeys].sort()); - }); - - it('returns ordered values for arrays and null for expired keys', async () => { - const actor = makeActor(8); - const now = Math.floor(Date.now() / 1000); - - await su.sudo(actor, () => kvStore.set({ key: 'a', value: 1 })); - await su.sudo(actor, () => kvStore.set({ key: 'b', value: 2, expireAt: now - 5 })); - await su.sudo(actor, () => kvStore.set({ key: 'c', value: 3 })); - - const results = await su.sudo(actor, () => kvStore.get({ key: ['c', 'b', 'a'] })); - - expect(results).toEqual([3, null, 1]); - }); - - it('flush clears all keys for the actor/app combination', async () => { - const actor = makeActor(9, 'flush-app'); - await su.sudo(actor, () => kvStore.set({ key: 'one', value: 1 })); - await su.sudo(actor, () => kvStore.set({ key: 'two', value: 2 })); - - const res = await su.sudo(actor, () => kvStore.flush()); - const remaining = await su.sudo(actor, () => kvStore.list({ as: 'entries' })); - - expect(res).toBe(true); - expect(remaining).toEqual([]); - }); - - it('expireAt and expire set timestamps that cause reads to return null', async () => { - const actor = makeActor(10); - const keyAt = 'expire-at'; - const keyTtl = 'expire-ttl'; - - await su.sudo(actor, () => kvStore.set({ key: keyAt, value: 'keep' })); - await su.sudo(actor, () => kvStore.set({ key: keyTtl, value: 'keep' })); - - await su.sudo(actor, () => kvStore.expireAt({ key: keyAt, timestamp: Math.floor(Date.now() / 1000) - 1 })); - await su.sudo(actor, () => kvStore.expire({ key: keyTtl, ttl: -1 })); - - const valAt = await su.sudo(actor, () => kvStore.get({ key: keyAt })); - const valTtl = await su.sudo(actor, () => kvStore.get({ key: keyTtl })); - - expect(valAt).toBeNull(); - expect(valTtl).toBeNull(); - }); - - it('updates nested paths and creates missing maps', async () => { - const actor = makeActor(12); - const key = 'update-key'; - - const updated = await su.sudo(actor, () => kvStore.update({ - key, - pathAndValueMap: { - 'profile.name': 'Ada', - 'profile.stats.score': 7, - 'active': true, - }, - })); - - expect(updated).toMatchObject({ - profile: { name: 'Ada', stats: { score: 7 } }, - active: true, - }); - - const stored = await su.sudo(actor, () => kvStore.get({ key })); - expect(stored).toMatchObject({ - profile: { name: 'Ada', stats: { score: 7 } }, - active: true, - }); - }); - - it('update can set ttl for the whole object', async () => { - const actor = makeActor(13); - const key = 'update-ttl'; - - await su.sudo(actor, () => kvStore.update({ - key, - pathAndValueMap: { 'count': 1 }, - ttl: -1, - })); - - const stored = await su.sudo(actor, () => kvStore.get({ key })); - expect(stored).toBeNull(); - }); - - it('supports list index paths when updating', async () => { - const actor = makeActor(17); - const key = 'update-list-index'; - - await su.sudo(actor, () => kvStore.set({ - key, - value: { a: { b: [1, 2] } }, - })); - - const updated = await su.sudo(actor, () => kvStore.update({ - key, - pathAndValueMap: { 'a.b[1]': 5 }, - })); - - expect((updated as { a?: { b?: number[] } }).a?.b).toEqual([1, 5]); - - const stored = await su.sudo(actor, () => kvStore.get({ key })); - expect((stored as { a?: { b?: number[] } }).a?.b).toEqual([1, 5]); - }); - - it('adds values to nested lists and creates missing maps', async () => { - const actor = makeActor(15); - const key = 'add-key'; - - const first = await su.sudo(actor, () => kvStore.add({ - key, - pathAndValueMap: { - 'a.b': 1, - }, - })); - - expect(first).toMatchObject({ a: { b: [1] } }); - - const second = await su.sudo(actor, () => kvStore.add({ - key, - pathAndValueMap: { - 'a.b': 2, - 'a.c': ['x', 'y'], - }, - })); - - expect(second).toMatchObject({ a: { b: [1, 2], c: ['x', 'y'] } }); - - const stored = await su.sudo(actor, () => kvStore.get({ key })); - expect(stored).toMatchObject({ a: { b: [1, 2], c: ['x', 'y'] } }); - }); - - it('supports list index paths when appending', async () => { - const actor = makeActor(18); - const key = 'add-list-index'; - - await su.sudo(actor, () => kvStore.set({ - key, - value: { a: { b: [[1], [2]] } }, - })); - - const updated = await su.sudo(actor, () => kvStore.add({ - key, - pathAndValueMap: { 'a.b[1]': 3 }, - })); - - expect((updated as { a?: { b?: number[][] } }).a?.b).toEqual([[1], [2, 3]]); - - const stored = await su.sudo(actor, () => kvStore.get({ key })); - expect((stored as { a?: { b?: number[][] } }).a?.b).toEqual([[1], [2, 3]]); - }); - - it('supports nested list indexing for add, update, remove, and incr', async () => { - const actor = makeActor(21); - const key = 'nested-list-index'; - - await su.sudo(actor, () => kvStore.set({ - key, - value: { a: [1, { b: { c: [1] } }, 2] }, - })); - - const added = await su.sudo(actor, () => kvStore.add({ - key, - pathAndValueMap: { 'a[1].b.c': 2 }, - })); - expect((added as { a?: Array }).a).toEqual([1, { b: { c: [1, 2] } }, 2]); - - const updated = await su.sudo(actor, () => kvStore.update({ - key, - pathAndValueMap: { 'a[1].b.c': [9] }, - })); - expect((updated as { a?: Array }).a).toEqual([1, { b: { c: [9] } }, 2]); - - const removed = await su.sudo(actor, () => kvStore.remove({ - key, - paths: ['a[1].b.c'], - })); - expect((removed as { a?: Array }).a).toEqual([1, { b: {} }, 2]); - - await su.sudo(actor, () => kvStore.set({ - key, - value: { a: [1, { b: { c: 1 } }, 2] }, - })); - const incrRes = await su.sudo(actor, () => kvStore.incr({ - key, - pathAndAmountMap: { 'a[1].b.c': 3 }, - })); - expect((incrRes as { a?: Array }).a).toEqual([1, { b: { c: 4 } }, 2]); - }); - - it('removes nested values including indexed list paths', async () => { - const actor = makeActor(19); - const key = 'remove-list-index'; - - await su.sudo(actor, () => kvStore.set({ - key, - value: { a: { b: [1, 2, 3], c: { d: 4 }, e: 'keep' } }, - })); - - const updated = await su.sudo(actor, () => kvStore.remove({ - key, - paths: ['a.b[1]', 'a.c'], - })); - - expect((updated as { a?: { b?: number[]; e?: string } }).a).toEqual({ b: [1, 3], e: 'keep' }); - - const stored = await su.sudo(actor, () => kvStore.get({ key })); - expect((stored as { a?: { b?: number[]; e?: string } }).a).toEqual({ b: [1, 3], e: 'keep' }); - }); - - it('rejects overlapping parent/child paths in a single request', async () => { - const actor = makeActor(20); - const key = 'overlap-paths'; - - await su.sudo(actor, () => kvStore.set({ - key, - value: { a: { b: { c: 1 } } }, - })); - - await expect(su.sudo(actor, () => kvStore.incr({ - key, - pathAndAmountMap: { 'a.b': 1, 'a.b.c': 1 }, - }))).rejects.toThrow(/paths overlap/i); - - await expect(su.sudo(actor, () => kvStore.add({ - key, - pathAndValueMap: { 'a.b': 1, 'a.b.c': 2 }, - }))).rejects.toThrow(/paths overlap/i); - - await expect(su.sudo(actor, () => kvStore.update({ - key, - pathAndValueMap: { 'a.b': 1, 'a.b.c': 2 }, - }))).rejects.toThrow(/paths overlap/i); - - await expect(su.sudo(actor, () => kvStore.remove({ - key, - paths: ['a.b', 'a.b.c'], - }))).resolves.not.toThrow(); - }); - - it('incr initializes nested maps for missing keys', async () => { - const actor = makeActor(14); - const key = 'incr-missing'; - - const first = await su.sudo(actor, () => kvStore.incr({ - key, - pathAndAmountMap: { 'a.b.c': 2, 'x': 1 }, - })); - - expect(first).toMatchObject({ a: { b: { c: 2 } }, x: 1 }); - - const second = await su.sudo(actor, () => kvStore.incr({ - key, - pathAndAmountMap: { 'a.b.c': 3 }, - })); - - expect(second).toMatchObject({ a: { b: { c: 5 } }, x: 1 }); - }); - - it('supports list index paths when incrementing', async () => { - const actor = makeActor(16); - const key = 'incr-list-index'; - - await su.sudo(actor, () => kvStore.set({ - key, - value: { a: { b: [1, 2] } }, - })); - - const updated = await su.sudo(actor, () => kvStore.incr({ - key, - pathAndAmountMap: { 'a.b[1]': 3 }, - })); - - expect((updated as { a?: { b?: number[] } }).a?.b).toEqual([1, 5]); - - const stored = await su.sudo(actor, () => kvStore.get({ key })); - expect((stored as { a?: { b?: number[] } }).a?.b).toEqual([1, 5]); - }); - - it('supports appUuid namespace isolation for non-app actors', async () => { - const actor = makeActor(22); - const key = 'override-key'; - const optConfig = { appUuid: 'override-app-a' }; - - await su.sudo(actor, () => kvStore.set({ key, value: 'default-value' })); - await su.sudo(actor, () => kvStore.set({ key, value: 'override-value', optConfig })); - - const defaultRead = await su.sudo(actor, () => kvStore.get({ key })); - const overrideRead = await su.sudo(actor, () => kvStore.get({ key, optConfig })); - const overrideList = await su.sudo(actor, () => kvStore.list({ as: 'keys', pattern: `${key}*`, optConfig })) as string[]; - - expect(defaultRead).toBe('default-value'); - expect(overrideRead).toBe('override-value'); - expect(overrideList).toContain(key); - - await su.sudo(actor, () => kvStore.del({ key, optConfig })); - const afterOverrideDelete = await su.sudo(actor, () => kvStore.get({ key, optConfig })); - const defaultAfterOverrideDelete = await su.sudo(actor, () => kvStore.get({ key })); - - expect(afterOverrideDelete).toBeNull(); - expect(defaultAfterOverrideDelete).toBe('default-value'); - }); - - it('flush with appUuid only clears that override namespace', async () => { - const actor = makeActor(23); - const overrideA = { appUuid: 'flush-override-a' }; - const overrideB = { appUuid: 'flush-override-b' }; - const keyA = 'flush-override-key-a'; - const keyB = 'flush-override-key-b'; - const defaultKey = 'flush-default-key'; - - await su.sudo(actor, () => kvStore.set({ key: keyA, value: 'A', optConfig: overrideA })); - await su.sudo(actor, () => kvStore.set({ key: keyB, value: 'B', optConfig: overrideB })); - await su.sudo(actor, () => kvStore.set({ key: defaultKey, value: 'default' })); - - await su.sudo(actor, () => kvStore.flush({ optConfig: overrideA })); - - expect(await su.sudo(actor, () => kvStore.get({ key: keyA, optConfig: overrideA }))).toBeNull(); - expect(await su.sudo(actor, () => kvStore.get({ key: keyB, optConfig: overrideB }))).toBe('B'); - expect(await su.sudo(actor, () => kvStore.get({ key: defaultKey }))).toBe('default'); - }); - - it('ignores appUuid when actor already has an app context', async () => { - const appActor = makeActor(24, 'real-app'); - const userActor = makeActor(24); - const key = 'app-context-override-ignore'; - - await su.sudo(appActor, () => kvStore.set({ - key, - value: 'from-app-context', - optConfig: { appUuid: 'fake-app' }, - })); - - const appRead = await su.sudo(appActor, () => kvStore.get({ key })); - const userReadFromFake = await su.sudo(userActor, () => kvStore.get({ key, optConfig: { appUuid: 'fake-app' } })); - const userReadFromReal = await su.sudo(userActor, () => kvStore.get({ key, optConfig: { appUuid: 'real-app' } })); - - expect(appRead).toBe('from-app-context'); - expect(userReadFromFake).toBeNull(); - expect(userReadFromReal).toBe('from-app-context'); - }); - - it('enforces key and value size limits', async () => { - const actor = makeActor(11); - const oversizedKey = 'a'.repeat(((config as unknown as Record).kv_max_key_size as number) + 1); - const oversizedValue = 'b'.repeat(((config as unknown as Record).kv_max_value_size as number) + 1); - - await expect(su.sudo(actor, () => kvStore.set({ key: oversizedKey, value: 'x' }))) - .rejects - .toThrow(/1024/i); - - await expect(su.sudo(actor, () => kvStore.set({ key: 'ok', value: oversizedValue }))) - .rejects - .toThrow(/has exceeded the maximum allowed size/i); - }); -}); diff --git a/src/backend/src/services/DynamoKVStore/DynamoKVStore.ts b/src/backend/src/services/DynamoKVStore/DynamoKVStore.ts deleted file mode 100644 index 73ea82b13..000000000 --- a/src/backend/src/services/DynamoKVStore/DynamoKVStore.ts +++ /dev/null @@ -1,894 +0,0 @@ -import { Actor, SystemActorType } from '@heyputer/backend/src/services/auth/Actor.js'; -import type { BaseDatabaseAccessService } from '@heyputer/backend/src/services/database/BaseDatabaseAccessService.js'; -import type { MeteringService } from '@heyputer/backend/src/services/MeteringService/MeteringService.js'; -import { RecursiveRecord } from '@heyputer/backend/src/services/MeteringService/types.js'; -import { Context } from '@heyputer/backend/src/util/context.js'; -import murmurhash from 'murmurhash'; -import type { DDBClient } from '../../clients/dynamodb/DDBClient.js'; -import { PUTER_KV_STORE_TABLE_DEFINITION } from './tableDefinition.js'; -import { Span } from '../../util/otelutil.js'; -import APIError from '../../api/APIError.js'; - -export class DynamoKVStore { - static GLOBAL_APP_KEY = 'os-global'; - static LEGACY_GLOBAL_APP_KEY = 'global'; - - #ddbClient: DDBClient; - #sqlClient: BaseDatabaseAccessService; - #meteringService: MeteringService; - #tableName = 'store-kv-v1'; - #pathCleanerRegex = /[:\-+/*]/g; - #enableMigrationFromSQL = false; - - constructor ({ ddbClient, sqlClient, tableName, meteringService }: { ddbClient: DDBClient, sqlClient: BaseDatabaseAccessService, tableName: string, meteringService: MeteringService }) { - this.#ddbClient = ddbClient; - this.#sqlClient = sqlClient; - this.#tableName = tableName; - this.#meteringService = meteringService; - this.#enableMigrationFromSQL = !this.#ddbClient.config?.aws; // TODO: disable via config after some time passes - } - - async createTableIfNotExists () { - if ( ! this.#enableMigrationFromSQL ) return; - await this.#ddbClient.createTableIfNotExists({ ...PUTER_KV_STORE_TABLE_DEFINITION, TableName: this.#tableName }, 'ttl'); - } - - #getNameSpace (actor: Actor, appUuidOverride?: string) { - if ( actor.type instanceof SystemActorType ) { - return 'v1:system'; - } else { - const appUuid = !actor.type?.app ? (appUuidOverride || undefined) : actor.type.app.uid; - const user = actor.type?.user ?? undefined; - if ( ! user ) throw new Error('User not found'); - - return `v1:${appUuid ? `${user.uuid}:${appUuid}` - : `${user.uuid}:${this.#enableMigrationFromSQL ? DynamoKVStore.LEGACY_GLOBAL_APP_KEY : DynamoKVStore.GLOBAL_APP_KEY}`}`; - } - } - - @Span('kv:get') - async get ({ key, optConfig }: { key: string | string[]; optConfig?: { appUuid?: string } }): Promise { - if ( key === '' ) { - throw APIError.create('field_empty', null, { - key: 'key', - }); - } - - const actor = Context.get('actor'); - const appUuid = !actor.type?.app ? optConfig?.appUuid : actor.type.app.uid; - const user = actor.type?.user ?? undefined; - - const namespace = this.#getNameSpace(actor, optConfig?.appUuid); - - const multi = Array.isArray(key); - const keys = multi ? key : [key]; - const values: unknown[] = []; - - let kvEntries; - let usage; - if ( multi ) { - const entriesAndUsage = (await this.#getBatches(namespace, keys)); - kvEntries = entriesAndUsage.kvEntries; - usage = entriesAndUsage.usage; - } else { - const res = await this.#ddbClient.get(this.#tableName, { namespace, key }); - kvEntries = res.Item ? [res.Item] : []; - usage = res.ConsumedCapacity?.CapacityUnits ?? 0; - } - - this.#meteringService.incrementUsage(actor, 'kv:read', usage || 0); - - for ( const key of keys ) { - const kv_entry = kvEntries?.find(e => e.key === key); - const time = Date.now() / 1000; - if ( kv_entry?.ttl && kv_entry.ttl <= (time) ) { - values.push(null); - continue; - } - if ( kv_entry?.value ) { - values.push(kv_entry.value); - continue; - } - - if ( this.#enableMigrationFromSQL ) { - const key_hash = murmurhash.v3(key); - const kv_row = await this.#sqlClient.read( - 'SELECT * FROM kv WHERE user_id=? AND app=? AND kkey_hash=? LIMIT 1', - [user?.id, appUuid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY, key_hash], - ); - - if ( kv_row[0]?.value ) { - // update and delete from this table - (async () => { - await this.set({ key: kv_row[0].key, value: kv_row[0].value }); - await this.#sqlClient.write( - 'DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash=?', - [user?.id, appUuid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY, key_hash], - ); - })(); - values.push(kv_row[0]?.value); - continue; - } - } - values.push(kv_entry?.value ?? null); - } - return multi ? values : values[0]; - } - /** - * - * @param {string} namespace - * @param {string[]} allKeys - * @returns - */ - async #getBatches (namespace: string, allKeys: string[]) { - - const batches: string[][] = []; - for ( let i = 0; i < allKeys.length; i += 100 ) { - batches.push(allKeys.slice(i, i + 100)); - } - const batchPromises = batches.map(async (keys) => { - const requests = [...new Set(keys)].map(k => ({ table: this.#tableName, items: { namespace, key: k } })); - const res = await this.#ddbClient.batchGet(requests); - const kvEntries = res.Responses?.[this.#tableName]; - const usage = res.ConsumedCapacity?.reduce((acc, curr) => acc + (curr.CapacityUnits ?? 0), 0); - return { kvEntries, usage }; - }); - - const batchGets = await Promise.all(batchPromises); - - return batchGets.reduce((acc, curr) => { - acc.kvEntries!.push(...curr?.kvEntries ?? []); - acc.usage! += curr.usage || 0; - return acc; - }, { kvEntries: [], usage: 0 }); - - } - - @Span('kv:set') - async set ({ key, value, expireAt, optConfig }: { key: string; value: unknown; expireAt?: number; optConfig?: { appUuid?: string } }): Promise { - const context = Context.get(); - const actor = context.get('actor'); - - if ( key === '' ) { - throw APIError.create('field_empty', undefined, { - key: 'key', - }); - } - - key = String(key); - if ( Buffer.byteLength(key, 'utf8') > 1024 ) { - throw new Error(`key is too large. Max size is ${1024}.`); - } - - if ( this.#enableMigrationFromSQL ) { - this.get({ key }); - } - - const namespace = this.#getNameSpace(actor, optConfig?.appUuid); - - const res = await this.#ddbClient.put(this.#tableName, { - namespace, - key, - value, - ttl: expireAt, - }); - - this.#meteringService.incrementUsage(actor, 'kv:write', res?.ConsumedCapacity?.CapacityUnits ?? 1); - return true; - } - - @Span('kv:batchPut') - async batchPut ({ - items, - optConfig, - }: { - items: Array<{ key: string; value: unknown; expireAt?: number }>; - optConfig?: { appUuid?: string }; - }): Promise { - const context = Context.get(); - const actor = context.get('actor'); - - if ( !Array.isArray(items) || items.length === 0 ) { - return true; - } - - const normalizedByKey = new Map(); - for ( const item of items ) { - const normalizedKey = String(item.key); - if ( normalizedKey === '' ) { - throw APIError.create('field_empty', undefined, { - key: 'key', - }); - } - - if ( Buffer.byteLength(normalizedKey, 'utf8') > 1024 ) { - throw new Error(`key is too large. Max size is ${1024}.`); - } - - normalizedByKey.set(normalizedKey, { - key: normalizedKey, - value: item.value, - expireAt: item.expireAt, - }); - } - - if ( this.#enableMigrationFromSQL ) { - for ( const key of normalizedByKey.keys() ) { - this.get({ key }); - } - } - - const namespace = this.#getNameSpace(actor, optConfig?.appUuid); - const putParams = Array.from(normalizedByKey.values()).map((item) => ({ - table: this.#tableName, - item: { - namespace, - key: item.key, - value: item.value, - ttl: item.expireAt, - }, - })); - const response = await this.#ddbClient.batchPut(putParams); - const usage = response.ConsumedCapacity?.reduce((acc, curr) => { - return acc + Number(curr.CapacityUnits ?? 0); - }, 0) ?? normalizedByKey.size; - - this.#meteringService.incrementUsage(actor, 'kv:write', usage || normalizedByKey.size); - return true; - } - - @Span('kv:del') - async del ({ key, optConfig}: { key: string;optConfig?: { appUuid?: string } }): Promise { - const actor = Context.get('actor'); - - const app = actor.type?.app ?? undefined; - const user = actor.type?.user ?? undefined; - if ( ! user ) throw new Error('User not found'); - - const namespace = this.#getNameSpace(actor, optConfig?.appUuid); - - const res = await this.#ddbClient.del(this.#tableName, { - namespace, - key, - }); - - this.#meteringService.incrementUsage(actor, 'kv:write', res?.ConsumedCapacity?.CapacityUnits ?? 1); - - if ( this.#enableMigrationFromSQL ) { - const key_hash = murmurhash.v3(key); - await this.#sqlClient.write( - 'DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash=?', - [user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY, key_hash], - ); - } - - return true; - } - - #encodeCursor (pageKey?: Record) { - if ( !pageKey || Object.keys(pageKey).length === 0 ) { - return undefined; - } - return Buffer.from(JSON.stringify(pageKey)).toString('base64'); - } - - #decodeCursor (cursor?: string | Record) { - if ( ! cursor ) { - return undefined; - } - if ( typeof cursor === 'object' ) { - return cursor; - } - if ( typeof cursor !== 'string' ) { - throw APIError.create('field_invalid', undefined, { - key: 'cursor', - }); - } - const trimmed = cursor.trim(); - if ( trimmed === '' ) { - return undefined; - } - try { - const decoded = Buffer.from(trimmed, 'base64').toString('utf8'); - return JSON.parse(decoded); - } catch ( e ) { - try { - return JSON.parse(trimmed); - } catch ( err ) { - throw APIError.create('field_invalid', undefined, { - key: 'cursor', - }); - } - } - } - - #normalizeLimit (limit?: number) { - if ( limit === undefined || limit === null ) { - return undefined; - } - const parsed = Number(limit); - if ( !Number.isFinite(parsed) || parsed <= 0 ) { - throw APIError.create('field_invalid', undefined, { - key: 'limit', - expected: 'positive number', - }); - } - return Math.floor(parsed); - } - - #normalizePattern (pattern?: string) { - if ( pattern === undefined || pattern === null ) { - return undefined; - } - if ( typeof pattern !== 'string' ) { - throw APIError.create('field_invalid', undefined, { - key: 'pattern', - }); - } - const trimmed = pattern.trim(); - if ( trimmed === '' ) { - return undefined; - } - if ( trimmed.endsWith('*') ) { - const prefix = trimmed.slice(0, -1); - return prefix === '' ? undefined : prefix; - } - return trimmed; - } - - @Span('kv:list') - async list ({ - as, - limit, - cursor, - pattern, - optConfig, - }: { - as?: 'keys' | 'values' | 'entries'; - limit?: number; - cursor?: string | Record; - pattern?: string; - optConfig?: { appUuid?: string } - }): Promise< - | string[] - | unknown[] - | { key: string; value: unknown; }[] - | { items: string[]; cursor?: string; } - | { items: unknown[]; cursor?: string; } - | { items: { key: string; value: unknown; }[]; cursor?: string; } - > { - const actor = Context.get('actor'); - - const app = actor.type?.app ?? undefined; - const user = actor.type?.user ?? undefined; - if ( ! user ) throw new Error('User not found'); - - const namespace = this.#getNameSpace(actor, optConfig?.appUuid); - - const normalizedLimit = this.#normalizeLimit(limit); - const pageKey = this.#decodeCursor(cursor); - const normalizedPattern = this.#normalizePattern(pattern); - const paginated = normalizedLimit !== undefined || pageKey !== undefined; - - const entriesRes = await this.#ddbClient.query( - this.#tableName, - { namespace }, - normalizedLimit ?? 0, - pageKey, - '', - false, - normalizedPattern ? { beginsWith: { key: 'key', value: normalizedPattern } } : undefined, - ); - - this.#meteringService.incrementUsage(actor, 'kv:read', entriesRes.ConsumedCapacity?.CapacityUnits ?? 1); - - let entries = entriesRes.Items ?? []; - - entries = entries?.filter(entry => { - if ( ! entry ) { - return false; - } - if ( entry.ttl && entry.ttl <= (Date.now() / 1000) ) { - return false; - } - return true; - }); - - if ( this.#enableMigrationFromSQL && !paginated ) { - const oldEntries = await this.#sqlClient.read( - 'SELECT * FROM kv WHERE user_id=? AND app=?', - [user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY], - ); - oldEntries.forEach(oldEntry => { - if ( normalizedPattern && !oldEntry.kkey?.startsWith(normalizedPattern) ) { - return; - } - if ( ! entries.find(e => e.key === oldEntry.kkey) ) { - if ( oldEntry.ttl && oldEntry.ttl <= (Date.now() / 1000) ) { - entries.push({ key: oldEntry.kkey, value: oldEntry.value }); - } - } - }); - } - - entries = entries?.map(entry => ({ - key: entry.key, - value: entry.value, - })); - - as = as || 'entries'; - - if ( ! ['keys', 'values', 'entries'].includes(as) ) { - throw APIError.create('field_invalid', undefined, { - key: 'as', - expected: '"keys", "values", or "entries"', - }); - } - - let items: string[] | unknown[] | { key: string; value: unknown; }[] = entries; - if ( as === 'keys' ) items = entries.map(entry => entry.key); - else if ( as === 'values' ) items = entries.map(entry => entry.value); - - if ( paginated ) { - const nextCursor = this.#encodeCursor(entriesRes.LastEvaluatedKey as Record | undefined); - if ( nextCursor ) { - return { items, cursor: nextCursor }; - } - return { items }; - } - - return items; - } - - @Span('kv:flush') - async flush ({ optConfig }: { optConfig?: { appUuid?: string } } = {}) { - const actor = Context.get('actor'); - - const app = actor.type.app ?? undefined; - const user = actor.type?.user ?? undefined; - if ( ! user ) throw new Error('User not found'); - - const namespace = this.#getNameSpace(actor, optConfig?.appUuid); - - // Query all keys - const entriesRes = await this.#ddbClient.query( - this.#tableName, - { namespace }, - ); - const entries = entriesRes.Items ?? []; - const readUsage = entriesRes?.ConsumedCapacity?.CapacityUnits ?? 0; - - // meter usage - this.#meteringService.incrementUsage(actor, 'kv:read', readUsage); - - // TODO DS: implement batch delete so its faster and less demanding on server - const allRes = (await Promise.all(entries.map(entry => { - try { - return this.#ddbClient.del(this.#tableName, { - namespace, - key: entry.key, - }); - } catch ( e ) { - console.error('Error deleting key', entry.key, e); - } - }))).filter(Boolean); - - const writeUsage = allRes.reduce((acc, curr) => acc + (curr?.ConsumedCapacity?.CapacityUnits ?? 0), 0); - - // meter usage - this.#meteringService.incrementUsage(actor, 'kv:write', writeUsage); - - if ( this.#enableMigrationFromSQL ) { - await this.#sqlClient.write( - 'DELETE FROM kv WHERE user_id=? AND app=?', - [user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY], - ); - } - - return !!allRes; - } - - @Span('kv:expireAt') - async expireAt ({ key, timestamp, optConfig}: { key: string; timestamp: number;optConfig?: { appUuid?: string } }): Promise { - if ( key === '' ) { - throw APIError.create('field_empty', null, { - key: 'key', - }); - } - - timestamp = Number(timestamp); - - return await this.#expireAt(key, timestamp, optConfig); - } - - @Span('kv:expire') - async expire ({ key, ttl, optConfig}: { key: string; ttl: number; optConfig?: { appUuid?: string } }): Promise { - if ( key === '' ) { - throw APIError.create('field_empty', null, { - key: 'key', - }); - } - - ttl = Number(ttl); - - // timestamp in seconds - let timestamp = Math.floor(Date.now() / 1000) + ttl; - - return await this.#expireAt(key, timestamp, optConfig); - } - - async #createPaths ( namespace: string, key: string, pathList: string[]) { - - const nestedMapValue = (() => { - const valueRoot: Record = {}; - let hasPaths = false; - pathList.forEach((valPath) => { - if ( ! valPath ) return; - hasPaths = true; - const chunks = valPath.split('.').filter(Boolean); - let cursor: Record = valueRoot; - for ( let i = 0; i < chunks.length - 1; i++ ) { - const chunk = chunks[i]; - const existing = cursor[chunk]; - if ( !existing || typeof existing !== 'object' || Array.isArray(existing) ) { - cursor[chunk] = {}; - } - cursor = cursor[chunk] as Record; - } - }); - return hasPaths ? valueRoot : null; - })(); - - if ( ! nestedMapValue ) { - return 0; - } - - const isPlainObject = (value: unknown): value is Record => { - return !!value && typeof value === 'object' && !Array.isArray(value); - }; - - const objectsEqual = (left: unknown, right: unknown): boolean => { - if ( left === right ) return true; - if ( !isPlainObject(left) || !isPlainObject(right) ) return false; - const leftKeys = Object.keys(left); - const rightKeys = Object.keys(right); - if ( leftKeys.length !== rightKeys.length ) return false; - for ( const key of leftKeys ) { - if ( ! rightKeys.includes(key) ) return false; - if ( ! objectsEqual(left[key], right[key]) ) return false; - } - return true; - }; - - // Collect all intermediate map paths for all entries - const allIntermediatePaths = new Set(); - pathList.forEach((valPath) => { - const chunks = ['value', ...valPath.split('.')].filter(Boolean); - // For each intermediate map (excluding the leaf) - for ( let i = 1; i < chunks.length; i++ ) { - const subPath = chunks.slice(0, i).join('.'); - allIntermediatePaths.add(subPath); - } - }); - - let writeUnits = 0; - // Ensure each intermediate map layer exists by issuing a separate DynamoDB update for each - const orderedPaths = [...allIntermediatePaths] - .sort((left, right) => left.split('.').length - right.split('.').length); - for ( const layerPath of orderedPaths ) { - // Build attribute names for the layer - const chunks = layerPath.split('.'); - const attrName = chunks.map((chunk) => `#${chunk}`.replaceAll(this.#pathCleanerRegex, '')).join('.'); - const expressionNames: Record = {}; - chunks.forEach((chunk) => { - const cleanedChunk = chunk.split(/\[\d*\]/g)[0]; - expressionNames[`#${cleanedChunk}`.replaceAll(this.#pathCleanerRegex, '')] = cleanedChunk; - }); - const isRootLayer = layerPath === 'value'; - const expressionValues = isRootLayer - ? { ':nestedMap': nestedMapValue } - : { ':emptyMap': {} }; - const valueToken = isRootLayer ? ':nestedMap' : ':emptyMap'; - // Issue update to set layer to {} if not exists - const layerUpsertRes = await this.#ddbClient.update( - this.#tableName, - { key, namespace }, - `SET ${attrName} = if_not_exists(${attrName}, ${valueToken})`, - expressionValues, - expressionNames, - ); - writeUnits += layerUpsertRes.ConsumedCapacity?.CapacityUnits ?? 0; - if ( isRootLayer && objectsEqual(layerUpsertRes.Attributes?.value, nestedMapValue) ) { - return writeUnits; - } - } - return writeUnits; - } - - // Ideally the paths support syntax like "a.b[2].c" - @Span('kv:incr') - async incr>({ key, pathAndAmountMap, optConfig }: { key: string; pathAndAmountMap: T;optConfig?: { appUuid?: string } }): Promise> { - if ( Object.values(pathAndAmountMap).find((v) => typeof v !== 'number') ) { - throw new Error('All values in pathAndAmountMap must be numbers'); - } - if ( key === '' ) { - throw APIError.create('field_empty', null, { - key: 'key', - }); - } - - if ( ! pathAndAmountMap ) { - throw new Error('invalid use of #incr: no pathAndAmountMap'); - } - - const actor = Context.get('actor'); - - const user = actor.type?.user ?? undefined; - if ( ! user ) throw new Error('User not found'); - - const namespace = this.#getNameSpace(actor, optConfig?.appUuid); - - if ( this.#enableMigrationFromSQL ) { - // trigger get to move element if exists - await this.get({ key }); - } - - const cleanerRegex = /[:\-+/*]/g; - - let writeUnits = await this.#createPaths(namespace, key, Object.keys(pathAndAmountMap)); - - const setStatements = Object.entries(pathAndAmountMap).map(([valPath, _amt], idx) => { - const path = ['value', ...valPath.split('.')].filter(Boolean).join('.'); - const attrName = path.split('.').map((chunk) => `#${chunk}`.replaceAll(cleanerRegex, '')).join('.'); - return `${attrName} = if_not_exists(${attrName}, :start${idx}) + :incr${idx}`; - }); - const valueAttributeValues = Object.entries(pathAndAmountMap).reduce((acc, [_path, amt], idx) => { - acc[`:incr${idx}`] = amt; - acc[`:start${idx}`] = 0; - return acc; - }, {} as Record); - const valueAttributeNames = Object.entries(pathAndAmountMap).reduce((acc, [valPath, _amt]) => { - const path = ['value', ...valPath.split('.')].filter(Boolean).join('.'); - path.split('.').forEach((chunk) => { - const cleanedChunk = chunk.split(/\[\d*\]/g)[0]; - acc[`#${cleanedChunk}`.replaceAll(cleanerRegex, '')] = cleanedChunk; - }); - return acc; - }, {} as Record); - - const res = await this.#ddbClient.update( - this.#tableName, - { key, namespace }, - `SET ${[...setStatements].join(', ')}`, - valueAttributeValues, - { ...valueAttributeNames, '#value': 'value' }, - ); - - writeUnits += res.ConsumedCapacity?.CapacityUnits ?? 0; - this.#meteringService.incrementUsage(actor, 'kv:write', writeUnits); - return res.Attributes?.value; - } - - async decr>({ key, pathAndAmountMap, optConfig }: { key: string; pathAndAmountMap: T; optConfig?: { appUuid?: string } }) { - return await this.incr({ key, pathAndAmountMap: Object.fromEntries(Object.entries(pathAndAmountMap).map(([k, v]) => [k, -v])) as T, optConfig }); - } - - @Span('kv:add') - async add ({ key, pathAndValueMap, optConfig}: { key: string; pathAndValueMap: Record; optConfig?: { appUuid?: string } }): Promise { - if ( !pathAndValueMap || Object.keys(pathAndValueMap).length === 0 ) { - throw new Error('invalid use of #add: no pathAndValueMap'); - } - if ( key === '' ) { - throw APIError.create('field_empty', null, { - key: 'key', - }); - } - - const actor = Context.get('actor'); - - const user = actor.type?.user ?? undefined; - if ( ! user ) throw new Error('User not found'); - - const namespace = this.#getNameSpace(actor, optConfig?.appUuid); - - if ( this.#enableMigrationFromSQL ) { - // trigger get to move element if exists - await this.get({ key }); - } - - const cleanerRegex = /[:\-+/*]/g; - - let writeUnits = await this.#createPaths(namespace, key, Object.keys(pathAndValueMap)); - - const setStatements = Object.entries(pathAndValueMap).map(([valPath, _val], idx) => { - const path = ['value', ...valPath.split('.')].filter(Boolean).join('.'); - const attrName = path.split('.').map((chunk) => `#${chunk}`.replaceAll(cleanerRegex, '')).join('.'); - return `${attrName} = list_append(if_not_exists(${attrName}, :emptyList${idx}), :append${idx})`; - }); - const valueAttributeValues = Object.entries(pathAndValueMap).reduce((acc, [_path, val], idx) => { - acc[`:append${idx}`] = Array.isArray(val) ? val : [val]; - acc[`:emptyList${idx}`] = []; - return acc; - }, {} as Record); - const valueAttributeNames = Object.entries(pathAndValueMap).reduce((acc, [valPath, _val]) => { - const path = ['value', ...valPath.split('.')].filter(Boolean).join('.'); - path.split('.').forEach((chunk) => { - const cleanedChunk = chunk.split(/\[\d*\]/g)[0]; - acc[`#${cleanedChunk}`.replaceAll(cleanerRegex, '')] = cleanedChunk; - }); - return acc; - }, {} as Record); - - const res = await this.#ddbClient.update( - this.#tableName, - { key, namespace }, - `SET ${[...setStatements].join(', ')}`, - valueAttributeValues, - { ...valueAttributeNames, '#value': 'value' }, - ); - - writeUnits += res.ConsumedCapacity?.CapacityUnits ?? 0; - this.#meteringService.incrementUsage(actor, 'kv:write', writeUnits); - return res.Attributes?.value; - } - - @Span('kv:remove') - async remove ({ key, paths, optConfig }: { key: string; paths: string[]; optConfig?: { appUuid?: string } }): Promise { - if ( !paths || paths.length === 0 ) { - throw new Error('invalid use of #remove: no paths'); - } - if ( key === '' ) { - throw APIError.create('field_empty', null, { - key: 'key', - }); - } - - const actor = Context.get('actor'); - - const user = actor.type?.user ?? undefined; - if ( ! user ) throw new Error('User not found'); - - const namespace = this.#getNameSpace(actor, optConfig?.appUuid); - - if ( this.#enableMigrationFromSQL ) { - // trigger get to move element if exists - await this.get({ key }); - } - - const cleanerRegex = /[:\-+/*]/g; - - const removeStatements = paths.map((valPath) => { - const path = ['value', ...valPath.split('.')].filter(Boolean).join('.'); - return path.split('.').map((chunk) => { - const cleanedChunk = chunk.split(/\[\d*\]/g)[0]; - const indexSuffix = chunk.slice(cleanedChunk.length); - return `${`#${cleanedChunk}`.replaceAll(cleanerRegex, '')}${indexSuffix}`; - }).join('.'); - }); - - const valueAttributeNames = paths.reduce((acc, valPath) => { - const path = ['value', ...valPath.split('.')].filter(Boolean).join('.'); - path.split('.').forEach((chunk) => { - const cleanedChunk = chunk.split(/\[\d*\]/g)[0]; - acc[`#${cleanedChunk}`.replaceAll(cleanerRegex, '')] = cleanedChunk; - }); - return acc; - }, {} as Record); - - try { - const res = await this.#ddbClient.update( - this.#tableName, - { key, namespace }, - `REMOVE ${removeStatements.join(', ')}`, - undefined, - { ...valueAttributeNames, '#value': 'value' }, - ); - - this.#meteringService.incrementUsage(actor, 'kv:write', res?.ConsumedCapacity?.CapacityUnits ?? 1); - return res.Attributes?.value; - } catch ( e ) { - const message = (e as Error)?.message ?? ''; - if ( (e as Error)?.name === 'ValidationException' && /document path|invalid updateexpression/i.test(message) ) { - this.#meteringService.incrementUsage(actor, 'kv:write', 1); - return await this.get({ key }); - } - throw e; - } - } - - @Span('kv:update') - async update ({ key, pathAndValueMap, ttl, optConfig }: { key: string; pathAndValueMap: Record; ttl?: number; optConfig?: { appUuid?: string } }): Promise { - if ( !pathAndValueMap || Object.keys(pathAndValueMap).length === 0 ) { - throw new Error('invalid use of #update: no pathAndValueMap'); - } - if ( key === '' ) { - throw APIError.create('field_empty', null, { - key: 'key', - }); - } - - const actor = Context.get('actor'); - - const user = actor.type?.user ?? undefined; - if ( ! user ) throw new Error('User not found'); - - const namespace = this.#getNameSpace(actor, optConfig?.appUuid); - - if ( this.#enableMigrationFromSQL ) { - // trigger get to move element if exists - await this.get({ key }); - } - - const cleanerRegex = /[:\-+/*]/g; - - let writeUnits = await this.#createPaths(namespace, key, Object.keys(pathAndValueMap)); - - const setStatements = Object.entries(pathAndValueMap).map(([valPath, _val], idx) => { - const path = ['value', ...valPath.split('.')].filter(Boolean).join('.'); - const attrName = path.split('.').map((chunk) => `#${chunk}`.replaceAll(cleanerRegex, '')).join('.'); - return `${attrName} = :value${idx}`; - }); - const valueAttributeValues = Object.entries(pathAndValueMap).reduce((acc, [_path, val], idx) => { - acc[`:value${idx}`] = val; - return acc; - }, {} as Record); - const valueAttributeNames = Object.entries(pathAndValueMap).reduce((acc, [valPath, _val]) => { - const path = ['value', ...valPath.split('.')].filter(Boolean).join('.'); - path.split('.').forEach((chunk) => { - const cleanedChunk = chunk.split(/\[\d*\]/g)[0]; - acc[`#${cleanedChunk}`.replaceAll(cleanerRegex, '')] = cleanedChunk; - }); - return acc; - }, {} as Record); - - if ( ttl !== undefined ) { - const ttlSeconds = Number(ttl); - if ( Number.isNaN(ttlSeconds) ) { - throw new Error('ttl must be a number'); - } - const timestamp = Math.floor(Date.now() / 1000) + ttlSeconds; - setStatements.push('#ttl = :ttl'); - valueAttributeValues[':ttl'] = timestamp; - valueAttributeNames['#ttl'] = 'ttl'; - } - - const res = await this.#ddbClient.update( - this.#tableName, - { key, namespace }, - `SET ${[...setStatements].join(', ')}`, - valueAttributeValues, - { ...valueAttributeNames, '#value': 'value' }, - ); - - writeUnits += res.ConsumedCapacity?.CapacityUnits ?? 0; - this.#meteringService.incrementUsage(actor, 'kv:write', writeUnits); - return res.Attributes?.value; - } - - async #expireAt (key: string, timestamp: number, optConfig?: { appUuid?: string }) { - - const actor = Context.get('actor'); - - const user = actor.type?.user ?? undefined; - if ( ! user ) throw new Error('User not found'); - - const namespace = this.#getNameSpace(actor, optConfig?.appUuid); - - // if possibly migrating from old SQL store, get entry first to move to dynamo - if ( this.#enableMigrationFromSQL ) { - await this.get({ key }); - } - - const res = await this.#ddbClient.update( - this.#tableName, - { key, namespace }, - 'SET #ttl = :ttl, #value = if_not_exists(#value, :defaultValue)', - { ':ttl': timestamp, ':defaultValue': null }, - { '#ttl': 'ttl', '#value': 'value' }, - ); - - // meter usage - this.#meteringService.incrementUsage(actor, 'kv:write', res?.ConsumedCapacity?.CapacityUnits ?? 1); - } - -} diff --git a/src/backend/src/services/DynamoKVStore/DynamoKVStoreWrapper.ts b/src/backend/src/services/DynamoKVStore/DynamoKVStoreWrapper.ts deleted file mode 100644 index d40e2e7f1..000000000 --- a/src/backend/src/services/DynamoKVStore/DynamoKVStoreWrapper.ts +++ /dev/null @@ -1,179 +0,0 @@ -import type { DDBClient } from '@heyputer/backend/src/clients/dynamodb/DDBClient.js'; -import { BaseService } from '@heyputer/backend/src/services/BaseService.js'; -import { randomUUID } from 'node:crypto'; -import { kv } from '../../util/kvSingleton.js'; -import { DynamoKVStore } from './DynamoKVStore.js'; - -const SECOND = 1000; -const DDB_OPERATION_LATENCY_FAIL_MS = 2 * SECOND; -const DDB_HEALTHCHECK_NAMESPACE = 'healthcheck'; -const DDB_HEALTHCHECK_RESULT_CACHE_WINDOW_SECONDS = 3; - -/** - * Wrapping implemenation for traits registration and use in our core structure - */ -class DynamoKVStoreServiceWrapper extends BaseService { - - kvStore!: DynamoKVStore; - ddbClient!: DDBClient; - kvStoreHealthcheckCache_: - { expiresAtMs: number; passed: boolean } | - null = null; - - async _init () { - const tableName = this.config.tableName || 'store-kv-v1'; - this.ddbClient = this.services.get('dynamo') as DDBClient; - this.kvStore = new DynamoKVStore({ - ddbClient: this.ddbClient, - sqlClient: this.services.get('database').get(), - meteringService: this.services.get('meteringService').meteringService, - tableName, - }); - await this.kvStore.createTableIfNotExists(); - - Object.getOwnPropertyNames(DynamoKVStore.prototype).forEach(fn => { - if ( fn === 'constructor' ) return; - this[fn] = (...args: unknown[]) => this.kvStore[fn](...args); - }); - - const checkDdbClientLatency = async () => { - const healthcheckCacheKey = `dynamodb:healthcheck:last-run:${tableName}`; - try { - const cachedHealthcheckResult = await kv.get(healthcheckCacheKey); - if ( cachedHealthcheckResult ) { - try { - const parsedCachedResult = JSON.parse(cachedHealthcheckResult); - if ( parsedCachedResult?.ok ) { - return parsedCachedResult; - } - throw new Error(parsedCachedResult?.error || 'cached dynamo kv healthcheck failure'); - } catch ( parseError ) { - if ( parseError instanceof SyntaxError ) { - // ignore invalid cache payload and run a fresh healthcheck - } else { - throw parseError; - } - } - } - } catch ( error ) { - if ( error instanceof Error ) throw error; - this.log.warn('unable to read dynamo healthcheck result cache; continuing', { - error: error instanceof Error ? error.message : String(error), - }); - } - - const healthcheckKey = randomUUID(); - const key = { - namespace: DDB_HEALTHCHECK_NAMESPACE, - key: healthcheckKey, - }; - const item = { - ...key, - value: Date.now(), - }; - const operationLatenciesMs = { - set: 0, - get: 0, - del: 0, - }; - const healthcheckResult: { - ok: boolean; - checkedAtMs: number; - operationLatenciesMs: typeof operationLatenciesMs; - error?: string; - } = { - ok: true, - checkedAtMs: Date.now(), - operationLatenciesMs, - }; - const writeHealthcheckResultToCache = async () => { - try { - await kv.set(healthcheckCacheKey, JSON.stringify(healthcheckResult), { - EX: DDB_HEALTHCHECK_RESULT_CACHE_WINDOW_SECONDS, - }); - } catch ( error ) { - this.log.warn('unable to write dynamo healthcheck result cache; continuing', { - error: error instanceof Error ? error.message : String(error), - }); - } - }; - - const runTimedOperation = async ( - operationName: keyof typeof operationLatenciesMs, - operation: () => Promise, - ) => { - const startedAt = Date.now(); - await Promise.race([ - operation(), - new Promise((_resolve, reject) => { - setTimeout(() => { - reject(new Error(`dynamo kv healthcheck ${operationName} timed out`)); - }, DDB_OPERATION_LATENCY_FAIL_MS); - }), - ]); - operationLatenciesMs[operationName] = Date.now() - startedAt; - }; - - try { - await runTimedOperation('set', async () => { - await this.ddbClient.put(tableName, item); - }); - await runTimedOperation('get', async () => { - await this.ddbClient.get(tableName, key); - }); - await runTimedOperation('del', async () => { - await this.ddbClient.del(tableName, key); - }); - } catch ( error ) { - healthcheckResult.ok = false; - healthcheckResult.error = error instanceof Error - ? error.message - : String(error); - await writeHealthcheckResultToCache(); - throw new Error(healthcheckResult.error); - } - - const exceededLatencyThreshold = Object.values(operationLatenciesMs) - .some(durationMs => durationMs > DDB_OPERATION_LATENCY_FAIL_MS); - if ( ! exceededLatencyThreshold ) { - await writeHealthcheckResultToCache(); - return healthcheckResult; - } - - healthcheckResult.ok = false; - healthcheckResult.error = - `dynamo kv healthcheck latency exceeded threshold ${DDB_OPERATION_LATENCY_FAIL_MS}ms`; - await writeHealthcheckResultToCache(); - throw new Error(healthcheckResult.error); - }; - - const svc_serverHealth = this.services.get('server-health'); - svc_serverHealth.add_check(`dynamo-kv:${tableName}`, async () => { - await checkDdbClientLatency(); - }).on_fail(async () => { - try { - await this.ddbClient.recreateClient(); - } catch ( recreateError ) { - this.log.error('failed to recreate dynamo client from server-health on_fail', { - error: recreateError instanceof Error ? recreateError.message : String(recreateError), - }); - } - }); - } - - static IMPLEMENTS = { - 'puter-kvstore': Object.getOwnPropertyNames(DynamoKVStore.prototype) - .filter(n => n !== 'constructor') - .reduce((acc, fn) => ({ - ...acc, - [fn]: async function (...a) { - return await (this as DynamoKVStoreServiceWrapper).kvStore[fn](...a); - }, - }), {}), - }; - -} - -export type IDynamoKVStoreWrapper = DynamoKVStoreServiceWrapper; - -export const DynamoKVStoreWrapper = DynamoKVStoreServiceWrapper as unknown as DynamoKVStore; diff --git a/src/backend/src/services/EmailService.js b/src/backend/src/services/EmailService.js deleted file mode 100644 index e5235d750..000000000 --- a/src/backend/src/services/EmailService.js +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const BaseService = require('./BaseService'); - -const TEMPLATES = { - 'new-referral': { - subject: 'You\'ve made a referral!', - html: ` -

Hi there,

-

A new user has used your referral code. Enjoy an extra {{storage_increase}} of storage, on the house!

-

Sincerely,

-

Puter

- `, - }, - 'approved-for-listing': { - subject: '\u{1f389} Your app has been approved for listing!', - html: ` -

Hi there,

-

-Exciting news! {{app_title}} is now approved and live on Puter App Center. It's now ready for users worldwide to discover and enjoy. -

-

-Next Step: As your app begins to gain traction with more users, we will conduct periodic reviews to assess its performance and user engagement. Once your app meets our criteria, we'll invite you to our Incentive Program. This exclusive program will allow you to earn revenue each time users open your app. So, keep an eye out for updates and stay tuned for this exciting opportunity! Make sure to share your app with your fans, friends and family to help it gain traction: https://puter.com/app/{{app_name}} -

- -

Best,
-The Puter Team -

- `, - }, - 'listing-rejected': { - subject: 'App Center Listing Request Rejected', - html: ` -

Hi{{#if owner_username}} {{owner_username}}{{/if}},

-

-Thanks for submitting {{app_title}} for the Puter App Center. We reviewed your listing and have rejected it for the following reason(s): -

-
{{{nl2br reason}}}
-

-Please update your app listing and resubmit when ready. If you have questions, just reply to this email. -

-

Best,
-The Puter Team -

- `, - }, - 'listing-update-request': { - subject: 'Update request for your app listing', - html: ` -

Hi{{#if owner_username}} {{owner_username}}{{/if}},

-

-Please update {{app_title}}. -

-

Requested updates:

-
{{message}}
-

Best,
-The Puter Team -

- `, - }, - 'email_change_request': { - subject: '\u{1f4dd} Confirm your email change', - html: ` -

Hi there,

-

-We received a request to link this email to the user "{{username}}" on Puter. If you made this request, please click the link below to confirm the change. If you did not make this request, please ignore this email. -

- -

-Confirm email change -

- `, - }, - 'email_change_notification': { - subject: '\u{1f4dd} Notification of email change', - html: ` -

Hi there,

-

-We're sending an email to let you know about a change to your account. -We have sent a confirmation to "{{new_email}}" to confirm an email change request. -If this was not you, please contact support@puter.com immediately. -

- `, - }, - 'password_change_notification': { - subject: '\u{1f511} Password change notification', - html: /*html*/` -

Hi there,

-

- We're sending an email to let you know about a change to your account. - Your password was recently changed. If this was not you, please contact - support@puter.com immediately. -

- `, - }, - 'email_verification_code': { - subject: '{{code}} is your confirmation code', - html: /*html*/` -

Hi there,

-

{{code}} is your email confirmation code.

-

Sincerely,

-

Puter

- `, - }, - 'email_verification_link': { - subject: 'Please confirm your email', - html: /*html*/` -

Hi there,

-

Please confirm your email address using this link: {{link}}.

-

Sincerely,

-

Puter

- `, - }, - 'email_password_recovery': { - subject: 'Password Recovery', - html: /*html*/` -

Hi there,

-

A password recovery request was issued for your account, please follow the link below to reset your password:

-

{{link}}

-

Sincerely,

-

Puter

- `, - }, - 'enabled_2fa': { - subject: '2FA Enabled on your Account', - html: ` -

Hi there,

-

We're sending you this email to let you know 2FA was successfully enabled - on your account

-

If you did not perform this action please contact support@puter.com - immediately

-

Sincerely,

-

Puter

- `, - }, - 'disabled_2fa': { - subject: '2FA Disabled on your Account', - html: ` -

Hi there,

-

We hope you did this on purpose! 2FA Was disabled on your account.

-

If you did not perform this action please contact support@puter.com - immediately

-

Sincerely,

-

Puter

- `, - }, - // TODO: revise email contents - 'share_by_username': { - subject: 'Puter share from {{susername}}', - html: /*html*/` -

Hi there {{rusername}},

-

You've received a share from {{susername}} on Puter.

-

Go to puter.com to check it out.

- {{#if message}} -

The following message was included:

-
{{message}}
- {{/if}} -

Sincerely,

-

Puter

- `, - }, - 'share_by_email': { - subject: 'share by email', - html: /*html*/` -

Hi there,

-

You've received a share from {{sender_name}} on Puter:

-

{{link}}

- {{#if message}} -

The following message was included:

-
{{message}}
- {{/if}} -

Sincerely,

-

Puter

- `, - }, -}; - -/** -* @class EmailService -* @extends BaseService -* @description The EmailService class handles the sending of emails using predefined templates. -* It utilizes the nodemailer library for sending emails and Handlebars for template rendering. -* The class includes methods for constructing and initializing the service, getting the email transport, -* and sending emails with provided templates and values. -*/ -class Emailservice extends BaseService { - static MODULES = { - nodemailer: require('nodemailer'), - handlebars: require('handlebars'), - dedent: require('dedent'), - }; - - /** - * Initializes the EmailService by compiling email templates. - * - * This method compiles the email templates using Handlebars and dedent - * to ensure that they are ready for use. It stores the compiled templates - * in an object for quick access. - * - * @returns {void} - */ - _construct () { - this.templates = TEMPLATES; - - const handlebars = this.modules.handlebars; - handlebars.registerHelper('nl2br', (text) => { - if ( text == null ) return ''; - const s = String(text) - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"'); - return new handlebars.SafeString(s.replace(/\n/g, '
')); - }); - - this.template_fns = {}; - for ( const k in this.templates ) { - const template = this.templates[k]; - this.template_fns[k] = values => { - const subject = this.modules.handlebars.compile(template.subject); - const html = - this.modules.handlebars.compile(this.modules.dedent(template.html)); - return { - ...template, - subject: subject(values), - html: html(values), - }; - }; - } - } - - /** - * Initializes the email service. - * This method is called during the initialization phase of the service. - * It sets up any necessary configurations or resources needed for the service to function correctly. - * - * @returns {void} - */ - _init () { - } - - /** - * Configures and initializes the email transport using Nodemailer. - * - * This method sets up the email transport configuration based on the provided settings and - * returns a configured Nodemailer transport object. - * - * @returns {Object} The configured Nodemailer transport object. - */ - get_transport_ () { - const nodemailer = this.modules.nodemailer; - - const config = { ...this.config }; - delete config.engine; - - let transport = nodemailer.createTransport(config); - - return transport; - } - - /** - * Sends an email using the configured transport and template. - * - * This method constructs an email message by applying the provided values to the specified template, - * then sends the email using the configured transport. - * - * @param {Object} user - The user object containing the email address. - * @param {string} template - The template key to use for constructing the email. - * @param {Object} values - The values to apply to the template. - * @returns {Promise} - A promise that resolves when the email is sent. - */ - async send_email (user, template, values) { - const email = user.email; - - const template_fn = this.template_fns[template]; - const { subject, html } = template_fn(values); - - const transporter = this.get_transport_(); - transporter.sendMail({ - from: '"Puter" no-reply@puter.com', // sender address - to: email, // list of receivers - subject, - html, - }); - } - - // simple passthrough to nodemailer - sendMail (params) { - const transporter = this.get_transport_(); - transporter.sendMail(params); - } -} - -module.exports = { - Emailservice, -}; diff --git a/src/backend/src/services/EntityStoreService.js b/src/backend/src/services/EntityStoreService.js deleted file mode 100644 index 4d28a6898..000000000 --- a/src/backend/src/services/EntityStoreService.js +++ /dev/null @@ -1,344 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../api/APIError'); -const { Entity } = require('../om/entitystorage/Entity'); -const { IdentifierUtil } = require('../om/IdentifierUtil'); -const { Null, And, Eq, PredicateUtil } = require('../om/query/query'); -const { Context } = require('../util/context'); -const BaseService = require('./BaseService'); - -/** -* EntityStoreService - A service class that manages entity-related operations in the backend of Puter. -* This class extends BaseService to provide methods for creating, reading, updating, selecting, -* upserting, and deleting entities. It interacts with an upstream data provider to perform these -* operations, ensuring consistency and providing context-aware functionality for entity management. -*/ -class EntityStoreService extends BaseService { - /** - * Initializes the EntityStoreService with necessary entity and upstream configurations. - * - * @param {Object} args - The initialization arguments. - * @param {string} args.entity - The name of the entity to operate on. Required. - * @param {Object} args.upstream - The upstream service to handle operations. - * - * @throws {Error} If `args.entity` is not provided. - * - * @returns {Promise} A promise that resolves when initialization is complete. - * - * @note This method sets up the context for the entity operations and provides it to the upstream service. - */ - async _init (args) { - if ( ! args.entity ) { - throw new Error('EntityStoreService requires an entity name'); - } - - this.upstream = args.upstream; - - const context = Context.get().sub({ services: this.services }); - const om = this.services.get('registry').get('om:mapping').get(args.entity); - this.om = om; - await this.upstream.provide_context({ - context, - om, - entity_name: args.entity, - }); - } - - static IMPLEMENTS = { - 'crud-q': { - async create ({ object, options }) { - if ( object.hasOwnProperty(this.om.primary_identifier) ) { - throw APIError.create('field_not_allowed_for_create', null, { - key: this.om.primary_identifier, - }); - } - const entity = await Entity.create({ om: this.om }, object); - return await this.create(entity, options); - }, - async update ({ object, id, options }) { - const entity = await Entity.create({ om: this.om }, object); - return await this.update(entity, id, options); - }, - async upsert ({ object, id, options }) { - const entity = await Entity.create({ om: this.om }, object); - return await this.upsert(entity, id, options); - }, - async read ({ uid, id, params = {} }) { - return await Context.sub({ - es_params: params, - }).arun(async () => { - if ( !uid && !id ) { - throw APIError.create('xor_field_missing', null, { - names: ['uid', 'id'], - }); - } - - const entity = await this.fetch_based_on_either_id_(uid, id); - if ( ! entity ) { - throw APIError.create('entity_not_found', null, { - identifier: uid, - }); - } - return await entity.get_client_safe(); - }); - }, - async select (options) { - return await Context.sub({ - es_params: options?.params ?? {}, - }).arun(async () => { - const entities = await this.select(options); - const promises = []; - for ( const entity of entities ) { - promises.push(entity.get_client_safe()); - } - const client_safe_entities = await Promise.all(promises); - return client_safe_entities; - }); - }, - async delete ({ uid, id }) { - if ( !uid && !id ) { - throw APIError.create('xor_field_missing', null, { - names: ['uid', 'id'], - }); - } - - if ( id && !uid ) { - const entity = await this.fetch_based_on_complex_id_(id); - if ( ! entity ) { - throw APIError.create('entity_not_found', null, { - identifier: id, - }); - } - uid = await entity.get(this.om.primary_identifier); - } - - return await this.delete(uid); - }, - }, - }; - - // TODO: can replace these with MethodProxyFeature - /** - * Create a new entity in the store. - * - * @param {Object} entity - The entity to add. - * @param {Object} options - Additional options for the update operation. - * @returns {Promise} The updated entity after the operation. - */ - async create (entity, options) { - return await this.upstream.upsert(entity, { old_entity: null, options }); - } - /** - * Reads an entity from the upstream data store using its unique identifier. - * - * @param {string} uid - The unique identifier of the entity to read. - * @returns {Promise} A promise that resolves to the entity object if found. - * @throws {APIError} If the entity with the given `uid` does not exist. - */ - async read (uid) { - return await this.upstream.read(uid); - } - /** - * Retrieves an entity by its unique identifier (UID). - * - * @param {string} uid - The unique identifier of the entity to retrieve. - * @returns {Promise} The entity associated with the given UID. - * @throws {Error} If the entity cannot be found or an error occurs during retrieval. - */ - async select ({ predicate, ...rest }) { - if ( ! predicate ) predicate = []; - if ( Array.isArray(predicate) ) { - const [p_op, ...p_args] = predicate; - predicate = await this.upstream.create_predicate(p_op, ...p_args); - } - if ( ! predicate ) predicate = new Null(); - return await this.upstream.select({ predicate, ...rest }); - } - /* Updates an existing entity in the store. - * - * @param {Object} entity - The entity to update with new values. - * @param {string|number} id - The identifier of the entity to update. Can be a string or number. - * @param {Object} options - Additional options for the update operation. - * @returns {Promise} The updated entity after the operation. - * @throws {APIError} If the entity to be updated is not found. - * - * @note This method first attempts to fetch the entity by its primary identifier. If not found, - * it uses `IdentifierUtil` to detect and fetch by other identifiers if provided. - * If the entity still isn't found, an error is thrown. The method ensures that the - * entity's primary identifier is updated to match the existing entity before performing - * the actual update through `this.upstream.update`. - */ - async update (entity, id, options) { - let old_entity = await this.read(await entity.get(this.om.primary_identifier)); - - if ( ! old_entity ) { - const idu = new IdentifierUtil({ - om: this.om, - }); - - const predicate = await idu.detect_identifier(id ?? {}, true); - if ( predicate ) { - const maybe_entity = await this.select({ predicate, limit: 1 }); - if ( maybe_entity.length ) { - old_entity = maybe_entity[0]; - } - } - - if ( ! old_entity ) { - throw APIError.create('entity_not_found', null, { - identifier: PredicateUtil.write_human_readable(predicate) - || await entity.get(this.om.primary_identifier), - }); - } - } - - // Set primary identifier's value of `entity` to that in `old_entity` - const id_prop = this.om.properties[this.om.primary_identifier]; - await entity.set(id_prop.name, await old_entity.get(id_prop.name)); - - return await this.upstream.upsert(entity, { old_entity, options }); - } - /** - * Updates an existing entity in the store or creates a new one. - * - * @param {Object} entity - The entity to update with new values. - * @param {string|number} id - The identifier of the entity to update. Can be a string or number. - * @param {Object} options - Additional options for the update operation. - * @returns {Promise} The updated entity after the operation. - * @throws {APIError} If the entity to be updated is not found. - * - * @note This method first attempts to fetch the entity by its primary identifier. If not found, - * it uses `IdentifierUtil` to detect and fetch by other identifiers if provided. - * If the entity still isn't found, an error is thrown. The method ensures that the - * entity's primary identifier is updated to match the existing entity before performing - * the actual update through `this.upstream.upsert`. - */ - async upsert (entity, id, options) { - let old_entity = await this.read(await entity.get(this.om.primary_identifier)); - - if ( ! old_entity ) { - const idu = new IdentifierUtil({ - om: this.om, - }); - - const predicate = await idu.detect_identifier(entity); - if ( predicate ) { - const maybe_entity = await this.select({ predicate, limit: 1 }); - if ( maybe_entity.length ) { - old_entity = maybe_entity[0]; - } - } - } - - if ( old_entity ) { - // Set primary identifier's value of `entity` to that in `old_entity` - const id_prop = this.om.properties[this.om.primary_identifier]; - await entity.set(id_prop.name, await old_entity.get(id_prop.name)); - } - - return await this.upstream.upsert(entity, { old_entity, options }); - } - /** - * Deletes an entity from the store. - * - * @param {string} uid - The unique identifier of the entity to delete. - * @returns {Promise} A promise that resolves when the entity is deleted. - * @throws {APIError} If the entity with the given `uid` is not found. - * - * This method first attempts to read the entity with the given `uid`. If the entity - * does not exist, it throws an `APIError` with the message 'entity_not_found'. - * If the entity exists, it calls the upstream service to delete the entity, - * passing along the old entity data for reference. - */ - async delete (uid) { - const old_entity = await this.read(uid); - if ( ! old_entity ) { - throw APIError.create('entity_not_found', null, { - identifier: uid, - }); - } - return await this.upstream.delete(uid, { old_entity }); - } - - async fetch_based_on_complex_id_ (id) { - // Ensure `id` is an object and get its keys - if ( !id || typeof id !== 'object' || Array.isArray(id) ) { - throw APIError.create('invalid_id', null, { id }); - } - - const id_keys = Object.keys(id); - // sort keys alphabetically - id_keys.sort(); - - // Ensure key set is valid based on redundant keys listing - const redundant_identifiers = this.om.redundant_identifiers ?? []; - - let match_found = false; - for ( let key of redundant_identifiers ) { - // Either a single key or a list - key = Array.isArray(key) ? key : [key]; - - // All keys in the list must be present in the id - for ( let i = 0 ; i < key.length ; i++ ) { - if ( ! id_keys.includes(key[i]) ) { - break; - } - if ( i === key.length - 1 ) { - match_found = true; - break; - } - } - } - - if ( ! match_found ) { - throw APIError.create('invalid_id', null, { id }); - } - - // Construct a query predicate based on the keys - const key_eqs = []; - for ( const key of id_keys ) { - key_eqs.push(new Eq({ - key, - value: id[key], - })); - } - let predicate = new And({ children: key_eqs }); - - // Perform a select - const entity = await this.read({ predicate }); - if ( ! entity ) { - return null; - } - - // Ensure there is only one result - return entity; - } - - async fetch_based_on_either_id_ (uid, id) { - if ( uid ) { - return await this.read(uid); - } - - return await this.fetch_based_on_complex_id_(id); - } -} - -module.exports = { - EntityStoreService, -}; diff --git a/src/backend/src/services/EntriService.js b/src/backend/src/services/EntriService.js deleted file mode 100644 index 953ecd853..000000000 --- a/src/backend/src/services/EntriService.js +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Copyright (C) 2025-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const BaseService = require('./BaseService'); - -const { Entity } = require('../om/entitystorage/Entity');; -const eggspress = require('../api/eggspress'); -// const { get_app, subdomain } = require("../helpers"); -let parseDomain ; -const { Eq } = require('../om/query/query'); -const { createHash } = require('crypto'); -const { NULL } = require('../om/proptypes/__all__'); -const APIError = require('../api/APIError'); - -// async function generateJWT(applicationId, secret, domain, ) { - -// return (await response.json()).auth_token; -// } - -class EntriService extends BaseService { - _init () { - - } - - async _construct () { - parseDomain = (await import('parse-domain')).parseDomain; - } - - '__on_install.routes' (_, { app }) { - app.use(eggspress('/entri/webhook', { - allowedMethods: ['POST', 'GET'], - /** - * - * @param {IncomingMessage} req - * @param {*} res - */ - }, async (req, res) => { - if ( createHash('sha256').update(req.body.id + this.config.secret).digest('hex') !== req.headers['entri-signature'] ) { - res.status(401).send('Lol'); - return; - } - if ( ! req.body.data.records_propagated ) { - return; - } - let rootDomain = false; - if ( req.body.data.records_propagated[0].type === 'A' ) { - rootDomain = true; - } - - let realDomain = (rootDomain ? '' : (`${req.body.subdomain }.`)) + req.body.domain; - const svc_su = this.services.get('su'); - - const es_subdomain = this.services.get('es:subdomain'); - - await svc_su.sudo(async () => { - const rows = (await es_subdomain.select({ predicate: new Eq({ key: 'domain', value: `in-progress:${ realDomain}` }) })); - for ( const row of rows ) { - const entity = await Entity.create({ om: es_subdomain.om }, { - uid: row.values_.uid, - domain: realDomain, - }); - await es_subdomain.upsert(entity); - - } - return true; - }); - - res.end('ok'); - })); - - const svc_web = this.services.get('web-server'); - svc_web.allow_undefined_origin('/entri/webhook', '/entri/webhook'); - } - - static IMPLEMENTS = { - 'entri': { - async getConfig ({ domain, userHostedSite }) { - const es_subdomain = this.services.get('es:subdomain'); - const svc_su = this.services.get('su'); - - let rootDomain = (parseDomain(domain)).icann.subDomains.length === 0; - - const exists = await svc_su.sudo(async () => { - const row = (await es_subdomain.select({ predicate: new Eq({ key: 'domain', value: domain }) }))[0] || (await es_subdomain.select({ predicate: new Eq({ key: 'domain', value: `in-progress:${ domain}` }) }))[0]; - if ( !!row && row.values_.subdomain === userHostedSite.replace('.puter.site', '') ) { - return false; - } - return !!row; - }); - - if ( exists ) { - throw APIError.create('already_in_use', null, { what: 'domain', value: domain }); - } - - const dnsRecords = rootDomain ? [{ - type: 'A', - host: '@', - value: '{ENTRI_SERVERS}', //This will be automatically replaced for the Entri servers IPs - ttl: 300, - applicationUrl: userHostedSite, - }] : [{ - type: 'CNAME', - value: 'power.goentri.com', // `{CNAME_TARGET}` will NOT automatically use the CNAME target as implied by the documentation - host: '{SUBDOMAIN}', // This will use the user inputted subdomain. If hostRequired is set to true, then this will default to "www" - ttl: 300, - applicationUrl: userHostedSite, - }]; - - const response = await fetch('https://api.goentri.com/token', { - method: 'POST', - body: JSON.stringify({ - applicationId: this.config.applicationId, - secret: this.config.secret, - domain, - // dnsRecords - }), - }); - - const row = (await es_subdomain.select({ predicate: new Eq({ key: 'subdomain', value: userHostedSite.replace('.puter.site', '') }) }))[0]; - const entity = await Entity.create({ om: es_subdomain.om }, { - uid: row.values_.uid, - domain: `in-progress:${ domain}`, - }); - - await es_subdomain.upsert(entity); - - return { - token: (await response.json()).auth_token, - applicationId: this.config.applicationId, - power: true, - dnsRecords, - prefilledDomain: domain, - hostRequired: false, - }; - - // let rootDomain = (parseDomain(domain)).icann.subDomains.length === 0; - - // const response = await fetch('https://api.goentri.com/power?' + new URLSearchParams({ - // domain, - // rootDomain - // }), { - // method: 'GET', - // headers: { - // 'Content-Type': 'application/json', - // 'Authorization': jwtForVerification, - // 'applicationId': this.config.applicationId - // } - // }); - - // const data = await response.json(); - // if (!data.eligible) { - // throw new APIError(); // figure this out later - // } - - }, - async deleteMapping ({ domain }) { - if ( domain.startsWith('in-progress') ) - { - throw APIError.create('field_invalid', null, { key: 'domain', expected: 'valid domain' }); - } - - /** @type {import("../om/entitystorage/SubdomainES")} */ - const es_subdomain = this.services.get('es:subdomain'); - - const row = (await es_subdomain.select({ predicate: new Eq({ key: 'domain', value: domain }) }))[0] || (await es_subdomain.select({ predicate: new Eq({ key: 'domain', value: `in-progress:${ domain}` }) }))[0]; - if ( ! row ) { - throw APIError.create('forbidden', null, {}); - } - - let inProgress = false; - if ( row.values_.domain.startsWith('in-progress:') ) { - inProgress = true; - } - - // Get token from Entri - const { auth_token } = await (fetch('https://api.goentri.com/token', { - method: 'POST', - body: JSON.stringify({ - applicationId: this.config.applicationId, - secret: this.config.secret, - }), - }).then(r => r.json())); - - const entity = await Entity.create({ om: es_subdomain.om }, { - uid: row.values_.uid, - domain: NULL, - }); - await es_subdomain.upsert(entity); - const errors = []; - // Even if the domain is in progress, still send the delete incase it's just propgation taking a while - const deleteRequest = await (fetch('https://api.goentri.com/power', { - method: 'DELETE', - headers: { - applicationId: this.config.applicationId, - 'Authorization': `Bearer ${ auth_token}`, - }, - body: JSON.stringify({ domain }), - })); - if ( deleteRequest.status !== 200 ) { - errors.push(await deleteRequest.text()); - } - - return { ok: true, errors }; - - }, - async fullyRegistered ({ domain, userHostedSite }) { - const es_subdomain = this.services.get('es:subdomain'); - const row = (await es_subdomain.select({ predicate: new Eq({ key: 'subdomain', value: userHostedSite.replace('.puter.site', '') }) }))[0]; - - }, - }, - }; - async '__on_driver.register.interfaces' () { - const svc_registry = this.services.get('registry'); - const col_interfaces = svc_registry.get('interfaces'); - - col_interfaces.set('entri', { - description: 'Execute code with various languages.', - methods: { - getConfig: { - description: 'get JWT for entri', - parameters: { - domain: { - type: 'string', - optional: false, - }, - userHostedSite: { - type: 'string', - optional: false, - }, - }, - result: { type: 'json' }, - }, - deleteMapping: { - description: 'delete domain mapping from entri', - parameters: { - domain: { - type: 'string', - optional: false, - }, - }, - result: { type: 'json' }, - }, - }, - }); - } -} - -module.exports = { - EntriService, -}; diff --git a/src/backend/src/services/EventService.js b/src/backend/src/services/EventService.js deleted file mode 100644 index e05fdb727..000000000 --- a/src/backend/src/services/EventService.js +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { Context } = require('../util/context'); -const BaseService = require('./BaseService'); - -/** - * A proxy to EventService or another scoped event bus, allowing for - * emitting or listening on a prefix (ex: `a.b.c`) without the user - * of the scoped bus needed to know what the prefix is. - */ -class ScopedEventBus { - constructor (event_bus, scope) { - this.event_bus = event_bus; - this.scope = scope; - } - - async emit (key, data) { - await this.event_bus.emit(`${this.scope }.${ key}`, data); - } - - on (key, callback) { - return this.event_bus.on(`${this.scope }.${ key}`, callback); - } -} - -/** -* Class representing the EventService, which extends the BaseService. -* This service is responsible for managing event listeners and emitting -* events within a scoped context, allowing for flexible event handling -* and decoupled communication between different parts of the application. -*/ -class EventService extends BaseService { - /** - * Initializes listeners and global listeners for the EventService. - * This method is called to set up the internal data structures needed - * for managing event listeners upon construction of the service. - * - * @async - * @returns {Promise} A promise that resolves when the initialization is complete. - */ - async _construct () { - this.listeners_ = {}; - this.global_listeners_ = []; - } - - async '__on_boot.ready' () { - this.emit('ready', {}, {}); - } - - async emit (key, data, meta) { - meta = meta ?? {}; - const parts = key.split('.'); - for ( let i = 0; i < parts.length; i++ ) { - const part = i === parts.length - 1 - ? parts.join('.') - : `${parts.slice(0, i + 1).join('.') }.*`; - - // actual emit - const listeners = this.listeners_[part]; - if ( ! listeners ) continue; - for ( const callback of listeners ) { - // IIAFE wrapper to catch errors without blocking - // event dispatch. - await Context.arun(async () => { - try { - await callback(key, data, meta); - } catch (e) { - this.errors.report('event-service.emit', { - source: e, - trace: true, - alarm: true, - }); - } - }); - } - } - - for ( const callback of this.global_listeners_ ) { - // IIAFE wrapper to catch errors without blocking - // event dispatch. - /** - * Invokes all registered global listeners for an event with the provided key, data, and meta - * information. Each callback is executed within a context that handles errors gracefully, - * ensuring that one failing listener does not disrupt subsequent invocations. - * - * @param {string} key - The event key to emit. - * @param {*} data - The data to be passed to the listeners. - * @param {Object} [meta={}] - Optional metadata related to the event. - * @returns {void} - */ - await Context.arun(async () => { - try { - await callback(key, data, meta); - } catch (e) { - this.errors.report('event-service.emit', { - source: e, - trace: true, - alarm: true, - }); - } - }); - } - - } - - /** - * Registers a callback function for the specified event selector. - * - * This method will push the provided callback onto the list of listeners - * for the event specified by the selector. It returns an object containing - * a detach method, which can be used to remove the listener. - * - * @param {string} selector - The event selector to listen for. - * @param {Function} callback - The function to be invoked when the event is emitted. - * @returns {Object} An object with a detach method to unsubscribe the listener. - */ - on (selector, callback) { - const listeners = this.listeners_[selector] || - (this.listeners_[selector] = []); - - listeners.push(callback); - - const det = { - detach: () => { - const idx = listeners.indexOf(callback); - if ( idx !== -1 ) { - listeners.splice(idx, 1); - } - }, - }; - - return det; - } - - on_all (callback) { - this.global_listeners_.push(callback); - } - - get_scoped (scope) { - return new ScopedEventBus(this, scope); - } -} - -module.exports = { - EventService, -}; diff --git a/src/backend/src/services/EventService.test.ts b/src/backend/src/services/EventService.test.ts deleted file mode 100644 index 37b2e6182..000000000 --- a/src/backend/src/services/EventService.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { createTestKernel } from '../../tools/test.mjs'; -import { EventService } from './EventService'; - -describe('EventService', async () => { - const testKernel = await createTestKernel({ - serviceMap: { - 'event-test': EventService, - }, - initLevelString: 'init', - }); - - const eventService = testKernel.services!.get('event-test') as EventService; - - it('should be instantiated', () => { - expect(eventService).toBeInstanceOf(EventService); - }); - - it('should emit and receive events', async () => { - let received = false; - eventService.on('test.event', () => { - received = true; - }); - - await eventService.emit('test.event', {}); - expect(received).toBe(true); - }); - - it('should pass data to event listeners', async () => { - let receivedData: any = null; - eventService.on('data.event', (key, data) => { - receivedData = data; - }); - - await eventService.emit('data.event', { value: 42 }); - expect(receivedData).toEqual({ value: 42 }); - }); - - it('should support wildcard listeners', async () => { - const received: string[] = []; - eventService.on('wild.*', (key) => { - received.push(key); - }); - - await eventService.emit('wild.test1', {}); - await eventService.emit('wild.test2', {}); - - expect(received).toContain('wild.test1'); - expect(received).toContain('wild.test2'); - }); - - it('should support multiple listeners on same event', async () => { - let count = 0; - eventService.on('multi.event', () => { count++; }); - eventService.on('multi.event', () => { count++; }); - - await eventService.emit('multi.event', {}); - expect(count).toBe(2); - }); - - it('should detach listeners', async () => { - let count = 0; - const det = eventService.on('detach.event', () => { count++; }); - - await eventService.emit('detach.event', {}); - expect(count).toBe(1); - - det.detach(); - await eventService.emit('detach.event', {}); - expect(count).toBe(1); // Should still be 1 - }); - - it('should support global listeners', async () => { - let globalReceived = false; - eventService.on_all(() => { - globalReceived = true; - }); - - await eventService.emit('any.event', {}); - expect(globalReceived).toBe(true); - }); - - it('should create scoped event bus', () => { - const scoped = eventService.get_scoped('test.scope'); - expect(scoped).toBeDefined(); - expect(scoped.scope).toBe('test.scope'); - }); - - it('should emit events through scoped bus', async () => { - let received = false; - eventService.on('scope.test.event', () => { - received = true; - }); - - const scoped = eventService.get_scoped('scope.test'); - await scoped.emit('event', {}); - expect(received).toBe(true); - }); -}); - diff --git a/src/backend/src/services/FeatureFlagService.js b/src/backend/src/services/FeatureFlagService.js deleted file mode 100644 index 03a90c4be..000000000 --- a/src/backend/src/services/FeatureFlagService.js +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { Context } = require('../util/context'); -const { PermissionUtil } = require('./auth/permissionUtils.mjs'); -const BaseService = require('./BaseService'); - -/** - * @class FeatureFlagService - * @extends BaseService - * - * FeatureFlagService is a way to let the client (frontend) know what features - * are enabled or disabled for the current user. - * - * A service that manages feature flags to control feature availability across the application. - * Provides methods to register, check, and retrieve feature flags based on user permissions and configurations. - * Integrates with the permission system to determine feature access for different users. - * Supports both static configuration flags and dynamic function-based feature flags. - */ -class FeatureFlagService extends BaseService { - /** - * Initializes the FeatureFlagService instance by setting up an empty Map for known flags - * @private - * @method - */ - _construct () { - this.known_flags = new Map(); - } - - /** - * Initializes the feature flag service by registering a provider with the whoami service. - * This provider adds feature flag information to user details when requested. - * - * @async - * @private - * @returns {Promise} - */ - async _init () { - const svc_detailProvider = this.services.get('whoami'); - svc_detailProvider.register_provider(async (context, out) => { - if ( ! context.actor ) return; - out.feature_flags = await this.get_summary(context.actor); - }); - } - - /** - * Registers a new feature flag with the service - * @param {string} name - The name/identifier of the feature flag - * @param {Object|boolean} spec - The specification for the flag. Can be a boolean value or an object with $ property indicating flag type - */ - register (name, spec) { - this.known_flags.set(name, spec); - } - - /** - * checks is a feature flag is enabled for the current user - * @return {boolean} true if the feature flag is enabled, false otherwise - * - * @example with a specified actor - * check({ actor }, 'flag-name'); - * @example with actor in context - * check('flag-name'); - */ - async check (...a) { - // allows binding call with multiple options objects; - // the last argument is the permission to check - const { options, value: permission } = (() => { - let value; - const options = {}; - for ( const arg of a ) { - if ( arg && typeof arg === 'object' && !Array.isArray(arg) ) { - Object.assign(options, arg); - continue; - } - value = arg; - break; - } - return { options, value }; - })(); - - if ( ! this.known_flags.has(permission) ) { - this.known_flags.set(permission, true); - } - - if ( this.known_flags.get(permission)?.$ === 'config-flag' ) { - return this.known_flags.get(permission)?.value; - } - - const actor = options.actor ?? Context.get('actor'); - - if ( this.known_flags.get(permission)?.$ === 'function-flag' ) { - return await this.known_flags.get(permission)?.fn({ - ...options, - actor, - }); - } - - const svc_permission = this.services.get('permission'); - const reading = await svc_permission.scan(actor, `feature:${permission}`); - const l = PermissionUtil.reading_to_options(reading); - if ( l.length === 0 ) return false; - return true; - } - - /** - * Gets a summary of all feature flags for a given actor - * @param {Object} actor - The actor to check feature flags for - * @returns {Promise} Object mapping feature flag names to their values: - * - For config flags: returns the configured value - * - For function flags: returns result of calling the flag function - * - For permission flags: returns true if actor has any matching permissions, false otherwise - */ - async get_summary (actor) { - const summary = {}; - for ( const [key, value] of this.known_flags.entries() ) { - if ( value.$ === 'config-flag' ) { - summary[key] = value.value; - continue; - } - if ( value.$ === 'function-flag' ) { - summary[key] = await value.fn({ actor }); - continue; - } - const svc_permission = this.services.get('permission'); - const reading = await svc_permission.scan(actor, `feature:${key}`); - const l = PermissionUtil.reading_to_options(reading); - summary[key] = l.length > 0; - } - - return summary; - } -} - -module.exports = { - FeatureFlagService, -}; diff --git a/src/backend/src/services/FeatureFlagService.test.ts b/src/backend/src/services/FeatureFlagService.test.ts deleted file mode 100644 index 42f0ca0ca..000000000 --- a/src/backend/src/services/FeatureFlagService.test.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { createTestKernel } from '../../tools/test.mjs'; -import { FeatureFlagService } from './FeatureFlagService'; - -describe('FeatureFlagService', async () => { - const testKernel = await createTestKernel({ - serviceMap: { - 'feature-flag': FeatureFlagService, - }, - initLevelString: 'init', - testCore: true, - }); - - const featureFlagService = testKernel.services!.get('feature-flag') as FeatureFlagService; - - it('should be instantiated', () => { - expect(featureFlagService).toBeInstanceOf(FeatureFlagService); - }); - - it('should register feature flags', () => { - featureFlagService.register('test-flag', true); - expect(featureFlagService.known_flags.has('test-flag')).toBe(true); - }); - - it('should register config flags', () => { - featureFlagService.register('config-flag', { $: 'config-flag', value: true }); - expect(featureFlagService.known_flags.get('config-flag')).toEqual({ $: 'config-flag', value: true }); - }); - - it('should check config flags', async () => { - featureFlagService.register('enabled-flag', { $: 'config-flag', value: true }); - const result = await featureFlagService.check('enabled-flag'); - expect(result).toBe(true); - }); - - it('should check disabled config flags', async () => { - featureFlagService.register('disabled-flag', { $: 'config-flag', value: false }); - const result = await featureFlagService.check('disabled-flag'); - expect(result).toBe(false); - }); - - it('should register function flags', () => { - featureFlagService.register('fn-flag', { - $: 'function-flag', - fn: async () => true, - }); - expect(featureFlagService.known_flags.has('fn-flag')).toBe(true); - }); - - it('should check function flags', async () => { - featureFlagService.register('dynamic-flag', { - $: 'function-flag', - fn: async ({ actor }) => actor?.type?.user?.username === 'test', - }); - - const result = await featureFlagService.check({ actor: { type: { user: { username: 'test' } } } }, 'dynamic-flag'); - expect(result).toBe(true); - }); - - it('should support function flags with different conditions', async () => { - featureFlagService.register('conditional-flag', { - $: 'function-flag', - fn: async ({ actor }) => actor?.type?.user?.username !== 'test', - }); - - const result = await featureFlagService.check({ actor: { type: { user: { username: 'other' } } } }, 'conditional-flag'); - expect(result).toBe(true); - }); - - it('should manage multiple flags', () => { - featureFlagService.register('multi-flag-1', { $: 'config-flag', value: true }); - featureFlagService.register('multi-flag-2', { $: 'config-flag', value: false }); - featureFlagService.register('multi-flag-3', { - $: 'function-flag', - fn: async () => true, - }); - - expect(featureFlagService.known_flags.has('multi-flag-1')).toBe(true); - expect(featureFlagService.known_flags.has('multi-flag-2')).toBe(true); - expect(featureFlagService.known_flags.has('multi-flag-3')).toBe(true); - expect(featureFlagService.known_flags.size).toBeGreaterThanOrEqual(3); - }); -}); - diff --git a/src/backend/src/services/FilesystemAPIService.js b/src/backend/src/services/FilesystemAPIService.js deleted file mode 100644 index 9f325945c..000000000 --- a/src/backend/src/services/FilesystemAPIService.js +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const BaseService = require('./BaseService'); - -/** -* @class FilesystemAPIService -* @extends BaseService -* @description This service handles all filesystem-related API routes, -* allowing for operations like file creation, deletion, -* reading, and searching through a structured set of -* endpoints. It integrates with the web server to expose -* these functionalities for client use. -*/ -class FilesystemAPIService extends BaseService { - /** - * Sets up the route handlers for the Filesystem API. - * This method registers various endpoints related to filesystem operations - * such as creating, deleting, reading, and updating files. It uses the - * web server's app instance to attach the corresponding routers. - * - * @async - * @function __on_install.routes - * @returns {Promise} A promise that resolves when the routes are set up. - */ - async '__on_install.routes' () { - const { app } = this.services.get('web-server'); - - // batch - app.use(require('../routers/filesystem_api/batch/all')); - - // v2 -- also in batch - app.use(require('../routers/filesystem_api/write')); - app.use(require('../routers/filesystem_api/mkdir')); - app.use(require('../routers/filesystem_api/delete')); - // v2 -- not in batch - app.use(require('../routers/filesystem_api/stat')); - app.use(require('../routers/filesystem_api/touch')); - app.use(require('../routers/filesystem_api/read')); - app.use(require('../routers/filesystem_api/token-read')); - app.use(require('../routers/filesystem_api/readdir')); - app.use((await import('../routers/filesystem_api/readdir-subdomains.mjs')).default); - app.use(require('../routers/filesystem_api/copy')); - app.use(require('../routers/filesystem_api/move')); - app.use(require('../routers/filesystem_api/rename')); - - app.use(require('../routers/filesystem_api/search')); - - // temporary or alpha - app.use(require('../routers/filesystem_api/update')); - - // v1 - app.use(require('../routers/writeFile')); - app.use(require('../routers/file')); - - // misc - app.use(require('../routers/df')); - - // cache - app.use(require('../routers/filesystem_api/cache')); - } -} - -module.exports = FilesystemAPIService; diff --git a/src/backend/src/services/GetUserService.js b/src/backend/src/services/GetUserService.js deleted file mode 100644 index f8e257e7d..000000000 --- a/src/backend/src/services/GetUserService.js +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { UserActorType } = require('./auth/Actor'); -const { PermissionImplicator } = require('./auth/permissionUtils.mjs'); -const BaseService = require('./BaseService'); -const { DB_READ } = require('./database/consts'); -const { UserRedisCacheSpace } = require('./UserRedisCacheSpace.js'); - -/** - * Get user by one of a variety of identifying properties. - * - * Pass `cached: false` to options to force a database read. - * Pass `force: true` to options to force a primary database read. - * - * This provides the functionality of `get_user` (helpers.js) - * as a service so that other services can register identifying - * properties for caching. - * - * The original `get_user` function now uses this service. - */ -class GetUserService extends BaseService { - /** - * Constructor for GetUserService. - * Initializes the set of identifying properties used to retrieve user data. - */ - _construct () { - this.id_properties = new Set(); - - this.id_properties.add('username'); - this.id_properties.add('uuid'); - this.id_properties.add('id'); - this.id_properties.add('email'); - this.id_properties.add('referral_code'); - } - - /** - * Initializes the GetUserService instance. - * This method prepares any necessary internal structures or states. - * It is called automatically upon instantiation of the service. - * - * @returns {Promise} A promise that resolves when the initialization is complete. - */ - async _init () { - - const svc_permission = this.services.get('permission'); - svc_permission.register_implicator(PermissionImplicator.create({ - id: 'user-set-own', - shortcut: true, - matcher: permission => { - return permission.startsWith('user:'); - }, - checker: async ({ actor, permission }) => { - if ( ! (actor.type instanceof UserActorType) ) { - return undefined; - } - if ( permission === `user:${ actor.type.user.uuid }:email:read` ) { - return {}; - } - }, - })); - } - - /** - * Retrieves a user object based on the provided options. - * - * This method queries the user from cache or database, - * depending on the caching options provided. If the user - * is found, it also calls the 'whoami' service to enrich - * the user details before returning. - * - * @param {Object} options - The options for retrieving the user. - * @param {boolean} [options.cached=true] - Indicates if caching should be used. - * @param {boolean} [options.force=false] - Forces a read from the database regardless of cache. - * @param {number?} [options.id] - Forces a read from the database regardless of cache. - * @param {string?} [options.uuid] - Forces a read from the database regardless of cache. - * @returns {Promise} The user object if found, else null. - */ - async get_user (options) { - const cached = options.cached ?? true; - - let user; - if ( cached && !options.force ) { - for ( const prop of this.id_properties ) { - if ( Object.prototype.hasOwnProperty.call(options, prop) ) { - const cachedUser = await UserRedisCacheSpace.getByProperty(prop, options[prop]); - if ( cachedUser ) { - user = cachedUser; - } - } - } - } - if ( ! user ) { - user = await this.get_user_(options); - } - if ( ! user ) return null; - - const svc_whoami = this.services.get('whoami'); - await svc_whoami.get_details({ user }, user); - - try { - UserRedisCacheSpace.setUser(user, { - props: Array.from(this.id_properties), - }); - } catch ( e ) { - console.error(e); - } - - return user; - } - - async refresh_actor (actor) { - if ( actor.type.user ) { - actor.type.user = await this.get_user({ - username: actor.type.user.username, - force: true, - }); - } - return actor; - } - - async get_user_ (options) { - const services = this.services; - - /** @type BaseDatabaseAccessService */ - const db = services.get('database').get(DB_READ, 'filesystem'); - - let user; - - if ( ! options.force ) { - for ( const prop of this.id_properties ) { - if ( Object.prototype.hasOwnProperty.call(options, prop) ) { - [user] = await db.read(`SELECT * FROM \`user\` WHERE \`${prop}\` = ? LIMIT 1`, [options[prop]]); - if ( user ) break; - } - } - } - - if ( !user || !user[0] ) { - for ( const prop of this.id_properties ) { - if ( Object.prototype.hasOwnProperty.call(options, prop) ) { - [user] = await db.pread(`SELECT * FROM \`user\` WHERE \`${prop}\` = ? LIMIT 1`, [options[prop]]); - if ( user ) break; - } - } - } - - if ( ! user ) return null; - - if ( user.metadata && typeof user.metadata === 'string' ) { - user.metadata = JSON.parse(user.metadata); - } else if ( ! user.metadata ) { - user.metadata = {}; - } - - return user; - } - - register_id_property (prop) { - this.id_properties.add(prop); - } -} - -module.exports = { GetUserService }; diff --git a/src/backend/src/services/HostDiskUsageService.js b/src/backend/src/services/HostDiskUsageService.js deleted file mode 100644 index 94df7dbf2..000000000 --- a/src/backend/src/services/HostDiskUsageService.js +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const BaseService = require('./BaseService'); -const { execSync } = require('child_process'); -const { Shescape } = require('shescape'); -const config = require('../config'); - -/** -* The HostDiskUsageService class extends BaseService to provide functionality for monitoring -* and reporting disk usage on the host system. This service identifies the mount point or drive -* where the current process is running, and performs disk usage checks for that specific location. -* It supports different operating systems like macOS and Linux, with placeholders for future -* Windows support. -* -* @extends BaseService -*/ -class HostDiskUsageService extends BaseService { - static DESCRIPTION = ` - This service is responsible for identifying the mountpoint/drive - on which the current process working directory is running, and then checking the - disk usage of that mountpoint/drive. - `; - - /** - * Initializes the service by determining the disk usage of the mountpoint/drive - * where the current working directory resides. - * - * @async - * @function - * @memberof HostDiskUsageService - * @instance - * @returns {Promise} A promise that resolves when initialization is complete. - * @throws {Error} If unable to determine disk usage for the platform. - */ - async _init () { - const current_platform = process.platform; - - // Setting the available space to a large number for unhandled platforms - var free_space = 1e+14; - - if ( current_platform == 'darwin' ) { - const mountpoint = this.get_darwin_mountpoint(process.cwd()); - free_space = this.get_disk_capacity_darwin(mountpoint); - } else if ( current_platform == 'linux' ) { - const mountpoint = this.get_linux_mountpint(process.cwd()); - free_space = this.get_disk_capacity_linux(mountpoint); - } else if ( current_platform == 'win32' ) { - this.log.warn('HostDiskUsageService: Windows is not supported yet'); - // TODO: Implement for windows systems - } - - config.available_device_storage = free_space; - } - - // TODO: TTL cache this value - /** - * Retrieves the current disk usage for the host system. - * - * This method checks the disk usage of the mountpoint or drive - * where the current process is running, based on the operating system. - * - * @returns {number} The amount of disk space used in bytes. - * - * @note This method does not cache its results and should be optimized - * with a TTL cache to prevent excessive system calls. - */ - get_host_usage () { - const current_platform = process.platform; - - let disk_use = 0; - if ( current_platform == 'darwin' ) { - const mountpoint = this.get_darwin_mountpoint(process.cwd()); - disk_use = this.get_disk_use_darwin(mountpoint); - } else if ( current_platform == 'linux' ) { - const mountpoint = this.get_linux_mountpint(process.cwd()); - disk_use = this.get_disk_use_linux(mountpoint); - } else if ( current_platform == 'win32' ) { - this.log.warn('HostDiskUsageService: Windows is not supported yet'); - // TODO: Implement for windows systems - } - return disk_use; - } - - // Called by the /df endpoint - /** - * Retrieves extra disk usage information for the host. - * This method is used by the /df endpoint to gather - * additional statistics on host disk usage. - * - * @returns {Object} An object containing the host's disk usage data. - */ - get_extra () { - return { - host_used: this.get_host_usage(), - }; - } - - // Get the mountpoint/drive of the current working directory in mac os - get_darwin_mountpoint (directory) { - const shescape = new Shescape({ shell: 'bash', quote: true }); - return execSync(`df -P ${shescape.escape(directory)} | awk 'NR==2 {print $6}'`, { encoding: 'utf-8' }).trim(); - } - - // Get the mountpoint/drive of the current working directory in linux - get_linux_mountpint (directory) { - const shescape = new Shescape({ shell: 'bash', quote: true }); - return execSync(`df -P ${shescape.escape(directory)} | awk 'NR==2 {print $6}'`, { encoding: 'utf-8' }).trim(); - // TODO: Implement for linux systems - } - - // Get the drive of the current working directory in windows - get_windows_drive (directory) { - // TODO: Implement for windows systems - } - - // Get the total drive capacity on the mountpoint/drive in mac os - get_disk_capacity_darwin (mountpoint) { - const shescape = new Shescape({ shell: 'bash', quote: true }); - const disk_info = execSync(`df -P ${shescape.escape(mountpoint)} | awk 'NR==2 {print $2}'`, { encoding: 'utf-8' }).trim().split(' '); - return parseInt(disk_info) * 512; - } - - // Get the total drive capacity on the mountpoint/drive in linux - get_disk_capacity_linux (mountpoint) { - const shescape = new Shescape({ shell: 'bash', quote: true }); - const disk_info = execSync(`df -P ${shescape.escape(mountpoint)} | awk 'NR==2 {print $2}'`, { encoding: 'utf-8' }).trim().split(' '); - return parseInt(disk_info) * 1024; - } - - // Get the total drive capacity on the drive in windows - get_disk_capacity_windows (drive) { - // TODO: Implement for windows systems - } - - // Get the free space on the mountpoint/drive in mac os - get_disk_use_darwin (mountpoint) { - const shescape = new Shescape({ shell: 'bash', quote: true }); - const disk_info = execSync(`df -P ${shescape.escape(mountpoint)} | awk 'NR==2 {print $4}'`, { encoding: 'utf-8' }).trim().split(' '); - return parseInt(disk_info) * 512; - } - - // Get the free space on the mountpoint/drive in linux - get_disk_use_linux (mountpoint) { - const shescape = new Shescape({ shell: 'bash', quote: true }); - const disk_info = execSync(`df -P ${shescape.escape(mountpoint)} | awk 'NR==2 {print $4}'`, { encoding: 'utf-8' }).trim().split(' '); - return parseInt(disk_info) * 1024; - } - - // Get the free space on the drive in windows - get_disk_use_windows (drive) { - // TODO: Implement for windows systems - } -} - -module.exports.HostDiskUsageService = HostDiskUsageService; diff --git a/src/backend/src/services/HostnameService.js b/src/backend/src/services/HostnameService.js deleted file mode 100644 index 581d74198..000000000 --- a/src/backend/src/services/HostnameService.js +++ /dev/null @@ -1,36 +0,0 @@ -const BaseService = require('./BaseService'); - -const os = require('os'); - -class HostnameService extends BaseService { - _construct () { - this.entries = {}; - } - - _init () { - if ( this.global_config.domain ) { - this.entries[this.global_config.domain] = { - scope: 'web', - }; - this.entries[`api.${this.global_config.domain}`] = { - scope: 'api', - }; - } - - const addresses = this.get_broadcast_addresses(); - - if ( ! this.global_config.no_nip ) { - // - } - } - - get_broadcast_addresses () { - const ifaces = os.networkInterfaces(); - - for ( const iface_key in ifaces ) { - console.log('iface_key', iface_key); - } - } -} - -module.exports = { HostnameService }; diff --git a/src/backend/src/services/HostnameService.test.ts b/src/backend/src/services/HostnameService.test.ts deleted file mode 100644 index 82c6c977b..000000000 --- a/src/backend/src/services/HostnameService.test.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { createTestKernel } from '../../tools/test.mjs'; -import { HostnameService } from './HostnameService'; - -describe('HostnameService', async () => { - const testKernel = await createTestKernel({ - serviceMap: { - hostname: HostnameService, - }, - initLevelString: 'init', - }); - - const hostnameService = testKernel.services!.get('hostname') as HostnameService; - - it('should be instantiated', () => { - expect(hostnameService).toBeInstanceOf(HostnameService); - }); - - it('should have entries object', () => { - expect(hostnameService.entries).toBeDefined(); - expect(typeof hostnameService.entries).toBe('object'); - }); - - it('should have entries as empty object by default', () => { - expect(hostnameService.entries).toBeDefined(); - expect(typeof hostnameService.entries).toBe('object'); - }); - - it('should have get_broadcast_addresses method', () => { - expect(typeof hostnameService.get_broadcast_addresses).toBe('function'); - }); - - it('should allow manual entry registration', () => { - hostnameService.entries['manual.test.com'] = { scope: 'test' }; - expect(hostnameService.entries['manual.test.com']).toBeDefined(); - expect(hostnameService.entries['manual.test.com'].scope).toBe('test'); - }); - - it('should maintain multiple entries', () => { - hostnameService.entries['first.test.com'] = { scope: 'web' }; - hostnameService.entries['second.test.com'] = { scope: 'api' }; - - expect(hostnameService.entries['first.test.com'].scope).toBe('web'); - expect(hostnameService.entries['second.test.com'].scope).toBe('api'); - }); -}); - diff --git a/src/backend/src/services/KernelInfoService.js b/src/backend/src/services/KernelInfoService.js deleted file mode 100644 index 9f8007f1c..000000000 --- a/src/backend/src/services/KernelInfoService.js +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const configurable_auth = require('../middleware/configurable_auth'); -const eggspress = require('../api/eggspress'); -const { Context } = require('../util/context'); -const BaseService = require('./BaseService'); -const { Interface } = require('./drivers/meta/Construct'); - -// Permission flag that grants access to view all services in the kernel info system -const PERM_SEE_ALL = 'kernel-info:see-all-services'; -// Permission flag that grants access to view all services in the kernel info system -const PERM_SEE_DRIVERS = 'kernel-info:see-all-drivers'; - -/** -* KernelInfoService class provides information about the kernel's services, modules, and interfaces. -* It handles listing available modules, services, and their implementations based on user permissions. -* The service exposes endpoints for querying kernel module information and manages access control -* through permission checks for viewing all services and drivers. -* @extends BaseService -*/ -class KernelInfoService extends BaseService { - async _init () { - } - - /** - * Installs routes for the kernel info service - * @param {*} _ Unused parameter - * @param {Object} param1 Object containing Express app instance - * @param {Express} param1.app Express application instance - * @private - */ - '__on_install.routes' (_, { app }) { - const router = (() => { - const require = this.require; - const express = require('express'); - return express.Router(); - })(); - - app.use('/', router); - - router.use(eggspress('/lsmod', { - allowedMethods: ['GET', 'POST'], - mw: [ - configurable_auth(), - ], - }, async (req, res) => { - const svc_permission = this.services.get('permission'); - - const actor = Context.get('actor'); - const can_see_all = actor && - await svc_permission.check(actor, PERM_SEE_ALL); - const can_see_drivers = actor && - await svc_permission.check(actor, PERM_SEE_DRIVERS); - - const interfaces = {}; - const svc_registry = this.services.get('registry'); - const col_interfaces = svc_registry.get('interfaces'); - for ( const interface_name of col_interfaces.keys() ) { - const iface = col_interfaces.get(interface_name); - if ( iface === undefined ) continue; - if ( iface.no_sdk ) continue; - interfaces[interface_name] = { - spec: (new Interface( - iface, - { name: interface_name }, - )).serialize(), - implementors: {}, - }; - } - - const services = []; - for ( const k in this.services.modules_ ) { - for ( const s_k of this.services.modules_[k].services_l ) { - const service_info = { - name: s_k, - traits: [], - }; - services.push(service_info); - - const service = this.services.get(s_k); - if ( service.list_traits ) { - const traits = service.list_traits(); - for ( const trait of traits ) { - const corresponding_iface = interfaces[trait]; - if ( ! corresponding_iface ) continue; - corresponding_iface.implementors[s_k] = {}; - } - service_info.traits = service.list_traits(); - } - } - } - - // If actor doesn't have permission to see all drivers, - // (granted by either "can_see_all" or "can_see_drivers") - if ( !can_see_all && !can_see_drivers ) { - // only show interfaces with at least one implementation - // that the actor has permission to use - for ( const iface_name in interfaces ) { - for ( const impl_name in interfaces[iface_name].implementors ) { - const perm = `service:${impl_name}:ii:${iface_name}`; - const can_see_this = actor && - await svc_permission.check(actor, perm); - if ( ! can_see_this ) { - delete interfaces[iface_name].implementors[impl_name]; - } - } - if ( Object.keys(interfaces[iface_name].implementors).length < 1 ) { - delete interfaces[iface_name]; - } - } - } - - res.json({ - interfaces, - ...(can_see_all ? { services } : {}), - }); - })); - } -} - -module.exports = { - KernelInfoService, -}; diff --git a/src/backend/src/services/LockService.js b/src/backend/src/services/LockService.js deleted file mode 100644 index 90eb3aab4..000000000 --- a/src/backend/src/services/LockService.js +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { RWLock } = require('../util/lockutil'); -const BaseService = require('./BaseService'); - -/** -* Represents the LockService class responsible for managing locks -* using reader-writer locks (RWLock). This service ensures that -* critical sections are properly handled by enforcing write locks -* exclusively, enabling safe concurrent access to shared resources -* while preventing race conditions and ensuring data integrity. -*/ -class LockService extends BaseService { - /** - * Initializes the LockService by setting up the locks object - * and registering the 'lock' commands. This method is called - * during the service initialization phase. - */ - async _construct () { - this.locks = {}; - } - /** - * Initializes the locks object to store lock instances. - * - * This method is called during the construction of the LockService - * instance to ensure that the locks property is ready for use. - * - * @returns {Promise} A promise that resolves when the - * initialization is complete. - */ - async _init () { - } - - /** - * Acquires a lock for the specified name, allowing for a callback to be executed while the lock is held. - * If the name is an array, all locks will be acquired in sequence. The method supports optional - * configurations, including a timeout feature. It returns the result of the callback execution. - * - * @param {string|string[]} name - The name(s) of the lock(s) to acquire. - * @param {Object} [opt_options] - Optional configuration options. - * @param {function} callback - The function to call while the lock is held. - * @returns {Promise} The result of the callback. - */ - async lock (name, opt_options, callback) { - if ( typeof opt_options === 'function' ) { - callback = opt_options; - opt_options = {}; - } - - // If name is an array, lock all of them - if ( Array.isArray(name) ) { - const names = name; - // TODO: verbose log option by service - const section = names.reduce((current_callback, name) => { - return async () => { - return await this.lock(name, opt_options, current_callback); - }; - }, callback); - - return await section(); - } - - if ( ! this.locks[name] ) { - const rwlock = new RWLock(); - this.locks[name] = rwlock; - } - - const handle = await this.locks[name].wlock(); - // TODO: verbose log option by service - // console.log(`\x1B[36;1mLOCK (${name})\x1B[0m`); - - let timeout, timed_out; - if ( opt_options.timeout ) { - timeout = setTimeout(() => { - handle.unlock(); - // TODO: verbose log option by service - // throw new Error(`lock ${name} timed out`); - }, opt_options.timeout); - } - - try { - return await callback(); - } finally { - if ( timeout ) { - clearTimeout(timeout); - } - if ( ! timed_out ) { - // TODO: verbose log option by service - // console.log(`\x1B[36;1mUNLOCK (${name})\x1B[0m`); - handle.unlock(); - } - } - } -} - -module.exports = { LockService }; \ No newline at end of file diff --git a/src/backend/src/services/LockService.test.ts b/src/backend/src/services/LockService.test.ts deleted file mode 100644 index 34dc8f4bf..000000000 --- a/src/backend/src/services/LockService.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { createTestKernel } from '../../tools/test.mjs'; -import { LockService } from './LockService'; - -describe('LockService', async () => { - const testKernel = await createTestKernel({ - serviceMap: { - lock: LockService, - }, - initLevelString: 'init', - testCore: true, - }); - - const lockService = testKernel.services!.get('lock') as LockService; - - it('should be instantiated', () => { - expect(lockService).toBeInstanceOf(LockService); - }); - - it('should acquire and release a lock', async () => { - let executed = false; - await lockService.lock('test-lock', async () => { - executed = true; - }); - expect(executed).toBe(true); - }); - - it('should execute callback within lock', async () => { - const result = await lockService.lock('test-lock-2', async () => { - return 'success'; - }); - expect(result).toBe('success'); - }); - - it('should handle multiple sequential locks', async () => { - const results: number[] = []; - - await lockService.lock('seq-lock', async () => { - results.push(1); - }); - - await lockService.lock('seq-lock', async () => { - results.push(2); - }); - - expect(results).toEqual([1, 2]); - }); - - it('should handle locks with options', async () => { - let executed = false; - await lockService.lock('opt-lock', { timeout: 5000 }, async () => { - executed = true; - }); - expect(executed).toBe(true); - }); - - it('should support array of lock names', async () => { - let executed = false; - await lockService.lock(['lock-a', 'lock-b'], async () => { - executed = true; - }); - expect(executed).toBe(true); - }); - - it('should maintain lock state', async () => { - await lockService.lock('state-lock', async () => { - expect(lockService.locks['state-lock']).toBeDefined(); - }); - // Lock should still exist after release - expect(lockService.locks['state-lock']).toBeDefined(); - }); - - it('should handle errors within lock callback', async () => { - await expect( - lockService.lock('error-lock', async () => { - throw new Error('Test error'); - }) - ).rejects.toThrow('Test error'); - }); -}); - diff --git a/src/backend/src/services/MakeProdDebuggingLessAwfulService.js b/src/backend/src/services/MakeProdDebuggingLessAwfulService.js deleted file mode 100644 index 13a35aa8e..000000000 --- a/src/backend/src/services/MakeProdDebuggingLessAwfulService.js +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { Context } = require('../util/context'); -const BaseService = require('./BaseService'); - -/** - * This service registers a middleware that will apply the value of - * header X-PUTER-DEBUG to the request's Context object. - * - * Consequentially, the value of X-PUTER-DEBUG will included in all - * log messages produced by the request. - */ -class MakeProdDebuggingLessAwfulService extends BaseService { - static USE = { - logutil: 'core.util.logutil', - }; - static MODULES = { - fs: require('fs'), - }; - /** - * Inner class that defines the modules required by the MakeProdDebuggingLessAwfulService. - * Currently includes the file system (fs) module for writing debug logs to files. - * @static - * @memberof MakeProdDebuggingLessAwfulService - */ - static ProdDebuggingMiddleware = class ProdDebuggingMiddleware { - /** - * Middleware class that handles production debugging functionality - * by capturing and processing the X-PUTER-DEBUG header value. - * - * This middleware extracts the debug header value and makes it - * available through the Context for logging and debugging purposes. - */ - constructor () { - this.header_name_ = 'x-puter-debug'; - } - install (app) { - app.use(this.run.bind(this)); - } - /** - * Installs the middleware into the Express application - * - * @param {Object} req - Express request object containing headers - * @param {Object} res - Express response object - * @param {Function} next - Express next middleware function - * @returns {void} - */ - async run (req, res, next) { - const x = Context.get(); - x.set('prod-debug', req.headers[this.header_name_]); - next(); - } - }; - - async _init () { - // Initialize express middleware - this.mw = new this.constructor.ProdDebuggingMiddleware(); - - // Add logger middleware - const svc_log = this.services.get('log-service'); - svc_log.register_log_middleware(async log_details => { - const { - context, - log_lvl, crumbs, message, fields, objects, - } = log_details; - - const maybe_debug_token = context.get('prod-debug'); - - if ( ! maybe_debug_token ) return; - - // Log to an additional log file so this is easier to find - const outfile = svc_log.get_log_file(`debug-${maybe_debug_token}.log`); - - try { - await this.modules.fs.promises.appendFile(outfile, - `${this.logutil.stringify_log_entry(log_details) }\n`); - } catch ( e ) { - console.error(e); - } - - // Add the prod_debug field to the log message - return { - fields: { - ...fields, - prod_debug: maybe_debug_token, - }, - }; - }); - } - /** - * Handles installation of the context-aware middleware for production debugging - * @param {*} _ Unused parameter - * @param {Object} options Installation options - * @param {Express} options.app Express application instance - * @returns {Promise} - */ - async '__on_install.middlewares.context-aware' (_, { app }) { - // Add express middleware - this.mw.install(app); - } -} - -module.exports = { - MakeProdDebuggingLessAwfulService, -}; diff --git a/src/backend/src/services/MeteringService/.gitignore b/src/backend/src/services/MeteringService/.gitignore deleted file mode 100644 index 4c43fe68f..000000000 --- a/src/backend/src/services/MeteringService/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.js \ No newline at end of file diff --git a/src/backend/src/services/MeteringService/MeteringService.test.ts b/src/backend/src/services/MeteringService/MeteringService.test.ts deleted file mode 100644 index 9f1af5539..000000000 --- a/src/backend/src/services/MeteringService/MeteringService.test.ts +++ /dev/null @@ -1,229 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { createTestKernel } from '../../../tools/test.mjs'; -import { Actor } from '../auth/Actor'; -import { DynamoKVStoreWrapper } from '../DynamoKVStore/DynamoKVStoreWrapper.js'; -import type { EventService } from '../EventService.js'; -import { GLOBAL_APP_KEY, PERIOD_ESCAPE } from './consts.js'; -import { COST_MAPS } from './costMaps/index.js'; -import { MeteringService } from './MeteringService'; -import { MeteringServiceWrapper } from './MeteringServiceWrapper.mjs'; - -describe('MeteringService', async () => { - const testKernel = await createTestKernel({ - serviceMap: { - meteringService: MeteringServiceWrapper, - 'puter-kvstore': DynamoKVStoreWrapper, - }, - initLevelString: 'init', - testCore: true, - serviceConfigOverrideMap: { - 'database': { - path: ':memory:', - }, - 'dynamo': { - path: ':memory:', - }, - }, - }); - - const testSubject = testKernel.services!.get('meteringService') as MeteringServiceWrapper; - const eventService = testKernel.services!.get('event') as EventService; - const makeActor = (userUuid: string, appUid?: string, email?: string) => { - const actor = { - type: { - user: { - uuid: userUuid, - ...(email ? { email } : {}), - }, - ...(appUid ? { app: { uid: appUid } } : {}), - }, - } as unknown as Actor; - return actor; - }; - - it('should be instantiated', () => { - expect(testSubject).toBeInstanceOf(MeteringServiceWrapper); - }); - - it('should contain a copy of the public methods of meteringService too', () => { - const meteringMethods = Object.getOwnPropertyNames(MeteringService.prototype) - .filter((name) => name !== 'constructor'); - const wrapperMethods = testSubject as unknown as Record; - const missing = meteringMethods.filter((name) => typeof wrapperMethods[name] !== 'function'); - - expect(missing).toEqual([]); - }); - - it('should have meteringService instantiated', async () => { - expect(testSubject.meteringService).toBeInstanceOf(MeteringService); - }); - - it('should record usage for an actor properly', async () => { - const usageType = 'aws-polly:standard:character'; - const costPerUnit = COST_MAPS[usageType]; - const res = await testSubject.meteringService.incrementUsage({ type: { user: { uuid: 'test-user-id' } } } as unknown as Actor, - usageType, - 1); - - expect(res.total).toBe(costPerUnit); - expect(res[usageType]).toMatchObject({ - cost: costPerUnit, - units: 1, - count: 1, - }); - }); - - it('utilRecordUsageObject delegates tracked usage to batchIncrementUsages', () => { - const actor = makeActor('util-user'); - const spy = vi.spyOn(testSubject.meteringService, 'batchIncrementUsages'); - - testSubject.meteringService.utilRecordUsageObject({ read: 2, write: 3 }, actor, 'kv', { write: 50 }); - - expect(spy).toHaveBeenCalledTimes(1); - expect(spy).toHaveBeenCalledWith(actor, [ - { usageType: 'kv:read', usageAmount: 2, costOverride: undefined }, - { usageType: 'kv:write', usageAmount: 3, costOverride: 50 }, - ]); - spy.mockRestore(); - }); - - it('batchIncrementUsages aggregates totals per usage type', async () => { - const actor = makeActor('batch-user', 'batch-app'); - - const res = await testSubject.meteringService.batchIncrementUsages(actor, [ - { usageType: 'kv:write', usageAmount: 2 }, - { usageType: 'kv:read', usageAmount: 3 }, - ]); - - expect(res.total).toBe(439); // (125 * 2) + (63 * 3) - expect(res['kv:write']).toMatchObject({ units: 2, cost: 250, count: 1 }); - expect(res['kv:read']).toMatchObject({ units: 3, cost: 189, count: 1 }); - }); - - it('getActorCurrentMonthUsageDetails groups current app and others', async () => { - const userId = 'usage-detail-user'; - const actorAppOne = makeActor(userId, 'app-one'); - const actorAppTwo = makeActor(userId, 'app-two'); - - await testSubject.meteringService.incrementUsage(actorAppOne, 'kv:write', 1); - await testSubject.meteringService.incrementUsage(actorAppTwo, 'kv:read', 2); - - const details = await testSubject.meteringService.getActorCurrentMonthUsageDetails(actorAppOne); - - expect(details.usage.total).toBe(251); - expect(details.appTotals['app-one']).toMatchObject({ total: 125, count: 1 }); - expect(details.appTotals.others).toMatchObject({ total: 126, count: 1 }); - }); - - it('getActorCurrentMonthAppUsageDetails returns per-app usage', async () => { - const actor = makeActor('app-usage-user', 'app-usage-app'); - await testSubject.meteringService.incrementUsage(actor, 'kv:write', 1); - - const usage = await testSubject.meteringService.getActorCurrentMonthAppUsageDetails(actor); - - expect(usage.total).toBe(125); - expect(usage['kv:write']).toMatchObject({ cost: 125, units: 1, count: 1 }); - }); - - it('getActorCurrentMonthAppUsageDetails rejects when actor queries another app', async () => { - const actor = makeActor('app-usage-user-2', 'app-one'); - await expect(testSubject.meteringService.getActorCurrentMonthAppUsageDetails(actor, 'app-two')) - .rejects - .toThrow('Actor can only get usage details for their own app or global app'); - }); - - it('getAllowedUsage respects subscription overrides and consumed usage', async () => { - const actor = makeActor('limited-user'); - const customPolicy = { id: 'tiny', monthUsageAllowance: 10, monthlyStorageAllowance: 0 }; - const detPolicies = eventService.on('metering:registerAvailablePolicies', (_key: string, data: Record) => { - data.availablePolicies.push(customPolicy); - }); - const detUserSub = eventService.on('metering:getUserSubscription', (_key: string, data: Record) => { - data.userSubscriptionId = customPolicy.id; - }); - - try { - await testSubject.meteringService.incrementUsage(actor, 'kv:write', 1); - const allowed = await testSubject.meteringService.getAllowedUsage(actor); - - expect(allowed.monthUsageAllowance).toBe(10); - expect(allowed.remaining).toBe(0); - expect(allowed.addons).toEqual({}); - expect(await testSubject.meteringService.hasAnyUsage(actor)).toBe(false); - expect(await testSubject.meteringService.hasEnoughCreditsFor(actor, 'kv:read', 1)).toBe(false); - expect(await testSubject.meteringService.hasEnoughCredits(actor, 1)).toBe(false); - } finally { - detPolicies.detach(); - detUserSub.detach(); - } - }); - - it('updateAddonCredit stores addon credits retrievable via getActorAddons', async () => { - const userId = 'addon-user'; - await testSubject.meteringService.updateAddonCredit(userId, 500); - - const addons = await testSubject.meteringService.getActorAddons(makeActor(userId)); - - expect(addons).toMatchObject({ purchasedCredits: 500 }); - }); - - it('getGlobalUsage aggregates totals across shards', async () => { - const actor = makeActor('global-user', 'global-app'); - const before = await testSubject.meteringService.getGlobalUsage(); - await testSubject.meteringService.incrementUsage(actor, 'kv:write', 1); - const after = await testSubject.meteringService.getGlobalUsage(); - - const beforeRecord = before['kv:write'] || { cost: 0, units: 0, count: 0 }; - const afterRecord = after['kv:write'] || { cost: 0, units: 0, count: 0 }; - - expect(after.total - before.total).toBe(125); - expect(afterRecord.cost - beforeRecord.cost).toBe(125); - expect(afterRecord.units - beforeRecord.units).toBe(1); - expect(afterRecord.count - beforeRecord.count).toBe(1); - }); - - it('getActorAppUsage rejects when actor is scoped to another app', async () => { - const actor = makeActor('app-usage-user-3', 'app-one'); - await expect(testSubject.meteringService.getActorAppUsage(actor, 'app-two')) - .rejects - .toThrow('Actor can only get usage for their own app'); - }); - - it('getActorAppUsage returns zeroed usage when none exists', async () => { - const actor = makeActor('app-usage-user-4'); - const usage = await testSubject.meteringService.getActorAppUsage(actor, GLOBAL_APP_KEY); - - expect(usage).toMatchObject({ total: 0 }); - }); - - it('should record usage for an actor when cost is overwritten', async () => { - const actor = makeActor('overridden-cost-user'); - const res = await testSubject.meteringService.incrementUsage(actor, - 'aws-polly:standard:character', - 10, - 12); - - expect(res.total).toBe(12); - expect(res['aws-polly:standard:character']).toMatchObject({ cost: 12, units: 10, count: 1 }); - }); - - it('applies the configured cost map rate for random samples of usage types', async () => { - const usageAmount = 2; - - const entries = Object.entries(COST_MAPS); - for ( let i = 0; i < entries.length; i += Math.ceil(Math.random() * entries.length / 10) ) { - - const [usageType, costPerUnit] = entries[i]; - const actor = makeActor(`cost-map-user-${usageType.replace(/[^a-zA-Z0-9]/g, '-')}`); - const result = await testSubject.meteringService.incrementUsage(actor, usageType, usageAmount); - const escapedUsageType = usageType.replace(/\./g, PERIOD_ESCAPE); - - expect(result.total).toBe(costPerUnit * usageAmount); - expect(result[escapedUsageType]).toMatchObject({ - cost: costPerUnit * usageAmount, - units: usageAmount, - count: 1, - }); - } - }, 30000); -}); diff --git a/src/backend/src/services/MeteringService/MeteringService.ts b/src/backend/src/services/MeteringService/MeteringService.ts deleted file mode 100644 index a1afcc109..000000000 --- a/src/backend/src/services/MeteringService/MeteringService.ts +++ /dev/null @@ -1,719 +0,0 @@ -import murmurhash from 'murmurhash'; -import type { AlarmService } from '../../modules/core/AlarmService.js'; -import { SystemActorType, type Actor } from '../auth/Actor.js'; -import type { DynamoKVStore } from '../DynamoKVStore/DynamoKVStore.js'; -import type { EventService } from '../EventService'; -import type { SUService } from '../SUService.js'; -import { DEFAULT_FREE_SUBSCRIPTION, DEFAULT_TEMP_SUBSCRIPTION, GLOBAL_APP_KEY, METRICS_PREFIX, PERIOD_ESCAPE, POLICY_PREFIX } from './consts.js'; -import { COST_MAPS } from './costMaps/index.js'; -import { SUB_POLICIES } from './subPolicies/index.js'; -import { AppTotals, MeteringServiceDeps, UsageAddons, UsageByType, UsageRecord } from './types.js'; -import { toMicroCents } from './utils.js'; -/** - * Handles usage metering and supports stubbs for billing methods for current scoped actor - */ -export class MeteringService { - - static GLOBAL_SHARD_COUNT = 10000; // number of global usage shards to spread writes across - static APP_SHARD_COUNT = 10000; // number of app usage shards to spread writes across - static MAX_GLOBAL_USAGE_PER_MINUTE = toMicroCents(.2); // 20 cents per minute max global usage to help detect abuse - #kvStore: DynamoKVStore; - #superUserService: SUService; - #alarmService: AlarmService; - #eventService: EventService; - constructor ({ kvStore, superUserService, alarmService, eventService }: MeteringServiceDeps) { - this.#superUserService = superUserService; - this.#kvStore = kvStore; - this.#alarmService = alarmService; - this.#eventService = eventService; - setInterval(() => { - this.#checkRateOfChange(); - }, 1000 * 60 * 16); // check every 16 minutes - } - - utilRecordUsageObject>(trackedUsageObject: T, actor: Actor, modelPrefix: string, costsOverrides?: Partial>) { - this.batchIncrementUsages(actor, Object.entries(trackedUsageObject).map(([usageKind, amount]) => { - const hasOverride = !!costsOverrides && Number.isFinite(costsOverrides[usageKind]); - return { - usageType: `${modelPrefix}:${usageKind}`, - usageAmount: amount, - costOverride: hasOverride ? costsOverrides![usageKind as keyof T] : undefined, - }; - })); - } - - #getMonthYearString () { - const now = new Date(); - return `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, '0')}`; - } - - /** - * Adds some randomized number from 0-999 to the usage key to help spread writes - * @param userId - * @param appId - * @returns - */ - #generateGloabalUsageKey (userId: string, appId: string, currentMonth: string) { - const hashOfUserAndApp = murmurhash.v3(`${userId}:${appId}`) % MeteringService.GLOBAL_SHARD_COUNT; - const key = `${METRICS_PREFIX}:puter:${hashOfUserAndApp}:${currentMonth}`; - return key; - } - - #generateAppUsageKey (appId: string, userId: string, currentMonth: string) { - const hashOfApp = murmurhash.v3(`${appId}${userId}`) % MeteringService.APP_SHARD_COUNT; - const key = `${METRICS_PREFIX}:app:${appId}:${hashOfApp}:${currentMonth}`; - return key; - } - - // TODO DS: track daily and hourly usage as well - async incrementUsage (actor: Actor, usageType: (keyof typeof COST_MAPS) | (string & {}), usageAmount: number, costOverride?: number) { - usageAmount = usageAmount < 0 ? 1 : usageAmount; - - const costOverrideRaw = costOverride; - costOverride = !Number.isFinite(costOverride) - ? undefined - : (costOverride as number) < 0 - ? 1 - : costOverride; - - if ( costOverrideRaw && costOverrideRaw < 0 ) { - this.#alarmService.create(`metering unexpected negative cost access to: ${usageType}`, 'negative cost abuse vector!', { - userId: actor.type?.user?.uuid, - username: actor.type?.user?.username, - appId: actor.type?.app?.uid, - usageType, - usageAmount, - costOverride, - }); - } - try { - if ( !usageAmount || !usageType || !actor ) { - // silent fail for now; - return { total: 0 } as UsageByType; - } - - if ( actor.type instanceof SystemActorType || actor.type?.user?.username === 'system' ) { - // Don't track for now since it will trigger infinite noise; - return { total: 0 } as UsageByType; - } - - const currentMonth = this.#getMonthYearString(); - - return this.#superUserService.sudo(async () => { - - const mappedCost = COST_MAPS[usageType as keyof typeof COST_MAPS]; - const totalCost = (((costOverride && costOverride < 0) ? 1 : costOverride) ?? ((mappedCost || 0) * usageAmount)); - - if ( totalCost === 0 && (mappedCost !== 0 && costOverride !== 0) ) { - // cost is zero but no explicit override to 0, so flag as potential abuse - this.#alarmService.create(`metering unexpected 0 cost access to: ${usageType}`, '0 cost abuse vector', { - userId: actor.type?.user?.uuid, - username: actor.type?.user?.username, - appId: actor.type?.app?.uid, - usageType, - usageAmount, - costOverride, - }); - } - - usageType = usageType.replace(/\./g, PERIOD_ESCAPE) as keyof typeof COST_MAPS; // replace dots with underscores for kvstore paths, TODO DS: map this back when reading - const appId = actor.type?.app?.uid || GLOBAL_APP_KEY; - const userId = actor.type?.user?.uuid!; - const pathAndAmountMap = { - 'total': totalCost, - [`${usageType}.units`]: usageAmount, - [`${usageType}.cost`]: totalCost, - [`${usageType}.count`]: 1, - }; - - const actorUsageKey = `${METRICS_PREFIX}:actor:${userId}:${currentMonth}`; - const actorUsagesPromise = this.#kvStore.incr({ - key: actorUsageKey, - pathAndAmountMap, - }) as unknown as Promise; - - const puterConsumptionKey = this.#generateGloabalUsageKey(userId, appId, currentMonth); // global consumption across all users and apps - this.#kvStore.incr({ - key: puterConsumptionKey, - pathAndAmountMap, - }).catch((e: Error) => { - console.warn(`Failed to increment aux usage data 'puterConsumptionKey' with error: ${ e.message} for userId: ${userId} appId: ${appId}`); - }); - - const actorAppUsageKey = `${METRICS_PREFIX}:actor:${userId}:app:${appId}:${currentMonth}`; - this.#kvStore.incr({ - key: actorAppUsageKey, - pathAndAmountMap, - }).catch((e: Error) => { - console.warn(`Failed to increment aux usage data 'actorAppUsageKey' with error: ${ e.message} for userId: ${userId} appId: ${appId}`); - }); - - if ( appId !== GLOBAL_APP_KEY ) { - const appUsageKey = this.#generateAppUsageKey(appId, userId, currentMonth); - this.#kvStore.incr({ - key: appUsageKey, - pathAndAmountMap, - }).catch((e: Error) => { - console.warn(`Failed to increment aux usage data 'appUsageKey' with error: ${ e.message} for userId: ${userId} appId: ${appId}`); - }); - } - - const actorAppTotalsKey = `${METRICS_PREFIX}:actor:${userId}:apps:${currentMonth}`; - this.#kvStore.incr({ - key: actorAppTotalsKey, - pathAndAmountMap: { - [`${appId}.total`]: totalCost, - [`${appId}.count`]: 1, - }, - }).catch((e: Error) => { - console.warn(`Failed to increment aux usage data 'actorAppTotalsKey' with error: ${ e.message} for userId: ${userId} appId: ${appId}`); - }); - - const lastUpdatedKey = `${METRICS_PREFIX}:actor:${userId}:lastUpdated`; - this.#kvStore.set({ - key: lastUpdatedKey, - value: Date.now(), - }).catch((e: Error) => { - console.warn('Failed to set lastUpdatedKey with error: ', e.message); - }); - - // update addon usage if we are over the allowance - const actorSubscriptionPromise = this.getActorSubscription(actor); - const actorAddonsPromise = this.getActorAddons(actor); - const [actorUsages, actorSubscription, actorAddons] = (await Promise.all([actorUsagesPromise, actorSubscriptionPromise, actorAddonsPromise])); - if ( actorUsages.total > actorSubscription.monthUsageAllowance && actorAddons.purchasedCredits && actorAddons.purchasedCredits > (actorAddons.consumedPurchaseCredits || 0) ) { - // if we are now over the allowance, start consuming purchased credits - const withinBoundsUsage = Math.max(0, actorSubscription.monthUsageAllowance - actorUsages.total + totalCost); - const overageUsage = totalCost - withinBoundsUsage; - - if ( overageUsage > 0 ) { - await this.#kvStore.incr({ - key: `${POLICY_PREFIX}:actor:${userId}:addons`, - pathAndAmountMap: { - consumedPurchaseCredits: Math.min(overageUsage, actorAddons.purchasedCredits - (actorAddons.consumedPurchaseCredits || 0)), // don't go over the purchased credits, technically a race condition here, but optimistically rare - }, - }); - } - } - // alert if significantly over allowance and no purchased credits left - const allowedUsageMultiple = Math.floor(actorUsages.total / actorSubscription.monthUsageAllowance); - const previousAllowedUsageMultiple = Math.floor((actorUsages.total - totalCost) / actorSubscription.monthUsageAllowance); - const isOver2x = allowedUsageMultiple >= 2; - const isChangeOverPastOverage = previousAllowedUsageMultiple < allowedUsageMultiple; - const hasNoAddonCredit = (actorAddons.purchasedCredits || 0) <= (actorAddons.consumedPurchaseCredits || 0); - if ( isOver2x && isChangeOverPastOverage && hasNoAddonCredit ) { - this.#alarmService.create(`metering usage exceeded by user: ${actor.type?.user?.username}`, `Actor ${userId} has exceeded their usage allowance significantly`, { - userId: actor.type?.user?.uuid, - username: actor.type?.user?.username, - appId: actor.type?.app?.uid, - usageType, - usageAmount, - costOverride, - totalUsage: actorUsages.total, - monthUsageAllowance: actorSubscription.monthUsageAllowance, - }); - } - return actorUsages; - }); - } catch ( e ) { - console.error('Metering: Failed to increment usage for actor', actor, 'usageType', usageType, 'usageAmount', usageAmount, e); - this.#alarmService.create(`metering service error for user: ${ actor.type?.user?.username} app: ${ actor.type.app?.uid}`, (e as Error).message, { - userId: actor.type?.user?.uuid, - username: actor.type?.user?.username, - appId: actor.type?.app?.uid, - error: e, - usageType, - usageAmount, - costOverride, - }); - return { total: 0 } as UsageByType; - } - } - - async batchIncrementUsages (actor: Actor, usages: { usageType: (keyof typeof COST_MAPS) | (string & {}), usageAmount: number, costOverride?: number }[]) { - try { - if ( !usages || usages.length === 0 || !actor ) { - // silent fail for now; - return { total: 0 } as UsageByType; - } - - if ( actor.type instanceof SystemActorType || actor.type?.user?.username === 'system' ) { - // Don't track for now since it will trigger infinite noise; - return { total: 0 } as UsageByType; - } - - const currentMonth = this.#getMonthYearString(); - - return this.#superUserService.sudo(async () => { - // Aggregate all pathAndAmountMap entries for all usages - const aggregatedPathAndAmountMap: Record = {}; - let totalBatchCost = 0; - let hasZeroCostWarning = false; - - // Process each usage and aggregate the pathAndAmountMap - for ( const usage of usages ) { - const { usageType, usageAmount: usageAmountRaw, costOverride: costOverrideRaw } = usage; - const usageAmount = (!Number.isFinite(usageAmountRaw) || usageAmountRaw < 0) ? 1 : usageAmountRaw; - const costOverride = !Number.isFinite(costOverrideRaw) - ? undefined - : (costOverrideRaw as number) < 0 - ? 1 - : costOverrideRaw; - - if ( !usageAmount || !usageType ) { - continue; // skip invalid entries - } - - if ( costOverrideRaw && costOverrideRaw < 0 ) { - this.#alarmService.create(`metering unexpected negative cost access to: ${usageType}`, 'negative cost abuse vector!', { - userId: actor.type?.user?.uuid, - username: actor.type?.user?.username, - appId: actor.type?.app?.uid, - usageType, - usageAmount, - costOverride, - costOverrideRaw, - }); - } - - const mappedCost = COST_MAPS[usageType as keyof typeof COST_MAPS]; - const totalCost = costOverride ?? ((mappedCost || 0) * usageAmount); - totalBatchCost += totalCost; - - // Check for zero cost warning (only flag once per batch) - if ( !hasZeroCostWarning && totalCost === 0 && (mappedCost !== 0 && costOverride !== 0 ) ) { - hasZeroCostWarning = true; - this.#alarmService.create(`metering unexpected 0 cost access to: ${usageType}`, '0 cost abuse vector', { - userId: actor.type?.user?.uuid, - username: actor.type?.user?.username, - appId: actor.type?.app?.uid, - usageType, - usageAmount, - costOverride, - costOverrideRaw, - }); - } - - const escapedUsageType = usageType.replace(/\./g, PERIOD_ESCAPE) as keyof typeof COST_MAPS; - - // Aggregate into the pathAndAmountMap - aggregatedPathAndAmountMap['total'] = (aggregatedPathAndAmountMap['total'] || 0) + totalCost; - aggregatedPathAndAmountMap[`${escapedUsageType}.units`] = (aggregatedPathAndAmountMap[`${escapedUsageType}.units`] || 0) + usageAmount; - aggregatedPathAndAmountMap[`${escapedUsageType}.cost`] = (aggregatedPathAndAmountMap[`${escapedUsageType}.cost`] || 0) + totalCost; - aggregatedPathAndAmountMap[`${escapedUsageType}.count`] = (aggregatedPathAndAmountMap[`${escapedUsageType}.count`] || 0) + 1; - } - - const appId = actor.type?.app?.uid || GLOBAL_APP_KEY; - const userId = actor.type?.user?.uuid!; - - const actorUsageKey = `${METRICS_PREFIX}:actor:${userId}:${currentMonth}`; - const actorUsagesPromise = this.#kvStore.incr({ - key: actorUsageKey, - pathAndAmountMap: aggregatedPathAndAmountMap, - }) as unknown as Promise; - - const puterConsumptionKey = this.#generateGloabalUsageKey(userId, appId, currentMonth); - this.#kvStore.incr({ - key: puterConsumptionKey, - pathAndAmountMap: aggregatedPathAndAmountMap, - }).catch((e: Error) => { - console.warn(`Failed to increment aux usage data 'puterConsumptionKey' with error: ${ e.message} for userId: ${userId} appId: ${appId}`); - }); - - const actorAppUsageKey = `${METRICS_PREFIX}:actor:${userId}:app:${appId}:${currentMonth}`; - this.#kvStore.incr({ - key: actorAppUsageKey, - pathAndAmountMap: aggregatedPathAndAmountMap, - }).catch((e: Error) => { - console.warn(`Failed to increment aux usage data 'actorAppUsageKey' with error: ${ e.message} for userId: ${userId} appId: ${appId}`); - }); - - const appUsageKey = this.#generateAppUsageKey(appId, userId, currentMonth); - this.#kvStore.incr({ - key: appUsageKey, - pathAndAmountMap: aggregatedPathAndAmountMap, - }).catch((e: Error) => { - console.warn(`Failed to increment aux usage data 'appUsageKey' with error: ${ e.message} for userId: ${userId} appId: ${appId}`); - }); - - const actorAppTotalsKey = `${METRICS_PREFIX}:actor:${userId}:apps:${currentMonth}`; - this.#kvStore.incr({ - key: actorAppTotalsKey, - pathAndAmountMap: { - [`${appId}.total`]: totalBatchCost, - [`${appId}.count`]: usages.length, - }, - }).catch((e: Error) => { - console.warn(`Failed to increment aux usage data 'actorAppTotalsKey' with error: ${ e.message} for userId: ${userId} appId: ${appId}`); - }); - - const lastUpdatedKey = `${METRICS_PREFIX}:actor:${userId}:lastUpdated`; - this.#kvStore.set({ - key: lastUpdatedKey, - value: Date.now(), - }).catch((e: Error) => { - console.warn(`Failed to set lastUpdatedKey with error: ${ e.message}`); - }); - - // update addon usage if we are over the allowance - const actorSubscriptionPromise = this.getActorSubscription(actor); - const actorAddonsPromise = this.getActorAddons(actor); - const [actorUsages, actorSubscription, actorAddons] = (await Promise.all([actorUsagesPromise, actorSubscriptionPromise, actorAddonsPromise])); - - if ( actorUsages.total > actorSubscription.monthUsageAllowance && actorAddons.purchasedCredits && actorAddons.purchasedCredits > (actorAddons.consumedPurchaseCredits || 0) ) { - // if we are now over the allowance, start consuming purchased credits - const withinBoundsUsage = Math.max(0, actorSubscription.monthUsageAllowance - actorUsages.total + totalBatchCost); - const overageUsage = totalBatchCost - withinBoundsUsage; - - if ( overageUsage > 0 ) { - await this.#kvStore.incr({ - key: `${POLICY_PREFIX}:actor:${userId}:addons`, - pathAndAmountMap: { - consumedPurchaseCredits: Math.min(overageUsage, actorAddons.purchasedCredits - (actorAddons.consumedPurchaseCredits || 0)), - }, - }); - } - } - - // alert if significantly over allowance and no purchased credits left - const allowedUsageMultiple = Math.floor(actorUsages.total / actorSubscription.monthUsageAllowance); - const previousAllowedUsageMultiple = Math.floor((actorUsages.total - totalBatchCost) / actorSubscription.monthUsageAllowance); - const isOver2x = allowedUsageMultiple >= 2; - const isChangeOverPastOverage = previousAllowedUsageMultiple < allowedUsageMultiple; - const hasNoAddonCredit = (actorAddons.purchasedCredits || 0) <= (actorAddons.consumedPurchaseCredits || 0); - - if ( isOver2x && isChangeOverPastOverage && hasNoAddonCredit ) { - this.#alarmService.create(`metering usage exceeded by user: ${actor.type?.user?.username}`, `Actor ${userId} has exceeded their usage allowance significantly`, { - userId: actor.type?.user?.uuid, - username: actor.type?.user?.username, - appId: actor.type?.app?.uid, - batchUsages: usages, - totalBatchCost, - totalUsage: actorUsages.total, - monthUsageAllowance: actorSubscription.monthUsageAllowance, - }); - } - - return actorUsages; - }); - } catch (e) { - console.error('Metering: Failed to batch increment usage for actor', actor, 'usages', usages, e); - this.#alarmService.create(`metering service error for user: ${ actor.type?.user?.username} app: ${ actor.type.app?.uid}`, (e as Error).message, { - userId: actor.type?.user?.uuid, - username: actor.type?.user?.username, - appId: actor.type?.app?.uid, - error: e, - actor, - batchUsages: usages, - }); - return { total: 0 } as UsageByType; - } - } - - async getActorCurrentMonthUsageDetails (actor: Actor) { - if ( ! actor.type?.user?.uuid ) { - throw new Error('Actor must be a user to get usage details'); - } - // batch get actor usage, per app usage, and actor app totals for the month - const currentMonth = this.#getMonthYearString(); - const keys = [ - `${METRICS_PREFIX}:actor:${actor.type.user.uuid}:${currentMonth}`, - `${METRICS_PREFIX}:actor:${actor.type.user.uuid}:apps:${currentMonth}`, - ]; - - return await this.#superUserService.sudo(async () => { - const [usage, appTotals] = await this.#kvStore.get({ key: keys }) as [UsageByType | null, Record | null]; - // only show details of app based on actor, aggregate all as others, except if app is global one or null, then show all - const appId = actor.type?.app?.uid; - if ( appTotals && appId ) { - const filteredAppTotals: Record = {}; - let othersTotal: AppTotals = {} as AppTotals; - Object.entries(appTotals).forEach(([appKey, appUsage]) => { - if ( appKey === appId ) { - filteredAppTotals[appKey] = appUsage; - } else { - Object.entries(appUsage).forEach(([usageKind, amount]) => { - if ( ! othersTotal[usageKind as keyof AppTotals] ) { - othersTotal[usageKind as keyof AppTotals] = 0; - } - othersTotal[usageKind as keyof AppTotals] += amount; - }); - } - }); - if ( othersTotal ) { - filteredAppTotals['others'] = othersTotal; - } - return { - usage: usage || { total: 0 }, - appTotals: filteredAppTotals, - }; - } - return { - usage: usage || { total: 0 }, - appTotals: appTotals || {}, - }; - }); - } - - async setActorCurrentMonthUsageTotal (actor: Actor, totalCost: number) { - if ( ! actor.type?.user?.uuid ) { - throw new Error('Actor must be a user to set usage details'); - } - if ( !Number.isFinite(totalCost) || totalCost < 0 ) { - throw new Error('Total cost must be a non-negative number'); - } - - const normalizedTotal = Math.round(totalCost); - const currentMonth = this.#getMonthYearString(); - const userId = actor.type.user.uuid; - const appId = actor.type?.app?.uid || GLOBAL_APP_KEY; - - return await this.#superUserService.sudo(async () => { - const actorUsageKey = `${METRICS_PREFIX}:actor:${userId}:${currentMonth}`; - const currentUsage = await this.#kvStore.get({ key: actorUsageKey }) as UsageByType | null; - const currentTotal = currentUsage?.total ?? 0; - const delta = normalizedTotal - currentTotal; - - if ( delta === 0 ) { - return currentUsage || { total: 0 } as UsageByType; - } - - const pathAndAmountMap = { - total: delta, - 'manual_adjustment.cost': delta, - 'manual_adjustment.units': delta, - 'manual_adjustment.count': 1, - }; - - const updatedUsage = await this.#kvStore.incr({ - key: actorUsageKey, - pathAndAmountMap, - }) as unknown as UsageByType; - - const puterConsumptionKey = this.#generateGloabalUsageKey(userId, appId, currentMonth); - this.#kvStore.incr({ - key: puterConsumptionKey, - pathAndAmountMap, - }).catch((e: Error) => { - console.warn(`Failed to increment aux usage data 'puterConsumptionKey' with error: ${ e.message} for userId: ${userId} appId: ${appId}`); - }); - - const actorAppUsageKey = `${METRICS_PREFIX}:actor:${userId}:app:${appId}:${currentMonth}`; - this.#kvStore.incr({ - key: actorAppUsageKey, - pathAndAmountMap, - }).catch((e: Error) => { - console.warn(`Failed to increment aux usage data 'actorAppUsageKey' with error: ${ e.message} for userId: ${userId} appId: ${appId}`); - }); - - const actorAppTotalsKey = `${METRICS_PREFIX}:actor:${userId}:apps:${currentMonth}`; - this.#kvStore.incr({ - key: actorAppTotalsKey, - pathAndAmountMap: { - [`${appId}.total`]: delta, - [`${appId}.count`]: 1, - }, - }).catch((e: Error) => { - console.warn('Failed to increment aux usage data \'actorAppTotalsKey\' with error: ', e); - }); - - const lastUpdatedKey = `${METRICS_PREFIX}:actor:${userId}:lastUpdated`; - this.#kvStore.set({ - key: lastUpdatedKey, - value: Date.now(), - }).catch((e: Error) => { - console.warn('Failed to set lastUpdatedKey with error: ', e); - }); - - return updatedUsage; - }); - } - - async getActorCurrentMonthAppUsageDetails (actor: Actor, appId?: string) { - if ( ! actor.type?.user?.uuid ) { - throw new Error('Actor must be a user to get usage details'); - } - appId = appId || actor.type?.app?.uid || GLOBAL_APP_KEY; - // batch get actor usage, per app usage, and actor app totals for the month - const currentMonth = this.#getMonthYearString(); - const key = `${METRICS_PREFIX}:actor:${actor.type.user.uuid}:app:${appId}:${currentMonth}`; - - return await this.#superUserService.sudo(async () => { - const usage = await this.#kvStore.get({ key }) as UsageByType | null; - // only show usage if actor app is the same or if global app ( null appId ) - const actorAppId = actor.type?.app?.uid; - if ( actorAppId && actorAppId !== appId && appId !== GLOBAL_APP_KEY ) { - throw new Error('Actor can only get usage details for their own app or global app'); - } - return usage || { total: 0 } as UsageByType; - }); - } - - async getRemainingUsage (actor: Actor) { - const allowedUsage = await this.getAllowedUsage(actor); - return allowedUsage.remaining || 0; - - } - - async getAllowedUsage (actor: Actor) { - const userSubscriptionPromise = this.getActorSubscription(actor); - const userAddonsPromise = this.getActorAddons(actor); - const currentUsagePromise = this.getActorCurrentMonthUsageDetails(actor); - - const [userSubscription, addons, currentMonthUsage] = await Promise.all([userSubscriptionPromise, userAddonsPromise, currentUsagePromise]); - return { - remaining: Math.max(0, (userSubscription.monthUsageAllowance || 0) + (addons?.purchasedCredits || 0) - (currentMonthUsage.usage.total || 0) - (addons?.consumedPurchaseCredits || 0)), - monthUsageAllowance: userSubscription.monthUsageAllowance, - addons, - }; - } - - async hasAnyUsage (actor: Actor) { - return (await this.getRemainingUsage(actor)) > 0; - } - - async hasEnoughCreditsFor (actor: Actor, usageType: keyof typeof COST_MAPS, usageAmount: number) { - const remainingUsage = await this.getRemainingUsage(actor); - const cost = (COST_MAPS[usageType] || 0) * (usageAmount < 0 ? 1 : usageAmount); - return remainingUsage >= cost; - } - - async hasEnoughCredits (actor: Actor, amount: number) { - const remainingUsage = await this.getRemainingUsage(actor); - return remainingUsage >= amount; - } - - async getActorSubscription (actor: Actor): Promise<(typeof SUB_POLICIES)[number]> { - // TODO DS: maybe allow non-user actors to have subscriptions eventually - if ( ! actor.type?.user?.uuid ) { - throw new Error('Actor must be a user to get policy'); - } - - const defaultUserSubscriptionId = (actor.type.user.email ? DEFAULT_FREE_SUBSCRIPTION : DEFAULT_TEMP_SUBSCRIPTION); - const defaultSubscriptionEvent = { actor, defaultSubscriptionId: '' }; - const availablePoliciesEvent = { actor, availablePolicies: [] as (typeof SUB_POLICIES)[number][] }; - const userSubscriptionEvent = { actor, userSubscriptionId: '' }; - - await Promise.allSettled([ - this.#eventService.emit('metering:overrideDefaultSubscription', defaultSubscriptionEvent), // can override default subscription based on actor properties - this.#eventService.emit('metering:registerAvailablePolicies', availablePoliciesEvent), // will add or modify available policies - this.#eventService.emit('metering:getUserSubscription', userSubscriptionEvent), // will set userSubscription property on event - ]); - - const defaultSubscriptionId = defaultSubscriptionEvent.defaultSubscriptionId as unknown as (typeof SUB_POLICIES)[number]['id'] || defaultUserSubscriptionId; - const availablePolicies = [...availablePoliciesEvent.availablePolicies, ...SUB_POLICIES]; - const userSubscriptionId = userSubscriptionEvent.userSubscriptionId as unknown as typeof SUB_POLICIES[number]['id'] || defaultSubscriptionId; - - return availablePolicies.find(({ id }) => id === userSubscriptionId) || availablePolicies.find(({ id }) => id === defaultSubscriptionId)!; - } - - async getActorAddons (actor: Actor) { - if ( ! actor.type?.user?.uuid ) { - throw new Error('Actor must be a user to get policy addons'); - } - const key = `${POLICY_PREFIX}:actor:${actor.type.user?.uuid}:addons`; - return this.#superUserService.sudo(async () => { - const addons = await this.#kvStore.get({ key }); - return (addons ?? {}) as UsageAddons; - }); - } - - async getActorAppUsage (actor: Actor, appId: string) { - if ( ! actor.type?.user?.uuid ) { - throw new Error('Actor must be a user to get app usage'); - } - - // only allow actor to get their own app usage - if ( actor.type?.app?.uid && actor.type?.app?.uid !== appId ) { - throw new Error('Actor can only get usage for their own app'); - } - - const currentMonth = this.#getMonthYearString(); - const key = `${METRICS_PREFIX}:actor:${actor.type.user.uuid}:app:${appId}:${currentMonth}`; - return this.#superUserService.sudo(async () => { - const usage = await this.#kvStore.get({ key }); - return (usage ?? { total: 0 }) as UsageByType; - }); - } - - async getGlobalUsage () { - - // TODO DS: add validation here? - - const currentMonth = this.#getMonthYearString(); - const keyPrefix = `${METRICS_PREFIX}:puter:`; - return this.#superUserService.sudo(async () => { - const keys: string[] = []; - for ( let shard = 0; shard < MeteringService.GLOBAL_SHARD_COUNT; shard++ ) { - keys.push(`${keyPrefix}${shard}:${currentMonth}`); - } - keys.push(`${keyPrefix}${currentMonth}`); // for initial unsharded data - const usages = await this.#kvStore.get({ key: keys }) as UsageByType[]; - const aggregatedUsage: UsageByType = { total: 0 } as UsageByType; - usages.filter(Boolean).forEach(({ total, ...usage } = {} as UsageByType) => { - aggregatedUsage.total += total || 0; - - Object.entries((usage || {}) as Record).forEach(([usageKind, record]) => { - if ( ! aggregatedUsage[usageKind] ) { - aggregatedUsage[usageKind] = { cost: 0, units: 0, count: 0 } as UsageRecord; - } - const aggregatedRecord = aggregatedUsage[usageKind] as UsageRecord; - aggregatedRecord.cost += record.cost; - aggregatedRecord.count += record.count; - aggregatedRecord.units += record.units; - }); - }); - return aggregatedUsage; - }); - } - - async updateAddonCredit (userId: string, tokenAmount: number) { - if ( ! userId ) { - throw new Error('User needed to update extra credits'); - } - const key = `${POLICY_PREFIX}:actor:${userId}:addons`; - return this.#superUserService.sudo(async () => { - await this.#kvStore.incr({ - key, - pathAndAmountMap: { - purchasedCredits: tokenAmount, - }, - }); - }); - } - - async #checkRateOfChange () { - const now = Date.now(); - const lastChange = await this.#superUserService.sudo(async () => { - return this.#kvStore.get({ key: `${METRICS_PREFIX}:lastGlobalUsageCheck` }) as Promise<{ total: number, timestamp: number } | null>; - }); - - if ( !lastChange || (now - lastChange.timestamp) > 14 * 60 * 1000 ) { - // only checked if more than 14 minutes from last check - const globalUsage = await this.getGlobalUsage(); - const currTotal = globalUsage.total; - - if ( lastChange ) { - const timeDelta = now - lastChange.timestamp; - const usageDelta = currTotal - lastChange.total; - const usagePerMinute = (usageDelta / (timeDelta / 60000)); - - if ( usagePerMinute > MeteringService.MAX_GLOBAL_USAGE_PER_MINUTE ) { - this.#alarmService.create('metering:excessiveGlobalUsageRate', `Global usage rate is excessive: ${usagePerMinute} micro-cents per minute`, { - usagePerMinute, - maxAllowedPerMinute: MeteringService.MAX_GLOBAL_USAGE_PER_MINUTE, - }); - } - } - await this.#superUserService.sudo(async () => { - await this.#kvStore.set({ - key: `${METRICS_PREFIX}:lastGlobalUsageCheck`, - value: { - total: currTotal, - timestamp: now, - }, - }); - }); - } - } -} diff --git a/src/backend/src/services/MeteringService/MeteringServiceWrapper.mjs b/src/backend/src/services/MeteringService/MeteringServiceWrapper.mjs deleted file mode 100644 index a33c21ce6..000000000 --- a/src/backend/src/services/MeteringService/MeteringServiceWrapper.mjs +++ /dev/null @@ -1,22 +0,0 @@ -import BaseService from '../BaseService.js'; -import { MeteringService } from './MeteringService.js'; - -export class MeteringServiceWrapper extends BaseService { - - /** @type {import('./MeteringService.js').MeteringService} */ - meteringService = undefined; - _init () { - this.meteringService = new MeteringService({ - kvStore: this.services.get('puter-kvstore').as('puter-kvstore'), - superUserService: this.services.get('su'), - alarmService: this.services.get('alarm'), - eventService: this.services.get('event'), - }); - // TODO DS: if we can pull this to an extension I don't need this - // for now this is util so you don't have to extract this.meteringService - Object.getOwnPropertyNames(MeteringService.prototype).forEach(fn => { - if ( fn === 'constructor' ) return; - this[fn] = (...args) => this.meteringService[fn](...args); - }); - } -} diff --git a/src/backend/src/services/MeteringService/README.md b/src/backend/src/services/MeteringService/README.md deleted file mode 100644 index 4f6f606c0..000000000 --- a/src/backend/src/services/MeteringService/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# Metering Service - -This service provides all metering functionality in puter. -It relies on our own KV infrastructure to track usage (note the implementation of kvStore affects performance, and atomicity, currently sqlite implementation is not atomic). - -It will also slowly add functionality around credit purchasing in the future, but for now it is just metering and usage. -This should be the primary, and ideally only, way to check for usage and record it. - -## Usage -### Within Core Modules -To use the metering service within core modules, you can access it via the `services` object. Here's an example of how to check if an actor has enough credits for a specific usage type: - -```typescript -class SomeCoreModule extends BaseService { - get #meteringService(): MeteringService { - return this.services.get('meteringService') as MeteringService; - } - - async someMeteredFunction(actor: Actor) { - const hasEnoughCredits = await this.#meteringService.hasEnoughCreditsFor(actor, 'someUsageKey:units', 1000); - - // ... - - const updatedUsage = await this.#meteringService.incrementUsage(actor, 'someUsageKey:units', 1000); - } -} -``` -Note you don't have to structure like that if you don't want, but it's a nice way to encapsulate the service access. You can also do: -```typescript -const meteringService = this.services.get('meteringService') as MeteringService; -// or -const meteringService = Context.get('services').get('meteringService') as MeteringService; -``` -or any other way you like to access services. -### Within Extensions -To use the metering service within extensions, you can import it using the extension's import service method - -```javascript -/** @type {import('@heyputer/backend/src/services/MeteringService/MeteringServiceWrapper.mjs').MeteringServiceWrapper} */ -const meteringService = extension.import('service:meteringService'); -``` - -### Note on imports -Due to the way we structure services, when importing the metering service in extensions, you get the `MeteringServiceWrapper` class. This is a bit of a middlestep while MeteringService is not an extension itself. Which is why you'll see some places doing: -```typescript -const meteringService = this.services.get('meteringService').meteringService as MeteringService -``` -but for usability, those same methods are exposed directly on the wrapper so you don't need to do that. - -## Cost maps -The metering service relies on cost maps to determine how much to charge for a given operation. -Cost maps are simple JSON objects that map a usage type to a cost per unit in microcents (1 millionth of a cent). -For example, a cost map for AWS Polly might look like this: - -```json -{ - "aws-polly:standard:character": 4, - "aws-polly:neural:character": 16 -} -``` - -We need to manually update these for now until we can automate it somehow. -You can add more costs to the cost map as needed. - -## Cost overrides -In some cases, you may want to override the default cost for a specific actor, or give a cost if not provided in the cost map. -you can do this by passing in the cost override when incrementing usage: - -```typescript -await meteringService.incrementUsage(actor, 'someUnmappedOperation:units', 1000, 5000000); // override cost to 5 cents = 5 million microcents for the whole 1000 units -``` - -## Other util methods -See [MeteringService.ts](./MeteringService.ts) for more details on how metering works. Its all typescript so you can always just get intellisense on the methods. - - -## Adding and Getting User Subscription Plans -Though the metering service itself doesn't handle subscriptions nor credit purchases (yet at least), it does emit events for extensions to provide them with the necessary data to limit usage for users. -These following events are emitted: -- `metering:overrideDefaultSubscription` - allows extension to override the default subscription plan for a user -- `metering:registerAvailablePolicies` - allows extension to register available subscription policies/plans -- `metering:getUserSubscription` - allows extension to provide the current subscription plan for a user -For example on these see the extension [meteringAndBilling](../../../../../extensions/meteringAndBilling/eventListeners/subscriptionEvents.js) for how to use these events to provide subscription plans. - -## Examples -### Core Module example -See OpenAI module for an example of how to use the metering service within a core module: [OpenAICompletionService.mjs](../../modules/puterai/OpenAiCompletionService/OpenAICompletionService.mjs) -### Extension example -See meteringAndBilling extension for an example of how to use the metering service within an extension: [usage.js](../../../../../extensions/meteringAndBilling/routes/usage.js) \ No newline at end of file diff --git a/src/backend/src/services/MeteringService/consts.ts b/src/backend/src/services/MeteringService/consts.ts deleted file mode 100644 index 14b8a4e41..000000000 --- a/src/backend/src/services/MeteringService/consts.ts +++ /dev/null @@ -1,7 +0,0 @@ - -export const GLOBAL_APP_KEY = 'os-global'; // TODO DS: this should be loaded from config or db eventually -export const METRICS_PREFIX = 'metering'; -export const POLICY_PREFIX = 'policy'; -export const PERIOD_ESCAPE = '_dot_'; // to replace dots in usage types for kvstore paths -export const DEFAULT_FREE_SUBSCRIPTION = 'user_free'; // TODO DS: this should be loaded from config or db eventually -export const DEFAULT_TEMP_SUBSCRIPTION = 'temp_free'; // TODO DS: this should be loaded from config or db eventually \ No newline at end of file diff --git a/src/backend/src/services/MeteringService/costMaps/awsPollyCostMap.ts b/src/backend/src/services/MeteringService/costMaps/awsPollyCostMap.ts deleted file mode 100644 index bf98a5561..000000000 --- a/src/backend/src/services/MeteringService/costMaps/awsPollyCostMap.ts +++ /dev/null @@ -1,24 +0,0 @@ -// AWS Polly Cost Map (character-based pricing for text-to-speech) -// -// This map defines per-character pricing (in microcents) for AWS Polly TTS engines. -// Pricing is based on the ENGINE_PRICING object from AWSPollyService.js. -// Each entry is the cost per character for the specified engine. -// -// Pattern: "aws-polly:{engine}:character" -// Example: "aws-polly:standard:character" → 400 microcents per character -// -// Note: This is per-character pricing for TTS engines, not token-based. - -export const AWS_POLLY_COST_MAP = { - // Standard engine: $4.00 per 1M characters (400 microcents per character) - 'aws-polly:standard:character': 400, - - // Neural engine: $16.00 per 1M characters (1600 microcents per character) - 'aws-polly:neural:character': 1600, - - // Long-form engine: $100.00 per 1M characters (10000 microcents per character) - 'aws-polly:long-form:character': 10000, - - // Generative engine: $30.00 per 1M characters (3000 microcents per character) - 'aws-polly:generative:character': 3000, -}; \ No newline at end of file diff --git a/src/backend/src/services/MeteringService/costMaps/awsTextractCostMap.ts b/src/backend/src/services/MeteringService/costMaps/awsTextractCostMap.ts deleted file mode 100644 index 4a0f078c5..000000000 --- a/src/backend/src/services/MeteringService/costMaps/awsTextractCostMap.ts +++ /dev/null @@ -1,15 +0,0 @@ -// AWS Textract Cost Map (page-based pricing for OCR) -// -// This map defines per-page pricing (in microcents) for AWS Textract OCR API. -// Pricing is based on the Detect Document Text API: $1.50 per 1,000 pages. -// Each entry is the cost per page for the specified API. -// -// Pattern: "aws-textract:{api}:page" -// Example: "aws-textract:detect-document-text:page" → 150 microcents per page -// -// Note: 1,000,000 microcents = $0.01 USD. $1.50 per 1,000 pages = $0.0015 per page = 0.15 cents per page = 150000 microcents per page. -// -export const AWS_TEXTRACT_COST_MAP = { - // Detect Document Text API: $1.50 per 1,000 pages (150000 microcents per page) - 'aws-textract:detect-document-text:page': 150000, -}; \ No newline at end of file diff --git a/src/backend/src/services/MeteringService/costMaps/claudeCostMap.ts b/src/backend/src/services/MeteringService/costMaps/claudeCostMap.ts deleted file mode 100644 index bc06f9d62..000000000 --- a/src/backend/src/services/MeteringService/costMaps/claudeCostMap.ts +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -export const CLAUDE_COST_MAP = { - // Claude Opus 4.6 - 'claude:claude-opus-4-6:input_tokens': 500, - 'claude:claude-opus-4-6:ephemeral_5m_input_tokens': 500 * 1.25, - 'claude:claude-opus-4-6:ephemeral_1h_input_tokens': 500 * 2, - 'claude:claude-opus-4-6:cache_read_input_tokens': 500 * 0.1, - 'claude:claude-opus-4-6:output_tokens': 2500, - - // Claude Opus 4.5 - 'claude:claude-opus-4-5-20251101:input_tokens': 500, - 'claude:claude-opus-4-5-20251101:ephemeral_5m_input_tokens': 500 * 1.25, - 'claude:claude-opus-4-5-20251101:ephemeral_1h_input_tokens': 500 * 2, - 'claude:claude-opus-4-5-20251101:cache_read_input_tokens': 500 * 0.1, - 'claude:claude-opus-4-5-20251101:output_tokens': 2500, - - // Claude Haiku 4.5 - 'claude:claude-haiku-4-5-20251001:input_tokens': 100, - 'claude:claude-haiku-4-5-20251001:ephemeral_5m_input_tokens': 100 * 1.25, - 'claude:claude-haiku-4-5-20251001:ephemeral_1h_input_tokens': 100 * 2, - 'claude:claude-haiku-4-5-20251001:cache_read_input_tokens': 100 * 0.1, - 'claude:claude-haiku-4-5-20251001:output_tokens': 500, - - // Claude Sonnet 4.5 - 'claude:claude-sonnet-4-5-20250929:input_tokens': 300, - 'claude:claude-sonnet-4-5-20250929:ephemeral_5m_input_tokens': 300 * 1.25, - 'claude:claude-sonnet-4-5-20250929:ephemeral_1h_input_tokens': 300 * 2, - 'claude:claude-sonnet-4-5-20250929:cache_read_input_tokens': 300 * 0.1, - 'claude:claude-sonnet-4-5-20250929:output_tokens': 1500, - - // Claude Opus 4.1 - 'claude:claude-opus-4-1-20250805:input_tokens': 1500, - 'claude:claude-opus-4-1-20250805:ephemeral_5m_input_tokens': 1500 * 1.25, - 'claude:claude-opus-4-1-20250805:ephemeral_1h_input_tokens': 1500 * 2, - 'claude:claude-opus-4-1-20250805:cache_read_input_tokens': 1500 * 0.1, - 'claude:claude-opus-4-1-20250805:output_tokens': 7500, - - // Claude Opus 4 - 'claude:claude-opus-4-20250514:input_tokens': 1500, - 'claude:claude-opus-4-20250514:ephemeral_5m_input_tokens': 1500 * 1.25, - 'claude:claude-opus-4-20250514:ephemeral_1h_input_tokens': 1500 * 2, - 'claude:claude-opus-4-20250514:cache_read_input_tokens': 1500 * 0.1, - 'claude:claude-opus-4-20250514:output_tokens': 7500, - - // Claude Sonnet 4 - 'claude:claude-sonnet-4-20250514:input_tokens': 300, - 'claude:claude-sonnet-4-20250514:ephemeral_5m_input_tokens': 300 * 1.25, - 'claude:claude-sonnet-4-20250514:ephemeral_1h_input_tokens': 300 * 2, - 'claude:claude-sonnet-4-20250514:cache_read_input_tokens': 300 * 0.1, - 'claude:claude-sonnet-4-20250514:output_tokens': 1500, - - // Claude 3.7 Sonnet - 'claude:claude-3-7-sonnet-20250219:input_tokens': 300, - 'claude:claude-3-7-sonnet-20250219:ephemeral_5m_input_tokens': 300 * 1.25, - 'claude:claude-3-7-sonnet-20250219:ephemeral_1h_input_tokens': 300 * 2, - 'claude:claude-3-7-sonnet-20250219:cache_read_input_tokens': 300 * 0.1, - 'claude:claude-3-7-sonnet-20250219:output_tokens': 1500, - - // Claude 3.5 Sonnet (Oct 2024) - 'claude:claude-3-5-sonnet-20241022:input_tokens': 300, - 'claude:claude-3-5-sonnet-20241022:ephemeral_5m_input_tokens': 300 * 1.25, - 'claude:claude-3-5-sonnet-20241022:ephemeral_1h_input_tokens': 300 * 2, - 'claude:claude-3-5-sonnet-20241022:cache_read_input_tokens': 300 * 0.1, - 'claude:claude-3-5-sonnet-20241022:output_tokens': 1500, - - // Claude 3.5 Sonnet (June 2024) - 'claude:claude-3-5-sonnet-20240620:input_tokens': 300, - 'claude:claude-3-5-sonnet-20240620:ephemeral_5m_input_tokens': 300 * 1.25, - 'claude:claude-3-5-sonnet-20240620:ephemeral_1h_input_tokens': 300 * 2, - 'claude:claude-3-5-sonnet-20240620:cache_read_input_tokens': 300 * 0.1, - 'claude:claude-3-5-sonnet-20240620:output_tokens': 1500, - - // Claude 3 Haiku - 'claude:claude-3-haiku-20240307:input_tokens': 25, - 'claude:claude-3-haiku-20240307:ephemeral_5m_input_tokens': 25 * 1.25, - 'claude:claude-3-haiku-20240307:ephemeral_1h_input_tokens': 25 * 2, - 'claude:claude-3-haiku-20240307:cache_read_input_tokens': 25 * 0.1, - 'claude:claude-3-haiku-20240307:output_tokens': 125, -}; \ No newline at end of file diff --git a/src/backend/src/services/MeteringService/costMaps/deepSeekCostMap.ts b/src/backend/src/services/MeteringService/costMaps/deepSeekCostMap.ts deleted file mode 100644 index e8512d5bd..000000000 --- a/src/backend/src/services/MeteringService/costMaps/deepSeekCostMap.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -export const DEEPSEEK_COST_MAP = { - // DeepSeek Chat - 'deepseek:deepseek-chat:prompt_tokens': 28, - 'deepseek:deepseek-chat:completion_tokens': 42, - 'deepseek:deepseek-chat:cached_tokens': 2.8, - - // DeepSeek Reasoner - 'deepseek:deepseek-reasoner:prompt_tokens': 28, - 'deepseek:deepseek-reasoner:completion_tokens': 42, - 'deepseek:deepseek-reasoner:cached_tokens': 2.8, -}; \ No newline at end of file diff --git a/src/backend/src/services/MeteringService/costMaps/elevenlabsCostMap.ts b/src/backend/src/services/MeteringService/costMaps/elevenlabsCostMap.ts deleted file mode 100644 index 29238064d..000000000 --- a/src/backend/src/services/MeteringService/costMaps/elevenlabsCostMap.ts +++ /dev/null @@ -1,16 +0,0 @@ -// ElevenLabs Text-to-Speech Cost Map -// -// Pricing for ElevenLabs voices varies by model and plan tier. We don't yet -// have public micro-cent pricing, so we record usage with a zero cost. This -// prevents metering alerts while still tracking character counts for future -// cost attribution once pricing is finalized. - -export const ELEVENLABS_COST_MAP = { - 'elevenlabs:eleven_multilingual_v2:character': 18000 * 0.9, // using scale costs per additional char * 0.9 - 'elevenlabs:eleven_turbo_v2_5:character': 18000 * 0.9, // using scale costs per additional char * 0.9 - 'elevenlabs:eleven_turbo_v2:character': 18000 * 0.9, // using scale costs per additional char * 0.9 - 'elevenlabs:eleven_flash_v2_5:character': 9000 * 0.9, // using scale costs per additional char * 0.9 - 'elevenlabs:eleven_v3:character': 18000 * 0.9, // using scale costs per additional char * 0.9 - 'elevenlabs:eleven_multilingual_sts_v2:second': 300000 * 0.9, // using scale costs unit * 0.9 - 'elevenlabs:eleven_english_sts_v2:second': 300000 * 0.9, // using scale costs unit * 0.9 -}; diff --git a/src/backend/src/services/MeteringService/costMaps/fileSystemCostMap.ts b/src/backend/src/services/MeteringService/costMaps/fileSystemCostMap.ts deleted file mode 100644 index f34367f46..000000000 --- a/src/backend/src/services/MeteringService/costMaps/fileSystemCostMap.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { toMicroCents } from '../utils.js'; - -export const FILE_SYSTEM_COST_MAP = { - 'filesystem:ingress:bytes': 0, - 'filesystem:delete:bytes': 0, - 'filesystem:egress:bytes': toMicroCents(0.12 / 1024 / 1024 / 1024), // $0.11 per GB ~> 0.12 per GiB - 'filesystem:cached-egress:bytes': toMicroCents(0.1 / 1024 / 1024 / 1024), // $0.09 per GB ~> 0.1 per GiB, -}; \ No newline at end of file diff --git a/src/backend/src/services/MeteringService/costMaps/geminiCostMap.ts b/src/backend/src/services/MeteringService/costMaps/geminiCostMap.ts deleted file mode 100644 index 6dd2a300a..000000000 --- a/src/backend/src/services/MeteringService/costMaps/geminiCostMap.ts +++ /dev/null @@ -1,29 +0,0 @@ - -// TODO DS: these should be loaded from config or db eventually -/** - * flat cost map based on usage types, numbers are in microcents (1/1 millionth of a cent) - * E.g. 1000000 microcents = 1 cent - * most services measure their prices in 1 million requests or tokens or whatever, so if that's the case you can simply use the cent val - * $0.63 per 1M reads = 63 microcents per read - * $1.25 per 1M writes = 125 microcents per write - */ -export const GEMINI_COST_MAP = { - // Gemini api usage types (costs per token in microcents) - 'gemini:gemini-1.5-flash:promptTokenCount': 7.5, - 'gemini:gemini-1.5-flash:candidatesTokenCount': 30, - 'gemini:gemini-2.0-flash:promptTokenCount': 10, - 'gemini:gemini-2.0-flash:candidatesTokenCount': 40, - 'gemini:gemini-2.0-flash-lite:promptTokenCount': 8, - 'gemini:gemini-2.0-flash-lite:candidatesTokenCount': 32, - 'gemini:gemini-2.5-flash:promptTokenCount': 12, - 'gemini:gemini-2.5-flash:candidatesTokenCount': 48, - 'gemini:gemini-2.5-flash-lite:promptTokenCount': 10, - 'gemini:gemini-2.5-flash-lite:candidatesTokenCount': 40, - 'gemini:gemini-2.5-pro:promptTokenCount': 15, - 'gemini:gemini-2.5-pro:candidatesTokenCount': 60, - 'gemini:gemini-3-pro-preview:promptTokenCount': 25, - 'gemini:gemini-3-pro-preview:candidatesTokenCount': 100, - 'gemini:gemini-2.5-flash-image-preview:1024x1024': 3_900_000, - 'gemini:gemini-3-pro-image-preview:1024x1024': 15_600_000, - 'gemini:gemini-3.1-flash-image-preview:1024x1024': 6_700_000, -}; diff --git a/src/backend/src/services/MeteringService/costMaps/groqCostMap.ts b/src/backend/src/services/MeteringService/costMaps/groqCostMap.ts deleted file mode 100644 index 3e680eb2c..000000000 --- a/src/backend/src/services/MeteringService/costMaps/groqCostMap.ts +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -export const GROQ_COST_MAP = { - // Gemma models - 'groq:gemma2-9b-it:prompt_tokens': 20, - 'groq:gemma2-9b-it:completion_tokens': 20, - 'groq:gemma-7b-it:prompt_tokens': 7, - 'groq:gemma-7b-it:completion_tokens': 7, - - // Llama 3 Groq Tool Use Preview - 'groq:llama3-groq-70b-8192-tool-use-preview:prompt_tokens': 89, - 'groq:llama3-groq-70b-8192-tool-use-preview:completion_tokens': 89, - 'groq:llama3-groq-8b-8192-tool-use-preview:prompt_tokens': 19, - 'groq:llama3-groq-8b-8192-tool-use-preview:completion_tokens': 19, - - // Llama 3.1 - 'groq:llama-3.1-70b-versatile:prompt_tokens': 59, - 'groq:llama-3.1-70b-versatile:completion_tokens': 79, - 'groq:llama-3.1-70b-specdec:prompt_tokens': 59, - 'groq:llama-3.1-70b-specdec:completion_tokens': 99, - 'groq:llama-3.1-8b-instant:prompt_tokens': 5, - 'groq:llama-3.1-8b-instant:completion_tokens': 8, - - // Llama Guard - 'groq:meta-llama/llama-guard-4-12b:prompt_tokens': 20, - 'groq:meta-llama/llama-guard-4-12b:completion_tokens': 20, - 'groq:llama-guard-3-8b:prompt_tokens': 20, - 'groq:llama-guard-3-8b:completion_tokens': 20, - - // Prompt Guard - 'groq:meta-llama/llama-prompt-guard-2-86m:prompt_tokens': 4, - 'groq:meta-llama/llama-prompt-guard-2-86m:completion_tokens': 4, - - // Llama 3.2 Preview - 'groq:llama-3.2-1b-preview:prompt_tokens': 4, - 'groq:llama-3.2-1b-preview:completion_tokens': 4, - 'groq:llama-3.2-3b-preview:prompt_tokens': 6, - 'groq:llama-3.2-3b-preview:completion_tokens': 6, - 'groq:llama-3.2-11b-vision-preview:prompt_tokens': 18, - 'groq:llama-3.2-11b-vision-preview:completion_tokens': 18, - 'groq:llama-3.2-90b-vision-preview:prompt_tokens': 90, - 'groq:llama-3.2-90b-vision-preview:completion_tokens': 90, - - // Llama 3 8k/70B - 'groq:llama3-70b-8192:prompt_tokens': 59, - 'groq:llama3-70b-8192:completion_tokens': 79, - 'groq:llama3-8b-8192:prompt_tokens': 5, - 'groq:llama3-8b-8192:completion_tokens': 8, - - // Mixtral - 'groq:mixtral-8x7b-32768:prompt_tokens': 24, - 'groq:mixtral-8x7b-32768:completion_tokens': 24, -}; \ No newline at end of file diff --git a/src/backend/src/services/MeteringService/costMaps/index.ts b/src/backend/src/services/MeteringService/costMaps/index.ts deleted file mode 100644 index 0bab195ec..000000000 --- a/src/backend/src/services/MeteringService/costMaps/index.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { AWS_POLLY_COST_MAP } from './awsPollyCostMap.js'; -import { AWS_TEXTRACT_COST_MAP } from './awsTextractCostMap.js'; -import { CLAUDE_COST_MAP } from './claudeCostMap.js'; -import { DEEPSEEK_COST_MAP } from './deepSeekCostMap.js'; -import { FILE_SYSTEM_COST_MAP } from './fileSystemCostMap.js'; -import { GEMINI_COST_MAP } from './geminiCostMap.js'; -import { GROQ_COST_MAP } from './groqCostMap.js'; -import { KV_COST_MAP } from './kvCostMap.js'; -import { MISTRAL_COST_MAP } from './mistralCostMap.js'; -import { OPENAI_COST_MAP } from './openAiCostMap.js'; -import { OPENAI_IMAGE_COST_MAP } from './openaiImageCostMap.js'; -import { OPENROUTER_COST_MAP } from './openrouterCostMap.js'; -import { OPENAI_VIDEO_COST_MAP } from './openaiVideoCostMap.js'; -import { TOGETHER_COST_MAP } from './togetherCostMap.js'; -import { XAI_COST_MAP } from './xaiCostMap.js'; -import { ELEVENLABS_COST_MAP } from './elevenlabsCostMap.js'; - -export const COST_MAPS = { - ...AWS_POLLY_COST_MAP, - ...AWS_TEXTRACT_COST_MAP, - ...CLAUDE_COST_MAP, - ...DEEPSEEK_COST_MAP, - ...ELEVENLABS_COST_MAP, - ...GEMINI_COST_MAP, - ...GROQ_COST_MAP, - ...KV_COST_MAP, - ...MISTRAL_COST_MAP, - ...OPENAI_COST_MAP, - ...OPENAI_IMAGE_COST_MAP, - ...OPENAI_VIDEO_COST_MAP, - ...OPENROUTER_COST_MAP, - ...TOGETHER_COST_MAP, - ...XAI_COST_MAP, - ...FILE_SYSTEM_COST_MAP, -}; diff --git a/src/backend/src/services/MeteringService/costMaps/kvCostMap.ts b/src/backend/src/services/MeteringService/costMaps/kvCostMap.ts deleted file mode 100644 index ba79456b7..000000000 --- a/src/backend/src/services/MeteringService/costMaps/kvCostMap.ts +++ /dev/null @@ -1,5 +0,0 @@ -export const KV_COST_MAP = { - // Map with unit to cost measurements in microcent - 'kv:read': 63, - 'kv:write': 125, -}; diff --git a/src/backend/src/services/MeteringService/costMaps/mistralCostMap.ts b/src/backend/src/services/MeteringService/costMaps/mistralCostMap.ts deleted file mode 100644 index 9e5976926..000000000 --- a/src/backend/src/services/MeteringService/costMaps/mistralCostMap.ts +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -export const MISTRAL_COST_MAP = { - // Mistral models (values in microcents/token, from MistralAIService.js) - 'mistral:mistral-large-latest:prompt_tokens': 200, - 'mistral:mistral-large-latest:completion_tokens': 600, - 'mistral:pixtral-large-latest:prompt_tokens': 200, - 'mistral:pixtral-large-latest:completion_tokens': 600, - 'mistral:mistral-small-latest:prompt_tokens': 20, - 'mistral:mistral-small-latest:completion_tokens': 60, - 'mistral:codestral-latest:prompt_tokens': 30, - 'mistral:codestral-latest:completion_tokens': 90, - 'mistral:ministral-8b-latest:prompt_tokens': 10, - 'mistral:ministral-8b-latest:completion_tokens': 10, - 'mistral:ministral-3b-latest:prompt_tokens': 4, - 'mistral:ministral-3b-latest:completion_tokens': 4, - 'mistral:pixtral-12b:prompt_tokens': 15, - 'mistral:pixtral-12b:completion_tokens': 15, - 'mistral:mistral-nemo:prompt_tokens': 15, - 'mistral:mistral-nemo:completion_tokens': 15, - 'mistral:open-mistral-7b:prompt_tokens': 25, - 'mistral:open-mistral-7b:completion_tokens': 25, - 'mistral:open-mixtral-8x7b:prompt_tokens': 7, - 'mistral:open-mixtral-8x7b:completion_tokens': 7, - 'mistral:open-mixtral-8x22b:prompt_tokens': 2, - 'mistral:open-mixtral-8x22b:completion_tokens': 6, - 'mistral:magistral-medium-latest:prompt_tokens': 200, - 'mistral:magistral-medium-latest:completion_tokens': 500, - 'mistral:magistral-small-latest:prompt_tokens': 10, - 'mistral:magistral-small-latest:completion_tokens': 10, - 'mistral:mistral-medium-latest:prompt_tokens': 40, - 'mistral:mistral-medium-latest:completion_tokens': 200, - 'mistral:mistral-moderation-latest:prompt_tokens': 10, - 'mistral:mistral-moderation-latest:completion_tokens': 10, - 'mistral:devstral-small-latest:prompt_tokens': 10, - 'mistral:devstral-small-latest:completion_tokens': 10, - 'mistral:mistral-saba-latest:prompt_tokens': 20, - 'mistral:mistral-saba-latest:completion_tokens': 60, - 'mistral:open-mistral-nemo:prompt_tokens': 10, - 'mistral:open-mistral-nemo:completion_tokens': 10, - 'mistral:mistral-ocr-latest:prompt_tokens': 100, - 'mistral:mistral-ocr-latest:completion_tokens': 300, - // OCR page-based pricing (values in microcents/page) - // $1 / 1000 pages -> $0.001 per page -> 100000 microcents - 'mistral-ocr:ocr:page': 100000, - // $3 / 1000 pages -> $0.003 per page -> 300000 microcents - 'mistral-ocr:annotations:page': 300000, -}; diff --git a/src/backend/src/services/MeteringService/costMaps/openAiCostMap.ts b/src/backend/src/services/MeteringService/costMaps/openAiCostMap.ts deleted file mode 100644 index 8cef6f2a1..000000000 --- a/src/backend/src/services/MeteringService/costMaps/openAiCostMap.ts +++ /dev/null @@ -1,102 +0,0 @@ - -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -export const OPENAI_COST_MAP = { - // GPT-5 models - 'openai:gpt-5.1:prompt_tokens': 125, - 'openai:gpt-5.1:cached_tokens': 13, - 'openai:gpt-5.1:completion_tokens': 1000, - 'openai:gpt-5.1-codex:prompt_tokens': 125, - 'openai:gpt-5.1-codex:cached_tokens': 13, - 'openai:gpt-5.1-codex:completion_tokens': 1000, - 'openai:gpt-5.1-codex-mini:prompt_tokens': 25, - 'openai:gpt-5.1-codex-mini:cached_tokens': 3, - 'openai:gpt-5.1-codex-mini:completion_tokens': 200, - 'openai:gpt-5.1-chat-latest:prompt_tokens': 125, - 'openai:gpt-5.1-chat-latest:cached_tokens': 13, - 'openai:gpt-5.1-chat-latest:completion_tokens': 1000, - 'openai:gpt-5-2025-08-07:prompt_tokens': 125, - 'openai:gpt-5-2025-08-07:cached_tokens': 13, - 'openai:gpt-5-2025-08-07:completion_tokens': 1000, - 'openai:gpt-5-mini-2025-08-07:prompt_tokens': 25, - 'openai:gpt-5-mini-2025-08-07:cached_tokens': 3, - 'openai:gpt-5-mini-2025-08-07:completion_tokens': 200, - 'openai:gpt-5-nano-2025-08-07:prompt_tokens': 5, - 'openai:gpt-5-nano-2025-08-07:cached_tokens': 1, - 'openai:gpt-5-nano-2025-08-07:completion_tokens': 40, - 'openai:gpt-5-chat-latest:prompt_tokens': 125, - 'openai:gpt-5-chat-latest:cached_tokens': 13, - 'openai:gpt-5-chat-latest:completion_tokens': 1000, - - // GPT-4o models - 'openai:gpt-4o:prompt_tokens': 250, - 'openai:gpt-4o:cached_tokens': 125, - 'openai:gpt-4o:completion_tokens': 1000, - 'openai:gpt-4o-mini:prompt_tokens': 15, - 'openai:gpt-4o-mini:cached_tokens': 8, - 'openai:gpt-4o-mini:completion_tokens': 60, - - // O1 models - 'openai:o1:prompt_tokens': 1500, - 'openai:o1:cached_tokens': 750, - 'openai:o1:completion_tokens': 6000, - 'openai:o1-mini:prompt_tokens': 110, - 'openai:o1-mini:completion_tokens': 440, - 'openai:o1-pro:prompt_tokens': 15000, - 'openai:o1-pro:completion_tokens': 60000, - - // O3 models - 'openai:o3:prompt_tokens': 200, - 'openai:o3:cached_tokens': 50, - 'openai:o3:completion_tokens': 800, - 'openai:o3-mini:prompt_tokens': 110, - 'openai:o3-mini:cached_tokens': 55, - 'openai:o3-mini:completion_tokens': 440, - - // O4 models - 'openai:o4-mini:prompt_tokens': 110, - 'openai:o4-mini:completion_tokens': 440, - - // GPT-4.1 models - 'openai:gpt-4.1:prompt_tokens': 200, - 'openai:gpt-4.1:cached_tokens': 50, - 'openai:gpt-4.1:completion_tokens': 800, - 'openai:gpt-4.1-mini:prompt_tokens': 40, - 'openai:gpt-4.1-mini:cached_tokens': 10, - 'openai:gpt-4.1-mini:completion_tokens': 160, - 'openai:gpt-4.1-nano:prompt_tokens': 10, - 'openai:gpt-4.1-nano:cached_tokens': 2, - 'openai:gpt-4.1-nano:completion_tokens': 40, - - // GPT-4.5 preview - 'openai:gpt-4.5-preview:prompt_tokens': 7500, - 'openai:gpt-4.5-preview:completion_tokens': 15000, - - // Text-to-speech models (per character, microcents) - 'openai:gpt-4o-mini-tts:character': 1500, - 'openai:tts-1:character': 1500, - 'openai:tts-1-hd:character': 3000, - - // Speech-to-text models (per second, microcents) - 'openai:gpt-4o-transcribe:second': 10000, - 'openai:gpt-4o-mini-transcribe:second': 5000, - 'openai:gpt-4o-transcribe-diarize:second': 10000, - 'openai:whisper-1:second': 10000, -}; diff --git a/src/backend/src/services/MeteringService/costMaps/openaiImageCostMap.ts b/src/backend/src/services/MeteringService/costMaps/openaiImageCostMap.ts deleted file mode 100644 index 00a481be6..000000000 --- a/src/backend/src/services/MeteringService/costMaps/openaiImageCostMap.ts +++ /dev/null @@ -1,54 +0,0 @@ -// OpenAI Image Generation Cost Map (microcents per image) -// Pricing for DALL-E 2 and DALL-E 3 models based on image dimensions. -// All costs are in microcents (1/1,000,000th of a cent). Example: 1,000,000 microcents = $0.01 USD.// -// Naming pattern: "openai:{model}:{size}" or "openai:{model}:hd:{size}" for HD images - -import { toMicroCents } from '../utils.js'; - -export const OPENAI_IMAGE_COST_MAP = { - // DALL-E 3 - 'openai:dall-e-3:1024x1024': toMicroCents(0.04), // $0.04 - 'openai:dall-e-3:1024x1792': toMicroCents(0.08), // $0.08 - 'openai:dall-e-3:1792x1024': toMicroCents(0.08), // $0.08 - 'openai:dall-e-3:hd:1024x1024': toMicroCents(0.08), // $0.08 - 'openai:dall-e-3:hd:1024x1792': toMicroCents(0.12), // $0.12 - 'openai:dall-e-3:hd:1792x1024': toMicroCents(0.12), // $0.12 - - // DALL-E 2 - 'openai:dall-e-2:1024x1024': toMicroCents(0.02), // $0.02 - 'openai:dall-e-2:512x512': toMicroCents(0.018), // $0.018 - 'openai:dall-e-2:256x256': toMicroCents(0.016), // $0.016 - - // gpt-image-1.5 - 'openai:gpt-image-1.5:low:1024x1024': toMicroCents(0.009), - 'openai:gpt-image-1.5:low:1024x1536': toMicroCents(0.013), - 'openai:gpt-image-1.5:low:1536x1024': toMicroCents(0.013), - 'openai:gpt-image-1.5:medium:1024x1024': toMicroCents(0.034), - 'openai:gpt-image-1.5:medium:1024x1536': toMicroCents(0.051), - 'openai:gpt-image-1.5:medium:1536x1024': toMicroCents(0.05), - 'openai:gpt-image-1.5:high:1024x1024': toMicroCents(0.133), - 'openai:gpt-image-1.5:high:1024x1536': toMicroCents(0.20), - 'openai:gpt-image-1.5:high:1536x1024': toMicroCents(0.199), - - // gpt-image-1 - 'openai:gpt-image-1:low:1024x1024': toMicroCents(0.011), - 'openai:gpt-image-1:low:1024x1536': toMicroCents(0.016), - 'openai:gpt-image-1:low:1536x1024': toMicroCents(0.016), - 'openai:gpt-image-1:medium:1024x1024': toMicroCents(0.042), - 'openai:gpt-image-1:medium:1024x1536': toMicroCents(0.063), - 'openai:gpt-image-1:medium:1536x1024': toMicroCents(0.063), - 'openai:gpt-image-1:high:1024x1024': toMicroCents(0.167), - 'openai:gpt-image-1:high:1024x1536': toMicroCents(0.25), - 'openai:gpt-image-1:high:1536x1024': toMicroCents(0.25), - - // gpt-image-1-mini - 'openai:gpt-image-1-mini:low:1024x1024': toMicroCents(0.005), - 'openai:gpt-image-1-mini:low:1024x1536': toMicroCents(0.006), - 'openai:gpt-image-1-mini:low:1536x1024': toMicroCents(0.006), - 'openai:gpt-image-1-mini:medium:1024x1024': toMicroCents(0.011), - 'openai:gpt-image-1-mini:medium:1024x1536': toMicroCents(0.015), - 'openai:gpt-image-1-mini:medium:1536x1024': toMicroCents(0.015), - 'openai:gpt-image-1-mini:high:1024x1024': toMicroCents(0.036), - 'openai:gpt-image-1-mini:high:1024x1536': toMicroCents(0.052), - 'openai:gpt-image-1-mini:high:1536x1024': toMicroCents(0.052), -}; \ No newline at end of file diff --git a/src/backend/src/services/MeteringService/costMaps/openaiVideoCostMap.ts b/src/backend/src/services/MeteringService/costMaps/openaiVideoCostMap.ts deleted file mode 100644 index 4d84e2617..000000000 --- a/src/backend/src/services/MeteringService/costMaps/openaiVideoCostMap.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { toMicroCents } from '../utils.js'; - -// Prices are per generated video-second. -export const OPENAI_VIDEO_COST_MAP = { - 'openai:sora-2:default': toMicroCents(0.10), - 'openai:sora-2-pro:default': toMicroCents(0.30), - 'openai:sora-2-pro:xl': toMicroCents(0.50), - 'openai:sora-2-pro:xxl': toMicroCents(0.70), -}; diff --git a/src/backend/src/services/MeteringService/costMaps/openrouterCostMap.ts b/src/backend/src/services/MeteringService/costMaps/openrouterCostMap.ts deleted file mode 100644 index 2117ece59..000000000 --- a/src/backend/src/services/MeteringService/costMaps/openrouterCostMap.ts +++ /dev/null @@ -1,760 +0,0 @@ -export const OPENROUTER_COST_MAP = { - 'openrouter:google/gemini-3-pro-preview:prompt': 200, - 'openrouter:google/gemini-3-pro-preview:completion': 1200, - 'openrouter:google/gemini-3-pro-preview:image': 825600, - 'openrouter:google/gemini-3-pro-preview:input_cache_read': 20, - 'openrouter:google/gemini-3-pro-preview:input_cache_write': 238, - 'openrouter:deepcogito/cogito-v2.1-671b:prompt': 125, - 'openrouter:deepcogito/cogito-v2.1-671b:completion': 125, - 'openrouter:openai/gpt-5.1:prompt': 125, - 'openrouter:openai/gpt-5.1:completion': 1000, - 'openrouter:openai/gpt-5.1:web_search': 1000000, - 'openrouter:openai/gpt-5.1:input_cache_read': 12, - 'openrouter:openai/gpt-5.1-chat:prompt': 125, - 'openrouter:openai/gpt-5.1-chat:completion': 1000, - 'openrouter:openai/gpt-5.1-chat:web_search': 1000000, - 'openrouter:openai/gpt-5.1-chat:input_cache_read': 12, - 'openrouter:openai/gpt-5.1-codex:prompt': 125, - 'openrouter:openai/gpt-5.1-codex:completion': 1000, - 'openrouter:openai/gpt-5.1-codex:input_cache_read': 12, - 'openrouter:openai/gpt-5.1-codex-mini:prompt': 25, - 'openrouter:openai/gpt-5.1-codex-mini:completion': 200, - 'openrouter:openai/gpt-5.1-codex-mini:input_cache_read': 3, - 'openrouter:moonshotai/kimi-linear-48b-a3b-instruct:prompt': 50, - 'openrouter:moonshotai/kimi-linear-48b-a3b-instruct:completion': 60, - 'openrouter:moonshotai/kimi-k2-thinking:prompt': 45, - 'openrouter:moonshotai/kimi-k2-thinking:completion': 235, - 'openrouter:amazon/nova-premier-v1:prompt': 250, - 'openrouter:amazon/nova-premier-v1:completion': 1250, - 'openrouter:amazon/nova-premier-v1:input_cache_read': 63, - 'openrouter:perplexity/sonar-pro-search:prompt': 300, - 'openrouter:perplexity/sonar-pro-search:completion': 1500, - 'openrouter:perplexity/sonar-pro-search:request': 1800000, - 'openrouter:mistralai/voxtral-small-24b-2507:prompt': 10, - 'openrouter:mistralai/voxtral-small-24b-2507:completion': 30, - 'openrouter:mistralai/voxtral-small-24b-2507:audio': 10000, - 'openrouter:openai/gpt-oss-safeguard-20b:prompt': 7, - 'openrouter:openai/gpt-oss-safeguard-20b:completion': 30, - 'openrouter:openai/gpt-oss-safeguard-20b:input_cache_read': 4, - 'openrouter:nvidia/nemotron-nano-12b-v2-vl:prompt': 20, - 'openrouter:nvidia/nemotron-nano-12b-v2-vl:completion': 60, - 'openrouter:minimax/minimax-m2:prompt': 26, - 'openrouter:minimax/minimax-m2:completion': 102, - 'openrouter:liquid/lfm2-8b-a1b:prompt': 5, - 'openrouter:liquid/lfm2-8b-a1b:completion': 10, - 'openrouter:liquid/lfm-2.2-6b:prompt': 5, - 'openrouter:liquid/lfm-2.2-6b:completion': 10, - 'openrouter:ibm-granite/granite-4.0-h-micro:prompt': 2, - 'openrouter:ibm-granite/granite-4.0-h-micro:completion': 11, - 'openrouter:deepcogito/cogito-v2-preview-llama-405b:prompt': 350, - 'openrouter:deepcogito/cogito-v2-preview-llama-405b:completion': 350, - 'openrouter:openai/gpt-5-image-mini:prompt': 250, - 'openrouter:openai/gpt-5-image-mini:completion': 200, - 'openrouter:openai/gpt-5-image-mini:image': 250, - 'openrouter:openai/gpt-5-image-mini:web_search': 1000000, - 'openrouter:openai/gpt-5-image-mini:input_cache_read': 25, - 'openrouter:anthropic/claude-haiku-4.5:prompt': 100, - 'openrouter:anthropic/claude-haiku-4.5:completion': 500, - 'openrouter:anthropic/claude-haiku-4.5:input_cache_read': 10, - 'openrouter:anthropic/claude-haiku-4.5:input_cache_write': 125, - 'openrouter:qwen/qwen3-vl-8b-thinking:prompt': 18, - 'openrouter:qwen/qwen3-vl-8b-thinking:completion': 210, - 'openrouter:qwen/qwen3-vl-8b-instruct:prompt': 8, - 'openrouter:qwen/qwen3-vl-8b-instruct:completion': 50, - 'openrouter:openai/gpt-5-image:prompt': 1000, - 'openrouter:openai/gpt-5-image:completion': 1000, - 'openrouter:openai/gpt-5-image:image': 1000, - 'openrouter:openai/gpt-5-image:web_search': 1000000, - 'openrouter:openai/gpt-5-image:input_cache_read': 125, - 'openrouter:openai/o3-deep-research:prompt': 1000, - 'openrouter:openai/o3-deep-research:completion': 4000, - 'openrouter:openai/o3-deep-research:image': 765000, - 'openrouter:openai/o3-deep-research:web_search': 1000000, - 'openrouter:openai/o3-deep-research:input_cache_read': 250, - 'openrouter:openai/o4-mini-deep-research:prompt': 200, - 'openrouter:openai/o4-mini-deep-research:completion': 800, - 'openrouter:openai/o4-mini-deep-research:image': 153000, - 'openrouter:openai/o4-mini-deep-research:web_search': 1000000, - 'openrouter:openai/o4-mini-deep-research:input_cache_read': 50, - 'openrouter:nvidia/llama-3.3-nemotron-super-49b-v1.5:prompt': 10, - 'openrouter:nvidia/llama-3.3-nemotron-super-49b-v1.5:completion': 40, - 'openrouter:baidu/ernie-4.5-21b-a3b-thinking:prompt': 7, - 'openrouter:baidu/ernie-4.5-21b-a3b-thinking:completion': 28, - 'openrouter:google/gemini-2.5-flash-image:prompt': 30, - 'openrouter:google/gemini-2.5-flash-image:completion': 250, - 'openrouter:google/gemini-2.5-flash-image:image': 123800, - 'openrouter:qwen/qwen3-vl-30b-a3b-thinking:prompt': 20, - 'openrouter:qwen/qwen3-vl-30b-a3b-thinking:completion': 100, - 'openrouter:qwen/qwen3-vl-30b-a3b-instruct:prompt': 15, - 'openrouter:qwen/qwen3-vl-30b-a3b-instruct:completion': 60, - 'openrouter:openai/gpt-5-pro:prompt': 1500, - 'openrouter:openai/gpt-5-pro:completion': 12000, - 'openrouter:openai/gpt-5-pro:web_search': 1000000, - 'openrouter:z-ai/glm-4.6:prompt': 40, - 'openrouter:z-ai/glm-4.6:completion': 175, - 'openrouter:z-ai/glm-4.6:exacto:prompt': 45, - 'openrouter:z-ai/glm-4.6:exacto:completion': 190, - 'openrouter:anthropic/claude-sonnet-4.5:prompt': 300, - 'openrouter:anthropic/claude-sonnet-4.5:completion': 1500, - 'openrouter:anthropic/claude-sonnet-4.5:input_cache_read': 30, - 'openrouter:anthropic/claude-sonnet-4.5:input_cache_write': 375, - 'openrouter:deepseek/deepseek-v3.2-exp:prompt': 27, - 'openrouter:deepseek/deepseek-v3.2-exp:completion': 40, - 'openrouter:thedrummer/cydonia-24b-v4.1:prompt': 30, - 'openrouter:thedrummer/cydonia-24b-v4.1:completion': 50, - 'openrouter:relace/relace-apply-3:prompt': 85, - 'openrouter:relace/relace-apply-3:completion': 125, - 'openrouter:google/gemini-2.5-flash-preview-09-2025:prompt': 30, - 'openrouter:google/gemini-2.5-flash-preview-09-2025:completion': 250, - 'openrouter:google/gemini-2.5-flash-preview-09-2025:image': 123800, - 'openrouter:google/gemini-2.5-flash-preview-09-2025:audio': 100, - 'openrouter:google/gemini-2.5-flash-preview-09-2025:input_cache_read': 7, - 'openrouter:google/gemini-2.5-flash-preview-09-2025:input_cache_write': 38, - 'openrouter:google/gemini-2.5-flash-lite-preview-09-2025:prompt': 10, - 'openrouter:google/gemini-2.5-flash-lite-preview-09-2025:completion': 40, - 'openrouter:qwen/qwen3-vl-235b-a22b-thinking:prompt': 30, - 'openrouter:qwen/qwen3-vl-235b-a22b-thinking:completion': 120, - 'openrouter:qwen/qwen3-vl-235b-a22b-instruct:prompt': 21, - 'openrouter:qwen/qwen3-vl-235b-a22b-instruct:completion': 190, - 'openrouter:qwen/qwen3-max:prompt': 120, - 'openrouter:qwen/qwen3-max:completion': 600, - 'openrouter:qwen/qwen3-max:input_cache_read': 24, - 'openrouter:qwen/qwen3-coder-plus:prompt': 100, - 'openrouter:qwen/qwen3-coder-plus:completion': 500, - 'openrouter:qwen/qwen3-coder-plus:input_cache_read': 10, - 'openrouter:openai/gpt-5-codex:prompt': 125, - 'openrouter:openai/gpt-5-codex:completion': 1000, - 'openrouter:openai/gpt-5-codex:input_cache_read': 12, - 'openrouter:deepseek/deepseek-v3.1-terminus:prompt': 23, - 'openrouter:deepseek/deepseek-v3.1-terminus:completion': 90, - 'openrouter:deepseek/deepseek-v3.1-terminus:exacto:prompt': 27, - 'openrouter:deepseek/deepseek-v3.1-terminus:exacto:completion': 100, - 'openrouter:x-ai/grok-4-fast:prompt': 20, - 'openrouter:x-ai/grok-4-fast:completion': 50, - 'openrouter:x-ai/grok-4-fast:input_cache_read': 5, - 'openrouter:alibaba/tongyi-deepresearch-30b-a3b:prompt': 9, - 'openrouter:alibaba/tongyi-deepresearch-30b-a3b:completion': 40, - 'openrouter:qwen/qwen3-coder-flash:prompt': 30, - 'openrouter:qwen/qwen3-coder-flash:completion': 150, - 'openrouter:qwen/qwen3-coder-flash:input_cache_read': 8, - 'openrouter:arcee-ai/afm-4.5b:prompt': 5, - 'openrouter:arcee-ai/afm-4.5b:completion': 15, - 'openrouter:opengvlab/internvl3-78b:prompt': 7, - 'openrouter:opengvlab/internvl3-78b:completion': 26, - 'openrouter:qwen/qwen3-next-80b-a3b-thinking:prompt': 15, - 'openrouter:qwen/qwen3-next-80b-a3b-thinking:completion': 120, - 'openrouter:qwen/qwen3-next-80b-a3b-instruct:prompt': 10, - 'openrouter:qwen/qwen3-next-80b-a3b-instruct:completion': 80, - 'openrouter:meituan/longcat-flash-chat:prompt': 15, - 'openrouter:meituan/longcat-flash-chat:completion': 75, - 'openrouter:qwen/qwen-plus-2025-07-28:prompt': 40, - 'openrouter:qwen/qwen-plus-2025-07-28:completion': 120, - 'openrouter:qwen/qwen-plus-2025-07-28:thinking:prompt': 40, - 'openrouter:qwen/qwen-plus-2025-07-28:thinking:completion': 400, - 'openrouter:nvidia/nemotron-nano-9b-v2:prompt': 4, - 'openrouter:nvidia/nemotron-nano-9b-v2:completion': 16, - 'openrouter:moonshotai/kimi-k2-0905:prompt': 39, - 'openrouter:moonshotai/kimi-k2-0905:completion': 190, - 'openrouter:moonshotai/kimi-k2-0905:exacto:prompt': 60, - 'openrouter:moonshotai/kimi-k2-0905:exacto:completion': 250, - 'openrouter:deepcogito/cogito-v2-preview-llama-70b:prompt': 88, - 'openrouter:deepcogito/cogito-v2-preview-llama-70b:completion': 88, - 'openrouter:deepcogito/cogito-v2-preview-llama-109b-moe:prompt': 18, - 'openrouter:deepcogito/cogito-v2-preview-llama-109b-moe:completion': 59, - 'openrouter:deepcogito/cogito-v2-preview-deepseek-671b:prompt': 125, - 'openrouter:deepcogito/cogito-v2-preview-deepseek-671b:completion': 125, - 'openrouter:stepfun-ai/step3:prompt': 57, - 'openrouter:stepfun-ai/step3:completion': 142, - 'openrouter:qwen/qwen3-30b-a3b-thinking-2507:prompt': 5, - 'openrouter:qwen/qwen3-30b-a3b-thinking-2507:completion': 34, - 'openrouter:x-ai/grok-code-fast-1:prompt': 20, - 'openrouter:x-ai/grok-code-fast-1:completion': 150, - 'openrouter:x-ai/grok-code-fast-1:input_cache_read': 2, - 'openrouter:nousresearch/hermes-4-70b:prompt': 11, - 'openrouter:nousresearch/hermes-4-70b:completion': 38, - 'openrouter:nousresearch/hermes-4-405b:prompt': 30, - 'openrouter:nousresearch/hermes-4-405b:completion': 120, - 'openrouter:google/gemini-2.5-flash-image-preview:prompt': 30, - 'openrouter:google/gemini-2.5-flash-image-preview:completion': 250, - 'openrouter:google/gemini-2.5-flash-image-preview:image': 123800, - 'openrouter:deepseek/deepseek-chat-v3.1:prompt': 20, - 'openrouter:deepseek/deepseek-chat-v3.1:completion': 80, - 'openrouter:openai/gpt-4o-audio-preview:prompt': 250, - 'openrouter:openai/gpt-4o-audio-preview:completion': 1000, - 'openrouter:openai/gpt-4o-audio-preview:audio': 4000, - 'openrouter:mistralai/mistral-medium-3.1:prompt': 40, - 'openrouter:mistralai/mistral-medium-3.1:completion': 200, - 'openrouter:baidu/ernie-4.5-21b-a3b:prompt': 7, - 'openrouter:baidu/ernie-4.5-21b-a3b:completion': 28, - 'openrouter:baidu/ernie-4.5-vl-28b-a3b:prompt': 14, - 'openrouter:baidu/ernie-4.5-vl-28b-a3b:completion': 56, - 'openrouter:z-ai/glm-4.5v:prompt': 60, - 'openrouter:z-ai/glm-4.5v:completion': 180, - 'openrouter:z-ai/glm-4.5v:input_cache_read': 11, - 'openrouter:ai21/jamba-mini-1.7:prompt': 20, - 'openrouter:ai21/jamba-mini-1.7:completion': 40, - 'openrouter:ai21/jamba-large-1.7:prompt': 200, - 'openrouter:ai21/jamba-large-1.7:completion': 800, - 'openrouter:openai/gpt-5-chat:prompt': 125, - 'openrouter:openai/gpt-5-chat:completion': 1000, - 'openrouter:openai/gpt-5-chat:web_search': 1000000, - 'openrouter:openai/gpt-5-chat:input_cache_read': 12, - 'openrouter:openai/gpt-5:prompt': 125, - 'openrouter:openai/gpt-5:completion': 1000, - 'openrouter:openai/gpt-5:web_search': 1000000, - 'openrouter:openai/gpt-5:input_cache_read': 12, - 'openrouter:openai/gpt-5-mini:prompt': 25, - 'openrouter:openai/gpt-5-mini:completion': 200, - 'openrouter:openai/gpt-5-mini:web_search': 1000000, - 'openrouter:openai/gpt-5-mini:input_cache_read': 3, - 'openrouter:openai/gpt-5-nano:prompt': 5, - 'openrouter:openai/gpt-5-nano:completion': 40, - 'openrouter:openai/gpt-5-nano:web_search': 1000000, - 'openrouter:openai/gpt-5-nano:input_cache_read': 1, - 'openrouter:openai/gpt-oss-120b:prompt': 4, - 'openrouter:openai/gpt-oss-120b:completion': 40, - 'openrouter:openai/gpt-oss-120b:exacto:prompt': 5, - 'openrouter:openai/gpt-oss-120b:exacto:completion': 24, - 'openrouter:openai/gpt-oss-20b:prompt': 3, - 'openrouter:openai/gpt-oss-20b:completion': 14, - 'openrouter:anthropic/claude-opus-4.1:prompt': 1500, - 'openrouter:anthropic/claude-opus-4.1:completion': 7500, - 'openrouter:anthropic/claude-opus-4.1:image': 2400000, - 'openrouter:anthropic/claude-opus-4.1:input_cache_read': 150, - 'openrouter:anthropic/claude-opus-4.1:input_cache_write': 1875, - 'openrouter:mistralai/codestral-2508:prompt': 30, - 'openrouter:mistralai/codestral-2508:completion': 90, - 'openrouter:qwen/qwen3-coder-30b-a3b-instruct:prompt': 6, - 'openrouter:qwen/qwen3-coder-30b-a3b-instruct:completion': 25, - 'openrouter:qwen/qwen3-30b-a3b-instruct-2507:prompt': 8, - 'openrouter:qwen/qwen3-30b-a3b-instruct-2507:completion': 33, - 'openrouter:z-ai/glm-4.5:prompt': 35, - 'openrouter:z-ai/glm-4.5:completion': 150, - 'openrouter:z-ai/glm-4.5-air:prompt': 13, - 'openrouter:z-ai/glm-4.5-air:completion': 85, - 'openrouter:qwen/qwen3-235b-a22b-thinking-2507:prompt': 11, - 'openrouter:qwen/qwen3-235b-a22b-thinking-2507:completion': 60, - 'openrouter:z-ai/glm-4-32b:prompt': 10, - 'openrouter:z-ai/glm-4-32b:completion': 10, - 'openrouter:qwen/qwen3-coder:prompt': 22, - 'openrouter:qwen/qwen3-coder:completion': 95, - 'openrouter:qwen/qwen3-coder:exacto:prompt': 38, - 'openrouter:qwen/qwen3-coder:exacto:completion': 153, - 'openrouter:bytedance/ui-tars-1.5-7b:prompt': 10, - 'openrouter:bytedance/ui-tars-1.5-7b:completion': 20, - 'openrouter:google/gemini-2.5-flash-lite:prompt': 10, - 'openrouter:google/gemini-2.5-flash-lite:completion': 40, - 'openrouter:google/gemini-2.5-flash-lite:input_cache_read': 1, - 'openrouter:google/gemini-2.5-flash-lite:input_cache_write': 18, - 'openrouter:qwen/qwen3-235b-a22b-2507:prompt': 8, - 'openrouter:qwen/qwen3-235b-a22b-2507:completion': 55, - 'openrouter:switchpoint/router:prompt': 85, - 'openrouter:switchpoint/router:completion': 340, - 'openrouter:moonshotai/kimi-k2:prompt': 50, - 'openrouter:moonshotai/kimi-k2:completion': 240, - 'openrouter:thudm/glm-4.1v-9b-thinking:prompt': 4, - 'openrouter:thudm/glm-4.1v-9b-thinking:completion': 14, - 'openrouter:mistralai/devstral-medium:prompt': 40, - 'openrouter:mistralai/devstral-medium:completion': 200, - 'openrouter:mistralai/devstral-small:prompt': 7, - 'openrouter:mistralai/devstral-small:completion': 28, - 'openrouter:x-ai/grok-4:prompt': 300, - 'openrouter:x-ai/grok-4:completion': 1500, - 'openrouter:x-ai/grok-4:input_cache_read': 75, - 'openrouter:tencent/hunyuan-a13b-instruct:prompt': 14, - 'openrouter:tencent/hunyuan-a13b-instruct:completion': 57, - 'openrouter:tngtech/deepseek-r1t2-chimera:prompt': 30, - 'openrouter:tngtech/deepseek-r1t2-chimera:completion': 120, - 'openrouter:morph/morph-v3-large:prompt': 90, - 'openrouter:morph/morph-v3-large:completion': 190, - 'openrouter:morph/morph-v3-fast:prompt': 80, - 'openrouter:morph/morph-v3-fast:completion': 120, - 'openrouter:baidu/ernie-4.5-vl-424b-a47b:prompt': 42, - 'openrouter:baidu/ernie-4.5-vl-424b-a47b:completion': 125, - 'openrouter:baidu/ernie-4.5-300b-a47b:prompt': 28, - 'openrouter:baidu/ernie-4.5-300b-a47b:completion': 110, - 'openrouter:thedrummer/anubis-70b-v1.1:prompt': 65, - 'openrouter:thedrummer/anubis-70b-v1.1:completion': 100, - 'openrouter:inception/mercury:prompt': 25, - 'openrouter:inception/mercury:completion': 100, - 'openrouter:mistralai/mistral-small-3.2-24b-instruct:prompt': 6, - 'openrouter:mistralai/mistral-small-3.2-24b-instruct:completion': 18, - 'openrouter:minimax/minimax-m1:prompt': 40, - 'openrouter:minimax/minimax-m1:completion': 220, - 'openrouter:google/gemini-2.5-flash:prompt': 30, - 'openrouter:google/gemini-2.5-flash:completion': 250, - 'openrouter:google/gemini-2.5-flash:image': 123800, - 'openrouter:google/gemini-2.5-flash:input_cache_read': 3, - 'openrouter:google/gemini-2.5-flash:input_cache_write': 38, - 'openrouter:google/gemini-2.5-pro:prompt': 125, - 'openrouter:google/gemini-2.5-pro:completion': 1000, - 'openrouter:google/gemini-2.5-pro:image': 516000, - 'openrouter:google/gemini-2.5-pro:input_cache_read': 12, - 'openrouter:google/gemini-2.5-pro:input_cache_write': 163, - 'openrouter:moonshotai/kimi-dev-72b:prompt': 29, - 'openrouter:moonshotai/kimi-dev-72b:completion': 115, - 'openrouter:openai/o3-pro:prompt': 2000, - 'openrouter:openai/o3-pro:completion': 8000, - 'openrouter:openai/o3-pro:image': 1530000, - 'openrouter:openai/o3-pro:web_search': 1000000, - 'openrouter:x-ai/grok-3-mini:prompt': 30, - 'openrouter:x-ai/grok-3-mini:completion': 50, - 'openrouter:x-ai/grok-3-mini:input_cache_read': 7, - 'openrouter:x-ai/grok-3:prompt': 300, - 'openrouter:x-ai/grok-3:completion': 1500, - 'openrouter:x-ai/grok-3:input_cache_read': 75, - 'openrouter:mistralai/magistral-small-2506:prompt': 50, - 'openrouter:mistralai/magistral-small-2506:completion': 150, - 'openrouter:mistralai/magistral-medium-2506:thinking:prompt': 200, - 'openrouter:mistralai/magistral-medium-2506:thinking:completion': 500, - 'openrouter:mistralai/magistral-medium-2506:prompt': 200, - 'openrouter:mistralai/magistral-medium-2506:completion': 500, - 'openrouter:google/gemini-2.5-pro-preview:prompt': 125, - 'openrouter:google/gemini-2.5-pro-preview:completion': 1000, - 'openrouter:google/gemini-2.5-pro-preview:image': 516000, - 'openrouter:google/gemini-2.5-pro-preview:input_cache_read': 31, - 'openrouter:google/gemini-2.5-pro-preview:input_cache_write': 163, - 'openrouter:deepseek/deepseek-r1-0528-qwen3-8b:prompt': 2, - 'openrouter:deepseek/deepseek-r1-0528-qwen3-8b:completion': 10, - 'openrouter:deepseek/deepseek-r1-0528:prompt': 20, - 'openrouter:deepseek/deepseek-r1-0528:completion': 450, - 'openrouter:anthropic/claude-opus-4:prompt': 1500, - 'openrouter:anthropic/claude-opus-4:completion': 7500, - 'openrouter:anthropic/claude-opus-4:image': 2400000, - 'openrouter:anthropic/claude-opus-4:input_cache_read': 150, - 'openrouter:anthropic/claude-opus-4:input_cache_write': 1875, - 'openrouter:anthropic/claude-sonnet-4:prompt': 300, - 'openrouter:anthropic/claude-sonnet-4:completion': 1500, - 'openrouter:anthropic/claude-sonnet-4:image': 480000, - 'openrouter:anthropic/claude-sonnet-4:input_cache_read': 30, - 'openrouter:anthropic/claude-sonnet-4:input_cache_write': 375, - 'openrouter:mistralai/devstral-small-2505:prompt': 6, - 'openrouter:mistralai/devstral-small-2505:completion': 12, - 'openrouter:google/gemma-3n-e4b-it:prompt': 2, - 'openrouter:google/gemma-3n-e4b-it:completion': 4, - 'openrouter:openai/codex-mini:prompt': 150, - 'openrouter:openai/codex-mini:completion': 600, - 'openrouter:openai/codex-mini:input_cache_read': 38, - 'openrouter:nousresearch/deephermes-3-mistral-24b-preview:prompt': 15, - 'openrouter:nousresearch/deephermes-3-mistral-24b-preview:completion': 59, - 'openrouter:mistralai/mistral-medium-3:prompt': 40, - 'openrouter:mistralai/mistral-medium-3:completion': 200, - 'openrouter:google/gemini-2.5-pro-preview-05-06:prompt': 125, - 'openrouter:google/gemini-2.5-pro-preview-05-06:completion': 1000, - 'openrouter:google/gemini-2.5-pro-preview-05-06:image': 516000, - 'openrouter:google/gemini-2.5-pro-preview-05-06:input_cache_read': 31, - 'openrouter:google/gemini-2.5-pro-preview-05-06:input_cache_write': 163, - 'openrouter:arcee-ai/spotlight:prompt': 18, - 'openrouter:arcee-ai/spotlight:completion': 18, - 'openrouter:arcee-ai/maestro-reasoning:prompt': 90, - 'openrouter:arcee-ai/maestro-reasoning:completion': 330, - 'openrouter:arcee-ai/virtuoso-large:prompt': 75, - 'openrouter:arcee-ai/virtuoso-large:completion': 120, - 'openrouter:arcee-ai/coder-large:prompt': 50, - 'openrouter:arcee-ai/coder-large:completion': 80, - 'openrouter:microsoft/phi-4-reasoning-plus:prompt': 7, - 'openrouter:microsoft/phi-4-reasoning-plus:completion': 35, - 'openrouter:inception/mercury-coder:prompt': 25, - 'openrouter:inception/mercury-coder:completion': 100, - 'openrouter:deepseek/deepseek-prover-v2:prompt': 50, - 'openrouter:deepseek/deepseek-prover-v2:completion': 218, - 'openrouter:meta-llama/llama-guard-4-12b:prompt': 18, - 'openrouter:meta-llama/llama-guard-4-12b:completion': 18, - 'openrouter:qwen/qwen3-30b-a3b:prompt': 6, - 'openrouter:qwen/qwen3-30b-a3b:completion': 22, - 'openrouter:qwen/qwen3-8b:prompt': 4, - 'openrouter:qwen/qwen3-8b:completion': 14, - 'openrouter:qwen/qwen3-14b:prompt': 5, - 'openrouter:qwen/qwen3-14b:completion': 22, - 'openrouter:qwen/qwen3-32b:prompt': 5, - 'openrouter:qwen/qwen3-32b:completion': 20, - 'openrouter:qwen/qwen3-235b-a22b:prompt': 18, - 'openrouter:qwen/qwen3-235b-a22b:completion': 54, - 'openrouter:tngtech/deepseek-r1t-chimera:prompt': 30, - 'openrouter:tngtech/deepseek-r1t-chimera:completion': 120, - 'openrouter:microsoft/mai-ds-r1:prompt': 30, - 'openrouter:microsoft/mai-ds-r1:completion': 120, - 'openrouter:openai/o4-mini-high:prompt': 110, - 'openrouter:openai/o4-mini-high:completion': 440, - 'openrouter:openai/o4-mini-high:image': 84150, - 'openrouter:openai/o4-mini-high:web_search': 1000000, - 'openrouter:openai/o4-mini-high:input_cache_read': 28, - 'openrouter:openai/o3:prompt': 200, - 'openrouter:openai/o3:completion': 800, - 'openrouter:openai/o3:image': 153000, - 'openrouter:openai/o3:web_search': 1000000, - 'openrouter:openai/o3:input_cache_read': 50, - 'openrouter:openai/o4-mini:prompt': 110, - 'openrouter:openai/o4-mini:completion': 440, - 'openrouter:openai/o4-mini:image': 84150, - 'openrouter:openai/o4-mini:web_search': 1000000, - 'openrouter:openai/o4-mini:input_cache_read': 28, - 'openrouter:qwen/qwen2.5-coder-7b-instruct:prompt': 3, - 'openrouter:qwen/qwen2.5-coder-7b-instruct:completion': 9, - 'openrouter:openai/gpt-4.1:prompt': 200, - 'openrouter:openai/gpt-4.1:completion': 800, - 'openrouter:openai/gpt-4.1:web_search': 1000000, - 'openrouter:openai/gpt-4.1:input_cache_read': 50, - 'openrouter:openai/gpt-4.1-mini:prompt': 40, - 'openrouter:openai/gpt-4.1-mini:completion': 160, - 'openrouter:openai/gpt-4.1-mini:web_search': 1000000, - 'openrouter:openai/gpt-4.1-mini:input_cache_read': 10, - 'openrouter:openai/gpt-4.1-nano:prompt': 10, - 'openrouter:openai/gpt-4.1-nano:completion': 40, - 'openrouter:openai/gpt-4.1-nano:web_search': 1000000, - 'openrouter:openai/gpt-4.1-nano:input_cache_read': 3, - 'openrouter:eleutherai/llemma_7b:prompt': 80, - 'openrouter:eleutherai/llemma_7b:completion': 120, - 'openrouter:alfredpros/codellama-7b-instruct-solidity:prompt': 80, - 'openrouter:alfredpros/codellama-7b-instruct-solidity:completion': 120, - 'openrouter:arliai/qwq-32b-arliai-rpr-v1:prompt': 3, - 'openrouter:arliai/qwq-32b-arliai-rpr-v1:completion': 11, - 'openrouter:x-ai/grok-3-mini-beta:prompt': 30, - 'openrouter:x-ai/grok-3-mini-beta:completion': 50, - 'openrouter:x-ai/grok-3-mini-beta:input_cache_read': 7, - 'openrouter:x-ai/grok-3-beta:prompt': 300, - 'openrouter:x-ai/grok-3-beta:completion': 1500, - 'openrouter:x-ai/grok-3-beta:input_cache_read': 75, - 'openrouter:nvidia/llama-3.1-nemotron-ultra-253b-v1:prompt': 60, - 'openrouter:nvidia/llama-3.1-nemotron-ultra-253b-v1:completion': 180, - 'openrouter:meta-llama/llama-4-maverick:prompt': 15, - 'openrouter:meta-llama/llama-4-maverick:completion': 60, - 'openrouter:meta-llama/llama-4-maverick:image': 66840, - 'openrouter:meta-llama/llama-4-scout:prompt': 8, - 'openrouter:meta-llama/llama-4-scout:completion': 30, - 'openrouter:meta-llama/llama-4-scout:image': 33420, - 'openrouter:qwen/qwen2.5-vl-32b-instruct:prompt': 5, - 'openrouter:qwen/qwen2.5-vl-32b-instruct:completion': 22, - 'openrouter:deepseek/deepseek-chat-v3-0324:prompt': 24, - 'openrouter:deepseek/deepseek-chat-v3-0324:completion': 84, - 'openrouter:openai/o1-pro:prompt': 15000, - 'openrouter:openai/o1-pro:completion': 60000, - 'openrouter:openai/o1-pro:image': 21675000, - 'openrouter:mistralai/mistral-small-3.1-24b-instruct:prompt': 5, - 'openrouter:mistralai/mistral-small-3.1-24b-instruct:completion': 22, - 'openrouter:allenai/olmo-2-0325-32b-instruct:prompt': 20, - 'openrouter:allenai/olmo-2-0325-32b-instruct:completion': 35, - 'openrouter:google/gemma-3-4b-it:prompt': 2, - 'openrouter:google/gemma-3-4b-it:completion': 7, - 'openrouter:google/gemma-3-12b-it:prompt': 3, - 'openrouter:google/gemma-3-12b-it:completion': 10, - 'openrouter:cohere/command-a:prompt': 250, - 'openrouter:cohere/command-a:completion': 1000, - 'openrouter:openai/gpt-4o-mini-search-preview:prompt': 15, - 'openrouter:openai/gpt-4o-mini-search-preview:completion': 60, - 'openrouter:openai/gpt-4o-mini-search-preview:request': 2750000, - 'openrouter:openai/gpt-4o-mini-search-preview:image': 21700, - 'openrouter:openai/gpt-4o-search-preview:prompt': 250, - 'openrouter:openai/gpt-4o-search-preview:completion': 1000, - 'openrouter:openai/gpt-4o-search-preview:request': 3500000, - 'openrouter:openai/gpt-4o-search-preview:image': 361300, - 'openrouter:google/gemma-3-27b-it:prompt': 7, - 'openrouter:google/gemma-3-27b-it:completion': 50, - 'openrouter:thedrummer/skyfall-36b-v2:prompt': 50, - 'openrouter:thedrummer/skyfall-36b-v2:completion': 80, - 'openrouter:microsoft/phi-4-multimodal-instruct:prompt': 5, - 'openrouter:microsoft/phi-4-multimodal-instruct:completion': 10, - 'openrouter:microsoft/phi-4-multimodal-instruct:image': 17685, - 'openrouter:perplexity/sonar-reasoning-pro:prompt': 200, - 'openrouter:perplexity/sonar-reasoning-pro:completion': 800, - 'openrouter:perplexity/sonar-reasoning-pro:web_search': 500000, - 'openrouter:perplexity/sonar-pro:prompt': 300, - 'openrouter:perplexity/sonar-pro:completion': 1500, - 'openrouter:perplexity/sonar-pro:web_search': 500000, - 'openrouter:perplexity/sonar-deep-research:prompt': 200, - 'openrouter:perplexity/sonar-deep-research:completion': 800, - 'openrouter:perplexity/sonar-deep-research:web_search': 500000, - 'openrouter:perplexity/sonar-deep-research:internal_reasoning': 300, - 'openrouter:qwen/qwq-32b:prompt': 15, - 'openrouter:qwen/qwq-32b:completion': 40, - 'openrouter:google/gemini-2.0-flash-lite-001:prompt': 7, - 'openrouter:google/gemini-2.0-flash-lite-001:completion': 30, - 'openrouter:anthropic/claude-3.7-sonnet:thinking:prompt': 300, - 'openrouter:anthropic/claude-3.7-sonnet:thinking:completion': 1500, - 'openrouter:anthropic/claude-3.7-sonnet:thinking:image': 480000, - 'openrouter:anthropic/claude-3.7-sonnet:thinking:input_cache_read': 30, - 'openrouter:anthropic/claude-3.7-sonnet:thinking:input_cache_write': 375, - 'openrouter:anthropic/claude-3.7-sonnet:prompt': 300, - 'openrouter:anthropic/claude-3.7-sonnet:completion': 1500, - 'openrouter:anthropic/claude-3.7-sonnet:image': 480000, - 'openrouter:anthropic/claude-3.7-sonnet:input_cache_read': 30, - 'openrouter:anthropic/claude-3.7-sonnet:input_cache_write': 375, - 'openrouter:mistralai/mistral-saba:prompt': 20, - 'openrouter:mistralai/mistral-saba:completion': 60, - 'openrouter:meta-llama/llama-guard-3-8b:prompt': 2, - 'openrouter:meta-llama/llama-guard-3-8b:completion': 6, - 'openrouter:openai/o3-mini-high:prompt': 110, - 'openrouter:openai/o3-mini-high:completion': 440, - 'openrouter:openai/o3-mini-high:input_cache_read': 55, - 'openrouter:google/gemini-2.0-flash-001:prompt': 10, - 'openrouter:google/gemini-2.0-flash-001:completion': 40, - 'openrouter:google/gemini-2.0-flash-001:image': 2580, - 'openrouter:google/gemini-2.0-flash-001:audio': 70, - 'openrouter:google/gemini-2.0-flash-001:input_cache_read': 3, - 'openrouter:google/gemini-2.0-flash-001:input_cache_write': 18, - 'openrouter:qwen/qwen-vl-plus:prompt': 21, - 'openrouter:qwen/qwen-vl-plus:completion': 63, - 'openrouter:qwen/qwen-vl-plus:image': 26880, - 'openrouter:aion-labs/aion-1.0:prompt': 400, - 'openrouter:aion-labs/aion-1.0:completion': 800, - 'openrouter:aion-labs/aion-1.0-mini:prompt': 70, - 'openrouter:aion-labs/aion-1.0-mini:completion': 140, - 'openrouter:aion-labs/aion-rp-llama-3.1-8b:prompt': 20, - 'openrouter:aion-labs/aion-rp-llama-3.1-8b:completion': 20, - 'openrouter:qwen/qwen-vl-max:prompt': 80, - 'openrouter:qwen/qwen-vl-max:completion': 320, - 'openrouter:qwen/qwen-vl-max:image': 102400, - 'openrouter:qwen/qwen-turbo:prompt': 5, - 'openrouter:qwen/qwen-turbo:completion': 20, - 'openrouter:qwen/qwen-turbo:input_cache_read': 2, - 'openrouter:qwen/qwen2.5-vl-72b-instruct:prompt': 8, - 'openrouter:qwen/qwen2.5-vl-72b-instruct:completion': 33, - 'openrouter:qwen/qwen-plus:prompt': 40, - 'openrouter:qwen/qwen-plus:completion': 120, - 'openrouter:qwen/qwen-plus:input_cache_read': 16, - 'openrouter:qwen/qwen-max:prompt': 160, - 'openrouter:qwen/qwen-max:completion': 640, - 'openrouter:qwen/qwen-max:input_cache_read': 64, - 'openrouter:openai/o3-mini:prompt': 110, - 'openrouter:openai/o3-mini:completion': 440, - 'openrouter:openai/o3-mini:input_cache_read': 55, - 'openrouter:mistralai/mistral-small-24b-instruct-2501:prompt': 5, - 'openrouter:mistralai/mistral-small-24b-instruct-2501:completion': 8, - 'openrouter:deepseek/deepseek-r1-distill-qwen-32b:prompt': 27, - 'openrouter:deepseek/deepseek-r1-distill-qwen-32b:completion': 27, - 'openrouter:deepseek/deepseek-r1-distill-qwen-14b:prompt': 15, - 'openrouter:deepseek/deepseek-r1-distill-qwen-14b:completion': 15, - 'openrouter:perplexity/sonar-reasoning:prompt': 100, - 'openrouter:perplexity/sonar-reasoning:completion': 500, - 'openrouter:perplexity/sonar-reasoning:request': 500000, - 'openrouter:perplexity/sonar:prompt': 100, - 'openrouter:perplexity/sonar:completion': 100, - 'openrouter:perplexity/sonar:request': 500000, - 'openrouter:deepseek/deepseek-r1-distill-llama-70b:prompt': 3, - 'openrouter:deepseek/deepseek-r1-distill-llama-70b:completion': 13, - 'openrouter:deepseek/deepseek-r1:prompt': 30, - 'openrouter:deepseek/deepseek-r1:completion': 120, - 'openrouter:minimax/minimax-01:prompt': 20, - 'openrouter:minimax/minimax-01:completion': 110, - 'openrouter:mistralai/codestral-2501:prompt': 30, - 'openrouter:mistralai/codestral-2501:completion': 90, - 'openrouter:microsoft/phi-4:prompt': 6, - 'openrouter:microsoft/phi-4:completion': 14, - 'openrouter:sao10k/l3.1-70b-hanami-x1:prompt': 300, - 'openrouter:sao10k/l3.1-70b-hanami-x1:completion': 300, - 'openrouter:deepseek/deepseek-chat:prompt': 30, - 'openrouter:deepseek/deepseek-chat:completion': 120, - 'openrouter:sao10k/l3.3-euryale-70b:prompt': 65, - 'openrouter:sao10k/l3.3-euryale-70b:completion': 75, - 'openrouter:openai/o1:prompt': 1500, - 'openrouter:openai/o1:completion': 6000, - 'openrouter:openai/o1:image': 2167500, - 'openrouter:openai/o1:input_cache_read': 750, - 'openrouter:cohere/command-r7b-12-2024:prompt': 4, - 'openrouter:cohere/command-r7b-12-2024:completion': 15, - 'openrouter:meta-llama/llama-3.3-70b-instruct:prompt': 13, - 'openrouter:meta-llama/llama-3.3-70b-instruct:completion': 38, - 'openrouter:amazon/nova-lite-v1:prompt': 6, - 'openrouter:amazon/nova-lite-v1:completion': 24, - 'openrouter:amazon/nova-lite-v1:image': 9000, - 'openrouter:amazon/nova-micro-v1:prompt': 4, - 'openrouter:amazon/nova-micro-v1:completion': 14, - 'openrouter:amazon/nova-pro-v1:prompt': 80, - 'openrouter:amazon/nova-pro-v1:completion': 320, - 'openrouter:amazon/nova-pro-v1:image': 120000, - 'openrouter:openai/gpt-4o-2024-11-20:prompt': 250, - 'openrouter:openai/gpt-4o-2024-11-20:completion': 1000, - 'openrouter:openai/gpt-4o-2024-11-20:image': 361300, - 'openrouter:openai/gpt-4o-2024-11-20:input_cache_read': 125, - 'openrouter:mistralai/mistral-large-2411:prompt': 200, - 'openrouter:mistralai/mistral-large-2411:completion': 600, - 'openrouter:mistralai/mistral-large-2407:prompt': 200, - 'openrouter:mistralai/mistral-large-2407:completion': 600, - 'openrouter:mistralai/pixtral-large-2411:prompt': 200, - 'openrouter:mistralai/pixtral-large-2411:completion': 600, - 'openrouter:mistralai/pixtral-large-2411:image': 288800, - 'openrouter:qwen/qwen-2.5-coder-32b-instruct:prompt': 4, - 'openrouter:qwen/qwen-2.5-coder-32b-instruct:completion': 16, - 'openrouter:raifle/sorcererlm-8x22b:prompt': 450, - 'openrouter:raifle/sorcererlm-8x22b:completion': 450, - 'openrouter:thedrummer/unslopnemo-12b:prompt': 40, - 'openrouter:thedrummer/unslopnemo-12b:completion': 40, - 'openrouter:anthropic/claude-3.5-haiku-20241022:prompt': 80, - 'openrouter:anthropic/claude-3.5-haiku-20241022:completion': 400, - 'openrouter:anthropic/claude-3.5-haiku-20241022:input_cache_read': 8, - 'openrouter:anthropic/claude-3.5-haiku-20241022:input_cache_write': 100, - 'openrouter:anthropic/claude-3.5-haiku:prompt': 80, - 'openrouter:anthropic/claude-3.5-haiku:completion': 400, - 'openrouter:anthropic/claude-3.5-haiku:web_search': 1000000, - 'openrouter:anthropic/claude-3.5-haiku:input_cache_read': 8, - 'openrouter:anthropic/claude-3.5-haiku:input_cache_write': 100, - 'openrouter:anthracite-org/magnum-v4-72b:prompt': 300, - 'openrouter:anthracite-org/magnum-v4-72b:completion': 500, - 'openrouter:anthropic/claude-3.5-sonnet:prompt': 300, - 'openrouter:anthropic/claude-3.5-sonnet:completion': 1500, - 'openrouter:anthropic/claude-3.5-sonnet:image': 480000, - 'openrouter:anthropic/claude-3.5-sonnet:input_cache_read': 30, - 'openrouter:anthropic/claude-3.5-sonnet:input_cache_write': 375, - 'openrouter:mistralai/ministral-8b:prompt': 10, - 'openrouter:mistralai/ministral-8b:completion': 10, - 'openrouter:mistralai/ministral-3b:prompt': 4, - 'openrouter:mistralai/ministral-3b:completion': 4, - 'openrouter:qwen/qwen-2.5-7b-instruct:prompt': 4, - 'openrouter:qwen/qwen-2.5-7b-instruct:completion': 10, - 'openrouter:nvidia/llama-3.1-nemotron-70b-instruct:prompt': 120, - 'openrouter:nvidia/llama-3.1-nemotron-70b-instruct:completion': 120, - 'openrouter:inflection/inflection-3-pi:prompt': 250, - 'openrouter:inflection/inflection-3-pi:completion': 1000, - 'openrouter:inflection/inflection-3-productivity:prompt': 250, - 'openrouter:inflection/inflection-3-productivity:completion': 1000, - 'openrouter:thedrummer/rocinante-12b:prompt': 17, - 'openrouter:thedrummer/rocinante-12b:completion': 43, - 'openrouter:meta-llama/llama-3.2-3b-instruct:prompt': 2, - 'openrouter:meta-llama/llama-3.2-3b-instruct:completion': 2, - 'openrouter:meta-llama/llama-3.2-1b-instruct:prompt': 3, - 'openrouter:meta-llama/llama-3.2-1b-instruct:completion': 20, - 'openrouter:meta-llama/llama-3.2-90b-vision-instruct:prompt': 35, - 'openrouter:meta-llama/llama-3.2-90b-vision-instruct:completion': 40, - 'openrouter:meta-llama/llama-3.2-90b-vision-instruct:image': 50580, - 'openrouter:meta-llama/llama-3.2-11b-vision-instruct:prompt': 5, - 'openrouter:meta-llama/llama-3.2-11b-vision-instruct:completion': 5, - 'openrouter:meta-llama/llama-3.2-11b-vision-instruct:image': 7948, - 'openrouter:qwen/qwen-2.5-72b-instruct:prompt': 7, - 'openrouter:qwen/qwen-2.5-72b-instruct:completion': 26, - 'openrouter:neversleep/llama-3.1-lumimaid-8b:prompt': 9, - 'openrouter:neversleep/llama-3.1-lumimaid-8b:completion': 60, - 'openrouter:mistralai/pixtral-12b:prompt': 10, - 'openrouter:mistralai/pixtral-12b:completion': 10, - 'openrouter:mistralai/pixtral-12b:image': 14450, - 'openrouter:cohere/command-r-08-2024:prompt': 15, - 'openrouter:cohere/command-r-08-2024:completion': 60, - 'openrouter:cohere/command-r-plus-08-2024:prompt': 250, - 'openrouter:cohere/command-r-plus-08-2024:completion': 1000, - 'openrouter:sao10k/l3.1-euryale-70b:prompt': 65, - 'openrouter:sao10k/l3.1-euryale-70b:completion': 75, - 'openrouter:qwen/qwen-2.5-vl-7b-instruct:prompt': 20, - 'openrouter:qwen/qwen-2.5-vl-7b-instruct:completion': 20, - 'openrouter:qwen/qwen-2.5-vl-7b-instruct:image': 14450, - 'openrouter:microsoft/phi-3.5-mini-128k-instruct:prompt': 10, - 'openrouter:microsoft/phi-3.5-mini-128k-instruct:completion': 10, - 'openrouter:nousresearch/hermes-3-llama-3.1-70b:prompt': 30, - 'openrouter:nousresearch/hermes-3-llama-3.1-70b:completion': 30, - 'openrouter:nousresearch/hermes-3-llama-3.1-405b:prompt': 100, - 'openrouter:nousresearch/hermes-3-llama-3.1-405b:completion': 100, - 'openrouter:openai/chatgpt-4o-latest:prompt': 500, - 'openrouter:openai/chatgpt-4o-latest:completion': 1500, - 'openrouter:openai/chatgpt-4o-latest:image': 722500, - 'openrouter:sao10k/l3-lunaris-8b:prompt': 4, - 'openrouter:sao10k/l3-lunaris-8b:completion': 5, - 'openrouter:openai/gpt-4o-2024-08-06:prompt': 250, - 'openrouter:openai/gpt-4o-2024-08-06:completion': 1000, - 'openrouter:openai/gpt-4o-2024-08-06:image': 361300, - 'openrouter:openai/gpt-4o-2024-08-06:input_cache_read': 125, - 'openrouter:meta-llama/llama-3.1-405b:prompt': 400, - 'openrouter:meta-llama/llama-3.1-405b:completion': 400, - 'openrouter:meta-llama/llama-3.1-8b-instruct:prompt': 2, - 'openrouter:meta-llama/llama-3.1-8b-instruct:completion': 3, - 'openrouter:meta-llama/llama-3.1-405b-instruct:prompt': 350, - 'openrouter:meta-llama/llama-3.1-405b-instruct:completion': 350, - 'openrouter:meta-llama/llama-3.1-70b-instruct:prompt': 40, - 'openrouter:meta-llama/llama-3.1-70b-instruct:completion': 40, - 'openrouter:mistralai/mistral-nemo:prompt': 2, - 'openrouter:mistralai/mistral-nemo:completion': 4, - 'openrouter:openai/gpt-4o-mini-2024-07-18:prompt': 15, - 'openrouter:openai/gpt-4o-mini-2024-07-18:completion': 60, - 'openrouter:openai/gpt-4o-mini-2024-07-18:image': 722500, - 'openrouter:openai/gpt-4o-mini-2024-07-18:input_cache_read': 7, - 'openrouter:openai/gpt-4o-mini:prompt': 15, - 'openrouter:openai/gpt-4o-mini:completion': 60, - 'openrouter:openai/gpt-4o-mini:image': 21700, - 'openrouter:openai/gpt-4o-mini:input_cache_read': 7, - 'openrouter:google/gemma-2-27b-it:prompt': 65, - 'openrouter:google/gemma-2-27b-it:completion': 65, - 'openrouter:google/gemma-2-9b-it:prompt': 3, - 'openrouter:google/gemma-2-9b-it:completion': 9, - 'openrouter:sao10k/l3-euryale-70b:prompt': 148, - 'openrouter:sao10k/l3-euryale-70b:completion': 148, - 'openrouter:nousresearch/hermes-2-pro-llama-3-8b:prompt': 3, - 'openrouter:nousresearch/hermes-2-pro-llama-3-8b:completion': 8, - 'openrouter:mistralai/mistral-7b-instruct:prompt': 3, - 'openrouter:mistralai/mistral-7b-instruct:completion': 5, - 'openrouter:mistralai/mistral-7b-instruct-v0.3:prompt': 20, - 'openrouter:mistralai/mistral-7b-instruct-v0.3:completion': 20, - 'openrouter:microsoft/phi-3-mini-128k-instruct:prompt': 10, - 'openrouter:microsoft/phi-3-mini-128k-instruct:completion': 10, - 'openrouter:microsoft/phi-3-medium-128k-instruct:prompt': 100, - 'openrouter:microsoft/phi-3-medium-128k-instruct:completion': 100, - 'openrouter:meta-llama/llama-guard-2-8b:prompt': 20, - 'openrouter:meta-llama/llama-guard-2-8b:completion': 20, - 'openrouter:openai/gpt-4o-2024-05-13:prompt': 500, - 'openrouter:openai/gpt-4o-2024-05-13:completion': 1500, - 'openrouter:openai/gpt-4o-2024-05-13:image': 722500, - 'openrouter:openai/gpt-4o:prompt': 250, - 'openrouter:openai/gpt-4o:completion': 1000, - 'openrouter:openai/gpt-4o:image': 361300, - 'openrouter:openai/gpt-4o:input_cache_read': 125, - 'openrouter:openai/gpt-4o:extended:prompt': 600, - 'openrouter:openai/gpt-4o:extended:completion': 1800, - 'openrouter:openai/gpt-4o:extended:image': 722500, - 'openrouter:meta-llama/llama-3-70b-instruct:prompt': 30, - 'openrouter:meta-llama/llama-3-70b-instruct:completion': 40, - 'openrouter:meta-llama/llama-3-8b-instruct:prompt': 3, - 'openrouter:meta-llama/llama-3-8b-instruct:completion': 6, - 'openrouter:mistralai/mixtral-8x22b-instruct:prompt': 200, - 'openrouter:mistralai/mixtral-8x22b-instruct:completion': 600, - 'openrouter:microsoft/wizardlm-2-8x22b:prompt': 48, - 'openrouter:microsoft/wizardlm-2-8x22b:completion': 48, - 'openrouter:openai/gpt-4-turbo:prompt': 1000, - 'openrouter:openai/gpt-4-turbo:completion': 3000, - 'openrouter:openai/gpt-4-turbo:image': 1445000, - 'openrouter:anthropic/claude-3-haiku:prompt': 25, - 'openrouter:anthropic/claude-3-haiku:completion': 125, - 'openrouter:anthropic/claude-3-haiku:image': 40000, - 'openrouter:anthropic/claude-3-haiku:input_cache_read': 3, - 'openrouter:anthropic/claude-3-haiku:input_cache_write': 30, - 'openrouter:anthropic/claude-3-opus:prompt': 1500, - 'openrouter:anthropic/claude-3-opus:completion': 7500, - 'openrouter:anthropic/claude-3-opus:image': 2400000, - 'openrouter:anthropic/claude-3-opus:input_cache_read': 150, - 'openrouter:anthropic/claude-3-opus:input_cache_write': 1875, - 'openrouter:mistralai/mistral-large:prompt': 200, - 'openrouter:mistralai/mistral-large:completion': 600, - 'openrouter:openai/gpt-3.5-turbo-0613:prompt': 100, - 'openrouter:openai/gpt-3.5-turbo-0613:completion': 200, - 'openrouter:openai/gpt-4-turbo-preview:prompt': 1000, - 'openrouter:openai/gpt-4-turbo-preview:completion': 3000, - 'openrouter:mistralai/mistral-small:prompt': 20, - 'openrouter:mistralai/mistral-small:completion': 60, - 'openrouter:mistralai/mistral-tiny:prompt': 25, - 'openrouter:mistralai/mistral-tiny:completion': 25, - 'openrouter:mistralai/mistral-7b-instruct-v0.2:prompt': 20, - 'openrouter:mistralai/mistral-7b-instruct-v0.2:completion': 20, - 'openrouter:mistralai/mixtral-8x7b-instruct:prompt': 54, - 'openrouter:mistralai/mixtral-8x7b-instruct:completion': 54, - 'openrouter:neversleep/noromaid-20b:prompt': 100, - 'openrouter:neversleep/noromaid-20b:completion': 175, - 'openrouter:alpindale/goliath-120b:prompt': 600, - 'openrouter:alpindale/goliath-120b:completion': 800, - 'openrouter:openrouter/auto:prompt': -100000000, - 'openrouter:openrouter/auto:completion': -100000000, - 'openrouter:openai/gpt-4-1106-preview:prompt': 1000, - 'openrouter:openai/gpt-4-1106-preview:completion': 3000, - 'openrouter:openai/gpt-3.5-turbo-instruct:prompt': 150, - 'openrouter:openai/gpt-3.5-turbo-instruct:completion': 200, - 'openrouter:mistralai/mistral-7b-instruct-v0.1:prompt': 11, - 'openrouter:mistralai/mistral-7b-instruct-v0.1:completion': 19, - 'openrouter:openai/gpt-3.5-turbo-16k:prompt': 300, - 'openrouter:openai/gpt-3.5-turbo-16k:completion': 400, - 'openrouter:mancer/weaver:prompt': 113, - 'openrouter:mancer/weaver:completion': 113, - 'openrouter:undi95/remm-slerp-l2-13b:prompt': 45, - 'openrouter:undi95/remm-slerp-l2-13b:completion': 65, - 'openrouter:gryphe/mythomax-l2-13b:prompt': 6, - 'openrouter:gryphe/mythomax-l2-13b:completion': 6, - 'openrouter:openai/gpt-4-0314:prompt': 3000, - 'openrouter:openai/gpt-4-0314:completion': 6000, - 'openrouter:openai/gpt-4:prompt': 3000, - 'openrouter:openai/gpt-4:completion': 6000, - 'openrouter:openai/gpt-3.5-turbo:prompt': 50, - 'openrouter:openai/gpt-3.5-turbo:completion': 150, -}; diff --git a/src/backend/src/services/MeteringService/costMaps/togetherCostMap.ts b/src/backend/src/services/MeteringService/costMaps/togetherCostMap.ts deleted file mode 100644 index c6884c57b..000000000 --- a/src/backend/src/services/MeteringService/costMaps/togetherCostMap.ts +++ /dev/null @@ -1,67 +0,0 @@ -// TogetherAI Cost Map - -export const TOGETHER_COST_MAP = { - // Test model (hardcoded) - 'together:model-fallback-test-1:input': 10, - 'together:model-fallback-test-1:output': 10, - - // Image generation placeholder (actual pricing is fetched dynamically via Together API) - 'together-image:default': 0, - 'together-image:ByteDance-Seed/Seedream-3.0': 0.018 * 100_000_000, - 'together-image:ByteDance-Seed/Seedream-4.0': 0.03 * 100_000_000, - 'together-image:HiDream-ai/HiDream-I1-Dev': 0.0045 * 100_000_000, - 'together-image:HiDream-ai/HiDream-I1-Fast': 0.0032 * 100_000_000, - 'together-image:HiDream-ai/HiDream-I1-Full': 0.009 * 100_000_000, - 'together-image:Lykon/DreamShaper': 0.0006 * 100_000_000, - 'together-image:Qwen/Qwen-Image': 0.0058 * 100_000_000, - 'together-image:RunDiffusion/Juggernaut-pro-flux': 0.0049 * 100_000_000, - 'together-image:Rundiffusion/Juggernaut-Lightning-Flux': 0.0017 * 100_000_000, - 'together-image:black-forest-labs/FLUX.1-kontext-max': 0.08 * 100_000_000, - 'together-image:black-forest-labs/FLUX.1-kontext-pro': 0.04 * 100_000_000, - 'together-image:black-forest-labs/FLUX.1-krea-dev': 0.025 * 100_000_000, - 'together-image:black-forest-labs/FLUX.1-schnell': 0.0027 * 100_000_000, - 'together-image:black-forest-labs/FLUX.1.1-pro': 0.04 * 100_000_000, - 'together-image:black-forest-labs/FLUX.2-pro': 0.03 * 100_000_000, - 'together-image:black-forest-labs/FLUX.2-flex': 0.03 * 100_000_000, - 'together-image:black-forest-labs/FLUX.2-dev': 0.0154 * 100_000_000, - 'together-image:black-forest-labs/FLUX.2-max': 0.07 * 100_000_000, - 'together-image:google/flash-image-2.5': 0.039 * 100_000_000, - 'together-image:google/flash-image-3.1': 0.067 * 100_000_000, - 'together-image:google/gemini-3-pro-image': 0.134 * 100_000_000, - 'together-image:google/imagen-4.0-fast': 0.02 * 100_000_000, - 'together-image:google/imagen-4.0-preview': 0.04 * 100_000_000, - 'together-image:google/imagen-4.0-ultra': 0.06 * 100_000_000, - 'together-image:ideogram/ideogram-3.0': 0.06 * 100_000_000, - 'together-image:openai/gpt-image-1.5': 0.034 * 100_000_000, - 'together-image:Qwen/Qwen-Image-2.0': 0.04 * 100_000_000, - 'together-image:Qwen/Qwen-Image-2.0-Pro': 0.08 * 100_000_000, - 'together-image:Wan-AI/Wan2.6-image': 0.03 * 100_000_000, - 'together-image:stabilityai/stable-diffusion-3-medium': 0.0019 * 100_000_000, - 'together-image:stabilityai/stable-diffusion-xl-base-1.0': 0.0019 * 100_000_000, - - // Video generation placeholder (per-video pricing). Update with real pricing when available. - 'together-video:default': 0, - 'together-video:ByteDance/Seedance-1.0-lite': 0.14 * 100_000_000, - 'together-video:ByteDance/Seedance-1.0-pro': 0.57 * 100_000_000, - 'together-video:Wan-AI/Wan2.2-I2V-A14B': 0.31 * 100_000_000, - 'together-video:Wan-AI/Wan2.2-T2V-A14B': 0.66 * 100_000_000, - 'together-video:Wan-AI/wan2.7-t2v': 0.10 * 100_000_000, - 'together-video:google/veo-2.0': 2.50 * 100_000_000, - 'together-video:google/veo-3.0': 1.60 * 100_000_000, - 'together-video:google/veo-3.0-audio': 3.20 * 100_000_000, - 'together-video:google/veo-3.0-fast': 0.80 * 100_000_000, - 'together-video:google/veo-3.0-fast-audio': 1.20 * 100_000_000, - 'together-video:kwaivgI/kling-1.6-pro': 0.32 * 100_000_000, - 'together-video:kwaivgI/kling-1.6-standard': 0.19 * 100_000_000, - 'together-video:kwaivgI/kling-2.0-master': 0.92 * 100_000_000, - 'together-video:kwaivgI/kling-2.1-master': 0.92 * 100_000_000, - 'together-video:kwaivgI/kling-2.1-pro': 0.32 * 100_000_000, - 'together-video:kwaivgI/kling-2.1-standard': 0.18 * 100_000_000, - 'together-video:minimax/hailuo-02': 0.49 * 100_000_000, - 'together-video:minimax/video-01-director': 0.28 * 100_000_000, - 'together-video:openai/sora-2': 0.80 * 100_000_000, - 'together-video:openai/sora-2-pro': 3.00 * 100_000_000, - 'together-video:pixverse/pixverse-v5': 0.30 * 100_000_000, - 'together-video:vidu/vidu-2.0': 0.28 * 100_000_000, - 'together-video:vidu/vidu-q1': 0.22 * 100_000_000, -}; diff --git a/src/backend/src/services/MeteringService/costMaps/xaiCostMap.ts b/src/backend/src/services/MeteringService/costMaps/xaiCostMap.ts deleted file mode 100644 index ee431c26b..000000000 --- a/src/backend/src/services/MeteringService/costMaps/xaiCostMap.ts +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -export const XAI_COST_MAP = { - // Grok Beta - 'xai:grok-beta:prompt_tokens': 500, - 'xai:grok-beta:completion-tokens': 1500, - - // Grok Vision Beta - 'xai:grok-vision-beta:prompt_tokens': 500, - 'xai:grok-vision-beta:completion-tokens': 1500, - 'xai:grok-vision-beta:image': 1000, - - // Grok 3 - 'xai:grok-3:prompt_tokens': 300, - 'xai:grok-3:completion-tokens': 1500, - - // Grok 3 Fast - 'xai:grok-3-fast:prompt_tokens': 500, - 'xai:grok-3-fast:completion-tokens': 2500, - - // Grok 3 Mini - 'xai:grok-3-mini:prompt_tokens': 30, - 'xai:grok-3-mini:completion-tokens': 50, - - // Grok 3 Mini Fast - 'xai:grok-3-mini-fast:prompt_tokens': 60, - 'xai:grok-3-mini-fast:completion-tokens': 400, - - // Grok 2 Vision - 'xai:grok-2-vision:prompt_tokens': 200, - 'xai:grok-2-vision:completion-tokens': 1000, - - // Grok 2 - 'xai:grok-2:prompt_tokens': 200, - 'xai:grok-2:completion-tokens': 1000, - - // Grok Image - 'xai:grok-2-image:output': 7_000_000, -}; diff --git a/src/backend/src/services/MeteringService/types.ts b/src/backend/src/services/MeteringService/types.ts deleted file mode 100644 index 4f46da20b..000000000 --- a/src/backend/src/services/MeteringService/types.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { AlarmService } from '../../modules/core/AlarmService'; -import type { DynamoKVStore } from '../DynamoKVStore/DynamoKVStore'; -import type { EventService } from '../EventService'; -import type { SUService } from '../SUService'; - -export interface UsageAddons { - purchasedCredits: number // total extra credits purchased - not expirable - consumedPurchaseCredits: number // total credits consumed from purchased ones - these are flattened upon new 'purchase' - purchasedStorage: number // TODO DS: not implemented yet - rateDiscounts: { - [usageType: string]: number | string // TODO DS: string to support graduated discounts eventually - } -} - -export interface RecursiveRecord { [k: string]: T | RecursiveRecord } - -export interface UsageRecord { - cost: number, - count: number, - units: number -} - -export type UsageByType = { total: number } & Partial, UsageRecord>>; - -export interface AppTotals { - total: number, - count: number -} -export interface MeteringServiceDeps { - kvStore: DynamoKVStore, - superUserService: SUService, - alarmService: AlarmService - eventService: EventService -} diff --git a/src/backend/src/services/MeteringService/utils.ts b/src/backend/src/services/MeteringService/utils.ts deleted file mode 100644 index 3a8dc208d..000000000 --- a/src/backend/src/services/MeteringService/utils.ts +++ /dev/null @@ -1 +0,0 @@ -export const toMicroCents = (dollars: number) => dollars * 1_000_000 * 100; diff --git a/src/backend/src/services/NotificationService.js b/src/backend/src/services/NotificationService.js deleted file mode 100644 index bccd273d2..000000000 --- a/src/backend/src/services/NotificationService.js +++ /dev/null @@ -1,310 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../api/APIError'); -const eggspress = require('../api/eggspress'); -const auth2 = require('../middleware/auth2'); -const { TeePromise } = require('@heyputer/putility').libs.promise; -const BaseService = require('./BaseService'); -const { DB_WRITE } = require('./database/consts'); - -const UsernameNotifSelector = username => async (self) => { - const svc_getUser = self.services.get('get-user'); - const user = await svc_getUser.get_user({ username }); - return [user.id]; -}; - -const UserIDNotifSelector = user_id => async (self) => { - return [user_id]; -}; - -/** -* @class NotificationService -* @extends BaseService -* -* The NotificationService class is responsible for managing notifications within the application. -* It handles creating, storing, and sending notifications to users, as well as updating the status of notifications -* (e.g., marking them as read or acknowledged). -* -* @property {Object} MODULES - Static object containing modules used by the service, such as uuidv4 and express. -* @property {Object} merged_on_user_connected_ - Object to track connected users and manage delayed actions. -* @property {Object} notifs_pending_write - Object to track pending write operations for notifications. -* -* @method _construct - Initializes the service's internal state. -* @method _init - Initializes the service, setting up database connections and event listeners. -* @method __on_install.routes - Registers API routes for notification-related endpoints. -* @method on_user_connected - Handles actions when a user connects to the application. -* @method do_on_user_connected - Queries and updates unread notifications for a connected user. -* @method on_sent_to_user - Updates the status of a notification when it is sent to a user. -* @method notify - Sends a notification to a list of users and persists it in the database. -* -* @example -* const notificationService = new NotificationService(); -* notificationService.notify(UsernameNotifSelector('user123'), { -* source: 'notification-testing', -* icon_source: 'builtin', -* icon: 'logo.svg', -* title: 'Test Notification', -* text: 'This is a test notification.' -* }); -*/ -class NotificationService extends BaseService { - static MODULES = { - uuidv4: require('uuid').v4, - express: require('express'), - }; - - /** - * Constructs the NotificationService instance. - * This method sets up the initial state of the service, including any necessary - * data structures or configurations. - * - * @private - */ - _construct () { - this.merged_on_user_connected_ = {}; - } - - /** - * Initializes the NotificationService by setting up necessary services, - * registering event listeners, and preparing the database connection. - * This method is called once during the service's lifecycle. - * @returns {Promise} A promise that resolves when initialization is complete. - */ - async _init () { - const svc_database = this.services.get('database'); - this.db = svc_database.get(DB_WRITE, 'notification'); - - const svc_script = this.services.get('script'); - svc_script.register('test-notification', async ({ log }, [username, summary]) => { - log(`creating notification: ${ summary}`); - - this.notify(UsernameNotifSelector(username), { - source: 'notification-testing', - icon_source: 'builtin', - icon: 'logo.svg', - title: summary, - text: summary, - }); - }); - - const svc_event = this.services.get('event'); - svc_event.on('web.socket.user-connected', (_, { user }) => { - this.on_user_connected({ user }); - }); - svc_event.on('sent-to-user.notif.message', (_, o) => { - this.on_sent_to_user(o); - }); - - this.notifs_pending_write = {}; - } - - '__on_install.routes' (_, { app }) { - const require = this.require; - const express = require('express'); - const router = express.Router(); - app.use('/notif', router); - - router.use(auth2); - - const svc_event = this.services.get('event'); - - [['ack', 'acknowledged'], ['read', 'read']].forEach(([ep_name, col_name]) => { - router.use(eggspress(`/mark-${ ep_name}`, { - allowedMethods: ['POST'], - }, async (req, res) => { - // TODO: validate uid - if ( typeof req.body.uid !== 'string' ) { - throw APIError.create('field_invalid', null, { - key: 'uid', - expected: 'a valid UUID', - got: 'non-string value', - }); - } - - const ack_ts = Math.floor(Date.now() / 1000); - await this.db.write( - `UPDATE \`notification\` SET ${ col_name } = ? ` + - 'WHERE uid = ? AND user_id = ? ' + - 'LIMIT 1', - [ack_ts, req.body.uid, req.user.id], - ); - - svc_event.emit('outer.gui.notif.ack', { - user_id_list: [req.user.id], - response: { - uid: req.body.uid, - }, - }); - - res.json({}); - })); - }); - } - - /** - * Handles the event when a user connects. - * - * This method checks if there is a timeout set for the user's connection event and clears it if it exists. - * If not, it sets a timeout to call `do_on_user_connected` after 2000 milliseconds. - * - * @param {object} params - The parameters object containing user data. - * @param {object} params.user - The user object with a `uuid` property. - * - * @returns {void} - */ - async on_user_connected ({ user }) { - if ( this.merged_on_user_connected_[user.uuid] ) { - clearTimeout(this.merged_on_user_connected_[user.uuid]); - } - this.merged_on_user_connected_[user.uuid] = - /** - * Schedules the `do_on_user_connected` method to be called after a delay. - * - * This method sets a timer to call `do_on_user_connected` after 2000 milliseconds. - * If a timer already exists for the user, it clears the existing timer before setting a new one. - */ - setTimeout(() => this.do_on_user_connected({ user }), 2000); - } - /** - * Handles the event when a user connects. - * Sets a timeout to delay the execution of the `do_on_user_connected` method by 2 seconds. - * This helps in merging multiple events that occur in a short period. - * - * @param {Object} obj - The event object containing user information. - * @param {Object} obj.user - The user object with a `uuid` property. - * @async - */ - async do_on_user_connected ({ user }) { - // query the users unread notifications - const notifications = await this.db.read( - 'SELECT * FROM `notification` ' + - 'WHERE user_id=? AND shown IS NULL AND acknowledged IS NULL ' + - 'ORDER BY created_at ASC', - [user.id], - ); - - // set all the notifications to "shown" - const shown_ts = Math.floor(Date.now() / 1000); - await this.db.write( - 'UPDATE `notification` ' + - 'SET shown = ? ' + - 'WHERE user_id=? AND shown IS NULL AND acknowledged IS NULL ', - [shown_ts, user.id], - ); - - for ( const n of notifications ) { - if ( !n.value || typeof (n.value) === 'string' ) { - n.value = JSON.parse(n.value || '{}'); - } - } - - const client_safe_notifications = []; - for ( const notif of notifications ) { - client_safe_notifications.push({ - uid: notif.uid, - notification: notif.value, - }); - } - - // send the unread notifications to gui - const svc_event = this.services.get('event'); - svc_event.emit('outer.gui.notif.unreads', { - user_id_list: [user.id], - response: { - unreads: client_safe_notifications, - }, - }); - } - - /** - * Handles the action when a notification is sent to a user. - * - * This method is triggered when a notification is sent to a user, - * updating the notification's status to 'shown' in the database. - * It logs the user ID and response, updates the 'shown' timestamp, - * and ensures the notification is written to the database. - * - * @param {Object} params - The parameters containing the user ID and response. - * @param {number} params.user_id - The ID of the user receiving the notification. - * @param {Object} params.response - The response object containing the notification details. - * @param {string} params.response.uid - The unique identifier of the notification. - */ - async on_sent_to_user ({ user_id, response }) { - const shown_ts = Math.floor(Date.now() / 1000); - if ( this.notifs_pending_write[response.uid] ) { - await this.notifs_pending_write[response.uid]; - } - await this.db.write(...ll([ - 'UPDATE `notification` ' + - 'SET shown = ? ' + - 'WHERE user_id=? AND uid=?', - [shown_ts, user_id, response.uid], - ])); - } - - /** - * Sends a notification to specified users. - * - * This method sends a notification to a list of users determined by the provided selector. - * It generates a unique identifier for the notification, emits an event to notify the GUI, - * and inserts the notification into the database. - * - * @param {Function} selector - A function that takes the service instance and returns a list of user IDs. - * @param {Object} notification - The notification details to be sent. - */ - async notify (selector, notification) { - const uid = this.modules.uuidv4(); - const svc_event = this.services.get('event'); - const user_id_list = await selector(this); - this.notifs_pending_write[uid] = new TeePromise(); - svc_event.emit('outer.gui.notif.message', { - user_id_list, - response: { - uid, - notification, - }, - }); - - (async () => { - for ( const user_id of user_id_list ) { - await this.db.write( - 'INSERT INTO `notification` ' + - '(`user_id`, `uid`, `value`) ' + - 'VALUES (?, ?, ?)', - [user_id, uid, JSON.stringify(notification)], - ); - } - const p = this.notifs_pending_write[uid]; - delete this.notifs_pending_write[uid]; - p.resolve(); - svc_event.emit('outer.gui.notif.persisted', { - user_id_list, - response: { - uid, - }, - }); - })(); - } -} - -module.exports = { - NotificationService, - UsernameNotifSelector, - UserIDNotifSelector, -}; diff --git a/src/backend/src/services/NotificationService.test.ts b/src/backend/src/services/NotificationService.test.ts deleted file mode 100644 index d6c3d04b2..000000000 --- a/src/backend/src/services/NotificationService.test.ts +++ /dev/null @@ -1,164 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { createTestKernel } from '../../tools/test.mjs'; -import * as config from '../config'; -import { NotificationService, UserIDNotifSelector, UsernameNotifSelector } from './NotificationService'; -import { ScriptService } from './ScriptService'; - -describe('NotificationService', async () => { - config.load_config({ - 'services': { - 'database': { - path: ':memory:', - }, - }, - }); - - const testKernel = await createTestKernel({ - serviceMap: { - 'script': ScriptService, - 'notification': NotificationService, - }, - initLevelString: 'init', - testCore: true, - }); - - const notificationService = testKernel.services!.get('notification') as any; - - it('should be instantiated', () => { - expect(notificationService).toBeInstanceOf(NotificationService); - }); - - it('should have db connection after init', () => { - expect(notificationService.db).toBeDefined(); - }); - - it('should have notifs_pending_write object', () => { - expect(notificationService.notifs_pending_write).toBeDefined(); - expect(typeof notificationService.notifs_pending_write).toBe('object'); - }); - - it('should have merged_on_user_connected_ object', () => { - expect(notificationService.merged_on_user_connected_).toBeDefined(); - expect(typeof notificationService.merged_on_user_connected_).toBe('object'); - }); - - it('should have on_user_connected method', () => { - expect(notificationService.on_user_connected).toBeDefined(); - expect(typeof notificationService.on_user_connected).toBe('function'); - }); - - it('should have do_on_user_connected method', () => { - expect(notificationService.do_on_user_connected).toBeDefined(); - expect(typeof notificationService.do_on_user_connected).toBe('function'); - }); - - it('should have on_sent_to_user method', () => { - expect(notificationService.on_sent_to_user).toBeDefined(); - expect(typeof notificationService.on_sent_to_user).toBe('function'); - }); - - it('should have notify method', () => { - expect(notificationService.notify).toBeDefined(); - expect(typeof notificationService.notify).toBe('function'); - }); - - it('should schedule do_on_user_connected on user connected', async () => { - vi.useFakeTimers(); - - const user = { uuid: 'test-uuid-123', id: 1 }; - - await notificationService.on_user_connected({ user }); - - expect(notificationService.merged_on_user_connected_[user.uuid]).toBeDefined(); - - vi.useRealTimers(); - }); - - it('should clear previous timeout on repeated user connected', async () => { - vi.useFakeTimers(); - - const user = { uuid: 'test-uuid-456', id: 2 }; - - await notificationService.on_user_connected({ user }); - const firstTimeout = notificationService.merged_on_user_connected_[user.uuid]; - - await notificationService.on_user_connected({ user }); - const secondTimeout = notificationService.merged_on_user_connected_[user.uuid]; - - expect(firstTimeout).toBeDefined(); - expect(secondTimeout).toBeDefined(); - // The timeout should have been replaced - - vi.useRealTimers(); - }); - - it('should handle notify with user ID selector', async () => { - const userId = 123; - const selector = UserIDNotifSelector(userId); - - const result = await selector(notificationService); - - expect(result).toEqual([userId]); - }); -}); - -describe('UsernameNotifSelector', () => { - it('should create a selector function', () => { - const selector = UsernameNotifSelector('testuser'); - - expect(selector).toBeDefined(); - expect(typeof selector).toBe('function'); - }); - - it('should return function that fetches user by username', async () => { - const mockGetUserService = { - get_user: vi.fn().mockResolvedValue({ id: 42, username: 'testuser' }), - }; - - const mockService = { - services: { - get: vi.fn().mockReturnValue(mockGetUserService), - }, - }; - - const selector = UsernameNotifSelector('testuser'); - const result = await selector(mockService as any); - - expect(mockService.services.get).toHaveBeenCalledWith('get-user'); - expect(mockGetUserService.get_user).toHaveBeenCalledWith({ username: 'testuser' }); - expect(result).toEqual([42]); - }); -}); - -describe('UserIDNotifSelector', () => { - it('should create a selector function', () => { - const selector = UserIDNotifSelector(123); - - expect(selector).toBeDefined(); - expect(typeof selector).toBe('function'); - }); - - it('should return array with user ID', async () => { - const userId = 456; - const selector = UserIDNotifSelector(userId); - - const result = await selector(null as any); - - expect(result).toEqual([userId]); - }); - - it('should work with different user IDs', async () => { - const selector1 = UserIDNotifSelector(100); - const selector2 = UserIDNotifSelector(200); - const selector3 = UserIDNotifSelector(300); - - const result1 = await selector1(null as any); - const result2 = await selector2(null as any); - const result3 = await selector3(null as any); - - expect(result1).toEqual([100]); - expect(result2).toEqual([200]); - expect(result3).toEqual([300]); - }); -}); - diff --git a/src/backend/src/services/OperationTraceService.js b/src/backend/src/services/OperationTraceService.js deleted file mode 100644 index 54e1bfd00..000000000 --- a/src/backend/src/services/OperationTraceService.js +++ /dev/null @@ -1,404 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('../../../putility'); -const { Context } = require('../util/context'); -const { ContextAwareFeature } = require('../traits/ContextAwareFeature'); -const { OtelFeature } = require('../traits/OtelFeature'); -const APIError = require('../api/APIError'); -const { AssignableMethodsFeature } = require('../traits/AssignableMethodsFeature'); - -// CONTEXT_KEY is used to create a unique context key for operation tracing -// and is utilized throughout the OperationTraceService to manage frames. -const CONTEXT_KEY = Context.make_context_key('operation-trace'); - -/** -* @class OperationFrame -* @description The `OperationFrame` class represents a frame within an operation trace. It is designed to manage the state, attributes, and hierarchy of frames within an operational context. This class provides methods to set status, calculate effective status, add tags, attributes, messages, errors, children, and describe the frame. It also includes methods to recursively search through frames to find attributes and handle frame completion. -*/ -class OperationFrame { - static LOG_DEBUG = true; - constructor ({ parent, label, x }) { - this.parent = parent; - this.label = label; - this.tags = []; - this.attributes = {}; - this.messages = []; - this.error_ = null; - this.children = []; - this.status_ = this.constructor.FRAME_STATUS_PENDING; - this.effective_status_ = this.status_; - this.id = require('uuid').v4(); - - this.log = (x ?? Context).get('services').get('log-service').create( - `frame:${this.id}`, - { concern: 'filesystem' }, - ); - } - - static FRAME_STATUS_PENDING = { label: 'pending' }; - static FRAME_STATUS_WORKING = { label: 'working' }; - static FRAME_STATUS_STUCK = { label: 'stuck' }; - static FRAME_STATUS_READY = { label: 'ready' }; - static FRAME_STATUS_DONE = { label: 'done' }; - - set status (status) { - this.status_ = status; - this._calc_effective_status(); - - this.log.debug( - `FRAME STATUS ${status.label} ${ - status !== this.effective_status_ - ? `(effective: ${this.effective_status_.label}) ` - : ''}`, - { - tags: this.tags, - ...this.attributes, - }, - ); - - if ( this.parent ) { - this.parent._calc_effective_status(); - } - } - /** - * Sets the status of the frame and updates the effective status. - * This method logs the status change and updates the parent frame's effective status if necessary. - * - * @param {Object} status - The new status to set. - */ - _calc_effective_status () { - for ( const child of this.children ) { - if ( child.status === OperationFrame.FRAME_STATUS_STUCK ) { - this.effective_status_ = OperationFrame.FRAME_STATUS_STUCK; - return; - } - } - - if ( this.status_ === OperationFrame.FRAME_STATUS_DONE ) { - for ( const child of this.children ) { - if ( child.status !== OperationFrame.FRAME_STATUS_DONE ) { - this.effective_status_ = OperationFrame.FRAME_STATUS_READY; - return; - } - } - } - - this.effective_status_ = this.status_; - if ( this.parent ) { - this.parent._calc_effective_status(); - } - - // TODO: operation trace service should hook a listener instead - if ( this.effective_status_ === OperationFrame.FRAME_STATUS_DONE ) { - const svc_operationTrace = Context.get('services').get('operationTrace'); - delete svc_operationTrace.ongoing[this.id]; - } - } - - /** - * Gets the effective status of the operation frame. - * - * This method returns the effective status of the current operation frame, - * considering the statuses of its children. The effective status is the - * aggregated status of the frame and its children, reflecting the current - * progress or state of the operation. - * - * @return {Object} The effective status of the operation frame. - */ - get status () { - return this.effective_status_; - } - - tag (...tags) { - this.tags.push(...tags); - return this; - } - - attr (key, value) { - this.attributes[key] = value; - return this; - } - - // recursively go through frames to find the attribute - get_attr (key) { - if ( this.attributes[key] ) return this.attributes[key]; - if ( this.parent ) return this.parent.get_attr(key); - } - - log (message) { - this.messages.push(message); - return this; - } - - error (err) { - this.error_ = err; - return this; - } - - push_child (frame) { - this.children.push(frame); - return this; - } - - /** - * Recursively traverses the frame hierarchy to find the root frame. - * - * @returns {OperationFrame} The root frame of the current frame hierarchy. - */ - get_root_frame () { - let frame = this; - while ( frame.parent ) { - frame = frame.parent; - } - return frame; - } - - /** - * Marks the operation frame as done. - * This method sets the status of the operation frame to 'done' and updates - * the effective status accordingly. It triggers a recalculation of the - * effective status for parent frames if necessary. - */ - done () { - this.status = OperationFrame.FRAME_STATUS_DONE; - } - - describe (show_tree, highlight_frame) { - let s = `${this.label } (${this.children.length})`; - if ( this.tags.length ) { - s += ` ${ this.tags.join(' ')}`; - } - if ( this.attributes ) { - s += ` ${ JSON.stringify(this.attributes)}`; - } - - if ( this.children.length == 0 ) return s; - - // It's ASCII box drawing time! - const prefix_child = '├─'; - const prefix_last = '└─'; - const prefix_deep = '│ '; - const prefix_deep_end = ' '; - - /** - * Recursively builds a string representation of the frame and its children. - * - * @param {boolean} show_tree - If true, includes the tree structure of child frames. - * @param {OperationFrame} highlight_frame - The frame to highlight in the output. - * @returns {string} - A string representation of the frame and its children. - */ - const recurse = (frame, prefix) => { - const children = frame.children; - for ( let i = 0; i < children.length; i++ ) { - const child = children[i]; - const is_last = i == children.length - 1; - if ( child === highlight_frame ) s += '\x1B[36;1m'; - s += `\n${ prefix }${is_last ? prefix_last : prefix_child }${child.describe()}`; - if ( child === highlight_frame ) s += '\x1B[0m'; - recurse(child, prefix + (is_last ? prefix_deep_end : prefix_deep)); - } - }; - - if ( show_tree ) recurse(this, ''); - return s; - } -} - -/** -* @class OperationTraceService -* @classdesc The OperationTraceService class manages operation frames and their statuses. -* It provides methods to add frames, track their progress, and handle their completion. -* This service is essential for monitoring and logging the lifecycle of operations within the system. -*/ -class OperationTraceService { - static CONCERN = 'filesystem'; - - constructor ({ services }) { - this.log = services.get('log-service').create('operation-trace', { - concern: this.constructor.CONCERN, - }); - - // TODO: replace with kv.js set - this.ongoing = {}; - } - - /** - * Adds a new operation frame to the trace. - * - * This method creates a new frame with the given label and context, - * and adds it to the ongoing operations. If a context is provided, - * it logs the context description. The frame is then added to the - * parent frame if one exists, and the frame's description is logged. - * - * @param {string} label - The label for the new operation frame. - * @param {?Object} [x] - The context for the operation frame. - * @returns {OperationFrame} The new operation frame. - */ - async add_frame (label) { - return this.add_frame_sync(label); - } - - add_frame_sync (label, x) { - if ( x ) { - this.log.debug(`add_frame_sync() called with explicit context: ${ - x.describe()}`); - } - let parent = (x ?? Context).get(this.ckey('frame')); - const frame = new OperationFrame({ - parent: parent || null, - label, - x, - }); - parent && parent.push_child(frame); - this.log.debug(`FRAME START ${ frame.describe()}`); - if ( ! parent ) { - // NOTE: only uncomment in local testing for now; - // this will cause a memory leak until frame - // done-ness is accurate - this.ongoing[frame.id] = frame; - } - return frame; - } - - ckey (key) { - return `${CONTEXT_KEY }:${ key}`; - } -} - -/** -* @class BaseOperation -* @extends AdvancedBase -* @description The BaseOperation class extends AdvancedBase and serves as the foundation for -* operations within the system. It integrates various features such as context awareness, -* observability through OpenTelemetry (OtelFeature), and assignable methods. This class is -* designed to be extended by specific operation classes to provide a common structure and -* functionality for running and tracing operations. -*/ -class BaseOperation extends AdvancedBase { - static FEATURES = [ - new ContextAwareFeature(), - new OtelFeature(['run']), - new AssignableMethodsFeature(), - ]; - - /** - * Executes the operation with the provided values. - * - * This method initiates an operation frame within the context, sets the operation status to working, - * executes the `_run` method, and handles post-run logic. It also manages the status of child frames - * and handles errors, updating the frame's attributes accordingly. - * - * @param {Object} firstArg - The values to be used in the operation. TODO DS: support multiple args with old state assignment? - * @param {...unknown} rest - rest of args passed in only to children - * @returns {Promise<*>} - The result of the operation. - * @throws {Error} - If the frame is missing or any other error occurs during the operation. - */ - async run (firstArg, ...rest) { - this.values = firstArg; - - firstArg.user = firstArg.user ?? - (firstArg.actor ? firstArg.actor.type.user : undefined); - - // getting context with a new operation frame - let x, frame; - x = Context.get(); - const operationTraceSvc = x.get('services').get('operationTrace'); - frame = await operationTraceSvc.add_frame(this.constructor.name); - x = x.sub({ [operationTraceSvc.ckey('frame')]: frame }); - - // the frame will be an explicit property as well as being in context - // (for convenience) - this.frame = frame; - - // let's make the logger for it too - this.log = x.get('services').get('log-service').create(this.constructor.name, { - operation: frame.id, - ...(this.constructor.CONCERN ? { - concern: this.constructor.CONCERN, - } : {}), - }); - - // Run operation in new context - try { - // Actual delegate call (this._run) with context and checkpoints - return await x.arun(async () => { - const x = Context.get(); - const operationTraceSvc = x.get('services').get('operationTrace'); - const frame = x.get(operationTraceSvc.ckey('frame')); - if ( ! frame ) { - throw new Error('missing frame'); - } - frame.status = OperationFrame.FRAME_STATUS_WORKING; - this.checkpoint('._run()'); - const res = await this._run(firstArg, ...rest); // TODO DS: simplify this, why are the passed in values being stored in class state? - this.checkpoint('._post_run()'); - const { any_async } = this._post_run(); - this.checkpoint('delegate .run_() returned'); - frame.status = any_async - ? OperationFrame.FRAME_STATUS_READY - : OperationFrame.FRAME_STATUS_DONE; - return res; - }); - } catch (e) { - if ( e instanceof APIError ) { - frame.attr('api-error', e.toString()); - } else { - frame.error(e); - } - throw e; - } - } - - checkpoint (name) { - this.frame.checkpoint = name; - } - - field (key, value) { - this.frame.attributes[key] = value; - } - - /** - * Actions to perform after running. - * - * If child operation frames think they're still pending, mark them as stuck; - * all child frames at least reach working state before the parent operation - * completes. - */ - _post_run () { - let any_async = false; - for ( const child of this.frame.children ) { - if ( child.status === OperationFrame.FRAME_STATUS_PENDING ) { - child.status = OperationFrame.FRAME_STATUS_STUCK; - } - - if ( child.status === OperationFrame.FRAME_STATUS_WORKING ) { - child.async = true; - any_async = true; - } - } - return { any_async }; - } -} - -module.exports = { - CONTEXT_KEY, - OperationTraceService, - BaseOperation, - OperationFrame, -}; diff --git a/src/backend/src/services/PeerService.js b/src/backend/src/services/PeerService.js deleted file mode 100644 index 6bec122b1..000000000 --- a/src/backend/src/services/PeerService.js +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import configurable_auth from '../middleware/configurable_auth.js'; -import eggspress from '../api/eggspress.js'; -import { Actor, UserActorType } from './auth/Actor.js'; -import BaseService from './BaseService.js'; - -function addDashesToUUID (i) { - return `${i.substr(0, 8) }-${ i.substr(8, 4) }-${ i.substr(12, 4) }-${ i.substr(16, 4) }-${ i.substr(20)}`; -} - -export class PeerService extends BaseService { - '__on_install.routes' (_, { app }) { - app.use(eggspress('/peer/signaller-info', { - allowedMethods: ['GET'], - subdomain: 'api', - }, async (req, res) => { - res.json({ - url: this.config.signaller_url, - fallbackIce: this.config.fallback_ice, - }); - })); - - app.use(eggspress('/peer/generate-turn', { - allowedMethods: ['POST'], - mw: [configurable_auth()], - subdomain: 'api', - }, async (req, res) => { - if ( ! this.config.cloudflare_turn ) { - res.status(500).send({ error: 'TURN is not configured' }); - return; - } - - // Build the custom identifier (short max length, we must compress it from hex to b64) - let customIdentifier = ''; - customIdentifier += Buffer.from(req.actor.type.user.uuid.replaceAll('-', ''), 'hex').toString('base64url'); - if ( req.actor.type?.app ) { - customIdentifier += `:${ Buffer.from(req.actor.type.app.uid.replace('app-', '').replaceAll('-', ''), 'hex').toString('base64url')}`; - } - let response = await fetch( - `https://rtc.live.cloudflare.com/v1/turn/keys/${this.config.cloudflare_turn.turn_key_id}/credentials/generate-ice-servers`, - { - headers: { - Authorization: `Bearer ${this.config.cloudflare_turn.turn_key_api_token}`, - 'Content-Type': 'application/json', - }, - method: 'POST', - body: JSON.stringify({ - ttl: this.config.cloudflare_turn.ttl_ms, - customIdentifier, - }), - }, - ); - - if ( ! response.ok ) { - res.status(500).send({ error: 'Failed to generate TURN credentials' }); - return; - } - - const { iceServers } = await response.json(); - - res.json({ - ttl: this.config.cloudflare_turn.ttl_ms, - iceServers, - }); - })); - - const svc_web = this.services.get('web-server'); - const meteringService = this.services.get('meteringService').meteringService; - svc_web.allow_undefined_origin('/turn/ingest-usage'); - - app.use(eggspress('/turn/ingest-usage', { - allowedMethods: ['POST'], - subdomain: 'api', - }, async (req, res) => { - if ( req.headers['x-puter-internal-auth'] !== this.config.turn_meter_secret ) { - res.status(403).send({ error: 'Failed to meter TURN credentials' }); - return; - } - /** @type {{timestamp: string, userId: string, origin: string, customIdentifier: Number, egressBytes: number, ingressBytes: number}[]} */ - const records = req.body.records; - for ( const record of records ) { - try { - const actor = await Actor.create(UserActorType, { - user_uid: addDashesToUUID(Buffer.from(record.userId, 'base64url').toString('hex')), - }); - const costInMicrocents = record.egressBytes * 0.005; - meteringService.incrementUsage(actor, 'turn:egress-bytes', record.egressBytes, costInMicrocents); - } catch (e) { - // failed to get user likely - console.error('TURN metering error: ', e); - } - res.send('ok'); - } - })); - } -} diff --git a/src/backend/src/services/PermissionAPIService.js b/src/backend/src/services/PermissionAPIService.js deleted file mode 100644 index 9150fbccb..000000000 --- a/src/backend/src/services/PermissionAPIService.js +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../api/APIError'); -const eggspress = require('../api/eggspress'); -const configurable_auth = require('../middleware/configurable_auth'); -const BaseService = require('./BaseService'); - -/** -* @class PermissionAPIService -* @extends BaseService -* @description Service class that handles API endpoints for permission management, including user-app permissions, -* user-user permissions, and group management. Provides functionality for creating groups, managing group memberships, -* granting/revoking various types of permissions, and checking access control lists (ACLs). Implements RESTful -* endpoints for group operations like creation, adding/removing users, and listing groups. -*/ -class PermissionAPIService extends BaseService { - static MODULES = { - express: require('express'), - }; - - /** - * Installs routes for authentication and permission management into the Express app - * @param {Object} _ Unused parameter - * @param {Object} options Installation options - * @param {Express} options.app Express application instance to install routes on - * @returns {Promise} - */ - async '__on_install.routes' (_, { app }) { - app.use(require('../routers/auth/get-user-app-token')); - app.use(require('../routers/auth/grant-user-app')); - app.use(require('../routers/auth/revoke-user-app')); - app.use(require('../routers/auth/grant-dev-app')); - app.use(require('../routers/auth/revoke-dev-app')); - app.use(require('../routers/auth/grant-user-user')); - app.use(require('../routers/auth/revoke-user-user')); - app.use(require('../routers/auth/grant-user-group')); - app.use(require('../routers/auth/revoke-user-group')); - app.use(require('../routers/auth/list-permissions').default); - app.use(require('../routers/auth/check-permissions.js')); - app.use(require('../routers/auth/request-app-root-dir')); - - const checkAppAclSpec = require('../routers/auth/check-app-acl.endpoint.js'); - app.use(eggspress('/auth/check-app-acl', { - allowedMethods: checkAppAclSpec.methods ?? ['GET'], - ...(checkAppAclSpec.subdomain ? { subdomain: checkAppAclSpec.subdomain } : {}), - ...(checkAppAclSpec.parameters ? { parameters: checkAppAclSpec.parameters } : {}), - ...(checkAppAclSpec.alias ? { alias: checkAppAclSpec.alias } : {}), - ...(checkAppAclSpec.mw ? { mw: checkAppAclSpec.mw } : {}), - ...checkAppAclSpec.otherOpts, - }, checkAppAclSpec.handler)); - - // track: scoping iife - /** - * Creates a scoped router for group-related endpoints using an IIFE pattern - * @private - * @returns {express.Router} Express router instance with isolated require scope - */ - const r_group = (() => { - const require = this.require; - const express = require('express'); - return express.Router(); - })(); - - this.install_group_endpoints_({ router: r_group }); - app.use('/group', r_group); - } - - install_group_endpoints_ ({ router }) { - router.use(eggspress('/create', { - allowedMethods: ['POST'], - mw: [configurable_auth()], - }, async (req, res) => { - const owner_user_id = req.user.id; - - const extra = req.body.extra ?? {}; - const metadata = req.body.metadata ?? {}; - if ( !extra || typeof extra !== 'object' || Array.isArray(extra) ) { - throw APIError.create('field_invalid', null, { - key: 'extra', - expected: 'object', - got: extra, - }); - } - if ( !metadata || typeof metadata !== 'object' || Array.isArray(metadata) ) { - throw APIError.create('field_invalid', null, { - key: 'metadata', - expected: 'object', - got: metadata, - }); - } - - const svc_group = this.services.get('group'); - const uid = await svc_group.create({ - owner_user_id, - // TODO: includeslist for allowed 'extra' fields - extra: {}, - // Metadata can be specified in request - metadata: metadata ?? {}, - }); - - res.json({ uid }); - })); - - router.use(eggspress('/add-users', { - allowedMethods: ['POST'], - mw: [configurable_auth()], - }, async (req, res) => { - const svc_group = this.services.get('group'); - - // TODO: validate string and uuid for request - - const group = await svc_group.get({ uid: req.body.uid }); - - if ( ! group ) { - throw APIError.create('entity_not_found', null, { - identifier: req.body.uid, - }); - } - - if ( group.owner_user_id !== req.user.id ) { - throw APIError.create('forbidden'); - } - - if ( ! Array.isArray(req.body.users) ) { - throw APIError.create('field_invalid', null, { - key: 'users', - expected: 'array', - got: req.body.users, - }); - } - - for ( let i = 0 ; i < req.body.users.length ; i++ ) { - const value = req.body.users[i]; - if ( typeof value === 'string' ) continue; - throw APIError.create('field_invalid', null, { - key: `users[${i}]`, - expected: 'string', - got: value, - }); - } - - await svc_group.add_users({ - uid: req.body.uid, - users: req.body.users, - }); - - res.json({}); - })); - - // TODO: DRY: add-users is very similar - router.use(eggspress('/remove-users', { - allowedMethods: ['POST'], - mw: [configurable_auth()], - }, async (req, res) => { - const svc_group = this.services.get('group'); - - // TODO: validate string and uuid for request - - const group = await svc_group.get({ uid: req.body.uid }); - - if ( ! group ) { - throw APIError.create('entity_not_found', null, { - identifier: req.body.uid, - }); - } - - if ( group.owner_user_id !== req.user.id ) { - throw APIError.create('forbidden'); - } - - if ( Array.isArray(req.body.users) ) { - throw APIError.create('field_invalid', null, { - key: 'users', - expected: 'array', - got: req.body.users, - }); - } - - for ( let i = 0 ; i < req.body.users.length ; i++ ) { - const value = req.body.users[i]; - if ( typeof value === 'string' ) continue; - throw APIError.create('field_invalid', null, { - key: `users[${i}]`, - expected: 'string', - got: value, - }); - } - - await svc_group.remove_users({ - uid: req.body.uid, - users: req.body.users, - }); - - res.json({}); - })); - - router.use(eggspress('/list', { - allowedMethods: ['GET'], - mw: [configurable_auth()], - }, async (req, res) => { - const svc_group = this.services.get('group'); - - // TODO: validate string and uuid for request - - const owned_groups = await svc_group.list_groups_with_owner({ owner_user_id: req.user.id }); - - const in_groups = await svc_group.list_groups_with_member({ user_id: req.user.id }); - - const public_groups = await svc_group.list_public_groups(); - - res.json({ - owned_groups: await Promise.all(owned_groups.map(g => g.get_client_value({ members: true }))), - in_groups: await Promise.all(in_groups.map(g => g.get_client_value({ members: true }))), - public_groups: await Promise.all(public_groups.map(g => g.get_client_value())), - }); - })); - - router.use(eggspress('/public-groups', { - allowedMethods: ['GET'], - mw: [configurable_auth()], - }, async (req, res) => { - res.json({ - user: this.global_config.default_user_group, - temp: this.global_config.default_temp_group, - }); - })); - } -} - -module.exports = { - PermissionAPIService, -}; diff --git a/src/backend/src/services/PuterAPIService.js b/src/backend/src/services/PuterAPIService.js deleted file mode 100644 index b66179342..000000000 --- a/src/backend/src/services/PuterAPIService.js +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import configurable_auth from '../middleware/configurable_auth.js'; -import eggspress from '../api/eggspress.js'; -import appsRouter from '../routers/apps.js'; -import authAppUidFromOriginRouter from '../routers/auth/app-uid-from-origin.js'; -import authCheckAppRouter from '../routers/auth/check-app.js'; -import configure2faRouter from '../routers/auth/configure-2fa.js'; -import createAccessTokenRouter from '../routers/auth/create-access-token.js'; -import listSessionsRouter from '../routers/auth/list-sessions.js'; -import { router as oidcRouter } from '../routers/auth/oidc.js'; -import revokeAccessTokenRouter from '../routers/auth/revoke-access-token.js'; -import revokeSessionRouter from '../routers/auth/revoke-session.js'; -import changeEmailRouter from '../routers/change_email.js'; -import changeUsernameRouter from '../routers/change_username.js'; -import confirmEmailRouter from '../routers/confirmEmail/confirm-email.js'; -import contactUsRouter from '../routers/contactUs.js'; -import deleteSiteRouter from '../routers/delete-site.js'; -import downRouter from '../routers/down.js'; -import driverCallRouter from '../routers/drivers/call.js'; -import driverListInterfacesRouter from '../routers/drivers/list-interfaces.js'; -import driverUsageRouter from '../routers/drivers/usage.js'; -import getDevProfileRouter from '../routers/get-dev-profile.js'; -import launchAppsHandler from '../routers/get-launch-apps.js'; -import healthcheckRouter from '../routers/healthcheck.js'; -import itemMetadataRouter from '../routers/itemMetadata.js'; -import kvstoreClearItemsRouter from '../routers/kvstore/clearItems.js'; -import kvstoreGetItemRouter from '../routers/kvstore/getItem.js'; -import kvstoreListItemsRouter from '../routers/kvstore/listItems.js'; -import kvstoreSetItemRouter from '../routers/kvstore/setItem.js'; -import loginRouter from '../routers/login.js'; -import logoutRouter from '../routers/logout.js'; -import openItemRouter from '../routers/open_item.js'; -import passwdRouter from '../routers/passwd.js'; -import appQueryRouter from '../routers/query/app.js'; -import recentAppOpensRouter from '../routers/recentAppOpens/rao.js'; -import saveAccountRouter from '../routers/save_account.js'; -import sendConfirmEmailRouter from '../routers/send-confirm-email.js'; -import sendPassRecoveryEmailRouter from '../routers/send-pass-recovery-email.js'; -import setDesktopBackgroundRouter from '../routers/set-desktop-bg.js'; -import setPassUsingTokenRouter from '../routers/set-pass-using-token.js'; -import setLayoutRouter from '../routers/set_layout.js'; -import setSortByRouter from '../routers/set_sort_by.js'; -import signRouter from '../routers/sign.js'; -import signupRouter from '../routers/signup.js'; -import suggestAppsRouter from '../routers/suggest_apps.js'; -import testRouter from '../routers/test.js'; -import updateTaskbarItemsRouter from '../routers/update-taskbar-items.js'; -import verifyPassRecoveryTokenRouter from '../routers/verify-pass-recovery-token.js'; -import BaseService from './BaseService.js'; -/** -* @class PuterAPIService -* @extends BaseService -* -* The PuterAPIService class is responsible for integrating various routes -* into the web server for the Puter application. It acts as a middleware -* support layer, providing necessary API endpoints for handling various -* functionality such as authentication, user management, and application -* operations. This class is designed to extend the core functionalities -* of BaseService, ensuring that all routes are properly configured and -* available for use. -*/ -export class PuterAPIService extends BaseService { - /** - * Sets up the routes for the Puter API service. - * This method registers various API endpoints with the web server. - * It does not return a value as it configures the server directly. - */ - async '__on_install.routes' () { - const svc_web = this.services.get('web-server'); - const { app } = svc_web; - svc_web.allow_undefined_origin('/healthcheck'); - - app.use(appsRouter); - app.use(appQueryRouter); - app.use(changeUsernameRouter); - changeEmailRouter(app); - app.use(listSessionsRouter); - app.use(revokeSessionRouter); - app.use(authCheckAppRouter); - app.use(authAppUidFromOriginRouter); - app.use(createAccessTokenRouter); - app.use(revokeAccessTokenRouter); - app.use(configure2faRouter); - app.use(driverCallRouter); - app.use(driverListInterfacesRouter); - app.use(driverUsageRouter); - app.use(confirmEmailRouter); - app.use(downRouter); - app.use(contactUsRouter); - app.use(deleteSiteRouter); - app.use(getDevProfileRouter); - app.use(kvstoreGetItemRouter); - app.use(kvstoreSetItemRouter); - app.use(kvstoreListItemsRouter); - app.use(kvstoreClearItemsRouter); - app.use(itemMetadataRouter); - app.use(loginRouter); - app.use(oidcRouter); - app.use(logoutRouter); - app.use(openItemRouter); - app.use(passwdRouter); - app.use(recentAppOpensRouter); - app.use(saveAccountRouter); - app.use(sendConfirmEmailRouter); - app.use(sendPassRecoveryEmailRouter); - app.use(setDesktopBackgroundRouter); - app.use(verifyPassRecoveryTokenRouter); - app.use(setPassUsingTokenRouter); - app.use(setLayoutRouter); - app.use(setSortByRouter); - app.use(signRouter); - app.use(signupRouter); - app.use(suggestAppsRouter); - app.use(healthcheckRouter); - app.use(testRouter); - app.use(updateTaskbarItemsRouter); - - app.use(eggspress('/get-launch-apps', { - allowedMethods: ['GET'], - mw: [configurable_auth()], - }, launchAppsHandler)); - - } -} diff --git a/src/backend/src/services/PuterHomepageService.js b/src/backend/src/services/PuterHomepageService.js deleted file mode 100644 index 5eb433247..000000000 --- a/src/backend/src/services/PuterHomepageService.js +++ /dev/null @@ -1,492 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { encode } from 'html-entities'; -import { LRUCache } from 'lru-cache'; -import fs from 'node:fs'; -import eggspress from '../api/eggspress.js'; -import { is_valid_url } from '../helpers.js'; -import { PathBuilder } from '../util/pathutil.js'; -import BaseService from './BaseService.js'; -/** - * PuterHomepageService serves the initial HTML page that loads the Puter GUI - * and all of its assets. - */ -export class PuterHomepageService extends BaseService { - - #outputCache = null; - - _construct () { - this.service_scripts = []; - this.gui_params = {}; - - this.#outputCache = new LRUCache({ - max: 200, - }); - } - - /** - * @description This method initializes the PuterHomepageService by loading the manifest file. - * It reads the manifest file located at the specified path and parses its JSON content. - * The parsed data is then assigned to the `manifest` property of the instance. - * @returns {Promise} A promise that resolves with the initialized PuterHomepageService instance. - */ - async _init () { - // Load manifest - const config = this.global_config; - const manifest_raw = fs.readFileSync( - PathBuilder - .add(config.assets.gui, { allow_traversal: true }) - .add('puter-gui.json') - .build(), - 'utf8', - ); - const manifest_data = JSON.parse(manifest_raw); - this.manifest = manifest_data[config.assets.gui_profile]; - } - - register_script (url) { - this.service_scripts.push(url); - } - - set_gui_param (key, val) { - this.gui_params[key] = val; - } - - async '__on_install.routes' (_, { app }) { - app.use(eggspress('/whoarewe', { - allowedMethods: ['GET'], - }, async (req, res) => { - // Get basic configuration information - const responseData = { - disable_user_signup: this.global_config.disable_user_signup, - disable_temp_users: this.global_config.disable_temp_users, - environmentInfo: { - env: this.global_config.env, - version: process.env.VERSION || 'development', - }, - }; - - // Add captcha requirement information - responseData.captchaRequired = { - login: req.captchaRequired, - signup: req.captchaRequired, - }; - - res.json(responseData); - })); - } - - /** - * This method sends the initial HTML page that loads the Puter GUI and its assets. - */ - async send ({ req, res, auth_user }, meta, launch_options) { - const config = this.global_config; - - if ( - req.query['puter.app_instance_id'] || - req.query['error_from_within_iframe'] - ) { - const easteregg = [ - 'puter in puter?', - 'Infinite recursion!', - 'what\'chu cookin\'?', - ]; - const message = req.query.message || - easteregg[ - Math.floor(Math.random(easteregg.length)) - ]; - - return res.send(this.generate_error_html({ - message, - })); - } - - // checkCaptcha middleware (in CaptchaService) sets req.captchaRequired - const captchaRequired = { - login: req.captchaRequired, - signup: req.captchaRequired, - }; - - // cloudflare turnstile site key - const turnstileSiteKey = config.services?.['cloudflare-turnstile']?.enabled ? config.services?.['cloudflare-turnstile']?.site_key : null; - - const cacheKey = (() => { - const cacheKeyObject = { - ...(meta ? { - title: meta.title, - app_name: meta?.app?.name, - } : {}), - }; - return JSON.stringify( - cacheKeyObject, - Object.keys(cacheKeyObject).sort(), - ); - })(); - - // Possibly send cached output - { - const maybeCachedOutputHTML = this.#outputCache.get(cacheKey); - if ( maybeCachedOutputHTML ) { - res.send(maybeCachedOutputHTML); - return; - } - } - - // Check if user is logged in - const logged_in_user = auth_user || null; - - const outputHTML = await this.generate_puter_page_html({ - req, - path: req.path, - - env: config.env, - - app_origin: config.origin, - api_origin: config.api_base_url, - use_bundled_gui: config.use_bundled_gui, - - manifest: this.manifest, - gui_path: config.assets.gui, - - // page meta - meta, - - // launch options - launch_options, - - // logged-in user info - logged_in_user, - - // gui parameters - gui_params: { - app_name_regex: config.app_name_regex, - app_name_max_length: config.app_name_max_length, - app_title_max_length: config.app_title_max_length, - hosting_domain: config.static_hosting_domain + - (config.pub_port !== 80 && config.pub_port !== 443 ? `:${config.pub_port}` : ''), - subdomain_regex: config.subdomain_regex, - subdomain_max_length: config.subdomain_max_length, - domain: config.domain, - protocol: config.protocol, - env: config.env, - api_base_url: config.api_base_url, - thumb_width: config.thumb_width, - thumb_height: config.thumb_height, - contact_email: config.contact_email, - max_fsentry_name_length: config.max_fsentry_name_length, - require_email_verification_to_publish_website: config.require_email_verification_to_publish_website, - short_description: config.short_description, - long_description: config.long_description, - disable_temp_users: config.disable_temp_users, - co_isolation_enabled: req.co_isolation_enabled, - // Add captcha requirements to GUI parameters - captchaRequired: captchaRequired, - turnstileSiteKey: turnstileSiteKey, - }, - }); - - // TODO: we will re-enable this shortly (within 24 hours) - // - // It is currently disabled so that we can determine the impact on - // performance of b687ba0 (not b687ba0 specifically but the subsequent - // fixed version of b687ba0) in isolation without confounding - // variables. - // - // this.#outputCache.set(cacheKey, outputHTML); - - res.send(outputHTML); - } - - async generate_puter_page_html ({ - req, - path, - env, - manifest, - gui_path: _gui_path, - use_bundled_gui, - app_origin, - api_origin, - meta, - launch_options, - logged_in_user, - gui_params, - }) { - - const eventService = this.services.get('event'); - - const e = encode; - - const { - title, - description, - short_description, - company, - canonical_url, - social_media_image, - } = meta; - - gui_params = { - ...meta, - ...gui_params, - ...this.gui_params, - launch_options, - app_origin, - api_origin, - gui_origin: app_origin, - }; - - const asset_dir = env === 'dev' - ? '/src' : '/dist'; - - gui_params.asset_dir = asset_dir; - - const bundled = env != 'dev' || use_bundled_gui; - - // check if social media image is a valid absolute URL - let is_social_media_image_valid = !!social_media_image; - if ( is_social_media_image_valid && !is_valid_url(social_media_image) ) { - is_social_media_image_valid = false; - } - - // check if social media image ends with a valid image extension - if ( is_social_media_image_valid && !/\.(png|jpg|jpeg|gif|webp)$/.test(social_media_image.toLowerCase()) ) { - is_social_media_image_valid = false; - } - - // set social media image to default if it is not valid - const social_media_image_url = is_social_media_image_valid ? social_media_image : `${asset_dir}/images/screenshot.png`; - - // Custom script tags to be added to the homepage by extensions - // an event is emitted to allow extensions to add their own script tags - // the event is emitted with an object containing a custom_script_tags array - // which extensions can push their script tags to - let custom_script_tags = []; - let custom_script_tags_str = ''; - process.emit('add_script_tags_to_homepage_html', { custom_script_tags }); - - for ( const tag of custom_script_tags ) { - custom_script_tags_str += tag; - } - - // emit extension event - const event = { - req: req, - path: path, - bodyContent: '', - headContent: '', - prependHeadContent: '', - logged_in_user: logged_in_user, - guiParams: { - ...gui_params, - }, - }; - await eventService.emit('puter.gui.addons', event); - - return ` - - - - ${e(title)} - - ${event.prependHeadContent || ''} - - - ${bundled - ? `` - : '' - } - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${(bundled) - ? `` : ''} - - - - - - - - ${((!bundled && manifest?.css_paths) - ? manifest.css_paths.map(path => `\n`) - : []).join('') - } - - - - ${event.headContent || ''} - - - - - - - ${custom_script_tags_str - } - ${bundled - ? '' - : '' - } - - - - - - - ${this.service_scripts - .map(path => `\n`) - .join('') - } - - - - ${event.bodyContent || ''} - - - - - `; - }; - - generate_error_html ({ message }) { - return ` - - - - - - -

${encode(message, { mode: 'nonAsciiPrintable' }) - }

- - - `; - } -} diff --git a/src/backend/src/services/PuterSiteService.js b/src/backend/src/services/PuterSiteService.js deleted file mode 100644 index 638ac2e9a..000000000 --- a/src/backend/src/services/PuterSiteService.js +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { NodeInternalIDSelector, NodeUIDSelector } = require('../deprecated/filesystem/node/selectors'); -const { SiteActorType } = require('./auth/Actor'); -const { PermissionUtil, PermissionRewriter, PermissionImplicator } = require('./auth/permissionUtils.mjs'); -const BaseService = require('./BaseService'); -const { DB_WRITE } = require('./database/consts'); - -/** -* The `PuterSiteService` class manages site-related operations within the Puter platform. -* This service extends `BaseService` to provide functionalities like: -* - Initializing database connections for site data. -* - Handling subdomain permissions and rewriting them as necessary. -* - Managing permissions for site files, ensuring that sites can access their own resources. -* - Retrieving subdomain information by name or unique identifier (UID). -* This class is crucial for controlling access and operations related to different sites hosted or managed by the Puter system. -*/ -class PuterSiteService extends BaseService { - /** - * Initializes the PuterSiteService by setting up database connections, - * registering permission rewriters and implicators, and preparing service dependencies. - * - * @returns {Promise} A promise that resolves when initialization is complete. - */ - async _init () { - const services = this.services; - this.db = services.get('database').get(DB_WRITE, 'sites'); - - const svc_fs = services.get('filesystem'); - - // Rewrite site permissions specified by name - const svc_permission = this.services.get('permission'); - svc_permission.register_rewriter(PermissionRewriter.create({ - matcher: permission => { - if ( ! permission.startsWith('site:') ) return false; - const [_, specifier] = PermissionUtil.split(permission); - if ( specifier.startsWith('uid#') ) return false; - return true; - }, - rewriter: async permission => { - const [_1, name, ...rest] = PermissionUtil.split(permission); - const sd = await this.get_subdomain(name); - return PermissionUtil.join(_1, `uid#${sd.uuid}`, ...rest); - }, - })); - - // Imply that sites can read their own files - svc_permission.register_implicator(PermissionImplicator.create({ - id: 'in-site', - matcher: permission => { - return permission.startsWith('fs:'); - }, - checker: async ({ actor, permission }) => { - if ( ! (actor.type instanceof SiteActorType) ) { - return undefined; - } - - const [_, uid, lvl] = PermissionUtil.split(permission); - const node = await svc_fs.node(new NodeUIDSelector(uid)); - - if ( ! ['read', 'list', 'see'].includes(lvl) ) { - return undefined; - } - - if ( ! await node.exists() ) { - return undefined; - } - - const site_node = await svc_fs.node(new NodeInternalIDSelector( - 'mysql', - actor.type.site.root_dir_id, - )); - - if ( await site_node.is(node) ) { - return {}; - } - if ( await site_node.is_above(node) ) { - return {}; - } - - return undefined; - }, - })); - } - - /** - * Retrieves subdomain information by its name. - * - * @param {string} subdomain - The name of the subdomain to retrieve. - * @returns {Promise} Returns an object with subdomain details or null if not found. - * @note In development environment, 'devtest' subdomain returns hardcoded values. - */ - async get_subdomain (subdomain, options) { - if ( subdomain === 'devtest' && this.global_config.env === 'dev' ) { - return { - user_id: null, - root_dir_id: this.config.devtest_directory, - }; - } - const rows = await this.db.read( - `SELECT * FROM subdomains WHERE ${ - options.is_custom_domain ? 'domain' : 'subdomain' - } = ? LIMIT 1`, - [subdomain], - ); - if ( rows.length === 0 ) return null; - return rows[0]; - } - - /** - * Retrieves a subdomain by its unique identifier (UID). - * - * @param {string} uid - The unique identifier of the subdomain to fetch. - * @returns {Promise} A promise that resolves to the subdomain object if found, or null if not found. - */ - async get_subdomain_by_uid (uid) { - const rows = await this.db.read( - 'SELECT * FROM subdomains WHERE uuid = ? LIMIT 1', - [uid], - ); - if ( rows.length === 0 ) return null; - return rows[0]; - } -} - -module.exports = { - PuterSiteService, -}; diff --git a/src/backend/src/services/PuterVersionService.js b/src/backend/src/services/PuterVersionService.js deleted file mode 100644 index 156f1d880..000000000 --- a/src/backend/src/services/PuterVersionService.js +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const BaseService = require('./BaseService'); - -/** -* Class representing the PuterVersionService. -* -* The PuterVersionService extends the BaseService and provides methods -* to initialize the service, handle routing for version information, -* and retrieve the current version of the application. It is responsible -* for managing version-related operations within the Puter framework. -*/ -class PuterVersionService extends BaseService { - /** - * Initializes the service by recording the current boot time. - * This method is called asynchronously to ensure that any necessary - * setup can be completed before the service begins handling requests. - */ - async _init () { - this.boot_time = Date.now(); - } - - /** - * Sets up the routes for the versioning API. - * This method registers the version router with the web server application. - * - * @async - * @returns {Promise} Resolves when the routes are successfully registered. - */ - async '__on_install.routes' () { - const { app } = this.services.get('web-server'); - app.use(require('../routers/version')); - } - - /** - * Retrieves the current version information of the application along with - * the environment and deployment details. The method fetches the version - * from the npm package or the local package.json file and returns an - * object containing the version, environment, server location, and - * deployment timestamp. - * - * @returns {Object} An object containing version details. - * @returns {string} return.version - The current application version. - * @returns {string} return.environment - The environment in which the app is running. - * @returns {string} return.location - The server ID where the application is deployed. - * @returns {number} return.deploy_timestamp - The timestamp when the application was deployed. - */ - get_version () { - const version = process.env.npm_package_version || - require('../../package.json').version; - return { - version, - environment: this.global_config.env, - location: this.global_config.server_id, - deploy_timestamp: this.boot_time, - }; - } -} - -module.exports = { - PuterVersionService, -}; \ No newline at end of file diff --git a/src/backend/src/services/PuterVersionService.test.ts b/src/backend/src/services/PuterVersionService.test.ts deleted file mode 100644 index 0ac4ade0a..000000000 --- a/src/backend/src/services/PuterVersionService.test.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { createTestKernel } from '../../tools/test.mjs'; -import { PuterVersionService } from './PuterVersionService'; - -describe('PuterVersionService', async () => { - const testKernel = await createTestKernel({ - serviceMap: { - 'puter-version': PuterVersionService, - }, - initLevelString: 'init', - }); - - const versionService = testKernel.services!.get('puter-version') as any; - - it('should be instantiated', () => { - expect(versionService).toBeInstanceOf(PuterVersionService); - }); - - it('should have boot_time set after init', () => { - expect(versionService.boot_time).toBeDefined(); - expect(typeof versionService.boot_time).toBe('number'); - expect(versionService.boot_time).toBeGreaterThan(0); - }); - - it('should return version info', () => { - const versionInfo = versionService.get_version(); - - expect(versionInfo).toBeDefined(); - expect(versionInfo).toHaveProperty('version'); - expect(versionInfo).toHaveProperty('environment'); - expect(versionInfo).toHaveProperty('location'); - expect(versionInfo).toHaveProperty('deploy_timestamp'); - }); - - it('should have valid version string', () => { - const versionInfo = versionService.get_version(); - - expect(typeof versionInfo.version).toBe('string'); - expect(versionInfo.version).toBeTruthy(); - }); - - it('should have deploy_timestamp matching boot_time', () => { - const versionInfo = versionService.get_version(); - - expect(versionInfo.deploy_timestamp).toBe(versionService.boot_time); - }); - - it('should have environment from config', () => { - const versionInfo = versionService.get_version(); - - // Environment might be undefined in test context - expect(versionInfo).toHaveProperty('environment'); - }); - - it('should have location from config', () => { - const versionInfo = versionService.get_version(); - - // Location might be undefined in test context - expect(versionInfo).toHaveProperty('location'); - }); - - it('should return consistent version info on multiple calls', () => { - const versionInfo1 = versionService.get_version(); - const versionInfo2 = versionService.get_version(); - - expect(versionInfo1.version).toBe(versionInfo2.version); - expect(versionInfo1.deploy_timestamp).toBe(versionInfo2.deploy_timestamp); - }); -}); - diff --git a/src/backend/src/services/ReferralCodeService.js b/src/backend/src/services/ReferralCodeService.js deleted file mode 100644 index 9ac09ddfb..000000000 --- a/src/backend/src/services/ReferralCodeService.js +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const seedrandom = require('seedrandom'); -const { generate_random_code } = require('../util/identifier'); -const { Context } = require('../util/context'); -const { get_user, invalidate_cached_user_by_id } = require('../helpers'); -const { DB_WRITE } = require('./database/consts'); -const BaseService = require('./BaseService'); -const { UserIDNotifSelector } = require('./NotificationService'); - -/** -* Class ReferralCodeService -* -* This class is responsible for managing the generation and handling of referral codes -* within the application. It extends the BaseService and provides methods to initialize -* referral code generation for users, verify referrals, and manage updates to user -* storage based on successful referrals. The service ensures that referral codes are -* unique and properly assigned during user interactions. -*/ -class ReferralCodeService extends BaseService { - _construct () { - this.REFERRAL_INCREASE_LEFT = 1 * 1024 * 1024 * 1024; // 1 GB - this.REFERRAL_INCREASE_RIGHT = 1 * 1024 * 1024 * 1024; // 1 GB - this.STORAGE_INCREASE_STRING = '1 GB'; - this.MAX_REFERRALS_PER_MONTH = 20; - this.MONTHLY_REFERRAL_KEY_PREFIX = 'referral:monthly'; - } - - /** - * Initializes the ReferralCodeService by setting up event listeners - * for user email confirmation. Listens for the 'user.email-confirmed' - * event and triggers the on_verified method when a user confirms their - * email address. - * - * @async - * @returns {Promise} A promise that resolves when initialization is complete. - */ - async _init () { - const svc_event = this.services.get('event'); - svc_event.on('user.email-confirmed', async (_, { user_uid }) => { - const user = await this.getUser({ uuid: user_uid }); - await this.on_verified(user); - }); - } - - /** - * Generates a unique referral code for the specified user. - * This method attempts to create a referral code and store it in the database. - * It retries the generation process up to a predefined number of attempts if - * any errors occur during the database write operation. - * - * @param {Object} user - The user for whom the referral code is being generated. - * @returns {Promise} The generated referral code. - * @throws Will throw an error if the user is missing or if the code generation fails after retries. - */ - async gen_referral_code (user) { - let iteration = 0; - let rng = seedrandom(`gen1-${user.id}`); - let referral_code = generate_random_code(8, { rng }); - - if ( !user || (user?.id == undefined) ) { - const err = new Error('missing user in gen_referral_code'); - this.errors.report('missing user in gen_referral_code', { - source: err, - trace: true, - alarm: true, - }); - throw err; - } - - // Constant representing the number of attempts to generate a unique referral code. - const TRIES = 5; - - const db = Context.get('services').get('database').get(DB_WRITE, 'referrals'); - - let last_error = null; - for ( let i = 0 ; i < TRIES; i++ ) { - this.log.debug(`trying referral code ${referral_code}`); - if ( i > 0 ) { - rng = seedrandom(`gen1-${user.id}-${++iteration}`); - referral_code = generate_random_code(8, { rng }); - } - try { - await db.write(` - UPDATE user SET referral_code=? WHERE id=? - `, [referral_code, user.id]); - invalidate_cached_user_by_id(user.id); - return referral_code; - } catch (e) { - last_error = e; - } - } - - this.errors.report('referral-service.gen-referral-code', { - source: last_error, - trace: true, - alarm: true, - }); - - throw last_error ?? new Error('unknown error from gen_referral_code'); - } - - /** - * Handles the logic when a user is verified. - * This method checks if the user has been referred by another user and updates - * the storage of both the referring user and the newly verified user accordingly. - * - * @param {Object} user - The user object representing the verified user. - * @returns {Promise} - A promise that resolves when the operation is complete. - */ - async on_verified (user) { - if ( ! user.referred_by ) return; - - const referred_by = await this.getUser({ id: user.referred_by }); - const monthlyReferralCount = await this.consumeMonthlyReferralSlot(referred_by.id); - if ( monthlyReferralCount > this.MAX_REFERRALS_PER_MONTH ) { - this.log.info( - `skipping referral rewards for user ${referred_by.id}: ` + - `monthly limit reached (${monthlyReferralCount}/${this.MAX_REFERRALS_PER_MONTH})`, - ); - return; - } - - // since this event handler is only called when the user is verified, - // we can assume that the `user` is already verified. - - // the referred_by user does not need to be verified at all - - // TODO: rename 'sizeService' to 'storage-capacity' - const svc_size = Context.get('services').get('sizeService'); - const meteringService = this.services.get('meteringService'); - - // For user that got referred to - await svc_size.add_storage( - user, - this.REFERRAL_INCREASE_RIGHT, - `user ${user.id} used referral code of user ${referred_by.id}`, - { - field_a: referred_by.referral_code, - field_b: 'REFER_R', - }, - ); - await meteringService.updateAddonCredit(user.uuid, 25 * 1_000_000); // give them 25 cents - - // For user who referred - await svc_size.add_storage( - referred_by, - this.REFERRAL_INCREASE_LEFT, - `user ${referred_by.id} referred user ${user.id}`, - { - field_a: referred_by.referral_code, - field_b: 'REFER_L', - }, - ); - await meteringService.updateAddonCredit(referred_by.uuid, 25 * 1_000_000); // give them 25 cents - - const svc_email = Context.get('services').get('email'); - await svc_email.send_email (referred_by, 'new-referral', { - storage_increase: this.STORAGE_INCREASE_STRING, - }); - - const svc_notification = Context.get('services').get('notification'); - svc_notification.notify(UserIDNotifSelector(referred_by.id), { - source: 'referral', - icon: 'c-check.svg', - text: `You have referred user ${user.username} and ` + - `have received ${this.STORAGE_INCREASE_STRING} of storage.`, - template: 'referral', - fields: { - storage_increase: this.STORAGE_INCREASE_STRING, - referred_username: user.username, - }, - }); - } - - getMonthlyReferralKey (referredByUserId, nowMs = Date.now()) { - const month = new Date(nowMs).toISOString().slice(0, 7); - return `${this.MONTHLY_REFERRAL_KEY_PREFIX}:user:${referredByUserId}:month:${month}`; - } - - getNextMonthTimestamp (nowMs = Date.now()) { - const now = new Date(nowMs); - return Math.floor(Date.UTC( - now.getUTCFullYear(), - now.getUTCMonth() + 1, - 1, - 0, - 0, - 0, - ) / 1000); - } - - async consumeMonthlyReferralSlot (referredByUserId) { - const su = this.services.get('su'); - const kvStore = this.services.get('puter-kvstore'); - const key = this.getMonthlyReferralKey(referredByUserId); - const expiryTimestamp = this.getNextMonthTimestamp(); - - return await su.sudo(async () => { - const counter = await kvStore.incr({ - key, - pathAndAmountMap: { total: 1 }, - }); - await kvStore.expireAt({ - key, - timestamp: expiryTimestamp, - }); - return Number(counter?.total ?? 0); - }); - } - - async getUser (query) { - return await get_user(query); - } -} - -module.exports = { - ReferralCodeService, -}; diff --git a/src/backend/src/services/ReferralCodeService.test.js b/src/backend/src/services/ReferralCodeService.test.js deleted file mode 100644 index 944da25bd..000000000 --- a/src/backend/src/services/ReferralCodeService.test.js +++ /dev/null @@ -1,152 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; - -vi.mock('./NotificationService.js', () => ({ - UserIDNotifSelector: vi.fn((userId) => ({ userId })), -})); - -import { Context } from '../util/context.js'; -const { ReferralCodeService } = require('./ReferralCodeService'); - -const createService = ({ monthlyCount, referredByUser }) => { - const kvStore = { - incr: vi.fn().mockResolvedValue({ total: monthlyCount }), - expireAt: vi.fn().mockResolvedValue(undefined), - }; - const suService = { - sudo: vi.fn().mockImplementation(async (runner) => await runner()), - }; - const meteringService = { - updateAddonCredit: vi.fn().mockResolvedValue(undefined), - }; - const sizeService = { - add_storage: vi.fn().mockResolvedValue(undefined), - }; - const emailService = { - send_email: vi.fn().mockResolvedValue(undefined), - }; - const notificationService = { - notify: vi.fn(), - }; - - const service = Object.create(ReferralCodeService.prototype); - service._construct(); - service.getUser = vi.fn().mockResolvedValue(referredByUser); - service.log = { - info: vi.fn(), - debug: vi.fn(), - }; - service.errors = { - report: vi.fn(), - }; - service.services = { - get: vi.fn((serviceName) => { - if ( serviceName === 'su' ) return suService; - if ( serviceName === 'puter-kvstore' ) return kvStore; - if ( serviceName === 'meteringService' ) return meteringService; - if ( serviceName === 'notification' ) return notificationService; - throw new Error(`unexpected service lookup: ${serviceName}`); - }), - }; - - Context.root.set('services', { - get: (serviceName) => { - if ( serviceName === 'sizeService' ) return sizeService; - if ( serviceName === 'email' ) return emailService; - if ( serviceName === 'notification' ) return notificationService; - throw new Error(`unexpected context service lookup: ${serviceName}`); - }, - }); - - return { - service, - kvStore, - suService, - meteringService, - sizeService, - emailService, - notificationService, - }; -}; - -describe('ReferralCodeService', () => { - let previousContextServices; - - beforeEach(() => { - vi.clearAllMocks(); - previousContextServices = Context.root.get('services'); - }); - - afterEach(() => { - Context.root.set('services', previousContextServices); - }); - - it('awards referral rewards when monthly count is within the 20-user cap', async () => { - const referredByUser = { - id: 200, - uuid: 'referrer-uuid', - referral_code: 'REF-200', - username: 'referrer', - }; - const { - service, - kvStore, - suService, - meteringService, - sizeService, - emailService, - notificationService, - } = createService({ monthlyCount: 20, referredByUser }); - - await service.on_verified({ - id: 201, - uuid: 'referred-uuid', - username: 'referred', - referred_by: 200, - }); - - expect(suService.sudo).toHaveBeenCalledTimes(1); - expect(kvStore.incr).toHaveBeenCalledWith({ - key: expect.stringContaining('referral:monthly:user:200:month:'), - pathAndAmountMap: { total: 1 }, - }); - expect(kvStore.expireAt).toHaveBeenCalledWith({ - key: expect.stringContaining('referral:monthly:user:200:month:'), - timestamp: expect.any(Number), - }); - const expiryTimestamp = kvStore.expireAt.mock.calls[0][0].timestamp; - expect(expiryTimestamp).toBeGreaterThan(Math.floor(Date.now() / 1000)); - expect(sizeService.add_storage).toHaveBeenCalledTimes(2); - expect(meteringService.updateAddonCredit).toHaveBeenCalledTimes(2); - expect(emailService.send_email).toHaveBeenCalledTimes(1); - expect(notificationService.notify).toHaveBeenCalledTimes(1); - }); - - it('skips referral rewards when monthly count exceeds the 20-user cap', async () => { - const referredByUser = { - id: 300, - uuid: 'referrer-uuid', - referral_code: 'REF-300', - username: 'referrer', - }; - const { - service, - sizeService, - meteringService, - emailService, - notificationService, - } = createService({ monthlyCount: 21, referredByUser }); - - await service.on_verified({ - id: 301, - uuid: 'referred-uuid', - username: 'referred', - referred_by: 300, - }); - - expect(sizeService.add_storage).not.toHaveBeenCalled(); - expect(meteringService.updateAddonCredit).not.toHaveBeenCalled(); - expect(emailService.send_email).not.toHaveBeenCalled(); - expect(notificationService.notify).not.toHaveBeenCalled(); - expect(service.log.info).toHaveBeenCalledTimes(1); - }); -}); diff --git a/src/backend/src/services/RefreshAssociationsService.js b/src/backend/src/services/RefreshAssociationsService.js deleted file mode 100644 index 21d219dec..000000000 --- a/src/backend/src/services/RefreshAssociationsService.js +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { Context } = require('../util/context'); -const BaseService = require('./BaseService'); - -/** -* Class RefreshAssociationsService -* -* This class is responsible for managing the refresh of associations in the system. -* It extends the BaseService and provides methods to handle the refreshing operations -* with context fallback capabilities to ensure reliability during the execution of tasks. -*/ -class RefreshAssociationsService extends BaseService { - /** - * Executes the consolidation process to refresh the associations cache. - * This method is triggered on the '__on_boot.consolidation' event and - * ensures that the cache is updated periodically. The first update occurs - * after a delay of 15 seconds, followed by continuous updates every 30 seconds. - * - * @async - * @returns {Promise} - A promise that resolves when the cache refresh process is complete. - */ - async '__on_boot.consolidation' () { - const { refresh_associations_cache } = require('../helpers'); - - /** - * Executes the consolidation process on boot, refreshing the associations cache. - * This method invokes the `refresh_associations_cache` function within a fallback context. - * The cache refresh is scheduled to run every 30 seconds after an initial delay of 15 seconds. - */ - await Context.allow_fallback(async () => { - refresh_associations_cache(); - }); - /** - * Executes the refresh associations cache function within a fallback context. - * This method ensures that the cache is refreshed properly, handling any - * potential errors that may occur during execution. It utilizes the Context - * utility to allow error handling without interrupting the main application flow. - */ - setTimeout(() => { - /** - * Schedules periodic refresh of associations cache after a timeout. - * - * This method initiates a cache refresh operation that is run at a specified interval. - * The initial refresh occurs after a delay, followed by regular refreshes every 30 seconds. - * - * @returns {Promise} A promise that resolves when the refresh process starts. - */ - setInterval(async () => { - /** - * Initializes a periodic refresh of associations in the cache. - * The method sets a timeout before starting an interval that calls - * the `refresh_associations_cache` function every 30 seconds. - * - * @returns {void} - */ - await Context.allow_fallback(async () => { - await refresh_associations_cache(); - }); - }, 32000); - }, 15000); - } -} - -module.exports = { RefreshAssociationsService }; diff --git a/src/backend/src/services/RegistrantService.js b/src/backend/src/services/RegistrantService.js deleted file mode 100644 index f3c5289f4..000000000 --- a/src/backend/src/services/RegistrantService.js +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { Mapping } = require('../om/definitions/Mapping'); -const { PropType } = require('../om/definitions/PropType'); -const { Context } = require('../util/context'); -const BaseService = require('./BaseService'); - -/** -* RegistrantService class handles the registration and initialization of property types and object mappings -* in the system registry. It extends BaseService and provides functionality to populate the registry with -* property types and their mappings, ensuring type validation and proper inheritance relationships. -* @extends BaseService -*/ -class RegistrantService extends BaseService { - /** - * If population fails, marks the system as invalid through system validation. - */ - async _init () { - const svc_systemValidation = this.services.get('system-validation'); - try { - await this._populate_registry(); - } catch ( e ) { - svc_systemValidation.mark_invalid('Failed to populate registry', - e); - } - } - /** - * Initializes the registrant service by populating the registry. - * Attempts to populate the registry with property types and mappings. - * If population fails, an error is thrown - * @throws {Error} Propagates any errors from registry population for system validation - * @returns {Promise} - */ - async _populate_registry () { - const svc_registry = this.services.get('registry'); - - // This context will be provided to the `create` methods - // that transform the raw data into objects. - /** - * Populates the registry with property types and object mappings. - * Loads property type definitions and mappings from configuration files, - * validates them for duplicates and dependencies, and registers them - * in the registry service. - * - * @throws {Error} If duplicate property types are found or if a property type - * references an undefined super type - * @private - */ - const ctx = Context.get().sub({ - registry: svc_registry, - }); - - // Register property types - { - const seen = new Set(); - - const collection = svc_registry.register_collection('om:proptype'); - const data = require('../om/proptypes/__all__'); - for ( const k in data ) { - if ( seen.has(k) ) { - throw new Error(`Duplicate property type "${k}"`); - } - if ( data[k].from && !seen.has(data[k].from) ) { - throw new Error(`Super type "${data[k].from}" not found for property type "${k}"`); - } - collection.set(k, PropType.create(ctx, data[k], k)); - seen.add(k); - } - } - - // Register object mappings - { - const collection = svc_registry.register_collection('om:mapping'); - const data = require('../om/mappings/__all__'); - for ( const k in data ) { - collection.set(k, Mapping.create(ctx, data[k])); - } - } - } -} - -module.exports = { - RegistrantService, -}; diff --git a/src/backend/src/services/RegistryService.js b/src/backend/src/services/RegistryService.js deleted file mode 100644 index 20d5b90fb..000000000 --- a/src/backend/src/services/RegistryService.js +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('@heyputer/putility'); -const BaseService = require('./BaseService'); -const uuidv4 = require('uuid').v4; - -/** -* @class MapCollection -* @extends AdvancedBase -* -* The `MapCollection` class extends the `AdvancedBase` class and is responsible for managing a collection of key-value pairs. -* It uses `uuid` library for generating unique identifiers for each key-value pair. -* This class provides methods for basic CRUD operations (create, read, update, delete) on the key-value pairs, as well as methods for checking the existence of a key and retrieving all keys in the collection. -*/ -class MapCollection extends AdvancedBase { - constructor () { - super(); - // We use kvjs instead of a plain object because it doesn't - // have a limit on the number of keys it can store. - this.map_id = uuidv4(); - this.map = new Map(); - } - - get (key) { - return this.map.get(this._mk_key(key)); - } - - exists (key) { - return this.map.has(this._mk_key(key)); - } - - set (key, value) { - return this.map.set(this._mk_key(key), value); - } - - del (key) { - return this.map.delete(this._mk_key(key)); - } - - /** - * Retrieves all keys in the map collection, excluding the prefix. - * - * This method fetches all keys that match the pattern for the current map collection. - * The prefix `registry:map:${this.map_id}:` is stripped from each key before returning. - * - * @returns {string[]} An array of keys without the prefix. - */ - keys () { - const keys = this.map.keys().find((k) => k.startsWith(`registry:map:${this.map_id}:`)); - return keys.map(k => k.slice(`registry:map:${this.map_id}:`.length)); - } - - _mk_key (key) { - return `registry:map:${this.map_id}:${key}`; - } -} - -/** -* @class RegistryService -* @extends BaseService -* @description The RegistryService class manages collections of key-value pairs, allowing for dynamic registration and retrieval of collections. -* It extends the BaseService class and provides methods to register new collections, retrieve existing collections, and handle consolidation tasks upon boot. -*/ -class RegistryService extends BaseService { - static MODULES = { - MapCollection, - }; - - /** - * Initializes the RegistryService by setting up the collections. - * - * This method is called during the construction phase of the service. - * It initializes an empty object to hold collections. - * - * @private - * @returns {void} - */ - _construct () { - this.collections_ = {}; - } - - /** - * Initializes the service by setting up the collections object. - * This method is called during the construction phase of the service. - * - * @private - */ - async '__on_boot.consolidation' () { - const services = this.services; - await services.emit('registry.collections'); - await services.emit('registry.entries'); - } - - register_collection (name) { - if ( this.collections_[name] ) { - throw Error(`collection ${name} already exists`); - } - this.collections_[name] = new this.modules.MapCollection(); - return this.collections_[name]; - } - - get (name) { - if ( ! this.collections_[name] ) { - throw Error(`collection ${name} does not exist`); - } - return this.collections_[name]; - } -} - -module.exports = { - RegistryService, -}; diff --git a/src/backend/src/services/RegistryService.test.ts b/src/backend/src/services/RegistryService.test.ts deleted file mode 100644 index ed6a6188a..000000000 --- a/src/backend/src/services/RegistryService.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { createTestKernel } from '../../tools/test.mjs'; -import { RegistryService } from './RegistryService'; - -describe('RegistryService', async () => { - - const testKernel = await createTestKernel({ - serviceMap: { - registry: RegistryService, - }, - initLevelString: 'init', - }); - - const registryService = testKernel.services!.get('registry') as RegistryService; - - it('should be instantiated', () => { - expect(registryService).toBeInstanceOf(RegistryService); - }); - - it('should register a collection', () => { - const collection = registryService.register_collection('test-collection'); - expect(collection).toBeDefined(); - }); - - it('should retrieve registered collection', () => { - registryService.register_collection('retrieve-collection'); - const collection = registryService.get('retrieve-collection'); - expect(collection).toBeDefined(); - }); - - it('should throw error when registering duplicate collection', () => { - registryService.register_collection('duplicate-collection'); - expect(() => { - registryService.register_collection('duplicate-collection'); - }).toThrow('collection duplicate-collection already exists'); - }); - - it('should throw error when getting non-existent collection', () => { - expect(() => { - registryService.get('non-existent-collection'); - }).toThrow('collection non-existent-collection does not exist'); - }); - - it('should allow setting values in collection', () => { - const collection = registryService.register_collection('value-collection'); - collection.set('key1', 'value1'); - expect(collection.get('key1')).toBe('value1'); - }); - - it('should allow checking existence in collection', () => { - const collection = registryService.register_collection('exists-collection'); - collection.set('existing-key', 'value'); - expect(collection.exists('existing-key')).toBeTruthy(); - expect(collection.exists('non-existing-key')).toBeFalsy(); - }); - - it('should allow deleting from collection', async () => { - const collection = registryService.register_collection('delete-collection'); - collection.set('delete-key', 'value'); - const res = collection.exists('delete-key'); - expect(collection.exists('delete-key')).toBeTruthy(); - collection.del('delete-key'); - expect(collection.exists('delete-key')).toBeFalsy(); - }); - - it('should support multiple independent collections', () => { - const collection1 = registryService.register_collection('coll1'); - const collection2 = registryService.register_collection('coll2'); - - collection1.set('key', 'value1'); - collection2.set('key', 'value2'); - - expect(collection1.get('key')).toBe('value1'); - expect(collection2.get('key')).toBe('value2'); - }); -}); diff --git a/src/backend/src/services/RequestMeasureService.js b/src/backend/src/services/RequestMeasureService.js deleted file mode 100644 index 9d8c1eb02..000000000 --- a/src/backend/src/services/RequestMeasureService.js +++ /dev/null @@ -1,24 +0,0 @@ -const BaseService = require('./BaseService'); - -class RequestMeasureService extends BaseService { - async '__on_install.middlewares.context-aware' (_, { app }) { - const svc_event = this.services.get('event'); - app.use(async (req, res, next) => { - next(); - const measurements = await req.measurements; - await svc_event.emit('request.measured', { - measurements, - req, - res, - ...(req.actor ? { actor: req.actor } : {}), - }); - }); - } - _init () { - // - } -} - -module.exports = { - RequestMeasureService, -}; diff --git a/src/backend/src/services/SUService.js b/src/backend/src/services/SUService.js deleted file mode 100644 index dc217bd67..000000000 --- a/src/backend/src/services/SUService.js +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { TeePromise } from '@heyputer/putility/src/libs/promise.js'; -import { Context } from '../util/context.js'; -import { Actor, UserActorType } from './auth/Actor.js'; -import BaseService from './BaseService.js'; - -/** -* "SUS"-Service (Super-User Service) -* Wherever you see this, be suspicious! (it escalates privileges) -* -* SUService is a specialized service that extends BaseService, -* designed to manage system user and actor interactions. It -* handles the initialization of system-level user and actor -* instances, providing methods to retrieve the system actor -* and perform actions with elevated privileges. -*/ -export class SUService extends BaseService { - /** - * Initializes the SUService instance, creating promises for system user - * and system actor. This method does not take any parameters and does - * not return a value. - */ - _construct () { - this.sys_user_ = new TeePromise(); - this.sys_actor_ = new TeePromise(); - } - - /** - * Resolves the system actor and user upon booting the service. - * This method fetches the system user and then creates an Actor - * instance for the user, resolving both promises. It's called - * automatically during the boot process. - * - * @async - * @returns {Promise} A promise that resolves when both the - * system user and actor have been set. - */ - async '__on_boot.consolidation' () { - const sys_user = await this.services.get('get-user').get_user({ username: 'system' }); - this.sys_user_.resolve(sys_user); - const sys_actor = new Actor({ - type: new UserActorType({ - user: sys_user, - }), - }); - this.sys_actor_.resolve(sys_actor); - } - - /** - * Retrieves the system user instance (resolved during consolidation). - * Prefer this over calling get_user({ username: 'system' }) to avoid re-fetching. - * - * @returns {Promise} A promise that resolves to the system user. - */ - async get_system_user () { - return this.sys_user_; - } - - /** - * Retrieves the system actor instance. - * - * This method returns a promise that resolves to the system actor. The actor - * represents the system user and is initialized during the boot process. - * - * @returns {Promise} A promise that resolves to the system actor. - */ - async get_system_actor () { - return this.sys_actor_; - } - - /** - * Super-User Do - * - * Performs an operation as a specified actor, allowing for callback execution - * within the context of that actor. If no actor is provided, the system actor - * is used by default. The adapted actor is then utilized to execute the callback - * under the appropriate user context. - * - * @overload - * @param {Actor} actor - The actor to perform the operation as. - * @param {() => Promise} callback - The callback function to execute as the specified actor. - * @returns {Promise} - * - * @overload - * @param {() => Promise} callback - The callback function to execute as the system actor. - * @returns {Promise} - * - * @template T - * @param {Actor|(() => Promise)} actor - The actor to perform the operation as, or the callback function if no actor is specified. - * @param {(() => Promise)} [callback] - The callback function to execute as the specified actor. - * @returns {Promise} A promise that resolves to the result of the callback function executed as the specified actor. - */ - async sudo (actor, callback) { - if ( ! callback ) { - callback = actor; - actor = await this.sys_actor_; - } - actor = Actor.adapt(actor); - return await Context.get().sub({ - actor, - user: actor.type.user, - }).arun(callback); - } -} \ No newline at end of file diff --git a/src/backend/src/services/ScriptService.js b/src/backend/src/services/ScriptService.js deleted file mode 100644 index 29b970fe6..000000000 --- a/src/backend/src/services/ScriptService.js +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const BaseService = require('./BaseService'); - -/** -* Class representing a service for managing and executing scripts. -* The ScriptService extends the BaseService and provides functionality -* to register scripts and execute them based on commands. -*/ -class BackendScript { - constructor (name, fn) { - this.name = name; - this.fn = fn; - } - - /** - * Executes the script function with the provided context and arguments. - * - * @async - * @param {Object} ctx - The context in which the script is run. - * @param {Array} args - The arguments to be passed to the script function. - * @returns {Promise} The result of the script function execution. - */ - async run (ctx, args) { - return await this.fn(ctx, args); - } - -} - -/** -* Class ScriptService extends BaseService to manage and execute scripts. -* It provides functionality to register scripts and run them through defined commands. -*/ -class ScriptService extends BaseService { - /** - * Initializes the service by registering script-related commands. - * - * This method retrieves the command service and sets up the commands - * related to script execution. It also defines a command handler that - * looks up and executes a script based on user input arguments. - * - * @async - * @function _init - */ - _construct () { - this.scripts = []; - } - - /** - * Initializes the script service by registering command handlers - * and setting up the environment for executing scripts. - * - * @async - * @returns {Promise} A promise that resolves when the initialization is complete. - */ - async _init () { - - } - - register (name, fn) { - this.scripts.push(new BackendScript(name, fn)); - } -} - -module.exports = { - ScriptService, - BackendScript, -}; diff --git a/src/backend/src/services/ScriptService.test.ts b/src/backend/src/services/ScriptService.test.ts deleted file mode 100644 index fd316aba2..000000000 --- a/src/backend/src/services/ScriptService.test.ts +++ /dev/null @@ -1,182 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { createTestKernel } from '../../tools/test.mjs'; -import { BackendScript, ScriptService } from './ScriptService'; - -describe('ScriptService', async () => { - const testKernel = await createTestKernel({ - serviceMap: { - 'script': ScriptService, - }, - initLevelString: 'construct', - }); - - const scriptService = testKernel.services!.get('script') as any; - - it('should be instantiated', () => { - expect(scriptService).toBeInstanceOf(ScriptService); - }); - - it('should have empty scripts array initially', () => { - expect(scriptService.scripts).toBeDefined(); - expect(Array.isArray(scriptService.scripts)).toBe(true); - }); - - it('should register a script', () => { - const initialLength = scriptService.scripts.length; - const scriptFn = async (ctx: any, args: any[]) => { - return 'result'; - }; - - scriptService.register('test-script', scriptFn); - - expect(scriptService.scripts.length).toBe(initialLength + 1); - }); - - it('should create BackendScript instance on registration', () => { - const service = testKernel.services!.get('script') as any; - const scriptFn = async (ctx: any, args: any[]) => {}; - - service.register('backend-script', scriptFn); - - const lastScript = service.scripts[service.scripts.length - 1]; - expect(lastScript).toBeInstanceOf(BackendScript); - expect(lastScript.name).toBe('backend-script'); - }); - - it('should store script function', () => { - const service = testKernel.services!.get('script') as any; - const scriptFn = async (ctx: any, args: any[]) => 'my-result'; - - service.register('fn-script', scriptFn); - - const lastScript = service.scripts[service.scripts.length - 1]; - expect(lastScript.fn).toBe(scriptFn); - }); - - it('should execute registered script', async () => { - const service = testKernel.services!.get('script') as any; - let executed = false; - - const scriptFn = async (ctx: any, args: any[]) => { - executed = true; - return 'executed'; - }; - - service.register('exec-script', scriptFn); - const script = service.scripts[service.scripts.length - 1]; - - const result = await script.run({}, []); - - expect(executed).toBe(true); - expect(result).toBe('executed'); - }); - - it('should pass context to script', async () => { - const service = testKernel.services!.get('script') as any; - let receivedCtx: any = null; - - const scriptFn = async (ctx: any, args: any[]) => { - receivedCtx = ctx; - }; - - service.register('ctx-script', scriptFn); - const script = service.scripts[service.scripts.length - 1]; - - const testCtx = { test: 'context' }; - await script.run(testCtx, []); - - expect(receivedCtx).toBe(testCtx); - }); - - it('should pass arguments to script', async () => { - const service = testKernel.services!.get('script') as any; - let receivedArgs: any[] = []; - - const scriptFn = async (ctx: any, args: any[]) => { - receivedArgs = args; - }; - - service.register('args-script', scriptFn); - const script = service.scripts[service.scripts.length - 1]; - - const testArgs = ['arg1', 'arg2', 'arg3']; - await script.run({}, testArgs); - - expect(receivedArgs).toEqual(testArgs); - }); - - it('should handle multiple script registrations', () => { - const service = testKernel.services!.get('script') as any; - - service.register('script1', async () => {}); - service.register('script2', async () => {}); - service.register('script3', async () => {}); - - const scriptNames = service.scripts.map((s: any) => s.name); - expect(scriptNames).toContain('script1'); - expect(scriptNames).toContain('script2'); - expect(scriptNames).toContain('script3'); - }); - - it('should allow scripts to return values', async () => { - const service = testKernel.services!.get('script') as any; - - service.register('return-script', async (ctx: any, args: any[]) => { - return { success: true, data: args[0] }; - }); - - const script = service.scripts[service.scripts.length - 1]; - const result = await script.run({}, ['test-data']); - - expect(result).toEqual({ success: true, data: 'test-data' }); - }); -}); - -describe('BackendScript', () => { - it('should create script with name and function', () => { - const fn = async () => {}; - const script = new BackendScript('test', fn); - - expect(script.name).toBe('test'); - expect(script.fn).toBe(fn); - }); - - it('should execute script function', async () => { - let executed = false; - const fn = async () => { executed = true; }; - const script = new BackendScript('exec', fn); - - await script.run({}, []); - - expect(executed).toBe(true); - }); - - it('should pass parameters to function', async () => { - let receivedCtx: any = null; - let receivedArgs: any = null; - - const fn = async (ctx: any, args: any) => { - receivedCtx = ctx; - receivedArgs = args; - }; - - const script = new BackendScript('params', fn); - const ctx = { test: true }; - const args = ['a', 'b']; - - await script.run(ctx, args); - - expect(receivedCtx).toBe(ctx); - expect(receivedArgs).toBe(args); - }); - - it('should return function result', async () => { - const fn = async () => 'result-value'; - const script = new BackendScript('return', fn); - - const result = await script.run({}, []); - - expect(result).toBe('result-value'); - }); -}); - diff --git a/src/backend/src/services/ServeGUIService.js b/src/backend/src/services/ServeGUIService.js deleted file mode 100644 index 1910e26d5..000000000 --- a/src/backend/src/services/ServeGUIService.js +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { static as static_ } from 'express'; -import { join } from 'path'; -import { catchAllRouter } from '../routers/_default.js'; -import { puterSiteMiddleware } from '../routers/hosting/puterSiteMiddleware.js'; -import BaseService from './BaseService.js'; -import { fileURLToPath } from 'url'; -import { dirname } from 'path'; -/** -* Class representing the ServeGUIService, which extends the BaseService. -* This service is responsible for setting up the GUI-related routes -* and serving static files for the Puter application. -*/ -export class ServeGUIService extends BaseService { - /** - * Handles the installation of GUI-related routes for the web server. - * This method sets up the routing for Puter site domains and other cases, - * including static file serving from the public directory. - * - * @async - * @returns {Promise} Resolves when routing is successfully set up. - */ - async '__on_install.routes-gui' () { - const { app } = this.services.get('web-server'); - - // is this a puter.site domain? - app.use(puterSiteMiddleware); - - // Router for all other cases - app.use(catchAllRouter); - - // Static files - const __filename = fileURLToPath(import.meta.url); - const __dirname = dirname(__filename); - - app.use(static_(join(__dirname, '../../public'))); - } -} diff --git a/src/backend/src/services/ServicePatch.js b/src/backend/src/services/ServicePatch.js deleted file mode 100644 index d557f22d6..000000000 --- a/src/backend/src/services/ServicePatch.js +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('@heyputer/putility'); - -/** -* Class ServicePatch -* -* This class extends the AdvancedBase class and provides functionality -* to apply patches to service methods dynamically. The patching mechanism -* ensures that the methods defined in the PATCH_METHODS static object -* are replaced with their respective patch implementations while maintaining -* a reference to the original service methods for potential fallback or -* additional processing. -*/ -class ServicePatch extends AdvancedBase { - patch ({ original_service }) { - const patch_methods = this._get_merged_static_object('PATCH_METHODS'); - for ( const k in patch_methods ) { - if ( typeof patch_methods[k] !== 'function' ) { - throw new Error(`Patch method ${k} to ${original_service.service_name} ` + - `from ${this.constructor.name} ` + - 'is not a function.'); - } - - const patch_method = patch_methods[k]; - - const patch_arguments = { - that: original_service, - original: original_service[k].bind(original_service), - }; - - original_service[k] = (...a) => { - return patch_method.call(this, patch_arguments, ...a); - }; - } - } -} - -module.exports = ServicePatch; diff --git a/src/backend/src/services/SessionService.js b/src/backend/src/services/SessionService.js deleted file mode 100644 index 1c85c2455..000000000 --- a/src/backend/src/services/SessionService.js +++ /dev/null @@ -1,465 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { redisClient } = require('../clients/redis/redisSingleton'); -const { UserRedisCacheSpace } = require('./UserRedisCacheSpace.js'); -const { get_user } = require('../helpers'); -const { v4: uuidv4 } = require('uuid'); -const SECOND = 1000; -const { BaseService } = require('./BaseService'); -const SESSION_CACHE_TTL_SECONDS = 5 * 60; -const SESSION_CACHE_KEY_PREFIX = 'session-cache'; -const SESSION_FLUSH_PENDING_SET_KEY = `${SESSION_CACHE_KEY_PREFIX}:flush-pending`; -const SESSION_USER_SESSIONS_KEY_PREFIX = `${SESSION_CACHE_KEY_PREFIX}:user-sessions`; -const SESSION_FLUSH_LOCK_KEY_PREFIX = `${SESSION_CACHE_KEY_PREFIX}:flush-lock`; -const SESSION_FLUSH_LOCK_TTL_SECONDS = 30; -const SESSION_FLUSH_INTERVAL_STEP_SECONDS = 5; -const SESSION_FLUSH_INTERVAL_MIN_STEPS = 1; -const SESSION_FLUSH_INTERVAL_MAX_STEPS = 12; - -/** - * This service is responsible for updating session activity - * timestamps and maintaining the number of active sessions. - */ -/** -* @class SessionService -* @description -* The SessionService class manages session-related operations within the Puter application. -* It handles the creation, retrieval, updating, and deletion of user sessions. This service: -* - Tracks session activity with timestamps. -* - Maintains a cache of active sessions. -* - Periodically updates session information in the database. -* - Ensures the integrity of session data across different parts of the application. -* - Provides methods to interact with sessions, including session creation, retrieval, and termination. -*/ -class SessionService extends BaseService { - getSessionCacheKey (uuid) { - return `${SESSION_CACHE_KEY_PREFIX}:${uuid}`; - } - - getSessionUserSetKey (userId) { - return `${SESSION_USER_SESSIONS_KEY_PREFIX}:${userId}`; - } - - getSessionFlushLockKey (uuid) { - return `${SESSION_FLUSH_LOCK_KEY_PREFIX}:${uuid}`; - } - - #getRandomFlushIntervalMs () { - const randomSteps = - Math.floor( - Math.random() * ( - SESSION_FLUSH_INTERVAL_MAX_STEPS - - SESSION_FLUSH_INTERVAL_MIN_STEPS - + 1 - ), - ) + SESSION_FLUSH_INTERVAL_MIN_STEPS; - return randomSteps * SESSION_FLUSH_INTERVAL_STEP_SECONDS * SECOND; - } - - #scheduleSessionFlushLoop () { - setTimeout(async () => { - try { - await this.#updateSessions(); - } catch (e) { - console.warn('session flush loop failed', { - reason: e?.message || String(e), - }); - } - this.#scheduleSessionFlushLoop(); - }, this.#getRandomFlushIntervalMs()); - } - - async cacheSession (session, options = {}) { - if ( ! session?.uuid ) return; - const flushState = options.flushState || 'unchanged'; - const normalizedSession = { - ...session, - flushPending: - flushState === 'pending' - ? true - : ( - flushState === 'flushed' - ? false - : !!session.flushPending - ), - }; - try { - await redisClient.set( - this.getSessionCacheKey(normalizedSession.uuid), - JSON.stringify(normalizedSession), - 'EX', - SESSION_CACHE_TTL_SECONDS, - ); - - if ( normalizedSession.user_id ) { - const userSessionSetKey = - this.getSessionUserSetKey(normalizedSession.user_id); - await redisClient.sadd(userSessionSetKey, normalizedSession.uuid); - await redisClient.expire(userSessionSetKey, SESSION_CACHE_TTL_SECONDS); - } - - if ( flushState === 'pending' ) { - await redisClient.sadd(SESSION_FLUSH_PENDING_SET_KEY, normalizedSession.uuid); - } else if ( flushState === 'flushed' ) { - await redisClient.srem(SESSION_FLUSH_PENDING_SET_KEY, normalizedSession.uuid); - } - } catch (e) { - console.warn('failed to cache session in redis', { - uuid: normalizedSession.uuid, - reason: e?.message || String(e), - }); - } - } - - async getCachedSession (uuid) { - let cachedSessionRaw; - try { - cachedSessionRaw = await redisClient.get(this.getSessionCacheKey(uuid)); - } catch (e) { - console.warn('failed to read session from redis', { - uuid, - reason: e?.message || String(e), - }); - return null; - } - if ( ! cachedSessionRaw ) return null; - - try { - const parsedSession = JSON.parse(cachedSessionRaw); - if ( !parsedSession || parsedSession.uuid !== uuid ) { - throw new Error('cached session payload mismatch'); - } - return parsedSession; - } catch { - await this.invalidateCachedSession(uuid); - return null; - } - } - - async invalidateCachedSession (uuid, userId) { - try { - await redisClient.del( - this.getSessionCacheKey(uuid), - ); - await redisClient.srem(SESSION_FLUSH_PENDING_SET_KEY, uuid); - if ( userId ) { - await redisClient.srem(this.getSessionUserSetKey(userId), uuid); - } - } catch (e) { - console.warn('failed to delete cached session from redis', { - uuid, - reason: e?.message || String(e), - }); - } - } - - /** - * Initializes the session storage by setting up the database connection - * and starting a periodic session update interval. - * - * @async - * @memberof SessionService - * @method _init - */ - async _init () { - this.db = await this.services.get('database').get(); - this.#scheduleSessionFlushLoop(); - } - - /** - * Creates a new session for the specified user and records metadata about - * the requestor. - * - * @async - * @returns {Promise} A new session object - */ - async create_session (user, meta) { - const unix_ts = Math.floor(Date.now() / 1000); - - meta = { - // clone - ...(meta || {}), - }; - meta.created = new Date().toISOString(); - meta.created_unix = unix_ts; - const uuid = uuidv4(); - await this.db.write( - 'INSERT INTO `sessions` ' + - '(`uuid`, `user_id`, `meta`, `last_activity`, `created_at`) ' + - 'VALUES (?, ?, ?, ?, ?)', - [uuid, user.id, JSON.stringify(meta), unix_ts, unix_ts], - ); - const session = { - last_touch: Date.now(), - last_store: Date.now(), - uuid, - user_uid: user.uuid, - user_id: user.id, - meta, - flushPending: false, - }; - await this.cacheSession(session); - - return session; - } - - /** - * Retrieves a session by its UUID, updates the session's last touch timestamp, - * and prepares the session data for external use by removing internal values. - * - * @param {string} uuid - The UUID of the session to retrieve. - * @returns {Object|undefined} The session object with internal values removed, or undefined if the session does not exist. - */ - async #getSession (uuid) { - let session = await this.getCachedSession(uuid); - if ( session ) { - session.last_touch = Date.now(); - return session; - } - ;[session] = await this.db.tryHardRead( - 'SELECT * FROM `sessions` WHERE `uuid` = ? LIMIT 1', - [uuid], - ); - if ( ! session ) return; - session.last_store = Date.now(); - - // MariaDB and SQLite store JSON as string. - if ( typeof session.meta === 'string' ) { - session.meta = JSON.parse(session.meta ?? '{}'); - } - - const user = await get_user({ id: session.user_id }); - session.user_uid = user?.uuid; - return session; - } - /** - * Retrieves a session by its UUID, updates its last touch time, and prepares it for external use. - * @param {string} uuid - The unique identifier for the session to retrieve. - * @returns {Promise} The session object with internal values removed, or undefined if not found. - */ - async getSession (uuid) { - const session = await this.#getSession(uuid); - if ( session ) { - session.last_touch = Date.now(); - session.meta = { - ...(session.meta || {}), - last_activity: (new Date()).toISOString(), - }; - await this.cacheSession(session, { - flushState: 'pending', - }); - } - return this.#removeInternalValues(session); - } - - #removeInternalValues (session) { - if ( session === undefined ) return; - - const copy = { - ...session, - }; - delete copy.last_touch; - delete copy.last_store; - delete copy.user_id; - delete copy.flushPending; - return copy; - } - - async get_user_sessions (user) { - if ( ! user?.id ) return []; - - let sessionUuids; - try { - sessionUuids = await redisClient.smembers( - this.getSessionUserSetKey(user.id), - ); - } catch (e) { - console.warn('failed to read user session set from redis', { - userId: user.id, - reason: e?.message || String(e), - }); - return []; - } - - if ( !Array.isArray(sessionUuids) || sessionUuids.length === 0 ) { - return []; - } - - const sessions = []; - for ( const sessionUuid of sessionUuids ) { - const session = await this.getCachedSession(sessionUuid); - if ( !session || session.user_id !== user.id ) { - await redisClient.srem(this.getSessionUserSetKey(user.id), sessionUuid); - continue; - } - sessions.push(session); - } - - return sessions.map(this.#removeInternalValues.bind(this)); - } - - /** - * Removes a session from Redis-backed cache state and the database. - * - * @param {string} uuid - The UUID of the session to remove. - * @returns {Promise} A promise that resolves to the result of the database write operation. - */ - async remove_session (uuid) { - const cachedSession = await this.getCachedSession(uuid); - const [dbSession] = await this.db.tryHardRead( - 'SELECT `user_id` FROM `sessions` WHERE `uuid` = ? LIMIT 1', - [uuid], - ); - await this.invalidateCachedSession(uuid, cachedSession?.user_id ?? dbSession?.user_id); - return await this.db.write( - 'DELETE FROM `sessions` WHERE `uuid` = ?', - [uuid], - ); - } - - async #updateSessions () { - const now = Date.now(); - let pendingSessionUuids; - try { - pendingSessionUuids = await redisClient.smembers(SESSION_FLUSH_PENDING_SET_KEY); - } catch (e) { - console.warn('failed to read pending session flush set from redis', { - reason: e?.message || String(e), - }); - return; - } - if ( !Array.isArray(pendingSessionUuids) || pendingSessionUuids.length === 0 ) { - return; - } - - const userUpdates = {}; - - for ( const sessionUuid of pendingSessionUuids ) { - const lockKey = this.getSessionFlushLockKey(sessionUuid); - let lockAcquired = false; - try { - lockAcquired = await redisClient.set( - lockKey, - '1', - 'EX', - SESSION_FLUSH_LOCK_TTL_SECONDS, - 'NX', - ); - if ( ! lockAcquired ) continue; - - const session = await this.getCachedSession(sessionUuid); - if ( ! session ) { - await redisClient.srem(SESSION_FLUSH_PENDING_SET_KEY, sessionUuid); - continue; - } - if ( ! session.flushPending ) { - await redisClient.srem(SESSION_FLUSH_PENDING_SET_KEY, sessionUuid); - continue; - } - - const lastTouch = typeof session.last_touch === 'number' - ? session.last_touch - : now; - const unixTs = Math.floor(lastTouch / 1000); - session.meta = { - ...(session.meta || {}), - last_activity: (new Date(lastTouch)).toISOString(), - }; - - const { anyRowsAffected } = await this.db.write( - 'UPDATE `sessions` ' + - 'SET `meta` = ?, `last_activity` = ? ' + - 'WHERE `uuid` = ? AND (`last_activity` IS NULL OR `last_activity` < ?)', - [JSON.stringify(session.meta), unixTs, session.uuid, unixTs], - ); - - if ( ! anyRowsAffected ) { - const [existingSession] = await this.db.tryHardRead( - 'SELECT `uuid` FROM `sessions` WHERE `uuid` = ? LIMIT 1', - [session.uuid], - ); - if ( ! existingSession ) { - await this.invalidateCachedSession(session.uuid, session.user_id); - continue; - } - } - - session.last_store = now; - await this.cacheSession({ - ...session, - flushPending: false, - }, { - flushState: 'flushed', - }); - - if ( - session.user_id && - ( - !userUpdates[session.user_id] - || userUpdates[session.user_id] < lastTouch - ) - ) { - userUpdates[session.user_id] = lastTouch; - } - } catch (e) { - console.warn('failed to flush session update to db', { - uuid: sessionUuid, - reason: e?.message || String(e), - }); - } finally { - if ( lockAcquired ) { - await redisClient.del(lockKey); - } - } - } - - for ( const [userIdRaw, lastTouch] of Object.entries(userUpdates) ) { - const userId = Number(userIdRaw); - const sql_ts = (date => - `${date.toISOString().split('T')[0] } ${ - date.toTimeString().split(' ')[0]}` - )(new Date(lastTouch)); - - await this.db.write( - 'UPDATE `user` ' + - 'SET `last_activity_ts` = ? ' + - 'WHERE `id` = ? AND (`last_activity_ts` IS NULL OR `last_activity_ts` < ?) LIMIT 1', - [sql_ts, userId, sql_ts], - ); - const cachedUser = await redisClient.get(UserRedisCacheSpace.key('id', userId)); - if ( cachedUser ) { - try { - const user = JSON.parse(cachedUser); - if ( - !user.last_activity_ts || - user.last_activity_ts < sql_ts - ) { - user.last_activity_ts = sql_ts; - UserRedisCacheSpace.setUser(user); - } - } catch ( e ) { - console.warn(e); - // ignore malformed cache entries - } - } - } - } -} - -module.exports = { SessionService }; diff --git a/src/backend/src/services/SessionService.test.js b/src/backend/src/services/SessionService.test.js deleted file mode 100644 index c5ee06e9d..000000000 --- a/src/backend/src/services/SessionService.test.js +++ /dev/null @@ -1,123 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { createTestKernel } from '../../tools/test.mjs'; -import { SessionService } from './SessionService.js'; -import { tmp_provide_services } from '../helpers.js'; -import { redisClient } from '../clients/redis/redisSingleton.js'; - -describe('SessionService', async () => { - const testKernel = await createTestKernel({ - initLevelString: 'init', - testCore: true, - serviceMap: { - session: SessionService, - }, - serviceConfigOverrideMap: { - database: { - path: ':memory:', - }, - }, - }); - - await tmp_provide_services(testKernel.services); - - const sessionService = testKernel.services.get('session'); - const db = testKernel.services.get('database').get('write', 'session-test'); - - const makeUnique = (prefix) => `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; - - const createUser = async () => { - const userUuid = makeUnique('user'); - const username = makeUnique('session-user'); - await db.write( - 'INSERT INTO `user` (`uuid`, `username`) VALUES (?, ?)', - [userUuid, username], - ); - const [user] = await db.read( - 'SELECT * FROM `user` WHERE `uuid` = ? LIMIT 1', - [userUuid], - ); - return user; - }; - - const clearSessionState = async (sessionUuid, userId) => { - if ( sessionUuid ) { - await redisClient.del(sessionService.getSessionCacheKey(sessionUuid)); - await redisClient.srem('session-cache:flush-pending', sessionUuid); - if ( userId ) { - await redisClient.srem( - sessionService.getSessionUserSetKey(userId), - sessionUuid, - ); - } - await db.write('DELETE FROM `sessions` WHERE `uuid` = ?', [sessionUuid]); - } - }; - - it('caches sessions in redis on create with five-minute ttl', async () => { - const user = await createUser(); - const session = await sessionService.create_session(user, {}); - try { - const cacheKey = sessionService.getSessionCacheKey(session.uuid); - const cached = await redisClient.get(cacheKey); - expect(cached).toBeTruthy(); - expect(JSON.parse(cached).uuid).toBe(session.uuid); - expect(await redisClient.ttl(cacheKey)).toBeGreaterThan(0); - expect(await redisClient.ttl(cacheKey)).toBeLessThanOrEqual(300); - const cachedUserSessionUuids = await redisClient.smembers( - sessionService.getSessionUserSetKey(user.id), - ); - expect(cachedUserSessionUuids).toContain(session.uuid); - } finally { - await clearSessionState(session.uuid, user.id); - } - }); - - it('loads sessions from redis cache before db on read', async () => { - const user = await createUser(); - const session = await sessionService.create_session(user, {}); - const dbReadSpy = vi.spyOn(sessionService.db, 'tryHardRead'); - try { - const loaded = await sessionService.getSession(session.uuid); - expect(dbReadSpy).not.toHaveBeenCalled(); - expect(loaded.user_uid).toBe(user.uuid); - const pendingSessions = await redisClient.smembers('session-cache:flush-pending'); - expect(pendingSessions).toContain(session.uuid); - } finally { - dbReadSpy.mockRestore(); - await clearSessionState(session.uuid, user.id); - } - }); - - it('invalidates redis cache when removing session', async () => { - const user = await createUser(); - const session = await sessionService.create_session(user, {}); - await sessionService.remove_session(session.uuid); - - const [dbSession] = await db.read( - 'SELECT * FROM `sessions` WHERE `uuid` = ? LIMIT 1', - [session.uuid], - ); - expect(await redisClient.get(`session-cache:${session.uuid}`)).toBeNull(); - expect(dbSession).toBeUndefined(); - const pendingSessions = await redisClient.smembers('session-cache:flush-pending'); - expect(pendingSessions).not.toContain(session.uuid); - const cachedUserSessionUuids = await redisClient.smembers( - sessionService.getSessionUserSetKey(user.id), - ); - expect(cachedUserSessionUuids).not.toContain(session.uuid); - }); - - it('loads session user uid using object lookup options', async () => { - const user = await createUser(); - const session = await sessionService.create_session(user, {}); - - await redisClient.del(`session-cache:${session.uuid}`); - - const loadedSession = await sessionService.getSession(session.uuid); - try { - expect(loadedSession.user_uid).toBe(user.uuid); - } finally { - await clearSessionState(session.uuid, user.id); - } - }); -}); diff --git a/src/backend/src/services/ShareService.js b/src/backend/src/services/ShareService.js deleted file mode 100644 index e2284f6f4..000000000 --- a/src/backend/src/services/ShareService.js +++ /dev/null @@ -1,398 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../api/APIError'); -const eggspress = require('../api/eggspress'); -const { get_user } = require('../helpers'); -const configurable_auth = require('../middleware/configurable_auth'); -const { Actor, UserActorType } = require('./auth/Actor'); -const BaseService = require('./BaseService'); -const { DB_WRITE } = require('./database/consts'); -const { UsernameNotifSelector } = require('./NotificationService'); - -class ShareService extends BaseService { - static MODULES = { - uuidv4: require('uuid').v4, - validator: require('validator'), - express: require('express'), - }; - - async _init () { - this.db = await this.services.get('database').get(DB_WRITE, 'share'); - - // registry "share" as a feature flag so gui is informed - // about whether or not a user has access to this feature - const svc_featureFlag = this.services.get('feature-flag'); - svc_featureFlag.register('share', { - $: 'function-flag', - fn: async ({ actor }) => { - const user = actor.type.user ?? null; - if ( ! user ) { - throw new Error('expected user'); - } - return !!user.email_confirmed; - }, - }); - - const svc_event = this.services.get('event'); - svc_event.on('user.email-confirmed', async (_, { user_uid, email }) => { - const user = await get_user({ uuid: user_uid }); - const relevant_shares = await this.db.read( - 'SELECT * FROM share WHERE recipient_email = ?', - [email], - ); - - for ( const share of relevant_shares ) { - if ( !share.data || typeof (share.data) === 'string' ) { - share.data = JSON.parse(share.data || '{}'); - } - - const issuer_user = await get_user({ - id: share.issuer_user_id, - }); - - if ( ! issuer_user ) { - continue; - } - - const issuer_actor = await Actor.create(UserActorType, { - user: issuer_user, - }); - - const svc_acl = this.services.get('acl'); - - for ( const permission of share.data.permissions ) { - await svc_acl.set_user_user(issuer_actor, user.username, permission, undefined, { only_if_higher: true }); - } - - await this.db.write( - 'DELETE FROM share WHERE uid = ?', - [share.uid], - ); - } - }); - } - - '__on_install.routes' (_, { app }) { - this.install_sharelink_endpoints({ app }); - this.install_share_endpoint({ app }); - } - - /** - * This method is responsible for processing the share link application request. - * It checks if the share token is valid and if the user making the request is the intended recipient. - * If both conditions are met, it grants the requested permissions to the user and deletes the share from the database. - * - * @param {Object} req - Express request object. - * @param {Object} res - Express response object. - * @returns {Promise} - */ - install_sharelink_endpoints ({ app }) { - // track: scoping iife - const router = (() => { - const require = this.require; - const express = require('express'); - return express.Router(); - })(); - - app.use('/sharelink', router); - - const svc_share = this.services.get('share'); - const svc_token = this.services.get('token'); - - router.use(eggspress('/check', { - allowedMethods: ['POST'], - }, async (req, res) => { - // Potentially confusing: - // The "share token" and "share cookie token" are different! - // -> "share token" is from the email link; - // it has a longer expiry time and can be used again - // if the share session expires. - // -> "share cookie token" lets the backend know it - // should grant permissions when the correct user - // is logged in. - - const share_token = req.body.token; - - if ( ! share_token ) { - throw APIError.create('field_missing', null, { - key: 'token', - }); - } - - const decoded = await svc_token.verify('share', share_token); - console.log('decoded?', decoded); - if ( decoded.$ !== 'token:share' ) { - throw APIError.create('invalid_token'); - } - - const share = await svc_share.get_share({ - uid: decoded.uid, - }); - - if ( ! share ) { - throw APIError.create('invalid_token'); - } - - res.json({ - $: 'api:share', - uid: share.uid, - email: share.recipient_email, - }); - })); - - router.use(eggspress('/apply', { - allowedMethods: ['POST'], - mw: [configurable_auth()], - }, async (req, res) => { - const share_uid = req.body.uid; - - const share = await svc_share.get_share({ - uid: share_uid, - }); - - if ( ! share ) { - throw APIError.create('share_expired'); - } - - if ( !share.data || typeof (share.data) === 'string' ) { - share.data = JSON.parse(share.data || '{}'); - } - - const actor = Actor.adapt(req.actor ?? req.user); - if ( ! actor ) { - // this shouldn't happen; auth should catch it - throw new Error('actor missing'); - } - - if ( ! actor.type.user.email_confirmed ) { - throw APIError.create('email_must_be_confirmed'); - } - - if ( actor.type.user.email !== share.recipient_email ) { - throw APIError.create('can_not_apply_to_this_user'); - } - - const issuer_user = await get_user({ - id: share.issuer_user_id, - }); - - if ( ! issuer_user ) { - throw APIError.create('share_expired'); - } - - const issuer_actor = await Actor.create(UserActorType, { - user: issuer_user, - }); - - const svc_permission = this.services.get('permission'); - - for ( const permission of share.data.permissions ) { - await svc_permission.grant_user_user_permission( - issuer_actor, - actor.type.user.username, - permission, - ); - } - - await this.db.write( - 'DELETE FROM share WHERE uid = ?', - [share.uid], - ); - - res.json({ - $: 'api:status-report', - status: 'success', - }); - })); - - router.use(eggspress('/request', { - allowedMethods: ['POST'], - mw: [configurable_auth()], - }, async (req, res) => { - const share_uid = req.body.uid; - - const share = await svc_share.get_share({ - uid: share_uid, - }); - - // track: null check before processing - if ( ! share ) { - throw APIError.create('share_expired'); - } - - if ( !share.data || typeof (share.data) === 'string' ) { - share.data = JSON.parse(share.data || '{}'); - } - - const actor = Actor.adapt(req.actor ?? req.user); - if ( ! actor ) { - // this shouldn't happen; auth should catch it - throw new Error('actor missing'); - } - - // track: opposite condition of sibling - // :: sibling: /apply endpoint - if ( - actor.type.user.email_confirmed && - actor.type.user.email === share.recipient_email - ) { - throw APIError.create('no_need_to_request'); - } - - const issuer_user = await get_user({ - id: share.issuer_user_id, - }); - - if ( ! issuer_user ) { - throw APIError.create('share_expired'); - } - - const svc_notification = this.services.get('notification'); - svc_notification.notify( - UsernameNotifSelector(issuer_user.username), - { - source: 'sharing', - title: `User ${actor.type.user.username} is ` + - `trying to open a share you sent to ${ - share.recipient_email}`, - template: 'user-requesting-share', - fields: { - username: actor.type.user.username, - intended_recipient: share.recipient_email, - permissions: share.data.permissions, - }, - }, - ); - res.json({ - $: 'api:status-report', - status: 'success', - }); - })); - } - - install_share_endpoint ({ app }) { - // track: scoping iife - const router = (() => { - const require = this.require; - const express = require('express'); - return express.Router(); - })(); - - app.use('/share', router); - - const share_sequence = require('../structured/sequence/share.js'); - router.use(eggspress('/', { - allowedMethods: ['POST'], - mw: [ - configurable_auth(), - // featureflag({ feature: 'share' }), - ], - }, async (req, res) => { - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check('verify-pass-recovery-token') ) { - return res.status(429).send('Too many requests.'); - } - - const actor = req.actor; - if ( ! (actor.type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - if ( ! actor.type.user.email_confirmed ) { - throw APIError.create('email_must_be_confirmed', null, { - action: 'share something', - }); - } - - return await share_sequence.call(this, { - actor, req, res, - }); - })); - } - - async get_share ({ uid }) { - const [share] = await this.db.read( - 'SELECT * FROM share WHERE uid = ?', - [uid], - ); - - return share; - } - - /** - * Method to handle the creation of a new share - * - * This method creates a new share and saves it to the database. - * It takes three parameters: the issuer of the share, the recipient's email address, and the data to be shared. - * The method returns the UID of the created share. - * - * @param {Actor} issuer - The actor who is creating the share - * @param {string} email - The email address of the recipient - * @param {object} data - The data to be shared - * @returns {string} - The UID of the created share - */ - async create_share ({ - issuer, - email, - data, - }) { - const require = this.require; - const validator = require('validator'); - - // track: type check - if ( typeof email !== 'string' ) { - throw new Error('email must be a string'); - } - // track: type check - if ( !data || typeof data !== 'object' || Array.isArray(data) ) { - throw new Error('data must be an object'); - } - - // track: adapt - issuer = Actor.adapt(issuer); - // track: type check - if ( ! (issuer instanceof Actor) ) { - throw new Error('expected issuer to be Actor'); - } - - // track: actor type - if ( ! (issuer.type instanceof UserActorType) ) { - throw new Error('only users are allowed to create shares'); - } - - if ( ! validator.isEmail(email) ) { - throw new Error('invalid email'); - } - - const uuid = this.modules.uuidv4(); - - await this.db.write( - 'INSERT INTO `share` ' + - '(`uid`, `issuer_user_id`, `recipient_email`, `data`) ' + - 'VALUES (?, ?, ?, ?)', - [uuid, issuer.type.user.id, email, JSON.stringify(data)], - ); - - return uuid; - } -} - -module.exports = { - ShareService, -}; diff --git a/src/backend/src/services/StrategizedService.js b/src/backend/src/services/StrategizedService.js deleted file mode 100644 index ed09e3399..000000000 --- a/src/backend/src/services/StrategizedService.js +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { quot } = require('@heyputer/putility').libs.string; - -/** - * An abstract service used to strategize services in confirguration, - * primarily used for thumbnail service selection, but it could be used - * to strategize any service. - */ -class StrategizedService { - constructor (service_resources, ...a) { - const { my_config, args, name } = service_resources; - - const key = args.strategy_key; - if ( !args.default_strategy && !my_config.hasOwnProperty(key) ) { - this.initError = new Error(`Must specify ${quot(key)} for service ${quot(name)}.`); - return; - } - - if ( ! args.hasOwnProperty('strategies') ) { - throw new Error('strategies not defined in service args'); - } - - const strategy_key = my_config[key] ?? args.default_strategy; - if ( ! args.strategies.hasOwnProperty(strategy_key) ) { - this.initError = new Error(`Invalid ${key} ${quot(strategy_key)} for service ${quot(name)}.`); - return; - } - const [cls, cls_args] = args.strategies[strategy_key]; - - const cls_resources = { - ...service_resources, - args: cls_args, - }; - this.strategy = new cls(cls_resources, ...a); - - return this.strategy; - } - - /** - * This method must be implemented by the delegate or an error will be thrown - */ - async init () { - throw this.initError; - } - - async construct () { - } -} - -module.exports = { - StrategizedService, -}; diff --git a/src/backend/src/services/SystemDataService.js b/src/backend/src/services/SystemDataService.js deleted file mode 100644 index b0c982557..000000000 --- a/src/backend/src/services/SystemDataService.js +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { LLRead } = require('../deprecated/filesystem/ll_operations/ll_read'); -const { Context } = require('../util/context'); -const { stream_to_buffer } = require('../util/streamutil'); -const BaseService = require('./BaseService'); - -/** -* The `SystemDataService` class extends `BaseService` to provide functionality for interpreting and dereferencing data structures. -* This service handles the recursive interpretation of complex data types including objects and arrays, as well as dereferencing -* JSON-address pointers to fetch and process data from file system nodes. It is designed to: -* - Interpret nested structures by recursively calling itself for each nested element. -* - Dereference JSON pointers, which involves reading from the filesystem, parsing JSON, and optionally selecting nested properties. -* - Manage different data types encountered during operations, ensuring proper handling or throwing errors for unrecognized types. -*/ -class SystemDataService extends BaseService { - async _init () { - } - - /** - * Interprets data, dereferencing JSON-address pointers if necessary. - * - * @param {Object|Array|string|number|boolean|null} data - The data to interpret. - * Can be an object, array, or primitive value. - * @returns {Promise} The interpreted data. - * For objects and arrays, this method recursively interprets each element. - * For special objects with a '$' property, it performs dereferencing. - */ - async interpret (data) { - if ( data?.$ ) { - return await this.#dereference(data); - } - - if ( Array.isArray(data) ) { - const new_a = []; - for ( const v of data ) { - new_a.push(await this.interpret(v)); - } - return new_a; - } - if ( data && typeof data === 'object' ) { - const new_o = {}; - for ( const k in data ) { - new_o[k] = await this.interpret(data[k]); - } - return new_o; - } - - return data; - } - - /** - * De-references a JSON address by reading the respective file and parsing - * the JSON contents. - * - * @param {Object|Array|*} data - The data to interpret, which can be of any type. - * @returns {Promise<*>} The interpreted result, which could be a primitive, object, or array. - */ - async #dereference (data) { - const svc_fs = this.services.get('filesystem'); - if ( data.$ === 'json-address' ) { - const node = await svc_fs.node(data.path); - const ll_read = new LLRead(); - const stream = await ll_read.run({ - actor: Context.get('actor'), - fsNode: node, - }); - const buffer = await stream_to_buffer(stream); - const json = buffer.toString('utf8'); - let result = JSON.parse(json); - result = await this.interpret(result); - if ( data.selector ) { - const parts = data.selector.split('.'); - for ( const part of parts ) { - result = result[part]; - } - } - return result; - } - throw new Error(`unrecognized data type: ${data.$}`); - } -} - -module.exports = { - SystemDataService, -}; diff --git a/src/backend/src/services/SystemValidationService.js b/src/backend/src/services/SystemValidationService.js deleted file mode 100644 index 12072c3b5..000000000 --- a/src/backend/src/services/SystemValidationService.js +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const BaseService = require('./BaseService'); - -/** -* SystemValidationService class. -* -* This class extends BaseService and is responsible for handling system validation -* and marking the server as invalid. It includes methods for reporting invalid -* system states, raising alarms, and managing the server's response in different -* environments (e.g., development and production). -* -* @class -* @extends BaseService -*/ -class SystemValidationService extends BaseService { - /** - * Marks the server as being in an invalid state. - * - * This method is used to indicate that the server is in a serious error state. It will attempt - * to alert the user and then shut down the server after 25 minutes. - * - * @param {string} message - A description of why mark_invalid was called. - * @param {Error} [source] - The error that caused the invalid state, if any. - */ - async mark_invalid (message, source) { - if ( ! source ) source = new Error('no source error'); - - // The system is in an invalid state. The server will do whatever it - // can to get our attention, and then it will shut down. - if ( ! this.errors ) { - console.error('SystemValidationService is trying to mark the system as invalid, but the error service is not available.', - message, - source); - - // We can't do anything else. The server will crash. - throw new Error('SystemValidationService is trying to mark the system as invalid, but the error service is not available.'); - } - - this.errors.report('INVALID SYSTEM STATE', { - source, - message, - trace: true, - alarm: true, - }); - - // If we're in dev mode... - if ( this.global_config.env === 'dev' ) { - const realConsole = globalThis.original_console_object ?? console; - realConsole.error('\n*** SYSTEM IS IN AN INVALID STATE ***'); - realConsole.error(message); - realConsole.error('Resolve the error above to clear this state.\n'); - return; - } - - // Raise further alarms if the system keeps running - for ( let i = 0; i < 5; i++ ) { - // After 5 minutes, raise another alarm - await new Promise(rslv => setTimeout(rslv, 60 * 5000)); - this.errors.report(`INVALID SYSTEM STATE (Reminder ${i + 1})`, { - source, - message, - trace: true, - alarm: true, - }); - } - } -} - -module.exports = { SystemValidationService }; diff --git a/src/backend/src/services/SystemValidationService.test.ts b/src/backend/src/services/SystemValidationService.test.ts deleted file mode 100644 index 5be5912e8..000000000 --- a/src/backend/src/services/SystemValidationService.test.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { createTestKernel } from '../../tools/test.mjs'; -import { SystemValidationService } from './SystemValidationService'; - -describe('SystemValidationService', async () => { - const testKernel = await createTestKernel({ - serviceMap: { - 'system-validation': SystemValidationService, - }, - initLevelString: 'init', - }); - - const systemValidationService = testKernel.services!.get('system-validation') as any; - - it('should be instantiated', () => { - expect(systemValidationService).toBeInstanceOf(SystemValidationService); - }); - - it('should have mark_invalid method', () => { - expect(systemValidationService.mark_invalid).toBeDefined(); - expect(typeof systemValidationService.mark_invalid).toBe('function'); - }); - - it('should handle mark_invalid in dev environment', async () => { - // Set up dev environment - const originalEnv = systemValidationService.global_config?.env; - if (systemValidationService.global_config) { - systemValidationService.global_config.env = 'dev'; - } - - // Mock the error service - const mockReport = vi.fn(); - systemValidationService.errors = { - report: mockReport, - }; - - try { - await systemValidationService.mark_invalid('test message', new Error('test error')); - - // Verify error was reported - expect(mockReport).toHaveBeenCalledWith('INVALID SYSTEM STATE', expect.objectContaining({ - message: 'test message', - trace: true, - alarm: true, - })); - } finally { - // Restore original environment - if (systemValidationService.global_config) { - systemValidationService.global_config.env = originalEnv; - } - } - }); - - it('should create source error if not provided', async () => { - const originalEnv = systemValidationService.global_config?.env; - if (systemValidationService.global_config) { - systemValidationService.global_config.env = 'dev'; - } - - const mockReport = vi.fn(); - systemValidationService.errors = { - report: mockReport, - }; - - try { - await systemValidationService.mark_invalid('test without source'); - - expect(mockReport).toHaveBeenCalledWith('INVALID SYSTEM STATE', expect.objectContaining({ - source: expect.any(Error), - })); - } finally { - if (systemValidationService.global_config) { - systemValidationService.global_config.env = originalEnv; - } - } - }); - - it('should report with correct parameters', async () => { - const originalEnv = systemValidationService.global_config?.env; - if (systemValidationService.global_config) { - systemValidationService.global_config.env = 'dev'; - } - - const mockReport = vi.fn(); - systemValidationService.errors = { - report: mockReport, - }; - - try { - const testError = new Error('specific error'); - await systemValidationService.mark_invalid('specific message', testError); - - expect(mockReport).toHaveBeenCalledWith('INVALID SYSTEM STATE', { - source: testError, - message: 'specific message', - trace: true, - alarm: true, - }); - } finally { - if (systemValidationService.global_config) { - systemValidationService.global_config.env = originalEnv; - } - } - }); -}); diff --git a/src/backend/src/services/TestService.js b/src/backend/src/services/TestService.js deleted file mode 100644 index 3ead9a358..000000000 --- a/src/backend/src/services/TestService.js +++ /dev/null @@ -1,12 +0,0 @@ -const BaseService = require('./BaseService'); - -/** - * TestService is a service for testing in the sense that it is a service - * that exists for the purpose of being testing or to be used for testing - * purposes. However, TestService is not a service that's meant to hold - * utility functions for testing. - */ -class TestService extends BaseService { -} - -module.exports = { TestService }; diff --git a/src/backend/src/services/TestService.test.ts b/src/backend/src/services/TestService.test.ts deleted file mode 100644 index 5a5384220..000000000 --- a/src/backend/src/services/TestService.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { TestKernel } from '../../tools/test.mjs'; -import { Core2Module } from '../modules/core/Core2Module.js'; -import { WebModule } from '../modules/web/WebModule.js'; -import { TestService } from './TestService.js'; - -describe('testing with TestKernel', () => { - it('can load TestService within TestKernel', () => { - const testKernel = new TestKernel(); - testKernel.add_module({ - install: (context) => { - const services = context.get('services'); - services.registerService('test', TestService); - }, - }); - - testKernel.boot(); - - const svc_test = testKernel.services?.get('test'); - - expect(svc_test).toBeInstanceOf(TestService); - }); - it('can load CoreModule within TestKernel', async () => { - const testKernel = new TestKernel(); - testKernel.add_module(new Core2Module()); - testKernel.add_module(new WebModule()); - testKernel.boot(); - - const { services } = testKernel; - await services?.ready; - - const svc_webServer = services?.get('web-server'); - - expect(svc_webServer.constructor.name).toBe('WebServerService'); - }); -}); diff --git a/src/backend/src/services/User.d.ts b/src/backend/src/services/User.d.ts deleted file mode 100644 index d10076398..000000000 --- a/src/backend/src/services/User.d.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { SUB_POLICIES } from './MeteringService/subPolicies'; - -export interface IUser { - id: number; - uuid: string; - username: string; - email?: string; - free_storage?: number | string | null; - actual_free_storage?: number | string | null; - subscription?: (typeof SUB_POLICIES)[number]['id'] & { - active: boolean; - tier: string; - }; - metadata?: Record & { hasDevAccountAccess?: boolean }; - repscore: number; - email_confirmed: 1 | 0; - requires_email_confirmation: 1 | 0; -} diff --git a/src/backend/src/services/UserRedisCacheSpace.js b/src/backend/src/services/UserRedisCacheSpace.js deleted file mode 100644 index 160b52837..000000000 --- a/src/backend/src/services/UserRedisCacheSpace.js +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { redisClient } from '../clients/redis/redisSingleton.js'; -import { deleteRedisKeys } from '../clients/redis/deleteRedisKeys.js'; - -const userKeyPrefix = 'users'; -const defaultUserIdProperties = ['username', 'uuid', 'email', 'id', 'referral_code']; -const DEFAULT_USER_CACHE_TTL_SECONDS = 15 * 60; - -const safeParseJson = (value, fallback = null) => { - if ( value === null || value === undefined ) return fallback; - try { - return JSON.parse(value); - } catch (e) { - return fallback; - } -}; - -const setKey = async (key, value, { ttlSeconds } = {}) => { - if ( ttlSeconds ) { - await redisClient.set(key, value, 'EX', ttlSeconds); - return; - } - await redisClient.set(key, value); -}; - -const userCacheKey = (prop, value) => `${userKeyPrefix}:${prop}:${value}`; - -const UserRedisCacheSpace = { - key: userCacheKey, - keysForUser: (user, props = defaultUserIdProperties) => { - if ( ! user ) return []; - return props - .filter(prop => user[prop] !== undefined && user[prop] !== null && user[prop] !== '') - .map(prop => userCacheKey(prop, user[prop])); - }, - getByProperty: async (prop, value) => safeParseJson(await redisClient.get(userCacheKey(prop, value))), - getById: async (id) => UserRedisCacheSpace.getByProperty('id', id), - setUser: async ( - user, - { props = defaultUserIdProperties, ttlSeconds = DEFAULT_USER_CACHE_TTL_SECONDS } = {}, - ) => { - if ( ! user ) return; - const serialized = JSON.stringify(user); - const writes = []; - const cacheKeys = []; - for ( const prop of props ) { - if ( user[prop] === undefined || user[prop] === null || user[prop] === '' ) continue; - const key = userCacheKey(prop, user[prop]); - cacheKeys.push(key); - writes.push(setKey(key, serialized, { ttlSeconds })); - } - if ( writes.length ) { - await Promise.all(writes); - } - }, - invalidateUser: async (user, props = defaultUserIdProperties) => { - const keys = UserRedisCacheSpace.keysForUser(user, props); - if ( keys.length ) { - await deleteRedisKeys(...keys); - } - }, - invalidateById: async (id, props = defaultUserIdProperties) => { - const user = await UserRedisCacheSpace.getById(id); - if ( ! user ) return; - await UserRedisCacheSpace.invalidateUser(user, props); - }, -}; - -export { UserRedisCacheSpace }; diff --git a/src/backend/src/services/UserService.d.ts b/src/backend/src/services/UserService.d.ts deleted file mode 100644 index b82337bd8..000000000 --- a/src/backend/src/services/UserService.d.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { BaseService } from './BaseService'; -import type { IUser } from './User'; - -export interface IInsertResult { - insertId: number; -} - -export class UserService extends BaseService { - get_system_dir (): unknown; - generate_default_fsentries (args: { user: IUser }): Promise; - updateUserMetadata (userId: string, updatedMetadata: Record): Promise; -} diff --git a/src/backend/src/services/UserService.js b/src/backend/src/services/UserService.js deleted file mode 100644 index 0edd0956d..000000000 --- a/src/backend/src/services/UserService.js +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { RootNodeSelector, NodeChildSelector } = require('../deprecated/filesystem/node/selectors'); -const { invalidate_cached_user, invalidate_cached_user_by_id } = require('../helpers'); -const BaseService = require('./BaseService'); -const { DB_WRITE } = require('./database/consts'); - -/** - * Lorem ipsum dolor sit amet - */ -class UserService extends BaseService { - static MODULES = { - uuidv4: require('uuid').v4, - }; - - async _init () { - this.db = this.services.get('database').get(DB_WRITE, 'user-service'); - this.dir_system = null; - } - - async '__on_filesystem.ready' () { - const svc_fs = this.services.get('filesystem'); - // Ensure system user has a home directory - const dir_system = await svc_fs.node(new NodeChildSelector( - new RootNodeSelector(), - 'system', - )); - - if ( ! await dir_system.exists() ) { - const svc_getUser = this.services.get('get-user'); - await this.generate_default_fsentries({ - user: await svc_getUser.get_user({ username: 'system' }), - }); - } - - this.dir_system = dir_system; - - this.services.emit('user.system-user-ready'); - } - - get_system_dir () { - return this.dir_system; - } - - /** - * This used to be called `generate_system_fsentries` - */ - async generate_default_fsentries ({ user }) { - - // Note: The comment below is outdated as we now do parallel writes for - // all filesystem operations. However, there may still be some - // performance hit so this requires further investigation. - - // Normally, it is recommended to use mkdir() to create new folders, - // but during signup this could result in multiple queries to the DB server - // and for servers in remote regions such as Asia this could result in a - // very long time for /signup to finish, sometimes up to 30-40 seconds! - // by combining as many queries as we can into one and avoiding multiple back-and-forth - // with the DB server, we can speed this process up significantly. - - const ts = Date.now() / 1000; - - // Generate UUIDs for all the default folders and files - const uuidv4 = this.modules.uuidv4; - - let home_uuid = uuidv4(); - let trash_uuid = uuidv4(); - let appdata_uuid = uuidv4(); - let desktop_uuid = uuidv4(); - let documents_uuid = uuidv4(); - let pictures_uuid = uuidv4(); - let videos_uuid = uuidv4(); - let public_uuid = uuidv4(); - - const insert_res = await this.db.write( - `INSERT INTO fsentries - (uuid, parent_uid, user_id, name, path, is_dir, created, modified, immutable) VALUES - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true) - `, - [ - // Home - home_uuid, null, user.id, user.username, `/${user.username}`, ts, ts, - // Trash - trash_uuid, home_uuid, user.id, 'Trash', `/${user.username}/Trash`, ts, ts, - // AppData - appdata_uuid, home_uuid, user.id, 'AppData', `/${user.username}/AppData`, ts, ts, - // Desktop - desktop_uuid, home_uuid, user.id, 'Desktop', `/${user.username}/Desktop`, ts, ts, - // Documents - documents_uuid, home_uuid, user.id, 'Documents', `/${user.username}/Documents`, ts, ts, - // Pictures - pictures_uuid, home_uuid, user.id, 'Pictures', `/${user.username}/Pictures`, ts, ts, - // Videos - videos_uuid, home_uuid, user.id, 'Videos', `/${user.username}/Videos`, ts, ts, - // Public - public_uuid, home_uuid, user.id, 'Public', `/${user.username}/Public`, ts, ts, - ], - ); - - // https://stackoverflow.com/a/50103616 - let trash_id = insert_res.insertId; - let appdata_id = insert_res.insertId + 1; - let desktop_id = insert_res.insertId + 2; - let documents_id = insert_res.insertId + 3; - let pictures_id = insert_res.insertId + 4; - let videos_id = insert_res.insertId + 5; - let public_id = insert_res.insertId + 6; - - // Asynchronously set the user's system folders uuids in database - // This is for caching purposes, so we don't have to query the DB every time we need to access these folders - // This is also possible because we know the user's system folders uuids will never change - - // TODO: pass to IIAFE manager to avoid unhandled promise rejection - // (IIAFE manager doesn't exist yet, hence this is a TODO) - this.db.write( - `UPDATE user SET - trash_uuid=?, appdata_uuid=?, desktop_uuid=?, documents_uuid=?, pictures_uuid=?, videos_uuid=?, public_uuid=?, - trash_id=?, appdata_id=?, desktop_id=?, documents_id=?, pictures_id=?, videos_id=?, public_id=? - WHERE id=?`, - [ - trash_uuid, appdata_uuid, desktop_uuid, documents_uuid, pictures_uuid, videos_uuid, public_uuid, - trash_id, appdata_id, desktop_id, documents_id, pictures_id, videos_id, public_id, - user.id, - ], - ); - invalidate_cached_user(user); - } - - async updateUserMetadata (userId, updatedMetadata) { - // Fetch current metadata - const [user] = await this.db.read('SELECT metadata FROM `user` WHERE uuid=?', [userId]); - let metadata = {}; - - if ( user?.metadata ) { - if ( typeof user.metadata === 'string' ) { - // SQLite stores as TEXT, need to parse JSON - try { - metadata = JSON.parse(user.metadata); - } catch { - // If parsing fails, start with empty object - metadata = {}; - } - } else { - // MySQL stores as JSON object - metadata = user.metadata; - } - } - - // Update fields - Object.assign(metadata, updatedMetadata); - - // Save back to DB - always stringify for compatibility with both databases - await this.db.write('UPDATE `user` SET metadata=? WHERE uuid=?', [JSON.stringify(metadata), userId]); - const refreshed_user = await this.services.get('get-user').get_user({ - uuid: userId, - force: true, - }); - if ( refreshed_user?.id ) { - invalidate_cached_user_by_id(refreshed_user.id); - } - } -} - -module.exports = { - UserService, -}; diff --git a/src/backend/src/services/VerifiedGroupService.js b/src/backend/src/services/VerifiedGroupService.js deleted file mode 100644 index 1bb2e69ad..000000000 --- a/src/backend/src/services/VerifiedGroupService.js +++ /dev/null @@ -1,28 +0,0 @@ -const { get_user } = require('../helpers'); -const BaseService = require('./BaseService'); - -class VerifiedGroupService extends BaseService { - async _init () { - const config = this.global_config; - - const svc_event = this.services.get('event'); - svc_event.on('user.email-confirmed', async (_, { user_uid }) => { - const user = await get_user({ uuid: user_uid }); - - // Update group - const svc_group = this.services.get('group'); - await svc_group.remove_users({ - uid: config.default_temp_group, - users: [user.username], - }); - await svc_group.add_users({ - uid: config.default_user_group, - users: [user.username], - }); - }); - } -} - -module.exports = { - VerifiedGroupService, -}; diff --git a/src/backend/src/services/WSPushService.js b/src/backend/src/services/WSPushService.js deleted file mode 100644 index e00f9c5ce..000000000 --- a/src/backend/src/services/WSPushService.js +++ /dev/null @@ -1,350 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const BaseService = require('./BaseService'); -const { Context } = require('../util/context'); -class WSPushService extends BaseService { - static LOG_DEBUG = true; - - /** - * Initializes the WSPushService by setting up event listeners for various file system operations. - * - * @param {Object} options - The configuration options for the service. - * @param {Object} options.services - An object containing service dependencies. - */ - async _init () { - this.svc_event = this.services.get('event'); - - this.svc_event.on('fs.create.*', this._on_fs_create.bind(this)); - this.svc_event.on('fs.write.*', this._on_fs_update.bind(this)); - this.svc_event.on('fs.move.*', this._on_fs_move.bind(this)); - this.svc_event.on('fs.pending.*', this._on_fs_pending.bind(this)); - this.svc_event.on( - 'fs.storage.upload-progress', - this._on_upload_progress.bind(this), - ); - this.svc_event.on( - 'fs.storage.progress.*', - this._on_upload_progress.bind(this), - ); - this.svc_event.on( - 'puter-exec.submission.done', - this._on_submission_done.bind(this), - ); - this.svc_event.on( - 'outer.gui.*', - this._on_outer_gui.bind(this), - ); - } - - async _on_fs_create (key, data) { - const { node, context } = data; - - const metadata = { - from_new_service: true, - }; - - const svc_operationTrace = context.get('services').get('operationTrace'); - const frame = context.get(svc_operationTrace.ckey('frame')); - const gui_metadata = frame.get_attr('gui_metadata') || {}; - Object.assign(metadata, gui_metadata); - - const response = await node.getSafeEntry({ thumbnail: true }); - - const user_id_list = await (async () => { - const user_id_set = new Set(); - if ( metadata.user_id ) user_id_set.add(metadata.user_id); - else user_id_set.add(await node.get('user_id')); - return Array.from(user_id_set); - })(); - - Object.assign(response, metadata); - - this.svc_event.emit('outer.gui.item.added', { - user_id_list, - response, - }); - - const ts = Date.now(); - await this._update_user_ts(user_id_list, ts, metadata); // Pass metadata - } - - /** - * Handles file system update events. - * - * @param {string} key - The event key. - * @param {Object} data - The event data containing node and context information. - * @returns {Promise} A promise that resolves when the update has been processed. - * - * @description - * This method is triggered when a file or directory is updated. It retrieves - * metadata from the context, fetches the updated node's entry, determines the - * relevant user IDs, and emits an event to notify the GUI of the update. - * - * @note - * - The method uses a set for user IDs to prepare for future multi-user dispatch. - * - If no specific user ID is provided in the metadata, it falls back to the node's user ID. - */ - async _on_fs_update (key, data) { - const { node, context } = data; - - const metadata = { - from_new_service: true, - }; - - const svc_operationTrace = context.get('services').get('operationTrace'); - const frame = context.get(svc_operationTrace.ckey('frame')); - const gui_metadata = frame?.get_attr?.('gui_metadata') || {}; - Object.assign(metadata, gui_metadata); - - const response = await node.getSafeEntry({ debug: 'hi', thumbnail: true }); - - const user_id_list = await (async () => { - const user_id_set = new Set(); - if ( metadata.user_id ) user_id_set.add(metadata.user_id); - else user_id_set.add(await node.get('user_id')); - return Array.from(user_id_set); - })(); - - Object.assign(response, metadata); - - this.svc_event.emit('outer.gui.item.updated', { - user_id_list, - response, - }); - - const ts = Date.now(); - await this._update_user_ts(user_id_list, ts, metadata); // Pass metadata - } - - /** - * Handles file system move events by emitting appropriate GUI update events. - * - * This method is triggered when a file or directory is moved within the file system. - * It collects necessary metadata, updates the response with the old path, and - * broadcasts the event to update the GUI for the affected users. - * - * @param {string} key - The event key triggering this method. - * @param {Object} data - An object containing details about the moved item: - * - {Node} moved - The moved file system node. - * - {string} old_path - The previous path of the moved item. - * - {Context} context - The context in which the move operation occurred. - * @returns {Promise} A promise that resolves when the event has been emitted. - */ - async _on_fs_move (key, data) { - const { moved, old_path, context } = data; - - const metadata = { - from_new_service: true, - }; - - const svc_operationTrace = context.get('services').get('operationTrace'); - const frame = context.get(svc_operationTrace.ckey('frame')); - const gui_metadata = frame.get_attr('gui_metadata') || {}; - Object.assign(metadata, gui_metadata); - - const response = await moved.getSafeEntry(); - - const user_id_list = await (async () => { - const user_id_set = new Set(); - if ( metadata.user_id ) user_id_set.add(metadata.user_id); - else user_id_set.add(await moved.get('user_id')); - return Array.from(user_id_set); - })(); - - response.old_path = old_path; - Object.assign(response, metadata); - - this.svc_event.emit('outer.gui.item.moved', { - user_id_list, - response, - }); - - const ts = Date.now(); - await this._update_user_ts(user_id_list, ts, metadata); // Pass metadata - } - - /** - * Handles the 'fs.pending' event, preparing and emitting data for items that are pending processing. - * - * @param {string} key - The event key, typically starting with 'fs.pending.'. - * @param {Object} data - An object containing the fsentry and context of the pending file system operation. - * @param {Object} data.fsentry - The file system entry that is pending. - * @param {Object} data.context - The operation context providing additional metadata. - * @fires svc_event#outer.gui.item.pending - Emitted with user ID list and entry details. - * - * @returns {Promise} Emits an event to update the GUI about the pending item. - */ - async _on_fs_pending (key, data) { - const { fsentry, context } = data; - - const metadata = { - from_new_service: true, - }; - - const response = { ...fsentry }; - - const svc_operationTrace = context.get('services').get('operationTrace'); - const frame = context.get(svc_operationTrace.ckey('frame')); - const gui_metadata = frame.get_attr('gui_metadata') || {}; - Object.assign(metadata, gui_metadata); - - const user_id_list = await (async () => { - const user_id_set = new Set(); - if ( metadata.user_id ) user_id_set.add(metadata.user_id); - return Array.from(user_id_set); - })(); - - Object.assign(response, metadata); - - this.svc_event.emit('outer.gui.item.pending', { - user_id_list, - response, - }); - - const ts = Date.now(); - await this._update_user_ts(user_id_list, ts, metadata); // Pass metadata - } - - /** - * Emits an upload or download progress event to the relevant user room. - * - * @param {string} key - The event key that triggered this method. - * @param {Object} data - Contains upload_tracker, context, and meta information. - * @param {Object} data.upload_tracker - Tracker for the upload/download progress. - * @param {Object} data.context - Context of the operation. - * @param {Object} data.meta - Additional metadata for the event. - * - * It emits a progress event to the room if it exists, otherwise, it does nothing. - */ - async _on_upload_progress (key, data) { - this.log.info('got upload progress event'); - const { upload_tracker, context, meta } = data; - - const metadata = { - ...meta, - from_new_service: true, - }; - - const svc_operationTrace = context.get('services').get('operationTrace'); - const frame = context.get(svc_operationTrace.ckey('frame')); - const gui_metadata = frame.get_attr('gui_metadata') || {}; - Object.assign(metadata, gui_metadata); - - const roomId = metadata.user_id ?? metadata.userId; - - if ( ! roomId ) { - console.warn('missing room id for upload progress', { metadata }); - return; - } - - const svc_socketio = context.get('services').get('socketio'); - - const ws_event_name = metadata.call_it_download - ? 'download.progress' : 'upload.progress'; - - upload_tracker.sub(delta => { - this.log.info('emitting progress event'); - svc_socketio.send({ room: roomId }, ws_event_name, { - ...metadata, - total: upload_tracker.total_, - loaded: upload_tracker.progress_, - loaded_diff: delta, - }); - }); - } - - async _on_submission_done (key, data) { - const { actor } = data; - const { id, output, summary, measures, aux_outputs } = data; - const user_id = actor.type.user.id; - - const response = { - id, - output, - summary, - measures, - aux_outputs, - }; - - this.svc_event.emit('outer.gui.submission.done', { - user_id_list: [user_id], - response, - }); - } - - /** - * Handles the 'outer.gui.*' event to emit GUI-related updates to specific users. - * - * @param {string} key - The event key with 'outer.gui.' prefix removed. - * @param {Object} data - Contains user_id_list and response to emit. - * @param {Object} meta - Additional metadata for the event. - * - * @note This method iterates over each user ID provided in the event data, - * checks if the user's socket room exists and has clients, then emits - * the event to the appropriate room. - */ - async _on_outer_gui (key, { user_id_list, response }, meta) { - key = key.slice('outer.gui.'.length); - - const svc_socketio = this.services.get('socketio'); - - for ( const user_id of user_id_list ) { - svc_socketio.send({ room: user_id }, key, response); - this.svc_event.emit(`sent-to-user.${key}`, { - user_id, - response, - meta, - }); - } - } - - /** - * Updates the timestamp for a list of users in the puter-kvstore. Emits an event to notify the GUI of the update. - * - * @param {string[]} user_id_list - The list of user IDs to update the timestamp for. - * @param {number} timestamp - The timestamp to update the users with. - * @returns {Promise} A promise that resolves when the timestamp has been updated. - */ - async _update_user_ts (user_id_list, timestamp, metadata = {}) { - for ( const user_id of user_id_list ) { - const ts = timestamp; - const key = `last_change_timestamp:${user_id}`; - - try { - /** @type {import('../clients/dynamodb/DynamoKVStore/DynamoKVStore.js').DynamoKVStore} */ - const kvStore = Context.get('services').get('puter-kvstore'); - await kvStore.set({ key: key, value: ts }); - } catch ( error ) { - console.error('Failed to update user timestamp in kvstore', { user_id, error: error.message }); - } - } - - this.svc_event.emit('outer.gui.cache.updated', { - user_id_list, - response: { - timestamp, - original_client_socket_id: metadata.original_client_socket_id, - }, - }); - } -} - -module.exports = { - WSPushService, -}; diff --git a/src/backend/src/services/WebDAV/WebDAVService.js b/src/backend/src/services/WebDAV/WebDAVService.js deleted file mode 100644 index ac032c039..000000000 --- a/src/backend/src/services/WebDAV/WebDAVService.js +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { NodePathSelector } = require('../../deprecated/filesystem/node/selectors'); -const eggspress = require('../../api/eggspress'); -const configurable_auth = require('../../middleware/configurable_auth'); -const BaseService = require('../BaseService'); -const bcrypt = require('bcrypt'); -const xmlparser = require('express-xml-bodyparser'); -let davMethodMap; -let unsupportedMethodHandler; -let COOKIE_NAME = null; - -const ROOT_WEB_DAV_RESPONSE_XML = ` - - - / - - - / - Fri, 03 Jan 2025 10:30:45 GMT - 2025-01-03T10:30:45Z - - "dav-folder-1735898444" - - - - - - - - - - - - 0 - - HTTP/1.1 200 OK - - - - / - - - dav - Fri, 03 Jan 2025 10:30:45 GMT - 2025-01-03T10:30:45Z - - "dav-folder-1735898445" - - - - - - - - - - - - 0 - - HTTP/1.1 200 OK - - -`; - -class WebDAVService extends BaseService { - async _construct () { - davMethodMap = (await import ( './methodHandlers/methodMap.mjs')).davMethodMap; - unsupportedMethodHandler = (await import('./methodHandlers/method.mjs')).unsupportedMethodHandler; - } - async _init () { - const svc_web = this.services.get('web-server'); - svc_web.allow_undefined_origin(/^\/dav(\/.*)?$/); - } - #extractHeaderToken = ( headerToken = '' ) => { - let headerLockToken = null; - let prefix = null; - const match = headerToken.match(/(.*)<(urn:uuid:[0-9a-fA-F-]{36})>/); - if ( match ) { - if ( match.length > 2 ) { - headerLockToken = match[2]; - prefix = match[1].trim().slice( 1, -1); // Remove surrounding parentheses - } else { - headerLockToken = match[1]; - } - } - return { headerLockToken, prefix }; - }; - async authenticateWebDavUser ( username, password, _req, res ) { - // Default implementation - you should override this method - // Return null to reject authentication - const svc_auth = this.services.get('auth'); - - const user = await this.services - .get('get-user') - .get_user( { username: username, cached: false }); - let otpToken = null; - let real_password = password; - - if ( username === '-token' ) { - return await svc_auth.authenticate_from_token(password); - } - - if ( user.otp_enabled ) { - real_password = password.slice(0, -6); - otpToken = password.slice(-6); - } - - if ( await bcrypt.compare(real_password, user.password) ) { - const { token } = await svc_auth.create_session_token(user); - if ( user.otp_enabled ) { - const svc_otp = this.services.get('otp'); - const ok = svc_otp.verify( - user.username, - user.otp_secret, - otpToken, - ); - if ( ! ok ) { - return null; - } - } - - res.cookie(COOKIE_NAME, token, { - sameSite: 'none', - secure: true, - httpOnly: true, - maxAge: 34560000000, // 400 days, chrome maximum - }); - return await svc_auth.authenticate_from_token(token); - } - return null; - } - async handleHttpBasicAuth ( actor, req, res ) { - if ( actor ) { - return actor; - } - // Check for Basic Authentication header - const authHeader = req.headers.authorization; - if ( authHeader && authHeader.startsWith('Basic ') ) { - try { - // Parse Basic auth credentials - const base64Credentials = authHeader.split(' ')[1]; - const credentials = Buffer.from( - base64Credentials, - 'base64', - ).toString( 'ascii'); - let [username, ...password] = credentials.split(':'); - password = password.join(':'); - - // Call user's authentication function - actor = await this.authenticateWebDavUser( - username, - password, - req, - res, - ); - if ( ! actor ) { - // Authentication failed - res.set({ - 'WWW-Authenticate': 'Basic realm="WebDAV"', - DAV: '1, 2', - 'MS-Author-Via': 'DAV', - }); - res.status(401).end( 'Unauthorized'); - return; - } else { - return actor; - } - } catch ( _e ) { - res.set({ - 'WWW-Authenticate': 'Basic realm="WebDAV"', - DAV: '1, 2', - 'MS-Author-Via': 'DAV', - }); - res.status(401).end( 'Unauthorized'); - return; - } - } else { - // No credentials provided, send challenge - res.set({ - 'WWW-Authenticate': 'Basic realm="WebDAV"', - DAV: '1, 2', - 'MS-Author-Via': 'DAV', - }); - res.status(401).end( 'Unauthorized'); - return; - } - } - async handleWebDavServer ( filePath, req, res ) { - const svc_fs = this.services.get('filesystem'); - const fileNode = await svc_fs.node(new NodePathSelector(filePath)); - // Extract the UUID from the If header (e.g., If: ()) - const ifHeader = req.headers['if']; - const { headerLockToken } = this.#extractHeaderToken(ifHeader); - - const methodHandler = - davMethodMap[req.method] ?? unsupportedMethodHandler; - - methodHandler(req, res, filePath, fileNode, headerLockToken); - } - '__on_install.routes' ( _, { app } ) { - COOKIE_NAME = this.global_config.cookie_name; - - const r_webdav = (() => { - const express = require('express'); - return express.Router(); - } )(); - r_webdav.use(xmlparser()); - - app.use('/', r_webdav); - - r_webdav.use(eggspress( - '/*', - { - subdomain: 'dav', - allowedMethods: [ - 'PROPFIND', - 'PROPPATCH', - 'MKCOL', - 'GET', - 'HEAD', - 'POST', - 'PUT', - 'DELETE', - 'COPY', - 'MOVE', - 'LOCK', - 'UNLOCK', - 'OPTIONS', - ], - mw: [configurable_auth({ optional: true })], - }, - /** - * - * @param {import("express").Request} req - * @param {import("express").Response} res - */ async ( req, res ) => { - if ( req.method === 'OPTIONS' ) { - this.handleWebDavServer('/', req, res); - return; - } - const svc_su = this.services.get('su'); - let actor = await this.handleHttpBasicAuth(req.actor, req, res); - if ( ! actor ) { - return; - } - let filePath = decodeURIComponent(req.path); - // Handle root path for WebDAV compatibility - if ( filePath === '/' || filePath === '' ) { - filePath = '/'; // Keep as root for WebDAV - } - - svc_su.sudo(actor, async () => { - this.handleWebDavServer(filePath, req, res); - }); - }, - )); - } -} - -module.exports = { - WebDavFS: WebDAVService, -}; diff --git a/src/backend/src/services/WebDAV/lockStore.mjs b/src/backend/src/services/WebDAV/lockStore.mjs deleted file mode 100644 index 73ac60aed..000000000 --- a/src/backend/src/services/WebDAV/lockStore.mjs +++ /dev/null @@ -1,160 +0,0 @@ -export const DAV_LOCK_DURATION = 30; // seconds - -/* - * @param {string} headerToken - * @returns - */ -export const extractHeaderToken = (headerToken = '') => { - let headerLockToken = null; - let prefix = null; - const match = headerToken.match(/(.*)<(urn:uuid:[0-9a-fA-F-]{36})>/); - if ( match ) { - if ( match.length > 2 ) { - headerLockToken = match[2]; - prefix = match[1].trim().slice( 1, -1); // Remove surrounding parentheses - } else { - headerLockToken = match[1]; - } - } - return { headerLockToken, prefix }; -}; - -const LOCK_PREFIX = 'locktoken:'; -/** - * @param {{sudo:Function}} suService - * @param {import('../../modules/kvstore/KVStoreInterfaceService.js').KVStoreInterface} kvStoreService - * @param {...string} lockTokens - * @returns {Promise<{path: string, lockScope: 'shared' | 'exclusive', lockType?: string}[]>} - */ -export const getLocksIfValid = (suService, kvStoreService, ...lockTokens) => { - return suService.sudo(async () => { - const res = (await kvStoreService.get({ - key: lockTokens.map(lockToken => `${LOCK_PREFIX}${lockToken}`), - })).filter(Boolean); - return res; - }); -}; - -/** - * @param {{sudo:Function}} suService - * @param {import('../../modules/kvstore/KVStoreInterfaceService.js').KVStoreInterface} kvStoreService - * @param {string} filePath - * @param {string} lockScope - * @param {string} lockType - * @returns {Promise} - */ -export const createLock = ( suService, kvStoreService, filePath, lockScope, lockType ) => { - return suService.sudo(async () => { - const lockToken = `urn:uuid:${crypto.randomUUID()}`; - const currentTokens = await getFileLocks(suService, kvStoreService, filePath); - kvStoreService.set({ - key: `${LOCK_PREFIX}${lockToken}`, - value: { path: filePath, lockScope, lockType }, - expireAt: (Date.now() / 1000) + DAV_LOCK_DURATION, - }); - kvStoreService.set({ - key: `${LOCK_PREFIX}${filePath}`, - value: { ...currentTokens, [lockToken]: { lockScope, lockType } }, - expireAt: (Date.now() / 1000) + DAV_LOCK_DURATION, - }); - return lockToken; - }); -}; - -/** - * @param {{sudo:Function}} suService - * @param {import('../../modules/kvstore/KVStoreInterfaceService.js').KVStoreInterface} kvStoreService - * @param {string} lockToken - * @param {string} filePath - * @returns {void} - */ -export const deleteLock = ( suService, kvStoreService, lockToken, filePath ) => { - return suService.sudo(async () => { - kvStoreService.del({ key: `${LOCK_PREFIX}${lockToken}` }); - kvStoreService.del({ key: `${LOCK_PREFIX}${filePath}` }); - }); -}; -/** - * @param {{sudo:Function}} suService - * @param {import('../../modules/kvstore/KVStoreInterfaceService.js').KVStoreInterface} kvStoreService - * @param {string} lockToken - * @param {string} filePath - * @returns - */ -export const refreshLock = ( suService, kvStoreService, lockToken, filePath ) => { - return suService.sudo(async () => { - kvStoreService.expireAt({ - key: `${LOCK_PREFIX}${lockToken}`, - timestamp: (Date.now() / 1000 ) + DAV_LOCK_DURATION, - }); - kvStoreService.expireAt({ - key: `${LOCK_PREFIX}${filePath}`, - timestamp: (Date.now() / 1000 ) + DAV_LOCK_DURATION, - }); - return lockToken; - }); -}; - -/** - * @param {{sudo:Function}} suService - * @param {import('../../modules/kvstore/KVStoreInterfaceService.js').KVStoreInterface} kvStoreService - * @param {string} filePath - * @returns {Promise<{lockToken: string, lockScope: 'shared' | 'exclusive', lockType?: string}[]>} - */ -export const getFileLocks = ( suService, kvStoreService, filePath ) => { - return suService.sudo(async () => { - const parentPaths = filePath.split('/'); - const filePaths = parentPaths.map((_, i, paths) => `${LOCK_PREFIX}${paths.slice(0, i + 1).join('/')}`).filter(Boolean); - const tokenMapList = await kvStoreService.get({ - key: filePaths.slice(2), - }); - return tokenMapList.flatMap(tokenMap => Object.entries(tokenMap ?? {}).map(([ lockToken, lockInfo ]) => ({ - lockToken: lockToken.replace(LOCK_PREFIX, ''), - ...lockInfo, - }))).filter(Boolean); - }); -}; - -/** - * @param {{sudo:Function}} suService - * @param {import('../../modules/kvstore/KVStoreInterfaceService.js').KVStoreInterface} kvStoreService - * @param {string} filePath - * @param {string} headerLockToken - * @returns {Promise} - */ -export const hasWritePermissionInDAV = async ( suService, kvStoreService, filePath, headerLockToken ) => { - - // if no lock on file, allow write - const locksOnFile = await getFileLocks(suService, kvStoreService, filePath); - if ( ! locksOnFile?.length ) { - return true; - } - - if ( ! headerLockToken ) { - return false; - } - - const existingFileFromLock = (await getLocksIfValid(suService, kvStoreService, headerLockToken))?.pop(); - if ( ! filePath.startsWith(existingFileFromLock.path) ) { - return false; - } - - const lock = locksOnFile.find(( l ) => l.lockToken === headerLockToken); - if ( ! lock ) { - return false; - } - - if ( lock.lockScope === 'exclusive' ) { - // only 1 exclusive lock can exist, and headerLockToken matches it, allow write - return true; - } - - // if lock(s) on file are shared locks, and headerLockToken is one of them, allow write - if ( lock.lockScope === 'shared' ) { - // this lock should not exist if there are any exclusive locks - return locksOnFile.find(( l ) => l.lockScope === 'exclusive') === undefined; - } - - // else, deny write - return false; -}; \ No newline at end of file diff --git a/src/backend/src/services/WebDAV/methodHandlers/COPY.mjs b/src/backend/src/services/WebDAV/methodHandlers/COPY.mjs deleted file mode 100644 index c59923f00..000000000 --- a/src/backend/src/services/WebDAV/methodHandlers/COPY.mjs +++ /dev/null @@ -1,111 +0,0 @@ -import path from 'path'; -import { NodePathSelector } from '../../../deprecated/filesystem/node/selectors.js'; -import { hasWritePermissionInDAV } from '../lockStore.mjs'; -import { fsOperations } from '../utils.mjs'; - -/** - * @type {import('./method.mjs').HandlerFunction} - */ -export const COPY = async ( req, res, _filePath, fileNode, headerLockToken ) => { - try { - const servicesForLocks = [req.services.get('su'), req.services.get('puter-kvstore').as('puter-kvstore')]; - - const svc_fs = req.services.get('filesystem'); - const exists = await fileNode?.exists(); - // Check if the resource exists - if ( ! exists ) { - res.status(404).end( 'Not Found'); - return; - } - - // Parse Destination header (required for COPY) - const destinationHeader = req.headers.destination; - if ( ! destinationHeader ) { - res.status(400).end( 'Bad Request: Destination header required'); - return; - } - - // Parse destination URI - extract path after /dav - let destinationPath; - try { - const destUrl = new URL(destinationHeader, `http://${req.headers.host}`); - destinationPath = destUrl.pathname; - if ( ! destinationPath.startsWith('/') ) { - destinationPath = `/${destinationPath}`; - } - } catch ( _e ) { - res.status(400).end( 'Bad Request: Invalid destination URI'); - return; - } - destinationPath = decodeURI(destinationPath); - - // Parse Overwrite header (T = true, F = false, default = T) - const overwriteHeader = req.headers.overwrite; - const overwrite = overwriteHeader !== 'F'; // Default to true unless explicitly F - - // Parse destination path to get parent and new name - const destParentPath = path.dirname(destinationPath); - const destName = path.basename(destinationPath); - - // Check if destination already exists - const destNode = await svc_fs.node(new NodePathSelector(destinationPath)); - const destExists = await destNode.exists(); - - if ( destExists && !overwrite ) { - res.status(412).end( 'Precondition Failed: Destination exists and Overwrite is F'); - return; - } - - // Get destination parent node - const destParentNode = await svc_fs.node(new NodePathSelector(destParentPath)); - const destParentExists = await destParentNode.exists(); - - if ( ! destParentExists ) { - res.status(409).end( 'Conflict: Destination parent does not exist'); - return; - } - - // Verify destination parent is a directory - const destParentStat = await fsOperations.stat(destParentNode); - if ( ! destParentStat.is_dir ) { - res.status(409).end( 'Conflict: Destination parent is not a directory'); - return; - } - - // check lock - const hasDestinationWriteAccess = await hasWritePermissionInDAV(...servicesForLocks, destinationPath, headerLockToken); - if ( ! hasDestinationWriteAccess ) { - // DAV lock in place blocking write to this file - res.status(423).end( 'Locked: No write access to destination'); - } - - // Perform the copy operation - await fsOperations.copy(fileNode, { - destinationNode: destParentNode, - new_name: destName, - overwrite: overwrite, - dedupe_name: false, // WebDAV should not auto-dedupe - }); - - // Set response headers - if ( destExists ) { - res.status(204).end(); // 204 No Content for overwrite - } else { - res.status(201).end(); // 201 Created for new resource - } - } catch ( error ) { - // Handle specific error types - if ( error.code === 'permission_denied' ) { - res.status(403).end( 'Forbidden'); - } else if ( error.code === 'item_with_same_name_exists' ) { - res.status(412).end( 'Precondition Failed: Destination exists'); - } else if ( error.code === 'immutable' ) { - res.status(403).end( 'Forbidden: Resource is immutable'); - } else if ( error.code === 'dest_does_not_exist' ) { - res.status(409).end( 'Conflict: Destination parent does not exist'); - } else { - console.error('LOCK error:', error); - res.status(500).end( 'Internal Server Error'); - } - } -}; diff --git a/src/backend/src/services/WebDAV/methodHandlers/DELETE.mjs b/src/backend/src/services/WebDAV/methodHandlers/DELETE.mjs deleted file mode 100644 index 0e443ae8b..000000000 --- a/src/backend/src/services/WebDAV/methodHandlers/DELETE.mjs +++ /dev/null @@ -1,43 +0,0 @@ -import { hasWritePermissionInDAV } from '../lockStore.mjs'; -import { fsOperations } from '../utils.mjs'; - -/** - * Handler for the DELETE HTTP method in WebDAV. - * @type {import('./method.mjs').HandlerFunction} - */ -export const DELETE = async ( req, res, filePath, fileNode, headerLockToken ) => { - try { - const servicesForLocks = [req.services.get('su'), req.services.get('puter-kvstore').as('puter-kvstore')]; - - const hasDestinationWriteAccess = await hasWritePermissionInDAV(...servicesForLocks, filePath, headerLockToken); - const exists = await fileNode?.exists(); - // Check if the resource exists - if ( ! exists ) { - res.status(404).end('Not Found'); - return; - } - - if ( ! hasDestinationWriteAccess ) { - // DAV lock in place blocking write to this file - res.status(423).end('Locked: No write access to destination'); - return; - } - // Delete the resource using operations.delete - await fsOperations.delete(fileNode); - - // Return success response - res.status(204).end(); // 204 No Content for successful deletion - } catch ( error ) { - // Handle specific error types - if ( error.code === 'permission_denied' ) { - res.status(403).end( 'Forbidden'); - } else if ( error.code === 'immutable' ) { - res.status(403).end( 'Forbidden'); - } else if ( error.code === 'dir_not_empty' ) { - res.status(409).end( 'Conflict'); - } else { - console.error('LOCK error:', error); - res.status(500).end( 'Internal Server Error'); - } - } -}; diff --git a/src/backend/src/services/WebDAV/methodHandlers/HEAD_GET.mjs b/src/backend/src/services/WebDAV/methodHandlers/HEAD_GET.mjs deleted file mode 100644 index 0b1b12987..000000000 --- a/src/backend/src/services/WebDAV/methodHandlers/HEAD_GET.mjs +++ /dev/null @@ -1,134 +0,0 @@ -import { fsOperations, getProperMimeType } from '../utils.mjs'; - -const parseRangeHeader = (rangeHeader) => { - // Check if this is a multipart range request - if ( rangeHeader.includes(',') ) { - // For now, we'll only serve the first range in multipart requests - // as the underlying storage layer doesn't support multipart responses - const firstRange = rangeHeader.split(',')[0].trim(); - const matches = firstRange.match(/bytes=(\d+)-(\d*)/); - if ( ! matches ) { - return null; - } - - const start = parseInt(matches[1], 10); - const end = matches[2] ? parseInt(matches[2], 10) : null; - - return { start, end, isMultipart: true }; - } - - // Single range request - const matches = rangeHeader.match(/bytes=(\d+)-(\d*)/); - if ( ! matches ) { - return null; - } - - const start = parseInt(matches[1], 10); - const end = matches[2] ? parseInt(matches[2], 10) : null; - - return { start, end, isMultipart: false }; -}; - -/** - * @type {import('./method.mjs').HandlerFunction} - */ -export const HEAD_GET = async (req, res, _filePath, fileNode, _headerLockToken) => { - try { - const exists = await fileNode?.exists(); - if ( ! exists ) { - res.status(404).end('File not found'); - return; - } - - // Get file stats for Content-Length and other headers - const fileStat = await fsOperations.stat(fileNode); - - // Set appropriate headers - const headers = { - 'Accept-Ranges': 'bytes', - }; - - // Set Content-Length for files (not directories) - if ( ! fileStat.is_dir ) { - headers['Content-Length'] = fileStat.size || 0; - headers['x-expected-entity-length'] = fileStat.size || 0; - headers['Content-Type'] = getProperMimeType(fileStat.type, fileStat.name); - } - - // Set last modified header - if ( fileStat.modified ) { - headers['Last-Modified'] = new Date(fileStat.modified * 1000).toUTCString(); - } - - // Set ETag - headers['ETag'] = `"${fileStat.uid}-${Math.floor(fileStat.modified)}"`; - - // For HEAD requests, only send headers, no body - if ( req.method === 'HEAD' ) { - res.status(200).end(); - return; - } - - // For GET requests, send the file content - if ( fileStat.is_dir ) { - res.status(400).end('Cannot GET a directory'); - return; - } - - const options = {}; - - if ( req.headers['range'] ) { - res.status(206); - options.range = req.headers['range']; - // Parse the Range header and set Content-Range - const rangeInfo = parseRangeHeader(req.headers['range']); - if ( rangeInfo ) { - const { start, end, isMultipart } = rangeInfo; - - // For open-ended ranges, we need to calculate the actual end byte - let actualEnd = end; - let fileSize = null; - - try { - fileSize = fileStat.size; - if ( end === null ) { - actualEnd = fileSize - 1; // File size is 1-based, end byte is 0-based - } - } catch ( _error ) { - // If we can't get file size, we'll let the storage layer handle it - // and not set Content-Range header - actualEnd = null; - fileSize = null; - } - - if ( actualEnd !== null ) { - const totalSize = fileSize !== null ? fileSize : '*'; - const contentRange = `bytes ${start}-${actualEnd}/${totalSize}`; - res.set('Content-Range', contentRange); - headers['Content-Length'] = (actualEnd - start) + 1; - } - - // If this was a multipart request, modify the range header to only include the first range - if ( isMultipart ) { - req.headers['range'] = end !== null ? `bytes=${start}-${end}` : `bytes=${start}-`; - } - } - } - res.set(headers); - - const stream = await fsOperations.read(fileNode, options); - stream.on('data', (data) => { - res.write(data); - }); - stream.on('end', () => { - res.end(); - }); - stream.on('error', (error) => { - console.error('Stream error:', error); - res.status(500).end('Internal server error'); - }); - } catch ( error ) { - console.error('HEAD or GET error:', error); - res.status(500).end('Internal Server Error'); - } -}; diff --git a/src/backend/src/services/WebDAV/methodHandlers/LOCK.mjs b/src/backend/src/services/WebDAV/methodHandlers/LOCK.mjs deleted file mode 100644 index 57c4565e5..000000000 --- a/src/backend/src/services/WebDAV/methodHandlers/LOCK.mjs +++ /dev/null @@ -1,103 +0,0 @@ -import { createLock, getFileLocks, getLocksIfValid, refreshLock } from '../lockStore.mjs'; -import { escapeXml } from '../utils.mjs'; - -/** - * - * @param {string} lockToken - * @param {string} lockScope - * @param {string} filePath - * @returns - */ -const getLockResponse = ( lockToken, lockScope, filePath ) => { - return ` - - - - - - 0 - - webdav-user - - Second-7200 - - ${lockToken} - - - ${escapeXml(encodeURI(filePath))} - - - -`; -}; -/** - * - * @param {import('express').Request} req - * @param {import('express').Response} res - * @param {string} filePath - * @param {import('../../../deprecated/filesystem/FSNodeContext')} fileNode - * @param {string} headerLockToken - * @returns - */ -export const LOCK = async ( req, res, filePath, fileNode, headerLockToken ) => { - try { - const servicesForLocks = [req.services.get('su'), req.services.get('puter-kvstore').as('puter-kvstore')]; - const exists = await fileNode.exists(); - - const lockScope = req.body.lockinfo?.lockscope?.[0]?.shared ? 'shared' : 'exclusive'; - const lockType = req.body.lockinfo?.locktype?.[0]?.write ? 'write' : null; - - const existingFileFromLock = (await getLocksIfValid(...servicesForLocks, headerLockToken)).pop(); - - // Check if the resource exists - if ( ! exists ) { - // handle non exsiting child folder if lock is present to refresh parent - if ( existingFileFromLock && filePath.startsWith(existingFileFromLock.path) ) { - filePath = existingFileFromLock.path; - } - // Though technically the resource does not exist, we'll make a lock so that other's can't write to it technically. - } - - const locksOnFile = await getFileLocks(...servicesForLocks, filePath); - // handle exclusive locks if theres any lock in place - if ( - lockScope === 'exclusive' && - locksOnFile?.length && - ( !headerLockToken || existingFileFromLock?.path !== `${filePath}` ) - ) { - res.status(423).end( 'Locked: Resource already locked'); - return; - } - // handle shared locks - if ( - locksOnFile?.length && - locksOnFile?.find(( lock ) => lock.lockScope === '') - && ( - !headerLockToken || existingFileFromLock?.path !== `${filePath}`) - ) { - res.status(423).end( 'Locked: Resource already locked'); - return; - } - - // Generate a UUID lock token - const lockToken = headerLockToken - ? await refreshLock(...servicesForLocks, headerLockToken, filePath) - : await createLock(...servicesForLocks, filePath, lockScope, lockType); - - // Set proper headers for WebDAV XML response - res.set({ - 'Content-Type': 'application/xml; charset=utf-8', - ...( headerLockToken && lockScope !== 'shared' ? {} : { 'Lock-Token': `<${lockToken}>` } ), - DAV: '1, 2', - 'MS-Author-Via': 'DAV', - }); - - // Return lock response - const lockResponse = getLockResponse(lockToken, lockScope, filePath); - res.status(!exists ? 201 : 200); - res.end(lockResponse); - } catch ( error ) { - console.error('LOCK error:', error); - res.status(500).end( 'Internal Server Error'); - } -}; diff --git a/src/backend/src/services/WebDAV/methodHandlers/MKCOL.mjs b/src/backend/src/services/WebDAV/methodHandlers/MKCOL.mjs deleted file mode 100644 index 95e500f84..000000000 --- a/src/backend/src/services/WebDAV/methodHandlers/MKCOL.mjs +++ /dev/null @@ -1,90 +0,0 @@ -import path from 'path'; -import { NodePathSelector } from '../../../deprecated/filesystem/node/selectors.js'; -import { hasWritePermissionInDAV } from '../lockStore.mjs'; -import { fsOperations } from '../utils.mjs'; - -/** - * @type {import('./method.mjs').HandlerFunction} - */ -export const MKCOL = async ( req, res, filePath, fileNode, headerLockToken ) => { - try { - const servicesForLocks = [req.services.get('su'), req.services.get('puter-kvstore').as('puter-kvstore')]; - const hasDestinationWriteAccess = await hasWritePermissionInDAV(...servicesForLocks, filePath, headerLockToken); - const exists = await fileNode?.exists(); - // Check if request has a body (not allowed for MKCOL) - const contentLength = req.headers['content-length']; - if ( contentLength && parseInt(contentLength) > 0 ) { - res.status(415).end( 'Unsupported Media Type'); - return; - } - - // Parse the path to get parent directory and target name - const targetPath = filePath; - const parentPath = path.dirname(targetPath); - const targetName = path.basename(targetPath); - - // Handle root directory case - if ( parentPath === '.' || targetPath === '/' ) { - res.status(403).end( 'Forbidden'); - return; - } - - // Check if target already exists - if ( exists ) { - res.status(405).end( 'Method Not Allowed'); - return; - } - - if ( ! hasDestinationWriteAccess ) { - // DAV lock in place blocking write to this file - res.status(423).end( 'Locked: No write access to destination'); - return; - } - - // Get parent directory node - const svc_fs = fileNode.services.get('filesystem'); - const parentNode = await svc_fs.node(new NodePathSelector(parentPath)); - const parentExists = await parentNode.exists(); - - if ( ! parentExists ) { - res.status(409).end( 'Conflict'); - return; - } - - // Verify parent is a directory - const parentStat = await fsOperations.stat(parentNode); - if ( ! parentStat.is_dir ) { - res.status(409).end( 'Conflict'); - return; - } - - // Create the directory - await fsOperations.mkdir(parentNode, { - name: targetName, - overwrite: false, - create_missing_parents: false, - }); - - // Set response headers - res.set({ - Location: `${targetPath}${targetPath.endsWith('/') ? '' : '/'}`, - 'Content-Length': '0', - }); - - res.status(201).end(); // 201 Created - } catch ( error ) { - // Handle specific error types - if ( error.code === 'item_with_same_name_exists' ) { - res.status(405).end( 'Method Not Allowed'); - } else if ( error.code === 'permission_denied' ) { - res.status(403).end( 'Forbidden'); - } else if ( error.code === 'dest_does_not_exist' ) { - res.status(409).end( 'Conflict'); - } else if ( error.code === 'invalid_file_name' ) { - res.status(400).end( 'Bad Request'); - } else { - console.error('MKCOL error:', error); - res.status(500).end( 'Internal Server Error'); - } - } -}; diff --git a/src/backend/src/services/WebDAV/methodHandlers/MOVE.mjs b/src/backend/src/services/WebDAV/methodHandlers/MOVE.mjs deleted file mode 100644 index 675cbc305..000000000 --- a/src/backend/src/services/WebDAV/methodHandlers/MOVE.mjs +++ /dev/null @@ -1,114 +0,0 @@ -import path from 'path'; -import { NodePathSelector } from '../../../deprecated/filesystem/node/selectors.js'; -import { hasWritePermissionInDAV } from '../lockStore.mjs'; -import { fsOperations } from '../utils.mjs'; - -/** - * MOVE method handler - * @type {import('./method.mjs').HandlerFunction} - */ -export const MOVE = async ( req, res, filePath, fileNode, headerLockToken ) => { - try { - const servicesForLocks = [req.services.get('su'), req.services.get('puter-kvstore').as('puter-kvstore')]; - const hasSourceWriteAccess = await hasWritePermissionInDAV(...servicesForLocks, filePath, headerLockToken); - const svc_fs = req.services.get('filesystem'); - const exists = await fileNode?.exists(); - // Check if the resource exists - if ( ! exists ) { - res.status(404).end( 'Not Found'); - return; - } - - // Parse Destination header (required for MOVE) - const destinationHeader = req.headers.destination; - if ( ! destinationHeader ) { - res.status(400).end( 'Bad Request: Destination header required'); - return; - } - - // Parse destination URI - extract path after /dav - let destinationPath; - try { - const destUrl = new URL(destinationHeader, `http://${req.headers.host}`); - destinationPath = destUrl.pathname; // Remove '/dav' prefix - if ( ! destinationPath.startsWith('/') ) { - destinationPath = `/${destinationPath}`; - } - } catch { - res.status(400).end( 'Bad Request: Invalid destination URI'); - return; - } - destinationPath = decodeURI(destinationPath); - - const hasDestinationWriteAccess = hasWritePermissionInDAV(destinationPath, headerLockToken); - - // Parse Overwrite header (T = true, F = false, default = T) - const overwriteHeader = req.headers.overwrite; - const overwrite = overwriteHeader !== 'F'; // Default to true unless explicitly F - - // Parse destination path to get parent and new name - const destParentPath = path.dirname(destinationPath); - const destName = path.basename(destinationPath); - - // Check if destination already exists - const destNode = await svc_fs.node(new NodePathSelector(destinationPath)); - const destExists = await destNode.exists(); - - if ( destExists && !overwrite ) { - res.status(412).end( 'Precondition Failed: Destination exists and Overwrite is F'); - return; - } - - // Get destination parent node - const destParentNode = await svc_fs.node(new NodePathSelector(destParentPath)); - const destParentExists = await destParentNode.exists(); - - if ( ! destParentExists ) { - res.status(409).end( 'Conflict: Destination parent does not exist'); - return; - } - - // Verify destination parent is a directory - const destParentStat = await fsOperations.stat(destParentNode); - if ( ! destParentStat.is_dir ) { - res.status(409).end( 'Conflict: Destination parent is not a directory'); - return; - } - - if ( !hasSourceWriteAccess || !hasDestinationWriteAccess ) { - // DAV lock in place blocking write to this file - res.status(423).end( 'Locked: No write access to source or destination'); - return; - } - - // Perform the move operation - await fsOperations.move(fileNode, { - destinationNode: destParentNode, - new_name: destName, - overwrite: overwrite, - dedupe_name: false, // WebDAV should not auto-dedupe - create_missing_parents: false, - }); - - // Set response headers - if ( destExists ) { - res.status(204).end(); // 204 No Content for overwrite - } else { - res.status(201).end(); // 201 Created for new resource - } - } catch ( error ) { - // Handle specific error types - if ( error.code === 'permission_denied' ) { - res.status(403).end( 'Forbidden'); - } else if ( error.code === 'item_with_same_name_exists' ) { - res.status(412).end( 'Precondition Failed: Destination exists'); - } else if ( error.code === 'immutable' ) { - res.status(403).end( 'Forbidden: Resource is immutable'); - } else if ( error.code === 'dest_does_not_exist' ) { - res.status(409).end( 'Conflict: Destination parent does not exist'); - } else { - console.error('LOCK error:', error); - res.status(500).end( 'Internal Server Error'); - } - } -}; diff --git a/src/backend/src/services/WebDAV/methodHandlers/OPTIONS.mjs b/src/backend/src/services/WebDAV/methodHandlers/OPTIONS.mjs deleted file mode 100644 index 6b5a909e7..000000000 --- a/src/backend/src/services/WebDAV/methodHandlers/OPTIONS.mjs +++ /dev/null @@ -1,14 +0,0 @@ -export const OPTIONS = async (_req, res) => { - res.set({ - 'Allow': 'OPTIONS, GET, HEAD, POST, PUT, DELETE, TRACE, COPY, MOVE, MKCOL, PROPFIND, PROPPATCH, LOCK, UNLOCK', - 'DAV': '1, 2, ordered-collections', // WebDAV compliance classes with ordered-collections for macOS - 'MS-Author-Via': 'DAV', // Microsoft compatibility - 'Server': 'Puter/WebDAV', // Server identification - 'Accept-Ranges': 'bytes', - 'Content-Type': 'text/plain; charset=utf-8', // Explicit content type - 'Content-Length': '0', - 'Cache-Control': 'no-cache', // Prevent caching issues - 'Connection': 'Keep-Alive', // Keep connection alive for macOS - }); - res.status(200).end(); -}; \ No newline at end of file diff --git a/src/backend/src/services/WebDAV/methodHandlers/PROPFIND.mjs b/src/backend/src/services/WebDAV/methodHandlers/PROPFIND.mjs deleted file mode 100644 index fb7945af2..000000000 --- a/src/backend/src/services/WebDAV/methodHandlers/PROPFIND.mjs +++ /dev/null @@ -1,177 +0,0 @@ -import { escapeXml, fsOperations } from '../utils.mjs'; - -const getProperMimeType = ( originalType, filename ) => { - if ( originalType && originalType !== 'application/octet-stream' ) { - return originalType; - } - const ext = filename.split('.').pop()?.toLowerCase(); - switch ( ext ) { - case 'js': - return 'application/javascript'; - case 'css': - return 'text/css'; - case 'html': - case 'htm': - return 'text/html'; - case 'txt': - return 'text/plain'; - case 'json': - return 'application/json'; - case 'xml': - return 'application/xml'; - case 'pdf': - return 'application/pdf'; - case 'png': - return 'image/png'; - case 'jpg': - case 'jpeg': - return 'image/jpeg'; - case 'gif': - return 'image/gif'; - case 'svg': - return 'image/svg+xml'; - default: - return 'application/octet-stream'; - } -}; - -const convertToWebDAVPropfindXML = ( fsEntry ) => { - const isDirectory = fsEntry.is_dir; - const lastModified = new Date(fsEntry.modified * 1000).toUTCString(); - const createdDate = new Date(fsEntry.created * 1000).toISOString(); - let href = fsEntry.path; - if ( isDirectory && !href.endsWith('/') ) { - href += '/'; - } - const xml = ` - - - ${escapeXml(encodeURI(href))} - - - ${escapeXml(fsEntry.name)} - ${lastModified} - ${createdDate} - ${ - isDirectory - ? '' - : ` - ${fsEntry.size || 0} - ${escapeXml(getProperMimeType(fsEntry.type, fsEntry.name))}` - } - "${fsEntry.uid}-${Math.floor(fsEntry.modified)}" - - - - - - - - - - - - 0 - - HTTP/1.1 200 OK - - -`; - return xml; -}; - -const convertMultipleToWebDAVPropfindXML = ( selfStat, fsEntries ) => { - fsEntries = [ selfStat, ...fsEntries ]; - const responses = fsEntries - .map(( fsEntry ) => { - const isDirectory = fsEntry.is_dir; - const lastModified = new Date(( fsEntry.modified || 0 ) * 1000).toUTCString(); - const createdDate = new Date(( fsEntry.created || 0 ) * 1000).toISOString(); - let href = fsEntry.path; - if ( isDirectory && !href.endsWith('/') ) { - href += '/'; - } - return ` - ${escapeXml(encodeURI(href))} - - - ${escapeXml(fsEntry.name)} - ${lastModified} - ${createdDate} - ${ - isDirectory - ? '' - : ` - ${fsEntry.size || 0} - ${escapeXml(getProperMimeType(fsEntry.type, fsEntry.name))}` - } - "${fsEntry.uid}-${Math.floor(fsEntry.modified)}" - - - - - - - - - - - - 0 - - HTTP/1.1 200 OK - - `; - }) - .join( '\n'); - - return ` - -${responses} -`; -}; - -export const PROPFIND = async ( req, res, filePath, fileNode, _headerLockToken ) => { - try { - res.set({ - 'Content-Type': 'application/xml; charset=utf-8', - DAV: '1, 2', - 'MS-Author-Via': 'DAV', - }); - - const exists = await fileNode?.exists(); - - // Handle special case for /dav/ root - return static response with only admin folder - if ( filePath === '/' || filePath === '' ) { - const stat = await fsOperations.stat(fileNode); - const entries = await fsOperations.readdir(fileNode); - res.status(207); - res.end(convertMultipleToWebDAVPropfindXML(stat, entries)); - return; - } - - // Check if file exists - if ( ! exists ) { - res.status(404).end( 'Not Found'); - return; - } - - // Handle Depth header (Windows WebDAV client compatibility) - const depth = req.headers.depth || '1'; - - const stat = await fsOperations.stat(fileNode); - - if ( stat.is_dir && depth !== '0' ) { - const entries = await fsOperations.readdir(fileNode); - res.status(207); - res.end(convertMultipleToWebDAVPropfindXML(stat, entries)); - } else { - res.status(207); - res.end(convertToWebDAVPropfindXML(stat)); - } - } catch ( error ) { - - console.error('PROPFIND error:', error); - res.status(500).end( 'Internal Server Error'); - } -}; diff --git a/src/backend/src/services/WebDAV/methodHandlers/PROPPATCH.mjs b/src/backend/src/services/WebDAV/methodHandlers/PROPPATCH.mjs deleted file mode 100644 index 7d7d4ea92..000000000 --- a/src/backend/src/services/WebDAV/methodHandlers/PROPPATCH.mjs +++ /dev/null @@ -1,52 +0,0 @@ -// WebDAV PROPPATCH handler for Puter -import { hasWritePermissionInDAV } from '../lockStore.mjs'; -import { escapeXml } from '../utils.mjs'; - -const getStubResponse = ( filePath ) => ` - - - ${escapeXml(encodeURI(filePath))} - - - HTTP/1.1 200 OK - - -`; - -/** - * Handles the WebDAV PROPPATCH method. - * Always returns a generic success response (no extended attributes supported) unless locked, which fails but doesn't matter anyway. - * - * @param {object} req - Express request object - * @param {object} res - Express response object - * @param {string} filePath - Path to the target file - * @param {object} fileNode - File node object (unused in stub) - * @param {string} headerLockToken - Lock token from headers (unused in stub) - */ -export const PROPPATCH = async ( req, res, filePath, _fileNode, headerLockToken ) => { - - try { - const servicesForLocks = [req.services.get('su'), req.services.get('puter-kvstore').as('puter-kvstore')]; - const hasDestinationWriteAccess = await hasWritePermissionInDAV(...servicesForLocks, filePath, headerLockToken); - - if ( ! hasDestinationWriteAccess ) { - // DAV lock in place blocking write to this file - res.status(423).end( 'Locked: No write access to destination'); - return; - } - res.set({ - 'Content-Type': 'application/xml; charset=utf-8', - DAV: '1, 2', - 'MS-Author-Via': 'DAV', - }); - // Generic success response (no real property update) - const stubResponse = getStubResponse(filePath); - - res.status(207); - res.end(stubResponse); - } catch ( error ) { - // Log error to console (can be replaced with service logger if needed) - console.error('PROPPATCH error:', error); - res.status(500).end( 'Internal Server Error'); - } -}; diff --git a/src/backend/src/services/WebDAV/methodHandlers/PUT.mjs b/src/backend/src/services/WebDAV/methodHandlers/PUT.mjs deleted file mode 100644 index a9ba43753..000000000 --- a/src/backend/src/services/WebDAV/methodHandlers/PUT.mjs +++ /dev/null @@ -1,109 +0,0 @@ -import path from 'path'; -import { hasWritePermissionInDAV } from '../lockStore.mjs'; -import { fsOperations } from '../utils.mjs'; - -/** - * @type {import('./method.mjs').HandlerFunction} - */ -export const PUT = async ( req, res, filePath, fileNode, headerLockToken ) => { - try { - const servicesForLocks = [req.services.get('su'), req.services.get('puter-kvstore').as('puter-kvstore')]; - const hasDestinationWriteAccess = await hasWritePermissionInDAV(...servicesForLocks, filePath, headerLockToken); - if ( ! hasDestinationWriteAccess ) { - // DAV lock in place blocking write to this file - res.status(423).end('Locked: No write access to destination'); - return; - } - // macOS loves polluting webdav directories with metadata which would be stored regularly in HFS+ or APFS. - // We will 422 all of these, because no one actually wants to see them. - const fileName = path.basename(filePath); - if ( - ( req.headers['user-agent'] && - req.headers['user-agent'].includes('Darwin/') && - fileName.toLowerCase() === '.ds_store' ) || - fileName.startsWith('._') - ) { - res.writeHead(422, { - 'Content-Type': 'application/xml; charset=utf-8', - }); - - res.end(` - - macOS metadata files not permitted -`); - return; - } - - // Handle Expect: 100-continue header - if ( req.headers.expect && req.headers.expect.toLowerCase() === '100-continue' ) { - res.writeContinue(); - } - - // Check Content-Length header to find length - // TODO: Allow partial uploads with Range header - // TODO: Allow uploads with no Content-Length - const contentLength = req.headers['content-length'] || req.headers['x-expected-entity-length']; // x-expected-entity-length is used by macOS Finder for some reason - if ( ! contentLength ) { - res.status(400).end( 'Content-Length header required'); - return; - } - - const fileSize = parseInt(contentLength); - if ( isNaN(fileSize) || fileSize < 0 ) { - res.status(400).end( 'Invalid Content-Length'); - return; - } - - // Check if file exists before writing (for proper status code) - const existedBefore = await fileNode.exists(); - - // Set Content-Type if provided - const contentType = req.headers['content-type']; - - // Prepare write options - const writeOptions = { - stream: req, // Express request object is a readable stream - size: fileSize, - overwrite: true, // PUT should always overwrite - create_missing_parents: true, // Create directories as needed - no_thumbnail: true, // Disable thumbnails for WebDAV - }; - - // If Content-Type is provided, include it in file metadata - if ( contentType ) { - writeOptions.file = { - mimetype: contentType, - }; - } - - // Write the file - const result = await fsOperations.write(fileNode, writeOptions); - - // Set response headers - res.set({ - ETag: `"${result.uid}-${Math.floor(result.modified)}"`, - 'Last-Modified': new Date(result.modified * 1000).toUTCString(), - }); - - // Return appropriate status code - if ( existedBefore ) { - res.status(204).end(); // 204 No Content for updated file - } else { - res.status(201).end(); // 201 Created for new file - } - } catch ( error ) { - // Handle specific error types - if ( error.code === 'item_with_same_name_exists' ) { - res.status(409).end( 'Conflict: Item already exists'); - } else if ( error.code === 'storage_limit_reached' ) { - res.status(507).end( 'Insufficient Storage'); - } else if ( error.code === 'permission_denied' ) { - res.status(403).end( 'Forbidden'); - } else if ( error.code === 'file_too_large' ) { - res.status(413).end( 'Request Entity Too Large'); - } else { - console.error('PUT error:', error); - res.status(500).end( 'Internal Server Error'); - } - } -}; diff --git a/src/backend/src/services/WebDAV/methodHandlers/UNLOCK.mjs b/src/backend/src/services/WebDAV/methodHandlers/UNLOCK.mjs deleted file mode 100644 index 2ed2665de..000000000 --- a/src/backend/src/services/WebDAV/methodHandlers/UNLOCK.mjs +++ /dev/null @@ -1,39 +0,0 @@ -import { deleteLock, extractHeaderToken, getLocksIfValid } from '../lockStore.mjs'; - -/** - * @type {import('./method.mjs').HandlerFunction} - */ -export const UNLOCK = async ( req, res, filePath, fileNode ) => { - try { - const servicesForLocks = [req.services.get('su'), req.services.get('puter-kvstore').as('puter-kvstore')]; - const exists = await fileNode?.exists(); - // Check if the resource exists - if ( ! exists ) { - res.status(204).end(); - return; - } - - // Check for Lock-Token header (normally required for UNLOCK) - const lockTokenHeader = req.headers['lock-token']; - const { headerLockToken } = extractHeaderToken(lockTokenHeader); - - if ( ! headerLockToken ) { - res.status(400).end( 'Bad Request: Lock-Token header required'); - return; - } - - const existingFileFromLock = (await getLocksIfValid(...servicesForLocks, headerLockToken)).pop(); - if ( existingFileFromLock ) { - if ( existingFileFromLock.path === filePath ) { - deleteLock(...servicesForLocks, headerLockToken, filePath); - return res.status(204).end(); // 204 No Content for successful unlock - } - return res.status(403).end(); // 403 Forbidden - lock token does not match - } else { - return res.status(409).end(); // 409 Conflict - no lock present - } - } catch ( error ) { - console.error('UNLOCK error:', error); - res.status(500).end( 'Internal Server Error'); - } -}; diff --git a/src/backend/src/services/WebDAV/methodHandlers/method.mjs b/src/backend/src/services/WebDAV/methodHandlers/method.mjs deleted file mode 100644 index 59dd57cd2..000000000 --- a/src/backend/src/services/WebDAV/methodHandlers/method.mjs +++ /dev/null @@ -1,28 +0,0 @@ -/** - * @typedef {import('express').Request & {services: import('../../BaseService.js')}} Request - * @typedef {import('express').Response} Response - * @typedef {import('../../../deprecated/filesystem/FSNodeContext')} FSNodeContext - */ - -/** - * @typedef {(req: Request, res: Response, filePath: string, fileNode: FSNodeContext, headerLockToken: string) => Promise} HandlerFunction - */ - -/** - * @type {HandlerFunction} - */ -export const unsupportedMethodHandler = async ( - req, - res, - _filePath, - _fileNode, - _headerLockToken, -) => { - res.set({ - Allow: - 'OPTIONS, GET, HEAD, POST, PUT, DELETE, COPY, MOVE, MKCOL, PROPFIND, PROPPATCH, LOCK, UNLOCK', - DAV: '1, 2', - 'MS-Author-Via': 'DAV', - }); - res.status(405).end( 'Method Not Allowed'); -}; diff --git a/src/backend/src/services/WebDAV/methodHandlers/methodMap.mjs b/src/backend/src/services/WebDAV/methodHandlers/methodMap.mjs deleted file mode 100644 index a2a7c14f1..000000000 --- a/src/backend/src/services/WebDAV/methodHandlers/methodMap.mjs +++ /dev/null @@ -1,30 +0,0 @@ -import { COPY } from './COPY.mjs'; -import { DELETE } from './DELETE.mjs'; -import { HEAD_GET } from './HEAD_GET.mjs'; -import { LOCK } from './LOCK.mjs'; -import { MKCOL } from './MKCOL.mjs'; -import { MOVE } from './MOVE.mjs'; -import { OPTIONS } from './OPTIONS.mjs'; -import { PROPFIND } from './PROPFIND.mjs'; -import { PROPPATCH } from './PROPPATCH.mjs'; -import { PUT } from './PUT.mjs'; -import { UNLOCK } from './UNLOCK.mjs'; - -/** - * Map of HTTP methods to their corresponding handler functions. - * @type {Record} - */ -export const davMethodMap = { - HEAD: HEAD_GET, - GET: HEAD_GET, - LOCK, - UNLOCK, - COPY, - MOVE, - DELETE, - PROPFIND, - PUT, - MKCOL, - PROPPATCH, - OPTIONS, -}; diff --git a/src/backend/src/services/WebDAV/utils.mjs b/src/backend/src/services/WebDAV/utils.mjs deleted file mode 100644 index 62cdefe63..000000000 --- a/src/backend/src/services/WebDAV/utils.mjs +++ /dev/null @@ -1,171 +0,0 @@ -import { HLCopy } from '../../deprecated/filesystem/hl_operations/hl_copy.js'; -import { HLMkdir } from '../../deprecated/filesystem/hl_operations/hl_mkdir.js'; -import { HLMove } from '../../deprecated/filesystem/hl_operations/hl_move.js'; -import { HLReadDir } from '../../deprecated/filesystem/hl_operations/hl_readdir.js'; -import { HLRemove } from '../../deprecated/filesystem/hl_operations/hl_remove.js'; -import { HLStat } from '../../deprecated/filesystem/hl_operations/hl_stat.js'; -import { HLWrite } from '../../deprecated/filesystem/hl_operations/hl_write.js'; -import { LLRead } from '../../deprecated/filesystem/ll_operations/ll_read.js'; -import { Context } from '../../util/context.js'; - -/** - * Small utility function to escape XML - * - * @param {string} text - * @returns - */ -export const escapeXml = ( text ) => { - if ( typeof text !== 'string' ) return text; - return text - .replace(/&/g, '&') - .replace( //g, '>') - .replace( /"/g, '"') - .replace( /'/g, '''); -}; - -// Small operations wrapper to make my life a bit easier. Generally it takes a FileNode and returns what puter.fs in puter.js would return. -export const fsOperations = { - stat: ( node ) => { - const hl_stat = new HLStat(); - return hl_stat.run({ - subject: node, - user: Context.get('actor'), - return_subdomains: false, - return_permissions: true, - return_shares: false, - return_versions: false, - return_size: true, - }); - }, - readdir: ( node ) => { - const hl_readdir = new HLReadDir(); - return hl_readdir.run({ - subject: node, - no_subdomains: true, - // user: Context.get("actor").type.user, - actor: Context.get('actor'), - recursive: false, - no_thumbs: true, - no_assocs: true, - }); - }, - read: ( node, options ) => { - const ll_read = new LLRead(); - return ll_read.run({ - fsNode: node, - actor: Context.get('actor'), - ...options, - }); - }, - write: ( node, options ) => { - const hl_write = new HLWrite(); - return hl_write.run({ - destination_or_parent: node, - actor: Context.get('actor'), - file: { - stream: options.stream, - size: options.size || 0, - ...options.file, // Allow additional file properties - }, - overwrite: options.overwrite !== undefined ? options.overwrite : true, // Default to true for WebDAV PUT - create_missing_parents: false, - dedupe_name: false, - user: Context.get('actor').type.user, - specified_name: options.name, // Optional filename if node is a directory - fallback_name: options.fallback_name, - shortcut_to: options.shortcut_to, - no_thumbnail: options.no_thumbnail || true, // Disable thumbnails for WebDAV by default - message: options.message, - app_id: options.app_id, - socket_id: options.socket_id, - operation_id: options.operation_id, - item_upload_id: options.item_upload_id, - offset: options.offset, // For partial/resume uploads - }); - }, - mkdir: ( node, options ) => { - const hl_mkdir = new HLMkdir(); - return hl_mkdir.run({ - parent: node, - path: options.path || options.name, // Support both path and name parameters - actor: Context.get('actor'), - overwrite: options.overwrite || false, // WebDAV MKCOL should not overwrite by default - create_missing_parents: - options.create_missing_parents !== undefined ? options.create_missing_parents : true, // Auto-create parent directories - shortcut_to: options.shortcut_to, // Support for shortcuts - user: Context.get('actor').type.user, // User context for permissions - }); - }, - delete: ( node ) => { - const hl_remove = new HLRemove(); - return hl_remove.run({ - target: node, - recursive: true, - user: Context.get('actor'), - }); - }, - move: ( sourceNode, options ) => { - const hl_move = new HLMove(); - return hl_move.run({ - source: sourceNode, // The source fileNode being moved - destination_or_parent: options.destinationNode, // The destination fileNode (could be parent dir or exact destination) - user: Context.get('actor').type.user, - actor: Context.get('actor'), - new_name: options.new_name, // New name in the destination folder - overwrite: options.overwrite !== undefined ? options.overwrite : false, // WebDAV overwrite is optional - dedupe_name: options.dedupe_name || false, // Handle name conflicts - create_missing_parents: options.create_missing_parents || false, // Whether to create missing parent directories - new_metadata: options.new_metadata, // Optional metadata updates - }); - }, - copy: ( sourceNode, options ) => { - const hl_copy = new HLCopy(); - return hl_copy.run({ - source: sourceNode, // The source fileNode being copied - destination_or_parent: options.destinationNode, // The destination fileNode (could be parent dir or exact destination) - user: Context.get('actor').type.user, - new_name: options.new_name, // New name in the destination folder - overwrite: options.overwrite !== undefined ? options.overwrite : false, // WebDAV overwrite is optional - dedupe_name: options.dedupe_name || false, // Handle name conflicts - }); - }, -}; - -export const getProperMimeType = ( originalType, filename ) => { - // If we have a type and it's not the generic octet-stream, use it - if ( originalType && originalType !== 'application/octet-stream' ) { - return originalType; - } - - // Otherwise, guess based on file extension - const ext = filename.split('.').pop()?.toLowerCase(); - switch ( ext ) { - case 'js': - return 'application/javascript'; - case 'css': - return 'text/css'; - case 'html': - case 'htm': - return 'text/html'; - case 'txt': - return 'text/plain'; - case 'json': - return 'application/json'; - case 'xml': - return 'application/xml'; - case 'pdf': - return 'application/pdf'; - case 'png': - return 'image/png'; - case 'jpg': - case 'jpeg': - return 'image/jpeg'; - case 'gif': - return 'image/gif'; - case 'svg': - return 'image/svg+xml'; - default: - return 'application/octet-stream'; - } -}; \ No newline at end of file diff --git a/src/backend/src/services/WispService.js b/src/backend/src/services/WispService.js deleted file mode 100644 index 7aad2bedd..000000000 --- a/src/backend/src/services/WispService.js +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const configurable_auth = require('../middleware/configurable_auth'); -const eggspress = require('../api/eggspress'); -const BaseService = require('./BaseService'); - -class WispService extends BaseService { - '__on_install.routes' (_, { app }) { - const r_wisp = (() => { - const require = this.require; - const express = require('express'); - return express.Router(); - })(); - - app.use('/wisp', r_wisp); - - r_wisp.use(eggspress('/relay-token/create', { - allowedMethods: ['POST'], - mw: [configurable_auth({ optional: true })], - }, async (req, res) => { - const svc_token = this.services.get('token'); - const actor = req.actor; - - if ( actor ) { - const token = svc_token.sign('wisp', { - $: 'token:wisp', - $v: '0.0.0', - user_uid: actor.type.user.uuid, - }, { - expiresIn: '1d', - }); - this.log.info('creating wisp token', { - actor: actor.uid, - token: token, - }); - res.json({ - token, - server: this.config.server, - }); - } else { - const token = svc_token.sign('wisp', { - $: 'token:wisp', - $v: '0.0.0', - guest: true, - }, { - expiresIn: '1d', - }); - res.json({ - token, - server: this.config.server, - }); - } - })); - - r_wisp.use(eggspress('/relay-token/verify', { - allowedMethods: ['POST'], - }, async (req, res) => { - const svc_token = this.services.get('token'); - const svc_apiError = this.services.get('api-error'); - const svc_event = this.services.get('event'); - - const decoded = (() => { - try { - const decoded = svc_token.verify('wisp', req.body.token); - if ( decoded.$ !== 'token:wisp' ) { - throw svc_apiError.create('invalid_token'); - } - return decoded; - } catch (e) { - throw svc_apiError.create('forbidden'); - } - })(); - - const svc_getUser = this.services.get('get-user'); - - const event = { - allow: true, - policy: { allow: true }, - guest: decoded.guest, - user: decoded.guest ? undefined : await svc_getUser.get_user({ - uuid: decoded.user_uid, - }), - }; - await svc_event.emit('wisp.get-policy', event); - if ( ! event.allow ) { - throw svc_apiError.create('forbidden'); - } - - res.json(event.policy); - })); - } -} - -module.exports = { - WispService, -}; diff --git a/src/backend/src/services/abuse-prevention/AuthAuditService.js b/src/backend/src/services/abuse-prevention/AuthAuditService.js deleted file mode 100644 index 7f48ef0ec..000000000 --- a/src/backend/src/services/abuse-prevention/AuthAuditService.js +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const BaseService = require('../BaseService'); -const { DB_WRITE } = require('../database/consts'); - -/** -* AuthAuditService Class -* -* The AuthAuditService class extends BaseService and is responsible for recording -* authentication audit logs. It handles the initialization of the database connection, -* recording audit events, and managing any errors that occur during the process. -* This class ensures that all authentication-related actions are logged for auditing -* and troubleshooting purposes. -*/ -class AuthAuditService extends BaseService { - static MODULES = { - uuidv4: require('uuid').v4, - }; - - async _init () { - this.db = this.services.get('database').get(DB_WRITE, 'auth:audit'); - } - - /** - * Records an audit entry for authentication actions. - * - * This method handles the recording of audit entries for various authentication actions. - * It captures the requester details, action, body, and any extra information. - * If an error occurs during the recording process, it reports the error with appropriate details. - * - * @param {Object} parameters - The parameters for the audit entry. - * @param {Object} parameters.requester - The requester object. - * @param {string} parameters.action - The action performed. - * @param {Object} parameters.body - The body of the request. - * @param {Object} [parameters.extra] - Any extra information. - * @returns {Promise} - A promise that resolves when the audit entry is recorded. - */ - async record (parameters) { - try { - await this._record(parameters); - } catch ( err ) { - this.errors.report('auth-audit-service.record', { - source: err, - trace: true, - alarm: true, - }); - } - } - - /** - * Records an authentication audit event. - * - * This method logs an authentication audit event with the provided parameters. - * It generates a unique identifier for the event, serializes the requester, - * body, and extra information, and writes the event to the database. - * - * @param {Object} params - The parameters for the authentication audit event. - * @param {Object} params.requester - The requester information. - * @param {string} params.requester.ip - The IP address of the requester. - * @param {string} params.requester.ua - The user-agent string of the requester. - * @param {Function} params.requester.serialize - A function to serialize the requester information. - * @param {string} params.action - The action performed during the authentication event. - * @param {Object} params.body - The body of the request. - * @param {Object} params.extra - Additional information related to the event. - * @returns {Promise} - A promise that resolves when the event is recorded. - */ - async _record ({ requester, action, body, extra }) { - const uid = `aas-${ this.modules.uuidv4()}`; - - const json_values = { - requester: requester.serialize(), - body: body, - extra: extra ?? {}, - }; - - let has_parse_error = 0; - - for ( const k in json_values ) { - let value = json_values[k]; - try { - value = JSON.stringify(value); - } catch ( err ) { - has_parse_error = 1; - value = { parse_error: err.message }; - } - json_values[k] = value; - } - - await this.db.write('INSERT INTO auth_audit (' + - 'uid, ip_address, ua_string, action, ' + - 'requester, body, extra, ' + - 'has_parse_error' + - ') VALUES ( ?, ?, ?, ?, ?, ?, ?, ? )', - [ - uid, - requester.ip, - requester.ua, - action, - JSON.stringify(requester.serialize()), - JSON.stringify(body), - JSON.stringify(extra ?? {}), - has_parse_error, - ]); - } -} - -module.exports = { - AuthAuditService, -}; diff --git a/src/backend/src/services/abuse-prevention/EdgeRateLimitService.js b/src/backend/src/services/abuse-prevention/EdgeRateLimitService.js deleted file mode 100644 index da7dded83..000000000 --- a/src/backend/src/services/abuse-prevention/EdgeRateLimitService.js +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { asyncSafeSetInterval } from '@heyputer/putility/src/libs/promise.js'; -import { Context } from '../../util/context.js'; -import { safeHasOwnProperty } from '../../util/safety.js'; -import { BaseService } from '../BaseService.js'; - -const MINUTE = 60 * 1000; -const HOUR = 60 * MINUTE; - -const DEFAULT_SCOPE = { - limit: 500, - window: 15 * MINUTE, -}; - -/* INCREMENTAL CHANGES - The first scopes are of the form 'name-of-endpoint', but later it was - decided that they're of the form `/path/to/endpoint`. New scopes should - follow the latter form. -*/ - -/** -* Class representing an edge rate limiting service that manages -* request limits for various scopes (e.g. login, signup) -* to prevent abuse. It keeps track of request timestamps -* and enforces limits based on a specified time window. -*/ -export class EdgeRateLimitService extends BaseService { - - scopes = { - 'oidc-general': { - limit: 100, - window: 15 * MINUTE, - }, - 'login': { - limit: 10, - window: 15 * MINUTE, - }, - 'signup': { - limit: 10, - window: 15 * MINUTE, - }, - 'contact-us': { - limit: 10, - window: 15 * MINUTE, - }, - 'share': { - limit: 30, - window: 1 * MINUTE, - }, - 'send-confirm-email': { - limit: 10, - window: HOUR, - }, - 'confirm-email': { - limit: 10, - window: HOUR, - }, - 'send-pass-recovery-email': { - limit: 10, - window: HOUR, - }, - 'verify-pass-recovery-token': { - limit: 10, - window: 15 * MINUTE, - }, - 'set-pass-using-token': { - limit: 10, - window: HOUR, - }, - 'save-account': { - limit: 10, - window: HOUR, - }, - 'change-email-start': { - limit: 10, - window: HOUR, - }, - 'change-email-confirm': { - limit: 10, - window: HOUR, - }, - 'passwd': { - limit: 10, - window: HOUR, - }, - '/user-protected/change-password': { - limit: 10, - window: HOUR, - }, - '/user-protected/change-email': { - limit: 10, - window: HOUR, - }, - '/user-protected/change-username': { - limit: 10, - window: HOUR, - }, - '/user-protected/disable-2fa': { - limit: 10, - window: HOUR, - }, - 'login-otp': { - limit: 15, - window: 30 * MINUTE, - }, - 'login-recovery': { - limit: 10, - window: HOUR, - }, - 'enable-2fa': { - limit: 10, - window: HOUR, - }, - - }; - requests = new Map(); - - /** - * Initializes the EdgeRateLimitService by setting up a periodic cleanup interval. - * This method sets an interval that calls the cleanup function every 5 minutes. - */ - async _init () { - asyncSafeSetInterval(() => this.cleanup(), 4.5 * MINUTE); - } - - check (scope, noIncrease = false) { - if ( ! Object.prototype.hasOwnProperty.call(this.scopes, scope) ) { - this.log.warn('unconfigured rate-limit scope', { scope }); - } - const scopeSpec = safeHasOwnProperty(this.scopes, scope) - ? this.scopes[scope] - : DEFAULT_SCOPE; - const { window, limit } = scopeSpec; - - const requester = Context.get('requester'); - const rl_identifier = requester.rl_identifier; - const key = `${scope}:${rl_identifier}`; - const now = Date.now(); - const windowStart = now - window; - - if ( ! this.requests.has(key) ) { - this.requests.set(key, []); - } - - // Access the timestamps of past requests for this scope and IP - const timestamps = this.requests.get(key); - - // Remove timestamps that are outside the current window - while ( timestamps.length > 0 && timestamps[0] < windowStart ) { - timestamps.shift(); - } - - // Check if the current request exceeds the rate limit - if ( timestamps.length >= limit ) { - return false; - } else { - // Add current timestamp and allow the request - if ( ! noIncrease ) { - timestamps.push(now); - } - return true; - } - } - - incr (scope) { - if ( ! Object.prototype.hasOwnProperty.call(this.scopes, scope) ) { - throw new Error(`unrecognized rate-limit scope: ${scope}`); - } - const requester = Context.get('requester'); - const rl_identifier = requester.rl_identifier; - const key = `${scope}:${rl_identifier}`; - const now = Date.now(); - - if ( ! this.requests.has(key) ) { - this.requests.set(key, []); - } - const timestamps = this.requests.get(key); - timestamps.push(now); - } - - /** - * Cleans up the rate limit request records by removing entries - * that have no associated timestamps. This method is intended - * to be called periodically to free up memory. - */ - cleanup () { - this.log.tick('edge rate-limit cleanup task'); - for ( const [key, timestamps] of this.requests.entries() ) { - if ( timestamps.length === 0 ) { - this.requests.delete(key); - } - } - } -} diff --git a/src/backend/src/services/abuse-prevention/IdentificationService.js b/src/backend/src/services/abuse-prevention/IdentificationService.js deleted file mode 100644 index 2523639e1..000000000 --- a/src/backend/src/services/abuse-prevention/IdentificationService.js +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('@heyputer/putility'); -const BaseService = require('../BaseService'); -const { Context } = require('../../util/context'); -const config = require('../../config'); -const isBot = require('isbot'); -/** -* @class Requester -* @classdesc This class represents a requester in the system. It encapsulates -* information about the requester's user-agent, IP address, origin, referer, and -* other relevant details. The class includes methods to create instances from -* request objects, check if the referer or origin is from Puter, and serialize -* the requester's information. It also includes a method to get a unique identifier -* based on the requester's IP address. -*/ -class Requester { - constructor (o) { - for ( const k in o ) this[k] = o[k]; - } - static create (o) { - return new Requester(o); - } - static from_request (req) { - - const has_referer = req.headers['referer'] !== undefined; - let referer_url; - let referer_origin; - if ( has_referer ) { - try { - referer_url = new URL(req.headers['referer']); - referer_origin = referer_url.origin; - } catch (e) { - // URL is invalid; referer_url and referer_origin will be undefined - } - } - - return new Requester({ - ua: req.headers['user-agent'], - ip: req.connection.remoteAddress, - ip_forwarded: req.headers['x-forwarded-for'], - ip_user: req.headers['x-forwarded-for'] || - req.connection.remoteAddress, - origin: req.headers['origin'], - referer: req.headers['referer'], - referer_origin, - }); - } - - /** - * Checks if the referer origin is from Puter. - * - * @returns {boolean} True if the referer origin matches any of the configured Puter origins, otherwise false. - */ - is_puter_referer () { - const puter_origins = [ - config.origin, - config.api_base_url, - ]; - return puter_origins.includes(this.referer_origin); - } - - /** - * Checks if the request origin is from a known Puter origin. - * - * @returns {boolean} - Returns true if the request origin matches one of the known Puter origins, false otherwise. - */ - is_puter_origin () { - const puter_origins = [ - config.origin, - config.api_base_url, - ]; - return puter_origins.includes(this.origin); - } - - /** - * @method get rl_identifier - * @description Retrieves the rate-limiter identifier, which is either the forwarded IP or the direct IP. - * @returns {string} The IP address used for rate-limiting purposes. - */ - get rl_identifier () { - return this.ip_forwarded || this.ip; - } - - /** - * Serializes the Requester object into a plain JavaScript object. - * - * This method converts the properties of the Requester instance into a plain object, - * making it suitable for serialization (e.g., for JSON). - * - * @returns {Object} The serialized representation of the Requester object. - */ - serialize () { - return { - ua: this.ua, - ip: this.ip, - ip_forwarded: this.ip_forwarded, - referer: this.referer, - referer_origin: this.referer_origin, - }; - } - -} - -// DRY: (3/3) - src/util/context.js; move install() to base class -/** -* @class RequesterIdentificationExpressMiddleware -* @extends AdvancedBase -* @description This class extends AdvancedBase and provides middleware functionality for identifying the requester in an Express application. -* It registers initializers, installs the middleware on the Express application, and runs the middleware to identify and log details about the requester. -* The class uses the 'isbot' module to determine if the requester is a bot. -*/ -class RequesterIdentificationExpressMiddleware extends AdvancedBase { - register_initializer (initializer) { - this.value_initializers_.push(initializer); - } - install (app) { - app.use(this.run.bind(this)); - } - async run (req, res, next) { - const x = Context.get(); - - const requester = Requester.from_request(req); - const is_bot = isBot(requester.ua); - requester.is_bot = is_bot; - - x.set('requester', requester); - req.requester = requester; - - next(); - } -} - -/** -* @class IdentificationService -* @extends BaseService -* @description The IdentificationService class is responsible for handling the identification of requesters in the application. -* It extends the BaseService class and utilizes the RequesterIdentificationExpressMiddleware to process and identify requesters. -* This service ensures that requester information is properly logged and managed within the application context. -*/ -class IdentificationService extends BaseService { - /** - * Constructs the IdentificationService instance. - * - * This method initializes the service by creating an instance of - * RequesterIdentificationExpressMiddleware and assigning it to the `mw` property. - * - * @returns {void} - */ - _construct () { - this.mw = new RequesterIdentificationExpressMiddleware(); - } - /** - * Initializes the middleware logger. - * - * This method sets the logger for the `RequesterIdentificationExpressMiddleware` instance. - * It does not take any parameters and does not return any value. - * - * @method - * @name _init - */ - _init () { - this.mw.log = this.log; - } - /** - * We need to listen to this event to install a context-aware middleware - */ - async '__on_install.middlewares.context-aware' (_, { app }) { - this.mw.install(app); - } -} - -module.exports = { - IdentificationService, -}; diff --git a/src/backend/src/services/abuse-prevention/concurrentRequestLimiter/.gitignore b/src/backend/src/services/abuse-prevention/concurrentRequestLimiter/.gitignore deleted file mode 100644 index 4e57eef88..000000000 --- a/src/backend/src/services/abuse-prevention/concurrentRequestLimiter/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.js -*.js.map diff --git a/src/backend/src/services/abuse-prevention/concurrentRequestLimiter/ConcurrentRequestLimiter.test.ts b/src/backend/src/services/abuse-prevention/concurrentRequestLimiter/ConcurrentRequestLimiter.test.ts deleted file mode 100644 index 02def72da..000000000 --- a/src/backend/src/services/abuse-prevention/concurrentRequestLimiter/ConcurrentRequestLimiter.test.ts +++ /dev/null @@ -1,173 +0,0 @@ -import { beforeEach, describe, expect, it } from 'vitest'; -import { redisClient } from '../../../clients/redis/redisSingleton.js'; -import { Context } from '../../../util/context.js'; -import { ConcurrentRequestLimiter } from './ConcurrentRequestLimiter.js'; - -const createId = () => - `${Date.now()}-${Math.random().toString(16).slice(2)}`; - -const setSubscriptionResolver = (subscriptionId = '') => { - Context.root.set('services', { - get: (serviceName: string) => { - if ( serviceName !== 'event' ) return undefined; - return { - emit: async ( - eventName: string, - payload: { userSubscriptionId?: string }, - ) => { - if ( eventName === 'metering:getUserSubscription' ) { - payload.userSubscriptionId = subscriptionId; - } - }, - }; - }, - }); -}; - -describe('ConcurrentRequestLimiter', () => { - beforeEach(() => { - setSubscriptionResolver(); - }); - - it('registers simple limit config', async () => { - const limiter = new ConcurrentRequestLimiter({ redis: redisClient }); - const key = `test.simple.${createId()}`; - limiter.registerLimitKey(key, { limit: 2 }); - - const first = await limiter.checkAndIncrementConcurrent({ - key, - actor: { - type: { - user: { - uuid: 'simple-user', - email: 'user@puter.dev', - email_confirmed: true, - password: 'hashed', - }, - }, - }, - }); - - expect(first.allowed).toBe(true); - await limiter.decrementConcurrent(first.permit); - }); - - it('enforces grouped limits from actor user group', async () => { - const limiter = new ConcurrentRequestLimiter({ redis: redisClient }); - const key = `test.grouped.${createId()}`; - limiter.registerLimitKey(key, { - temp_free: { limit: 1 }, - user_free: { limit: 2 }, - default: { limit: 2 }, - }); - - const tmpActor = { - type: { - user: { - uuid: 'tmp-user', - email: null, - password: null, - }, - }, - }; - - const first = await limiter.checkAndIncrementConcurrent({ - key, - actor: tmpActor, - }); - const second = await limiter.checkAndIncrementConcurrent({ - key, - actor: tmpActor, - }); - - expect(first.allowed).toBe(true); - expect(second.allowed).toBe(false); - - await limiter.decrementConcurrent(first.permit); - }); - - it('decrementConcurrent releases permit for later calls', async () => { - const limiter = new ConcurrentRequestLimiter({ redis: redisClient }); - const key = `test.release.${createId()}`; - limiter.registerLimitKey(key, { - user_free: { limit: 1 }, - default: { limit: 1 }, - }); - - const actor = { - type: { - user: { - uuid: 'free-user', - email: 'free@puter.dev', - email_confirmed: true, - password: 'hashed', - }, - }, - }; - - const first = await limiter.checkAndIncrementConcurrent({ - key, - actor, - }); - expect(first.allowed).toBe(true); - - const blocked = await limiter.checkAndIncrementConcurrent({ - key, - actor, - }); - expect(blocked.allowed).toBe(false); - - await limiter.decrementConcurrent(first.permit); - - const allowedAgain = await limiter.checkAndIncrementConcurrent({ - key, - actor, - }); - expect(allowedAgain.allowed).toBe(true); - - await limiter.decrementConcurrent(allowedAgain.permit); - }); - - it('maps paid group from active paid subscription tier', async () => { - const limiter = new ConcurrentRequestLimiter({ redis: redisClient }); - const key = `test.paid.${createId()}`; - setSubscriptionResolver('basic'); - - limiter.registerLimitKey(key, { - temp_free: { limit: 1 }, - user_free: { limit: 1 }, - basic: { limit: 2 }, - default: { limit: 1 }, - }); - - const actor = { - type: { - user: { - uuid: 'paid-user', - email: 'paid@puter.dev', - email_confirmed: false, - }, - }, - }; - - const first = await limiter.checkAndIncrementConcurrent({ - key, - actor, - }); - const second = await limiter.checkAndIncrementConcurrent({ - key, - actor, - }); - const third = await limiter.checkAndIncrementConcurrent({ - key, - actor, - }); - - expect(first.allowed).toBe(true); - expect(second.allowed).toBe(true); - expect(third.allowed).toBe(false); - - await limiter.decrementConcurrent(first.permit); - await limiter.decrementConcurrent(second.permit); - }); -}); diff --git a/src/backend/src/services/abuse-prevention/concurrentRequestLimiter/ConcurrentRequestLimiter.ts b/src/backend/src/services/abuse-prevention/concurrentRequestLimiter/ConcurrentRequestLimiter.ts deleted file mode 100644 index 63722580a..000000000 --- a/src/backend/src/services/abuse-prevention/concurrentRequestLimiter/ConcurrentRequestLimiter.ts +++ /dev/null @@ -1,245 +0,0 @@ -import crypto from 'crypto'; -import { Cluster } from 'ioredis'; -import { redisClient } from '../../../clients/redis/redisSingleton.js'; -import { Context } from '../../../util/context.js'; -import { Actor } from '../../auth/Actor.js'; -import { DEFAULT_FREE_SUBSCRIPTION, DEFAULT_TEMP_SUBSCRIPTION } from '../../MeteringService/consts.js'; -import type { - CheckAndIncrementConcurrentOptions, - ConcurrentLimitConfig, - ConcurrentPermit, - GroupLimitConfig, - SimpleLimitConfig, -} from './types.js'; - -const defaultLeaseMs = 60 * 1000; -const maxAcquireAttempts = 5; - -const tempGroup = DEFAULT_TEMP_SUBSCRIPTION; -const freeGroup = DEFAULT_FREE_SUBSCRIPTION; - -const hasOwn = (object: unknown, key: string): boolean => { - if ( !object || typeof object !== 'object' ) return false; - return Object.prototype.hasOwnProperty.call(object, key); -}; - -const isPositiveFiniteNumber = (value: unknown): value is number => - Number.isFinite(value) && Number(value) > 0; - -const isSimpleLimitConfig = ( - config: ConcurrentLimitConfig, -): config is SimpleLimitConfig => - hasOwn(config, 'limit') && - isPositiveFiniteNumber((config as SimpleLimitConfig).limit); - -const isGroupLimitConfig = ( - config: ConcurrentLimitConfig, -): config is GroupLimitConfig => { - if ( typeof config !== 'object' || config === null || Array.isArray(config) ) { - return false; - } - if ( hasOwn(config, 'limit') ) { - return false; - } - const groups = Object.keys(config); - if ( groups.length === 0 ) return false; - for ( const group of groups ) { - const groupConfig = (config as GroupLimitConfig)[group]; - if ( !groupConfig || !isPositiveFiniteNumber(groupConfig.limit) ) { - return false; - } - } - return true; -}; - -const cloneLimitConfig = (config: ConcurrentLimitConfig): ConcurrentLimitConfig => - JSON.parse(JSON.stringify(config)) as ConcurrentLimitConfig; - -// TODO DS: expand this to block at middleware layer -export class ConcurrentRequestLimiter { - #redis: Cluster; - #limitsByKey: Map; - - get #eventService () { - return Context.get('services').get('event'); - } - - constructor ({ redis = redisClient }: { redis?: Cluster } = {}) { - this.#redis = redis; - this.#limitsByKey = new Map(); - } - - #isTemporaryUser (actor: Actor) { - const user = actor?.type?.user; - if ( ! user ) return true; - return !(user.email) || !(user.email_confirmed); - }; - - async #getActorUserGroup (actor: Actor, noSub = false) { - const userSubscriptionEvent = { actor, userSubscriptionId: '' }; - if ( ! noSub ) { - await this.#eventService.emit('metering:getUserSubscription', userSubscriptionEvent); // will set userSubscription property on event - } - - if ( userSubscriptionEvent.userSubscriptionId && !noSub ) { - return userSubscriptionEvent.userSubscriptionId; - } - - if ( this.#isTemporaryUser(actor) ) { - return tempGroup; - } - - return freeGroup; - }; - - registerLimitKey (key: string, config: ConcurrentLimitConfig): void { - if ( typeof key !== 'string' || key.length === 0 ) { - throw new TypeError('key must be a non-empty string'); - } - - if ( !isSimpleLimitConfig(config) && !isGroupLimitConfig(config) ) { - throw new TypeError( - 'config must be {limit:number} or {[userGroup]:{limit:number}}', - ); - } - - this.#limitsByKey.set(key, cloneLimitConfig(config)); - } - - hasLimitKey (key: string): boolean { - return this.#limitsByKey.has(key); - } - - async checkAndIncrementConcurrent ( - options: CheckAndIncrementConcurrentOptions, - ) { - const { actor, key } = options; - const leaseMs = options.leaseMs ?? defaultLeaseMs; - - if ( typeof key !== 'string' || key.length === 0 ) { - throw new TypeError('key must be a non-empty string'); - } - if ( ! isPositiveFiniteNumber(leaseMs) ) { - throw new TypeError('leaseMs must be a positive number'); - } - - const userId = actor?.type?.user?.uuid; - if ( ! userId ) { - throw new Error('actor user id is required for concurrency checks'); - } - - const userGroup = await this.#getActorUserGroup(actor); - const limit = this.#resolveLimit({ key, userGroup }); - const redisKey = this.#toRedisKey({ key, userId }); - const token = this.#createToken(); - - for ( let attempt = 0; attempt < maxAcquireAttempts; attempt++ ) { - const now = Date.now(); - const expiresAt = now + leaseMs; - - await this.#redis.zremrangebyscore(redisKey, '-inf', now); - await this.#redis.watch(redisKey); - try { - const activeCountRaw = await this.#redis.zcard(redisKey); - const activeCount = Number(activeCountRaw) || 0; - if ( activeCount >= limit ) { - await this.#redis.unwatch(); - return { - allowed: false, - limit, - activeCount, - userGroup, - }; - } - - const transaction = this.#redis.multi(); - transaction.zadd(redisKey, expiresAt, token); - transaction.pexpire(redisKey, leaseMs); - const transactionResult = await transaction.exec(); - - if ( transactionResult === null ) { - continue; - } - - const permit: ConcurrentPermit = { - key, - redisKey, - token, - userId, - userGroup, - limit, - expiresAt, - }; - - return { - allowed: true, - limit, - activeCount: activeCount + 1, - userGroup, - permit, - }; - } catch ( error: unknown ) { - await this.#redis.unwatch(); - throw error; - } - } - - throw new Error( - `failed to acquire concurrency permit for ${key} after ${maxAcquireAttempts} attempts`, - ); - } - - async decrementConcurrent ( - permit: ConcurrentPermit | null | undefined, - ): Promise { - if ( ! permit ) return; - if ( !permit.redisKey || !permit.token ) return; - await this.#redis.zrem(permit.redisKey, permit.token); - } - - #resolveLimit ({ - key, - userGroup, - }: { - key: string; - userGroup: string; - }): number { - const config = this.#limitsByKey.get(key); - if ( ! config ) { - throw new Error(`no concurrent limit config for key: ${key}`); - } - - if ( isSimpleLimitConfig(config) ) { - return config.limit; - } - - if ( hasOwn(config, userGroup) ) { - return config[userGroup].limit; - } - - if ( hasOwn(config, 'default') ) { - return config.default.limit; - } - - throw new Error( - `no concurrent limit group config for key: ${key} and userGroup: ${userGroup}`, - ); - } - - #toRedisKey ({ - key, - userId, - }: { - key: string; - userId: string; - }): string { - return `concurrency:${encodeURIComponent(key)}:${encodeURIComponent(userId)}`; - } - - #createToken (): string { - if ( typeof crypto.randomUUID === 'function' ) { - return crypto.randomUUID(); - } - return crypto.randomBytes(16).toString('hex'); - } -} diff --git a/src/backend/src/services/abuse-prevention/concurrentRequestLimiter/index.ts b/src/backend/src/services/abuse-prevention/concurrentRequestLimiter/index.ts deleted file mode 100644 index cff3dd37b..000000000 --- a/src/backend/src/services/abuse-prevention/concurrentRequestLimiter/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { ConcurrentRequestLimiter } from './ConcurrentRequestLimiter.js'; - -export const concurrentRequestLimiter = new ConcurrentRequestLimiter(); diff --git a/src/backend/src/services/abuse-prevention/concurrentRequestLimiter/types.ts b/src/backend/src/services/abuse-prevention/concurrentRequestLimiter/types.ts deleted file mode 100644 index 37d05238d..000000000 --- a/src/backend/src/services/abuse-prevention/concurrentRequestLimiter/types.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Actor } from '../../auth/Actor'; - -export interface SimpleLimitConfig { - limit: number; -} - -export type GroupLimitConfig = { default: SimpleLimitConfig } & Record; - -export type ConcurrentLimitConfig = SimpleLimitConfig | GroupLimitConfig; - -export interface CheckAndIncrementConcurrentOptions { - actor: Actor; - key: string; - leaseMs?: number; -} - -export interface ConcurrentPermit { - key: string; - redisKey: string; - token: string; - userId: string; - userGroup: string; - limit: number; - expiresAt: number; -} - -export interface CheckAndIncrementConcurrentResult { - allowed: boolean; - limit: number; - activeCount: number; - userGroup: string; - permit?: ConcurrentPermit; -} diff --git a/src/backend/src/services/ai/AIInterfaceService.js b/src/backend/src/services/ai/AIInterfaceService.js deleted file mode 100644 index 933e7fa4e..000000000 --- a/src/backend/src/services/ai/AIInterfaceService.js +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const BaseService = require('../BaseService'); - -/** -* Service class that manages AI interface registrations and configurations. -* Handles registration of various AI services including OCR, chat completion, -* image generation, and text-to-speech interfaces. Each interface defines -* its available methods, parameters, and expected results. -* @extends BaseService -*/ -class AIInterfaceService extends BaseService { - /** - * Service class for managing AI interface registrations and configurations. - * Extends the base service to provide AI-related interface management. - * Handles registration of OCR, chat completion, image generation, and TTS interfaces. - */ - async '__on_driver.register.interfaces' () { - const svc_registry = this.services.get('registry'); - const col_interfaces = svc_registry.get('interfaces'); - - col_interfaces.set('puter-ocr', { - description: 'Optical character recognition', - methods: { - recognize: { - description: 'Recognize text in an image or document.', - parameters: { - source: { - type: 'file', - }, - model: { - type: 'string', - optional: true, - }, - pages: { - type: 'json', - subtype: 'array', - optional: true, - }, - includeImageBase64: { - type: 'flag', - optional: true, - }, - imageLimit: { - type: 'number', - optional: true, - }, - imageMinSize: { - type: 'number', - optional: true, - }, - bboxAnnotationFormat: { - type: 'json', - optional: true, - }, - documentAnnotationFormat: { - type: 'json', - optional: true, - }, - }, - result: { - type: { - $: 'stream', - content_type: 'image', - }, - }, - }, - }, - }); - - col_interfaces.set('puter-chat-completion', { - description: 'Chatbot.', - methods: { - models: { - description: 'List supported models and their details.', - result: { type: 'json' }, - parameters: {}, - }, - list: { - description: 'List supported models', - result: { type: 'json' }, - parameters: {}, - }, - complete: { - description: 'Get completions for a chat log.', - parameters: { - messages: { type: 'json' }, - tools: { type: 'json' }, - vision: { type: 'flag' }, - stream: { type: 'flag' }, - response: { type: 'json' }, - reasoning: { type: 'json', optional: true }, - reasoning_effort: { type: 'string', optional: true }, - text: { type: 'json', optional: true }, - verbosity: { type: 'string', optional: true }, - model: { type: 'string' }, - provider: { type: 'string', optional: true }, - temperature: { type: 'number' }, - max_tokens: { type: 'number' }, - image_config: { type: 'json', optional: true }, - }, - result: { type: 'json' }, - }, - }, - }); - - col_interfaces.set('puter-image-generation', { - description: 'AI Image Generation.', - methods: { - generate: { - description: 'Generate an image from a prompt.', - parameters: { - prompt: { type: 'string' }, - quality: { type: 'string' }, - model: { type: 'string' }, - provider: { type: 'string', optional: true }, - ratio: { type: 'json' }, - width: { type: 'number', optional: true }, - height: { type: 'number', optional: true }, - aspect_ratio: { type: 'string', optional: true }, - steps: { type: 'number', optional: true }, - seed: { type: 'number', optional: true }, - negative_prompt: { type: 'string', optional: true }, - n: { type: 'number', optional: true }, - input_image: { type: 'string', optional: true }, - input_image_mime_type: { type: 'string', optional: true }, - input_images: { type: 'json', optional: true }, - image_url: { type: 'string', optional: true }, - image_base64: { type: 'string', optional: true }, - mask_image_url: { type: 'string', optional: true }, - mask_image_base64: { type: 'string', optional: true }, - prompt_strength: { type: 'number', optional: true }, - guidance: { type: 'number', optional: true }, - output_quality: { type: 'number', optional: true }, - output_megapixels: { type: 'string', optional: true }, - go_fast: { type: 'json', optional: true }, - disable_safety_checker: { type: 'flag', optional: true }, - response_format: { type: 'string', optional: true }, - }, - result_choices: [ - { - names: ['image'], - type: { - $: 'stream', - content_type: 'image', - }, - }, - { - names: ['url'], - type: { - $: 'string:url:web', - content_type: 'image', - }, - }, - ], - result: { - description: 'URL of the generated image.', - type: 'string', - }, - }, - }, - }); - - col_interfaces.set('puter-video-generation', { - description: 'AI Video Generation.', - methods: { - generate: { - description: 'Generate a video from a prompt.', - parameters: { - prompt: { type: 'string' }, - model: { type: 'string', optional: true }, - seconds: { type: 'number', optional: true }, - duration: { type: 'number', optional: true }, - size: { type: 'string', optional: true }, - resolution: { type: 'string', optional: true }, - width: { type: 'number', optional: true }, - height: { type: 'number', optional: true }, - fps: { type: 'number', optional: true }, - steps: { type: 'number', optional: true }, - guidance_scale: { type: 'number', optional: true }, - seed: { type: 'number', optional: true }, - output_format: { type: 'string', optional: true }, - output_quality: { type: 'number', optional: true }, - negative_prompt: { type: 'string', optional: true }, - reference_images: { type: 'json', optional: true }, - frame_images: { type: 'json', optional: true }, - metadata: { type: 'json', optional: true }, - input_reference: { type: 'file', optional: true }, - no_extra_params: { type: 'flag', optional: true }, - }, - result_choices: [ - { - names: ['url'], - type: { - $: 'string:url:web', - content_type: 'video', - }, - }, - { - names: ['video'], - type: { - $: 'stream', - content_type: 'video', - }, - }, - ], - result: { - description: 'Video asset descriptor or URL for the generated video.', - type: 'json', - }, - }, - }, - }); - - col_interfaces.set('puter-tts', { - description: 'Text-to-speech.', - methods: { - list_voices: { - description: 'List available voices.', - parameters: { - engine: { type: 'string', optional: true }, - provider: { type: 'string', optional: true }, - }, - }, - list_engines: { - description: 'List available TTS engines with pricing information.', - parameters: { - provider: { type: 'string', optional: true }, - }, - result: { type: 'json' }, - }, - synthesize: { - description: 'Synthesize speech from text.', - parameters: { - text: { type: 'string' }, - voice: { type: 'string' }, - language: { type: 'string' }, - ssml: { type: 'flag' }, - engine: { type: 'string', optional: true }, - model: { type: 'string', optional: true }, - response_format: { type: 'string', optional: true }, - instructions: { type: 'string', optional: true }, - provider: { type: 'string', optional: true }, - }, - result_choices: [ - { - names: ['audio'], - type: { - $: 'stream', - content_type: 'audio', - }, - }, - ], - }, - }, - }); - - col_interfaces.set('puter-speech2speech', { - description: 'Speech to speech voice conversion (voice changer).', - methods: { - convert: { - description: 'Convert input audio to a target voice.', - parameters: { - audio: { type: 'file' }, - voice: { type: 'string', optional: true }, - voice_id: { type: 'string', optional: true }, - model: { type: 'string', optional: true }, - output_format: { type: 'string', optional: true }, - voice_settings: { type: 'json', optional: true }, - seed: { type: 'number', optional: true }, - remove_background_noise: { type: 'flag', optional: true }, - file_format: { type: 'string', optional: true }, - optimize_streaming_latency: { type: 'number', optional: true }, - enable_logging: { type: 'flag', optional: true }, - }, - result_choices: [ - { - names: ['audio'], - type: { - $: 'stream', - content_type: 'audio', - }, - }, - ], - }, - }, - }); - - col_interfaces.set('puter-speech2txt', { - description: 'Speech to text transcription and translation.', - methods: { - list_models: { - description: 'List available speech-to-text models.', - result: { type: 'json' }, - }, - transcribe: { - description: 'Transcribe audio into text.', - parameters: { - file: { type: 'file' }, - model: { type: 'string', optional: true }, - response_format: { type: 'string', optional: true }, - language: { type: 'string', optional: true }, - prompt: { type: 'string', optional: true }, - temperature: { type: 'number', optional: true }, - logprobs: { type: 'flag', optional: true }, - timestamp_granularities: { type: 'json', optional: true }, - stream: { type: 'flag', optional: true }, - chunking_strategy: { type: 'string', optional: true }, - known_speaker_names: { type: 'json', optional: true }, - known_speaker_references: { type: 'json', optional: true }, - extra_body: { type: 'json', optional: true }, - }, - result: { type: 'json' }, - }, - translate: { - description: 'Translate audio into English text.', - parameters: { - file: { type: 'file' }, - model: { type: 'string', optional: true }, - response_format: { type: 'string', optional: true }, - prompt: { type: 'string', optional: true }, - temperature: { type: 'number', optional: true }, - logprobs: { type: 'flag', optional: true }, - timestamp_granularities: { type: 'json', optional: true }, - stream: { type: 'flag', optional: true }, - extra_body: { type: 'json', optional: true }, - }, - result: { type: 'json' }, - }, - }, - }); - } -} - -module.exports = { - AIInterfaceService, -}; diff --git a/src/backend/src/services/ai/README.md b/src/backend/src/services/ai/README.md deleted file mode 100644 index 9261fcfad..000000000 --- a/src/backend/src/services/ai/README.md +++ /dev/null @@ -1,276 +0,0 @@ -# AI Services - -CoreModule registers the backend AI services directly. -These services cover chat, image generation, video generation, speech, and OCR. -Some providers are only registered when the corresponding configuration is present, -including AWS, OpenAI, and ElevenLabs integrations. - -## Services - -### AIChatService - -AIChatService class extends BaseService to provide AI chat completion functionality. -Manages multiple AI providers, models, and fallback mechanisms for chat interactions. -Handles model registration, usage tracking, cost calculation, content moderation, -and implements the puter-chat-completion driver interface. Supports streaming responses -and maintains detailed model information including pricing and capabilities. - -#### Listeners - -##### `boot.consolidation` - -Handles consolidation during service boot by registering service aliases -and populating model lists/maps from providers. - -Registers each provider as an 'ai-chat' service alias and fetches their -available models and pricing information. Populates: -- simple_model_list: Basic list of supported models -- detail_model_list: Detailed model info including costs -- detail_model_map: Maps model IDs/aliases to their details - -#### Methods - -##### `register_provider` - - - -##### `moderate` - -Moderates chat messages for inappropriate content using OpenAI's moderation service - -###### Parameters - -- **params:** The parameters object -- **params.messages:** Array of chat messages to moderate - -##### `get_delegate` - -Gets the appropriate delegate service for handling chat completion requests. -If the intended service is this service (ai-chat), returns undefined. -Otherwise returns the intended service wrapped as a puter-chat-completion interface. - -##### `get_fallback_model` - -Find an appropriate fallback model by sorting the list of models -by the euclidean distance of the input/output prices and selecting -the first one that is not in the tried list. - -###### Parameters - -- **param0:** null - -##### `get_model_from_request` - - - -### AIInterfaceService - -Service class that manages AI interface registrations and configurations. -Handles registration of various AI services including OCR, chat completion, -image generation, and text-to-speech interfaces. Each interface defines -its available methods, parameters, and expected results. - -#### Listeners - -##### `driver.register.interfaces` - -Service class for managing AI interface registrations and configurations. -Extends the base service to provide AI-related interface management. -Handles registration of OCR, chat completion, image generation, and TTS interfaces. - -### AITestModeService - -Service class that handles AI test mode functionality. -Extends BaseService to register test services for AI chat completions. -Used for testing and development of AI-related features by providing -a mock implementation of the chat completion service. - -### AWSPollyService - -AWSPollyService class provides text-to-speech functionality using Amazon Polly. -Extends BaseService to integrate with AWS Polly for voice synthesis operations. -Implements voice listing, speech synthesis, and voice selection based on language. -Includes caching for voice descriptions and supports both text and SSML inputs. - -#### Methods - -##### `describe_voices` - -Describes available AWS Polly voices and caches the results - -##### `synthesize_speech` - -Synthesizes speech from text using AWS Polly - -###### Parameters - -- **text:** The text to synthesize -- **options:** Synthesis options -- **options.format:** Output audio format (e.g. 'mp3') - -### AWSTextractService - -AWSTextractService class - Provides OCR (Optical Character Recognition) functionality using AWS Textract -Extends BaseService to integrate with AWS Textract for document analysis and text extraction. -Implements driver capabilities and puter-ocr interface for document recognition. -Handles both S3-stored and buffer-based document processing with automatic region management. - -#### Methods - -##### `analyze_document` - -Analyzes a document using AWS Textract to extract text and layout information - -###### Parameters - -- **file_facade:** Interface to access the document file - -#### Methods - -##### `get_system_prompt` - -Service that emulates Claude's behavior using alternative AI models - -##### `adapt_model` - - - -### ClaudeService - -ClaudeService class extends BaseService to provide integration with Anthropic's Claude AI models. -Implements the puter-chat-completion interface for handling AI chat interactions. -Manages message streaming, token limits, model selection, and API communication with Claude. -Supports system prompts, message adaptation, and usage tracking. - -#### Methods - -##### `get_default_model` - -Returns the default model identifier for Claude API interactions - -### FakeChatService - -FakeChatService - A mock implementation of a chat service that extends BaseService. -Provides fake chat completion responses using Lorem Ipsum text generation. -Used for testing and development purposes when a real chat service is not needed. -Implements the 'puter-chat-completion' interface with list() and complete() methods. - -### GroqAIService - -Service class for integrating with Groq AI's language models. -Extends BaseService to provide chat completion capabilities through the Groq API. -Implements the puter-chat-completion interface for model management and text generation. -Supports both streaming and non-streaming responses, handles multiple models including -various versions of Llama, Mixtral, and Gemma, and manages usage tracking. - -#### Methods - -##### `get_default_model` - -Returns the default model ID for the Groq AI service - -### MistralAIService - -MistralAIService class extends BaseService to provide integration with the Mistral AI API. -Implements chat completion functionality with support for various Mistral models including -mistral-large, pixtral, codestral, and ministral variants. Handles both streaming and -non-streaming responses, token usage tracking, and model management. Provides cost information -for different models and implements the puter-chat-completion interface. - -#### Methods - -##### `get_default_model` - -Populates the internal models array with available Mistral AI models and their metadata -Fetches model data from the API, filters based on cost configuration, and stores -model objects containing ID, name, aliases, context length, capabilities, and pricing - -### OpenAICompletionService - -OpenAICompletionService class provides an interface to OpenAI's chat completion API. -Extends BaseService to handle chat completions, message moderation, token counting, -and streaming responses. Implements the puter-chat-completion interface and manages -OpenAI API interactions with support for multiple models including GPT-4 variants. -Handles usage tracking, spending records, and content moderation. - -#### Methods - -##### `get_default_model` - -Gets the default model identifier for OpenAI completions - -##### `check_moderation` - -Checks text content against OpenAI's moderation API for inappropriate content - -###### Parameters - -- **text:** The text content to check for moderation - -##### `complete` - -Completes a chat conversation using OpenAI's API - -###### Parameters - -- **messages:** Array of message objects or strings representing the conversation -- **options:** Configuration options -- **options.stream:** Whether to stream the response -- **options.moderation:** Whether to perform content moderation -- **options.model:** The model to use for completion - -### OpenAIImageGenerationService - -Service class for generating images using OpenAI's DALL-E API. -Extends BaseService to provide image generation capabilities through -the puter-image-generation interface. Supports different aspect ratios -(square, portrait, landscape) and handles API authentication, request -validation, and spending tracking. - -#### Methods - -##### `generate` - - - -### TogetherAIService - -TogetherAIService class provides integration with Together AI's language models. -Extends BaseService to implement chat completion functionality through the -puter-chat-completion interface. Manages model listings, chat completions, -and streaming responses while handling usage tracking and model fallback testing. - -#### Methods - -##### `get_default_model` - -Returns the default model ID for the Together AI service - -### XAIService - -XAIService class - Provides integration with X.AI's API for chat completions -Extends BaseService to implement the puter-chat-completion interface. -Handles model management, message adaptation, streaming responses, -and usage tracking for X.AI's language models like Grok. - -#### Methods - -##### `get_system_prompt` - -Gets the system prompt used for AI interactions - -##### `adapt_model` - - - -##### `get_default_model` - -Returns the default model identifier for the XAI service - -## Notes - -### Outside Imports - -This module has external relative imports. When these are -removed it may become possible to move this module to an -extension. diff --git a/src/backend/src/services/ai/chat/.gitignore b/src/backend/src/services/ai/chat/.gitignore deleted file mode 100644 index aa4a6da26..000000000 --- a/src/backend/src/services/ai/chat/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.js -*.js.map \ No newline at end of file diff --git a/src/backend/src/services/ai/chat/AIChatRedisCacheSpace.ts b/src/backend/src/services/ai/chat/AIChatRedisCacheSpace.ts deleted file mode 100644 index 83835a1c8..000000000 --- a/src/backend/src/services/ai/chat/AIChatRedisCacheSpace.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -export const fallbackModelsKey = (modelId: string) => `aichat:fallbacks:${modelId}`; diff --git a/src/backend/src/services/ai/chat/AIChatService.ts b/src/backend/src/services/ai/chat/AIChatService.ts deleted file mode 100644 index 74e10fd91..000000000 --- a/src/backend/src/services/ai/chat/AIChatService.ts +++ /dev/null @@ -1,870 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { createId as cuid2 } from '@paralleldrive/cuid2'; -import { PassThrough } from 'stream'; -import { APIError } from '../../../api/APIError.js'; -import { setRedisCacheValue } from '../../../clients/redis/cacheUpdate.js'; -import { redisClient } from '../../../clients/redis/redisSingleton.js'; -import { ErrorService } from '../../../modules/core/ErrorService.js'; -import { Context } from '../../../util/context.js'; -import { concurrentRequestLimiter } from '../../abuse-prevention/concurrentRequestLimiter/index.js'; -import type { GroupLimitConfig } from '../../abuse-prevention/concurrentRequestLimiter/types.js'; -import BaseService from '../../BaseService.js'; -import { BaseDatabaseAccessService } from '../../database/BaseDatabaseAccessService.js'; -import { DriverService } from '../../drivers/DriverService.js'; -import { TypedValue } from '../../drivers/meta/Runtime.js'; -import { EventService } from '../../EventService.js'; -import { MeteringService } from '../../MeteringService/MeteringService.js'; -import { AsModeration } from '../moderation/AsModeration.js'; -import { normalize_tools_object } from '../utils/FunctionCalling.js'; -import { extract_text, normalize_messages, normalize_single_message } from '../utils/Messages.js'; -import Streaming from '../utils/Streaming.js'; -import { fallbackModelsKey } from './AIChatRedisCacheSpace.js'; -import { ClaudeProvider } from './providers/ClaudeProvider/ClaudeProvider.js'; -import { DeepSeekProvider } from './providers/DeepSeekProvider/DeepSeekProvider.js'; -import { FakeChatProvider } from './providers/FakeChatProvider.js'; -import { GeminiChatProvider } from './providers/GeminiProvider/GeminiChatProvider.js'; -import { GroqAIProvider } from './providers/GroqAiProvider/GroqAIProvider.js'; -import { MistralAIProvider } from './providers/MistralAiProvider/MistralAiProvider.js'; -import { OllamaChatProvider } from './providers/OllamaProvider.js'; -import { OpenAiChatProvider } from './providers/OpenAiProvider/OpenAiChatCompletionsProvider.js'; -import { OpenAiResponsesChatProvider } from './providers/OpenAiProvider/OpenAiChatResponsesProvider.js'; -import { OpenRouterProvider } from './providers/OpenRouterProvider/OpenRouterProvider.js'; -import { TogetherAIProvider } from './providers/TogetherAiProvider/TogetherAIProvider.js'; -import { IChatModel, IChatProvider, ICompleteArguments } from './providers/types.js'; -import { XAIProvider } from './providers/XAIProvider/XAIProvider.js'; - -// Maximum number of fallback attempts when a model fails, including the first attempt -const MAX_FALLBACKS = 3 + 1; // includes first attempt -const aiChatConcurrentLimitKey = 'ai-chat.complete'; -const defaultAiChatConcurrentLeaseMs = 2 * 60 * 1000; - -export class AIChatService extends BaseService { - - static SERVICE_NAME = 'ai-chat'; - - static DEFAULT_PROVIDER = 'openai-completion'; - - get meteringService (): MeteringService { - return this.services.get('meteringService').meteringService; - } - - get db (): BaseDatabaseAccessService { - return this.services.get('database').get(); - } - - get errorService (): ErrorService { - return this.services.get('error-service') as ErrorService; - } - - get eventService (): EventService { - return this.services.get('event'); - } - - get driverService (): DriverService { - return this.services.get('driver') as DriverService; - } - - getProvider (name: string): IChatProvider | undefined { - return this.#providers[name]; - } - - #providers: Record = {}; - #modelIdMap: Record = {}; - - #toLimitValue (rawLimit: unknown): number | null { - if ( typeof rawLimit === 'number' && Number.isFinite(rawLimit) && rawLimit > 0 ) { - return rawLimit; - } - - if ( rawLimit && typeof rawLimit === 'object' && 'limit' in rawLimit ) { - const nestedLimit = Number((rawLimit as { limit?: unknown }).limit); - if ( Number.isFinite(nestedLimit) && nestedLimit > 0 ) { - return nestedLimit; - } - } - - return null; - } - - #getAiChatConcurrentLimitConfig (): GroupLimitConfig { - const limitConfig: GroupLimitConfig = { - default: { limit: 3 }, - temp_free: { limit: 3 }, - user_free: { limit: 5 }, - }; - - const subscriptionLimits = this.config?.concurrentRequests?.subscriptionLimits; - if ( !subscriptionLimits || typeof subscriptionLimits !== 'object' || Array.isArray(subscriptionLimits) ) { - return limitConfig; - } - - for ( const [subscriptionId, rawLimit] of Object.entries(subscriptionLimits) ) { - const parsedLimit = this.#toLimitValue(rawLimit); - if ( ! parsedLimit ) { - continue; - } - limitConfig[subscriptionId] = { limit: parsedLimit }; - } - - return limitConfig; - } - - #getAiChatConcurrentLeaseMs (): number { - const rawLeaseMs = this.config?.concurrentRequests?.leaseMs; - const leaseMs = Number(rawLeaseMs); - if ( Number.isFinite(leaseMs) && leaseMs > 0 ) { - return leaseMs; - } - return defaultAiChatConcurrentLeaseMs; - } - - /** Driver interfaces */ - static IMPLEMENTS = { - 'driver-capabilities': { - supports_test_mode (iface: string, method_name: string) { - return iface === 'puter-chat-completion' && - method_name === 'complete'; - }, - }, - 'puter-chat-completion': { - - async models () { - return await (this as unknown as AIChatService).models(); - }, - - async list () { - return await (this as unknown as AIChatService).list(); - }, - - async complete (...parameters: Parameters) { - return await (this as unknown as AIChatService).complete(...parameters); - }, - }, - }; - - getModel ({ modelId, provider }: { modelId: string, provider?: string }) { - const models = this.#modelIdMap[modelId]; - - if ( ! models ) { - throw new Error('Model not found, please try one of the following models listed here: https://developer.puter.com/ai/models/'); - } - if ( ! provider ) { - return models[0]; - } - const model = models.find(m => m.provider === provider); - return model ?? models[0]; - } - - private async registerProviders () { - const claudeConfig = this.config.providers?.['claude'] || this.global_config?.services?.['claude']; - if ( claudeConfig && claudeConfig.apiKey ) { - this.#providers['claude'] = new ClaudeProvider(this.meteringService, claudeConfig, this.errorService); - } - const openAiConfig = this.config.providers?.['openai-completion'] || this.global_config?.services?.['openai-completion'] || this.global_config?.openai; - if ( openAiConfig && (openAiConfig.apiKey || openAiConfig.secret_key) ) { - this.#providers['openai-completion'] = new OpenAiChatProvider(this.meteringService, openAiConfig); - this.#providers['openai-responses'] = new OpenAiResponsesChatProvider(this.meteringService, openAiConfig); - } - const geminiConfig = this.config.providers?.['gemini'] || this.global_config?.services?.['gemini']; - if ( geminiConfig && geminiConfig.apiKey ) { - this.#providers['gemini'] = new GeminiChatProvider(this.meteringService, geminiConfig); - } - const groqConfig = this.config.providers?.['groq'] || this.global_config?.services?.['groq']; - if ( groqConfig && groqConfig.apiKey ) { - this.#providers['groq'] = new GroqAIProvider(groqConfig, this.meteringService); - } - const deepSeekConfig = this.config.providers?.['deepseek'] || this.global_config?.services?.['deepseek']; - if ( deepSeekConfig && deepSeekConfig.apiKey ) { - this.#providers['deepseek'] = new DeepSeekProvider(deepSeekConfig, this.meteringService); - } - const mistralConfig = this.config.providers?.['mistral'] || this.global_config?.services?.['mistral']; - if ( mistralConfig && mistralConfig.apiKey ) { - this.#providers['mistral'] = new MistralAIProvider(mistralConfig, this.meteringService); - } - const xaiConfig = this.config.providers?.['xai'] || this.global_config?.services?.['xai']; - if ( xaiConfig && xaiConfig.apiKey ) { - this.#providers['xai'] = new XAIProvider(xaiConfig, this.meteringService); - } - const openrouterConfig = this.config.providers?.['openrouter'] || this.global_config?.services?.['openrouter']; - if ( openrouterConfig && openrouterConfig.apiKey ) { - this.#providers['openrouter'] = new OpenRouterProvider(openrouterConfig, this.meteringService); - } - const togetherConfig = this.config.providers?.['together-ai'] || this.global_config?.services?.['together-ai']; - if ( togetherConfig && togetherConfig.apiKey ) { - this.#providers['together-ai'] = new TogetherAIProvider(togetherConfig, this.meteringService); - } - - // ollama if local instance detected - - // Autodiscover Ollama service and then check if its disabled in the config - // if config.services.ollama.enabled is undefined, it means the user hasn't set it, so we should default to true - const ollamaConfig = this.config.providers?.['ollama'] || this.global_config?.services?.ollama; - const ollama_available = await fetch('http://localhost:11434/api/tags').then(resp => resp.json()).then(_data => { - if ( ollamaConfig?.enabled === undefined ) { - return true; - } - return ollamaConfig?.enabled; - }).catch(_err => { - return false; - }); - // User can disable ollama in the config, but by default it should be enabled if discovery is successful - if ( ollama_available || ollamaConfig?.enabled ) { - console.log('🦙 Ollama support detected! Enabling local AI support'); - this.#providers['ollama'] = new OllamaChatProvider(ollamaConfig, this.meteringService); - } - - // fake providers last - this.#providers['fake-chat'] = new FakeChatProvider(); - - // emit event for extensions to add providers - const extensionProviders = {} as Record; - await this.eventService.emit('ai.chat.registerProviders', extensionProviders); - for ( const providerName in extensionProviders ) { - if ( this.#providers[providerName] ) { - console.warn('AIChatService: provider name conflict for ', providerName, ' registering with -extension suffix'); - this.#providers[`${providerName}-extension`] = extensionProviders[providerName]; - continue; - } - this.#providers[providerName] = extensionProviders[providerName]; - } - } - - protected async '__on_boot.consolidation' () { - // register - concurrentRequestLimiter.registerLimitKey( - aiChatConcurrentLimitKey, - this.#getAiChatConcurrentLimitConfig(), - ); - - // register chat providers here - await this.registerProviders(); - - // build model id map - for ( const providerName in this.#providers ) { - const provider = this.#providers[providerName]; - - // alias all driver requests to go here to support legacy routing - this.driverService.register_service_alias( - AIChatService.SERVICE_NAME, - providerName, - { iface: 'puter-chat-completion' }, - ); - - // build model id map - for ( const model of await provider.models() ) { - model.id = model.id.trim().toLowerCase(); - if ( ! this.#modelIdMap[model.id] ) { - this.#modelIdMap[model.id] = []; - } - this.#modelIdMap[model.id].push({ ...model, provider: providerName }); - if ( model.puterId ) { - if ( model.aliases ) { - model.aliases.push(model.puterId); - } else { - model.aliases = [model.puterId]; - } - } - - let exists = false; - if ( model.aliases ) { - for ( let alias of model.aliases ) { - if ( this.#modelIdMap[alias] && this.#modelIdMap[alias] !== this.#modelIdMap[model.id] ) { - if ( providerName === 'together-ai' || providerName === 'openrouter' ) { - if ( this.#modelIdMap[alias].find(m => m.provider === 'gemini') ) { - // enable openrouter gemini for now since exposing some tools we don't - continue; - } - delete this.#modelIdMap[model.id]; - exists = true; - break; - } - } - } - } - if ( exists ) { - continue; - } - - if ( model.aliases ) { - for ( let alias of model.aliases ) { - alias = alias.trim().toLowerCase(); - // join arrays which are aliased the same - if ( ! this.#modelIdMap[alias] ) { - this.#modelIdMap[alias] = this.#modelIdMap[model.id]; - continue; - } - if ( this.#modelIdMap[alias] !== this.#modelIdMap[model.id] ) { - this.#modelIdMap[alias].push({ ...model, provider: providerName }); - this.#modelIdMap[model.id] = this.#modelIdMap[alias]; - continue; - } - } - } - this.#modelIdMap[model.id].sort((a, b) => { - // Sort togetherai provider models last - if ( a.provider === 'together-ai' && b.provider !== 'together-ai' ) { - return 1; - } - if ( b.provider === 'together-ai' && a.provider !== 'together-ai' ) { - return -1; - } - - if ( a.costs[a.input_cost_key || 'input_tokens'] === b.costs[b.input_cost_key || 'input_tokens'] ) { - return a.id.length - b.id.length; // use shorter id since its likely the official one - } - return a.costs[a.input_cost_key || 'input_tokens'] - b.costs[b.input_cost_key || 'input_tokens']; - }); - } - } - } - - models () { - const seen = new Set(); - return Object.entries(this.#modelIdMap) - .map(([_, models]) => models) - .flat() - .filter(model => { - if ( seen.has(model.id) ) { - return false; - } - seen.add(model.id); - return true; - }) - .sort((a, b) => { - if ( a.provider === b.provider ) { - return a.id.localeCompare(b.id); - } - return a.provider!.localeCompare(b.provider!); - }); - } - - list () { - return this.models().map(m => (m.puterId || m.id)).sort(); - } - - async complete (parameters: ICompleteArguments) { - const clientDriverCall = Context.get('client_driver_call') as { - test_mode?: boolean; - response_metadata?: Record; - intended_service?: string; - } | undefined; - const fallbackDriverCall = { - test_mode: false, - response_metadata: {}, - intended_service: undefined, - } as { - test_mode?: boolean; - response_metadata?: Record; - intended_service?: string; - }; - let { test_mode: testMode, response_metadata: resMetadata, intended_service: legacyProviderName } = - clientDriverCall ?? fallbackDriverCall; - resMetadata = (resMetadata ?? {}) as Record; - const actor = Context.get('actor'); - - const concurrentRequestAllowance = await concurrentRequestLimiter.checkAndIncrementConcurrent({ - actor, - key: aiChatConcurrentLimitKey, - leaseMs: this.#getAiChatConcurrentLeaseMs(), - }); - if ( ! concurrentRequestAllowance.allowed ) { - throw APIError.create('too_many_requests', undefined, { - message: `Concurrent request limit reached (${concurrentRequestAllowance.activeCount}/${concurrentRequestAllowance.limit})`, - }); - } - - let concurrentPermit = concurrentRequestAllowance.permit; - const releaseConcurrentPermit = async () => { - if ( ! concurrentPermit ) return; - const permit = concurrentPermit; - concurrentPermit = undefined; - await concurrentRequestLimiter.decrementConcurrent(permit); - }; - - try { - let intendedProvider = parameters.provider || (legacyProviderName === AIChatService.SERVICE_NAME ? '' : legacyProviderName); // should now all go through here - - if ( !parameters.model && !intendedProvider ) { - intendedProvider = AIChatService.DEFAULT_PROVIDER; - } - if ( !parameters.model && intendedProvider ) { - parameters.model = this.#providers[intendedProvider].getDefaultModel(); - } - let model = this.getModel({ modelId: parameters.model, provider: intendedProvider }) || await this.getFallbackModel(parameters.model, [], []); - const abuseModel = this.getModel({ modelId: 'abuse' }); - - const completionId = cuid2(); - const event = { - actor, - completionId, - allow: true, - intended_service: intendedProvider || '', - parameters, - } as Record; - - // If we reach here with a suspended user, block and log; this shouldn't happen - const user = actor.type.user ?? actor.type?.authorizer?.type?.user ?? Context.get('user'); - if ( ! user ) { - this.errors.report('this should not happen: no user in AIChatService', { - trace: true, - }); - throw APIError.create('permission_denied'); - } - const svc_getUser = this.services.get('get-user'); - const nocache_user = await svc_getUser.get_user({ id: user.id, force: true }); - if ( nocache_user?.suspended ) { - this.errors.report('this should not happen: reached AIChatService with suspended user', { - trace: true, - }); - throw APIError.create('account_suspended'); - } - if ( user.requires_email_confirmation && !user.email_confirmed ) { - throw APIError.create('email_must_be_confirmed', null, { - action: 'use this service', - }); - } - - await this.eventService.emit('ai.prompt.validate', event); - if ( ! event.allow ) { - testMode = true; - if ( event.custom ) parameters.custom = event.custom; - } - - if ( parameters.messages ) { - parameters.messages = - normalize_messages(parameters.messages); - } - - // Skip moderation for Ollama (local service) and other local services - const should_moderate = !testMode && - parameters.provider !== 'ollama'; - - if ( should_moderate && !await this.moderate(parameters) ) { - testMode = true; - throw APIError.create('moderation_failed'); - } - - // Only set moderated flag if we actually ran moderation - if ( !testMode && should_moderate ) { - Context.set('moderated', true); - } - - if ( testMode ) { - if ( event.abuse ) { - model = abuseModel; - } - } - - if ( parameters.tools ) { - normalize_tools_object(parameters.tools); - } - - if ( ! model ) { - // TODO DS: route them to new endpoints once ready - const availableModelsUrl = `${this.global_config.origin }/puterai/chat/models`; - - throw APIError.create('field_invalid', undefined, { - key: 'model', - expected: `a valid model name from ${availableModelsUrl}`, - got: model, - }); - } - - const inputTokenCost = model.costs[model.input_cost_key || 'input_tokens'] as number; - const outputTokenCost = model.costs[model.output_cost_key || 'output_tokens'] as number; - const maxTokens = model.max_tokens; - const text = extract_text(parameters.messages); - const approximateTokenCount = Math.floor(((text.length / 4) + (text.split(/\s+/).length * (4 / 3))) / 2); // see https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them - const approximateInputCost = approximateTokenCount * inputTokenCost; - const minimumCredits = Math.max(model.minimumCredits || 1); - const usageAllowed = await this.meteringService.hasEnoughCredits(actor, Math.max(approximateInputCost, minimumCredits)); - - // Handle usage limits reached case - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds', new Error('No usage left for request.'), { - delegate: 'usage-limited-chat', - message: 'No usage left for request.', - }); - } - - // block non subscriber only models for non-subscribers - if ( model.subscriberOnly ) { - const eventObject = { actor, userSubscriptionId: '' }; - await this.eventService.emit('metering:getUserSubscription', eventObject); - if ( ! eventObject.userSubscriptionId ) { - //TODO DS: register checker events when we add more of these exclusions - throw APIError.create('permission_denied', undefined, { - message: `The model ${model.id} is only available to subscribers. Please subscribe to access this model.`, - }); - } - } - - const availableCredits = await this.meteringService.getRemainingUsage(actor); - const maxAllowedOutput = - availableCredits - approximateInputCost; - - const maxAllowedOutputTokens = - maxAllowedOutput / outputTokenCost; - - if ( maxAllowedOutputTokens ) { - parameters.max_tokens = Math.floor(Math.min( - parameters.max_tokens ?? Number.POSITIVE_INFINITY, - maxAllowedOutputTokens, - maxTokens - approximateTokenCount, - )); - if ( parameters.max_tokens < 1 ) { - parameters.max_tokens = undefined; - } - } - - // call model provider; - let res: Awaited>; - const provider = this.#providers[model.provider!]; - if ( ! provider ) { - throw new Error(`no provider found for model ${model.id}`); - } - const attempts: { model: string; provider: string; error: string }[] = []; - try { - res = await provider.complete({ - ...parameters, - model: model.id, - provider: model.provider, - }); - } catch (e) { - const tried: string[] = []; - const triedProviders: string[] = []; - - tried.push(model.id); - triedProviders.push(model.provider!); - - let error = e as Error; - attempts.push({ - model: model.id, - provider: model.provider!, - error: error?.message ?? String(e), - }); - - while ( error ) { - - // TODO: simplify our error handling - // Distinguishing between user errors and service errors - // is very messy because of different conventions between - // services. This is a best-effort attempt to catch user - // errors and throw them as 400s. - const isRequestError = (() => { - if ( error instanceof APIError ) { - return true; - } - if ( (error as unknown as { type: string }).type === 'invalid_request_error' ) { - return true; - } - })(); - - if ( isRequestError ) { - console.error((error as Error)); - throw APIError.create('error_400_from_delegate', error as Error, { - delegate: model.provider, - message: (error as Error).message, - }); - } - - if ( this.config.disable_fallback_mechanisms ) { - console.error((error as Error)); - throw error; - } - - console.error('error calling ai chat provider for model: ', model, '\n trying fallbacks...'); - - // No fallbacks for pseudo-models - if ( model.provider === 'fake-chat' ) { - break; - } - - const fallback = await this.getFallbackModel(model.id, tried, triedProviders); - - if ( ! fallback ) { - throw APIError.create('ai_chat_all_providers_failed', null, { - attempts, - }); - } - - const { - fallbackModelId, - fallbackProvider, - } = fallback; - - console.warn('model fallback', { - fallbackModelId, - fallbackProvider, - }); - - let fallBackModel = this.getModel({ modelId: fallbackModelId, provider: fallbackProvider }); - - tried.push(fallbackModelId); - triedProviders.push(fallbackProvider); - - if ( tried.length > MAX_FALLBACKS ) { - console.error('max fallbacks reached', { tried, triedProviders }); - break; - } - - const fallbackUsageAllowed = await this.meteringService.hasEnoughCredits(actor, 1); // we checked earlier, assume same costs - - if ( ! fallbackUsageAllowed ) { - throw APIError.create('insufficient_funds', new Error('No usage left for request.'), { - delegate: 'usage-limited-chat', - message: 'No usage left for request.', - }); - } - - const provider = this.#providers[fallBackModel.provider!]; - if ( ! provider ) { - throw new Error(`no provider found for model ${fallBackModel.id}`); - } - try { - res = await provider.complete({ - ...parameters, - model: fallBackModel.id, - provider: fallBackModel.provider, - }); - model = fallBackModel; - break; // success - } catch (e) { - console.error('error during fallback selection: ', e); - error = e as Error; - attempts.push({ - model: fallBackModel.id, - provider: fallBackModel.provider!, - error: error?.message ?? String(e), - }); - } - } - } - - resMetadata.service_used = model.provider; // legacy field - resMetadata.providerUsed = model.id; - - const username = actor.type?.user?.username; - - if ( ! res! ) { - throw APIError.create('ai_chat_all_providers_failed', null, { - attempts, - }); - } - - res.via_ai_chat_service = true; // legacy field always true now - if ( res.stream ) { - const originalFinallyFn = res.finally_fn; - res.finally_fn = async () => { - try { - if ( originalFinallyFn ) { - await originalFinallyFn(); - } - } finally { - await releaseConcurrentPermit(); - } - }; - - if ( res.init_chat_stream ) { - const stream = new PassThrough(); - // TODO DS: simplify how we handle streaming responses and remove custom runtime types - const retval = new TypedValue({ - $: 'stream', - content_type: 'application/x-ndjson', - chunked: true, - }, stream); - - const chatStream = new Streaming.AIChatStream({ - stream, - }); - - (async () => { - try { - await res.init_chat_stream({ chatStream }); - } catch (e) { - this.errors.report('error during stream response', { - source: e, - }); - stream.write(`${JSON.stringify({ - type: 'error', - message: (e as Error).message, - }) }\n`); - stream.end(); - } finally { - if ( res.finally_fn ) { - await res.finally_fn(); - } - } - })(); - - return retval; - } - - return res; - } - await this.eventService.emit('ai.prompt.complete', { - username, - intended_service: intendedProvider, - parameters, - result: res, - model_used: model.id, - service_used: model.provider, - }); - - if ( parameters.response?.normalize ) { - res = { - ...res, - message: normalize_single_message(res.message), - normalized: true, - }; - } - await releaseConcurrentPermit(); - return res; - } catch ( error ) { - await releaseConcurrentPermit(); - throw error; - } - } - - async moderate ({ messages }: { messages: Array; }) { - if ( process.env.TEST_MODERATION_FAILURE ) return false; - const fulltext = extract_text(messages); - let mod_last_error; - let mod_result: Awaited>; - try { - const openaiProvider = this.#providers['openai-completion']; - mod_result = await openaiProvider.checkModeration(fulltext); - if ( mod_result.flagged ) return false; - return true; - } catch (e) { - console.error(e); - mod_last_error = e; - } - try { - const claudeChatProvider = this.#providers['claude']; - const mod = new AsModeration({ - chatProvider: claudeChatProvider, - model: 'claude-3-haiku-20240307', - }); - if ( ! await mod.moderate(fulltext) ) { - return false; - } - mod_last_error = null; - return true; - } catch (e) { - console.error(e); - mod_last_error = e; - } - - if ( mod_last_error ) { - this.log.error('moderation error', { - fulltext, - mod_last_error, - }); - throw new Error('no working moderation service'); - } - return true; - } - - /** - * Find an appropriate fallback model by sorting the list of models - * by the euclidean distance of the input/output prices and selecting - * the first one that is not in the tried list. - * - * @param {*} param0 - * @returns - */ - async getFallbackModel (modelId: string, triedIds: string[], triedProviders: string[]) { - const models = this.#modelIdMap[modelId]; - - if ( ! models ) { - this.log.error('could not find model', { modelId }); - throw new Error('could not find model'); - } - - const targetModel = models[0]; - - // First see if any models with the same id but different provider exist - for ( const model of models ) { - if ( triedProviders.includes(model.provider!) ) continue; - if ( model.provider === 'fake-chat' ) continue; - return { - fallbackProvider: model.provider, - fallbackModelId: model.id, - }; - } - - // First check KV for the sorted list - let potentialFallbacks; - const cached_fallbacks = await redisClient.get(fallbackModelsKey(targetModel.id)); - if ( cached_fallbacks ) { - try { - potentialFallbacks = JSON.parse(cached_fallbacks); - } catch (e) { - // no-op cache in invalid state - } - } - - if ( ! potentialFallbacks ) { - // Calculate the sorted list - const models = this.models(); - - let aiProvider, modelToSearch; - if ( targetModel.id.startsWith('openrouter:') || targetModel.id.startsWith('togetherai:') ) { - [aiProvider, modelToSearch] = targetModel.id.replace('openrouter:', '').replace('togetherai:', '').toLowerCase().split('/'); - } else { - [aiProvider, modelToSearch] = [targetModel.provider!.toLowerCase().replace('gemini', 'google').replace('openai-completion', 'openai').replace('openai-responses', 'openai'), targetModel.id.toLowerCase()]; - } - - const potentialMatches = models.filter(model => { - const possibleModelNames = [`openrouter:${aiProvider}/${modelToSearch}`, - `togetherai:${aiProvider}/${modelToSearch}`, ...(targetModel.aliases?.map((alias) => [`openrouter:${aiProvider}/${alias}`, - `togetherai:${aiProvider}/${alias}`])?.flat() ?? [])]; - - return !!possibleModelNames.find(possibleName => model.id.toLowerCase() === possibleName); - }).slice(0, MAX_FALLBACKS); - - await setRedisCacheValue( - fallbackModelsKey(modelId), - JSON.stringify(potentialMatches), - { eventData: potentialMatches }, - ); - potentialFallbacks = potentialMatches; - } - - for ( const model of potentialFallbacks ) { - if ( triedIds.includes(model.id) ) continue; - if ( model.provider === 'fake-chat' ) continue; - - return { - fallbackProvider: model.provider, - fallbackModelId: model.id, - }; - } - - // No fallbacks available - console.error('no fallbacks', { - potentialFallbacks, - triedIds, - triedProviders, - }); - } -} diff --git a/src/backend/src/services/ai/chat/providers/ChatProvider.ts b/src/backend/src/services/ai/chat/providers/ChatProvider.ts deleted file mode 100644 index 78c400944..000000000 --- a/src/backend/src/services/ai/chat/providers/ChatProvider.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { ModerationCreateResponse } from 'openai/resources/moderations.js'; -import { IChatModel, IChatProvider, ICompleteArguments } from './types'; - -/** - * Abstract base class for AI chat providers, and default hollow implementation; - */ -export class ChatProvider implements IChatProvider { - getDefaultModel (): string { - return ''; - } - models (): IChatModel[] | Promise { - return []; - } - list (): string[] | Promise { - return []; - } - async checkModeration (_text: string): ReturnType { - return { - flagged: false, - results: {} as ModerationCreateResponse, - }; - } - async complete (_arg: ICompleteArguments): ReturnType { - throw new Error('Method not implemented.'); - } -} \ No newline at end of file diff --git a/src/backend/src/services/ai/chat/providers/ClaudeProvider/ClaudeProvider.test.ts b/src/backend/src/services/ai/chat/providers/ClaudeProvider/ClaudeProvider.test.ts deleted file mode 100644 index ea10f17b3..000000000 --- a/src/backend/src/services/ai/chat/providers/ClaudeProvider/ClaudeProvider.test.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { describe, expect, it, test } from 'vitest'; -import { createTestKernel } from '../../../../../../tools/test.mjs'; -import { SUService } from '../../../../SUService.js'; -import { ClaudeProvider } from './ClaudeProvider.js'; - -describe('ClaudeProvider ', async () => { - const testKernel = await createTestKernel({ - initLevelString: 'init', - testCore: true, - serviceConfigOverrideMap: { - 'database': { - path: ':memory:', - }, - 'dynamo': { - path: ':memory:', - }, - }, - }); - - const target = new ClaudeProvider(testKernel.services!.get('meteringService'), { apiKey: process.env.PUTER_CLAUDE_API_KEY || '' }, testKernel.services?.get('error-service')); - const su = testKernel.services!.get('su') as SUService; - - it('should have all models have cost in models json', async () => { - const models = target.models(); - - for ( const model of models ) { - expect(model.input_cost_key).toBeTruthy(); - expect(model.costs[model.input_cost_key!]).not.toBeNullable(); - expect(model.output_cost_key).toBeTruthy(); - expect(model.costs[model.output_cost_key!]).not.toBeNullable(); - } - }); - - test.skipIf(!process.env.PUTER_CLAUDE_API_KEY)('should return flat response from claude if token provided', async () => { - - const response = await su.sudo(async () => await target.complete({ - messages: [ - { role: 'user', content: 'Only reply: "hi"' }, - ], - model: 'claude-haiku-4-5-20251001', - max_tokens: 15, - })); - - expect(response.message.id).toBeDefined(); - expect(response.message.content.length).toBeGreaterThan(0); - expect(response.message.content[0].text).include('hi'); - expect(response.message.model).toEqual('claude-haiku-4-5-20251001'); - expect(response.message.usage).toBeDefined(); - expect(response.message.usage.output_tokens).toBeLessThan(15); - expect(response.finish_reason).toBe('stop'); - }); - -}); diff --git a/src/backend/src/services/ai/chat/providers/ClaudeProvider/ClaudeProvider.ts b/src/backend/src/services/ai/chat/providers/ClaudeProvider/ClaudeProvider.ts deleted file mode 100644 index 0f4919e85..000000000 --- a/src/backend/src/services/ai/chat/providers/ClaudeProvider/ClaudeProvider.ts +++ /dev/null @@ -1,504 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import Anthropic, { toFile } from '@anthropic-ai/sdk'; -import { Message } from '@anthropic-ai/sdk/resources'; -import { BetaUsage } from '@anthropic-ai/sdk/resources/beta.js'; -import { MessageCreateParams as BetaMessageCreateParams } from '@anthropic-ai/sdk/resources/beta/messages/messages.js'; -import { MessageCreateParams, Usage } from '@anthropic-ai/sdk/resources/messages.js'; -import mime from 'mime-types'; -import FSNodeParam from '../../../../../api/filesystem/FSNodeParam.js'; -import { LLRead } from '../../../../../deprecated/filesystem/ll_operations/ll_read.js'; -import { ErrorService } from '../../../../../modules/core/ErrorService.js'; -import { Context } from '../../../../../util/context.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import { make_claude_tools } from '../../../utils/FunctionCalling.js'; -import { extract_and_remove_system_messages } from '../../../utils/Messages.js'; -import { AIChatStream, AIChatTextStream, AIChatToolUseStream } from '../../../utils/Streaming.js'; -import { IChatProvider, ICompleteArguments } from '../types.js'; -import { CLAUDE_MODELS } from './models.js'; -export class ClaudeProvider implements IChatProvider { - anthropic: Anthropic; - - #meteringService: MeteringService; - - errorService: ErrorService; - - constructor (meteringService: MeteringService, config: { apiKey: string }, errorService: ErrorService) { - - this.#meteringService = meteringService; - this.errorService = errorService; - this.anthropic = new Anthropic({ - apiKey: config.apiKey, - // 10 minutes is the default; we need to override the timeout to - // disable an "aggressive" preemptive error that's thrown - // erroneously by the SDK. - // (https://github.com/anthropics/anthropic-sdk-typescript/issues/822) - timeout: 10 * 60 * 1001, - }); - } - getDefaultModel () { - return 'claude-haiku-4-5-20251001'; - } - - async list () { - const models = this.models(); - const model_names: string[] = []; - for ( const model of models ) { - model_names.push(model.id); - if ( model.aliases ) { - model_names.push(...model.aliases); - } - } - return model_names; - } - - async complete ({ messages, stream, model, tools, max_tokens, temperature, reasoning, reasoning_effort }: ICompleteArguments): ReturnType { - tools = make_claude_tools(tools); - - let system_prompts: string | any[]; - // unsure why system_prompts is an array but it always seems to only have exactly one element, - // and the real array of system_prompts seems to be the [0].content -- NS - [system_prompts, messages] = extract_and_remove_system_messages(messages); - - // Apply the cache control tag to all content blocks - if ( - system_prompts.length > 0 && - system_prompts[0].cache_control && - system_prompts[0]?.content - ) { - system_prompts[0].content = system_prompts[0].content.map((prompt: { cache_control: unknown }) => { - prompt.cache_control = system_prompts[0].cache_control; - return prompt; - }); - } - - messages = messages.map(message => { - if ( message.cache_control ) { - message.content[0].cache_control = message.cache_control; - } - delete message.cache_control; - return message; - }); - - // Convert OpenAI-style tool calls/results to Claude format. - messages = messages.map(message => { - if ( message.tool_calls && Array.isArray(message.tool_calls) ) { - if ( ! Array.isArray(message.content) ) { - message.content = message.content ? [message.content] : []; - } - for ( const toolCall of message.tool_calls ) { - message.content.push({ - type: 'tool_use', - id: toolCall.id, - name: toolCall.function?.name, - input: toolCall.function?.arguments ?? {}, - }); - } - delete message.tool_calls; - } - - if ( message.role !== 'tool' ) return message; - - const toolUseId = message.tool_call_id || message.tool_use_id; - const contentValue = (() => { - if ( Array.isArray(message.content) ) { - const toolResultBlock = message.content.find((part: any) => part?.type === 'tool_result'); - if ( toolResultBlock ) { - return toolResultBlock.content ?? toolResultBlock.text ?? ''; - } - return message.content.map((part: any) => { - if ( typeof part === 'string' ) return part; - if ( part && typeof part.text === 'string' ) return part.text; - if ( part && typeof part.content === 'string' ) return part.content; - return ''; - }).join(''); - } - if ( typeof message.content === 'string' ) return message.content; - if ( message.content && typeof message.content.text === 'string' ) return message.content.text; - if ( message.content && typeof message.content.content === 'string' ) return message.content.content; - return ''; - })(); - - return { - role: 'user', - content: [ - { - type: 'tool_result', - tool_use_id: toolUseId, - content: contentValue, - }, - ], - }; - }); - - // Claude requires tool_use.input to be a dictionary, not a JSON string. - messages = messages.map(message => { - if ( ! Array.isArray(message.content) ) return message; - message.content = message.content.map((part: any) => { - if ( part?.type !== 'tool_use' ) return part; - if ( typeof part.input === 'string' ) { - try { - part.input = JSON.parse(part.input); - } catch { - part.input = {}; - } - } else if ( part.input === undefined || part.input === null ) { - part.input = {}; - } - return part; - }); - return message; - }); - - const modelUsed = this.models().find(m => [m.id, ...(m.aliases || [])].includes(model)) || this.models().find(m => m.id === this.getDefaultModel())!; - const requestedReasoningEffort = reasoning_effort ?? reasoning?.effort; - const thinkingConfig = this.#buildThinkingConfig({ - modelId: modelUsed.id, - reasoningEffort: requestedReasoningEffort, - maxTokens: max_tokens, - }); - // Opus 4.7 errors on non-default sampling params; omit temperature entirely. - // Other models require temperature=1 when thinking is enabled. - const isOpus47 = modelUsed.id === 'claude-opus-4-7'; - const resolvedTemperature = isOpus47 ? undefined : (thinkingConfig ? 1 : (temperature ?? 0)); - const supportsEffort = [ - 'claude-opus-4-7', - 'claude-opus-4-6', - 'claude-sonnet-4-6', - ].includes(modelUsed.id); - const sdkParams: MessageCreateParams = { - model: modelUsed.id, - max_tokens: Math.floor(max_tokens || - (( - model === 'claude-3-5-sonnet-20241022' - || model === 'claude-3-5-sonnet-20240620' - ) ? 8192 : this.models().filter(e => (e.name === model || e.aliases?.includes(model)))[0]?.max_tokens || 4096)), //required - ...(resolvedTemperature !== undefined ? { temperature: resolvedTemperature } : {}), - ...( (system_prompts && system_prompts[0]?.content) ? { - system: system_prompts[0]?.content, - } : {}), - tool_choice: { - type: 'auto', - disable_parallel_tool_use: true, - }, - messages, - ...(tools ? { tools } : {}), - ...(thinkingConfig ? { thinking: thinkingConfig } : {}), - ...(supportsEffort && requestedReasoningEffort ? { output_config: { effort: requestedReasoningEffort } } : {}), - } as MessageCreateParams; - - let beta_mode = false; - - // Perform file uploads - const file_delete_tasks: { file_id: string }[] = []; - const actor = Context.get('actor'); - const { user } = actor.type; - - const file_input_tasks: any[] = []; - for ( const message of messages ) { - // We can assume `message.content` is not undefined because - // Messages.normalize_single_message ensures this. - for ( const contentPart of message.content ) { - if ( ! contentPart.puter_path ) continue; - file_input_tasks.push({ - node: await (new FSNodeParam(contentPart.puter_path)).consolidate({ - req: { user }, - getParam: () => contentPart.puter_path, - }), - contentPart, - }); - } - } - - const promises: Promise[] = []; - for ( const task of file_input_tasks ) { - promises.push((async () => { - const ll_read = new LLRead(); - const stream = await ll_read.run({ - actor: Context.get('actor'), - fsNode: task.node, - }); - - const mimeType = mime.contentType(await task.node.get('name')); - - beta_mode = true; - const fileUpload = await this.anthropic.beta.files.upload({ - file: await toFile(stream, undefined, { type: mimeType as string }), - }, { - betas: ['files-api-2025-04-14'], - } as Parameters[1]); - - file_delete_tasks.push({ file_id: fileUpload.id }); - // We have to copy a table from the documentation here: - // https://docs.anthropic.com/en/docs/build-with-claude/files - const contentBlockTypeForFileBasedOnMime = (() => { - if ( mimeType && mimeType.startsWith('image/') ) { - return 'image'; - } - if ( mimeType && mimeType.startsWith('text/') ) { - return 'document'; - } - if ( mimeType && mimeType === 'application/pdf' || mimeType === 'application/x-pdf' ) { - return 'document'; - } - return 'container_upload'; - })(); - - delete task.contentPart.puter_path; - task.contentPart.type = contentBlockTypeForFileBasedOnMime; - task.contentPart.source = { - type: 'file', - file_id: fileUpload.id, - }; - })()); - } - await Promise.all(promises); - - const cleanup_files = async () => { - const promises: Promise[] = []; - for ( const task of file_delete_tasks ) { - promises.push((async () => { - try { - await this.anthropic.beta.files.delete( - task.file_id, - { betas: ['files-api-2025-04-14'] }, - ); - } catch (e) { - this.errorService.report('claude:file-delete-task', { - source: e, - trace: true, - alarm: true, - extra: { file_id: task.file_id }, - }); - } - })()); - } - await Promise.all(promises); - }; - - if ( beta_mode ) { - (sdkParams as BetaMessageCreateParams).betas = ['files-api-2025-04-14']; - } - const anthropic = (beta_mode ? this.anthropic.beta : this.anthropic) as Anthropic; - - if ( stream ) { - const init_chat_stream = async ({ chatStream }: { chatStream: AIChatStream }) => { - const completion = await anthropic.messages.stream(sdkParams as MessageCreateParams); - const usageSum: Record = {}; - - let message, contentBlock; - let currentContentBlockType: string | null = null; - for await ( const event of completion ) { - - if ( event.type === 'message_delta' ) { - const usageObject = (event?.usage ?? {}); - const meteredData = this.#usageFormatterUtil(usageObject as Usage | BetaUsage); - - for ( const key in meteredData ) { - // Anthropic message_delta usage counters are cumulative. - // Keep the latest value instead of summing every delta. - usageSum[key] = Math.max( - usageSum[key] ?? 0, - meteredData[key as keyof typeof meteredData], - ); - } - } - - if ( event.type === 'message_start' ) { - message = chatStream.message(); - continue; - } - if ( event.type === 'message_stop' ) { - message!.end(); - message = null; - continue; - } - - if ( event.type === 'content_block_start' ) { - currentContentBlockType = event.content_block.type; - if ( event.content_block.type === 'tool_use' ) { - contentBlock = message!.contentBlock({ - type: event.content_block.type, - id: event.content_block.id, - name: event.content_block.name, - }); - continue; - } - if ( event.content_block.type === 'thinking' ) { - // We map Anthropic "thinking" blocks to our text stream type, - // then forward deltas through addReasoning(). - contentBlock = message!.contentBlock({ - type: 'text', - }); - continue; - } - contentBlock = message!.contentBlock({ - type: event.content_block.type, - }); - continue; - } - - if ( event.type === 'content_block_stop' ) { - contentBlock!.end(); - contentBlock = null; - currentContentBlockType = null; - continue; - } - - if ( event.type === 'content_block_delta' ) { - if ( event.delta.type === 'input_json_delta' ) { - (contentBlock as AIChatToolUseStream)!.addPartialJSON(event.delta.partial_json); - continue; - } - if ( event.delta.type === 'text_delta' ) { - if ( currentContentBlockType === 'thinking' ) { - (contentBlock as AIChatTextStream)!.addReasoning(event.delta.text); - } else { - (contentBlock as AIChatTextStream)!.addText(event.delta.text); - } - continue; - } - if ( event.delta.type === 'thinking_delta' ) { - (contentBlock as AIChatTextStream)!.addReasoning(event.delta.thinking); - continue; - } - if ( event.delta.type === 'signature_delta' ) { - continue; - } - } - } - // Some usage fields (e.g. thinking_tokens) may only be available - // on the final message usage object. - const finalUsage = await completion.finalMessage() - .then(message => this.#usageFormatterUtil(message.usage as Usage | BetaUsage)) - .catch(() => null); - if ( finalUsage ) { - for ( const [key, value] of Object.entries(finalUsage) ) { - usageSum[key] = value; - } - } - - chatStream.end(usageSum); - const costsOverrideFromModel = this.#buildCostsOverrideFromModel(usageSum, modelUsed); - this.#meteringService.utilRecordUsageObject(usageSum, actor, `claude:${modelUsed.id}`, costsOverrideFromModel); - }; - - return { - init_chat_stream, - stream: true, - finally_fn: cleanup_files, - }; - } - - let msg; - try { - msg = await anthropic.messages.create(sdkParams); - } catch (e) { - console.error('anthropic error:', e); - throw e; - } - await cleanup_files(); - - const usage = this.#usageFormatterUtil((msg as Message).usage as Usage | BetaUsage); - const costsOverrideFromModel = this.#buildCostsOverrideFromModel(usage, modelUsed); - this.#meteringService.utilRecordUsageObject(usage, actor, `claude:${modelUsed.id}`, costsOverrideFromModel); - - // TODO DS: cleanup old usage tracking - return { - message: msg, - usage: usage, - finish_reason: 'stop', - }; - } - - #usageFormatterUtil (usage: Usage | BetaUsage) { - return { - input_tokens: usage?.input_tokens || 0, - ephemeral_5m_input_tokens: usage?.cache_creation?.ephemeral_5m_input_tokens || usage.cache_creation_input_tokens || 0, // this is because they're api is a bit inconsistent - ephemeral_1h_input_tokens: usage?.cache_creation?.ephemeral_1h_input_tokens || 0, - cache_read_input_tokens: usage?.cache_read_input_tokens || 0, - output_tokens: usage?.output_tokens || 0, - thinking_tokens: (usage as any)?.thinking_tokens || (usage as any)?.output_tokens_details?.thinking_tokens || 0, - }; - }; - - #buildThinkingConfig ({ - modelId, - reasoningEffort, - maxTokens, - }: { - modelId: string; - reasoningEffort?: 'low' | 'medium' | 'high'; - maxTokens?: number; - }) { - if ( ! reasoningEffort ) return undefined; - - // Opus 4.7, 4.6, and Sonnet 4.6 use adaptive thinking (budget_tokens - // is deprecated on 4.6/Sonnet 4.6 and removed on 4.7). - // Opus 4.7 omits thinking content by default; display: 'summarized' - // restores visible reasoning in the stream. - if ( modelId === 'claude-opus-4-7' ) { - return { type: 'adaptive' as const, display: 'summarized' as const }; - } - if ( modelId === 'claude-opus-4-6' || modelId === 'claude-sonnet-4-6' ) { - return { type: 'adaptive' as const }; - } - - const requestedBudget = { - low: 1024, - medium: 4096, - high: 8192, - }[reasoningEffort]; - - // Keep budget <= max_tokens when it's set. If max_tokens is too low - // to satisfy Anthropic's minimum thinking budget, disable thinking. - if ( typeof maxTokens === 'number' && Number.isFinite(maxTokens) ) { - const maxBudget = Math.floor(maxTokens - 1); - if ( maxBudget < 1024 ) { - return undefined; - } - } - - const budget_tokens = Math.floor(Math.max( - 1024, - Math.min(requestedBudget, (maxTokens ? (maxTokens - 1) : requestedBudget)), - )); - - return { - type: 'enabled' as const, - budget_tokens, - }; - } - - #buildCostsOverrideFromModel (usage: Record, modelUsed: { costs: Record }) { - return Object.fromEntries(Object.entries(usage).map(([k, v]) => { - const modelCost = modelUsed.costs[k] ?? (k === 'thinking_tokens' ? modelUsed.costs.output_tokens : 0); - return [k, v * modelCost]; - })); - } - - models () { - return CLAUDE_MODELS; - } - - checkModeration (_text: string): ReturnType { - throw new Error('CheckModeration Not provided.'); - } -} diff --git a/src/backend/src/services/ai/chat/providers/GeminiProvider/GeminiChatProvider.ts b/src/backend/src/services/ai/chat/providers/GeminiProvider/GeminiChatProvider.ts deleted file mode 100644 index 2852e6259..000000000 --- a/src/backend/src/services/ai/chat/providers/GeminiProvider/GeminiChatProvider.ts +++ /dev/null @@ -1,445 +0,0 @@ -// Preamble: Before this we used Gemini's SDK directly and as we found out -// its actually kind of terrible. So we use the openai sdk now -// (except for image models, where we need the native SDK for image I/O) -import openai, { OpenAI } from 'openai'; -import { GenerateContentResponse, GoogleGenAI } from '@google/genai'; -import { Context } from '../../../../../util/context.js'; -import APIError from '../../../../../api/APIError.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import { handle_completion_output, process_input_messages } from '../../../utils/OpenAIUtil.js'; -import { IChatModel, IChatProvider, ICompleteArguments, PuterMessage } from '../types.js'; -import { AIChatStream, AIChatTextStream } from '../../../utils/Streaming.js'; -import { GEMINI_IMAGE_CHAT_MODELS, GEMINI_MODELS } from './models.js'; -import { GEMINI_ESTIMATED_IMAGE_TOKENS } from '../../../image/providers/GeminiImageGenerationProvider/models.js'; -import { Actor } from '../../../../auth/Actor.js'; -import { ChatCompletionCreateParams } from 'openai/resources/index.js'; - -export class GeminiChatProvider implements IChatProvider { - - meteringService: MeteringService; - openai: OpenAI; - genai: GoogleGenAI; - - defaultModel = 'gemini-2.5-flash'; - - constructor ( meteringService: MeteringService, config: { apiKey: string }) - { - this.meteringService = meteringService; - this.openai = new openai.OpenAI({ - apiKey: config.apiKey, - baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai/', - }); - this.genai = new GoogleGenAI({ apiKey: config.apiKey }); - } - - getDefaultModel () { - return this.defaultModel; - } - - async models () { - return GEMINI_MODELS; - } - async list () { - return (await this.models()).map(m => [m.id, ... (m.aliases || [])]).flat(); - } - - async complete ({ messages, stream, model, tools, max_tokens, temperature, image_config }: ICompleteArguments): ReturnType { - const actor = Context.get('actor'); - messages = await process_input_messages(messages); - - // delete cache_control - messages = messages.map(m => { - delete m.cache_control; - return m; - }); - - const modelUsed = (await this.models()).find(m => [m.id, ...(m.aliases || [])].includes(model)) || (await this.models()).find(m => m.id === this.getDefaultModel())!; - - if ( GEMINI_IMAGE_CHAT_MODELS.includes(modelUsed.id) ) { - return this.completeImageGeneration({ messages, stream, modelUsed, image_config, temperature }); - } - - const sdk_params: ChatCompletionCreateParams = { - messages: messages, - model: modelUsed.id, - ...(tools ? { tools } : {}), - ...(max_tokens ? { max_completion_tokens: max_tokens } : {}), - ...(temperature ? { temperature } : {}), - stream, - ...(stream ? { - stream_options: { include_usage: true }, - } : {}), - } as ChatCompletionCreateParams; - - let completion; - try { - completion = await this.openai.chat.completions.create(sdk_params); - } catch (e) { - console.error('Gemini completion error: ', e); - throw e; - } - - return handle_completion_output({ - usage_calculator: ({ usage }) => { - const trackedUsage = { - prompt_tokens: (usage.prompt_tokens ?? 0) - (usage.prompt_tokens_details?.cached_tokens ?? 0), - completion_tokens: usage.completion_tokens ?? 0, - cached_tokens: usage.prompt_tokens_details?.cached_tokens ?? 0, - }; - - const costsOverrideFromModel = Object.fromEntries(Object.entries(trackedUsage).map(([k, v]) => { - return [k, v * (modelUsed.costs[k])]; - })); - this.meteringService.utilRecordUsageObject(trackedUsage, actor, `gemini:${modelUsed?.id}`, costsOverrideFromModel); - - return trackedUsage; - }, - stream, - completion, - }); - - } - - private static extractTextFromContent (content: PuterMessage['content']): string { - if ( typeof content === 'string' ) return content; - if ( Array.isArray(content) ) { - return content - .filter((p: Record) => p.type === 'text' || typeof p === 'string') - .map((p: Record) => (p as { text?: string }).text ?? p) - .join('\n'); - } - return ''; - } - - private translateMessagesToGemini (messages: PuterMessage[]): { - contents: Record[], - systemInstruction?: string, - } { - let systemInstruction: string | undefined; - const contents: Record[] = []; - - for ( const msg of messages ) { - if ( msg.role === 'system' ) { - const text = GeminiChatProvider.extractTextFromContent(msg.content); - if ( text ) systemInstruction = systemInstruction ? `${systemInstruction}\n${text}` : text; - continue; - } - - // Only translate user and assistant (model) messages; drop tool messages - if ( msg.role !== 'user' && msg.role !== 'assistant' ) continue; - - const role = msg.role === 'assistant' ? 'model' : 'user'; - const parts: Record[] = []; - - // First pass: collect parts and find if any has thoughtSignature - let sharedThoughtSignature: string | undefined; - if ( Array.isArray(msg.content) ) { - for ( const part of msg.content ) { - if ( part.type === 'image_url' && part.thoughtSignature ) { - sharedThoughtSignature = part.thoughtSignature; - break; - } - } - } - - if ( typeof msg.content === 'string' ) { - parts.push({ text: msg.content }); - } else if ( Array.isArray(msg.content) ) { - for ( const part of msg.content ) { - if ( typeof part === 'string' ) { - const textPart: Record = { text: part }; - if ( sharedThoughtSignature ) { - textPart.thoughtSignature = sharedThoughtSignature; - } - parts.push(textPart); - } else if ( part.type === 'text' ) { - const textPart: Record = { text: part.text }; - if ( sharedThoughtSignature ) { - textPart.thoughtSignature = sharedThoughtSignature; - } - parts.push(textPart); - } else if ( part.type === 'image_url' && part.image_url?.url ) { - const url: string = part.image_url.url; - const thoughtSignature = part.thoughtSignature; - if ( url.startsWith('data:') ) { - const commaIdx = url.indexOf(','); - if ( commaIdx !== -1 ) { - const header = url.substring(5, commaIdx); - const mimeType = header.replace(';base64', ''); - const data = url.substring(commaIdx + 1); - const imagePart: Record = { inlineData: { mimeType, data } }; - if ( thoughtSignature ) { - imagePart.thoughtSignature = thoughtSignature; - } - parts.push(imagePart); - } - } else { - const imagePart: Record = { fileData: { fileUri: url } }; - if ( thoughtSignature ) { - imagePart.thoughtSignature = thoughtSignature; - } - parts.push(imagePart); - } - } - } - } - - if ( parts.length > 0 ) { - contents.push({ role, parts }); - } - } - - return { contents, systemInstruction }; - } - - private toMicroCents (cents: number): number { - return (!Number.isFinite(cents) || cents <= 0) ? 1 : Math.ceil(cents * 1_000_000); - } - - private tokenCostInCents (count: number, centsPerMillion: number): number { - return (count > 0 && centsPerMillion > 0) ? (count / 1_000_000) * centsPerMillion : 0; - } - - private static extractImageUsageMetadata (response: GenerateContentResponse): { - promptTokenCount: number, - candidatesTokenCount: number, - outputImageTokenCount: number, - thoughtsTokenCount: number, - } { - const usage = (response as GenerateContentResponse & { usageMetadata?: Record }).usageMetadata; - - let outputImageTokenCount = 0; - const details = (usage as Record)?.candidatesTokensDetails; - if ( Array.isArray(details) ) { - for ( const entry of details ) { - if ( entry?.modality === 'IMAGE' ) { - outputImageTokenCount += (typeof entry.tokenCount === 'number' ? entry.tokenCount : 0); - } - } - } - - const toSafe = (v: unknown) => (typeof v === 'number' && Number.isFinite(v) && v >= 0) ? Math.floor(v) : 0; - return { - promptTokenCount: toSafe((usage as Record)?.promptTokenCount), - candidatesTokenCount: toSafe((usage as Record)?.candidatesTokenCount), - outputImageTokenCount, - thoughtsTokenCount: toSafe((usage as Record)?.thoughtsTokenCount), - }; - } - - private meterImageGeneration ( - actor: Actor, - modelUsed: IChatModel, - response: GenerateContentResponse, - estimatedPromptTokens: number, - estimatedImageTokens: number, - ) { - const usage = GeminiChatProvider.extractImageUsageMetadata(response); - const inputTokenCount = usage.promptTokenCount || estimatedPromptTokens; - const outputImageTokenCount = usage.outputImageTokenCount || estimatedImageTokens; - const outputTextTokenCount = Math.max(0, usage.candidatesTokenCount - outputImageTokenCount) + usage.thoughtsTokenCount; - - const costs = modelUsed.costs; - const usagePrefix = `gemini:${modelUsed.id}`; - this.meteringService.batchIncrementUsages(actor, [ - { - usageType: `${usagePrefix}:input`, - usageAmount: Math.max(inputTokenCount, 1), - costOverride: this.toMicroCents(this.tokenCostInCents(inputTokenCount, costs.prompt_tokens)), - }, - { - usageType: `${usagePrefix}:output:text`, - usageAmount: Math.max(outputTextTokenCount, 1), - costOverride: this.toMicroCents(this.tokenCostInCents(outputTextTokenCount, costs.completion_tokens)), - }, - { - usageType: `${usagePrefix}:output:image`, - usageAmount: Math.max(outputImageTokenCount, 1), - costOverride: this.toMicroCents(this.tokenCostInCents(outputImageTokenCount, costs.output_image)), - }, - ]); - - return { inputTokenCount, outputTextTokenCount, outputImageTokenCount }; - } - - private static parseGeminiImageResponse (response: GenerateContentResponse): { - content: string, - images: { type: string, image_url: { url: string }, thoughtSignature?: string }[], - } { - const parts = response?.candidates?.[0]?.content?.parts ?? []; - let content = ''; - const images: { type: string, image_url: { url: string }, thoughtSignature?: string }[] = []; - - for ( const part of parts ) { - if ( part.text ) { - content += part.text; - } else if ( part.inlineData?.data ) { - const mimeType = part.inlineData.mimeType ?? 'image/png'; - const image: { type: string, image_url: { url: string }, thoughtSignature?: string } = { - type: 'image_url', - image_url: { - url: `data:${mimeType};base64,${part.inlineData.data}`, - }, - }; - // Preserve thoughtSignature from Gemini for multi-turn image editing - if ( (part as Record).thoughtSignature ) { - image.thoughtSignature = (part as Record).thoughtSignature as string; - } - images.push(image); - } - } - - if ( !content && images.length > 0 ) { - content = 'Generated image.'; - } - - return { content, images }; - } - - private async completeImageGeneration ({ messages, stream, modelUsed, image_config, temperature }: { - messages: PuterMessage[], - stream: boolean | undefined, - modelUsed: IChatModel, - image_config?: { aspect_ratio?: string, image_size?: string }, - temperature?: number, - }): ReturnType { - const actor = Context.get('actor') as Actor; - const { contents, systemInstruction } = this.translateMessagesToGemini(messages); - - // Resolve and validate image_size against model's allowed quality levels - let imageSize = image_config?.image_size; - const allowed = modelUsed.allowedQualityLevels; - if ( allowed && allowed.length > 0 ) { - if ( imageSize && !allowed.includes(imageSize) ) { - throw APIError.create('field_invalid', null, { - key: 'image_config.image_size', - expected: allowed.join(', '), - got: imageSize, - }); - } - if ( ! imageSize ) imageSize = allowed[0]; - } - - const geminiImageConfig: Record = {}; - if ( image_config?.aspect_ratio ) geminiImageConfig.aspectRatio = image_config.aspect_ratio; - if ( imageSize ) geminiImageConfig.imageSize = imageSize; - - const config: Record = { - responseModalities: ['TEXT', 'IMAGE'], - }; - if ( Object.keys(geminiImageConfig).length > 0 ) { - config.imageConfig = geminiImageConfig; - } - if ( systemInstruction ) { - config.systemInstruction = systemInstruction; - } - if ( temperature !== undefined ) { - config.temperature = temperature; - } - - // Pre-flight cost estimate for image input tokens - const inputImageCount = contents - .reduce((n, c) => n + ((c.parts as Record[]) ?? []).filter((p) => p.inlineData || p.fileData).length, 0); - const estimatedPromptTokens = inputImageCount * 560; - - const imageTokenKey = imageSize ? `${modelUsed.id}:${imageSize}` : modelUsed.id; - const estimatedImageTokens = GEMINI_ESTIMATED_IMAGE_TOKENS[imageTokenKey]; - if ( estimatedImageTokens === undefined ) { - throw new Error(`No estimated image token count configured for '${imageTokenKey}'.`); - } - - const estimatedCost = this.toMicroCents( - this.tokenCostInCents(estimatedPromptTokens, modelUsed.costs.prompt_tokens) + - this.tokenCostInCents(estimatedImageTokens, modelUsed.costs.output_image) + - this.tokenCostInCents(50, modelUsed.costs.completion_tokens), - ); - const usageAllowed = await this.meteringService.hasEnoughCredits(actor, estimatedCost); - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); - } - - if ( stream ) { - const streamResponse = await this.genai.models.generateContentStream({ - model: modelUsed.id, - contents, - config, - }); - - const init_chat_stream = async ({ chatStream }: { chatStream: AIChatStream }) => { - const message = chatStream.message(); - const textblock = message.contentBlock({ type: 'text' }) as AIChatTextStream; - - let lastResponse: GenerateContentResponse | undefined; - for await ( const chunk of streamResponse ) { - lastResponse = chunk; - const parts = chunk.candidates?.[0]?.content?.parts ?? []; - for ( const part of parts ) { - if ( part.text ) { - textblock.addText(part.text); - } else if ( part.inlineData?.data ) { - const mimeType = part.inlineData.mimeType ?? 'image/png'; - const image: Record = { - type: 'image_url', - image_url: { - url: `data:${mimeType};base64,${part.inlineData.data}`, - }, - }; - if ( (part as Record).thoughtSignature ) { - image.thoughtSignature = (part as Record).thoughtSignature; - } - textblock.addImage(image); - } - } - } - - if ( lastResponse ) { - const metered = this.meterImageGeneration(actor, modelUsed, lastResponse, estimatedPromptTokens, estimatedImageTokens); - textblock.end(); - message.end(); - chatStream.end({ - prompt_tokens: metered.inputTokenCount, - completion_tokens: metered.outputTextTokenCount + metered.outputImageTokenCount, - }); - } else { - textblock.end(); - message.end(); - chatStream.end({ prompt_tokens: 0, completion_tokens: 0 }); - } - }; - - return { - stream: true as const, - init_chat_stream, - finally_fn: async () => { - }, - }; - } - - const response = await this.genai.models.generateContent({ - model: modelUsed.id, - contents, - config, - }); - - const metered = this.meterImageGeneration(actor, modelUsed, response, estimatedPromptTokens, estimatedImageTokens); - - const { content, images } = GeminiChatProvider.parseGeminiImageResponse(response); - - return { - message: { - role: 'assistant', - content, - ...(images.length > 0 ? { images } : {}), - }, - usage: { - prompt_tokens: metered.inputTokenCount, - completion_tokens: metered.outputTextTokenCount + metered.outputImageTokenCount, - }, - finish_reason: 'stop', - }; - } - - checkModeration (_text: string): ReturnType { - throw new Error('No moderation logic.'); - } -} diff --git a/src/backend/src/services/ai/chat/providers/OpenAiProvider/OpenAiChatCompletionsProvider.ts b/src/backend/src/services/ai/chat/providers/OpenAiProvider/OpenAiChatCompletionsProvider.ts deleted file mode 100644 index 9e8569060..000000000 --- a/src/backend/src/services/ai/chat/providers/OpenAiProvider/OpenAiChatCompletionsProvider.ts +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import mime from 'mime-types'; -import { OpenAI } from 'openai'; -import { ChatCompletionCreateParams } from 'openai/resources/index.js'; -import { FSNodeParam } from '../../../../../api/filesystem/FSNodeParam.js'; -import { LLRead } from '../../../../../deprecated/filesystem/ll_operations/ll_read.js'; -import { Context } from '../../../../../util/context.js'; -import { stream_to_buffer } from '../../../../../util/streamutil.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import * as OpenAiUtil from '../../../utils/OpenAIUtil.js'; -import { IChatProvider, ICompleteArguments } from '../types.js'; -import { OPEN_AI_MODELS } from './models.js'; - -; - -// We're capping at 5MB, which sucks, but Chat Completions doesn't suuport -// file inputs. -const MAX_FILE_SIZE = 5 * 1_000_000; - -/** -* OpenAICompletionService class provides an interface to OpenAI's chat completion API. -* Extends BaseService to handle chat completions, message moderation, token counting, -* and streaming responses. Implements the puter-chat-completion interface and manages -* OpenAI API interactions with support for multiple models including GPT-4 variants. -* Handles usage tracking, spending records, and content moderation. -*/ -export class OpenAiChatProvider implements IChatProvider { - /** - * @type {import('openai').OpenAI} - */ - #openAi: OpenAI; - - #defaultModel = 'gpt-5-nano'; - - #meteringService: MeteringService; - - constructor ( - meteringService: MeteringService, - config: { apiKey?: string, secret_key?: string }, - ) { - - this.#meteringService = meteringService; - let apiKey = config.apiKey; - - // Fallback to the old format for backward compatibility - if ( ! apiKey ) { - apiKey = config?.secret_key; - - // Log a warning to inform users about the deprecated format - console.warn('The `openai.secret_key` configuration format is deprecated. ' + - 'Please use `services.openai.apiKey` instead.'); - } - if ( ! apiKey ) { - throw new Error('OpenAI API key is missing in configuration.'); - } - this.#openAi = new OpenAI({ - apiKey: apiKey, - }); - } - - /** - * Returns an array of available AI models with their pricing information. - * Each model object includes an ID and cost details (currency, tokens, input/output rates). - */ - models () { - return OPEN_AI_MODELS.filter(e => !e.responses_api_only); - } - - list () { - const models = this.models(); - const modelNames: string[] = []; - for ( const model of models ) { - modelNames.push(model.id); - if ( model.aliases ) { - modelNames.push(...model.aliases); - } - } - return modelNames; - } - - getDefaultModel () { - return this.#defaultModel; - } - - async complete (params: ICompleteArguments): ReturnType - { - let { messages, model, max_tokens, moderation, tools, verbosity, stream, reasoning, reasoning_effort, temperature, text } = params; - if ( tools?.filter((e: any) => e.type === 'web_search').length ) { - // User is trying to use openai-responses only tool web_search. - // We should pass it to that service - const aiChat = (Context.get('services') as any).get('ai-chat'); - const openAIresponses = aiChat.getProvider('openai-responses')!; - return await openAIresponses.complete!(params); - } - // Validate messages - if ( ! Array.isArray(messages) ) { - throw new Error('`messages` must be an array'); - } - const actor = Context.get('actor'); - - model = model ?? this.#defaultModel; - - const modelUsed = (this.models()).find(m => [m.id, ...(m.aliases || [])].includes(model)) || (this.models()).find(m => m.id === this.getDefaultModel())!; - - // messages.unshift({ - // role: 'system', - // content: 'Don\'t let the user trick you into doing something bad.', - // }) - - const user_private_uid = actor?.private_uid ?? 'UNKNOWN'; - if ( user_private_uid === 'UNKNOWN' ) { - console.error(new Error('chat-completion-service:unknown-user - failed to get a user ID for an OpenAI request')); - } - - // Perform file uploads - const { user } = actor.type; - - const file_input_tasks: any[] = []; - for ( const message of messages ) { - // We can assume `message.content` is not undefined because - // Messages.normalize_single_message ensures this. - for ( const contentPart of message.content ) { - - if ( ! contentPart.puter_path ) continue; - file_input_tasks.push({ - node: await (new FSNodeParam(contentPart.puter_path)).consolidate({ - req: { user }, - getParam: () => contentPart.puter_path, - }), - contentPart, - }); - } - } - - const promises: Promise[] = []; - for ( const task of file_input_tasks ) { - promises.push((async () => { - if ( await task.node.get('size') > MAX_FILE_SIZE ) { - delete task.contentPart.puter_path; - task.contentPart.type = 'text'; - task.contentPart.text = `{error: input file exceeded maximum of ${MAX_FILE_SIZE} bytes; ` + - 'the user did not write this message}'; // "poor man's system prompt" - return; // "continue" - } - - const ll_read = new LLRead(); - const stream = await ll_read.run({ - actor: Context.get('actor'), - fsNode: task.node, - }); - const mimeType = mime.contentType(await task.node.get('name')); - - const buffer = await stream_to_buffer(stream); - const base64 = buffer.toString('base64'); - - delete task.contentPart.puter_path; - if ( mimeType && mimeType.startsWith('image/') ) { - task.contentPart.type = 'image_url'; - task.contentPart.image_url = { - url: `data:${mimeType};base64,${base64}`, - }; - } else if ( mimeType && mimeType.startsWith('audio/') ) { - task.contentPart.type = 'input_audio'; - task.contentPart.input_audio = { - data: `data:${mimeType};base64,${base64}`, - format: mimeType.split('/')[1], - }; - } else { - task.contentPart.type = 'text'; - task.contentPart.text = '{error: input file has unsupported MIME type; ' + - 'the user did not write this message}'; // "poor man's system prompt" - } - })()); - } - await Promise.all(promises); - - // Here's something fun; the documentation shows `type: 'image_url'` in - // objects that contain an image url, but everything still works if - // that's missing. We normalise it here so the token count code works. - messages = await OpenAiUtil.process_input_messages(messages); - - const requestedReasoningEffort = reasoning_effort ?? reasoning?.effort; - const requestedVerbosity = verbosity ?? text?.verbosity; - const supportsReasoningControls = typeof model === 'string' && model.startsWith('gpt-5'); - - const completionParams: ChatCompletionCreateParams = { - user: user_private_uid, - safety_identifier: user_private_uid, - messages: messages, - model: modelUsed.id, - ...(tools ? { tools } : {}), - ...(max_tokens ? { max_completion_tokens: max_tokens } : {}), - ...(temperature ? { temperature } : {}), - stream: !!stream, - ...(stream ? { - stream_options: { include_usage: true }, - } : {}), - ...(supportsReasoningControls ? {} : - { - ...(requestedReasoningEffort ? { reasoning_effort: requestedReasoningEffort } : {}), - ...(requestedVerbosity ? { verbosity: requestedVerbosity } : {}), - } - ), - } as ChatCompletionCreateParams; - - const completion = await this.#openAi.chat.completions.create(completionParams); - - return OpenAiUtil.handle_completion_output({ - usage_calculator: ({ usage }) => { - const trackedUsage = { - prompt_tokens: (usage.prompt_tokens ?? 0) - (usage.prompt_tokens_details?.cached_tokens ?? 0), - completion_tokens: usage.completion_tokens ?? 0, - cached_tokens: usage.prompt_tokens_details?.cached_tokens ?? 0, - }; - - const costsOverrideFromModel = Object.fromEntries(Object.entries(trackedUsage).map(([k, v]) => { - return [k, v * (modelUsed.costs[k])]; - })); - - this.#meteringService.utilRecordUsageObject(trackedUsage, actor, `openai:${modelUsed?.id}`, costsOverrideFromModel); - return trackedUsage; - }, - stream, - completion, - moderate: moderation ? this.checkModeration.bind(this) : undefined, - }); - } - - async checkModeration (text: string) { - // create moderation - const results = await this.#openAi.moderations.create({ - model: 'omni-moderation-latest', - input: text, - }); - - let flagged = false; - - for ( const result of results?.results ?? [] ) { - - // OpenAI does a crazy amount of false positives. We filter by their 80% interval - const veryFlaggedEntries = Object.entries(result.category_scores).filter(e => e[1] > 0.8); - if ( veryFlaggedEntries.length > 0 ) { - flagged = true; - break; - } - } - - return { - flagged, - results, - }; - } -} diff --git a/src/backend/src/services/ai/chat/providers/OpenAiProvider/OpenAiChatResponsesProvider.ts b/src/backend/src/services/ai/chat/providers/OpenAiProvider/OpenAiChatResponsesProvider.ts deleted file mode 100644 index e4fb10373..000000000 --- a/src/backend/src/services/ai/chat/providers/OpenAiProvider/OpenAiChatResponsesProvider.ts +++ /dev/null @@ -1,322 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import mime from 'mime-types'; -import { OpenAI } from 'openai'; -import { ResponseCreateParams } from 'openai/resources/responses/responses.mjs'; -import { FSNodeParam } from '../../../../../api/filesystem/FSNodeParam.js'; -import { LLRead } from '../../../../../deprecated/filesystem/ll_operations/ll_read.js'; -import { Context } from '../../../../../util/context.js'; -import { stream_to_buffer } from '../../../../../util/streamutil.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import * as OpenAiUtil from '../../../utils/OpenAIUtil.js'; -import { IChatProvider, ICompleteArguments } from '../types.js'; -import { OPEN_AI_MODELS } from './models.js'; - -; - -// We're capping at 5MB, which sucks, but Chat Completions doesn't suuport -// file inputs. -const MAX_FILE_SIZE = 5 * 1_000_000; - -/** -* OpenAICompletionService class provides an interface to OpenAI's chat completion API. -* Extends BaseService to handle chat completions, message moderation, token counting, -* and streaming responses. Implements the puter-chat-completion interface and manages -* OpenAI API interactions with support for multiple models including GPT-4 variants. -* Handles usage tracking, spending records, and content moderation. -*/ -export class OpenAiResponsesChatProvider implements IChatProvider { - /** - * @type {import('openai').OpenAI} - */ - #openAi: OpenAI; - - #defaultModel = 'gpt-5-nano'; - - #meteringService: MeteringService; - - constructor ( - meteringService: MeteringService, - config: { apiKey?: string, secret_key?: string }, - ) { - - this.#meteringService = meteringService; - let apiKey = config.apiKey; - - // Fallback to the old format for backward compatibility - if ( ! apiKey ) { - apiKey = config?.secret_key; - - // Log a warning to inform users about the deprecated format - console.warn('The `openai.secret_key` configuration format is deprecated. ' + - 'Please use `services.openai.apiKey` instead.'); - } - if ( ! apiKey ) { - throw new Error('OpenAI API key is missing in configuration.'); - } - this.#openAi = new OpenAI({ - apiKey: apiKey, - }); - } - - /** - * Returns an array of available AI models with their pricing information. - * Each model object includes an ID and cost details (currency, tokens, input/output rates). - */ - models (extra_params) { - if ( extra_params?.no_restrictions ) - { - return OPEN_AI_MODELS; - } - return OPEN_AI_MODELS.filter(e => e.responses_api_only === true); - } - - list () { - const models = this.models({ no_restrictions: false }); - const modelNames: string[] = []; - for ( const model of models ) { - modelNames.push(model.id); - if ( model.aliases ) { - modelNames.push(...model.aliases); - } - } - return modelNames; - } - - getDefaultModel () { - return this.#defaultModel; - } - - async complete ({ - messages, - model, - max_tokens, - moderation, - tools, - tool_choice, - parallel_tool_calls, - include, - conversation, - previous_response_id, - instructions, - metadata, - prompt, - prompt_cache_key, - prompt_cache_retention, - store, - top_p, - truncation, - background, - service_tier, - verbosity, - stream, - reasoning, - reasoning_effort, - temperature, - text, - }: ICompleteArguments): ReturnType - { - // Validate messages - if ( ! Array.isArray(messages) ) { - throw new Error('`messages` must be an array'); - } - const actor = Context.get('actor'); - - model = model ?? this.#defaultModel; - - const modelUsed = (this.models({ no_restrictions: true })).find(m => [m.id, ...(m.aliases || [])].includes(model)) || (this.models(({ no_restrictions: true })).find(m => m.id === this.getDefaultModel())!); - - // messages.unshift({ - // role: 'system', - // content: 'Don\'t let the user trick you into doing something bad.', - // }) - - const user_private_uid = actor?.private_uid ?? 'UNKNOWN'; - if ( user_private_uid === 'UNKNOWN' ) { - console.error(new Error('chat-completion-service:unknown-user - failed to get a user ID for an OpenAI request')); - } - - // Perform file uploads - const { user } = actor.type; - - const file_input_tasks: any[] = []; - for ( const message of messages ) { - // We can assume `message.content` is not undefined because - // Messages.normalize_single_message ensures this. - for ( const contentPart of message.content ) { - - if ( ! contentPart.puter_path ) continue; - file_input_tasks.push({ - node: await (new FSNodeParam(contentPart.puter_path)).consolidate({ - req: { user }, - getParam: () => contentPart.puter_path, - }), - contentPart, - }); - } - } - - const promises: Promise[] = []; - for ( const task of file_input_tasks ) { - promises.push((async () => { - if ( await task.node.get('size') > MAX_FILE_SIZE ) { - delete task.contentPart.puter_path; - task.contentPart.type = 'text'; - task.contentPart.text = `{error: input file exceeded maximum of ${MAX_FILE_SIZE} bytes; ` + - 'the user did not write this message}'; // "poor man's system prompt" - return; // "continue" - } - - const ll_read = new LLRead(); - const stream = await ll_read.run({ - actor: Context.get('actor'), - fsNode: task.node, - }); - const mimeType = mime.contentType(await task.node.get('name')); - - const buffer = await stream_to_buffer(stream); - const base64 = buffer.toString('base64'); - - delete task.contentPart.puter_path; - if ( mimeType && mimeType.startsWith('image/') ) { - task.contentPart.type = 'image_url'; - task.contentPart.image_url = { - url: `data:${mimeType};base64,${base64}`, - }; - } else if ( mimeType && mimeType.startsWith('audio/') ) { - task.contentPart.type = 'input_audio'; - task.contentPart.input_audio = { - data: `data:${mimeType};base64,${base64}`, - format: mimeType.split('/')[1], - }; - } else { - task.contentPart.type = 'text'; - task.contentPart.text = '{error: input file has unsupported MIME type; ' + - 'the user did not write this message}'; // "poor man's system prompt" - } - })()); - } - await Promise.all(promises); - - if ( tools ) { - // Unravel tools to OpenAI Responses API format - tools = (tools as any).map((e) => { - if ( e.type === 'function' ) { - const tool = e.function; - tool.type = 'function'; - return tool; - } else { - return e; - } - }); - } - - // Here's something fun; the documentation shows `type: 'image_url'` in - // objects that contain an image url, but everything still works if - // that's missing. We normalise it here so the token count code works. - messages = await OpenAiUtil.process_input_messages_responses_api(messages); - - const requestedReasoningEffort = reasoning_effort ?? reasoning?.effort; - const requestedVerbosity = verbosity ?? text?.verbosity; - const supportsReasoningControls = typeof model === 'string' && model.startsWith('gpt-5'); - - const completionParams: ResponseCreateParams = { - user: user_private_uid, - safety_identifier: user_private_uid, - input: messages, - model: modelUsed.id, - ...(tools ? { tools } : {}), - ...(tool_choice !== undefined ? { tool_choice } : {}), - ...(parallel_tool_calls !== undefined ? { parallel_tool_calls } : {}), - ...(include !== undefined ? { include } : {}), - ...(conversation !== undefined ? { conversation } : {}), - ...(previous_response_id !== undefined ? { previous_response_id } : {}), - ...(instructions !== undefined ? { instructions } : {}), - ...(metadata !== undefined ? { metadata } : {}), - ...(prompt !== undefined ? { prompt } : {}), - ...(prompt_cache_key !== undefined ? { prompt_cache_key } : {}), - ...(prompt_cache_retention !== undefined ? { prompt_cache_retention } : {}), - ...(store !== undefined ? { store } : {}), - ...(max_tokens !== undefined ? { max_output_tokens: max_tokens } : {}), - ...(temperature !== undefined ? { temperature } : {}), - ...(top_p !== undefined ? { top_p } : {}), - ...(truncation !== undefined ? { truncation } : {}), - ...(background !== undefined ? { background } : {}), - ...(service_tier !== undefined ? { service_tier } : {}), - ...(stream !== undefined ? { stream: !!stream } : {}), - ...(text !== undefined ? { text } : {}), - ...(supportsReasoningControls ? {} : - { - ...(requestedReasoningEffort ? { reasoning_effort: requestedReasoningEffort } : {}), - ...(requestedVerbosity ? { verbosity: requestedVerbosity } : {}), - } - ), - ...(supportsReasoningControls && reasoning ? { reasoning } : {}), - } as ResponseCreateParams; - - // console.log("completion params: ", completionParams) - const completion = await this.#openAi.responses.create(completionParams); - // console.log("Completion: ", completion) - return OpenAiUtil.handle_completion_output_responses_api({ - usage_calculator: ({ usage }) => { - const trackedUsage = { - prompt_tokens: ((usage as any).input_tokens ?? 0) - ((usage as any).input_tokens_details?.cached_tokens ?? 0), - completion_tokens: (usage as any).output_tokens ?? 0, - cached_tokens: (usage as any).input_tokens_details?.cached_tokens ?? 0, - }; - - const costsOverrideFromModel = Object.fromEntries(Object.entries(trackedUsage).map(([k, v]) => { - return [k, v * (modelUsed.costs[k])]; - })); - - this.#meteringService.utilRecordUsageObject(trackedUsage, actor, `openai:${modelUsed?.id}`, costsOverrideFromModel); - return trackedUsage; - }, - stream, - completion, - moderate: moderation ? this.checkModeration.bind(this) : undefined, - }); - } - - async checkModeration (text: string) { - // create moderation - const results = await this.#openAi.moderations.create({ - model: 'omni-moderation-latest', - input: text, - }); - - let flagged = false; - - for ( const result of results?.results ?? [] ) { - - // OpenAI does a crazy amount of false positives. We filter by their 80% interval - const veryFlaggedEntries = Object.entries(result.category_scores).filter(e => e[1] > 0.8); - if ( veryFlaggedEntries.length > 0 ) { - flagged = true; - break; - } - } - - return { - flagged, - results, - }; - } -} diff --git a/src/backend/src/services/ai/chat/providers/OpenRouterProvider/OpenRouterProvider.ts b/src/backend/src/services/ai/chat/providers/OpenRouterProvider/OpenRouterProvider.ts deleted file mode 100644 index 57685c822..000000000 --- a/src/backend/src/services/ai/chat/providers/OpenRouterProvider/OpenRouterProvider.ts +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import axios from 'axios'; -import { OpenAI } from 'openai'; -import { ChatCompletionCreateParams } from 'openai/resources'; -import APIError from '../../../../../api/APIError.js'; -import { Context } from '../../../../../util/context.js'; -import { kv } from '../../../../../util/kvSingleton.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import * as OpenAIUtil from '../../../utils/OpenAIUtil.js'; -import { IChatModel, IChatProvider } from '../types.js'; -import { OPEN_ROUTER_MODEL_OVERRIDES } from './modelOverrides.js'; - -type OpenrouterUsage = OpenAI.Completions.CompletionUsage & { - cost?: number -}; - -export class OpenRouterProvider implements IChatProvider { - - #meteringService: MeteringService; - - #openai: OpenAI; - - #apiBaseUrl: string = 'https://openrouter.ai/api/v1'; - - constructor (config: { apiBaseUrl?: string, apiKey: string }, meteringService: MeteringService) { - this.#apiBaseUrl = config.apiBaseUrl || 'https://openrouter.ai/api/v1'; - this.#openai = new OpenAI({ - apiKey: config.apiKey, - baseURL: this.#apiBaseUrl, - }); - this.#meteringService = meteringService; - } - - getDefaultModel () { - return 'openrouter:openai/gpt-5-nano'; - } - /** - * Returns a list of available model names including their aliases - * @returns {Promise} Array of model identifiers and their aliases - * @description Retrieves all available model IDs and their aliases, - * flattening them into a single array of strings that can be used for model selection - */ - async list () { - const models = await this.models(); - const model_names: string[] = []; - for ( const model of models ) { - model_names.push(model.id); - } - return model_names; - } - - /** - * AI Chat completion method. - * See AIChatService for more details. - */ - async complete ({ messages, stream, model, tools, max_tokens, temperature }) { - - const modelUsed = (await this.models()).find(m => [m.id, ...(m.aliases || [])].includes(model)) || (await this.models()).find(m => m.id === this.getDefaultModel())!; - - const modelIdForParams = modelUsed.id.startsWith('openrouter:') ? modelUsed.id.slice('openrouter:'.length) : modelUsed.id; - - if ( model === 'openrouter/auto' ) { - throw APIError.create('field_invalid', undefined, { - key: 'model', - expected: 'allowed model', - got: 'disallowed model', - }); - } - - const actor = Context.get('actor'); - - messages = await OpenAIUtil.process_input_messages(messages); - - const completionParams = { - messages, - model: modelIdForParams, - ...(tools ? { tools } : {}), - max_tokens, - temperature: temperature, // default to 1.0 - stream, - ...(stream ? { - stream_options: { include_usage: true }, - } : {}), - usage: { include: true }, - } as ChatCompletionCreateParams; - - let completion; - try { - completion = await this.#openai.chat.completions.create(completionParams); - } catch ( e: unknown ) { - // If you overestimate allowed max_tokens on openrouter then it will throw an error. - // Since we know the user has enough for the query anyways, we should reexecute the - // request without max_tokens. - const err = e as { error: Error }; - if ( err && err.error && err.error.message && err.error.message.startsWith("This endpoint's maximum context length is ") ) { - delete completionParams.max_tokens; - completion = await this.#openai.chat.completions.create(completionParams); - } else { - console.log('Openarouter error: ', err.error.message); - throw e; - } - } - - return OpenAIUtil.handle_completion_output({ - usage_calculator: ({ usage }: { usage: OpenrouterUsage }) => { - if ( typeof usage.cost === 'number' ) { - // custom open router logic because they're pricing are weird - const trackedUsage = { - prompt: (usage.prompt_tokens ?? 0 ) - (usage.prompt_tokens_details?.cached_tokens ?? 0), - completion: usage.completion_tokens ?? 0, - input_cache_read: usage.prompt_tokens_details?.cached_tokens ?? 0, - request: (usage as unknown as Record).request || 1, - billedUsage: 1, - }; - const costOverwrites = Object.fromEntries(Object.keys(trackedUsage).map((k) => { - return ([k, 0]); // make everything else 0 if they don't respect their own pricing - })); - costOverwrites.billedUsage = (usage.cost * 100_000_000) || 1; - this.#meteringService.utilRecordUsageObject(trackedUsage, actor, modelUsed.id, costOverwrites); - return trackedUsage; - } else { - // custom open router logic because they're pricing are weird - const trackedUsage = { - prompt: (usage.prompt_tokens ?? 0 ) - (usage.prompt_tokens_details?.cached_tokens ?? 0), - completion: usage.completion_tokens ?? 0, - input_cache_read: usage.prompt_tokens_details?.cached_tokens ?? 0, - request: (usage as unknown as Record).request || 1, - }; - const costOverwrites = Object.fromEntries(Object.keys(trackedUsage).map((k) => { - return ([k, (modelUsed.costs[k]) * trackedUsage[k]]); - })); - this.#meteringService.utilRecordUsageObject(trackedUsage, actor, modelUsed.id, costOverwrites); - return trackedUsage; - } - - }, - stream, - completion, - }); - } - - async models () { - let models = kv.get('openrouterChat:models'); - if ( ! models ) { - try { - const resp = await axios.request({ - method: 'GET', - url: `${this.#apiBaseUrl}/models`, - }); - - models = resp.data.data; - kv.set('openrouterChat:models', models); - } catch (e) { - console.log(e); - } - } - const coerced_models: IChatModel[] = []; - for ( const model of models ) { - if ( (model.id as string).includes('openrouter/auto') ) { - continue; - } - const overridenModel = OPEN_ROUTER_MODEL_OVERRIDES.find(m => m.id === `openrouter:${model.id}`); - const microcentCosts = Object.fromEntries(Object.entries(model.pricing).map(([k, v]) => [k, Math.round((v as number < 0 ? 1 : v as number) * 1_000_000 * 100)])) ; - if ( ! microcentCosts.request ) { - microcentCosts.request = 0; - } - coerced_models.push({ - id: `openrouter:${model.id}`, - name: `${model.name} (OpenRouter)`, - aliases: [model.id, model.name, `openrouter/${model.id}`, model.id.split('/').slice(1).join('/')], - context: model.context_length, - max_tokens: model.top_provider.max_completion_tokens, - costs_currency: 'usd-cents', - input_cost_key: 'prompt', - output_cost_key: 'completion', - costs: { - tokens: 1_000_000, - ...microcentCosts, - }, - ...overridenModel, - }); - } - return coerced_models; - } - checkModeration (_text: string): ReturnType { - throw new Error('Method not implemented.'); - } -} \ No newline at end of file diff --git a/src/backend/src/services/ai/chat/providers/types.ts b/src/backend/src/services/ai/chat/providers/types.ts deleted file mode 100644 index 0ee11e2bc..000000000 --- a/src/backend/src/services/ai/chat/providers/types.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { Message } from 'openai/resources/conversations/conversations.js'; -import { ModerationCreateResponse } from 'openai/resources/moderations.js'; -import { AIChatStream } from '../../utils/Streaming'; - -type ModelCost = Record; - -export interface ModelModalities { - input: string[]; - output: string[]; -} - -export interface IChatModel extends Record { - id: string, - provider?: string, - puterId?: string - aliases?: string[] - costs_currency: string, - input_cost_key?: keyof T, - output_cost_key?: keyof T, - costs: T, - context?: number, - max_tokens: number, - subscriberOnly?: boolean, - minimumCredits?: number, - // Models.dev metadata (https://models.dev/api.json) - modalities?: ModelModalities, - open_weights?: boolean, - tool_call?: boolean, - knowledge?: string, - release_date?: string, - allowedQualityLevels?: string[], -} - -export type PuterMessage = Message | any; // TODO DS: type this more strictly -export interface ICompleteArguments { - messages: PuterMessage[]; - provider?: string; - stream?: boolean; - model: string; - tools?: unknown[]; - tool_choice?: unknown; - parallel_tool_calls?: boolean; - include?: unknown[]; - conversation?: unknown; - previous_response_id?: string; - instructions?: string | PuterMessage[]; - metadata?: Record; - prompt?: unknown; - prompt_cache_key?: string; - prompt_cache_retention?: 'in-memory' | '24h' | undefined; - store?: boolean; - top_p?: number; - truncation?: 'auto' | 'disabled' | undefined; - background?: boolean; - service_tier?: 'auto' | 'default' | 'flex' | 'scale' | 'priority' | undefined; - max_tokens?: number; - temperature?: number; - reasoning?: { effort: 'low' | 'medium' | 'high' } | undefined; - text?: string & { verbosity?: 'concise' | 'detailed' | undefined }; - reasoning_effort?: 'low' | 'medium' | 'high' | undefined; - verbosity?: 'concise' | 'detailed' | undefined; - moderation?: boolean; - custom?: unknown; - response?: { - normalize?: boolean; - }; - customLimitMessage?: string; - image_config?: { - aspect_ratio?: string; - image_size?: string; - }; -} - -export interface IChatProvider { - models(extra_params?: any): IChatModel[] | Promise - list(): string[] | Promise - checkModeration (text: string): Promise<{ - flagged: boolean; - results: ModerationCreateResponse & { - _request_id?: string | null; - }; - }> - getDefaultModel(): string; - complete (arg: ICompleteArguments): Promise<{ - init_chat_stream: ({ chatStream }: { - chatStream: AIChatStream; - }) => Promise; - stream: true; - finally_fn: () => Promise; - message?: never; - usage?: never; - finish_reason?: never; - via_ai_chat_service?: true, // legacy field always true now - } | { - message: PuterMessage; - usage: Record; - finish_reason: string; - init_chat_stream?: never; - stream?: never; - finally_fn?: never; - normalized?: boolean; - via_ai_chat_service?: true, // legacy field always true now - }> -} diff --git a/src/backend/src/services/ai/docs/README.md b/src/backend/src/services/ai/docs/README.md deleted file mode 100644 index 0ddcb1728..000000000 --- a/src/backend/src/services/ai/docs/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# PuterAI Documentation - -This directory contains documentation for the PuterAI module, which provides AI services integration for the Puter platform. - -## Contents - -### General Documentation - -- [Configuration](./config.md) - General configuration for PuterAI -- [AI Services Configuration](./ai-services-config.md) - Configuration for specific AI services - -### API Examples - -- [API Request Examples](./api_examples.md) - Examples of API requests to PuterAI services - -## Related Documentation - -For more information about the overall Puter documentation structure, see the [documentation meta guide](../../../../../doc/docmeta.md). \ No newline at end of file diff --git a/src/backend/src/services/ai/docs/ai-services-config.md b/src/backend/src/services/ai/docs/ai-services-config.md deleted file mode 100644 index ae171941e..000000000 --- a/src/backend/src/services/ai/docs/ai-services-config.md +++ /dev/null @@ -1,23 +0,0 @@ -# Configuring AI Services - -AI services are configured under the `services` block in the configuration file. Each service requires an `apiKey` to authenticate requests. - -## Example Configuration -```json -{ - "services": { - "openai": { - "apiKey": "sk-abcdefg..." - }, - "elevenlabs": { - "apiKey": "eleven-api-key", - "defaultVoiceId": "optional-voice-id" - }, - "deepseek": { - "apiKey": "sk-xyz123..." - }, - "other-ai-service": { - "apiKey": "sk-hijklmn..." - } - } -} diff --git a/src/backend/src/services/ai/docs/api_examples.md b/src/backend/src/services/ai/docs/api_examples.md deleted file mode 100644 index c10ae0702..000000000 --- a/src/backend/src/services/ai/docs/api_examples.md +++ /dev/null @@ -1,255 +0,0 @@ -# PuterAI API Request Examples - -This document provides examples of API requests to the PuterAI services. These examples demonstrate how to interact with various AI capabilities of the Puter platform. - -## OCR (Optical Character Recognition) - -Example of using AWS Textract for OCR: - -```javascript -await (await fetch("http://api.puter.localhost:4100/drivers/call", { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - interface: 'puter-ocr', - driver: 'aws-textract', - method: 'recognize', - args: { - source: '~/Desktop/testocr.png', - }, - }), - "method": "POST", -})).json(); -``` - -## Chat Completion - -Example of using OpenAI for chat completion: - -```javascript -await (await fetch("http://api.puter.localhost:4100/drivers/call", { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - interface: 'puter-chat-completion', - driver: 'openai-completion', - method: 'complete', - args: { - messages: [ - { - role: 'system', - content: 'Act like Spongebob' - }, - { - role: 'user', - content: 'How do I make my code run faster?' - }, - ] - }, - }), - "method": "POST", -})).json(); -``` - -## Image Generation - -Example of using OpenAI for image generation: - -```javascript -URL.createObjectURL(await (await fetch("http://api.puter.localhost:4100/drivers/call", { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - interface: 'puter-image-generation', - driver: 'openai-image-generation', - method: 'generate', - args: { - prompt: 'photorealistic teapot made of swiss cheese', - } - }), - "method": "POST", -})).blob()); -``` - -## Tool Use - -Example of using tool functions with AI: - -```javascript -await puter.ai.chat('What\'s the weather like in Vancouver?', { - tools: [ - { - type: 'function', - 'function': { - name: 'get_weather', - description: 'A string describing the weather', - parameters: { - type: 'object', - properties: { - location: { - type: 'string', - description: 'city', - }, - }, - required: ['location'], - additionalProperties: false, - }, - strict: true - }, - } - ] -}) -``` - -Example with tool response: - -```javascript -await puter.ai.chat([ - { content: `What's the weather like in Vancouver?` }, - { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "call_vcfEOmDczXq7KGMirPGGiNEe", - "type": "function", - "function": { - "name": "get_weather", - "arguments": "{\"location\":\"Vancouver\"}" - } - } - ], - "refusal": null - }, - { - role: 'tool', - tool_call_id: 'call_vcfEOmDczXq7KGMirPGGiNEe', - content: 'Sunny with a chance of rain' - }, -], { - tools: [ - { - type: 'function', - 'function': { - name: 'get_weather', - description: 'A string describing the weather', - parameters: { - type: 'object', - properties: { - location: { - type: 'string', - description: 'city', - }, - }, - required: ['location'], - additionalProperties: false, - }, - strict: true - }, - } - ] -}) -``` - -## Claude Tool Use with Streaming - -Example of using Claude with streaming: - -```javascript -gen = await puter.ai.chat('What\'s the weather like in Vancouver?', { - model: 'claude', - stream: true, - tools: [ - { - type: 'function', - 'function': { - name: 'get_weather', - description: 'A string describing the weather', - parameters: { - type: 'object', - properties: { - location: { - type: 'string', - description: 'city', - }, - }, - required: ['location'], - additionalProperties: false, - }, - strict: true - }, - } - ] -}) -for await ( const thing of gen ) { console.log('thing', thing) } -``` - -Last item in the stream looks like this: -```json -{ - "tool_use": { - "type": "tool_use", - "id": "toolu_01Y4naZhXygjUVRjGBvrL9z8", - "name": "get_weather", - "input": { - "location": "Vancouver" - } - } -} -``` - -Responding to tool use: - -```javascript -gen = await puter.ai.chat([ - { role: 'user', content: `What's the weather like in Vancouver?` }, - { - "role": "assistant", - "content": [ - { type: 'text', text: "I'll check the weather in Vancouver for you." }, - { type: 'tool_use', name: 'get_weather', id: 'toolu_01Y4naZhXygjUVRjGBvrL9z8', input: { location: 'Vancouver' } }, - ] - }, - { - role: 'user', - content: [ - { - type: 'tool_result', - tool_use_id: 'toolu_01Y4naZhXygjUVRjGBvrL9z8', - content: 'Sunny with a chance of rain' - } - ] - }, -], { - model: 'claude', - stream: true, - tools: [ - { - type: 'function', - 'function': { - name: 'get_weather', - description: 'A string describing the weather', - parameters: { - type: 'object', - properties: { - location: { - type: 'string', - description: 'city', - }, - }, - required: ['location'], - additionalProperties: false, - }, - strict: true - }, - } - ] -}) -for await ( const item of gen ) { console.log(item) } -``` \ No newline at end of file diff --git a/src/backend/src/services/ai/docs/config.md b/src/backend/src/services/ai/docs/config.md deleted file mode 100644 index 1828882da..000000000 --- a/src/backend/src/services/ai/docs/config.md +++ /dev/null @@ -1,2 +0,0 @@ -## AI Services Configuration -For details on configuring AI services, see [AI Services Configuration](ai-services-config.md). \ No newline at end of file diff --git a/src/backend/src/services/ai/image/.gitignore b/src/backend/src/services/ai/image/.gitignore deleted file mode 100644 index aa4a6da26..000000000 --- a/src/backend/src/services/ai/image/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.js -*.js.map \ No newline at end of file diff --git a/src/backend/src/services/ai/image/AIImageGenerationService.ts b/src/backend/src/services/ai/image/AIImageGenerationService.ts deleted file mode 100644 index 78d7e2841..000000000 --- a/src/backend/src/services/ai/image/AIImageGenerationService.ts +++ /dev/null @@ -1,336 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { APIError } from '../../../api/APIError.js'; -import { ErrorService } from '../../../modules/core/ErrorService.js'; -import { Context } from '../../../util/context.js'; -import BaseService from '../../BaseService.js'; -import { BaseDatabaseAccessService } from '../../database/BaseDatabaseAccessService.js'; -import { DriverService } from '../../drivers/DriverService.js'; -import { TypedValue } from '../../drivers/meta/Runtime.js'; -import { EventService } from '../../EventService.js'; -import { MeteringService } from '../../MeteringService/MeteringService.js'; -import { CloudflareImageGenerationProvider } from './providers/CloudflareImageGenerationProvider/CloudflareImageGenerationProvider.js'; -import { GeminiImageGenerationProvider } from './providers/GeminiImageGenerationProvider/GeminiImageGenerationProvider.js'; -import { OpenAiImageGenerationProvider } from './providers/OpenAiImageGenerationProvider/OpenAiImageGenerationProvider.js'; -import { TogetherImageGenerationProvider } from './providers/TogetherImageGenerationProvider/TogetherImageGenerationProvider.js'; -import { IGenerateParams, IImageModel, IImageProvider } from './providers/types.js'; -import { ReplicateImageGenerationProvider } from './providers/ReplicateImageGenerationProvider/ReplicateImageGenerationProvider.js'; -import { XAIImageGenerationProvider } from './providers/XAIImageGenerationProvider/XAIImageGenerationProvider.js'; - -export class AIImageGenerationService extends BaseService { - - static SERVICE_NAME = 'ai-image'; - - static DEFAULT_PROVIDER = 'openai-image-generation'; - - get meteringService (): MeteringService { - return this.services.get('meteringService').meteringService; - } - - get db (): BaseDatabaseAccessService { - return this.services.get('database').get(); - } - - get errorService (): ErrorService { - return this.services.get('error-service'); - } - - get eventService (): EventService { - return this.services.get('event'); - } - - get driverService (): DriverService { - return this.services.get('driver'); - } - - getProvider (name: string): IImageProvider | undefined { - return this.#providers[name]; - } - - #providers: Record = {}; - #modelIdMap: Record = {}; - - /** Driver interfaces */ - static IMPLEMENTS = { - 'driver-capabilities': { - supports_test_mode (iface: string, method_name: string) { - return iface === 'puter-image-generation' && - method_name === 'generate'; - }, - }, - 'puter-image-generation': { - - async generate (...parameters: Parameters) { - return (this as unknown as AIImageGenerationService).generate(...parameters); - }, - }, - }; - - getModel ({ modelId, provider }: { modelId: string, provider?: string }) { - const models = this.#modelIdMap[modelId]; - if ( ! models ) { - return undefined; - } - - if ( provider ) { - const model = models.find(m => m.provider === provider); - return model ?? models[0]; - } - - // If no provider is specified, prefer a model whose puterId exactly matches the requested modelId. - const exactPuterIdMatch = models.find(m => m.puterId === modelId); - if ( exactPuterIdMatch ) { - return exactPuterIdMatch; - } - - return models[0]; - } - - private async registerProviders () { - - const openAiConfig = this.config.providers?.['openai-image-generation'] || this.global_config?.services?.['openai'] || this.global_config?.openai; - if ( openAiConfig && (openAiConfig.apiKey || openAiConfig.secret_key) ) { - this.#providers['openai-image-generation'] = new OpenAiImageGenerationProvider({ apiKey: openAiConfig.apiKey || openAiConfig.secret_key }, this.meteringService, this.errorService); - } - - const geminiConfig = this.config.providers?.['gemini-image-generation'] || this.global_config?.services?.gemini; - if ( geminiConfig && (geminiConfig.apiKey || geminiConfig.secret_key) ) { - this.#providers['gemini-image-generation'] = new GeminiImageGenerationProvider({ apiKey: geminiConfig.apiKey || geminiConfig.secret_key }, this.meteringService, this.errorService); - } - - const togetherConfig = this.config.providers?.['together-image-generation'] || this.global_config?.services?.['together-ai']; - if ( togetherConfig && (togetherConfig.apiKey || togetherConfig.secret_key) ) { - this.#providers['together-image-generation'] = new TogetherImageGenerationProvider({ apiKey: togetherConfig.apiKey || togetherConfig.secret_key }, this.meteringService, this.errorService, this.eventService); - } - - const xaiConfig = this.config.providers?.['xai-image-generation'] || this.config.providers?.['xai'] || this.global_config?.services?.['xai']; - if ( xaiConfig && (xaiConfig.apiKey || xaiConfig.secret_key) ) { - this.#providers['xai-image-generation'] = new XAIImageGenerationProvider({ apiKey: xaiConfig.apiKey || xaiConfig.secret_key }, this.meteringService, this.errorService); - } - - const cloudflareImageConfig = this.config.providers?.['cloudflare-image-generation'] || - this.config.providers?.['cloudflare-workers-ai-image'] || - this.global_config?.services?.['cloudflare-image-generation'] || - this.global_config?.services?.['cloudflare-workers-ai-image'] || - this.global_config?.services?.['cloudflare-workers-ai']; - if ( cloudflareImageConfig && (cloudflareImageConfig.apiToken || cloudflareImageConfig.apiKey || cloudflareImageConfig.secret_key) && (cloudflareImageConfig.accountId || cloudflareImageConfig.account_id) ) { - this.#providers['cloudflare-image-generation'] = new CloudflareImageGenerationProvider({ - apiToken: cloudflareImageConfig.apiToken || cloudflareImageConfig.apiKey || cloudflareImageConfig.secret_key, - accountId: cloudflareImageConfig.accountId || cloudflareImageConfig.account_id, - apiBaseUrl: cloudflareImageConfig.apiBaseUrl, - }, this.meteringService, this.errorService, this.eventService); - } - - const replicateConfig = this.config.providers?.['replicate-image-generation'] || this.global_config?.services?.['replicate']; - if ( replicateConfig && (replicateConfig.apiKey || replicateConfig.secret_key) ) { - this.#providers['replicate-image-generation'] = new ReplicateImageGenerationProvider( - { apiKey: replicateConfig.apiKey || replicateConfig.secret_key }, - this.meteringService, - this.errorService, - ); - } - - // emit event for extensions to add providers - const extensionProviders = {} as Record; - await this.eventService.emit('ai.image.registerProviders', extensionProviders); - for ( const providerName in extensionProviders ) { - if ( this.#providers[providerName] ) { - console.warn('AIChatService: provider name conflict for ', providerName, ' registering with -extension suffix'); - this.#providers[`${providerName}-extension`] = extensionProviders[providerName]; - continue; - } - this.#providers[providerName] = extensionProviders[providerName]; - } - } - - protected async '__on_boot.consolidation' () { - // register chat providers here - await this.registerProviders(); - - // build model id map - for ( const providerName in this.#providers ) { - const provider = this.#providers[providerName]; - - // alias all driver requests to go here to support legacy routing - this.driverService.register_service_alias( - AIImageGenerationService.SERVICE_NAME, - providerName, - { iface: 'puter-image-generation' }, - ); - - // build model id map - for ( const model of await provider.models() ) { - model.id = model.id.trim().toLowerCase(); - if ( model.puterId ) { - model.puterId = model.puterId.trim().toLowerCase(); - } - if ( model.aliases ) { - model.aliases = model.aliases.map(alias => alias.trim().toLowerCase()); - } - if ( ! this.#modelIdMap[model.id] ) { - this.#modelIdMap[model.id] = []; - } - this.#modelIdMap[model.id].push({ ...model, provider: providerName }); - - if ( model.puterId ) { - if ( model.aliases ) { - model.aliases.push(model.puterId); - } else { - model.aliases = [model.puterId]; - } - } - - if ( model.aliases ) { - for ( let alias of model.aliases ) { - alias = alias.trim().toLowerCase(); - // join arrays which are aliased the same - if ( ! this.#modelIdMap[alias] ) { - this.#modelIdMap[alias] = this.#modelIdMap[model.id]; - continue; - } - if ( this.#modelIdMap[alias] !== this.#modelIdMap[model.id] ) { - this.#modelIdMap[alias].push({ ...model, provider: providerName }); - this.#modelIdMap[model.id] = this.#modelIdMap[alias]; - continue; - } - } - } - this.#modelIdMap[model.id].sort((a, b) => a.costs[a.index_cost_key || Object.keys(a.costs)[0]] - b.costs[b.index_cost_key || Object.keys(b.costs)[0]]); - } - } - } - - models () { - const seen = new Set(); - return Object.entries(this.#modelIdMap) - .map(([_, models]) => models) - .flat() - .filter(model => { - const identity = `${model.provider}:${model.puterId || model.id}`; - if ( seen.has(identity) ) { - return false; - } - seen.add(identity); - return true; - }) - .sort((a, b) => { - if ( a.provider === b.provider ) { - return a.id.localeCompare(b.id); - } - return a.provider!.localeCompare(b.provider!); - }); - } - - list () { - return this.models().map(m => (m.puterId || m.id)).sort(); - } - - async generate (parameters: IGenerateParams) { - const clientDriverCall = Context.get('client_driver_call'); - let { test_mode: testMode, intended_service: legacyProviderName } = clientDriverCall as { test_mode?: boolean; response_metadata: Record; intended_service?: string }; - - if ( parameters.model ) { - parameters.model = parameters.model.trim().toLowerCase(); - } - - const configuredProviders = Object.keys(this.#providers); - if ( configuredProviders.length === 0 ) { - throw new Error('no image generation providers configured'); - } - - let intendedProvider = (parameters.provider || (legacyProviderName === AIImageGenerationService.SERVICE_NAME ? '' : legacyProviderName)) ?? ''; - if ( intendedProvider === 'xai' ) { - intendedProvider = 'xai-image-generation'; - } - - if ( !parameters.model && !intendedProvider ) { - intendedProvider = configuredProviders.includes(AIImageGenerationService.DEFAULT_PROVIDER) - ? AIImageGenerationService.DEFAULT_PROVIDER - : configuredProviders[0]; - } - - if ( intendedProvider && !this.#providers[intendedProvider] ) { - intendedProvider = configuredProviders[0]; - } - - if ( !parameters.model && intendedProvider ) { - parameters.model = this.#providers[intendedProvider].getDefaultModel(); - } - - const model = parameters.model ? this.getModel({ modelId: parameters.model, provider: intendedProvider }) : undefined; - - if ( ! model ) { - const availableModelsUrl = `${this.global_config.origin }/puterai/image/models`; - - throw APIError.create('field_invalid', undefined, { - key: 'model', - expected: `a valid model name from ${availableModelsUrl}`, - got: model, - }); - } - - // call model provider; - const provider = this.#providers[model.provider!]; - if ( ! provider ) { - throw new Error(`no provider found for model ${model.id}`); - } - - if ( model.allowedRatios?.length ) { - if ( parameters.ratio ) { - const isValidRatio = model.allowedRatios.some(r => r.w === parameters.ratio!.w && r.h === parameters.ratio!.h); - if ( ! isValidRatio ) { - parameters.ratio = model.allowedRatios[0]; - } - } else { - parameters.ratio = model.allowedRatios[0]; - } - } - - if ( ! parameters.ratio ) { - parameters.ratio = { w: 1024, h: 1024 }; - } - - if ( model.allowedQualityLevels?.length ) { - if ( parameters.quality ) { - if ( ! model.allowedQualityLevels.includes(parameters.quality) ) { - parameters.quality = model.allowedQualityLevels[0]; - } - } else { - parameters.quality = model.allowedQualityLevels[0]; - } - } - - const url = await provider.generate({ - ...parameters, - model: model.id, - provider: model.provider, - test_mode: testMode, - }); - - const isDataUrl = url.startsWith('data:'); - const image = new TypedValue({ - $: isDataUrl ? 'string:url:data' : 'string:url:web', - content_type: 'image', - }, url); - - return image; - - } -} diff --git a/src/backend/src/services/ai/image/providers/CloudflareImageGenerationProvider/CloudflareImageGenerationProvider.ts b/src/backend/src/services/ai/image/providers/CloudflareImageGenerationProvider/CloudflareImageGenerationProvider.ts deleted file mode 100644 index b38926023..000000000 --- a/src/backend/src/services/ai/image/providers/CloudflareImageGenerationProvider/CloudflareImageGenerationProvider.ts +++ /dev/null @@ -1,431 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import APIError from '../../../../../api/APIError.js'; -import { ErrorService } from '../../../../../modules/core/ErrorService.js'; -import { Context } from '../../../../../util/context.js'; -import { EventService } from '../../../../EventService.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import { IGenerateParams, IImageModel, IImageProvider } from '../types.js'; -import { CLOUDFLARE_IMAGE_GENERATION_MODELS, CloudflareImageModel } from './models.js'; - -type CloudflareGenerateParams = IGenerateParams & { - steps?: number; - num_steps?: number; - seed?: number; - guidance?: number; - negative_prompt?: string; - output_format?: 'jpeg' | 'png' | 'webp'; - image?: string; -}; - -interface CostComponent { - key: string; - usageAmount: number; - totalCostMicroCents: number; -}; - -const DEFAULT_MODEL = '@cf/black-forest-labs/flux-1-schnell'; -const DEFAULT_RATIO = { w: 1024, h: 1024 }; - -export class CloudflareImageGenerationProvider implements IImageProvider { - #apiToken: string; - #accountId: string; - #apiBaseUrl: string; - #meteringService: MeteringService; - #errors: ErrorService; - #eventService: EventService; - - constructor ( - config: { - apiToken?: string; - apiKey?: string; - secret_key?: string; - accountId?: string; - account_id?: string; - apiBaseUrl?: string; - }, - meteringService: MeteringService, - errorService: ErrorService, - eventService: EventService, - ) { - const apiToken = config.apiToken || config.apiKey || config.secret_key; - if ( ! apiToken ) { - throw new Error('Cloudflare image generation requires `apiToken` (or `apiKey`)'); - } - - const accountId = config.accountId || config.account_id; - if ( ! accountId ) { - throw new Error('Cloudflare image generation requires `accountId`'); - } - - this.#apiToken = apiToken; - this.#accountId = accountId; - this.#apiBaseUrl = config.apiBaseUrl || 'https://api.cloudflare.com/client/v4'; - this.#meteringService = meteringService; - this.#errors = errorService; - this.#eventService = eventService; - } - - models (): IImageModel[] { - return CLOUDFLARE_IMAGE_GENERATION_MODELS; - } - - getDefaultModel (): string { - return DEFAULT_MODEL; - } - - async generate (params: IGenerateParams): Promise { - const options = params as CloudflareGenerateParams; - const { prompt, test_mode } = options; - const ratio = this.#normalizeRatio(options.ratio); - const selectedModel = this.#getModel(options.model); - - await this.#eventService.emit('ai.log.image', { - actor: Context.get('actor'), - parameters: params, - completionId: '0', - intended_service: selectedModel.id, - }); - - if ( test_mode ) { - return 'https://puter-sample-data.puter.site/image_example.png'; - } - - if ( typeof prompt !== 'string' || prompt.trim().length === 0 ) { - throw new Error('`prompt` must be a non-empty string'); - } - - const actor = Context.get('actor'); - if ( ! actor ) { - this.#errors.report('cloudflare-image-generation:unknown-actor', { - message: 'failed to resolve actor for Cloudflare image generation', - trace: true, - }); - throw new Error('actor not found in context'); - } - - const steps = this.#resolveSteps(selectedModel, options); - const costComponents = this.#estimateCost(selectedModel, ratio, steps, { - hasInputImage: typeof options.image === 'string' && options.image.trim() !== '', - }); - const totalCostInMicroCents = costComponents.reduce((acc, component) => acc + component.totalCostMicroCents, 0); - const usageAllowed = await this.#meteringService.hasEnoughCredits(actor, totalCostInMicroCents); - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); - } - - const response = await this.#runModel(selectedModel, { - ...options, - ratio, - steps, - }); - - this.#meteringService.batchIncrementUsages(actor, costComponents - .filter(component => component.usageAmount > 0 && component.totalCostMicroCents > 0) - .map(component => ({ - usageType: `cloudflare:${this.#getMeteringModelKey(selectedModel)}:${component.key}`, - usageAmount: component.usageAmount, - costOverride: component.totalCostMicroCents, - }))); - - return response; - } - - #getModel (model?: string): CloudflareImageModel { - const models = CLOUDFLARE_IMAGE_GENERATION_MODELS; - const found = models.find(m => m.id === model || m.aliases?.includes(model ?? '')); - return found || models.find(m => m.id === DEFAULT_MODEL)!; - } - - #normalizeRatio (ratio?: { w: number; h: number }) { - const width = Number(ratio?.w); - const height = Number(ratio?.h); - if ( Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0 ) { - return { w: Math.max(64, Math.round(width)), h: Math.max(64, Math.round(height)) }; - } - return { ...DEFAULT_RATIO }; - } - - #resolveSteps (model: CloudflareImageModel, options: CloudflareGenerateParams): number { - const input = Number(options.steps ?? options.num_steps ?? model.defaultSteps ?? 25); - const fallback = model.defaultSteps ?? 25; - if ( ! Number.isFinite(input) ) return fallback; - return Math.max(1, Math.min(50, Math.round(input))); - } - - // Cloudflare models have *really exact* billing needs. They pretty much bill based on exactly what the model does - // If a model is a diffusion model, thing flux-2-dev, we actually need to calculate how many steps they take to - // Denoise the model and calculate based on that. It's pretty annoying and we'll have to keep updating this table - // in the future likely. It's VERY easy to screw this up. I would not recommend touching any step based calculations - // unless you actually know what you're doing here, or you might regret it! - // Signed -- NS - #estimateCost ( - model: CloudflareImageModel, - ratio: { w: number; h: number }, - steps: number, - options?: { hasInputImage?: boolean }, - ): CostComponent[] { - const tiles = this.#tileCount(ratio); - const pixels = ratio.w * ratio.h; - const megapixels = this.#megapixels(ratio); - - switch ( model.billingScheme ) { - case 'tile-plus-step': - return [ - { - key: 'tile_512', - usageAmount: tiles, - totalCostMicroCents: this.#costForUnits(tiles, model.costs.tile_512), - }, - { - key: 'step', - usageAmount: steps, - totalCostMicroCents: this.#costForUnits(steps, model.costs.step), - }, - ]; - case 'step-only': - return [ - { - key: 'step', - usageAmount: steps, - totalCostMicroCents: this.#costForUnits(steps, model.costs.step), - }, - ]; - case 'flux2-dev-tile-step': - return [ - { - key: 'input_tile_512_per_step', - usageAmount: tiles * steps, - totalCostMicroCents: this.#costForUnits(tiles * steps, model.costs.input_tile_512_per_step), - }, - { - key: 'output_tile_512_per_step', - usageAmount: tiles * steps, - totalCostMicroCents: this.#costForUnits(tiles * steps, model.costs.output_tile_512_per_step), - }, - ]; - case 'flux2-klein-4b-tile': - return [ - { - key: 'input_tile_512', - usageAmount: tiles, - totalCostMicroCents: this.#costForUnits(tiles, model.costs.input_tile_512), - }, - { - key: 'output_tile_512', - usageAmount: tiles, - totalCostMicroCents: this.#costForUnits(tiles, model.costs.output_tile_512), - }, - ]; - case 'flux2-klein-9b-mp': { - const firstMP = Math.min(megapixels, 1); - const subsequentMP = Math.max(0, megapixels - firstMP); - const firstPixels = Math.min(pixels, 1_000_000); - const subsequentPixels = Math.max(0, pixels - firstPixels); - const inputImageMP = options?.hasInputImage ? megapixels : 0; - return [ - { - key: 'first_mp', - usageAmount: firstMP, - totalCostMicroCents: this.#costForMillionUnits(firstPixels, model.costs.first_mp), - }, - { - key: 'subsequent_mp', - usageAmount: subsequentMP, - totalCostMicroCents: this.#costForMillionUnits(subsequentPixels, model.costs.subsequent_mp), - }, - { - key: 'input_image_mp', - usageAmount: inputImageMP, - totalCostMicroCents: options?.hasInputImage - ? this.#costForMillionUnits(pixels, model.costs.input_image_mp) - : 0, - }, - ]; - } - default: - return []; - } - } - - async #runModel (model: CloudflareImageModel, params: CloudflareGenerateParams & { ratio: { w: number; h: number }, steps: number }) { - const endpoint = `${this.#apiBaseUrl}/accounts/${this.#accountId}/ai/run/${model.id}`; - const headers: Record = { - Authorization: `Bearer ${this.#apiToken}`, - }; - - let body; - if ( model.requiresMultipart ) { - const formData = new FormData(); - formData.append('prompt', params.prompt); - formData.append('width', String(params.ratio.w)); - formData.append('height', String(params.ratio.h)); - formData.append('steps', String(params.steps)); - - if ( Number.isFinite(params.seed) ) formData.append('seed', String(Math.round(params.seed as number))); - if ( Number.isFinite(params.guidance) ) formData.append('guidance', String(params.guidance)); - if ( typeof params.negative_prompt === 'string' ) formData.append('negative_prompt', params.negative_prompt); - if ( typeof params.output_format === 'string' ) formData.append('output_format', params.output_format); - if ( typeof params.image === 'string' ) formData.append('image', params.image); - body = formData; - } else { - headers['Content-Type'] = 'application/json'; - body = JSON.stringify({ - prompt: params.prompt, - width: params.ratio.w, - height: params.ratio.h, - steps: params.steps, - num_steps: params.steps, - ...(Number.isFinite(params.seed) ? { seed: Math.round(params.seed as number) } : {}), - ...(Number.isFinite(params.guidance) ? { guidance: params.guidance } : {}), - ...(typeof params.negative_prompt === 'string' ? { negative_prompt: params.negative_prompt } : {}), - ...(typeof params.output_format === 'string' ? { output_format: params.output_format } : {}), - }); - } - - const response = await fetch(endpoint, { - method: 'POST', - headers, - body, - }); - - const contentType = (response.headers.get('content-type') || '').toLowerCase(); - if ( contentType.startsWith('image/') ) { - const imageBuffer = Buffer.from(await response.arrayBuffer()); - return `data:${contentType};base64,${imageBuffer.toString('base64')}`; - } - - const text = await response.text(); - let payload: unknown; - try { - payload = text ? JSON.parse(text) : {}; - } catch { - payload = { raw: text }; - } - - if ( ! response.ok ) { - const message = - this.#extractErrorMessage(payload) || - `Cloudflare image generation failed with status ${response.status}`; - throw new Error(message); - } - - if ( typeof payload === 'object' && payload !== null ) { - const envelope = payload as Record; - if ( envelope.success === false ) { - const message = - this.#extractErrorMessage(payload) || - 'Cloudflare image generation failed'; - throw new Error(message); - } - } - - const imageString = this.#extractImageString(payload); - if ( ! imageString ) { - throw new Error('Cloudflare image generation response did not include image data'); - } - - if ( imageString.startsWith('data:image/') || imageString.startsWith('http://') || imageString.startsWith('https://') ) { - return imageString; - } - - const mime = this.#mimeForFormat(params.output_format); - return `data:${mime};base64,${imageString}`; - } - - #extractImageString (payload: unknown): string | undefined { - if ( typeof payload === 'string' ) return payload; - if ( !payload || typeof payload !== 'object' ) return undefined; - - const record = payload as Record; - if ( typeof record.image === 'string' ) return record.image; - if ( typeof record.output === 'string' ) return record.output; - if ( Array.isArray(record.images) && typeof record.images[0] === 'string' ) return record.images[0]; - if ( Array.isArray(record.images) && typeof record.images[0] === 'object' && record.images[0] !== null ) { - const firstImage = record.images[0] as Record; - if ( typeof firstImage.image === 'string' ) return firstImage.image; - } - if ( Array.isArray(record.output) && typeof record.output[0] === 'string' ) return record.output[0]; - - if ( record.result ) { - const nested = this.#extractImageString(record.result); - if ( nested ) return nested; - } - if ( record.response ) { - const nested = this.#extractImageString(record.response); - if ( nested ) return nested; - } - return undefined; - } - - #extractErrorMessage (payload: unknown): string | undefined { - if ( !payload || typeof payload !== 'object' ) return undefined; - const record = payload as Record; - - if ( typeof record.error === 'string' ) return record.error; - if ( typeof record.message === 'string' ) return record.message; - if ( Array.isArray(record.errors) && record.errors.length > 0 ) { - const first = record.errors[0] as Record; - if ( typeof first?.message === 'string' ) return first.message; - if ( typeof first?.error === 'string' ) return first.error; - } - return undefined; - } - - #tileCount ({ w, h }: { w: number; h: number }) { - return Math.ceil(w / 512) * Math.ceil(h / 512); - } - - #megapixels ({ w, h }: { w: number; h: number }) { - return (w * h) / 1_000_000; - } - - #mimeForFormat (format?: string) { - if ( format === 'jpeg' ) return 'image/jpeg'; - if ( format === 'webp' ) return 'image/webp'; - return 'image/png'; - } - - #costForUnits (units: number, microCentsPerUnit?: number) { - if ( !Number.isFinite(units) || units <= 0 ) return 0; - if ( !Number.isFinite(microCentsPerUnit) || (microCentsPerUnit as number) <= 0 ) return 0; - return Math.round(units * (microCentsPerUnit as number)); - } - - // `numerator` is in millionths of a unit (e.g. pixels out of 1,000,000 for MP-based pricing). - #costForMillionUnits (numerator: number, microCentsPerMillion?: number) { - if ( !Number.isFinite(numerator) || numerator <= 0 ) return 0; - if ( !Number.isFinite(microCentsPerMillion) || (microCentsPerMillion as number) <= 0 ) return 0; - return Math.round((numerator * (microCentsPerMillion as number)) / 1_000_000); - } - - #getMeteringModelKey (model: CloudflareImageModel) { - if ( model.puterId && typeof model.puterId === 'string' ) { - return model.puterId; - } - - if ( model.id.startsWith('@cf/') ) { - return `workers-ai:${model.id.slice('@cf/'.length)}`; - } - - return model.id.replace(/^@+/, ''); - } - -} diff --git a/src/backend/src/services/ai/image/providers/GeminiImageGenerationProvider/GeminiImageGenerationProvider.ts b/src/backend/src/services/ai/image/providers/GeminiImageGenerationProvider/GeminiImageGenerationProvider.ts deleted file mode 100644 index a9ca08fe0..000000000 --- a/src/backend/src/services/ai/image/providers/GeminiImageGenerationProvider/GeminiImageGenerationProvider.ts +++ /dev/null @@ -1,430 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { GenerateContentResponse, GoogleGenAI } from '@google/genai'; -import APIError from '../../../../../api/APIError.js'; -import { ErrorService } from '../../../../../modules/core/ErrorService.js'; -import { Context } from '../../../../../util/context.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import { GEMINI_DEFAULT_RATIO, GEMINI_ESTIMATED_IMAGE_TOKENS, GEMINI_IMAGE_GENERATION_MODELS, IGeminiImageModel } from './models.js'; -import { IGenerateParams, IImageModel, IImageProvider } from '../types.js'; - -const MIME_SIGNATURES: Record = { - '/9j/': 'image/jpeg', - 'iVBOR': 'image/png', - 'UklGR': 'image/webp', -}; - -interface GeminiUsageMetadata { - promptTokenCount: number; - candidatesTokenCount: number; - candidatesTextTokenCount: number; - candidatesImageTokenCount: number; - thoughtsTokenCount: number; -} - -export class GeminiImageGenerationProvider implements IImageProvider { - #meteringService: MeteringService; - #client: GoogleGenAI; - #errors: ErrorService; - - constructor (config: { apiKey: string }, meteringService: MeteringService, errorService: ErrorService) { - if ( ! config.apiKey ) { - throw new Error('Gemini image generation requires an API key'); - } - this.#meteringService = meteringService; - this.#client = new GoogleGenAI({ apiKey: config.apiKey }); - this.#errors = errorService; - } - - models (): IImageModel[] { - return GEMINI_IMAGE_GENERATION_MODELS; - } - - getDefaultModel (): string { - return GEMINI_IMAGE_GENERATION_MODELS[0].id; - } - - async generate (params: IGenerateParams): Promise { - const { prompt, test_mode, input_image, input_image_mime_type, model, quality } = params; - let { ratio, input_images } = params; - - const selectedModel = (this.models() as IGeminiImageModel[]).find(m => m.id === model) - || (this.models() as IGeminiImageModel[]).find(m => m.id === this.getDefaultModel())!; - - if ( test_mode ) { - return 'https://puter-sample-data.puter.site/image_example.png'; - } - - if ( typeof prompt !== 'string' || prompt.trim().length === 0 ) { - throw new Error('`prompt` must be a non-empty string'); - } - - if ( selectedModel.apiType === 'generateImages' ) { - return this.#generateWithImagen(prompt, selectedModel, params); - } - - const allowedRatios = selectedModel.allowedRatios ?? [GEMINI_DEFAULT_RATIO]; - ratio = ratio && this.#isValidRatio(ratio, allowedRatios) ? ratio : allowedRatios[0]; - - // Backwards compat: merge singular input_image into input_images - if ( input_image && (!input_images || input_images.length === 0) ) { - input_images = [input_image]; - } - - // Validate input images have detectable MIME types - if ( input_images?.length ) { - for ( const img of input_images ) { - const mime = this.#detectMimeType(img) ?? input_image_mime_type; - if ( ! mime ) { - throw new Error('Could not detect MIME type for an input image. Provide a known image format (JPEG, PNG, WebP) or set `input_image_mime_type`.'); - } - } - } - - const actor = Context.get('actor'); - const user_private_uid = actor?.private_uid ?? 'UNKNOWN'; - if ( user_private_uid === 'UNKNOWN' ) { - this.#errors.report('gemini-image-generation:unknown-user', { - message: 'failed to get a user ID for a Gemini request', - alarm: true, - trace: true, - }); - } - - // --- Pre-flight cost estimation --- - const inputImageCount = input_images?.length ?? 0; - const estimatedImageInputTokens = inputImageCount * 560; // https://ai.google.dev/gemini-api/docs/pricing#gemini-3-pro-image-preview - const estimatedPromptTokenCount = this.#estimatePromptTokenCount(prompt) + estimatedImageInputTokens; - const estimatedInputCostInCents = this.#calculateTokenCostInCents(estimatedPromptTokenCount, selectedModel.costs.input); - - // Estimate output image tokens - const imageTokenKey = quality ? `${selectedModel.id}:${quality}` : selectedModel.id; - const estimatedOutputImageTokens = GEMINI_ESTIMATED_IMAGE_TOKENS[imageTokenKey] ?? GEMINI_ESTIMATED_IMAGE_TOKENS[selectedModel.id]; - if ( estimatedOutputImageTokens === undefined ) { - throw new Error(`No estimated image token count configured for '${imageTokenKey}'.`); - } - const estimatedOutputImageCostInCents = this.#calculateTokenCostInCents(estimatedOutputImageTokens, selectedModel.costs.output_image); - const estimatedOutputTextCostInCents = this.#calculateTokenCostInCents(50, selectedModel.costs.output); // small text overhead estimate - const estimatedOutputCostInCents = estimatedOutputImageCostInCents + estimatedOutputTextCostInCents; - - const estimatedTotalCostInMicroCents = this.#toMicroCents(estimatedInputCostInCents + estimatedOutputCostInCents); - const usageAllowed = await this.#meteringService.hasEnoughCredits(actor, estimatedTotalCostInMicroCents); - - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); - } - - // --- API call --- - const contents = this.#buildContents(prompt, input_images, input_image_mime_type); - const aspectRatio = `${ratio.w}:${ratio.h}`; - - const imageConfig: Record = { aspectRatio }; - if ( quality && selectedModel.allowedQualityLevels?.includes(quality) ) { - imageConfig.imageSize = quality; - } - - const response = await this.#client.models.generateContent({ - model: selectedModel.id, - contents, - config: { - responseModalities: ['TEXT', 'IMAGE'], - imageConfig, - }, - }); - - // --- Actual cost calculation from response usage --- - const usage = this.#extractUsageMetadata(response); - const inputTokenCount = usage.promptTokenCount || estimatedPromptTokenCount; - - const outputTextTokenCount = usage.candidatesTextTokenCount + usage.thoughtsTokenCount; - const outputImageTokenCount = usage.candidatesImageTokenCount || estimatedOutputImageTokens; - - const inputCostInCents = this.#calculateTokenCostInCents(inputTokenCount, selectedModel.costs.input); - const outputTextCostInCents = this.#calculateTokenCostInCents(outputTextTokenCount, selectedModel.costs.output); - const outputImageCostInCents = this.#calculateTokenCostInCents(outputImageTokenCount, selectedModel.costs.output_image); - const outputCostInCents = outputTextCostInCents + outputImageCostInCents; - - const totalOutputTokenCount = outputTextTokenCount + outputImageTokenCount; - const usagePrefix = `gemini:${selectedModel.id}`; - this.#meteringService.batchIncrementUsages(actor, [ - { - usageType: `${usagePrefix}:input`, - usageAmount: Math.max(inputTokenCount, 1), - costOverride: this.#toMicroCents(inputCostInCents), - }, - { - usageType: `${usagePrefix}:output:text`, - usageAmount: Math.max(outputTextTokenCount, 1), - costOverride: this.#toMicroCents(outputTextCostInCents), - }, - { - usageType: `${usagePrefix}:output:image`, - usageAmount: Math.max(outputImageTokenCount, 1), - costOverride: this.#toMicroCents(outputImageCostInCents), - }, - ]); - - this.#setResponseCostMetadata({ - model: selectedModel.id, - quality, - ratio, - inputCostInCents, - outputCostInCents, - inputTokenCount, - outputTokenCount: totalOutputTokenCount, - outputTextTokenCount, - outputImageTokenCount, - }); - - const url = this.#extractImageUrl(response); - - if ( ! url ) { - throw new Error('Failed to extract image URL from Gemini response'); - } - - return url; - } - - async #generateWithImagen (prompt: string, selectedModel: IGeminiImageModel, params: IGenerateParams): Promise { - const actor = Context.get('actor'); - if ( ! actor ) { - throw new Error('actor not found in context'); - } - const costCents = selectedModel.costs?.['per-image']; - if ( costCents === undefined ) { - throw new Error(`No per-image cost configured for model '${selectedModel.id}'`); - } - const costInMicroCents = Math.ceil(costCents * 1_000_000); - - const usageAllowed = await this.#meteringService.hasEnoughCredits(actor, costInMicroCents); - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); - } - - const allowedRatios = selectedModel.allowedRatios ?? [GEMINI_DEFAULT_RATIO]; - const ratio = params.ratio && this.#isValidRatio(params.ratio, allowedRatios) - ? params.ratio : allowedRatios[0]; - const aspectRatio = `${ratio.w}:${ratio.h}`; - - const config: Record = { - numberOfImages: 1, - aspectRatio, - }; - - if ( params.quality && selectedModel.allowedQualityLevels?.includes(params.quality) ) { - config.imageSize = params.quality; - } - - const response = await this.#client.models.generateImages({ - model: selectedModel.id, - prompt, - config, - }); - - const generated = response?.generatedImages; - if ( !generated || generated.length === 0 ) { - throw new Error('Imagen response did not include an image'); - } - - const entry = generated[0]; - if ( entry.raiFilteredReason ) { - throw new Error(`Image was filtered: ${entry.raiFilteredReason}`); - } - - const image = entry.image; - if ( ! image?.imageBytes ) { - throw new Error('Imagen response did not include image bytes'); - } - - const usageKey = `gemini:${selectedModel.id}`; - await this.#meteringService.incrementUsage(actor, usageKey, 1, costInMicroCents); - - const mimeType = image.mimeType ?? 'image/png'; - return `data:${mimeType};base64,${image.imageBytes}`; - } - - #buildContents (prompt: string, input_images?: string[], input_image_mime_type?: string) { - const parts: Record[] = [{ text: prompt }]; - - if ( input_images?.length ) { - for ( const img of input_images ) { - const parsed = this.#parseDataUri(img); - const mimeType = parsed?.mimeType ?? this.#detectMimeType(img) ?? input_image_mime_type ?? 'image/png'; - const rawBase64 = parsed?.base64 ?? img; - parts.push({ - inlineData: { - mimeType, - data: rawBase64, - }, - }); - } - } - - return parts; - } - - #setResponseCostMetadata ({ - model, - quality, - ratio, - inputCostInCents, - outputCostInCents, - inputTokenCount, - outputTokenCount, - outputTextTokenCount, - outputImageTokenCount, - }: { - model: string; - quality?: string; - ratio: { w: number; h: number }; - inputCostInCents: number; - outputCostInCents: number; - inputTokenCount: number; - outputTokenCount: number; - outputTextTokenCount: number; - outputImageTokenCount: number; - }) { - const clientDriverCall = Context.get('client_driver_call') as { response_metadata?: Record } | undefined; - const responseMetadata = clientDriverCall?.response_metadata; - if ( ! responseMetadata ) return; - - const totalCostInCents = inputCostInCents + outputCostInCents; - responseMetadata.cost = { - currency: 'usd-cents', - input: inputCostInCents, - output: outputCostInCents, - total: totalCostInCents, - }; - responseMetadata.cost_components = { - provider: 'gemini-image-generation', - model, - quality, - ratio: `${ratio.w}x${ratio.h}`, - input_tokens: inputTokenCount, - output_tokens: outputTokenCount, - output_text_tokens: outputTextTokenCount, - output_image_tokens: outputImageTokenCount, - input_microcents: this.#toMicroCents(inputCostInCents), - output_microcents: this.#toMicroCents(outputCostInCents), - total_microcents: this.#toMicroCents(totalCostInCents), - }; - } - - #extractUsageMetadata (response: GenerateContentResponse): GeminiUsageMetadata { - const usage = (response as GenerateContentResponse & { usageMetadata?: Record }).usageMetadata; - - let candidatesImageTokenCount = 0; - - const details = usage?.candidatesTokensDetails; - if ( Array.isArray(details) ) { - for ( const entry of details ) { - if ( entry?.modality === 'IMAGE' ) { - candidatesImageTokenCount += this.#toSafeCount(entry.tokenCount); - } - } - } - - // api only returns modality image, so calculate text tokens as candidates (output) - image tokens - const candidatesTokenCount = this.#toSafeCount(usage?.candidatesTokenCount); - const candidatesTextTokenCount = Math.max(0, candidatesTokenCount - candidatesImageTokenCount); - - return { - promptTokenCount: this.#toSafeCount(usage?.promptTokenCount), - candidatesTokenCount, - candidatesTextTokenCount, - candidatesImageTokenCount, - thoughtsTokenCount: this.#toSafeCount(usage?.thoughtsTokenCount), - }; - } - - #estimatePromptTokenCount (prompt: string): number { - const text = prompt.trim(); - if ( text.length === 0 ) return 0; - - // Same approximation used by chat billing flow. - return Math.max(1, Math.floor(((text.length / 4) + (text.split(/\s+/).length * (4 / 3))) / 2)); - } - - #calculateTokenCostInCents (tokenCount: number, centsPerMillion?: number): number { - if ( !Number.isFinite(tokenCount) || tokenCount <= 0 ) return 0; - if ( !Number.isFinite(centsPerMillion) || (centsPerMillion ?? 0) <= 0 ) return 0; - - return (tokenCount / 1_000_000) * (centsPerMillion as number); - } - - #toMicroCents (cents: number): number { - if ( !Number.isFinite(cents) || cents <= 0 ) return 1; - return Math.ceil(cents * 1_000_000); - } - - #toSafeCount (value: unknown): number { - if ( typeof value !== 'number' || !Number.isFinite(value) || value < 0 ) return 0; - return Math.floor(value); - } - - #extractImageUrl (response: GenerateContentResponse): string | undefined { - const parts = response?.candidates?.[0]?.content?.parts; - if ( ! Array.isArray(parts) ) { - return undefined; - } - - for ( const part of parts ) { - if ( part?.inlineData?.data ) { - const mimeType = part.inlineData.mimeType ?? 'image/png'; - return `data:${mimeType};base64,${ part.inlineData.data}`; - } - } - return undefined; - } - - #detectMimeType (data: string): string | undefined { - // Handle data URIs like "data:image/jpeg;base64,..." - const parsed = this.#parseDataUri(data); - if ( parsed ) { - return parsed.mimeType; - } - - for ( const [signature, mimeType] of Object.entries(MIME_SIGNATURES) ) { - if ( data.startsWith(signature) ) { - return mimeType; - } - } - return undefined; - } - - #parseDataUri (data: string): { mimeType: string; base64: string } | undefined { - if ( ! data.startsWith('data:image/') ) return undefined; - - const commaIdx = data.indexOf(','); - if ( commaIdx === -1 ) return undefined; - - const header = data.substring(5, commaIdx); // after "data:" up to "," - if ( ! header.endsWith(';base64') ) return undefined; - - const mimeType = header.substring(0, header.length - 7); // strip ";base64" - if ( mimeType.length === 0 ) return undefined; - - return { mimeType, base64: data.substring(commaIdx + 1) }; - } - - #isValidRatio (ratio: { w: number; h: number }, allowedRatios: { w: number; h: number }[]) { - return allowedRatios.some(r => r.w === ratio.w && r.h === ratio.h); - } -} diff --git a/src/backend/src/services/ai/image/providers/OpenAiImageGenerationProvider/OpenAiImageGenerationProvider.ts b/src/backend/src/services/ai/image/providers/OpenAiImageGenerationProvider/OpenAiImageGenerationProvider.ts deleted file mode 100644 index 3147c076f..000000000 --- a/src/backend/src/services/ai/image/providers/OpenAiImageGenerationProvider/OpenAiImageGenerationProvider.ts +++ /dev/null @@ -1,549 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import openai, { OpenAI } from 'openai'; -import { ImageGenerateParamsNonStreaming, ImagesResponse } from 'openai/resources/images.js'; -import APIError from '../../../../../api/APIError.js'; -import { ErrorService } from '../../../../../modules/core/ErrorService.js'; -import { Context } from '../../../../../util/context.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import { IGenerateParams, IImageModel, IImageProvider } from '../types.js'; -import { OPEN_AI_IMAGE_GENERATION_MODELS } from './models.js'; - -interface OpenAIImageUsage { - inputTokens: number; - outputTokens: number; - inputTextTokens: number; - inputImageTokens: number; - cachedInputTokens: number; - cachedInputTextTokens: number; - cachedInputImageTokens: number; -} - -/** -* Service class for generating images using OpenAI's DALL-E API. -* Extends BaseService to provide image generation capabilities through -* the puter-image-generation interface. Supports different aspect ratios -* (square, portrait, landscape) and handles API authentication, request -* validation, and spending tracking. -*/ -export class OpenAiImageGenerationProvider implements IImageProvider { - #meteringService: MeteringService; - #openai: OpenAI; - #errors: ErrorService; - - static #NON_SIZE_COST_KEYS = [ - 'text_input', - 'text_cached_input', - 'text_output', - 'image_input', - 'image_cached_input', - 'image_output', - ]; - - constructor (config: { apiKey: string }, meteringService: MeteringService, errorService: ErrorService) { - this.#meteringService = meteringService; - this.#openai = new openai.OpenAI({ - apiKey: config.apiKey, - }); - this.#errors = errorService; - } - - models () { - return OPEN_AI_IMAGE_GENERATION_MODELS; - } - - getDefaultModel (): string { - return 'dall-e-2'; - } - - async generate ({ prompt, quality, test_mode, model, ratio }: IGenerateParams) { - - const selectedModel = this.models().find(m => m.id === model) || this.models().find(m => m.id === this.getDefaultModel())!; - - if ( test_mode ) { - return 'https://puter-sample-data.puter.site/image_example.png'; - } - - if ( typeof prompt !== 'string' ) { - throw new Error('`prompt` must be a string'); - } - - const validRatios = selectedModel?.allowedRatios; - if ( validRatios ) { - if ( !ratio || !validRatios.some(r => r.w === ratio.w && r.h === ratio.h) ) { - ratio = validRatios[0]; // Default to the first allowed ratio - } - } else { - // Open-ended size models (gpt-image-2): conform to OpenAI's size - // rules (16px multiples, 3840 cap, 3:1 ratio, pixel budget). - ratio = this.#normalizeGptImage2Ratio(ratio); - } - - if ( ! ratio ) { - ratio = { w: 1024, h: 1024 }; // Fallback ratio - } - - const validQualities = selectedModel?.allowedQualityLevels; - if ( validQualities && (!quality || !validQualities.includes(quality)) ) { - quality = validQualities[0]; // Default to the first allowed quality - } - - const size = `${ratio.w}x${ratio.h}`; - const price_key = this.#buildPriceKey(selectedModel.id, quality!, size); - let outputPriceInCents: number | undefined = selectedModel?.costs[price_key]; - if ( outputPriceInCents === undefined ) { - outputPriceInCents = this.#estimateOutputCostFromTokens(selectedModel, ratio, quality); - } - if ( outputPriceInCents === undefined ) { - const availableSizes = Object.keys(selectedModel?.costs) - .filter(key => !OpenAiImageGenerationProvider.#NON_SIZE_COST_KEYS.includes(key)); - throw APIError.create('field_invalid', undefined, { - key: 'size/quality combination', - expected: `one of: ${ availableSizes.join(', ')}`, - got: price_key, - }); - } - - const actor = Context.get('actor'); - const user_private_uid = actor?.private_uid ?? 'UNKNOWN'; - if ( user_private_uid === 'UNKNOWN' ) { - this.#errors.report('chat-completion-service:unknown-user', { - message: 'failed to get a user ID for an OpenAI request', - alarm: true, - trace: true, - }); - } - - const estimatedPromptTokenCount = this.#estimatePromptTokenCount(prompt); - const estimatedInputCostInCents = this.#calculateInputCostInCents(selectedModel, { - inputTokens: estimatedPromptTokenCount, - inputTextTokens: estimatedPromptTokenCount, - inputImageTokens: 0, - cachedInputTokens: 0, - cachedInputTextTokens: 0, - cachedInputImageTokens: 0, - } as OpenAIImageUsage); - const estimatedOutputCostInCents = outputPriceInCents; - const estimatedTotalCostInMicroCents = this.#toMicroCents(estimatedInputCostInCents + estimatedOutputCostInCents); - const usageAllowed = await this.#meteringService.hasEnoughCredits(actor, estimatedTotalCostInMicroCents); - - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); - } - - // Build API parameters based on model - const apiParams = this.#buildApiParams(selectedModel.id, { - user: user_private_uid, - prompt, - size, - quality, - } as Partial); - - const result = await this.#openai.images.generate(apiParams); - - const usage = this.#extractUsage(result); - const hasInputTokenUsage = - usage.inputTokens > 0 || - usage.inputTextTokens > 0 || - usage.inputImageTokens > 0; - const hasOutputTokenUsage = usage.outputTokens > 0; - - const billableUsage = hasInputTokenUsage ? usage : { - ...usage, - inputTokens: estimatedPromptTokenCount, - inputTextTokens: estimatedPromptTokenCount, - }; - - const inputCostInCents = hasInputTokenUsage - ? this.#calculateInputCostInCents(selectedModel, billableUsage) - : estimatedInputCostInCents; - const outputCostInCents = this.#calculateOutputCostInCents(selectedModel, usage, outputPriceInCents); - - const usageType = `openai:${selectedModel.id}:${price_key}`; - const usageEntries: Array<{ usageType: string; usageAmount: number; costOverride: number }> = []; - if ( inputCostInCents > 0 ) { - usageEntries.push({ - usageType: `${usageType}:input`, - usageAmount: Math.max(billableUsage.inputTokens || estimatedPromptTokenCount, 1), - costOverride: this.#toMicroCents(inputCostInCents), - }); - } - if ( outputCostInCents > 0 ) { - usageEntries.push({ - usageType: `${usageType}:output`, - usageAmount: Math.max(usage.outputTokens, 1), - costOverride: this.#toMicroCents(outputCostInCents), - }); - } - if ( usageEntries.length ) { - this.#meteringService.batchIncrementUsages(actor, usageEntries); - } - - this.#setResponseCostMetadata({ - model: selectedModel.id, - quality, - ratio, - inputCostInCents, - outputCostInCents, - usage: billableUsage, - inputUsageSource: hasInputTokenUsage ? 'token-usage' : 'prompt-estimate', - outputUsageSource: hasOutputTokenUsage ? 'token-usage' : 'per-image-fallback', - outputPriceInCents, - }); - - const url = result.data?.[0]?.url || (result.data?.[0]?.b64_json ? `data:image/png;base64,${ result.data[0].b64_json}` : null); - - if ( ! url ) { - throw new Error('Failed to extract image URL from OpenAI response'); - } - - return url; - } - - #extractUsage (result: ImagesResponse): OpenAIImageUsage { - const usage = (result.usage ?? {}) as ImagesResponse.Usage & Record; - const inputTokens = this.#toSafeCount(usage.input_tokens); - const outputTokens = this.#toSafeCount(usage.output_tokens); - - const inputDetails = (usage.input_tokens_details ?? {}) as unknown as Record; - const inputTextTokens = this.#toSafeCount(inputDetails.text_tokens); - const inputImageTokens = this.#toSafeCount(inputDetails.image_tokens); - - const cachedInputTokens = Math.max( - this.#toSafeCount((usage as Record).cached_input_tokens), - this.#toSafeCount(inputDetails.cached_tokens), - ); - - const cachedDetails = ((inputDetails.cached_tokens_details || inputDetails.cache_tokens_details) ?? {}) as Record; - const cachedInputTextTokens = this.#toSafeCount(cachedDetails.text_tokens); - const cachedInputImageTokens = this.#toSafeCount(cachedDetails.image_tokens); - - return { - inputTokens, - outputTokens, - inputTextTokens, - inputImageTokens, - cachedInputTokens, - cachedInputTextTokens, - cachedInputImageTokens, - }; - } - - #calculateInputCostInCents (selectedModel: IImageModel, usage: OpenAIImageUsage): number { - if ( ! this.#isGptImageModel(selectedModel.id) ) { - return 0; - } - - const textInputRate = this.#getCostRate(selectedModel, 'text_input'); - const textCachedInputRate = this.#getCostRate(selectedModel, 'text_cached_input') ?? textInputRate; - const imageInputRate = this.#getCostRate(selectedModel, 'image_input'); - const imageCachedInputRate = this.#getCostRate(selectedModel, 'image_cached_input') ?? imageInputRate; - - if ( textInputRate === undefined && imageInputRate === undefined ) { - return 0; - } - - const totalInputTokens = Math.max(usage.inputTokens, usage.inputTextTokens + usage.inputImageTokens); - let textTokens = usage.inputTextTokens; - let imageTokens = usage.inputImageTokens; - - // Current image generate calls are usually text-only prompts. - if ( textTokens + imageTokens === 0 && totalInputTokens > 0 ) { - textTokens = totalInputTokens; - } - - const knownInputTokens = textTokens + imageTokens; - let cachedInputTokens = Math.min(usage.cachedInputTokens, knownInputTokens || totalInputTokens); - - let cachedTextTokens = Math.min(usage.cachedInputTextTokens, textTokens); - let cachedImageTokens = Math.min(usage.cachedInputImageTokens, imageTokens); - - let cachedRemaining = Math.max(0, cachedInputTokens - (cachedTextTokens + cachedImageTokens)); - if ( cachedRemaining > 0 ) { - const availableText = Math.max(textTokens - cachedTextTokens, 0); - const availableImage = Math.max(imageTokens - cachedImageTokens, 0); - const availableTotal = availableText + availableImage; - - if ( availableTotal > 0 ) { - const proportionalText = Math.min(availableText, Math.round((availableText / availableTotal) * cachedRemaining)); - cachedTextTokens += proportionalText; - cachedRemaining -= proportionalText; - - const proportionalImage = Math.min(availableImage, cachedRemaining); - cachedImageTokens += proportionalImage; - cachedRemaining -= proportionalImage; - } - - if ( cachedRemaining > 0 && textTokens > cachedTextTokens ) { - const extraText = Math.min(textTokens - cachedTextTokens, cachedRemaining); - cachedTextTokens += extraText; - cachedRemaining -= extraText; - } - - if ( cachedRemaining > 0 && imageTokens > cachedImageTokens ) { - const extraImage = Math.min(imageTokens - cachedImageTokens, cachedRemaining); - cachedImageTokens += extraImage; - cachedRemaining -= extraImage; - } - } - - const uncachedTextTokens = Math.max(textTokens - cachedTextTokens, 0); - const uncachedImageTokens = Math.max(imageTokens - cachedImageTokens, 0); - - return this.#costForTokens(uncachedTextTokens, textInputRate) - + this.#costForTokens(cachedTextTokens, textCachedInputRate) - + this.#costForTokens(uncachedImageTokens, imageInputRate) - + this.#costForTokens(cachedImageTokens, imageCachedInputRate); - } - - #calculateOutputCostInCents (selectedModel: IImageModel, usage: OpenAIImageUsage, fallbackPriceInCents: number): number { - if ( ! this.#isGptImageModel(selectedModel.id) ) { - return fallbackPriceInCents; - } - - if ( usage.outputTokens <= 0 ) { - return fallbackPriceInCents; - } - - const imageOutputRate = this.#getCostRate(selectedModel, 'image_output'); - if ( imageOutputRate !== undefined ) { - return this.#costForTokens(usage.outputTokens, imageOutputRate); - } - - const textOutputRate = this.#getCostRate(selectedModel, 'text_output'); - if ( textOutputRate !== undefined ) { - return this.#costForTokens(usage.outputTokens, textOutputRate); - } - - return fallbackPriceInCents; - } - - #setResponseCostMetadata ({ - model, - quality, - ratio, - inputCostInCents, - outputCostInCents, - usage, - inputUsageSource, - outputUsageSource, - outputPriceInCents, - }: { - model: string; - quality?: string; - ratio: { w: number; h: number }; - inputCostInCents: number; - outputCostInCents: number; - usage: OpenAIImageUsage; - inputUsageSource: 'token-usage' | 'prompt-estimate'; - outputUsageSource: 'token-usage' | 'per-image-fallback'; - outputPriceInCents: number; - }) { - const clientDriverCall = Context.get('client_driver_call') as { response_metadata?: Record } | undefined; - const responseMetadata = clientDriverCall?.response_metadata; - if ( ! responseMetadata ) return; - - const totalCostInCents = inputCostInCents + outputCostInCents; - responseMetadata.cost = { - currency: 'usd-cents', - input: inputCostInCents, - output: outputCostInCents, - total: totalCostInCents, - }; - responseMetadata.cost_components = { - provider: 'openai-image-generation', - model, - quality, - ratio: `${ratio.w}x${ratio.h}`, - input_usage_source: inputUsageSource, - output_usage_source: outputUsageSource, - output_image_price_cents: outputPriceInCents, - input_tokens: usage.inputTokens, - output_tokens: usage.outputTokens, - input_text_tokens: usage.inputTextTokens, - input_image_tokens: usage.inputImageTokens, - cached_input_tokens: usage.cachedInputTokens, - cached_input_text_tokens: usage.cachedInputTextTokens, - cached_input_image_tokens: usage.cachedInputImageTokens, - input_microcents: this.#toMicroCents(inputCostInCents), - output_microcents: this.#toMicroCents(outputCostInCents), - total_microcents: this.#toMicroCents(totalCostInCents), - }; - } - - #estimatePromptTokenCount (prompt: string): number { - const text = prompt.trim(); - if ( text.length === 0 ) return 0; - - // Same approximation used by chat and Gemini image billing flows. - return Math.max(1, Math.floor(((text.length / 4) + (text.split(/\s+/).length * (4 / 3))) / 2)); - } - - #getCostRate (selectedModel: IImageModel, key: string): number | undefined { - const value = selectedModel.costs[key]; - if ( ! Number.isFinite(value) ) { - return undefined; - } - return value; - } - - #costForTokens (tokenCount: number, centsPerMillion?: number): number { - if ( !Number.isFinite(tokenCount) || tokenCount <= 0 ) return 0; - if ( !Number.isFinite(centsPerMillion) || (centsPerMillion ?? 0) <= 0 ) return 0; - return (tokenCount / 1_000_000) * (centsPerMillion as number); - } - - #toMicroCents (cents: number): number { - if ( !Number.isFinite(cents) || cents <= 0 ) return 1; - return Math.ceil(cents * 1_000_000); - } - - #toSafeCount (value: unknown): number { - if ( typeof value !== 'number' || !Number.isFinite(value) || value < 0 ) return 0; - return Math.floor(value); - } - - #isGptImageModel (model: string) { - // Covers gpt-image-1, gpt-image-1-mini, gpt-image-1.5, gpt-image-2 and future variants. - return model.startsWith('gpt-image-'); - } - - // gpt-image-2 size rules: each edge in [16, 3840] and a multiple of 16, - // long:short ratio ≤ 3:1, pixel count in [655360, 8294400]. Silently - // clamps/snaps rather than throwing so arbitrary user input is accepted. - // https://developers.openai.com/api/docs/guides/image-generation - #normalizeGptImage2Ratio (ratio?: { w: number; h: number }) { - const MIN_EDGE = 16; - const MAX_EDGE = 3840; - const STEP = 16; - const MAX_RATIO = 3; - const MIN_PIXELS = 655_360; - const MAX_PIXELS = 8_294_400; - - let w = Number(ratio?.w); - let h = Number(ratio?.h); - if ( !Number.isFinite(w) || !Number.isFinite(h) || w <= 0 || h <= 0 ) { - return { w: 1024, h: 1024 }; - } - - // 1. Clamp long:short ratio to MAX_RATIO by shrinking the longer edge. - if ( w / h > MAX_RATIO ) w = h * MAX_RATIO; - else if ( h / w > MAX_RATIO ) h = w * MAX_RATIO; - - // 2. Cap each edge at MAX_EDGE, preserving aspect ratio. - if ( w > MAX_EDGE ) { - const s = MAX_EDGE / w; w = MAX_EDGE; h *= s; - } - if ( h > MAX_EDGE ) { - const s = MAX_EDGE / h; h = MAX_EDGE; w *= s; - } - - // 3. Scale uniformly into the pixel budget. - const prescaledPixels = w * h; - if ( prescaledPixels < MIN_PIXELS ) { - const s = Math.sqrt(MIN_PIXELS / prescaledPixels); - w *= s; h *= s; - } else if ( prescaledPixels > MAX_PIXELS ) { - const s = Math.sqrt(MAX_PIXELS / prescaledPixels); - w *= s; h *= s; - } - - // 4. Snap to STEP. Bias rounding direction so snap doesn't push pixels - // back out of the budget. - const dir = prescaledPixels < MIN_PIXELS ? 1 - : prescaledPixels > MAX_PIXELS ? -1 - : 0; - const snap = (v: number) => { - const snapped = dir > 0 ? Math.ceil(v / STEP) * STEP - : dir < 0 ? Math.floor(v / STEP) * STEP - : Math.round(v / STEP) * STEP; - return Math.max(MIN_EDGE, Math.min(MAX_EDGE, snapped)); - }; - w = snap(w); h = snap(h); - - // 5. If snap rounding pushed ratio above MAX_RATIO, trim the longer - // edge by one STEP. Pixel budget had headroom from step 3 so this - // won't drop below MIN_PIXELS. - if ( Math.max(w, h) / Math.min(w, h) > MAX_RATIO ) { - if ( w >= h ) w = Math.max(MIN_EDGE, w - STEP); - else h = Math.max(MIN_EDGE, h - STEP); - } - return { w, h }; - } - - // extracted from calculator at https://developers.openai.com/api/docs/guides/image-generation#cost-and-latency - #estimateGptImage2OutputTokens (width: number, height: number, quality?: string): number { - const FACTORS: Record = { low: 16, medium: 48, high: 96 }; - const factor = FACTORS[quality ?? ''] ?? FACTORS.medium; - const longEdge = Math.max(width, height); - const shortEdge = Math.min(width, height); - const shortLatent = Math.round(factor * shortEdge / longEdge); - const latentW = width >= height ? factor : shortLatent; - const latentH = width >= height ? shortLatent : factor; - const baseArea = latentW * latentH; - return Math.ceil(baseArea * (2_000_000 + width * height) / 4_000_000); - } - - #estimateOutputCostFromTokens ( - selectedModel: IImageModel, - ratio: { w: number; h: number }, - quality?: string, - ): number | undefined { - if ( ! selectedModel.id.startsWith('gpt-image-2') ) return undefined; - const rate = this.#getCostRate(selectedModel, 'image_output'); - if ( rate === undefined ) return undefined; - const tokens = this.#estimateGptImage2OutputTokens(ratio.w, ratio.h, quality); - return this.#costForTokens(tokens, rate); - } - - #buildPriceKey (model: string, quality: string, size: string) { - if ( this.#isGptImageModel(model) ) { - // GPT image models use format: "quality:size" - default to low if not specified - const qualityLevel = quality || 'low'; - return `${qualityLevel}:${size}`; - } - - // DALL-E models use format: "hd:size" or just "size" - return (quality === 'hd' ? 'hd:' : '') + size; - } - - #buildApiParams (model: string, baseParams: Partial): ImageGenerateParamsNonStreaming { - const apiParams = { - user: baseParams.user, - prompt: baseParams.prompt, - size: baseParams.size, - } as ImageGenerateParamsNonStreaming; - - if ( this.#isGptImageModel(model) ) { - // GPT image models require the model parameter and use quality mapping - apiParams.model = model; - // Default to low quality if not specified, consistent with _buildPriceKey - apiParams.quality = baseParams.quality || 'low'; - } else { - // dall-e models - apiParams.model = model; - if ( baseParams.quality === 'hd' ) { - apiParams.quality = 'hd'; - } - } - - return apiParams; - } -} diff --git a/src/backend/src/services/ai/image/providers/ReplicateImageGenerationProvider/ReplicateImageGenerationProvider.ts b/src/backend/src/services/ai/image/providers/ReplicateImageGenerationProvider/ReplicateImageGenerationProvider.ts deleted file mode 100644 index e0289c771..000000000 --- a/src/backend/src/services/ai/image/providers/ReplicateImageGenerationProvider/ReplicateImageGenerationProvider.ts +++ /dev/null @@ -1,281 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import Replicate from 'replicate'; -import sharp from 'sharp'; -import APIError from '../../../../../api/APIError.js'; -import { ErrorService } from '../../../../../modules/core/ErrorService.js'; -import { Context } from '../../../../../util/context.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import { IGenerateParams, IImageModel, IImageProvider } from '../types.js'; -import { REPLICATE_IMAGE_GENERATION_MODELS, ReplicateImageModel } from './models.js'; - -const DEFAULT_MODEL = 'black-forest-labs/flux-schnell'; -const DEFAULT_RATIO = { w: 1024, h: 1024 }; - -export class ReplicateImageGenerationProvider implements IImageProvider { - #client: Replicate; - #meteringService: MeteringService; - #errors: ErrorService; - - constructor ( - config: { apiKey: string }, - meteringService: MeteringService, - errorService: ErrorService, - ) { - if ( ! config.apiKey ) { - throw new Error('Replicate image generation requires an API key'); - } - this.#client = new Replicate({ auth: config.apiKey }); - this.#meteringService = meteringService; - this.#errors = errorService; - } - - models (): IImageModel[] { - return REPLICATE_IMAGE_GENERATION_MODELS; - } - - getDefaultModel (): string { - return DEFAULT_MODEL; - } - - async generate (params: IGenerateParams): Promise { - const { prompt, test_mode } = params; - const selectedModel = this.#getModel(params.model); - const ratio = this.#normalizeRatio(params.ratio); - - if ( test_mode ) { - return 'https://puter-sample-data.puter.site/image_example.png'; - } - - if ( typeof prompt !== 'string' || prompt.trim().length === 0 ) { - throw new Error('`prompt` must be a non-empty string'); - } - - const actor = Context.get('actor'); - if ( ! actor ) { - this.#errors.report('replicate-image-generation:unknown-actor', { - message: 'failed to resolve actor for Replicate image generation', - trace: true, - }); - throw new Error('actor not found in context'); - } - - const extra = params as unknown as Record; - - const goFast = selectedModel.supportsGoFast - ? (extra.go_fast !== undefined ? !!extra.go_fast : (selectedModel.goFastDefault ?? false)) - : false; - - const inputImages: string[] = []; - if ( selectedModel.imageInputKey ) { - if ( params.input_image ) inputImages.push(params.input_image); - if ( params.input_images?.length ) inputImages.push(...params.input_images); - } - const singleImage = selectedModel.singleImageInputKey ? params.input_image : undefined; - const allInputUrls = singleImage ? [singleImage] : inputImages; - const inputMp = allInputUrls.length > 0 - ? await this.#measureInputMegapixels(allInputUrls) - : 0; - - const outputMp = this.#resolveOutputMegapixels(extra.output_megapixels as string | undefined); - - const totalCostMicroCents = this.#estimateCost(selectedModel, outputMp, goFast, inputMp); - if ( totalCostMicroCents <= 0 ) { - throw new Error(`Error calculating cost for Replicate model ${selectedModel.id}`); - } - const usageAllowed = await this.#meteringService.hasEnoughCredits(actor, totalCostMicroCents); - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); - } - - const input: Record = { - prompt, - aspect_ratio: this.#toAspectRatio(ratio), - disable_safety_checker: !!extra.disable_safety_checker, - }; - if ( selectedModel.supportsGoFast ) { - input.go_fast = goFast; - } - if ( inputImages.length && selectedModel.imageInputKey ) { - input[selectedModel.imageInputKey] = inputImages; - } else if ( singleImage && selectedModel.singleImageInputKey ) { - input[selectedModel.singleImageInputKey] = singleImage; - } - if ( Number.isFinite(extra.seed) ) input.seed = Math.round(extra.seed as number); - if ( Number.isFinite(extra.steps) ) input.num_inference_steps = Math.round(extra.steps as number); - if ( Number.isFinite(extra.guidance) ) input.guidance = extra.guidance; - if ( Number.isFinite(extra.output_quality) ) input.output_quality = Math.round(extra.output_quality as number); - if ( typeof extra.output_megapixels === 'string' && selectedModel.resolutionInputKey ) { - const val = extra.output_megapixels + (selectedModel.resolutionSuffix ?? ''); - input[selectedModel.resolutionInputKey] = val; - } else if ( typeof extra.output_megapixels === 'string' ) { - input.megapixels = extra.output_megapixels; - } - if ( Number.isFinite(extra.prompt_strength) ) input.prompt_strength = extra.prompt_strength; - if ( typeof extra.negative_prompt === 'string' ) input.negative_prompt = extra.negative_prompt; - if ( typeof extra.response_format === 'string' ) input.output_format = extra.response_format; - - const output = await this.#client.run( - selectedModel.replicateId as `${string}/${string}`, - { input }, - ); - - const url = this.#extractUrl(output); - if ( ! url ) { - throw new Error('Failed to extract image URL from Replicate response'); - } - - this.#recordUsage(actor, selectedModel, outputMp, goFast, inputMp); - - return url; - } - - #getModel (model?: string): ReplicateImageModel { - const models = REPLICATE_IMAGE_GENERATION_MODELS; - const found = models.find(m => m.id === model || m.aliases?.includes(model ?? '')); - return found || models.find(m => m.id === DEFAULT_MODEL)!; - } - - #normalizeRatio (ratio?: { w: number; h: number }) { - const w = Number(ratio?.w); - const h = Number(ratio?.h); - if ( Number.isFinite(w) && Number.isFinite(h) && w > 0 && h > 0 ) { - return { w: Math.round(w), h: Math.round(h) }; - } - return { ...DEFAULT_RATIO }; - } - - #toAspectRatio (ratio: { w: number; h: number }): string { - const g = this.#gcd(ratio.w, ratio.h); - return `${ratio.w / g}:${ratio.h / g}`; - } - - #gcd (a: number, b: number): number { - return b === 0 ? a : this.#gcd(b, a % b); - } - - #resolveOutputMegapixels (userValue?: string): number { - if ( typeof userValue === 'string' ) { - const parsed = parseFloat(userValue); - if ( Number.isFinite(parsed) && parsed > 0 ) return parsed; - } - return 1; - } - - async #measureInputMegapixels (imageUrls: string[]): Promise { - let totalMp = 0; - for ( const url of imageUrls ) { - try { - const res = await fetch(url); - const buffer = Buffer.from(await res.arrayBuffer()); - const meta = await sharp(buffer).metadata(); - if ( meta.width && meta.height ) { - totalMp += Math.ceil((meta.width * meta.height) / 1_000_000); - } - } catch { - totalMp += 1; - } - } - return totalMp; - } - - #resolveCosts (model: ReplicateImageModel, goFast: boolean): Record { - return (goFast && model.costs_go_fast) ? model.costs_go_fast : model.costs; - } - - #estimateCost (model: ReplicateImageModel, outputMp: number, goFast: boolean, inputMp: number): number { - const costs = this.#resolveCosts(model, goFast); - - if ( model.billingScheme === 'per-image' ) { - const cents = costs.output; - if ( !cents || cents <= 0 ) { - throw new Error(`Replicate model ${model.id} has no valid per-image cost configured`); - } - return Math.round(cents * 1_000_000); - } - - const runCents = costs.run ?? 0; - const outputMpCents = costs.output_mp; - if ( !outputMpCents || outputMpCents <= 0 ) { - throw new Error(`Replicate model ${model.id} has no valid output_mp cost configured`); - } - const inputMpCents = (costs.input_mp ?? 0) * inputMp; - return Math.round((runCents + outputMpCents * outputMp + inputMpCents) * 1_000_000); - } - - #recordUsage (actor: any, model: ReplicateImageModel, outputMp: number, goFast: boolean, inputMp: number) { - const prefix = `replicate:${model.id}`; - const costs = this.#resolveCosts(model, goFast); - - if ( model.billingScheme === 'per-image' ) { - const cents = costs.output; - if ( !cents || cents <= 0 ) return; - this.#meteringService.incrementUsage(actor, `${prefix}:output`, 1, Math.round(cents * 1_000_000)); - return; - } - const components: { usageType: string; usageAmount: number; costOverride: number }[] = []; - - const runCents = costs.run ?? 0; - if ( runCents > 0 ) { - components.push({ - usageType: `${prefix}:run`, - usageAmount: 1, - costOverride: Math.round(runCents * 1_000_000), - }); - } - - const outputMpCents = costs.output_mp ?? 0; - if ( outputMpCents > 0 ) { - components.push({ - usageType: `${prefix}:output_mp`, - usageAmount: outputMp, - costOverride: Math.round(outputMpCents * outputMp * 1_000_000), - }); - } - - const inputMpCents = costs.input_mp ?? 0; - if ( inputMpCents > 0 && inputMp > 0 ) { - components.push({ - usageType: `${prefix}:input_mp`, - usageAmount: inputMp, - costOverride: Math.round(inputMpCents * inputMp * 1_000_000), - }); - } - - if ( components.length > 0 ) { - this.#meteringService.batchIncrementUsages(actor, components); - } - } - - #extractUrl (output: unknown): string | undefined { - if ( typeof output === 'string' ) return output; - if ( Array.isArray(output) ) { - const first = output[0]; - if ( typeof first === 'string' ) return first; - if ( first && typeof first === 'object' ) { - return String(first); - } - } - if ( output && typeof output === 'object' ) { - return String(output); - } - return undefined; - } -} diff --git a/src/backend/src/services/ai/moderation/AsModeration.js b/src/backend/src/services/ai/moderation/AsModeration.js deleted file mode 100644 index f15a89d48..000000000 --- a/src/backend/src/services/ai/moderation/AsModeration.js +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { default: dedent } = require('dedent'); - -class AsModeration { - - /** @type {import('../chat/providers/ChatProvider').IChatProvider} */ - #chatProvider; - - /** @type {string} */ - #model; - - /** - * @param {object} args - * @param {import('../chat/providers/ChatProvider').IChatProvider} args.chatProvider - * @param {string} args.model - */ - constructor ({ chatProvider, model }) { - this.#chatProvider = chatProvider; - this.#model = model; - } - - async moderate (text) { - // copied from Anthropic documentation - // https://docs.anthropic.com/en/docs/about-claude/use-case-guides/content-moderation - const unsafe_categories = [ - 'Child Exploitation', - 'Conspiracy Theories', - 'Hate', - 'Indiscriminate Weapons', - 'Intellectual Property', - 'Non-Violent Crimes', - 'Privacy', - 'Self-Harm', - 'Sex Crimes', - 'Sexual Content', - 'Specialized Advice', - 'Violent Crimes', - ]; - - const unsafe_category_str = unsafe_categories.join('\n'); - - const assessment_prompt = dedent(` - Determine whether the following message warrants moderation, - based on the unsafe categories outlined below. - - Message: - - ${text.replace('<', '<').replace('>', '>')} - - - Unsafe Categories: - - ${unsafe_category_str} - - - Respond with ONLY a JSON object, using the format below: - {{ - "violation": , - "categories": [Comma-separated list of violated categories], - "explanation": [Optional. Only include if there is a violation.] - }} - `); - - const result = await this.#chatProvider.complete({ - messages: [ - { - role: 'user', - content: assessment_prompt, - }, - ], - model: this.#model, - }); - const str = result.message?.content?.[0]?.text ?? - result.messages?.[0]?.content?.[0]?.text ?? - '{ "violation": true }'; - - const parsed = JSON.parse(str); - return !parsed.violation; - } -} - -module.exports = { - AsModeration, -}; diff --git a/src/backend/src/services/ai/ocr/AWSTextractService.js b/src/backend/src/services/ai/ocr/AWSTextractService.js deleted file mode 100644 index 0e6e6ab1b..000000000 --- a/src/backend/src/services/ai/ocr/AWSTextractService.js +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { TextractClient, AnalyzeDocumentCommand, InvalidS3ObjectException } = require('@aws-sdk/client-textract'); - -const BaseService = require('../../BaseService'); -const APIError = require('../../../api/APIError'); -const { Context } = require('../../../util/context'); - -/** -* AWSTextractService class - Provides OCR (Optical Character Recognition) functionality using AWS Textract -* Extends BaseService to integrate with AWS Textract for document analysis and text extraction. -* Implements driver capabilities and puter-ocr interface for document recognition. -* Handles both S3-stored and buffer-based document processing with automatic region management. -*/ -class AWSTextractService extends BaseService { - /** @type {import('../../MeteringService/MeteringService').MeteringService} */ - get meteringService () { - return this.services.get('meteringService').meteringService; - } - /** - * AWS Textract service for OCR functionality - * Provides document analysis capabilities using AWS Textract API - * Implements interfaces for OCR recognition and driver capabilities - * @extends BaseService - */ - _construct () { - this.clients_ = {}; - } - - static IMPLEMENTS = { - 'driver-capabilities': { - supports_test_mode (iface, method_name) { - return iface === 'puter-ocr' && method_name === 'recognize'; - }, - }, - 'puter-ocr': { - /** - * Performs OCR recognition on a document using AWS Textract - * @param {Object} params - Recognition parameters - * @param {Object} params.source - The document source to analyze - * @param {boolean} params.test_mode - If true, returns sample test output instead of processing - * @returns {Promise} Recognition results containing blocks of text with confidence scores - */ - async recognize ({ source, test_mode }) { - if ( test_mode ) { - return { - blocks: [ - { - type: 'text/textract:WORD', - confidence: 0.9999998807907104, - text: 'Hello', - }, - { - type: 'text/puter:sample-output', - confidence: 1, - text: 'The test_mode flag is set to true. This is a sample output.', - }, - ], - }; - } - - const resp = await this.analyze_document(source); - - // Simplify the response for common interface - const puter_response = { - blocks: [], - }; - - for ( const block of resp.Blocks ) { - if ( block.BlockType === 'PAGE' ) continue; - if ( block.BlockType === 'CELL' ) continue; - if ( block.BlockType === 'TABLE' ) continue; - if ( block.BlockType === 'MERGED_CELL' ) continue; - if ( block.BlockType === 'LAYOUT_FIGURE' ) continue; - if ( block.BlockType === 'LAYOUT_TEXT' ) continue; - - const puter_block = { - type: `text/textract:${block.BlockType}`, - confidence: block.Confidence, - text: block.Text, - }; - puter_response.blocks.push(puter_block); - } - - return puter_response; - }, - }, - }; - - /** - * Creates AWS credentials object for authentication - * @private - * @returns {Object} Object containing AWS access key ID and secret access key - */ - _create_aws_credentials () { - return { - accessKeyId: this.config.aws.access_key, - secretAccessKey: this.config.aws.secret_key, - }; - } - - _get_client (region) { - if ( ! region ) { - region = this.config.aws?.region ?? this.global_config.aws?.region - ?? 'us-west-2'; - } - if ( this.clients_[region] ) return this.clients_[region]; - - this.clients_[region] = new TextractClient({ - credentials: this._create_aws_credentials(), - region, - }); - - return this.clients_[region]; - } - - /** - * Analyzes a document using AWS Textract to extract text and layout information - * @param {FileFacade} file_facade - Interface to access the document file - * @returns {Promise} The raw Textract API response containing extracted text blocks - * @throws {Error} If document analysis fails or no suitable input format is available - * @description Processes document through Textract's AnalyzeDocument API with LAYOUT feature. - * Will attempt to use S3 direct access first, falling back to buffer upload if needed. - */ - async analyze_document (file_facade) { - const { - client, document, using_s3, - } = await this._get_client_and_document(file_facade); - - const actor = Context.get('actor'); - const usageType = 'aws-textract:detect-document-text:page'; - - const usageAllowed = await this.meteringService.hasEnoughCreditsFor(actor, usageType, 1); // allow them to pass if they have enough for 1 page atleast - - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); - } - - const command = new AnalyzeDocumentCommand({ - Document: document, - FeatureTypes: [ - // 'TABLES', - // 'FORMS', - // 'SIGNATURES', - 'LAYOUT', - ], - }); - - let textractResp; - try { - textractResp = await client.send(command); - } catch (e) { - if ( using_s3 && e instanceof InvalidS3ObjectException ) { - const { client, document } = - await this._get_client_and_document(file_facade, true); - const command = new AnalyzeDocumentCommand({ - Document: document, - FeatureTypes: [ - 'LAYOUT', - ], - }); - textractResp = await client.send(command); - } else { - throw e; - } - } - - // Metering integration for Textract OCR usage - // AWS Textract metering: track page count, block count, cost, document size if available - let pageCount = 0; - if ( textractResp.Blocks ) { - for ( const block of textractResp.Blocks ) { - if ( block.BlockType === 'PAGE' ) pageCount += 1; - } - } - this.meteringService.incrementUsage(actor, usageType, pageCount || 1); - - return textractResp; - } - - /** - * Gets AWS client and document configuration for Textract processing - * @param {Object} file_facade - File facade object containing document source info - * @param {boolean} [force_buffer] - If true, forces using buffer instead of S3 - * @returns {Promise} Object containing: - * - client: Configured AWS Textract client - * - document: Document configuration for Textract - * - using_s3: Boolean indicating if using S3 source - * @throws {APIError} If file does not exist - * @throws {Error} If no suitable input format is available - */ - async _get_client_and_document (file_facade, force_buffer) { - const try_s3info = await file_facade.get('s3-info'); - if ( try_s3info && !force_buffer ) { - console.log('S3 INFO', try_s3info); - return { - using_s3: true, - client: this._get_client(try_s3info.bucket_region), - document: { - S3Object: { - Bucket: try_s3info.bucket, - Name: try_s3info.key, - }, - }, - }; - } - - const try_buffer = await file_facade.get('buffer'); - if ( try_buffer ) { - return { - client: this._get_client(), - document: { - Bytes: try_buffer, - }, - }; - } - - const fsNode = await file_facade.get('fs-node'); - if ( fsNode && !await fsNode.exists() ) { - throw APIError.create('subject_does_not_exist'); - } - - throw new Error('No suitable input for Textract'); - } -} - -module.exports = { - AWSTextractService, -}; diff --git a/src/backend/src/services/ai/ocr/MistralOCRService.js b/src/backend/src/services/ai/ocr/MistralOCRService.js deleted file mode 100644 index 5a607e607..000000000 --- a/src/backend/src/services/ai/ocr/MistralOCRService.js +++ /dev/null @@ -1,291 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { Context } from '@heyputer/putility/src/libs/context.js'; -import { Mistral } from '@mistralai/mistralai'; -import mime from 'mime-types'; -import { APIError } from 'openai'; -import path from 'path'; -import BaseService from '../../BaseService.js'; - -/** -* MistralAIService class extends BaseService to provide integration with the Mistral AI API. -* Implements chat completion functionality with support for various Mistral models including -* mistral-large, pixtral, codestral, and ministral variants. Handles both streaming and -* non-streaming responses, token usage tracking, and model management. Provides cost information -* for different models and implements the puter-chat-completion interface. -*/ -export class MistralOCRService extends BaseService { - /** @type {import('../../MeteringService/MeteringService.js').MeteringService} */ - meteringService; - /** - * Initializes the service's cost structure for different Mistral AI models. - * Sets up pricing information for various models including token costs for input/output. - * Each model entry specifies currency (usd-cents) and costs per million tokens. - * @private - */ - - models = [ - { id: 'mistral-ocr-latest', - aliases: ['mistral-ocr-2505', 'mistral-ocr'], - cost: { - currency: 'usd-cents', - pages: 1000, - input: 100, - output: 300, - }, - }, - ]; - - static IMPLEMENTS = { - 'driver-capabilities': { - supports_test_mode (iface, method_name) { - return iface === 'puter-ocr' && method_name === 'recognize'; - }, - }, - 'puter-ocr': { - async recognize (...params) { - return this.recognize(...params); - }, - }, - }; - - /** - * Initializes the service's cost structure for different Mistral AI models. - * Sets up pricing information for various models including token costs for input/output. - * Each model entry specifies currency (USD cents) and costs per million tokens. - * @private - */ - async _init () { - this.api_base_url = 'https://api.mistral.ai/v1'; - this.client = new Mistral({ - apiKey: this.config.apiKey, - }); - - this.meteringService = this.services.get('meteringService').meteringService; - } - - async recognize ({ - source, - model, - pages, - includeImageBase64, - imageLimit, - imageMinSize, - bboxAnnotationFormat, - documentAnnotationFormat, - test_mode, - }) { - if ( test_mode ) { - return this.#sampleOcrResponse(); - } - if ( ! source ) { - throw APIError.create('missing_required_argument', { - interface_name: 'puter-ocr', - method_name: 'recognize', - arg_name: 'source', - }); - } - - const document = await this._buildDocumentChunkFromSource(source); - const payload = { - model: model ?? 'mistral-ocr-latest', - document, - }; - if ( Array.isArray(pages) ) { - payload.pages = pages; - } - if ( typeof includeImageBase64 === 'boolean' ) { - payload.includeImageBase64 = includeImageBase64; - } - if ( typeof imageLimit === 'number' ) { - payload.imageLimit = imageLimit; - } - if ( typeof imageMinSize === 'number' ) { - payload.imageMinSize = imageMinSize; - } - if ( bboxAnnotationFormat !== undefined ) { - payload.bboxAnnotationFormat = bboxAnnotationFormat; - } - if ( documentAnnotationFormat !== undefined ) { - payload.documentAnnotationFormat = documentAnnotationFormat; - } - - const response = await this.client.ocr.process(payload); - const annotationsRequested = ( - payload.documentAnnotationFormat !== undefined || - payload.bboxAnnotationFormat !== undefined - ); - this.#recordOcrUsage(response, payload.model, { - annotationsRequested, - }); - return this.#normalizeOcrResponse(response); - } - - async _buildDocumentChunkFromSource (fileFacade) { - const dataUrl = await this._safeFileValue(fileFacade, 'data_url'); - const webUrl = await this._safeFileValue(fileFacade, 'web_url'); - const filePath = await this._safeFileValue(fileFacade, 'path'); - const fsNode = await this._safeFileValue(fileFacade, 'fs-node'); - const fileName = filePath ? path.basename(filePath) : fsNode?.name; - const inferredMime = this._inferMimeFromName(fileName); - - if ( webUrl ) { - return this._chunkFromUrl(webUrl, fileName, inferredMime); - } - if ( dataUrl ) { - const mimeFromUrl = this._extractMimeFromDataUrl(dataUrl) ?? inferredMime; - return this._chunkFromUrl(dataUrl, fileName, mimeFromUrl); - } - - const buffer = await this._safeFileValue(fileFacade, 'buffer'); - if ( ! buffer ) { - throw APIError.create('field_invalid', null, { - key: 'source', - expected: 'file, data URL, or web URL', - }); - } - const mimeType = inferredMime ?? 'application/octet-stream'; - const generatedDataUrl = this._createDataUrl(buffer, mimeType); - return this._chunkFromUrl(generatedDataUrl, fileName, mimeType); - } - - async _safeFileValue (fileFacade, key) { - if ( !fileFacade || typeof fileFacade.get !== 'function' ) return undefined; - const maybeCache = fileFacade.values?.values; - if ( maybeCache && Object.prototype.hasOwnProperty.call(maybeCache, key) ) { - return maybeCache[key]; - } - try { - return await fileFacade.get(key); - } catch (e) { - return undefined; - } - } - - _chunkFromUrl (url, fileName, mimeType) { - const lowerName = fileName?.toLowerCase(); - const urlLooksPdf = /\.pdf($|\?)/i.test(url); - const mimeLooksPdf = mimeType?.includes('pdf'); - const isPdf = mimeLooksPdf || urlLooksPdf || (lowerName ? lowerName.endsWith('.pdf') : false); - - if ( isPdf ) { - const chunk = { - type: 'document_url', - documentUrl: url, - }; - if ( fileName ) { - chunk.documentName = fileName; - } - return chunk; - } - - return { - type: 'image_url', - imageUrl: { - url, - }, - }; - } - - _inferMimeFromName (name) { - if ( ! name ) return undefined; - return mime.lookup(name) || undefined; - } - - _extractMimeFromDataUrl (url) { - if ( typeof url !== 'string' ) return undefined; - const match = url.match(/^data:([^;,]+)[;,]/); - return match ? match[1] : undefined; - } - - _createDataUrl (buffer, mimeType) { - return `data:${mimeType || 'application/octet-stream'};base64,${buffer.toString('base64')}`; - } - - #normalizeOcrResponse (response) { - if ( ! response ) return {}; - const normalized = { - model: response.model, - pages: response.pages ?? [], - usage_info: response.usageInfo, - }; - const blocks = []; - if ( Array.isArray(response.pages) ) { - for ( const page of response.pages ) { - if ( typeof page?.markdown !== 'string' ) continue; - const lines = page.markdown.split('\n').map(line => line.trim()).filter(Boolean); - for ( const line of lines ) { - blocks.push({ - type: 'text/mistral:LINE', - text: line, - page: page.index, - }); - } - } - } - normalized.blocks = blocks; - if ( blocks.length ) { - normalized.text = blocks.map(block => block.text).join('\n'); - } else if ( Array.isArray(response.pages) ) { - normalized.text = response.pages.map(page => page?.markdown || '').join('\n\n').trim(); - } - return normalized; - } - - #recordOcrUsage (response, model, { annotationsRequested } = {}) { - try { - if ( ! this.meteringService ) return; - const actor = Context.get('actor'); - if ( ! actor ) return; - const pagesProcessed = - response?.usageInfo?.pagesProcessed ?? - (Array.isArray(response?.pages) ? response.pages.length : 1); - this.meteringService.incrementUsage(actor, 'mistral-ocr:ocr:page', pagesProcessed); - if ( annotationsRequested ) { - this.meteringService.incrementUsage(actor, 'mistral-ocr:annotations:page', pagesProcessed); - } - } catch (e) { - // ignore metering failures to avoid blocking OCR results - } - } - - #sampleOcrResponse () { - const markdown = 'Sample OCR output (test mode).'; - return { - model: 'mistral-ocr-latest', - pages: [ - { - index: 0, - markdown, - images: [], - dimensions: null, - }, - ], - blocks: [ - { - type: 'text/mistral:LINE', - text: markdown, - page: 0, - }, - ], - text: markdown, - }; - } -} diff --git a/src/backend/src/services/ai/sts/ElevenLabsVoiceChangerService.js b/src/backend/src/services/ai/sts/ElevenLabsVoiceChangerService.js deleted file mode 100644 index 9756a5db1..000000000 --- a/src/backend/src/services/ai/sts/ElevenLabsVoiceChangerService.js +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { Readable } = require('stream'); -const APIError = require('../../../api/APIError'); -const BaseService = require('../../BaseService'); -const { TypedValue } = require('../../drivers/meta/Runtime'); -const { FileFacade } = require('../../drivers/FileFacade'); -const { Context } = require('../../../util/context'); - -const DEFAULT_MODEL = 'eleven_multilingual_sts_v2'; -const DEFAULT_VOICE_ID = '21m00Tcm4TlvDq8ikWAM'; -const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3'; -const MAX_AUDIO_FILE_SIZE = 25 * 1024 * 1024; -const DEFAULT_OUTPUT_FORMAT = 'mp3_44100_128'; - -/** - * ElevenLabs voice changer (speech-to-speech). - */ -class ElevenLabsVoiceChangerService extends BaseService { - /** @type {import('../../MeteringService/MeteringService').MeteringService} */ - get meteringService () { - return this.services.get('meteringService').meteringService; - } - - static MODULES = { - mime: require('mime-types'), - musicMetadata: require('music-metadata'), - path: require('path'), - }; - - static IMPLEMENTS = { - 'driver-capabilities': { - supports_test_mode (iface, method_name) { - return iface === 'puter-speech2speech' && method_name === 'convert'; - }, - }, - 'puter-speech2speech': { - async convert (params) { - return this.convert(params); - }, - }, - }; - - async _init () { - const svcConfig = this.global_config?.services?.elevenlabs ?? - this.config?.services?.elevenlabs ?? - this.config?.elevenlabs; - - this.apiKey = svcConfig?.apiKey ?? svcConfig?.api_key ?? svcConfig?.key; - this.baseUrl = svcConfig?.baseUrl ?? 'https://api.elevenlabs.io'; - this.defaultVoiceId = svcConfig?.defaultVoiceId ?? svcConfig?.voiceId ?? DEFAULT_VOICE_ID; - this.defaultModelId = svcConfig?.speechToSpeechModelId ?? svcConfig?.stsModelId ?? DEFAULT_MODEL; - - if ( ! this.apiKey ) { - throw new Error('ElevenLabs API key not configured'); - } - } - - async convert (params) { - const { - audio, - voice, - voice_id, - voiceId, - model, - model_id, - voice_settings, - voiceSettings, - seed, - remove_background_noise, - output_format, - file_format, - optimize_streaming_latency, - enable_logging, - test_mode, - } = params ?? {}; - - if ( test_mode ) { - return new TypedValue({ - $: 'string:url:web', - content_type: 'audio', - }, SAMPLE_AUDIO_URL); - } - - if ( ! audio ) { - throw APIError.create('field_required', null, { key: 'audio' }); - } - - if ( ! (audio instanceof FileFacade) ) { - throw APIError.create('field_invalid', null, { - key: 'audio', - expected: 'file reference', - }); - } - - const { - buffer, - filename, - mimeType, - estimatedSeconds, - } = await this._prepareAudioBuffer(audio); - - const modelId = model_id || model || this.defaultModelId || DEFAULT_MODEL; - const selectedVoiceId = voice_id || voiceId || voice || this.defaultVoiceId; - - if ( ! selectedVoiceId ) { - throw APIError.create('field_required', null, { key: 'voice' }); - } - - const actor = Context.get('actor'); - const usageKey = `elevenlabs:${modelId}:second`; - const usageAllowed = await this.meteringService.hasEnoughCreditsFor(actor, usageKey, estimatedSeconds); - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); - } - - const formData = new FormData(); - const blob = new Blob([buffer], { type: mimeType || 'application/octet-stream' }); - formData.append('audio', blob, filename); - formData.append('model_id', modelId); - - const mergedVoiceSettings = voice_settings ?? voiceSettings; - if ( mergedVoiceSettings !== undefined && mergedVoiceSettings !== null ) { - const serializedSettings = typeof mergedVoiceSettings === 'string' - ? mergedVoiceSettings - : JSON.stringify(mergedVoiceSettings); - formData.append('voice_settings', serializedSettings); - } - - if ( seed !== undefined && seed !== null ) { - formData.append('seed', seed); - } - - if ( typeof remove_background_noise === 'boolean' ) { - formData.append('remove_background_noise', String(remove_background_noise)); - } - - if ( file_format ) { - formData.append('file_format', file_format); - } - - const searchParams = new URLSearchParams(); - const desiredOutputFormat = output_format || DEFAULT_OUTPUT_FORMAT; - if ( desiredOutputFormat ) { - searchParams.set('output_format', desiredOutputFormat); - } - if ( optimize_streaming_latency !== undefined && optimize_streaming_latency !== null ) { - searchParams.set('optimize_streaming_latency', optimize_streaming_latency); - } - if ( enable_logging !== undefined && enable_logging !== null ) { - searchParams.set('enable_logging', enable_logging); - } - - const url = new URL(`/v1/speech-to-speech/${selectedVoiceId}`, this.baseUrl); - const search = searchParams.toString(); - if ( search ) { - url.search = search; - } - - const response = await fetch(url, { - method: 'POST', - headers: { - 'xi-api-key': this.apiKey, - }, - body: formData, - }); - - if ( ! response.ok ) { - let detail = null; - try { - detail = await response.json(); - } catch ( e ) { - // ignore - } - this.log.error('ElevenLabs voice changer request failed', { - status: response.status, - detail, - }); - throw APIError.create('internal_server_error', null, { - provider: 'elevenlabs', - status: response.status, - }); - } - - const arrayBuffer = await response.arrayBuffer(); - const responseBuffer = Buffer.from(arrayBuffer); - const stream = Readable.from(responseBuffer); - - this.meteringService.incrementUsage(actor, usageKey, estimatedSeconds); - - return new TypedValue({ - $: 'stream', - content_type: response.headers.get('content-type') || 'audio/mpeg', - }, stream); - } - - async _prepareAudioBuffer (file) { - const buffer = await file.get('buffer'); - if ( !buffer || !buffer.length ) { - throw APIError.create('field_invalid', null, { - key: 'audio', - expected: 'non-empty audio file', - }); - } - - if ( buffer.length > MAX_AUDIO_FILE_SIZE ) { - throw APIError.create('file_too_large', null, { - max_size: MAX_AUDIO_FILE_SIZE, - }); - } - - let filename = 'audio'; - let mimeType; - - const pathValue = await file.get('path'); - if ( pathValue ) { - filename = this.modules.path.basename(pathValue); - } else { - const url = await file.get('web_url'); - if ( url ) { - try { - const parsed = new URL(url); - const candidate = this.modules.path.basename(parsed.pathname); - if ( candidate ) filename = candidate; - } catch (_) { - // Ignore URL parsing errors; we'll fall back to defaults. - } - } - } - - const dataUrl = await file.get('data_url'); - if ( dataUrl ) { - const match = /^data:([^;,]+)[;,]/.exec(dataUrl); - if ( match ) { - mimeType = match[1]; - } - } - - if ( ! mimeType ) { - const guessedMime = this.modules.mime.lookup(filename); - if ( guessedMime ) { - mimeType = guessedMime; - } - } - - if ( ! filename.includes('.') ) { - const extension = mimeType ? this.modules.mime.extension(mimeType) : 'mp3'; - filename = `${filename}.${extension || 'mp3'}`; - } - - let estimatedSeconds = Math.ceil(buffer.length / 16000); - try { - const metadata = await this.modules.musicMetadata.parseBuffer(buffer, { - mimeType, - size: buffer.length, - }); - if ( metadata?.format?.duration ) { - estimatedSeconds = Math.ceil(metadata.format.duration); - } - } catch (e) { - if ( process.env.DEBUG_AUDIO_METADATA === '1' ) { - console.warn('Failed to parse audio metadata for duration estimation:', e.message); - } - } - - estimatedSeconds = Math.max(1, estimatedSeconds); - - return { - buffer, - filename, - mimeType, - estimatedSeconds, - }; - } -} - -module.exports = { - ElevenLabsVoiceChangerService, -}; diff --git a/src/backend/src/services/ai/stt/OpenAISpeechToTextService.js b/src/backend/src/services/ai/stt/OpenAISpeechToTextService.js deleted file mode 100644 index 8c4343d85..000000000 --- a/src/backend/src/services/ai/stt/OpenAISpeechToTextService.js +++ /dev/null @@ -1,406 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const BaseService = require('../../BaseService'); -const APIError = require('../../../api/APIError'); -const { Context } = require('../../../util/context'); -const { FileFacade } = require('../../drivers/FileFacade'); - -const MAX_AUDIO_FILE_SIZE = 25 * 1024 * 1024; // 25 MB per OpenAI limits -const DEFAULT_TRANSCRIBE_MODEL = 'gpt-4o-mini-transcribe'; -const DEFAULT_TRANSLATE_MODEL = 'whisper-1'; -const SAMPLE_TRANSCRIPT = { - text: 'Hello! This is a sample transcription returned while test mode is enabled.', - language: 'en', - duration_seconds: 2, - words: [ - { start: 0.0, end: 0.5, text: 'Hello' }, - { start: 0.5, end: 0.9, text: '!' }, - { start: 1.1, end: 2.0, text: 'This is a sample transcription.' }, - ], -}; - -const TRANSCRIPTION_MODEL_CAPABILITIES = { - 'gpt-4o-mini-transcribe': { - canPrompt: true, - canLogprobs: true, - responseFormats: ['json', 'text'], - }, - 'gpt-4o-transcribe': { - canPrompt: true, - canLogprobs: true, - responseFormats: ['json', 'text'], - }, - 'gpt-4o-transcribe-diarize': { - canPrompt: false, - canLogprobs: false, - responseFormats: ['json', 'text', 'diarized_json'], - requiresChunkingOverThirtySeconds: true, - diarization: true, - }, - 'whisper-1': { - canPrompt: true, - canLogprobs: false, - responseFormats: ['json', 'text', 'srt', 'verbose_json', 'vtt'], - timestampGranularities: true, - }, -}; - -class OpenAISpeechToTextService extends BaseService { - /** @type {import('../../MeteringService/MeteringService').MeteringService} */ - get meteringService () { - return this.services.get('meteringService').meteringService; - } - - static MODULES = { - openai: require('openai'), - musicMetadata: require('music-metadata'), - mime: require('mime-types'), - path: require('path'), - }; - - async _init () { - let apiKey = - this.config?.services?.openai?.apiKey ?? - this.global_config?.services?.openai?.apiKey; - - if ( ! apiKey ) { - apiKey = - this.config?.openai?.secret_key ?? - this.global_config.openai?.secret_key; - - if ( apiKey ) { - console.warn('The `openai.secret_key` configuration format is deprecated. ' + - 'Please use `services.openai.apiKey` instead.'); - } - } - - if ( ! apiKey ) { - throw new Error('OpenAI API key not configured'); - } - - this.openai = new this.modules.openai.OpenAI({ apiKey }); - } - - static IMPLEMENTS = { - 'driver-capabilities': { - supports_test_mode (iface, method_name) { - return iface === 'puter-speech2txt' && - (method_name === 'transcribe' || method_name === 'translate'); - }, - }, - 'puter-speech2txt': { - async list_models () { - return this.listModels(); - }, - async transcribe (params) { - return this._handleTranscription({ ...params, translate: false }); - }, - async translate (params) { - return this._handleTranscription({ ...params, translate: true }); - }, - }, - }; - - listModels () { - return [ - { - id: 'gpt-4o-mini-transcribe', - name: 'GPT-4o mini (Transcribe)', - type: 'transcription', - response_formats: TRANSCRIPTION_MODEL_CAPABILITIES['gpt-4o-mini-transcribe'].responseFormats, - supports_prompt: true, - supports_logprobs: true, - }, - { - id: 'gpt-4o-transcribe', - name: 'GPT-4o (Transcribe)', - type: 'transcription', - response_formats: TRANSCRIPTION_MODEL_CAPABILITIES['gpt-4o-transcribe'].responseFormats, - supports_prompt: true, - supports_logprobs: true, - }, - { - id: 'gpt-4o-transcribe-diarize', - name: 'GPT-4o (Transcribe + Diarization)', - type: 'transcription', - response_formats: TRANSCRIPTION_MODEL_CAPABILITIES['gpt-4o-transcribe-diarize'].responseFormats, - supports_prompt: false, - supports_logprobs: false, - supports_diarization: true, - }, - { - id: 'whisper-1', - name: 'Whisper 1', - type: 'translation', - response_formats: TRANSCRIPTION_MODEL_CAPABILITIES['whisper-1'].responseFormats, - supports_prompt: true, - supports_logprobs: false, - supports_timestamp_granularities: true, - }, - ]; - } - - async _handleTranscription ({ - file, - translate = false, - model, - response_format, - language, - prompt, - temperature, - logprobs, - timestamp_granularities, - chunking_strategy, - known_speaker_names, - known_speaker_references, - extra_body, - stream, - test_mode, - }) { - if ( test_mode ) { - return { - ...SAMPLE_TRANSCRIPT, - model: model || (translate ? DEFAULT_TRANSLATE_MODEL : DEFAULT_TRANSCRIBE_MODEL), - }; - } - - if ( stream ) { - throw APIError.create('not_yet_supported', null, { - message: 'Streaming transcription is not yet supported.', - }); - } - - if ( ! file ) { - throw APIError.create('field_missing', null, { key: 'file' }); - } - - if ( ! (file instanceof FileFacade) ) { - throw APIError.create('field_invalid', null, { - key: 'file', - expected: 'file reference', - }); - } - - const { - buffer, - filename, - mimeType, - estimatedSeconds, - } = await this._prepareAudioBuffer(file); - - const selectedModel = model || (translate ? DEFAULT_TRANSLATE_MODEL : DEFAULT_TRANSCRIBE_MODEL); - const capabilities = TRANSCRIPTION_MODEL_CAPABILITIES[selectedModel]; - - if ( ! capabilities ) { - throw APIError.create('field_invalid', null, { - key: 'model', - expected: Object.keys(TRANSCRIPTION_MODEL_CAPABILITIES).join(', '), - got: selectedModel, - }); - } - - if ( response_format && !capabilities.responseFormats.includes(response_format) ) { - throw APIError.create('field_invalid', null, { - key: 'response_format', - expected: capabilities.responseFormats.join(', '), - got: response_format, - }); - } - - if ( prompt && !capabilities.canPrompt ) { - throw APIError.create('field_invalid', null, { - key: 'prompt', - expected: `Not supported for model ${selectedModel}`, - }); - } - - if ( logprobs && !capabilities.canLogprobs ) { - throw APIError.create('field_invalid', null, { - key: 'logprobs', - expected: `Not supported for model ${selectedModel}`, - }); - } - - if ( timestamp_granularities && !capabilities.timestampGranularities ) { - throw APIError.create('field_invalid', null, { - key: 'timestamp_granularities', - expected: 'Only supported on models that provide timestamp granularity (such as whisper-1).', - }); - } - - let diarizationChunkingStrategy = chunking_strategy; - if ( capabilities.diarization ) { - if ( ! response_format ) { - response_format = 'diarized_json'; - } - if ( !diarizationChunkingStrategy && capabilities.requiresChunkingOverThirtySeconds && estimatedSeconds > 30 ) { - diarizationChunkingStrategy = 'auto'; - } - } - - const actor = Context.get('actor'); - const usageType = `openai:${selectedModel}:second`; - const usageAllowed = await this.meteringService.hasEnoughCreditsFor(actor, usageType, estimatedSeconds); - - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); - } - - const openaiFile = await this.modules.openai.toFile( - buffer, - filename, - mimeType ? { type: mimeType } : undefined, - ); - const payload = { - file: openaiFile, - model: selectedModel, - }; - - if ( response_format ) payload.response_format = response_format; - if ( language ) payload.language = language; - if ( typeof temperature === 'number' ) payload.temperature = temperature; - if ( prompt && capabilities.canPrompt ) payload.prompt = prompt; - if ( logprobs && capabilities.canLogprobs ) payload.logprobs = logprobs; - if ( timestamp_granularities && capabilities.timestampGranularities ) payload.timestamp_granularities = timestamp_granularities; - if ( diarizationChunkingStrategy ) payload.chunking_strategy = diarizationChunkingStrategy; - - if ( capabilities.diarization && (known_speaker_names || known_speaker_references) ) { - payload.extra_body = { - ...(extra_body || {}), - ...(known_speaker_names ? { known_speaker_names } : {}), - ...(known_speaker_references ? { known_speaker_references } : {}), - }; - } else if ( extra_body ) { - payload.extra_body = extra_body; - } - - let transcription; - if ( translate ) { - transcription = await this.openai.audio.translations.create(payload); - } else { - transcription = await this.openai.audio.transcriptions.create(payload); - } - - this.meteringService.incrementUsage(actor, usageType, estimatedSeconds); - - return this._formatResponse(transcription, response_format); - } - - async _prepareAudioBuffer (file) { - const buffer = await file.get('buffer'); - if ( !buffer || !buffer.length ) { - throw APIError.create('field_invalid', null, { - key: 'file', - expected: 'non-empty audio file', - }); - } - - if ( buffer.length > MAX_AUDIO_FILE_SIZE ) { - throw APIError.create('file_too_large', null, { - max_size: MAX_AUDIO_FILE_SIZE, - }); - } - - let filename = 'audio'; - let mimeType; - - const pathValue = await file.get('path'); - if ( pathValue ) { - filename = this.modules.path.basename(pathValue); - } else { - const url = await file.get('web_url'); - if ( url ) { - try { - const parsed = new URL(url); - const candidate = this.modules.path.basename(parsed.pathname); - if ( candidate ) filename = candidate; - } catch (_) { - // Ignore URL parsing errors; we'll fall back to defaults. - } - } - } - - const dataUrl = await file.get('data_url'); - if ( dataUrl ) { - const match = /^data:([^;,]+)[;,]/.exec(dataUrl); - if ( match ) { - mimeType = match[1]; - } - } - - if ( ! mimeType ) { - const guessedMime = this.modules.mime.lookup(filename); - if ( guessedMime ) { - mimeType = guessedMime; - } - } - - if ( ! filename.includes('.') ) { - let extension = mimeType ? this.modules.mime.extension(mimeType) : 'mp3'; - // No one uses mpga but mime resolves audio/mpeg to mpga - if ( extension === 'mpga' ) { - extension = 'mp3'; - } - filename = `${filename}.${extension || 'mp3'}`; - } - - let estimatedSeconds = Math.ceil(buffer.length / 16000); - try { - const metadata = await this.modules.musicMetadata.parseBuffer(buffer, { - mimeType, - size: buffer.length, - }); - if ( metadata?.format?.duration ) { - estimatedSeconds = Math.ceil(metadata.format.duration); - } - } catch (e) { - // When metadata parsing fails we fall back to the byte-size estimate. - if ( process.env.DEBUG_AUDIO_METADATA === '1' ) { - console.warn('Failed to parse audio metadata for duration estimation:', e.message); - } - } - - estimatedSeconds = Math.max(1, estimatedSeconds); - - return { - buffer, - filename, - mimeType, - estimatedSeconds, - }; - } - - _formatResponse (result, response_format) { - if ( response_format === 'text' && typeof result === 'string' ) { - return result; - } - if ( typeof result === 'string' ) { - return result; - } - if ( response_format === 'text' && result && typeof result.text === 'string' ) { - return result.text; - } - return result; - } -} - -module.exports = { - OpenAISpeechToTextService, -}; diff --git a/src/backend/src/services/ai/tts/AWSPollyService.js b/src/backend/src/services/ai/tts/AWSPollyService.js deleted file mode 100644 index aa5bb99ba..000000000 --- a/src/backend/src/services/ai/tts/AWSPollyService.js +++ /dev/null @@ -1,329 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { PollyClient, SynthesizeSpeechCommand, DescribeVoicesCommand } = require('@aws-sdk/client-polly'); -const BaseService = require('../../BaseService'); -const { TypedValue } = require('../../drivers/meta/Runtime'); -const APIError = require('../../../api/APIError'); -const { Context } = require('../../../util/context'); -const { redisClient } = require('../../../clients/redis/redisSingleton'); -const { setRedisCacheValue } = require('../../../clients/redis/cacheUpdate.js'); -const { PollyRedisCacheKeys } = require('./PollyRedisCacheKeys.js'); - -// Polly price calculation per engine -const ENGINE_PRICING = { - 'standard': 400, // $4.00 per 1M characters - 'neural': 1600, // $16.00 per 1M characters - 'long-form': 10000, // $100.00 per 1M characters - 'generative': 3000, // $30.00 per 1M characters -}; - -// Valid engine types -const VALID_ENGINES = ['standard', 'neural', 'long-form', 'generative']; - -/** -* AWSPollyService class provides text-to-speech functionality using Amazon Polly. -* Extends BaseService to integrate with AWS Polly for voice synthesis operations. -* Implements voice listing, speech synthesis, and voice selection based on language. -* Includes caching for voice descriptions and supports both text and SSML inputs. -* Supports multiple TTS engines: Standard, Neural, Long-form, and Generative. -* @extends BaseService -*/ -class AWSPollyService extends BaseService { - - /** @type {import('../../MeteringService/MeteringService').MeteringService} */ - get meteringService () { - return this.services.get('meteringService').meteringService; - } - /** - * Initializes the service by creating an empty clients object. - * This method is called during service construction to set up - * the internal state needed for AWS Polly client management. - * @returns {Promise} - */ - async _construct () { - this.clients_ = {}; - } - - static IMPLEMENTS = { - 'driver-capabilities': { - supports_test_mode (iface, method_name) { - return iface === 'puter-tts' && method_name === 'synthesize'; - }, - }, - 'puter-tts': { - /** - * Implements the driver interface methods for text-to-speech functionality - * Contains methods for listing available voices and synthesizing speech - * @interface - * @property {Object} list_voices - Lists available Polly voices with language info - * @property {Object} synthesize - Converts text to speech using specified voice/language - * @property {Function} supports_test_mode - Indicates test mode support for methods - */ - async list_voices ({ engine } = {}) { - const polly_voices = await this.describe_voices(); - - let voices = polly_voices.Voices; - - if ( engine ) { - if ( VALID_ENGINES.includes(engine) ) { - voices = voices.filter((voice) => voice.SupportedEngines?.includes(engine)); - } else { - throw APIError.create('invalid_engine', null, { engine, valid_engines: VALID_ENGINES }); - } - } - - voices = voices.map((voice) => ({ - id: voice.Id, - name: voice.Name, - language: { - name: voice.LanguageName, - code: voice.LanguageCode, - }, - supported_engines: voice.SupportedEngines || ['standard'], - })); - - return voices; - }, - async list_engines () { - return VALID_ENGINES.map(engine => ({ - id: engine, - name: engine.charAt(0).toUpperCase() + engine.slice(1), - pricing_per_million_chars: ENGINE_PRICING[engine] / 100, // Convert microcents to dollars - })); - }, - async synthesize ({ - text, voice, - ssml, language, - engine = 'standard', - test_mode, - }) { - if ( test_mode ) { - const url = 'https://puter-sample-data.puter.site/tts_example.mp3'; - return new TypedValue({ - $: 'string:url:web', - content_type: 'audio', - }, url); - } - - // Validate engine - if ( ! VALID_ENGINES.includes(engine) ) { - throw APIError.create('invalid_engine', null, { engine, valid_engines: VALID_ENGINES }); - } - - const actor = Context.get('actor'); - - const usageType = `aws-polly:${engine}:character`; - - const usageAllowed = await this.meteringService.hasEnoughCreditsFor(actor, usageType, text.length); - - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); - } - - const polly_speech = await this.synthesize_speech(text, { - format: 'mp3', - voice_id: voice, - text_type: ssml ? 'ssml' : 'text', - language, - engine, - }); - - // AWS Polly TTS metering: track character count, voice, engine, cost, audio duration if available - this.meteringService.incrementUsage(actor, usageType, text.length); - - const speech = new TypedValue({ - $: 'stream', - content_type: 'audio/mpeg', - }, polly_speech.AudioStream); - - return speech; - }, - }, - }; - - /** - * Creates AWS credentials object for authentication - * @private - * @returns {Object} Object containing AWS access key ID and secret access key - */ - _create_aws_credentials () { - return { - accessKeyId: this.config.aws.access_key, - secretAccessKey: this.config.aws.secret_key, - }; - } - - _get_client (region) { - if ( ! region ) { - region = this.config.aws?.region ?? this.global_config.aws?.region - ?? 'us-west-2'; - } - if ( this.clients_[region] ) return this.clients_[region]; - - this.clients_[region] = new PollyClient({ - credentials: this._create_aws_credentials(), - region, - }); - - return this.clients_[region]; - } - - /** - * Describes available AWS Polly voices and caches the results - * @returns {Promise} Response containing array of voice details in Voices property - * @description Fetches voice information from AWS Polly API and caches it for 10 minutes - * Uses KV store for caching to avoid repeated API calls - */ - async describe_voices () { - const cached_voices = await redisClient.get(PollyRedisCacheKeys.voices); - if ( cached_voices ) { - try { - const voices = JSON.parse(cached_voices); - this.log.debug('voices cache hit'); - return voices; - } catch (e) { - // no op cache is in an invalid state - } - } - - this.log.debug('voices cache miss'); - - const client = this._get_client(this.config.aws.region); - - const params = {}; - - const command = new DescribeVoicesCommand(params); - - const response = await client.send(command); - - await setRedisCacheValue(PollyRedisCacheKeys.voices, JSON.stringify(response), { - ttlSeconds: 60 * 10, - eventData: response, - }); - - return response; - } - - /** - * Synthesizes speech from text using AWS Polly - * @param {string} text - The text to synthesize - * @param {Object} options - Synthesis options - * @param {string} options.format - Output audio format (e.g. 'mp3') - * @param {string} [options.voice_id] - AWS Polly voice ID to use - * @param {string} [options.language] - Language code (e.g. 'en-US') - * @param {string} [options.text_type] - Type of input text ('text' or 'ssml') - * @param {string} [options.engine] - TTS engine to use ('standard', 'neural', 'long-form', 'generative') - * @returns {Promise} The synthesized speech response - */ - async synthesize_speech (text, { format, voice_id, language, text_type, engine = 'standard' }) { - const client = this._get_client(this.config.aws.region); - - let voice = voice_id ?? undefined; - - if ( !voice && language ) { - this.log.debug('getting language appropriate voice', { language, engine }); - voice = await this.maybe_get_language_appropriate_voice_(language, engine); - } - - if ( ! voice ) { - // Get a default voice that supports the specified engine - voice = await this.get_default_voice_for_engine_(engine); - } - - this.log.debug('using voice', { voice, engine }); - - const params = { - Engine: engine, - OutputFormat: format, - Text: text, - VoiceId: voice, - LanguageCode: language ?? 'en-US', - TextType: text_type ?? 'text', - }; - - const command = new SynthesizeSpeechCommand(params); - - const response = await client.send(command); - - return response; - } - - /** - * Attempts to find an appropriate voice for the given language code and engine - * @param {string} language - The language code to find a voice for (e.g. 'en-US') - * @param {string} engine - The TTS engine to use - * @returns {Promise} The voice ID if found, null if no matching voice exists - * @private - */ - async maybe_get_language_appropriate_voice_ (language, engine = 'standard') { - const voices = await this.describe_voices(); - - const voice = voices.Voices.find((voice) => { - return voice.LanguageCode === language && - voice.SupportedEngines && - voice.SupportedEngines.includes(engine); - }); - - if ( ! voice ) return null; - - return voice.Id; - } - - /** - * Gets a default voice that supports the specified engine - * @param {string} engine - The TTS engine to use - * @returns {Promise} The default voice ID for the engine - * @private - */ - async get_default_voice_for_engine_ (engine = 'standard') { - const voices = await this.describe_voices(); - - // Common default voices for each engine - const default_voices = { - 'standard': ['Salli', 'Joanna', 'Matthew'], - 'neural': ['Joanna', 'Matthew', 'Salli'], - 'long-form': ['Joanna', 'Matthew'], - 'generative': ['Joanna', 'Matthew', 'Salli'], - }; - - const preferred_voices = default_voices[engine] || ['Salli']; - - for ( const voice_name of preferred_voices ) { - const voice = voices.Voices.find((v) => - v.Id === voice_name && - v.SupportedEngines && - v.SupportedEngines.includes(engine)); - if ( voice ) { - return voice.Id; - } - } - - // Fallback: find any voice that supports the engine - const fallback_voice = voices.Voices.find((voice) => - voice.SupportedEngines && - voice.SupportedEngines.includes(engine)); - - return fallback_voice ? fallback_voice.Id : 'Salli'; - } -} - -module.exports = { - AWSPollyService, -}; diff --git a/src/backend/src/services/ai/tts/ElevenLabsTTSService.js b/src/backend/src/services/ai/tts/ElevenLabsTTSService.js deleted file mode 100644 index c4e4c860c..000000000 --- a/src/backend/src/services/ai/tts/ElevenLabsTTSService.js +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { Readable } = require('stream'); -const APIError = require('../../../api/APIError'); -const BaseService = require('../../BaseService'); -const { TypedValue } = require('../../drivers/meta/Runtime'); -const { Context } = require('../../../util/context'); - -const DEFAULT_MODEL = 'eleven_multilingual_v2'; -const DEFAULT_VOICE_ID = '21m00Tcm4TlvDq8ikWAM'; // Common public "Rachel" sample voice -const DEFAULT_OUTPUT_FORMAT = 'mp3_44100_128'; -const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3'; - -const ELEVENLABS_TTS_MODELS = [ - { id: DEFAULT_MODEL, name: 'Eleven Multilingual v2' }, - { id: 'eleven_flash_v2_5', name: 'Eleven Flash v2.5' }, - { id: 'eleven_turbo_v2_5', name: 'Eleven Turbo v2.5' }, - { id: 'eleven_v3', name: 'Eleven v3 Alpha' }, -]; - -/** - * ElevenLabs text-to-speech provider. - * Implements the `puter-tts` interface so the AI module can synthesize speech - * using ElevenLabs voices. - */ -class ElevenLabsTTSService extends BaseService { - /** @type {import('../../MeteringService/MeteringService').MeteringService} */ - get meteringService () { - return this.services.get('meteringService').meteringService; - } - - static IMPLEMENTS = { - 'driver-capabilities': { - supports_test_mode (iface, method_name) { - return iface === 'puter-tts' && method_name === 'synthesize'; - }, - }, - 'puter-tts': { - async list_voices () { - return this.listVoices(); - }, - async list_engines () { - return this.listEngines(); - }, - async synthesize (params) { - return this.synthesize(params); - }, - }, - }; - - async _init () { - const svcThere = this.global_config?.services?.elevenlabs ?? this.config?.services?.elevenlabs ?? this.config?.elevenlabs; - - this.apiKey = svcThere?.apiKey ?? svcThere?.api_key ?? svcThere?.key; - this.baseUrl = svcThere?.baseUrl ?? 'https://api.elevenlabs.io'; - this.defaultVoiceId = svcThere?.defaultVoiceId ?? svcThere?.voiceId ?? DEFAULT_VOICE_ID; - - if ( ! this.apiKey ) { - throw new Error('ElevenLabs API key not configured'); - } - } - - async request (path, { method = 'GET', body, headers = {} } = {}) { - const response = await fetch(`${this.baseUrl}${path}`, { - method, - headers: { - 'xi-api-key': this.apiKey, - ...(body ? { 'Content-Type': 'application/json' } : {}), - ...headers, - }, - body: body ? JSON.stringify(body) : undefined, - }); - - if ( response.ok ) { - return response; - } - - let detail = null; - try { - detail = await response.json(); - } catch ( e ) { - // ignore - } - this.log.error('ElevenLabs request failed', { path, status: response.status, detail }); - throw APIError.create('internal_server_error', null, { provider: 'elevenlabs', status: response.status }); - } - - async listVoices () { - const res = await this.request('/v1/voices'); - const data = await res.json(); - const voices = Array.isArray(data?.voices) ? data.voices : Array.isArray(data) ? data : []; - - return voices - .map(voice => ({ - id: voice.voice_id || voice.voiceId || voice.id, - name: voice.name, - description: voice.description, - category: voice.category, - provider: 'elevenlabs', - labels: voice.labels, - supported_models: ELEVENLABS_TTS_MODELS.map(model => model.id), - })) - .filter(v => v.id && v.name); - } - - async listEngines () { - return ELEVENLABS_TTS_MODELS.map(model => ({ - id: model.id, - name: model.name, - provider: 'elevenlabs', - pricing_per_million_chars: 0, - })); - } - - async synthesize (params) { - const { - text, - voice, - model, - response_format, - output_format, - voice_settings, - voiceSettings, - test_mode, - } = params; - if ( test_mode ) { - return new TypedValue({ - $: 'string:url:web', - content_type: 'audio', - }, SAMPLE_AUDIO_URL); - } - - if ( typeof text !== 'string' || !text.trim() ) { - throw APIError.create('field_required', null, { key: 'text' }); - } - - const voiceId = voice || this.defaultVoiceId; - const modelId = model || DEFAULT_MODEL; - const desiredFormat = output_format || response_format || DEFAULT_OUTPUT_FORMAT; - - const actor = Context.get('actor'); - const usageKey = `elevenlabs:${modelId}:character`; - const usageAllowed = await this.meteringService.hasEnoughCreditsFor(actor, usageKey, text.length); - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); - } - - const payload = { - text, - model_id: modelId, - output_format: desiredFormat, - }; - - const finalVoiceSettings = voice_settings ?? voiceSettings; - if ( finalVoiceSettings ) { - payload.voice_settings = finalVoiceSettings; - } - - const response = await this.request(`/v1/text-to-speech/${voiceId}`, { - method: 'POST', - body: payload, - }); - - const arrayBuffer = await response.arrayBuffer(); - const buffer = Buffer.from(arrayBuffer); - const stream = Readable.from(buffer); - - this.meteringService.incrementUsage(actor, usageKey, text.length); - - return new TypedValue({ - $: 'stream', - content_type: response.headers.get('content-type') || 'audio/mpeg', - }, stream); - } -} - -module.exports = { - ElevenLabsTTSService, -}; diff --git a/src/backend/src/services/ai/tts/OpenAITTSService.js b/src/backend/src/services/ai/tts/OpenAITTSService.js deleted file mode 100644 index fc94b2c78..000000000 --- a/src/backend/src/services/ai/tts/OpenAITTSService.js +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { Readable } = require('stream'); -const APIError = require('../../../api/APIError'); -const BaseService = require('../../BaseService'); -const { TypedValue } = require('../../drivers/meta/Runtime'); -const { Context } = require('../../../util/context'); - -const DEFAULT_MODEL = 'gpt-4o-mini-tts'; -const DEFAULT_VOICE = 'alloy'; -const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3'; - -const RESPONSE_CONTENT_TYPES = { - mp3: 'audio/mpeg', - opus: 'audio/opus', - aac: 'audio/aac', - flac: 'audio/flac', - wav: 'audio/wav', - pcm: 'audio/pcm', -}; - -const OPENAI_TTS_VOICES = [ - { id: 'alloy', name: 'Alloy' }, - { id: 'ash', name: 'Ash' }, - { id: 'ballad', name: 'Ballad' }, - { id: 'coral', name: 'Coral' }, - { id: 'echo', name: 'Echo' }, - { id: 'fable', name: 'Fable' }, - { id: 'nova', name: 'Nova' }, - { id: 'onyx', name: 'Onyx' }, - { id: 'sage', name: 'Sage' }, - { id: 'shimmer', name: 'Shimmer' }, -]; - -const OPENAI_TTS_MODELS = [ - { - id: DEFAULT_MODEL, - name: 'GPT-4o mini TTS', - pricing_per_million_chars: 15, - }, - { - id: 'tts-1', - name: 'TTS 1', - pricing_per_million_chars: 15, - }, - { - id: 'tts-1-hd', - name: 'TTS 1 HD', - pricing_per_million_chars: 30, - }, -]; - -/** - * Service that connects the puter-tts driver interface with OpenAI Text-to-Speech API. - * Provides voice synthesis, engine discovery, and test-mode behaviour consistent with - * the AWS Polly implementation. - */ -class OpenAITTSService extends BaseService { - /** @type {import('../../MeteringService/MeteringService').MeteringService} */ - get meteringService () { - return this.services.get('meteringService').meteringService; - } - - static MODULES = { - openai: require('openai'), - }; - - async _init () { - let apiKey = - this.config?.services?.openai?.apiKey ?? - this.global_config?.services?.openai?.apiKey; - - if ( ! apiKey ) { - apiKey = - this.config?.openai?.secret_key ?? - this.global_config.openai?.secret_key; - - if ( apiKey ) { - console.warn('The `openai.secret_key` configuration format is deprecated. ' + - 'Please use `services.openai.apiKey` instead.'); - } - } - - if ( ! apiKey ) { - throw new Error('OpenAI API key not configured'); - } - - this.openai = new this.modules.openai.OpenAI({ apiKey }); - } - - static IMPLEMENTS = { - 'driver-capabilities': { - supports_test_mode (iface, method_name) { - return iface === 'puter-tts' && method_name === 'synthesize'; - }, - }, - 'puter-tts': { - async list_voices ({ provider } = {}) { - if ( provider && provider !== 'openai' ) { - return []; - } - - return OPENAI_TTS_VOICES.map((voice) => ({ - id: voice.id, - name: voice.name, - language: { - name: 'English', - code: 'en', - }, - provider: 'openai', - supported_models: OPENAI_TTS_MODELS.map(model => model.id), - })); - }, - async list_engines ({ provider } = {}) { - if ( provider && provider !== 'openai' ) { - return []; - } - - return OPENAI_TTS_MODELS.map(model => ({ - id: model.id, - name: model.name, - pricing_per_million_chars: model.pricing_per_million_chars, - provider: 'openai', - })); - }, - async synthesize (params) { - return this.synthesize(params); - }, - }, - }; - - async synthesize ({ - text, - voice, - model, - response_format, - instructions, - test_mode, - }) { - if ( test_mode ) { - return new TypedValue({ - $: 'string:url:web', - content_type: 'audio', - }, SAMPLE_AUDIO_URL); - } - - if ( typeof text !== 'string' || text.trim() === '' ) { - throw APIError.create('field_required', null, { key: 'text' }); - } - - model = model || DEFAULT_MODEL; - if ( ! OPENAI_TTS_MODELS.find(({ id }) => id === model) ) { - throw APIError.create('field_invalid', null, { - key: 'model', - expected: OPENAI_TTS_MODELS.map(({ id }) => id).join(', '), - got: model, - }); - } - - voice = voice || DEFAULT_VOICE; - if ( ! OPENAI_TTS_VOICES.find(({ id }) => id === voice) ) { - throw APIError.create('field_invalid', null, { - key: 'voice', - expected: OPENAI_TTS_VOICES.map(({ id }) => id).join(', '), - got: voice, - }); - } - - const format = response_format || 'mp3'; - const contentType = RESPONSE_CONTENT_TYPES[format] || RESPONSE_CONTENT_TYPES.mp3; - - const actor = Context.get('actor'); - const usageType = `openai:${model}:character`; - - const usageAllowed = await this.meteringService.hasEnoughCreditsFor(actor, usageType, text.length); - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); - } - - const payload = { - model, - voice, - input: text, - }; - - if ( instructions ) { - payload.instructions = instructions; - } - - if ( response_format ) { - payload.response_format = response_format; - } - - const response = await this.openai.audio.speech.create(payload); - const arrayBuffer = await response.arrayBuffer(); - const buffer = Buffer.from(arrayBuffer); - const stream = Readable.from(buffer); - - this.meteringService.incrementUsage(actor, usageType, text.length); - - return new TypedValue({ - $: 'stream', - content_type: contentType, - }, stream); - } -} - -module.exports = { - OpenAITTSService, -}; diff --git a/src/backend/src/services/ai/tts/PollyRedisCacheKeys.js b/src/backend/src/services/ai/tts/PollyRedisCacheKeys.js deleted file mode 100644 index 377ae51ed..000000000 --- a/src/backend/src/services/ai/tts/PollyRedisCacheKeys.js +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const PollyRedisCacheKeys = { - voices: 'svc:polly:voices', -}; - -export { PollyRedisCacheKeys }; diff --git a/src/backend/src/services/ai/utils/FunctionCalling.js b/src/backend/src/services/ai/utils/FunctionCalling.js deleted file mode 100644 index c72f2054b..000000000 --- a/src/backend/src/services/ai/utils/FunctionCalling.js +++ /dev/null @@ -1,130 +0,0 @@ - -export const normalize_json_schema = (schema) => { - if ( ! schema ) return schema; - - if ( schema.type === 'object' ) { - if ( ! schema.properties ) { - return schema; - } - - const keys = Object.keys(schema.properties); - for ( const key of keys ) { - schema.properties[key] = normalize_json_schema(schema.properties[key]); - } - } - - if ( schema.type === 'array' ) { - if ( ! schema.items ) { - schema.items = {}; - } else { - schema.items = normalize_json_schema(schema.items); - } - } - - return schema; -}; - -/** - * Normalizes the 'tools' object in-place. - * - * This function will accept an array of tools provided by the - * user, and produce a normalized object that can then be - * converted to the apprpriate representation for another - * service. - * - * We will accept conventions from either service that a user - * might expect to work, prioritizing the OpenAI convention - * when conflicting conventions are present. - * - * @param {*} tools - */ -export const normalize_tools_object = (tools) => { - for ( let i = 0 ; i < tools.length ; i++ ) { - const tool = tools[i]; - - if ( tool.type === 'web_search' ) { - // OpenAI Responses specific - continue; - } - let normalized_tool = {}; - - const normalize_function = fn => { - const normal_fn = {}; - let parameters = - fn.parameters || - fn.input_schema; - - if ( !parameters || typeof parameters !== 'object' ) { - parameters = { type: 'object' }; - } else if ( ! parameters.type ) { - parameters.type = 'object'; - } - - normal_fn.parameters = parameters; - - if ( parameters.properties ) { - parameters = normalize_json_schema(parameters); - } - - if ( fn.name ) { - normal_fn.name = fn.name; - } - - if ( fn.description ) { - normal_fn.description = fn.description; - } - - return normal_fn; - }; - - if ( tool.input_schema ) { - normalized_tool = { - type: 'function', - function: normalize_function(tool), - }; - } else if ( tool.type === 'function' ) { - normalized_tool = { - type: 'function', - function: normalize_function(tool.function || tool), - }; - } else { - normalized_tool = { - type: 'function', - function: normalize_function(tool), - }; - } - - tools[i] = normalized_tool; - } - return tools; -}; - -/** - * This function will convert a normalized tools object to the - * format expected by OpenAI. - * - * @param {*} tools - * @returns - */ -export const make_openai_tools = (tools) => { - return tools; -}; - -/** - * This function will convert a normalized tools object to the - * format expected by Claude. - * - * @param {*} tools - * @returns - */ -export const make_claude_tools = (tools) => { - if ( ! tools ) return undefined; - return tools.map(tool => { - const { name, description, parameters } = tool.function; - return { - name, - description, - input_schema: parameters, - }; - }); -}; diff --git a/src/backend/src/services/ai/utils/Messages.js b/src/backend/src/services/ai/utils/Messages.js deleted file mode 100644 index 677a98b68..000000000 --- a/src/backend/src/services/ai/utils/Messages.js +++ /dev/null @@ -1,209 +0,0 @@ - -/** - * Normalizes a single message into a standardized format with role and content array. - * Converts string messages to objects, ensures content is an array of content blocks, - * transforms tool_calls into tool_use content blocks, and coerces content items into objects. - * - * @param {string|Object} message - The message to normalize, either a string or message object - * @param {Object} params - Optional parameters including default role - * @returns {Object} Normalized message with role and content array - * @throws {Error} If message is not a string or object - * @throws {Error} If message has no content property and no tool_calls - * @throws {Error} If any content item is not a string or object - */ -export const normalize_single_message = (message, params = {}) => { - params = Object.assign({ - role: 'user', - }, params); - - if ( typeof message === 'string' ) { - message = { - content: [message], - }; - } - if ( !message || typeof message !== 'object' || Array.isArray(message) ) { - throw new Error('each message must be a string or object'); - } - if ( ! message.role ) { - message.role = params.role; - } - if ( ! message.content ) { - if ( message.tool_calls ) { - message.content = []; - for ( let i = 0 ; i < message.tool_calls.length ; i++ ) { - const tool_call = message.tool_calls[i]; - message.content.push({ - type: 'tool_use', - id: tool_call.id, - name: tool_call.function.name, - input: tool_call.function.arguments, - }); - } - delete message.tool_calls; - } else if ( !message.role === 'tool' ) { - throw new Error('each message must have a \'content\' property'); - } - } - - // Normalize OpenAI-style tool results into internal tool_result blocks - if ( message.role === 'tool' ) { - const tool_use_id = message.tool_call_id || message.tool_use_id || message.id; - const tool_content = message.content; - message.tool_use_id = tool_use_id; - message.content = [ - { - type: 'tool_result', - tool_use_id, - content: typeof tool_content === 'string' - ? tool_content - : JSON.stringify(tool_content ?? {}), - }, - ]; - } - if ( ! Array.isArray(message.content) ) { - message.content = [message.content]; - } - // Coerce each content block into an object - for ( let i = 0 ; i < message.content.length ; i++ ) { - if ( typeof message.content[i] === 'string' ) { - message.content[i] = { - type: 'text', - text: message.content[i], - }; - } - if ( !message || typeof message.content[i] !== 'object' || Array.isArray(message.content[i]) ) { - throw new Error('each message content item must be a string or object'); - } - if ( typeof message.content[i].text === 'string' && !message.content[i].type ) { - message.content[i].type = 'text'; - } - } - - // Remove "text" properties from content blocks with type=tool_result - for ( let i = 0 ; i < message.content.length ; i++ ) { - if ( message.content[i].type !== 'tool_use' ) { - continue; - } - if ( Object.prototype.hasOwnProperty.call(message.content[i], 'text') ) { - delete message.content[i].text; - } - } - - return message; -}; - -/** - * Normalizes an array of messages by applying normalize_single_message to each, - * then splits messages with multiple content blocks into separate messages, - * and finally merges consecutive messages from the same role. - * - * @param {Array} messages - Array of messages to normalize - * @param {Object} params - Optional parameters passed to normalize_single_message - * @returns {Array} Normalized and merged array of messages - */ -export const normalize_messages = (messages, params = {}) => { - for ( let i = 0 ; i < messages.length ; i++ ) { - messages[i] = normalize_single_message(messages[i], params); - } - - // Split messages with multiple content blocks into separate messages. - // Keep assistant tool_use blocks together to preserve OpenAI tool-call ordering. - // TODO: unit test this - messages = [...messages]; - for ( let i = 0 ; i < messages.length ; i++ ) { - let message = messages[i]; - let separated_messages = []; - const has_tool_use = message.role === 'assistant' && - message.content?.some(c => c?.type === 'tool_use'); - if ( has_tool_use ) { - separated_messages.push(message); - messages.splice(i, 1, ...separated_messages); - continue; - } - for ( let j = 0 ; j < message.content.length ; j++ ) { - separated_messages.push({ - ...message, - content: [message.content[j]], - }); - } - messages.splice(i, 1, ...separated_messages); - } - - // If multiple messages are from the same role, merge them - // but avoid merging tool_use/tool_result messages, since order matters - const hasToolContent = (message) => { - if ( !message || !Array.isArray(message.content) ) return false; - return message.content.some((part) => - part && (part.type === 'tool_use' || part.type === 'tool_result')); - }; - let merged_messages = []; - let current_role = null; - for ( let i = 0 ; i < messages.length ; i++ ) { - const can_merge = current_role === messages[i].role && - !hasToolContent(messages[i]) && - !hasToolContent(merged_messages[merged_messages.length - 1]); - if ( can_merge ) { - merged_messages[merged_messages.length - 1].content.push(...messages[i].content); - } else { - merged_messages.push(messages[i]); - current_role = messages[i].role; - } - } - - return merged_messages; -}; - -/** - * Separates system messages from other messages in the array. - * - * @param {Array} messages - Array of messages to process - * @returns {Array} Tuple containing [system_messages, non_system_messages] - */ -export const extract_and_remove_system_messages = (messages) => { - let system_messages = []; - let new_messages = []; - for ( let i = 0 ; i < messages.length ; i++ ) { - if ( messages[i].role === 'system' ) { - system_messages.push(messages[i]); - } else { - new_messages.push(messages[i]); - } - } - return [system_messages, new_messages]; -}; - -/** - * Extracts all text content from messages, handling various message formats. - * Processes strings, objects with content arrays, and nested content structures, - * joining all text with spaces. - * - * @param {Array} messages - Array of messages to extract text from - * @returns {string} Concatenated text content from all messages - * @throws {Error} If text content is not a string - */ -export const extract_text = (messages) => { - return messages.map(m => { - if ( typeof m === 'string' ) { - return m; - } - if ( !m || typeof m !== 'object' || Array.isArray(m) ) { - return ''; - } - if ( Array.isArray(m.content) ) { - return m.content.map(c => c.text).join(' '); - } - if ( typeof m.content === 'string' ) { - return m.content; - } else { - const is_text_type = m.content.type === 'text' || - !Object.prototype.hasOwnProperty.call(m.content, 'type'); - if ( is_text_type ) { - if ( typeof m.content.text !== 'string' ) { - throw new Error('text content must be a string'); - } - return m.content.text; - } - return ''; - } - }).join(' '); -}; diff --git a/src/backend/src/services/ai/utils/OpenAIUtil.d.ts b/src/backend/src/services/ai/utils/OpenAIUtil.d.ts deleted file mode 100644 index 117bb8b7c..000000000 --- a/src/backend/src/services/ai/utils/OpenAIUtil.d.ts +++ /dev/null @@ -1,129 +0,0 @@ -import type { - ChatCompletion, - ChatCompletionChunk, - ChatCompletionContentPart, - ChatCompletionMessageParam, - ChatCompletionMessageToolCall, -} from 'openai/resources/chat/completions'; -import type { CompletionUsage } from 'openai/resources/completions'; -import { IChatModel, IChatProvider } from '../chat/providers/types'; - -export interface ToolUseContent { - type: 'tool_use'; - id: string; - name: string; - input: unknown; - extra_content?: unknown; -} - -export interface ToolResultContent { - type: 'tool_result'; - tool_use_id: string; - content: unknown; -} - -export type NormalizedContent = - | ChatCompletionContentPart - | ToolUseContent - | ToolResultContent - | ({ type?: 'image_url'; image_url: unknown; [key: string]: unknown }); - -export interface NormalizedMessage extends Partial { - role?: ChatCompletionMessageParam['role'] | string; - content?: NormalizedContent[] | null; - tool_calls?: ChatCompletionMessageToolCall[]; - tool_call_id?: string; - [key: string]: unknown; -} - -export type UsageCalculator = (args: { usage: CompletionUsage }) => Record; - -export interface ChatStream { - message(): { - contentBlock: (params: { type: 'text' } | { type: 'tool_use'; id: string; name: string; extra_content?: unknown }) => { - addText?(text: string): void; - addReasoning?(reasoning: string): void; - addExtraContent?(extra_content: unknown): void; - addPartialJSON?(partial_json: string): void; - end(): void; - }; - end(): void; - }; - end(): void; -} - -export type StreamingToolCall = ChatCompletionChunk.Choice.Delta.ToolCall & { extra_content?: unknown }; - -export type CompletionChunk = Omit & { - choices: Array< - Omit & { - delta: ChatCompletionChunk['choices'][number]['delta'] & { - reasoning_content?: string | null; - reasoning?: string | null; - extra_content?: unknown; - tool_calls?: StreamingToolCall[]; - }; - } - >; - usage?: CompletionUsage | null; -}; - -export interface StreamDeviations { - index_usage_from_stream_chunk?: (chunk: CompletionChunk) => Partial | null | undefined; - chunk_but_like_actually?: (chunk: CompletionChunk) => Partial; - index_tool_calls_from_stream_choice?: (choice: CompletionChunk['choices'][number]) => StreamingToolCall[] | undefined; -} - -export interface CompletionDeviations { - coerce_completion_usage?: (completion: TCompletion) => Partial; - chunk_but_like_actually?: (chunk: CompletionChunk) => Partial; - index_tool_calls_from_stream_choice?: (choice: CompletionChunk['choices'][number]) => StreamingToolCall[] | undefined; - index_usage_from_stream_chunk?: (chunk: CompletionChunk) => Partial | null | undefined; - -} - -export function process_input_messages (messages: TMessage[]): Promise; -export function process_input_messages_responses_api (messages: TMessage[]): Promise; - -export function create_usage_calculator (params: { model_details: IChatModel }): UsageCalculator; - -export function extractMeteredUsage (usage: { - prompt_tokens?: number | null; - completion_tokens?: number | null; - prompt_tokens_details?: { cached_tokens?: number | null } | null; -}): { - prompt_tokens: number; - completion_tokens: number; - cached_tokens: number; -}; - -export function create_chat_stream_handler (params: { - deviations?: StreamDeviations; - completion: AsyncIterable; - usage_calculator?: UsageCalculator; -}): (args: { chatStream: ChatStream }) => Promise; - -type CompletionChoice = TCompletion extends { choices: Array } - ? Choice - : ChatCompletion['choices'][number]; - -export function handle_completion_output (params: { - deviations?: CompletionDeviations; - stream?: boolean; - completion: AsyncIterable | TCompletion; - moderate?: (text: string) => Promise<{ flagged: boolean }>; - usage_calculator?: UsageCalculator; - finally_fn?: () => Promise; -}): ReturnType; - -export function handle_completion_output_responses_api (params: { - deviations?: CompletionDeviations; - stream?: boolean; - completion: AsyncIterable | TCompletion; - moderate?: (text: string) => Promise<{ flagged: boolean }>; - usage_calculator?: UsageCalculator; - finally_fn?: () => Promise; -}): ReturnType; - - - diff --git a/src/backend/src/services/ai/utils/OpenAIUtil.js b/src/backend/src/services/ai/utils/OpenAIUtil.js deleted file mode 100644 index bab8f537f..000000000 --- a/src/backend/src/services/ai/utils/OpenAIUtil.js +++ /dev/null @@ -1,486 +0,0 @@ -/** - * Process input messages from Puter's normalized format to OpenAI's format - * May make changes in-place. - * - * @param {Array} messages - array of normalized messages - * @returns {Array} - array of messages in OpenAI format - */ -export const process_input_messages = async (messages) => { - for ( const msg of messages ) { - if ( ! msg.content ) continue; - if ( typeof msg.content !== 'object' ) continue; - - const content = msg.content; - - for ( const o of content ) { - if ( o['image_url'] && !o.type ) { - o.type = 'image_url'; - } - if ( o['video_url'] && !o.type ) { - o.type = 'video_url'; - } - } - - // coerce tool calls - let is_tool_call = false; - for ( let i = content.length - 1 ; i >= 0 ; i-- ) { - const content_block = content[i]; - - if ( content_block.type === 'tool_use' ) { - if ( ! msg.tool_calls ) { - msg.tool_calls = []; - is_tool_call = true; - } - msg.tool_calls.push({ - id: content_block.id, - type: 'function', - function: { - name: content_block.name, - arguments: JSON.stringify(content_block.input), - }, - ...(content_block.extra_content ? { extra_content: content_block.extra_content } : {}), - }); - content.splice(i, 1); - } - } - - if ( is_tool_call ) msg.content = null; - - // coerce tool results - // (we assume multiple tool results were already split into separate messages) - for ( let i = content.length - 1 ; i >= 0 ; i-- ) { - const content_block = content[i]; - if ( content_block.type !== 'tool_result' ) continue; - msg.role = 'tool'; - msg.tool_call_id = content_block.tool_use_id; - msg.content = content_block.content; - } - } - - return messages; -}; - -export const process_input_messages_responses_api = async (messages) => { - for ( const msg of messages ) { - const content_as_string = (content) => { - if ( content === undefined || content === null ) return ''; - if ( typeof content === 'string' ) return content; - if ( Array.isArray(content) ) { - return content.map((part) => { - if ( typeof part === 'string' ) return part; - if ( part && typeof part.text === 'string' ) return part.text; - if ( part && typeof part.content === 'string' ) return part.content; - return ''; - }).join(''); - } - if ( content && typeof content.text === 'string' ) return content.text; - if ( content && typeof content.content === 'string' ) return content.content; - return ''; - }; - - if ( msg.role === 'tool' ) { - msg.type = 'function_call_output'; - msg.call_id = msg.tool_call_id || msg.tool_use_id; - msg.output = content_as_string(msg.content); - delete msg.role; - delete msg.content; - delete msg.tool_call_id; - delete msg.tool_use_id; - delete msg.tool_calls; - continue; - } - - if ( ! msg.content ) continue; - if ( typeof msg.content !== 'object' ) continue; - - const content = msg.content; - - for ( const o of content ) { - if ( o['image_url'] && !o.type ) { - o.type = 'image_url'; - } - if ( o['video_url'] && !o.type ) { - o.type = 'video_url'; - } - } - - // coerce tool calls - let is_tool_call = false; - for ( let i = content.length - 1; i >= 0; i-- ) { - const content_block = content[i]; - if ( content_block.type === 'text' && (msg.role === 'user' || msg.role === 'system') ) { - content_block.type = 'input_text'; - } - if ( content_block.type === 'text' && (msg.role === 'assistant') ) { - content_block.type = 'output_text'; - } - - if ( content_block.type === 'tool_use' ) { - if ( ! msg.tool_calls ) { - msg.tool_calls = []; - is_tool_call = true; - } - msg.tool_calls.push({ - id: content_block.id, - canonical_id: content_block.canonical_id, - type: 'function', - function: { - name: content_block.name, - arguments: JSON.stringify(content_block.input), - }, - ...(content_block.extra_content ? { extra_content: content_block.extra_content } : {}), - }); - - content.splice(i, 1); - } - } - - // Right now this does NOT support parallel tool calls! - // We only allow sequential toolcalling right now so this shouldn't be an issue right now - // but this probably needs to be changed in the future to split "one completions message" - // into multiple responses inputs. - if ( is_tool_call ) { - msg.call_id = msg.tool_calls[0].id; - msg.id = msg.tool_calls[0].canonical_id; - msg.name = msg.tool_calls[0].function.name; - msg.arguments = msg.tool_calls[0].function.arguments; - msg.type = 'function_call'; - - delete msg.role; - delete msg.content; - delete msg.tool_calls; - } - - // coerce tool results - for ( let i = content.length - 1; i >= 0; i-- ) { - const content_block = content[i]; - if ( content_block.type !== 'tool_result' ) continue; - msg.type = 'function_call_output'; - msg.call_id = content_block.tool_use_id; - msg.output = content_block.content; - - delete msg.role; - delete msg.content; - } - } - - return messages; -}; - -export const create_usage_calculator = ({ model_details }) => { - return ({ usage }) => { - const tokens = []; - - tokens.push({ - type: 'prompt', - model: model_details.id, - amount: usage.prompt_tokens, - cost: model_details.cost.input * usage.prompt_tokens, - }); - - tokens.push({ - type: 'completion', - model: model_details.id, - amount: usage.completion_tokens, - cost: model_details.cost.output * usage.completion_tokens, - }); - - return tokens; - }; -}; - -export const extractMeteredUsage = (usage) => { - return { - prompt_tokens: usage.prompt_tokens ?? 0, - completion_tokens: usage.completion_tokens ?? 0, - cached_tokens: usage.prompt_tokens_details?.cached_tokens ?? 0, - }; -}; - -export const create_chat_stream_handler = ({ - deviations, - completion, - usage_calculator, -}) => async ({ chatStream }) => { - deviations = Object.assign({ - // affected by: Groq - index_usage_from_stream_chunk: chunk => chunk.usage, - // affected by: Mistral - chunk_but_like_actually: chunk => chunk, - index_tool_calls_from_stream_choice: choice => choice.delta.tool_calls, - }, deviations); - - const message = chatStream.message(); - let textblock = message.contentBlock({ type: 'text' }); - let toolblock = null; - let mode = 'text'; - const tool_call_blocks = []; - - let last_usage = null; - for await ( let chunk of completion ) { - chunk = deviations.chunk_but_like_actually(chunk); - const chunk_usage = deviations.index_usage_from_stream_chunk(chunk); - if ( chunk_usage ) last_usage = chunk_usage; - if ( chunk.choices.length < 1 ) continue; - - const choice = chunk.choices[0]; - - // Deepseek returns choice.delta.reasoning_content, openrouter returns choice.delta.reasoning. - if ( choice.delta.reasoning_content || choice.delta.reasoning ) { - textblock.addReasoning(choice.delta.reasoning_content || choice.delta.reasoning); - // Q: Why don't "continue" to next chunk here? - // A: For now, reasoning_content and content never appear together, but I’m not sure if they’ll always be mutually exclusive. - } - - if ( choice.delta.content ) { - if ( mode === 'tool' ) { - toolblock.end(); - mode = 'text'; - textblock = message.contentBlock({ type: 'text' }); - } - textblock.addText(choice.delta.content); - continue; - } - - if ( choice.delta.extra_content ) { - // Gemini specific thing for metadata, we will basically be appending onto the current message by abusing .addText a little - // Apps have to choose to handle extra_content themselves, it doesn't seem like theres a way we can do it in a backwards - // compatible fashion since most streaming apps will handle chat history by continuously updating content themselves - // This doesn't present us a chance to add in an extra object for gemini's chat continuing features - textblock.addExtraContent(choice.delta.extra_content); - } - - const tool_calls = deviations.index_tool_calls_from_stream_choice(choice); - if ( tool_calls ) { - if ( mode === 'text' ) { - mode = 'tool'; - textblock.end(); - } - for ( const tool_call of tool_calls ) { - if ( ! tool_call_blocks[tool_call.index] ) { - toolblock = message.contentBlock({ - type: 'tool_use', - id: tool_call.id, - name: tool_call.function.name, - ...(tool_call.extra_content ? { extra_content: tool_call.extra_content } : {}), - }); - tool_call_blocks[tool_call.index] = toolblock; - } else { - toolblock = tool_call_blocks[tool_call.index]; - } - toolblock.addPartialJSON(tool_call.function.arguments); - } - } - } - - // TODO DS: this is a bit too abstracted... this is basically just doing the metering now - const usage = usage_calculator({ usage: last_usage }); - - if ( mode === 'text' ) textblock.end(); - if ( mode === 'tool' ) toolblock.end(); - - message.end(); - chatStream.end(usage); -}; - -export const create_chat_stream_handler_responses_api = ({ - deviations, - completion, - usage_calculator, -}) => async ({ chatStream }) => { - deviations = Object.assign({ - // affected by: Groq - index_usage_from_stream_chunk: chunk => chunk.usage, - // affected by: Mistral - chunk_but_like_actually: chunk => chunk, - index_tool_calls_from_stream_choice: choice => choice.delta.tool_calls, - }, deviations); - - const message = chatStream.message(); - let textblock = message.contentBlock({ type: 'text' }); - let toolblock = null; - let mode = 'text'; - - let last_usage = null; - for await ( let chunk of completion ) { - - if ( chunk.type === 'response.output_text.delta' ) { - textblock.addText(chunk.delta); - continue; - } - - if ( chunk.type === 'response.completed' ) { - last_usage = chunk.response.usage; - } - - if ( chunk.type === 'response.output_item.done' && chunk.item?.type === 'function_call' ) { - const tool_call = chunk.item; - toolblock = message.contentBlock({ - type: 'tool_use', - canonical_id: tool_call.id, - id: tool_call.call_id, - name: tool_call.name, - ...(tool_call.extra_content ? { extra_content: tool_call.extra_content } : {}), - }); - toolblock.addPartialJSON(tool_call.arguments); - toolblock.end(); - } - } - - // TODO DS: this is a bit too abstracted... this is basically just doing the metering now - const usage = usage_calculator({ usage: last_usage }); - - if ( mode === 'text' ) textblock.end(); - if ( mode === 'tool' ) toolblock.end(); - - message.end(); - chatStream.end(usage); -}; - -/** - * - * @param {object} params - * @param {(args: {usage: import("openai/resources/completions.mjs").CompletionUsage})=> unknown } params.usage_calculator - * @returns - */ -export const handle_completion_output = async ({ - deviations, - stream, - completion, - moderate, - usage_calculator, - finally_fn, -}) => { - deviations = Object.assign({ - // affected by: Mistral - coerce_completion_usage: completion => completion.usage, - }, deviations); - - if ( stream ) { - const init_chat_stream = - create_chat_stream_handler({ - deviations, - completion, - usage_calculator, - }); - - return { - stream: true, - init_chat_stream, - finally_fn, - }; - } - - if ( finally_fn ) await finally_fn(); - - // We need to moderate the completion too - const mod_text = completion.choices[0].message.content; - if ( moderate && mod_text !== null ) { - const moderation_result = await moderate(mod_text); - if ( moderation_result.flagged ) { - throw new Error('message is not allowed'); - } - } - - const ret = completion.choices[0]; - const completion_usage = deviations.coerce_completion_usage(completion); - ret.usage = usage_calculator ? usage_calculator({ - ...completion, - usage: completion_usage, - }) : { - input_tokens: completion_usage.prompt_tokens, - output_tokens: completion_usage.completion_tokens, - }; - return ret; -}; - -/** - * - * @param {object} params - * @param {(args: {usage: import("openai/resources/completions.mjs").CompletionUsage})=> unknown } params.usage_calculator - * @returns - */ -export const handle_completion_output_responses_api = async ({ - deviations, - stream, - completion, - moderate, - usage_calculator, - finally_fn, -}) => { - deviations = Object.assign({ - // affected by: Mistral - coerce_completion_usage: completion => completion.usage, - }, deviations); - - if ( stream ) { - const init_chat_stream = - create_chat_stream_handler_responses_api({ - deviations, - completion, - usage_calculator, - }); - - return { - stream: true, - init_chat_stream, - finally_fn, - }; - } - - if ( finally_fn ) await finally_fn(); - - const output = Array.isArray(completion.output) ? completion.output : []; - const responseToolCalls = output - .filter(item => item?.type === 'function_call') - .map(item => ({ - id: item.call_id, - type: 'function', - function: { - name: item.name, - arguments: item.arguments, - }, - ...(item.id ? { canonical_id: item.id } : {}), - })); - - const is_empty = completion.output_text.trim() === ''; - if ( is_empty && responseToolCalls.length < 1 ) { - // GPT refuses to generate an empty response if you ask it to, - // so this will probably only happen on an error condition. - throw new Error('an empty response was generated'); - } - - // We need to moderate the completion too - const mod_text = completion.output_text; - if ( moderate && mod_text !== null ) { - const moderation_result = await moderate(mod_text); - if ( moderation_result.flagged ) { - throw new Error('message is not allowed'); - } - } - - const ret = { - finish_reason: 'stop', - index: 0, - message: { - content: completion.output_text, - reasoning: null, // Fix later to add proper reasoning - refusal: null, - role: 'assistant', - ...(responseToolCalls.length ? { tool_calls: responseToolCalls } : {}), - }, - }; - ret.role = output.find(item => item?.role)?.role ?? 'assistant'; - - delete ret.type; - - ret.usage = usage_calculator ? usage_calculator({ - ...completion, - usage: completion.usage, - }) : { - input_tokens: completion.usage.input_tokens, - output_tokens: completion.usage.output_tokens, - }; - return ret; - -}; diff --git a/src/backend/src/services/ai/utils/messages.test.js b/src/backend/src/services/ai/utils/messages.test.js deleted file mode 100644 index fb16fcb94..000000000 --- a/src/backend/src/services/ai/utils/messages.test.js +++ /dev/null @@ -1,184 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import * as Messages from './Messages.js'; -import * as OpenAIUtil from './OpenAIUtil.js'; - -describe('Messages', () => { - describe('normalize_single_message', () => { - const cases = [ - { - name: 'string message', - input: 'Hello, world!', - output: { - role: 'user', - content: [ - { - type: 'text', - text: 'Hello, world!', - }, - ], - }, - }, - ]; - for ( const tc of cases ) { - it(`should normalize ${tc.name}`, () => { - const output = Messages.normalize_single_message(tc.input); - expect(output).toEqual(tc.output); - }); - } - }); - describe('extract_text', () => { - const cases = [ - { - name: 'string message', - input: ['Hello, world!'], - output: 'Hello, world!', - }, - { - name: 'object message', - input: [{ - content: [ - { - type: 'text', - text: 'Hello, world!', - }, - ], - }], - output: 'Hello, world!', - }, - { - name: 'irregular messages', - input: [ - 'First Part', - { - content: [ - { - type: 'text', - text: 'Second Part', - }, - ], - }, - { - content: 'Third Part', - }, - ], - output: 'First Part Second Part Third Part', - }, - ]; - for ( const tc of cases ) { - it(`should extract text from ${tc.name}`, () => { - const output = Messages.extract_text(tc.input); - expect(output).toBe(tc.output); - }); - } - }); - describe('normalize OpenAI tool calls', () => { - const cases = [ - { - name: 'string message', - input: { - role: 'assistant', - tool_calls: [ - { - id: 'tool-1', - type: 'function', - function: { - name: 'tool-1-function', - arguments: {}, - }, - }, - ], - }, - output: { - role: 'assistant', - content: [ - { - type: 'tool_use', - id: 'tool-1', - name: 'tool-1-function', - input: {}, - }, - ], - }, - }, - ]; - for ( const tc of cases ) { - it(`should normalize ${tc.name}`, () => { - const output = Messages.normalize_single_message(tc.input); - expect(output).toEqual(tc.output); - }); - } - }); - describe('normalize Claude tool calls', () => { - const cases = [ - { - name: 'string message', - input: { - role: 'assistant', - content: [ - { - type: 'tool_use', - id: 'tool-1', - name: 'tool-1-function', - input: '{}', - }, - ], - }, - output: { - role: 'assistant', - content: [ - { - type: 'tool_use', - id: 'tool-1', - name: 'tool-1-function', - input: '{}', - }, - ], - }, - }, - ]; - for ( const tc of cases ) { - it(`should normalize ${tc.name}`, () => { - const output = Messages.normalize_single_message(tc.input); - expect(output).toEqual(tc.output); - }); - } - }); - describe('OpenAI-ify normalized tool calls', () => { - const cases = [ - { - name: 'string message', - input: [{ - role: 'assistant', - content: [ - { - type: 'tool_use', - id: 'tool-1', - name: 'tool-1-function', - input: {}, - }, - ], - }], - output: [{ - role: 'assistant', - content: null, - tool_calls: [ - { - id: 'tool-1', - type: 'function', - function: { - name: 'tool-1-function', - arguments: '{}', - }, - }, - ], - }], - }, - ]; - for ( const tc of cases ) { - it(`should normalize ${tc.name}`, async () => { - const output = await OpenAIUtil.process_input_messages(tc.input); - expect(output).toEqual(tc.output); - }); - } - }); -}); \ No newline at end of file diff --git a/src/backend/src/services/ai/video/.gitignore b/src/backend/src/services/ai/video/.gitignore deleted file mode 100644 index aa4a6da26..000000000 --- a/src/backend/src/services/ai/video/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.js -*.js.map \ No newline at end of file diff --git a/src/backend/src/services/ai/video/AIVideoGenerationService.ts b/src/backend/src/services/ai/video/AIVideoGenerationService.ts deleted file mode 100644 index 6d2ae5b91..000000000 --- a/src/backend/src/services/ai/video/AIVideoGenerationService.ts +++ /dev/null @@ -1,316 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { APIError } from '../../../api/APIError.js'; -import { Context } from '../../../util/context.js'; -import BaseService from '../../BaseService.js'; -import { DriverService } from '../../drivers/DriverService.js'; -import { EventService } from '../../EventService.js'; -import { MeteringService } from '../../MeteringService/MeteringService.js'; -import { GeminiVideoGenerationProvider } from './providers/GeminiVideoGenerationProvider/GeminiVideoGenerationProvider.js'; -import { OpenAIVideoGenerationProvider } from './providers/OpenAIVideoGenerationProvider/OpenAIVideoGenerationProvider.js'; -import { TogetherVideoGenerationProvider } from './providers/TogetherVideoGenerationProvider/TogetherVideoGenerationProvider.js'; -import { IGenerateVideoParams, IVideoModel, IVideoProvider } from './providers/types.js'; - -export class AIVideoGenerationService extends BaseService { - - static SERVICE_NAME = 'ai-video'; - - static DEFAULT_PROVIDER = 'openai-video-generation'; - - get meteringService (): MeteringService { - return this.services.get('meteringService').meteringService; - } - - get eventService (): EventService { - return this.services.get('event'); - } - - get driverService (): DriverService { - return this.services.get('driver'); - } - - getProvider (name: string): IVideoProvider | undefined { - return this.#providers[name]; - } - - #providers: Record = {}; - #modelIdMap: Record = {}; - - static IMPLEMENTS = { - 'driver-capabilities': { - supports_test_mode (iface: string, method_name: string) { - return iface === 'puter-video-generation' && - method_name === 'generate'; - }, - }, - 'puter-video-generation': { - async generate (...parameters: Parameters) { - return (this as unknown as AIVideoGenerationService).generate(...parameters); - }, - }, - }; - - getModel ({ modelId, provider }: { modelId: string, provider?: string }) { - const models = this.#modelIdMap[modelId]; - if ( ! models ) { - return undefined; - } - - if ( provider ) { - const model = models.find(m => m.provider === provider); - return model ?? models[0]; - } - - // Prefer exact primary ID match over alias matches - const exactIdMatch = models.find(m => m.id === modelId); - if ( exactIdMatch ) { - return exactIdMatch; - } - - const exactPuterIdMatch = models.find(m => m.puterId === modelId); - if ( exactPuterIdMatch ) { - return exactPuterIdMatch; - } - - return models[0]; - } - - private async registerProviders () { - const openAiConfig = this.config.providers?.['openai-video-generation'] || this.global_config?.services?.['openai'] || this.global_config?.openai; - if ( openAiConfig && (openAiConfig.apiKey || openAiConfig.secret_key) ) { - this.#providers['openai-video-generation'] = new OpenAIVideoGenerationProvider( - { apiKey: openAiConfig.apiKey || openAiConfig.secret_key }, - this.meteringService, - ); - } - - const togetherConfig = this.config.providers?.['together-video-generation'] || this.global_config?.services?.['together-ai']; - if ( togetherConfig && (togetherConfig.apiKey || togetherConfig.secret_key) ) { - this.#providers['together-video-generation'] = new TogetherVideoGenerationProvider( - { apiKey: togetherConfig.apiKey || togetherConfig.secret_key }, - this.meteringService, - ); - } - - const geminiVideoConfig = this.config.providers?.['gemini-video-generation'] || this.global_config?.services?.gemini; - if ( geminiVideoConfig && (geminiVideoConfig.apiKey || geminiVideoConfig.secret_key) ) { - this.#providers['gemini-video-generation'] = new GeminiVideoGenerationProvider( - { - apiKey: geminiVideoConfig.apiKey || geminiVideoConfig.secret_key, - origin: this.global_config?.api_base_url, - urlSignatureSecret: this.global_config?.url_signature_secret, - }, - this.meteringService, - ); - } - - // emit event for extensions to add providers - const extensionProviders = {} as Record; - await this.eventService.emit('ai.video.registerProviders', extensionProviders); - for ( const providerName in extensionProviders ) { - if ( this.#providers[providerName] ) { - console.warn('AIVideoGenerationService: provider name conflict for ', providerName, ' registering with -extension suffix'); - this.#providers[`${providerName}-extension`] = extensionProviders[providerName]; - continue; - } - this.#providers[providerName] = extensionProviders[providerName]; - } - } - - protected async '__on_boot.consolidation' () { - await this.registerProviders(); - - for ( const providerName in this.#providers ) { - const provider = this.#providers[providerName]; - - // alias all driver requests to go here to support legacy routing - this.driverService.register_service_alias( - AIVideoGenerationService.SERVICE_NAME, - providerName, - { iface: 'puter-video-generation' }, - ); - - // build model id map - for ( const model of await provider.models() ) { - model.id = model.id.trim().toLowerCase(); - if ( model.puterId ) { - model.puterId = model.puterId.trim().toLowerCase(); - } - if ( model.aliases ) { - model.aliases = model.aliases.map(alias => alias.trim().toLowerCase()); - } - if ( ! this.#modelIdMap[model.id] ) { - this.#modelIdMap[model.id] = []; - } - this.#modelIdMap[model.id].push({ ...model, provider: providerName }); - - if ( model.puterId ) { - if ( model.aliases ) { - model.aliases.push(model.puterId); - } else { - model.aliases = [model.puterId]; - } - - // Derive standard alias forms from puterId for model singularity: - // puterId "service:org/model" -> "org/model" and "model" - const withoutService = model.puterId.includes(':') - ? model.puterId.slice(model.puterId.indexOf(':') + 1) - : model.puterId; - if ( ! model.aliases.includes(withoutService) ) { - model.aliases.push(withoutService); - } - const shortName = withoutService.includes('/') - ? withoutService.slice(withoutService.indexOf('/') + 1) - : withoutService; - if ( shortName !== withoutService && !model.aliases.includes(shortName) ) { - model.aliases.push(shortName); - } - } - - if ( model.aliases ) { - for ( let alias of model.aliases ) { - alias = alias.trim().toLowerCase(); - if ( ! this.#modelIdMap[alias] ) { - this.#modelIdMap[alias] = this.#modelIdMap[model.id]; - continue; - } - if ( this.#modelIdMap[alias] !== this.#modelIdMap[model.id] ) { - this.#modelIdMap[alias].push({ ...model, provider: providerName }); - this.#modelIdMap[model.id] = this.#modelIdMap[alias]; - continue; - } - } - } - this.#modelIdMap[model.id].sort((a, b) => { - const aCostKey = a.index_cost_key || a.output_cost_key || Object.keys(a.costs || {})[0]; - const bCostKey = b.index_cost_key || b.output_cost_key || Object.keys(b.costs || {})[0]; - const aCost = a.costs?.[aCostKey] ?? Infinity; - const bCost = b.costs?.[bCostKey] ?? Infinity; - return aCost - bCost; - }); - } - } - } - - models () { - const seen = new Set(); - return Object.entries(this.#modelIdMap) - .map(([_, models]) => models) - .flat() - .filter(model => { - const identity = `${model.provider}:${model.puterId || model.id}`; - if ( seen.has(identity) ) { - return false; - } - seen.add(identity); - return true; - }) - .sort((a, b) => { - if ( a.provider === b.provider ) { - return a.id.localeCompare(b.id); - } - return a.provider!.localeCompare(b.provider!); - }); - } - - list () { - return this.models().map(m => (m.puterId || m.id)).sort(); - } - - async generate (parameters: IGenerateVideoParams) { - const clientDriverCall = Context.get('client_driver_call'); - let { test_mode: testMode, intended_service: legacyProviderName } = clientDriverCall as { test_mode?: boolean; response_metadata: Record; intended_service?: string }; - - if ( parameters.model ) { - parameters.model = parameters.model.trim().toLowerCase(); - } - - const configuredProviders = Object.keys(this.#providers); - if ( configuredProviders.length === 0 ) { - throw new Error('no video generation providers configured'); - } - - let intendedProvider = (parameters.provider || (legacyProviderName === AIVideoGenerationService.SERVICE_NAME ? '' : legacyProviderName)) ?? ''; - - if ( !parameters.model && !intendedProvider ) { - intendedProvider = configuredProviders.includes(AIVideoGenerationService.DEFAULT_PROVIDER) - ? AIVideoGenerationService.DEFAULT_PROVIDER - : configuredProviders[0]; - } - - if ( intendedProvider && !this.#providers[intendedProvider] ) { - intendedProvider = configuredProviders[0]; - } - - if ( !parameters.model && intendedProvider ) { - parameters.model = this.#providers[intendedProvider].getDefaultModel(); - } - - const model = parameters.model ? this.getModel({ modelId: parameters.model, provider: intendedProvider }) : undefined; - - if ( ! model ) { - const availableModelsUrl = `${this.global_config.origin}/puterai/video/models`; - - throw APIError.create('field_invalid', undefined, { - key: 'model', - expected: `a valid model name from ${availableModelsUrl}`, - got: parameters.model, - }); - } - - const provider = this.#providers[model.provider!]; - if ( ! provider ) { - throw new Error(`no provider found for model ${model.id}`); - } - - if ( model.durationSeconds?.length ) { - const requestedSeconds = parameters.seconds ?? parameters.duration; - const normalizedSeconds = typeof requestedSeconds === 'string' - ? Number.parseInt(requestedSeconds, 10) - : requestedSeconds; - const validSeconds = model.durationSeconds.includes(Number(normalizedSeconds)) - ? normalizedSeconds - : model.durationSeconds[0]; - parameters.seconds = validSeconds; - parameters.duration = validSeconds; - } - - if ( model.dimensions?.length ) { - const requestedResolution = typeof parameters.size === 'string' && parameters.size.trim() - ? parameters.size - : typeof parameters.resolution === 'string' && parameters.resolution.trim() - ? parameters.resolution - : undefined; - - const normalizedResolution = requestedResolution && model.dimensions.includes(requestedResolution) - ? requestedResolution - : model.dimensions[0]; - parameters.size = normalizedResolution; - parameters.resolution = normalizedResolution; - } - - return await provider.generate({ - ...parameters, - model: model.id, - provider: model.provider, - test_mode: testMode, - }); - } -} diff --git a/src/backend/src/services/ai/video/providers/GeminiVideoGenerationProvider/GeminiVideoGenerationProvider.ts b/src/backend/src/services/ai/video/providers/GeminiVideoGenerationProvider/GeminiVideoGenerationProvider.ts deleted file mode 100644 index 10c414e94..000000000 --- a/src/backend/src/services/ai/video/providers/GeminiVideoGenerationProvider/GeminiVideoGenerationProvider.ts +++ /dev/null @@ -1,309 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { GoogleGenAI, GenerateVideosOperation, GenerateVideosParameters } from '@google/genai'; -import { sha256 } from 'js-sha256'; -import APIError from '../../../../../api/APIError.js'; -import { Context } from '../../../../../util/context.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import { IGenerateVideoParams, IVideoModel, IVideoProvider } from '../types.js'; -import { TypedValue } from '../../../../drivers/meta/Runtime.js'; -import { GEMINI_VIDEO_GENERATION_MODELS, IGeminiVideoModel } from './models.js'; - -const DEFAULT_TEST_VIDEO_URL = 'https://assets.puter.site/txt2vid.mp4'; -const POLL_INTERVAL_MS = 10_000; -const DEFAULT_TIMEOUT_MS = 10 * 60 * 1000; - -const DIMENSION_MAP: Record = { - '1280x720': { aspectRatio: '16:9', resolution: '720p' }, - '720x1280': { aspectRatio: '9:16', resolution: '720p' }, - '1920x1080': { aspectRatio: '16:9', resolution: '1080p' }, - '1080x1920': { aspectRatio: '9:16', resolution: '1080p' }, - '3840x2160': { aspectRatio: '16:9', resolution: '4k' }, - '2160x3840': { aspectRatio: '9:16', resolution: '4k' }, -}; - -export class GeminiVideoGenerationProvider implements IVideoProvider { - #client: GoogleGenAI; - #meteringService: MeteringService; - #origin: string; - #urlSignatureSecret: string; - - constructor (config: { apiKey: string, origin?: string, urlSignatureSecret?: string }, meteringService: MeteringService) { - if ( ! config.apiKey ) { - throw new Error('Gemini video generation requires an API key'); - } - this.#client = new GoogleGenAI({ apiKey: config.apiKey }); - this.#meteringService = meteringService; - this.#origin = config.origin || 'https://api.puter.com'; - this.#urlSignatureSecret = config.urlSignatureSecret || ''; - } - - getDefaultModel (): string { - return GEMINI_VIDEO_GENERATION_MODELS[0].id; - } - - async models (): Promise { - return GEMINI_VIDEO_GENERATION_MODELS.map(model => ({ - ...model, - aliases: [model.id, `google/${model.id}`], - })); - } - - async generate (params: IGenerateVideoParams): Promise { - const { - prompt, - model: requestedModel, - seconds, - duration, - size, - resolution, - negative_prompt: negativePrompt, - reference_images: referenceImages, - input_reference: inputReference, - last_frame: lastFrame, - test_mode: testMode, - } = params ?? {}; - - if ( typeof prompt !== 'string' || !prompt.trim() ) { - throw APIError.create('field_invalid', null, { - key: 'prompt', - expected: 'a non-empty string', - got: prompt, - }); - } - - const selectedModel = this.#getModel(requestedModel); - - if ( testMode ) { - return new TypedValue({ - $: 'string:url:web', - content_type: 'video', - }, DEFAULT_TEST_VIDEO_URL); - } - - const hasFirstFrame = selectedModel.supportsImageInput - && typeof inputReference === 'string' && inputReference.trim().length > 0; - const hasRefImages = selectedModel.supportsReferenceImages - && Array.isArray(referenceImages) && referenceImages.length > 0; - - const { aspectRatio, videoResolution } = this.#resolveAspectAndResolution(size, selectedModel); - - // 1080p and 4K require duration=8 - const isHighRes = videoResolution === '1080p' || videoResolution === '4k'; - let durationSeconds = this.#coercePositiveInteger(seconds ?? duration) - ?? selectedModel.durationSeconds?.[0] ?? 8; - if ( isHighRes || hasRefImages ) { - durationSeconds = 8; - } - - const is4K = videoResolution === '4k'; - const is1080p = videoResolution === '1080p'; - const perSecondCents = is4K - ? selectedModel.costs?.['per-second-4k'] ?? selectedModel.costs?.['per-second'] - : is1080p - ? selectedModel.costs?.['per-second-1080p'] ?? selectedModel.costs?.['per-second'] - : selectedModel.costs?.['per-second']; - if ( perSecondCents === undefined ) { - throw new Error(`No per-second cost configured for video model '${selectedModel.id}'`); - } - const costCents = perSecondCents * durationSeconds; - const costInMicroCents = Math.ceil(costCents * 1_000_000); - - const actor = Context.get('actor'); - if ( ! actor ) { - throw new Error('actor not found in context'); - } - - const usageAllowed = await this.#meteringService.hasEnoughCredits(actor, costInMicroCents); - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); - } - - const config: Record = { - numberOfVideos: 1, - durationSeconds, - }; - - if ( aspectRatio ) config.aspectRatio = aspectRatio; - if ( videoResolution && selectedModel.resolutions.length > 0 ) { - config.resolution = videoResolution; - } - if ( typeof negativePrompt === 'string' && negativePrompt.trim() ) { - config.negativePrompt = negativePrompt; - } - - // Reference images (Veo 3.1 supports up to 3) - // When referenceImages is set, image (first frame), video, and lastFrame are not supported. - if ( hasRefImages ) { - const validImages = referenceImages - .filter((img: string) => typeof img === 'string' && img.trim().length > 0) - .slice(0, 3); - config.referenceImages = validImages.map((img: string) => ({ - image: this.#parseImageInput(img), - referenceType: 'asset', - })); - } - - if ( !hasRefImages && typeof lastFrame === 'string' && lastFrame.trim() ) { - config.lastFrame = this.#parseImageInput(lastFrame); - } - - const generateParams: GenerateVideosParameters = { - model: selectedModel.id, - prompt, - config, - }; - - // First frame (image-to-video) - if ( hasFirstFrame && !hasRefImages ) { - generateParams.image = this.#parseImageInput(inputReference as string); - } - - let operation: GenerateVideosOperation; - try { - operation = await this.#client.models.generateVideos(generateParams); - } catch (e) { - console.error('Gemini video generation error:', e); - throw e; - } - - const completed = await this.#pollUntilComplete(operation); - - const generatedVideos = completed.response?.generatedVideos; - if ( !generatedVideos || generatedVideos.length === 0 ) { - const filtered = completed.response?.raiMediaFilteredCount ?? 0; - if ( filtered > 0 ) { - const reasons = completed.response?.raiMediaFilteredReasons?.join(', ') || 'content policy'; - throw new Error(`Video was filtered due to ${reasons}`); - } - throw new Error('Gemini response did not include a video'); - } - - const video = generatedVideos[0].video; - if ( ! video ) { - throw new Error('Gemini response video entry was empty'); - } - - const resTier = is4K ? ':4k' : is1080p && selectedModel.costs?.['per-second-1080p'] ? ':1080p' : ''; - const usageKey = `gemini:${selectedModel.id}${resTier}`; - await this.#meteringService.incrementUsage(actor, usageKey, durationSeconds, costInMicroCents); - - if ( video.uri ) { - const fileIdMatch = video.uri.match(/\/files\/([^/:]+)/); - if ( ! fileIdMatch ) { - throw new Error('Could not extract file ID from Gemini video URI'); - } - const fileId = fileIdMatch[1]; - const expires = Math.ceil(Date.now() / 1000) + 48 * 3600; // from google docs: Generated videos are stored on the server for 2 days - const signature = sha256(`${fileId}/video-proxy/${this.#urlSignatureSecret}/${expires}`); - const proxyUrl = `${this.#origin}/puterai/video/proxy?fileId=${encodeURIComponent(fileId)}&provider=gemini&expires=${expires}&signature=${signature}`; - return new TypedValue({ - $: 'string:url:web', - content_type: 'video', - }, proxyUrl); - } - - if ( video.videoBytes ) { - const mimeType = video.mimeType ?? 'video/mp4'; - const dataUri = `data:${mimeType};base64,${video.videoBytes}`; - return new TypedValue({ - $: 'string:url:data', - content_type: 'video', - }, dataUri); - } - - throw new Error('Gemini video response contained neither uri nor videoBytes'); - } - - async #pollUntilComplete (operation: GenerateVideosOperation): Promise { - let op = operation; - const start = Date.now(); - - while ( !op.done ) { - if ( Date.now() - start > DEFAULT_TIMEOUT_MS ) { - throw new Error('Timed out waiting for Gemini video generation to complete'); - } - - await this.#delay(POLL_INTERVAL_MS); - op = await this.#client.operations.getVideosOperation({ operation: op }); - } - - if ( op.error ) { - const msg = (op.error as Record).message ?? JSON.stringify(op.error); - throw new Error(`Gemini video generation failed: ${msg}`); - } - - return op; - } - - #parseImageInput (input: string): { imageBytes: string; mimeType: string } { - if ( input.startsWith('data:') ) { - const commaIdx = input.indexOf(','); - if ( commaIdx !== -1 ) { - const header = input.substring(5, commaIdx); - if ( header.endsWith(';base64') ) { - const mimeType = header.substring(0, header.length - 7); - if ( mimeType.length > 0 ) { - return { imageBytes: input.substring(commaIdx + 1), mimeType }; - } - } - } - } - return { imageBytes: input, mimeType: 'image/png' }; - } - - #getModel (requestedModel?: string): IGeminiVideoModel { - return GEMINI_VIDEO_GENERATION_MODELS.find(m => m.id === requestedModel) - ?? GEMINI_VIDEO_GENERATION_MODELS[0]; - } - - #resolveAspectAndResolution ( - size: string | undefined, - model: IGeminiVideoModel, - ): { aspectRatio: string; videoResolution: string | undefined } { - if ( size && DIMENSION_MAP[size] ) { - return { - aspectRatio: DIMENSION_MAP[size].aspectRatio, - videoResolution: DIMENSION_MAP[size].resolution, - }; - } - - return { - aspectRatio: model.aspectRatios[0], - videoResolution: model.resolutions[0], - }; - } - - #coercePositiveInteger (value: unknown): number | undefined { - if ( typeof value === 'number' && Number.isFinite(value) ) { - const rounded = Math.round(value); - return rounded > 0 ? rounded : undefined; - } - if ( typeof value === 'string' ) { - const numeric = Number.parseInt(value, 10); - return Number.isFinite(numeric) && numeric > 0 ? numeric : undefined; - } - return undefined; - } - - async #delay (ms: number): Promise { - return await new Promise(resolve => setTimeout(resolve, ms)); - } -} diff --git a/src/backend/src/services/ai/video/providers/OpenAIVideoGenerationProvider/OpenAIVideoGenerationProvider.ts b/src/backend/src/services/ai/video/providers/OpenAIVideoGenerationProvider/OpenAIVideoGenerationProvider.ts deleted file mode 100644 index 94d0fe64e..000000000 --- a/src/backend/src/services/ai/video/providers/OpenAIVideoGenerationProvider/OpenAIVideoGenerationProvider.ts +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import OpenAI from 'openai'; -import APIError from '../../../../../api/APIError.js'; -import { Context } from '../../../../../util/context.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import { IGenerateVideoParams, IVideoModel, IVideoProvider } from '../types.js'; -import { TypedValue } from '../../../../drivers/meta/Runtime.js'; -import { Readable } from 'stream'; -import { OPENAI_VIDEO_MODELS, OPENAI_VIDEO_ALLOWED_SECONDS } from './models.js'; - -const DEFAULT_TEST_VIDEO_URL = 'https://assets.puter.site/txt2vid.mp4'; -const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000; -const POLL_INTERVAL_MS = 5_000; -const DEFAULT_DURATION_SECONDS = 4; - -export class OpenAIVideoGenerationProvider implements IVideoProvider { - #openai: OpenAI; - #meteringService: MeteringService; - - constructor (config: { apiKey: string }, meteringService: MeteringService) { - if ( ! config.apiKey ) { - throw new Error('OpenAI video generation requires an API key'); - } - this.#openai = new OpenAI({ apiKey: config.apiKey }); - this.#meteringService = meteringService; - } - - getDefaultModel (): string { - return OPENAI_VIDEO_MODELS[0].id; - } - - async models (): Promise { - return OPENAI_VIDEO_MODELS; - } - - async generate (params: IGenerateVideoParams): Promise { - const { - prompt, - model: requestedModel, - duration, - seconds, - size, - resolution, - input_reference: inputReference, - test_mode: testMode, - } = params ?? {}; - - if ( typeof prompt !== 'string' || !prompt.trim() ) { - throw APIError.create('field_invalid', null, { - key: 'prompt', - expected: 'a non-empty string', - got: prompt, - }); - } - - const selectedModel = await this.#selectModel(requestedModel); - - if ( ! selectedModel ) { - throw new Error(`Unknown video model: ${requestedModel}`); - } - - if ( testMode ) { - return new TypedValue({ - $: 'string:url:web', - content_type: 'video', - }, DEFAULT_TEST_VIDEO_URL); - } - - const defaultSize = selectedModel.dimensions?.[0] ?? '720x1280'; - const normalizedSize = this.#normalizeSize(size ?? resolution, selectedModel) ?? defaultSize; - const normalizedSeconds = this.#normalizeSeconds(seconds ?? duration) ?? String(DEFAULT_DURATION_SECONDS); - - const sizeTier = this.#determineSizeTier(selectedModel, normalizedSize); - const costPerSecondCents = this.#getCostPerSecond(selectedModel, sizeTier); - - if ( ! costPerSecondCents ) { - throw new Error(`No pricing configured for model ${selectedModel.id} at size ${normalizedSize}`); - } - - const estimatedUnits = this.#parseSeconds(normalizedSeconds) ?? DEFAULT_DURATION_SECONDS; - const actor = Context.get('actor'); - const costInMicroCents = costPerSecondCents * 1_000_000; - const usageAllowed = await this.#meteringService.hasEnoughCredits(actor, costInMicroCents * estimatedUnits); - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); - } - - const createParams: OpenAI.VideoCreateParams = { - prompt, - model: selectedModel.id, - seconds: normalizedSeconds as OpenAI.VideoSeconds, - size: normalizedSize as OpenAI.VideoSize, - }; - - if ( inputReference ) { - createParams.input_reference = inputReference as OpenAI.VideoCreateParams['input_reference']; - } - - const createResponse = await this.#openai.videos.create(createParams); - const finalJob = await this.#pollUntilComplete(createResponse); - - if ( finalJob.status === 'failed' ) { - const errorMessage = finalJob.error?.message ?? 'Video generation failed'; - throw new Error(errorMessage); - } - - const finalResolution = this.#normalizeSize(finalJob.size, selectedModel) ?? normalizedSize; - const finalTier = this.#determineSizeTier(selectedModel, finalResolution); - const finalCostPerSecondCents = this.#getCostPerSecond(selectedModel, finalTier); - - if ( ! finalCostPerSecondCents ) { - throw new Error(`No pricing configured for model ${selectedModel.id} at size ${finalResolution}`); - } - - const finalCostInMicroCents = finalCostPerSecondCents * 1_000_000; - const actualSeconds = this.#parseSeconds(finalJob.seconds) ?? estimatedUnits; - - const downloadResponse = await this.#openai.videos.downloadContent(finalJob.id); - const contentType = downloadResponse.headers.get('content-type') ?? 'video/mp4'; - - let stream: any = downloadResponse.body; - if ( stream && typeof stream.getReader === 'function' ) { - stream = Readable.fromWeb(stream as any); - } - - if ( ! stream ) { - const arrayBuffer = await downloadResponse.arrayBuffer(); - stream = Readable.from(Buffer.from(arrayBuffer)); - } - - const finalUsageKey = this.#getUsageKey(selectedModel, finalTier); - await this.#meteringService.incrementUsage(actor, finalUsageKey, actualSeconds, finalCostInMicroCents * actualSeconds); - - return new TypedValue({ - $: 'stream', - content_type: contentType, - }, stream); - } - - async #selectModel (requestedModel?: string): Promise { - const allModels = await this.models(); - return allModels.find(m => m.id.toLowerCase() === requestedModel?.toLowerCase()); - } - - async #pollUntilComplete (initialJob: OpenAI.Video): Promise { - let job = initialJob; - const start = Date.now(); - - while ( job.status === 'queued' || job.status === 'in_progress' ) { - if ( Date.now() - start > DEFAULT_TIMEOUT_MS ) { - throw new Error('Timed out waiting for Sora video generation to complete'); - } - - await this.#delay(POLL_INTERVAL_MS); - job = await this.#openai.videos.retrieve(job.id); - } - - return job; - } - - async #delay (ms: number): Promise { - return await new Promise(resolve => setTimeout(resolve, ms)); - } - - #normalizeSize (candidate: unknown, model: IVideoModel): string | undefined { - if ( ! candidate ) return undefined; - const normalized = this.#normalizeResolution(candidate); - if ( normalized && model.dimensions?.includes(normalized) ) { - return normalized; - } - return undefined; - } - - #normalizeSeconds (value: unknown): string | undefined { - if ( value === null || value === undefined ) { - return undefined; - } - const parsed = typeof value === 'number' ? String(Math.round(value)) : typeof value === 'string' ? value.trim() : undefined; - if ( parsed && OPENAI_VIDEO_ALLOWED_SECONDS.includes(Number(parsed) as typeof OPENAI_VIDEO_ALLOWED_SECONDS[number]) ) { - return parsed; - } - return undefined; - } - - #determineSizeTier (model: IVideoModel, size: string): string { - if ( model.id === 'sora-2-pro' ) { - if ( size === '1080x1920' || size === '1920x1080' ) return 'xxl'; - if ( size === '1024x1792' || size === '1792x1024' ) return 'xl'; - } - return 'default'; - } - - #getCostPerSecond (model: IVideoModel, tier: string): number | undefined { - const key = tier === 'default' ? 'per-second' : `per-second-${tier}`; - return model.costs?.[key]; - } - - #getUsageKey (model: IVideoModel, tier: string): string { - return `openai:${model.id}:${tier}`; - } - - #normalizeResolution (value: unknown): string | undefined { - if ( ! value ) return undefined; - if ( typeof value === 'string' ) { - const match = value.match(/(\d+)\s*x\s*(\d+)/i); - if ( match ) { - const w = Number.parseInt(match[1], 10); - const h = Number.parseInt(match[2], 10); - if ( Number.isFinite(w) && Number.isFinite(h) ) { - return `${w}x${h}`; - } - } - } - return undefined; - } - - #parseSeconds (value: unknown): number | undefined { - if ( value === null || value === undefined ) return undefined; - if ( typeof value === 'number' && Number.isFinite(value) ) { - return Math.round(value); - } - if ( typeof value === 'string' ) { - const numeric = Number.parseInt(value, 10); - return Number.isFinite(numeric) ? numeric : undefined; - } - return undefined; - } -} diff --git a/src/backend/src/services/ai/video/providers/TogetherVideoGenerationProvider/TogetherVideoGenerationProvider.ts b/src/backend/src/services/ai/video/providers/TogetherVideoGenerationProvider/TogetherVideoGenerationProvider.ts deleted file mode 100644 index 5320ac44e..000000000 --- a/src/backend/src/services/ai/video/providers/TogetherVideoGenerationProvider/TogetherVideoGenerationProvider.ts +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { Together } from 'together-ai'; -import APIError from '../../../../../api/APIError.js'; -import { Context } from '../../../../../util/context.js'; -import { MeteringService } from '../../../../MeteringService/MeteringService.js'; -import { IGenerateVideoParams, IVideoModel, IVideoProvider } from '../types.js'; -import { TypedValue } from '../../../../drivers/meta/Runtime.js'; -import { TOGETHER_VIDEO_GENERATION_MODELS } from './models.js'; - -const DEFAULT_TEST_VIDEO_URL = 'https://assets.puter.site/txt2vid.mp4'; -const POLL_INTERVAL_MS = 5_000; -const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000; -const DEFAULT_MODEL = 'minimax/video-01-director'; -const DEFAULT_DURATION_SECONDS = 6; - -export class TogetherVideoGenerationProvider implements IVideoProvider { - #client: Together; - #meteringService: MeteringService; - - constructor (config: { apiKey: string }, meteringService: MeteringService) { - if ( ! config.apiKey ) { - throw new Error('Together AI video generation requires an API key'); - } - this.#client = new Together({ apiKey: config.apiKey }); - this.#meteringService = meteringService; - } - - getDefaultModel (): string { - return 'togetherai:minimax/video-01-director'; - } - - async models (): Promise { - return TOGETHER_VIDEO_GENERATION_MODELS.map((model) => ({ - ...model, - aliases: [model.model], - durationSeconds: model.durationSeconds ?? undefined, - dimensions: model.dimensions ?? undefined, - fps: model.fps ?? undefined, - keyframes: model.keyframes ?? undefined, - promptLength: model.promptLength ?? undefined, - promptSupported: model.promptSupported ?? undefined, - })); - } - - async generate (params: IGenerateVideoParams): Promise { - const { - prompt, - model: requestedModel, - seconds, - no_extra_params, - duration, - width, - height, - fps, - steps, - guidance_scale: guidanceScale, - seed, - output_format: outputFormat, - output_quality: outputQuality, - negative_prompt: negativePrompt, - reference_images: referenceImages, - frame_images: frameImages, - metadata, - test_mode: testMode, - } = params ?? {}; - - if ( typeof prompt !== 'string' || !prompt.trim() ) { - throw APIError.create('field_invalid', null, { - key: 'prompt', - expected: 'a non-empty string', - got: prompt, - }); - } - - const selectedModel = await this.#getModel(requestedModel); - const model = selectedModel?.model ?? this.#stripTogetherPrefix(requestedModel ?? DEFAULT_MODEL); - - if ( testMode ) { - return new TypedValue({ - $: 'string:url:web', - content_type: 'video', - }, DEFAULT_TEST_VIDEO_URL); - } - - const costPerVideoCents = selectedModel?.costs?.['per-video']; - if ( ! costPerVideoCents ) { - throw new Error(`No pricing configured for video model ${model}`); - } - const costInMicroCents = costPerVideoCents * 1_000_000; - - let normalizedSeconds = this.#coercePositiveInteger(seconds ?? duration); - - if ( ! no_extra_params ) { - normalizedSeconds ??= DEFAULT_DURATION_SECONDS; - } - - const actor = Context.get('actor'); - if ( ! actor ) { - throw new Error('actor not found in context'); - } - - const usageAllowed = await this.#meteringService.hasEnoughCredits(actor, costInMicroCents); - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); - } - - const createPayload: Together.VideoCreateParams & { metadata?: object } = { - prompt, - model, - }; - - if ( normalizedSeconds ) { - createPayload.seconds = String(normalizedSeconds); - } - if ( this.#isFiniteNumber(width) ) { - createPayload.width = Number(width); - } - if ( this.#isFiniteNumber(height) ) { - createPayload.height = Number(height); - } - if ( this.#isFiniteNumber(fps) ) { - createPayload.fps = Number(fps); - } - if ( this.#isFiniteNumber(steps) ) { - createPayload.steps = Number(steps); - } - if ( this.#isFiniteNumber(guidanceScale) ) { - createPayload.guidance_scale = Number(guidanceScale); - } - if ( this.#isFiniteNumber(seed) ) { - createPayload.seed = Number(seed); - } - if ( typeof outputFormat === 'string' && outputFormat.trim() ) { - createPayload.output_format = outputFormat.trim() as Together.VideoCreateParams['output_format']; - } - if ( this.#isFiniteNumber(outputQuality) ) { - createPayload.output_quality = Number(outputQuality); - } - if ( typeof negativePrompt === 'string' && negativePrompt.trim() ) { - createPayload.negative_prompt = negativePrompt; - } - if ( Array.isArray(referenceImages) && referenceImages.length > 0 ) { - createPayload.reference_images = referenceImages.filter((item: string) => typeof item === 'string' && item.trim().length > 0); - } - if ( Array.isArray(frameImages) && frameImages.length > 0 ) { - createPayload.frame_images = frameImages.filter((frame: any) => frame && typeof frame === 'object' && typeof frame.input_image === 'string') as Together.VideoCreateParams['frame_images']; - } - if ( metadata && typeof metadata === 'object' ) { - createPayload.metadata = metadata; - } - - const job = await this.#client.videos.create(createPayload); - const finalJob = await this.#pollUntilComplete(job.id); - - if ( finalJob.status === 'failed' ) { - const errorMessage = finalJob?.info?.errors?.[0]?.message ?? - finalJob?.info?.errors?.message ?? - finalJob?.info?.errors ?? - 'Video generation failed'; - throw new Error(errorMessage); - } - - if ( finalJob.status === 'cancelled' ) { - throw new Error('Video generation was cancelled'); - } - - const usageKey = `together-video:${model}`; - await this.#meteringService.incrementUsage(actor, usageKey, 1, costInMicroCents); - - const videoUrl = finalJob?.outputs?.video_url; - if ( typeof videoUrl === 'string' && videoUrl.trim() ) { - return new TypedValue({ - $: 'string:url:web', - content_type: 'video', - }, videoUrl); - } - - throw new Error('Together AI response did not include a video URL'); - } - - async #pollUntilComplete (jobId: string): Promise { - // any here because sdk types are wrong https://docs.together.ai/docs/videos-overview -> "Job Status Reference" - let job = await (this.#client as any).videos.retrieve(jobId); - const start = Date.now(); - - while ( job.status === 'queued' || job.status === 'in_progress' ) { - if ( Date.now() - start > DEFAULT_TIMEOUT_MS ) { - throw new Error('Timed out waiting for Together AI video generation to complete'); - } - - await this.#delay(POLL_INTERVAL_MS); - job = await (this.#client as any).videos.retrieve(jobId); - } - - return job; - } - - async #delay (ms: number): Promise { - return await new Promise(resolve => setTimeout(resolve, ms)); - } - - async #getModel (requestedModel?: string): Promise { - const bareModel = this.#stripTogetherPrefix(requestedModel ?? DEFAULT_MODEL); - const allModels = await this.models(); - return allModels.find(m => m.model?.toLowerCase() === bareModel.toLowerCase()); - } - - #stripTogetherPrefix (model: string): string { - if ( typeof model === 'string' && model.startsWith('togetherai:') ) { - return model.slice('togetherai:'.length); - } - return model; - } - - #coercePositiveInteger (value: unknown): number | undefined { - if ( typeof value === 'number' && Number.isFinite(value) ) { - const rounded = Math.round(value); - return rounded > 0 ? rounded : undefined; - } - if ( typeof value === 'string' ) { - const numeric = Number.parseInt(value, 10); - return Number.isFinite(numeric) && numeric > 0 ? numeric : undefined; - } - return undefined; - } - - #isFiniteNumber (value: unknown): boolean { - if ( typeof value === 'number' ) { - return Number.isFinite(value); - } - if ( typeof value === 'string' ) { - const numeric = Number(value); - return Number.isFinite(numeric); - } - return false; - } -} diff --git a/src/backend/src/services/auth/ACLService.js b/src/backend/src/services/auth/ACLService.js deleted file mode 100644 index c9e467b6e..000000000 --- a/src/backend/src/services/auth/ACLService.js +++ /dev/null @@ -1,649 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const FSNodeParam = require('../../api/filesystem/FSNodeParam'); -const eggspress = require('../../api/eggspress'); -const { NodePathSelector } = require('../../deprecated/filesystem/node/selectors'); -const { get_user } = require('../../helpers'); -const configurable_auth = require('../../middleware/configurable_auth'); -const { Context } = require('../../util/context'); -const { BaseService } = require('../BaseService'); -const { AppUnderUserActorType, UserActorType, Actor, SystemActorType, AccessTokenActorType } = require('./Actor'); -const { DB_READ } = require('../database/consts'); -const { MANAGE_PERM_PREFIX } = require('./permissionConts.mjs'); -const { PermissionUtil } = require('./permissionUtils.mjs'); - -/** -* ACLService class handles Access Control List functionality for the Puter filesystem. -* Extends BaseService to provide permission management, access control checks, and ACL operations. -* Manages user-to-user permissions, filesystem node access, and handles special cases like -* public folders, app data access, and system actor privileges. Provides methods for -* checking permissions, setting ACLs, and managing access control hierarchies. -* @extends BaseService -*/ -class ACLService extends BaseService { - static MODULES = { - express: require('express'), - }; - - /** - * Initializes the ACLService by registering the 'public-folders' feature flag - * with the feature flag service. The flag's value is determined by the - * global_config.enable_public_folders setting. - * - * @async - * @private - * @returns {Promise} - */ - async _init () { - const svc_featureFlag = this.services.get('feature-flag'); - svc_featureFlag.register('public-folders', { - $: 'config-flag', - value: this.global_config.enable_public_folders ?? false, - }); - } - /** - * Checks if an actor has permission to perform a specific mode of access on a resource - * - * @param {Actor} actor - The actor requesting access (user, system, app, etc) - * @param {FSNode} resource - The filesystem resource being accessed - * @param {('see'| 'list'| 'read'| 'write')} mode - The access mode being requested ('read', 'write', etc) - * @returns {Promise} True if access is allowed, false otherwise - */ - async check (actor, resource, mode) { - if ( resource && typeof resource.path === 'string' && !resource.get_selector_of_type ) { - return await this.checkResource(actor, resource, mode); - } - return await this._check_fsNode(actor, resource, mode); - } - - /** - * Checks if an actor has permission for a specific mode on a filesystem node. - * Handles various actor types (System, User, AppUnderUser, AccessToken) and - * enforces access control rules including public folder access and app data permissions. - * - * @param {Actor} actor - The actor requesting access - * @param {FSNode} fsNode - The filesystem node to check permissions on - * @param {string} mode - The permission mode to check ('see', 'list', 'read', 'write') - * @returns {Promise} True if actor has permission, false otherwise - * @private - */ - async '__on_install.routes' (_, { app }) { - /** - * Handles route installation for ACL service endpoints. - * Sets up routes for user-to-user permission management including: - * - /acl/stat-user-user: Get permissions between users - * - /acl/set-user-user: Set permissions between users - * - * @param {*} _ Unused parameter - * @param {Object} options Installation options - * @param {Express} options.app Express app instance to attach routes to - * @returns {Promise} - */ - const r_acl = (() => { - const require = this.require; - const express = require('express'); - return express.Router(); - })(); - - app.use('/acl', r_acl); - - r_acl.use(eggspress('/stat-user-user', { - allowedMethods: ['POST'], - mw: [configurable_auth()], - }, async (req, res) => { - // Only user actor is allowed - if ( ! (req.actor.type instanceof UserActorType) ) { - return res.status(403).json({ - error: 'forbidden', - }); - } - - const holder_user = await get_user({ - username: req.body.user, - }); - - if ( ! holder_user ) { - throw APIError.create('user_does_not_exist', null, { - username: req.body.user, - }); - } - - const issuer = req.actor; - const holder = new Actor({ - type: new UserActorType({ - user: holder_user, - }), - }); - - const node = await (new FSNodeParam('path')).consolidate({ - req, - getParam: () => req.body.resource, - }); - - const permissions = await this.stat_user_user(issuer, holder, node); - - res.json({ permissions }); - })); - - r_acl.use(eggspress('/set-user-user', { - allowedMethods: ['POST'], - mw: [configurable_auth()], - }, async (req, res) => { - // Only user actor is allowed - if ( ! (req.actor.type instanceof UserActorType) ) { - return res.status(403).json({ - error: 'forbidden', - }); - } - - const holder_user = await get_user({ - username: req.body.user, - }); - - if ( ! holder_user ) { - throw APIError.create('user_does_not_exist', null, { - username: req.body.user, - }); - } - - const issuer = req.actor; - const holder = new Actor({ - type: new UserActorType({ - user: holder_user, - }), - }); - - const node = await (new FSNodeParam('path')).consolidate({ - req, - getParam: () => req.body.resource, - }); - - await this.set_user_user(issuer, holder, node, req.body.mode, req.body.options ?? {}); - - res.json({}); - })); - } - - /** - * Sets user-to-user permissions for a filesystem resource - * @param {Actor} issuer - The user granting the permission - * @param {Actor|string} holder - The user receiving the permission, or their username - * @param {FSNode|string} resource - The filesystem resource or permission string - * @param {string} mode - The permission mode to set - * @param {Object} [options={}] - Additional options - * @param {boolean} [options.only_if_higher] - Only set permission if no higher mode exists - * @returns {Promise} False if permission already exists or higher mode present - * @throws {Error} If issuer or holder is not a UserActorType - */ - async set_user_user (issuer, holder, resource, mode, options = {}) { - const svc_perm = this.services.get('permission'); - const svc_fs = this.services.get('filesystem'); - - if ( typeof holder === 'string' ) { - const holder_user = await get_user({ username: holder }); - if ( ! holder_user ) { - throw APIError.create('user_does_not_exist', null, { username: holder }); - } - - holder = new Actor({ - type: new UserActorType({ user: holder_user }), - }); - } - - let uid; - - if ( typeof resource === 'string' && mode === undefined ) { - const perm_parts = PermissionUtil.split(resource); - const isManage = PermissionUtil.isManage(resource); - uid = perm_parts.at(isManage ? -1 : -2); // always will end with fs:uid:mode - mode = isManage ? MANAGE_PERM_PREFIX : perm_parts.at(-1); - resource = await svc_fs.node(new NodePathSelector(uid)); - if ( ! resource ) { - throw APIError.create('subject_does_not_exist'); - } - } - - if ( ! (issuer.type instanceof UserActorType) ) { - throw new Error('issuer must be a UserActorType'); - } - if ( ! (holder.type instanceof UserActorType) ) { - throw new Error('holder must be a UserActorType'); - } - - const stat = await this.stat_user_user(issuer, holder, resource); - - const perms_on_this = stat[await resource.get('path')] ?? []; - - const mode_parts = perms_on_this.map(perm => PermissionUtil.isManage(perm) ? MANAGE_PERM_PREFIX : PermissionUtil.split(perm).at(-1)); - - // If mode already present, do nothing - if ( mode_parts.includes(mode) ) { - return false; - } - - // If higher mode already present, do nothing - if ( options.only_if_higher ) { - const higher_modes = this._higher_modes(mode); - if ( mode_parts.some(m => m === MANAGE_PERM_PREFIX || higher_modes.includes(m)) ) { - return false; - } - } - - uid = uid ?? await resource.get('uid'); - - // If mode not present, add it - await svc_perm.grant_user_user_permission(issuer, holder.type.user.username, mode === MANAGE_PERM_PREFIX ? PermissionUtil.join(MANAGE_PERM_PREFIX, 'fs', uid) : PermissionUtil.join('fs', uid, mode)); - - // Remove other modes - for ( const perm of perms_on_this ) { - const existingPermMode = PermissionUtil.isManage(perm) ? MANAGE_PERM_PREFIX : PermissionUtil.split(perm).at(-1); - if ( existingPermMode === mode ) continue; - - await svc_perm.revoke_user_user_permission(issuer, holder.type.user.username, perm); - } - } - - /** - * Sets user-to-user permissions for a filesystem resource - * @param {Actor} issuer - The user granting the permission - * @param {Actor|string} holder - The user receiving the permission, or their username - * @param {FSNode|string} resource - The filesystem resource or permission string - * @param {string} mode - The permission mode to set - * @param {Object} [options={}] - Additional options - * @param {boolean} [options.only_if_higher] - Only set permission if no higher mode exists - * @returns {Promise} False if permission already exists or higher mode present - * @throws {Error} If issuer or holder is not a UserActorType - */ - async stat_user_user (issuer, holder, resource) { - const svc_perm = this.services.get('permission'); - - if ( ! (issuer.type instanceof UserActorType) ) { - throw new Error('issuer must be a UserActorType'); - } - if ( ! (holder.type instanceof UserActorType) ) { - throw new Error('holder must be a UserActorType'); - } - - const permissions = {}; - - let perm_fsNode = resource; - while ( !await perm_fsNode.get('is-root') ) { - const prefix = PermissionUtil.join('fs', await perm_fsNode.get('uid')); - - const these_permissions = await - svc_perm.query_issuer_holder_permissions_by_prefix(issuer, holder, prefix); - - if ( these_permissions.length > 0 ) { - permissions[await perm_fsNode.get('path')] = these_permissions; - } - - perm_fsNode = await perm_fsNode.getParent(); - } - - return permissions; - } - - /** - * Checks filesystem node permissions for a given actor and mode - * - * @param {Actor} actor - The actor requesting access (User, System, AccessToken, or AppUnderUser) - * @param {FSNode} fsNode - The filesystem node to check permissions for - * @param {'see'| 'list' | 'read' | 'write' | 'manage'} mode - The permission mode to check ('see', 'list', 'read', 'write', 'manage) - * @returns {Promise} True if actor has permission, false otherwise - * - * @description - * Evaluates access permissions by checking: - * - System actors always have access - * - Public folder access rules - * - Access token authorizer permissions - * - App data directory special cases - * - Explicit permissions in the ACL hierarchy - */ - async _check_fsNode (actor, fsNode, mode) { - const context = Context.get(); - - actor = Actor.adapt(actor); - - if ( actor.type instanceof SystemActorType ) { - return true; - } - - const path_selector = fsNode.get_selector_of_type(NodePathSelector); - if ( path_selector && path_selector.value === '/' ) { - if ( ['list', 'see', 'read'].includes(mode) ) { - return true; - } - return false; - } - - // PERF: Short-circuit the permission check for users accessing their own files. - // Since the filesystem structure guarantees ownership within a user's home directory, - // we can safely grant access without a database lookup for the fsentry. - if ( actor.type instanceof UserActorType ) { - const username = actor.type.user.username; - const path_selector = fsNode.get_selector_of_type(NodePathSelector); - - if ( path_selector ) { - const path = path_selector.value; - // If the path starts with the user's own home directory, grant access immediately. - if ( path === `/${username}` || path.startsWith(`/${username}/`) ) { - return true; - } - } - } - - // PERF: Short-circuit for apps accessing their own AppData directory. - if ( actor.type instanceof AppUnderUserActorType ) { - const username = actor.type.user.username; - const app_uid = actor.type.app.uid; - let path_selector = fsNode.get_selector_of_type(NodePathSelector); - - // PATCH: Path selector must be obtained here due to a bug (#2295) - if ( ! path_selector ) { - path_selector = new NodePathSelector(await fsNode.get('path')); - } - - if ( path_selector ) { - const path = path_selector.value; - const appDataPath = `/${username}/AppData/${app_uid}`; - if ( path === appDataPath || path.startsWith(`${appDataPath}/`) ) { - return true; - } - } - } - - // Hard rule: anyone and anything can read /user/public directories - if ( this.global_config.enable_public_folders ) { - const public_modes = Object.freeze(['read', 'list', 'see']); - let is_public; - /** - * Checks if a given mode is allowed for a public folder path - * - * @param {Actor} actor - The actor requesting access - * @param {FSNode} fsNode - The filesystem node to check - * @param {string} mode - The access mode being requested (read/write/etc) - * @returns {Promise} True if access is allowed, false otherwise - * - * Handles special case for /user/public directories when public folders are enabled. - * Only allows read, list, and see modes for public folders, and only if the folder - * owner has confirmed their email (except for admin user). - */ - await (async () => { - if ( ! public_modes.includes(mode) ) return; - if ( ! (await fsNode.isPublic()) ) return; - - const svc_getUser = this.services.get('get-user'); - - const username = await fsNode.getUserPart(); - const user = await svc_getUser.get_user({ username }); - if ( ! (user.email_confirmed || user.username === 'admin') ) { - return; - } - - is_public = true; - })(); - if ( is_public ) return true; - } - - // Access tokens: allow if token has permission via DB and authorizer has permission - if ( actor.type instanceof AccessTokenActorType ) { - const { authorizer, token } = actor.type; - const authorizer_perm = await this._check_fsNode(authorizer, fsNode, mode); - if ( ! authorizer_perm ) return false; - - // We check access token permissions manually here and skip PermissionService - const db = this.services.get('database').get(DB_READ, 'auth'); - let perm_fsNode = fsNode; - - // Iterate up the directory tree (towards root directory) - while ( !(await perm_fsNode.get('is-root')) ) { - const uid = await perm_fsNode.get('uid'); - // DRY: second occurance of this code - const permission = mode === MANAGE_PERM_PREFIX - ? PermissionUtil.join(MANAGE_PERM_PREFIX, 'fs', uid) - : PermissionUtil.join('fs', uid, mode); - const rows = await db.read( - 'SELECT * FROM `access_token_permissions` WHERE `token_uid` = ? AND `permission` = ?', - [token, permission], - ); - - // We already checked that the authorizer has the required permission, - // so if the access token has the required permission as well we can - // return true immediately. - if ( rows[0] ) return true; - - // ...iterate - perm_fsNode = await perm_fsNode.getParent(); - } - - // If we reach here, the authorizer has permission to access the requested - // file/directory but the access token does not - return false; - } - - // Hard rule: if app-under-user is accessing appdata directory, allow - if ( actor.type instanceof AppUnderUserActorType ) { - const appdata_path = `/${actor.type.user.username}/AppData/${actor.type.app.uid}`; - const svc_fs = await context.get('services').get('filesystem'); - const appdata_node = await svc_fs.node(new NodePathSelector(appdata_path)); - - if ( - await appdata_node.is(fsNode) || - await appdata_node.is_above(fsNode) - ) { - return true; - } - } - - // app-under-user only works if the user also has permission - if ( actor.type instanceof AppUnderUserActorType ) { - const user_actor = new Actor({ - type: new UserActorType({ user: actor.type.user }), - }); - const user_perm = await this._check_fsNode(user_actor, fsNode, mode); - - if ( ! user_perm ) return false; - } - - // Hard rule: if app-under-user is accessing appdata directory - // under a **different user**, allow, - // IFF that appdata directory is shared with user - // (by "user also has permission" check above) - /** - * Checks if an actor has permission to perform a specific mode of access on a filesystem node. - * Handles various actor types (System, AccessToken, AppUnderUser) and special cases like - * public folders and app data directories. - * - * @param {Actor} actor - The actor requesting access - * @param {FSNode} fsNode - The filesystem node to check access for - * @param {string} mode - The access mode to check ('see', 'list', 'read', 'write') - * @returns {Promise} True if access is allowed, false otherwise - * @private - */ - if ( await (async () => { - if ( ! (actor.type instanceof AppUnderUserActorType) ) { - return false; - } - if ( await fsNode.getUserPart() === actor.type.user.username ) { - return false; - } - const components = await fsNode.getPathComponents(); - if ( components[1] !== 'AppData' ) return false; - if ( components[2] !== actor.type.app.uid ) return false; - return true; - })() ) return true; - - /** - * @type {import('../../services/auth/PermissionService').PermissionService} - */ - const svc_permission = await context.get('services').get('permission'); - - let perm_fsNode = fsNode; - while ( !await perm_fsNode.get('is-root') ) { - const uid = await perm_fsNode.get('uid'); - const permissionsToCheck = [mode === MANAGE_PERM_PREFIX ? PermissionUtil.join(MANAGE_PERM_PREFIX, 'fs', uid) : PermissionUtil.join('fs', uid, mode)]; - const reading = await svc_permission.scan(actor, permissionsToCheck); - const options = PermissionUtil.reading_to_options(reading); - if ( options.length > 0 ) { - return true; - } - perm_fsNode = await perm_fsNode.getParent(); - } - - return false; - } - - async checkResource (actor, resource, mode) { - const context = Context.get(); - - actor = Actor.adapt(actor); - - if ( actor.type instanceof SystemActorType ) { - return true; - } - - if ( resource.path === '/' ) { - return ['list', 'see', 'read'].includes(mode); - } - - const components = resource.path.slice(1).split('/'); - - if ( actor.type instanceof UserActorType ) { - const username = actor.type.user.username; - if ( resource.path === `/${username}` || resource.path.startsWith(`/${username}/`) ) { - return true; - } - } - - if ( actor.type instanceof AppUnderUserActorType ) { - const username = actor.type.user.username; - const appUid = actor.type.app.uid; - const appDataPath = `/${username}/AppData/${appUid}`; - if ( resource.path === appDataPath || resource.path.startsWith(`${appDataPath}/`) ) { - return true; - } - } - - if ( this.global_config.enable_public_folders ) { - const publicModes = ['read', 'list', 'see']; - if ( publicModes.includes(mode) && components.length > 1 && components[1] === 'Public' ) { - const svcGetUser = this.services.get('get-user'); - const username = components[0]; - const user = await svcGetUser.get_user({ username }); - if ( user && (user.email_confirmed || user.username === 'admin') ) { - return true; - } - } - } - - if ( actor.type instanceof AccessTokenActorType ) { - const { authorizer, token } = actor.type; - if ( ! (await this.checkResource(authorizer, resource, mode)) ) { - return false; - } - - const db = this.services.get('database').get(DB_READ, 'auth'); - const ancestors = await resource.resolveAncestors(); - for ( const ancestor of ancestors ) { - const permission = mode === MANAGE_PERM_PREFIX - ? PermissionUtil.join(MANAGE_PERM_PREFIX, 'fs', ancestor.uid) - : PermissionUtil.join('fs', ancestor.uid, mode); - const rows = await db.read( - 'SELECT * FROM `access_token_permissions` WHERE `token_uid` = ? AND `permission` = ?', - [token, permission], - ); - if ( rows[0] ) return true; - } - return false; - } - - if ( actor.type instanceof AppUnderUserActorType ) { - const userActor = new Actor({ - type: new UserActorType({ user: actor.type.user }), - }); - if ( ! (await this.checkResource(userActor, resource, mode)) ) { - return false; - } - } - - if ( actor.type instanceof AppUnderUserActorType ) { - if ( components[0] !== actor.type.user.username - && components[1] === 'AppData' - && components[2] === actor.type.app.uid ) { - return true; - } - } - - const svcPermission = context.get('services').get('permission'); - const ancestors = await resource.resolveAncestors(); - for ( const ancestor of ancestors ) { - const permissionsToCheck = [ - mode === MANAGE_PERM_PREFIX - ? PermissionUtil.join(MANAGE_PERM_PREFIX, 'fs', ancestor.uid) - : PermissionUtil.join('fs', ancestor.uid, mode), - ]; - const reading = await svcPermission.scan(actor, permissionsToCheck); - const options = PermissionUtil.reading_to_options(reading); - if ( options.length > 0 ) { - return true; - } - } - - return false; - } - - async get_safe_acl_error (actor, resource, _mode) { - const can_see = await this.check(actor, resource, 'see'); - if ( ! can_see ) { - return APIError.create('subject_does_not_exist'); - } - - return APIError.create('forbidden'); - } - - // If any logic depends on knowledge of the highest ACL mode, it should use - // this method in case a higher mode is added (ex: might add 'config' mode) - /** - * Gets the highest permission mode in the ACL system - * - * @returns {string} Returns 'write' as the highest permission mode - * - * @remarks - * This method should be used by any logic that depends on knowing the highest ACL mode, - * in case higher modes are added in the future (e.g. a potential 'config' mode). - * Currently 'write' is the highest mode in the hierarchy: see > list > read > write - */ - get_highest_mode () { - return 'write'; - } - - // TODO: DRY: Also in FilesystemService - _higher_modes (mode) { - // If you want to X, you can do so with any of [...Y] - if ( mode === 'see' ) return ['see', 'list', 'read', 'write']; - if ( mode === 'list' ) return ['list', 'read', 'write']; - if ( mode === 'read' ) return ['read', 'write']; - if ( mode === 'write' ) return ['write']; - } -} - -module.exports = { - ACLService, -}; diff --git a/src/backend/src/services/auth/Actor.d.ts b/src/backend/src/services/auth/Actor.d.ts deleted file mode 100644 index 107a52bd1..000000000 --- a/src/backend/src/services/auth/Actor.d.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { IUser } from '../User'; - -export interface ActorLogFields { - uid: string; - username?: string; -} - -export class SystemActorType { - constructor (_o?: Record); - get uid (): string; - get_related_type (_type_class: unknown): SystemActorType; -} - -export class UserActorType { - constructor (_params: { user: IUser; session?: { uuid: string }; hasHttpOnlyCookie?: boolean }); - user: IUser; - /** When true, this actor can access user-protected HTTP endpoints (e.g. change password). GUI tokens set this false. */ - hasHttpOnlyCookie: boolean; - get uid (): string; - get_related_type (_type_class: unknown): UserActorType; -} - -export class AppUnderUserActorType { - constructor (_params: { user: IUser, app: { id?: number; uid: string } }); - user: IUser; - app: { id?: number; uid: string }; - get uid (): string; - get_related_type (_type_class: unknown): UserActorType | AppUnderUserActorType; -} - -export class AccessTokenActorType { - constructor (_params: { authorizer: Actor, authorized?: Actor, token: string }); - authorizer: Actor; - authorized?: Actor; - token: string; - get uid (): string; - get_related_actor (): never; -} - -export class SiteActorType { - constructor (_params: { site: { name: string } }); - site: { name: string }; - get uid (): string; -} - -export type ActorType = - | SystemActorType - | UserActorType - | AppUnderUserActorType - | AccessTokenActorType - | SiteActorType; - -export interface ActorInit { - type: ActorType; -} - -export class Actor { - constructor (_init: ActorInit); - type: ActorType & { - user?: IUser; - app?: { id?: number; uid: string; timestamp?: Date }; - authorizer?: Actor; - }; - get uid (): string; - get private_uid (): string; - toLogFields (): ActorLogFields; - clone (): Actor; - get_related_actor (_type_class: unknown): Actor; - static create ( - _type: new (_params?: Record) => ActorType, - _params?: { - user_uid?: string; - app_uid?: string; - user?: IUser; - app?: { uid: string }; - [key: string]: unknown; - }, - ): Promise; - static get_system_actor (): Actor; - static adapt (_actor?: Actor | { username?: string, uuid?: string }): Actor; -} diff --git a/src/backend/src/services/auth/Actor.js b/src/backend/src/services/auth/Actor.js deleted file mode 100644 index 6afde7077..000000000 --- a/src/backend/src/services/auth/Actor.js +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import crypto from 'crypto'; -import { v5 as uuidv5 } from 'uuid'; -import { AdvancedBase } from '../../../../putility/index.js'; -import * as config from '../../config.js'; -import { get_app, get_user } from '../../helpers.js'; -import { Context } from '../../util/context.js'; -// TODO: add these to configuration; production deployments should change these! - -const PRIVATE_UID_NAMESPACE = config.private_uid_namespace - ?? crypto.randomUUID(); -const PRIVATE_UID_SECRET = config.private_uid_secret - ?? crypto.randomBytes(24).toString('hex'); - -/** - * Base class for all actor types in the system. - * Provides common initialization functionality for actor type instances. - */ -export class ActorType { - /** - * Initializes the ActorType with the provided properties. - * - * @param {Object} o - Object containing properties to assign to this instance. - */ - constructor (o) { - for ( const k in o ) { - this[k] = o[k]; - } - } -} - -/** - * Class representing the system actor type within the actor framework. - * This type serves as a specific implementation of an actor that - * represents a system-level entity and provides methods for UID retrieval - * and related type management. - */ -export class SystemActorType extends ActorType { - /** - * Gets the unique identifier for the system actor. - * - * @returns {string} Always returns 'system'. - */ - get uid () { - return 'system'; - } - - /** - * Gets a related actor type for the system actor. - * - * @param {Function} type_class - The ActorType class to get a related type for. - * @returns {SystemActorType} Returns this instance if type_class is SystemActorType. - * @throws {Error} If the requested type_class is not supported. - */ - get_related_type (type_class) { - if ( type_class === SystemActorType ) { - return this; - } - throw new Error(`cannot get ${type_class.name} from ${this.constructor.name}`); - } -} - -/** - * Represents an Actor in the system, extending functionality from AdvancedBase. - * The Actor class is responsible for managing actor instances, including - * creating new actors, generating unique identifiers, and handling related types - * that represent different roles within the context of the application. - */ -export class Actor extends AdvancedBase { - /** @type {ActorType} */ - type; - - static system_actor_ = null; - - /** - * Retrieves the system actor instance, creating it if it doesn't exist. - * This static method ensures that there is only one instance of the system actor. - * If the system actor has not yet been created, it will be instantiated with a - * new SystemActorType. - * - * @returns {Actor} The system actor instance. - */ - static get_system_actor () { - if ( ! this.system_actor_ ) { - this.system_actor_ = new Actor({ - type: new SystemActorType(), - }); - } - return this.system_actor_; - } - - /** - * Creates a new Actor instance with the specified type and parameters. - * Resolves user and app references from UIDs if provided in the parameters. - * - * @param {Function} type - The ActorType constructor to instantiate. - * @param {Object} params - Parameters for the actor type. - * @param {string} [params.user_uid] - UUID of the user to resolve. - * @param {string} [params.app_uid] - UID of the app to resolve. - * @returns {Promise} A new Actor instance. - */ - static async create (type, params) { - params = { ...params }; - if ( params.user_uid ) { - params.user = await get_user({ uuid: params.user_uid }); - } - if ( params.app_uid ) { - params.app = await get_app({ uid: params.app_uid }); - } - return new Actor({ - type: new type(params), - }); - } - - /** - * Initializes the Actor instance with the provided parameters. - * This constructor assigns object properties from the input object to the instance. - * - * @param {Object} o - The object containing actor parameters. - * @param {...any} a - Additional arguments passed to the parent class constructor. - */ - constructor (o, ...a) { - super(o, ...a); - for ( const k in o ) { - this[k] = o[k]; - } - } - - /** - * Gets the unique identifier for this actor. - * - * @returns {string} The actor's UID from its type. - */ - get uid () { - return this.type.uid; - } - - /** - * Returns fields suitable for logging this actor. - * - * @returns {Object} Object containing UID and optionally username for logging. - */ - toLogFields () { - return { - uid: this.type.uid, - ...(this.type.user ? { - username: this.type.user.username, - } : {}), - }; - } - - /** - * Generates a cryptographically-secure deterministic UUID - * from an actor's UID. The generated UUID is derived by - * applying SHA-256 HMAC to the actor's UID using a secret, - * then formatting the result as a UUID V5. - * - * @returns {string} The derived UUID corresponding to the actor's UID. - */ - get private_uid () { - // Pass the UUID through SHA-2 first because UUIDv5 - // is not cryptographically secure (it uses SHA-1) - const hmac = crypto.createHmac('sha256', PRIVATE_UID_SECRET) - .update(this.uid) - .digest('hex'); - - // Generate a UUIDv5 from the HMAC - // Note: this effectively does an additional SHA-1 hash, - // but this is done only to format the result as a UUID - // and not for cryptographic purposes - let str = uuidv5(hmac, PRIVATE_UID_NAMESPACE); - - // Uppercase UUID to avoid inference of what uuid library is being used - str = (`${str}`).toUpperCase(); - return str; - } - - /** - * Clones the current Actor instance, returning a new Actor object with the same type. - * - * @returns {Actor} A new Actor instance that is a copy of the current one. - */ - clone () { - return new Actor({ - type: this.type, - }); - } - - /** - * Creates a related actor of the specified type based on the current actor. - * - * @param {Function} type_class - The ActorType class to create a related actor for. - * @returns {Actor} A new Actor instance with the related type. - */ - get_related_actor (type_class) { - const actor = this.clone(); - actor.type = this.type.get_related_type(type_class); - return actor; - } -} - -/** - * Represents the type of a User Actor in the system, allowing operations and relations - * specific to user actors. This class extends the base functionality to uniquely identify - * user actors and define how they relate to other types of actors within the system. - */ -export class UserActorType extends ActorType { - constructor (o) { - super(o); - if ( this.hasHttpOnlyCookie === undefined ) { - this.hasHttpOnlyCookie = false; - } - } - - /** - * Gets the unique identifier for the user actor. - * - * @returns {string} The UID in format 'user:{uuid}'. - */ - get uid () { - return `user:${this.user.uuid}`; - } - - /** - * Gets a related actor type for the user actor. - * - * @param {Function} type_class - The ActorType class to get a related type for. - * @returns {UserActorType} Returns this instance if type_class is UserActorType. - * @throws {Error} If the requested type_class is not supported. - */ - get_related_type (type_class) { - if ( type_class === UserActorType ) { - return this; - } - throw new Error(`cannot get ${type_class.name} from ${this.constructor.name}`); - } -} - -/** - * Represents a user actor type in the application. This class defines the structure - * and behavior specific to user actors, including obtaining unique identifiers and - * retrieving related actor types. It extends the base actor type functionality - * to cater to user-specific needs. - */ -export class AppUnderUserActorType extends ActorType { - /** - * Gets the unique identifier for the app-under-user actor. - * - * @returns {string} The UID in format 'app-under-user:{user_uuid}:{app_uid}'. - */ - get uid () { - return `app-under-user:${this.user.uuid}:${this.app.uid}`; - } - - /** - * Gets a related actor type for the app-under-user actor. - * - * @param {Function} type_class - The ActorType class to get a related type for. - * @returns {UserActorType|AppUnderUserActorType} The related actor type instance. - * @throws {Error} If the requested type_class is not supported. - */ - get_related_type (type_class) { - if ( type_class === UserActorType ) { - return new UserActorType({ user: this.user }); - } - if ( type_class === AppUnderUserActorType ) { - return this; - } - throw new Error(`cannot get ${type_class.name} from ${this.constructor.name}`); - } -} - -/** - * Represents the type of access tokens in the system. - * An AccessTokenActorType associates an authorizer and an authorized actor - * with a string token, facilitating permission checks and identity management. - */ -export class AccessTokenActorType extends ActorType { - // authorizer: an Actor who authorized the token - // authorized: an Actor who is authorized by the token - // token: a string - - /** - * Gets the unique identifier for the access token actor. - * The UID is constructed based on the authorizer's UID, the authorized actor's UID (if available), - * and the token string. This UID format is useful for identifying the access token's context. - * - * @returns {string} The generated UID for the access token. - */ - get uid () { - return `access-token:${this.authorizer.uid - }:${this.authorized?.uid ?? '' - }:${this.token}`; - } - - /** - * Throws an error as getting related actors is not supported for access tokens. - * This would be dangerous because of ambiguity between authorizer and authorized. - * - * @throws {Error} Always throws an error indicating this operation is not supported. - */ - get_related_actor () { - // This would be dangerous because of ambiguity - // between authorizer and authorized - throw new Error(`cannot call get_related_actor on ${this.constructor.name}`); - } -} - -/** - * Represents a Site Actor Type, which encapsulates information about a site-specific actor. - * This class is used to manage details related to the site and implement functionalities - * pertinent to site-level operations and interactions in the actor framework. - */ -export class SiteActorType { - /** - * Constructor for the SiteActorType class. - * Initializes a new instance of SiteActorType with the provided properties. - * - * @param {Object} o - The properties to initialize the SiteActorType with. - * @param {...*} a - Additional arguments. - */ - constructor (o, ..._a) { - for ( const k in o ) { - this[k] = o[k]; - } - } - - /** - * Gets the unique identifier for the site actor. - * - * @returns {string} The UID in format 'site:{site_name}'. - */ - get uid () { - return `site:${this.site.name}`; - } -} - -/** - * Adapts various input types to a proper Actor instance. - * If no actor is provided, attempts to get one from the current context. - * Handles legacy user objects by wrapping them in UserActorType. - * - * @param {Actor|Object} [actor] - The actor to adapt, or undefined to use context. - * @returns {Actor} A properly formatted Actor instance. - */ -Actor.adapt = function (actor) { - actor = actor || Context.get('actor'); - - if ( actor?.username ) { - const user = actor; - actor = new Actor({ - type: new UserActorType({ user }), - }); - } - // Legacy: if actor is undefined, use the user in the context - if ( ! actor ) { - const user = Context.get('user'); - actor = new Actor({ - type: new UserActorType({ user }), - }); - } - - return actor; -}; \ No newline at end of file diff --git a/src/backend/src/services/auth/AntiCSRFService.js b/src/backend/src/services/auth/AntiCSRFService.js deleted file mode 100644 index 5222aef54..000000000 --- a/src/backend/src/services/auth/AntiCSRFService.js +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const eggspress = require('../../api/eggspress'); -const config = require('../../config'); -const { subdomain } = require('../../helpers'); -const BaseService = require('../BaseService'); -const { redisClient } = require('../../clients/redis/redisSingleton'); - -const REDIS_KEY_PREFIX = 'anticsrf:'; -const MAX_TOKENS_PER_SESSION = 10; -const TOKEN_TTL_SECONDS = 60 * 60; -// Sub-millisecond tie-breaker so rapid-fire create_token calls get strictly -// increasing ZSET scores instead of falling back to lexical token ordering. -const SCORE_TIEBREAKER_MOD = 1000; - -/** -* Class AntiCSRFService extends BaseService to manage and protect against Cross-Site Request Forgery (CSRF) attacks. -* Tokens are stored in Redis as a per-session sorted set (score = creation timestamp) -* so state is shared across backend instances. Only the most recent -* MAX_TOKENS_PER_SESSION tokens are retained; keys expire after TOKEN_TTL_SECONDS. -*/ -class AntiCSRFService extends BaseService { - _construct () { - this.score_tiebreaker_ = 0; - } - - /** - * Sets up the route handler for getting anti-CSRF tokens. - * Registers the '/get-anticsrf-token' endpoint that returns a new token for authenticated users. - * - * @returns {void} - */ - '__on_install.routes' () { - const { app } = this.services.get('web-server'); - - app.use(eggspress('/get-anticsrf-token', { - auth2: true, - allowedMethods: ['GET'], - }, async (req, res) => { - // We disallow `api.` because it has a more relaxed CORS policy - const subdomain_check = config.experimental_no_subdomain || - (subdomain(req) !== 'api'); - if ( ! subdomain_check ) { - return res.status(404).send('Hey, stop that!'); - } - - if ( ! req.user ) { - res.status(403).send({}); - return; - } - - // TODO: session uuid instead of user - const token = await this.create_token(req.user.uuid); - res.send({ token }); - })); - } - - /** - * Creates a new anti-CSRF token for the specified session and stores it in Redis. - * Only the most recent MAX_TOKENS_PER_SESSION tokens are retained per session. - * - * @param {string} session - The session identifier - * @returns {Promise} The newly created token - */ - async create_token (session) { - const token = this.generate_token_(); - const key = this.redis_key_(session); - this.score_tiebreaker_ = (this.score_tiebreaker_ + 1) % SCORE_TIEBREAKER_MOD; - const score = Date.now() * SCORE_TIEBREAKER_MOD + this.score_tiebreaker_; - const pipeline = redisClient.pipeline(); - pipeline.zadd(key, score, token); - pipeline.zremrangebyrank(key, 0, -(MAX_TOKENS_PER_SESSION + 1)); - pipeline.expire(key, TOKEN_TTL_SECONDS); - await pipeline.exec(); - return token; - } - - /** - * Attempts to consume (validate and remove) a token for the specified session. - * Uses an atomic ZREM so concurrent consumers can't double-spend a token. - * - * @param {string} session - The session identifier - * @param {string} token - The token to consume - * @returns {Promise} True if the token was valid and consumed, false otherwise - */ - async consume_token (session, token) { - if ( ! token ) return false; - const removed = await redisClient.zrem(this.redis_key_(session), token); - return removed > 0; - } - - redis_key_ (session) { - return `${REDIS_KEY_PREFIX}${session}`; - } - - /** - * Generates a secure random token as a hexadecimal string. - * The token is created using cryptographic random bytes to ensure uniqueness - * and security for Anti-CSRF purposes. - * - * @returns {string} The generated token. - */ - generate_token_ () { - return require('crypto').randomBytes(32).toString('hex'); - } - -} - -module.exports = { - AntiCSRFService, -}; diff --git a/src/backend/src/services/auth/AntiCSRFService.test.ts b/src/backend/src/services/auth/AntiCSRFService.test.ts deleted file mode 100644 index 346922cac..000000000 --- a/src/backend/src/services/auth/AntiCSRFService.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { createTestKernel } from '../../../tools/test.mjs'; -import { AntiCSRFService } from './AntiCSRFService.js'; - -describe('AntiCSRFService', () => { - it('should handle token generation, expiration, and consumption correctly', async () => { - const testKernel = await createTestKernel({ - serviceMap: { - 'anti-csrf': AntiCSRFService, - }, - }); - - const antiCSRFService = testKernel.services!.get('anti-csrf') as AntiCSRFService; - - // Do this several times, like a user would - for ( let i = 0 ; i < 30 ; i++ ) { - const session = `session-${i}`; - // Generate 30 tokens - const tokens = []; - for ( let j = 0 ; j < 30 ; j++ ) { - tokens.push(await antiCSRFService.create_token(session)); - } - // Only the last 10 should be valid - const results_for_stale_tokens = []; - for ( let j = 0 ; j < 20 ; j++ ) { - const result = await antiCSRFService.consume_token(session, tokens[j]); - results_for_stale_tokens.push(result); - } - expect(results_for_stale_tokens.every(v => v === false)).toBe(true); - // The last 10 should be valid - const results_for_valid_tokens = []; - for ( let j = 20 ; j < 30 ; j++ ) { - const result = await antiCSRFService.consume_token(session, tokens[j]); - results_for_valid_tokens.push(result); - } - expect(results_for_valid_tokens.every(v => v === true)).toBe(true); - // A completely arbitrary token should not be valid - expect(await antiCSRFService.consume_token(session, 'arbitrary')).toBe(false); - } - }); -}); diff --git a/src/backend/src/services/auth/AuthService.js b/src/backend/src/services/auth/AuthService.js deleted file mode 100644 index 404053456..000000000 --- a/src/backend/src/services/auth/AuthService.js +++ /dev/null @@ -1,1763 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { Actor, UserActorType, AppUnderUserActorType, AccessTokenActorType, SiteActorType } = require('./Actor'); -const { BaseService } = require('../BaseService'); -const { get_user, get_app } = require('../../helpers'); -const { Context } = require('../../util/context'); -const { kv } = require('../../util/kvSingleton'); -const APIError = require('../../api/APIError'); -const { setRedisCacheValue } = require('../../clients/redis/cacheUpdate.js'); -const { deleteRedisKeys } = require('../../clients/redis/deleteRedisKeys.js'); -const { redisClient } = require('../../clients/redis/redisSingleton.js'); -const { DB_READ, DB_WRITE } = require('../database/consts'); -const { UUIDFPE } = require('../../util/uuidfpe'); -const uuidLib = require('uuid'); -const crypto = require('crypto'); -// This constant defines the namespace used for generating app UUIDs from their origins -const APP_ORIGIN_UUID_NAMESPACE = '33de3768-8ee0-43e9-9e73-db192b97a5d8'; -const APP_ORIGIN_CACHE_KEY_PREFIX = 'auth:appOriginCanonicalization:origin'; -const APP_ORIGIN_LOCAL_CACHE_KEY_PREFIX = 'auth:appOriginCanonicalization:local'; -const DEFAULT_APP_ORIGIN_CANONICAL_CACHE_TTL_SECONDS = 300; -const DEFAULT_PRIVATE_APP_ASSET_TOKEN_TTL_SECONDS = 60 * 60; -const DEFAULT_PRIVATE_APP_ASSET_COOKIE_NAME = 'puter.private.asset.token'; -const DEFAULT_PUBLIC_HOSTED_ACTOR_TOKEN_TTL_SECONDS = 15 * 60; -const DEFAULT_PUBLIC_HOSTED_ACTOR_COOKIE_NAME = 'puter.public.hosted.actor.token'; - -const LegacyTokenError = class extends Error { -}; - -/** -* @class AuthService -* This class is responsible for handling authentication and authorization tasks for the application. -*/ -class AuthService extends BaseService { - - async _init () { - this.db = await this.services.get('database').get(DB_WRITE, 'auth'); - this.sessionService = await this.services.get('session'); - - const svc_feature_flag = await this.services.get('feature-flag'); - svc_feature_flag.register('temp-users-disabled', { - $: 'config-flag', - value: this.global_config.disable_temp_users ?? false, - }); - - svc_feature_flag.register('user-signup-disabled', { - $: 'config-flag', - value: this.global_config.disable_user_signup ?? false, - }); - - // "FPE" stands for "Format Preserving Encryption" - // The `uuid_fpe_key` is a key for creating encrypted alternatives - // to UUIDs and decrypting them back to the original UUIDs - // - // We do this to avoid exposing the internal UUID for sessions. - const uuid_fpe_key = this.config.uuid_fpe_key - ? UUIDFPE.uuidToBuffer(this.config.uuid_fpe_key) - : crypto.randomBytes(16); - this.uuid_fpe = new UUIDFPE(uuid_fpe_key); - - this.sessions = {}; - - this.tokenService = await this.services.get('token'); - - this.appOriginCanonicalizationLocalCacheNamespace = this.createAppOriginLocalCacheNamespace(); - - const eventService = await this.services.get('event'); - eventService.on('app.changed', async (_meta, event = {}) => { - await this.invalidateCanonicalAppUidCacheFromAppChangeEvent(event); - }); - } - - /** - * This method authenticates a user or app using a token. - * It checks the token's type (session, app-under-user, access-token) and decodes it. - * Depending on the token type, it returns the corresponding user/app actor. - * @param {string} token - The token to authenticate. - * @returns {Promise} The authenticated user or app actor. - */ - async authenticate_from_token (token) { - const decoded = this.tokenService.verify( - 'auth', - token, - ); - - if ( ! Object.prototype.hasOwnProperty.call(decoded, 'type') ) { - throw new LegacyTokenError(); - } - - if ( decoded.type === 'session' ) { - const session = await this.sessionService.getSession(decoded.uuid); - - if ( ! session ) { - throw APIError.create('token_auth_failed'); - } - - const user = await get_user({ uuid: decoded.user_uid }); - - if ( ! user ) { - throw APIError.create('user_not_found'); - } - - const actor_type = new UserActorType({ - user, - session: session.uuid, - hasHttpOnlyCookie: true, - }); - - return new Actor({ - user_uid: decoded.user_uid, - type: actor_type, - }); - } - - if ( decoded.type === 'gui' ) { - const session = await this.sessionService.getSession(decoded.uuid); - - if ( ! session ) { - throw APIError.create('token_auth_failed'); - } - - const user = await get_user({ uuid: decoded.user_uid }); - - if ( ! user ) { - throw APIError.create('user_not_found'); - } - - const actor_type = new UserActorType({ - user, - session: session.uuid, - hasHttpOnlyCookie: false, - }); - - return new Actor({ - user_uid: decoded.user_uid, - type: actor_type, - }); - } - - if ( decoded.type === 'app-under-user' ) { - let session; - if ( decoded.session ) { - const session_uuid = this.uuid_fpe.decrypt(decoded.session); - session = await this.sessionService.getSession(session_uuid); - - if ( ! session ) { - throw APIError.create('token_auth_failed'); - } - } - - const user = await get_user({ uuid: decoded.user_uid }); - if ( ! user ) { - throw APIError.create('token_auth_failed'); - } - - const app = await get_app({ uid: decoded.app_uid }); - if ( ! app ) { - throw APIError.create('token_auth_failed'); - } - - const actor_type = new AppUnderUserActorType({ - user, - app, - session, - }); - - return new Actor({ - user_uid: decoded.user_uid, - app_uid: decoded.app_uid, - type: actor_type, - }); - } - - if ( decoded.type === 'access-token' ) { - const token = decoded.token_uid; - if ( ! token ) { - throw APIError.create('token_auth_failed'); - } - - const user_uid = decoded.user_uid; - if ( ! user_uid ) { - throw APIError.create('token_auth_failed'); - } - - const app_uid = decoded.app_uid; - - const authorizer = ( user_uid && app_uid ) - ? await Actor.create(AppUnderUserActorType, { user_uid, app_uid }) - : await Actor.create(UserActorType, { user_uid }); - - const authorized = Context.get('actor'); - - const actor_type = new AccessTokenActorType({ - token, authorizer, authorized, - }); - - return new Actor({ - user_uid, - app_uid, - type: actor_type, - }); - } - - if ( decoded.type === 'actor-site' ) { - const site_uid = decoded.site_uid; - const svc_puterSite = this.services.get('puter-site'); - const site = - await svc_puterSite.get_subdomain_by_uid(site_uid); - return Actor.create(SiteActorType, { - site, - iat: decoded.iat, - }); - } - - throw APIError.create('token_auth_failed'); - } - - get_user_app_token (app_uid) { - const actor = Context.get('actor'); - const actor_type = actor.type; - - if ( ! (actor_type instanceof UserActorType) ) { - throw APIError.create('forbidden'); - } - - this.log.debug(`generating user-app token for app ${app_uid} and user ${actor_type.user.uuid}`, { - app_uid, - user_uid: actor_type.user.uuid, - }); - - const token = this.tokenService.sign( - 'auth', - { - type: 'app-under-user', - version: '0.0.0', - user_uid: actor_type.user.uuid, - app_uid, - ...(actor_type.session ? { session: this.uuid_fpe.encrypt(actor_type.session) } : {}), - }, - ); - - return token; - } - - get_site_app_token ({ site_uid }) { - const token = this.tokenService.sign( - 'auth', - { - type: 'actor-site', - version: '0.0.0', - site_uid, - }, - { expiresIn: '1h' }, - ); - - return token; - } - - resolvePositiveInteger (value, fallback) { - const parsed = Number(value); - if ( !Number.isFinite(parsed) || parsed <= 0 ) { - return fallback; - } - return Math.floor(parsed); - } - - getPrivateAssetTokenTtlSeconds () { - return this.resolvePositiveInteger( - this.global_config.private_app_asset_token_ttl_seconds, - DEFAULT_PRIVATE_APP_ASSET_TOKEN_TTL_SECONDS, - ); - } - - getPrivateAssetCookieName () { - const configuredCookieName = this.global_config.private_app_asset_cookie_name; - if ( typeof configuredCookieName === 'string' && configuredCookieName.trim() ) { - return configuredCookieName.trim(); - } - return DEFAULT_PRIVATE_APP_ASSET_COOKIE_NAME; - } - - getPublicHostedActorTokenTtlSeconds () { - return this.resolvePositiveInteger( - this.global_config.public_hosted_actor_token_ttl_seconds, - DEFAULT_PUBLIC_HOSTED_ACTOR_TOKEN_TTL_SECONDS, - ); - } - - getPublicHostedActorCookieName () { - const configuredCookieName = this.global_config.public_hosted_actor_cookie_name; - if ( typeof configuredCookieName === 'string' && configuredCookieName.trim() ) { - return configuredCookieName.trim(); - } - return DEFAULT_PUBLIC_HOSTED_ACTOR_COOKIE_NAME; - } - - normalizeHostnameForCookieDomain (hostnameValue) { - if ( typeof hostnameValue !== 'string' ) return null; - const trimmedHostname = hostnameValue.trim().toLowerCase().replace(/^\./, ''); - if ( ! trimmedHostname ) return null; - try { - return new URL(`http://${trimmedHostname}`).hostname.toLowerCase(); - } catch { - return trimmedHostname.split(':')[0] || null; - } - } - - isCookieDomainHostEligible (hostnameValue) { - if ( typeof hostnameValue !== 'string' || !hostnameValue ) return false; - if ( hostnameValue === 'localhost' ) return false; - if ( hostnameValue.includes(':') ) return false; - if ( ! hostnameValue.includes('.') ) return false; - if ( /^\d{1,3}(?:\.\d{1,3}){3}$/.test(hostnameValue) ) return false; - return true; - } - - getConfiguredPrivateCookieDomains () { - const configuredDomains = []; - for ( const configuredDomainCandidate of [ - this.global_config.private_app_hosting_domain, - this.global_config.private_app_hosting_domain_alt, - ] ) { - const normalizedDomain = this.normalizeHostnameForCookieDomain(configuredDomainCandidate); - if ( normalizedDomain ) { - configuredDomains.push(normalizedDomain); - } - } - return [...new Set(configuredDomains)]; - } - - getConfiguredHostedCookieDomains () { - const configuredDomains = []; - for ( const configuredDomainCandidate of [ - this.global_config.static_hosting_domain, - this.global_config.static_hosting_domain_alt, - this.global_config.private_app_hosting_domain, - this.global_config.private_app_hosting_domain_alt, - ] ) { - const normalizedDomain = this.normalizeHostnameForCookieDomain(configuredDomainCandidate); - if ( normalizedDomain ) { - configuredDomains.push(normalizedDomain); - } - } - return [...new Set(configuredDomains)]; - } - - resolvePrivateAssetCookieDomain ({ requestHostname } = {}) { - const configuredDomains = this.getConfiguredPrivateCookieDomains(); - const normalizedRequestHost = this.normalizeHostnameForCookieDomain(requestHostname); - - if ( normalizedRequestHost ) { - const matchedConfiguredDomain = configuredDomains - .sort((domainA, domainB) => domainB.length - domainA.length) - .find(configuredDomain => - normalizedRequestHost === configuredDomain || - normalizedRequestHost.endsWith(`.${configuredDomain}`)); - if ( this.isCookieDomainHostEligible(matchedConfiguredDomain) ) { - return `.${matchedConfiguredDomain}`; - } - return undefined; - } - - const normalizedConfiguredPrimaryDomain = this.normalizeHostnameForCookieDomain( - this.global_config.private_app_hosting_domain, - ); - if ( this.isCookieDomainHostEligible(normalizedConfiguredPrimaryDomain) ) { - return `.${normalizedConfiguredPrimaryDomain}`; - } - return undefined; - } - - getPrivateAssetCookieOptions ({ ttlSeconds, requestHostname } = {}) { - const effectiveTtlSeconds = this.resolvePositiveInteger( - ttlSeconds, - this.getPrivateAssetTokenTtlSeconds(), - ); - - const cookieOptions = { - sameSite: 'none', - secure: true, - httpOnly: true, - path: '/', - maxAge: effectiveTtlSeconds * 1000, - }; - - const cookieDomain = this.resolvePrivateAssetCookieDomain({ requestHostname }); - if ( cookieDomain ) { - cookieOptions.domain = cookieDomain; - } - - return cookieOptions; - } - - resolvePublicHostedActorCookieDomain ({ requestHostname } = {}) { - const configuredDomains = this.getConfiguredHostedCookieDomains(); - const normalizedRequestHost = this.normalizeHostnameForCookieDomain(requestHostname); - - if ( normalizedRequestHost ) { - const matchedConfiguredDomain = configuredDomains - .sort((domainA, domainB) => domainB.length - domainA.length) - .find(configuredDomain => - normalizedRequestHost === configuredDomain || - normalizedRequestHost.endsWith(`.${configuredDomain}`)); - if ( this.isCookieDomainHostEligible(matchedConfiguredDomain) ) { - return `.${matchedConfiguredDomain}`; - } - return undefined; - } - - const [firstConfiguredDomain] = configuredDomains; - if ( this.isCookieDomainHostEligible(firstConfiguredDomain) ) { - return `.${firstConfiguredDomain}`; - } - return undefined; - } - - getPublicHostedActorCookieOptions ({ ttlSeconds, requestHostname } = {}) { - const effectiveTtlSeconds = this.resolvePositiveInteger( - ttlSeconds, - this.getPublicHostedActorTokenTtlSeconds(), - ); - - const cookieOptions = { - sameSite: 'none', - secure: true, - httpOnly: true, - path: '/', - maxAge: effectiveTtlSeconds * 1000, - }; - - const cookieDomain = this.resolvePublicHostedActorCookieDomain({ - requestHostname, - }); - if ( cookieDomain ) { - cookieOptions.domain = cookieDomain; - } - - return cookieOptions; - } - - normalizePrivateAssetSubdomain (subdomain) { - if ( typeof subdomain !== 'string' ) return undefined; - const normalizedSubdomain = subdomain.trim().toLowerCase(); - return normalizedSubdomain || undefined; - } - - normalizePrivateAssetHost (privateHost) { - if ( typeof privateHost !== 'string' ) return undefined; - const normalizedPrivateHost = privateHost.trim().toLowerCase().replace(/^\./, ''); - if ( ! normalizedPrivateHost ) return undefined; - return normalizedPrivateHost; - } - - createPrivateAssetToken ({ appUid, userUid, sessionUuid, subdomain, privateHost, ttlSeconds } = {}) { - if ( typeof appUid !== 'string' || !appUid.trim() ) { - throw new Error('appUid is required to create private asset token.'); - } - if ( typeof userUid !== 'string' || !userUid.trim() ) { - throw new Error('userUid is required to create private asset token.'); - } - if ( sessionUuid !== undefined && (typeof sessionUuid !== 'string' || !sessionUuid.trim()) ) { - throw new Error('sessionUuid must be a non-empty string when provided.'); - } - const normalizedSubdomain = this.normalizePrivateAssetSubdomain(subdomain); - if ( subdomain !== undefined && !normalizedSubdomain ) { - throw new Error('subdomain must be a non-empty string when provided.'); - } - const normalizedPrivateHost = this.normalizePrivateAssetHost(privateHost); - if ( privateHost !== undefined && !normalizedPrivateHost ) { - throw new Error('privateHost must be a non-empty string when provided.'); - } - - const effectiveTtlSeconds = this.resolvePositiveInteger( - ttlSeconds, - this.getPrivateAssetTokenTtlSeconds(), - ); - - const payload = { - type: 'app-private-asset', - version: '0.0.0', - app_uid: appUid.trim(), - user_uid: userUid.trim(), - ...(sessionUuid ? { session: this.uuid_fpe.encrypt(sessionUuid) } : {}), - ...(normalizedSubdomain ? { subdomain: normalizedSubdomain } : {}), - ...(normalizedPrivateHost ? { private_host: normalizedPrivateHost } : {}), - }; - - return this.tokenService.sign('auth', payload, { - expiresIn: effectiveTtlSeconds, - }); - } - - createPublicHostedActorToken ({ appUid, userUid, sessionUuid, subdomain, host, ttlSeconds } = {}) { - if ( typeof appUid !== 'string' || !appUid.trim() ) { - throw new Error('appUid is required to create public hosted actor token.'); - } - if ( typeof userUid !== 'string' || !userUid.trim() ) { - throw new Error('userUid is required to create public hosted actor token.'); - } - if ( sessionUuid !== undefined && (typeof sessionUuid !== 'string' || !sessionUuid.trim()) ) { - throw new Error('sessionUuid must be a non-empty string when provided.'); - } - const normalizedSubdomain = this.normalizePrivateAssetSubdomain(subdomain); - if ( subdomain !== undefined && !normalizedSubdomain ) { - throw new Error('subdomain must be a non-empty string when provided.'); - } - const normalizedHost = this.normalizePrivateAssetHost(host); - if ( host !== undefined && !normalizedHost ) { - throw new Error('host must be a non-empty string when provided.'); - } - - const effectiveTtlSeconds = this.resolvePositiveInteger( - ttlSeconds, - this.getPublicHostedActorTokenTtlSeconds(), - ); - - const payload = { - type: 'app-public-hosted-actor', - version: '0.0.0', - app_uid: appUid.trim(), - user_uid: userUid.trim(), - ...(sessionUuid ? { session: this.uuid_fpe.encrypt(sessionUuid) } : {}), - ...(normalizedSubdomain ? { subdomain: normalizedSubdomain } : {}), - ...(normalizedHost ? { host: normalizedHost } : {}), - }; - - return this.tokenService.sign('auth', payload, { - expiresIn: effectiveTtlSeconds, - }); - } - - verifyPrivateAssetToken ( - token, - { expectedAppUid, expectedUserUid, expectedSessionUuid, expectedSubdomain, expectedPrivateHost } = {}, - ) { - let decoded; - try { - decoded = this.tokenService.verify('auth', token); - } catch (e) { - throw APIError.create('token_auth_failed'); - } - - if ( - !decoded || - decoded.type !== 'app-private-asset' || - typeof decoded.app_uid !== 'string' || - !decoded.app_uid || - typeof decoded.user_uid !== 'string' || - !decoded.user_uid - ) { - throw APIError.create('token_auth_failed'); - } - - let sessionUuid; - if ( decoded.session !== undefined ) { - if ( typeof decoded.session !== 'string' || !decoded.session ) { - throw APIError.create('token_auth_failed'); - } - try { - sessionUuid = this.uuid_fpe.decrypt(decoded.session); - } catch (e) { - throw APIError.create('token_auth_failed'); - } - } - - let subdomain; - if ( decoded.subdomain !== undefined ) { - if ( typeof decoded.subdomain !== 'string' || !decoded.subdomain.trim() ) { - throw APIError.create('token_auth_failed'); - } - subdomain = decoded.subdomain.trim().toLowerCase(); - } - let privateHost; - if ( decoded.private_host !== undefined ) { - if ( typeof decoded.private_host !== 'string' || !decoded.private_host.trim() ) { - throw APIError.create('token_auth_failed'); - } - privateHost = decoded.private_host.trim().toLowerCase(); - } - - if ( expectedAppUid && decoded.app_uid !== expectedAppUid ) { - throw APIError.create('token_auth_failed'); - } - if ( expectedUserUid && decoded.user_uid !== expectedUserUid ) { - throw APIError.create('token_auth_failed'); - } - if ( expectedSessionUuid ) { - if ( !sessionUuid || sessionUuid !== expectedSessionUuid ) { - throw APIError.create('token_auth_failed'); - } - } - const normalizedExpectedSubdomain = this.normalizePrivateAssetSubdomain(expectedSubdomain); - if ( expectedSubdomain !== undefined && !normalizedExpectedSubdomain ) { - throw APIError.create('token_auth_failed'); - } - if ( normalizedExpectedSubdomain ) { - if ( !subdomain || subdomain !== normalizedExpectedSubdomain ) { - throw APIError.create('token_auth_failed'); - } - } - const normalizedExpectedPrivateHost = this.normalizePrivateAssetHost(expectedPrivateHost); - if ( expectedPrivateHost !== undefined && !normalizedExpectedPrivateHost ) { - throw APIError.create('token_auth_failed'); - } - if ( normalizedExpectedPrivateHost ) { - if ( !privateHost || privateHost !== normalizedExpectedPrivateHost ) { - throw APIError.create('token_auth_failed'); - } - } - - return { - appUid: decoded.app_uid, - userUid: decoded.user_uid, - sessionUuid, - subdomain, - privateHost, - exp: decoded.exp, - iat: decoded.iat, - }; - } - - verifyPublicHostedActorToken ( - token, - { expectedAppUid, expectedUserUid, expectedSessionUuid, expectedSubdomain, expectedHost } = {}, - ) { - let decoded; - try { - decoded = this.tokenService.verify('auth', token); - } catch (e) { - throw APIError.create('token_auth_failed'); - } - - if ( - !decoded || - decoded.type !== 'app-public-hosted-actor' || - typeof decoded.app_uid !== 'string' || - !decoded.app_uid || - typeof decoded.user_uid !== 'string' || - !decoded.user_uid - ) { - throw APIError.create('token_auth_failed'); - } - - let sessionUuid; - if ( decoded.session !== undefined ) { - if ( typeof decoded.session !== 'string' || !decoded.session ) { - throw APIError.create('token_auth_failed'); - } - try { - sessionUuid = this.uuid_fpe.decrypt(decoded.session); - } catch (e) { - throw APIError.create('token_auth_failed'); - } - } - - let subdomain; - if ( decoded.subdomain !== undefined ) { - if ( typeof decoded.subdomain !== 'string' || !decoded.subdomain.trim() ) { - throw APIError.create('token_auth_failed'); - } - subdomain = decoded.subdomain.trim().toLowerCase(); - } - - let host; - if ( decoded.host !== undefined ) { - if ( typeof decoded.host !== 'string' || !decoded.host.trim() ) { - throw APIError.create('token_auth_failed'); - } - host = decoded.host.trim().toLowerCase(); - } - - if ( expectedAppUid && decoded.app_uid !== expectedAppUid ) { - throw APIError.create('token_auth_failed'); - } - if ( expectedUserUid && decoded.user_uid !== expectedUserUid ) { - throw APIError.create('token_auth_failed'); - } - if ( expectedSessionUuid ) { - if ( !sessionUuid || sessionUuid !== expectedSessionUuid ) { - throw APIError.create('token_auth_failed'); - } - } - - const normalizedExpectedSubdomain = this.normalizePrivateAssetSubdomain(expectedSubdomain); - if ( expectedSubdomain !== undefined && !normalizedExpectedSubdomain ) { - throw APIError.create('token_auth_failed'); - } - if ( normalizedExpectedSubdomain ) { - if ( !subdomain || subdomain !== normalizedExpectedSubdomain ) { - throw APIError.create('token_auth_failed'); - } - } - - const normalizedExpectedHost = this.normalizePrivateAssetHost(expectedHost); - if ( expectedHost !== undefined && !normalizedExpectedHost ) { - throw APIError.create('token_auth_failed'); - } - if ( normalizedExpectedHost ) { - if ( !host || host !== normalizedExpectedHost ) { - throw APIError.create('token_auth_failed'); - } - } - - return { - appUid: decoded.app_uid, - userUid: decoded.user_uid, - sessionUuid, - subdomain, - host, - exp: decoded.exp, - iat: decoded.iat, - }; - } - - resolvePrivateBootstrapSessionUuid (decoded) { - if ( !decoded || typeof decoded !== 'object' ) { - return null; - } - - if ( decoded.type === 'session' || decoded.type === 'gui' ) { - if ( typeof decoded.uuid !== 'string' || !decoded.uuid ) { - return null; - } - return decoded.uuid; - } - - if ( decoded.type === 'app-under-user' ) { - if ( typeof decoded.session !== 'string' || !decoded.session ) { - return null; - } - try { - return this.uuid_fpe.decrypt(decoded.session); - } catch (e) { - return null; - } - } - - return null; - } - - async resolvePrivateBootstrapIdentityFromToken (token, { expectedAppUid, expectedAppUids } = {}) { - let decoded; - try { - decoded = this.tokenService.verify('auth', token); - } catch (e) { - throw new Error('Token decode error'); - } - - const userUid = typeof decoded?.user_uid === 'string' - ? decoded.user_uid - : null; - if ( ! userUid ) { - throw new Error('Token missing uuid'); - } - - const allowedTypes = new Set(['session', 'gui', 'app-under-user']); - if ( ! allowedTypes.has(decoded.type) ) { - throw new Error(`Token wrong type: ${ decoded.type}`); - } - const bootstrapAppUid = typeof decoded?.app_uid === 'string' - ? decoded.app_uid - : null; - const expectedAppUidCandidates = new Set(); - if ( typeof expectedAppUid === 'string' && expectedAppUid ) { - expectedAppUidCandidates.add(expectedAppUid); - } - if ( Array.isArray(expectedAppUids) ) { - for ( const appUidCandidate of expectedAppUids ) { - if ( typeof appUidCandidate === 'string' && appUidCandidate ) { - expectedAppUidCandidates.add(appUidCandidate); - } - } - } - if ( - bootstrapAppUid - && expectedAppUidCandidates.size > 0 - && !expectedAppUidCandidates.has(bootstrapAppUid) - ) { - throw new Error(`Token app uuid: ${ bootstrapAppUid } doesn't match expected appUuid candidates: ${ JSON.stringify(expectedAppUidCandidates)}`); - } - - const sessionUuid = this.resolvePrivateBootstrapSessionUuid(decoded); - if ( ! sessionUuid ) { - throw new Error('Token missing sessionUuid'); - } - - const session = await this.sessionService.getSession(sessionUuid); - if ( ! session ) { - throw new Error('Token missing session'); - } - - const sessionUserUid = typeof session.user_uid === 'string' - ? session.user_uid - : null; - if ( !sessionUserUid || sessionUserUid !== userUid ) { - throw new Error('Token mismatch userId'); - } - - return { - userUid, - sessionUuid: session.uuid || sessionUuid, - }; - } - - /** - * Internal method for creating a session. - * - * If a request object is provided in the metadata, it will be used to - * extract information about the requestor and include it in the - * session's metadata. - */ - async create_session_ (user, meta = {}) { - this.log.debug('CREATING SESSION'); - - if ( meta.req ) { - const req = meta.req; - delete meta.req; - - const ip = this.global_config.fowarded - ? req.headers['x-forwarded-for'] || - req.connection.remoteAddress - : req.connection.remoteAddress - ; - - meta.ip = ip; - - meta.server = this.global_config.server_id; - - if ( req.headers['user-agent'] ) { - meta.user_agent = req.headers['user-agent']; - } - - if ( req.headers['referer'] ) { - meta.referer = req.headers['referer']; - } - - if ( req.headers['origin'] ) { - const origin = this._origin_from_url(req.headers['origin']); - if ( origin ) { - meta.origin = origin; - } - } - - if ( req.headers['host'] ) { - const host = this._origin_from_url(req.headers['host']); - if ( host ) { - meta.host = host; - } - } - } - - return await this.sessionService.create_session(user, meta); - } - - /** - * Creates a session token using TokenService's sign method - * with type 'session' using a newly created session for the - * specified user. - * @param {*} user - * @param {*} meta - * @returns - */ - async create_session_token (user, meta) { - const session = await this.create_session_(user, meta); - - const token = this.tokenService.sign('auth', { - type: 'session', - version: '0.0.0', - uuid: session.uuid, - // meta: session.meta, - user_uid: user.uuid, - }); - - return { session, token }; - } - - /** - * Creates a GUI token bound to the same session as the given session object. - * GUI tokens create a UserActorType with hasHttpOnlyCookie false, so they cannot - * access user-protected HTTP endpoints (e.g. change password). The GUI receives - * only this token, not the full session token. - * - * @param {*} user - User object (must have .uuid). - * @param {{ uuid: string }} session - Session object (must have .uuid). - * @returns {string} JWT GUI token. - */ - create_gui_token (user, session) { - return this.tokenService.sign('auth', { - type: 'gui', - version: '0.0.0', - uuid: session.uuid, - user_uid: user.uuid, - }); - } - - /** - * Creates a session token (hasHttpOnlyCookie) for an existing session. - * Used when the client authenticated with a GUI token (e.g. QR login via - * ?auth_token=) so we can set the HTTP-only cookie and allow user-protected - * endpoints (change password, email, username, etc.) to work. - * - * @param {*} user - User object (must have .uuid). - * @param {string} session_uuid - Existing session UUID. - * @returns {string} JWT session token. - */ - create_session_token_for_session (user, session_uuid) { - return this.tokenService.sign('auth', { - type: 'session', - version: '0.0.0', - uuid: session_uuid, - user_uid: user.uuid, - }); - } - - /** - * This method checks if the provided session token is valid and returns the associated user and token. - * If the token is not a valid session token or it does not exist in the database, it returns an empty object. - * - * @param {string} cur_token - The session token to be checked. - * @param {object} meta - Additional metadata associated with the token. - * @returns {object} Object containing the user and token if the token is valid, otherwise an empty object. - */ - async check_session (cur_token, meta) { - const decoded = this.tokenService.verify('auth', cur_token); - - console.debug('\x1B[36;1mDECODED SESSION', decoded); - - if ( decoded.type && decoded.type !== 'session' && decoded.type !== 'gui' ) { - return {}; - } - - const is_legacy = !decoded.type; - - const user = await get_user({ uuid: - is_legacy ? decoded.uuid : decoded.user_uid, - }); - if ( ! user ) { - return {}; - } - - if ( ! is_legacy ) { - // Ensure session exists - const session = await this.sessionService.getSession(decoded.uuid); - if ( ! session ) { - return {}; - } - - // Return GUI token to client (if they sent session token, exchange for GUI token) - const gui_token = decoded.type === 'gui' - ? cur_token - : this.create_gui_token(user, session); - return { user, token: gui_token }; - } - - this.log.info('UPGRADING SESSION'); - - // Upgrade legacy token - // TODO: phase this out - const { session, token: session_token } = await this.create_session_token(user, meta); - const gui_token = this.create_gui_token(user, session); - - const actor_type = new UserActorType({ - user, - session, - hasHttpOnlyCookie: true, - }); - - const actor = new Actor({ - user_uid: user.uuid, - type: actor_type, - }); - - // token = GUI token for client (response body); session_token = for HTTP-only cookie - return { actor, user, token: gui_token, session_token }; - } - - /** - * Removes a session with the specified token - * - * @param {string} token - The token to be authenticated. - * @returns {Promise} - */ - async remove_session_by_token (token) { - const decoded = this.tokenService.verify('auth', token); - - if ( decoded.type !== 'session' && decoded.type !== 'gui' ) { - return; - } - - await this.sessionService.remove_session(decoded.uuid); - } - - /** - * This method is used to create an access token for a user or an application. - * - * Access tokens aren't currently used by any of Puter's features. - * The feature is kept here for future-use. - * - * @param {1} authorizer - The actor that is creating the access token. - * @param {*} permissions - The permissions to be granted to the access token. - * @returns - */ - async create_access_token (authorizer, permissions, options) { - const jwt_obj = {}; - const authorizer_obj = {}; - if ( authorizer.type instanceof UserActorType ) { - Object.assign(authorizer_obj, { - authorizer_user_id: authorizer.type.user.id, - }); - const user = await get_user({ id: authorizer.type.user.id }); - jwt_obj.user_uid = user.uuid; - } - else if ( authorizer.type instanceof AppUnderUserActorType ) { - Object.assign(authorizer_obj, { - authorizer_user_id: authorizer.type.user.id, - authorizer_app_id: authorizer.type.app.id, - }); - const user = await get_user({ id: authorizer.type.user.id }); - jwt_obj.user_uid = user.uuid; - const app = await get_app({ id: authorizer.type.app.id }); - jwt_obj.app_uid = app.uid; - } - else { - throw APIError.create('forbidden'); - } - - const uuid = uuidLib.v4(); - - const jwt = this.tokenService.sign('auth', { - type: 'access-token', - version: '0.0.0', - token_uid: uuid, - ...jwt_obj, - }, options); - - for ( const permmission_spec of permissions ) { - let [permission, extra] = permmission_spec; - - const svc_permission = await Context.get('services').get('permission'); - permission = await svc_permission._rewrite_permission(permission); - - const insert_object = { - token_uid: uuid, - ...authorizer_obj, - permission, - extra: JSON.stringify(extra ?? {}), - }; - const cols = Object.keys(insert_object).join(', '); - const vals = Object.values(insert_object).map(() => '?').join(', '); - await this.db.write( - 'INSERT INTO `access_token_permissions` ' + - `(${cols}) VALUES (${vals})`, - Object.values(insert_object), - ); - } - - console.log('token uuid?', uuid); - - return jwt; - } - - /** - * Revokes an access token by removing it from the database. - * Accepts either the access token JWT or the token UUID. - * - * @param {string} tokenOrUuid - The access token JWT or the token UUID. - * @returns {Promise} - */ - async revoke_access_token (tokenOrUuid) { - let token_uid; - const isJwt = typeof tokenOrUuid === 'string' && - /^[\w-]*\.[\w-]*\.[\w-]*$/.test(tokenOrUuid.trim()); - if ( isJwt ) { - const decoded = this.tokenService.verify('auth', tokenOrUuid); - if ( decoded.type !== 'access-token' || !decoded.token_uid ) { - throw APIError.create('token_auth_failed'); - } - token_uid = decoded.token_uid; - } else { - token_uid = tokenOrUuid; - } - - await this.db.write( - 'DELETE FROM `access_token_permissions` WHERE `token_uid` = ?', - [token_uid], - ); - } - - /** - * Get the session list for the specified actor. - * - * This is primarily used by the `/list-sessions` API endpoint - * for the Session Manager in Puter's settings window. - * - * @param {*} actor - The actor for which to list sessions. - * @returns {Promise} - A list of sessions for the actor. - */ - async list_sessions (actor) { - const seen = new Set(); - const sessions = []; - - const cache_sessions = await this.sessionService.get_user_sessions(actor.type.user); - for ( const session of cache_sessions ) { - seen.add(session.uuid); - sessions.push(session); - } - - // We won't take the cached sessions here because it's - // possible the user has sessions on other servers - const db_sessions = await this.db.read( - 'SELECT uuid, meta FROM `sessions` WHERE `user_id` = ?', - [actor.type.user.id], - ); - - for ( const session of db_sessions ) { - if ( seen.has(session.uuid) ) { - continue; - } - - if ( !session.meta || typeof (session.meta) === 'string' ) { - session.meta = JSON.parse(session.meta || '{}'); - } - sessions.push(session); - }; - - for ( const session of sessions ) { - if ( session.uuid === actor.type.session ) { - session.current = true; - } - } - - return sessions; - } - - /** - * Revokes a session by UUID. The actor is ignored but should be provided - * for future use. - * - * @param {*} actor - * @param {*} uuid - */ - async revoke_session (actor, uuid) { - delete this.sessions[uuid]; - this.sessionService.remove_session(uuid); - } - - /** - * This method is used to create or obtain a user-app token deterministically - * from an origin at which puter.js might be embedded. - * - * @param {*} origin - The origin URL at which puter.js is embedded. - * @returns - */ - async get_user_app_token_from_origin (origin) { - origin = this._origin_from_url(origin); - if ( origin === null ) { - throw APIError.create('no_origin_for_app'); - } - - const canonicalAppUid = await this.resolveCanonicalAppUidFromOrigin(origin); - const app_uid = canonicalAppUid ?? await this._app_uid_from_origin(origin); - - // Determine if the app exists - const apps = await this.db.read( - 'SELECT * FROM `apps` WHERE `uid` = ? LIMIT 1', - [app_uid], - ); - - if ( apps[0] ) { - return this.get_user_app_token(app_uid); - } - - this.log.info(`creating app ${app_uid} from origin ${origin}`); - - const name = app_uid; - const title = app_uid; - const description = `App created from origin ${origin}`; - const index_url = origin; - const owner_user_id = null; - - // Create the app - await this.db.write( - 'INSERT INTO `apps` ' + - '(`uid`, `name`, `title`, `description`, `index_url`, `owner_user_id`) ' + - 'VALUES (?, ?, ?, ?, ?, ?)', - [app_uid, name, title, description, index_url, owner_user_id], - ); - - await this.invalidateCanonicalAppUidCacheForOrigins([origin]); - - return this.get_user_app_token(app_uid); - } - - /** - * Generates a deterministic app uuid from an origin - * - * @param {*} origin - * @returns - */ - async app_uid_from_origin (origin) { - origin = this._origin_from_url(origin); - if ( origin === null ) { - throw APIError.create('no_origin_for_app'); - } - const canonicalAppUid = await this.resolveCanonicalAppUidFromOrigin(origin); - if ( canonicalAppUid ) { - return canonicalAppUid; - } - return await this._app_uid_from_origin(origin); - } - - getAppOriginCanonicalCacheTtlSeconds () { - return this.resolvePositiveInteger( - this.global_config.app_origin_canonical_cache_ttl_seconds, - DEFAULT_APP_ORIGIN_CANONICAL_CACHE_TTL_SECONDS, - ); - } - - buildAppOriginCanonicalCacheKey ({ origin }) { - const encodedOrigin = encodeURIComponent(origin); - return `${APP_ORIGIN_CACHE_KEY_PREFIX}:${encodedOrigin}`; - } - - createAppOriginLocalCacheNamespace () { - return `${APP_ORIGIN_LOCAL_CACHE_KEY_PREFIX}:${uuidLib.v4()}`; - } - - getAppOriginLocalCacheNamespace () { - if ( - typeof this.appOriginCanonicalizationLocalCacheNamespace !== 'string' - || !this.appOriginCanonicalizationLocalCacheNamespace - ) { - this.appOriginCanonicalizationLocalCacheNamespace = this.createAppOriginLocalCacheNamespace(); - } - return this.appOriginCanonicalizationLocalCacheNamespace; - } - - buildLocalCanonicalAppUidCacheKey (origin) { - const encodedOrigin = encodeURIComponent(origin); - return `${this.getAppOriginLocalCacheNamespace()}:${encodedOrigin}`; - } - - readLocalCanonicalAppUidFromCache (origin) { - const localCacheKey = this.buildLocalCanonicalAppUidCacheKey(origin); - const cachedResolution = kv.get(localCacheKey); - if ( !cachedResolution || typeof cachedResolution !== 'object' ) { - return undefined; - } - if ( ! Object.prototype.hasOwnProperty.call(cachedResolution, 'appUid') ) { - return undefined; - } - return cachedResolution.appUid; - } - - writeLocalCanonicalAppUidToCache (origin, appUid) { - const ttlSeconds = this.getAppOriginCanonicalCacheTtlSeconds(); - const localCacheKey = this.buildLocalCanonicalAppUidCacheKey(origin); - kv.set(localCacheKey, { - appUid: appUid ?? null, - }, { EX: ttlSeconds }); - } - - async readCanonicalAppUidFromRedisCache (origin) { - const cacheKey = this.buildAppOriginCanonicalCacheKey({ - origin, - }); - - try { - const cachedPayload = await redisClient.get(cacheKey); - if ( typeof cachedPayload !== 'string' || cachedPayload === '' ) { - return undefined; - } - - const parsedPayload = JSON.parse(cachedPayload); - if ( !parsedPayload || typeof parsedPayload !== 'object' ) { - return undefined; - } - if ( ! Object.prototype.hasOwnProperty.call(parsedPayload, 'appUid') ) { - return undefined; - } - return parsedPayload.appUid ?? null; - } catch { - return undefined; - } - } - - async writeCanonicalAppUidToRedisCache (origin, appUid) { - const cacheKey = this.buildAppOriginCanonicalCacheKey({ - origin, - }); - - await setRedisCacheValue( - cacheKey, - JSON.stringify({ appUid: appUid ?? null }), - { ttlSeconds: this.getAppOriginCanonicalCacheTtlSeconds() }, - ); - } - - async resolveCanonicalAppUidFromOrigin (origin) { - const normalizedOrigin = this._origin_from_url(origin); - if ( normalizedOrigin === null ) return null; - - const isFirstPartyHostedOrigin = this.isHostedOriginOnConfiguredDomain(normalizedOrigin); - const canonicalOrigin = this.canonicalizeHostedAppOriginForUid(normalizedOrigin); - const localCachedAppUid = this.readLocalCanonicalAppUidFromCache(canonicalOrigin); - if ( localCachedAppUid !== undefined ) { - return localCachedAppUid; - } - - const redisCachedAppUid = await this.readCanonicalAppUidFromRedisCache(canonicalOrigin); - if ( redisCachedAppUid !== undefined ) { - this.writeLocalCanonicalAppUidToCache(canonicalOrigin, redisCachedAppUid); - return redisCachedAppUid; - } - - const canonicalAppUid = await this.lookupCanonicalAppUidFromOrigin(canonicalOrigin, { - allowHostedOwnerlessLookup: isFirstPartyHostedOrigin, - restrictToOriginHost: isFirstPartyHostedOrigin, - }); - this.writeLocalCanonicalAppUidToCache(canonicalOrigin, canonicalAppUid); - try { - await this.writeCanonicalAppUidToRedisCache(canonicalOrigin, canonicalAppUid); - } catch { - // Redis cache writes are best-effort. - } - return canonicalAppUid; - } - - normalizeOriginForCanonicalAppUidCache (originCandidate) { - const normalizedOrigin = this._origin_from_url(originCandidate); - if ( normalizedOrigin === null ) return null; - return this.canonicalizeHostedAppOriginForUid(normalizedOrigin); - } - - collectCanonicalCacheOriginsFromAppChangeEvent (event = {}) { - const originCandidates = []; - if ( event?.app?.index_url ) { - originCandidates.push(event.app.index_url); - } - if ( event?.old_app?.index_url ) { - originCandidates.push(event.old_app.index_url); - } - if ( event?.index_url ) { - originCandidates.push(event.index_url); - } - if ( event?.old_index_url ) { - originCandidates.push(event.old_index_url); - } - - const canonicalOrigins = new Set(); - for ( const originCandidate of originCandidates ) { - const normalizedCanonicalOrigin = this.normalizeOriginForCanonicalAppUidCache(originCandidate); - if ( normalizedCanonicalOrigin ) { - canonicalOrigins.add(normalizedCanonicalOrigin); - } - } - - return [...canonicalOrigins]; - } - - async invalidateCanonicalAppUidCacheForOrigins (originCandidates = []) { - const canonicalOrigins = new Set(); - for ( const originCandidate of originCandidates ) { - const normalizedCanonicalOrigin = this.normalizeOriginForCanonicalAppUidCache(originCandidate); - if ( normalizedCanonicalOrigin ) { - canonicalOrigins.add(normalizedCanonicalOrigin); - } - } - - if ( canonicalOrigins.size === 0 ) return; - - const localCacheKeys = []; - const redisCacheKeys = []; - for ( const canonicalOrigin of canonicalOrigins ) { - localCacheKeys.push(this.buildLocalCanonicalAppUidCacheKey(canonicalOrigin)); - redisCacheKeys.push(this.buildAppOriginCanonicalCacheKey({ origin: canonicalOrigin })); - } - - if ( localCacheKeys.length > 0 ) { - kv.del(...localCacheKeys); - } - if ( redisCacheKeys.length > 0 ) { - try { - await deleteRedisKeys(redisCacheKeys); - } catch { - // best-effort invalidation; cache TTL bounds stale reads. - } - } - } - - async invalidateCanonicalAppUidCacheFromAppChangeEvent (event = {}) { - const canonicalOrigins = this.collectCanonicalCacheOriginsFromAppChangeEvent(event); - await this.invalidateCanonicalAppUidCacheForOrigins(canonicalOrigins); - } - - buildIndexUrlCandidatesFromOrigin (origin, options = {}) { - const includeHostedAliases = options?.includeHostedAliases !== false; - try { - const parsedOrigin = new URL(origin); - const hostCandidates = new Set(); - hostCandidates.add(parsedOrigin.host.toLowerCase()); - - const hostedSubdomain = this.extractHostedAppSubdomainFromHostname(parsedOrigin.hostname); - if ( hostedSubdomain && includeHostedAliases ) { - const hostedDomainCandidates = this.getHostedAppDomainCandidatesForMatch(); - for ( const hostedDomainCandidate of hostedDomainCandidates ) { - if ( hostedDomainCandidate?.host ) { - hostCandidates.add(`${hostedSubdomain}.${hostedDomainCandidate.host}`); - } - } - } - - const indexUrlCandidates = []; - for ( const hostCandidate of hostCandidates ) { - const baseUrl = `${parsedOrigin.protocol}//${hostCandidate}`; - indexUrlCandidates.push(baseUrl); - indexUrlCandidates.push(`${baseUrl}/`); - indexUrlCandidates.push(`${baseUrl}/index.html`); - } - - return [...new Set(indexUrlCandidates)]; - } catch { - return []; - } - } - - async getHostedSubdomainOwnerUserId (subdomain) { - if ( typeof subdomain !== 'string' || !subdomain ) return null; - try { - const databaseService = this.services.get('database'); - const dbReadSites = databaseService.get(DB_READ, 'sites'); - const rows = await dbReadSites.read( - 'SELECT user_id FROM subdomains WHERE subdomain = ? LIMIT 1', - [subdomain], - ); - const ownerUserId = Number(rows?.[0]?.user_id); - if ( Number.isInteger(ownerUserId) && ownerUserId > 0 ) { - return ownerUserId; - } - return null; - } catch { - return null; - } - } - - isOriginBootstrapAppRow (appRow) { - if ( !appRow || typeof appRow !== 'object' ) return false; - const appUid = typeof appRow.uid === 'string' ? appRow.uid : ''; - if ( ! appUid ) return false; - if ( appRow.name !== appUid ) return false; - if ( appRow.title !== appUid ) return false; - const appDescription = typeof appRow.description === 'string' - ? appRow.description - : ''; - return appDescription.startsWith('App created from origin '); - } - - async queryCanonicalAppUidForIndexUrlCandidates ({ - indexUrlCandidates, - ownerUserId, - preferNonBootstrap = false, - }) { - if ( !Array.isArray(indexUrlCandidates) || indexUrlCandidates.length === 0 ) { - return null; - } - - const placeholders = indexUrlCandidates.map(() => '?').join(', '); - const parameters = []; - let whereClause = `index_url IN (${placeholders})`; - parameters.push(...indexUrlCandidates); - - if ( Number.isInteger(ownerUserId) && ownerUserId > 0 ) { - whereClause = `owner_user_id = ? AND ${whereClause}`; - parameters.unshift(ownerUserId); - } - - try { - const dbReadApps = this.services.get('database').get(DB_READ, 'apps'); - const rows = await dbReadApps.read( - `SELECT uid, name, title, description - FROM apps - WHERE ${whereClause} - ORDER BY timestamp ASC, id ASC`, - parameters, - ); - - const oldestAppUid = rows?.[0]?.uid; - if ( typeof oldestAppUid !== 'string' || !oldestAppUid ) { - return null; - } - - if ( ! preferNonBootstrap ) return oldestAppUid; - - const preferredAppRow = rows.find(appRow => !this.isOriginBootstrapAppRow(appRow)); - const preferredAppUid = preferredAppRow?.uid; - if ( typeof preferredAppUid === 'string' && preferredAppUid ) { - return preferredAppUid; - } - - return oldestAppUid; - } catch { - return null; - } - } - - isHostedOriginOnConfiguredDomain (origin) { - try { - const parsedOrigin = new URL(origin); - const hostedSubdomain = this.extractHostedAppSubdomainFromHostname(parsedOrigin.hostname); - if ( ! hostedSubdomain ) return false; - - const firstPartyHostedDomain = this.normalizeHostedDomainCandidate(this.global_config.domain); - if ( ! firstPartyHostedDomain?.hostname ) return false; - - const normalizedOriginHostname = parsedOrigin.hostname.trim().toLowerCase(); - return ( - normalizedOriginHostname === firstPartyHostedDomain.hostname || - normalizedOriginHostname.endsWith(`.${firstPartyHostedDomain.hostname}`) - ); - } catch { - return false; - } - } - - async lookupCanonicalAppUidFromOrigin (origin, options = {}) { - const allowHostedOwnerlessLookup = options?.allowHostedOwnerlessLookup === true; - const restrictToOriginHost = options?.restrictToOriginHost === true; - const indexUrlCandidates = this.buildIndexUrlCandidatesFromOrigin(origin, { - includeHostedAliases: !restrictToOriginHost, - }); - if ( indexUrlCandidates.length === 0 ) return null; - - try { - const parsedOrigin = new URL(origin); - const hostedSubdomain = this.extractHostedAppSubdomainFromHostname(parsedOrigin.hostname); - - if ( hostedSubdomain ) { - if ( allowHostedOwnerlessLookup || this.isHostedOriginOnConfiguredDomain(origin) ) { - return await this.queryCanonicalAppUidForIndexUrlCandidates({ - indexUrlCandidates, - preferNonBootstrap: true, - }); - } - - const hostedSubdomainOwnerUserId = await this.getHostedSubdomainOwnerUserId(hostedSubdomain); - if ( ! hostedSubdomainOwnerUserId ) { - return null; - } - return await this.queryCanonicalAppUidForIndexUrlCandidates({ - ownerUserId: hostedSubdomainOwnerUserId, - indexUrlCandidates, - preferNonBootstrap: true, - }); - } - - return await this.queryCanonicalAppUidForIndexUrlCandidates({ - indexUrlCandidates, - }); - } catch { - return null; - } - } - - normalizeHostedDomainCandidate (domainValue) { - if ( typeof domainValue !== 'string' ) return null; - - const normalizedDomainValue = domainValue.trim().toLowerCase().replace(/^\./, ''); - if ( ! normalizedDomainValue ) return null; - - try { - const parsedDomain = new URL(`http://${normalizedDomainValue}`); - return { - host: parsedDomain.host.toLowerCase(), - hostname: parsedDomain.hostname.toLowerCase(), - }; - } catch { - const [hostname] = normalizedDomainValue.split(':'); - if ( ! hostname ) return null; - return { - host: normalizedDomainValue, - hostname, - }; - } - } - - getHostedAppDomainCandidatesForMatch () { - const hostedDomainCandidates = []; - const seenHostnames = new Set(); - - for ( const domainCandidate of [ - this.global_config.static_hosting_domain, - this.global_config.static_hosting_domain_alt, - this.global_config.private_app_hosting_domain, - this.global_config.private_app_hosting_domain_alt, - this.global_config.domain, - ] ) { - const normalizedDomainCandidate = this.normalizeHostedDomainCandidate(domainCandidate); - if ( ! normalizedDomainCandidate ) continue; - if ( seenHostnames.has(normalizedDomainCandidate.hostname) ) continue; - seenHostnames.add(normalizedDomainCandidate.hostname); - hostedDomainCandidates.push(normalizedDomainCandidate); - } - - return hostedDomainCandidates; - } - - getCanonicalHostedAppDomain () { - for ( const domainCandidate of [ - this.global_config.static_hosting_domain, - this.global_config.static_hosting_domain_alt, - this.global_config.private_app_hosting_domain, - this.global_config.private_app_hosting_domain_alt, - ] ) { - const normalizedDomainCandidate = this.normalizeHostedDomainCandidate(domainCandidate); - if ( normalizedDomainCandidate?.host ) { - return normalizedDomainCandidate.host; - } - } - return null; - } - - extractHostedAppSubdomainFromHostname (hostname) { - if ( typeof hostname !== 'string' ) return null; - const normalizedHostname = hostname.trim().toLowerCase(); - if ( ! normalizedHostname ) return null; - - const hostedDomainCandidates = this.getHostedAppDomainCandidatesForMatch() - .sort((domainCandidateA, domainCandidateB) => - domainCandidateB.hostname.length - domainCandidateA.hostname.length); - - for ( const hostedDomainCandidate of hostedDomainCandidates ) { - if ( normalizedHostname === hostedDomainCandidate.hostname ) { - return null; - } - const hostedDomainSuffix = `.${hostedDomainCandidate.hostname}`; - if ( normalizedHostname.endsWith(hostedDomainSuffix) ) { - const subdomain = normalizedHostname.slice( - 0, - normalizedHostname.length - hostedDomainSuffix.length, - ); - return subdomain || null; - } - } - - return null; - } - - canonicalizeHostedAppOriginForUid (origin) { - try { - if ( this.isHostedOriginOnConfiguredDomain(origin) ) return origin; - - const parsedOrigin = new URL(origin); - const hostedSubdomain = this.extractHostedAppSubdomainFromHostname(parsedOrigin.hostname); - if ( ! hostedSubdomain ) return origin; - - const canonicalHostedDomain = this.getCanonicalHostedAppDomain(); - if ( ! canonicalHostedDomain ) return origin; - - return `${parsedOrigin.protocol}//${hostedSubdomain}.${canonicalHostedDomain}`; - } catch { - return origin; - } - } - - async _app_uid_from_origin (origin) { - const canonicalOrigin = this.canonicalizeHostedAppOriginForUid(origin); - const event = { origin: canonicalOrigin }; - const eventService = this.services.get('event'); - await eventService.emit('app.from-origin', event); - // UUIDV5 - const uuid = uuidLib.v5(event.origin, APP_ORIGIN_UUID_NAMESPACE); - return `app-${uuid}`; - } - - _origin_from_url ( url ) { - try { - const parsedUrl = new URL(url); - // Origin is protocol + hostname + port - return `${parsedUrl.protocol}//${parsedUrl.hostname}${parsedUrl.port ? `:${parsedUrl.port}` : ''}`; - } catch ( error ) { - console.error('Invalid URL:', error.message); - return null; - } - } - - /** - * Registers GET /get-gui-token. Must be called from the GUI origin (no api. subdomain) - * so the HTTP-only session cookie is sent. Returns the GUI token for use in Authorization headers. - */ - '__on_install.routes' () { - const { app } = this.services.get('web-server'); - const eggspress = require('../../api/eggspress'); - const config = require('../../config'); - const configurable_auth = require('../../middleware/configurable_auth'); - const svc_auth = this; - - app.use(eggspress('/get-gui-token', { - allowedMethods: ['GET'], - mw: [configurable_auth()], - }, async (req, res) => { - if ( ! req.user ) { - return res.status(401).json({}); - } - - const actor = Context.get('actor'); - if ( ! (actor.type instanceof UserActorType) ) { - return res.status(403).json({}); - } - if ( ! actor.type.session ) { - return res.status(400).json({ error: 'No session bound to this actor' }); - } - - const gui_token = svc_auth.create_gui_token(actor.type.user, { uuid: actor.type.session }); - return res.json({ token: gui_token }); - })); - - // Sync HTTP-only session cookie to the user implied by the request's auth token. - // Used when switching users in the UI: client sends Authorization with the new user's - // GUI token; we set the session cookie so cookie-based (e.g. user-protected) requests match. - app.use(eggspress('/session/sync-cookie', { - allowedMethods: ['GET'], - mw: [configurable_auth()], - }, async (req, res) => { - if ( ! req.user ) { - return res.status(401).end(); - } - const actor = Context.get('actor'); - if ( !(actor.type instanceof UserActorType) || !actor.type.session ) { - return res.status(400).end(); - } - const session_token = svc_auth.create_session_token_for_session( - actor.type.user, - actor.type.session, - ); - res.cookie(config.cookie_name, session_token, { - sameSite: 'none', - secure: true, - httpOnly: true, - }); - return res.status(204).end(); - })); - } -} - -module.exports = { - AuthService, - LegacyTokenError, -}; diff --git a/src/backend/src/services/auth/AuthService.privateAssetToken.test.ts b/src/backend/src/services/auth/AuthService.privateAssetToken.test.ts deleted file mode 100644 index aa60433af..000000000 --- a/src/backend/src/services/auth/AuthService.privateAssetToken.test.ts +++ /dev/null @@ -1,588 +0,0 @@ -import { randomUUID } from 'node:crypto'; -import { describe, expect, it } from 'vitest'; -import { createTestKernel } from '../../../tools/test.mjs'; -import { tmp_provide_services } from '../../helpers.js'; -import * as jwt from 'jsonwebtoken'; -import { AuthService } from './AuthService.js'; - -type AuthServiceForPrivateTokenTests = AuthService & { - global_config: { - jwt_secret: string; - private_app_asset_token_ttl_seconds: number; - private_app_asset_cookie_name: string; - app_origin_canonical_cache_ttl_seconds?: number; - public_hosted_actor_token_ttl_seconds?: number; - public_hosted_actor_cookie_name?: string; - static_hosting_domain: string; - static_hosting_domain_alt?: string; - private_app_hosting_domain: string; - private_app_hosting_domain_alt?: string; - domain?: string; - }; - tokenService: { - sign: (scope: string, payload: unknown, options?: jwt.SignOptions) => string; - verify: (scope: string, token: string) => jwt.JwtPayload & Record; - }; - uuid_fpe: { - encrypt: (value: string) => string; - decrypt: (value: string) => string; - }; - services: { - get: (name: string) => unknown; - }; - sessionService: { - getSession: (uuid: string) => Promise<{ uuid: string; user_uid?: string } | undefined>; - create_session: (user: { id: number; uuid: string }, meta?: Record) => Promise<{ uuid: string }>; - }; - appOriginCanonicalizationLocalCacheNamespace?: string; -}; - -const testKernel = await createTestKernel({ - initLevelString: 'init', - testCore: true, - serviceConfigOverrideMap: { - database: { - path: ':memory:', - }, - }, -}); -await tmp_provide_services(testKernel.services); - -const authService = testKernel.services.get('auth') as AuthServiceForPrivateTokenTests; -const db = testKernel.services.get('database').get('write', 'auth-private-asset-test'); - -const applyDefaultAuthConfig = () => { - authService.global_config.jwt_secret = 'private-asset-test-secret'; - authService.global_config.private_app_asset_token_ttl_seconds = 3600; - authService.global_config.private_app_asset_cookie_name = 'puter.private.asset.token'; - authService.global_config.app_origin_canonical_cache_ttl_seconds = 300; - authService.global_config.public_hosted_actor_token_ttl_seconds = 900; - authService.global_config.public_hosted_actor_cookie_name = 'puter.public.hosted.actor.token'; - authService.global_config.static_hosting_domain = 'puter.site'; - authService.global_config.static_hosting_domain_alt = 'puter.host'; - authService.global_config.private_app_hosting_domain = 'app.puter.localhost'; - authService.global_config.private_app_hosting_domain_alt = 'puter.dev'; - authService.global_config.domain = 'puter.com'; - (authService.tokenService as { secret: string }).secret = authService.global_config.jwt_secret; - authService.appOriginCanonicalizationLocalCacheNamespace = - authService.createAppOriginLocalCacheNamespace(); -}; - -const createAuthService = (): AuthServiceForPrivateTokenTests => { - applyDefaultAuthConfig(); - return authService; -}; - -const insertUser = async () => { - const userUuid = randomUUID(); - const username = `u_${Math.random().toString(36).slice(2, 10)}`; - await db.write( - 'INSERT INTO `user` (`uuid`, `username`) VALUES (?, ?)', - [userUuid, username], - ); - const [user] = await db.read( - 'SELECT * FROM `user` WHERE `uuid` = ? LIMIT 1', - [userUuid], - ); - return user as { id: number; uuid: string; username: string }; -}; - -const insertApp = async ({ - uid, - name, - title, - indexUrl, - ownerUserId = null, -}: { - uid: string; - name: string; - title: string; - indexUrl: string; - ownerUserId?: number | null; -}) => { - await db.write( - 'INSERT INTO `apps` (`uid`, `name`, `title`, `description`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?, ?)', - [uid, name, title, `desc-${name}`, indexUrl, ownerUserId], - ); -}; - -const tamperTokenSignature = (token: string): string => { - const parts = token.split('.'); - if ( parts.length !== 3 ) return `${token}x`; - const signature = parts[2]; - if ( signature.length === 0 ) { - parts[2] = 'x'; - return parts.join('.'); - } - const lastChar = signature[signature.length - 1]; - const replacement = lastChar === 'a' ? 'b' : 'a'; - parts[2] = `${signature.slice(0, -1)}${replacement}`; - return parts.join('.'); -}; - -describe('AuthService private asset token helpers', () => { - it('creates and verifies private asset tokens with expected claims', () => { - const authService = createAuthService(); - const appUid = 'app-7e2d3016-8d36-456a-9dc7-b75b0f4f7683'; - const userUid = '4b0cecf8-dd6a-4eb5-bcc4-c76cc7e8d7f0'; - const sessionUuid = 'f9000804-2fd3-4da5-819b-afc5296f90f7'; - const subdomain = 'beans'; - const privateHost = 'beans.puter.dev'; - - const token = authService.createPrivateAssetToken({ - appUid, - userUid, - sessionUuid, - subdomain, - privateHost, - ttlSeconds: 120, - }); - - const claims = authService.verifyPrivateAssetToken(token, { - expectedAppUid: appUid, - expectedUserUid: userUid, - expectedSessionUuid: sessionUuid, - expectedSubdomain: subdomain, - expectedPrivateHost: privateHost, - }); - - expect(claims.appUid).toBe(appUid); - expect(claims.userUid).toBe(userUid); - expect(claims.sessionUuid).toBe(sessionUuid); - expect(claims.subdomain).toBe(subdomain); - expect(claims.privateHost).toBe(privateHost); - expect(typeof claims.exp).toBe('number'); - }); - - it('rejects tokens when expected user or app does not match', () => { - const authService = createAuthService(); - const token = authService.createPrivateAssetToken({ - appUid: 'app-9f1c10e3-9a7f-43fb-8671-af4918e65407', - userUid: '9885b80e-1a14-4c8d-9e3f-4fa5915b1136', - subdomain: 'beans', - privateHost: 'beans.puter.dev', - }); - - expect(() => authService.verifyPrivateAssetToken(token, { - expectedAppUid: 'app-aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', - })).toThrow(); - - expect(() => authService.verifyPrivateAssetToken(token, { - expectedUserUid: 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', - })).toThrow(); - - expect(() => authService.verifyPrivateAssetToken(token, { - expectedSubdomain: 'other-app', - })).toThrow(); - - expect(() => authService.verifyPrivateAssetToken(token, { - expectedPrivateHost: 'other.puter.dev', - })).toThrow(); - }); - - it('rejects non private-asset tokens', () => { - const authService = createAuthService(); - const token = jwt.sign({ - type: 'session', - uuid: '245f33f0-c07e-40e2-be22-5215752e3462', - user_uid: '6cce4692-3855-4ef8-af7d-5c2a02e6b6d8', - }, authService.global_config.jwt_secret, { expiresIn: 60 }); - - expect(() => authService.verifyPrivateAssetToken(token)).toThrow(); - }); - - it('rejects private asset tokens with tampered signatures', () => { - const authService = createAuthService(); - const token = authService.createPrivateAssetToken({ - appUid: 'app-9f1c10e3-9a7f-43fb-8671-af4918e65407', - userUid: '9885b80e-1a14-4c8d-9e3f-4fa5915b1136', - }); - const tampered = tamperTokenSignature(token); - - expect(() => authService.verifyPrivateAssetToken(tampered)).toThrow(); - }); - - it('returns hardened cookie options with config-driven ttl and domain', () => { - const authService = createAuthService(); - const options = authService.getPrivateAssetCookieOptions(); - - expect(authService.getPrivateAssetCookieName()).toBe('puter.private.asset.token'); - expect(options.sameSite).toBe('none'); - expect(options.secure).toBe(true); - expect(options.httpOnly).toBe(true); - expect(options.path).toBe('/'); - expect(options.maxAge).toBe(3_600_000); - expect(options.domain).toBe('.app.puter.localhost'); - }); - - it('creates and verifies public hosted actor tokens with expected claims', () => { - const authService = createAuthService(); - const appUid = 'app-d18f4a26-1e9a-4e9d-89dd-d3476f9efab4'; - const userUid = '1a8600ea-25a7-4ac6-95be-3a9f84e95f17'; - const sessionUuid = 'f6bb30b0-f9d8-4bd6-94ea-0bfcf48e1ba8'; - const subdomain = 'beans'; - const host = 'beans.puter.dev'; - - const token = authService.createPublicHostedActorToken({ - appUid, - userUid, - sessionUuid, - subdomain, - host, - ttlSeconds: 180, - }); - - const claims = authService.verifyPublicHostedActorToken(token, { - expectedAppUid: appUid, - expectedUserUid: userUid, - expectedSessionUuid: sessionUuid, - expectedSubdomain: subdomain, - expectedHost: host, - }); - - expect(claims.appUid).toBe(appUid); - expect(claims.userUid).toBe(userUid); - expect(claims.sessionUuid).toBe(sessionUuid); - expect(claims.subdomain).toBe(subdomain); - expect(claims.host).toBe(host); - expect(typeof claims.exp).toBe('number'); - }); - - it('returns public hosted actor cookie options with matched hosted domain', () => { - const authService = createAuthService(); - authService.global_config.static_hosting_domain = 'site.puter.localhost'; - authService.global_config.static_hosting_domain_alt = 'site.puter.dev'; - authService.global_config.private_app_hosting_domain = 'app.puter.localhost'; - authService.global_config.private_app_hosting_domain_alt = 'puter.dev'; - authService.global_config.public_hosted_actor_token_ttl_seconds = 1200; - authService.global_config.public_hosted_actor_cookie_name = 'puter.public.hosted.actor'; - - const options = authService.getPublicHostedActorCookieOptions({ - requestHostname: 'beans.puter.dev', - }); - - expect(authService.getPublicHostedActorCookieName()).toBe('puter.public.hosted.actor'); - expect(options.sameSite).toBe('none'); - expect(options.secure).toBe(true); - expect(options.httpOnly).toBe(true); - expect(options.path).toBe('/'); - expect(options.maxAge).toBe(1_200_000); - expect(options.domain).toBe('.puter.dev'); - }); - - it('uses the matched request host private domain when provided', () => { - const authService = createAuthService(); - authService.global_config.private_app_hosting_domain = 'app.puter.localhost'; - authService.global_config.private_app_hosting_domain_alt = 'puter.dev'; - - const options = authService.getPrivateAssetCookieOptions({ - requestHostname: 'beans.puter.dev', - }); - - expect(options.domain).toBe('.puter.dev'); - }); - - it('omits domain when request host does not match configured private domains', () => { - const authService = createAuthService(); - authService.global_config.private_app_hosting_domain = 'puter.app'; - authService.global_config.private_app_hosting_domain_alt = 'puter.app'; - - const options = authService.getPrivateAssetCookieOptions({ - requestHostname: 'beans.puter.dev', - }); - - expect(options.domain).toBeUndefined(); - }); - - it('resolves bootstrap identity from app-under-user token without app lookup', async () => { - const authService = createAuthService(); - const user = await insertUser(); - const session = await authService.sessionService.create_session(user, {}); - const token = authService.tokenService.sign('auth', { - type: 'app-under-user', - version: '0.0.0', - user_uid: user.uuid, - app_uid: 'app-7e2d3016-8d36-456a-9dc7-b75b0f4f7683', - session: authService.uuid_fpe.encrypt(session.uuid), - }, { expiresIn: 60 }); - - const identity = await authService.resolvePrivateBootstrapIdentityFromToken(token); - - expect(identity).toEqual({ - userUid: user.uuid, - sessionUuid: session.uuid, - }); - }); - - it('rejects bootstrap identity when session owner does not match token user', async () => { - const authService = createAuthService(); - const claimedUser = await insertUser(); - const actualSessionOwner = await insertUser(); - const actualSession = await authService.sessionService.create_session(actualSessionOwner, {}); - const token = authService.tokenService.sign('auth', { - type: 'app-under-user', - version: '0.0.0', - user_uid: claimedUser.uuid, - app_uid: 'app-7e2d3016-8d36-456a-9dc7-b75b0f4f7683', - session: authService.uuid_fpe.encrypt(actualSession.uuid), - }, { expiresIn: 60 }); - - await expect(authService.resolvePrivateBootstrapIdentityFromToken(token)) - .rejects - .toThrow(); - }); - - it('rejects bootstrap identity when expected app uid does not match token app uid', async () => { - const authService = createAuthService(); - const user = await insertUser(); - const session = await authService.sessionService.create_session(user, {}); - const token = authService.tokenService.sign('auth', { - type: 'app-under-user', - version: '0.0.0', - user_uid: user.uuid, - app_uid: 'app-7e2d3016-8d36-456a-9dc7-b75b0f4f7683', - session: authService.uuid_fpe.encrypt(session.uuid), - }, { expiresIn: 60 }); - - await expect(authService.resolvePrivateBootstrapIdentityFromToken(token, { - expectedAppUid: 'app-aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', - })) - .rejects - .toThrow(); - }); - - it('accepts bootstrap identity when expected app uid candidates include token app uid', async () => { - const authService = createAuthService(); - const user = await insertUser(); - const session = await authService.sessionService.create_session(user, {}); - const appUid = 'app-7e2d3016-8d36-456a-9dc7-b75b0f4f7683'; - const token = authService.tokenService.sign('auth', { - type: 'app-under-user', - version: '0.0.0', - user_uid: user.uuid, - app_uid: appUid, - session: authService.uuid_fpe.encrypt(session.uuid), - }, { expiresIn: 60 }); - - const identity = await authService.resolvePrivateBootstrapIdentityFromToken(token, { - expectedAppUids: ['app-aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', appUid], - }); - - expect(identity).toEqual({ - userUid: user.uuid, - sessionUuid: session.uuid, - }); - }); - - it('rejects bootstrap identity token when signature is tampered', async () => { - const authService = createAuthService(); - const user = await insertUser(); - const session = await authService.sessionService.create_session(user, {}); - const token = authService.tokenService.sign('auth', { - type: 'app-under-user', - version: '0.0.0', - user_uid: user.uuid, - app_uid: 'app-7e2d3016-8d36-456a-9dc7-b75b0f4f7683', - session: authService.uuid_fpe.encrypt(session.uuid), - }, { expiresIn: 60 }); - const tampered = tamperTokenSignature(token); - - await expect(authService.resolvePrivateBootstrapIdentityFromToken(tampered)) - .rejects - .toThrow(); - }); - - it('prefers oldest owner-matched app for hosted subdomain origins', async () => { - const authService = createAuthService(); - const owner = await insertUser(); - const otherOwner = await insertUser(); - const subdomain = `beans${Math.random().toString(36).slice(2, 9)}`; - - await db.write( - 'INSERT INTO `subdomains` (`uuid`, `subdomain`, `user_id`) VALUES (?, ?, ?)', - [randomUUID(), subdomain, owner.id], - ); - - await insertApp({ - uid: 'app-oldest-owner-match', - name: `oldest-owner-${subdomain}`, - title: `oldest-owner-${subdomain}`, - indexUrl: `https://${subdomain}.puter.dev/`, - ownerUserId: owner.id, - }); - await insertApp({ - uid: 'app-newer-owner-match', - name: `newer-owner-${subdomain}`, - title: `newer-owner-${subdomain}`, - indexUrl: `https://${subdomain}.puter.dev/index.html`, - ownerUserId: owner.id, - }); - await insertApp({ - uid: `app-other-owner-${randomUUID()}`, - name: `other-owner-${subdomain}`, - title: `other-owner-${subdomain}`, - indexUrl: `https://${subdomain}.puter.dev/`, - ownerUserId: otherOwner.id, - }); - - const appUid = await authService.app_uid_from_origin(`https://${subdomain}.puter.dev`); - - expect(appUid).toBe('app-oldest-owner-match'); - }); - - it('prefers non-bootstrap app for hosted subdomain origins when both exist', async () => { - const authService = createAuthService(); - authService.global_config.domain = 'puter.com'; - const owner = await insertUser(); - const subdomain = `music${Math.random().toString(36).slice(2, 9)}`; - const bootstrapUid = `app-bootstrap-${randomUUID()}`; - const canonicalUid = `app-canonical-${randomUUID()}`; - - await db.write( - 'INSERT INTO `subdomains` (`uuid`, `subdomain`, `user_id`) VALUES (?, ?, ?)', - [randomUUID(), subdomain, owner.id], - ); - - await db.write( - 'INSERT INTO `apps` (`uid`, `name`, `title`, `description`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?, ?)', - [ - bootstrapUid, - bootstrapUid, - bootstrapUid, - `App created from origin https://${subdomain}.puter.com`, - `https://${subdomain}.puter.com`, - owner.id, - ], - ); - await insertApp({ - uid: canonicalUid, - name: `music-player-${subdomain}`, - title: `music-player-${subdomain}`, - indexUrl: `https://${subdomain}.puter.com/index.html`, - ownerUserId: owner.id, - }); - - const appUid = await authService.app_uid_from_origin(`https://${subdomain}.puter.com`); - - expect(appUid).toBe(canonicalUid); - }); - - it('prefers canonical first-party app for hosted subdomain without subdomain owner record', async () => { - const authService = createAuthService(); - authService.global_config.domain = 'puter.com'; - const owner = await insertUser(); - const subdomain = `music${Math.random().toString(36).slice(2, 9)}`; - const bootstrapUid = `app-bootstrap-${randomUUID()}`; - const canonicalUid = `app-canonical-${randomUUID()}`; - - await db.write( - 'INSERT INTO `apps` (`uid`, `name`, `title`, `description`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?, ?)', - [ - bootstrapUid, - bootstrapUid, - bootstrapUid, - `App created from origin https://${subdomain}.puter.com`, - `https://${subdomain}.puter.com`, - null, - ], - ); - await insertApp({ - uid: canonicalUid, - name: `music-player-${subdomain}`, - title: `music-player-${subdomain}`, - indexUrl: `https://${subdomain}.puter.com/`, - ownerUserId: owner.id, - }); - - const appUid = await authService.app_uid_from_origin(`https://${subdomain}.puter.com`); - - expect(appUid).toBe(canonicalUid); - }); - - it('falls back to deterministic origin uid when hosted subdomain owner cannot be resolved', async () => { - const authService = createAuthService(); - const subdomain = `beans${Math.random().toString(36).slice(2, 9)}`; - const uidFromPrivateAlias = await authService.app_uid_from_origin(`https://${subdomain}.puter.dev`); - const uidFromStaticAlias = await authService.app_uid_from_origin(`https://${subdomain}.puter.site`); - - expect(uidFromPrivateAlias).toBe(uidFromStaticAlias); - expect(uidFromPrivateAlias.startsWith('app-')).toBe(true); - }); - - it('prefers oldest app for non-hosted origins', async () => { - const authService = createAuthService(); - const host = `${Math.random().toString(36).slice(2, 10)}.example.com`; - const origin = `https://${host}`; - - await insertApp({ - uid: 'app-oldest-external', - name: `oldest-external-${host}`, - title: `oldest-external-${host}`, - indexUrl: `${origin}/`, - }); - await insertApp({ - uid: 'app-newer-external', - name: `newer-external-${host}`, - title: `newer-external-${host}`, - indexUrl: `${origin}/index.html`, - }); - - const appUid = await authService.app_uid_from_origin(origin); - expect(appUid).toBe('app-oldest-external'); - }); - - it('collects canonical cache origins from app change payloads', () => { - const authService = createAuthService(); - authService.global_config.static_hosting_domain = 'puter.site'; - authService.global_config.static_hosting_domain_alt = 'puter.host'; - authService.global_config.private_app_hosting_domain = 'puter.app'; - authService.global_config.private_app_hosting_domain_alt = 'puter.dev'; - - const canonicalOrigins = authService.collectCanonicalCacheOriginsFromAppChangeEvent({ - app: { - index_url: 'https://beans.puter.dev/index.html', - }, - old_app: { - index_url: 'https://beans.puter.site/', - }, - old_index_url: 'https://example.com', - }); - - expect(canonicalOrigins).toContain('https://beans.puter.site'); - expect(canonicalOrigins).toContain('https://example.com'); - expect(canonicalOrigins.filter(origin => origin === 'https://beans.puter.site')).toHaveLength(1); - }); - - it('keeps puter.com distinct while deriving same uid for non-domain hosted aliases', async () => { - const authService = createAuthService(); - authService.global_config.static_hosting_domain = 'puter.site'; - authService.global_config.static_hosting_domain_alt = 'puter.host'; - authService.global_config.private_app_hosting_domain = 'puter.app'; - authService.global_config.private_app_hosting_domain_alt = 'puter.dev'; - authService.global_config.domain = 'puter.com'; - - const uidSite = await authService.app_uid_from_origin('https://beans.puter.site'); - const uidStaticAlt = await authService.app_uid_from_origin('https://beans.puter.host'); - const uidPrivatePrimary = await authService.app_uid_from_origin('https://beans.puter.app'); - const uidPrivateAlt = await authService.app_uid_from_origin('https://beans.puter.dev'); - const uidMainDomain = await authService.app_uid_from_origin('https://beans.puter.com'); - - expect(uidSite).toBe(uidStaticAlt); - expect(uidSite).toBe(uidPrivatePrimary); - expect(uidSite).toBe(uidPrivateAlt); - expect(uidMainDomain).not.toBe(uidSite); - }); - - it('keeps distinct app uid per subdomain under hosted alias canonicalization', async () => { - const authService = createAuthService(); - authService.global_config.static_hosting_domain = 'puter.site'; - authService.global_config.static_hosting_domain_alt = 'puter.host'; - authService.global_config.private_app_hosting_domain = 'puter.app'; - authService.global_config.private_app_hosting_domain_alt = 'puter.dev'; - - const uidBeans = await authService.app_uid_from_origin('https://beans.puter.dev'); - const uidCats = await authService.app_uid_from_origin('https://cats.puter.site'); - - expect(uidBeans).not.toBe(uidCats); - }); -}); diff --git a/src/backend/src/services/auth/GroupRedisCacheSpace.js b/src/backend/src/services/auth/GroupRedisCacheSpace.js deleted file mode 100644 index 26c4debec..000000000 --- a/src/backend/src/services/auth/GroupRedisCacheSpace.js +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const GroupRedisCacheSpace = { - publicGroupsKey: kvKey => `${kvKey}:public-groups`, -}; - -export { GroupRedisCacheSpace }; diff --git a/src/backend/src/services/auth/GroupService.js b/src/backend/src/services/auth/GroupService.js deleted file mode 100644 index a39c1fb3e..000000000 --- a/src/backend/src/services/auth/GroupService.js +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const { redisClient } = require('../../clients/redis/redisSingleton'); -const { setRedisCacheValue } = require('../../clients/redis/cacheUpdate.js'); -const { GroupRedisCacheSpace } = require('./GroupRedisCacheSpace.js'); -const BaseService = require('../BaseService'); -const { DB_WRITE } = require('../database/consts'); -const { v4: uuidv4 } = require('uuid'); - -const create_group_entity = (svc_group, values) => ({ - values, - async fetch_members () { - if ( Object.prototype.hasOwnProperty.call(this.values, 'members') ) { - return this.values.members; - } - - const members = await svc_group.list_members({ uid: this.values.uid }); - this.values.members = members; - return members; - }, - async get_client_value (options = {}) { - if ( options.members ) { - await this.fetch_members(); - } - - return { - uid: this.values.uid, - metadata: this.values.metadata, - ...(options.members ? { members: this.values.members } : {}), - }; - }, -}); -/** -* The GroupService class provides functionality for managing groups within the Puter application. -* It extends the BaseService to handle group-related operations such as creation, retrieval, -* listing members, adding or removing users from groups, and more. This service interacts with -* the database to perform CRUD operations on group entities, ensuring proper management -* of user permissions and group metadata. -*/ -class GroupService extends BaseService { - - /** - * Initializes the GroupService by setting up the database connection and registering - * - * @memberof GroupService - * @instance - */ - _init () { - this.db = this.services.get('database').get(DB_WRITE, 'permissions'); - this.kvkey = uuidv4(); - } - - /** - * Retrieves a group by its unique identifier (UID). - * - * @param {Object} params - The parameters object. - * @param {string} params.uid - The unique identifier of the group. - * @returns {Promise} The group object if found, otherwise undefined. - * @throws {Error} If there's an issue with the database query. - * - * This method fetches a group from the database using its UID. If the group - * does not exist, it returns undefined. The 'extra' and 'metadata' fields are - * parsed from JSON strings to objects if not using MySQL, otherwise they remain - * as strings. - */ - async get ({ uid }) { - const [group] = - await this.db.read('SELECT * FROM `group` WHERE uid=?', [uid]); - if ( ! group ) return; - - if ( !group.extra || typeof (group.extra) === 'string' ) { - group.extra = JSON.parse(group.extra || '{}'); - } - if ( !group.metadata || typeof (group.metadata) === 'string' ) { - group.metadata = JSON.parse(group.metadata || '{}'); - } - return group; - } - - /** - * Creates a new group with the provided owner, extra data, and metadata. - * This method performs rate limiting checks to prevent abuse, generates a unique identifier for the group, - * and handles the database insertion of the group details. - * - * @param {Object} options - The options object for creating a group. - * @param {string} options.owner_user_id - The ID of the user who owns the group. - * @param {Object} [options.extra] - Additional data associated with the group. - * @param {Object} [options.metadata] - Metadata for the group, which can be used for various purposes. - * @returns {Promise} - A promise that resolves to the unique identifier of the newly created group. - * @throws {APIError} If the rate limit is exceeded. - */ - async create ({ owner_user_id, extra, metadata }) { - extra = extra ?? {}; - metadata = metadata ?? {}; - - const uid = uuidv4(); - - const [{ n_groups }] = await this.db.read( - 'SELECT COUNT(*) AS n_groups FROM `group` WHERE ' + - `owner_user_id=? AND created_at >= ${ - this.db.case({ - sqlite: "datetime('now', '-1 hour')", - otherwise: 'NOW() - INTERVAL 1 HOUR', - })}`, - [owner_user_id], - ); - - if ( Number(n_groups) > 20 ) { - throw APIError.create('too_many_requests'); - } - - await this.db.write( - 'INSERT INTO `group` ' + - '(`uid`, `owner_user_id`, `extra`, `metadata`) ' + - 'VALUES (?, ?, ?, ?)', - [ - uid, owner_user_id, - JSON.stringify(extra), - JSON.stringify(metadata), - ], - ); - - return uid; - } - - /** - * Lists all groups where the specified user is a member. - * - * This method queries the database to find groups associated with the given user_id through the junction table `jct_user_group`. - * Each group's `extra` and `metadata` fields are parsed based on the database type to ensure compatibility. - * - * @param {Object} params - Parameters for the query. - * @param {string} params.user_id - The ID of the user whose groups are to be listed. - * @returns {Promise>} A promise that resolves to an array of Group objects representing groups the user is a member of. - */ - async list_groups_with_owner ({ owner_user_id }) { - const groups = await this.db.read( - 'SELECT * FROM `group` WHERE owner_user_id=?', - [owner_user_id], - ); - for ( const group of groups ) { - if ( !group.extra || typeof (group.extra) === 'string' ) { - group.extra = JSON.parse(group.extra || '{}'); - } - if ( !group.metadata || typeof (group.metadata) === 'string' ) { - group.metadata = JSON.parse(group.metadata || '{}'); - } - } - return groups.map(g => create_group_entity(this, g)); - } - - /** - * Lists all groups where the specified user is a member. - * - * @param {Object} options - The options object. - * @param {string} options.user_id - The ID of the user whose group memberships are to be listed. - * @returns {Promise} A promise that resolves to an array of Group objects representing the groups the user is a member of. - */ - async list_groups_with_member ({ user_id }) { - const groups = await this.db.read( - 'SELECT * FROM `group` WHERE id IN (' + - 'SELECT group_id FROM `jct_user_group` WHERE user_id=?)', - [user_id], - ); - for ( const group of groups ) { - if ( !group.extra || typeof (group.extra) === 'string' ) { - group.extra = JSON.parse(group.extra || '{}'); - } - if ( !group.metadata || typeof (group.metadata) === 'string' ) { - group.metadata = JSON.parse(group.metadata || '{}'); - } - } - return groups.map(g => create_group_entity(this, g)); - } - - /** - * Lists public groups. May get groups from kv.js cache. - */ - async list_public_groups () { - const public_group_uids = [ - this.global_config.default_user_group, - this.global_config.default_temp_group, - ]; - - const cacheKey = GroupRedisCacheSpace.publicGroupsKey(this.kvkey); - const cached_groups = await redisClient.get(cacheKey); - if ( cached_groups ) { - try { - return JSON.parse(cached_groups).map(g => create_group_entity(this, g)); - } catch (e) { - // no op cache is in an invalid state - } - } - - let groups = await this.db.read( - `SELECT * FROM \`group\` WHERE uid IN (${ - public_group_uids.map(() => '?').join(', ') - })`, - public_group_uids, - ); - for ( const group of groups ) { - if ( !group.metadata || typeof (group.metadata) === 'string' ) { - group.metadata = JSON.parse(group.metadata || '{}'); - } - if ( !group.extra || typeof (group.extra) === 'string' ) { - group.extra = JSON.parse(group.extra || '{}'); - } - } - const group_entities = groups.map(g => create_group_entity(this, g)); - await setRedisCacheValue(cacheKey, JSON.stringify(groups), { - ttlSeconds: 60, - eventData: groups, - }); - return group_entities; - } - - /** - * Lists the members of a group by their username. - * - * @param {Object} options - The options object. - * @param {string} options.uid - The unique identifier of the group. - * @returns {Promise} A promise that resolves to an array of usernames of the group members. - */ - async list_members ({ uid }) { - const users = await this.db.read( - 'SELECT u.username FROM user u ' + - 'JOIN (SELECT user_id FROM `jct_user_group` WHERE group_id = ' + - '(SELECT id FROM `group` WHERE uid=?)) ug ' + - 'ON u.id = ug.user_id', - [uid], - ); - return users.map(u => u.username); - } - - /** - * Adds specified users to a group. - * - * @param {Object} options - The options object. - * @param {string} options.uid - The unique identifier of the group. - * @param {string[]} options.users - An array of usernames to add to the group. - * @returns {Promise} A promise that resolves when the users have been added. - * @throws {APIError} If there's an issue with the database operation or if the group does not exist. - */ - async add_users ({ uid, users }) { - const question_marks = - `(${ Array(users.length).fill('?').join(', ') })`; - await this.db.write( - 'INSERT INTO `jct_user_group` ' + - '(user_id, group_id) ' + - 'SELECT u.id, g.id FROM user u ' + - 'JOIN (SELECT id FROM `group` WHERE uid=?) g ON 1=1 ' + - `WHERE u.username IN ${ - question_marks}`, - [uid, ...users], - ); - } - - /** - * Removes specified users from a group. - * - * This method deletes the association between users and a group from the junction table. - * It uses the group's uid to identify the group and an array of usernames to remove. - * - * @param {Object} params - The parameters for the operation. - * @param {string} params.uid - The unique identifier of the group. - * @param {string[]} params.users - An array of usernames to be removed from the group. - * @returns {Promise} A promise that resolves when the operation is complete. - */ - async remove_users ({ uid, users }) { - const question_marks = - `(${ Array(users.length).fill('?').join(', ') })`; - /* -DELETE FROM `jct_user_group` -WHERE group_id = 1 -AND user_id IN ( - SELECT u.id - FROM user u - WHERE u.username IN ('user_that_shares', 'user_that_gets_shared_to') -); - */ - await this.db.write( - 'DELETE FROM `jct_user_group` ' + - 'WHERE group_id = (SELECT id FROM `group` WHERE uid=?) ' + - 'AND user_id IN (' + - 'SELECT u.id FROM user u ' + - `WHERE u.username IN ${ - question_marks - })`, - [uid, ...users], - ); - } -} - -module.exports = { - GroupService, -}; diff --git a/src/backend/src/services/auth/OIDCService.js b/src/backend/src/services/auth/OIDCService.js deleted file mode 100644 index c903deba0..000000000 --- a/src/backend/src/services/auth/OIDCService.js +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -'use strict'; -import jwt from 'jsonwebtoken'; -import { username_exists } from '../../helpers.js'; -import { generate_identifier } from '../../util/identifier.js'; -import { OutcomeObject } from '../../util/outcomeutil.js'; -import BaseService from '../BaseService.js'; -import { DB_WRITE } from '../database/consts.js'; -import { CreatedUserOutcome } from './SignupService.js'; - -const GOOGLE_DISCOVERY_URL = 'https://accounts.google.com/.well-known/openid-configuration'; -const GOOGLE_SCOPES = 'openid email profile'; -const STATE_EXPIRY_SEC = 600; // 10 minutes - -const VALID_OIDC_FLOWS = ['login', 'signup', 'revalidate']; - -async function generate_random_username () { - let username; - do { - username = generate_identifier(); - } while ( await username_exists(username) ); - return username; -} - -/** - * OIDC/OAuth2 service for sign-in with Google (and extensible to other providers). - * Uses config.oidc.providers only; no environment variables. - */ -export class OIDCService extends BaseService { - #googleDiscovery; - - async _init () { - this.db = await this.services.get('database').get(DB_WRITE, 'auth'); - this.providers = this.config.providers ?? {}; - this.#googleDiscovery = null; - } - - /** - * Get provider config from config.oidc.providers. For Google, resolve endpoints from discovery. - * @param {string} providerId - e.g. 'google' - * @returns {Promise} Config with client_id, client_secret, authorization_endpoint, token_endpoint, userinfo_endpoint, scopes - */ - async getProviderConfig (providerId) { - const providers = this.providers; - const raw = providers[providerId]; - if ( !raw || typeof raw !== 'object' || !raw.client_id || !raw.client_secret ) { - return null; - } - if ( providerId === 'google' ) { - const discovery = await this.#getGoogleDiscovery(); - if ( ! discovery ) return null; - return { - client_id: raw.client_id, - client_secret: raw.client_secret, - authorization_endpoint: discovery.authorization_endpoint, - token_endpoint: discovery.token_endpoint, - userinfo_endpoint: discovery.userinfo_endpoint, - scopes: raw.scopes ?? GOOGLE_SCOPES, - }; - } - if ( raw.authorization_endpoint && raw.token_endpoint && raw.userinfo_endpoint ) { - return { - ...raw, - scopes: raw.scopes ?? 'openid email profile', - }; - } - return null; - } - - async #getGoogleDiscovery () { - if ( this.#googleDiscovery ) return this.#googleDiscovery; - try { - const res = await fetch(GOOGLE_DISCOVERY_URL); - if ( ! res.ok ) return null; - this.#googleDiscovery = await res.json(); - return this.#googleDiscovery; - } catch ( e ) { - this.log?.warn?.('OIDC: Google discovery fetch failed', e); - return null; - } - } - - /** - * Return the OAuth callback URL for a given flow. Structure: /auth/oidc/callback/ - * @param {string} flow - e.g. 'login' or 'signup' - * @returns {string|null} Full callback URL, or null if flow is invalid - */ - getCallbackUrlForFlow (flow) { - if ( !flow || !VALID_OIDC_FLOWS.includes(flow) ) return null; - const base = this.global_config.origin || ''; - const callback_url = `${base.replace(/\/$/, '')}/auth/oidc/callback/${flow}`; - this.log.noticeme('CALLBACK URL???', { callback_url }); - return callback_url; - } - - /** - * Build authorization URL for the provider. Callback URL is /auth/oidc/callback/ when flow is provided. - */ - async getAuthorizationUrl (providerId, state, flow) { - const config = await this.getProviderConfig(providerId); - if ( ! config ) return null; - const base = this.getCallbackUrlForFlow(flow) ?? `${this.global_config.api_base_url}/auth/oidc/callback`; - const params = new URLSearchParams({ - client_id: config.client_id, - redirect_uri: base, - response_type: 'code', - scope: config.scopes, - state, - }); - return `${config.authorization_endpoint}?${params.toString()}`; - } - - /** - * Sign state payload for CSRF protection (short-lived JWT). - */ - signState (payload) { - return jwt.sign(payload, - this.global_config.jwt_secret, - { expiresIn: STATE_EXPIRY_SEC }); - } - - verifyState (token) { - try { - return jwt.verify(token, this.global_config.jwt_secret); - } catch ( e ) { - return null; - } - } - - /** - * Exchange authorization code for tokens. redirectUri must match the URL used in getAuthorizationUrl (e.g. /auth/oidc/callback/:flow). - */ - async exchangeCodeForTokens (providerId, code, redirectUri) { - const config = await this.getProviderConfig(providerId); - if ( ! config ) return null; - const base = redirectUri ?? `${this.global_config.api_base_url}/auth/oidc/callback`; - const body = new URLSearchParams({ - grant_type: 'authorization_code', - code, - redirect_uri: base, - client_id: config.client_id, - client_secret: config.client_secret, - }); - const res = await fetch(config.token_endpoint, { - method: 'POST', - headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, - body: body.toString(), - }); - if ( ! res.ok ) { - const text = await res.text(); - this.log?.warn?.('OIDC token exchange failed', { status: res.status, body: text }); - return null; - } - return await res.json(); - } - - /** - * Get userinfo from provider (e.g. Google userinfo endpoint). - */ - async getUserInfo (providerId, accessToken) { - const config = await this.getProviderConfig(providerId); - if ( !config || !config.userinfo_endpoint ) return null; - const res = await fetch(config.userinfo_endpoint, { - headers: { Authorization: `Bearer ${accessToken}` }, - }); - if ( ! res.ok ) return null; - return await res.json(); - } - - /** - * Find Puter user by provider and IdP subject. Returns user object or null. - */ - async findUserByProviderSub (providerId, providerSub) { - const rows = await this.db.pread('SELECT user_id FROM user_oidc_providers WHERE provider = ? AND provider_sub = ? LIMIT 1', - [providerId, providerSub]); - if ( !rows || rows.length === 0 ) return null; - const svc_get_user = this.services.get('get-user'); - return await svc_get_user.get_user({ id: rows[0].user_id, cached: false }); - } - - /** - * Link an existing Puter user to an OIDC provider identity. - */ - async linkProviderToUser (userId, providerId, providerSub, refreshToken = null) { - try { - await this.db.write('INSERT INTO user_oidc_providers (user_id, provider, provider_sub, refresh_token) VALUES (?, ?, ?, ?)', - [userId, providerId, providerSub, refreshToken]); - } catch ( e ) { - if ( e.message?.includes('UNIQUE') || e.code === 'SQLITE_CONSTRAINT' ) { - // already linked - return; - } - throw e; - } - } - - /** - * Create a new Puter user from OIDC claims and link the provider. Delegates to signup_create_new_user. - */ - async createUserFromOIDC (providerId, claims) { - if ( claims.email_verified === false ) { - // This should never happen; Google always sends verified emails. - const outcome = new OutcomeObject(new CreatedUserOutcome()); - return outcome.fail( - 'Provider did not verify this email address.', - 'oidc.email_not_verified', - ); - } - const svc_signup = this.services.get('signup'); - const outcome = await svc_signup.create_new_user({ - username: await generate_random_username(), - email: claims?.email ?? null, - password: null, - oidc_only: true, - assume_email_ownership: true, - }); - const { user_id } = outcome.infoObject; - if ( outcome.succeeded ) { - await this.linkProviderToUser(user_id, providerId, claims.sub, null); - } - return outcome; - } - - /** - * List provider ids that have valid config (for frontend to show "Sign in with Google" etc.). - */ - async getEnabledProviderIds () { - const providers = this.providers ?? {}; - const ids = []; - for ( const id of Object.keys(providers) ) { - const cfg = await this.getProviderConfig(id); - if ( cfg ) ids.push(id); - } - return ids; - } -} diff --git a/src/backend/src/services/auth/OTPService.js b/src/backend/src/services/auth/OTPService.js deleted file mode 100644 index 0a167fbf4..000000000 --- a/src/backend/src/services/auth/OTPService.js +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const BaseService = require('../BaseService'); - -/** -* Represents the OTP (One-Time Password) service. -* This class provides functionalities to create OTP secrets, recovery codes, -* and verify OTPs against given secrets and codes, using the 'otpauth' and 'crypto' libraries. -*/ -class OTPService extends BaseService { - static MODULES = { - otpauth: require('otpauth'), - crypto: require('crypto'), - 'hi-base32': require('hi-base32'), - }; - - create_secret (label) { - const require = this.require; - const otpauth = require('otpauth'); - - const secret = this.gen_otp_secret_(); - const totp = new otpauth.TOTP({ - issuer: 'puter.com', - label, - algorithm: 'SHA1', - digits: 6, - secret, - }); - - return { - url: totp.toString(), - secret, - }; - } - - /** - * Creates a recovery code for the user. - * Generates a random byte sequence, encodes it in base32, - * and returns a unique 8-character recovery code. - * - * @returns {string} The generated recovery code. - */ - create_recovery_code () { - const require = this.require; - const crypto = require('crypto'); - const { encode } = require('hi-base32'); - - const buffer = crypto.randomBytes(6); - const code = encode(buffer).replace(/=/g, '').substring(0, 8); - return code; - } - - verify (label, secret, code) { - const require = this.require; - const otpauth = require('otpauth'); - - const totp = new otpauth.TOTP({ - issuer: 'puter.com', - label, - algorithm: 'SHA1', - digits: 6, - secret, - }); - - const allowed = [-1, 0, 1]; - - const delta = totp.validate({ token: code }); - if ( delta === null ) return false; - if ( ! allowed.includes(delta) ) return false; - return true; - } - - /** - * Generates a random OTP secret. - * This method creates a 15-byte random buffer and encodes it into a base32 string. - * The resulting string is trimmed to a maximum length of 24 characters. - * - * @returns {string} The generated OTP secret in base32 format. - */ - gen_otp_secret_ () { - const require = this.require; - const crypto = require('crypto'); - const { encode } = require('hi-base32'); - - const buffer = crypto.randomBytes(15); - const base32 = encode(buffer).replace(/=/g, '').substring(0, 24); - return base32; - }; -}; - -module.exports = { OTPService }; diff --git a/src/backend/src/services/auth/PermissionScanRedisCacheSpace.js b/src/backend/src/services/auth/PermissionScanRedisCacheSpace.js deleted file mode 100644 index a8aca78e0..000000000 --- a/src/backend/src/services/auth/PermissionScanRedisCacheSpace.js +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const PermissionScanRedisCacheSpace = { - key: ({ actorUid, permissionOptions, joinPermissionParts }) => ( - joinPermissionParts('permission-scan', actorUid, 'options-list', ...permissionOptions) - ), -}; - -export { PermissionScanRedisCacheSpace }; diff --git a/src/backend/src/services/auth/PermissionScanRedisCacheSpace.test.js b/src/backend/src/services/auth/PermissionScanRedisCacheSpace.test.js deleted file mode 100644 index 094657187..000000000 --- a/src/backend/src/services/auth/PermissionScanRedisCacheSpace.test.js +++ /dev/null @@ -1,39 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { PermissionScanRedisCacheSpace } from './PermissionScanRedisCacheSpace.js'; -import { PermissionUtil } from './permissionUtils.mjs'; - -describe('PermissionScanRedisCacheSpace', () => { - it('builds cache keys for actor and permission options', () => { - const actorUid = 'app-under-user:user-123:app-456'; - const permissionOptions = ['fs:node-1:read']; - const key = PermissionScanRedisCacheSpace.key({ - actorUid, - permissionOptions, - joinPermissionParts: PermissionUtil.join, - }); - - expect(key).toBe(PermissionUtil.join( - 'permission-scan', - actorUid, - 'options-list', - ...permissionOptions, - )); - }); - - it('builds stable exact keys for app-under-user + one permission', () => { - const actorUid = 'app-under-user:user-123:app-456'; - const permissionOptions = ['flag:app-is-authenticated']; - const key = PermissionScanRedisCacheSpace.key({ - actorUid, - permissionOptions, - joinPermissionParts: PermissionUtil.join, - }); - - expect(key).toBe(PermissionUtil.join( - 'permission-scan', - actorUid, - 'options-list', - ...permissionOptions, - )); - }); -}); diff --git a/src/backend/src/services/auth/PermissionService.js b/src/backend/src/services/auth/PermissionService.js deleted file mode 100644 index 4626999bd..000000000 --- a/src/backend/src/services/auth/PermissionService.js +++ /dev/null @@ -1,1324 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const { hardcoded_user_group_permissions } = require('../../data/hardcoded-permissions.js'); -const { ECMAP } = require('../../deprecated/filesystem/ECMAP'); -const { get_user, get_app } = require('../../helpers'); -const { reading_has_terminal } = require('../../unstructured/permission-scan-lib'); -const { trace } = require('@opentelemetry/api'); -const BaseService = require('../BaseService'); -const { DB_WRITE } = require('../database/consts'); -const { UserActorType, Actor } = require('./Actor'); -const { PERM_KEY_PREFIX, MANAGE_PERM_PREFIX } = require('./permissionConts.mjs'); -const { PermissionUtil, PermissionExploder, PermissionImplicator, PermissionRewriter } = require('./permissionUtils.mjs'); -const { spanify } = require('../../util/otelutil'); -const { deleteRedisKeys } = require('../../clients/redis/deleteRedisKeys.js'); -const { setRedisCacheValue } = require('../../clients/redis/cacheUpdate.js'); -const { redisClient } = require('../../clients/redis/redisSingleton'); -const { PermissionScanRedisCacheSpace } = require('./PermissionScanRedisCacheSpace.js'); -const { Context } = require('../../util/context'); - -const defaultPermissionRedisTimeoutMs = 200; -const formatErrorMessage = (error) => error instanceof Error ? error.message : String(error); -const withTimeout = async (operationPromise, timeoutMs, timeoutMessage) => { - let timeout; - try { - return await Promise.race([ - operationPromise, - new Promise((_, reject) => { - timeout = setTimeout(() => { - reject(new Error(timeoutMessage)); - }, timeoutMs); - }), - ]); - } finally { - if ( timeout ) clearTimeout(timeout); - } -}; - -/** -* @class PermissionService -* @extends BaseService -* @description -* The PermissionService class manages and enforces permissions within the application. It provides methods to: -* - Check, grant, and revoke permissions for users and applications. -* - Scan for existing permissions. -* - Handle permission implications, rewriting, and explosion to support complex permission hierarchies. -* This service interacts with the database to manage permissions and logs actions for auditing purposes. -*/ -class PermissionService extends BaseService { - static CONCERN = 'permissions'; - /** - * Initializes the PermissionService by setting up internal arrays for permission handling. - * - * This method is called during the construction of the PermissionService instance to - * prepare it for handling permissions, rewriters, implicators, and exploders. - */ - _construct () { - this._permission_rewriters = []; - this._permission_implicators = []; - this._permission_exploders = []; - this._PERMISSION_SCAN_CACHE_TTL_SECONDS = 20; - } - - /** - * Registers a permission exploder which expands permissions into their component parts or related permissions. - * - * @param {PermissionExploder} exploder - The PermissionExploder instance to register. - * @throws {Error} If the provided exploder is not an instance of PermissionExploder. - */ - async _init () { - /** - * @type {import('../../modules/kvstore/KVStoreInterfaceService.js').KVStoreInterface} db - */ - this.kvService = this.services.get('puter-kvstore').as('puter-kvstore'); - this.db = this.services.get('database').get(DB_WRITE, 'permissions'); - this.kvAvgTimes = { count: 0, avg: 0, max: 0 }; - this.dbAvgTimes = { count: 0, avg: 0, max: 0 }; - } - - async '__on_boot.consolidation' () { - const svc_event = this.services.get('event'); - // Event to allow extensions to add permissions - { - const event = {}; - event.grant_to_everyone = permission => { - /* eslint-disable */ - hardcoded_user_group_permissions - .system - [this.global_config.default_temp_group] - [permission] - = {}; - hardcoded_user_group_permissions - .system - [this.global_config.default_user_group] - [permission] - = {}; - /* eslint-enable */ - }; - event.grant_to_users = permission => { - /* eslint-disable */ - hardcoded_user_group_permissions - [this.global_config.default_user_group] - [permission] - = {}; - /* eslint-enable */ - }; - svc_event.emit('create.permissions', event); - } - } - - /** - * Rewrites the given permission string based on registered PermissionRewriters. - * - * @param {string} permission - The original permission string to be rewritten. - * @returns {Promise} A promise that resolves to the rewritten permission string. - * - * @note This method iterates through all registered rewriters. If a rewriter matches the permission, - * it applies the rewrite transformation. The process continues until no more matches are found. - */ - async _rewrite_permission (permission) { - for ( const rewriter of this._permission_rewriters ) { - if ( ! rewriter.matches(permission) ) continue; - permission = await rewriter.rewrite(permission); - } - return permission; - } - - /** - * Checks if the actor has any of the specified permissions. - * - * @param {Actor} actor - The actor to check permissions for. - * @param {string[]|string} permission_options - The permissions to check against. - * Can be a single permission string or an array of permission strings. - * @returns {Promise} - True if the actor has at least one of the permissions, false otherwise. - */ - check = spanify('permission:check', async (actor, permission_options, scan_options = {}) => { - const reading = await this.scan(actor, permission_options, undefined, undefined, scan_options); - const options = PermissionUtil.reading_to_options(reading); - return options.length > 0; - }); - /** - * Checks if the actor has grant access to any of the specified permissions. - * - * @param {Actor} actor - The actor to check if they can manage a permission. - * @param {string} permission - The permission to check against. - * @returns {Promise} - True if the actor has at least one of the permissions, false otherwise. - */ - canManagePermission = spanify('permission:check', async (actor, permission) => { - const managePermission = PermissionUtil.join(MANAGE_PERM_PREFIX, ...PermissionUtil.split(permission)); - const reading = await this.scan(actor, managePermission); - const options = PermissionUtil.reading_to_options(reading); - return options.length > 0; - }); - - /** - * Scans the permissions for an actor against specified permission options. - * - * This method performs a comprehensive scan of permissions, considering: - * - Direct permissions - * - Implicit permissions - * - Permission rewriters - * - * @param {Actor} actor - The actor whose permissions are being checked. - * @param {string|string[]} permission_options - One or more permission strings to check against. - * @param {*} _reserved - Reserved for future use, currently not utilized. - * @param {Object} state - State object to manage recursion and prevent cycles. - * - * @returns {Promise} A promise that resolves to an array of permission readings. - */ - scan = spanify('permission:scan', async (actor, permission_options, _reserved, state, scan_options = {}) => { - const activeSpan = trace.getActiveSpan(); - if ( activeSpan ) { - const options = Array.isArray(permission_options) - ? permission_options - : [permission_options]; - activeSpan.setAttribute('permission_options', options); - if ( actor?.uid != null ) { - activeSpan.setAttribute('actor', actor.uid); - } - } - return await ECMAP.arun(async () => { - return await this.#scan(actor, permission_options, _reserved, state, scan_options); - }); - }); - - async #scan (actor, permission_options, _reserved, state, scan_options = {}) { - if ( ! state ) { - this.log.debug('scan', { - actor: actor.uid, - permission_options, - }); - } - const reading = []; - - if ( ! state ) { - state = { - anti_cycle_actors: [actor], - }; - } - - if ( ! Array.isArray(permission_options) ) { - permission_options = [permission_options]; - } - - const cacheKey = PermissionScanRedisCacheSpace.key({ - actorUid: actor.uid, - permissionOptions: permission_options, - joinPermissionParts: PermissionUtil.join, - }); - - const permissionRedisTimeoutMs = Number(this.global_config?.services?.permission?.redis_timeout_ms) - || defaultPermissionRedisTimeoutMs; - let cached; - if ( ! scan_options.no_cache ) { - try { - cached = await withTimeout( - redisClient.get(cacheKey), - permissionRedisTimeoutMs, - `permission scan cache read timed out after ${permissionRedisTimeoutMs}ms`, - ); - } catch ( error ) { - this.log.warn('permission scan cache read failed; continuing without cache', { - actorUid: actor.uid, - cacheKey, - error: formatErrorMessage(error), - }); - } - } - if ( cached && !scan_options.no_cache ) { - try { - return JSON.parse(cached); - } catch (e) { - // no op cache is in an invalid state - } - } - - // TODO: command to enable these logs - // const l = get_a_letter(); - // cylog(l, 'ACT & PERM:', actor.uid, permission_options); - - const start_ts = Date.now(); - await require('../../structured/sequence/scan-permission.mjs').default - .call(this, { - actor, - permission_options, - reading, - state, - }); - const end_ts = Date.now(); - - // TODO: command to enable these logs - // cylog(l, 'READING', JSON.stringify(reading, null, ' ')); - - reading.push({ - $: 'time', - value: end_ts - start_ts, - }); - - try { - await withTimeout( - setRedisCacheValue(cacheKey, JSON.stringify(reading), { - ttlSeconds: this._PERMISSION_SCAN_CACHE_TTL_SECONDS, - eventData: reading, - }), - permissionRedisTimeoutMs, - `permission scan cache write timed out after ${permissionRedisTimeoutMs}ms`, - ); - } catch ( error ) { - this.log.warn('permission scan cache write failed; continuing without cache', { - actorUid: actor.uid, - cacheKey, - error: formatErrorMessage(error), - }); - } - - return reading; - } - - /** - * Removes a specific permission-scan cache entry for a single app-under-user actor. - * This targets only the exact key for (user_uuid, app_uid, permission). - * - * @param {string} user_uuid - The user's UUID. - * @param {string} app_uid - The app UID. - * @param {string} permission - The permission string used in scan. - * @returns {Promise} - */ - async invalidate_permission_scan_cache_for_app_under_user (user_uuid, app_uid, permission) { - const actorUid = `app-under-user:${user_uuid}:${app_uid}`; - const cacheKey = PermissionScanRedisCacheSpace.key({ - actorUid, - permissionOptions: [permission], - joinPermissionParts: PermissionUtil.join, - }); - await deleteRedisKeys(cacheKey); - } - - async validateUserPerms ({ actor, permissions }) { - - const flatPermsReading = await this.#flat_validateUserPerms({ actor, permissions }); - const linkedPermsReadingPromise = this.#linked_validateUserPerms({ actor, permissions, state: { anti_cycle_actors: [actor] } }); - - if ( flatPermsReading && flatPermsReading.length > 0 ) { - return flatPermsReading[0].deleted ? [] : flatPermsReading; - } - - const linkedPermsReading = await linkedPermsReadingPromise; - const options = PermissionUtil.reading_to_options(linkedPermsReading); - - options.forEach((perm) => { - if ( perm.permission ) { - this.kvService.set({ - key: PermissionUtil.join(PERM_KEY_PREFIX, actor.type.user.id, perm.permission), - value: { - permission: perm.permission, - issuer_user_id: perm.data?.[0]?.issuer_user_id, - data: perm.data, - }, - }); - } - }); - return flatPermsReading; - } - - async #flat_validateUserPerms ({ actor, permissions }) { - /** @type {Promise[]>} */ - const validPerms = (await this.services.get('su').sudo(() => ( - this.kvService.get({ - key: [...new Set(permissions.map(perm => PermissionUtil.join(PERM_KEY_PREFIX, actor.type.user.id, perm)))], - }) - ))).filter(Boolean); - - let permDeleted = false; - // We no longer fetch up the tree, if user was given this perm, then they have it - for ( const validPerm of validPerms ) { - const { permission, issuer_user_id, deleted, ...extra } = validPerm; - if ( deleted ) { - permDeleted = true; - continue; - } - const issuer_actor = new Actor({ - type: new UserActorType({ - user: await get_user({ id: issuer_user_id }), - }), - }); - // return first perm that allows them in here - return [{ - $: 'option', - via: 'user', - has_terminal: true, - permission: permission, - data: extra, - holder_username: actor.type.user.username, - issuer_username: issuer_actor.type.user.username, - issuer_user_id: issuer_actor.type.user.uuid, - reading: [], - }]; - - } - return permDeleted ? [{ - deleted: true, - }] : []; - - } - async #linked_validateUserPerms ({ actor, permissions, state }) { - let sqlPermQuery = permissions.map(_perm => { - return '`permission` = ?'; - }).join(' OR '); - - if ( permissions.length > 1 ) { - sqlPermQuery = `(${sqlPermQuery})`; - } - - const rows = await this.db.read( - 'SELECT * FROM `user_to_user_permissions` ' + - `WHERE \`holder_user_id\` = ? AND ${ - sqlPermQuery}`, - [ - actor.type.user.id, - ...permissions, - ], - ); - - const readings = []; - // Return the first matching permission where the - // issuer also has the permission granted - for ( const row of rows ) { - if ( !row.extra || typeof (row.extra) === 'string' ) { - row.extra = JSON.parse(row.extra || '{}'); - } - - const issuer_actor = new Actor({ - type: new UserActorType({ - user: await get_user({ id: row.issuer_user_id }), - }), - }); - - let should_continue = false; - for ( const seen_actor of state.anti_cycle_actors ) { - if ( seen_actor.type.user.id === issuer_actor.type.user.id ) { - should_continue = true; - break; - } - } - - if ( should_continue ) continue; - - const issuer_reading = await this.scan(issuer_actor, row.permission, undefined, state); - - const has_terminal = reading_has_terminal({ reading: issuer_reading }); - - readings.push({ - $: 'path', - via: 'user', - has_terminal, - permission: row.permission, - data: row.extra, - holder_username: actor.type.user.username, - issuer_username: issuer_actor.type.user.username, - issuer_user_id: issuer_actor.type.user.uuid, - reading: issuer_reading, - }); - } - return readings; - } - - /** - * Grants a user permission to an app the user is working with if the user has permission. - * - * @param {Actor} actor - The actor granting the permission (must be a user). - * @param {string} app_uid - The unique identifier or name of the app. - * @param {string} permission - The permission string to grant. - * @param {Object} [extra={}] - Additional metadata or conditions for the permission. - * @param {Object} [meta] - Metadata for logging or auditing purposes. - * @throws {Error} If the user to grant permission to is not found or if attempting to grant permissions to oneself. - * @returns {Promise} - */ - async grant_user_app_permission (actor, app_uid, permission, extra = {}, meta) { - // We add 'is_grant_user_app_permission' to guard against any logic - // error that might cause unintended access being granted to users. - permission = await Context.sub({ - is_grant_user_app_permission: true, - }).arun(async () => await this._rewrite_permission(permission)); - - let app = await get_app({ uid: app_uid }); - if ( ! app ) app = await get_app({ name: app_uid }); - - if ( ! app ) { - throw APIError.create('entity_not_found', null, { - identifier: `app:${app_uid}`, - }); - } - - const app_id = app.id; - - // Skip if already granted (avoids redundant writes and invalidation when e.g. get-user-app-token or open_item is called many times for the same permission). - const existing = await this.db.read( - 'SELECT 1 FROM `user_to_app_permissions` WHERE `user_id` = ? AND `app_id` = ? AND `permission` = ? LIMIT 1', - [actor.type.user.id, app_id, permission], - ); - if ( existing && existing.length > 0 ) return; - - // UPSERT permission - await this.db.write( - 'INSERT INTO `user_to_app_permissions` (`user_id`, `app_id`, `permission`, `extra`) ' + - `VALUES (?, ?, ?, ?) ${ - this.db.case({ - mysql: 'ON DUPLICATE KEY UPDATE `extra` = ?', - otherwise: 'ON CONFLICT(`user_id`, `app_id`, `permission`) DO UPDATE SET `extra` = ?', - })}`, - [ - actor.type.user.id, - app_id, - permission, - JSON.stringify(extra), - JSON.stringify(extra), - ], - ); - - // INSERT audit table - const audit_values = { - user_id: actor.type.user.id, - user_id_keep: actor.type.user.id, - app_id: app_id, - app_id_keep: app_id, - permission, - action: 'grant', - reason: meta?.reason || 'granted via PermissionService', - }; - - const sql_cols = Object.keys(audit_values).map((key) => `\`${key}\``).join(', '); - const sql_vals = Object.keys(audit_values).map(() => '?').join(', '); - - this.db.write( - `INSERT INTO \`audit_user_to_app_permissions\` (${sql_cols}) ` + - `VALUES (${sql_vals})`, - Object.values(audit_values), - ); - - // Invalidate permission-scan cache for this app-under-user so the next check sees the grant. - this.invalidate_permission_scan_cache_for_app_under_user(actor.type.user.uuid, app_uid, permission); - } - - /** - * Grants an app a permission for any user, as long as the user granting the - * permission can manage permission. - * - * @param {Actor} actor - The actor granting the permission (must be a user). - * @param {string} app_uid - The unique identifier or name of the app. - * @param {string} permission - The permission string to grant. - * @param {Object} [extra={}] - Additional metadata or conditions for the permission. - * @param {Object} [meta] - Metadata for logging or auditing purposes. - * @throws {Error} If the user to grant permission to is not found or if attempting to grant permissions to oneself. - * @returns {Promise} - */ - async grant_dev_app_permission (actor, app_uid, permission, extra = {}, meta) { - permission = await this._rewrite_permission(permission); - - let app = await get_app({ uid: app_uid }); - if ( ! app ) app = await get_app({ name: app_uid }); - - if ( ! app ) { - throw APIError.create('entity_not_found', null, { - identifier: `app:${app_uid}`, - }); - } - - const app_id = app.id; - - const canManagePerms = await this.canManagePermission(actor, permission); - if ( ! canManagePerms ) { - throw APIError.create('permission_denied', null, { - permission, - }); - } - - // UPSERT permission - await this.db.write( - 'INSERT INTO `dev_to_app_permissions` (`user_id`, `app_id`, `permission`, `extra`) ' + - `VALUES (?, ?, ?, ?) ${ - this.db.case({ - mysql: 'ON DUPLICATE KEY UPDATE `extra` = ?', - otherwise: 'ON CONFLICT(`user_id`, `app_id`, `permission`) DO UPDATE SET `extra` = ?', - })}`, - [ - actor.type.user.id, - app_id, - permission, - JSON.stringify(extra), - JSON.stringify(extra), - ], - ); - - // INSERT audit table - const audit_values = { - user_id: actor.type.user.id, - user_id_keep: actor.type.user.id, - app_id: app_id, - app_id_keep: app_id, - permission, - action: 'grant', - reason: meta?.reason || 'granted via PermissionService', - }; - - const sql_cols = Object.keys(audit_values).map((key) => `\`${key}\``).join(', '); - const sql_vals = Object.keys(audit_values).map(() => '?').join(', '); - - this.db.write( - `INSERT INTO \`audit_dev_to_app_permissions\` (${sql_cols}) ` + - `VALUES (${sql_vals})`, - Object.values(audit_values), - ); - } - async revoke_dev_app_permission (actor, app_uid, permission, meta) { - permission = await this._rewrite_permission(permission); - - // For now, actor MUST be a user - if ( ! (actor.type instanceof UserActorType) ) { - throw new Error('actor must be a user'); - } - - let app = await get_app({ uid: app_uid }); - if ( ! app ) app = await get_app({ name: app_uid }); - if ( ! app ) { - throw APIError.create('entity_not_found', null, { - identifier: `app${app_uid}`, - }); - } - const app_id = app.id; - - // DELETE permission - await this.db.write( - 'DELETE FROM `dev_to_app_permissions` ' + - 'WHERE `user_id` = ? AND `app_id` = ? AND `permission` = ?', - [ - actor.type.user.id, - app_id, - permission, - ], - ); - - // INSERT audit table - const audit_values = { - user_id: actor.type.user.id, - user_id_keep: actor.type.user.id, - app_id: app_id, - app_id_keep: app_id, - permission, - action: 'revoke', - reason: meta?.reason || 'revoked via PermissionService', - }; - - const sql_cols = Object.keys(audit_values).map((key) => `\`${key}\``).join(', '); - const sql_vals = Object.keys(audit_values).map(() => '?').join(', '); - - this.db.write( - `INSERT INTO \`audit_dev_to_app_permissions\` (${sql_cols}) ` + - `VALUES (${sql_vals})`, - Object.values(audit_values), - ); - } - async revoke_dev_app_all (actor, app_uid, meta) { - // For now, actor MUST be a user - if ( ! (actor.type instanceof UserActorType) ) { - throw new Error('actor must be a user'); - } - - let app = await get_app({ uid: app_uid }); - if ( ! app ) app = await get_app({ name: app_uid }); - const app_id = app.id; - - // DELETE permissions - await this.db.write( - 'DELETE FROM `dev_to_app_permissions` ' + - 'WHERE `user_id` = ? AND `app_id` = ?', - [ - actor.type.user.id, - app_id, - ], - ); - - // INSERT audit table - const audit_values = { - user_id: actor.type.user.id, - user_id_keep: actor.type.user.id, - app_id: app_id, - app_id_keep: app_id, - permission: '*', - action: 'revoke', - reason: meta?.reason || 'revoked all via PermissionService', - }; - - const sql_cols = Object.keys(audit_values).map((key) => `\`${key}\``).join(', '); - const sql_vals = Object.keys(audit_values).map(() => '?').join(', '); - - this.db.write( - `INSERT INTO \`audit_dev_to_app_permissions\` (${sql_cols}) ` + - `VALUES (${sql_vals})`, - Object.values(audit_values), - ); - } - - /** - * Grants a permission to a user for a specific app. - * - * @param {Actor} actor - The actor granting the permission, must be a user. - * @param {string} app_uid - The unique identifier or name of the app. - * @param {string} permission - The permission string to be granted. - * @param {Object} [extra={}] - Additional data associated with the permission. - * @param {Object} [meta] - Metadata for the operation, including a reason for the grant. - * - * @throws {Error} If the actor is not a user or if the app is not found. - * - * @returns {Promise} A promise that resolves when the permission is granted and logged. - */ - async revoke_user_app_permission (actor, app_uid, permission, meta) { - permission = await this._rewrite_permission(permission); - - // For now, actor MUST be a user - if ( ! (actor.type instanceof UserActorType) ) { - throw new Error('actor must be a user'); - } - - let app = await get_app({ uid: app_uid }); - if ( ! app ) app = await get_app({ name: app_uid }); - if ( ! app ) { - throw APIError.create('entity_not_found', null, { - identifier: `app${app_uid}`, - }); - } - const app_id = app.id; - - // DELETE permission - await this.db.write( - 'DELETE FROM `user_to_app_permissions` ' + - 'WHERE `user_id` = ? AND `app_id` = ? AND `permission` = ?', - [ - actor.type.user.id, - app_id, - permission, - ], - ); - - // INSERT audit table - const audit_values = { - user_id: actor.type.user.id, - user_id_keep: actor.type.user.id, - app_id: app_id, - app_id_keep: app_id, - permission, - action: 'revoke', - reason: meta?.reason || 'revoked via PermissionService', - }; - - const sql_cols = Object.keys(audit_values).map((key) => `\`${key}\``).join(', '); - const sql_vals = Object.keys(audit_values).map(() => '?').join(', '); - - this.db.write( - `INSERT INTO \`audit_user_to_app_permissions\` (${sql_cols}) ` + - `VALUES (${sql_vals})`, - Object.values(audit_values), - ); - } - - /** - * Revokes all permissions for a user on a specific app. - * - * @param {Actor} actor - The actor performing the revocation, must be a user. - * @param {string} app_uid - The unique identifier or name of the app for which permissions are being revoked. - * @param {Object} meta - Metadata for logging the revocation action. - * @throws {Error} If the actor is not a user. - */ - async revoke_user_app_all (actor, app_uid, meta) { - // For now, actor MUST be a user - if ( ! (actor.type instanceof UserActorType) ) { - throw new Error('actor must be a user'); - } - - let app = await get_app({ uid: app_uid }); - if ( ! app ) app = await get_app({ name: app_uid }); - const app_id = app.id; - - // DELETE permissions - await this.db.write( - 'DELETE FROM `user_to_app_permissions` ' + - 'WHERE `user_id` = ? AND `app_id` = ?', - [ - actor.type.user.id, - app_id, - ], - ); - - // INSERT audit table - const audit_values = { - user_id: actor.type.user.id, - user_id_keep: actor.type.user.id, - app_id: app_id, - app_id_keep: app_id, - permission: '*', - action: 'revoke', - reason: meta?.reason || 'revoked all via PermissionService', - }; - - const sql_cols = Object.keys(audit_values).map((key) => `\`${key}\``).join(', '); - const sql_vals = Object.keys(audit_values).map(() => '?').join(', '); - - this.db.write( - `INSERT INTO \`audit_user_to_app_permissions\` (${sql_cols}) ` + - `VALUES (${sql_vals})`, - Object.values(audit_values), - ); - } - - /** - * Grants a permission from one user to another. - * - * This method handles the process of granting permissions between users, - * ensuring that the permission is correctly formatted, the users exist, - * and that self-granting is not allowed. - * - * @param {Actor} actor - * @param {string} username - * @param {string} permission - * @param {object} extra - * @param {object} meta - * @throws {Error} Throws if the user is not found or if attempting to grant permissions to oneself. - * @returns {Promise} - */ - async grant_user_user_permission (actor, username, permission, extra = {}, meta) { - permission = await this._rewrite_permission(permission); - const user = await get_user({ username }); - if ( ! user ) { - throw APIError.create('user_does_not_exist', null, { - username, - }); - } - - // Don't allow granting permissions to yourself - if ( user.id === actor.type.user.id ) { - throw new Error('cannot grant permissions to yourself'); - } - - const canManagePerms = await this.canManagePermission(actor, permission); - if ( ! canManagePerms ) { - throw APIError.create('permission_denied', null, { - permission, - }); - } - - const flatRes = this.#flat_grant_user_user_permission(actor, user, permission, extra); - // shoot this async - this.#linked_grant_user_user_permission(actor, user, permission, extra, meta); - return flatRes; - - } - - /** - * @param {Actor} actor - * @param {User} user - * @param {string} permission - * @param {object} extra - * @throws {Error} Throws if the user is not found or if attempting to grant permissions to oneself. - * @returns {Promise} - */ - async #flat_grant_user_user_permission (actor, user, permission, extra = {}) { - // UPSERT permission - await this.services - .get('su') - .sudo(() => this.kvService.set({ - key: PermissionUtil.join(PERM_KEY_PREFIX, user.id, permission), - value: { - ...extra, - issuer_user_id: actor.type.user.id, - permission, - deleted: false, - }, - })); - - } - - /** - * @param {Actor} actor - * @param {User} user - * @param {string} permission - * @param {object} extra - * @param {object} meta - * @throws {Error} Throws if the user is not found or if attempting to grant permissions to oneself. - * @returns {Promise} - */ - async #linked_grant_user_user_permission (actor, user, permission, extra = {}, meta) { - // UPSERT permission - await this.db.write( - 'INSERT INTO `user_to_user_permissions` (`holder_user_id`, `issuer_user_id`, `permission`, `extra`) ' + - `VALUES (?, ?, ?, ?) ${ - this.db.case({ - mysql: 'ON DUPLICATE KEY UPDATE `extra` = ?', - otherwise: 'ON CONFLICT(`holder_user_id`, `issuer_user_id`, `permission`) DO UPDATE SET `extra` = ?', - })}`, - [ - user.id, - actor.type.user.id, - permission, - JSON.stringify(extra), - JSON.stringify(extra), - ], - ); - - // INSERT audit table - this.db.write( - 'INSERT INTO `audit_user_to_user_permissions` (' + - '`holder_user_id`, `holder_user_id_keep`, `issuer_user_id`, `issuer_user_id_keep`, ' + - '`permission`, `action`, `reason`) ' + - 'VALUES (?, ?, ?, ?, ?, ?, ?)', - [ - user.id, - user.id, - actor.type.user.id, - actor.type.user.id, - permission, - 'grant', - meta?.reason || 'granted via PermissionService', - ], - ); - } - - /** - * Grants a user permission to interact with a specific group. - * - * @param {Actor} actor - The actor granting the permission. - * @param {string} gid - The group identifier (UID or name). - * @param {string} permission - The permission string to be granted. - * @param {Object} [extra={}] - Additional metadata for the permission. - * @param {Object} [meta] - Metadata about the grant action, including the reason. - * @returns {Promise} - * - * @note This method ensures the group exists before granting permission. - * @note The permission is first rewritten using any registered rewriters. - * @note If the permission already exists, its extra data is updated. - */ - async grant_user_group_permission (actor, gid, permission, extra = {}, meta) { - permission = await this._rewrite_permission(permission); - const svc_group = this.services.get('group'); - const group = await svc_group.get({ uid: gid }); - if ( ! group ) { - throw APIError.create('entity_not_found', null, { - identifier: `group:${gid}`, - }); - } - - const canManagePerms = await this.canManagePermission(actor, permission); - if ( ! canManagePerms ) { - throw APIError.create('permission_denied', null, { - permission, - }); - } - - await this.db.write( - 'INSERT INTO `user_to_group_permissions` (`user_id`, `group_id`, `permission`, `extra`) ' + - `VALUES (?, ?, ?, ?) ${ - this.db.case({ - mysql: 'ON DUPLICATE KEY UPDATE `extra` = ?', - otherwise: 'ON CONFLICT(`user_id`, `group_id`, `permission`) DO UPDATE SET `extra` = ?', - })}`, - [ - actor.type.user.id, - group.id, - permission, - JSON.stringify(extra), - JSON.stringify(extra), - ], - ); - - // INSERT audit table - this.db.write( - 'INSERT INTO `audit_user_to_group_permissions` (' + - '`user_id`, `user_id_keep`, `group_id`, `group_id_keep`, ' + - '`permission`, `action`, `reason`) ' + - 'VALUES (?, ?, ?, ?, ?, ?, ?)', - [ - actor.type.user.id, - actor.type.user.id, - group.id, - group.id, - permission, - 'grant', - meta?.reason || 'granted via PermissionService', - ], - ); - } - - /** - * @typedef {Object} RevokeUserUserPermissionParams - * @property {Actor} actor - The actor performing the revocation - * @property {string} username - The username of the user whose permission is being revoked - * @property {string} permission - The specific permission string to revoke - * @property {Object} meta - Metadata for the revocation action - */ - - /** - * Revokes a specific user-to-user permission - * - * @param {RevokeUserUserPermissionParams} params - Parameters for revoking permission - * @throws {Error} If the specified user is not found - * @returns {Promise} A promise that resolves when the permission has been revoked and audit logs updated - */ - async revoke_user_user_permission (actor, username, permission, meta) { - const flatRes = this.#flat_revoke_user_user_permission(actor, username, permission, meta); - // shoot this async - this.#linked_revoke_user_user_permission(actor, username, permission, meta); - return flatRes; - } - - /** - * @param {RevokeUserUserPermissionParams} params - Parameters for revoking permission - * @throws {Error} If the specified user is not found - * @returns {Promise} A promise that resolves when the permission has been revoked and audit logs updated - */ - async #flat_revoke_user_user_permission (actor, username, permission, _meta) { - permission = await this._rewrite_permission(permission); - - const user = await get_user({ username }); - if ( ! user ) { - if ( ! user ) { - throw APIError.create('user_does_not_exist', null, { - username, - }); - } - } - - const canManagePerms = await this.canManagePermission(actor, permission); - - if ( ! canManagePerms ) { - throw APIError.create('permission_denied', null, { - permission, - }); - } - - // DELETE permission - await this.services.get('su').sudo(() => - this.kvService.del({ key: PermissionUtil.join(PERM_KEY_PREFIX, user.id, permission) })); - - } - /** - * @param {RevokeUserUserPermissionParams} params - Parameters for revoking permission - * @throws {Error} If the specified user is not found - * @returns {Promise} A promise that resolves when the permission has been revoked and audit logs updated - */ - async #linked_revoke_user_user_permission (actor, username, permission, meta) { - permission = await this._rewrite_permission(permission); - - const user = await get_user({ username }); - if ( ! user ) { - if ( ! user ) { - throw APIError.create('user_does_not_exist', null, { - username, - }); - } - } - - // DELETE permission - await this.db.write( - 'DELETE FROM `user_to_user_permissions` ' + - 'WHERE `holder_user_id` = ? AND `permission` = ?', - [ - user.id, - permission, - ], - ); - - // INSERT audit table - this.db.write( - 'INSERT INTO `audit_user_to_user_permissions` (' + - '`holder_user_id`, `holder_user_id_keep`, `issuer_user_id`, `issuer_user_id_keep`, ' + - '`permission`, `action`, `reason`) ' + - 'VALUES (?, ?, ?, ?, ?, ?, ?)', - [ - user.id, - user.id, - actor.type.user.id, - actor.type.user.id, - permission, - 'revoke', - meta?.reason || 'revoked via PermissionService', - ], - ); - } - - /** - * Revokes a specific permission granted by the actor to a group. - * - * This method removes the specified permission from the `user_to_group_permissions` table, - * ensuring that the actor no longer has that permission for the specified group. - * - * @param {Actor} actor - The actor revoking the permission. - * @param {string} gid - The group ID for which the permission is being revoked. - * @param {string} permission - The permission string to revoke. - * @param {Object} meta - Metadata for the revocation action, including reason. - * @returns {Promise} A promise that resolves when the revocation is complete. - */ - async revoke_user_group_permission (actor, gid, permission, meta) { - permission = await this._rewrite_permission(permission); - const svc_group = this.services.get('group'); - const group = await svc_group.get({ uid: gid }); - if ( ! group ) { - throw APIError.create('entity_not_found', null, { - identifier: `group:${gid}`, - }); - } - - // DELETE permission - await this.db.write( - 'DELETE FROM `user_to_group_permissions` ' + - 'WHERE `user_id` = ? AND `group_id` = ? AND `permission` = ?', - [ - actor.type.user.id, - group.id, - permission, - ], - ); - - // INSERT audit table - this.db.write( - 'INSERT INTO `audit_user_to_group_permissions` (' + - '`user_id`, `user_id_keep`, `group_id`, `group_id_keep`, ' + - '`permission`, `action`, `reason`) ' + - 'VALUES (?, ?, ?, ?, ?, ?, ?)', - [ - actor.type.user.id, - actor.type.user.id, - group.id, - group.id, - permission, - 'revoke', - meta?.reason || 'revoked via PermissionService', - ], - ); - } - - /** - * List the users that have any permissions granted to the - * specified user. - * - * This is a "flat" (non-cascading) view. - * - * Use History: - * - This was written for use in ll_listusers to display - * home directories of users that shared files with the - * current user. - * - * @param {Object} user - The user whose permission issuers are to be listed. - * @returns {Promise} A promise that resolves to an array of user objects. - */ - async list_user_permission_issuers (user) { - const rows = await this.db.read( - 'SELECT DISTINCT issuer_user_id FROM `user_to_user_permissions` ' + - 'WHERE `holder_user_id` = ?', - [user.id], - ); - - const users = []; - for ( const row of rows ) { - users.push(await get_user({ id: row.issuer_user_id })); - } - - return users; - } - - /** - * List the permissions that the specified actor (the "issuer") - * has granted to all other users which have some specified - * prefix in the permission key (ex: "fs:FILE-UUID") - * - * Note that if the prefix contains a literal '%' character - * the behavior may not be as expected. - * - * This is a "flat" (non-cascading) view. - * - * Use History: - * - This was written for FSNodeContext.fetchShares to query - * all the "shares" associated with a file. - * - * This method retrieves permissions from the database where the permission key starts with a specified prefix. - * It is designed for "flat" (non-cascading) queries. - * - * @param {Object} issuer - The actor granting the permissions. - * @param {string} prefix - The prefix to match in the permission key. - * @returns {Object} An object containing arrays of user and app permissions matching the prefix. - */ - async query_issuer_permissions_by_prefix (issuer, prefix) { - const user_perms = await this.db.read( - 'SELECT DISTINCT holder_user_id, permission ' + - 'FROM `user_to_user_permissions` ' + - 'WHERE issuer_user_id = ? ' + - 'AND permission LIKE ?', - [issuer.id, `${prefix}%`], - ); - - const app_perms = await this.db.read( - 'SELECT DISTINCT app_id, permission ' + - 'FROM `user_to_app_permissions` ' + - 'WHERE user_id = ? ' + - 'AND permission LIKE ?', - [issuer.id, `${prefix}%`], - ); - - const retval = { users: [], apps: [] }; - - for ( const user_perm of user_perms ) { - const { holder_user_id, permission } = user_perm; - retval.users.push({ - user: await get_user({ id: holder_user_id }), - permission, - }); - } - - for ( const app_perm of app_perms ) { - const { app_id, permission } = app_perm; - retval.apps.push({ - app: await get_app({ id: app_id }), - permission, - }); - } - - return retval; - } - - /** - * List the permissions that the specified actor (the "issuer") - * has granted to the specified user (the "holder") which have - * some specified prefix in the permission key (ex: "fs:FILE-UUID") - * - * Note that if the prefix contains a literal '%' character - * the behavior may not be as expected. - * - * This is a "flat" (non-cascading) view. - * - * @param {Object} issuer - The actor granting the permissions. - * @param {Object} holder - The actor receiving the permissions. - * @param {string} prefix - The prefix of the permission keys to match. - * @returns {Promise>} An array of permission strings matching the prefix. - */ - async query_issuer_holder_permissions_by_prefix (issuer, holder, prefix) { - const user_perms = await this.db.read( - 'SELECT permission ' + - 'FROM `user_to_user_permissions` ' + - 'WHERE issuer_user_id = ? ' + - 'AND holder_user_id = ? ' + - 'AND permission LIKE ?', - [issuer.type.user.id, holder.type.user.id, `${prefix}%`], - ); - - return user_perms.map(row => row.permission); - } - - /** - * Retrieves permissions granted by an issuer to a specific holder with a given prefix. - * - * @param {Actor} issuer - The actor granting the permissions. - * @param {Actor} holder - The actor receiving the permissions. - * @param {string} prefix - The prefix to filter permissions by. - * @returns {Promise>} A promise that resolves to an array of permission strings. - * - * @note This method performs a database query to fetch permissions. It does not handle - * recursion or implication of permissions, providing only a direct, flat list. - */ - async get_higher_permissions (permission) { - const higher_perms = new Set(); - higher_perms.add(permission); - - const parent_perms = this.get_parent_permissions(permission); - for ( const parent_perm of parent_perms ) { - higher_perms.add(parent_perm); - for ( const exploder of this._permission_exploders ) { - if ( ! exploder.matches(parent_perm) ) continue; - const perms = await exploder.explode({ - permission: parent_perm, - }); - for ( const perm of perms ) higher_perms.add(perm); - } - } - return Array.from(higher_perms); - } - - get_parent_permissions (permission) { - const parent_perms = []; - { - // We don't use PermissionUtil.split here because it unescapes - // components; we want to keep the components escaped for matching. - const parts = permission.split(':'); - - // Add sub-permissions - for ( let i = 0 ; i < parts.length ; i++ ) { - parent_perms.push(parts.slice(0, i + 1).join(':')); - } - } - parent_perms.reverse(); - return parent_perms; - } - - /** - * Register a permission rewriter. For details see the documentation on the - * PermissionRewriter class. - * - * @param {PermissionRewriter} rewriter - The permission rewriter to register - */ - register_rewriter (rewriter) { - const is_permission_rewriter = rewriter instanceof PermissionRewriter - // Hack for ESM/CJS interop issue in unit tests. - || rewriter?.constructor?.name === 'PermissionRewriter'; - if ( ! is_permission_rewriter ) { - throw new Error('rewriter must be a PermissionRewriter'); - } - - this._permission_rewriters.push(rewriter); - } - - /** - * Register a permission implicator. For details see the documentation on the - * PermissionImplicator class. - * - * @param {PermissionImplicator} implicator - The permission implicator to register - */ - register_implicator (implicator) { - if ( ! (implicator instanceof PermissionImplicator) ) { - throw new Error('implicator must be a PermissionImplicator'); - } - - this._permission_implicators.push(implicator); - } - - /** - * Register a permission exploder. For details see the documentation on the - * PermissionExploder class. - * - * @param {PermissionExploder} exploder - The permission exploder to register - */ - register_exploder (exploder) { - if ( ! (exploder instanceof PermissionExploder) ) { - throw new Error('exploder must be a PermissionExploder'); - } - - this._permission_exploders.push(exploder); - } -} - -module.exports = { - PermissionService, -}; diff --git a/src/backend/src/services/auth/PermissionShortcutService.js b/src/backend/src/services/auth/PermissionShortcutService.js deleted file mode 100644 index 3617e9b1a..000000000 --- a/src/backend/src/services/auth/PermissionShortcutService.js +++ /dev/null @@ -1,30 +0,0 @@ -const BaseService = require('../BaseService'); -const { PermissionImplicator } = require('./permissionUtils.mjs'); - -class PermissionShortcutService extends BaseService { - _init () { - const svc_permission = this.services.get('permission'); - - svc_permission.register_implicator(PermissionImplicator.create({ - id: 'kv permissions are easy', - shortcut: true, - matcher: permission => { - return permission === 'service:puter-kvstore:ii:puter-kvstore'; - }, - checker: async ({ actor: _actor }) => { - return { - policy: { - 'rate-limit': { - max: 200, - period: 10000, - }, - }, - }; - }, - })); - } -} - -module.exports = { - PermissionShortcutService, -}; diff --git a/src/backend/src/services/auth/PreAuthService.js b/src/backend/src/services/auth/PreAuthService.js deleted file mode 100644 index c7c1c9e18..000000000 --- a/src/backend/src/services/auth/PreAuthService.js +++ /dev/null @@ -1,12 +0,0 @@ -const configurable_auth = require('../../middleware/configurable_auth'); -const BaseService = require('../BaseService'); - -class PreAuthService extends BaseService { - async '__on_install.middlewares.early' (_, { app }) { - app.use(configurable_auth({ optional: true })); - } -} - -module.exports = { - PreAuthService, -}; diff --git a/src/backend/src/services/auth/SignupService.js b/src/backend/src/services/auth/SignupService.js deleted file mode 100644 index 8e7d31586..000000000 --- a/src/backend/src/services/auth/SignupService.js +++ /dev/null @@ -1,286 +0,0 @@ -//@ts-check -import bcrypt from 'bcrypt'; -import { v4 as uuidv4 } from 'uuid'; -import { generate_random_username, send_email_verification_code, send_email_verification_token, username_exists } from '../../helpers.js'; -import { Context } from '../../util/context.js'; -import { OutcomeObject } from '../../util/outcomeutil.js'; -import { validate_nonEmpty_string } from '../../util/validutil.js'; -import BaseService from '../BaseService.js'; -import { DB_WRITE } from '../database/consts.js'; - -export class CreatedUserOutcome { - /** - * @type {number|null} - */ - user_id = null; -} - -export class SignupService extends BaseService { - /** - * Creates a new user. - * @async - * @param {object} params - The parameters for creating a new user. - * @param {object} [params.req] - The request object. If not specified, - * the request will be obtained from the context. If specified as null, request - * information will not be included for this signup. - * @param {boolean} [params.temporary] - Whether the user is a temporary user. - * @param {boolean} [params.oidc_only] - Whether the user created with OIDC - * @param {boolean} [params.send_confirmation_code] - Whether to send a confirmation code instead of a token by email - * @param {boolean} [params.assume_email_ownership] - If true, set email_confirmed=1 without sending verification (e.g. OIDC provider already verified). - * @param {string|null} params.username - The username of the user. - * @param {string|null} params.email - The email of the user. - * @param {string|null} params.password - The password of the user. - * @returns {Promise>} The outcome of the user creation. - */ - async create_new_user ({ - req, - temporary = false, - oidc_only = false, - send_confirmation_code = false, - assume_email_ownership = false, - username = null, - email = null, - password = null, - }) { - const outcome = new OutcomeObject(new CreatedUserOutcome()); - - if ( !req && req !== null ) { - req = Context.get('req'); - } - - let raw_email = email; - - if ( ! username ) { - throw new TypeError('username is a required parameter of create_new_user'); - } - if ( !temporary && !validate_nonEmpty_string(email) ) { - throw new TypeError('email is a required parameter of create_new_user'); - } - - // Temp users get default values; they cannot have emails or passwords - if ( temporary ) { - username = username ?? await generate_random_username(); - email = email ?? `${username}@nonexis.com`; - password = 'login-is-not-enabled'; // arbitrary, but accurate - } - - // Some installations of Puter are configured to disable - // signup or temporary users. In these cases, we will specify - // a failure message and abort creating a user. - { - const svc_featureFlag = this.services.get('feature-flag'); - const is_temp_users_disabled = - await svc_featureFlag.check('temp-users-disabled'); - const is_user_signup_disabled = - await svc_featureFlag.check('user-signup-disabled'); - - if ( is_user_signup_disabled && is_temp_users_disabled ) { - return outcome.fail( - 'User signup and Temporary users are disabled.', - 'signup.signup_and_temp_users_disabled', - ); - } - - if ( temporary && is_temp_users_disabled ) { - return outcome.fail( - 'Temporary users are disabled.', - 'signup.temp_users_disabled', - ); - } - - if ( !temporary && is_user_signup_disabled ) { - return outcome.fail( - 'User signup is disabled.', - 'signup.user_signup_disabled', - ); - } - } - - // Emit the `puter.signup` event - // NOTICE: conditional early return - { - const svc_event = this.services.get('event'); - const event = { allow: true, outcome }; - - if ( req ) { - event.ip = req.headers?.['x-forwarded-for'] || - req.connection?.remoteAddress; - event.user_agent = req.headers?.['user-agent']; - event.body = req.body; - } - - await svc_event.emit('puter.signup', event); - - if ( ! event.allow ) { - outcome.log('disallowed by a puter.signup listener'); - return outcome; - } - } - - if ( await username_exists(username) ) { - return outcome.fail( - 'Username already exists', - 'username_already_exists', - ); - } - - // These checks are required for non-temporary users - if ( ! temporary ) { - const db = this.services.get('database').get(DB_WRITE, 'create-user:not-temp-checks'); - const svc_cleanEmail = this.services.get('clean-email'); - raw_email = email; - - if ( ! email ) { - return outcome.fail( - 'An email address is required', - 'email_required', - ); - } - - email = svc_cleanEmail.clean(email); - if ( ! await svc_cleanEmail.validate(email) ) { - return outcome.fail( - 'This email cannot be used. Please try a different email address', - 'email_invalid', - ); - } - - let rows2 = await db.read(`SELECT EXISTS( - SELECT 1 FROM user WHERE (email=? OR clean_email=?) AND email_confirmed=1 AND password IS NOT NULL - ) AS email_exists`, [raw_email, email]); - if ( rows2[0].email_exists ) - { - return outcome.fail( - 'Email is already verified for another account', - 'email_already_exists', - ); - } - } - - // TODO: this is where referral goes. We might drop - // referral, so I'm leaving it out here for now. - - const user_uuid = uuidv4(); - const email_confirm_token = uuidv4(); - // TODO: `Math.random()` is not crypto-secure - const email_confirm_code = `${Math.floor(100000 + Math.random() * 900000)}`; - - const audit_metadata = {}; - if ( req ) { - audit_metadata.ip = req.connection.remoteAddress; - audit_metadata.ip_fwd = req.headers['x-forwarded-for']; - audit_metadata.user_agent = req.headers['user-agent']; - audit_metadata.origin = req.headers['origin']; - audit_metadata.server = this.global_config.server_id; - } - - { - const db = this.services.get('database').get(DB_WRITE, 'create-user:main-insert'); - - const insert_res = await db.write( - `INSERT INTO user - ( - username, email, clean_email, password, uuid, referrer, - email_confirm_code, email_confirm_token, email_confirmed, free_storage, - referred_by, audit_metadata, signup_ip, signup_ip_forwarded, - signup_user_agent, signup_origin, signup_server - ) - VALUES - (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - [ - // username - username, - // email - temporary ? null : raw_email, - // normalized email - temporary ? null : email, - // password - (temporary || oidc_only) ? null : await bcrypt.hash(password, 8), - // uuid - user_uuid, - // referrer - req?.body?.referrer ?? null, - // email_confirm_code - email_confirm_code, - // email_confirm_token - email_confirm_token, - // email_confirmed (1 when assume_email_ownership, else 0) - assume_email_ownership ? 1 : 0, - // free_storage - this.global_config.storage_capacity, - // referred_by - // TODO: we might remove referalls so I'mm leaving out - // the value for the `referred_by` field for now - null, - // audit_metadata - JSON.stringify(audit_metadata), - // signup_ip - req?.connection?.remoteAddress ?? null, - // signup_ip_fwd - req?.headers?.['x-forwarded-for'] ?? null, - // signup_user_agent - req?.headers?.['user-agent'] ?? null, - // signup_origin - req?.headers?.['origin'] ?? null, - // signup_server - this.global_config.server_id ?? null, - ], - ); - - // record activity (asynchronously) - db.write( - 'UPDATE `user` SET `last_activity_ts` = now() WHERE id=? LIMIT 1', - [insert_res.insertId], - ); - - // TODO: it would be VERY NICE if this was a calculated - // group membership instead of something we store in the DB - const svc_group = this.services.get('group'); - await svc_group.add_users({ - uid: temporary - ? this.global_config.default_temp_group - : this.global_config.default_user_group, - users: [username], - }); - - const user_id = insert_res.insertId; - outcome.infoObject.user_id = user_id; - - const [user] = await db.pread( - 'SELECT * FROM `user` WHERE `id` = ? LIMIT 1', - [user_id], - ); - - // TODO(???): should user login happen here or by caller? - { - // const { token } = await svc_auth.create_session_token(user, { - // req, - // }); - } - - if ( ! assume_email_ownership ) { - if ( send_confirmation_code ) { - send_email_verification_code(email_confirm_code, email); - } else { - send_email_verification_token(email_confirm_token, email, user_uuid); - } - } - - // TODO: This is where sending the referral code would - // usually happen but we might remove referral so I'm - // leaving it out for now. - const svc_user = this.services.get('user'); - await svc_user.generate_default_fsentries({ user }); - - // NOTE: `res.cookie` happens here in @signup.js but this - // should be handled by the caller over here. - - { - const svc_event = this.services.get('event'); - svc_event.emit('user.save_account', { user }); - } - - return outcome.success(); - } - } -} diff --git a/src/backend/src/services/auth/TokenService.js b/src/backend/src/services/auth/TokenService.js deleted file mode 100644 index 92629bd1c..000000000 --- a/src/backend/src/services/auth/TokenService.js +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const BaseService = require('../BaseService'); - -const def = o => { - for ( let k in o ) { - if ( typeof o[k] === 'string' ) { - o[k] = { short: o[k] }; - } - } - return { - fullkey_to_info: o, - short_to_fullkey: Object.keys(o).reduce((acc, key) => { - acc[o[key].short] = key; - return acc; - }, {}), - }; -}; - -const defv = o => { - return { - to_short: o, - to_long: Object.keys(o).reduce((acc, key) => { - acc[o[key]] = key; - return acc; - }, {}), - }; -}; - -const uuid_compression = prefix => ({ - encode: v => { - if ( prefix ) { - if ( ! v.startsWith(prefix) ) { - throw new Error(`Expected ${prefix} prefix`); - } - v = v.slice(prefix.length); - } - - const undecorated = v.replace(/-/g, ''); - const base64 = Buffer - .from(undecorated, 'hex') - .toString('base64'); - return base64; - }, - decode: v => { - // if already a uuid, return that - if ( v.includes('-') ) return v; - - const undecorated = Buffer - .from(v, 'base64') - .toString('hex'); - return (prefix ?? '') + [ - undecorated.slice(0, 8), - undecorated.slice(8, 12), - undecorated.slice(12, 16), - undecorated.slice(16, 20), - undecorated.slice(20), - ].join('-'); - }, -}); - -const compression = { - auth: def({ - uuid: { - short: 'u', - ...uuid_compression(), - }, - session: { - short: 's', - ...uuid_compression(), - }, - version: 'v', - type: { - short: 't', - values: defv({ - 'session': 's', - 'access-token': 't', - 'app-under-user': 'au', - }), - }, - user_uid: { - short: 'uu', - ...uuid_compression(), - }, - app_uid: { - short: 'au', - ...uuid_compression('app-'), - }, - }), -}; - -/** -* TokenService class for managing token creation and verification. -* This service extends the BaseService class and provides methods -* for signing and verifying JWTs, as well as compressing and decompressing -* payloads to and from a compact format. -*/ -class TokenService extends BaseService { - static MODULES = { - jwt: require('jsonwebtoken'), - }; - - /** - * Constructs a new TokenService instance and initializes the compression settings. - * This method is called when a TokenService object is created. - * - * @returns {void} - */ - _construct () { - this.compression = compression; - } - - /** - * Initializes the TokenService instance by setting the JWT secret - * from the global configuration. - * - * @function - * @returns {void} - * @throws {Error} Throws an error if the jwt_secret is not defined in global_config. - */ - _init () { - // TODO: move to service config - this.secret = this.global_config.jwt_secret; - } - - sign (scope, payload, options) { - const require = this.require; - - const jwt = require('jwt'); - const secret = this.secret; - return jwt.sign(payload, secret, options); - } - - verify (scope, token) { - const require = this.require; - - const jwt = require('jwt'); - const secret = this.secret; - - const context = this.compression[scope]; - const payload = jwt.verify(token, secret); - - const decoded = this._decompress_payload(context, payload); - return decoded; - } - - _compress_payload (context, payload) { - if ( ! context ) return payload; - - const fullkey_to_info = context.fullkey_to_info; - - const compressed = {}; - - for ( let fullkey in payload ) { - if ( ! fullkey_to_info[fullkey] ) { - compressed[fullkey] = payload[fullkey]; - continue; - } - - let k = fullkey, v = payload[fullkey]; - const compress_info = fullkey_to_info[fullkey]; - - if ( compress_info.short ) k = compress_info.short; - if ( compress_info.values && compress_info.values.to_short[v] ) { - v = compress_info.values.to_short[v]; - } else if ( compress_info.encode ) { - v = compress_info.encode(v); - } - - compressed[k] = v; - } - - return compressed; - } - - _decompress_payload (context, payload) { - if ( ! context ) return payload; - - const fullkey_to_info = context.fullkey_to_info; - const short_to_fullkey = context.short_to_fullkey; - - const decompressed = {}; - - for ( let short in payload ) { - if ( ! short_to_fullkey[short] ) { - decompressed[short] = payload[short]; - continue; - } - - let k = short, v = payload[short]; - const fullkey = short_to_fullkey[short]; - const compress_info = fullkey_to_info[fullkey]; - - if ( compress_info.short ) k = fullkey; - if ( compress_info.values && compress_info.values.to_long[v] ) { - v = compress_info.values.to_long[v]; - } else if ( compress_info.decode ) { - v = compress_info.decode(v); - } - - decompressed[k] = v; - } - - return decompressed; - } - -} - -module.exports = { TokenService }; diff --git a/src/backend/src/services/auth/TokenService.test.ts b/src/backend/src/services/auth/TokenService.test.ts deleted file mode 100644 index 4eb17623d..000000000 --- a/src/backend/src/services/auth/TokenService.test.ts +++ /dev/null @@ -1,153 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import * as jwt from 'jsonwebtoken'; -import { createTestKernel } from '../../../tools/test.mjs'; -import { TokenService } from './TokenService.js'; - -// Helper function to match the uuid_compression logic from TokenService -const uuid_compression = (prefix?: string) => ({ - encode: (v: string) => { - if ( prefix ) { - if ( ! v.startsWith(prefix) ) { - throw new Error(`Expected ${prefix} prefix`); - } - v = v.slice(prefix.length); - } - - const undecorated = v.replace(/-/g, ''); - const base64 = Buffer - .from(undecorated, 'hex') - .toString('base64'); - return base64; - }, - decode: (v: string) => { - // if already a uuid, return that - if ( v.includes('-') ) return v; - - const undecorated = Buffer - .from(v, 'base64') - .toString('hex'); - return (prefix ?? '') + [ - undecorated.slice(0, 8), - undecorated.slice(8, 12), - undecorated.slice(12, 16), - undecorated.slice(16, 20), - undecorated.slice(20), - ].join('-'); - }, -}); - -describe('TokenService', () => { - it('signs auth tokens using uncompressed claim names', async () => { - const testKernel = await createTestKernel({ - serviceMap: { - 'token': TokenService, - }, - }); - - const tokenService = testKernel.services!.get('token') as TokenService; - tokenService.secret = 'test-token-service-secret'; - const payload = { - type: 'session', - version: '0.0.0', - uuid: '843f1d83-3c30-48c7-8964-62aff1a912d0', - user_uid: '42e9c36b-8a53-4c3e-8e18-fe549b10a44d', - app_uid: 'app-c22ef816-edb6-47c5-8c41-31c6520fa9e6', - }; - - const token = tokenService.sign('auth', payload); - const decoded = jwt.verify(token, tokenService.secret as string) as jwt.JwtPayload & Record; - - expect(decoded.type).toBe(payload.type); - expect(decoded.version).toBe(payload.version); - expect(decoded.uuid).toBe(payload.uuid); - expect(decoded.user_uid).toBe(payload.user_uid); - expect(decoded.app_uid).toBe(payload.app_uid); - expect(decoded.t).toBeUndefined(); - expect(decoded.u).toBeUndefined(); - expect(decoded.uu).toBeUndefined(); - expect(decoded.au).toBeUndefined(); - }); - - it('verifies legacy compressed auth tokens', async () => { - const testKernel = await createTestKernel({ - serviceMap: { - 'token': TokenService, - }, - }); - - const tokenService = testKernel.services!.get('token') as TokenService; - tokenService.secret = 'test-token-service-secret'; - const payload = { - uuid: '843f1d83-3c30-48c7-8964-62aff1a912d0', - type: 'session', - user_uid: '42e9c36b-8a53-4c3e-8e18-fe549b10a44d', - app_uid: 'app-c22ef816-edb6-47c5-8c41-31c6520fa9e6', - }; - - const compressedPayload = tokenService._compress_payload(tokenService.compression!.auth, payload); - const token = jwt.sign(compressedPayload, tokenService.secret as string); - const decoded = tokenService.verify('auth', token); - - expect(decoded.uuid).toBe(payload.uuid); - expect(decoded.type).toBe(payload.type); - expect(decoded.user_uid).toBe(payload.user_uid); - expect(decoded.app_uid).toBe(payload.app_uid); - }); - - it('should compress and decompress payloads correctly', async () => { - const testKernel = await createTestKernel({ - serviceMap: { - 'token': TokenService, - }, - }); - - const tokenService = testKernel.services!.get('token') as TokenService; - tokenService.secret = 'test-token-service-secret'; - - const U1 = '843f1d83-3c30-48c7-8964-62aff1a912d0'; - const U2 = '42e9c36b-8a53-4c3e-8e18-fe549b10a44d'; - const U3 = 'app-c22ef816-edb6-47c5-8c41-31c6520fa9e6'; - - // Test compression - { - const context = tokenService.compression!.auth; - const payload = { - uuid: U1, - type: 'session', - user_uid: U2, - app_uid: U3, - }; - - const compressed = tokenService._compress_payload(context, payload); - expect(compressed.u).toBe(uuid_compression().encode(U1)); - expect(compressed.t).toBe('s'); - expect(compressed.uu).toBe(uuid_compression().encode(U2)); - expect(compressed.au).toBe(uuid_compression('app-').encode(U3)); - } - - // Test decompression - { - const context = tokenService.compression!.auth; - const payload = { - u: uuid_compression().encode(U1), - t: 's', - uu: uuid_compression().encode(U2), - au: uuid_compression('app-').encode(U3), - }; - - const decompressed = tokenService._decompress_payload(context, payload); - expect(decompressed.uuid).toBe(U1); - expect(decompressed.type).toBe('session'); - expect(decompressed.user_uid).toBe(U2); - expect(decompressed.app_uid).toBe(U3); - } - - // Test UUID preservation - { - const payload = { uuid: U1 }; - const compressed = tokenService._compress_payload(tokenService.compression!.auth, payload); - const decompressed = tokenService._decompress_payload(tokenService.compression!.auth, compressed); - expect(decompressed.uuid).toBe(U1); - } - }); -}); diff --git a/src/backend/src/services/auth/VirtualGroupService.js b/src/backend/src/services/auth/VirtualGroupService.js deleted file mode 100644 index f56b3f577..000000000 --- a/src/backend/src/services/auth/VirtualGroupService.js +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const BaseService = require('../BaseService'); - -/** -* Class representing a VirtualGroupService. -* This service extends the BaseService and provides methods to manage virtual groups, -* allowing for the registration of membership implicators and the retrieval of virtual group data. -*/ -class VirtualGroupService extends BaseService { - _construct () { - this.groups_ = {}; - this.membership_implicators_ = []; - } - - /** - * Registers a function that reports one or more groups that an actor - * should be considered a member of. - * - * @note this only applies to virtual groups, not persistent groups. - * - * @param {*} implicator - */ - register_membership_implicator (implicator) { - this.membership_implicators_.push(implicator); - } - - add_group (group) { - this.groups_[group.id] = group; - } - - /** - * Retrieves a list of virtual groups based on the provided actor, - * utilizing registered membership implicators to determine group membership. - * - * @param {Object} params - The parameters object. - * @param {Object} params.actor - The actor to check against the membership implicators. - * @returns {Array} An array of virtual group objects that the actor is a member of. - */ - get_virtual_groups ({ actor }) { - const groups_set = {}; - - for ( const implicator of this.membership_implicators_ ) { - const groups = implicator.run({ actor }); - for ( const group of groups ) { - groups_set[group] = true; - } - } - - const groups = Object.keys(groups_set).map( - id => this.groups_[id]); - - return groups; - } -} - -module.exports = { VirtualGroupService }; diff --git a/src/backend/src/services/auth/permissionConts.mjs b/src/backend/src/services/auth/permissionConts.mjs deleted file mode 100644 index 5a090dbe7..000000000 --- a/src/backend/src/services/auth/permissionConts.mjs +++ /dev/null @@ -1,2 +0,0 @@ -export const MANAGE_PERM_PREFIX = 'manage'; -export const PERM_KEY_PREFIX = 'perm'; \ No newline at end of file diff --git a/src/backend/src/services/auth/permissionUtils.bench.js b/src/backend/src/services/auth/permissionUtils.bench.js deleted file mode 100644 index 1a8dcbaf3..000000000 --- a/src/backend/src/services/auth/permissionUtils.bench.js +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { bench, describe } from 'vitest'; -import { PermissionUtil } from './permissionUtils.mjs'; - -// Sample permission strings for benchmarking -const simplePermissions = [ - 'fs:read', - 'fs:write', - 'app:execute', - 'user:profile:view', -]; - -const complexPermissions = [ - 'fs:aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee:read', - 'app:my-app-name:config:update', - 'user:john_doe:profile:avatar:upload', - 'service:database:table:users:column:email:read', -]; - -const escapedPermissions = [ - 'fs:path\\Cwith\\Ccolons:read', - 'app:name\\Cwith\\Cmany\\Ccolons:execute', - 'user:email\\Cexample@test.com:verify', -]; - -// Generate large batch of permissions for bulk testing -const generatePermissions = (count) => { - const perms = []; - for ( let i = 0; i < count; i++ ) { - perms.push(`service:svc${i}:action${i % 10}:resource${i % 100}`); - } - return perms; -}; - -const bulkPermissions = generatePermissions(100); - -describe('PermissionUtil.split()', () => { - bench('split simple permissions', () => { - for ( const perm of simplePermissions ) { - PermissionUtil.split(perm); - } - }); - - bench('split complex permissions', () => { - for ( const perm of complexPermissions ) { - PermissionUtil.split(perm); - } - }); - - bench('split escaped permissions', () => { - for ( const perm of escapedPermissions ) { - PermissionUtil.split(perm); - } - }); - - bench('split bulk permissions (100)', () => { - for ( const perm of bulkPermissions ) { - PermissionUtil.split(perm); - } - }); -}); - -describe('PermissionUtil.join()', () => { - const simpleComponents = [['fs', 'read'], ['app', 'execute'], ['user', 'view']]; - const complexComponents = [ - ['fs', 'uuid-here', 'read'], - ['service', 'database', 'table', 'users', 'read'], - ['app', 'my-app', 'config', 'setting', 'update'], - ]; - const needsEscaping = [ - ['fs', 'path:with:colons', 'read'], - ['user', 'email:test@example.com', 'verify'], - ]; - - bench('join simple components', () => { - for ( const comps of simpleComponents ) { - PermissionUtil.join(...comps); - } - }); - - bench('join complex components', () => { - for ( const comps of complexComponents ) { - PermissionUtil.join(...comps); - } - }); - - bench('join components needing escaping', () => { - for ( const comps of needsEscaping ) { - PermissionUtil.join(...comps); - } - }); -}); - -describe('PermissionUtil.escape_permission_component()', () => { - const noEscape = ['simple', 'another_one', 'with-dashes', 'CamelCase']; - const needsEscape = ['has:colon', 'multiple:colons:here', ':starts:with', 'ends:']; - - bench('escape components without special chars', () => { - for ( const comp of noEscape ) { - PermissionUtil.escape_permission_component(comp); - } - }); - - bench('escape components with colons', () => { - for ( const comp of needsEscape ) { - PermissionUtil.escape_permission_component(comp); - } - }); -}); - -describe('PermissionUtil.unescape_permission_component()', () => { - const noUnescape = ['simple', 'another_one', 'with-dashes']; - const needsUnescape = ['has\\Ccolon', 'multiple\\Ccolons\\Chere', '\\Cstarts\\Cwith']; - - bench('unescape components without escape sequences', () => { - for ( const comp of noUnescape ) { - PermissionUtil.unescape_permission_component(comp); - } - }); - - bench('unescape components with escape sequences', () => { - for ( const comp of needsUnescape ) { - PermissionUtil.unescape_permission_component(comp); - } - }); -}); - -describe('PermissionUtil roundtrip (split then join)', () => { - bench('roundtrip simple permissions', () => { - for ( const perm of simplePermissions ) { - const parts = PermissionUtil.split(perm); - PermissionUtil.join(...parts); - } - }); - - bench('roundtrip complex permissions', () => { - for ( const perm of complexPermissions ) { - const parts = PermissionUtil.split(perm); - PermissionUtil.join(...parts); - } - }); -}); - -describe('PermissionUtil vs native string operations (baseline)', () => { - const perm = 'service:database:table:users:column:email:read'; - - bench('PermissionUtil.split()', () => { - PermissionUtil.split(perm); - }); - - bench('native String.split() (baseline, no unescaping)', () => { - perm.split(':'); - }); - - bench('PermissionUtil.join()', () => { - PermissionUtil.join('service', 'database', 'table', 'users'); - }); - - bench('native Array.join() (baseline, no escaping)', () => { - ['service', 'database', 'table', 'users'].join(':'); - }); -}); diff --git a/src/backend/src/services/auth/permissionUtils.mjs b/src/backend/src/services/auth/permissionUtils.mjs deleted file mode 100644 index 737b53dc7..000000000 --- a/src/backend/src/services/auth/permissionUtils.mjs +++ /dev/null @@ -1,279 +0,0 @@ -import { MANAGE_PERM_PREFIX } from './permissionConts.mjs'; - -/** - * De-facto placeholder permission for permission rewrites that do not grant any access. - */ -export const PERMISSION_FOR_NOTHING_IN_PARTICULAR = 'permission-for-nothing-in-particular'; - -/** -* The PermissionUtil class provides utility methods for handling -* permission strings and operations, including splitting, joining, -* escaping, and unescaping permission components. It also includes -* functionality to convert permission reading structures into options. -*/ -export const PermissionUtil = { - /** - * Unescapes a permission component string, converting escape sequences to their literal characters. - * @param {string} component - The escaped permission component string. - * @returns {string} The unescaped permission component. - */ - unescape_permission_component (component) { - let unescaped_str = ''; - // Constant for unescaped permission component string - const STATE_NORMAL = {}; - // Constant for escaping special characters in permission strings - const STATE_ESCAPE = {}; - let state = STATE_NORMAL; - const const_escapes = { C: ':' }; - for ( let i = 0 ; i < component.length ; i++ ) { - const c = component[i]; - if ( state === STATE_NORMAL ) { - if ( c === '\\' ) { - state = STATE_ESCAPE; - } else { - unescaped_str += c; - } - } else if ( state === STATE_ESCAPE ) { - unescaped_str += Object.prototype.hasOwnProperty.call(const_escapes, c) - ? const_escapes[c] : c; - state = STATE_NORMAL; - } - } - return unescaped_str; - }, - - /** - * Escapes special characters in a permission component string for safe joining. - * @param {string} component - The permission component string to escape. - * @returns {string} The escaped permission component. - */ - escape_permission_component (component) { - let escaped_str = ''; - for ( let i = 0 ; i < component.length ; i++ ) { - const c = component[i]; - if ( c === ':' ) { - escaped_str += '\\C'; - continue; - } - escaped_str += c; - } - return escaped_str; - }, - - /** - * Splits a permission string into its component parts, unescaping each component. - * @param {string} permission - The permission string to split. - * @returns {string[]} Array of unescaped permission components. - */ - split (permission) { - return permission - .split(':') - .map(PermissionUtil.unescape_permission_component) - ; - }, - - /** - * Joins permission components into a single permission string, escaping as needed. - * @param {...string} components - The permission components to join. - * @returns {string} The escaped, joined permission string. - */ - join (...components) { - return components - .map(PermissionUtil.escape_permission_component) - .join(':') - ; - }, - - /** - * Exact key prefix for permission-scan cache entries belonging to a given app-under-user actor. - * Cache keys are built as join('permission-scan', actor.uid, 'options-list', ...); - * for app-under-user, actor.uid is 'app-under-user:{user_uuid}:{app_uid}' (colon-escaped in the key). - * Use with Redis SCAN MATCH prefix + '*' to delete only that actor's cache entries. - * - * @param {string} user_uuid - The user's UUID. - * @param {string} app_uid - The app UID. - * @returns {string} The exact key prefix for that actor's permission-scan cache keys. - */ - permission_scan_cache_prefix_for_app_under_user (user_uuid, app_uid) { - const actor_uid = `app-under-user:${user_uuid}:${app_uid}`; - return this.join('permission-scan', actor_uid, 'options-list'); - }, - - /** - * Converts a permission reading structure into an array of option objects. - * Recursively traverses the reading tree to collect all options with their associated path and data. - * @param {Array} reading - The permission reading structure to convert. - * @param {Object} [parameters={}] - Optional parameters for the conversion. - * @param {Array} [options=[]] - Accumulator for options (used internally for recursion). - * @param {Array} [extras=[]] - Extra data to include (used internally for recursion). - * @param {Array} [path=[]] - Current path in the reading tree (used internally for recursion). - * @returns {Array} Array of option objects with path and data. - */ - reading_to_options ( - // actual arguments - reading, - parameters = {}, - // recursion state - options = [], - extras = [], - path = [], - ) { - const to_path_item = finding => ({ - key: finding.key, - holder: finding.holder_username, - data: finding.data, - }); - for ( let finding of reading ) { - if ( finding.$ === 'option' ) { - path = [to_path_item(finding), ...path]; - options.push({ - ...finding, - data: [ - ...(finding.data ? [finding.data] : []), - ...extras, - ], - path, - }); - } - if ( finding.$ === 'path' ) { - if ( finding.has_terminal === false ) continue; - const new_extras = ( finding.data ) ? [ - finding.data, - ...extras, - ] : []; - const new_path = [to_path_item(finding), ...path]; - this.reading_to_options(finding.reading, parameters, options, new_extras, new_path); - } - } - return options; - }, - /** @type {(permission:string)=>boolean} */ - isManage (permission ) { - return permission.startsWith(`${MANAGE_PERM_PREFIX }:`); - }, -}; - -/** - * Permission rewriters are used to map one set of permission strings to another. - * These are invoked during permission scanning and when permissions are granted or revoked. - * - * For example, Puter's filesystem uses this to map 'fs:/some/path:mode' to - * 'fs:SOME-UUID:mode'. - * - * A rewriter is constructed using the static method PermissionRewriter.create({ matcher, rewriter }). - * The matcher is a function that takes a permission string and returns true if the rewriter should be applied. - * The rewriter is a function that takes a permission string and returns the rewritten permission string. - */ -export class PermissionRewriter { - static create ({ id, matcher, rewriter }) { - return new PermissionRewriter({ id, matcher, rewriter }); - } - - constructor ({ id, matcher, rewriter }) { - this.id = id; - this.matcher = matcher; - this.rewriter = rewriter; - } - - matches (permission) { - return this.matcher(permission); - } - - /** - * Determines if the given permission matches the criteria set for this rewriter. - * - * @param {string} permission - The permission string to check. - * @returns {boolean} - True if the permission matches, false otherwise. - */ - async rewrite (permission) { - return await this.rewriter(permission); - } -} - -/** - * Permission implicators are used to manage implicit permissions. - * It defines a method to check if a given permission is implicitly granted to an actor. - * - * For example, Puter's filesystem uses this to grant permission to a file if the specified - * 'actor' is the owner of the file. - * - * An implicator is constructed using the static method PermissionImplicator.create({ matcher, checker }). - * `matcher is a function that takes a permission string and returns true if the implicator should be applied. - * `checker` is a function that takes an actor and a permission string and returns true if the permission is implied. - * The actor and permission are passed to checker({ actor, permission }) as an object. - */ -export class PermissionImplicator { - static create ({ id, matcher, checker, ...options }) { - return new PermissionImplicator({ id, matcher, checker, options }); - } - - constructor ({ id, matcher, checker, options }) { - this.id = id; - this.matcher = matcher; - this.checker = checker; - this.options = options; - } - - matches (permission) { - return this.matcher(permission); - } - - /** - * Check if the permission is implied by this implicator - * @param {Actor} actor - * @param {string} permission - * @returns - */ - /** - * Rewrites a permission string if it matches any registered rewriter. - * @param {string} permission - The permission string to potentially rewrite. - * @returns {Promise} The possibly rewritten permission string. - */ - async check ({ actor, permission, recurse }) { - return await this.checker({ actor, permission, recurse }); - } -} - -/** - * Permission exploders are used to map any permission to a list of permissions - * which are considered to imply the specified permission. - * - * It uses a matcher function to determine if a permission should be exploded - * and an exploder function to perform the expansion. - * - * The exploder is constructed using the static method PermissionExploder.create({ matcher, explode }). - * The `matcher` is a function that takes a permission string and returns true if the exploder should be applied. - * The `explode` is a function that takes an actor and a permission string and returns a list of implied permissions. - * The actor and permission are passed to explode({ actor, permission }) as an object. - */ -export class PermissionExploder { - static create ({ id, matcher, exploder }) { - return new PermissionExploder({ id, matcher, exploder }); - } - - constructor ({ id, matcher, exploder }) { - this.id = id; - this.matcher = matcher; - this.exploder = exploder; - } - - matches (permission) { - return this.matcher(permission); - } - - /** - * Explodes a permission into a set of implied permissions. - * - * This method takes a permission string and an actor object, - * then uses the associated exploder function to derive additional - * permissions that are implied by the given permission. - * - * @param {Object} options - The options object containing: - * @param {Actor} options.actor - The actor requesting the permission explosion. - * @param {string} options.permission - The base permission to be exploded. - * @returns {Promise>} A promise resolving to an array of implied permissions. - */ - async explode ({ actor, permission }) { - return await this.exploder({ actor, permission }); - } -} \ No newline at end of file diff --git a/src/backend/src/services/database/BaseDatabaseAccessService.js b/src/backend/src/services/database/BaseDatabaseAccessService.js deleted file mode 100644 index 65a5b897f..000000000 --- a/src/backend/src/services/database/BaseDatabaseAccessService.js +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { BaseService } from '../BaseService.js'; -import { DB_WRITE, DB_READ } from './consts.js'; -import { spanify } from '../../util/otelutil.js'; - -/** -* BaseDatabaseAccessService class extends BaseService to provide -* an abstraction layer for database access, enabling operations -* like reading, writing, and inserting data while managing -* different database configurations and optimizations. -*/ -export class BaseDatabaseAccessService extends BaseService { - static DB_WRITE = DB_WRITE; - static DB_READ = DB_READ; - - case ( choices ) { - const engine_name = this.constructor.ENGINE_NAME; - if ( Object.prototype.hasOwnProperty.call(choices, engine_name) ) { - return choices[engine_name]; - } - return choices.otherwise; - } - - /** - * Retrieves the current instance of the service. - * This method currently returns `this`, but it is designed - * to allow for future enhancements such as auditing behavior - * or implementing service-specific optimizations for database - * interactions. - * - * @returns {BaseDatabaseAccessService} The current instance of the service. - */ - get () { - return this; - } - - read = spanify('database:read', async (query, params) => { - return await this._read(query, params); - }); - - /** - * requireRead will fallback to the primary database - * when a read-replica configuration is in use; - * otherwise it behaves the same as `read()`. - * - * @param {string} query - * @param {array} params - * @returns {Promise<*>} - */ - async tryHardRead (query, params) { - return this._tryHardRead(query, params); - } - - /** - * requireRead will fallback to the primary database - * when a read-replica configuration is in use by - * delegating to `tryHardRead()`. - * If the query returns no results, an error is thrown. - * - * @param {string} query - * @param {array} params - * @returns {Promise<*>} - */ - async requireRead (query, params) { - const results = this._tryHardRead(query, params); - if ( results.length === 0 ) { - throw new Error(`required read failed: ${ query}`); - } - return results; - } - - pread = spanify('database:pread', async (query, params) => { - return await this._read(query, params, { use_primary: true }); - }); - - write = spanify('database:write', async (query, params) => { - return await this._write(query, params); - }); - - async insert (table_name, data) { - const values = Object.values(data); - const sql = this._gen_insert_sql(table_name, data); - return this.write(sql, values); - } - - _gen_insert_sql (table_name, data) { - const cols = Object.keys(data); - return `INSERT INTO \`${ table_name }\` ` + - `(${ cols.map(str => `\`${ str }\``).join(', ') }) ` + - `VALUES (${ cols.map(() => '?').join(', ') })`; - } - - batch_write (statements) { - return this._batch_write(statements); - } -} \ No newline at end of file diff --git a/src/backend/src/services/database/SqliteDatabaseAccessService.js b/src/backend/src/services/database/SqliteDatabaseAccessService.js deleted file mode 100644 index 40d6849be..000000000 --- a/src/backend/src/services/database/SqliteDatabaseAccessService.js +++ /dev/null @@ -1,378 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { Context } = require('../../util/context'); -const { CompositeError } = require('../../util/errorutil'); -const structutil = require('../../util/structutil'); -const { BaseDatabaseAccessService } = require('./BaseDatabaseAccessService'); - -class SqliteDatabaseAccessService extends BaseDatabaseAccessService { - static ENGINE_NAME = 'sqlite'; - - static MODULES = { - // Documentation calls it 'Database'; it's new-able so - // I'll stick with their convention over ours. - Database: require('better-sqlite3'), - }; - - /** - * @description Method to handle database schema upgrades. - * This method checks the current database version against the available migration scripts and performs any necessary upgrades. - * @param {void} - * @returns {void} - */ - async _init () { - const require = this.require; - const Database = require('better-sqlite3'); - - const fs = require('fs'); - const path_ = require('path'); - const do_setup = this.config.path === ':memory:' || !fs.existsSync(this.config.path); - - this.db = new Database(this.config.path); - - const upgrade_files = []; - - const available_migrations = [ - [-1, [ - '0001_create-tables.sql', - '0002_add-default-apps.sql', - ]], - [0, [ - '0003_user-permissions.sql', - ]], - [1, [ - '0004_sessions.sql', - ]], - [2, [ - '0005_background-apps.sql', - ]], - [3, [ - '0006_update-apps.sql', - ]], - [4, [ - '0007_sessions.sql', - ]], - [5, [ - '0008_otp.sql', - ]], - [6, [ - '0009_app-prefix-fix.sql', - ]], - [7, [ - '0010_add-git-app.sql', - ]], - [8, [ - '0011_notification.sql', - ]], - [9, [ - '0012_appmetadata.sql', - ]], - [10, [ - '0013_protected-apps.sql', - ]], - [11, [ - '0014_share.sql', - ]], - [12, [ - '0015_group.sql', - ]], - [13, [ - '0016_group-permissions.sql', - ]], - [14, [ - '0017_publicdirs.sql', - ]], - [15, [ - '0018_fix-0003.sql', - ]], - [16, [ - '0019_fix-0016.sql', - ]], - [17, [ - '0020_dev-center.sql', - ]], - [18, [ - '0021_app-owner-id.sql', - ]], - [19, [ - '0022_dev-center-max.sql', - ]], - [20, [ - '0023_fix-kv.sql', - ]], - [21, [ - '0024_default-groups.sql', - ]], - [22, [ - '0025_system-user.dbmig.js', - ]], - [23, [ - '0026_user-groups.dbmig.js', - ]], - [25, [ - '0028_clean-email.sql', - ]], - [27, [ - '0030_comments.sql', - ]], - [28, [ - '0031_audit-meta.sql', - ]], - [29, [ - '0032_signup_metadata.sql', - ]], - [30, [ - '0033_ai-usage.sql', - ]], - [31, [ - '0034_app-redirect.sql', - ]], - [32, [ - '0035_threads.sql', - ]], - [33, [ - '0036_dev-to-app.sql', - ]], - [34, [ - '0038_custom-domains.sql', - ]], - [35, [ - '0039_add-expireAt-to-kv-store.sql', - ]], - [36, [ - '0040_add_user_metadata.sql', - ]], - [37, [ - '0041_add_unique_constraint_user_uuid.sql', - ]], - [38, [ - '0042_add_cloudflare_d1.sql', - ]], - [39, [ - '0043_add_dt.sql', - ]], - [40, [ - '0044_dev-center-godmode.sql', - ]], - [41, [ - '0045_user_oidc_providers.sql', - ]], - [42, [ - '0046_is-private-apps.sql', - ]], - ]; - - // Database upgrade logic - const HIGHEST_VERSION = - available_migrations[available_migrations.length - 1][0] + 1; - /** - * Upgrades the database schema to the specified version. - * - * @param {number} targetVersion - The target version to upgrade the database to. - * @returns {Promise} A promise that resolves when the database has been upgraded. - */ - const TARGET_VERSION = (() => { - const args = Context.get('args'); - if ( args?.['database-target-version'] ) { - return parseInt(args['database-target-version']); - } - return HIGHEST_VERSION; - })(); - - const [{ user_version }] = do_setup - ? [{ user_version: -1 }] - : await this._read('PRAGMA user_version'); - this.log.info(`database version: ${ user_version}`); - - for ( const [v_lt_or_eq, files] of available_migrations ) { - if ( v_lt_or_eq + 1 >= TARGET_VERSION && TARGET_VERSION !== HIGHEST_VERSION ) { - console.warn(`Early exit: target version set to ${TARGET_VERSION}`); - break; - } - if ( user_version <= v_lt_or_eq ) { - upgrade_files.push(...files); - } - } - - if ( upgrade_files.length > 0 ) { - console.debug(`Database out of date: ${this.config.path}`); - console.debug(`UPGRADING DATABASE: ${user_version} -> ${TARGET_VERSION}`); - console.debug(`${upgrade_files.length} .sql files to apply`); - - const sql_files = upgrade_files.map(p => path_.join(__dirname, 'sqlite_setup', p)); - const fs = require('fs'); - for ( const filename of sql_files ) { - const basename = path_.basename(filename); - const contents = fs.readFileSync(filename, 'utf8'); - switch ( path_.extname(filename) ) { - case '.sql': - { - const stmts = contents.split(/;\s*\n/); - for ( let i = 0; i < stmts.length; i++ ) { - if ( stmts[i].trim() === '' ) continue; - const stmt = `${stmts[i] };`; - try { - this.db.exec(stmt); - } catch ( e ) { - throw new CompositeError(`failed to apply: ${basename} at line ${i}`, e); - } - } - break; - } - case '.js': - try { - await this.run_js_migration_({ - filename, contents, - }); - } catch ( e ) { - throw new CompositeError(`failed to apply: ${basename}`, e); - } - break; - default: - throw new Error(`unrecognized migration type: ${filename}`); - } - } - - // Update version number - await this.db.exec(`PRAGMA user_version = ${TARGET_VERSION};`); - - this.log.info(`Database has been updated to version ${TARGET_VERSION}`); - } - - const svc_serverHealth = this.services.get('server-health'); - - /** - * Register a health check to ensure the SQLite schema matches the expected version. - */ - svc_serverHealth.add_check('sqlite', async () => { - const [{ user_version }] = await this.requireRead('PRAGMA user_version'); - if ( user_version !== TARGET_VERSION ) { - throw new Error(`Database version mismatch: expected ${TARGET_VERSION}, ` + - `got ${user_version}`); - } - }); - } - - async '__on_boot.consolidation' () { - } - - /** - * Implementation for prepared statements for READ operations. - */ - async _read (query, params = []) { - query = this.sqlite_transform_query_(query); - params = this.sqlite_transform_params_(params); - return this.db.prepare(query).all(...params); - } - - /** - * Implementation for prepared statements for READ operations. - * This method may perform additional steps to obtain the data, which - * is not applicable to the SQLite implementation. - */ - async _tryHardRead (query, params) { - return await this._read(query, params); - } - - /** - * Implementation for prepared statements for WRITE operations. - */ - async _write (query, params) { - query = this.sqlite_transform_query_(query); - params = this.sqlite_transform_params_(params); - - const stmt = this.db.prepare(query); - const info = stmt.run(...params); - - return { - insertId: info.lastInsertRowid, - anyRowsAffected: info.changes > 0, - }; - } - - /** - * This method initializes the SQLite database by checking if it exists, setting up the connection, and performing any necessary database upgrades based on the current version. - * - * @param {object} config - The configuration object for the database. - * @returns {Promise} A promise that resolves when the database is initialized. - */ - async _batch_write (entries) { - /** - * @description This method is used to execute SQL queries in batch mode. - * It accepts an array of objects, where each object contains a SQL query as the `statement` property and an array of parameters as the `values` property. - * The method executes each SQL query in the transaction block, ensuring that all operations are atomic. - * @param {Array<{statement: string, values: any[]}>} entries - An array of SQL queries and their corresponding parameters. - * @return {void} This method does not return any value. - */ - this.db.transaction(() => { - for ( let { statement, values } of entries ) { - statement = this.sqlite_transform_query_(statement); - values = this.sqlite_transform_params_(values); - this.db.prepare(statement).run(values); - } - })(); - } - - sqlite_transform_query_ (query) { - // replace `now()` with `datetime('now')` - query = query.replace(/now\(\)/g, 'datetime(\'now\')'); - - return query; - } - - sqlite_transform_params_ (params) { - return params.map(p => { - if ( typeof p === 'boolean' ) { - return p ? 1 : 0; - } - return p; - }); - } - - /** - * @description This method is responsible for performing database upgrades. It checks the current database version against the available versions and applies any necessary migrations. - * @param {object} options - Optional parameters for the method. - * @returns {Promise} A promise that resolves when the database upgrade is complete. - */ - async run_js_migration_ ({ filename: _filename, contents }) { - /** - * Method to run JavaScript migrations. This method is used to apply JavaScript code to the SQLite database during the upgrade process. - * - * @param {Object} options - An object containing the following properties: - * - `filename`: The name of the JavaScript file containing the migration code. - * - `contents`: The contents of the JavaScript file. - * - * @returns {Promise} A promise that resolves when the migration is completed. - */ - contents = `(async () => {${contents}})()`; - const vm = require('vm'); - const context = vm.createContext({ - read: this.read.bind(this), - write: this.write.bind(this), - log: this.log, - structutil, - }); - await vm.runInContext(contents, context); - } - -} - -module.exports = { - SqliteDatabaseAccessService, -}; diff --git a/src/backend/src/services/database/constructs.js b/src/backend/src/services/database/constructs.js deleted file mode 100644 index 6c34a7de3..000000000 --- a/src/backend/src/services/database/constructs.js +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -/** - * Statement simply holds a string that represents a SQL statement - * and an array of parameters to be used with the statement. - * - * This is meant to be used via the database access service when - * performing batch operations. - */ -const Statement = function Statement ({ statement, values }) { - // For now we just return an identical object. - return { - statement, values, - }; -}; - -module.exports = { - Statement, -}; diff --git a/src/backend/src/services/database/consts.js b/src/backend/src/services/database/consts.js deleted file mode 100644 index 43e24d67d..000000000 --- a/src/backend/src/services/database/consts.js +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -export const DB_READ = Symbol('DB_READ'); -export const DB_WRITE = Symbol('DB_WRITE'); diff --git a/src/backend/src/services/database/sqlite_setup/0026_user-groups.dbmig.js b/src/backend/src/services/database/sqlite_setup/0026_user-groups.dbmig.js deleted file mode 100644 index 1b0d9f205..000000000 --- a/src/backend/src/services/database/sqlite_setup/0026_user-groups.dbmig.js +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { insertId: temp_group_id } = await write('INSERT INTO `group` (`uid`, `owner_user_id`, `extra`, `metadata`) ' + - 'VALUES (?, ?, ?, ?)', -[ - 'b7220104-7905-4985-b996-649fdcdb3c8f', - 1, - '{"critical": true, "type": "default", "name": "temp"}', - '{"title": "Guest", "color": "#777777"}', -]); diff --git a/src/backend/src/services/drivers/CoercionService.js b/src/backend/src/services/drivers/CoercionService.js deleted file mode 100644 index 8ccb59407..000000000 --- a/src/backend/src/services/drivers/CoercionService.js +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const BaseService = require('../BaseService'); -const { TypeSpec } = require('./meta/Construct'); -const { TypedValue } = require('./meta/Runtime'); -const { secureAxiosRequest } = require('../../util/securehttp'); - -/** -* CoercionService class is responsible for handling coercion operations -* between TypedValue instances and their target TypeSpec representations. -* It provides functionality to construct and initialize coercions that -* can convert one type into another, based on specified produces and -* consumes specifications. -*/ -class CoercionService extends BaseService { - static MODULES = { - axios: require('axios'), - }; - - /** - * Attempt to coerce a TypedValue to a target TypeSpec. - * This method checks if the current TypedValue can be adapted to the specified target TypeSpec, - * using the available coercions defined in the service. It implements caching for previously calculated coercions. - * - * @param {*} target - the target TypeSpec - * @param {*} typed_value - the TypedValue to coerce - * @returns {TypedValue|undefined} - the coerced TypedValue, or undefined if coercion cannot be performed - */ - async _construct () { - this.coercions_ = []; - } - - /** - * Initializes the coercion service by populating the coercions_ array - * with predefined coercion rules that specify how TypedValues should - * be processed. This method should be called before any coercion - * operations are performed. - */ - async _init () { - this.coercions_.push({ - produces: { - $: 'stream', - content_type: 'image', - }, - consumes: { - $: 'string:url:web', - content_type: 'image', - }, - coerce: async typed_value => { - console.debug('coercion is running!'); - - const response = await secureAxiosRequest( - CoercionService.MODULES.axios, - typed_value.value, - { - responseType: 'stream', - }, - ); - - return new TypedValue({ - $: 'stream', - content_type: response.headers['content-type'], - }, response.data); - }, - }); - - this.coercions_.push({ - produces: { - $: 'stream', - content_type: 'video', - }, - consumes: { - $: 'string:url:web', - content_type: 'video', - }, - coerce: async typed_value => { - const response = await secureAxiosRequest( - CoercionService.MODULES.axios, - typed_value.value, - { - responseType: 'stream', - }, - ); - - return new TypedValue({ - $: 'stream', - content_type: response.headers['content-type'] ?? 'video/mp4', - }, response.data); - }, - }); - - // Add coercion for data URLs to streams - this.coercions_.push({ - produces: { - $: 'stream', - content_type: 'image', - }, - consumes: { - $: 'string:url:data', - content_type: 'image', - }, - coerce: async typed_value => { - const data_url = typed_value.value; - const data = data_url.split(',')[1]; - const buffer = Buffer.from(data, 'base64'); - - const { PassThrough } = require('stream'); - const stream = new PassThrough(); - stream.end(buffer); - - // Extract content type from data URL - const contentType = data_url.match(/data:([^;]+)/)?.[1] || 'image/png'; - - return new TypedValue({ - $: 'stream', - content_type: contentType, - }, stream); - }, - }); - } - - /** - * Attempt to coerce a TypedValue to a target TypeSpec. - * - * This method first adapts the target and the current type of the - * TypedValue. If they are equal, it returns the original TypedValue. - * Otherwise, it checks if the coercion has been calculated before, - * retrieves applicable coercions, and applies them to the TypedValue. - * - * DRY: this is implemented similarly to MultiValue.get. - * @param {*} target - the target TypeSpec - * @param {*} typed_value - the TypedValue to coerce - * @returns {TypedValue|undefined} - the coerced TypedValue, or undefined - */ - async coerce (target, typed_value) { - target = TypeSpec.adapt(target); - const target_hash = target.hash(); - - const current_type = TypeSpec.adapt(typed_value.type); - - if ( target.equals(current_type) ) { - return typed_value; - } - - if ( typed_value.calculated_coercions_[target_hash] ) { - return typed_value.calculated_coercions_[target_hash]; - } - - const coercions = this.coercions_.filter(coercion => { - const produces = TypeSpec.adapt(coercion.produces); - return target.equals(produces); - }); - - for ( const coercion of coercions ) { - const available = await this.coerce(coercion.consumes, typed_value); - if ( ! available ) continue; - const coerced = await coercion.coerce(available); - typed_value.calculated_coercions_[target_hash] = coerced; - return coerced; - } - - return undefined; - } -} - -module.exports = { CoercionService }; diff --git a/src/backend/src/services/drivers/DriverError.js b/src/backend/src/services/drivers/DriverError.js deleted file mode 100644 index a5c240735..000000000 --- a/src/backend/src/services/drivers/DriverError.js +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -/** -* Represents an error that occurs within the Driver system of Puter. -* This class provides a structured way to handle, report, and serialize errors -* originating from various drivers or backend services in Puter. -* @class DriverError -*/ -class DriverError { - static create (source) { - return new DriverError({ source }); - } - constructor ({ source, message }) { - this.source = source; - this.message = source?.message || message; - } - - /** - * Serializes the DriverError instance into a standardized object format. - * @returns {Object} An object with keys '$' for type identification and 'message' for error details. - * @note The method uses a custom type identifier for compatibility with Puter's error handling system. - */ - serialize () { - return { - $: 'heyputer:api/DriverError', - message: this.message, - }; - } -} - -module.exports = { - DriverError, -}; diff --git a/src/backend/src/services/drivers/DriverService.js b/src/backend/src/services/drivers/DriverService.js deleted file mode 100644 index 47c3b09b6..000000000 --- a/src/backend/src/services/drivers/DriverService.js +++ /dev/null @@ -1,679 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { Context } = require('../../util/context'); -const APIError = require('../../api/APIError'); -const { DriverError } = require('./DriverError'); -const { TypedValue } = require('./meta/Runtime'); -const BaseService = require('../BaseService'); -const { PermissionUtil } = require('../auth/permissionUtils.mjs'); -const { Invoker } = require('../../../../putility/src/libs/invoker'); -const { get_user } = require('../../helpers'); -const { AdvancedBase } = require('@heyputer/putility'); -const { span } = require('../../util/otelutil'); - -const strutil = require('@heyputer/putility').libs.string; - -/** - * DriverService provides the functionality of Puter drivers. - * This class is responsible for managing and interacting with Puter drivers. - * It provides methods for registering drivers, calling driver methods, and handling driver errors. - */ -class DriverService extends BaseService { - static CONCERN = 'drivers'; - - static MODULES = { - types: require('./types'), - }; - - // 'IMPLEMENTS' here makes DriverService itself a driver - static IMPLEMENTS = { - driver: { - async usage () { - const actor = Context.get('actor'); - - const usages = { - user: {}, // map[str(iface:method)]{date,count,max} - apps: {}, // []{app,map[str(iface:method)]{date,count,max}} - app_objects: {}, - usages: [], - }; - - const event = { - actor, - usages: [], - }; - const svc_event = this.services.get('event'); - await svc_event.emit('usages.query', event); - usages.usages = event.usages; - - for ( const k in usages.apps ) { - usages.apps[k] = Object.values(usages.apps[k]); - } - - return { - // Usage endpoint reports these, but the driver doesn't need to - // user: Object.values(usages.user), - // apps: usages.apps, - // app_objects: usages.app_objects, - - // This is the main "usages" object - usages: usages.usages, - }; - }, - }, - }; - - _construct () { - this.drivers = {}; - this.interface_to_implementation = {}; - this.interface_to_test_service = {}; - this.service_aliases = {}; - this.interface_service_aliases = {}; - } - - _init () { - const svc_registry = this.services.get('registry'); - svc_registry.register_collection(''); - - const { quot } = strutil; - const svc_apiError = this.services.get('api-error'); - - /** - * There are registered into the new APIErrorService which allows for - * better sepration of concerns between APIError and the services which. - * depend on it. - */ - svc_apiError.register({ - 'missing_required_argument': { - status: 400, - message: ({ interface_name, method_name, arg_name }) => - `Missing required argument ${quot(arg_name)} for method ${quot(method_name)} on interface ${quot(interface_name)}`, - }, - 'argument_consolidation_failed': { - status: 400, - message: ({ interface_name, method_name, arg_name, message }) => - `Failed to parse or process argument ${quot(arg_name)} for method ${quot(method_name)} on interface ${quot(interface_name)}: ${message}`, - }, - 'interface_not_found': { - status: 404, - message: ({ interface_name }) => `Interface not found: ${quot(interface_name)}`, - }, - 'method_not_found': { - status: 404, - message: ({ interface_name, method_name }) => `Method not found: ${quot(method_name)} on interface ${quot(interface_name)}`, - }, - 'no_implementation_available': { - status: 502, - message: ({ iface, interface_name, driver }) => { - const has_interface = (iface ?? interface_name) !== undefined; - const target_type = has_interface ? 'interface' : 'driver'; - const target_name = quot(iface ?? interface_name ?? driver); - return `No implementation available for ${target_type} ${target_name}.`; - }, - }, - }); - } - - async '__on_boot.consolidation' () { - const svc_registry = this.services.get('registry'); - const svc_event = this.services.get('event'); - - { - const col_interfaces = svc_registry.get('interfaces'); - const event = { - createInterface (name, definition) { - col_interfaces.set(name, definition); - }, - }; - await svc_event.emit('create.interfaces', event); - } - - { - const col_drivers = svc_registry.get('drivers'); - const event = { - createDriver (ifaceName, implName, definition) { - col_drivers.set(`${ifaceName}:${implName}`, definition); - }, - }; - await svc_event.emit('create.drivers', event); - } - } - - /** - * This method is responsible for registering collections in the service registry. - * It registers 'interfaces', 'drivers', and 'types' collections. - */ - async '__on_registry.collections' () { - const svc_registry = this.services.get('registry'); - svc_registry.register_collection('interfaces'); - svc_registry.register_collection('drivers'); - svc_registry.register_collection('types'); - } - /** - * This method is responsible for initializing the collections in the driver service registry. - * It registers 'interfaces', 'drivers', and 'types' collections. - * It also populates the 'interfaces' collection with default interfaces and registers the collections with the driver service registry. - */ - async '__on_registry.entries' () { - const services = this.services; - const svc_registry = services.get('registry'); - const col_interfaces = svc_registry.get('interfaces'); - const col_drivers = svc_registry.get('drivers'); - const col_types = svc_registry.get('types'); - { - const types = this.modules.types; - for ( const k in types ) { - col_types.set(k, types[k]); - } - } - await services.emit( - 'driver.register.interfaces', - { col_interfaces }, - ); - - await services.emit( - 'driver.register.drivers', - { col_drivers }, - ); - } - - // This is a bit meta: we register the "driver" driver interface. - // This allows DriverService to be a driver called "driver". - // The driver drivers allows checking metered usage for drivers, - // and in the future may provide other driver-related functions. - async '__on_driver.register.interfaces' () { - const svc_registry = this.services.get('registry'); - const col_interfaces = svc_registry.get('interfaces'); - - col_interfaces.set('driver', { - description: 'provides functions for managing Puter drivers', - methods: { - usage: { - description: 'get usage information for drivers', - parameters: {}, - result: { type: 'json' }, - }, - }, - }); - } - - register_driver (interface_name, implementation) { - this.interface_to_implementation[interface_name] = implementation; - } - - register_test_service (interface_name, service_name) { - this.interface_to_test_service[interface_name] = service_name; - } - - register_service_alias (service_name, alias, options = {}) { - const iface = options.iface; - if ( iface ) { - if ( ! this.interface_service_aliases[iface] ) { - this.interface_service_aliases[iface] = {}; - } - this.interface_service_aliases[iface][alias] = service_name; - return; - } - this.service_aliases[alias] = service_name; - } - - get_default_implementation (interface_name) { - // If there's a hardcoded implementation, use that - // (^ temporary, until all are migrated) - if ( Object.prototype.hasOwnProperty.call(this.interface_to_implementation, interface_name) ) { - return this.interface_to_implementation[interface_name]; - } - } - - /** - * This method is responsible for calling the specified driver method with the given arguments. - * It first processes the arguments to ensure they are in the correct format, then it checks if the driver and method exist, - * and if the user has the necessary permissions to call them. If all checks pass, it calls the method and returns the result. - * If any check fails, it throws an error or returns an error response. - * - * @param {Object} o - An object containing the driver name, interface name, method name, and arguments. - * @returns {Promise} A promise that resolves to an object containing the result of the method call, - * or rejects with an error if any check fails. - */ - async call (o) { - try { - return await this._call(o); - } catch ( e ) { - this.log.error(`Driver error response: ${ e.toString().slice(0, 100)}${e.toString().length > 100 ? '...' : ''}`); - if ( ! (e instanceof APIError) ) { - this.errors.report('driver', { - source: e, - trace: true, - }); - } - return this._driver_response_from_error(e); - } - } - - /** - * This method is responsible for making a call to a driver using its implementation and interface. - * It handles various aspects such as argument processing, permission checks, and invoking the driver's method. - * It returns a promise that resolves to an object containing the result, metadata, and an error if one occurred. - */ - async _call ({ driver, iface, method, args }) { - const processed_args = await this._process_args(iface, method, args); - const test_mode = Context.get('test_mode'); - if ( test_mode ) { - processed_args.test_mode = true; - } - - const actor = Context.get('actor'); - if ( ! actor ) { - throw Error('actor not found in context'); - } - - // There used to be only an 'interface' parameter but no 'driver' - // parameter. To support outdated clients we use this hard-coded - // table to map interfaces to default drivers. - const iface_to_driver = { - 'puter-ocr': 'aws-textract', - 'puter-tts': 'aws-polly', - 'puter-speech2speech': 'elevenlabs-voice-changer', - 'puter-speech2txt': 'openai-speech2txt', - 'puter-chat-completion': 'openai-completion', - 'puter-image-generation': 'openai-image-generation', - 'puter-video-generation': 'ai-video', - 'puter-apps': 'es:app', - 'puter-subdomains': 'es:subdomain', - 'puter-notifications': 'es:notification', - }; - - driver = driver ?? iface_to_driver[iface] ?? iface; - - // For these ones, the interface specified actually specifies the - // specificc driver to use. - const iface_to_iface = { - 'puter-apps': 'crud-q', - 'puter-subdomains': 'crud-q', - 'puter-notifications': 'crud-q', - }; - iface = iface_to_iface[iface] ?? iface; - - let skip_usage = false; - if ( test_mode && this.interface_to_test_service[iface] ) { - driver = this.interface_to_test_service[iface]; - } - - const client_driver_call = { - intended_service: driver, - response_metadata: {}, - test_mode, - }; - const iface_aliases = this.interface_service_aliases[iface]; - if ( iface_aliases && iface_aliases[driver] ) { - driver = iface_aliases[driver]; - } else { - driver = this.service_aliases[driver] ?? driver; - } - - const service = this.get_service_or_throw_(driver, iface); - - const caps = service.as('driver-capabilities'); - if ( test_mode && caps && caps.supports_test_mode(iface, method) ) { - skip_usage = true; - } - - const svc_event = this.services.get('event'); - const event = {}; - event.call_details = { - service: driver, - iface, - method, - args, - skip_usage, - }; - event.context = Context.sub({ - client_driver_call, - call_details: event.call_details, - }); - - svc_event.emit('driver.create-call-context', event); - - return await span(`driver:${driver}:${iface}:${method}`, async () => { - return event.context.arun(async () => { - const result = await this.call_new_({ - actor, - service, - service_name: driver, - iface, - method, - args: processed_args, - skip_usage, - }); - result.metadata = client_driver_call.response_metadata; - return result; - }); - }); - } - - /** - * Reserved for future implementation of "best policy" selection. - * For now, it just returns the first root option's path. - */ - async get_policies_for_option_ (option) { - // NOT FINAL: before implementing cascading monthly usage, - // this return will be removed and the code below it will - // be uncommented - return option.path; - /* - const svc_systemData = this.services.get('system-data'); - const svc_su = this.services.get('su'); - - const policies = await Promise.all(option.path.map(async path_node => { - const policy = await svc_su.sudo(async () => { - return await svc_systemData.interpret(option.data); - }); - return { - ...path_node, - policy, - }; - })); - return policies; - */ - } - - /** - * Reserved for future implementation of "best policy" selection. - * For now, this just returns the first option of a list of options. - * - * @param {*} options - * @returns - */ - async select_best_option_ (options) { - return options[0]; - } - - /** - * This method is used to call a driver method with provided arguments. - * It first processes the arguments to ensure they are of the correct type and format. - * Then it checks if the method exists in the interface and if the driver service for that interface is available. - * If the method exists and the driver service is available, it calls the method using the driver service. - * If the method does not exist or the driver service is not available, it throws an error. - * @param {object} o - Object containing driver, interface, method and arguments - * @returns {Promise} - Promise that resolves to an object containing the result of the driver method call - */ - async call_new_ ({ - actor, - service, - service_name, - iface, method, args, - _skip_usage, - }) { - if ( ! service ) { - service = this.services.get(service_name); - } - - const svc_permission = this.services.get('permission'); - const reading = await svc_permission.scan( - actor, - PermissionUtil.join('service', service_name, 'ii', iface), - ); - const options = PermissionUtil.reading_to_options(reading); - if ( options.length <= 0 ) { - throw APIError.create('forbidden'); - } - const option = await this.select_best_option_(options); - const policies = await this.get_policies_for_option_(option); - - // NOT FINAL: For now we apply monthly usage logic - // to the first holder of the permission. Later this - // will be changed so monthly usage can cascade across - // multiple actors. I decided not to implement this - // immediately because it's a hefty time sink and it's - // going to be some time before we can offer this feature - // to the end-user either way. - - let effective_policy = null; - for ( const policy of policies ) { - if ( policy.holder ) { - effective_policy = policy; - break; - } - } - - if ( ! effective_policy ) { - throw new Error('policies with no effective user are not yet ' + - 'supported'); - } - - const policy_holder = await get_user({ username: effective_policy.holder }); - - // NOT FINAL: this will be handled by 'get_policies_for_option_' - // when cascading monthly usage is implemented. - const svc_systemData = this.services.get('system-data'); - const svc_su = this.services.get('su'); - effective_policy = await svc_su.sudo(async () => { - return await svc_systemData.interpret(effective_policy.data); - }); - - effective_policy = effective_policy.policy; - - this.log.debug('Invoking Driver Call', { - service_name, - iface, - method, - policy: effective_policy, - }); - - const invoker = Invoker.create({ - decorators: [ - { - name: 'enforce logical rate-limit', - on_call: async args => { - if ( ! effective_policy?.['rate-limit'] ) return args; - const svc_su = this.services.get('su'); - const svc_rateLimit = this.services.get('rate-limit'); - - await svc_su.sudo(policy_holder, async () => { - await svc_rateLimit.check_and_increment( - `V1:${service_name}:${iface}:${method}`, - effective_policy['rate-limit'].max, - effective_policy['rate-limit'].period, - ); - }); - return args; - }, - }, - { - name: 'add metadata', - on_return: async result => { - const service_meta = {}; - if ( service.list_traits().includes('version') ) { - service_meta.version = service.as('version').get_version(); - } - return { - success: true, - service: { - ...service_meta, - name: service_name, - }, - result, - }; - }, - }, - { - name: 'result coercion', - on_return: async (result) => { - if ( result instanceof TypedValue ) { - const svc_registry = this.services.get('registry'); - const c_interfaces = svc_registry.get('interfaces'); - - const interface_ = c_interfaces.get(iface); - const method_spec = interface_.methods[method]; - let desired_type = - method_spec.result_choices - ? method_spec.result_choices[0].type - : method_spec.result.type - ; - const svc_coercion = this.services.get('coercion'); - const coerced = await svc_coercion.coerce(desired_type, result); - if ( coerced ) { - result = coerced; - } - } - return result; - }, - }, - ], - delegate: async (args) => { - return await service.as(iface)[method](args); - }, - }); - return await invoker.run(args); - } - - /** - * This method converts an error into an appropriate driver response. - */ - async _driver_response_from_error (e, meta) { - let serializable = (e instanceof APIError) || (e instanceof DriverError); - return { - success: false, - ...meta, - error: serializable ? e.serialize() : e.message, - }; - } - - /** - * Processes arguments according to the argument types specified - * on the interface (in interfaces.js). The behavior of types is - * defined in types.js - * @param {*} interface_name - the name of the interface - * @param {*} method_name - the name of the method - * @param {*} args - raw argument values from request body - * @returns - */ - async _process_args (interface_name, method_name, args) { - const svc_registry = this.services.get('registry'); - const c_interfaces = svc_registry.get('interfaces'); - const c_types = svc_registry.get('types'); - - const svc_apiError = this.services.get('api-error'); - - // Note: 'interface' is a strict mode reserved word. - const interface_ = c_interfaces.get(interface_name); - if ( ! interface_ ) { - throw svc_apiError.create('interface_not_found', { interface_name }); - } - - const processed_args = {}; - const method = interface_.methods[method_name]; - if ( ! method ) { - throw svc_apiError.create('method_not_found', { interface_name, method_name }); - } - - if ( Object.prototype.hasOwnProperty.call(method, 'default_parameter') && (typeof args !== 'object' || Array.isArray(args)) ) { - args = { [method.default_parameter]: args }; - } - - for ( const [arg_name, arg_descriptor] of Object.entries(method.parameters) ) { - const arg_value = arg_name === '*' ? args : args[arg_name]; - const arg_behaviour = c_types.get(arg_descriptor.type); - - // TODO: eventually put this in arg behaviour base class. - // There's a particular way I want to do this that involves - // a trait for extensible behaviour. - if ( arg_value === undefined && arg_descriptor.required ) { - throw svc_apiError.create('missing_required_argument', { - interface_name, - method_name, - arg_name, - }); - } - - const ctx = Context.get(); - - try { - processed_args[arg_name] = await arg_behaviour.consolidate(ctx, arg_value, { arg_descriptor, arg_name }); - } catch ( e ) { - throw svc_apiError.create('argument_consolidation_failed', { - interface_name, - method_name, - arg_name, - message: e.message, - }); - } - } - - if ( typeof processed_args['*'] === 'object' ) { - for ( const k in processed_args['*'] ) { - processed_args[k] = processed_args['*'][k]; - } - delete processed_args['*']; - } - - return processed_args; - } - - /** - * This method retrieves the driver service for the provided interface name. - * It first checks if the driver service already exists in the registry, - * and if not, it throws an error. - * - * @param {string} interfaceName - The name of the interface for which to retrieve the driver service. - * @returns {DriverService} The driver service instance for the provided interface. - */ - get_service_or_throw_ (name, iface) { - let driver_service_exists = (() => { - return this.services.has(name) && - this.services.get(name).list_traits() - .includes(iface); - })(); - - if ( driver_service_exists ) { - return this.services.get(name); - } - - const svc_registry = this.services.get('registry'); - const col_drivers = svc_registry.get('drivers'); - let maybe_driver = col_drivers.get(`${iface}:${name}`); - if ( maybe_driver ) { - const org = maybe_driver; - const impl = Object.create(org); - - // TraitsFeature also uses `in `, so this should cover - // all the methods that would get re-"`bind`'d" - for ( const k in org ) { - if ( ! (typeof org[k] === 'function') ) continue; - impl[k] = org[k].bind(org); - } - maybe_driver = class extends AdvancedBase { - static IMPLEMENTS = { - [iface]: impl, - }; - }; - Object.defineProperty(maybe_driver, 'name', { - value: `driver:${iface}:${name}`, - }); - return new maybe_driver(); - } - - const svc_apiError = this.services.get('api-error'); - throw svc_apiError.create('no_implementation_available', { iface }); - } -} - -module.exports = { - DriverService, -}; diff --git a/src/backend/src/services/drivers/DriverUsagePolicyService.js b/src/backend/src/services/drivers/DriverUsagePolicyService.js deleted file mode 100644 index 0d58b8c61..000000000 --- a/src/backend/src/services/drivers/DriverUsagePolicyService.js +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { PermissionUtil } = require('../auth/permissionUtils.mjs'); -const BaseService = require('../BaseService'); - -// DO WE HAVE enough information to get the policy for the newer drivers? -// - looks like it: service:: - -/** -* Class representing the DriverUsagePolicyService. -* This service manages the retrieval and application of usage policies -* for drivers, handling permission checks and policy interpretation -* using the provided service architecture. -*/ -class DriverUsagePolicyService extends BaseService { - /** - * Retrieves the usage policies for a given option. - * - * This method takes an option containing a path and returns the corresponding - * policies. Note that the implementation is not final and may include cascading - * monthly usage logic in the future. - * - * @param {Object} option - The option for which policies are to be retrieved. - * @param {Array} option.path - The path representing the request to get policies. - * @returns {Promise} A promise that resolves to the policies associated with the given option. - */ - async get_policies_for_option_ (option) { - // NOT FINAL: before implementing cascading monthly usage, - // this return will be removed and the code below it will - // be uncommented - return option.path; - /* - const svc_systemData = this.services.get('system-data'); - const svc_su = this.services.get('su'); - - const policies = await Promise.all(option.path.map(async path_node => { - const policy = await svc_su.sudo(async () => { - return await svc_systemData.interpret(option.data); - }); - return { - ...path_node, - policy, - }; - })); - return policies; - */ - } - - /** - * Selects the best option from the provided list of options. - * - * This method assumes that the options array is not empty and will - * return the first option found. It does not perform any sorting - * or decision-making beyond this. - * - * @param {Array} options - An array of options to select from. - * @returns {Object} The best option from the provided list. - */ - async select_best_option_ (options) { - return options[0]; - } - - // TODO: DRY: This is identical to the method of the same name in - // DriverService, except after the line with a comment containing - // the string "[DEVIATION]". - /** - * Retrieves the effective policy for a given actor, service name, and trait name. - * This method checks for permissions associated with the provided actor and then generates - * a list of policies based on the permissions read. If no policies are found, it returns - * `undefined`. Otherwise, it selects the best option and retrieves the corresponding - * policies. - * - * @param {Object} parameters - The parameters for the method. - * @param {string} parameters.actor - The actor for which the policy is being requested. - * @param {string} parameters.service_name - The name of the service to which the policy applies. - * @param {string} parameters.trait_name - The name of the trait for which the effective policy is needed. - * @returns {Object|undefined} - Returns the effective policy object or `undefined` if no policies are available. - */ - async get_effective_policy ({ actor, service_name, trait_name }) { - const svc_permission = this.services.get('permission'); - const reading = await svc_permission.scan( - actor, - PermissionUtil.join('service', service_name, 'ii', trait_name), - ); - const options = PermissionUtil.reading_to_options(reading); - if ( options.length <= 0 ) { - return undefined; - } - const option = await this.select_best_option_(options); - const policies = await this.get_policies_for_option_(option); - - // NOT FINAL: For now we apply monthly usage logic - // to the first holder of the permission. Later this - // will be changed so monthly usage can cascade across - // multiple actors. I decided not to implement this - // immediately because it's a hefty time sink and it's - // going to be some time before we can offer this feature - // to the end-user either way. - - let effective_policy = null; - for ( const policy of policies ) { - if ( policy.holder ) { - effective_policy = policy; - break; - } - } - - // === [DEVIATION] In DriverService, this is part of call_new_ === - const svc_systemData = this.services.get('system-data'); - const svc_su = this.services.get('su'); - /** - * Retrieves and interprets the effective policy for a given holder. - * Utilizes system data and super-user privileges to interpret the policy data. - * - * @param {Object} effective_policy - The policy object for the current holder. - * @returns {Promise} - The interpreted policy object after applying the necessary logic. - */ - effective_policy = await svc_su.sudo(async () => { - return await svc_systemData.interpret(effective_policy.data); - }); - - effective_policy = effective_policy.policy; - - return effective_policy; - } -} - -module.exports = { - DriverUsagePolicyService, -}; diff --git a/src/backend/src/services/drivers/FileFacade.js b/src/backend/src/services/drivers/FileFacade.js deleted file mode 100644 index 6093b283e..000000000 --- a/src/backend/src/services/drivers/FileFacade.js +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('../../../../putility'); -const { Context } = require('../../util/context'); -const { MultiValue } = require('../../util/multivalue'); -const { stream_to_buffer } = require('../../util/streamutil'); -const { PassThrough } = require('stream'); -const { LLRead } = require('../../deprecated/filesystem/ll_operations/ll_read'); -const { secureAxiosRequest } = require('../../util/securehttp'); - -/** -* @class FileFacade -* This class is used to provide a unified interface for -* passing files through the Puter Driver API, and avoiding -* unnecessary work such as downloading the file from S3 -* (when a Puter file is specified) in case the underlying -* implementation can accept S3 bucket information instead -* of the file's contents. -* @extends AdvancedBase -* @description This class provides a unified interface for passing files through the Puter Driver API. It aims to avoid unnecessary operations such as downloading files from S3 when a Puter file is specified, especially if the underlying implementation can accept S3 bucket information instead of the file's contents. -*/ -class FileFacade extends AdvancedBase { - static OUT_TYPES = { - S3_INFO: { key: 's3-info' }, - STREAM: { key: 'stream' }, - }; - - static MODULES = { - axios: require('axios'), - }; - - constructor (...a) { - super(...a); - - this.values = new MultiValue(); - - this.values.add_factory('fs-node', 'uid', async uid => { - const context = Context.get(); - const services = context.get('services'); - const svc_filesystem = services.get('filesystem'); - const fsNode = await svc_filesystem.node({ uid }); - return fsNode; - }); - - this.values.add_factory('fs-node', 'path', async path => { - const context = Context.get(); - const services = context.get('services'); - const svc_filesystem = services.get('filesystem'); - const fsNode = await svc_filesystem.node({ path }); - return fsNode; - }); - - this.values.add_factory('s3-info', 'fs-node', async fsNode => { - try { - return await fsNode.get('s3:location'); - } catch (e) { - return null; - } - }); - - this.values.add_factory('stream', 'fs-node', async fsNode => { - if ( ! await fsNode.exists() ) return null; - - const context = Context.get(); - - const ll_read = new LLRead(); - const stream = await ll_read.run({ - actor: context.get('actor'), - fsNode, - }); - - return stream; - }); - - this.values.add_factory('stream', 'web_url', async web_url => { - const response = await secureAxiosRequest( - FileFacade.MODULES.axios, - web_url, - { - responseType: 'stream', - }, - ); - - return response.data; - }); - - this.values.add_factory('stream', 'data_url', async data_url => { - const data = data_url.split(',')[1]; - const buffer = Buffer.from(data, 'base64'); - const stream = new PassThrough(); - stream.end(buffer); - return stream; - }); - - this.values.add_factory('buffer', 'stream', async stream => { - return await stream_to_buffer(stream); - }); - } - - set (k, v) { - this.values.set(k, v); - } - get (k) { - return this.values.get(k); - } - -} - -module.exports = { - FileFacade, -}; diff --git a/src/backend/src/services/drivers/meta/Construct.js b/src/backend/src/services/drivers/meta/Construct.js deleted file mode 100644 index 005a7ca41..000000000 --- a/src/backend/src/services/drivers/meta/Construct.js +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { BasicBase } = require('../../../../../putility/src/bases/BasicBase'); -const types = require('../types'); -const { hash_serializable_object, stringify_serializable_object } = require('../../../util/datautil'); - -/** -* @class Construct -* @extends BasicBase -* @classdesc The Construct class is a base class for building various types of constructs. -* It extends the BasicBase class and provides a framework for processing and serializing -* constructs. This class includes methods for processing raw data and serializing the -* constructed object into a JSON-compatible format. -*/ -class Construct extends BasicBase { - constructor (json, { name } = {}) { - super(); - this.name = name; - this.raw = json; - this.__process(); - } - - /** - * Processes the raw JSON data to initialize the object's properties. - * If a process function is defined, it will be executed with the raw JSON data. - */ - __process () { - if ( this._process ) this._process(this.raw); - } - - /** - * Serializes the properties of the object into a JSON-compatible format. - * - * This method iterates over the properties defined in the static `PROPERTIES` - * object and serializes each property according to its type. - * - * @returns {Object} The serialized representation of the object. - */ - serialize () { - const props = this._get_merged_static_object('PROPERTIES'); - const serialized = {}; - for ( const prop_name in props ) { - const prop = props[prop_name]; - - if ( prop.type === 'object' ) { - serialized[prop_name] = this[prop_name]?.serialize?.() ?? null; - } else if ( prop.type === 'map' ) { - serialized[prop_name] = {}; - for ( const key in this[prop_name] ) { - const object = this[prop_name][key]; - serialized[prop_name][key] = object.serialize(); - } - } else { - serialized[prop_name] = this[prop_name]; - } - } - return serialized; - } -} - -/** -* @class Parameter -* @extends Construct -* @description The Parameter class extends the Construct class and is used to define a parameter in a method. -* It includes properties such as type, whether it's optional, and a description. -* The class processes raw data to initialize these properties. -*/ -class Parameter extends Construct { - static PROPERTIES = { - type: { type: 'object' }, - optional: { type: 'boolean' }, - description: { type: 'string' }, - }; - - _process (raw) { - this.type = types[raw.type]; - } -} - -/** -* @class Method -* @extends Construct -* @description Represents a method in the system, including its description, parameters, and result. -* This class processes raw method data and structures it into a usable format. -*/ -class Method extends Construct { - static PROPERTIES = { - description: { type: 'string' }, - parameters: { type: 'map' }, - result: { type: 'object' }, - }; - - _process (raw) { - this.description = raw.description; - this.parameters = {}; - - for ( const parameter_name in raw.parameters ) { - const parameter = raw.parameters[parameter_name]; - this.parameters[parameter_name] = new Parameter(parameter, { name: parameter_name }); - } - - if ( raw.result ) { - this.result = new Parameter(raw.result, { name: 'result' }); - } - } -} - -/** -* @class Interface -* @extends Construct -* @description The Interface class represents a collection of methods and their descriptions. -* It extends the Construct class and defines static properties and methods to process raw data -* into a structured format. Each method in the Interface is an instance of the Method class, -* which in turn contains Parameter instances for its parameters and result. -*/ -class Interface extends Construct { - static PROPERTIES = { - description: { type: 'string' }, - methods: { type: 'map' }, - }; - - _process (raw) { - this.description = raw.description; - this.methods = {}; - - for ( const method_name in raw.methods ) { - const method = raw.methods[method_name]; - this.methods[method_name] = new Method(method, { name: method_name }); - } - } -} - -/** -* @class TypeSpec -* @extends BasicBase -* @description The TypeSpec class is used to represent a type specification. -* It provides methods to adapt raw data into a TypeSpec instance, check equality, -* convert the raw data to a string, and generate a hash of the raw data. -*/ -class TypeSpec extends BasicBase { - static adapt (raw) { - if ( raw instanceof TypeSpec ) return raw; - return new TypeSpec(raw); - } - constructor (raw) { - super(); - this.raw = raw; - } - - equals (other) { - return this.raw.$ === other.raw.$; - } - - /** - * Converts the TypeSpec object to its string representation. - * - * @returns {string} The string representation of the TypeSpec object. - */ - toString () { - return stringify_serializable_object(this.raw); - } - - /** - * Generates a hash value for the serialized object. - * - * This method uses the `hash_serializable_object` utility function to create a hash - * from the internal `raw` object. This hash can be used for comparison or indexing. - * - * @returns {string} The hash value of the serialized object. - */ - hash () { - return hash_serializable_object(this.raw); - } -} - -// NEXT: class Type extends Construct - -module.exports = { - Construct, - Parameter, - Method, - Interface, - TypeSpec, -}; diff --git a/src/backend/src/services/drivers/meta/Runtime.js b/src/backend/src/services/drivers/meta/Runtime.js deleted file mode 100644 index 45f2c881f..000000000 --- a/src/backend/src/services/drivers/meta/Runtime.js +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { BasicBase } = require('../../../../../putility/src/bases/BasicBase'); -const { TypeSpec } = require('./Construct'); - -/** -* Represents an entity in the runtime environment that extends the BasicBase class. -* This class serves as a foundational type for creating various runtime constructs -* within the drivers subsystem, enabling the implementation of specialized behaviors -* and properties. -*/ -class RuntimeEntity extends BasicBase { -} - -/** -* Represents a base runtime entity that extends functionality -* from the BasicBase class. This entity can be used as a -* foundation for creating more specific runtime objects -* within the application, enabling consistent behavior across -* derived entities. -*/ -class TypedValue extends RuntimeEntity { - constructor (type, value) { - super(); - this.type = TypeSpec.adapt(type); - this.value = value; - this.calculated_coercions_ = {}; - } -} - -module.exports = { - TypedValue, -}; diff --git a/src/backend/src/services/drivers/types.js b/src/backend/src/services/drivers/types.js deleted file mode 100644 index d18805745..000000000 --- a/src/backend/src/services/drivers/types.js +++ /dev/null @@ -1,316 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { is_valid_url, is_valid_uuid4, is_valid_path } = require('../../helpers'); -const { FileFacade } = require('./FileFacade'); -const APIError = require('../../api/APIError'); -const { AdvancedBase } = require('@heyputer/putility'); - -/** -* @class BaseType -* @extends AdvancedBase -* @description Base class for all type validators in the Puter type system. -* Extends AdvancedBase to provide core functionality for type checking and validation. -* Serves as the foundation for specialized type classes like String, Flag, NumberType, etc. -* Each type has a consolidate method that takes an input value and -* returns a sanitized or coerced value appropriate for that input. -*/ -class BaseType extends AdvancedBase { -} - -/** -* @class String -* @extends AdvancedBase -* @description A class that handles string values in the type system. -*/ -class String extends BaseType { - /** - * Consolidates input into a string value - * @param {Object} ctx - The context object - * @param {*} input - The input value to consolidate - * @returns {string|undefined} The consolidated string value, or undefined if input is null/undefined - */ - async consolidate (ctx, input) { - // undefined means the optional parameter was not provided, - // which is different from an empty string. - return ( - input === undefined || - input === null - ) ? undefined : `${ input}`; - } - - /** - * Serializes the type to a string representation - * @returns {string} Always returns 'string' to identify this as a string type - */ - serialize () { - return 'string'; - } -} - -/** -* @class Flag -* @description A class that handles boolean flag values in the type system. -* Converts any input value to a boolean using double negation, -* making it useful for command line flags and boolean parameters. -* Extends BaseType to integrate with the type validation system. -*/ -class Flag extends BaseType { - /** - * Consolidates input into a boolean flag value - * @param {Object} ctx - The context object - * @param {*} input - The input value to consolidate - * @returns {boolean} The consolidated boolean value, using double negation to coerce to boolean - */ - async consolidate (ctx, input) { - return !!input; - } - - /** - * Serializes the Flag type to a string representation - * @returns {string} Returns 'flag' as the type identifier - */ - serialize () { - return 'flag'; - } -} - -/** -* @class NumberType -* @extends BaseType -* @description Represents a number type validator and consolidator for API parameters. -* Handles both regular and unsigned numbers, performs type checking, and validates -* numeric constraints. Supports optional values and throws appropriate API errors -* for invalid inputs. -*/ -class NumberType extends BaseType { - /** - * Validates and consolidates number inputs for API parameters - * @param {Object} ctx - The context object - * @param {*} input - The input value to validate - * @param {Object} options - Options object containing arg_name and arg_descriptor - * @param {string} options.arg_name - Name of the argument being validated - * @param {Object} options.arg_descriptor - Descriptor containing validation rules - * @returns {number|undefined} The validated number or undefined if input was undefined - * @throws {APIError} If input is not a valid number or violates unsigned constraint - */ - async consolidate (ctx, input, { arg_name, arg_descriptor }) { - // Case for optional values - if ( input === undefined ) return undefined; - - if ( typeof input !== 'number' ) { - throw APIError.create('field_invalid', null, { - key: arg_name, - expected: 'number', - }); - } - - if ( arg_descriptor.unsigned && input < 0 ) { - throw APIError.create('field_invalid', null, { - key: arg_name, - expected: 'unsigned number', - }); - } - - return input; - } - - /** - * Validates and consolidates a number input value - * @param {Object} ctx - The context object - * @param {number} input - The input number to validate - * @param {Object} options - Options object containing arg_name and arg_descriptor - * @param {string} options.arg_name - The name of the argument being validated - * @param {Object} options.arg_descriptor - Descriptor containing validation rules like 'unsigned' - * @returns {number|undefined} The validated number or undefined if input was undefined - * @throws {APIError} If input is not a valid number or violates unsigned constraint - */ - serialize () { - return 'number'; - } -} - -/** -* @class File -* @description Represents a file type that can handle various input formats for files in the Puter system. -* Accepts and processes multiple file reference formats including: -* - Puter filepaths -* - Filesystem UUIDs -* - URLs -* - Base64 encoded data strings -* Converts these inputs into a FileFacade instance for standardized file handling. -* @extends BaseType -*/ -class File extends BaseType { - static DOC_INPUT_FORMATS = [ - 'A puter filepath, like /home/user/file.txt', - 'A puter filesystem UUID, like 12345678-1234-1234-1234-123456789abc', - 'A URL, like https://example.com/file.txt', - 'A base64-encoded string, like data:image/png;base64,iVBORw0K...', - ]; - static DOC_INTERNAL_TYPE = 'An instance of FileFacade'; - - static MODULES = { - _path: require('path'), - }; - - /** - * Validates and consolidates file input into a FileFacade instance. - * Handles multiple input formats including: - * - Puter filepaths - * - Filesystem UUIDs - * - URLs (web and data URLs) - * - Existing FileFacade instances - * Resolves home directory (~) references for authenticated users. - * - * @param {Object} ctx - Context object containing user info - * @param {string|FileFacade} input - The file input to consolidate - * @param {Object} options - Options object - * @param {string} options.arg_name - Name of the argument for error messages - * @returns {Promise} A FileFacade instance representing the file - * @throws {APIError} If input format is invalid - */ - async consolidate (ctx, input, { arg_name }) { - if ( input === undefined ) return undefined; - - if ( input instanceof FileFacade ) { - return input; - } - - const result = new FileFacade(); - // DRY: Part of this is duplicating FSNodeParam, but FSNodeParam is - // subject to change in PR #647, so this should be updated later. - - if ( ! ['/', '.', '~'].includes(input[0]) ) { - if ( is_valid_uuid4(input) ) { - result.set('uid', input); - return result; - } - - if ( is_valid_url(input) ) { - if ( input.startsWith('data:') ) { - result.set('data_url', input); - return result; - } - result.set('web_url', input); - return result; - } - - } - - if ( input.startsWith('~') ) { - const user = ctx.get('user'); - if ( ! user ) { - throw new Error('Cannot use ~ without a user'); - } - const homedir = `/${user.username}`; - input = homedir + input.slice(1); - } - - if ( ! is_valid_path(input) ) { - throw APIError.create('field_invalid', null, { - key: arg_name, - expected: 'unix-style path or UUID', - }); - } - - result.set('path', this.modules._path.resolve('/', input)); - return result; - } - - /** - * Serializes the File type identifier - * @returns {string} Returns 'file' as the type identifier for File parameters - */ - serialize () { - return 'file'; - } -} - -/** -* @class JSONType -* @extends BaseType -* @description Handles JSON data type validation and consolidation. This class validates JSON input -* against specified subtypes (array, object, string, etc) if provided in the argument descriptor. -* It ensures type safety for JSON data structures while allowing null and undefined values when -* appropriate. The class supports optional parameters and performs type checking against the -* specified subtype constraint. -*/ -class JSONType extends BaseType { - /** - * Validates and processes JSON input values according to specified type constraints - * @param {Context} ctx - The execution context - * @param {*} input - The input value to validate and process - * @param {Object} options - Validation options - * @param {string} options.arg_descriptor - Descriptor containing subtype constraints - * @param {string} options.arg_name - Name of the argument being validated - * @returns {*} The validated input value, or undefined if input is undefined - * @throws {APIError} If input type doesn't match specified subtype constraint - */ - async consolidate (ctx, input, { arg_descriptor, arg_name }) { - if ( input === undefined ) return undefined; - - if ( arg_descriptor.subtype ) { - const input_json_type = - Array.isArray(input) ? 'array' : - input === null ? 'null' : - typeof input; - - if ( input_json_type === 'null' || input_json_type === 'undefined' ) { - return input; - } - - if ( input_json_type !== arg_descriptor.subtype ) { - throw APIError.create('field_invalid', null, { - key: arg_name, - expected: `JSON value of type ${arg_descriptor.subtype}`, - got: `JSON value of type ${input_json_type}`, - }); - } - } - return input; - } - - /** - * Serializes the type identifier for JSON type parameters - * @returns {string} Returns 'json' as the type identifier - */ - serialize () { - return 'json'; - } -} - -/** -* @class WebURLString -* @extends BaseType -* @description A class for validating and handling web URL strings. This class extends BaseType -* and is designed to specifically handle and validate web-based URL strings. Currently commented -* out in the codebase, it would provide functionality for ensuring URLs conform to web standards -* and protocols (http/https). -*/ -// class WebURLString extends BaseType { -// } - -module.exports = { - file: new File(), - string: new String(), - flag: new Flag(), - json: new JSONType(), - number: new NumberType(), - // 'string:url:web': WebURLString, -}; diff --git a/src/backend/src/services/fs/FSLockService.js b/src/backend/src/services/fs/FSLockService.js deleted file mode 100644 index 9a8c032c5..000000000 --- a/src/backend/src/services/fs/FSLockService.js +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { RWLock } = require('../../util/lockutil'); -const BaseService = require('../BaseService'); - -// Constant representing the read lock mode used for distinguishing between read and write operations. -const MODE_READ = Symbol('read'); -// Constant representing the read mode for locks, used to distinguish between read and write operations. -const MODE_WRITE = Symbol('write'); - -// TODO: DRY: could use LockService now -/** -* FSLockService is a service class that manages file system locks using read-write locks. -* It provides functionality to create, list, and manage locks on file paths, -* allowing concurrent read and exclusive write operations. -*/ -class FSLockService extends BaseService { - static LOG_DEBUG = true; - - async _construct () { - this.locks = {}; - } - /** - * Initializes the FSLockService by setting up the locks object. - * This method should be called before using the service to ensure - * that the locks property is properly instantiated. - * - * @returns {Promise} A promise that resolves when the initialization is complete. - */ - async _init () { - } - - /** - * Lock a file by parent path and child node name. - * - * @param {string} path - The path to lock. - * @param {string} name - The name of the resource to lock. - * @param {symbol} mode - The mode of the lock (read or write). - * @returns {Promise} A promise that resolves when the lock is acquired. - * @throws {Error} Throws an error if an invalid mode is provided. - */ - async lock_child (path, name, mode) { - if ( path.endsWith('/') ) path = path.slice(0, -1); - return await this.lock_path(`${path }/${ name}`, mode); - } - - /** - * Lock a file by path. - * - * @param {string} path - The path to lock. - * @param {symbol} mode - The mode of the lock (read or write). - * @returns {Promise} A promise that resolves when the lock is acquired. - * @throws {Error} Throws an error if an invalid mode is provided. - */ - async lock_path (path, mode) { - // TODO: Why??? - // if ( this.locks === undefined ) this.locks = {}; - - if ( ! this.locks[path] ) { - const rwlock = new RWLock(); - /** - * Acquires a lock for the specified path and mode. If the lock does not exist, - * a new RWLock instance is created and associated with the path. The lock is - * released when there are no more active locks. - * - * @param {string} path - The path for which to acquire the lock. - * @param {Symbol} mode - The mode of the lock, either MODE_READ or MODE_WRITE. - * @returns {Promise} A promise that resolves once the lock is successfully acquired. - * @throws {Error} Throws an error if the mode provided is invalid. - */ - rwlock.on_empty_ = () => { - delete this.locks[path]; - }; - this.locks[path] = rwlock; - } - - this.log.info(`WAITING FOR LOCK: ${ path } ${ - mode.toString()}`); - - if ( mode === MODE_READ ) { - return await this.locks[path].rlock(); - } - - if ( mode === MODE_WRITE ) { - return await this.locks[path].wlock(); - } - - throw new Error('Invalid mode'); - } -} - -module.exports = { - MODE_READ, - MODE_WRITE, - FSLockService, -}; diff --git a/src/backend/src/services/sla/RateLimitRedisCacheSpace.js b/src/backend/src/services/sla/RateLimitRedisCacheSpace.js deleted file mode 100644 index d14844281..000000000 --- a/src/backend/src/services/sla/RateLimitRedisCacheSpace.js +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const RateLimitRedisCacheSpace = { - keyPrefix: consumerScopedKey => `rate-limit:${consumerScopedKey}`, - windowStartKey: consumerScopedKey => `${RateLimitRedisCacheSpace.keyPrefix(consumerScopedKey)}:window_start`, - countKey: consumerScopedKey => `${RateLimitRedisCacheSpace.keyPrefix(consumerScopedKey)}:count`, -}; - -export { RateLimitRedisCacheSpace }; diff --git a/src/backend/src/services/sla/RateLimitService.js b/src/backend/src/services/sla/RateLimitService.js deleted file mode 100644 index c053b6766..000000000 --- a/src/backend/src/services/sla/RateLimitService.js +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const { Context } = require('../../util/context'); -const BaseService = require('../BaseService'); -const { SyncFeature } = require('../../traits/SyncFeature'); -const { DB_WRITE } = require('../database/consts'); -const { redisClient } = require('../../clients/redis/redisSingleton'); -const { RateLimitRedisCacheSpace } = require('./RateLimitRedisCacheSpace.js'); - -const toSqlTimestamp = (timestampMs) => Math.floor(timestampMs / 1000); -const fromSqlTimestamp = (timestampSec) => timestampSec * 1000; -const defaultRateLimitRedisTimeoutMs = 200; -const formatErrorMessage = (error) => error instanceof Error ? error.message : String(error); -const withTimeout = async (operationPromise, timeoutMs, timeoutMessage) => { - let timeout; - try { - return await Promise.race([ - operationPromise, - new Promise((_, reject) => { - timeout = setTimeout(() => { - reject(new Error(timeoutMessage)); - }, timeoutMs); - }), - ]); - } finally { - if ( timeout ) clearTimeout(timeout); - } -}; - -/** -* RateLimitService class handles rate limiting functionality for API requests. -* Implements a fixed window counter strategy to track and limit request rates -* per user/consumer. Manages rate limit data both in memory (KV store) and -* persistent storage (database). Extends BaseService and includes SyncFeature -* for synchronized rate limit checking and incrementing. -*/ -class RateLimitService extends BaseService { - static FEATURES = [ - new SyncFeature([ - 'check_and_increment', - ]), - ]; - - /** - * Initializes the service by setting up the database connection - * for rate limiting operations. Gets a database instance from - * the database service using the 'rate-limit' namespace. - * @private - * @returns {Promise} - */ - async _init () { - this.db = this.services.get('database').get(DB_WRITE, 'rate-limit'); - } - - async #checkAndIncrementInDb ({ dbKey, max, period, methodName }) { - const rows = await this.db.read( - 'SELECT * FROM `rl_usage_fixed_window` WHERE `key` = ?', - [dbKey], - ); - - let windowStart = 0; - let currentCount = 0; - - if ( rows.length === 0 ) { - windowStart = Date.now(); - this.db.write( - 'INSERT INTO `rl_usage_fixed_window` (`key`, `window_start`, `count`) VALUES (?, ?, ?)', - [dbKey, toSqlTimestamp(windowStart), 0], - ); - } else { - const row = rows[0]; - windowStart = fromSqlTimestamp(row.window_start); - currentCount = Number.isFinite(Number(row.count)) ? Number(row.count) : 0; - } - - if ( windowStart + period < Date.now() ) { - windowStart = Date.now(); - currentCount = 0; - this.db.write( - 'UPDATE `rl_usage_fixed_window` SET `window_start` = ?, `count` = ? WHERE `key` = ?', - [toSqlTimestamp(windowStart), 0, dbKey], - ); - } - - if ( currentCount >= max ) { - throw APIError.create('rate_limit_exceeded', null, { - method_name: methodName, - rate_limit: { max, period }, - }); - } - - this.db.write( - 'UPDATE `rl_usage_fixed_window` SET `count` = `count` + 1 WHERE `key` = ?', - [dbKey], - ); - } - - /** - * Checks if a rate limit has been exceeded and increments the counter - * @param {string} key - The rate limit key/identifier - * @param {number} max - Maximum number of requests allowed in the period - * @param {number} period - Time window in milliseconds - * @param {Object} [options={}] - Additional options - * @param {boolean} [options.global] - Whether this is a global rate limit across servers - * @throws {APIError} When rate limit is exceeded - */ - async check_and_increment (key, max, period, options = {}) { - const consumerId = this._get_consumer_id(); - const methodName = key; - const rateLimitKey = `${consumerId}:${key}`; - const windowStartKey = RateLimitRedisCacheSpace.windowStartKey(rateLimitKey); - const countKey = RateLimitRedisCacheSpace.countKey(rateLimitKey); - const dbKey = options.global - ? rateLimitKey - : `${this.global_config.server_id}:${rateLimitKey}`; - const rateLimitRedisTimeoutMs = Number(this.global_config?.services?.['rate-limit']?.redis_timeout_ms) - || defaultRateLimitRedisTimeoutMs; - const runRedis = async (operationName, operationPromise) => { - try { - const value = await withTimeout( - operationPromise, - rateLimitRedisTimeoutMs, - `rate-limit redis ${operationName} timed out after ${rateLimitRedisTimeoutMs}ms`, - ); - return { ok: true, value }; - } catch ( error ) { - this.log.warn('rate-limit redis operation failed; continuing with db fallback', { - operationName, - rateLimitKey, - error: formatErrorMessage(error), - }); - return { ok: false, value: null }; - } - }; - - // Fixed window counter strategy (see devlog 2023-11-21) - const windowStartRead = await runRedis('window-start-read', redisClient.get(windowStartKey)); - if ( ! windowStartRead.ok ) { - await this.#checkAndIncrementInDb({ dbKey, max, period, methodName }); - return; - } - let windowStart = Number.isFinite(Number(windowStartRead.value)) ? Number(windowStartRead.value) : 0; - if ( windowStart === 0 ) { - // Try database - const rows = await this.db.read( - 'SELECT * FROM `rl_usage_fixed_window` WHERE `key` = ?', - [dbKey], - ); - - if ( rows.length !== 0 ) { - const row = rows[0]; - windowStart = fromSqlTimestamp(row.window_start); - const count = row.count; - - void Promise.all([ - runRedis('window-start-seed', redisClient.set(windowStartKey, windowStart)), - runRedis('count-seed', redisClient.set(countKey, count)), - ]); - } - } - - if ( windowStart === 0 ) { - windowStart = Date.now(); - void Promise.all([ - runRedis('window-start-init', redisClient.set(windowStartKey, windowStart)), - runRedis('count-init', redisClient.set(countKey, 0)), - ]); - - this.db.write( - 'INSERT INTO `rl_usage_fixed_window` (`key`, `window_start`, `count`) VALUES (?, ?, ?)', - [dbKey, toSqlTimestamp(windowStart), 0], - ); - - this.log.debug( - 'create windowStart and count', - { windowStart, count: 0 }, - ); - } - - if ( windowStart + period < Date.now() ) { - windowStart = Date.now(); - Promise.all([ - runRedis('window-start-reset', redisClient.set(windowStartKey, windowStart)), - runRedis('count-reset', redisClient.set(countKey, 0)), - ]); - - this.db.write( - 'UPDATE `rl_usage_fixed_window` SET `window_start` = ?, `count` = ? WHERE `key` = ?', - [toSqlTimestamp(windowStart), 0, dbKey], - ); - } - - const currentRead = await runRedis('count-read', redisClient.get(countKey)); - if ( ! currentRead.ok ) { - await this.#checkAndIncrementInDb({ dbKey, max, period, methodName }); - return; - } - const current = Number.isFinite(Number(currentRead.value)) ? Number(currentRead.value) : 0; - if ( current >= max ) { - throw APIError.create('rate_limit_exceeded', null, { - method_name: methodName, - rate_limit: { max, period }, - }); - } - - runRedis('count-incr', redisClient.incr(countKey)); - this.db.write( - 'UPDATE `rl_usage_fixed_window` SET `count` = `count` + 1 WHERE `key` = ?', - [dbKey], - ); - } - - /** - * Gets the consumer ID for rate limiting based on the current user context - * @returns {string} Consumer ID in format 'user:{id}' if user exists, or 'missing' if no user - * @private - */ - _get_consumer_id () { - const context = Context.get(); - const user = context.get('user'); - return user ? `user:${user.id}` : 'missing'; - } -} - -module.exports = { - RateLimitService, -}; diff --git a/src/backend/src/services/sla/SLAService.js b/src/backend/src/services/sla/SLAService.js deleted file mode 100644 index 4b50fddbd..000000000 --- a/src/backend/src/services/sla/SLAService.js +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const BaseService = require('../BaseService'); - -/** - * SLAService is responsible for getting the appropriate SLA for a given - * driver or service endpoint, including limits with respect to the actor - * and server-wide limits. - */ -/** -* @class SLAService -* @extends BaseService -* @description Service class responsible for managing Service Level Agreement (SLA) configurations. -* Handles rate limiting and usage quotas for various API endpoints and drivers. Provides access -* to system-wide limits, user-specific limits (both verified and unverified), and maintains -* hardcoded limits for different service categories. Extends BaseService to integrate with -* the core service infrastructure. -*/ -class SLAService extends BaseService { - /** - * Initializes the service by setting up hardcoded SLA limits for different categories and endpoints. - * Contains rate limits and monthly usage limits for various driver implementations. - * @private - * @async - * @returns {Promise} - */ - async _construct () { - // I'm not putting this in config for now until we have checks - // for production configuration. - EAD - this.hardcoded_limits = { - system: { - 'driver:impl:public-helloworld:greet': { - rate_limit: { - max: 1000, - period: 30000, - }, - }, - 'driver:impl:public-aws-textract:recognize': { - rate_limit: { - max: 10, - period: 30000, - }, - }, - }, - // app_default: { - // 'driver:impl:public-aws-textract:recognize': { - // rate_limit: { - // max: 40, - // period: 30000, - // }, - // monthly_limit: 1000, - // }, - // 'driver:impl:public-openai-chat-completion:complete': { - // rate_limit: { - // max: 30, - // period: 1000 * 60 * 60, - // }, - // monthly_limit: 600, - // }, - // 'driver:impl:public-openai-image-generation:generate': { - // rate_limit: { - // max: 30, - // period: 1000 * 60 * 60, - // }, - // monthly_limit: 10000, - // }, - // }, - user_unverified: { - 'driver:impl:public-aws-textract:recognize': { - rate_limit: { - max: 40, - period: 30000, - }, - }, - 'driver:impl:public-openai-chat-completion:complete': { - rate_limit: { - max: 40, - period: 30000, - }, - }, - 'driver:impl:public-openai-image-generation:generate': { - rate_limit: { - max: 40, - period: 30000, - }, - }, - }, - user_verified: { - 'driver:impl:public-aws-textract:recognize': { - rate_limit: { - max: 40, - period: 30000, - }, - }, - 'driver:impl:public-openai-chat-completion:complete': { - rate_limit: { - max: 40, - period: 30000, - }, - }, - 'driver:impl:public-openai-image-generation:generate': { - rate_limit: { - max: 40, - period: 30000, - }, - }, - }, - }; - } - - get (category, key) { - return this.hardcoded_limits[category]?.[key]; - } -} - -module.exports = { - SLAService, -}; diff --git a/src/backend/src/services/web/UserProtectedEndpointsService.js b/src/backend/src/services/web/UserProtectedEndpointsService.js deleted file mode 100644 index 037bf95c0..000000000 --- a/src/backend/src/services/web/UserProtectedEndpointsService.js +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { get_user } = require('../../helpers'); -const { Context } = require('../../util/context'); -const BaseService = require('../BaseService'); -const { UserActorType } = require('../auth/Actor'); -const APIError = require('../../api/APIError.js'); -const configurable_auth = require('../../middleware/configurable_auth.js'); -const config = require('../../config'); -const jwt = require('jsonwebtoken'); - -const REVALIDATION_COOKIE_NAME = 'puter_revalidation'; - -/** -* @class UserProtectedEndpointsService -* @extends BaseService -* @classdesc -* This service manages endpoints that are protected by password authentication, -* excluding login. It ensures that only authenticated user sessions can access -* these endpoints, which typically involve actions affecting security settings -* such as changing passwords, email addresses, or disabling two-factor authentication. -* The service also handles middleware for rate limiting, session validation, -* and password verification for security-critical operations. -*/ -class UserProtectedEndpointsService extends BaseService { - static MODULES = { - express: require('express'), - }; - - async #revalidateUrlFields (req, user) { - const origin = (config.origin || '').replace(/\/$/, ''); - const svc_oidc = req.services.get('oidc'); - const providers = await svc_oidc.getEnabledProviderIds(); - const provider = providers && providers[0]; - if ( ! provider ) return {}; - return { revalidate_url: `${origin}/auth/oidc/${provider}/start?flow=revalidate&user_id=${user.id}` }; - } - - /** - * Sets up and configures routes for user-protected endpoints. - * This method initializes an Express router, applies middleware for authentication, - * rate limiting, and session validation, and attaches user-specific endpoints. - * - * @memberof UserProtectedEndpointsService - * @instance - * @method __on_install.routes - */ - '__on_install.routes' () { - const router = (() => { - const require = this.require; - const express = require('express'); - return express.Router(); - })(); - - const { app } = this.services.get('web-server'); - app.use('/user-protected', router); - - // Apply edge (unauthenticated) rate-limiting - router.use((req, res, next) => { - if ( req.method === 'OPTIONS' ) return next(); - - const svc_edgeRateLimit = req.services.get('edge-rate-limit'); - if ( ! svc_edgeRateLimit.check(req.baseUrl + req.path) ) { - return APIError.create('too_many_requests').write(res); - } - next(); - }); - - // Require authenticated session; bypass user cache to enforce suspension reliably - router.use(configurable_auth({ no_options_auth: true, allow_cached_user: false })); - - // Only allow user sessions with HTTP powers (session token), not GUI tokens or API tokens - router.use((req, res, next) => { - if ( req.method === 'OPTIONS' ) return next(); - - const actor = Context.get('actor'); - if ( ! (actor.type instanceof UserActorType) ) { - return APIError.create('user_tokens_only').write(res); - } - if ( ! actor.type.hasHttpOnlyCookie ) { - return APIError.create('session_required').write(res); - } - next(); - }); - - // Prioritize consistency for user object - router.use(async (req, res, next) => { - if ( req.method === 'OPTIONS' ) return next(); - const user = await get_user({ id: req.user.id, force: true }); - req.user = user; - next(); - }); - - // Do not allow temporary users (except for delete-own-user, which allows them) - router.use(async (req, res, next) => { - if ( req.method === 'OPTIONS' ) return next(); - if ( req.path === '/delete-own-user' ) return next(); - - if ( req.user.password === null && req.user.email === null ) { - return APIError.create('temporary_account').write(res); - } - next(); - }); - - /** - * Middleware to validate identity: either password (bcrypt) or a valid OIDC revalidation cookie. - * OIDC-only accounts (user.password === null) must use revalidation; password accounts may use either. - * Temporary users (no password, no email) are allowed only for delete-own-user. - */ - router.use(async (req, res, next) => { - if ( req.method === 'OPTIONS' ) return next(); - - const user = await get_user({ id: req.user.id, force: true }); - const revalidationCookie = req.cookies && req.cookies[REVALIDATION_COOKIE_NAME]; - - if ( user.password === null && user.email === null ) { - return next(); - } - - if ( req.body.password ) { - if ( user.password === null ) { - return (APIError.create('oidc_revalidation_required', null, await this.#revalidateUrlFields(req, user))).write(res); - } - const bcrypt = (() => { - const require = this.require; - return require('bcrypt'); - })(); - const isMatch = await bcrypt.compare(req.body.password, user.password); - if ( ! isMatch ) { - return APIError.create('password_mismatch').write(res); - } - return next(); - } - - if ( revalidationCookie ) { - try { - const payload = jwt.verify(revalidationCookie, config.jwt_secret); - if ( payload.purpose === 'revalidate' && payload.user_id === req.user.id ) { - return next(); - } - } catch ( e ) { - // invalid or expired - } - } - - if ( user.password === null ) { - return (APIError.create('oidc_revalidation_required', null, await this.#revalidateUrlFields(req, user))).write(res); - } - return (APIError.create('password_required')).write(res); - }); - - router.use(require('../../routers/user-protected/change-password.js')); - router.use(require('../../routers/user-protected/change-email.js')); - router.use(require('../../routers/user-protected/change-username.js')); - router.use(require('../../routers/user-protected/disable-2fa.js')); - router.use(require('../../routers/user-protected/delete-own-user.js')); - } -} - -module.exports = { - UserProtectedEndpointsService, -}; diff --git a/src/backend/src/services/worker/.gitignore b/src/backend/src/services/worker/.gitignore deleted file mode 100644 index 77738287f..000000000 --- a/src/backend/src/services/worker/.gitignore +++ /dev/null @@ -1 +0,0 @@ -dist/ \ No newline at end of file diff --git a/src/backend/src/services/worker/README.md b/src/backend/src/services/worker/README.md deleted file mode 100644 index 3c525f604..000000000 --- a/src/backend/src/services/worker/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# Worker Service - -This directory contains the worker service components for Puter's server-to-web (s2w) worker functionality. - -## Build Process - -The `dist/workerPreamble.js` file is **generated** by webpack and c-preprocessor and should not be edited directly. Instead, edit the source files in the `src/` directory and rebuild. - -### Building - -To build the worker preamble: - -```bash -# From this directory -npm install -npm run build -``` - -Or from the backend root: - -```bash -npm run build:worker -``` - -### Development - -For development with auto-rebuild: - -```bash -npm run build:watch -``` - -This will watch for changes in the source files and automatically rebuild the `workerPreamble.js`. - -## Source Files - -- `template/puter-portable.js` - Puter portable API wrapper -- `src/s2w-router.js` - Server-to-web router implementation -- `src/index.js` - Main entry point that combines both components - -## Dependencies - -- `path-to-regexp` - URL pattern matching library used by the s2w router - -## Generated Output - -The webpack build process creates `dist/workerPreamble.js` which contains: -1. The bundled `path-to-regexp` library -2. The puter portable API -3. The s2w router with proper initialization -4. Initialization code that sets up both systems - -This file is then read by `WorkerService.js` and injected into worker environments. \ No newline at end of file diff --git a/src/backend/src/services/worker/WorkerService.js b/src/backend/src/services/worker/WorkerService.js deleted file mode 100644 index 77277af11..000000000 --- a/src/backend/src/services/worker/WorkerService.js +++ /dev/null @@ -1,421 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const configurable_auth = require('../../middleware/configurable_auth'); -const BaseService = require('../BaseService'); -const fs = require('node:fs'); - -const { createWorker, setCloudflareKeys, deleteWorker } = require('./workerUtils/cloudflareDeploy'); -const { getUserInfo } = require('./workerUtils/puterUtils'); -const { LLRead } = require('../../deprecated/filesystem/ll_operations/ll_read'); -const { Context } = require('../../util/context'); -const { NodePathSelector, NodeUIDSelector } = require('../../deprecated/filesystem/node/selectors'); -const { calculateWorkerNameNew } = require('./workerUtils/nameUtils'); -const { Entity } = require('../../om/entitystorage/Entity'); -const { SKIP_ES_VALIDATION } = require('../../om/entitystorage/consts'); -const { Eq, StartsWith } = require('../../om/query/query'); -const { get_app, subdomain } = require('../../helpers'); -const { UsernameNotifSelector } = require('../NotificationService'); -const APIError = require('../../api/APIError'); -const FSNodeParam = require('../../api/filesystem/FSNodeParam'); -const { UserActorType } = require('../auth/Actor'); - -async function readPuterFile (actor, filePath) { - try { - const svc_fs = this.services.get('filesystem'); - const node = await svc_fs.node(new NodePathSelector(filePath)); - const ll_read = new LLRead(); - const stream = await ll_read.run({ - fsNode: node, - actor, - }); - const chunks = []; - let bytes = 0; - stream.on('data', (data) => { - chunks.push(data); - bytes += data.byteLength; - if ( bytes > 10 ** 7 ) { - const err = Error('Worker source code must not exceed 10MB'); - stream.emit('error', err); - throw err; - } - }); - return new Promise((res, rej) => { - stream.on('error', (e) => { - rej(e.toString()); - }); - stream.on('end', () => { - res(Buffer.concat(chunks)); - }); - }); - } catch (e) { - console.error(e); - } - -} -// This file is generated by webpack. To rebuild: cd to this directory and run `npm run build` -let preamble; -try { - preamble = fs.readFileSync(`${__dirname }/dist/workerPreamble.js`, 'utf-8'); -} catch (e) { - preamble = ''; - console.error('WORKERS ERROR: Preamble has not been built! Workers will not have access to puter.js\nTo fix this cd into src/backend/src/worker and run npm run build'); -} -const PREAMBLE_LENGTH = preamble.split('\n').length - 1; -class WorkerService extends BaseService { - _init () { - setCloudflareKeys(this.config); - - // Services used - const svc_event = this.services.get('event'); - const svc_su = this.services.get('su'); - const es_subdomain = this.services.get('es:subdomain'); - const svc_auth = this.services.get('auth'); - const svc_notification = this.services.get('notification'); - - svc_event.on('fs.write.file', async (_key, data, meta) => { - // Code should only run on the same server as the write - if ( meta.from_outside ) return; - // There seems to be some bug in file writes where uid is null. We will check for this - if ( !data.node.uid || data.node.uid === '' ) return; - - // Check if the file that was written correlates to a worker - const results = await svc_su.sudo(async () => { - return await es_subdomain.select({ predicate: new Eq({ key: 'root_dir', value: data.node }) }); - }); - if ( !results || results.length === 0 ) - { - return; - } - - for ( const result of results ) { - // Person who just wrote file (not necessarily file owner) - const actor = Context.get('actor'); - - const /** @type {string} */ workerFullName = (await result.get('subdomain')); - if ( ! workerFullName.startsWith('workers.puter.') ) { - continue; - } - - // Worker data - const fileData = (await readPuterFile(Context.get('actor'), data.node.path)).toString(); - const workerName = workerFullName.split('.').pop(); - - // Get appropriate deploy time auth token to give to the worker - let authToken; - const appOwner = await result.get('app_owner'); - if ( appOwner ) { // If the deployer is an app... - const appID = await appOwner.get('uid'); - authToken = await svc_su.sudo(await data.node.get('owner'), async () => { - return await svc_auth.get_user_app_token(appID); - }); - } else { // If the deployer is not attached to any application - authToken = (await svc_auth.create_session_token((await data.node.get('owner')).type.user)).token; - } - - // svc_notification.notify( - // UsernameNotifSelector(actor.type.user.username), - // { - // source: 'worker', - // title: `Deploying CF worker ${workerName}`, - // template: 'user-requesting-share', - // fields: { - // username: actor.type.user.username, - // }, - // } - // ); - try { - // Create the worker - const cfData = await createWorker((await data.node.get('owner')).type.user, authToken, workerName, preamble + fileData, PREAMBLE_LENGTH); - - // Send user the appropriate notification - if ( cfData.success ) { - svc_notification.notify( - UsernameNotifSelector(actor.type.user.username), - { - source: 'worker', - title: `Succesfully deployed ${cfData.url}`, - template: 'user-requesting-share', - fields: { - username: actor.type.user.username, - }, - }, - ); - } else { - svc_notification.notify( - UsernameNotifSelector(actor.type.user.username), - { - source: 'worker', - title: `Failed to deploy ${workerName}! ${cfData.errors}`, - template: 'user-requesting-share', - fields: { - username: actor.type.user.username, - }, - }, - ); - } - - } catch (e) { - svc_notification.notify( - UsernameNotifSelector(actor.type.user.username), - { - source: 'worker', - title: `Failed to deploy ${workerName}!!\n ${e}`, - template: 'user-requesting-share', - fields: { - username: actor.type.user.username, - }, - }, - ); - } - } - }); - } - static IMPLEMENTS = { - 'workers': { - /** - * - * @param {{filePath: string, workerName: string, authorization: string}} param0 - * @returns {any} - */ - async create ({ filePath, workerName, authorization, appId }) { - try { - workerName = workerName.toLocaleLowerCase(); // just incase - const svc_su = this.services.get('su'); - const es_subdomain = this.services.get('es:subdomain'); - const svc_auth = this.services.get('auth'); - - const currentDomains = await svc_su.sudo(Context.get('actor').get_related_actor(UserActorType), async () => { - return (await es_subdomain.select({ predicate: new StartsWith({ key: 'subdomain', value: 'workers.puter.' }) })); - }); - - if ( appId ) { - const app = await get_app({ uid: appId }); - if ( Context.get('actor').type.user.id !== app.owner_user_id ) - { - throw APIError.create('no_suitable_app', null, { entry_name: workerName }); - } - - authorization = await svc_auth.get_user_app_token(appId); - } - - if ( currentDomains.length >= 100 ) { - throw APIError.create('subdomain_limit_reached', null, { isWorker: true, limit: 100 }); - } - - if ( this.global_config.reserved_words.includes(workerName) ) { - throw APIError.create('subdomain_reserved', null, { - subdomain: workerName, - }); - } - - if ( ! (/^[a-zA-Z0-9_-]+$/.test(workerName)) ) return; - - filePath = await (await (new FSNodeParam('path')).consolidate({ - req: { user: Context.get('actor').type.user }, - getParam: () => filePath, - })).get('path'); - - const userData = await getUserInfo(authorization, this.global_config.api_base_url); - const actor = Context.get('actor'); - if ( appId ) { - await svc_su.sudo(await svc_auth.authenticate_from_token(authorization), async () => { - await Context.sub({ [SKIP_ES_VALIDATION]: true }).arun(async () => { - const entity = await Entity.create({ om: es_subdomain.om }, { - subdomain: `workers.puter.${ calculateWorkerNameNew(userData, workerName)}`, - root_dir: filePath, - }); - await es_subdomain.upsert(entity); - }); - }); - } else { - await Context.sub({ [SKIP_ES_VALIDATION]: true }).arun(async () => { - const entity = await Entity.create({ om: es_subdomain.om }, { - subdomain: `workers.puter.${ calculateWorkerNameNew(userData, workerName)}`, - root_dir: filePath, - }); - await es_subdomain.upsert(entity); - }); - } - - const fileData = (await readPuterFile(actor, filePath)).toString(); - const cfData = await createWorker(userData, authorization, calculateWorkerNameNew(userData.uuid, workerName), preamble + fileData, PREAMBLE_LENGTH); - - return cfData; - } catch (e) { - if ( e instanceof APIError ) - { - throw e; - } - console.error(e); - return { success: false, errors: e }; - } - }, - async destroy ({ workerName, authorization }) { - try { - workerName = workerName.toLocaleLowerCase(); // just incase - const svc_su = this.services.get('su'); - const es_subdomain = this.services.get('es:subdomain'); - - const userData = await getUserInfo(authorization, this.global_config.api_base_url); - - const [result] = (await es_subdomain.select({ predicate: new Eq({ key: 'subdomain', value: `workers.puter.${ calculateWorkerNameNew(undefined, workerName)}` }) })); - - if ( result.values_.owner.uuid !== userData.uuid ) { - throw new Error('This is not your worker!'); - } - - const cfData = await deleteWorker(userData, authorization, workerName); - - await es_subdomain.delete(await result.get('uid')); - return cfData; - - } catch (e) { - if ( e instanceof APIError ) - { - throw e; - } - console.error(e); - return { success: false, e }; - } - }, - async getFilePaths ({ workerName }) { - try { - const es_subdomain = this.services.get('es:subdomain'); - let currentDomains; - if ( typeof (workerName) !== 'string' ) { - currentDomains = (await es_subdomain.select({ predicate: new StartsWith({ key: 'subdomain', value: 'workers.puter.' }) })); - } else { - currentDomains = (await es_subdomain.select({ predicate: new Eq({ key: 'subdomain', value: `workers.puter.${ workerName}` }) })); - } - - const domainToPath = []; - for ( const domain of currentDomains ) { - const node = await domain.get('root_dir'); - const subdomainString = (await domain.get('subdomain')); - let file_path = null; - let file_uid = null; - try { - file_path = await node.get('path'); - file_uid = await node.get('uid'); - } catch (e) { - } - const name = subdomainString.split('.').pop(); - const url = `https://${name}.puter.work`; - domainToPath.push({ name, url, file_path, file_uid, created_at: (new Date(await domain.get('created_at'))).toISOString() }); - } - - return domainToPath; - } catch (e) { - console.error(e); - } - - }, - async startLogs ({ workerName, authorization }) { - return await this.exec_({ runtime, code }); - }, - async endLogs ({ workerName, authorization }) { - return await this.exec_({ runtime, code }); - }, - async getLoggingUrl ({ }) { - return this.config.loggingUrl; - }, - }, - }; - async '__on_driver.register.interfaces' () { - const svc_registry = this.services.get('registry'); - const col_interfaces = svc_registry.get('interfaces'); - - col_interfaces.set('workers', { - description: 'Execute code with various languages.', - methods: { - getFilePaths: { - description: 'get paths for your workers', - parameters: { - workerName: { - type: 'string', - description: 'Optionally, the name of the worker you want the path for', - }, - }, - result: { type: 'json' }, - }, - create: { - description: 'Create a backend worker', - parameters: { - filePath: { - type: 'string', - description: 'The path of the code of the worker to upload', - }, - workerName: { - type: 'string', - description: 'The name of the worker you want to upload', - }, - authorization: { - type: 'string', - description: 'Puter token', - }, - appId: { - type: 'string', - description: 'App ID to tie a worker to', - }, - }, - result: { type: 'json' }, - }, - startLogs: { - description: 'Get logs for your backend worker', - parameters: { - workerName: { - type: 'string', - description: 'The name of the worker you want the logs of', - }, - authorization: { - type: 'string', - description: 'Puter token', - }, - }, - result: { type: 'json' }, - }, - getLoggingUrl: { - description: 'Get logging endpoint for your backend worker', - parameters: { - }, - result: { type: 'string' }, - }, - destroy: { - description: 'Get rid of your backend worker', - parameters: { - workerName: { - type: 'string', - description: 'The name of the worker you want to destroy', - }, - authorization: { - type: 'string', - description: 'Puter token', - }, - }, - result: { type: 'json' }, - }, - }, - }); - } -} - -module.exports = { - WorkerService, -}; diff --git a/src/backend/src/services/worker/package.json b/src/backend/src/services/worker/package.json deleted file mode 100644 index e91793aa3..000000000 --- a/src/backend/src/services/worker/package.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "@heyputer/worker-service", - "version": "1.0.0", - "description": "Worker service components for Puter", - "main": "src/index.js", - "scripts": { - "build": "webpack --mode production && npm run preprocess", - "preprocess": "c-preprocessor template/puter-portable.js dist/workerPreamble.js" - }, - "dependencies": { - "c-preprocessor": "^0.2.13", - "path-to-regexp": "^8.2.0" - }, - "devDependencies": { - "imports-loader": "^5.0.0", - "raw-loader": "^4.0.2", - "script-loader": "^0.7.2", - "terser-webpack-plugin": "^5.3.14", - "webpack": "^5.88.2", - "webpack-cli": "^5.1.1" - }, - "author": "Puter Technologies Inc.", - "license": "AGPL-3.0-only" -} diff --git a/src/backend/src/services/worker/src/index.js b/src/backend/src/services/worker/src/index.js deleted file mode 100644 index 899cdea40..000000000 --- a/src/backend/src/services/worker/src/index.js +++ /dev/null @@ -1,3 +0,0 @@ -import inits2w from './s2w-router.js'; -// Initialize s2w router -inits2w(); diff --git a/src/backend/src/services/worker/src/s2w-router.js b/src/backend/src/services/worker/src/s2w-router.js deleted file mode 100644 index 94a7abd26..000000000 --- a/src/backend/src/services/worker/src/s2w-router.js +++ /dev/null @@ -1,133 +0,0 @@ -import { match } from 'path-to-regexp'; - -function inits2w () { - // s2w router itself: Not part of any package, just a simple router. - const router = { - routing: true, - handleCors: true, - map: new Map(), - custom (eventName, route, eventListener) { - const matchExp = match(route); - if ( ! this.map.has(eventName) ) { - this.map.set(eventName, [[matchExp, eventListener]]); - } else { - this.map.get(eventName).push([matchExp, eventListener]); - } - }, - get (...args) { - this.custom('GET', ...args); - }, - post (...args) { - this.custom('POST', ...args); - }, - options (...args) { - this.custom('OPTIONS', ...args); - }, - put (...args) { - this.custom('PUT', ...args); - }, - delete (...args) { - this.custom('DELETE', ...args); - }, - async handleOptions (request) { - const corsHeaders = { - 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Methods': 'GET,HEAD,POST,OPTIONS', - 'Access-Control-Max-Age': '86400', - }; - if ( - request.headers.get('Origin') !== null && - request.headers.get('Access-Control-Request-Method') !== null && - request.headers.get('Access-Control-Request-Headers') !== null - ) { - // Handle CORS preflight requests. - return new Response(null, { - headers: { - ...corsHeaders, - 'Access-Control-Allow-Headers': request.headers.get('Access-Control-Request-Headers'), - }, - }); - } else { - // Handle standard OPTIONS request. - return new Response(null, { - headers: { - Allow: 'GET, HEAD, POST, OPTIONS', - }, - }); - } - }, - /** - * - * @param {FetchEvent } event - * @returns - */ - async route (event) { - if ( ! globalThis.me ) { - globalThis.me = { puter: init_puter_portable(globalThis.puter_auth, globalThis.puter_endpoint || 'https://api.puter.com', 'userPuter') }; - globalThis.my = me; - globalThis.myself = me; - } - if ( event.request.headers.has('puter-auth') ) { - event.requestor = { puter: init_puter_portable(event.request.headers.get('puter-auth'), globalThis.puter_endpoint || 'https://api.puter.com', 'userPuter') }; - event.user = event.requestor; - } - - const mappings = this.map.get(event.request.method); - if ( this.handleCors && event.request.method === 'OPTIONS' && !mappings ) { - return this.handleOptions(event.request); - } - if ( ! mappings ) { - return new Response(`No routes for given request type ${event.request.method}`, { status: 404 }); - } - const url = new URL(event.request.url); - try { - for ( const mapping of mappings ) { - // return new Response(JSON.stringify(mapping)) - const results = mapping[0](url.pathname); - if ( results ) { - event.params = results.params; - let response = await mapping[1](event); - if ( ! (response instanceof Response) ) { - try { - if ( response instanceof Blob || - response instanceof ArrayBuffer || - response instanceof Uint8Array.__proto__ || - response instanceof ReadableStream || - response instanceof URLSearchParams || - typeof (response) === 'string' ) { - response = new Response(response); - } else { - response = new Response(JSON.stringify(response), { headers: { 'content-type': 'application/json' } }); - } - } catch (e) { - throw new Error('Returned response by handler was neither a Response object nor an object which can implicitly be converted into a Response object'); - } - } - if ( this.handleCors && !response.headers.has('access-control-allow-origin') ) { - response.headers.set('Access-Control-Allow-Origin', '*'); - } - return response; - } - } - } catch (e) { - const response = new Response(e, { status: 500, statusText: 'Server Error' }); - if ( this.handleCors && !response.headers.has('access-control-allow-origin') ) { - response.headers.set('Access-Control-Allow-Origin', '*'); - } - return response; - } - - return new Response('Path not found', { status: 404, statusText: 'Not found' }); - }, - }; - globalThis.router = router; - self.addEventListener('fetch', (event) => { - if ( ! router.routing ) - { - return false; - } - event.respondWith(router.route(event)); - }); -} - -export default inits2w; \ No newline at end of file diff --git a/src/backend/src/services/worker/template/puter-portable.js b/src/backend/src/services/worker/template/puter-portable.js deleted file mode 100644 index 53a958084..000000000 --- a/src/backend/src/services/worker/template/puter-portable.js +++ /dev/null @@ -1,49 +0,0 @@ -// This file is not actually in the webpack project, it is handled seperately. - -if (globalThis.Cloudflare) { - // Cloudflare Workers has a faulty EventTarget implementation which doesn't bind "this" to the event handler - // This is a workaround to bind "this" to the event handler - // https://github.com/cloudflare/workerd/issues/4453 - const __cfEventTarget = EventTarget; - globalThis.EventTarget = class EventTarget extends __cfEventTarget { - constructor(...args) { - super(...args) - } - addEventListener(type, listener, options) { - super.addEventListener(type, listener.bind(this), options); - } - } -} - -globalThis.init_puter_portable = (auth, apiOrigin, type) => { - // Who put C in my JS?? - /* - * This is a hack to include the puter.js file. - * It is not a good idea to do this, but it is the only way to get the puter.js file to work. - * The puter.js file is handled by the C preprocessor here because webpack cant behave with already minified files. - * The C preprocessor basically just includes the file and then we can use the puter.js file in the worker. - */ - if (type === "userPuter") { - const goodContext = {} - Object.getOwnPropertyNames(globalThis).forEach(name => { try { goodContext[name] = globalThis[name]; } catch {} }) - goodContext.globalThis = goodContext; - goodContext.WorkerGlobalScope = WorkerGlobalScope; - goodContext.ServiceWorkerGlobalScope = ServiceWorkerGlobalScope; - goodContext.location = new URL("https://puter.work"); - goodContext.addEventListener = ()=>{}; - // @ts-ignore - with (goodContext) { - #include "../../../../../puter-js/dist/puter.js" - } - goodContext.puter.setAPIOrigin(apiOrigin); - goodContext.puter.setAuthToken(auth); - return goodContext.puter; - } else { - #include "../../../../../puter-js/dist/puter.js" - - puter.setAPIOrigin(apiOrigin); - puter.setAuthToken(auth); - } -} -#include "../dist/webpackPreamplePart.js" - diff --git a/src/backend/src/services/worker/workerUtils/cloudflareDeploy.js b/src/backend/src/services/worker/workerUtils/cloudflareDeploy.js deleted file mode 100644 index 636d9cef4..000000000 --- a/src/backend/src/services/worker/workerUtils/cloudflareDeploy.js +++ /dev/null @@ -1,100 +0,0 @@ -const fs = require('fs'); -const { calculateWorkerNameNew } = require('./nameUtils.js'); -let config = {}; -// Constants -const CF_BASE_URL = 'https://api.cloudflare.com/'; -let WORKERS_BASE_URL; -// Workers for Platforms support - -function cfFetch (url, method = 'GET', body, givenHeaders) { - const headers = { 'Authorization': `Bearer ${ config['XAUTHKEY']}` }; - if ( givenHeaders ) { - for ( const header of givenHeaders ) { - headers[header[0]] = header[1]; - } - } - return fetch(url, { headers, method, body }); -} -async function getWorker (userData, authorization, workerId) { - await cfFetch(`${WORKERS_BASE_URL}/scripts/${calculateWorkerNameNew(userData.uuid, workerId)}`, 'GET'); -} -async function createWorker (userData, authorization, workerName, body, PREAMBLE_LENGTH) { - const formData = new FormData(); - - const workerMetaData = { - - body_part: 'swCode', - compatibility_flags: ['global_fetch_strictly_public'], - compatibility_date: '2025-07-15', - bindings: [ - { - type: 'secret_text', - name: 'puter_auth', - text: authorization, - }, - { - type: 'plain_text', - name: 'puter_endpoint', - text: config.internetExposedUrl || 'https://api.puter.com', - }, - - ], - - }; - formData.append('metadata', JSON.stringify(workerMetaData)); - formData.append('swCode', body); - const cfReturnCodes = await (await cfFetch(`${WORKERS_BASE_URL}/scripts/${workerName}/`, 'PUT', formData)).json(); - - if ( cfReturnCodes.success ) { - return { success: true, errors: [], url: `https://${workerName}.puter.work` }; - } else { - const parsedErrors = []; - for ( const error of cfReturnCodes.errors ) { - const message = error.message; - let finalMessage = ''; - const lines = message.split('\n'); - finalMessage += `${lines.shift() }\n`; - try { - // throw new Error("test") - for ( const line of lines ) { - if ( line.includes('at worker.js:') ) { - let positions = line.trimStart().replace('at worker.js:', '').split(':'); - positions[0] = parseInt(positions[0]) - PREAMBLE_LENGTH; - finalMessage += ` at worker.js:${positions.join(':')}\n`; - } else { - finalMessage += `${line }\n`; - } - } - } catch (e) { - console.error(`Failed to parse V8 Stack trace\n${ message}`); - finalMessage = message; - } - - parsedErrors.push(finalMessage); - } - return { success: false, errors: parsedErrors, url: null, body }; - } -} -function setPreambleLength (length) { - -} -function setCloudflareKeys (givenConfig) { - config = givenConfig; - WORKERS_BASE_URL = `${CF_BASE_URL }client/v4/accounts/${config.ACCOUNTID}/workers`; - if ( config.namespace ) { - WORKERS_BASE_URL += `/dispatch/namespaces/${config.namespace}`; - } - -} - -async function deleteWorker (userData, authorization, workerId) { - return await (await cfFetch(`${WORKERS_BASE_URL}/scripts/${calculateWorkerNameNew(userData.uuid, workerId)}/`, 'DELETE')).json(); - -} - -module.exports = { - createWorker, - deleteWorker, - getWorker, - setCloudflareKeys, -}; diff --git a/src/backend/src/services/worker/workerUtils/nameUtils.js b/src/backend/src/services/worker/workerUtils/nameUtils.js deleted file mode 100644 index 269db2b6d..000000000 --- a/src/backend/src/services/worker/workerUtils/nameUtils.js +++ /dev/null @@ -1,15 +0,0 @@ -// import crypto from 'node:crypto' -const crypto = require('node:crypto'); - -function sha1 (input) { - return crypto.createHash('sha1').update(input, 'utf8').digest().toString('hex').slice(0, 7); -} - -function calculateWorkerNameNew (uuid, workerId) { - - return `${workerId}`; // Used to be ${workerId}-${uuid.replaceAll("-", "")} -} -module.exports = { - sha1, - calculateWorkerNameNew, -}; \ No newline at end of file diff --git a/src/backend/src/services/worker/workerUtils/puterUtils.js b/src/backend/src/services/worker/workerUtils/puterUtils.js deleted file mode 100644 index ae5aad245..000000000 --- a/src/backend/src/services/worker/workerUtils/puterUtils.js +++ /dev/null @@ -1,14 +0,0 @@ -function getUserInfo (authorization, apiBase = 'https://puter.com') { - return fetch(`${apiBase }/whoami`, { headers: { authorization, origin: 'https://docs.puter.com' } }).then(async res => { - if ( res.status != 200 ) { - throw (`User data endpoint returned error code ${ await res.text()}`); - return; - } - - return res.json(); - }); -} - -module.exports = { - getUserInfo, -}; \ No newline at end of file diff --git a/src/backend/src/structured/README.md b/src/backend/src/structured/README.md deleted file mode 100644 index 07d6673e6..000000000 --- a/src/backend/src/structured/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Structured Code - -Each directory in this directory represents some type of -structured code. For example, everything in the directory -`./sequence` (relative to this file's location) is a -cjs module that exports an instance of [Sequence](../codex/Sequence.js). diff --git a/src/backend/src/structured/sequence/scan-permission.mjs b/src/backend/src/structured/sequence/scan-permission.mjs deleted file mode 100644 index 9c875de8c..000000000 --- a/src/backend/src/structured/sequence/scan-permission.mjs +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { Sequence } from '../../codex/Sequence.js'; -import { UserActorType } from '../../services/auth/Actor.js'; -import { PERMISSION_SCANNERS } from '../../unstructured/permission-scanners.js'; - -const permissionSequence = new Sequence([ - async function grant_if_system (a) { - const reading = a.get('reading'); - const { actor, permission_options } = a.values(); - if ( ! (actor.type instanceof UserActorType) ) { - return; - } - if ( actor.type.user.username === 'system' ) { - reading.push({ - $: 'option', - key: 'sys', - permission: permission_options[0], - source: 'implied', - by: 'system', - data: {}, - }); - return a.stop({}); - } - }, - async function rewrite_permission (a) { - let { reading, permission_options } = a.values(); - for ( let i = 0 ; i < permission_options.length ; i++ ) { - const old_perm = permission_options[i]; - const permission = await a.icall('_rewrite_permission', old_perm); - if ( permission === old_perm ) continue; - permission_options[i] = permission; - reading.push({ - $: 'rewrite', - from: old_perm, - to: permission, - }); - } - }, - async function explode_permission (a) { - let { reading, permission_options } = a.values(); - - // VERY nasty bugs can happen if this array is not cloned! - // (this was learned the hard way) - permission_options = [...permission_options]; - - for ( let i = 0 ; i < permission_options.length ; i++ ) { - const permission = permission_options[i]; - permission_options[i] = - await a.icall('get_higher_permissions', permission); - if ( permission_options[i].length > 1 ) { - reading.push({ - $: 'explode', - from: permission, - to: permission_options[i], - }); - } - } - a.set('permission_options', permission_options.flat()); - }, - async function handle_shortcuts (a) { - const reading = a.get('reading'); - const { actor, permission_options } = a.values(); - - const _permission_implicators = a.iget('_permission_implicators'); - - for ( const permission of permission_options ) - { - for ( const implicator of _permission_implicators ) { - if ( ! implicator.options?.shortcut ) continue; - - // TODO: is it possible to DRY this with concurrent implicators in permission-scanners.js? - if ( ! implicator.matches(permission) ) { - continue; - } - const implied = await implicator.check({ - actor, - permission, - }); - if ( implied ) { - reading.push({ - $: 'option', - permission, - source: 'implied', - by: implicator.id, - data: implied, - ...((actor.type.user) - ? { holder_username: actor.type.user.username } - : {}), - }); - if ( implicator.options?.shortcut ) { - a.stop(); - return; - } - } - } - } - }, - async function run_scanners (a) { - const scanners = PERMISSION_SCANNERS; - const ps = []; - for ( const scanner of scanners ) { - ps.push(scanner.scan(a)); - } - await Promise.all(ps); - }, -]); - -export default permissionSequence; diff --git a/src/backend/src/structured/sequence/share.js b/src/backend/src/structured/sequence/share.js deleted file mode 100644 index 0662d3123..000000000 --- a/src/backend/src/structured/sequence/share.js +++ /dev/null @@ -1,261 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require('../../api/APIError'); -const { Sequence } = require('../../codex/Sequence'); -const config = require('../../config'); -const { WorkList } = require('../../util/workutil'); -const { processSharesSequence } = require('./share/process_shares.js'); -const { UsernameNotifSelector } = require('../../services/NotificationService'); -const { quot } = require('@heyputer/putility').libs.string; - -/* - This code is optimized for editors supporting folding. - Fold at Level 2 to conveniently browse sequence steps. - Fold at Level 3 after opening an inner-sequence. - - If you're using VSCode { - typically "Ctrl+K, Ctrl+2" or "⌘K, ⌘2"; - to revert "Ctrl+K, Ctrl+J" or "⌘K, ⌘J"; - https://stackoverflow.com/questions/30067767 - } -*/ - -module.exports = new Sequence([ - require('./share/validate.js'), - function initialize_result_object (a) { - a.set('result', { - $: 'api:share', - $version: 'v0.0.0', - status: null, - recipients: - Array(a.get('req_recipients').length).fill(null), - shares: - Array(a.get('req_shares').length).fill(null), - serialize () { - const result = this; - for ( let i = 0 ; i < result.recipients.length ; i++ ) { - if ( ! result.recipients[i] ) continue; - if ( result.recipients[i] instanceof APIError ) { - result.status = 'mixed'; - result.recipients[i] = result.recipients[i].serialize(); - } - } - for ( let i = 0 ; i < result.shares.length ; i++ ) { - if ( ! result.shares[i] ) continue; - if ( result.shares[i] instanceof APIError ) { - result.status = 'mixed'; - result.shares[i] = result.shares[i].serialize(); - } - } - delete result.serialize; - return result; - }, - }); - }, - function initialize_worklists (a) { - const recipients_work = new WorkList(); - const shares_work = new WorkList(); - - const { req_recipients, req_shares } = a.values(); - - // track: common operations on multiple items - - for ( let i = 0 ; i < req_recipients.length ; i++ ) { - const value = req_recipients[i]; - recipients_work.push({ i, value }); - } - - for ( let i = 0 ; i < req_shares.length ; i++ ) { - const value = req_shares[i]; - shares_work.push({ i, value }); - } - - recipients_work.lockin(); - shares_work.lockin(); - - a.values({ recipients_work, shares_work }); - }, - require('./share/process_recipients.js'), - processSharesSequence, - function abort_on_error_if_mode_is_strict (a) { - const strict_mode = a.get('strict_mode'); - if ( ! strict_mode ) return; - - const result = a.get('result'); - if ( - result.recipients.some(v => v !== null) || - result.shares.some(v => v !== null) - ) { - result.serialize(); - result.status = 'aborted'; - const res = a.get('res'); - res.status(218).send(result); - a.stop(); - } - }, - function early_return_on_dry_run (a) { - if ( ! a.get('req').body.dry_run ) return; - - const { res, result, recipients_work } = a.values(); - for ( const item of recipients_work.list() ) { - result.recipients[item.i] = - { $: 'api:status-report', status: 'success' }; - } - - result.serialize(); - result.status = 'success'; - result.dry_run = true; - res.send(result); - a.stop(); - }, - async function grant_permissions_to_existing_users (a) { - const { - req, result, recipients_work, shares_work, - } = a.values(); - - const svc_permission = a.iget('services').get('permission'); - const svc_acl = a.iget('services').get('acl'); - const svc_notification = a.iget('services').get('notification'); - const svc_email = a.iget('services').get('email'); - - const actor = a.get('actor'); - - for ( const recipient_item of recipients_work.list() ) { - if ( recipient_item.type !== 'username' ) continue; - - const username = recipient_item.user.username; - - for ( const share_item of shares_work.list() ) { - const permissions = share_item.share_intent.permissions; - for ( const perm of permissions ) { - if ( perm.startsWith('fs:') || perm.startsWith('manage:fs:') ) { - await svc_acl.set_user_user(actor, - username, - perm, - undefined, - { only_if_higher: true }); - } else { - await svc_permission.grant_user_user_permission(actor, - username, - perm); - } - } - } - - const files = []; { - for ( const item of shares_work.list() ) { - if ( item.thing.$ !== 'fs-share' ) continue; - files.push(await item.node.getSafeEntry()); - } - } - - const metadata = a.get('req').body.metadata || {}; - - svc_notification.notify(UsernameNotifSelector(username), { - source: 'sharing', - icon: 'shared.svg', - title: 'Files were shared with you!', - template: 'file-shared-with-you', - fields: { - metadata, - username: actor.type.user.username, - files, - }, - text: `The user ${quot(req.user.username)} shared ` + - `${files.length} ${ - files.length === 1 ? 'file' : 'files' } ` + - 'with you.', - }); - - // Working on notifications - // Email should have a link to a shared file, right? - // .. how do I make those URLs? (gui feature) - if ( recipient_item.user.email && recipient_item.user.email_confirmed ) { - await svc_email.send_email({ - email: recipient_item.user.email, - }, 'share_by_username', { - // link: // TODO: create a link to the shared file - susername: actor.type.user.username, - rusername: username, - message: metadata.message, - }); - } - - result.recipients[recipient_item.i] = - { $: 'api:status-report', status: 'success' }; - } - }, - async function email_the_email_recipients (a) { - const { actor, recipients_work, shares_work } = a.values(); - - const svc_share = a.iget('services').get('share'); - const svc_token = a.iget('services').get('token'); - const svc_email = a.iget('services').get('email'); - - for ( const recipient_item of recipients_work.list() ) { - if ( recipient_item.type !== 'email' ) continue; - - const email = recipient_item.value; - - // data that gets stored in the `data` column of the share - const metadata = a.get('req').body.metadata || {}; - const data = { - $: 'internal:share', - $v: 'v0.0.0', - permissions: [], - metadata, - }; - - for ( const share_item of shares_work.list() ) { - const permissions = share_item.share_intent.permissions; - data.permissions.push(...permissions); - } - - // track: scoping iife - const share_token = await (async () => { - const share_uid = await svc_share.create_share({ - issuer: actor, - email, - data, - }); - return svc_token.sign('share', { - $: 'token:share', - $v: '0.0.0', - uid: share_uid, - }, { - expiresIn: '14d', - }); - })(); - - const email_link = - `${config.origin}?share_token=${share_token}`; - - await svc_email.send_email({ email }, 'share_by_email', { - link: email_link, - sender_name: actor.type.user.username, - message: metadata.message, - }); - } - }, - function send_result (a) { - const { res, result } = a.values(); - result.serialize(); - res.send(result); - }, -]); diff --git a/src/backend/src/structured/sequence/share/process_recipients.js b/src/backend/src/structured/sequence/share/process_recipients.js deleted file mode 100644 index befa0b52d..000000000 --- a/src/backend/src/structured/sequence/share/process_recipients.js +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const APIError = require('../../../api/APIError'); -const { Sequence } = require('../../../codex/Sequence'); -const config = require('../../../config'); - -const validator = require('validator'); -const { get_user } = require('../../../helpers'); - -/* - This code is optimized for editors supporting folding. - Fold at Level 2 to conveniently browse sequence steps. - Fold at Level 3 after opening an inner-sequence. - - If you're using VSCode { - typically "Ctrl+K, Ctrl+2" or "⌘K, ⌘2"; - to revert "Ctrl+K, Ctrl+J" or "⌘K, ⌘J"; - https://stackoverflow.com/questions/30067767 - } -*/ - -module.exports = new Sequence({ - name: 'process recipients', - after_each (a) { - const { recipients_work } = a.values(); - recipients_work.clear_invalid(); - }, -}, [ - function valid_username_or_email (a) { - const { result, recipients_work } = a.values(); - for ( const item of recipients_work.list() ) { - const { value, i } = item; - - if ( typeof value !== 'string' ) { - item.invalid = true; - result.recipients[i] = - APIError.create('invalid_username_or_email', null, { - value, - }); - continue; - } - - if ( value.match(config.username_regex) ) { - item.type = 'username'; - continue; - } - if ( validator.isEmail(value) ) { - item.type = 'email'; - continue; - } - - item.invalid = true; - result.recipients[i] = - APIError.create('invalid_username_or_email', null, { - value, - }); - } - }, - async function check_existing_users_for_email_shares (a) { - const { recipients_work } = a.values(); - for ( const recipient_item of recipients_work.list() ) { - if ( recipient_item.type !== 'email' ) continue; - const user = await get_user({ - email: recipient_item.value, - }); - if ( ! user ) continue; - recipient_item.type = 'username'; - recipient_item.value = user.username; - } - }, - async function check_username_specified_users_exist (a) { - const { result, recipients_work } = a.values(); - for ( const item of recipients_work.list() ) { - if ( item.type !== 'username' ) continue; - - const user = await get_user({ username: item.value }); - if ( ! user ) { - item.invalid = true; - result.recipients[item.i] = - APIError.create('user_does_not_exist', null, { - username: item.value, - }); - continue; - } - item.user = user; - } - }, - function return_state (a) { - return a; - }, -]); diff --git a/src/backend/src/structured/sequence/share/process_shares.js b/src/backend/src/structured/sequence/share/process_shares.js deleted file mode 100644 index f68506510..000000000 --- a/src/backend/src/structured/sequence/share/process_shares.js +++ /dev/null @@ -1,343 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import APIError from '../../../api/APIError.js'; -import { Sequence } from '../../../codex/Sequence.js'; -import config from '../../../config.js'; -import { get_user, get_app } from '../../../helpers.js'; -import { PermissionUtil } from '../../../services/auth/permissionUtils.mjs'; -import FSNodeParam from '../../../api/filesystem/FSNodeParam.js'; -import { TYPE_DIRECTORY } from '../../../deprecated/filesystem/FSNodeContext.js'; -import { MANAGE_PERM_PREFIX } from '../../../services/auth/permissionConts.mjs'; - -/* - This code is optimized for editors supporting folding. - Fold at Level 2 to conveniently browse sequence steps. - Fold at Level 3 after opening an inner-sequence. - - If you're using VSCode { - typically "Ctrl+K, Ctrl+2" or "⌘K, ⌘2"; - to revert "Ctrl+K, Ctrl+J" or "⌘K, ⌘J"; - https://stackoverflow.com/questions/30067767 - } -*/ - -// TODO DS: simplify these into the method -const is_plain_object = (value) => - value !== null && typeof value === 'object' && !Array.isArray(value); - -const error = (code, message) => ({ $: 'error', code, message }); - -const normalize_body = (body) => { - if ( body === undefined ) return {}; - if ( is_plain_object(body) ) return body; - return { value: body }; -}; - -const normalize_meta = (meta) => is_plain_object(meta) ? meta : {}; - -const to_standard = (type, body = {}, meta = {}) => { - if ( ! type ) { - return error('missing-type-param', 'type parameter is missing'); - } - - const prefixed_meta = Object.fromEntries(Object.entries(meta).map(([k, v]) => [`$${k}`, v])); - - return { $: type, ...prefixed_meta, ...body }; -}; - -const process_array = (value) => { - if ( value.length <= 1 || value.length > 3 ) { - return error( - 'invalid-array-length', - 'tag-typed arrays should have 1-3 elements', - ); - } - - const [type, raw_body, raw_meta] = value; - return to_standard(type, normalize_body(raw_body), normalize_meta(raw_meta)); -}; - -const process_structured = (value) => { - if ( ! Object.prototype.hasOwnProperty.call(value, 'type') ) { - return error('missing-type-property', 'missing "type" property'); - } - - return to_standard( - value.type, - normalize_body(value.body), - normalize_meta(value.meta), - ); -}; - -const process_standard = (value) => { - const meta = {}; - const body = {}; - - for ( const [key, val] of Object.entries(value) ) { - if ( key === '$' ) continue; - if ( key.startsWith('$') ) { - meta[key.slice(1)] = val; - } else { - body[key] = val; - } - } - - return to_standard(value.$, body, meta); -}; - -const parseTypeTagged = (value) => { - const is_object_like = value !== null && typeof value === 'object'; - if ( !is_object_like && !Array.isArray(value) ) { - return error('invalid-type', 'should be object or array'); - } - - if ( Array.isArray(value) ) { - return process_array(value); - } - - if ( value.$ === '$meta-body' ) { - return process_structured(value); - } - - return process_standard(value); -}; - -export const processSharesSequence = new Sequence({ - name: 'process shares', - beforeEach (a) { - const { shares_work } = a.values(); - shares_work.clear_invalid(); - }, -}, [ - function validate_share_types (a) { - const { result, shares_work } = a.values(); - - for ( const item of shares_work.list() ) { - const { i } = item; - let { value } = item; - - const thing = parseTypeTagged(value); - if ( thing.$ === 'error' ) { - item.invalid = true; - result.shares[i] = - APIError.create('format_error', null, { - message: thing.message, - }); - continue; - } - - const allowed_things = ['fs-share', 'app-share']; - if ( ! allowed_things.includes(thing.$) ) { - item.invalid = true; - result.shares[i] = - APIError.create('disallowed_thing', null, { - thing: thing.$, - accepted: allowed_things, - }); - continue; - } - - item.thing = thing; - } - }, - function create_file_share_intents (a) { - const { result, shares_work } = a.values(); - for ( const item of shares_work.list() ) { - const { thing } = item; - if ( thing.$ !== 'fs-share' ) continue; - - item.type = 'fs'; - const errors = []; - if ( ! thing.path ) { - errors.push('`path` is required'); - } - let access = thing.access; - if ( access ) { - if ( ! ['read', 'write', MANAGE_PERM_PREFIX].includes(access) ) { - errors.push('`access` should be `read` or `write`'); - } - } else access = 'read'; - - if ( errors.length ) { - item.invalid = true; - result.shares[item.i] = - APIError.create('field_errors', null, { - key: `shares[${item.i}]`, - errors, - }); - continue; - } - - item.path = thing.path; - item.share_intent = { - $: 'share-intent:file', - permissions: access === MANAGE_PERM_PREFIX ? [PermissionUtil.join(access, 'fs', thing.path)] : [PermissionUtil.join('fs', thing.path, access)], - }; - } - }, - function create_app_share_intents (a) { - const { result, shares_work } = a.values(); - for ( const item of shares_work.list() ) { - const { thing } = item; - if ( thing.$ !== 'app-share' ) continue; - - item.type = 'app'; - const errors = []; - if ( !thing.uid && !thing.name ) { - errors.push('`uid` or `name` is required'); - } - - if ( errors.length ) { - item.invalid = true; - result.shares[item.i] = - APIError.create('field_errors', null, { - key: `shares[${item.i}]`, - errors, - }); - continue; - } - - const app_selector = thing.uid - ? `uid#${thing.uid}` : thing.name; - - item.share_intent = { - $: 'share-intent:app', - permissions: [ - PermissionUtil.join('app', app_selector, 'access'), - ], - }; - continue; - } - }, - async function fetch_nodes_for_file_shares (a) { - const { req, result, shares_work } = a.values(); - for ( const item of shares_work.list() ) { - if ( item.type !== 'fs' ) continue; - const node = await (new FSNodeParam('path')).consolidate({ - req, getParam: () => item.path, - }); - - if ( ! await node.exists() ) { - item.invalid = true; - result.shares[item.i] = APIError.create('subject_does_not_exist', { - path: item.path, - }); - continue; - } - - item.node = node; - let email_path = item.path; - let is_dir = true; - if ( await node.get('type') !== TYPE_DIRECTORY ) { - is_dir = false; - // remove last component - email_path = email_path.slice(0, item.path.lastIndexOf('/') + 1); - } - - if ( email_path.startsWith('/') ) email_path = email_path.slice(1); - const email_link = `${config.origin}/show/${email_path}`; - item.is_dir = is_dir; - item.email_link = email_link; - } - }, - async function fetch_apps_for_app_shares (a) { - const { result, shares_work } = a.values(); - const db = a.iget('db'); - - for ( const item of shares_work.list() ) { - if ( item.type !== 'app' ) continue; - const { thing } = item; - - const app = await get_app(thing.uid ? - { uid: thing.uid } : { name: thing.name }); - if ( ! app ) { - item.invalid = true; - result.shares[item.i] = - // note: since we're reporting `entity_not_found` - // we will report the id as an entity-storage-compatible - // identifier. - APIError.create('entity_not_found', null, { - identifier: thing.uid - ? { uid: thing.uid } - : { id: { name: thing.name } }, - }); - } - - if ( !app.metadata || typeof (app.metadata) === 'string' ) { - app.metadata = JSON.parse(app.metadata || '{}'); - } - - item.app = app; - } - }, - async function add_subdomain_permissions (a) { - const { shares_work } = a.values(); - const actor = a.get('actor'); - const db = a.iget('db'); - - for ( const item of shares_work.list() ) { - if ( item.type !== 'app' ) continue; - const [subdomain] = await db.read( - 'SELECT * FROM subdomains WHERE associated_app_id = ? ' + - 'AND user_id = ? LIMIT 1', - [item.app.id, actor.type.user.id], - ); - if ( ! subdomain ) continue; - - // The subdomain is also owned by this user, so we'll - // add a permission for that as well - - const site_selector = `uid#${subdomain.uuid}`; - item.share_intent.permissions.push(PermissionUtil.join('site', site_selector, 'access')); - } - }, - async function add_appdata_permissions (a) { - const { shares_work } = a.values(); - for ( const item of shares_work.list() ) { - if ( item.type !== 'app' ) continue; - if ( ! item.app.metadata?.shared_appdata ) continue; - - const app_owner = await get_user({ id: item.app.owner_user_id }); - - const appdatadir = - `/${app_owner.username}/AppData/${item.app.uid}`; - const appdatadir_perm = - PermissionUtil.join('fs', appdatadir, 'write'); - - item.share_intent.permissions.push(appdatadir_perm); - } - }, - function apply_success_status_to_shares (a) { - const { result, shares_work } = a.values(); - for ( const item of shares_work.list() ) { - result.shares[item.i] = - { - $: 'api:status-report', - status: 'success', - fields: { - permission: item.permission, - }, - }; - } - }, - function return_state (a) { - return a; - }, -]); diff --git a/src/backend/src/structured/sequence/share/validate.js b/src/backend/src/structured/sequence/share/validate.js deleted file mode 100644 index 074e4bc02..000000000 --- a/src/backend/src/structured/sequence/share/validate.js +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const APIError = require('../../../api/APIError'); -const { Sequence } = require('../../../codex/Sequence'); - -/* - This code is optimized for editors supporting folding. - Fold at Level 2 to conveniently browse sequence steps. - Fold at Level 3 after opening an inner-sequence. - - If you're using VSCode { - typically "Ctrl+K, Ctrl+2" or "⌘K, ⌘2"; - to revert "Ctrl+K, Ctrl+J" or "⌘K, ⌘J"; - https://stackoverflow.com/questions/30067767 - } -*/ - -module.exports = new Sequence({ - name: 'validate request', -}, [ - function validate_metadata (a) { - const req = a.get('req'); - const metadata = req.body.metadata; - - if ( ! metadata ) return; - - if ( typeof metadata !== 'object' ) { - throw APIError.create('field_invalid', null, { - key: 'metadata', - expected: 'object', - got: metadata, - }); - } - - const MAX_KEYS = 20; - const MAX_STRING = 255; - const MAX_MESSAGE_STRING = 10 * 1024; - - if ( Object.keys(metadata).length > MAX_KEYS ) { - throw APIError.create('field_invalid', null, { - key: 'metadata', - expected: `at most ${MAX_KEYS} keys`, - got: `${Object.keys(metadata).length} keys`, - }); - } - - for ( const key in metadata ) { - const value = metadata[key]; - if ( typeof value !== 'string' && typeof value !== 'number' ) { - throw APIError.create('field_invalid', null, { - key: `metadata.${key}`, - expected: 'string or number', - got: value, - }); - } - if ( key === 'message' ) { - if ( typeof value !== 'string' ) { - throw APIError.create('field_invalid', null, { - key: `metadata.${key}`, - expected: 'string', - got: value, - }); - } - if ( value.length > MAX_MESSAGE_STRING ) { - throw APIError.create('field_invalid', null, { - key: `metadata.${key}`, - expected: `at most ${MAX_MESSAGE_STRING} characters`, - got: `${value.length} characters`, - }); - } - continue; - } - if ( typeof value === 'string' && value.length > MAX_STRING ) { - throw APIError.create('field_invalid', null, { - key: `metadata.${key}`, - expected: `at most ${MAX_STRING} characters`, - got: `${value.length} characters`, - }); - } - } - }, - function validate_mode (a) { - const req = a.get('req'); - const mode = req.body.mode; - - if ( mode === 'strict' ) { - a.set('strict_mode', true); - return; - } - if ( !mode || mode === 'best-effort' ) { - a.set('strict_mode', false); - return; - } - throw APIError.create('field_invalid', null, { - key: 'mode', - expected: '`strict`, `best-effort`, or undefined', - }); - }, - function validate_recipients (a) { - const req = a.get('req'); - let recipients = req.body.recipients; - - // A string can be adapted to an array of one string - if ( typeof recipients === 'string' ) { - recipients = [recipients]; - } - // Must be an array - if ( ! Array.isArray(recipients) ) { - throw APIError.create('field_invalid', null, { - key: 'recipients', - expected: 'array or string', - got: typeof recipients, - }); - } - // At least one recipient - if ( recipients.length < 1 ) { - throw APIError.create('field_invalid', null, { - key: 'recipients', - expected: 'at least one', - got: 'none', - }); - } - a.set('req_recipients', recipients); - }, - function validate_shares (a) { - const req = a.get('req'); - let shares = req.body.shares; - - if ( ! Array.isArray(shares) ) { - shares = [shares]; - } - - // At least one share - if ( shares.length < 1 ) { - throw APIError.create('field_invalid', null, { - key: 'shares', - expected: 'at least one', - got: 'none', - }); - } - - a.set('req_shares', shares); - }, - function return_state (a) { - return a; - }, -]); diff --git a/src/backend/src/traits/AssignableMethodsFeature.js b/src/backend/src/traits/AssignableMethodsFeature.js deleted file mode 100644 index 003867b4f..000000000 --- a/src/backend/src/traits/AssignableMethodsFeature.js +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -class AssignableMethodsFeature { - install_in_instance (instance) { - const methods = instance._get_merged_static_object('METHODS'); - - for ( const k in methods ) { - instance[k] = methods[k]; - } - } -} - -module.exports = { - AssignableMethodsFeature, -}; diff --git a/src/backend/src/traits/AsyncProviderFeature.js b/src/backend/src/traits/AsyncProviderFeature.js deleted file mode 100644 index f937c8e7b..000000000 --- a/src/backend/src/traits/AsyncProviderFeature.js +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -class AsyncProviderFeature { - install_in_instance (instance) { - instance.valueListeners_ = {}; - instance.valueFactories_ = {}; - instance.values_ = {}; - instance.rejections_ = {}; - - instance.provideValue = AsyncProviderFeature.prototype.provideValue; - instance.rejectValue = AsyncProviderFeature.prototype.rejectValue; - instance.awaitValue = AsyncProviderFeature.prototype.awaitValue; - instance.onValue = AsyncProviderFeature.prototype.onValue; - instance.setFactory = AsyncProviderFeature.prototype.setFactory; - } - - provideValue (key, value) { - this.values_[key] = value; - - let listeners = this.valueListeners_[key]; - if ( ! listeners ) return; - - delete this.valueListeners_[key]; - - for ( let listener of listeners ) { - if ( Array.isArray(listener) ) listener = listener[0]; - listener(value); - } - } - - rejectValue (key, err) { - this.rejections_[key] = err; - - let listeners = this.valueListeners_[key]; - if ( ! listeners ) return; - - delete this.valueListeners_[key]; - - for ( let listener of listeners ) { - if ( ! Array.isArray(listener) ) continue; - if ( ! listener[1] ) continue; - listener = listener[1]; - - listener(err); - } - } - - awaitValue (key) { - return new Promise ((rslv, rjct) => { - this.onValue(key, rslv, rjct); - }); - } - - onValue (key, fn, rjct) { - if ( this.values_[key] ) { - fn(this.values_[key]); - return; - } - - if ( this.rejections_[key] ) { - if ( rjct ) { - rjct(this.rejections_[key]); - } else throw this.rejections_[key]; - return; - } - - if ( ! this.valueListeners_[key] ) { - this.valueListeners_[key] = []; - } - this.valueListeners_[key].push([fn, rjct]); - - if ( this.valueFactories_[key] ) { - const fn = this.valueFactories_[key]; - delete this.valueFactories_[key]; - (async () => { - try { - const value = await fn(); - this.provideValue(key, value); - } catch (e) { - this.rejectValue(key, e); - } - })(); - } - } - - async setFactory (key, factoryFn) { - if ( this.valueListeners_[key] ) { - let v; - try { - v = await factoryFn(); - } catch (e) { - this.rejectValue(key, e); - } - this.provideValue(key, v); - return; - } - - this.valueFactories_[key] = factoryFn; - } -} - -module.exports = { - AsyncProviderFeature, -}; \ No newline at end of file diff --git a/src/backend/src/traits/ChannelFeature.js b/src/backend/src/traits/ChannelFeature.js deleted file mode 100644 index 1818f9439..000000000 --- a/src/backend/src/traits/ChannelFeature.js +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -// name: 'Channel' does not behave the same as Golang's channel construct; it -// behaves more like an EventEmitter. -class Channel { - constructor () { - this.listeners_ = []; - } - - // compare(EventService): EventService has an 'on' method, - // but it accepts a 'selector' argument to narrow the scope of events - on (callback) { - // wet: EventService also creates an object like this - const det = { - detach: () => { - const idx = this.listeners_.indexOf(callback); - if ( idx !== -1 ) { - this.listeners_.splice(idx, 1); - } - }, - }; - - this.listeners_.push(callback); - - return det; - } - - emit (...a) { - for ( const lis of this.listeners_ ) { - lis(...a); - } - } -} - -class ChannelFeature { - install_in_instance (instance) { - const channels = instance._get_merged_static_array('CHANNELS'); - - instance.channels = {}; - for ( const name of channels ) { - instance.channels[name] = new Channel(name); - } - } -} - -module.exports = { - ChannelFeature, -}; diff --git a/src/backend/src/traits/ContextAwareFeature.js b/src/backend/src/traits/ContextAwareFeature.js deleted file mode 100644 index c701af0d4..000000000 --- a/src/backend/src/traits/ContextAwareFeature.js +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { Context } = require('../util/context'); - -class ContextAwareFeature { - install_in_instance (instance) { - instance.context = Context.get(); - instance.x = instance.context; - } -} - -module.exports = { - ContextAwareFeature, -}; diff --git a/src/backend/src/traits/OtelFeature.js b/src/backend/src/traits/OtelFeature.js deleted file mode 100644 index 71d1e007f..000000000 --- a/src/backend/src/traits/OtelFeature.js +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { Context } = require('../util/context'); -const { getTracer } = require('../util/otelutil'); - -class OtelFeature { - constructor (method_include_list) { - this.method_include_list = method_include_list; - } - install_in_instance (instance) { - for ( const method_name of this.method_include_list ) { - const original_method = instance[method_name]; - instance[method_name] = async (...args) => { - const context = Context.get(); - // This happens when internal services call, such as PuterVersionService - if ( ! context ) return; - - const class_name = instance.constructor.name; - - const tracer = getTracer(); - let result; - await tracer.startActiveSpan(`${class_name}:${method_name}`, async span => { - result = await original_method.call(instance, ...args); - span.end(); - }); - return result; - }; - } - } -} - -module.exports = { - OtelFeature, -}; diff --git a/src/backend/src/traits/SyncFeature.js b/src/backend/src/traits/SyncFeature.js deleted file mode 100644 index 9bc785059..000000000 --- a/src/backend/src/traits/SyncFeature.js +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { Lock } = require('@heyputer/putility').libs.promise; - -class SyncFeature { - constructor (method_include_list) { - this.method_include_list = method_include_list; - } - - install_in_instance (instance) { - for ( const method_name of this.method_include_list ) { - const original_method = instance[method_name]; - const lock = new Lock(); - instance[method_name] = async (...args) => { - return await lock.acquire(async () => { - return await original_method.call(instance, ...args); - }); - }; - } - } -} - -module.exports = { - SyncFeature, -}; diff --git a/src/backend/src/traits/WeakConstructorFeature.js b/src/backend/src/traits/WeakConstructorFeature.js deleted file mode 100644 index 5d2c9704d..000000000 --- a/src/backend/src/traits/WeakConstructorFeature.js +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -class WeakConstructorFeature { - install_in_instance (instance, { parameters }) { - for ( const key in parameters ) { - instance[key] = parameters[key]; - } - } -} - -module.exports = { - WeakConstructorFeature, -}; diff --git a/src/backend/src/unstructured/permission-scan-lib.js b/src/backend/src/unstructured/permission-scan-lib.js deleted file mode 100644 index 712fe9952..000000000 --- a/src/backend/src/unstructured/permission-scan-lib.js +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -/** - * Filters a permission reading so that it does not contain paths through the - * specified user. This operation is performed recursively on all paths in the - * reading. - * - * This does not prevent all possible cycles. To prevent all cycles, this filter - * must by applied on each reading for a permission holder, specifying the - * permission issuer as the user to filter out. - */ -const remove_paths_through_user = ({ reading, user }) => { - const no_cycle_reading = []; - - for ( const node of reading ) { - if ( node.$ === 'path' ) { - if ( - node.issuer_username === user.username - ) { - continue; - } - - node.reading = remove_paths_through_user({ - reading: node.reading, - user, - }); - } - - no_cycle_reading.push(node); - } - - return no_cycle_reading; -}; - -const reading_has_terminal = ({ reading }) => { - for ( const node of reading ) { - if ( node.has_terminal ) { - return true; - } - if ( node.$ === 'option' ) { - return true; - } - } - - return false; -}; - -module.exports = { - remove_paths_through_user, - reading_has_terminal, -}; diff --git a/src/backend/src/unstructured/permission-scanners.js b/src/backend/src/unstructured/permission-scanners.js deleted file mode 100644 index c7588de2d..000000000 --- a/src/backend/src/unstructured/permission-scanners.js +++ /dev/null @@ -1,460 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { - default_implicit_user_app_permissions, - implicit_user_app_permissions, - hardcoded_user_group_permissions, -} = require('../data/hardcoded-permissions'); -const { get_user } = require('../helpers'); -const { Actor, UserActorType, AppUnderUserActorType, AccessTokenActorType } = require('../services/auth/Actor'); -const { reading_has_terminal } = require('./permission-scan-lib'); - -/* - OPTIMAL FOLD LEVEL: 3 - - "Ctrl+K, Ctrl+3" or "⌘K, ⌘3"; - "Ctrl+K, Ctrl+J" or "⌘K, ⌘J"; -*/ - -/** - * - * @type { {name:string, documentation:string, scan: (a:import('../codex/Sequence.js').A)=>Promise }[]} - * Permission Scanners - * @usedBy scan-permission.js - * - * These are all the different ways an entity (user or app) can have a permission. - * This list of scanners is iterated over and invoked by scan-permission.js. - * - * Each `scan` function is passed a sequence scope. The instance attached to the - * sequence scope is PermissionService itself, so any `a.iget('something')` is - * accessing the member 'something' of the PermissionService instance. - */ -const PERMISSION_SCANNERS = [ - { - name: 'implied', - documentation: ` - Scans for permissions that are implied by "permission implicators". - - Permission implicators are added by other services via - PermissionService's \`register_implicator\` method. - `, - async scan (a) { - const reading = a.get('reading'); - const { actor, permission_options } = a.values(); - - const _permission_implicators = a.iget('_permission_implicators'); - - for ( const permission of permission_options ) - { - for ( const implicator of _permission_implicators ) { - if ( implicator.options?.shortcut ) continue; - - if ( ! implicator.matches(permission) ) { - continue; - } - const implied = await implicator.check({ - actor, - permission, - }); - if ( implied ) { - reading.push({ - $: 'option', - permission, - source: 'implied', - by: implicator.id, - data: implied, - ...((actor.type.user) - ? { holder_username: actor.type.user.username } - : {}), - }); - if ( implicator.options?.shortcut ) { - a.stop(); - return; - } - } - } - } - }, - }, - { - name: 'access-token', - documentation: ` - Permissoins for access tokens - `, - async scan (a) { - const { reading, actor, permission_options } = a.values(); - - if ( ! (actor.type instanceof AccessTokenActorType) ) return; - - const { authorizer: issuer_actor, token } = actor.type; - - for ( const permission of permission_options ) { - const issuer_reading = - await a.icall('scan', issuer_actor, permission); - const has_terminal = reading_has_terminal({ reading: issuer_reading }); - - const db = a.iget('db'); - const rows = await db.read( - 'SELECT * FROM `access_token_permissions` ' + - 'WHERE `token_uid` = ? AND `permission` = ?', - [ - token, - permission, - ], - ); - - // Token must have permission - if ( ! rows[0] ) continue; - - reading.push({ - $: 'path', - via: 'access-token', - has_terminal, - permission, - reading: issuer_reading, - }); - } - }, - }, - { - name: 'user-user', - documentation: ` - User-to-User permissions are permission granted form one user to another. - `, - async scan (a) { - const { reading, actor, permission_options, state } = a.values(); - if ( ! (actor.type instanceof UserActorType) ) { - return; - } - const subReadings = await a.icall('validateUserPerms', { actor, permissions: permission_options, state }); - reading.push(...subReadings); - - }, - }, - { - name: 'hc-user-group-user', - documentation: ` - These are user-to-group permissions that are defined in the - hardcoded_user_group_permissions section of "hardcoded-permissions.js". - - These are typically used to grant permissions from the system user to - the default groups: "admin", "user", and "temp". - `, - async scan (a) { - const { reading, actor, permission_options } = a.values(); - if ( ! (actor.type instanceof UserActorType) ) { - return; - } - - const svc_group = await a.iget('services').get('group'); - const groups = await svc_group.list_groups_with_member({ user_id: actor.type.user.id }); - const group_uids = {}; - for ( const group of groups ) { - group_uids[group.values.uid] = group; - } - - for ( const issuer_username in hardcoded_user_group_permissions ) { - const issuer_actor = new Actor({ - type: new UserActorType({ - user: await get_user({ username: issuer_username }), - }), - }); - const issuer_groups = - hardcoded_user_group_permissions[issuer_username]; - for ( const group_uid in issuer_groups ) { - if ( ! group_uids[group_uid] ) continue; - const issuer_group = issuer_groups[group_uid]; - for ( const permission of permission_options ) { - if ( ! Object.prototype.hasOwnProperty.call(issuer_group, permission) ) continue; - const issuer_reading = - await a.icall('scan', issuer_actor, permission); - - const has_terminal = reading_has_terminal({ reading: issuer_reading }); - - reading.push({ - $: 'path', - via: 'hc-user-group', - has_terminal, - permission, - data: issuer_group[permission], - holder_username: actor.type.user.username, - issuer_username, - reading: issuer_reading, - group_id: group_uids[group_uid].id, - }); - } - } - } - }, - }, - { - name: 'user-group-user', - documentation: ` - This scans for permissions that are granted to the user because a - group they are a member of was granted this permission by another - user. - `, - async scan (a) { - const { reading, actor, permission_options } = a.values(); - if ( ! (actor.type instanceof UserActorType) ) { - return; - } - const db = a.iget('db'); - - let sql_perm = permission_options.map(() => - 'p.permission = ?').join(' OR '); - - if ( permission_options.length > 1 ) { - sql_perm = `(${sql_perm})`; - } - const rows = await db.read( - 'SELECT p.permission, p.user_id, p.group_id, p.extra FROM `user_to_group_permissions` p ' + - 'JOIN `jct_user_group` ug ON p.group_id = ug.group_id ' + - `WHERE ug.user_id = ? AND ${sql_perm}`, - [ - actor.type.user.id, - ...permission_options, - ], - ); - - for ( const row of rows ) { - if ( !row.extra || typeof (row.extra) === 'string' ) { - row.extra = JSON.parse(row.extra || '{}'); - } - - const issuer_actor = new Actor({ - type: new UserActorType({ - user: await get_user({ id: row.user_id }), - }), - }); - - const issuer_reading = await a.icall('scan', issuer_actor, row.permission); - - const has_terminal = reading_has_terminal({ reading: issuer_reading }); - - reading.push({ - $: 'path', - via: 'user-group', - has_terminal, - // issuer: issuer_actor, - permission: row.permission, - data: row.extra, - holder_username: actor.type.user.username, - issuer_username: issuer_actor.type.user.username, - reading: issuer_reading, - group_id: row.group_id, - }); - } - }, - }, - { - name: 'user-virtual-group-user', - documentation: ` - These are groups with computed membership. Permissions are not granted - to these groups; instead the groups are defined with a list of - permissions that are granted to the group members. - - Services can define "virtual groups" via the "virtual-group" service. - Services can also register membership implicators for virtual groups - which will compute on the fly whether or not an actor should be - considered a member of the group. - `, - async scan (a) { - const svc_virtualGroup = await a.iget('services').get('virtual-group'); - const { reading, actor, permission_options } = a.values(); - const groups = svc_virtualGroup.get_virtual_groups({ actor }); - - for ( const group of groups ) { - for ( const perm_entry of group.permissions ) { - const { permission, data } = perm_entry; - if ( ! permission_options.includes(permission) ) { - continue; - } - reading.push({ - $: 'option', - permission, - data, - holder_username: actor.type.user.username, - source: 'virtual-group', - vgroup_id: group.id, - }); - } - } - }, - }, - { - name: 'user-app-implied', - documentation: ` - Some permissions are implied for apps as long as the user also has - these permissions. - `, - async scan (a) { - const { reading, actor, permission_options } = a.values(); - if ( ! (actor.type instanceof AppUnderUserActorType) ) { - return; - } - const issuer_actor = actor.get_related_actor(UserActorType); - const issuer_reading = await a.icall('scan', issuer_actor, permission_options); - const has_terminal = reading_has_terminal({ reading: issuer_reading }); - const app_uid = actor.type.app.uid; - for ( const permission of permission_options ) { - { - - const implied = default_implicit_user_app_permissions[permission]; - if ( implied ) { - reading.push({ - $: 'path', - permission, - has_terminal, - source: 'user-app-implied', - by: 'user-app-hc-1', - data: implied, - issuer_username: actor.type.user.username, - reading: issuer_reading, - }); - } - } { - const implicit_permissions = {}; - for ( const implicit_permission of implicit_user_app_permissions ) { - if ( implicit_permission.apps.includes(app_uid) ) { - implicit_permissions[permission] = implicit_permission.permissions[permission]; - } - } - if ( implicit_permissions[permission] ) { - reading.push({ - $: 'path', - permission, - has_terminal, - source: 'user-app-implied', - by: 'user-app-hc-2', - data: implicit_permissions[permission], - issuer_username: actor.type.user.username, - reading: issuer_reading, - }); - } - } - } - - }, - }, - { - name: 'user-app', - documentation: ` - If the actor is an app, this scans for permissions granted to the app - because the user has the permission and granted it to the app. - `, - async scan (a) { - const { reading, actor, permission_options } = a.values(); - if ( ! (actor.type instanceof AppUnderUserActorType) ) { - return; - } - const db = a.iget('db'); - - let sql_perm = permission_options.map(() => - '`permission` = ?').join(' OR '); - if ( permission_options.length > 1 ) sql_perm = `(${sql_perm})`; - - // SELECT permission - const rows = await db.read( - 'SELECT * FROM `user_to_app_permissions` ' + - `WHERE \`user_id\` = ? AND \`app_id\` = ? AND ${ - sql_perm}`, - [ - actor.type.user.id, - actor.type.app.id, - ...permission_options, - ], - ); - - if ( rows[0] ) { - const row = rows[0]; - if ( !row.extra || typeof (row.extra) === 'string' ) { - row.extra = JSON.parse(row.extra || '{}'); - } - const issuer_actor = actor.get_related_actor(UserActorType); - const issuer_reading = await a.icall('scan', issuer_actor, row.permission); - const has_terminal = reading_has_terminal({ reading: issuer_reading }); - reading.push({ - $: 'path', - via: 'user-app', - permission: row.permission, - has_terminal, - data: row.extra, - issuer_username: actor.type.user.username, - reading: issuer_reading, - }); - } - }, - }, - { - name: 'user-app', - documentation: ` - If the actor is an app, this scans for permissions granted to the app - because any other user has the permission and granted it to the app - for all users of the app. - `, - async scan (a) { - const { reading, actor, permission_options } = a.values(); - if ( ! (actor.type instanceof AppUnderUserActorType) ) { - return; - } - const db = a.iget('db'); - - let sql_perm = permission_options.map(() => - '`permission` = ?').join(' OR '); - if ( permission_options.length > 1 ) sql_perm = `(${sql_perm})`; - - // SELECT permission - const rows = await db.read( - 'SELECT * FROM `dev_to_app_permissions` ' + - `WHERE \`app_id\` = ? AND ${ - sql_perm}`, - [ - actor.type.app.id, - ...permission_options, - ], - ); - - if ( rows[0] ) { - const row = rows[0]; - if ( !row.extra || typeof (row.extra) === 'string' ) { - row.extra = JSON.parse(row.extra || '{}'); - } - const issuer_user = await get_user({ id: row.user_id }); - const issuer_actor = Actor.adapt(issuer_user); - const issuer_reading = await a.icall('scan', issuer_actor, row.permission); - const has_terminal = reading_has_terminal({ reading: issuer_reading }); - reading.push({ - $: 'path', - via: 'dev-app', - permission: row.permission, - has_terminal, - data: row.extra, - issuer_username: actor.type.user.username, - reading: issuer_reading, - }); - } - }, - }, -]; - -module.exports = { - PERMISSION_SCANNERS, -}; diff --git a/src/backend/src/util/.gitignore b/src/backend/src/util/.gitignore deleted file mode 100644 index c6ac5cd28..000000000 --- a/src/backend/src/util/.gitignore +++ /dev/null @@ -1 +0,0 @@ -outcomeutil.js diff --git a/src/backend/src/util/CircularQueue.bench.js b/src/backend/src/util/CircularQueue.bench.js deleted file mode 100644 index d100eae03..000000000 --- a/src/backend/src/util/CircularQueue.bench.js +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { bench, describe } from 'vitest'; -import { CircularQueue } from './CircularQueue.js'; - -/** - * Naive array-based implementation for comparison (no Map optimization). - * This serves as a baseline to demonstrate the performance improvement - * of the Map-optimized CircularQueue. - */ -class NaiveCircularQueue { - constructor (size) { - this.size = size; - this.queue = []; - this.index = 0; - } - - push (item) { - this.queue[this.index] = item; - this.index = (this.index + 1) % this.size; - } - - get (index) { - return this.queue[(this.index + index) % this.size]; - } - - has (item) { - return this.queue.includes(item); - } - - maybe_consume (item) { - const index = this.queue.indexOf(item); - if ( index !== -1 ) { - this.queue[index] = null; - return true; - } - return false; - } -} - -// Generate test tokens -const generateToken = () => Math.random().toString(36).substring(2, 15); - -describe('CircularQueue - push() operations', () => { - bench('push() with size=50', () => { - const queue = new CircularQueue(50); - for ( let i = 0; i < 1000; i++ ) { - queue.push(generateToken()); - } - }); - - bench('push() with size=500', () => { - const queue = new CircularQueue(500); - for ( let i = 0; i < 1000; i++ ) { - queue.push(generateToken()); - } - }); - - bench('NaiveCircularQueue push() with size=50 (baseline)', () => { - const queue = new NaiveCircularQueue(50); - for ( let i = 0; i < 1000; i++ ) { - queue.push(generateToken()); - } - }); -}); - -describe('CircularQueue - has() operations', () => { - const setupQueue = (QueueClass, size) => { - const queue = new QueueClass(size); - const tokens = []; - for ( let i = 0; i < size; i++ ) { - const token = generateToken(); - tokens.push(token); - queue.push(token); - } - return { queue, tokens }; - }; - - bench('has() on existing items - CircularQueue', () => { - const { queue, tokens } = setupQueue(CircularQueue, 100); - for ( let i = 0; i < 1000; i++ ) { - queue.has(tokens[i % tokens.length]); - } - }); - - bench('has() on existing items - NaiveCircularQueue (baseline)', () => { - const { queue, tokens } = setupQueue(NaiveCircularQueue, 100); - for ( let i = 0; i < 1000; i++ ) { - queue.has(tokens[i % tokens.length]); - } - }); - - bench('has() on non-existing items - CircularQueue', () => { - const { queue } = setupQueue(CircularQueue, 100); - for ( let i = 0; i < 1000; i++ ) { - queue.has(`nonexistent-token-${ i}`); - } - }); - - bench('has() on non-existing items - NaiveCircularQueue (baseline)', () => { - const { queue } = setupQueue(NaiveCircularQueue, 100); - for ( let i = 0; i < 1000; i++ ) { - queue.has(`nonexistent-token-${ i}`); - } - }); -}); - -describe('CircularQueue - maybe_consume() operations', () => { - bench('maybe_consume() on existing items', () => { - const queue = new CircularQueue(100); - const tokens = []; - for ( let i = 0; i < 100; i++ ) { - const token = generateToken(); - tokens.push(token); - queue.push(token); - } - for ( const token of tokens ) { - queue.maybe_consume(token); - } - }); - - bench('maybe_consume() mixed existing/non-existing', () => { - const queue = new CircularQueue(100); - const tokens = []; - for ( let i = 0; i < 100; i++ ) { - const token = generateToken(); - tokens.push(token); - queue.push(token); - } - for ( let i = 0; i < 200; i++ ) { - if ( i % 2 === 0 && i / 2 < tokens.length ) { - queue.maybe_consume(tokens[i / 2]); - } else { - queue.maybe_consume(`fake-token-${ i}`); - } - } - }); -}); - -describe('CircularQueue - real-world usage pattern', () => { - bench('CSRF token lifecycle: generate, validate, consume', () => { - const queue = new CircularQueue(50); - const activeTokens = []; - - for ( let i = 0; i < 500; i++ ) { - // Generate new token - const token = generateToken(); - queue.push(token); - activeTokens.push(token); - - // Occasionally validate tokens - if ( i % 3 === 0 && activeTokens.length > 0 ) { - const checkToken = activeTokens[Math.floor(Math.random() * activeTokens.length)]; - queue.has(checkToken); - } - - // Occasionally consume tokens - if ( i % 5 === 0 && activeTokens.length > 0 ) { - const consumeToken = activeTokens.shift(); - queue.maybe_consume(consumeToken); - } - } - }); -}); diff --git a/src/backend/src/util/CircularQueue.js b/src/backend/src/util/CircularQueue.js deleted file mode 100644 index f29ddf33a..000000000 --- a/src/backend/src/util/CircularQueue.js +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -/** - * A utility class to manage a circular queue with O(1) lookup. - * Uses a Map for fast membership checks and a circular array for storage. - * - * Items expire when they are evicted from the queue (when the queue is full - * and a new item is pushed). - */ -export class CircularQueue { - /** - * Creates a new CircularQueue instance with the specified size. - * - * @param {number} size - The maximum number of items the queue can hold - */ - constructor (size) { - this.size = size; - this.queue = []; - this.index = 0; - this.map = new Map(); - } - - /** - * Adds an item to the queue. If the queue is full, the oldest item is removed. - * - * @param {*} item - The item to add to the queue - */ - push (item) { - if ( this.queue[this.index] ) { - this.map.delete(this.queue[this.index]); - } - this.queue[this.index] = item; - this.map.set(item, this.index); - this.index = (this.index + 1) % this.size; - } - - /** - * Retrieves an item from the queue at the specified relative index. - * - * @param {number} index - The relative index from the current position - * @returns {*} The item at the specified index - */ - get (index) { - return this.queue[(this.index + index) % this.size]; - } - - /** - * Checks if the queue contains the specified item. - * - * @param {*} item - The item to check for - * @returns {boolean} True if the item exists in the queue, false otherwise - */ - has (item) { - return this.map.has(item); - } - - /** - * Attempts to consume (remove) an item from the queue if it exists. - * - * @param {*} item - The item to consume - * @returns {boolean} True if the item was found and consumed, false otherwise - */ - maybe_consume (item) { - if ( this.has(item) ) { - const index = this.map.get(item); - this.map.delete(item); - this.queue[index] = null; - return true; - } - return false; - } -} diff --git a/src/backend/src/util/asyncutil.js b/src/backend/src/util/asyncutil.js deleted file mode 100644 index b6f9158f2..000000000 --- a/src/backend/src/util/asyncutil.js +++ /dev/null @@ -1,18 +0,0 @@ -const sleep = async ms => { - await new Promise(rslv => setTimeout(rslv, ms)); -}; - -const atimeout = async (ms, p) => { - return await Promise.race([ - p, - new Promise(async (rslv, rjct) => { - await sleep(ms); - rjct('timeout'); - }), - ]); -}; - -module.exports = { - sleep, - atimeout, -}; diff --git a/src/backend/src/util/configutil.js b/src/backend/src/util/configutil.js deleted file mode 100644 index e0ff1d67b..000000000 --- a/src/backend/src/util/configutil.js +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -let memoized_common_template_vars_ = null; -const get_common_template_vars = () => { - const path_ = require('path'); - if ( memoized_common_template_vars_ !== null ) { - return memoized_common_template_vars_; - } - - const code_root = path_.resolve(__dirname, '../../'); - - memoized_common_template_vars_ = { - code_root, - }; - - return memoized_common_template_vars_; -}; - -module.exports = { - get_common_template_vars, -}; diff --git a/src/backend/src/util/consolelog.js b/src/backend/src/util/consolelog.js deleted file mode 100644 index fa952b8d9..000000000 --- a/src/backend/src/util/consolelog.js +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -class ConsoleLogManager { - static instance_; - - static getInstance () { - if ( this.instance_ ) return this.instance_; - return this.instance_ = new ConsoleLogManager(); - } - - static CONSOLE_METHODS = [ - 'log', 'error', 'warn', - ]; - - static PROXY_METHOD = function (method, ...args) { - const decorators = this.get_log_decorators_(method); - - // TODO: Add this feature later - // const pre_listeners = self.get_log_pre_listeners_(method); - // const post_listeners = self.get_log_post_listeners_(method); - - const replace = (...newargs) => { - args = newargs; - }; - for ( const dec of decorators ) { - dec({ - manager: this, - replace, - }, ...args); - } - - this.__original_methods[method](...args); - - const post_hooks = this.get_post_hooks_(method); - for ( const fn of post_hooks ) { - fn(); - } - }; - - get_log_decorators_ (method) { - return this.__log_decorators[method]; - } - - get_post_hooks_ (method) { - return this.__log_hooks_post[method]; - } - - constructor () { - const THIS = this.constructor; - this.__original_console = console; - this.__original_methods = {}; - for ( const k of THIS.CONSOLE_METHODS ) { - this.__original_methods[k] = console[k]; - } - this.__proxy_methods = {}; - this.__log_decorators = {}; - this.__log_hooks_post = {}; - - // TODO: Add this feature later - // this.__log_pre_listeners = {}; - // this.__log_post_listeners = {}; - } - - initialize_proxy_methods (methods) { - const THIS = this.constructor; - methods = methods || THIS.CONSOLE_METHODS; - for ( const k of methods ) { - this.__proxy_methods[k] = THIS.PROXY_METHOD.bind(this, k); - console[k] = this.__proxy_methods[k]; - this.__log_decorators[k] = []; - this.__log_hooks_post[k] = []; - } - } - - decorate (method, dec_fn) { - this.__log_decorators[method] = dec_fn; - } - - decorate_all (dec_fn) { - const THIS = this.constructor; - for ( const k of THIS.CONSOLE_METHODS ) { - this.__log_decorators[k].push(dec_fn); - } - } - - post_all (post_fn) { - const THIS = this.constructor; - for ( const k of THIS.CONSOLE_METHODS ) { - this.__log_hooks_post[k].push(post_fn); - } - } - - log_raw (method, ...args) { - this.__original_methods[method](...args); - } -} - -module.exports = { - consoleLogManager: ConsoleLogManager.getInstance(), -}; diff --git a/src/backend/src/util/context.bench.js b/src/backend/src/util/context.bench.js deleted file mode 100644 index db07172ce..000000000 --- a/src/backend/src/util/context.bench.js +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { bench, describe } from 'vitest'; -import { Context } from './context.js'; - -describe('Context - Creation', () => { - bench('create empty context', () => { - Context.create({}); - }); - - bench('create context with single value', () => { - Context.create({ user: 'testuser' }); - }); - - bench('create context with multiple values', () => { - Context.create({ - user: 'testuser', - requestId: '12345', - timestamp: Date.now(), - metadata: { key: 'value' }, - }); - }); - - bench('create 100 contexts', () => { - for ( let i = 0; i < 100; i++ ) { - Context.create({ index: i }); - } - }); -}); - -describe('Context - Sub-context creation', () => { - const parentContext = Context.create({ parent: 'value' }); - - bench('create sub-context (empty)', () => { - parentContext.sub({}); - }); - - bench('create sub-context with values', () => { - parentContext.sub({ child: 'childValue' }); - }); - - bench('create sub-context with name', () => { - parentContext.sub({}, 'named-context'); - }); - - bench('create deeply nested sub-contexts (5 levels)', () => { - let ctx = parentContext; - for ( let i = 0; i < 5; i++ ) { - ctx = ctx.sub({ level: i }); - } - }); - - bench('create deeply nested sub-contexts (10 levels)', () => { - let ctx = parentContext; - for ( let i = 0; i < 10; i++ ) { - ctx = ctx.sub({ level: i }); - } - }); -}); - -describe('Context - Get/Set operations', () => { - const ctx = Context.create({ - key1: 'value1', - key2: 'value2', - key3: { nested: 'object' }, - }); - - bench('get existing key', () => { - ctx.get('key1'); - }); - - bench('get non-existing key', () => { - ctx.get('nonexistent'); - }); - - bench('get nested object', () => { - ctx.get('key3'); - }); - - bench('set new value', () => { - ctx.set('dynamic', Math.random()); - }); - - bench('get/set cycle (100 operations)', () => { - for ( let i = 0; i < 100; i++ ) { - ctx.set(`key_${i}`, i); - ctx.get(`key_${i}`); - } - }); -}); - -describe('Context - Prototype chain lookup', () => { - // Create a deep context chain - let deepCtx = Context.create({ root: 'rootValue' }); - for ( let i = 0; i < 10; i++ ) { - deepCtx = deepCtx.sub({ [`level${i}`]: `value${i}` }); - } - - bench('get value from root (10 levels up)', () => { - deepCtx.get('root'); - }); - - bench('get value from middle (5 levels up)', () => { - deepCtx.get('level5'); - }); - - bench('get value from current level', () => { - deepCtx.get('level9'); - }); -}); - -describe('Context - arun async execution', () => { - const ctx = Context.create({ test: 'value' }); - - bench('arun with simple callback', async () => { - await ctx.arun(async () => { - return 'result'; - }); - }); - - bench('arun with Context.get inside', async () => { - await ctx.arun(async () => { - Context.get('test'); - return 'result'; - }); - }); - - bench('nested arun calls (3 levels)', async () => { - await ctx.arun(async () => { - const subCtx = Context.get().sub({ level: 1 }); - await subCtx.arun(async () => { - const subSubCtx = Context.get().sub({ level: 2 }); - await subSubCtx.arun(async () => { - return Context.get('level'); - }); - }); - }); - }); -}); - -describe('Context - abind', () => { - const ctx = Context.create({ bound: 'value' }); - - bench('create bound function', () => { - ctx.abind(() => 'result'); - }); - - bench('execute bound function', async () => { - const boundFn = ctx.abind(async () => Context.get('bound')); - await boundFn(); - }); -}); - -describe('Context - describe/debug', () => { - const ctx = Context.create({ test: 'value' }, 'test-context'); - const deepCtx = ctx.sub({ level: 1 }, 'sub1').sub({ level: 2 }, 'sub2'); - - bench('describe shallow context', () => { - ctx.describe(); - }); - - bench('describe deep context', () => { - deepCtx.describe(); - }); -}); - -describe('Context - unlink (memory cleanup)', () => { - bench('create and unlink context', () => { - const ctx = Context.create({ - user: 'test', - data: { large: 'object' }, - }); - ctx.unlink(); - }); -}); - -describe('Context - Real-world simulation', () => { - bench('HTTP request context lifecycle', async () => { - // Simulate creating a context for an HTTP request - const reqCtx = Context.create({ - req: { method: 'GET', path: '/api/test' }, - res: {}, - trace_request: 'uuid-here', - }, 'req'); - - await reqCtx.arun(async () => { - // Simulate middleware adding data - const ctx = Context.get(); - ctx.set('user', { id: 1, name: 'test' }); - - // Simulate sub-operation - const opCtx = ctx.sub({ operation: 'readFile' }); - await opCtx.arun(async () => { - Context.get('user'); - Context.get('operation'); - }); - }); - }); -}); diff --git a/src/backend/src/util/context.d.ts b/src/backend/src/util/context.d.ts deleted file mode 100644 index 196bd616e..000000000 --- a/src/backend/src/util/context.d.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { AsyncLocalStorage } from 'async_hooks'; -import { Actor } from '../services/auth/Actor'; -import type { ServiceResources } from '../services/BaseService'; - -type AnyRecord = Record; - -interface ContextCreateHookPayload { - values: AnyRecord; - name?: string; -} - -interface ContextArunHookPayload { - hints: AnyRecord; - name?: string; - trace_name?: string; - replace_callback: (cb: () => unknown | Promise) => void; - callback: () => unknown | Promise; -} - -declare interface IContext { - get (): Context; - get (k: 'actor', options?: { allow_fallback?: boolean }): Actor; - get (k: 'services', options?: { allow_fallback?: boolean }): ServiceResources['services']; - get(k?: string, options?: { allow_fallback?: boolean }): T; -} - -declare class Context { - static USE_NAME_FALLBACK: Record; - static next_name_: number; - static other_next_names_: Record; - static context_hooks_: { - pre_create: Array<(payload: ContextCreateHookPayload) => void>; - post_create: unknown[]; - pre_arun: Array<(payload: ContextArunHookPayload) => void>; - }; - static contextAsyncLocalStorage: AsyncLocalStorage>; - static __last_context_key: number; - static make_context_key (opt_human_readable?: string): string; - static create(values: T, opt_name?: string): Context; - static get: IContext['get']; - static set (k: string, v: unknown): void; - static root: Context; - static describe (): string; - static arun(...args: unknown[]): Promise; - static sub (values: AnyRecord | string, opt_name?: string): Context; - - trace_name?: string; - name?: string; - - constructor (imm_values: AnyRecord, opt_parent?: Context, opt_name?: string); - unlink (): void; - get: IContext['get']; - set (k: string, v: unknown): void; - sub (values: AnyRecord | string, opt_name?: string): Context; - get values (): AnyRecord; - get_proxy_object (): AnyRecord; - arun(...args: unknown[]): Promise; - abind(cb: (...args: unknown[]) => T | Promise): (...args: unknown[]) => Promise; - describe (): string; - describe_ (): string; - static allow_fallback(cb: () => Promise | T): Promise; -} - -declare class ContextExpressMiddleware { - constructor (args: { parent: Context }); - install (app: { use: (handler: (...args: unknown[]) => void) => void }): void; - run (req: AnyRecord, res: AnyRecord, next: (...args: unknown[]) => void): Promise; -} - -declare const context_config: { strict?: boolean } & AnyRecord; - -export { Context, context_config, ContextExpressMiddleware }; diff --git a/src/backend/src/util/context.js b/src/backend/src/util/context.js deleted file mode 100644 index 8c78c5634..000000000 --- a/src/backend/src/util/context.js +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { AsyncLocalStorage } from 'async_hooks'; -import { randomUUID } from 'crypto'; -import { v4 as uuidv4 } from 'uuid'; - -// Singleton pattern to ensure ESM and CJS loads share the same class instance in vitest -const CONTEXT_SINGLETON_KEY = Symbol.for('puter.context.module'); - -let Context; -let ContextExpressMiddleware; -let context_config; - -if ( globalThis[CONTEXT_SINGLETON_KEY] ) { - // Use existing singleton - ({ Context, ContextExpressMiddleware, context_config } = globalThis[CONTEXT_SINGLETON_KEY]); -} else { - // Define classes for the first time - context_config = {}; - - Context = class Context { - static testId = randomUUID(); - - static USE_NAME_FALLBACK = {}; - static next_name_ = 0; - static other_next_names_ = {}; - - // Context hooks should be registered via service (ContextService.js) - static context_hooks_ = { - pre_create: [], - post_create: [], - pre_arun: [], - }; - - static contextAsyncLocalStorage = new AsyncLocalStorage(); - static __last_context_key = 0; - static make_context_key (opt_human_readable) { - let k = `_:${++this.__last_context_key}`; - if ( opt_human_readable ) { - k += `:${opt_human_readable}`; - } - return k; - } - static create (values, opt_name) { - return new Context(values, undefined, opt_name); - } - static get (key, { allow_fallback } = {}) { - const existingContext = this.contextAsyncLocalStorage.getStore()?.get('context'); - if ( ! existingContext ) { - if ( context_config.strict && !allow_fallback ) { - throw new Error('FAILED TO GET THE CORRECT CONTEXT'); - } - const rootFallback = this.root.sub({}, this.USE_NAME_FALLBACK); - if ( key ) { - return rootFallback.get(key); - } - return rootFallback; - } - if ( key ) { - return existingContext.get(key); - } - return existingContext; - } - static set (k, v) { - const x = this.contextAsyncLocalStorage.getStore()?.get('context'); - if ( x ) return x.set(k, v); - } - static root = new Context({}, undefined, 'root'); - static describe () { - return this.get().describe(); - } - static arun (...a) { - return this.get().arun(...a); - } - static sub (values, opt_name) { - return this.get().sub(values, opt_name); - } - - #dead = false; - - /** - * Clears this context's values and unlinks from its parent. This context - * will become empty. This is to ensure contexts that aren't used anymore - * get garbage collected. This was added to prevent memory leaks due to - * ECMAP, where currently we're not sure what's holding a reference back - * to the ECMAP (or perhaps its subcontext). - */ - unlink () { - // Settings `values_` to an empty object should clear any references - // that were inside it while avoiding errors if .get() happens to be - // called by a lingering asynchronous function. - this.values_ = {}; - this.#dead = true; - } - - get (k) { - return this.values_[k]; - } - set (k, v) { - if ( this.#dead ) return; - this.values_[k] = v; - } - sub (values, opt_name) { - if ( typeof values === 'string' ) { - opt_name = values; - values = {}; - } - const name = opt_name ?? this.name ?? this.get('name'); - for ( const hook of this.constructor.context_hooks_.pre_create ) { - hook({ values, name }); - } - return new Context(values, this, opt_name); - } - get values () { - return this.values_; - } - - /** - * @untested - */ - get_proxy_object () { - return new Proxy(this.values_, { - get: (target, prop) => { - return this.get(prop); - }, - set: (target, prop, value) => { - this.set(prop, value); - return true; - }, - }); - } - - constructor (imm_values, opt_parent, opt_name) { - const values = { ...imm_values }; - imm_values = null; - - opt_parent = opt_parent || Context.root; - - this.trace_name = opt_name ?? undefined; - this.name = (() => { - if ( opt_name === this.constructor.USE_NAME_FALLBACK ) { - opt_name = 'F'; - } - if ( opt_name ) { - const name_numbers = this.constructor.other_next_names_; - if ( ! Object.prototype.hasOwnProperty.call(name_numbers, opt_name) ) { - name_numbers[opt_name] = 0; - } - const num = ++name_numbers[opt_name]; - return `{${opt_name}:${num}}`; - } - return `${++this.constructor.next_name_}`; - })(); - this.parent_ = opt_parent; - - if ( opt_parent ) { - Object.setPrototypeOf(values, opt_parent.values_); - for ( const k in values ) { - const parent_val = opt_parent.values_[k]; - if ( parent_val instanceof Context ) { - if ( ! (values[k] instanceof Context) ) { - values[k] = parent_val.sub(values[k]); - } - } - } - } - - this.values_ = values; - } - async arun (...args) { - let cb = args.shift(); - - let hints = {}; - if ( typeof cb === 'object' ) { - hints = cb; - cb = args.shift(); - } - - if ( typeof cb === 'string' ) { - const sub_context = this.sub(cb); - return await sub_context.arun({ trace: true }, ...args); - } - - const replace_callback = new_cb => { - cb = new_cb; - }; - - for ( const hook of this.constructor.context_hooks_.pre_arun ) { - hook({ - hints, - name: this.name ?? this.get('name'), - trace_name: this.trace_name, - replace_callback, - callback: cb, - }); - } - - const als = this.constructor.contextAsyncLocalStorage; - return await als.run(new Map(), async () => { - als.getStore().set('context', this); - return await cb(); - }); - } - abind (cb) { - return async (...args) => { - return await this.arun(async () => { - return await cb(...args); - }); - }; - } - - describe () { - return `Context(${this.describe_()})`; - } - describe_ () { - if ( ! this.parent_ ) return '[R]'; - return `${this.parent_.describe_()}->${this.name}`; - } - - static async allow_fallback (cb) { - const x = this.get(undefined, { allow_fallback: true }); - return await x.arun(async () => { - return await cb(); - }); - } - }; - - ContextExpressMiddleware = class ContextExpressMiddleware { - constructor ({ parent }) { - this.parent_ = parent; - } - install (app) { - app.use(this.run.bind(this)); - } - async run (req, res, next) { - return await this.parent_.sub({ - req, - res, - trace_request: uuidv4(), - }, 'req').arun(async () => { - const ctx = Context.get(); - req.ctx = ctx; - res.locals.ctx = ctx; - next(); - }); - } - }; - - // Store singleton - globalThis[CONTEXT_SINGLETON_KEY] = { Context, ContextExpressMiddleware, context_config }; -} - -export { Context, context_config, ContextExpressMiddleware }; diff --git a/src/backend/src/util/datautil.bench.js b/src/backend/src/util/datautil.bench.js deleted file mode 100644 index f9391a581..000000000 --- a/src/backend/src/util/datautil.bench.js +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { bench, describe } from 'vitest'; -import { hash_serializable_object, stringify_serializable_object } from './datautil.js'; - -// Test data generators -const createFlatObject = (size) => { - const obj = {}; - for ( let i = 0; i < size; i++ ) { - obj[`key${i}`] = `value${i}`; - } - return obj; -}; - -const createNestedObject = (depth, breadth) => { - if ( depth === 0 ) { - return { leaf: 'value' }; - } - const obj = {}; - for ( let i = 0; i < breadth; i++ ) { - obj[`level${depth}_child${i}`] = createNestedObject(depth - 1, breadth); - } - return obj; -}; - -const createMixedObject = () => ({ - string: 'hello world', - number: 42, - boolean: true, - null: null, - array: [1, 2, 3, { nested: 'array' }], - nested: { - deep: { - value: 'found', - numbers: [1, 2, 3], - }, - }, -}); - -// Objects with different key orderings (should produce same hash) -const objA = { z: 1, a: 2, m: 3 }; -const objB = { a: 2, m: 3, z: 1 }; -const objC = { m: 3, z: 1, a: 2 }; - -describe('stringify_serializable_object - Flat objects', () => { - const small = createFlatObject(5); - const medium = createFlatObject(20); - const large = createFlatObject(100); - - bench('small flat object (5 keys)', () => { - stringify_serializable_object(small); - }); - - bench('medium flat object (20 keys)', () => { - stringify_serializable_object(medium); - }); - - bench('large flat object (100 keys)', () => { - stringify_serializable_object(large); - }); -}); - -describe('stringify_serializable_object - Nested objects', () => { - const shallow = createNestedObject(2, 3); // depth 2, 3 children each - const medium = createNestedObject(3, 3); // depth 3, 3 children each - const deep = createNestedObject(4, 2); // depth 4, 2 children each - - bench('shallow nested (depth=2, breadth=3)', () => { - stringify_serializable_object(shallow); - }); - - bench('medium nested (depth=3, breadth=3)', () => { - stringify_serializable_object(medium); - }); - - bench('deep nested (depth=4, breadth=2)', () => { - stringify_serializable_object(deep); - }); -}); - -describe('stringify_serializable_object - Mixed types', () => { - const mixed = createMixedObject(); - - bench('mixed type object', () => { - stringify_serializable_object(mixed); - }); - - bench('primitives', () => { - stringify_serializable_object('string'); - stringify_serializable_object(42); - stringify_serializable_object(true); - stringify_serializable_object(null); - stringify_serializable_object(undefined); - }); -}); - -describe('stringify_serializable_object - Key ordering normalization', () => { - bench('objects with different key orderings', () => { - // All should produce the same output - stringify_serializable_object(objA); - stringify_serializable_object(objB); - stringify_serializable_object(objC); - }); -}); - -describe('stringify_serializable_object vs JSON.stringify', () => { - const obj = createFlatObject(20); - - bench('stringify_serializable_object', () => { - stringify_serializable_object(obj); - }); - - bench('JSON.stringify (baseline, no key sorting)', () => { - JSON.stringify(obj); - }); - - bench('JSON.stringify with sorted keys (manual)', () => { - const sortedObj = {}; - Object.keys(obj).sort().forEach(k => { - sortedObj[k] = obj[k]; - }); - JSON.stringify(sortedObj); - }); -}); - -describe('hash_serializable_object', () => { - const small = createFlatObject(5); - const medium = createFlatObject(20); - const mixed = createMixedObject(); - - bench('hash small object', () => { - hash_serializable_object(small); - }); - - bench('hash medium object', () => { - hash_serializable_object(medium); - }); - - bench('hash mixed object', () => { - hash_serializable_object(mixed); - }); - - bench('hash objects with different key orderings (should be equal)', () => { - hash_serializable_object(objA); - hash_serializable_object(objB); - hash_serializable_object(objC); - }); -}); diff --git a/src/backend/src/util/datautil.js b/src/backend/src/util/datautil.js deleted file mode 100644 index 7c12b32dd..000000000 --- a/src/backend/src/util/datautil.js +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -/** - * Stringify an object in such a way that objects with differing - * key orderings will still be considered equal. - * @param {*} obj - */ -const stringify_serializable_object = obj => { - if ( obj === undefined ) return '[undefined]'; - if ( obj === null ) return '[null]'; - if ( typeof obj === 'function' ) return '[function]'; - if ( typeof obj !== 'object' ) return JSON.stringify(obj); - - // ensure an error is thrown if the object is not serializable. - // (instead of failing with a stack overflow) - JSON.stringify(obj); - - const keys = Object.keys(obj).sort(); - const pairs = keys.map(key => { - const value = stringify_serializable_object(obj[key]); - const outer_json = JSON.stringify({ [key]: value }); - return outer_json.slice(1, -1); - }); - - return `{${ pairs.join(',') }}`; -}; - -const hash_serializable_object = obj => { - const crypto = require('crypto'); - const str = stringify_serializable_object(obj); - return crypto.createHash('sha1').update(str).digest('hex'); -}; - -module.exports = { - stringify_serializable_object, - hash_serializable_object, -}; diff --git a/src/backend/src/util/debugutil.js b/src/backend/src/util/debugutil.js deleted file mode 100644 index de3aab021..000000000 --- a/src/backend/src/util/debugutil.js +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const LETTERS = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N']; - -let curr_letter_ = 0; - -const ind = () => { - let v = curr_letter_; - curr_letter_++; - curr_letter_ = curr_letter_ % LETTERS.length; - return v; -}; - -module.exports = { - get_a_letter: () => LETTERS[ind()], - cylog: (...a) => { - console.log('\x1B[36;1m', ...a); - }, -}; diff --git a/src/backend/src/util/errorutil.js b/src/backend/src/util/errorutil.js deleted file mode 100644 index dd17e8cf6..000000000 --- a/src/backend/src/util/errorutil.js +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const log_http_error = e => { - console.log(`\x1B[31;1m${ e.message }\x1B[0m`); - - console.log('HTTP Method: ', e.config.method.toUpperCase()); - console.log('URL: ', e.config.url); - - if ( e.config.params ) { - console.log('URL Parameters: ', e.config.params); - } - - if ( e.config.method.toLowerCase() === 'post' && e.config.data ) { - console.log('Post body: ', e.config.data); - } - - console.log('Request Headers: ', JSON.stringify(e.config.headers, null, 2)); - - if ( e.response ) { - console.log('Response Status: ', e.response.status); - console.log('Response Headers: ', JSON.stringify(e.response.headers, null, 2)); - console.log('Response body: ', e.response.data); - } - - console.log(`\x1B[31;1m${ e.message }\x1B[0m`); -}; - -const better_error_printer = e => { - if ( e.request ) { - log_http_error(e); - return; - } - - console.error(e); -}; - -/** - * This class is used to wrap an error when the error has - * already been sent to ErrorService. This prevents higher-level - * error handlers from sending it to ErrorService again. - */ -class ManagedError extends Error { - constructor (source, extra = {}) { - super(source?.message ?? source); - this.source = source; - this.name = `Managed(${source?.name ?? 'Error'})`; - this.extra = extra; - } -} - -module.exports = { - ManagedError, - better_error_printer, - - // We export CompositeError from 'composite-error' here - // in case we want to change the implementation later. - // i.e. it's under the MIT license so it would be easier - // to just copy the class to this file than maintain a fork. - CompositeError: require('composite-error'), -}; diff --git a/src/backend/src/util/esmcontext.js b/src/backend/src/util/esmcontext.js deleted file mode 100644 index 3b7bb8ebb..000000000 --- a/src/backend/src/util/esmcontext.js +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2026-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -// Bridge file to ensure ES modules and CommonJS modules use the same Context instance -// This file uses require() to load context.js, ensuring compatibility with -// CommonJS modules that also require() context.js -const { Context, ContextExpressMiddleware } = require('./context.js'); - -module.exports = { - Context, - ContextExpressMiddleware, -}; diff --git a/src/backend/src/util/expressutil.js b/src/backend/src/util/expressutil.js deleted file mode 100644 index e6ca4a590..000000000 --- a/src/backend/src/util/expressutil.js +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const eggspress = require('../api/eggspress'); - -/** - * @deprecated Use eggspress directly - * @param {any} spec - * @param {any} handler - * @returns {any} - */ -const Endpoint = function Endpoint (spec, handler) { - return { - attach (route) { - const eggspress_options = { - allowedMethods: spec.methods ?? ['GET'], - ...(spec.subdomain ? { subdomain: spec.subdomain } : {}), - ...(spec.parameters ? { parameters: spec.parameters } : {}), - ...(spec.alias ? { alias: spec.alias } : {}), - ...(spec.mw ? { mw: spec.mw } : {}), - ...spec.otherOpts, - }; - const eggspress_router = eggspress( - spec.route, - eggspress_options, - handler ?? spec.handler, - ); - route.use(eggspress_router); - }, - but (newSpec) { - // TODO: add merge with '$' behaviors (like config has) - return Endpoint({ - ...spec, - ...newSpec, - }); - }, - }; -}; - -module.exports = { - Endpoint, -}; diff --git a/src/backend/src/util/fnutil.js b/src/backend/src/util/fnutil.js deleted file mode 100644 index 1fd2f2052..000000000 --- a/src/backend/src/util/fnutil.js +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const UtilFn = fn => { - /** - * A null-coalescing call - */ - fn.if = function utilfn_if (v) { - if ( v === null || v === undefined ) return v; - return this(v); - }; - return fn; -}; - -const OnlyOnceFn = fn => { - let called = false; - return function onlyoncefn_call (...args) { - if ( called ) return; - called = true; - return fn(...args); - }; -}; - -module.exports = { - UtilFn, - OnlyOnceFn, -}; diff --git a/src/backend/src/util/fuzz.js b/src/backend/src/util/fuzz.js deleted file mode 100644 index f50e65c47..000000000 --- a/src/backend/src/util/fuzz.js +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -/** -* Rounds numbers to human-friendly thresholds commonly used for displaying metrics. -* -* This function implements a stepwise rounding system: -* - For small numbers (1-99): Uses specific thresholds (1+, 10+, 50+) to avoid showing exact small counts -* - For hundreds (100-999): Rounds to 100+ or 500+ -* - For thousands (1K-999K): Uses K+ notation with 1K, 5K, 10K, 50K, 100K, 500K thresholds -* - For millions (1M-999M): Uses M+ notation with 1M, 5M, 10M, 50M, 100M, 500M thresholds -* - For billions: Shows as 1B+ -* -* The rounding is always down to the nearest threshold to ensure the "+" symbol -* accurately indicates there are at least that many items. -* -* @param {number} num - The number to be rounded -* @returns {number} The rounded number according to the threshold rules -* (without the "+" symbol, which should be added by display logic) -* -* @example -* fuzz_number(7) // returns 1 (displays as "1+") -* fuzz_number(45) // returns 10 (displays as "10+") -* fuzz_number(2500) // returns 1000 (displays as "1K+") -* fuzz_number(7500000) // returns 5000000 (displays as "5M+") -*/ - -function fuzz_number (num) { - // If the number is 0, return 0 - if ( num === 0 ) return 0; - - // For 1-9 - if ( num < 10 ) return 1; - - // For 10-49 - if ( num < 50 ) return 10; - - // For 50-99 - if ( num < 100 ) return 50; - - // For 100-499 - if ( num < 500 ) return 100; - - // For 500-999 - if ( num < 1000 ) return 500; - - // For 1K-4.99K - if ( num < 5000 ) return 1000; - - // For 5K-9.99K - if ( num < 10000 ) return 5000; - - // For 10K-49.99K - if ( num < 50000 ) return 10000; - - // For 50K-99.99K - if ( num < 100000 ) return 50000; - - // For 100K-499.99K - if ( num < 500000 ) return 100000; - - // For 500K-999.99K - if ( num < 1000000 ) return 500000; - - // For 1M-4.99M - if ( num < 5000000 ) return 1000000; - - // For 5M-9.99M - if ( num < 10000000 ) return 5000000; - - // For 10M-49.99M - if ( num < 50000000 ) return 10000000; - - // For 50M-99.99M - if ( num < 100000000 ) return 50000000; - - // For 100M-499.99M - if ( num < 500000000 ) return 100000000; - - // For 500M-999.99M - if ( num < 1000000000 ) return 500000000; - - // For 1B+ - return 1000000000; -} - -module.exports = { - fuzz_number, -}; \ No newline at end of file diff --git a/src/backend/src/util/gcutil.js b/src/backend/src/util/gcutil.js deleted file mode 100644 index 507385f01..000000000 --- a/src/backend/src/util/gcutil.js +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -/** - * gc_friendly_rslv is based on a hunch about how the garbage collector works. - */ -const NOOP = () => { -}; -const gc_friendly_rslv = (rslv) => { - return (value) => { - rslv(value); - rslv = NOOP; - }; -}; - -module.exports = { - NOOP, - gc_friendly_rslv, -}; diff --git a/src/backend/src/util/hl_types.js b/src/backend/src/util/hl_types.js deleted file mode 100644 index a74c06e37..000000000 --- a/src/backend/src/util/hl_types.js +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { quot } = require('@heyputer/putility').libs.string; - -const hl_type_definitions = { - flag: { - fallback: false, - required_check: v => { - if ( v === undefined || v === '' ) { - return false; - } - return true; - }, - adapt: (v) => { - if ( typeof v === 'string' ) { - if ( - v === 'true' || v === '1' || v === 'yes' - ) return true; - - if ( - v === 'false' || v === '0' || v === 'no' - ) return false; - - throw new Error(`could not adapt string to boolean: ${quot(v)}`); - } - - if ( typeof v === 'boolean' ) { - return v; - } - - if ( v === 1 ) return true; - if ( v === 0 ) return false; - if ( typeof v === 'object' ) { - return v !== null; - } - - throw new Error(`could not adapt value to boolean: ${quot(v)}`); - }, - }, -}; - -class HLTypeFacade { - static REQUIRED = {}; - static convert (type, value, opt_default) { - const type_definition = hl_type_definitions[type]; - const has_value = type_definition.required_check(value); - if ( ! has_value ) { - if ( opt_default === HLTypeFacade.REQUIRED ) { - throw new Error('required value is missing'); - } - return opt_default ?? type_definition.fallback; - } - return type_definition.adapt(value); - } -} - -module.exports = { - hl_type_definitions, - HLTypeFacade, - boolify: HLTypeFacade.convert.bind(HLTypeFacade, 'flag'), -}; diff --git a/src/backend/src/util/hl_types.test.js b/src/backend/src/util/hl_types.test.js deleted file mode 100644 index fe3b27874..000000000 --- a/src/backend/src/util/hl_types.test.js +++ /dev/null @@ -1,17 +0,0 @@ -import { describe, it, expect } from 'vitest'; -const { boolify } = require('./hl_types'); - -describe('hl_types', () => { - it('boolify falsy values', () => { - expect(boolify(undefined)).toBe(false); - expect(boolify(0)).toBe(false); - expect(boolify('')).toBe(false); - expect(boolify(null)).toBe(false); - }); - it('boolify truthy values', () => { - expect(boolify(true)).toBe(true); - expect(boolify(1)).toBe(true); - expect(boolify('1')).toBe(true); - expect(boolify({})).toBe(true); - }); -}); diff --git a/src/backend/src/util/identifier.bench.js b/src/backend/src/util/identifier.bench.js deleted file mode 100644 index a3dad3b73..000000000 --- a/src/backend/src/util/identifier.bench.js +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { bench, describe } from 'vitest'; -import { generate_identifier, generate_random_code } from './identifier.js'; - -describe('generate_identifier - Basic generation', () => { - bench('generate single identifier (default separator)', () => { - generate_identifier(); - }); - - bench('generate identifier with hyphen separator', () => { - generate_identifier('-'); - }); - - bench('generate identifier with empty separator', () => { - generate_identifier(''); - }); - - bench('generate 100 identifiers', () => { - for ( let i = 0; i < 100; i++ ) { - generate_identifier(); - } - }); - - bench('generate 1000 identifiers', () => { - for ( let i = 0; i < 1000; i++ ) { - generate_identifier(); - } - }); -}); - -describe('generate_identifier - With custom RNG', () => { - // Seeded pseudo-random for reproducibility - const seededRng = () => { - let seed = 12345; - return () => { - seed = (seed * 1103515245 + 12345) & 0x7fffffff; - return seed / 0x7fffffff; - }; - }; - - bench('generate with Math.random (default)', () => { - generate_identifier('_', Math.random); - }); - - bench('generate with seeded RNG', () => { - const rng = seededRng(); - generate_identifier('_', rng); - }); -}); - -describe('generate_random_code - Various lengths', () => { - bench('generate 4-char code', () => { - generate_random_code(4); - }); - - bench('generate 8-char code', () => { - generate_random_code(8); - }); - - bench('generate 16-char code', () => { - generate_random_code(16); - }); - - bench('generate 32-char code', () => { - generate_random_code(32); - }); - - bench('generate 64-char code', () => { - generate_random_code(64); - }); -}); - -describe('generate_random_code - Custom character sets', () => { - const numericOnly = '0123456789'; - const hexChars = '0123456789ABCDEF'; - const alphaLower = 'abcdefghijklmnopqrstuvwxyz'; - const fullAlphanumeric = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; - - bench('numeric only (10 chars)', () => { - generate_random_code(10, { chars: numericOnly }); - }); - - bench('hex chars (16 chars)', () => { - generate_random_code(16, { chars: hexChars }); - }); - - bench('lowercase alpha (10 chars)', () => { - generate_random_code(10, { chars: alphaLower }); - }); - - bench('full alphanumeric (16 chars)', () => { - generate_random_code(16, { chars: fullAlphanumeric }); - }); -}); - -describe('generate_random_code - Batch generation', () => { - bench('generate 100 codes (8 chars each)', () => { - for ( let i = 0; i < 100; i++ ) { - generate_random_code(8); - } - }); - - bench('generate 1000 codes (8 chars each)', () => { - for ( let i = 0; i < 1000; i++ ) { - generate_random_code(8); - } - }); -}); - -describe('Comparison with alternatives', () => { - bench('generate_identifier', () => { - generate_identifier(); - }); - - bench('generate_random_code (8 chars)', () => { - generate_random_code(8); - }); - - bench('Math.random().toString(36).slice(2, 10)', () => { - Math.random().toString(36).slice(2, 10); - }); - - bench('Date.now().toString(36)', () => { - Date.now().toString(36); - }); -}); - -describe('Real-world usage patterns', () => { - bench('generate username suggestion', () => { - // Pattern: adjective_noun_number - generate_identifier('_'); - }); - - bench('generate session token (32 chars)', () => { - generate_random_code(32); - }); - - bench('generate verification code (6 chars, numeric)', () => { - generate_random_code(6, { chars: '0123456789' }); - }); - - bench('generate file suffix (8 chars)', () => { - generate_random_code(8); - }); -}); diff --git a/src/backend/src/util/identifier.js b/src/backend/src/util/identifier.js deleted file mode 100644 index f638966b7..000000000 --- a/src/backend/src/util/identifier.js +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const adjectives = [ - 'amazing', 'ambitious', 'articulate', 'cool', 'bubbly', 'mindful', 'noble', 'savvy', 'serene', - 'sincere', 'sleek', 'sparkling', 'spectacular', 'splendid', 'spotless', 'stunning', - 'awesome', 'beaming', 'bold', 'brilliant', 'cheerful', 'modest', 'motivated', - 'friendly', 'fun', 'funny', 'generous', 'gifted', 'graceful', 'grateful', - 'passionate', 'patient', 'peaceful', 'perceptive', 'persistent', - 'helpful', 'sensible', 'loyal', 'honest', 'clever', 'capable', - 'calm', 'smart', 'genius', 'bright', 'charming', 'creative', 'diligent', 'elegant', 'fancy', - 'colorful', 'avid', 'active', 'gentle', 'happy', 'intelligent', - 'jolly', 'kind', 'lively', 'merry', 'nice', 'optimistic', 'polite', - 'quiet', 'relaxed', 'silly', 'witty', 'young', - 'strong', 'brave', 'agile', 'bold', 'confident', 'daring', - 'fearless', 'heroic', 'mighty', 'powerful', 'valiant', 'wise', 'wonderful', 'zealous', - 'warm', 'swift', 'neat', 'tidy', 'nifty', 'lucky', 'keen', - 'blue', 'red', 'aqua', 'green', 'orange', 'pink', 'purple', 'cyan', 'magenta', 'lime', - 'teal', 'lavender', 'beige', 'maroon', 'navy', 'olive', 'silver', 'gold', 'ivory', -]; - -const nouns = [ - 'street', 'roof', 'floor', 'tv', 'idea', 'morning', 'game', 'wheel', 'bag', 'clock', 'pencil', 'pen', - 'magnet', 'chair', 'table', 'house', 'room', 'book', 'car', 'tree', 'candle', 'light', 'planet', - 'flower', 'bird', 'fish', 'sun', 'moon', 'star', 'cloud', 'rain', 'snow', 'wind', 'mountain', - 'river', 'lake', 'sea', 'ocean', 'island', 'bridge', 'road', 'train', 'plane', 'ship', 'bicycle', - 'circle', 'square', 'garden', 'harp', 'grass', 'forest', 'rock', 'cake', 'pie', 'cookie', 'candy', - 'butterfly', 'computer', 'phone', 'keyboard', 'mouse', 'cup', 'plate', 'glass', 'door', - 'window', 'key', 'wallet', 'pillow', 'bed', 'blanket', 'soap', 'towel', 'lamp', 'mirror', - 'camera', 'hat', 'shirt', 'pants', 'shoes', 'watch', 'ring', - 'necklace', 'ball', 'toy', 'doll', 'kite', 'balloon', 'guitar', 'violin', 'piano', 'drum', - 'trumpet', 'flute', 'viola', 'cello', 'harp', 'banjo', 'tuba', -]; - -const words = { - adjectives, - nouns, -}; - -const randomItem = (arr, random) => arr[Math.floor((random ?? Math.random)() * arr.length)]; - -/** - * A function that generates a unique identifier by combining a random adjective, a random noun, and a random number (between 0 and 9999). - * The result is returned as a string with components separated by the specified separator. - * It is useful when you need to create unique identifiers that are also human-friendly. - * - * @param {string} [separator='_'] - The character used to separate the adjective, noun, and number. Defaults to '_' if not provided. - * @returns {string} A unique, human-friendly identifier. - * - * @example - * - * let identifier = window.generate_identifier(); - * // identifier would be something like 'clever-idea-123' - * - */ -function generate_identifier (separator = '_', rng = Math.random) { - // return a random combination of first_adj + noun + number (between 0 and 9999) - // e.g. clever-idea-123 - return [ - randomItem(adjectives, rng), - randomItem(nouns, rng), - Math.floor(rng() * 10000), - ].join(separator); -} - -const HUMAN_READABLE_CASE_INSENSITIVE = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; - -function generate_random_code (n, { - rng = Math.random, - chars = HUMAN_READABLE_CASE_INSENSITIVE, -} = {}) { - let code = ''; - for ( let i = 0 ; i < n ; i++ ) { - code += randomItem(chars, rng); - } - return code; -} - -/** - * - * @param {*} n length of output code - * @param {*} mask - a string of characters to start with - * @param {*} value - a number to be converted to base-36 and put on the right - */ -function compose_code (mask, value) { - const right_str = value.toString(36); - let out_str = mask; - console.log('right_str', right_str); - console.log('out_str', out_str); - for ( let i = 0 ; i < right_str.length ; i++ ) { - out_str[out_str.length - 1 - i] = right_str[right_str.length - 1 - i]; - } - - out_str = out_str.toUpperCase(); - return out_str; -} - -module.exports = { - generate_identifier, - generate_random_code, -}; diff --git a/src/backend/src/util/lockutil.bench.js b/src/backend/src/util/lockutil.bench.js deleted file mode 100644 index c92344562..000000000 --- a/src/backend/src/util/lockutil.bench.js +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { bench, describe } from 'vitest'; -import { RWLock } from './lockutil.js'; - -describe('RWLock - Creation', () => { - bench('create RWLock', () => { - new RWLock(); - }); - - bench('create 100 RWLocks', () => { - for ( let i = 0; i < 100; i++ ) { - new RWLock(); - } - }); -}); - -describe('RWLock - Mode checking', () => { - const lock = new RWLock(); - - bench('check effective_mode (idle)', () => { - void lock.effective_mode; - }); -}); - -describe('RWLock - Read locks (no contention)', () => { - bench('single rlock/unlock cycle', async () => { - const lock = new RWLock(); - const handle = await lock.rlock(); - handle.unlock(); - }); - - bench('10 sequential rlock/unlock cycles', async () => { - const lock = new RWLock(); - for ( let i = 0; i < 10; i++ ) { - const handle = await lock.rlock(); - handle.unlock(); - } - }); - - bench('concurrent read locks (5 readers)', async () => { - const lock = new RWLock(); - const handles = await Promise.all([ - lock.rlock(), - lock.rlock(), - lock.rlock(), - lock.rlock(), - lock.rlock(), - ]); - for ( const handle of handles ) { - handle.unlock(); - } - }); - - bench('concurrent read locks (10 readers)', async () => { - const lock = new RWLock(); - const promises = []; - for ( let i = 0; i < 10; i++ ) { - promises.push(lock.rlock()); - } - const handles = await Promise.all(promises); - for ( const handle of handles ) { - handle.unlock(); - } - }); -}); - -describe('RWLock - Write locks (no contention)', () => { - bench('single wlock/unlock cycle', async () => { - const lock = new RWLock(); - const handle = await lock.wlock(); - handle.unlock(); - }); - - bench('10 sequential wlock/unlock cycles', async () => { - const lock = new RWLock(); - for ( let i = 0; i < 10; i++ ) { - const handle = await lock.wlock(); - handle.unlock(); - } - }); -}); - -describe('RWLock - Mixed read/write patterns', () => { - bench('read then write then read', async () => { - const lock = new RWLock(); - - const r1 = await lock.rlock(); - r1.unlock(); - - const w = await lock.wlock(); - w.unlock(); - - const r2 = await lock.rlock(); - r2.unlock(); - }); - - bench('write then multiple reads', async () => { - const lock = new RWLock(); - - const w = await lock.wlock(); - w.unlock(); - - const handles = await Promise.all([ - lock.rlock(), - lock.rlock(), - lock.rlock(), - ]); - for ( const h of handles ) { - h.unlock(); - } - }); - - bench('alternating read/write (10 cycles)', async () => { - const lock = new RWLock(); - for ( let i = 0; i < 10; i++ ) { - if ( i % 2 === 0 ) { - const h = await lock.rlock(); - h.unlock(); - } else { - const h = await lock.wlock(); - h.unlock(); - } - } - }); -}); - -describe('RWLock - Contention patterns', () => { - bench('readers waiting for writer', async () => { - const lock = new RWLock(); - - // Writer goes first - const writePromise = (async () => { - const h = await lock.wlock(); - // Simulate work - h.unlock(); - })(); - - // Readers queue up - const readerPromises = []; - for ( let i = 0; i < 5; i++ ) { - readerPromises.push((async () => { - const h = await lock.rlock(); - h.unlock(); - })()); - } - - await Promise.all([writePromise, ...readerPromises]); - }); - - bench('writer waiting for readers', async () => { - const lock = new RWLock(); - - // Readers go first - const readerPromises = []; - for ( let i = 0; i < 5; i++ ) { - readerPromises.push((async () => { - const h = await lock.rlock(); - h.unlock(); - })()); - } - - // Writer queues up - const writePromise = (async () => { - const h = await lock.wlock(); - h.unlock(); - })(); - - await Promise.all([...readerPromises, writePromise]); - }); -}); - -describe('RWLock - Queue behavior', () => { - bench('check_queue_ with empty queue', () => { - const lock = new RWLock(); - lock.check_queue_(); - }); -}); - -describe('RWLock - on_empty_ callback', () => { - bench('set on_empty_ callback', () => { - const lock = new RWLock(); - lock.on_empty_ = () => { - }; - }); - - bench('trigger on_empty_ via lock cycle', async () => { - const lock = new RWLock(); - lock.on_empty_ = () => { - }; - - const h = await lock.rlock(); - h.unlock(); - // on_empty_ should be called - }); -}); - -describe('Real-world patterns', () => { - bench('cache read pattern (10 concurrent readers)', async () => { - const lock = new RWLock(); - const promises = []; - - for ( let i = 0; i < 10; i++ ) { - promises.push((async () => { - const h = await lock.rlock(); - // Simulate cache read - h.unlock(); - })()); - } - - await Promise.all(promises); - }); - - bench('cache invalidation pattern', async () => { - const lock = new RWLock(); - - // Some readers first - const readerPromises = []; - for ( let i = 0; i < 3; i++ ) { - readerPromises.push((async () => { - const h = await lock.rlock(); - h.unlock(); - })()); - } - - // Invalidation (write) - const invalidatePromise = (async () => { - const h = await lock.wlock(); - // Simulate cache clear - h.unlock(); - })(); - - // New readers after invalidation - for ( let i = 0; i < 3; i++ ) { - readerPromises.push((async () => { - const h = await lock.rlock(); - h.unlock(); - })()); - } - - await Promise.all([...readerPromises, invalidatePromise]); - }); - - bench('file access pattern (mostly reads, occasional write)', async () => { - const lock = new RWLock(); - const operations = []; - - for ( let i = 0; i < 20; i++ ) { - if ( i % 5 === 0 ) { - // Write every 5th operation - operations.push((async () => { - const h = await lock.wlock(); - h.unlock(); - })()); - } else { - // Read otherwise - operations.push((async () => { - const h = await lock.rlock(); - h.unlock(); - })()); - } - } - - await Promise.all(operations); - }); -}); diff --git a/src/backend/src/util/lockutil.js b/src/backend/src/util/lockutil.js deleted file mode 100644 index 49a62934c..000000000 --- a/src/backend/src/util/lockutil.js +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { TeePromise } = require('@heyputer/putility').libs.promise; - -/** - * RWLock is a read-write lock that allows multiple readers or a single writer. - */ -class RWLock { - static TYPE_READ = Symbol('read'); - static TYPE_WRITE = Symbol('write'); - - constructor () { - this.queue = []; - - this.readers_ = 0; - this.writer_ = false; - - this.on_empty_ = () => { - }; - - this.mode = this.constructor.TYPE_READ; - } - get effective_mode () { - if ( this.readers_ > 0 ) return this.constructor.TYPE_READ; - if ( this.writer_ ) return this.constructor.TYPE_WRITE; - return undefined; - } - push_ (item) { - if ( this.readers_ === 0 && !this.writer_ ) { - this.mode = item.type; - } - this.queue.push(item); - this.check_queue_(); - } - check_queue_ () { - if ( this.queue.length === 0 ) { - if ( this.readers_ === 0 && !this.writer_ ) { - this.on_empty_(); - } - return; - } - - const peek = () => this.queue[0]; - - if ( this.readers_ === 0 && !this.writer_ ) { - this.mode = peek().type; - } - - if ( this.mode === this.constructor.TYPE_READ ) { - while ( peek()?.type === this.constructor.TYPE_READ ) { - const item = this.queue.shift(); - this.readers_++; - (async () => { - await item.p_unlock; - this.readers_--; - this.check_queue_(); - })(); - item.p_operation.resolve(); - } - return; - } - - if ( this.writer_ ) return; - - const item = this.queue.shift(); - this.writer_ = true; - (async () => { - await item.p_unlock; - this.writer_ = false; - this.check_queue_(); - })(); - item.p_operation.resolve(); - } - async rlock () { - const p_read = new TeePromise(); - const p_unlock = new TeePromise(); - const handle = { - unlock: () => { - p_unlock.resolve(); - }, - }; - - this.push_({ - type: this.constructor.TYPE_READ, - p_operation: p_read, - p_unlock, - }); - await p_read; - - return handle; - } - - async wlock () { - const p_write = new TeePromise(); - const p_unlock = new TeePromise(); - const handle = { - unlock: () => { - p_unlock.resolve(); - }, - }; - - this.push_({ - type: this.constructor.TYPE_WRITE, - p_operation: p_write, - p_unlock, - }); - await p_write; - - return handle; - } - -} - -module.exports = { - RWLock, -}; diff --git a/src/backend/src/util/modutil.js b/src/backend/src/util/modutil.js deleted file mode 100644 index 8475cf2a2..000000000 --- a/src/backend/src/util/modutil.js +++ /dev/null @@ -1,61 +0,0 @@ -const fs = require('fs').promises; -const path = require('path'); - -async function prependToJSFiles (directory, snippet) { - const jsExtensions = new Set(['.js', '.cjs', '.mjs', '.ts']); - - async function processDirectory (dir) { - try { - const entries = await fs.readdir(dir, { withFileTypes: true }); - const promises = []; - - for ( const entry of entries ) { - const fullPath = path.join(dir, entry.name); - - if ( entry.isDirectory() ) { - // Skip common directories that shouldn't be modified - if ( ! shouldSkipDirectory(entry.name) ) { - promises.push(processDirectory(fullPath)); - } - } else if ( entry.isFile() && jsExtensions.has(path.extname(entry.name)) ) { - promises.push(prependToFile(fullPath, snippet)); - } - } - - await Promise.all(promises); - } catch ( error ) { - throw new Error(`error processing directory ${dir}`, { - cause: error, - }); - } - } - - function shouldSkipDirectory (dirName) { - const skipDirs = new Set([ - 'node_modules', - 'gui', - ]); - if ( skipDirs.has(dirName) ) return true; - if ( dirName.startsWith('.') ) return true; - return false; - } - - async function prependToFile (filePath, snippet) { - try { - const content = await fs.readFile(filePath, 'utf8'); - if ( content.startsWith('//!no-prepend') ) return; - const newContent = snippet + content; - await fs.writeFile(filePath, newContent, 'utf8'); - } catch ( error ) { - throw new Error(`error processing file ${filePath}`, { - cause: error, - }); - } - } - - await processDirectory(directory); -} - -module.exports = { - prependToJSFiles, -}; diff --git a/src/backend/src/util/multivalue.js b/src/backend/src/util/multivalue.js deleted file mode 100644 index afcc1e78c..000000000 --- a/src/backend/src/util/multivalue.js +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('../../../putility'); - -/** - * MutliValue represents a subject with multiple values or a value with multiple - * formats/types. It can be used for lazy evaluation of values and prioritizing - * equally-suitable outputs with lower resource cost. - * - * For example, a MultiValue representing a file could have a key called - * `stream` as well as a key called `s3-info`. It would always be possible - * to obtain a `stream` but when the `s3-info` is available and applicable - * it will be less costly to obtain. - */ -class MultiValue extends AdvancedBase { - constructor () { - super(); - this.factories = {}; - this.values = {}; - } - - async add_factory (key_desired, key_available, fn, cost) { - if ( ! this.factories[key_desired] ) { - this.factories[key_desired] = []; - } - this.factories[key_desired].push({ - key_available, - fn, - cost, - }); - } - - async get (key) { - return this._get(key); - } - - set (key, value) { - this.values[key] = value; - } - - async _get (key) { - if ( this.values[key] ) { - return this.values[key]; - } - const factories = this.factories[key]; - if ( !factories || !factories.length ) { - console.log('no factory for key', key); - return undefined; - } - for ( const factory of factories ) { - const available = await this._get(factory.key_available); - if ( ! available ) { - console.log('no available for key', key, factory.key_available); - continue; - } - const value = await factory.fn(available); - this.values[key] = value; - return value; - } - return undefined; - } -} - -module.exports = { - MultiValue, -}; diff --git a/src/backend/src/util/objutil.js b/src/backend/src/util/objutil.js deleted file mode 100644 index ec2303422..000000000 --- a/src/backend/src/util/objutil.js +++ /dev/null @@ -1,43 +0,0 @@ -const DO_NOT_DEFINE = Symbol('DO_NOT_DEFINE'); - -const createTransformedValues = (input, options = {}, state = {}) => { - // initialize state - if ( ! state.keys ) state.keys = []; - - if ( Array.isArray(input) ) { - if ( options.doNotProcessArrays ) { - return DO_NOT_DEFINE; - } - const output = []; - for ( let i = 0 ; i < input.length; i++ ) { - const value = input[i]; - state.keys.push(i); - output.push(createTransformedValues(value, options, state)); - state.keys.pop(); - } - return output; - } - if ( input && typeof input === 'object' && !Array.isArray(input) ) { - const output = {}; - Object.setPrototypeOf(output, input); - for ( const k in input ) { - state.keys.push(k); - const new_value = createTransformedValues(input[k], options, state); - if ( new_value !== DO_NOT_DEFINE ) { - output[k] = new_value; - } - state.keys.pop(); - } - return output; - } - let value = input; - if ( options.mutateValue ) { - value = options.mutateValue(value, { options, state }); - } - return value; -}; - -module.exports = { - createTransformedValues, - DO_NOT_DEFINE, -}; diff --git a/src/backend/src/util/opmath.bench.js b/src/backend/src/util/opmath.bench.js deleted file mode 100644 index 80aab31fb..000000000 --- a/src/backend/src/util/opmath.bench.js +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { bench, describe } from 'vitest'; -import { EWMA, MovingMode, TimeWindow, normalize } from './opmath.js'; - -describe('EWMA - Exponential Weighted Moving Average', () => { - bench('EWMA put() with constant alpha', () => { - const ewma = new EWMA({ initial: 0, alpha: 0.2 }); - for ( let i = 0; i < 1000; i++ ) { - ewma.put(Math.random() * 100); - } - }); - - bench('EWMA put() with function alpha', () => { - const ewma = new EWMA({ initial: 0, alpha: () => 0.2 }); - for ( let i = 0; i < 1000; i++ ) { - ewma.put(Math.random() * 100); - } - }); - - bench('EWMA get() after many puts', () => { - const ewma = new EWMA({ initial: 0, alpha: 0.2 }); - for ( let i = 0; i < 100; i++ ) { - ewma.put(i); - } - for ( let i = 0; i < 1000; i++ ) { - ewma.get(); - } - }); -}); - -describe('MovingMode - Mode calculation with sliding window', () => { - bench('MovingMode put() with window_size=30', () => { - const mode = new MovingMode({ initial: 0, window_size: 30 }); - for ( let i = 0; i < 1000; i++ ) { - mode.put(Math.floor(Math.random() * 10)); - } - }); - - bench('MovingMode put() with window_size=100', () => { - const mode = new MovingMode({ initial: 0, window_size: 100 }); - for ( let i = 0; i < 1000; i++ ) { - mode.put(Math.floor(Math.random() * 10)); - } - }); - - bench('MovingMode with high cardinality values', () => { - const mode = new MovingMode({ initial: 0, window_size: 50 }); - for ( let i = 0; i < 1000; i++ ) { - mode.put(Math.floor(Math.random() * 1000)); - } - }); - - bench('MovingMode with low cardinality values', () => { - const mode = new MovingMode({ initial: 0, window_size: 50 }); - for ( let i = 0; i < 1000; i++ ) { - mode.put(Math.floor(Math.random() * 3)); - } - }); -}); - -describe('TimeWindow - Time-based sliding window', () => { - bench('TimeWindow add() and get()', () => { - let fakeTime = 0; - const tw = new TimeWindow({ - window_duration: 1000, - reducer: values => values.reduce((a, b) => a + b, 0), - now: () => fakeTime, - }); - for ( let i = 0; i < 1000; i++ ) { - fakeTime += 10; - tw.add(Math.random()); - } - }); - - bench('TimeWindow with stale entry removal', () => { - let fakeTime = 0; - const tw = new TimeWindow({ - window_duration: 100, - reducer: values => values.length, - now: () => fakeTime, - }); - for ( let i = 0; i < 1000; i++ ) { - fakeTime += 50; // Fast time progression causes stale removal - tw.add(i); - tw.get(); - } - }); -}); - -describe('normalize - Exponential normalization', () => { - bench('normalize() single value', () => { - for ( let i = 0; i < 10000; i++ ) { - normalize({ high_value: 0.001 }, Math.random()); - } - }); - - bench('normalize() with varying high_value', () => { - const high_values = [0.001, 0.01, 0.1, 1, 10]; - for ( let i = 0; i < 10000; i++ ) { - const hv = high_values[i % high_values.length]; - normalize({ high_value: hv }, Math.random() * 100); - } - }); -}); diff --git a/src/backend/src/util/opmath.js b/src/backend/src/util/opmath.js deleted file mode 100644 index 68568a0db..000000000 --- a/src/backend/src/util/opmath.js +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -class Getter { - static adapt (v) { - if ( typeof v === 'function' ) return v; - return () => v; - } -} - -const LinearByCountGetter = ({ initial, slope, pre = false }) => { - let value = initial; - return () => { - if ( pre ) value += slope; - let v = value; - if ( ! pre ) value += slope; - return v; - }; -}; - -const ConstantGetter = ({ initial }) => () => initial; - -// bind function for parameterized functions -const Bind = (fn, important_parameters) => { - return (given_parameters) => { - return fn({ - ...given_parameters, - ...important_parameters, - }); - }; -}; - -/** - * SwitchByCountGetter - * - * @example - * const getter = SwitchByCountGetter({ - * initial: 0, - * body: { - * 0: Bind(LinearByCountGetter, { slop: 1 }), - * 5: ConstantGetter, - * } - * }); // 0, 1, 2, 3, 4, 4, 4, ... - */ -const SwitchByCountGetter = ({ initial, body }) => { - let value = initial ?? 0; - let count = 0; - let getter; - if ( ! body.hasOwnProperty(count) ) { - throw new Error('body of SwitchByCountGetter must have an entry for count 0'); - } - return () => { - if ( body.hasOwnProperty(count) ) { - getter = body[count]({ initial: value }); - console.log('getter is', getter); - } - value = getter(); - count++; - return value; - }; -}; - -class StreamReducer { - constructor (initial) { - this.value = initial; - } - - put (v) { - this._put(v); - } - - get () { - return this._get(); - } - - _put (v) { - throw new Error('Not implemented'); - } - - _get () { - return this.value; - } -} - -class EWMA extends StreamReducer { - constructor ({ initial, alpha }) { - super(initial ?? 0); - this.alpha = Getter.adapt(alpha); - } - - _put (v) { - this.value = this.alpha() * v + (1 - this.alpha()) * this.value; - } -} - -class MovingMode extends StreamReducer { - constructor ({ initial, window_size }) { - super(initial ?? 0); - this.window_size = window_size ?? 30; - this.window = []; - } - - _put (v) { - this.window.push(v); - if ( this.window.length > this.window_size ) { - this.window.shift(); - } - this.value = this._get_mode(); - } - - _get_mode () { - let counts = {}; - for ( let v of this.window ) { - if ( ! counts.hasOwnProperty(v) ) counts[v] = 0; - counts[v]++; - } - let max = 0; - let mode = null; - for ( let v in counts ) { - if ( counts[v] > max ) { - max = counts[v]; - mode = v; - } - } - return mode; - } -} - -class TimeWindow { - constructor ({ window_duration, reducer, now }) { - this.window_duration = window_duration; - this.reducer = reducer; - this.entries_ = []; - this.now = now ?? Date.now; - } - - add (value) { - this.remove_stale_entries_(); - - const timestamp = this.now(); - this.entries_.push({ - timestamp, - value, - }); - } - - get () { - this.remove_stale_entries_(); - - const values = this.entries_.map(entry => entry.value); - if ( ! this.reducer ) return values; - - return this.reducer(values); - } - - get_entries () { - return [...this.entries_]; - } - - remove_stale_entries_ () { - let i = 0; - const current_ts = this.now(); - for ( ; i < this.entries_.length ; i++ ) { - const entry = this.entries_[i]; - // as soon as an entry is in the window we can break, - // since entries will always be in ascending order by timestamp - if ( current_ts - entry.timestamp < this.window_duration ) { - break; - } - } - - this.entries_ = this.entries_.slice(i); - } -} - -const normalize = ({ - high_value, -}, value) => { - const k = -1 * (1 / high_value); - return 1 - Math.pow(Math.E, k * value); -}; - -module.exports = { - Getter, - LinearByCountGetter, - SwitchByCountGetter, - ConstantGetter, - Bind, - StreamReducer, - EWMA, - MovingMode, - TimeWindow, - normalize, -}; diff --git a/src/backend/src/util/opmath.test.js b/src/backend/src/util/opmath.test.js deleted file mode 100644 index 97c0619a6..000000000 --- a/src/backend/src/util/opmath.test.js +++ /dev/null @@ -1,40 +0,0 @@ -import { describe, it, expect } from 'vitest'; - -describe('opmath', () => { - describe('TimeWindow', () => { - it('clears old entries', () => { - const { TimeWindow } = require('./opmath'); - let now_value = 0; - const now = () => now_value; - const window = new TimeWindow({ window_duration: 1000, now }); - - window.add(1); - window.add(2); - window.add(3); - - now_value = 900; - - window.add(4); - window.add(5); - window.add(6); - - expect(window.get()).toEqual([1, 2, 3, 4, 5, 6]); - - now_value = 1100; - - window.add(7); - window.add(8); - window.add(9); - - expect(window.get()).toEqual([4, 5, 6, 7, 8, 9]); - - now_value = 2000; - - expect(window.get()).toEqual([7, 8, 9]); - - now_value = 2200; - - expect(window.get()).toEqual([]); - }); - }); -}); \ No newline at end of file diff --git a/src/backend/src/util/otelutil.js b/src/backend/src/util/otelutil.js deleted file mode 100644 index 07807e659..000000000 --- a/src/backend/src/util/otelutil.js +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -// The OpenTelemetry SDK provides a very error-prone API for creating -// spans. This is a wrapper around the SDK that makes it convenient -// to create spans correctly. The path of least resistance should -// be the correct path, not a way to shoot yourself in the foot. - -import { context, trace, SpanStatusCode } from '@opentelemetry/api'; -import { TeePromise } from '@heyputer/putility/src/libs/promise.js'; - -/* -parallel span example from GPT-4: - -promises.push(tracer.startActiveSpan(`job:${job.id}`, (span) => { - return context.with(trace.setSpan(context.active(), span), async () => { - try { - await job.run(); - } catch (error) { - span.setStatus({ code: SpanStatusCode.ERROR, message: error.message }); - throw error; - } finally { - span.end(); - } - }); -})); -*/ - -export const DEFAULT_TRACER_NAME = 'puter-tracer'; - -export const getTracer = (name = DEFAULT_TRACER_NAME) => - trace.getTracer(name ?? DEFAULT_TRACER_NAME); - -const resolveTracer = (tracer, name) => - tracer ?? getTracer(name ?? DEFAULT_TRACER_NAME); - -/** @type {(label:string, fn:T, options?: object | unknown, tracer?: unknown)=> T} */ -export const spanify = (label, fn, options, tracer) => async function (...args) { - if ( options && typeof options.startActiveSpan === 'function' && !tracer ) { - tracer = options; - options = undefined; - } - - const resolvedTracer = resolveTracer(tracer); - let result; - const spanArgs = [label]; - if ( options !== null && typeof options === 'object' ) { - spanArgs.push(options); - } - spanArgs.push(async span => { - try { - // eslint-disable-next-line no-invalid-this - result = await fn.apply(this, args); - span.setStatus({ code: SpanStatusCode.OK }); - return result; - } catch (e) { - span.recordException(e); - span.setStatus({ code: SpanStatusCode.ERROR, message: e.message }); - throw e; - } finally { - span.end(); - } - }); - return await resolvedTracer.startActiveSpan(...spanArgs); -}; - -/** @type {(label:string, fn:T, options?: object | unknown, tracer?: unknown)=> ReturnType} */ -export const span = async (label, fn, options, tracer) => - await spanify(label, fn, options, tracer)(); - -/** @type {(label: string, options?: object | unknown, tracer?: unknown) => MethodDecorator} */ -export const Span = (label, options, tracer) => (_target, _propertyKey, descriptor) => { - if ( !descriptor || typeof descriptor.value !== 'function' ) return descriptor; - descriptor.value = spanify(label, descriptor.value, options, tracer); - return descriptor; -}; - -export const abtest = async (label, impls) => { - const tracer = getTracer(); - let result; - const impl_keys = Object.keys(impls); - const impl_i = Math.floor(Math.random() * impl_keys.length); - const impl_name = impl_keys[impl_i]; - const impl = impls[impl_name]; - - await tracer.startActiveSpan(`${label }:${ impl_name}`, async span => { - span.setAttribute('abtest.impl', impl_name); - result = await impl(); - span.end(); - }); - return result; -}; - -export class ParallelTasks { - constructor ({ tracer, max } = {}) { - this.tracer = tracer ?? getTracer(); - this.max = max ?? Infinity; - this.promises = []; - - this.queue_ = []; - this.ongoing_ = 0; - } - - add (name, fn, flags) { - if ( this.ongoing_ >= this.max && !flags?.force ) { - const p = new TeePromise(); - this.promises.push(p); - this.queue_.push([name, fn, p]); - return; - } - - this.promises.push(this.run_(name, fn)); - } - - run_ (name, fn) { - this.ongoing_++; - const span = this.tracer.startSpan(name); - return context.with(trace.setSpan(context.active(), span), async () => { - try { - const res = await fn(); - this.ongoing_--; - this.check_queue_(); - return res; - } catch ( error ) { - span.setStatus({ code: SpanStatusCode.ERROR, message: error.message }); - throw error; - } finally { - span.end(); - } - }); - } - - check_queue_ () { - while ( this.ongoing_ < this.max && this.queue_.length > 0 ) { - const [name, fn, p] = this.queue_.shift(); - const run_p = this.run_(name, fn); - run_p.then(p.resolve.bind(p), p.reject.bind(p)); - } - } - - async awaitAll () { - await Promise.all(this.promises); - } - - async awaitAllAndDeferThrow () { - const results = await Promise.allSettled(this.promises); - const errors = []; - for ( const result of results ) { - if ( result.status === 'rejected' ) { - errors.push(result.reason); - } - } - if ( errors.length !== 0 ) { - throw new AggregateError(errors); - } - } -} diff --git a/src/backend/src/util/outcomeutil.ts b/src/backend/src/util/outcomeutil.ts deleted file mode 100644 index d7c723cf7..000000000 --- a/src/backend/src/util/outcomeutil.ts +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Represents the outcome of a task that might fail or succeed. - */ -export class OutcomeObject { - /** - * If the task was not successful, this will be the message a user - * sees. - */ - userMessage = null; - - /** - * If the task was not successful, this will be the i18n key for - * the message a user sees. - */ - userMessageKey = null; - - /** - * If the task was not successful, this will be values used for - * a message template that is identified using `userMessageKey`. - */ - userMessageFields = {}; - - /** - * If the task being performed failed - */ - failed = false; - - messages: Record[] = []; - fields = {}; - - /** - * Whether the task being performed has ended, - * either successfully or unsuccessfully. - */ - ended = false; - - infoObject: T; - - constructor (infoObject: T) { - this.failed = true; - this.userMessageFields = {}; - this.infoObject = infoObject; - } - log (text, fields?: unknown) { - this.messages.push({ text, fields }); - } - - get succeeded () { - return this.ended && !this.failed; - } - - /** - * Records a failure message. - * Returns the outcome object for chaining with a return statement. - * - * @example - * return outcome.fail( - * 'User already exists', - * 'signup.user_already_exists', - * { username: 'john_doe' } - * ); - * - * @param {*} message - message the user sees without i18n - * @param {*} i18nKey - i18n key for the message - * @param {*} fields - fields for i18n-key-identified template - */ - fail (message, i18nKey, fields = {}) { - this.userMessage = message; - this.userMessageKey = i18nKey; - this.userMessageFields = fields; - this.ended = true; - this.failed = true; - return this; - } - - success () { - this.ended = true; - this.failed = false; - return this; - } -} diff --git a/src/backend/src/util/pathutil.bench.js b/src/backend/src/util/pathutil.bench.js deleted file mode 100644 index 972a1b8fc..000000000 --- a/src/backend/src/util/pathutil.bench.js +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { bench, describe } from 'vitest'; -import { PathBuilder } from './pathutil.js'; - -describe('PathBuilder - Creation', () => { - bench('create PathBuilder (default)', () => { - PathBuilder.create(); - }); - - bench('create PathBuilder (puterfs mode)', () => { - PathBuilder.create({ puterfs: true }); - }); - - bench('create via new', () => { - new PathBuilder(); - }); -}); - -describe('PathBuilder - Static add', () => { - bench('static add single fragment', () => { - PathBuilder.add('directory'); - }); - - bench('static add with traversal prevention', () => { - PathBuilder.add('../../../etc/passwd'); - }); - - bench('static add with allow_traversal', () => { - PathBuilder.add('../parent', { allow_traversal: true }); - }); -}); - -describe('PathBuilder - Static resolve', () => { - bench('resolve simple path', () => { - PathBuilder.resolve('/home/user/file.txt'); - }); - - bench('resolve relative path', () => { - PathBuilder.resolve('./relative/path'); - }); - - bench('resolve with puterfs', () => { - PathBuilder.resolve('/home/user/file.txt', { puterfs: true }); - }); - - bench('resolve complex path', () => { - PathBuilder.resolve('/a/b/c/../d/./e/f'); - }); -}); - -describe('PathBuilder - Instance add', () => { - bench('add single fragment', () => { - const builder = PathBuilder.create(); - builder.add('directory'); - }); - - bench('add multiple fragments (chain)', () => { - PathBuilder.create() - .add('home') - .add('user') - .add('documents') - .add('file.txt'); - }); - - bench('add 10 fragments', () => { - const builder = PathBuilder.create(); - for ( let i = 0; i < 10; i++ ) { - builder.add(`dir${i}`); - } - }); -}); - -describe('PathBuilder - Traversal prevention', () => { - bench('sanitize parent traversal (..)', () => { - PathBuilder.create().add('..'); - }); - - bench('sanitize multiple parent traversals', () => { - PathBuilder.create().add('../../..'); - }); - - bench('sanitize mixed traversal patterns', () => { - PathBuilder.create().add('../foo/../../bar/../baz'); - }); - - bench('sanitize with backslash traversal', () => { - PathBuilder.create().add('..\\..\\..\\etc\\passwd'); - }); - - bench('allow_traversal option', () => { - PathBuilder.create().add('../parent/child', { allow_traversal: true }); - }); -}); - -describe('PathBuilder - Build', () => { - bench('build empty path', () => { - PathBuilder.create().build(); - }); - - bench('build simple path', () => { - PathBuilder.create() - .add('home') - .add('user') - .build(); - }); - - bench('build long path', () => { - const builder = PathBuilder.create(); - for ( let i = 0; i < 20; i++ ) { - builder.add(`directory${i}`); - } - builder.build(); - }); -}); - -describe('PathBuilder - Complete workflows', () => { - bench('create, add, build (simple)', () => { - PathBuilder.create() - .add('home') - .add('user') - .add('file.txt') - .build(); - }); - - bench('create, add, build (with sanitization)', () => { - PathBuilder.create() - .add('../attempt') - .add('actual') - .add('path') - .build(); - }); - - bench('puterfs path building', () => { - PathBuilder.create({ puterfs: true }) - .add('username') - .add('documents') - .add('report.pdf') - .build(); - }); -}); - -describe('PathBuilder - Batch operations', () => { - const fragments = ['home', 'user', 'documents', 'projects', 'puter']; - - bench('build 10 paths', () => { - for ( let i = 0; i < 10; i++ ) { - const builder = PathBuilder.create(); - for ( const frag of fragments ) { - builder.add(frag); - } - builder.build(); - } - }); - - bench('build 100 paths', () => { - for ( let i = 0; i < 100; i++ ) { - const builder = PathBuilder.create(); - for ( const frag of fragments ) { - builder.add(frag); - } - builder.build(); - } - }); -}); - -describe('Comparison with native path operations', () => { - const path = require('path'); - - bench('PathBuilder.resolve', () => { - PathBuilder.resolve('/home/user/file.txt'); - }); - - bench('native path.resolve', () => { - path.resolve('/home/user/file.txt'); - }); - - bench('PathBuilder chain vs path.join', () => { - PathBuilder.create() - .add('home') - .add('user') - .add('file.txt') - .build(); - }); - - bench('native path.join', () => { - path.join('home', 'user', 'file.txt'); - }); -}); diff --git a/src/backend/src/util/pathutil.js b/src/backend/src/util/pathutil.js deleted file mode 100644 index 533a716bc..000000000 --- a/src/backend/src/util/pathutil.js +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require('../../../putility'); - -/** - * PathBuilder implements the builder pattern for building paths. - * This makes it clear which path fragments are allowed to traverse - * to parent directories. - */ -class PathBuilder extends AdvancedBase { - static MODULES = { - path: require('path'), - }; - - constructor (parameters = {}) { - super(); - if ( parameters.puterfs ) { - this.modules.path = - this.modules.path.posix; - } - this.path_ = ''; - } - - static create (parameters) { - return new PathBuilder(parameters); - } - - static add (fragment, options) { - return PathBuilder.create().add(fragment, options); - } - - static resolve (fragment, parameters = {}) { - const { puterfs } = parameters; - - const p = PathBuilder.create(parameters); - const require = p.require; - const node_path = require('path'); - fragment = node_path.resolve(fragment); - if ( process.platform === 'win32' && !parameters.puterfs ) { - fragment = `/${ fragment.slice('c:\\'.length)}`; // >:-( - } - let result = p.add(fragment).build(); - if ( puterfs && process.platform === 'win32' && - result.startsWith('\\') - ) { - result = `/${ result.slice(1)}`; - } - return result; - } - - add (fragment, options) { - const require = this.require; - const node_path = require('path'); - - options = options || {}; - if ( ! options.allow_traversal ) { - fragment = node_path.normalize(fragment); - fragment = fragment.replace(/(\.+\/|\.+\\)/g, ''); - if ( fragment === '..' ) { - fragment = ''; - } - } - - this.path_ = this.path_ - ? node_path.join(this.path_, fragment) - : fragment; - - return this; - } - - build () { - return this.path_; - } -} - -module.exports = { - PathBuilder, -}; diff --git a/src/backend/src/util/retryutil.js b/src/backend/src/util/retryutil.js deleted file mode 100644 index a46d62928..000000000 --- a/src/backend/src/util/retryutil.js +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -/** - * Retries a function a maximum number of times, with a given interval between each try. - * @param {Function} func - The function to retry - * @param {Number} max_tries - The maximum number of tries - * @param {Number} interval - The interval between each try - * @returns {Promise<[Error, Boolean, any]>} - A promise that resolves to an - * array containing the last error, a boolean indicating whether the function - * eventually succeeded, and the return value of the function - */ -const simple_retry = async function simple_retry (func, max_tries, interval) { - let tries = 0; - let last_error = null; - - if ( max_tries === undefined ) { - throw new Error('simple_retry: max_tries is undefined'); - } - if ( interval === undefined ) { - throw new Error('simple_retry: interval is undefined'); - } - - while ( tries < max_tries ) { - try { - return [last_error, true, await func()]; - } catch ( error ) { - last_error = error; - tries++; - await new Promise((resolve) => setTimeout(resolve, interval)); - } - } - if ( last_error === null ) { - last_error = new Error('simple_retry: failed, but error is null'); - } - return [last_error, false]; -}; - -const poll = async function poll ({ poll_fn, schedule_fn }) { - let delay; - - while ( true ) { - const is_done = await poll_fn(); - if ( is_done ) { - return; - } - delay = schedule_fn(delay); - await new Promise((resolve) => setTimeout(resolve, delay)); - } -}; - -module.exports = { - simple_retry, - poll, -}; diff --git a/src/backend/src/util/safety.js b/src/backend/src/util/safety.js deleted file mode 100644 index 733ec35a3..000000000 --- a/src/backend/src/util/safety.js +++ /dev/null @@ -1,16 +0,0 @@ -/** - * Instead of `myObject.hasOwnProperty(k)`, always write: - * `safeHasOwnProperty(myObject, k)`. - * - * This is a less verbose way to call `Object.prototype.hasOwnProperty.call`. - * This prevents unexpected behavior when `hasOwnProperty` is overridden, - * which is especially possible for objects parsed from user-sent JSON. - * - * explanation: https://eslint.org/docs/latest/rules/no-prototype-builtins - * @param {*} o - * @param {...any} a - * @returns - */ -export const safeHasOwnProperty = (o, ...a) => { - return Object.prototype.hasOwnProperty.call(o, ...a); -}; \ No newline at end of file diff --git a/src/backend/src/util/securehttp.js b/src/backend/src/util/securehttp.js deleted file mode 100644 index 4c33939bd..000000000 --- a/src/backend/src/util/securehttp.js +++ /dev/null @@ -1,273 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const http = require('http'); -const https = require('https'); -const dns = require('dns'); -const net = require('net'); -const { URL } = require('url'); -const APIError = require('../api/APIError'); - -// Cloudflare's malware-blocking DNS server -const SECURE_DNS_SERVER = '1.1.1.3'; - -/** - * Validates that a URL does not contain an IP address (IPv4 or IPv6). - * Only domain names are allowed to prevent SSRF attacks. - * - * This is NOT the only validation required to prevent SSRF attacks. - * - * @param {string} url - The URL to validate - * @throws {APIError} If the URL contains an IP address - */ -function validateUrlNoIP (url) { - const parsedUrl = new URL(url); - - const hostname = parsedUrl.hostname; - - // Remove brackets from IPv6 addresses for validation - const hostnameForValidation = hostname.startsWith('[') && hostname.endsWith(']') - ? hostname.slice(1, -1) - : hostname; - - // Disallow specifying the host by IP address directly. - // (we want to always use CloudFlare DNS here) - const ipVersion = net.isIP(hostnameForValidation); - if ( ipVersion === 4 || ipVersion === 6 ) { - throw APIError.create('ip_not_allowed'); - } - - // This is not necessary, but there's no reason not to disallow this - if ( hostnameForValidation === 'localhost' ) { - throw APIError.create('ip_not_allowed'); - } -} - -/** - * Creates a custom DNS lookup function that uses 1.1.1.3 for DNS resolution. - * This function resolves hostnames using Node.js's built-in Resolver with the secure DNS server. - * @param {string} hostname - The hostname to resolve - * @param {Object|number|Function} options - Lookup options, family number, or callback - * @param {Function} callback - Callback function (err, address, family) or (err, addresses[]) - */ -function secureDNSLookup (hostname, options, callback) { - // Overloading (possible call signatures) - if ( typeof options === 'function' ) { - callback = options; - options = { family: 0, all: false }; - } else if ( typeof options === 'number' ) { - options = { family: options, all: false }; - } else if ( ! options ) { - options = { family: 0, all: false }; - } - - const family = options.family || 0; // 0 = both, 4 = IPv4, 6 = IPv6 - const all = options.all || false; - - const hostnameForValidation = hostname.startsWith('[') && hostname.endsWith(']') - ? hostname.slice(1, -1) - : hostname; - - // Ensure IP addresses don't reach this DNS lookup - // (already checked in validateUrlNoIP, but double-check in - // case this ever is called elsewhere) - const ipVersion = net.isIP(hostnameForValidation); - if ( ipVersion === 4 || ipVersion === 6 ) { - return callback(new Error('IP addresses not allowed')); - } - - // Use Resolver with 1.1.1.3 to resolve the hostname - const resolver = new dns.Resolver(); - resolver.setServers([SECURE_DNS_SERVER]); - - const resolveAddresses = (err, addresses, addrFamily) => { - if ( err || !addresses || addresses.length === 0 ) { - console.error(`[securehttp] Failed to resolve ${hostname}:`, err || 'No addresses found'); - return callback(err || new Error('No addresses found')); - } - - if ( all ) { - const result = addresses.map(addr => ({ address: addr, family: addrFamily })); - callback(null, result); - } else { - callback(null, addresses[0], addrFamily); - } - }; - - if ( family === 4 || family === 0 ) { - resolver.resolve4(hostname, (err, addresses) => { - if ( !err && addresses && addresses.length > 0 ) { - console.log(`[securehttp] Resolved ${hostname} to ${addresses[0]} via 1.1.1.3 (IPv4)`); - resolveAddresses(null, addresses, 4); - } else if ( family === 4 ) { - // If we only wanted IPv4 and it failed, return error - resolveAddresses(err || new Error('No IPv4 addresses found'), null, 4); - } else { - // Try IPv6 as fallback - resolver.resolve6(hostname, (err6, addresses6) => { - if ( !err6 && addresses6 && addresses6.length > 0 ) { - console.log(`[securehttp] Resolved ${hostname} to ${addresses6[0]} via 1.1.1.3 (IPv6)`); - resolveAddresses(null, addresses6, 6); - } else { - resolveAddresses(err6 || err || new Error('No addresses found'), null, 0); - } - }); - } - }); - } else if ( family === 6 ) { - // IPv6 only - resolver.resolve6(hostname, (err, addresses) => { - if ( !err && addresses && addresses.length > 0 ) { - console.log(`[securehttp] Resolved ${hostname} to ${addresses[0]} via 1.1.1.3 (IPv6)`); - resolveAddresses(null, addresses, 6); - } else { - resolveAddresses(err || new Error('No IPv6 addresses found'), null, 6); - } - }); - } else { - callback(new Error('Invalid family')); - } -} - -/** - * Creates secure HTTP and HTTPS agents with custom DNS lookup and no redirects. - * @returns {Object} Object containing httpAgent and httpsAgent - */ -function createSecureAgents () { - const httpAgent = new http.Agent({ - lookup: secureDNSLookup, - keepAlive: false, - }); - - const httpsAgent = new https.Agent({ - lookup: secureDNSLookup, - keepAlive: false, - }); - - return { httpAgent, httpsAgent }; -} - -/** - * Makes a secure HTTP request using axios with SSRF protections: - * - Validates URL does not contain IP addresses - * - Disables redirects - * - Uses secure DNS resolution (1.1.1.3) - * @param {Object} axios - The axios instance - * @param {string} url - The URL to request - * @param {Object} options - Additional axios options - * @returns {Promise} Axios response - */ -async function secureAxiosRequest (axios, url, options = {}) { - // Validate URL doesn't contain IP addresses - validateUrlNoIP(url); - - // Create secure agents - const { httpAgent, httpsAgent } = createSecureAgents(); - - // Merge options with security settings - const secureOptions = { - ...options, - maxRedirects: 0, // Disable redirects - axios will return 3xx responses without following - httpAgent, - httpsAgent, - validateStatus: (_status) => { - // Accept all status codes so we can check for redirects - return true; - }, - }; - - try { - const parsedUrl = new URL(url); - if ( parsedUrl.protocol !== 'data:' && globalThis.global_config.services.secureCorsProxy.url ) { - url = globalThis.global_config.services.secureCorsProxy.url + url; - if ( ! secureOptions.headers ) { - secureOptions.headers = {}; - } - secureOptions.headers['x-cors-proxy-auth-secret'] = globalThis.global_config.services.secureCorsProxy.secret; - - } - const response = await axios.get(url, secureOptions); - - // Check if the response is a redirect (maxRedirects: 0 means axios returns but doesn't follow) - if ( response.status >= 300 && response.status < 400 ) { - throw APIError.create('field_invalid', null, { - key: 'url', - expected: 'web URL (redirects not allowed)', - got: `redirect to ${response.headers.location || 'unknown'}`, - }); - } - - // Log different information based on URL type - - if ( parsedUrl.protocol === 'data:' ) { - // Extract data format from data URL - const dataFormat = url.split(',')[0].split(':')[1] || 'unknown format'; - console.log(`[securehttp] Successfully processed data URL with format: ${dataFormat}`); - } else { - console.log(`[securehttp] Successfully fetched ${url} (status: ${response.status})`); - } - return response; - } catch (e) { - // Re-throw APIError if it's already one (e.g., from validateUrlNoIP or redirect check) - if ( e instanceof APIError || (e.constructor && e.constructor.name === 'APIError') ) { - throw e; - } - - // Log different information based on URL type - const parsedUrl = new URL(url); - if ( parsedUrl.protocol === 'data:' ) { - // Extract data format from data URL - const dataFormat = url.split(',')[0].split(':')[1] || 'unknown format'; - console.error(`[securehttp] Request failed for data URL with format: ${dataFormat}:`, e); - } else { - console.error(`[securehttp] Request failed for ${url}:`, e); - } - - // Handle redirect errors in catch block (in case axios throws for redirects) - if ( e.response && (e.response.status === 301 || e.response.status === 302 || - e.response.status === 303 || e.response.status === 307 || e.response.status === 308) ) { - throw APIError.create('field_invalid', null, { - key: 'url', - expected: 'web URL (redirects not allowed)', - got: `redirect to ${e.response.headers.location || 'unknown'}`, - }); - } - - // Provide more detailed error messages - let errorMessage = e.message; - if ( e.code === 'ENOTFOUND' || e.code === 'EAI_AGAIN' ) { - errorMessage = `DNS resolution failed: ${e.message}`; - } else if ( e.code === 'ECONNREFUSED' ) { - errorMessage = `Connection refused: ${e.message}`; - } else if ( e.code === 'ETIMEDOUT' ) { - errorMessage = `Connection timeout: ${e.message}`; - } - - throw APIError.create('field_invalid', null, { - key: 'url', - expected: 'web URL', - got: errorMessage, - }); - } -} - -module.exports = { - validateUrlNoIP, - createSecureAgents, - secureAxiosRequest, -}; diff --git a/src/backend/src/util/stdioutil.js b/src/backend/src/util/stdioutil.js deleted file mode 100644 index 385e8bd4b..000000000 --- a/src/backend/src/util/stdioutil.js +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -/** - * Strip ANSI escape sequences from a string (e.g. color codes) - * and then return the length of the resulting string. - * - * @param {*} str - */ -const visible_length = (str) => { - // eslint-disable-next-line no-control-regex - return str.replace(/\x1b\[[0-9;]*m/g, '').length; -}; - -/** - * Split a string into lines according to the terminal width, - * preserving ANSI escape sequences, and return an array of lines. - * - * @param {*} str - */ -const split_lines = (str) => { - const lines = []; - let line = ''; - let line_length = 0; - for ( const c of str ) { - line += c; - if ( c === '\n' ) { - lines.push(line); - line = ''; - line_length = 0; - } else { - line_length++; - if ( line_length >= process.stdout.columns ) { - lines.push(line); - line = ''; - line_length = 0; - } - } - } - if ( line.length ) { - lines.push(line); - } - return lines; -}; - -module.exports = { - visible_length, - split_lines, -}; diff --git a/src/backend/src/util/streamutil.js b/src/backend/src/util/streamutil.js deleted file mode 100644 index 633c2c706..000000000 --- a/src/backend/src/util/streamutil.js +++ /dev/null @@ -1,569 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { PassThrough, Readable, Transform } = require('stream'); -const { TeePromise } = require('@heyputer/putility').libs.promise; -const crypto = require('crypto'); - -class StreamBuffer extends TeePromise { - constructor () { - super(); - - this.stream = new PassThrough(); - this.buffer_ = ''; - - this.stream.on('data', (chunk) => { - this.buffer_ += chunk.toString(); - }); - - this.stream.on('end', () => { - this.resolve(this.buffer_); - }); - - this.stream.on('error', (err) => { - this.reject(err); - }); - } -} - -const stream_to_the_void = stream => { - stream.on('data', () => { - }); - stream.on('end', () => { - }); - stream.on('error', () => { - }); -}; - -/** - * This will split a stream (on the read side) into `n` streams. - * The slowest reader will determine the speed the the source stream - * is consumed at to avoid buffering. - * - * @param {*} source - * @param {*} n - * @returns - */ -const pausing_tee = (source, n) => { - const { PassThrough } = require('stream'); - - const ready_ = []; - const streams_ = []; - let first_ = true; - for ( let i = 0 ; i < n ; i++ ) { - ready_.push(true); - const stream = new PassThrough(); - streams_.push(stream); - stream.on('drain', () => { - ready_[i] = true; - if ( first_ ) { - source.resume(); - first_ = false; - } - if ( ready_.every(v => !!v) ) source.resume(); - }); - } - - source.on('data', (chunk) => { - ready_.forEach((v, i) => { - ready_[i] = streams_[i].write(chunk); - }); - if ( ! ready_.every(v => !!v) ) { - source.pause(); - return; - } - }); - - source.on('end', () => { - for ( let i = 0 ; i < n ; i++ ) { - streams_[i].end(); - } - }); - - source.on('error', (err) => { - for ( let i = 0 ; i < n ; i++ ) { - streams_[i].emit('error', err); - } - }); - - return streams_; -}; - -/** - * A debugging stream transform that logs the data it receives. - */ -class LoggingStream extends Transform { - constructor (options) { - super(options); - this.count = 0; - } - - _transform (chunk, encoding, callback) { - const stream_id = this.id ?? 'unknown'; - console.log(`[DATA@${stream_id}] :: ${chunk.length} (${this.count++})`); - this.push(chunk); - callback(); - } -} - -// logs stream activity -const logging_stream = source => { - const stream = new LoggingStream(); - if ( source.id ) stream.id = source.id; - source.pipe(stream); - return stream; -}; - -/** - * Returns a readable stream that emits the data from `originalDataStream`, - * replacing the data at position `offset` with the data from `newDataStream`. - * When the `newDataStream` is consumed, the `originalDataStream` will continue - * emitting data. - * - * Note: `originalDataStream` will be paused until `newDataStream` is consumed. - * - * @param {*} originalDataStream - * @param {*} newDataStream - * @param {*} offset - */ -const offset_write_stream = ({ - originalDataStream, newDataStream, offset, - replace_length = 0, -}) => { - - const passThrough = new PassThrough(); - let remaining = offset; - let new_end = false; - let org_end = false; - let replaced_bytes = 0; - let defer_buffer = Buffer.alloc(0); - let new_stream_early_buffer = Buffer.alloc(0); - let implied; - const STATE_ORIGINAL_STREAM = { - on_enter: () => { - console.log('STATE_ORIGINAL_STREAM'); - newDataStream.pause(); - }, - }; - - const STATE_NEW_STREAM = { - on_enter: () => { - console.log('STATE_NEW_STREAM'); - originalDataStream.pause(); - originalDataStream.off('data', original_stream_on_data); - newDataStream.resume(); - }, - }; - - const STATE_END = { - on_enter: () => { - console.log('STATE_END'); - passThrough.end(); - }, - }; - - const STATE_CONTINUE = { - on_enter: () => { - console.log('STATE_CONTINUE'); - if ( defer_buffer.length > 0 ) { - const remaining_replacement = replace_length - replaced_bytes; - if ( replaced_bytes < replace_length ) { - if ( defer_buffer.length <= remaining_replacement ) { - console.log('skipping deferred', defer_buffer.toString()); - replaced_bytes += defer_buffer.length; - defer_buffer = Buffer.alloc(0); - } else { - console.log('skipping deferred', defer_buffer.slice(0, remaining_replacement).toString()); - defer_buffer = defer_buffer.slice(remaining_replacement); - replaced_bytes += remaining_replacement; - } - } - console.log('pushing deferred:', defer_buffer.toString()); - passThrough.push(defer_buffer); - } - // originalDataStream.pipe(passThrough); - originalDataStream.on('data', original_stream_on_data); - originalDataStream.resume(); - }, - }; - - function original_stream_on_data (chunk) { - console.log('original stream data', chunk.length, implied.state); - console.log('received from original:', chunk.toString()); - - if ( implied.state === STATE_NEW_STREAM ) { - console.warn('original stream is not paused'); - defer_buffer = Buffer.concat([defer_buffer, chunk]); - return; - } - - if ( - implied.state === STATE_ORIGINAL_STREAM && - chunk.length >= remaining - ) { - defer_buffer = chunk.slice(remaining); - console.log('deferred:', defer_buffer.toString()); - chunk = chunk.slice(0, remaining); - } - - if ( - implied.state === STATE_CONTINUE && - replaced_bytes < replace_length - ) { - const remaining_replacement = replace_length - replaced_bytes; - if ( chunk.length <= remaining_replacement ) { - console.log('skipping chunk', chunk.toString()); - replaced_bytes += chunk.length; - return; // skip the chunk - } - console.log('skipping part of chunk', chunk.slice(0, remaining_replacement).toString()); - chunk = chunk.slice(remaining_replacement); - - // `+= remaining_replacement` and `= replace_length` are equivalent - // at this point. - replaced_bytes += remaining_replacement; - } - - remaining -= chunk.length; - console.log('pushing from org stream:', chunk.toString()); - passThrough.push(chunk); - implied.state; - }; - - let last_state = null; - implied = { - get state () { - const state = - remaining > 0 ? STATE_ORIGINAL_STREAM : - new_end && org_end ? STATE_END : - new_end ? STATE_CONTINUE : - STATE_NEW_STREAM ; - // (comment to reset indentation) - if ( state !== last_state ) { - last_state = state; - if ( state.on_enter ) state.on_enter(); - } - return state; - }, - }; - - implied.state; - - originalDataStream.on('data', original_stream_on_data); - originalDataStream.on('end', () => { - console.log('original stream end'); - org_end = true; - implied.state; - }); - - newDataStream.on('data', chunk => { - console.log('new stream data', chunk.toString()); - - if ( implied.state === STATE_NEW_STREAM ) { - console.log('pushing from new stream', chunk.toString()); - passThrough.push(chunk); - return; - } - - console.warn('new stream is not paused'); - new_stream_early_buffer = Buffer.concat([new_stream_early_buffer, chunk]); - }); - newDataStream.on('end', () => { - console.log('new stream end', implied.state); - - new_end = true; - implied.state; - }); - - return passThrough; -}; - -class ProgressReportingStream extends Transform { - constructor (options, { total, progress_callback }) { - super(options); - this.total = total; - this.loaded = 0; - this.progress_callback = progress_callback; - } - - _transform (chunk, encoding, callback) { - this.loaded += chunk.length; - this.progress_callback({ - loaded: this.loaded, - uploaded: this.loaded, - total: this.total, - }); - this.push(chunk); - callback(); - } -} - -const progress_stream = (source, { total, progress_callback }) => { - const stream = new ProgressReportingStream({}, { total, progress_callback }); - source.pipe(stream); - return stream; -}; - -class SizeLimitingStream extends Transform { - constructor (options, { limit }) { - super(options); - this.limit = limit; - this.loaded = 0; - } - - _transform (chunk, encoding, callback) { - this.loaded += chunk.length; - if ( this.loaded > this.limit ) { - const excess = this.loaded - this.limit; - chunk = chunk.slice(0, chunk.length - excess); - } - this.push(chunk); - if ( this.loaded >= this.limit ) { - this.end(); - } - callback(); - } -} - -const size_limit_stream = (source, { limit }) => { - const stream = new SizeLimitingStream({}, { limit }); - source.pipe(stream); - return stream; -}; - -class SizeMeasuringStream extends Transform { - constructor (options, probe) { - super(options); - this.probe = probe; - this.loaded = 0; - } - - _transform (chunk, encoding, callback) { - this.loaded += chunk.length; - this.probe.amount = this.loaded; - this.push(chunk); - callback(); - } -} - -/** - * Pass in a source stream and a probe object. The source stream you pass - * will be the return value for chaining stream transforms/controllers. - * The probe object will have the property `probe.amount` set to a number - * of bytes consumed so far each time a chunk is read from the stream. When - * the stream is consumed fully `probe.amount` will contain the total number - * of bytes read. - * @param {*} source - source stream - * @param {*} probe - probe object with `amount` property (you make this) - * @returns source - */ -const size_measure_stream = (source, probe = {}) => { - const stream = new SizeMeasuringStream({}, probe); - source.pipe(stream); - return stream; -}; - -class StuckDetectorStream extends Transform { - constructor (options, { - timeout, - on_stuck, - on_unstuck, - }) { - super(options); - this.timeout = timeout; - this.stuck_ = false; - this.on_stuck = on_stuck; - this.on_unstuck = on_unstuck; - this.last_chunk_time = Date.now(); - - this._start_timer(); - } - - _start_timer () { - if ( this.timer ) clearTimeout(this.timer); - this.timer = setTimeout(() => { - if ( this.stuck_ ) return; - this.stuck_ = true; - this.on_stuck(); - }, this.timeout); - } - - _transform (chunk, encoding, callback) { - if ( this.stuck_ ) { - this.stuck_ = false; - this.on_unstuck(); - } - this._start_timer(); - this.push(chunk); - callback(); - } - - _flush (callback) { - clearTimeout(this.timer); - callback(); - } -} - -const stuck_detector_stream = (source, { - timeout, - on_stuck, - on_unstuck, -}) => { - const stream = new StuckDetectorStream({}, { - timeout, - on_stuck, - on_unstuck, - }); - source.pipe(stream); - return stream; -}; - -const string_to_stream = (str, chunk_size) => { - const s = new Readable(); - s._read = () => { - }; // redundant? see update below - // split string into chunks - const chunks = []; - for ( let i = 0; i < str.length; i += chunk_size ) { - chunks.push(str.slice(i, Math.min(i + chunk_size, str.length))); - } - // push each chunk onto the readable stream - chunks.forEach((chunk) => { - s.push(chunk); - }); - s.push(null); - return s; -}; - -async function* chunk_stream ( - stream, - chunk_size = 1024 * 1024 * 5, - expected_chunk_time, -) { - let buffer = Buffer.alloc(chunk_size); - let offset = 0; - - const chunk_time_ewma = expected_chunk_time !== undefined - ? expected_chunk_time - : null; - - for await ( const chunk of stream ) { - if ( globalThis.average_chunk_size ) { - globalThis.average_chunk_size.put(chunk.length); - } - let remaining = chunk_size - offset; - let amount = Math.min(remaining, chunk.length); - - chunk.copy(buffer, offset, 0, amount); - offset += amount; - - while ( offset >= chunk_size ) { - yield buffer; - - buffer = Buffer.alloc(chunk_size); - offset = 0; - - if ( amount < chunk.length ) { - const leftover = chunk.length - amount; - const next_amount = Math.min(leftover, chunk_size); - chunk.copy(buffer, offset, amount, amount + next_amount); - offset += next_amount; - amount += next_amount; - } - } - - if ( chunk_time_ewma !== null ) { - const chunk_time = chunk_time_ewma.get(); - const sleep_time = (chunk.length / chunk_size) * chunk_time / 2; - await new Promise(resolve => setTimeout(resolve, sleep_time)); - } - } - - if ( offset > 0 ) { - yield buffer.subarray(0, offset); // Yield remaining chunk if it's not empty. - } -} - -const stream_to_buffer = async (stream) => { - const chunks = []; - for await ( const chunk of stream ) { - chunks.push(chunk); - } - return Buffer.concat(chunks); -}; - -const buffer_to_stream = (buffer) => { - const stream = new Readable(); - stream.push(buffer); - stream.push(null); - return stream; -}; - -const hashing_stream = (source) => { - const hash = crypto.createHash('sha256'); - const hashPromise = new TeePromise(); - - const stream = new Transform({ - transform (chunk, encoding, callback) { - hash.update(chunk); - this.push(chunk); - callback(); - }, - // This behaviour used to be on `source.on('end', ...)`; it is assumed - // that the 'end' event caused a race condition where `hash.update` was - // called after `hash.digest` when the server was under sufficient load. - // Using the `flush` callback on Transform should avoid this issue. - flush (callback) { - hashPromise.resolve(hash.digest('hex')); - callback(); - }, - }); - - source.pipe(stream); - - source.on('error', (err) => { - stream.destroy(err); - hashPromise.reject(err); - }); - - stream.on('error', (err) => { - hashPromise.reject(err); - }); - - return { - stream, - hashPromise, - }; -}; - -module.exports = { - StreamBuffer, - stream_to_the_void, - pausing_tee, - logging_stream, - offset_write_stream, - progress_stream, - size_limit_stream, - size_measure_stream, - stuck_detector_stream, - string_to_stream, - chunk_stream, - stream_to_buffer, - buffer_to_stream, - hashing_stream, -}; diff --git a/src/backend/src/util/structutil.bench.js b/src/backend/src/util/structutil.bench.js deleted file mode 100644 index 2980a4e4b..000000000 --- a/src/backend/src/util/structutil.bench.js +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import { bench, describe } from 'vitest'; -import { apply_keys, cart_product } from './structutil.js'; - -describe('cart_product - Small inputs', () => { - bench('2 keys, 2 values each', () => { - cart_product({ - a: [1, 2], - b: ['x', 'y'], - }); - }); - - bench('3 keys, 2 values each', () => { - cart_product({ - a: [1, 2], - b: ['x', 'y'], - c: [true, false], - }); - }); - - bench('2 keys, 3 values each', () => { - cart_product({ - a: [1, 2, 3], - b: ['x', 'y', 'z'], - }); - }); -}); - -describe('cart_product - Medium inputs', () => { - bench('4 keys, 2 values each (16 combinations)', () => { - cart_product({ - a: [1, 2], - b: [3, 4], - c: [5, 6], - d: [7, 8], - }); - }); - - bench('3 keys, 3 values each (27 combinations)', () => { - cart_product({ - a: [1, 2, 3], - b: [4, 5, 6], - c: [7, 8, 9], - }); - }); - - bench('5 keys, 2 values each (32 combinations)', () => { - cart_product({ - a: [1, 2], - b: [3, 4], - c: [5, 6], - d: [7, 8], - e: [9, 10], - }); - }); -}); - -describe('cart_product - Large inputs', () => { - bench('3 keys, 5 values each (125 combinations)', () => { - cart_product({ - a: [1, 2, 3, 4, 5], - b: [6, 7, 8, 9, 10], - c: [11, 12, 13, 14, 15], - }); - }); - - bench('4 keys, 4 values each (256 combinations)', () => { - cart_product({ - a: [1, 2, 3, 4], - b: [5, 6, 7, 8], - c: [9, 10, 11, 12], - d: [13, 14, 15, 16], - }); - }); - - bench('6 keys, 2 values each (64 combinations)', () => { - cart_product({ - a: [1, 2], - b: [3, 4], - c: [5, 6], - d: [7, 8], - e: [9, 10], - f: [11, 12], - }); - }); -}); - -describe('cart_product - Single values', () => { - bench('3 keys, 1 value each (1 combination)', () => { - cart_product({ - a: 1, - b: 2, - c: 3, - }); - }); - - bench('mixed single and array values', () => { - cart_product({ - a: 1, - b: [2, 3], - c: 4, - d: [5, 6], - }); - }); -}); - -describe('cart_product - Edge cases', () => { - bench('empty object', () => { - cart_product({}); - }); - - bench('single key with array', () => { - cart_product({ - only: [1, 2, 3, 4, 5], - }); - }); - - bench('many keys with single values', () => { - cart_product({ - a: 1, - b: 2, - c: 3, - d: 4, - e: 5, - f: 6, - g: 7, - h: 8, - i: 9, - j: 10, - }); - }); -}); - -describe('apply_keys - Basic operations', () => { - const keys = ['a', 'b', 'c']; - - bench('apply to single entry', () => { - apply_keys(keys, [1, 2, 3]); - }); - - bench('apply to 5 entries', () => { - apply_keys(keys, - [1, 2, 3], - [4, 5, 6], - [7, 8, 9], - [10, 11, 12], - [13, 14, 15]); - }); - - bench('apply to 10 entries', () => { - const entries = []; - for ( let i = 0; i < 10; i++ ) { - entries.push([i * 3, i * 3 + 1, i * 3 + 2]); - } - apply_keys(keys, ...entries); - }); -}); - -describe('apply_keys - Varying key counts', () => { - bench('2 keys', () => { - apply_keys(['a', 'b'], [1, 2], [3, 4], [5, 6]); - }); - - bench('5 keys', () => { - apply_keys(['a', 'b', 'c', 'd', 'e'], - [1, 2, 3, 4, 5], - [6, 7, 8, 9, 10]); - }); - - bench('10 keys', () => { - const keys = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']; - const entry = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - apply_keys(keys, entry, entry, entry); - }); -}); - -describe('Combined cart_product + apply_keys workflow', () => { - bench('generate and label small product', () => { - const product = cart_product({ - size: ['small', 'medium', 'large'], - color: ['red', 'blue'], - }); - apply_keys(['size', 'color'], ...product); - }); - - bench('generate and label medium product', () => { - const product = cart_product({ - a: [1, 2, 3], - b: [4, 5, 6], - c: [7, 8, 9], - }); - apply_keys(['a', 'b', 'c'], ...product); - }); -}); - -describe('Real-world configuration generation', () => { - bench('test matrix generation (browser x OS)', () => { - const matrix = cart_product({ - browser: ['chrome', 'firefox', 'safari'], - os: ['windows', 'macos', 'linux'], - }); - apply_keys(['browser', 'os'], ...matrix); - }); - - bench('feature flag combinations', () => { - cart_product({ - featureA: [true, false], - featureB: [true, false], - featureC: [true, false], - featureD: [true, false], - }); - }); - - bench('API endpoint parameter combinations', () => { - const combinations = cart_product({ - method: ['GET', 'POST'], - auth: ['none', 'token', 'session'], - format: ['json', 'xml'], - }); - apply_keys(['method', 'auth', 'format'], ...combinations); - }); -}); diff --git a/src/backend/src/util/structutil.js b/src/backend/src/util/structutil.js deleted file mode 100644 index 368298e4e..000000000 --- a/src/backend/src/util/structutil.js +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const cart_product = (obj) => { - // Get array of keys - let keys = Object.keys(obj); - - // Generate the Cartesian Product - return keys.reduce((acc, key) => { - let appendArrays = Array.isArray(obj[key]) ? obj[key] : [obj[key]]; - - let newAcc = []; - acc.forEach(arr => { - appendArrays.forEach(item => { - newAcc.push([...arr, item]); - }); - }); - - return newAcc; - }, [[]]); // start with the "empty product" -}; - -const apply_keys = (keys, ...entries) => { - const l = []; - for ( const entry of entries ) { - const o = {}; - for ( let i = 0 ; i < keys.length ; i++ ) { - o[keys[i]] = entry[i]; - } - l.push(o); - } - return l; -}; - -module.exports = { - cart_product, - apply_keys, -}; diff --git a/src/backend/src/util/urlutil.js b/src/backend/src/util/urlutil.js deleted file mode 100644 index a5ad98696..000000000 --- a/src/backend/src/util/urlutil.js +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const origin_from_url = url => { - try { - const parsedUrl = new URL(url); - // Origin is protocol + hostname + port - return `${parsedUrl.protocol}//${parsedUrl.hostname}${parsedUrl.port ? `:${parsedUrl.port}` : ''}`; - } catch ( error ) { - console.error('Invalid URL:', error.message); - return null; - } -}; - -module.exports = { - origin_from_url, -}; diff --git a/src/backend/src/util/uuidfpe.bench.js b/src/backend/src/util/uuidfpe.bench.js deleted file mode 100644 index 4c44904a7..000000000 --- a/src/backend/src/util/uuidfpe.bench.js +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import crypto from 'crypto'; -import { bench, describe } from 'vitest'; -import { UUIDFPE } from './uuidfpe.js'; - -// Test data -const testKey = Buffer.from('0123456789abcdef'); // 16-byte key -const testUuid = '550e8400-e29b-41d4-a716-446655440000'; -const fpe = new UUIDFPE(testKey); -const encryptedUuid = fpe.encrypt(testUuid); - -// Pre-generate UUIDs for batch tests -const uuids = []; -for ( let i = 0; i < 100; i++ ) { - uuids.push(crypto.randomUUID()); -} - -describe('UUIDFPE - Construction', () => { - bench('create UUIDFPE instance', () => { - new UUIDFPE(testKey); - }); - - bench('create with random key', () => { - const key = crypto.randomBytes(16); - new UUIDFPE(key); - }); -}); - -describe('UUIDFPE - Static utilities', () => { - bench('uuidToBuffer', () => { - UUIDFPE.uuidToBuffer(testUuid); - }); - - bench('bufferToUuid', () => { - const buffer = Buffer.from('550e8400e29b41d4a716446655440000', 'hex'); - UUIDFPE.bufferToUuid(buffer); - }); - - bench('round-trip buffer conversion', () => { - const buffer = UUIDFPE.uuidToBuffer(testUuid); - UUIDFPE.bufferToUuid(buffer); - }); -}); - -describe('UUIDFPE - Encryption', () => { - bench('encrypt single UUID', () => { - fpe.encrypt(testUuid); - }); - - bench('encrypt 10 UUIDs', () => { - for ( let i = 0; i < 10; i++ ) { - fpe.encrypt(uuids[i]); - } - }); - - bench('encrypt 100 UUIDs', () => { - for ( const uuid of uuids ) { - fpe.encrypt(uuid); - } - }); -}); - -describe('UUIDFPE - Decryption', () => { - bench('decrypt single UUID', () => { - fpe.decrypt(encryptedUuid); - }); - - // Pre-encrypt for decryption benchmarks - const encryptedUuids = uuids.map(uuid => fpe.encrypt(uuid)); - - bench('decrypt 10 UUIDs', () => { - for ( let i = 0; i < 10; i++ ) { - fpe.decrypt(encryptedUuids[i]); - } - }); - - bench('decrypt 100 UUIDs', () => { - for ( const encrypted of encryptedUuids ) { - fpe.decrypt(encrypted); - } - }); -}); - -describe('UUIDFPE - Round-trip', () => { - bench('encrypt then decrypt (single)', () => { - const encrypted = fpe.encrypt(testUuid); - fpe.decrypt(encrypted); - }); - - bench('encrypt then decrypt (10 UUIDs)', () => { - for ( let i = 0; i < 10; i++ ) { - const encrypted = fpe.encrypt(uuids[i]); - fpe.decrypt(encrypted); - } - }); -}); - -describe('UUIDFPE - Comparison with alternatives', () => { - bench('UUIDFPE encrypt', () => { - fpe.encrypt(testUuid); - }); - - bench('native crypto.randomUUID (for comparison)', () => { - crypto.randomUUID(); - }); - - bench('SHA256 hash of UUID (for comparison)', () => { - crypto.createHash('sha256').update(testUuid).digest('hex'); - }); -}); - -describe('UUIDFPE - Different keys', () => { - const keys = []; - for ( let i = 0; i < 10; i++ ) { - keys.push(crypto.randomBytes(16)); - } - - bench('encrypt with 10 different keys', () => { - for ( const key of keys ) { - const instance = new UUIDFPE(key); - instance.encrypt(testUuid); - } - }); -}); - -describe('Real-world patterns', () => { - bench('obfuscate user ID', () => { - // Simulate hiding internal UUID from external API - fpe.encrypt(testUuid); - }); - - bench('de-obfuscate incoming ID', () => { - // Simulate receiving obfuscated ID and decrypting - fpe.decrypt(encryptedUuid); - }); - - bench('API response transformation (10 items)', () => { - // Simulate transforming a list of items with obfuscated IDs - uuids.slice(0, 10).map(uuid => ({ - id: fpe.encrypt(uuid), - name: 'item', - })); - }); -}); diff --git a/src/backend/src/util/uuidfpe.js b/src/backend/src/util/uuidfpe.js deleted file mode 100644 index 6f143b60a..000000000 --- a/src/backend/src/util/uuidfpe.js +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const crypto = require('crypto'); - -class UUIDFPE { - static ALGORITHM = 'aes-128-ecb'; - - constructor (key) { - if ( !key || key.length !== 16 ) { - throw new Error('Key must be a 16-byte Buffer.'); - } - this.key = key; - } - - static uuidToBuffer (uuidStr) { - const hexStr = uuidStr.replace(/-/g, ''); - return Buffer.from(hexStr, 'hex'); - } - static bufferToUuid (buffer) { - const hexStr = buffer.toString('hex'); - return [ - hexStr.substring(0, 8), - hexStr.substring(8, 12), - hexStr.substring(12, 16), - hexStr.substring(16, 20), - hexStr.substring(20), - ].join('-'); - } - - encrypt (uuidStr) { - const plaintext = this.constructor.uuidToBuffer(uuidStr); - - const cipher = crypto.createCipheriv(this.constructor.ALGORITHM, - this.key, - null); - cipher.setAutoPadding(false); - - const encrypted = Buffer.concat([ - cipher.update(plaintext), - cipher.final(), - ]); - return this.constructor.bufferToUuid(encrypted); - } - - decrypt (encryptedUuidStr) { - const encrypted = this.constructor.uuidToBuffer(encryptedUuidStr); - const decipher = crypto.createDecipheriv(this.constructor.ALGORITHM, - this.key, - null); - decipher.setAutoPadding(false); - - const decrypted = Buffer.concat([decipher.update(encrypted), decipher.final()]); - return this.constructor.bufferToUuid(decrypted); - } -} - -module.exports = { - UUIDFPE, -}; diff --git a/src/backend/src/util/validutil.js b/src/backend/src/util/validutil.js deleted file mode 100644 index e90bc7779..000000000 --- a/src/backend/src/util/validutil.js +++ /dev/null @@ -1,73 +0,0 @@ -const APIError = require('../api/APIError'); - -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const valid_file_size = v => { - v = Number(v); - if ( ! Number.isInteger(v) ) { - return { ok: false, v }; - } - if ( v < 0 ) { - return { ok: false, v }; - } - return { ok: true, v }; -}; - -const validate_fields = (fields, values) => { - // First, check for missing fields (undefined) - const missing_fields = Object.keys(fields).filter(field => !fields[field].optional && values[field] === undefined); - if ( missing_fields.length > 0 ) { - throw APIError.create('fields_missing', null, { keys: missing_fields }); - } - - // Next, check for invalid fields (based on ) - const invalid_fields = Object.entries(fields).filter(([field, field_def]) => { - if ( field_def.type === 'string' ) { - return typeof values[field] !== 'string'; - } - if ( field_def.type === 'number' ) { - return typeof values[field] !== 'number'; - } - }); - if ( invalid_fields.length > 0 ) { - throw APIError.create('fields_invalid', null, { - errors: invalid_fields.map(([field, field_def]) => ({ - key: field, - expected: field_def.type, - got: typeof values[field], - })), - }); - } -}; - -const validate_nonEmpty_string = value => { - if ( typeof value !== 'string' ) { - return false; - } - if ( value.length === 0 ) { - return false; - } - return true; -}; - -module.exports = { - valid_file_size, - validate_nonEmpty_string, - validate_fields, -}; diff --git a/src/backend/src/util/validutil.test.js b/src/backend/src/util/validutil.test.js deleted file mode 100644 index 8223638be..000000000 --- a/src/backend/src/util/validutil.test.js +++ /dev/null @@ -1,248 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -const { valid_file_size, validate_fields } = require('./validutil'); -const APIError = require('../api/APIError'); - -describe('valid_file_size', () => { - it('returns ok for positive integer', () => { - const result = valid_file_size(100); - expect(result).toEqual({ ok: true, v: 100 }); - }); - - it('returns ok for zero', () => { - const result = valid_file_size(0); - expect(result).toEqual({ ok: true, v: 0 }); - }); - - it('converts string to number and validates', () => { - const result = valid_file_size('42'); - expect(result).toEqual({ ok: true, v: 42 }); - }); - - it('returns not ok for negative number', () => { - const result = valid_file_size(-1); - expect(result).toEqual({ ok: false, v: -1 }); - }); - - it('returns not ok for floating point number', () => { - const result = valid_file_size(3.14); - expect(result).toEqual({ ok: false, v: 3.14 }); - }); - - it('returns not ok for NaN', () => { - const result = valid_file_size(NaN); - expect(result.ok).toBe(false); - expect(Number.isNaN(result.v)).toBe(true); - }); - - it('returns not ok for non-numeric string', () => { - const result = valid_file_size('abc'); - expect(result.ok).toBe(false); - expect(Number.isNaN(result.v)).toBe(true); - }); - - it('returns not ok for Infinity', () => { - const result = valid_file_size(Infinity); - expect(result).toEqual({ ok: false, v: Infinity }); - }); -}); - -describe('validate_fields', () => { - describe('missing fields', () => { - it('throws fields_missing error when required field is undefined', () => { - const fields = { - name: { type: 'string' }, - }; - const values = {}; - - expect(() => validate_fields(fields, values)) - .toThrow(APIError); - }); - - it('throws with correct keys for multiple missing fields', () => { - const fields = { - name: { type: 'string' }, - age: { type: 'number' }, - }; - const values = {}; - - try { - validate_fields(fields, values); - expect.fail('Expected error to be thrown'); - } catch (e) { - expect(e).toBeInstanceOf(APIError); - expect(e.fields.keys).toContain('name'); - expect(e.fields.keys).toContain('age'); - } - }); - - it('does not throw for optional undefined fields when they have no type check', () => { - const fields = { - name: { type: 'string' }, - nickname: { optional: true }, // No type defined - }; - const values = { name: 'John' }; - - expect(() => validate_fields(fields, values)).not.toThrow(); - }); - - // Note: Current implementation validates type even for optional undefined fields - // This test documents that behavior - optional fields must still pass type validation - it('throws for optional undefined fields if type validation is defined', () => { - const fields = { - name: { type: 'string' }, - nickname: { type: 'string', optional: true }, - }; - const values = { name: 'John' }; - - // Current behavior: type validation runs on optional undefined fields - expect(() => validate_fields(fields, values)).toThrow(APIError); - }); - - it('accepts optional fields when provided with correct type', () => { - const fields = { - name: { type: 'string' }, - nickname: { type: 'string', optional: true }, - }; - const values = { name: 'John', nickname: 'Johnny' }; - - expect(() => validate_fields(fields, values)).not.toThrow(); - }); - - it('does not throw when all required fields are present', () => { - const fields = { - name: { type: 'string' }, - age: { type: 'number' }, - }; - const values = { name: 'John', age: 25 }; - - expect(() => validate_fields(fields, values)).not.toThrow(); - }); - }); - - describe('invalid fields', () => { - it('throws fields_invalid error when string field receives number', () => { - const fields = { - name: { type: 'string' }, - }; - const values = { name: 123 }; - - expect(() => validate_fields(fields, values)) - .toThrow(APIError); - }); - - it('throws fields_invalid error when number field receives string', () => { - const fields = { - age: { type: 'number' }, - }; - const values = { age: '25' }; - - expect(() => validate_fields(fields, values)) - .toThrow(APIError); - }); - - it('throws with correct error details for invalid fields', () => { - const fields = { - age: { type: 'number' }, - }; - const values = { age: 'not a number' }; - - try { - validate_fields(fields, values); - expect.fail('Expected error to be thrown'); - } catch (e) { - expect(e).toBeInstanceOf(APIError); - expect(e.fields.errors).toBeDefined(); - expect(e.fields.errors[0].key).toBe('age'); - expect(e.fields.errors[0].expected).toBe('number'); - expect(e.fields.errors[0].got).toBe('string'); - } - }); - - it('validates multiple fields and reports all invalid ones', () => { - const fields = { - name: { type: 'string' }, - age: { type: 'number' }, - }; - const values = { name: 42, age: 'twenty-five' }; - - try { - validate_fields(fields, values); - expect.fail('Expected error to be thrown'); - } catch (e) { - expect(e).toBeInstanceOf(APIError); - expect(e.fields.errors.length).toBe(2); - } - }); - }); - - describe('valid inputs', () => { - it('accepts valid string fields', () => { - const fields = { - name: { type: 'string' }, - }; - const values = { name: 'John' }; - - expect(() => validate_fields(fields, values)).not.toThrow(); - }); - - it('accepts valid number fields', () => { - const fields = { - age: { type: 'number' }, - }; - const values = { age: 25 }; - - expect(() => validate_fields(fields, values)).not.toThrow(); - }); - - it('accepts mixed valid string and number fields', () => { - const fields = { - name: { type: 'string' }, - age: { type: 'number' }, - }; - const values = { name: 'John', age: 25 }; - - expect(() => validate_fields(fields, values)).not.toThrow(); - }); - - it('accepts empty string as valid string', () => { - const fields = { - name: { type: 'string' }, - }; - const values = { name: '' }; - - expect(() => validate_fields(fields, values)).not.toThrow(); - }); - - it('accepts zero as valid number', () => { - const fields = { - count: { type: 'number' }, - }; - const values = { count: 0 }; - - expect(() => validate_fields(fields, values)).not.toThrow(); - }); - }); - - describe('priority of errors', () => { - it('throws fields_missing before checking invalid fields', () => { - const fields = { - name: { type: 'string' }, - age: { type: 'number' }, - }; - // name is missing, age is invalid - const values = { age: 'not a number' }; - - try { - validate_fields(fields, values); - expect.fail('Expected error to be thrown'); - } catch (e) { - expect(e).toBeInstanceOf(APIError); - // Should throw fields_missing, not fields_invalid - expect(e.fields.keys).toBeDefined(); - expect(e.fields.keys).toContain('name'); - } - }); - }); -}); - diff --git a/src/backend/src/util/versionutil.js b/src/backend/src/util/versionutil.js deleted file mode 100644 index f8e1819b9..000000000 --- a/src/backend/src/util/versionutil.js +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Select the object with the highest version. - * Objects are of the form: - * { version: '1.2.0' } - * - * Semver is assumed. - * - * @param {*} objects - */ -const find_highest_version = (objects) => { - let highest = [0, 0, 0]; - let highest_obj = null; - - for ( const obj of objects ) { - const parts = obj.version.split('.'); - for ( let i = 0; i < 3; i++ ) { - const part = parseInt(parts[i]); - if ( part > highest[i] ) { - highest = parts; - highest_obj = obj; - break; - } else if ( part < highest[i] ) { - break; - }1; - } - } - - return highest_obj; -}; - -module.exports = { - find_highest_version, -}; diff --git a/src/backend/src/util/versionutil.test.js b/src/backend/src/util/versionutil.test.js deleted file mode 100644 index 283a0155a..000000000 --- a/src/backend/src/util/versionutil.test.js +++ /dev/null @@ -1,18 +0,0 @@ -import { describe, it, expect } from 'vitest'; - -describe('versionutil', () => { - it('works', () => { - const objects = [ - { version: '1.2.0' }, - { version: '3.0.2' }, - { version: '1.2.1' }, - { version: '1.2.0' }, - { version: '3.1.0', h: true }, - { version: '1.2.2' }, - ]; - - const { find_highest_version } = require('./versionutil'); - const highest_object = find_highest_version(objects); - expect(highest_object).toEqual({ version: '3.1.0', h: true }); - }); -}); \ No newline at end of file diff --git a/src/backend/src/util/workutil.js b/src/backend/src/util/workutil.js deleted file mode 100644 index 4852b28fa..000000000 --- a/src/backend/src/util/workutil.js +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -class WorkList { - constructor () { - this.locked_ = false; - this.items = []; - } - - list () { - return [...this.items]; - } - - clear_invalid () { - const new_items = []; - for ( const item of this.items ) { - if ( item.invalid ) continue; - new_items.push(item); - } - this.items = new_items; - } - - push (item) { - if ( this.locked_ ) { - throw new Error('work items were already locked in; what are you doing?'); - } - this.items.push(item); - } - - lockin () { - this.locked_ = true; - } -} - -module.exports = { - WorkList, -}; diff --git a/src/backend/src/validation.js b/src/backend/src/validation.js deleted file mode 100644 index cec37987c..000000000 --- a/src/backend/src/validation.js +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -export { is_valid_path } from './deprecated/filesystem/validation.js'; - -export const is_valid_uuid = (uuid) => { - let s = `${ uuid }`; - s = s.match(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}$/i); - return !!s; -}; - -export const is_valid_uuid4 = (uuid) => { - return is_valid_uuid(uuid); -}; - -export const is_specifically_uuidv4 = (uuid) => { - let s = `${ uuid }`; - - s = s.match(/^[0-9A-F]{8}-[0-9A-F]{4}-[4][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i); - if ( ! s ) { - return false; - } - return true; -}; - -export const is_valid_url = (url) => { - let s = `${ url }`; - - try { - new URL(s); - return true; - } catch (e) { - return false; - } -}; \ No newline at end of file diff --git a/src/backend/stores/app/AppStore.js b/src/backend/stores/app/AppStore.js new file mode 100644 index 000000000..dab204ad6 --- /dev/null +++ b/src/backend/stores/app/AppStore.js @@ -0,0 +1,1261 @@ +import { v4 as uuidv4 } from 'uuid'; +import { PuterStore } from '../types'; + +/** + * Persistence + cache for the `apps` table. + * + * CRUD over app rows with multi-key caching (uid, name, id). No + * validation or permission logic — those live in AppDriver. Callers + * that need enforcement should go through the driver; callers that + * just need data (internal services) can hit the store directly. + */ + +const CACHE_KEY_PREFIX = 'apps'; +const CACHE_TTL_SECONDS = 24 * 60 * 60; +const LIST_CACHE_KEY_PREFIX = `${CACHE_KEY_PREFIX}:list`; +const LIST_CACHE_TRACKER_KEY = `${LIST_CACHE_KEY_PREFIX}:keys`; +const LIST_CACHE_TTL_SECONDS = 15 * 60; +const FILETYPE_CACHE_KEY_PREFIX = 'apps:by-filetype'; +const FILETYPE_CACHE_TTL_SECONDS = 60; +const APP_ID_PROPERTIES = ['id', 'uid', 'name']; +// Cap on placeholders per `IN (?, ?, …)` query. SQLite's default parameter +// limit is 999; staying well under that keeps `getByIds` portable across +// backends without splitting the cap by driver. +const BULK_QUERY_CHUNK_SIZE = 200; + +// Top-level all-time open/user counts: hot path, slow to compute, refreshed +// periodically by one instance and read via MGET on every app list/read. +const STATS_CACHE_TTL_SECONDS = 30 * 60; +const STATS_REFRESH_INTERVAL_MS = 15 * 60 * 1000; +const STATS_REFRESH_LOCK_KEY = 'appStatsLastRefresh'; + +// Period helpers for detailed/grouped stats. Ported from v1 +// AppInformationService — queries go straight to ClickHouse/MySQL on demand +// (no cache) since they're UI-driven and rarely repeated with identical args. +const MYSQL_DATE_FORMATS = { + hour: '%Y-%m-%d %H:00:00', + day: '%Y-%m-%d', + week: '%Y-%U', + month: '%Y-%m', + year: '%Y', +}; +const CLICKHOUSE_GROUP_BY_FORMATS = { + hour: 'toStartOfHour(fromUnixTimestamp(ts))', + day: 'toStartOfDay(fromUnixTimestamp(ts))', + week: 'toStartOfWeek(fromUnixTimestamp(ts))', + month: 'toStartOfMonth(fromUnixTimestamp(ts))', + year: 'toStartOfYear(fromUnixTimestamp(ts))', +}; + +// Columns that may not be set through `create` / `update` from user input +// or from any patch map forwarded into the store. Defence-in-depth against +// future callers (admin routes, extensions, new REST endpoints) that might +// forward `req.body` straight to `update`: the driver's #validateInput is +// an allow-list upstream, but the store is the last line before SQL. +// +// Categories: +// - identity: `id`, `uid` +// - system timestamps / admin review: `timestamp`, `last_review` +// - admin-only flags: `approved_for_*`, `godmode` +// - ownership: `owner_user_id`, `app_owner` — set via `create`'s second +// argument, never from a patch map. Re-assigning via update would +// hand an app to another user. +// - access gates: `protected`, `is_private` — flipping these silently +// bypasses #canReadApp / leaks a private app's index_url via +// #toClient. If an admin flow ever needs to toggle these it must +// call a purpose-built method, not a generic patch. +// +// `index_url` is intentionally NOT here — it's legitimately user-editable +// (the whole point of updating an app). XSS-unsafe schemes are rejected +// upstream by validateUrl's scheme allow-list. +const READ_ONLY_COLUMNS = new Set([ + 'id', + 'uid', + 'timestamp', + 'last_review', + 'approved_for_listing', + 'approved_for_opening_items', + 'approved_for_incentive_program', + 'godmode', + 'owner_user_id', + 'app_owner', + 'protected', + 'is_private', +]); + +export class AppStore extends PuterStore { + #appStatsInterval; + // ── Reads ──────────────────────────────────────────────────────── + + async getByUid(uid) { + return this.#getByProperty('uid', uid); + } + async getById(id) { + return this.#getByProperty('id', id); + } + async getByName(name) { + return this.#getByProperty('name', name); + } + + /** + * Batched lookup by id. Dedupes input ids, reads cache via a pipelined + * MGET, and resolves remaining misses with a single + * `SELECT … WHERE id IN (…)` per chunk. Use this in place of + * `Promise.all(ids.map(getById))` to avoid one connection per row on + * large id sets. + * + * Missing ids (no DB row) are simply absent from the returned map. + */ + async getByIds(ids) { + const result = new Map(); + const uniqueIds = [ + ...new Set( + (Array.isArray(ids) ? ids : []).filter( + (id) => id !== null && id !== undefined, + ), + ), + ]; + if (uniqueIds.length === 0) return result; + + const missingIds = []; + try { + const pipeline = this.clients.redis.pipeline(); + for (const id of uniqueIds) { + pipeline.get(this.#cacheKey('id', id)); + } + const cacheResults = (await pipeline.exec()) ?? []; + for (let i = 0; i < uniqueIds.length; i++) { + const id = uniqueIds[i]; + const raw = cacheResults[i]?.[1]; + if (typeof raw === 'string') { + try { + result.set(id, JSON.parse(raw)); + continue; + } catch { + // Fall through to DB on any parse failure. + } + } + missingIds.push(id); + } + } catch { + missingIds.push(...uniqueIds); + } + + for ( + let offset = 0; + offset < missingIds.length; + offset += BULK_QUERY_CHUNK_SIZE + ) { + const chunk = missingIds.slice( + offset, + offset + BULK_QUERY_CHUNK_SIZE, + ); + const placeholders = chunk.map(() => '?').join(', '); + const rows = await this.clients.db.read( + `SELECT * FROM \`apps\` WHERE \`id\` IN (${placeholders})`, + chunk, + ); + for (const row of rows) { + const app = this.#normalizeRow(row); + if (!app) continue; + result.set(app.id, app); + this.#writeCache(app).catch(() => {}); + } + } + + return result; + } + + async existsByName(name) { + const rows = await this.clients.db.read( + 'SELECT `id` FROM `apps` WHERE `name` = ? LIMIT 1', + [name], + ); + return rows.length > 0; + } + + async existsByIndexUrl(indexUrl) { + const rows = await this.clients.db.read( + 'SELECT `id` FROM `apps` WHERE `index_url` = ? LIMIT 1', + [indexUrl], + ); + return rows.length > 0; + } + + /** + * Find the oldest app whose `index_url` matches one of `candidates`. + * Used by the driver to detect duplicate puter-hosted index_url rows + * (origin-bootstrap apps + same-owner duplicates) so they can be + * merged on `create` / `update`. Returns the minimal row shape + * needed by the merge path; falsy when nothing matches. + */ + async findByIndexUrlCandidates(candidates, { excludeAppId } = {}) { + if (!Array.isArray(candidates) || candidates.length === 0) return null; + const placeholders = candidates.map(() => '?').join(', '); + const params = [...candidates]; + let sql = `SELECT \`id\`, \`uid\`, \`owner_user_id\`, \`index_url\` FROM \`apps\` WHERE \`index_url\` IN (${placeholders})`; + if (Number.isInteger(excludeAppId) && excludeAppId > 0) { + sql += ' AND `id` != ?'; + params.push(excludeAppId); + } + sql += ' ORDER BY `timestamp` ASC, `id` ASC LIMIT 1'; + const rows = await this.clients.db.read(sql, params); + return rows[0] ?? null; + } + + /** + * Set `owner_user_id` on an app row only if it has no current owner. + * Used by the merge path to claim ownership of an origin-bootstrap + * app row (auto-created with no owner) before merging into it. + * Returns true iff the row was modified. + */ + async claimOwnership(appId, userId) { + if (!Number.isInteger(appId) || appId <= 0) return false; + if (!Number.isInteger(userId) || userId <= 0) return false; + const result = await this.clients.db.write( + 'UPDATE `apps` SET `owner_user_id` = ? WHERE `id` = ? AND `owner_user_id` IS NULL', + [userId, appId], + ); + const affected = (result?.affectedRows ?? result?.changes ?? 0) > 0; + if (affected) { + const fresh = await this.getById(appId); + if (fresh) { + await this.invalidate(fresh); + } + } + return affected; + } + + /** + * List apps with optional filters. Returns raw normalised rows — + * the driver is responsible for permission filtering + icon URL + * resolution when serving to clients. + * + * @param {object} filters + * @param {number} [filters.ownerUserId] — only apps owned by this user + * @param {number} [filters.appOwner] — only apps created by another app + * @param {string} [filters.name] — exact name match + * @param {number} [filters.limit=500] + */ + async list(filters = {}) { + const where = []; + const params = []; + + if (filters.ownerUserId !== undefined) { + where.push('`owner_user_id` = ?'); + params.push(filters.ownerUserId); + } + if (filters.appOwner !== undefined) { + where.push('`app_owner` = ?'); + params.push(filters.appOwner); + } + if (filters.name !== undefined) { + where.push('`name` = ?'); + params.push(filters.name); + } + + const whereClause = where.length ? `WHERE ${where.join(' AND ')}` : ''; + const limit = filters.limit ?? 500; + const cacheKey = this.#listCacheKey(whereClause, params, limit); + + try { + const cached = await this.clients.redis.get(cacheKey); + if (cached) { + const parsed = JSON.parse(cached); + if (Array.isArray(parsed)) { + return parsed.map((r) => this.#normalizeRow(r)); + } + } + } catch { + // Fall through to DB on any cache failure. + } + + const rows = await this.clients.db.read( + `SELECT * FROM \`apps\` ${whereClause} LIMIT ?`, + [...params, limit], + ); + const apps = rows.map((r) => this.#normalizeRow(r)); + + this.#writeListCache(cacheKey, apps).catch(() => {}); + return apps; + } + + // ── Writes ─────────────────────────────────────────────────────── + + /** + * Create a new app row. + * + * `fields` is the post-validation column map from the driver. + * `ownerUserId` and `appOwner` come in as a separate arg rather + * than living on `fields` so user-derived patches can never fake + * ownership — the store's READ_ONLY_COLUMNS filter would strip + * them from `fields` anyway; putting them here makes the + * privileged contract obvious at the call site. + * + * Returns the created app row. + */ + async create(fields, { ownerUserId, appOwner = null } = {}) { + if (typeof ownerUserId !== 'number') { + throw new Error('AppStore.create requires a numeric ownerUserId'); + } + + const uid = `app-${uuidv4()}`; + const allowed = this.#filterEditable(fields); + allowed.owner_user_id = ownerUserId; + if (appOwner !== null && appOwner !== undefined) { + allowed.app_owner = appOwner; + } + + const columns = ['uid', ...Object.keys(allowed)]; + const values = [uid, ...Object.values(allowed)]; + + const placeholders = columns.map(() => '?').join(', '); + const colList = columns.map((c) => `\`${c}\``).join(', '); + + const result = await this.clients.db.write( + `INSERT INTO \`apps\` (${colList}) VALUES (${placeholders})`, + values, + ); + const insertId = result?.insertId; + if (!insertId) + throw new Error('Failed to create app — no insertId returned'); + + const fresh = await this.getById(insertId); + await this.#invalidateListCachesForApps([fresh]); + return fresh; + } + + /** Updates + refreshes cache (local + peers) with the post-update row. */ + async update(appId, patch) { + const allowed = this.#filterEditable(patch); + const keys = Object.keys(allowed); + if (keys.length === 0) return this.getById(appId); + + const before = await this.#readFromDb('id', appId); + const setClause = keys.map((k) => `\`${k}\` = ?`).join(', '); + const values = keys.map((k) => allowed[k]); + + await this.clients.db.write( + `UPDATE \`apps\` SET ${setClause} WHERE \`id\` = ?`, + [...values, appId], + ); + + const fresh = await this.#readFromDb('id', appId); + await this.#invalidateListCachesForApps([before, fresh]); + if (fresh) { + await this.#refreshCache({ ...fresh, ...allowed }); + } + return fresh; + } + + /** + * Delete an app row. Also deletes its filetype associations. + * Invalidates cache. + */ + async delete(appId) { + const app = await this.getById(appId); + if (!app) return false; + + await this.clients.db.write( + 'DELETE FROM `app_filetype_association` WHERE `app_id` = ?', + [appId], + ); + await this.clients.db.write('DELETE FROM `apps` WHERE `id` = ?', [ + appId, + ]); + await this.invalidate(app); + return true; + } + + // ── Filetype associations ──────────────────────────────────────── + + async getFiletypeAssociations(appId) { + const rows = await this.clients.db.read( + 'SELECT `type` FROM `app_filetype_association` WHERE `app_id` = ?', + [appId], + ); + return rows.map((r) => r.type); + } + + /** + * Batch sibling of {@link getFiletypeAssociations} — one query for many + * app ids, returns `Map` (every requested id is + * present, with `[]` for apps that have no associations). + */ + async getFiletypeAssociationsByIds(appIds) { + const ids = Array.isArray(appIds) + ? Array.from( + new Set( + appIds.filter((id) => id !== null && id !== undefined), + ), + ) + : []; + const out = new Map(); + for (const id of ids) out.set(id, []); + if (ids.length === 0) return out; + + const placeholders = ids.map(() => '?').join(','); + const rows = await this.clients.db.read( + `SELECT \`app_id\`, \`type\` FROM \`app_filetype_association\` + WHERE \`app_id\` IN (${placeholders})`, + ids, + ); + for (const row of rows) { + const list = out.get(row.app_id); + if (list) list.push(row.type); + } + return out; + } + + async getAppsByFiletype(extension) { + // Cache-on-read: first request after a miss pays the join, subsequent + // reads inside the TTL window hit redis. `setFiletypeAssociations` + // invalidates the affected extension explicitly so changes show up + // immediately. + const cacheKey = `${FILETYPE_CACHE_KEY_PREFIX}:${extension}`; + try { + const cached = await this.clients.redis.get(cacheKey); + if (cached) { + const parsed = JSON.parse(cached); + if (Array.isArray(parsed)) return parsed; + } + } catch { + // Fall through to DB on any cache failure. + } + + const rows = await this.clients.db.read( + `SELECT a.* FROM \`apps\` a + INNER JOIN \`app_filetype_association\` fa ON fa.\`app_id\` = a.\`id\` + WHERE fa.\`type\` = ?`, + [extension], + ); + const apps = rows.map((r) => this.#normalizeRow(r)); + + this.clients.redis + .set( + cacheKey, + JSON.stringify(apps), + 'EX', + FILETYPE_CACHE_TTL_SECONDS, + ) + .catch(() => { + // Best-effort cache write. + }); + + return apps; + } + + async getRecentAppOpens(userId, { limit = 10 } = {}) { + const rows = await this.clients.db.read( + `SELECT DISTINCT \`app_uid\` FROM \`app_opens\` + WHERE \`user_id\` = ? + GROUP BY \`app_uid\` + ORDER BY MAX(\`_id\`) DESC + LIMIT ${limit}`, + [userId], + ); + return rows.map((r) => r.app_uid); + } + + async setFiletypeAssociations(appId, types) { + // Replace-all semantics. Capture the previous extension set so we + // can drop their cached app lists in addition to the new ones. + const previous = await this.getFiletypeAssociations(appId); + const newTypes = Array.isArray(types) ? types : []; + + // DELETE + multi-row INSERT in one transactional batch — partial + // success would otherwise leave the row's filetype set in a state + // that doesn't match either `previous` or `newTypes`. + const entries = [ + { + statement: + 'DELETE FROM `app_filetype_association` WHERE `app_id` = ?', + values: [appId], + }, + ]; + if (newTypes.length > 0) { + const placeholders = newTypes.map(() => '(?, ?)').join(', '); + const values = newTypes.flatMap((t) => [appId, t]); + entries.push({ + statement: `INSERT INTO \`app_filetype_association\` (\`app_id\`, \`type\`) VALUES ${placeholders}`, + values, + }); + } + await this.clients.db.batchWrite(entries); + + const affected = new Set([...previous, ...newTypes]); + if (affected.size === 0) return; + const keys = [...affected].map( + (t) => `${FILETYPE_CACHE_KEY_PREFIX}:${t}`, + ); + await this.publishCacheKeys({ keys, broadcast: true }); + } + + // ── Cache invalidation ─────────────────────────────────────────── + + async invalidate(app) { + const keys = this.#cacheKeysForApp(app); + await this.publishCacheKeys({ keys, broadcast: true }); + await this.#invalidateListCachesForApps([app]); + } + + async invalidateById(id) { + const app = + (await this.#readCache('id', id)) ?? + (await this.#readFromDb('id', id)); + if (app) await this.invalidate(app); + } + + async invalidateByUid(uid) { + const app = + (await this.#readCache('uid', uid)) ?? + (await this.#readFromDb('uid', uid)); + if (app) await this.invalidate(app); + } + + /** Resolve an app by either uid or name; tries uid first, then name. */ + async resolveApp(identifier) { + return ( + (await this.getByUid(identifier)) ?? + (await this.getByName(identifier)) + ); + } + + // ── Internals ──────────────────────────────────────────────────── + + async #getByProperty(prop, value) { + if (value === undefined || value === null) return null; + + const cached = await this.#readCache(prop, value); + if (cached) return cached; + + const normalized = await this.#readFromDb(prop, value); + if (!normalized) return null; + + this.#writeCache(normalized).catch(() => {}); + return normalized; + } + + async #readFromDb(prop, value) { + const rows = await this.clients.db.read( + `SELECT * FROM \`apps\` WHERE \`${prop}\` = ? LIMIT 1`, + [value], + ); + if (rows.length === 0) return null; + return this.#normalizeRow(rows[0]); + } + + #cacheKey(prop, value) { + return `${CACHE_KEY_PREFIX}:${prop}:${value}`; + } + + #listCacheKey(whereClause, params, limit) { + return `${LIST_CACHE_KEY_PREFIX}:${JSON.stringify([ + whereClause, + params, + limit, + ])}`; + } + + #parseListCacheKey(cacheKey) { + if (!cacheKey.startsWith(`${LIST_CACHE_KEY_PREFIX}:`)) return null; + try { + const raw = cacheKey.slice(LIST_CACHE_KEY_PREFIX.length + 1); + const parsed = JSON.parse(raw); + if (!Array.isArray(parsed) || parsed.length !== 3) return null; + return { whereClause: parsed[0], params: parsed[1] }; + } catch { + return null; + } + } + + #listCacheMatchesApp(cacheKey, app) { + if (!app) return false; + const parsed = this.#parseListCacheKey(cacheKey); + if (!parsed) return true; + + const { whereClause, params } = parsed; + if (!whereClause) return true; + if (!Array.isArray(params)) return true; + + const columns = whereClause + .replace(/^WHERE\s+/u, '') + .split(' AND ') + .map((part) => part.match(/^`([^`]+)` = \?$/u)?.[1]); + + if (columns.some((column) => !column)) return true; + + for (let i = 0; i < columns.length; i++) { + if (app[columns[i]] !== params[i]) return false; + } + return true; + } + + #cacheKeysForApp(app) { + const keys = []; + for (const prop of APP_ID_PROPERTIES) { + if (app[prop] !== undefined && app[prop] !== null) { + keys.push(this.#cacheKey(prop, app[prop])); + } + } + return keys; + } + + async #readCache(prop, value) { + try { + const raw = await this.clients.redis.get( + this.#cacheKey(prop, value), + ); + return raw ? this.#normalizeRow(JSON.parse(raw)) : null; + } catch { + return null; + } + } + + async #writeCache(app) { + const keys = this.#cacheKeysForApp(app); + if (keys.length === 0) return; + const serialized = JSON.stringify(app); + await Promise.all( + keys.map((k) => + this.clients.redis.set(k, serialized, 'EX', CACHE_TTL_SECONDS), + ), + ); + } + + async #writeListCache(cacheKey, apps) { + const pipeline = this.clients.redis.pipeline(); + pipeline.set( + cacheKey, + JSON.stringify(apps), + 'EX', + LIST_CACHE_TTL_SECONDS, + ); + pipeline.sadd(LIST_CACHE_TRACKER_KEY, cacheKey); + pipeline.expire(LIST_CACHE_TRACKER_KEY, LIST_CACHE_TTL_SECONDS); + await pipeline.exec(); + } + + async #invalidateListCachesForApps(apps) { + let keys = []; + try { + keys = await this.clients.redis.smembers(LIST_CACHE_TRACKER_KEY); + } catch { + return; + } + if (!Array.isArray(keys)) keys = []; + keys = keys.filter((key) => + apps.some((app) => this.#listCacheMatchesApp(key, app)), + ); + if (keys.length === 0) return; + await this.publishCacheKeys({ + keys, + broadcast: true, + }); + } + + async #refreshCache(app) { + const keys = this.#cacheKeysForApp(app); + if (keys.length === 0) return; + await this.publishCacheKeys({ + keys, + serializedData: JSON.stringify(app), + ttlSeconds: CACHE_TTL_SECONDS, + broadcast: true, + }); + } + + #filterEditable(fields) { + const out = {}; + for (const [k, v] of Object.entries(fields)) { + if (READ_ONLY_COLUMNS.has(k)) continue; + out[k] = v; + } + return out; + } + + #normalizeRow(row) { + if (!row) return null; + // Coerce booleans + for (const key of [ + 'godmode', + 'background', + 'maximize_on_start', + 'protected', + 'is_private', + 'approved_for_listing', + 'approved_for_opening_items', + 'approved_for_incentive_program', + ]) { + if (row[key] !== undefined) row[key] = Boolean(row[key]); + } + // Parse metadata + if (typeof row.metadata === 'string') { + try { + row.metadata = JSON.parse(row.metadata); + } catch { + row.metadata = null; + } + } + // Alias created_at + if (row.timestamp !== undefined && row.created_at === undefined) { + row.created_at = row.timestamp; + } + return row; + } + + appStatsCachePrefix = 'appstats:'; + + #openCountCacheKey(uid) { + return `${this.appStatsCachePrefix}open:${uid}`; + } + #userCountCacheKey(uid) { + return `${this.appStatsCachePrefix}user:${uid}`; + } + + /** + * Batched, cached all-time { open_count, user_count } for a set of apps. + * + * Flow: pipelined redis MGET → on miss, one ClickHouse (or MySQL) query + * for all misses at once → backfill cache. Apps with no rows in + * `app_opens` resolve to zero counts. Returns `Map`. + */ + async getAppsStats(appUids) { + const uids = Array.isArray(appUids) + ? [...new Set(appUids.filter((u) => typeof u === 'string' && u))] + : []; + const stats = new Map(); + if (uids.length === 0) return stats; + + let cacheResults = []; + try { + const pipeline = this.clients.redis.pipeline(); + for (const uid of uids) { + pipeline.get(this.#openCountCacheKey(uid)); + pipeline.get(this.#userCountCacheKey(uid)); + } + cacheResults = (await pipeline.exec()) ?? []; + } catch { + // Fall through — treat everything as a miss. + } + + const missing = []; + for (let i = 0; i < uids.length; i++) { + const uid = uids[i]; + const openEntry = cacheResults[i * 2]; + const userEntry = cacheResults[i * 2 + 1]; + const openVal = openEntry?.[1]; + const userVal = userEntry?.[1]; + if (openVal != null && userVal != null) { + const o = parseInt(openVal, 10); + const u = parseInt(userVal, 10); + if (!Number.isNaN(o) && !Number.isNaN(u)) { + stats.set(uid, { + open_count: o, + user_count: u, + referral_count: null, + }); + continue; + } + } + missing.push(uid); + } + + if (missing.length > 0) { + const fresh = await this.#queryStatsForUids(missing); + const writePipe = this.clients.redis.pipeline(); + for (const uid of missing) { + const row = fresh.get(uid) ?? { open_count: 0, user_count: 0 }; + stats.set(uid, { ...row, referral_count: null }); + writePipe.set( + this.#openCountCacheKey(uid), + String(row.open_count), + 'EX', + STATS_CACHE_TTL_SECONDS, + ); + writePipe.set( + this.#userCountCacheKey(uid), + String(row.user_count), + 'EX', + STATS_CACHE_TTL_SECONDS, + ); + } + writePipe.exec().catch(() => {}); + } + + return stats; + } + + /** + * Detailed / period-filtered / grouped stats for a single app. + * Deliberately uncached — UI-driven, rarely repeated with the exact same + * args, and ClickHouse is fast enough at interactive latency. + * + * @param {object} [options] + * @param {string} [options.period='all'] — today, yesterday, 7d, 30d, + * this_week, last_week, this_month, last_month, this_year, last_year, + * 12m, all + * @param {string} [options.grouping] — hour, day, week, month, year + * @param {number|string|Date} [options.createdAt] — app creation ts; + * used to bound the `all` period + */ + async getAppStatsDetailed(appUid, options = {}) { + const period = options.period ?? 'all'; + const grouping = options.grouping; + const timeRange = this.#computeTimeRange(period, options.createdAt); + const clickhouse = globalThis.clickhouseClient; + + if (grouping) { + if (!MYSQL_DATE_FORMATS[grouping]) { + throw new Error( + `Invalid grouping: ${grouping}. Supported: hour, day, week, month, year`, + ); + } + return this.#queryGroupedStats( + appUid, + timeRange, + grouping, + clickhouse, + ); + } + + return this.#querySingleStats(appUid, timeRange, clickhouse); + } + + // ── Stats internals ────────────────────────────────────────────── + + async #queryStatsForUids(uids) { + const out = new Map(); + if (uids.length === 0) return out; + + const clickhouse = globalThis.clickhouseClient; + if (clickhouse) { + const res = await clickhouse.query({ + query: ` + SELECT app_uid, + count(_id) AS open_count, + count(DISTINCT user_id) AS user_count + FROM app_opens + WHERE app_uid IN {uids:Array(String)} + GROUP BY app_uid + `, + query_params: { uids: uids.map((u) => String(u)) }, + format: 'JSONEachRow', + }); + const rows = await res.json(); + for (const row of rows) { + out.set(row.app_uid, { + open_count: parseInt(row.open_count, 10) || 0, + user_count: parseInt(row.user_count, 10) || 0, + }); + } + return out; + } + + const placeholders = uids.map(() => '?').join(','); + const rows = await this.clients.db.read( + `SELECT app_uid, + COUNT(_id) AS open_count, + COUNT(DISTINCT user_id) AS user_count + FROM app_opens + WHERE app_uid IN (${placeholders}) + GROUP BY app_uid`, + uids, + ); + for (const row of rows) { + out.set(row.app_uid, { + open_count: parseInt(row.open_count, 10) || 0, + user_count: parseInt(row.user_count, 10) || 0, + }); + } + return out; + } + + async #querySingleStats(appUid, timeRange, clickhouse) { + if (clickhouse) { + const query_params = { appUid: String(appUid) }; + let timeCond = ''; + if (timeRange) { + query_params.tsStart = Math.floor(timeRange.start / 1000); + query_params.tsEnd = Math.floor(timeRange.end / 1000); + timeCond = 'AND ts >= {tsStart:Int64} AND ts < {tsEnd:Int64}'; + } + const res = await clickhouse.query({ + query: ` + SELECT count(_id) AS open_count, + count(DISTINCT user_id) AS user_count + FROM app_opens + WHERE app_uid = {appUid:String} + ${timeCond} + `, + query_params, + format: 'JSONEachRow', + }); + const rows = await res.json(); + const row = rows[0] ?? { open_count: 0, user_count: 0 }; + return { + open_count: parseInt(row.open_count, 10) || 0, + user_count: parseInt(row.user_count, 10) || 0, + }; + } + + // ts is stored as unix seconds; timeRange.start/end are ms. + const params = timeRange + ? [appUid, timeRange.start / 1000, timeRange.end / 1000] + : [appUid]; + const where = timeRange ? 'AND ts >= ? AND ts < ?' : ''; + const rows = await this.clients.db.read( + `SELECT COUNT(_id) AS open_count, + COUNT(DISTINCT user_id) AS user_count + FROM app_opens + WHERE app_uid = ? ${where}`, + params, + ); + const row = rows[0] ?? { open_count: 0, user_count: 0 }; + return { + open_count: parseInt(row.open_count, 10) || 0, + user_count: parseInt(row.user_count, 10) || 0, + referral_count: null, + }; + } + + async #queryGroupedStats(appUid, timeRange, grouping, clickhouse) { + const allPeriods = this.#generateAllPeriods( + new Date(timeRange.start), + new Date(timeRange.end), + grouping, + ); + + if (clickhouse) { + const groupBy = CLICKHOUSE_GROUP_BY_FORMATS[grouping]; + const res = await clickhouse.query({ + query: ` + SELECT ${groupBy} AS period, + count(_id) AS open_count, + count(DISTINCT user_id) AS user_count + FROM app_opens + WHERE app_uid = {appUid:String} + AND ts >= {tsStart:Int64} AND ts < {tsEnd:Int64} + GROUP BY period + ORDER BY period + `, + query_params: { + appUid: String(appUid), + tsStart: Math.floor(timeRange.start / 1000), + tsEnd: Math.floor(timeRange.end / 1000), + }, + format: 'JSONEachRow', + }); + const rows = await res.json(); + const processed = rows.map((r) => ({ + period: new Date(r.period), + open_count: parseInt(r.open_count, 10) || 0, + user_count: parseInt(r.user_count, 10) || 0, + })); + return this.#assembleGroupedResult(processed, allPeriods, grouping); + } + + const timeFormat = MYSQL_DATE_FORMATS[grouping]; + const params = [appUid, timeRange.start / 1000, timeRange.end / 1000]; + // ts is stored as unix seconds — no /1000 in the date conversion. + const periodExpr = this.clients.db.case({ + mysql: `DATE_FORMAT(FROM_UNIXTIME(ts), '${timeFormat}')`, + sqlite: `STRFTIME('${timeFormat}', datetime(ts, 'unixepoch'))`, + otherwise: `DATE_FORMAT(FROM_UNIXTIME(ts), '${timeFormat}')`, + }); + const rows = await this.clients.db.read( + `SELECT ${periodExpr} AS period, + COUNT(_id) AS open_count, + COUNT(DISTINCT user_id) AS user_count + FROM app_opens + WHERE app_uid = ? AND ts >= ? AND ts < ? + GROUP BY period + ORDER BY period`, + params, + ); + const processed = rows.map((r) => ({ + period: r.period, + open_count: parseInt(r.open_count, 10) || 0, + user_count: parseInt(r.user_count, 10) || 0, + })); + return this.#assembleGroupedResult(processed, allPeriods, grouping); + } + + #assembleGroupedResult(rows, allPeriods, grouping) { + // Totals come from raw rows so they survive even if a row's + // period key fails to match an `allPeriods` entry (e.g. week + // grouping format mismatches, timezone edge cases). Matches v1 + // AppInformationService.get_stats behaviour. + let totalOpen = 0; + let totalUser = 0; + for (const r of rows) { + totalOpen += r.open_count; + totalUser += r.user_count; + } + const dataMap = new Map( + rows.map((r) => [this.#normalizePeriodKey(r.period, grouping), r]), + ); + const open = []; + const user = []; + for (const p of allPeriods) { + const match = dataMap.get(p.period); + open.push({ period: p.period, count: match?.open_count ?? 0 }); + user.push({ period: p.period, count: match?.user_count ?? 0 }); + } + return { + open_count: totalOpen, + user_count: totalUser, + grouped_stats: { open_count: open, user_count: user }, + referral_count: null, + }; + } + + #normalizePeriodKey(period, grouping) { + if (!(period instanceof Date)) return period; + switch (grouping) { + case 'hour': + return `${period.toISOString().slice(0, 13)}:00:00`; + case 'day': + return period.toISOString().slice(0, 10); + case 'week': { + const wn = String(this.#getWeekNumber(period)).padStart(2, '0'); + return `${period.getFullYear()}-${wn}`; + } + case 'month': + return period.toISOString().slice(0, 7); + case 'year': + return period.getFullYear().toString(); + default: + return period.toISOString(); + } + } + + #computeTimeRange(period, createdAt) { + const now = new Date(); + const today = new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + ); + switch (period) { + case 'today': + return { start: today.getTime(), end: now.getTime() }; + case 'yesterday': { + const y = new Date(today); + y.setDate(y.getDate() - 1); + return { start: y.getTime(), end: today.getTime() - 1 }; + } + case '7d': { + const s = new Date(now); + s.setDate(s.getDate() - 7); + return { start: s.getTime(), end: now.getTime() }; + } + case '30d': { + const s = new Date(now); + s.setDate(s.getDate() - 30); + return { start: s.getTime(), end: now.getTime() }; + } + case 'this_week': { + const s = new Date( + now.getFullYear(), + now.getMonth(), + now.getDate() - now.getDay(), + ); + return { start: s.getTime(), end: now.getTime() }; + } + case 'last_week': { + const s = new Date( + now.getFullYear(), + now.getMonth(), + now.getDate() - now.getDay() - 7, + ); + const e = new Date( + now.getFullYear(), + now.getMonth(), + now.getDate() - now.getDay(), + ); + return { start: s.getTime(), end: e.getTime() - 1 }; + } + case 'this_month': { + const s = new Date(now.getFullYear(), now.getMonth(), 1); + return { start: s.getTime(), end: now.getTime() }; + } + case 'last_month': { + const s = new Date(now.getFullYear(), now.getMonth() - 1, 1); + const e = new Date(now.getFullYear(), now.getMonth(), 1); + return { start: s.getTime(), end: e.getTime() - 1 }; + } + case 'this_year': { + const s = new Date(now.getFullYear(), 0, 1); + return { start: s.getTime(), end: now.getTime() }; + } + case 'last_year': { + const s = new Date(now.getFullYear() - 1, 0, 1); + const e = new Date(now.getFullYear(), 0, 1); + return { start: s.getTime(), end: e.getTime() - 1 }; + } + case '12m': { + const s = new Date(now); + s.setMonth(s.getMonth() - 12); + return { start: s.getTime(), end: now.getTime() }; + } + case 'all': { + const start = createdAt ? new Date(createdAt).getTime() : 0; + return { start, end: now.getTime() }; + } + default: + return { start: 0, end: now.getTime() }; + } + } + + #generateAllPeriods(startDate, endDate, grouping) { + const out = []; + const cur = new Date(startDate); + if (Number.isNaN(cur.getTime())) return out; + while (cur <= endDate) { + let period; + switch (grouping) { + case 'hour': + period = `${cur.toISOString().slice(0, 13)}:00:00`; + cur.setHours(cur.getHours() + 1); + break; + case 'day': + period = cur.toISOString().slice(0, 10); + cur.setDate(cur.getDate() + 1); + break; + case 'week': { + const wn = String(this.#getWeekNumber(cur)).padStart( + 2, + '0', + ); + period = `${cur.getFullYear()}-${wn}`; + cur.setDate(cur.getDate() + 7); + break; + } + case 'month': + period = cur.toISOString().slice(0, 7); + cur.setMonth(cur.getMonth() + 1); + break; + case 'year': + period = cur.getFullYear().toString(); + cur.setFullYear(cur.getFullYear() + 1); + break; + default: + return out; + } + out.push({ period, count: 0 }); + } + return out; + } + + #getWeekNumber(date) { + const target = new Date(date.valueOf()); + const dayNumber = (date.getDay() + 6) % 7; + target.setDate(target.getDate() - dayNumber + 3); + const firstThursday = target.valueOf(); + target.setMonth(0, 1); + if (target.getDay() !== 4) { + target.setMonth(0, 1 + ((4 - target.getDay() + 7) % 7)); + } + return 1 + Math.ceil((firstThursday - target) / 604800000); + } + + // ── Refresh loop ───────────────────────────────────────────────── + + async #refreshAppStats() { + // Cross-instance lock: if another node refreshed within the window, + // skip. TTL slightly longer than the interval so the guard survives + // jitter in the setInterval drift. + try { + const last = parseInt( + (await this.clients.redis.get(STATS_REFRESH_LOCK_KEY)) || '0', + 10, + ); + const now = Date.now(); + if (now - last < STATS_REFRESH_INTERVAL_MS) return; + await this.clients.redis.set( + STATS_REFRESH_LOCK_KEY, + String(now), + 'EX', + Math.floor(STATS_REFRESH_INTERVAL_MS / 1000) + 60, + ); + } catch { + // Keep going — better to over-refresh than miss updates entirely. + } + + const clickhouse = globalThis.clickhouseClient; + let rows; + try { + if (clickhouse) { + const res = await clickhouse.query({ + query: ` + SELECT app_uid, + count(_id) AS open_count, + count(DISTINCT user_id) AS user_count + FROM app_opens + GROUP BY app_uid + `, + format: 'JSONEachRow', + }); + rows = await res.json(); + } else { + rows = await this.clients.db.read( + `SELECT app_uid, + COUNT(_id) AS open_count, + COUNT(DISTINCT user_id) AS user_count + FROM app_opens + GROUP BY app_uid`, + ); + } + } catch (e) { + console.warn('[AppStore] refresh app stats failed:', e); + return; + } + + if (!rows?.length) return; + + try { + const pipeline = this.clients.redis.pipeline(); + for (const row of rows) { + if (!row.app_uid) continue; + pipeline.set( + this.#openCountCacheKey(row.app_uid), + String(parseInt(row.open_count, 10) || 0), + 'EX', + STATS_CACHE_TTL_SECONDS, + ); + pipeline.set( + this.#userCountCacheKey(row.app_uid), + String(parseInt(row.user_count, 10) || 0), + 'EX', + STATS_CACHE_TTL_SECONDS, + ); + } + await pipeline.exec(); + } catch (e) { + console.warn('[AppStore] refresh app stats cache write failed:', e); + } + } + + onServerStart() { + // Kick off one refresh immediately so the cache is warm before the + // first client request (failing silently is fine — getAppsStats + // falls back to an on-demand query). + this.#refreshAppStats().catch(() => {}); + // Jitter prevents every node refreshing on the same tick when a + // cluster boots together. + this.#appStatsInterval = setInterval( + () => { + this.#refreshAppStats().catch(() => {}); + }, + STATS_REFRESH_INTERVAL_MS + Math.floor(Math.random() * 500), + ); + } + + onServerShutdown() { + if (this.#appStatsInterval) { + clearInterval(this.#appStatsInterval); + this.#appStatsInterval = undefined; + } + } +} diff --git a/extensions/fsv2/src/types/FSEntry.ts b/src/backend/stores/fs/FSEntry.ts similarity index 88% rename from extensions/fsv2/src/types/FSEntry.ts rename to src/backend/stores/fs/FSEntry.ts index bb7a8c0b6..629a4b550 100644 --- a/extensions/fsv2/src/types/FSEntry.ts +++ b/src/backend/stores/fs/FSEntry.ts @@ -28,6 +28,16 @@ export interface FSEntry { size: number | null; symlinkPath: string | null; isSymlink: boolean; + subdomains: FSEntrySubdomain[]; + workers: FSEntrySubdomain[]; + hasWebsite?: boolean; + suggestedApps: unknown[]; // TODO DS: type with app row +} + +export interface FSEntrySubdomain { + uuid: string; + address: string; // `${config.protocol}://${subdomain}.${'puter.site'|'puter.work'}` depending on wether dir or file + subdomain: string; } export interface FSEntryWriteInput { diff --git a/src/backend/stores/fs/FSEntryStore.ts b/src/backend/stores/fs/FSEntryStore.ts new file mode 100644 index 000000000..6ae312319 --- /dev/null +++ b/src/backend/stores/fs/FSEntryStore.ts @@ -0,0 +1,2578 @@ +import { statfs } from 'node:fs/promises'; +import { posix as pathPosix } from 'node:path'; +import { v4 as uuidv4 } from 'uuid'; +import { HttpError } from '../../core/http/HttpError.js'; +import type { LayerInstances } from '../../types.js'; +import { runWithConcurrencyLimit } from '../../util/concurrency.js'; +import type { puterStores } from '../index.js'; +import { PuterStore } from '../types.js'; +import { + FSEntry, + FSEntryCreateInput, + FSEntrySubdomain, + PendingUploadCreateInput, + PendingUploadSession, +} from './FSEntry.js'; +import { + normalizePendingUploadSession, + PendingUploadSessionStatus, + toPendingUploadSession, + toPendingUploadSessionExpiresAtSeconds, + toPendingUploadSessionKey, + withPendingUploadSessionStatus, +} from './pendingUploadSessionHelpers.js'; +import type { + FSEntryRow, + NormalizedEntryWrite, + ReadEntriesByPathsOptions, +} from './types.js'; + +const ENTRY_CACHE_TTL_SECONDS = 60; +const BULK_QUERY_CHUNK_SIZE = 200; +const DEFAULT_DB_CHUNK_CONCURRENCY = 4; + +/** + * Store backing the `fsentries` table. Owns DB CRUD over filesystem entries, + * Redis caching keyed by uuid/path/id, and pending-upload-session state in + * the system KV store. Constructed by the store registry; depends on `kv`. + */ +export class FSEntryStore extends PuterStore { + declare protected stores: LayerInstances; + + #insertIgnoreIntoFsentriesSql(): string { + return this.clients.db.case({ + sqlite: 'INSERT OR IGNORE INTO fsentries', + otherwise: 'INSERT IGNORE INTO fsentries', + }); + } + + // JSON aggregation of associated subdomain rows, keyed on fsentries.id. + // SQLite uses `json_group_array` + `json_object`; MySQL/MariaDB use + // `JSON_ARRAYAGG` + `JSON_OBJECT`. Correlated subquery keeps the row + // count 1:1 with fsentries and avoids a GROUP BY on the outer query. + #subdomainsAggSql(): string { + return this.clients.db.case({ + sqlite: `( + SELECT json_group_array( + json_object('uuid', sd.uuid, 'subdomain', sd.subdomain) + ) + FROM subdomains sd + WHERE sd.root_dir_id = fsentries.id + )`, + otherwise: `( + SELECT JSON_ARRAYAGG( + JSON_OBJECT('uuid', sd.uuid, 'subdomain', sd.subdomain) + ) + FROM subdomains sd + WHERE sd.root_dir_id = fsentries.id + )`, + }); + } + + // Projection shared by every read path: all fsentries columns plus the + // aggregated subdomains JSON aliased to `subdomains_agg`. Callers append + // their own `FROM fsentries ...` tail. + #selectFsentriesColumns(): string { + return `fsentries.*, ${this.#subdomainsAggSql()} AS subdomains_agg`; + } + + #parseSubdomainsAgg( + raw: unknown, + ): Array<{ uuid: string; subdomain: string }> { + if (raw === null || raw === undefined) { + return []; + } + let parsed: unknown = raw; + if (typeof raw === 'string') { + if (raw.length === 0) { + return []; + } + try { + parsed = JSON.parse(raw); + } catch { + return []; + } + } + if (!Array.isArray(parsed)) { + return []; + } + const result: Array<{ uuid: string; subdomain: string }> = []; + for (const item of parsed) { + if (item === null || typeof item !== 'object') { + continue; + } + const record = item as Record; + if ( + typeof record.uuid !== 'string' || + typeof record.subdomain !== 'string' + ) { + continue; + } + result.push({ uuid: record.uuid, subdomain: record.subdomain }); + } + return result; + } + + // Static hosting for user sites lives at `*.{static_hosting_domain}` + // (typically `puter.site`). Workers deploy as subdomain rows prefixed + // `workers.puter.` and are exposed at `.puter.work` — that + // mapping is hardcoded in `WorkerDriver`, mirrored here so the URL we + // return matches the deployment domain. + #buildFsEntrySubdomains(rows: Array<{ uuid: string; subdomain: string }>): { + subdomains: FSEntrySubdomain[]; + workers: FSEntrySubdomain[]; + } { + const protocol = this.config.protocol ?? 'https'; + const siteDomain = this.config.static_hosting_domain ?? 'puter.site'; + const workerPrefix = 'workers.puter.'; + const workerDomain = 'puter.work'; + + const subdomains: FSEntrySubdomain[] = []; + const workers: FSEntrySubdomain[] = []; + for (const row of rows) { + if (row.subdomain.startsWith(workerPrefix)) { + const workerName = row.subdomain.slice(workerPrefix.length); + workers.push({ + uuid: row.uuid, + subdomain: row.subdomain, + address: `${protocol}://${workerName}.${workerDomain}`, + }); + continue; + } + subdomains.push({ + uuid: row.uuid, + subdomain: row.subdomain, + address: `${protocol}://${row.subdomain}.${siteDomain}`, + }); + } + return { subdomains, workers }; + } + + #normalizePath(path: string): string { + const trimmed = path.trim(); + if (trimmed.length === 0) { + throw new HttpError(400, 'Path cannot be empty'); + } + + let normalized = pathPosix.normalize(trimmed); + if (!normalized.startsWith('/')) { + normalized = `/${normalized}`; + } + if (normalized.length > 1 && normalized.endsWith('/')) { + normalized = normalized.slice(0, -1); + } + + return normalized; + } + + #toBoolean(value: number | boolean | null | undefined): boolean { + if (typeof value === 'boolean') { + return value; + } + return Number(value ?? 0) === 1; + } + + #toNullableBoolean( + value: number | boolean | null | undefined, + ): boolean | null { + if (value === null || value === undefined) { + return null; + } + return this.#toBoolean(value); + } + + #mapFSEntryRow(row: FSEntryRow): FSEntry { + const { subdomains, workers } = this.#buildFsEntrySubdomains( + this.#parseSubdomainsAgg(row.subdomains_agg), + ); + // Legacy NULL-path rows surface here with `path: null`. Async lineage + // heal happens at the call site via `#healEntriesWithMissingPathsInPlace` + // — this mapper stays sync because mapping fans out from many places + // that batch their DB reads (per-chunk). + return { + id: Number(row.id), + uuid: row.uuid, + uid: row.uuid, + userId: Number(row.user_id), + parentId: row.parent_id === null ? null : Number(row.parent_id), + parentUid: row.parent_uid, + path: row.path, + name: row.name, + isDir: this.#toBoolean(row.is_dir), + bucket: row.bucket, + bucketRegion: row.bucket_region, + publicToken: row.public_token, + fileRequestToken: row.file_request_token, + isShortcut: this.#toBoolean(row.is_shortcut), + shortcutTo: row.shortcut_to, + associatedAppId: row.associated_app_id, + layout: row.layout, + sortBy: row.sort_by, + sortOrder: row.sort_order, + isPublic: this.#toNullableBoolean(row.is_public), + thumbnail: row.thumbnail, + immutable: this.#toBoolean(row.immutable), + metadata: row.metadata, + modified: Number(row.modified), + created: row.created === null ? null : Number(row.created), + accessed: row.accessed === null ? null : Number(row.accessed), + size: row.size === null ? null : Number(row.size), + symlinkPath: row.symlink_path, + isSymlink: this.#toBoolean(row.is_symlink), + subdomains, + workers, + hasWebsite: subdomains.length > 0, + // Populated by `SuggestedAppsService` in the request path — kept + // empty here so the field is always present on the type. + suggestedApps: [], + }; + } + + #entryCacheKeys(entry: FSEntry): string[] { + return [ + `prodfsv2:fsentry:id:${entry.id}`, + `prodfsv2:fsentry:uuid:${entry.uuid}`, + `prodfsv2:fsentry:path:any:${entry.path}`, + ]; + } + + async #readEntryFromCache(cacheKey: string): Promise { + try { + const cached = await this.clients.redis.get(cacheKey); + if (!cached) { + return null; + } + return JSON.parse(cached) as FSEntry; + } catch { + return null; + } + } + + async #writeEntryToCache(entry: FSEntry): Promise { + try { + const serialized = JSON.stringify(entry); + await Promise.all( + this.#entryCacheKeys(entry).map((cacheKey) => { + return this.clients.redis.setex( + cacheKey, + ENTRY_CACHE_TTL_SECONDS, + serialized, + ); + }), + ); + } catch { + // Best effort cache write. + } + } + + async #invalidateEntryCache(entry: FSEntry): Promise { + // Broadcasts; read-path `#writeEntryToCache` stays local to avoid + // fanning backfills over the network. + const keys = this.#entryCacheKeys(entry); + await this.publishCacheKeys({ keys }); + } + + async invalidateEntryCacheByPathForUser( + _userId: number, + path: string, + ): Promise { + const normalizedPath = this.#normalizePath(path); + const cacheKeys: string[] = [ + `prodfsv2:fsentry:path:any:${normalizedPath}`, + ]; + + const rows = (await this.clients.db.read( + `SELECT ${this.#selectFsentriesColumns()} FROM fsentries WHERE path = ? LIMIT 1`, + [normalizedPath], + )) as unknown as FSEntryRow[]; + const row = rows[0]; + + if (row) { + const entry = this.#mapFSEntryRow(row); + await this.#invalidateEntryCache(entry); + return; + } + + await this.publishCacheKeys({ keys: cacheKeys }); + } + + async invalidateEntryCacheByUuid(uuid: string): Promise { + if (typeof uuid !== 'string' || uuid.length === 0) { + return; + } + + const rows = (await this.clients.db.read( + `SELECT ${this.#selectFsentriesColumns()} FROM fsentries WHERE uuid = ? LIMIT 1`, + [uuid], + )) as unknown as FSEntryRow[]; + const row = rows[0]; + + if (row) { + const entry = this.#mapFSEntryRow(row); + await this.#invalidateEntryCache(entry); + return; + } + + const cached = await this.#readEntryFromCache( + `prodfsv2:fsentry:uuid:${uuid}`, + ); + if (cached) { + await this.#invalidateEntryCache(cached); + return; + } + + await this.publishCacheKeys({ + keys: [`prodfsv2:fsentry:uuid:${uuid}`], + }); + } + + #chunk(values: T[], size: number): T[][] { + if (values.length === 0) { + return []; + } + const chunks: T[][] = []; + for (let index = 0; index < values.length; index += size) { + chunks.push(values.slice(index, index + size)); + } + return chunks; + } + + async #writePendingUploadSessions( + sessions: PendingUploadSession[], + operationName: string, + ): Promise { + if (sessions.length === 0) { + return; + } + + try { + await this.stores.kv.batchPut({ + items: sessions.map((session) => ({ + key: toPendingUploadSessionKey(session.sessionId), + value: session, + expireAt: toPendingUploadSessionExpiresAtSeconds( + session.expiresAt, + ), + })), + }); + } catch (error) { + if (error instanceof Error) { + throw error; + } + throw new Error(`Failed to ${operationName}`); + } + } + + async #getPendingUploadSessionsBySessionIds( + sessionIds: string[], + ): Promise> { + const uniqueSessionIds = Array.from(new Set(sessionIds)); + const sessionsById = new Map(); + if (uniqueSessionIds.length === 0) { + return sessionsById; + } + + const { res: rawValues } = await this.stores.kv.get({ + key: uniqueSessionIds.map((sessionId) => + toPendingUploadSessionKey(sessionId), + ), + }); + if (!Array.isArray(rawValues)) { + return sessionsById; + } + + for (let index = 0; index < uniqueSessionIds.length; index++) { + const sessionId = uniqueSessionIds[index]; + const rawValue = rawValues[index]; + if (!sessionId) { + continue; + } + + const normalizedSession = normalizePendingUploadSession( + rawValue, + sessionId, + ); + if (normalizedSession) { + sessionsById.set(sessionId, normalizedSession); + } + } + + return sessionsById; + } + + async #markPendingSessionsWithStatus( + sessionIds: string[], + status: PendingUploadSessionStatus, + reason: string | null, + ): Promise { + if (sessionIds.length === 0) { + return; + } + + const sessionsById = + await this.#getPendingUploadSessionsBySessionIds(sessionIds); + const now = Date.now(); + const updatedSessions = Array.from(new Set(sessionIds)) + .map((sessionId) => { + const session = sessionsById.get(sessionId); + if (!session) { + return null; + } + + return withPendingUploadSessionStatus( + session, + status, + reason, + now, + ); + }) + .filter((session): session is PendingUploadSession => + Boolean(session), + ); + + await this.#writePendingUploadSessions( + updatedSessions, + `mark pending upload sessions as ${status}`, + ); + } + + // Paths are globally unique because they're prefixed with the owner's + // username (`//...`); the SQL `WHERE` clause can't filter by + // user_id — doing so breaks older accounts where the stored `user_id` + // doesn't line up with the current id. Matches v1, which only ever + // queried `WHERE path = ?`. The userId param is accepted for call-site + // clarity but intentionally unused. + async #readEntriesByPathsForUser( + _userId: number, + paths: string[], + options: ReadEntriesByPathsOptions = {}, + ): Promise> { + const useTryHardRead = Boolean(options.useTryHardRead); + const skipCache = Boolean(options.skipCache); + const normalizedPaths = Array.from( + new Set( + paths + .map((path) => this.#normalizePath(path)) + .filter((path) => path.length > 0), + ), + ); + const entriesByPath = new Map(); + if (normalizedPaths.length === 0) { + return entriesByPath; + } + + const missingPaths: string[] = []; + if (skipCache) { + missingPaths.push(...normalizedPaths); + } else { + const cacheReads = await Promise.all( + normalizedPaths.map(async (path) => { + const cacheKey = `prodfsv2:fsentry:path:any:${path}`; + const cachedEntry = + await this.#readEntryFromCache(cacheKey); + return { path, cachedEntry }; + }), + ); + + for (const cacheRead of cacheReads) { + if (cacheRead.cachedEntry) { + entriesByPath.set(cacheRead.path, cacheRead.cachedEntry); + } else { + missingPaths.push(cacheRead.path); + } + } + } + + const chunks = this.#chunk(missingPaths, BULK_QUERY_CHUNK_SIZE); + const chunkResults = await runWithConcurrencyLimit( + chunks, + DEFAULT_DB_CHUNK_CONCURRENCY, + async (chunk) => { + if (chunk.length === 0) { + return []; + } + + const placeholders = chunk.map(() => '?').join(', '); + const selectSql = `SELECT ${this.#selectFsentriesColumns()} FROM fsentries WHERE path IN (${placeholders})`; + const rows = (useTryHardRead + ? await this.clients.db.tryHardRead(selectSql, chunk) + : await this.clients.db.read( + selectSql, + chunk, + )) as unknown as FSEntryRow[]; + + const entries = rows.map((row) => this.#mapFSEntryRow(row)); + if (entries.length > 0) { + await Promise.all( + entries.map((entry) => this.#writeEntryToCache(entry)), + ); + } + return entries; + }, + ); + for (const chunkEntries of chunkResults) { + for (const entry of chunkEntries) { + entriesByPath.set(entry.path, entry); + } + } + + // Per-path lineage fallback for entries the bulk SELECT missed — + // legacy rows with a NULL `path` column (see #resolveEntryByPathLineage). + // Sequential to keep the round-trip cost predictable; the common case + // is a single missing path. + const stillMissing = normalizedPaths.filter( + (path) => !entriesByPath.has(path), + ); + for (const path of stillMissing) { + const entry = await this.#resolveEntryByPathLineage(path); + if (entry) { + entriesByPath.set(path, entry); + } + } + + return entriesByPath; + } + + #pathDepth(path: string): number { + return path.split('/').filter(Boolean).length; + } + + async #ensureDirectoryPathsForUser( + userId: number, + requiredPaths: string[], + ): Promise<{ + requiredEntryMap: Map; + createdEntryMap: Map; + }> { + const normalizedRequiredPaths = Array.from( + new Set( + requiredPaths + .map((path) => this.#normalizePath(path)) + .filter((path) => path !== '/'), + ), + ); + const requiredEntryMap = new Map(); + const createdEntryMap = new Map(); + if (normalizedRequiredPaths.length === 0) { + return { + requiredEntryMap, + createdEntryMap, + }; + } + + const candidateDirSet = new Set(); + for (const requiredPath of normalizedRequiredPaths) { + let cursor = requiredPath; + while (cursor !== '/') { + candidateDirSet.add(cursor); + cursor = pathPosix.dirname(cursor); + } + } + + const candidatePaths = Array.from(candidateDirSet); + const allEntries = await this.#readEntriesByPathsForUser( + userId, + candidatePaths, + ); + for (const path of candidatePaths) { + const entry = allEntries.get(path); + if (entry && !entry.isDir) { + throw new HttpError(409, `Path is not a directory: ${path}`); + } + } + + const missingPaths = candidatePaths + .filter((path) => !allEntries.has(path)) + .sort( + (pathA, pathB) => + this.#pathDepth(pathA) - this.#pathDepth(pathB), + ); + if (missingPaths.length > 0) { + const uniqueDepths = Array.from( + new Set(missingPaths.map((path) => this.#pathDepth(path))), + ).sort((depthA, depthB) => depthA - depthB); + + for (const depth of uniqueDepths) { + const pathsAtDepth = missingPaths.filter( + (path) => this.#pathDepth(path) === depth, + ); + if (pathsAtDepth.length === 0) { + continue; + } + + const now = Math.floor(Date.now() / 1000); + const insertRows: unknown[] = []; + const valuePlaceholders: string[] = []; + const expectedUuidByPath = new Map(); + for (const dirPath of pathsAtDepth) { + const parentPath = pathPosix.dirname(dirPath); + const parentEntry = + parentPath === '/' ? null : allEntries.get(parentPath); + if (parentPath !== '/' && !parentEntry) { + throw new Error( + `Parent directory not resolved while creating ${dirPath}`, + ); + } + if (parentEntry && !parentEntry.isDir) { + throw new HttpError( + 409, + `Path is not a directory: ${parentPath}`, + ); + } + + const expectedUuid = uuidv4(); + expectedUuidByPath.set(dirPath, expectedUuid); + valuePlaceholders.push( + '(?, ?, ?, ?, ?, ?, 1, ?, ?, ?, 0, 0)', + ); + insertRows.push( + expectedUuid, + userId, + parentEntry ? parentEntry.id : null, + parentEntry ? parentEntry.uuid : null, + pathPosix.basename(dirPath), + dirPath, + now, + now, + now, + ); + } + + try { + await this.clients.db.write( + `${this.#insertIgnoreIntoFsentriesSql()} ( + uuid, + user_id, + parent_id, + parent_uid, + name, + path, + is_dir, + created, + modified, + accessed, + immutable, + size + ) VALUES ${valuePlaceholders.join(', ')}`, + insertRows, + ); + } catch { + // Concurrent create may have already inserted some/all rows. + } + + const insertedEntries = await this.#readEntriesByPathsForUser( + userId, + pathsAtDepth, + { useTryHardRead: true }, + ); + for (const path of pathsAtDepth) { + let insertedEntry = insertedEntries.get(path); + if (!insertedEntry) { + insertedEntry = await this.#ensureDirectoryPath( + path, + userId, + true, + ); + } + if (!insertedEntry.isDir) { + throw new HttpError( + 409, + `Path is not a directory: ${path}`, + ); + } + if (expectedUuidByPath.get(path) === insertedEntry.uuid) { + createdEntryMap.set(path, insertedEntry); + } + allEntries.set(path, insertedEntry); + } + } + } + + for (const requiredPath of normalizedRequiredPaths) { + const entry = allEntries.get(requiredPath); + if (!entry) { + throw new Error( + `Failed to resolve directory path: ${requiredPath}`, + ); + } + if (!entry.isDir) { + throw new HttpError( + 409, + `Path is not a directory: ${requiredPath}`, + ); + } + requiredEntryMap.set(requiredPath, entry); + } + + return { + requiredEntryMap, + createdEntryMap, + }; + } + + async #ensureDirectoryPath( + path: string, + userId: number, + createPaths: boolean, + ): Promise { + const normalizedPath = this.#normalizePath(path); + + const existingEntry = await this.getEntryByPath(normalizedPath); + if (existingEntry) { + if (!existingEntry.isDir) { + throw new HttpError( + 409, + `Path is not a directory: ${normalizedPath}`, + ); + } + return existingEntry; + } + + if (!createPaths) { + throw new HttpError( + 404, + `Parent path does not exist: ${normalizedPath}`, + ); + } + + if (normalizedPath === '/') { + throw new HttpError(400, 'Cannot create root directory'); + } + + const parentPath = pathPosix.dirname(normalizedPath); + const parentEntry = + parentPath === '/' + ? null + : await this.#ensureDirectoryPath(parentPath, userId, true); + const dirName = pathPosix.basename(normalizedPath); + const now = Math.floor(Date.now() / 1000); + + let insertError: unknown = null; + try { + await this.clients.db.write( + `${this.#insertIgnoreIntoFsentriesSql()} ( + uuid, + user_id, + parent_id, + parent_uid, + name, + path, + is_dir, + created, + modified, + accessed, + immutable, + size + ) VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, 0, 0)`, + [ + uuidv4(), + userId, + parentEntry ? parentEntry.id : null, + parentEntry ? parentEntry.uuid : null, + dirName, + normalizedPath, + now, + now, + now, + ], + ); + } catch (err) { + // Most often a concurrent create raced us — the SELECT below + // will pick up the row the other request inserted. If the + // SELECT then comes back empty, surface this error so we can + // tell a real INSERT failure apart from the race. + insertError = err; + } + + const resolvedEntries = await this.#readEntriesByPathsForUser( + userId, + [normalizedPath], + { useTryHardRead: true }, + ); + const resolvedEntry = resolvedEntries.get(normalizedPath) ?? null; + if (!resolvedEntry) { + if (insertError) { + throw new Error( + `Failed to create directory ${normalizedPath}: ${ + insertError instanceof Error + ? insertError.message + : String(insertError) + }`, + ); + } + throw new Error( + `Failed to resolve directory path: ${normalizedPath}`, + ); + } + if (!resolvedEntry.isDir) { + throw new HttpError( + 409, + `Path is not a directory: ${normalizedPath}`, + ); + } + + return resolvedEntry; + } + + #serializeMetadata(input: FSEntryCreateInput): string | null { + if (typeof input.metadata === 'string') { + return input.metadata; + } + + const metadataObject: Record = + input.metadata && typeof input.metadata === 'object' + ? { ...input.metadata } + : {}; + + if (input.contentType) { + metadataObject.contentType = input.contentType; + } + if (input.checksumSha256) { + metadataObject.checksumSha256 = input.checksumSha256; + } + + if (Object.keys(metadataObject).length === 0) { + return null; + } + + return JSON.stringify(metadataObject); + } + + async getEntryByPath( + path: string, + options: ReadEntriesByPathsOptions = {}, + ): Promise { + if (options.useTryHardRead || options.skipCache) { + const normalizedPath = this.#normalizePath(path); + const entriesByPath = await this.#readEntriesByPathsForUser( + 0, + [normalizedPath], + options, + ); + return entriesByPath.get(normalizedPath) ?? null; + } + + const normalizedPath = this.#normalizePath(path); + const cacheKey = `prodfsv2:fsentry:path:any:${normalizedPath}`; + const cached = await this.#readEntryFromCache(cacheKey); + if (cached) { + return cached; + } + + const rows = (await this.clients.db.read( + `SELECT ${this.#selectFsentriesColumns()} FROM fsentries WHERE path = ? LIMIT 1`, + [normalizedPath], + )) as unknown as FSEntryRow[]; + const row = rows[0]; + if (row) { + const entry = this.#mapFSEntryRow(row); + await this.#writeEntryToCache(entry); + return entry; + } + + // Fallback for legacy rows whose `path` column was never populated + // (very old accounts pre-date the path-cache write). Mirrors v1's + // `convert_path_to_fsentry` lineage walk: split the path into + // segments, anchor at `parent_uid IS NULL` matching the first + // segment, then walk down via `parent_uid` chains. Backfills the + // `path` column on every resolved row so subsequent calls take + // the fast `WHERE path = ?` branch above. + return this.#resolveEntryByPathLineage(normalizedPath); + } + + /** + * Recursive-CTE lineage resolver. Returns the entry at `path`, or null + * if any segment is unresolvable. On success, backfills the `path` + * column on every row in the lineage (one UPDATE per row, gated on + * `path IS NULL OR path != expected` so we don't write the same value + * back on warm rows). + * + */ + async #resolveEntryByPathLineage( + normalizedPath: string, + ): Promise { + if (normalizedPath === '/') return null; + const segments = normalizedPath.split('/').filter(Boolean); + if (segments.length === 0) return null; + + // Build the segments CTE as an UNION ALL chain of literal SELECTs. + // Both SQLite and MySQL support recursive CTEs of this shape. + const segmentRows = segments + .map((_, i) => + i === 0 + ? 'SELECT 1 AS depth, ? AS name' + : `UNION ALL SELECT ${i + 1}, ?`, + ) + .join(' '); + const sql = ` + WITH RECURSIVE + segments(depth, name) AS ( + ${segmentRows} + ), + lineage(id, uuid, depth) AS ( + SELECT f.id, f.uuid, 1 + FROM fsentries f + INNER JOIN segments s ON s.depth = 1 + WHERE f.parent_uid IS NULL AND f.name = s.name + + UNION ALL + + SELECT f.id, f.uuid, l.depth + 1 + FROM fsentries f + INNER JOIN lineage l ON f.parent_uid = l.uuid + INNER JOIN segments s ON s.depth = l.depth + 1 + WHERE f.name = s.name + ) + SELECT id, uuid, depth FROM lineage ORDER BY depth ASC, id ASC + `; + + const rows = (await this.clients.db.read(sql, segments)) as { + id: number | string; + uuid: string; + depth: number | string; + }[]; + + // Take the lowest-id row at each depth (deterministic pick that + // matches v1's `LIMIT 1` per-segment behaviour). + const byDepth = new Map(); + for (const r of rows) { + const depth = Number(r.depth); + if (!byDepth.has(depth)) { + byDepth.set(depth, { id: Number(r.id), uuid: r.uuid }); + } + } + if (byDepth.size !== segments.length) { + return null; + } + + await this.#backfillLineagePaths(byDepth, segments); + + const target = byDepth.get(segments.length); + if (!target) return null; + return this.getEntryByUuid(target.uuid); + } + + async #backfillLineagePaths( + byDepth: Map, + segments: string[], + ): Promise { + for (let depth = 1; depth <= segments.length; depth++) { + const row = byDepth.get(depth); + if (!row) continue; + const expected = '/' + segments.slice(0, depth).join('/'); + await this.clients.db.write( + `UPDATE fsentries SET path = ? + WHERE id = ? AND (path IS NULL OR path <> ?)`, + [expected, row.id, expected], + ); + await this.invalidateEntryCacheByUuid(row.uuid); + } + } + + /** + * Upward-walk variant of the lineage resolver. Used by uuid/id lookups + * where we already have the row but its `path` column is NULL — same + * legacy state the path-based resolver heals from the other direction. + * + * Walks `parent_uid` chains from the entry up to the home row + * (`parent_uid IS NULL`), then reuses `#backfillLineagePaths` to write + * the correct `path` on every ancestor + the entry itself. Returns the + * re-fetched target entry so the caller doesn't need a second query. + * + * Returns null if the chain is broken (an intermediate `parent_uid` + * doesn't resolve to a row) — those rows are unrecoverable from this + * direction and the caller should fall back to whatever degraded + * behaviour it had before. + * + * Same security posture as `#resolveEntryByPathLineage`: this method + * only resolves + heals data, it does not authorize access. Callers + * still gate via ACL on the returned entry. + */ + async #healEntryPathByLineageUp( + targetUuid: string, + ): Promise { + const sql = ` + WITH RECURSIVE + ancestors(id, uuid, parent_uid, name, depth) AS ( + SELECT id, uuid, parent_uid, name, 0 + FROM fsentries + WHERE uuid = ? + + UNION ALL + + SELECT f.id, f.uuid, f.parent_uid, f.name, a.depth + 1 + FROM fsentries f + INNER JOIN ancestors a ON f.uuid = a.parent_uid + ) + SELECT id, uuid, parent_uid, name, depth + FROM ancestors + ORDER BY depth ASC + `; + const rows = (await this.clients.db.read(sql, [targetUuid])) as Array<{ + id: number | string; + uuid: string; + parent_uid: string | null; + name: string; + depth: number | string; + }>; + if (rows.length === 0) return null; + + // Highest-depth row should be the home (parent_uid IS NULL). If it + // isn't, the chain dead-ends mid-walk — bail rather than write a + // bogus path. + const root = rows[rows.length - 1]; + if (root.parent_uid !== null) return null; + + // Reverse to root → target order so segments[0] is the home name + // and segments[N-1] is the target name. Then map into the + // (depth → row) shape `#backfillLineagePaths` expects (depth=1 is + // home, depth=N is target). + const ordered = [...rows].reverse(); + const segments = ordered.map((r) => r.name); + const byDepth = new Map(); + for (let i = 0; i < ordered.length; i++) { + byDepth.set(i + 1, { + id: Number(ordered[i].id), + uuid: ordered[i].uuid, + }); + } + + await this.#backfillLineagePaths(byDepth, segments); + + // Re-fetch the target so the caller gets an entry with the now + // populated `path` column (and the full column projection). + const refreshed = (await this.clients.db.read( + `SELECT ${this.#selectFsentriesColumns()} FROM fsentries WHERE uuid = ? LIMIT 1`, + [targetUuid], + )) as unknown as FSEntryRow[]; + if (!refreshed[0]) return null; + return this.#mapFSEntryRow(refreshed[0]); + } + + /** + * Heal in place: for every entry in `entries` whose `path` column is NULL + * (legacy rows that pre-date the path-cache write), run the lineage-up + * resolver and replace the slot with the healed entry. Heals run in + * parallel — typical workloads have zero such rows so the loop is a + * no-op; rare bursts (a freshly migrated account) only pay one + * recursive-CTE round-trip per row instead of N sequential. + * + * Mutates the array in place. + */ + async #healEntriesWithMissingPathsInPlace( + entries: FSEntry[], + ): Promise { + const healJobs: Promise[] = []; + for (let index = 0; index < entries.length; index++) { + if (entries[index].path) continue; + const slot = index; + healJobs.push( + this.#healEntryPathByLineageUp(entries[slot].uuid).then( + (healed) => { + if (healed) entries[slot] = healed; + }, + ), + ); + } + if (healJobs.length === 0) return; + await Promise.all(healJobs); + } + + async getEntriesByPaths(paths: string[]): Promise> { + const normalizedPaths = Array.from( + new Set( + paths + .map((path) => this.#normalizePath(path)) + .filter((path) => path.length > 0), + ), + ); + const entriesByPath = new Map(); + if (normalizedPaths.length === 0) { + return entriesByPath; + } + + const missingPaths: string[] = []; + const cacheReads = await Promise.all( + normalizedPaths.map(async (path) => { + const cacheKey = `prodfsv2:fsentry:path:any:${path}`; + const cachedEntry = await this.#readEntryFromCache(cacheKey); + return { path, cachedEntry }; + }), + ); + for (const { path, cachedEntry } of cacheReads) { + if (cachedEntry) { + entriesByPath.set(path, cachedEntry); + } else { + missingPaths.push(path); + } + } + + if (missingPaths.length > 0) { + const chunks = this.#chunk(missingPaths, BULK_QUERY_CHUNK_SIZE); + const chunkResults = await runWithConcurrencyLimit( + chunks, + DEFAULT_DB_CHUNK_CONCURRENCY, + async (chunk) => { + if (chunk.length === 0) { + return []; + } + const placeholders = chunk.map(() => '?').join(', '); + const rows = (await this.clients.db.read( + `SELECT ${this.#selectFsentriesColumns()} FROM fsentries WHERE path IN (${placeholders})`, + chunk, + )) as unknown as FSEntryRow[]; + const entries = rows.map((row) => this.#mapFSEntryRow(row)); + await Promise.all( + entries.map((entry) => this.#writeEntryToCache(entry)), + ); + return entries; + }, + ); + for (const chunkEntries of chunkResults) { + for (const entry of chunkEntries) { + entriesByPath.set(entry.path, entry); + } + } + } + + return entriesByPath; + } + + async getEntryByUuid(id: string): Promise { + const cacheKey = `prodfsv2:fsentry:uuid:${id}`; + const cached = await this.#readEntryFromCache(cacheKey); + // Treat a cached row with no `path` as a miss — the cache may + // have captured a legacy NULL-path row pre-heal. Falling through + // to the DB read + heal lets the next caller hit the warm cache. + if (cached?.path) return cached; + + const rows = (await this.clients.db.read( + `SELECT ${this.#selectFsentriesColumns()} FROM fsentries WHERE uuid = ? LIMIT 1`, + [id], + )) as unknown as FSEntryRow[]; + const row = rows[0]; + if (!row) { + return null; + } + let entry = this.#mapFSEntryRow(row); + if (!entry.path) { + const healed = await this.#healEntryPathByLineageUp(entry.uuid); + if (healed) entry = healed; + } + await this.#writeEntryToCache(entry); + return entry; + } + + async getEntryById(id: number): Promise { + const cacheKey = `prodfsv2:fsentry:id:${id}`; + const cached = await this.#readEntryFromCache(cacheKey); + if (cached?.path) return cached; + + const rows = (await this.clients.db.read( + `SELECT ${this.#selectFsentriesColumns()} FROM fsentries WHERE id = ? LIMIT 1`, + [id], + )) as unknown as FSEntryRow[]; + const row = rows[0]; + if (!row) { + return null; + } + let entry = this.#mapFSEntryRow(row); + if (!entry.path) { + const healed = await this.#healEntryPathByLineageUp(entry.uuid); + if (healed) entry = healed; + } + await this.#writeEntryToCache(entry); + return entry; + } + + /** + * Batched lookup by id. Dedupes input ids, reads cache via per-id GETs, + * and resolves remaining misses with a single + * `SELECT … WHERE id IN (…)` per chunk. Use this in place of + * `Promise.all(ids.map(getEntryById))` to avoid one connection per row + * on large id sets. + * + * Missing ids (no DB row) are simply absent from the returned map. + */ + async getEntriesByIds(ids: number[]): Promise> { + const result = new Map(); + const uniqueIds = [ + ...new Set( + (Array.isArray(ids) ? ids : []).filter( + (id): id is number => typeof id === 'number', + ), + ), + ]; + if (uniqueIds.length === 0) return result; + + const missingIds: number[] = []; + const cacheReads = await Promise.all( + uniqueIds.map(async (id) => { + const entry = await this.#readEntryFromCache( + `prodfsv2:fsentry:id:${id}`, + ); + return { id, entry }; + }), + ); + for (const { id, entry } of cacheReads) { + // Same NULL-path-cache fallthrough as getEntryById/Uuid. + if (entry?.path) { + result.set(id, entry); + } else { + missingIds.push(id); + } + } + + if (missingIds.length === 0) return result; + + const chunks = this.#chunk(missingIds, BULK_QUERY_CHUNK_SIZE); + const chunkResults = await runWithConcurrencyLimit( + chunks, + DEFAULT_DB_CHUNK_CONCURRENCY, + async (chunk) => { + if (chunk.length === 0) return []; + const placeholders = chunk.map(() => '?').join(', '); + const rows = (await this.clients.db.read( + `SELECT ${this.#selectFsentriesColumns()} FROM fsentries WHERE id IN (${placeholders})`, + chunk, + )) as unknown as FSEntryRow[]; + const entries = rows.map((row) => this.#mapFSEntryRow(row)); + // Heal any legacy NULL-path rows in this chunk before caching. + await this.#healEntriesWithMissingPathsInPlace(entries); + await Promise.all( + entries.map((entry) => this.#writeEntryToCache(entry)), + ); + return entries; + }, + ); + for (const chunkEntries of chunkResults) { + for (const entry of chunkEntries) { + result.set(entry.id, entry); + } + } + + return result; + } + + async updateEntryThumbnailByUuidForUser( + userId: number, + uuid: string, + thumbnail: string | null, + ): Promise { + const now = Math.floor(Date.now() / 1000); + const writeResult = await this.clients.db.write( + `UPDATE fsentries + SET thumbnail = ?, + modified = ?, + accessed = ? + WHERE uuid = ? AND user_id = ?`, + [thumbnail, now, now, uuid, userId], + ); + if (typeof writeResult === 'object' && writeResult !== null) { + const writeResultRecord = writeResult as unknown as Record< + string, + unknown + >; + const anyRowsAffected = writeResultRecord.anyRowsAffected; + if (typeof anyRowsAffected === 'boolean' && !anyRowsAffected) { + throw new HttpError( + 404, + 'File entry was not found for thumbnail update', + ); + } + + const affectedRowsRaw = writeResultRecord.affectedRows; + const affectedRows = Number(affectedRowsRaw); + if ( + affectedRowsRaw !== undefined && + Number.isFinite(affectedRows) && + affectedRows <= 0 + ) { + throw new HttpError( + 404, + 'File entry was not found for thumbnail update', + ); + } + } + + const refreshedRows = (await this.clients.db.tryHardRead( + `SELECT ${this.#selectFsentriesColumns()} FROM fsentries WHERE uuid = ? AND user_id = ? LIMIT 1`, + [uuid, userId], + )) as unknown as FSEntryRow[]; + const refreshedRow = refreshedRows[0]; + if (!refreshedRow) { + throw new HttpError( + 404, + 'File entry was not found for thumbnail update', + ); + } + + const updatedEntry = this.#mapFSEntryRow(refreshedRow); + await this.#invalidateEntryCache(updatedEntry); + await this.#writeEntryToCache(updatedEntry); + return updatedEntry; + } + + async resolveParentDirectory( + userId: number, + parentPath: string, + createPaths: boolean, + ): Promise { + return this.#ensureDirectoryPath(parentPath, userId, createPaths); + } + + async getEntriesByPathsForUser( + userId: number, + paths: string[], + options: ReadEntriesByPathsOptions = {}, + ): Promise<(FSEntry | null)[]> { + const username = options.crossNamespace + ? null + : ((await this.stores.user.getById(userId))?.username ?? null); + const entriesByPath = await this.#readEntriesByPathsForUser( + userId, + paths, + options, + ); + return paths.map((path) => { + const normalizedPath = this.#normalizePath(path); + // Namespace check applied per-path so a single batch can mix + // in/out-of-namespace inputs without leaking out-of-namespace hits. + if ( + !options.crossNamespace && + !this.#pathInUserNamespace(normalizedPath, username) + ) { + return null; + } + return entriesByPath.get(normalizedPath) ?? null; + }); + } + + #pathInUserNamespace(path: string, username: string | null): boolean { + if (!username) return false; + const normalized = this.#normalizePath(path); + const root = `/${username}`; + return normalized === root || normalized.startsWith(`${root}/`); + } + + async resolveParentDirectoriesBatch( + userId: number, + requests: { parentPath: string; createPaths: boolean }[], + ): Promise { + const { parentEntries } = + await this.resolveParentDirectoriesBatchWithCreated( + userId, + requests, + ); + return parentEntries; + } + + async resolveParentDirectoriesBatchWithCreated( + userId: number, + requests: { parentPath: string; createPaths: boolean }[], + ): Promise<{ + parentEntries: FSEntry[]; + createdDirectoryEntries: FSEntry[]; + }> { + if (requests.length === 0) { + return { + parentEntries: [], + createdDirectoryEntries: [], + }; + } + + const parentPathsToEnsure = requests + .filter((request) => request.createPaths) + .map((request) => request.parentPath); + const { createdEntryMap } = await this.#ensureDirectoryPathsForUser( + userId, + parentPathsToEnsure, + ); + + const allParentPaths = requests.map((request) => request.parentPath); + const parentEntriesByPath = await this.#readEntriesByPathsForUser( + userId, + allParentPaths, + ); + const parentEntries = allParentPaths.map((path) => { + const normalizedPath = this.#normalizePath(path); + const parentEntry = parentEntriesByPath.get(normalizedPath); + if (!parentEntry) { + throw new HttpError( + 404, + `Parent path does not exist: ${normalizedPath}`, + ); + } + if (!parentEntry.isDir) { + throw new HttpError( + 409, + `Path is not a directory: ${normalizedPath}`, + ); + } + return parentEntry; + }); + + return { + parentEntries, + createdDirectoryEntries: Array.from(createdEntryMap.values()), + }; + } + + async ensureDirectoriesForUser( + userId: number, + requests: { path: string; createPaths: boolean }[], + ): Promise { + const { entries } = await this.ensureDirectoriesForUserWithCreated( + userId, + requests, + ); + return entries; + } + + async ensureDirectoriesForUserWithCreated( + userId: number, + requests: { path: string; createPaths: boolean }[], + ): Promise<{ + entries: FSEntry[]; + createdDirectoryEntries: FSEntry[]; + }> { + if (requests.length === 0) { + return { + entries: [], + createdDirectoryEntries: [], + }; + } + + const normalizedRequests = requests.map((request) => { + const normalizedPath = this.#normalizePath(request.path); + if (normalizedPath === '/') { + throw new HttpError(400, 'Cannot create root directory'); + } + return { + path: normalizedPath, + createPaths: request.createPaths, + }; + }); + + const pathsToEnsure = normalizedRequests + .filter((request) => request.createPaths) + .map((request) => request.path); + const { createdEntryMap } = await this.#ensureDirectoryPathsForUser( + userId, + pathsToEnsure, + ); + + const allPaths = normalizedRequests.map((request) => request.path); + const entriesByPath = await this.#readEntriesByPathsForUser( + userId, + allPaths, + ); + + const entries = normalizedRequests.map((request) => { + const entry = entriesByPath.get(request.path); + if (!entry) { + throw new HttpError( + 404, + `Directory path does not exist: ${request.path}`, + ); + } + if (!entry.isDir) { + throw new HttpError( + 409, + `Path is not a directory: ${request.path}`, + ); + } + return entry; + }); + + return { + entries, + createdDirectoryEntries: Array.from(createdEntryMap.values()), + }; + } + + async createEntry( + fsEntry: FSEntryCreateInput, + createPaths = true, + ): Promise { + const [entry] = await this.batchCreateEntries([fsEntry], createPaths); + if (!entry) { + throw new Error('Failed to create entry'); + } + return entry; + } + + async batchCreateEntries( + entries: FSEntryCreateInput[], + createPaths = true, + ): Promise { + if (entries.length === 0) { + return []; + } + + const normalizedEntries: NormalizedEntryWrite[] = entries.map( + (entryInput, index) => { + const targetPath = this.#normalizePath(entryInput.path); + if (targetPath === '/') { + throw new HttpError(400, 'Cannot write to root path'); + } + + const parentPath = this.#normalizePath( + pathPosix.dirname(targetPath), + ); + if (parentPath === '/') { + throw new HttpError( + 400, + 'Cannot write directly under root path', + ); + } + + const size = Number(entryInput.size); + if (Number.isNaN(size) || size < 0) { + throw new HttpError( + 400, + `Invalid size for path ${targetPath}`, + ); + } + + return { + index, + input: entryInput, + userId: entryInput.userId, + targetPath, + parentPath, + fileName: pathPosix.basename(targetPath), + metadataJson: this.#serializeMetadata(entryInput), + bucket: entryInput.bucket ?? null, + bucketRegion: entryInput.bucketRegion ?? null, + size, + createPaths: entryInput.createMissingParents ?? createPaths, + }; + }, + ); + + const duplicatePathSet = new Set(); + for (const normalizedEntry of normalizedEntries) { + const dedupeKey = `${normalizedEntry.userId}:${normalizedEntry.targetPath}`; + if (duplicatePathSet.has(dedupeKey)) { + throw new HttpError( + 409, + `Batch contains duplicate target path: ${normalizedEntry.targetPath}`, + ); + } + duplicatePathSet.add(dedupeKey); + } + + const entriesByUser = new Map(); + for (const normalizedEntry of normalizedEntries) { + const userEntries = entriesByUser.get(normalizedEntry.userId) ?? []; + userEntries.push(normalizedEntry); + entriesByUser.set(normalizedEntry.userId, userEntries); + } + + const resultsByIndex = new Map(); + for (const [userId, userEntries] of entriesByUser) { + const parentEntries = await this.resolveParentDirectoriesBatch( + userId, + userEntries.map((entry) => ({ + parentPath: entry.parentPath, + createPaths: entry.createPaths, + })), + ); + const parentByPath = new Map(); + for (const parentEntry of parentEntries) { + parentByPath.set(parentEntry.path, parentEntry); + } + + const existingEntriesByPath = await this.#readEntriesByPathsForUser( + userId, + userEntries.map((entry) => entry.targetPath), + { + useTryHardRead: true, + skipCache: true, + }, + ); + + const now = Math.floor(Date.now() / 1000); + const updateOperations: Array<{ + existingEntry: FSEntry; + updatedEntry: FSEntry; + promise: Promise; + }> = []; + const updatedResultsByIndex = new Map(); + const insertCandidates: NormalizedEntryWrite[] = []; + + for (const entry of userEntries) { + const parentEntry = parentByPath.get(entry.parentPath); + if (!parentEntry) { + throw new Error( + `Failed to resolve parent directory for ${entry.targetPath}`, + ); + } + + const existingEntry = existingEntriesByPath.get( + entry.targetPath, + ); + if (existingEntry) { + if (!entry.input.overwrite) { + throw new HttpError( + 409, + `Entry already exists at ${entry.targetPath}`, + ); + } + if (existingEntry.isDir) { + throw new HttpError( + 409, + `Cannot overwrite a directory at ${entry.targetPath}`, + ); + } + + const updatedEntry = { + ...existingEntry, + bucket: entry.bucket, + bucketRegion: entry.bucketRegion, + parentId: parentEntry.id, + parentUid: parentEntry.uuid, + associatedAppId: entry.input.associatedAppId ?? null, + isPublic: + entry.input.isPublic === undefined + ? null + : Boolean(entry.input.isPublic), + thumbnail: entry.input.thumbnail ?? null, + immutable: Boolean(entry.input.immutable), + name: entry.fileName, + path: entry.targetPath, + metadata: entry.metadataJson, + modified: now, + accessed: now, + size: entry.size, + }; + updateOperations.push({ + existingEntry, + updatedEntry, + promise: this.clients.db.write( + `UPDATE fsentries + SET bucket = ?, + bucket_region = ?, + parent_id = ?, + parent_uid = ?, + associated_app_id = ?, + is_public = ?, + thumbnail = ?, + immutable = ?, + name = ?, + path = ?, + metadata = ?, + modified = ?, + accessed = ?, + size = ? + WHERE id = ?`, + [ + entry.bucket, + entry.bucketRegion, + parentEntry.id, + parentEntry.uuid, + entry.input.associatedAppId ?? null, + entry.input.isPublic === undefined + ? null + : entry.input.isPublic + ? 1 + : 0, + entry.input.thumbnail ?? null, + entry.input.immutable ? 1 : 0, + entry.fileName, + entry.targetPath, + entry.metadataJson, + now, + now, + entry.size, + existingEntry.id, + ], + ), + }); + updatedResultsByIndex.set(entry.index, updatedEntry); + continue; + } + + insertCandidates.push(entry); + } + + if (updateOperations.length > 0) { + const updateResults = await Promise.allSettled( + updateOperations.map((operation) => operation.promise), + ); + const successfulUpdateOperations = updateResults.flatMap( + (result, index) => { + if (result.status !== 'fulfilled') { + return []; + } + const operation = updateOperations[index]; + return operation ? [operation] : []; + }, + ); + if (successfulUpdateOperations.length > 0) { + await Promise.all( + successfulUpdateOperations.map((operation) => { + return this.#invalidateEntryCache( + operation.existingEntry, + ); + }), + ); + await Promise.all( + successfulUpdateOperations.map((operation) => { + return this.#writeEntryToCache( + operation.updatedEntry, + ); + }), + ); + } + + const failedUpdate = updateResults.find( + (result) => result.status === 'rejected', + ); + if (failedUpdate?.status === 'rejected') { + throw failedUpdate.reason instanceof Error + ? failedUpdate.reason + : new Error('Failed to update fsentries batch'); + } + } + + const insertChunks = this.#chunk( + insertCandidates, + BULK_QUERY_CHUNK_SIZE, + ); + await runWithConcurrencyLimit( + insertChunks, + DEFAULT_DB_CHUNK_CONCURRENCY, + async (insertChunk) => { + if (insertChunk.length === 0) { + return; + } + + const valuePlaceholders: string[] = []; + const values: unknown[] = []; + for (const entry of insertChunk) { + const parentEntry = parentByPath.get(entry.parentPath); + if (!parentEntry) { + throw new Error( + `Failed to resolve parent directory for ${entry.targetPath}`, + ); + } + + valuePlaceholders.push( + '(?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', + ); + values.push( + entry.input.uuid, + entry.bucket, + entry.bucketRegion, + userId, + parentEntry.id, + parentEntry.uuid, + entry.input.associatedAppId ?? null, + entry.input.isPublic === undefined + ? null + : entry.input.isPublic + ? 1 + : 0, + entry.input.thumbnail ?? null, + entry.input.immutable ? 1 : 0, + entry.fileName, + entry.targetPath, + entry.metadataJson, + now, + now, + now, + entry.size, + ); + } + + await this.clients.db.write( + `INSERT INTO fsentries ( + uuid, + bucket, + bucket_region, + user_id, + parent_id, + parent_uid, + associated_app_id, + is_dir, + is_public, + thumbnail, + immutable, + name, + path, + metadata, + modified, + created, + accessed, + size + ) VALUES ${valuePlaceholders.join(', ')}`, + values, + ); + }, + ); + + const insertedEntriesByUuid = new Map(); + if (insertCandidates.length > 0) { + const insertUuidChunks = this.#chunk( + insertCandidates.map((entry) => entry.input.uuid), + BULK_QUERY_CHUNK_SIZE, + ); + + const insertedChunkResults = await runWithConcurrencyLimit( + insertUuidChunks, + DEFAULT_DB_CHUNK_CONCURRENCY, + async (insertUuidChunk) => { + if (insertUuidChunk.length === 0) { + return []; + } + + const placeholders = insertUuidChunk + .map(() => '?') + .join(', '); + const rows = (await this.clients.db.tryHardRead( + `SELECT ${this.#selectFsentriesColumns()} FROM fsentries WHERE user_id = ? AND uuid IN (${placeholders})`, + [userId, ...insertUuidChunk], + )) as unknown as FSEntryRow[]; + + const insertedEntries = rows.map((row) => + this.#mapFSEntryRow(row), + ); + if (insertedEntries.length > 0) { + await Promise.all( + insertedEntries.map((entry) => + this.#writeEntryToCache(entry), + ), + ); + } + return insertedEntries; + }, + ); + for (const insertedEntries of insertedChunkResults) { + for (const insertedEntry of insertedEntries) { + insertedEntriesByUuid.set( + insertedEntry.uuid, + insertedEntry, + ); + } + } + } + + for (const entry of userEntries) { + const updatedResult = updatedResultsByIndex.get(entry.index); + if (updatedResult) { + resultsByIndex.set(entry.index, updatedResult); + continue; + } + + const insertedResult = insertedEntriesByUuid.get( + entry.input.uuid, + ); + if (insertedResult) { + resultsByIndex.set(entry.index, insertedResult); + continue; + } + + throw new Error( + `Failed to load final entry for ${entry.targetPath}`, + ); + } + } + + const createdEntries: FSEntry[] = []; + for (let index = 0; index < entries.length; index++) { + const entry = resultsByIndex.get(index); + if (!entry) { + throw new Error( + `Failed to resolve entry result at index ${index}`, + ); + } + createdEntries.push(entry); + } + return createdEntries; + } + + async createPendingEntry( + entry: PendingUploadCreateInput, + ): Promise { + const [createdEntry] = await this.batchCreatePendingEntries([entry]); + if (!createdEntry) { + throw new Error('Failed to create pending upload entry'); + } + return createdEntry; + } + + async batchCreatePendingEntries( + entries: PendingUploadCreateInput[], + ): Promise { + if (entries.length === 0) { + return []; + } + const now = Date.now(); + const pendingSessions = entries.map((entry) => + toPendingUploadSession(entry, now), + ); + await this.#writePendingUploadSessions( + pendingSessions, + 'create pending upload sessions', + ); + return pendingSessions; + } + + async getPendingEntryBySessionId( + sessionId: string, + ): Promise { + // SystemKVStore returns `{ res, usage }`. Hand the raw value (res) + // to the normalizer — passing the envelope would trip + // `isPendingUploadSession` and silently 404 the session. + const { res } = await this.stores.kv.get({ + key: toPendingUploadSessionKey(sessionId), + }); + return normalizePendingUploadSession(res, sessionId); + } + + async getPendingEntriesBySessionIds( + sessionIds: string[], + ): Promise<(PendingUploadSession | null)[]> { + if (sessionIds.length === 0) { + return []; + } + + const entriesBySessionId = + await this.#getPendingUploadSessionsBySessionIds(sessionIds); + return sessionIds.map( + (sessionId) => entriesBySessionId.get(sessionId) ?? null, + ); + } + + async markPendingEntryCompleted(sessionId: string): Promise { + await this.#markPendingSessionsWithStatus( + [sessionId], + 'completed', + null, + ); + } + + async markPendingEntryFailed( + sessionId: string, + reason: string, + ): Promise { + await this.#markPendingSessionsWithStatus( + [sessionId], + 'failed', + reason, + ); + } + + async markPendingEntriesFailed( + sessionIds: string[], + reason: string, + ): Promise { + await this.#markPendingSessionsWithStatus(sessionIds, 'failed', reason); + } + + async abortPendingEntry(sessionId: string, reason: string): Promise { + await this.#markPendingSessionsWithStatus( + [sessionId], + 'aborted', + reason, + ); + } + + async completePendingEntry( + sessionId: string, + finalData: FSEntryCreateInput, + ): Promise { + const [completedEntry] = await this.batchCompletePendingEntries([ + { sessionId, finalData }, + ]); + if (!completedEntry) { + throw new Error('Failed to complete pending entry'); + } + return completedEntry; + } + + async batchCompletePendingEntries( + entries: { sessionId: string; finalData: FSEntryCreateInput }[], + ): Promise { + if (entries.length === 0) { + return []; + } + + const completedEntries = await this.batchCreateEntries( + entries.map((entry) => entry.finalData), + true, + ); + + await this.#markPendingSessionsWithStatus( + entries.map((entry) => entry.sessionId), + 'completed', + null, + ); + + return completedEntries; + } + + // ── Non-file entry creation (dirs, shortcuts, symlinks, touch) ────── + + /** + * Create a single non-file entry: directory, shortcut, or symlink. + * + * Unlike `batchCreateEntries` (which is geared to S3-backed files), + * these rows carry no bucket metadata. The caller is responsible for + * parent/name conflict resolution — this method assumes the parent + * exists and the target name is free. + * + * Returns the inserted entry with a refreshed row read. Throws 409 on + * a unique-key collision (caller should pre-check and dedupe). + */ + async createNonFileEntry(input: { + userId: number; + parent: FSEntry; + name: string; + kind: 'directory' | 'shortcut' | 'symlink' | 'empty-file'; + shortcutTo?: number | null; + symlinkPath?: string | null; + associatedAppId?: number | null; + metadata?: string | null; + immutable?: boolean; + isPublic?: boolean | null; + thumbnail?: string | null; + }): Promise { + const uuid = uuidv4(); + const now = Math.floor(Date.now() / 1000); + const parentPath = this.#normalizePath(input.parent.path); + const path = + parentPath === '/' + ? `/${input.name}` + : `${parentPath}/${input.name}`; + + const isDir = input.kind === 'directory' ? 1 : 0; + const isShortcut = input.kind === 'shortcut' ? 1 : 0; + const isSymlink = input.kind === 'symlink' ? 1 : 0; + + await this.clients.db.write( + `INSERT INTO fsentries ( + uuid, + user_id, + parent_id, + parent_uid, + name, + path, + is_dir, + is_shortcut, + shortcut_to, + is_symlink, + symlink_path, + associated_app_id, + metadata, + thumbnail, + immutable, + is_public, + created, + modified, + accessed, + size + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + uuid, + input.userId, + input.parent.id, + input.parent.uuid, + input.name, + path, + isDir, + isShortcut, + input.shortcutTo ?? null, + isSymlink, + input.symlinkPath ?? null, + input.associatedAppId ?? null, + input.metadata ?? null, + input.thumbnail ?? null, + input.immutable ? 1 : 0, + input.isPublic === undefined || input.isPublic === null + ? null + : input.isPublic + ? 1 + : 0, + now, + now, + now, + 0, + ], + ); + + const rows = (await this.clients.db.tryHardRead( + `SELECT ${this.#selectFsentriesColumns()} FROM fsentries WHERE uuid = ? LIMIT 1`, + [uuid], + )) as unknown as FSEntryRow[]; + const row = rows[0]; + if (!row) { + throw new HttpError(500, 'Failed to read created entry'); + } + const entry = this.#mapFSEntryRow(row); + await this.#writeEntryToCache(entry); + return entry; + } + + /** + * Update accessed/modified/created timestamps in place. Used by `touch` + * for entries that already exist. + */ + async touchEntryTimestamps( + uuid: string, + options: { + setAccessed?: boolean; + setModified?: boolean; + setCreated?: boolean; + }, + ): Promise { + const now = Math.floor(Date.now() / 1000); + const assignments: string[] = []; + const values: unknown[] = []; + if (options.setAccessed) { + assignments.push('accessed = ?'); + values.push(now); + } + if (options.setModified) { + assignments.push('modified = ?'); + values.push(now); + } + if (options.setCreated) { + assignments.push('created = ?'); + values.push(now); + } + if (assignments.length === 0) { + // Default: touch all three. + assignments.push('accessed = ?', 'modified = ?', 'created = ?'); + values.push(now, now, now); + } + await this.clients.db.write( + `UPDATE fsentries SET ${assignments.join(', ')} WHERE uuid = ?`, + [...values, uuid], + ); + const entry = await this.getEntryByUuid(uuid); + if (!entry) throw new HttpError(404, 'Entry not found after touch'); + await this.#invalidateEntryCache(entry); + await this.#writeEntryToCache(entry); + return entry; + } + + // ── Listing / descendants / search ────────────────────────────────── + + // Children of a directory (direct children only). Paginated + sortable. + async listChildren( + parentUid: string, + options: { + limit?: number; + offset?: number; + sortBy?: 'name' | 'modified' | 'type' | 'size' | null; + sortOrder?: 'asc' | 'desc' | null; + } = {}, + ): Promise { + const limit = Number.isFinite(options.limit) + ? Math.max(1, Math.min(10_000, Number(options.limit))) + : 10_000; + const offset = Number.isFinite(options.offset) + ? Math.max(0, Number(options.offset)) + : 0; + + // Map sort field to a safe column name; reject anything else. + const sortColumn = (() => { + switch (options.sortBy) { + case 'modified': + return 'modified'; + case 'size': + return 'size'; + case 'type': + return 'is_dir'; // directories first when DESC + case 'name': + default: + return 'name'; + } + })(); + const sortDirection = options.sortOrder === 'desc' ? 'DESC' : 'ASC'; + + const rows = (await this.clients.db.read( + `SELECT ${this.#selectFsentriesColumns()} + FROM fsentries + WHERE parent_uid = ? + ORDER BY ${sortColumn} ${sortDirection} + LIMIT ? OFFSET ?`, + [parentUid, limit, offset], + )) as unknown as FSEntryRow[]; + const entries = rows.map((row) => this.#mapFSEntryRow(row)); + // Heal NULL-path rows before they reach the controller — readdir + // consumers (GUI, suggested-apps, downstream `pathPosix.dirname` + // callers) all expect a populated path on every child. + await this.#healEntriesWithMissingPathsInPlace(entries); + await Promise.all( + entries.map((entry) => this.#writeEntryToCache(entry)), + ); + return entries; + } + + // Escape a string for safe use inside a LIKE pattern. We use backslash as + // the LIKE escape char so `%` and `_` in user paths aren't treated as wildcards. + // Uses `!` as the LIKE escape character — both MySQL and SQLite treat `!` as + // a plain character inside string literals, so no dialect-specific quoting. + #escapeLikePattern(value: string): string { + return value.replace(/([!%_])/g, '!$1'); + } + + // All descendants of a directory path (recursive). Paths in fsentries are + // absolute and don't carry a trailing slash, so the prefix pattern is + // `${prefix}/%`. Scoped by user_id to keep the index tight. + async listDescendantsByPath( + userId: number, + pathPrefix: string, + ): Promise { + const normalizedPrefix = this.#normalizePath(pathPrefix); + if (normalizedPrefix === '/') { + // Refuse to list all user entries this way — caller must mean something else. + throw new HttpError(400, 'Refusing to list descendants of root'); + } + const likePattern = `${this.#escapeLikePattern(normalizedPrefix)}/%`; + const rows = (await this.clients.db.read( + `SELECT ${this.#selectFsentriesColumns()} FROM fsentries WHERE user_id = ? AND path LIKE ? ESCAPE '!' ORDER BY path ASC`, + [userId, likePattern], + )) as unknown as FSEntryRow[]; + return rows.map((row) => this.#mapFSEntryRow(row)); + } + + async countDescendantsByPath( + userId: number, + pathPrefix: string, + ): Promise { + const normalizedPrefix = this.#normalizePath(pathPrefix); + if (normalizedPrefix === '/') return 0; + const likePattern = `${this.#escapeLikePattern(normalizedPrefix)}/%`; + const rows = (await this.clients.db.read( + "SELECT COUNT(*) AS n FROM fsentries WHERE user_id = ? AND path LIKE ? ESCAPE '!'", + [userId, likePattern], + )) as unknown as { n: number | string }[]; + return Number(rows[0]?.n ?? 0); + } + + // Sum of sizes under a path (inclusive). Files only — dirs have null size. + // NOTE: linear scan under the path prefix index; optimize later if it + // becomes hot (e.g., incremental size counters or materialized totals). + async getSubtreeSize(userId: number, pathPrefix: string): Promise { + const normalizedPrefix = this.#normalizePath(pathPrefix); + const likePattern = + normalizedPrefix === '/' + ? '/%' + : `${this.#escapeLikePattern(normalizedPrefix)}/%`; + const rows = (await this.clients.db.read( + "SELECT COALESCE(SUM(size), 0) AS total FROM fsentries WHERE user_id = ? AND (path = ? OR path LIKE ? ESCAPE '!')", + [userId, normalizedPrefix, likePattern], + )) as unknown as { total: number | string }[]; + return Number(rows[0]?.total ?? 0); + } + + // Simple case-insensitive substring search on name, scoped to one user. + async searchByNameForUser( + userId: number, + query: string, + limit = 200, + ): Promise { + const q = query.trim(); + if (q.length === 0) return []; + const likePattern = `%${this.#escapeLikePattern(q)}%`; + const capped = Math.max(1, Math.min(1000, Math.floor(limit))); + const rows = (await this.clients.db.read( + `SELECT ${this.#selectFsentriesColumns()} FROM fsentries + WHERE user_id = ? AND name LIKE ? ESCAPE '!' + ORDER BY modified DESC + LIMIT ${capped}`, + [userId, likePattern], + )) as unknown as FSEntryRow[]; + const entries = rows.map((row) => this.#mapFSEntryRow(row)); + // Search matches by `name`, so legacy NULL-path rows can land here. + // Heal before returning so clients render `path`/`dirname` correctly. + await this.#healEntriesWithMissingPathsInPlace(entries); + return entries; + } + + // ── Mutation ─────────────────────────────────────────────────────── + + // Generic single-entry update. Only a narrow set of columns are patchable + // through this method; the caller provides the JS-shaped patch and we map. + async updateEntry( + uuid: string, + patch: { + name?: string; + path?: string; + parentId?: number | null; + parentUid?: string | null; + thumbnail?: string | null; + metadata?: string | null; + isPublic?: boolean | null; + immutable?: boolean; + associatedAppId?: number | null; + layout?: string | null; + sortBy?: 'name' | 'modified' | 'type' | 'size' | null; + sortOrder?: 'asc' | 'desc' | null; + size?: number | null; + accessed?: number | null; + modified?: number | null; + }, + ): Promise { + const assignments: string[] = []; + const values: unknown[] = []; + const push = (column: string, value: unknown) => { + assignments.push(`${column} = ?`); + values.push(value); + }; + + if (patch.name !== undefined) push('name', patch.name); + if (patch.path !== undefined) push('path', patch.path); + if (patch.parentId !== undefined) push('parent_id', patch.parentId); + if (patch.parentUid !== undefined) push('parent_uid', patch.parentUid); + if (patch.thumbnail !== undefined) push('thumbnail', patch.thumbnail); + if (patch.metadata !== undefined) push('metadata', patch.metadata); + if (patch.isPublic !== undefined) + push( + 'is_public', + patch.isPublic === null ? null : patch.isPublic ? 1 : 0, + ); + if (patch.immutable !== undefined) + push('immutable', patch.immutable ? 1 : 0); + if (patch.associatedAppId !== undefined) + push('associated_app_id', patch.associatedAppId); + if (patch.layout !== undefined) push('layout', patch.layout); + if (patch.sortBy !== undefined) push('sort_by', patch.sortBy); + if (patch.sortOrder !== undefined) push('sort_order', patch.sortOrder); + if (patch.size !== undefined) push('size', patch.size); + if (patch.accessed !== undefined) push('accessed', patch.accessed); + // Always bump modified unless caller provides it explicitly. + push('modified', patch.modified ?? Math.floor(Date.now() / 1000)); + + if (assignments.length === 0) { + const existing = await this.getEntryByUuid(uuid); + if (!existing) throw new HttpError(404, 'Entry not found'); + return existing; + } + + await this.clients.db.write( + `UPDATE fsentries SET ${assignments.join(', ')} WHERE uuid = ?`, + [...values, uuid], + ); + + const refreshedRows = (await this.clients.db.tryHardRead( + `SELECT ${this.#selectFsentriesColumns()} FROM fsentries WHERE uuid = ? LIMIT 1`, + [uuid], + )) as unknown as FSEntryRow[]; + const row = refreshedRows[0]; + if (!row) { + throw new HttpError(404, 'Entry not found after update'); + } + const updated = this.#mapFSEntryRow(row); + await this.#invalidateEntryCache(updated); + await this.#writeEntryToCache(updated); + return updated; + } + + // Authoritative root lookup. A user's home directory is the row with + // `parent_uid IS NULL AND user_id = ?`, regardless of what `path`/`name` + // currently are — legacy rows may have drifted (e.g. stale username after + // a rename that didn't cascade). Callers use this to heal or rename. + async getRootEntryForUser(userId: number): Promise { + const rows = (await this.clients.db.read( + `SELECT ${this.#selectFsentriesColumns()} FROM fsentries + WHERE user_id = ? AND parent_uid IS NULL AND is_dir = 1 + ORDER BY id ASC LIMIT 1`, + [userId], + )) as unknown as FSEntryRow[]; + const row = rows[0]; + if (!row) return null; + const entry = this.#mapFSEntryRow(row); + await this.#writeEntryToCache(entry); + return entry; + } + + // Heal a user's home tree to `/{username}`: if the root entry's path/name + // already match, no-op; otherwise rewrite the root row and cascade the + // prefix to descendants. Used by the username change flow AND by the + // on-read backfill for legacy users whose root row drifted (or whose + // path column was never populated). + async renameUserHome( + userId: number, + newUsername: string, + ): Promise { + const root = await this.getRootEntryForUser(userId); + if (!root) return null; + + const newPath = `/${newUsername}`; + if (root.path === newPath && root.name === newUsername) { + return root; + } + + const oldPath = root.path; + const now = Math.floor(Date.now() / 1000); + + await this.clients.db.write( + `UPDATE fsentries SET name = ?, path = ?, modified = ? + WHERE id = ?`, + [newUsername, newPath, now, root.id], + ); + + if (oldPath && oldPath !== '/' && oldPath !== newPath) { + const likePattern = `${this.#escapeLikePattern(oldPath)}/%`; + const oldLen = oldPath.length; + await this.clients.db.write( + `UPDATE fsentries + SET path = CONCAT(?, SUBSTR(path, ?)), + modified = ? + WHERE user_id = ? AND path LIKE ? ESCAPE '!'`, + [newPath, oldLen + 1, now, userId, likePattern], + ); + } + + // Invalidate root cache under both old and new keys; descendants + // rely on TTL (60s) to refresh — username rename is rare enough + // that a broad subtree invalidation isn't worth the round-trips. + await this.#invalidateEntryCache(root); + const refreshedRows = (await this.clients.db.tryHardRead( + `SELECT ${this.#selectFsentriesColumns()} FROM fsentries WHERE id = ? LIMIT 1`, + [root.id], + )) as unknown as FSEntryRow[]; + const refreshed = refreshedRows[0] + ? this.#mapFSEntryRow(refreshedRows[0]) + : null; + if (refreshed) { + await this.#invalidateEntryCache(refreshed); + await this.#writeEntryToCache(refreshed); + } + return refreshed; + } + + // Rewrites path column for every descendant of `oldPrefix` to use `newPrefix`. + // Used by move/rename when a directory is relocated. Cache for affected + // entries is invalidated coarsely afterwards by the caller. + async updatePathPrefixForUser( + userId: number, + oldPrefix: string, + newPrefix: string, + ): Promise { + const normalizedOld = this.#normalizePath(oldPrefix); + const normalizedNew = this.#normalizePath(newPrefix); + if (normalizedOld === '/' || normalizedNew === '/') { + throw new HttpError(400, 'Cannot rewrite path prefix to/from root'); + } + if (normalizedOld === normalizedNew) return 0; + + const likePattern = `${this.#escapeLikePattern(normalizedOld)}/%`; + const now = Math.floor(Date.now() / 1000); + + // CONCAT(?, SUBSTR(path, ? + 1)) to rewrite just the prefix portion. + const oldPrefixLen = normalizedOld.length; + const result = await this.clients.db.write( + `UPDATE fsentries + SET path = CONCAT(?, SUBSTR(path, ?)), + modified = ? + WHERE user_id = ? AND path LIKE ? ESCAPE '!'`, + [normalizedNew, oldPrefixLen + 1, now, userId, likePattern], + ); + const affected = this.#affectedRows(result); + return affected; + } + + async deleteEntry(entry: FSEntry): Promise { + await this.clients.db.write('DELETE FROM fsentries WHERE id = ?', [ + entry.id, + ]); + await this.#invalidateEntryCache(entry); + } + + async deleteEntries(entries: FSEntry[]): Promise { + if (entries.length === 0) return; + const chunks = this.#chunk(entries, BULK_QUERY_CHUNK_SIZE); + await runWithConcurrencyLimit( + chunks, + DEFAULT_DB_CHUNK_CONCURRENCY, + async (chunk) => { + const ids = chunk.map((entry) => entry.id); + const placeholders = ids.map(() => '?').join(', '); + await this.clients.db.write( + `DELETE FROM fsentries WHERE id IN (${placeholders})`, + ids, + ); + }, + ); + // Invalidate caches for all removed entries (best effort). + await Promise.all( + entries.map((entry) => this.#invalidateEntryCache(entry)), + ); + } + + #affectedRows(writeResult: unknown): number { + if (typeof writeResult !== 'object' || writeResult === null) return 0; + const record = writeResult as Record; + const affected = Number(record.affectedRows ?? record.changes ?? 0); + return Number.isFinite(affected) ? affected : 0; + } + + async getUserStorageAllowance( + userId: number, + ): Promise<{ curr: number; max: number }> { + const [usageRows, userRows] = await Promise.all([ + this.clients.db.read( + 'SELECT COALESCE(SUM(size), 0) AS totalUsage FROM fsentries WHERE user_id = ?', + [userId], + ) as Promise<{ totalUsage: number }[]>, + this.clients.db.read( + 'SELECT free_storage AS freeStorage FROM user WHERE id = ? LIMIT 1', + [userId], + ) as Promise<{ freeStorage: number | null }[]>, + ]); + const usageRow = usageRows[0]; + const userRow = userRows[0]; + + const curr = Number(usageRow?.totalUsage ?? 0); + let max = Number( + userRow?.freeStorage ?? this.config.storage_capacity ?? 0, + ); + + const event: { userId: number; extra: number } = { userId, extra: 0 }; + try { + await this.clients.event.emitAndWait( + 'storage.quota.bonus', + event, + {}, + ); + } catch { + /* best-effort */ + } + if (Number.isFinite(event.extra) && event.extra > 0) { + max += event.extra; + } + + if (!this.config.is_storage_limited) { + const availableDeviceStorage = Number( + this.config.available_device_storage ?? 0, + ); + if (availableDeviceStorage > 0) { + max = availableDeviceStorage; + } else { + const freeOnDisk = await this.#getFreeDeviceBytes(); + max = + freeOnDisk !== null + ? curr + freeOnDisk + : Number.MAX_SAFE_INTEGER; + } + } + + return { curr, max }; + } + + async #getFreeDeviceBytes(): Promise { + try { + const stats = await statfs(process.cwd()); + return Number(stats.bavail) * Number(stats.bsize); + } catch { + return null; + } + } +} diff --git a/src/backend/stores/fs/S3ObjectStore.ts b/src/backend/stores/fs/S3ObjectStore.ts new file mode 100644 index 000000000..48deeee62 --- /dev/null +++ b/src/backend/stores/fs/S3ObjectStore.ts @@ -0,0 +1,607 @@ +import { + AbortMultipartUploadCommand, + CompleteMultipartUploadCommand, + CopyObjectCommand, + CreateMultipartUploadCommand, + DeleteObjectCommand, + DeleteObjectsCommand, + GetObjectCommand, + PutObjectCommand, + type S3Client, + UploadPartCommand, +} from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { Readable } from 'node:stream'; +import type { + CopyObjectInput, + DeleteObjectsInput, + GetObjectInput, + GetObjectResult, + MultipartCompleteInput, + ServerUploadInput, + SignedMultipartPartUrlsInput, + SignedUploadInput, + SignedUploadPart, + SignedUploadResult, +} from './s3Types.js'; +import { PuterStore } from '../types.js'; + +/** + * Store that owns S3 object I/O for fsentries: signed-URL minting, multipart + * lifecycle, server-driven uploads, and object reads/copies/deletes. Wraps + * the regional `S3Client` pool exposed by `clients.s3`. + */ +export class S3ObjectStore extends PuterStore { + #getClientForRegion(region: string): S3Client { + return this.clients.s3.get(region); + } + + // Older entries (migrated from v1) can have a null bucketRegion; callers + // use this to fall back to the configured default instead of erroring. + resolveRegion(region?: string | null): string { + return ( + region || this.config.s3_region || this.config.region || 'us-west-2' + ); + } + + // Same story for `bucket`: older rows may be null. Fall back to the + // configured default bucket. + resolveBucket(bucket?: string | null): string { + return bucket || this.config.s3_bucket || 'puter-local'; + } + + getMaxSingleUploadSize(): number { + return this.clients.s3.maxSingleUploadSize; + } + + getMultipartPartSize(): number { + return this.clients.s3.partSize; + } + + #resolveMultipartPartSize(requestedPartSize?: number): number { + return Math.max( + this.getMaxSingleUploadSize(), + requestedPartSize ?? this.getMultipartPartSize(), + ); + } + + async createSignedUploadUrl( + fileMetadata: SignedUploadInput, + region: string, + ): Promise { + const [result] = await this.batchCreateSignedUploadUrls( + [fileMetadata], + region, + ); + if (!result) { + throw new Error('Failed to create signed upload url'); + } + return result; + } + + async batchCreateSignedUploadUrls( + filesMetadata: SignedUploadInput[], + region: string, + ): Promise { + const client = this.#getClientForRegion(region); + const now = Date.now(); + const settledResults = await Promise.allSettled( + filesMetadata.map(async (fileMetadata) => { + const expiresInSeconds = Math.max( + 60, + Math.min(60 * 60, fileMetadata.expiresInSeconds), + ); + const expiresAt = now + expiresInSeconds * 1000; + const maxSingleUploadSize = this.getMaxSingleUploadSize(); + const shouldUseSingleUpload = + fileMetadata.uploadMode === 'single' && + fileMetadata.size <= maxSingleUploadSize; + + if (shouldUseSingleUpload) { + const command = new PutObjectCommand({ + Bucket: fileMetadata.bucket, + Key: fileMetadata.objectKey, + ContentType: fileMetadata.contentType, + }); + const url = await getSignedUrl(client, command, { + expiresIn: expiresInSeconds, + }); + return { + uploadMode: 'single' as const, + expiresAt, + url, + }; + } + + const multipartPartSize = this.#resolveMultipartPartSize( + fileMetadata.multipartPartSize, + ); + const multipartPartCount = Math.max( + 1, + Math.ceil(fileMetadata.size / multipartPartSize), + ); + let multipartUploadId: string | undefined; + + try { + const multipartResult = await client.send( + new CreateMultipartUploadCommand({ + Bucket: fileMetadata.bucket, + Key: fileMetadata.objectKey, + ContentType: fileMetadata.contentType, + }), + ); + + if (!multipartResult.UploadId) { + throw new Error( + 'Failed to initialize multipart upload', + ); + } + + multipartUploadId = multipartResult.UploadId; + const partUrls = await this.createSignedMultipartPartUrls( + { + bucket: fileMetadata.bucket, + objectKey: fileMetadata.objectKey, + multipartUploadId, + partNumbers: Array.from( + { length: multipartPartCount }, + (_, index) => index + 1, + ), + expiresInSeconds, + }, + region, + ); + + return { + uploadMode: 'multipart' as const, + expiresAt, + multipartUploadId, + multipartPartSize, + multipartPartCount, + multipartPartUrls: partUrls, + }; + } catch (error) { + if (multipartUploadId) { + try { + await this.abortMutipartUpload( + multipartUploadId, + region, + fileMetadata.bucket, + fileMetadata.objectKey, + ); + } catch { + // Best effort cleanup for partially initialized multipart uploads. + } + } + throw error; + } + }), + ); + + const failedResults = settledResults.filter( + (result) => result.status === 'rejected', + ); + if (failedResults.length > 0) { + await Promise.allSettled( + settledResults.map((result, index) => { + if (result.status !== 'fulfilled') { + return Promise.resolve(); + } + if ( + result.value.uploadMode !== 'multipart' || + !result.value.multipartUploadId + ) { + return Promise.resolve(); + } + const fileMetadata = filesMetadata[index]; + if (!fileMetadata) { + return Promise.resolve(); + } + return this.abortMutipartUpload( + result.value.multipartUploadId, + region, + fileMetadata.bucket, + fileMetadata.objectKey, + ); + }), + ); + + const firstFailure = failedResults[0]?.reason; + if (firstFailure instanceof Error) { + throw firstFailure; + } + throw new Error('Failed to create signed upload urls'); + } + + return settledResults.map((result) => { + if (result.status !== 'fulfilled') { + throw new Error('Failed to create signed upload urls'); + } + return result.value; + }); + } + + async createSignedMultipartPartUrls( + input: SignedMultipartPartUrlsInput, + region: string, + ): Promise { + const client = this.#getClientForRegion(region); + const expiresInSeconds = Math.max( + 60, + Math.min(60 * 60, input.expiresInSeconds), + ); + + return Promise.all( + input.partNumbers.map(async (partNumber) => { + const command = new UploadPartCommand({ + Bucket: input.bucket, + Key: input.objectKey, + UploadId: input.multipartUploadId, + PartNumber: partNumber, + }); + const url = await getSignedUrl(client, command, { + expiresIn: expiresInSeconds, + }); + return { + partNumber, + url, + }; + }), + ); + } + + async completeMultipartUpload( + input: MultipartCompleteInput, + region: string, + ): Promise { + const client = this.#getClientForRegion(region); + await client.send( + new CompleteMultipartUploadCommand({ + Bucket: input.bucket, + Key: input.objectKey, + UploadId: input.multipartUploadId, + MultipartUpload: { + Parts: [...input.parts] + .sort( + (partA, partB) => + partA.partNumber - partB.partNumber, + ) + .map((part) => ({ + PartNumber: part.partNumber, + ETag: part.etag, + })), + }, + }), + ); + } + + async abortMutipartUpload( + uploadId: string, + region: string, + bucket: string, + objectKey: string, + ): Promise { + const client = this.#getClientForRegion(region); + await client.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: objectKey, + UploadId: uploadId, + }), + ); + } + + async uploadFromServer( + input: ServerUploadInput, + region: string, + ): Promise { + const client = this.#getClientForRegion(region); + const maxSingleUploadSize = this.getMaxSingleUploadSize(); + const resolvedContentLength = this.#resolveContentLength(input); + const shouldUseMultipart = + input.body instanceof Readable + ? resolvedContentLength === undefined || + resolvedContentLength > maxSingleUploadSize + : resolvedContentLength !== undefined && + resolvedContentLength > maxSingleUploadSize; + + if (!shouldUseMultipart) { + await client.send( + new PutObjectCommand({ + Bucket: input.bucket, + Key: input.objectKey, + ContentType: input.contentType, + Body: input.body, + // Use the *resolved* length (input.contentLength → sizeHint → + // Buffer.byteLength). Without this, a Readable body whose + // length we know only via `sizeHint` falls into the SDK's + // chunked-streaming path and emits + // `x-amz-decoded-content-length: undefined`, which the HTTP + // layer rejects as an invalid header value. + ...(resolvedContentLength !== undefined + ? { ContentLength: resolvedContentLength } + : {}), + }), + ); + return; + } + + await this.#uploadFromServerMultipart( + input, + region, + this.#resolveMultipartPartSize(), + ); + } + + async deleteObject( + bucket: string, + objectKey: string, + region: string, + ): Promise { + const client = this.#getClientForRegion(region); + await client.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: objectKey, + }), + ); + } + + // Batch delete up to 1000 objects per S3 API limit; callers chunk if needed. + async deleteObjects( + input: DeleteObjectsInput, + region: string, + ): Promise { + if (input.objectKeys.length === 0) return; + const client = this.#getClientForRegion(region); + const MAX_BATCH = 1000; + for ( + let offset = 0; + offset < input.objectKeys.length; + offset += MAX_BATCH + ) { + const chunk = input.objectKeys.slice(offset, offset + MAX_BATCH); + await client.send( + new DeleteObjectsCommand({ + Bucket: input.bucket, + Delete: { + Objects: chunk.map((key) => ({ Key: key })), + Quiet: true, + }, + }), + ); + } + } + + // Server-side copy. Avoids downloading/re-uploading bytes. + async copyObject(input: CopyObjectInput, region: string): Promise { + const client = this.#getClientForRegion(region); + await client.send( + new CopyObjectCommand({ + Bucket: input.destinationBucket, + Key: input.destinationKey, + CopySource: `${input.sourceBucket}/${encodeURIComponent(input.sourceKey)}`, + ...(input.contentType + ? { ContentType: input.contentType } + : {}), + ...(input.metadataDirective + ? { MetadataDirective: input.metadataDirective } + : {}), + }), + ); + } + + async getObjectStream( + input: GetObjectInput, + region: string, + ): Promise { + const client = this.#getClientForRegion(region); + const response = await client.send( + new GetObjectCommand({ + Bucket: input.bucket, + Key: input.objectKey, + ...(input.range ? { Range: input.range } : {}), + }), + ); + + const body = response.Body; + if (!body) { + throw new Error('S3 getObject returned no body'); + } + // AWS SDK v3 returns body as Readable in Node.js; other runtimes return a web stream. + const stream = + body instanceof Readable + ? body + : Readable.fromWeb( + body as unknown as import('node:stream/web').ReadableStream, + ); + + return { + body: stream, + contentLength: response.ContentLength ?? null, + contentType: response.ContentType ?? null, + contentRange: response.ContentRange ?? null, + etag: response.ETag ?? null, + lastModified: response.LastModified ?? null, + }; + } + + #resolveContentLength(input: ServerUploadInput): number | undefined { + if ( + Number.isFinite(input.contentLength) && + Number(input.contentLength) >= 0 + ) { + return Number(input.contentLength); + } + if (Number.isFinite(input.sizeHint) && Number(input.sizeHint) >= 0) { + return Number(input.sizeHint); + } + if (Buffer.isBuffer(input.body) || input.body instanceof Uint8Array) { + return input.body.byteLength; + } + if (typeof input.body === 'string') { + return Buffer.byteLength(input.body); + } + return undefined; + } + + #toBuffer(chunk: unknown): Buffer { + if (Buffer.isBuffer(chunk)) { + return chunk; + } + if (chunk instanceof Uint8Array) { + return Buffer.from( + chunk.buffer, + chunk.byteOffset, + chunk.byteLength, + ); + } + if (typeof chunk === 'string') { + return Buffer.from(chunk); + } + throw new Error('Unsupported chunk type for multipart upload'); + } + + async #uploadFromServerMultipart( + input: ServerUploadInput, + region: string, + partSize: number, + ): Promise { + const client = this.#getClientForRegion(region); + const createResult = await client.send( + new CreateMultipartUploadCommand({ + Bucket: input.bucket, + Key: input.objectKey, + ContentType: input.contentType, + }), + ); + + const uploadId = createResult.UploadId; + if (!uploadId) { + throw new Error('Failed to initialize multipart upload'); + } + + const completedParts: Array<{ ETag: string; PartNumber: number }> = []; + let partNumber = 1; + + const uploadPart = async (partBody: Buffer) => { + const uploadPartResult = await client.send( + new UploadPartCommand({ + Bucket: input.bucket, + Key: input.objectKey, + UploadId: uploadId, + PartNumber: partNumber, + Body: partBody, + ContentLength: partBody.byteLength, + }), + ); + if (!uploadPartResult.ETag) { + throw new Error( + `Multipart upload returned no ETag for part ${partNumber}`, + ); + } + completedParts.push({ + ETag: uploadPartResult.ETag, + PartNumber: partNumber, + }); + partNumber++; + }; + + try { + if ( + Buffer.isBuffer(input.body) || + input.body instanceof Uint8Array + ) { + const bufferBody = this.#toBuffer(input.body); + for ( + let offset = 0; + offset < bufferBody.byteLength; + offset += partSize + ) { + const partBody = bufferBody.subarray( + offset, + offset + partSize, + ); + await uploadPart(partBody); + } + } else if (typeof input.body === 'string') { + const bufferBody = Buffer.from(input.body); + for ( + let offset = 0; + offset < bufferBody.byteLength; + offset += partSize + ) { + const partBody = bufferBody.subarray( + offset, + offset + partSize, + ); + await uploadPart(partBody); + } + } else if (input.body instanceof Readable) { + let pendingChunk: Buffer = Buffer.alloc(0); + for await (const chunk of input.body) { + const chunkBuffer = this.#toBuffer(chunk); + if (chunkBuffer.byteLength === 0) { + continue; + } + pendingChunk = + pendingChunk.byteLength === 0 + ? chunkBuffer + : Buffer.concat([pendingChunk, chunkBuffer]); + while (pendingChunk.byteLength >= partSize) { + const partBody = pendingChunk.subarray(0, partSize); + await uploadPart(partBody); + pendingChunk = pendingChunk.subarray(partSize); + } + } + if (pendingChunk.byteLength > 0) { + await uploadPart(pendingChunk); + } + } else { + throw new Error('Unsupported body type for multipart upload'); + } + + if (completedParts.length === 0) { + await client.send( + new AbortMultipartUploadCommand({ + Bucket: input.bucket, + Key: input.objectKey, + UploadId: uploadId, + }), + ); + await client.send( + new PutObjectCommand({ + Bucket: input.bucket, + Key: input.objectKey, + ContentType: input.contentType, + Body: Buffer.alloc(0), + ContentLength: 0, + }), + ); + return; + } + + await client.send( + new CompleteMultipartUploadCommand({ + Bucket: input.bucket, + Key: input.objectKey, + UploadId: uploadId, + MultipartUpload: { + Parts: completedParts, + }, + }), + ); + } catch (error) { + await client + .send( + new AbortMultipartUploadCommand({ + Bucket: input.bucket, + Key: input.objectKey, + UploadId: uploadId, + }), + ) + .catch(() => undefined); + throw error; + } + } +} diff --git a/extensions/fsv2/src/repositories/pendingUploadSessionHelpers.ts b/src/backend/stores/fs/pendingUploadSessionHelpers.ts similarity index 59% rename from extensions/fsv2/src/repositories/pendingUploadSessionHelpers.ts rename to src/backend/stores/fs/pendingUploadSessionHelpers.ts index df839233c..b7e18a763 100644 --- a/extensions/fsv2/src/repositories/pendingUploadSessionHelpers.ts +++ b/src/backend/stores/fs/pendingUploadSessionHelpers.ts @@ -1,21 +1,24 @@ -import { - PendingUploadCreateInput, - PendingUploadSession, -} from '../types/FSEntry.js'; +import { PendingUploadCreateInput, PendingUploadSession } from './FSEntry.js'; -export type PendingUploadSessionStatus = 'pending' | 'completed' | 'failed' | 'aborted'; +export type PendingUploadSessionStatus = + | 'pending' + | 'completed' + | 'failed' + | 'aborted'; export const PENDING_UPLOAD_SESSION_KEY_PREFIX = 'prodfsv2:upload-session:'; -export function toPendingUploadSessionKey (sessionId: string): string { +export function toPendingUploadSessionKey(sessionId: string): string { return `${PENDING_UPLOAD_SESSION_KEY_PREFIX}${sessionId}`; } -export function toPendingUploadSessionExpiresAtSeconds (expiresAtMs: number): number { +export function toPendingUploadSessionExpiresAtSeconds( + expiresAtMs: number, +): number { return Math.max(1, Math.ceil(expiresAtMs / 1000)); } -export function toPendingUploadSession ( +export function toPendingUploadSession( input: PendingUploadCreateInput, now: number, ): PendingUploadSession { @@ -51,28 +54,30 @@ export function toPendingUploadSession ( }; } -export function isPendingUploadSession (value: unknown): value is PendingUploadSession { - if ( !value || typeof value !== 'object' || Array.isArray(value) ) { +export function isPendingUploadSession( + value: unknown, +): value is PendingUploadSession { + if (!value || typeof value !== 'object' || Array.isArray(value)) { return false; } const candidate = value as Record; return ( - typeof candidate.sessionId === 'string' - && typeof candidate.userId === 'number' - && typeof candidate.status === 'string' - && typeof candidate.expiresAt === 'number' - && typeof candidate.objectKey === 'string' - && typeof candidate.parentPath === 'string' - && typeof candidate.targetPath === 'string' + typeof candidate.sessionId === 'string' && + typeof candidate.userId === 'number' && + typeof candidate.status === 'string' && + typeof candidate.expiresAt === 'number' && + typeof candidate.objectKey === 'string' && + typeof candidate.parentPath === 'string' && + typeof candidate.targetPath === 'string' ); } -export function normalizePendingUploadSession ( +export function normalizePendingUploadSession( value: unknown, sessionId: string, ): PendingUploadSession | null { - if ( ! isPendingUploadSession(value) ) { + if (!isPendingUploadSession(value)) { return null; } @@ -84,28 +89,35 @@ export function normalizePendingUploadSession ( consumedAt?: unknown; completedAt?: unknown; }; - const createdAt = typeof record.createdAt === 'number' ? record.createdAt : Date.now(); - const updatedAt = typeof record.updatedAt === 'number' ? record.updatedAt : createdAt; + const createdAt = + typeof record.createdAt === 'number' ? record.createdAt : Date.now(); + const updatedAt = + typeof record.updatedAt === 'number' ? record.updatedAt : createdAt; return { ...record, id: typeof record.id === 'number' ? record.id : 0, sessionId, - failureReason: typeof record.failureReason === 'string' ? record.failureReason : null, + failureReason: + typeof record.failureReason === 'string' + ? record.failureReason + : null, createdAt, updatedAt, - consumedAt: typeof record.consumedAt === 'number' ? record.consumedAt : null, - completedAt: typeof record.completedAt === 'number' ? record.completedAt : null, + consumedAt: + typeof record.consumedAt === 'number' ? record.consumedAt : null, + completedAt: + typeof record.completedAt === 'number' ? record.completedAt : null, }; } -export function withPendingUploadSessionStatus ( +export function withPendingUploadSessionStatus( session: PendingUploadSession, status: PendingUploadSessionStatus, reason: string | null, now: number, ): PendingUploadSession { - if ( status === 'completed' ) { + if (status === 'completed') { return { ...session, status, @@ -116,7 +128,7 @@ export function withPendingUploadSessionStatus ( }; } - if ( status === 'failed' || status === 'aborted' ) { + if (status === 'failed' || status === 'aborted') { return { ...session, status, diff --git a/extensions/fsv2/src/repositories/s3Types.ts b/src/backend/stores/fs/s3Types.ts similarity index 66% rename from extensions/fsv2/src/repositories/s3Types.ts rename to src/backend/stores/fs/s3Types.ts index 2cdb0c2a3..912d833f8 100644 --- a/extensions/fsv2/src/repositories/s3Types.ts +++ b/src/backend/stores/fs/s3Types.ts @@ -53,3 +53,32 @@ export interface ServerUploadInput { contentLength?: number; sizeHint?: number; } + +export interface GetObjectInput { + bucket: string; + objectKey: string; + range?: string; +} + +export interface GetObjectResult { + body: Readable; + contentLength: number | null; + contentType: string | null; + contentRange: string | null; + etag: string | null; + lastModified: Date | null; +} + +export interface CopyObjectInput { + sourceBucket: string; + sourceKey: string; + destinationBucket: string; + destinationKey: string; + contentType?: string; + metadataDirective?: 'COPY' | 'REPLACE'; +} + +export interface DeleteObjectsInput { + bucket: string; + objectKeys: string[]; +} diff --git a/extensions/fsv2/src/repositories/types.ts b/src/backend/stores/fs/types.ts similarity index 64% rename from extensions/fsv2/src/repositories/types.ts rename to src/backend/stores/fs/types.ts index 0d3e944b1..d2bb7aa2e 100644 --- a/extensions/fsv2/src/repositories/types.ts +++ b/src/backend/stores/fs/types.ts @@ -1,4 +1,4 @@ -import type { FSEntryCreateInput } from '../types/FSEntry.js'; +import type { FSEntryCreateInput } from './FSEntry.js'; export interface FSEntryRow { id: number; @@ -29,6 +29,9 @@ export interface FSEntryRow { symlink_path: string | null; is_symlink: number | boolean; path: string; + // Aggregated JSON produced by the subdomains subquery. SQLite returns a + // JSON text string; MySQL/MariaDB drivers may parse it into an array. + subdomains_agg?: string | unknown[] | null; } export interface NormalizedEntryWrite { @@ -48,4 +51,13 @@ export interface NormalizedEntryWrite { export interface ReadEntriesByPathsOptions { useTryHardRead?: boolean; skipCache?: boolean; + /** + * Opt-out of the user-namespace check applied by `getEntriesByPathsForUser`. + * Default `false` — paths must live under `//...` for the supplied + * userId, otherwise the entry is dropped from the result. Set `true` only + * for FSService internals that legitimately resolve cross-namespace entries + * (collision checks against paths in shared folders the writer has been + * granted access to). + */ + crossNamespace?: boolean; } diff --git a/src/backend/stores/group/GroupStore.ts b/src/backend/stores/group/GroupStore.ts new file mode 100644 index 000000000..e43f502d3 --- /dev/null +++ b/src/backend/stores/group/GroupStore.ts @@ -0,0 +1,213 @@ +import { v4 as uuidv4 } from 'uuid'; +import { PuterStore } from '../types'; + +// ── Types ──────────────────────────────────────────────────────────── + +export interface GroupRow { + id: number; + uid: string; + owner_user_id: number; + extra: Record; + metadata: Record; + [k: string]: unknown; +} + +// ── Constants ──────────────────────────────────────────────────────── + +const CREATE_RATE_LIMIT_PER_HOUR = 20; +const PUBLIC_GROUPS_CACHE_TTL_SECONDS = 10 * 60; + +// ── GroupStore ─────────────────────────────────────────────────────── + +/** + * Persistence layer for persistent user groups. + * + * Owns CRUD over the `group` table and the `jct_user_group` junction table, + * plus a per-process redis cache for the (small, frequently-read) set of + * public groups (the hardcoded default user + temp groups from config). + * + * Returns plain rows. Callers that need the members of a group can call + * `listMemberUsernames(uid)` explicitly. + */ +export class GroupStore extends PuterStore { + /** + * Random per-process cache namespace so restart-staleness can't cross + * processes. Populated in `onServerStart`. + */ + private redisNamespace: string = ''; + + override onServerStart(): void { + this.redisNamespace = uuidv4(); + } + + // ── Reads ──────────────────────────────────────────────────────── + + async getByUid(uid: string): Promise { + const rows = await this.clients.db.read( + 'SELECT * FROM `group` WHERE `uid` = ? LIMIT 1', + [uid], + ); + return rows[0] ? this.#decodeGroup(rows[0]) : null; + } + + async listGroupsWithOwner(ownerUserId: number): Promise { + const rows = await this.clients.db.read( + 'SELECT * FROM `group` WHERE `owner_user_id` = ?', + [ownerUserId], + ); + return rows.map((r) => this.#decodeGroup(r)); + } + + async listGroupsWithMember(userId: number): Promise { + const rows = await this.clients.db.read( + 'SELECT * FROM `group` WHERE `id` IN (' + + 'SELECT `group_id` FROM `jct_user_group` WHERE `user_id` = ?)', + [userId], + ); + return rows.map((r) => this.#decodeGroup(r)); + } + + /** + * Lists the two default public groups (user + temp). Redis-cached for + * 60s per-process. Falls back to DB on cache miss or decode failure. + */ + async listPublicGroups(): Promise { + const userGroupUid = this.config.default_user_group; + const tempGroupUid = this.config.default_temp_group; + const publicUids = [userGroupUid, tempGroupUid].filter( + (v): v is string => typeof v === 'string' && v.length > 0, + ); + if (publicUids.length === 0) return []; + + const cacheKey = this.#publicGroupsCacheKey(); + try { + const cached = await this.clients.redis.get(cacheKey); + if (cached) { + const parsed = JSON.parse(cached) as GroupRow[]; + if (Array.isArray(parsed)) return parsed; + } + } catch { + // fall through to DB read + } + + const placeholders = publicUids.map(() => '?').join(', '); + const rows = await this.clients.db.read( + `SELECT * FROM \`group\` WHERE \`uid\` IN (${placeholders})`, + publicUids, + ); + const decoded = rows.map((r) => this.#decodeGroup(r)); + + try { + await this.clients.redis.set( + cacheKey, + JSON.stringify(decoded), + 'EX', + PUBLIC_GROUPS_CACHE_TTL_SECONDS, + ); + } catch { + // cache writes are best-effort + } + return decoded; + } + + /** Usernames of the group's members. */ + async listMemberUsernames(uid: string): Promise { + const rows = await this.clients.db.read( + 'SELECT u.username FROM `user` u ' + + 'JOIN (SELECT user_id FROM `jct_user_group` WHERE group_id = ' + + '(SELECT id FROM `group` WHERE uid = ?)) ug ' + + 'ON u.id = ug.user_id', + [uid], + ); + return rows.map((r) => String(r.username)); + } + + // ── Writes ─────────────────────────────────────────────────────── + + /** + * Creates a new group owned by `ownerUserId`. Enforces a 20/hour per-owner + * rate limit (throws `Error('too_many_requests')` if exceeded). + */ + async create({ + ownerUserId, + extra = {}, + metadata = {}, + }: { + ownerUserId: number; + extra?: Record; + metadata?: Record; + }): Promise { + const windowClause = this.clients.db.case({ + sqlite: "datetime('now', '-1 hour')", + otherwise: 'NOW() - INTERVAL 1 HOUR', + }); + const [countRow] = await this.clients.db.read( + `SELECT COUNT(*) AS n_groups FROM \`group\` WHERE \`owner_user_id\` = ? AND \`created_at\` >= ${windowClause}`, + [ownerUserId], + ); + if (Number(countRow?.n_groups ?? 0) >= CREATE_RATE_LIMIT_PER_HOUR) { + throw new Error('too_many_requests'); + } + + const uid = uuidv4(); + await this.clients.db.write( + 'INSERT INTO `group` (`uid`, `owner_user_id`, `extra`, `metadata`) VALUES (?, ?, ?, ?)', + [uid, ownerUserId, JSON.stringify(extra), JSON.stringify(metadata)], + ); + return uid; + } + + /** Adds users (by username) to the group identified by `uid`. No-op if `usernames` is empty. */ + async addUsers(uid: string, usernames: string[]): Promise { + if (usernames.length === 0) return; + const placeholders = `(${usernames.map(() => '?').join(', ')})`; + await this.clients.db.write( + 'INSERT INTO `jct_user_group` (`user_id`, `group_id`) ' + + 'SELECT u.id, g.id FROM `user` u ' + + 'JOIN (SELECT id FROM `group` WHERE uid = ?) g ON 1 = 1 ' + + `WHERE u.username IN ${placeholders}`, + [uid, ...usernames], + ); + } + + /** Removes users (by username) from the group identified by `uid`. No-op if `usernames` is empty. */ + async removeUsers(uid: string, usernames: string[]): Promise { + if (usernames.length === 0) return; + const placeholders = `(${usernames.map(() => '?').join(', ')})`; + await this.clients.db.write( + 'DELETE FROM `jct_user_group` ' + + 'WHERE `group_id` = (SELECT id FROM `group` WHERE uid = ?) ' + + 'AND `user_id` IN (' + + 'SELECT u.id FROM `user` u ' + + `WHERE u.username IN ${placeholders})`, + [uid, ...usernames], + ); + } + + // ── Internals ──────────────────────────────────────────────────── + + #publicGroupsCacheKey(): string { + return `${this.redisNamespace}:group:public-groups`; + } + + #decodeGroup(row: Record): GroupRow { + const parse = (v: unknown): Record => { + if (v == null) return {}; + if (typeof v === 'object') return v as Record; + try { + return JSON.parse(String(v)); + } catch { + return {}; + } + }; + const extra = this.clients.db.case<() => Record>({ + mysql: () => (row.extra as Record) ?? {}, + otherwise: () => parse(row.extra), + })(); + const metadata = this.clients.db.case<() => Record>({ + mysql: () => (row.metadata as Record) ?? {}, + otherwise: () => parse(row.metadata), + })(); + return { ...row, extra, metadata } as unknown as GroupRow; + } +} diff --git a/src/backend/stores/index.ts b/src/backend/stores/index.ts new file mode 100644 index 000000000..7ef8efd59 --- /dev/null +++ b/src/backend/stores/index.ts @@ -0,0 +1,35 @@ +import { AppStore } from './app/AppStore.js'; +import { FSEntryStore } from './fs/FSEntryStore.js'; +import { GroupStore } from './group/GroupStore.js'; +import { NotificationStore } from './notification/NotificationStore.js'; +import { OIDCStore } from './oidc/OIDCStore.js'; +import { PermissionStore } from './permission/PermissionStore.js'; +import { S3ObjectStore } from './fs/S3ObjectStore.js'; +import { SessionStore } from './session/SessionStore.js'; +import { ShareStore } from './share/ShareStore.js'; +import { SubdomainStore } from './subdomain/SubdomainStore.js'; +import { SystemKVStore } from './systemKv/SystemKVStore.js'; +import { UserStore } from './user/UserStore.js'; +import type { IPuterStoreRegistry } from './types.js'; + +// Ordering matters: stores declared later see earlier ones as peers. +// PermissionStore depends on `kv`, so `kv` must come first. +// UserStore / AppStore are leaves (db + redis only); sit early so other +// stores/services can lean on them for cached lookups. +// FSEntryStore depends on `kv` (pending-upload sessions live there). +// S3ObjectStore is a leaf (clients.s3 only). +// SessionStore / ShareStore are leaves — only use clients.db. +export const puterStores = { + kv: SystemKVStore, + user: UserStore, + app: AppStore, + fsEntry: FSEntryStore, + s3Object: S3ObjectStore, + subdomain: SubdomainStore, + notification: NotificationStore, + share: ShareStore, + group: GroupStore, + permission: PermissionStore, + session: SessionStore, + oidc: OIDCStore, +} satisfies IPuterStoreRegistry; diff --git a/src/backend/stores/notification/NotificationStore.js b/src/backend/stores/notification/NotificationStore.js new file mode 100644 index 000000000..4a3bc9f26 --- /dev/null +++ b/src/backend/stores/notification/NotificationStore.js @@ -0,0 +1,140 @@ +import { v4 as uuidv4 } from 'uuid'; +import { PuterStore } from '../types'; + +// `markShown` intentionally doesn't invalidate unack count — it doesn't move it. + +const UNACK_CACHE_KEY_PREFIX = 'notifications:unack'; +const UNACK_CACHE_TTL_SECONDS = 5 * 60; + +export class NotificationStore extends PuterStore { + // ── Reads ──────────────────────────────────────────────────────── + + async getByUid(uid, { userId } = {}) { + const where = + userId !== undefined + ? 'WHERE `uid` = ? AND `user_id` = ?' + : 'WHERE `uid` = ?'; + const params = userId !== undefined ? [uid, userId] : [uid]; + const rows = await this.clients.db.read( + `SELECT * FROM \`notification\` ${where} LIMIT 1`, + params, + ); + return this.#normalizeRow(rows[0]) ?? null; + } + + /** @param {number} userId @param {{ limit?: number, onlyUnacknowledged?: boolean, filter?: string }} [opts] */ + async listByUserId( + userId, + { limit = 200, onlyUnacknowledged = false, filter = undefined } = {}, + ) { + let extraWhere = ''; + if (onlyUnacknowledged || filter === 'unacknowledged') { + extraWhere = 'AND `acknowledged` IS NULL'; + } else if (filter === 'unseen') { + extraWhere = 'AND `shown` IS NULL AND `acknowledged` IS NULL'; + } else if (filter === 'acknowledged') { + extraWhere = 'AND `acknowledged` IS NOT NULL'; + } + const rows = await this.clients.db.read( + `SELECT * FROM \`notification\` + WHERE \`user_id\` = ? ${extraWhere} + ORDER BY \`created_at\` DESC + LIMIT ${limit}`, + [userId], + ); + return rows.map((r) => this.#normalizeRow(r)); + } + + async countUnacknowledged(userId) { + if (!userId) return 0; + + const cacheKey = this.#unackCacheKey(userId); + try { + const raw = await this.clients.redis.get(cacheKey); + if (raw !== null && raw !== undefined) { + const parsed = Number(raw); + if (Number.isFinite(parsed)) return parsed; + } + } catch { + // Fall through to DB. + } + + const rows = await this.clients.db.read( + 'SELECT COUNT(*) AS n FROM `notification` WHERE `user_id` = ? AND `acknowledged` IS NULL', + [userId], + ); + const count = Number(rows[0]?.n ?? 0); + + this.clients.redis + .set(cacheKey, String(count), 'EX', UNACK_CACHE_TTL_SECONDS) + .catch(() => {}); + return count; + } + + // ── Writes ─────────────────────────────────────────────────────── + + async create({ userId, value }) { + if (!userId) throw new Error('create: userId is required'); + const uid = uuidv4(); + const serialized = + typeof value === 'string' ? value : JSON.stringify(value ?? {}); + await this.clients.db.write( + 'INSERT INTO `notification` (`uid`, `user_id`, `value`) VALUES (?, ?, ?)', + [uid, userId, serialized], + ); + await this.#invalidateUnack(userId); + return this.getByUid(uid, { userId }); + } + + async markAcknowledged(uid, userId) { + const now = Math.floor(Date.now() / 1000); + const result = await this.clients.db.write( + 'UPDATE `notification` SET `acknowledged` = ? WHERE `uid` = ? AND `user_id` = ? AND `acknowledged` IS NULL', + [now, uid, userId], + ); + const changed = (result?.affectedRows ?? result?.changes ?? 0) > 0; + if (changed) await this.#invalidateUnack(userId); + return changed; + } + + async markShown(uid, userId) { + const now = Math.floor(Date.now() / 1000); + const result = await this.clients.db.write( + 'UPDATE `notification` SET `shown` = ? WHERE `uid` = ? AND `user_id` = ? AND `shown` IS NULL', + [now, uid, userId], + ); + return (result?.affectedRows ?? result?.changes ?? 0) > 0; + } + + async deleteByUid(uid, userId) { + const result = await this.clients.db.write( + 'DELETE FROM `notification` WHERE `uid` = ? AND `user_id` = ?', + [uid, userId], + ); + const changed = (result?.affectedRows ?? result?.changes ?? 0) > 0; + if (changed) await this.#invalidateUnack(userId); + return changed; + } + + // ── Internals ──────────────────────────────────────────────────── + + #unackCacheKey(userId) { + return `${UNACK_CACHE_KEY_PREFIX}:${userId}`; + } + + async #invalidateUnack(userId) { + await this.publishCacheKeys({ keys: [this.#unackCacheKey(userId)] }); + } + + #normalizeRow(row) { + if (!row) return null; + if (typeof row.value === 'string') { + try { + row.value = JSON.parse(row.value); + } catch { + /* keep string */ + } + } + return row; + } +} diff --git a/src/backend/stores/oidc/OIDCStore.js b/src/backend/stores/oidc/OIDCStore.js new file mode 100644 index 000000000..59a49064c --- /dev/null +++ b/src/backend/stores/oidc/OIDCStore.js @@ -0,0 +1,63 @@ +import { PuterStore } from '../types'; + +/** + * CRUD over the `user_oidc_providers` table. + * + * Columns: id, user_id, provider, provider_sub, refresh_token, created_at. + * UNIQUE(provider, provider_sub). + */ +export class OIDCStore extends PuterStore { + // ── Reads ──────────────────────────────────────────────────────── + + async getByProviderSub(provider, providerSub) { + const rows = await this.clients.db.read( + 'SELECT * FROM `user_oidc_providers` WHERE `provider` = ? AND `provider_sub` = ? LIMIT 1', + [provider, providerSub], + ); + return rows[0] ?? null; + } + + async listByUserId(userId) { + return this.clients.db.read( + 'SELECT * FROM `user_oidc_providers` WHERE `user_id` = ?', + [userId], + ); + } + + // ── Writes ─────────────────────────────────────────────────────── + + async link(userId, provider, providerSub, refreshToken = null) { + try { + await this.clients.db.write( + 'INSERT INTO `user_oidc_providers` (`user_id`, `provider`, `provider_sub`, `refresh_token`) VALUES (?, ?, ?, ?)', + [userId, provider, providerSub, refreshToken], + ); + return; + } catch (e) { + const isUnique = + e.message?.includes('UNIQUE') || + e.code === 'SQLITE_CONSTRAINT' || + e.code === 'ER_DUP_ENTRY'; + if (!isUnique) throw e; + } + + // UNIQUE(provider, provider_sub) collision — either we're re-linking + // the same (user, provider, sub) triple (idempotent no-op), or the + // sub already belongs to a DIFFERENT user. The latter must fail loudly + // so callers don't assume success and act on an unrelated account. + const existing = await this.getByProviderSub(provider, providerSub); + if (!existing) return; + if (existing.user_id !== userId) { + throw new Error( + `OIDC link conflict: (${provider}, ${providerSub}) already bound to user ${existing.user_id}`, + ); + } + } + + async unlinkByUserId(userId, provider) { + await this.clients.db.write( + 'DELETE FROM `user_oidc_providers` WHERE `user_id` = ? AND `provider` = ?', + [userId, provider], + ); + } +} diff --git a/src/backend/stores/permission/PermissionStore.ts b/src/backend/stores/permission/PermissionStore.ts new file mode 100644 index 000000000..6b9bdb90d --- /dev/null +++ b/src/backend/stores/permission/PermissionStore.ts @@ -0,0 +1,756 @@ +import { PuterStore } from '../types'; +import type { LayerInstances } from '../../types'; +import type { puterStores } from '../index'; +import { PermissionUtil } from '../../services/permission/permissionUtil'; +import { + PERM_KEY_PREFIX, + PERMISSION_SCAN_CACHE_TTL_SECONDS, +} from '../../services/permission/consts'; +import type { UserRow } from '../user/UserStore'; + +// Short TTLs: FK CASCADE on user/app delete + PermissionService rewriters +// can touch rows this store's mutators never see. +const U2A_CACHE_TTL_SECONDS = 5 * 60; +const U2U_CACHE_TTL_SECONDS = 5 * 60; +const TOKEN_CACHE_TTL_SECONDS = 10 * 60; + +// Re-export for back-compat — PermissionService et al. import `UserRow` from here. +// The canonical definition lives in `UserStore`, which owns the user table. +export type { UserRow }; + +// ── Types ──────────────────────────────────────────────────────────── + +export interface FlatPermValue { + permission?: string; + issuer_user_id?: number; + deleted?: boolean; + [k: string]: unknown; +} + +export interface LinkedUserUserPermRow { + holder_user_id: number; + issuer_user_id: number; + permission: string; + extra: Record; + [k: string]: unknown; +} + +export interface LinkedUserAppPermRow { + user_id: number; + app_id: number; + permission: string; + extra: Record; + [k: string]: unknown; +} + +export interface LinkedUserGroupPermRow { + user_id: number; + group_id: number; + permission: string; + extra: Record; + [k: string]: unknown; +} + +export interface AccessTokenPermRow { + token_uid: string; + permission: string; + [k: string]: unknown; +} + +export interface AuditEntry { + action: 'grant' | 'revoke'; + reason: string; + [k: string]: unknown; +} + +/** + * PermissionStore owns the *persistence* side of permissions: + * - SQL CRUD + audit inserts for all permission tables + * - Flat KV reads/writes under `PERM_KEY_PREFIX` (system namespace) + * - Redis scan-cache get/set/invalidate + * + * It does NOT own semantics — rewriters, implicators, exploders, and the + * `scan()` algorithm all live on PermissionService. This store is just I/O. + */ +export class PermissionStore extends PuterStore { + declare protected stores: LayerInstances; + + // ── Flat view (KV under system namespace) ──────────────────────── + + /** + * Read the flat user-to-user permissions for a holder across a set of + * permission strings. Returns the KV values that exist; missing keys are + * filtered out. + */ + async getFlatUserPerms( + holderUserId: number, + permissions: string[], + ): Promise { + if (permissions.length === 0) return []; + const keys = [ + ...new Set( + permissions.map((p) => + PermissionUtil.join( + PERM_KEY_PREFIX, + String(holderUserId), + p, + ), + ), + ), + ]; + const { res } = await this.stores.kv.get({ key: keys }); + const values = Array.isArray(res) ? res : [res]; + return values.filter( + (v): v is FlatPermValue => v !== null && typeof v === 'object', + ); + } + + /** Write a single flat user-to-user permission entry to KV. */ + async setFlatUserPerm( + holderUserId: number, + permission: string, + value: FlatPermValue, + ): Promise { + const key = PermissionUtil.join( + PERM_KEY_PREFIX, + String(holderUserId), + permission, + ); + await this.stores.kv.set({ key, value }); + } + + /** Delete a single flat user-to-user permission entry from KV. */ + async delFlatUserPerm( + holderUserId: number, + permission: string, + ): Promise { + const key = PermissionUtil.join( + PERM_KEY_PREFIX, + String(holderUserId), + permission, + ); + await this.stores.kv.del({ key }); + } + + // ── SQL: user-to-user permissions ─────────────────────────────── + + async readLinkedUserUserPerms( + holderUserId: number, + permissions: string[], + ): Promise { + if (permissions.length === 0) return []; + const all = await this.#readAllUserUserPermsForHolder(holderUserId); + const wanted = new Set(permissions); + return all.filter((row) => wanted.has(row.permission)); + } + + async upsertUserUserPerm( + holderUserId: number, + issuerUserId: number, + permission: string, + extra: Record, + ): Promise { + const upsertClause = this.clients.db.case({ + mysql: 'ON DUPLICATE KEY UPDATE `extra` = ?', + otherwise: + 'ON CONFLICT(`holder_user_id`, `issuer_user_id`, `permission`) DO UPDATE SET `extra` = ?', + }); + await this.clients.db.write( + 'INSERT INTO `user_to_user_permissions` (`holder_user_id`, `issuer_user_id`, `permission`, `extra`) ' + + `VALUES (?, ?, ?, ?) ${upsertClause}`, + [ + holderUserId, + issuerUserId, + permission, + JSON.stringify(extra), + JSON.stringify(extra), + ], + ); + await this.publishCacheKeys({ + keys: [this.#u2uCacheKey(holderUserId)], + }); + } + + async deleteUserUserPermByHolder( + holderUserId: number, + permission: string, + ): Promise { + await this.clients.db.write( + 'DELETE FROM `user_to_user_permissions` WHERE `holder_user_id` = ? AND `permission` = ?', + [holderUserId, permission], + ); + await this.publishCacheKeys({ + keys: [this.#u2uCacheKey(holderUserId)], + }); + } + + async auditUserUserPerm( + entry: AuditEntry & { + holder_user_id: number; + issuer_user_id: number; + permission: string; + }, + ): Promise { + await this.clients.db.write( + 'INSERT INTO `audit_user_to_user_permissions` (' + + '`holder_user_id`, `holder_user_id_keep`, `issuer_user_id`, `issuer_user_id_keep`, ' + + '`permission`, `action`, `reason`) VALUES (?, ?, ?, ?, ?, ?, ?)', + [ + entry.holder_user_id, + entry.holder_user_id, + entry.issuer_user_id, + entry.issuer_user_id, + entry.permission, + entry.action, + entry.reason, + ], + ); + } + + async listUserPermissionIssuerIds(holderUserId: number): Promise { + const rows = await this.clients.db.read( + 'SELECT DISTINCT issuer_user_id FROM `user_to_user_permissions` WHERE `holder_user_id` = ?', + [holderUserId], + ); + return rows.map((r) => Number(r.issuer_user_id)); + } + + // ── SQL: user-to-app permissions ──────────────────────────────── + + async readUserAppPerms( + userId: number, + appId: number, + permissions: string[], + ): Promise { + if (permissions.length === 0) return []; + const all = await this.#readAllUserAppPerms(userId, appId); + const wanted = new Set(permissions); + return all.filter((row) => wanted.has(row.permission)); + } + + async hasUserAppPerm( + userId: number, + appId: number, + permission: string, + ): Promise { + const all = await this.#readAllUserAppPerms(userId, appId); + return all.some((row) => row.permission === permission); + } + + async upsertUserAppPerm( + userId: number, + appId: number, + permission: string, + extra: Record, + ): Promise { + const upsertClause = this.clients.db.case({ + mysql: 'ON DUPLICATE KEY UPDATE `extra` = ?', + otherwise: + 'ON CONFLICT(`user_id`, `app_id`, `permission`) DO UPDATE SET `extra` = ?', + }); + await this.clients.db.write( + 'INSERT INTO `user_to_app_permissions` (`user_id`, `app_id`, `permission`, `extra`) ' + + `VALUES (?, ?, ?, ?) ${upsertClause}`, + [ + userId, + appId, + permission, + JSON.stringify(extra), + JSON.stringify(extra), + ], + ); + await this.publishCacheKeys({ + keys: [this.#u2aCacheKey(userId, appId)], + }); + } + + async deleteUserAppPerm( + userId: number, + appId: number, + permission: string, + ): Promise { + await this.clients.db.write( + 'DELETE FROM `user_to_app_permissions` WHERE `user_id` = ? AND `app_id` = ? AND `permission` = ?', + [userId, appId, permission], + ); + await this.publishCacheKeys({ + keys: [this.#u2aCacheKey(userId, appId)], + }); + } + + async deleteUserAppAll(userId: number, appId: number): Promise { + await this.clients.db.write( + 'DELETE FROM `user_to_app_permissions` WHERE `user_id` = ? AND `app_id` = ?', + [userId, appId], + ); + await this.publishCacheKeys({ + keys: [this.#u2aCacheKey(userId, appId)], + }); + } + + async auditUserAppPerm( + entry: AuditEntry & { + user_id: number; + app_id: number; + permission: string; + }, + ): Promise { + await this.clients.db.write( + 'INSERT INTO `audit_user_to_app_permissions` (' + + '`user_id`, `user_id_keep`, `app_id`, `app_id_keep`, ' + + '`permission`, `action`, `reason`) VALUES (?, ?, ?, ?, ?, ?, ?)', + [ + entry.user_id, + entry.user_id, + entry.app_id, + entry.app_id, + entry.permission, + entry.action, + entry.reason, + ], + ); + } + + // ── SQL: dev-to-app permissions ───────────────────────────────── + + async readDevAppPerms( + appId: number, + permissions: string[], + ): Promise { + if (permissions.length === 0) return []; + let permClause = permissions.map(() => '`permission` = ?').join(' OR '); + if (permissions.length > 1) permClause = `(${permClause})`; + const rows = await this.clients.db.read( + 'SELECT * FROM `dev_to_app_permissions` ' + + `WHERE \`app_id\` = ? AND ${permClause}`, + [appId, ...permissions], + ); + return rows.map((row) => this.#decodeExtra(row)); + } + + async upsertDevAppPerm( + userId: number, + appId: number, + permission: string, + extra: Record, + ): Promise { + const upsertClause = this.clients.db.case({ + mysql: 'ON DUPLICATE KEY UPDATE `extra` = ?', + otherwise: + 'ON CONFLICT(`user_id`, `app_id`, `permission`) DO UPDATE SET `extra` = ?', + }); + await this.clients.db.write( + 'INSERT INTO `dev_to_app_permissions` (`user_id`, `app_id`, `permission`, `extra`) ' + + `VALUES (?, ?, ?, ?) ${upsertClause}`, + [ + userId, + appId, + permission, + JSON.stringify(extra), + JSON.stringify(extra), + ], + ); + } + + async deleteDevAppPerm( + userId: number, + appId: number, + permission: string, + ): Promise { + await this.clients.db.write( + 'DELETE FROM `dev_to_app_permissions` WHERE `user_id` = ? AND `app_id` = ? AND `permission` = ?', + [userId, appId, permission], + ); + } + + async deleteDevAppAll(userId: number, appId: number): Promise { + await this.clients.db.write( + 'DELETE FROM `dev_to_app_permissions` WHERE `user_id` = ? AND `app_id` = ?', + [userId, appId], + ); + } + + async auditDevAppPerm( + entry: AuditEntry & { + user_id: number; + app_id: number; + permission: string; + }, + ): Promise { + await this.clients.db.write( + 'INSERT INTO `audit_dev_to_app_permissions` (' + + '`user_id`, `user_id_keep`, `app_id`, `app_id_keep`, ' + + '`permission`, `action`, `reason`) VALUES (?, ?, ?, ?, ?, ?, ?)', + [ + entry.user_id, + entry.user_id, + entry.app_id, + entry.app_id, + entry.permission, + entry.action, + entry.reason, + ], + ); + } + + // ── SQL: user-to-group permissions ────────────────────────────── + + /** + * Reads group permissions granted to groups the user is a member of, for + * a given set of permission strings. Result already joined against + * `jct_user_group` so callers don't need group membership resolution. + */ + async readUserGroupPerms( + userId: number, + permissions: string[], + ): Promise { + if (permissions.length === 0) return []; + let permClause = permissions.map(() => 'p.permission = ?').join(' OR '); + if (permissions.length > 1) permClause = `(${permClause})`; + const rows = await this.clients.db.read( + 'SELECT p.permission, p.user_id, p.group_id, p.extra FROM `user_to_group_permissions` p ' + + 'JOIN `jct_user_group` ug ON p.group_id = ug.group_id ' + + `WHERE ug.user_id = ? AND ${permClause}`, + [userId, ...permissions], + ); + return rows.map((row) => + this.#decodeExtra(row), + ); + } + + async upsertUserGroupPerm( + userId: number, + groupId: number, + permission: string, + extra: Record, + ): Promise { + const upsertClause = this.clients.db.case({ + mysql: 'ON DUPLICATE KEY UPDATE `extra` = ?', + otherwise: + 'ON CONFLICT(`user_id`, `group_id`, `permission`) DO UPDATE SET `extra` = ?', + }); + await this.clients.db.write( + 'INSERT INTO `user_to_group_permissions` (`user_id`, `group_id`, `permission`, `extra`) ' + + `VALUES (?, ?, ?, ?) ${upsertClause}`, + [ + userId, + groupId, + permission, + JSON.stringify(extra), + JSON.stringify(extra), + ], + ); + } + + async deleteUserGroupPerm( + userId: number, + groupId: number, + permission: string, + ): Promise { + await this.clients.db.write( + 'DELETE FROM `user_to_group_permissions` WHERE `user_id` = ? AND `group_id` = ? AND `permission` = ?', + [userId, groupId, permission], + ); + } + + async auditUserGroupPerm( + entry: AuditEntry & { + user_id: number; + group_id: number; + permission: string; + }, + ): Promise { + await this.clients.db.write( + 'INSERT INTO `audit_user_to_group_permissions` (' + + '`user_id`, `user_id_keep`, `group_id`, `group_id_keep`, ' + + '`permission`, `action`, `reason`) VALUES (?, ?, ?, ?, ?, ?, ?)', + [ + entry.user_id, + entry.user_id, + entry.group_id, + entry.group_id, + entry.permission, + entry.action, + entry.reason, + ], + ); + } + + // ── SQL: access token permissions ─────────────────────────────── + + async hasAccessTokenPerm( + tokenUid: string, + permission: string, + ): Promise { + const all = await this.#readAccessTokenPerms(tokenUid); + return all.includes(permission); + } + + /** Call from AuthService after it mutates `access_token_permissions`. */ + async invalidateAccessTokenPerms(tokenUid: string): Promise { + await this.publishCacheKeys({ + keys: [this.#tokenCacheKey(tokenUid)], + }); + } + + // ── SQL: issuer-prefix queries (share discovery, etc.) ────────── + + async queryIssuerUserPermsByPrefix( + issuerUserId: number, + prefix: string, + ): Promise> { + const rows = await this.clients.db.read( + 'SELECT DISTINCT holder_user_id, permission FROM `user_to_user_permissions` ' + + 'WHERE issuer_user_id = ? AND permission LIKE ?', + [issuerUserId, `${prefix}%`], + ); + return rows.map((r) => ({ + holder_user_id: Number(r.holder_user_id), + permission: String(r.permission), + })); + } + + async queryIssuerAppPermsByPrefix( + issuerUserId: number, + prefix: string, + ): Promise> { + const rows = await this.clients.db.read( + 'SELECT DISTINCT app_id, permission FROM `user_to_app_permissions` ' + + 'WHERE user_id = ? AND permission LIKE ?', + [issuerUserId, `${prefix}%`], + ); + return rows.map((r) => ({ + app_id: Number(r.app_id), + permission: String(r.permission), + })); + } + + async queryIssuerHolderPermsByPrefix( + issuerUserId: number, + holderUserId: number, + prefix: string, + ): Promise { + const rows = await this.clients.db.read( + 'SELECT permission FROM `user_to_user_permissions` ' + + 'WHERE issuer_user_id = ? AND holder_user_id = ? AND permission LIKE ?', + [issuerUserId, holderUserId, `${prefix}%`], + ); + return rows.map((r) => String(r.permission)); + } + + // ── Scan cache (redis) ────────────────────────────────────────── + + buildScanCacheKey(actorUid: string, permissionOptions: string[]): string { + return PermissionUtil.join( + 'permission-scan', + actorUid, + 'options-list', + ...permissionOptions, + ); + } + + async getScanCache(cacheKey: string): Promise { + const raw = await this.clients.redis.get(cacheKey); + if (!raw) return null; + try { + return JSON.parse(raw); + } catch { + return null; + } + } + + async setScanCache( + cacheKey: string, + value: unknown, + ttlSeconds: number = PERMISSION_SCAN_CACHE_TTL_SECONDS, + ): Promise { + await this.clients.redis.set( + cacheKey, + JSON.stringify(value), + 'EX', + ttlSeconds, + ); + } + + async invalidateScanCache(cacheKey: string): Promise { + await this.publishCacheKeys({ keys: [cacheKey] }); + } + + // ── Per-permission check cache (for `checkMany`) ───────────────── + // + // Cached as `1`/`0` per (actor, permission) pair so a batch lookup + // reduces to a single MGET. Same TTL as the scan cache — the + // underlying perm tables already publish invalidations via + // `publishCacheKeys` for grant/revoke writes, but those don't reach + // these keys, so we keep the TTL short and rely on it for staleness. + #checkCacheKey(actorUid: string, permission: string): string { + return PermissionUtil.join( + 'permission-check', + actorUid, + 'p', + permission, + ); + } + + async getMultiCheckCache( + actorUid: string, + permissions: string[], + ): Promise> { + const out = new Map(); + if (permissions.length === 0) return out; + const keys = permissions.map((p) => this.#checkCacheKey(actorUid, p)); + let raw: Array = []; + try { + raw = (await this.clients.redis.mget(...keys)) as Array< + string | null + >; + } catch { + return out; + } + for (let i = 0; i < permissions.length; i++) { + const v = raw[i]; + if (v === '1') out.set(permissions[i], true); + else if (v === '0') out.set(permissions[i], false); + } + return out; + } + + async setMultiCheckCache( + actorUid: string, + entries: Array<{ permission: string; granted: boolean }>, + ttlSeconds: number = PERMISSION_SCAN_CACHE_TTL_SECONDS, + ): Promise { + if (entries.length === 0) return; + const pipeline = this.clients.redis.pipeline(); + for (const { permission, granted } of entries) { + pipeline.set( + this.#checkCacheKey(actorUid, permission), + granted ? '1' : '0', + 'EX', + ttlSeconds, + ); + } + try { + await pipeline.exec(); + } catch { + // Best-effort cache write. + } + } + + // ── Internals ─────────────────────────────────────────────────── + + #u2uCacheKey(holderUserId: number): string { + return `perms:u2u:holder:${holderUserId}`; + } + + #u2aCacheKey(userId: number, appId: number): string { + return `perms:u2a:${userId}:${appId}`; + } + + #tokenCacheKey(tokenUid: string): string { + return `perms:token:${tokenUid}`; + } + + async #readAllUserUserPermsForHolder( + holderUserId: number, + ): Promise { + const cacheKey = this.#u2uCacheKey(holderUserId); + try { + const raw = await this.clients.redis.get(cacheKey); + if (raw) { + const parsed = JSON.parse(raw); + if (Array.isArray(parsed)) return parsed; + } + } catch { + // Fall through to DB. + } + + const rows = await this.clients.db.read( + 'SELECT * FROM `user_to_user_permissions` WHERE `holder_user_id` = ?', + [holderUserId], + ); + const decoded = rows.map((row) => + this.#decodeExtra(row), + ); + + this.clients.redis + .set(cacheKey, JSON.stringify(decoded), 'EX', U2U_CACHE_TTL_SECONDS) + .catch(() => {}); + return decoded; + } + + async #readAllUserAppPerms( + userId: number, + appId: number, + ): Promise { + const cacheKey = this.#u2aCacheKey(userId, appId); + try { + const raw = await this.clients.redis.get(cacheKey); + if (raw) { + const parsed = JSON.parse(raw); + if (Array.isArray(parsed)) return parsed; + } + } catch { + // Fall through to DB. + } + + const rows = await this.clients.db.read( + 'SELECT * FROM `user_to_app_permissions` WHERE `user_id` = ? AND `app_id` = ?', + [userId, appId], + ); + const decoded = rows.map((row) => + this.#decodeExtra(row), + ); + + this.clients.redis + .set(cacheKey, JSON.stringify(decoded), 'EX', U2A_CACHE_TTL_SECONDS) + .catch(() => {}); + return decoded; + } + + async #readAccessTokenPerms(tokenUid: string): Promise { + const cacheKey = this.#tokenCacheKey(tokenUid); + try { + const raw = await this.clients.redis.get(cacheKey); + if (raw) { + const parsed = JSON.parse(raw); + if (Array.isArray(parsed)) return parsed; + } + } catch { + // Fall through to DB. + } + + const rows = await this.clients.db.read( + 'SELECT `permission` FROM `access_token_permissions` WHERE `token_uid` = ?', + [tokenUid], + ); + const perms = rows.map((r) => String(r.permission)); + + this.clients.redis + .set(cacheKey, JSON.stringify(perms), 'EX', TOKEN_CACHE_TTL_SECONDS) + .catch(() => {}); + return perms; + } + + /** Parse the JSON `extra` column into an object. */ + #decodeExtra>( + row: Record, + ): T { + const extra = this.clients.db.case<() => unknown>({ + mysql: () => row.extra, + otherwise: () => { + if (row.extra == null) return {}; + if (typeof row.extra === 'object') return row.extra; + try { + return JSON.parse(String(row.extra)); + } catch { + return {}; + } + }, + })(); + return { + ...row, + extra: (extra ?? {}) as Record, + } as unknown as T; + } +} diff --git a/src/backend/stores/session/SessionStore.js b/src/backend/stores/session/SessionStore.js new file mode 100644 index 000000000..842720fab --- /dev/null +++ b/src/backend/stores/session/SessionStore.js @@ -0,0 +1,204 @@ +import { v4 as uuidv4 } from 'uuid'; +import { PuterStore } from '../types'; + +// `updateActivity` / `updateUserActivity` intentionally skip cache +// invalidation — they're throttled UPDATEs, stale `last_activity` in +// the cached row doesn't gate anything, and eating a Redis write per +// request isn't worth it. + +const CACHE_KEY_PREFIX = 'sessions'; +const CACHE_TTL_SECONDS = 15 * 60; +// Min interval between successive activity flushes per session/user. +// In-memory throttle keeps DB writes bounded; multi-node duplicates +// are harmless because the SQL guard `last_activity < ?` makes the +// UPDATE idempotent. +const TOUCH_THROTTLE_MS = 60 * 1000; +// Hard cap to keep the throttle map from growing unbounded for +// long-lived processes. Clearing only loses throttling — at worst a +// brief burst of redundant UPDATEs. +const TOUCH_THROTTLE_MAX_ENTRIES = 10000; + +const sqlTimestamp = (ms) => + new Date(ms).toISOString().slice(0, 19).replace('T', ' '); + +export class SessionStore extends PuterStore { + #lastSessionTouchMs = new Map(); + #lastUserTouchMs = new Map(); + + /** Look up a session by its uuid. Returns `null` if not found. */ + async getByUuid(uuid) { + if (!uuid) return null; + + const cached = await this.#readCache(uuid); + if (cached) return cached; + + const rows = await this.clients.db.read( + 'SELECT * FROM `sessions` WHERE `uuid` = ? LIMIT 1', + [uuid], + ); + const normalized = this.#normalizeRow(rows[0]); + if (!normalized) return null; + + this.#writeCache(normalized).catch(() => { + // Best-effort backfill — local only. + }); + return normalized; + } + + /** Get all sessions for a user. */ + async getByUserId(userId) { + const rows = await this.clients.db.read( + 'SELECT * FROM `sessions` WHERE `user_id` = ?', + [userId], + ); + return rows.map((r) => this.#normalizeRow(r)).filter(Boolean); + } + + /** + * Create a new session. + * + * @param userId - User ID (numeric) + * @param meta - Metadata object (IP, user-agent, etc.) + * @returns The created session row + */ + async create(userId, meta = {}) { + const uuid = uuidv4(); + const now = Math.floor(Date.now() / 1000); + + meta.created = new Date().toISOString(); + meta.created_unix = now; + + await this.clients.db.write( + 'INSERT INTO `sessions` (`uuid`, `user_id`, `meta`, `last_activity`, `created_at`) VALUES (?, ?, ?, ?, ?)', + [uuid, userId, JSON.stringify(meta), now, now], + ); + + return { + uuid, + user_id: userId, + meta, + created_at: now, + last_activity: now, + }; + } + + /** Delete a session by uuid. Invalidates cache on this node + peers. */ + async removeByUuid(uuid) { + await this.clients.db.write('DELETE FROM `sessions` WHERE `uuid` = ?', [ + uuid, + ]); + await this.publishCacheKeys({ keys: [this.#cacheKey(uuid)] }); + } + + /** Update session activity timestamp. */ + async updateActivity(uuid, lastActivity) { + await this.clients.db.write( + 'UPDATE `sessions` SET `last_activity` = ? WHERE `uuid` = ? AND (`last_activity` IS NULL OR `last_activity` < ?)', + [lastActivity, uuid, lastActivity], + ); + } + + /** Update user-level last activity timestamp. */ + async updateUserActivity(userId, lastActivityTs) { + await this.clients.db.write( + 'UPDATE `user` SET `last_activity_ts` = ? WHERE `id` = ? AND (`last_activity_ts` IS NULL OR `last_activity_ts` < ?) LIMIT 1', + [lastActivityTs, userId, lastActivityTs], + ); + } + + /** + * Best-effort throttled activity touch. Updates the session row's + * `last_activity` column and the owning user's `user.last_activity_ts` + * if either hasn't been touched within `TOUCH_THROTTLE_MS`. + * + * Callers fire-and-forget — failures are swallowed. + */ + async touch({ uuid, userId } = {}) { + const nowMs = Date.now(); + + const sessionDue = + uuid && + nowMs - (this.#lastSessionTouchMs.get(uuid) ?? 0) >= + TOUCH_THROTTLE_MS; + const userDue = + userId && + nowMs - (this.#lastUserTouchMs.get(userId) ?? 0) >= + TOUCH_THROTTLE_MS; + + if (!sessionDue && !userDue) return; + + // Reserve the throttle slot before awaiting so concurrent + // callers on the same node coalesce. + if (sessionDue) { + if (this.#lastSessionTouchMs.size >= TOUCH_THROTTLE_MAX_ENTRIES) { + this.#lastSessionTouchMs.clear(); + } + this.#lastSessionTouchMs.set(uuid, nowMs); + } + if (userDue) { + if (this.#lastUserTouchMs.size >= TOUCH_THROTTLE_MAX_ENTRIES) { + this.#lastUserTouchMs.clear(); + } + this.#lastUserTouchMs.set(userId, nowMs); + } + + const tasks = []; + if (sessionDue) { + tasks.push( + this.updateActivity(uuid, Math.floor(nowMs / 1000)).catch( + () => {}, + ), + ); + } + if (userDue) { + tasks.push( + this.updateUserActivity(userId, sqlTimestamp(nowMs)).catch( + () => {}, + ), + ); + } + await Promise.all(tasks); + } + + // ── Internals ─────────────────────────────────────────────────── + + #cacheKey(uuid) { + return `${CACHE_KEY_PREFIX}:uuid:${uuid}`; + } + + async #readCache(uuid) { + try { + const raw = await this.clients.redis.get(this.#cacheKey(uuid)); + return raw ? JSON.parse(raw) : null; + } catch { + return null; + } + } + + async #writeCache(session) { + if (!session?.uuid) return; + try { + await this.clients.redis.set( + this.#cacheKey(session.uuid), + JSON.stringify(session), + 'EX', + CACHE_TTL_SECONDS, + ); + } catch { + // Best-effort local backfill. + } + } + + #normalizeRow(row) { + if (!row) return null; + // Meta may be stored as JSON string (SQLite) or already parsed + if (typeof row.meta === 'string') { + try { + row.meta = JSON.parse(row.meta); + } catch { + row.meta = {}; + } + } + return row; + } +} diff --git a/src/backend/stores/share/ShareStore.js b/src/backend/stores/share/ShareStore.js new file mode 100644 index 000000000..48dbb3e45 --- /dev/null +++ b/src/backend/stores/share/ShareStore.js @@ -0,0 +1,88 @@ +import { v4 as uuidv4 } from 'uuid'; +import { PuterStore } from '../types'; + +/** + * CRUD over the `share` table. + * + * Columns: id, uid (unique), issuer_user_id, recipient_email, data (JSON), + * created_at. + * + * Shares are pending permission grants sent to an email address. Once + * the recipient applies the share, the permissions are granted and the + * row is deleted. + */ +export class ShareStore extends PuterStore { + // ── Reads ──────────────────────────────────────────────────────── + + async getByUid(uid) { + const rows = await this.clients.db.read( + 'SELECT * FROM `share` WHERE `uid` = ? LIMIT 1', + [uid], + ); + return this.#normalizeRow(rows[0]) ?? null; + } + + async listByRecipientEmail(email) { + const rows = await this.clients.db.read( + 'SELECT * FROM `share` WHERE `recipient_email` = ? ORDER BY `created_at` DESC', + [email], + ); + return rows.map((r) => this.#normalizeRow(r)); + } + + async listByIssuer(issuerUserId) { + const rows = await this.clients.db.read( + 'SELECT * FROM `share` WHERE `issuer_user_id` = ? ORDER BY `created_at` DESC', + [issuerUserId], + ); + return rows.map((r) => this.#normalizeRow(r)); + } + + // ── Writes ─────────────────────────────────────────────────────── + + async create({ issuerUserId, recipientEmail, data }) { + if (!issuerUserId || !recipientEmail) { + throw new Error( + 'create: issuerUserId and recipientEmail are required', + ); + } + const uid = uuidv4(); + const serialized = + typeof data === 'string' ? data : JSON.stringify(data ?? {}); + await this.clients.db.write( + 'INSERT INTO `share` (`uid`, `issuer_user_id`, `recipient_email`, `data`) VALUES (?, ?, ?, ?)', + [uid, issuerUserId, recipientEmail, serialized], + ); + return this.getByUid(uid); + } + + async deleteByUid(uid) { + const result = await this.clients.db.write( + 'DELETE FROM `share` WHERE `uid` = ?', + [uid], + ); + return (result?.affectedRows ?? result?.changes ?? 0) > 0; + } + + async deleteByRecipientEmail(email) { + const result = await this.clients.db.write( + 'DELETE FROM `share` WHERE `recipient_email` = ?', + [email], + ); + return (result?.affectedRows ?? result?.changes ?? 0) > 0; + } + + // ── Internals ──────────────────────────────────────────────────── + + #normalizeRow(row) { + if (!row) return null; + if (typeof row.data === 'string') { + try { + row.data = JSON.parse(row.data); + } catch { + /* keep string */ + } + } + return row; + } +} diff --git a/src/backend/stores/subdomain/SubdomainStore.js b/src/backend/stores/subdomain/SubdomainStore.js new file mode 100644 index 000000000..2cdaf7221 --- /dev/null +++ b/src/backend/stores/subdomain/SubdomainStore.js @@ -0,0 +1,317 @@ +import { v4 as uuidv4 } from 'uuid'; +import { PuterStore } from '../types'; + +// Columns that may not be set through an `update` patch map. Defence-in-depth +// against future callers (admin routes, extensions, new REST handlers) that +// might forward `req.body` straight into the store: the driver's update +// builds its patch from a strict allow-list upstream, but the store is the +// last line before SQL. +// +// Categories: +// - identity: `id`, `uuid` +// - system timestamps: `ts` +// - name / identity: `subdomain` — v1 marks this `immutable: true`; rename +// would orphan DNS + ACL wiring tied to the old name. +// - ownership: `user_id`, `app_owner` — set via `create`, never via patch. +// Flipping either hands the site to another user / app. +// - access gate: `protected` — clearing this lets a future caller delete +// or rename a protected site (e.g. `puter-app-icons`). The driver itself +// honours the flag only via a read check in `delete`. +// - external resource link: `database_id` — repointing a Cloudflare D1 +// binding could route another site's traffic / writes to attacker DB. +// +// `root_dir_id`, `associated_app_id`, and `domain` are intentionally NOT +// here — they're legitimately editable through the driver with their own +// access checks (FS permission, app ownership, custom-domain validation). +const READ_ONLY_COLUMNS = new Set([ + 'id', + 'uuid', + 'ts', + 'subdomain', + 'user_id', + 'app_owner', + 'protected', + 'database_id', +]); + +const CACHE_KEY_PREFIX = 'subdomains'; +const CACHE_TTL_SECONDS = 60 * 60; +// Sentinel so 404s on the same public subdomain don't hit the DB repeatedly. +const NEGATIVE_CACHE_MARKER = '__none__'; +const NEGATIVE_CACHE_TTL_SECONDS = 60; + +export class SubdomainStore extends PuterStore { + // ── Reads ──────────────────────────────────────────────────────── + + async getByUuid(uuid, { userId } = {}) { + const where = + userId !== undefined + ? 'WHERE `uuid` = ? AND `user_id` = ?' + : 'WHERE `uuid` = ?'; + const params = userId !== undefined ? [uuid, userId] : [uuid]; + const rows = await this.clients.db.read( + `SELECT * FROM \`subdomains\` ${where} LIMIT 1`, + params, + ); + return rows[0] ?? null; + } + + async getBySubdomain(subdomain) { + if (!subdomain) return null; + + const cacheKey = this.#cacheKey(subdomain); + try { + const raw = await this.clients.redis.get(cacheKey); + if (raw === NEGATIVE_CACHE_MARKER) return null; + if (raw) { + const parsed = JSON.parse(raw); + if (parsed) return parsed; + } + } catch { + /* fall through */ + } + + const rows = await this.clients.db.read( + 'SELECT * FROM `subdomains` WHERE `subdomain` = ? LIMIT 1', + [subdomain], + ); + const row = rows[0] ?? null; + + if (row) { + this.clients.redis + .set(cacheKey, JSON.stringify(row), 'EX', CACHE_TTL_SECONDS) + .catch(() => {}); + } else { + this.clients.redis + .set( + cacheKey, + NEGATIVE_CACHE_MARKER, + 'EX', + NEGATIVE_CACHE_TTL_SECONDS, + ) + .catch(() => {}); + } + return row; + } + + async listByUserId(userId, { limit = 500 } = {}) { + const rows = await this.clients.db.read( + `SELECT * FROM \`subdomains\` WHERE \`user_id\` = ? LIMIT ?`, + [userId, limit], + ); + return rows; + } + + async listAll({ limit = 5000 } = {}) { + const rows = await this.clients.db.read( + `SELECT * FROM \`subdomains\` LIMIT ?`, + [limit], + ); + return rows; + } + + async existsBySubdomain(subdomain) { + // Reuse the positive/negative cache populated by getBySubdomain — + // creation uniqueness checks and the Workers quota path would + // otherwise punch through to the DB on every call. + const row = await this.getBySubdomain(subdomain); + return row != null; + } + + async countByUserId(userId) { + const rows = await this.clients.db.read( + 'SELECT COUNT(*) AS n FROM `subdomains` WHERE `user_id` = ?', + [userId], + ); + return rows[0]?.n ?? 0; + } + + async getByDomain(domain) { + const rows = await this.clients.db.read( + 'SELECT * FROM `subdomains` WHERE `domain` = ? LIMIT 1', + [domain], + ); + return rows[0] ?? null; + } + + async listByDomain(domain) { + return this.clients.db.read( + 'SELECT * FROM `subdomains` WHERE `domain` = ?', + [domain], + ); + } + + async listByUserIdAndPrefix(userId, prefix, extra = {}) { + if (!userId || prefix == null) return []; + + const like = `${prefix}%`; + let rows; + if (!extra.appId) { + rows = await this.clients.db.read( + 'SELECT * FROM `subdomains` WHERE `user_id` = ? AND `subdomain` LIKE ?', + [userId, like], + ); + } else { + rows = await this.clients.db.read( + 'SELECT * FROM `subdomains` WHERE `user_id` = ? AND `app_owner` = ? AND `subdomain` LIKE ?', + [userId, extra.appId, like], + ); + } + + return rows; + } + + // ── Writes ─────────────────────────────────────────────────────── + + /** @param {{ userId: number, subdomain: string, rootDirId?: number|null, associatedAppId?: number|null, appOwner?: number|null }} opts */ + async create({ + userId, + subdomain, + rootDirId = null, + associatedAppId = null, + appOwner = null, + }) { + if (!userId || !subdomain) { + throw new Error('create: userId and subdomain are required'); + } + const uuid = uuidv4(); + await this.clients.db.write( + `INSERT INTO \`subdomains\` + (\`uuid\`, \`subdomain\`, \`user_id\`, \`root_dir_id\`, \`associated_app_id\`, \`app_owner\`) + VALUES (?, ?, ?, ?, ?, ?)`, + [ + uuid, + subdomain, + userId, + rootDirId ?? null, + associatedAppId, + appOwner, + ], + ); + + const row = { + uuid, + subdomain, + user_id: userId, + root_dir_id: rootDirId ?? null, + associated_app_id: associatedAppId, + app_owner: appOwner, + }; + await this.#refreshCache(row); + await this.#invalidatePrefixListsForUser(userId); + + return row; + } + + async update(uuid, patch, { userId } = {}) { + const allowed = {}; + for (const [k, v] of Object.entries(patch)) { + if (READ_ONLY_COLUMNS.has(k)) continue; + allowed[k] = v; + } + const keys = Object.keys(allowed); + if (keys.length === 0) return this.getByUuid(uuid, { userId }); + + // Rename drops the old cache key. + const before = await this.getByUuid(uuid, { userId }); + + const setClause = keys.map((k) => `\`${k}\` = ?`).join(', '); + const values = keys.map((k) => allowed[k]); + + const where = + userId !== undefined + ? 'WHERE `uuid` = ? AND `user_id` = ?' + : 'WHERE `uuid` = ?'; + const whereParams = userId !== undefined ? [uuid, userId] : [uuid]; + + await this.clients.db.write( + `UPDATE \`subdomains\` SET ${setClause} ${where}`, + [...values, ...whereParams], + ); + + const after = await this.getByUuid(uuid, { userId }); + if (before?.subdomain && before.subdomain !== after?.subdomain) { + await this.publishCacheKeys({ + keys: [this.#cacheKey(before.subdomain)], + broadcast: true, + }); + } + if (after) { + await this.#refreshCache({ ...after, ...allowed }); + } + // A patched root_dir_id / associated_app_id / domain changes the rows + // the prefix-list cache would return, so drop those caches for the + // owning user(s). Covers pre- and post-rename owners in case the + // caller ever allowed re-assignment (currently we don't, but cheap). + const affectedUsers = new Set( + [before?.user_id, after?.user_id].filter((v) => v != null), + ); + for (const uid of affectedUsers) { + await this.#invalidatePrefixListsForUser(uid); + } + return after; + } + + async deleteByUuid(uuid, { userId } = {}) { + const row = await this.getByUuid(uuid, { userId }); + + const where = + userId !== undefined + ? 'WHERE `uuid` = ? AND `user_id` = ?' + : 'WHERE `uuid` = ?'; + const params = userId !== undefined ? [uuid, userId] : [uuid]; + + const result = await this.clients.db.write( + `DELETE FROM \`subdomains\` ${where}`, + params, + ); + const affected = (result?.affectedRows ?? result?.changes ?? 0) > 0; + if (affected && row?.subdomain) { + await this.publishCacheKeys({ + keys: [this.#cacheKey(row.subdomain)], + broadcast: true, + }); + if (row.user_id != null) { + await this.#invalidatePrefixListsForUser(row.user_id); + } + } + return affected; + } + + // ── Internals ──────────────────────────────────────────────────── + + #cacheKey(subdomain) { + return `${CACHE_KEY_PREFIX}:name:${subdomain}`; + } + + #prefixListTrackerKey(userId) { + return `${CACHE_KEY_PREFIX}:listByUserPrefixKeys:${userId}`; + } + + async #refreshCache(row) { + if (!row?.subdomain) return; + await this.publishCacheKeys({ + keys: [this.#cacheKey(row.subdomain)], + serializedData: JSON.stringify(row), + ttlSeconds: CACHE_TTL_SECONDS, + broadcast: true, + }); + } + + async #invalidatePrefixListsForUser(userId) { + if (userId == null) return; + const trackerKey = this.#prefixListTrackerKey(userId); + let cacheKeys = []; + try { + cacheKeys = await this.clients.redis.smembers(trackerKey); + } catch { + return; + } + const keysToInvalidate = [...cacheKeys, trackerKey]; + if (keysToInvalidate.length === 0) return; + await this.publishCacheKeys({ + keys: keysToInvalidate, + broadcast: true, + }); + } +} diff --git a/src/backend/stores/systemKv/SystemKVStore.ts b/src/backend/stores/systemKv/SystemKVStore.ts new file mode 100644 index 000000000..fdd2f3596 --- /dev/null +++ b/src/backend/stores/systemKv/SystemKVStore.ts @@ -0,0 +1,907 @@ +import { PuterStore } from '../types'; +import type { Actor } from '../../core/actor'; +import { + isSystemActor, + SYSTEM_ACTOR, + SYSTEM_ACTOR_UUID, +} from '../../core/actor'; +import { PUTER_KV_STORE_TABLE_DEFINITION } from './tableDefinition'; + +// ── Types ──────────────────────────────────────────────────────────── + +/** DynamoDB consumed-capacity units split by operation kind. */ +export interface KVUsage { + read: number; + write: number; +} + +/** Standard return envelope: `res` is the operation result, `usage` is the + * DynamoDB consumed capacity so callers can meter if they choose to. */ +export interface KVResult { + res: T; + usage: KVUsage; +} + +export interface KVOpts { + /** Optional actor — defaults to the system actor. */ + actor?: Actor; + /** Optional app uuid override for non-app-scoped actors. */ + appUuid?: string; +} + +export interface RecursiveRecord { + [k: string]: T | RecursiveRecord; +} + +// ── Helpers ────────────────────────────────────────────────────────── + +const GLOBAL_APP_KEY = 'os-global'; +const SYSTEM_NAMESPACE = `v1:${SYSTEM_ACTOR_UUID}:${GLOBAL_APP_KEY}`; +const MAX_KEY_BYTES = 1024; +const BATCH_GET_CHUNK = 100; +const PATH_CLEANER_REGEX = /[:\-+/*]/g; + +const emptyUsage = (): KVUsage => ({ read: 0, write: 0 }); + +const readUsage = (units: number | undefined): KVUsage => ({ + read: Number(units ?? 0), + write: 0, +}); + +const writeUsage = (units: number | undefined): KVUsage => ({ + read: 0, + write: Number(units ?? 0), +}); + +const addUsage = (a: KVUsage, b: KVUsage): KVUsage => ({ + read: a.read + b.read, + write: a.write + b.write, +}); + +const ensureActor = (opts?: KVOpts): Actor => opts?.actor ?? SYSTEM_ACTOR; + +const getNamespace = (actor: Actor, appUuidOverride?: string): string => { + if (isSystemActor(actor)) return SYSTEM_NAMESPACE; + const appUuid = actor.app?.uid ?? appUuidOverride ?? GLOBAL_APP_KEY; + return `v1:${actor.user.uuid}:${appUuid}`; +}; + +const assertKey = (key: string): void => { + if (key === '') throw new Error('kv: key is empty'); + if (Buffer.byteLength(key, 'utf8') > MAX_KEY_BYTES) { + throw new Error(`kv: key exceeds ${MAX_KEY_BYTES} byte limit`); + } +}; + +const encodeCursor = ( + pageKey?: Record, +): string | undefined => { + if (!pageKey || Object.keys(pageKey).length === 0) return undefined; + return Buffer.from(JSON.stringify(pageKey)).toString('base64'); +}; + +const decodeCursor = ( + cursor?: string | Record, +): Record | undefined => { + if (!cursor) return undefined; + if (typeof cursor === 'object') return cursor; + const trimmed = cursor.trim(); + if (trimmed === '') return undefined; + try { + return JSON.parse(Buffer.from(trimmed, 'base64').toString('utf8')); + } catch { + try { + return JSON.parse(trimmed); + } catch { + throw new Error('kv: invalid cursor'); + } + } +}; + +const normalizeLimit = (limit?: number): number | undefined => { + if (limit === undefined || limit === null) return undefined; + const parsed = Number(limit); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error('kv: limit must be a positive number'); + } + return Math.floor(parsed); +}; + +const normalizePattern = (pattern?: string): string | undefined => { + if (pattern === undefined || pattern === null) return undefined; + if (typeof pattern !== 'string') + throw new Error('kv: pattern must be a string'); + const trimmed = pattern.trim(); + if (trimmed === '') return undefined; + if (trimmed.endsWith('*')) { + const prefix = trimmed.slice(0, -1); + return prefix === '' ? undefined : prefix; + } + return trimmed; +}; + +const isPlainObject = (value: unknown): value is Record => + !!value && typeof value === 'object' && !Array.isArray(value); + +const objectsEqual = (left: unknown, right: unknown): boolean => { + if (left === right) return true; + if (!isPlainObject(left) || !isPlainObject(right)) return false; + const leftKeys = Object.keys(left); + const rightKeys = Object.keys(right); + if (leftKeys.length !== rightKeys.length) return false; + for (const key of leftKeys) { + if (!rightKeys.includes(key)) return false; + if (!objectsEqual(left[key], right[key])) return false; + } + return true; +}; + +const cleanAttrName = (chunk: string): string => + `#${chunk}`.replaceAll(PATH_CLEANER_REGEX, ''); + +// ── SystemKVStore ──────────────────────────────────────────────────── + +/** + * Underlying key-value store. Housed at the store layer so both services + * (permissions, metering) and drivers (`puter-kvstore`) can share it. + * + * Every method returns `{ res, usage }` — `res` is the operation result, + * `usage` is the DynamoDB consumed capacity split into read/write units so + * callers can meter at the driver level when needed. No metering happens + * inside the store itself. + * + * If `opts.actor` is omitted, operations are scoped to the system namespace. + */ +export class SystemKVStore extends PuterStore { + private tableName = 'store-kv-v1'; + private initialized: Promise | null = null; + + override async onServerStart(): Promise { + // For local/dynalite runs we need to create the table up front. + // For real AWS we assume the table already exists. + const ddbConfig = this.config.dynamo ?? {}; + if (ddbConfig.aws) return; + + this.initialized = this.clients.dynamo.createTableIfNotExists( + { ...PUTER_KV_STORE_TABLE_DEFINITION, TableName: this.tableName }, + 'ttl', + ); + await this.initialized; + } + + // ── Public API ─────────────────────────────────────────────────── + + async get( + { key }: { key: string | string[] }, + opts?: KVOpts, + ): Promise> { + const actor = ensureActor(opts); + const namespace = getNamespace(actor, opts?.appUuid); + const multi = Array.isArray(key); + const keys = multi ? key : [key]; + + for (const k of keys) assertKey(k); + + let kvEntries: Array<{ key: string; value?: unknown; ttl?: number }> = + []; + let usage = emptyUsage(); + + if (multi) { + const { entries, usage: u } = await this.getBatches( + namespace, + keys, + ); + kvEntries = entries; + usage = u; + } else { + const response = await this.clients.dynamo.get(this.tableName, { + namespace, + key, + }); + kvEntries = response.Item + ? [response.Item as (typeof kvEntries)[number]] + : []; + usage = readUsage( + response.ConsumedCapacity?.CapacityUnits as number | undefined, + ); + } + + const now = Date.now() / 1000; + const values = keys.map((k) => { + const entry = kvEntries.find((e) => e.key === k); + if (!entry) return null; + if (entry.ttl && entry.ttl <= now) return null; + return entry.value ?? null; + }); + + return { res: multi ? values : values[0], usage }; + } + + async set( + { + key, + value, + expireAt, + }: { key: string; value: unknown; expireAt?: number }, + opts?: KVOpts, + ): Promise> { + assertKey(key); + const actor = ensureActor(opts); + const namespace = getNamespace(actor, opts?.appUuid); + + const response = await this.clients.dynamo.put(this.tableName, { + namespace, + key, + value, + ttl: expireAt, + }); + + return { + res: true, + usage: writeUsage( + response.ConsumedCapacity?.CapacityUnits as number | undefined, + ), + }; + } + + async batchPut( + { + items, + }: { items: Array<{ key: string; value: unknown; expireAt?: number }> }, + opts?: KVOpts, + ): Promise> { + if (!Array.isArray(items) || items.length === 0) { + return { res: true, usage: emptyUsage() }; + } + + const byKey = new Map< + string, + { key: string; value: unknown; expireAt?: number } + >(); + for (const item of items) { + const k = String(item.key); + assertKey(k); + byKey.set(k, { + key: k, + value: item.value, + expireAt: item.expireAt, + }); + } + + const actor = ensureActor(opts); + const namespace = getNamespace(actor, opts?.appUuid); + + const putParams = Array.from(byKey.values()).map((item) => ({ + table: this.tableName, + item: { + namespace, + key: item.key, + value: item.value, + ttl: item.expireAt, + }, + })); + + const response = await this.clients.dynamo.batchPut(putParams); + const units = + response.ConsumedCapacity?.reduce( + (acc, curr) => acc + Number(curr.CapacityUnits ?? 0), + 0, + ) ?? byKey.size; + + return { res: true, usage: writeUsage(units || byKey.size) }; + } + + async del( + { key }: { key: string }, + opts?: KVOpts, + ): Promise> { + const actor = ensureActor(opts); + const namespace = getNamespace(actor, opts?.appUuid); + + const response = await this.clients.dynamo.del(this.tableName, { + namespace, + key, + }); + return { + res: true, + usage: writeUsage( + (response.ConsumedCapacity?.CapacityUnits as + | number + | undefined) ?? 1, + ), + }; + } + + async list( + { + as, + limit, + cursor, + pattern, + }: { + as?: 'keys' | 'values' | 'entries'; + limit?: number; + cursor?: string | Record; + pattern?: string; + }, + opts?: KVOpts, + ): Promise< + KVResult< + | string[] + | unknown[] + | { key: string; value: unknown }[] + | { items: string[]; cursor?: string } + | { items: unknown[]; cursor?: string } + | { items: { key: string; value: unknown }[]; cursor?: string } + > + > { + const actor = ensureActor(opts); + const namespace = getNamespace(actor, opts?.appUuid); + + const normalizedLimit = normalizeLimit(limit); + const pageKey = decodeCursor(cursor); + const normalizedPattern = normalizePattern(pattern); + const paginated = + normalizedLimit !== undefined || pageKey !== undefined; + + const response = await this.clients.dynamo.query( + this.tableName, + { namespace }, + normalizedLimit ?? 0, + pageKey, + '', + false, + normalizedPattern + ? { beginsWith: { key: 'key', value: normalizedPattern } } + : undefined, + ); + + const usage = readUsage( + (response.ConsumedCapacity?.CapacityUnits as number | undefined) ?? + 1, + ); + + const now = Date.now() / 1000; + const entries = (response.Items ?? []) + .filter((e) => e && (!e.ttl || e.ttl > now)) + .map((e) => ({ key: e!.key as string, value: e!.value })); + + const kind = as ?? 'entries'; + if (!['keys', 'values', 'entries'].includes(kind)) { + throw new Error('kv: list "as" must be keys, values, or entries'); + } + + let items: string[] | unknown[] | { key: string; value: unknown }[] = + entries; + if (kind === 'keys') items = entries.map((e) => e.key); + else if (kind === 'values') items = entries.map((e) => e.value); + + if (paginated) { + const nextCursor = encodeCursor( + response.LastEvaluatedKey as + | Record + | undefined, + ); + return { + res: nextCursor ? { items, cursor: nextCursor } : { items }, + usage, + }; + } + + return { res: items, usage }; + } + + async flush(opts?: KVOpts): Promise> { + const actor = ensureActor(opts); + const namespace = getNamespace(actor, opts?.appUuid); + + const response = await this.clients.dynamo.query(this.tableName, { + namespace, + }); + let usage = readUsage( + response.ConsumedCapacity?.CapacityUnits as number | undefined, + ); + + const entries = response.Items ?? []; + const results = ( + await Promise.all( + entries.map(async (entry) => { + try { + return await this.clients.dynamo.del(this.tableName, { + namespace, + key: entry.key, + }); + } catch (e) { + console.error('[kv] flush delete failed', entry.key, e); + return null; + } + }), + ) + ).filter(Boolean); + + const deleteUnits = results.reduce( + (acc, r) => acc + Number(r?.ConsumedCapacity?.CapacityUnits ?? 0), + 0, + ); + usage = addUsage(usage, writeUsage(deleteUnits)); + + return { res: true, usage }; + } + + async expireAt( + { key, timestamp }: { key: string; timestamp: number }, + opts?: KVOpts, + ): Promise> { + assertKey(key); + const actor = ensureActor(opts); + const namespace = getNamespace(actor, opts?.appUuid); + const usage = await this.rawExpireAt(namespace, key, Number(timestamp)); + return { res: undefined, usage }; + } + + async expire( + { key, ttl }: { key: string; ttl: number }, + opts?: KVOpts, + ): Promise> { + assertKey(key); + const actor = ensureActor(opts); + const namespace = getNamespace(actor, opts?.appUuid); + const timestamp = Math.floor(Date.now() / 1000) + Number(ttl); + const usage = await this.rawExpireAt(namespace, key, timestamp); + return { res: undefined, usage }; + } + + async incr>( + { key, pathAndAmountMap }: { key: string; pathAndAmountMap: T }, + opts?: KVOpts, + ): Promise< + KVResult> + > { + assertKey(key); + if (!pathAndAmountMap) + throw new Error('kv: incr requires pathAndAmountMap'); + if ( + Object.values(pathAndAmountMap).some((v) => typeof v !== 'number') + ) { + throw new Error( + 'kv: all values in pathAndAmountMap must be numbers', + ); + } + + const actor = ensureActor(opts); + const namespace = getNamespace(actor, opts?.appUuid); + + const createPathsUsage = await this.createPaths( + namespace, + key, + Object.keys(pathAndAmountMap), + ); + + const setStatements = Object.entries(pathAndAmountMap).map( + ([valPath, _amt], idx) => { + const attrName = ['value', ...valPath.split('.')] + .filter(Boolean) + .map(cleanAttrName) + .join('.'); + return `${attrName} = if_not_exists(${attrName}, :start${idx}) + :incr${idx}`; + }, + ); + const valueAttributeValues = Object.entries(pathAndAmountMap).reduce( + (acc, [_path, amt], idx) => { + acc[`:incr${idx}`] = amt; + acc[`:start${idx}`] = 0; + return acc; + }, + {} as Record, + ); + const valueAttributeNames = Object.entries(pathAndAmountMap).reduce( + (acc, [valPath]) => { + ['value', ...valPath.split('.')] + .filter(Boolean) + .forEach((chunk) => { + const cleanedChunk = chunk.split(/\[\d*\]/g)[0]; + acc[cleanAttrName(cleanedChunk)] = cleanedChunk; + }); + return acc; + }, + {} as Record, + ); + + const response = await this.clients.dynamo.update( + this.tableName, + { key, namespace }, + `SET ${setStatements.join(', ')}`, + valueAttributeValues, + { ...valueAttributeNames, '#value': 'value' }, + ); + + const usage = writeUsage( + Number(response.ConsumedCapacity?.CapacityUnits ?? 0) + + createPathsUsage, + ); + + return { res: response.Attributes?.value, usage }; + } + + async decr>( + { key, pathAndAmountMap }: { key: string; pathAndAmountMap: T }, + opts?: KVOpts, + ): Promise< + KVResult> + > { + const negated = Object.fromEntries( + Object.entries(pathAndAmountMap).map(([k, v]) => [k, -v]), + ) as T; + return this.incr({ key, pathAndAmountMap: negated }, opts); + } + + async add( + { + key, + pathAndValueMap, + }: { key: string; pathAndValueMap: Record }, + opts?: KVOpts, + ): Promise> { + assertKey(key); + if (!pathAndValueMap || Object.keys(pathAndValueMap).length === 0) { + throw new Error('kv: add requires pathAndValueMap'); + } + + const actor = ensureActor(opts); + const namespace = getNamespace(actor, opts?.appUuid); + + const createPathsUsage = await this.createPaths( + namespace, + key, + Object.keys(pathAndValueMap), + ); + + const setStatements = Object.entries(pathAndValueMap).map( + ([valPath], idx) => { + const attrName = ['value', ...valPath.split('.')] + .filter(Boolean) + .map(cleanAttrName) + .join('.'); + return `${attrName} = list_append(if_not_exists(${attrName}, :emptyList${idx}), :append${idx})`; + }, + ); + const valueAttributeValues = Object.entries(pathAndValueMap).reduce( + (acc, [_path, val], idx) => { + acc[`:append${idx}`] = Array.isArray(val) ? val : [val]; + acc[`:emptyList${idx}`] = []; + return acc; + }, + {} as Record, + ); + const valueAttributeNames = Object.entries(pathAndValueMap).reduce( + (acc, [valPath]) => { + ['value', ...valPath.split('.')] + .filter(Boolean) + .forEach((chunk) => { + const cleanedChunk = chunk.split(/\[\d*\]/g)[0]; + acc[cleanAttrName(cleanedChunk)] = cleanedChunk; + }); + return acc; + }, + {} as Record, + ); + + const response = await this.clients.dynamo.update( + this.tableName, + { key, namespace }, + `SET ${setStatements.join(', ')}`, + valueAttributeValues, + { ...valueAttributeNames, '#value': 'value' }, + ); + + const usage = writeUsage( + Number(response.ConsumedCapacity?.CapacityUnits ?? 0) + + createPathsUsage, + ); + + return { res: response.Attributes?.value, usage }; + } + + async remove( + { key, paths }: { key: string; paths: string[] }, + opts?: KVOpts, + ): Promise> { + assertKey(key); + if (!paths || paths.length === 0) { + throw new Error('kv: remove requires paths'); + } + + const actor = ensureActor(opts); + const namespace = getNamespace(actor, opts?.appUuid); + + const removeStatements = paths.map((valPath) => { + return ['value', ...valPath.split('.')] + .filter(Boolean) + .map((chunk) => { + const cleanedChunk = chunk.split(/\[\d*\]/g)[0]; + const indexSuffix = chunk.slice(cleanedChunk.length); + return `${cleanAttrName(cleanedChunk)}${indexSuffix}`; + }) + .join('.'); + }); + const valueAttributeNames = paths.reduce( + (acc, valPath) => { + ['value', ...valPath.split('.')] + .filter(Boolean) + .forEach((chunk) => { + const cleanedChunk = chunk.split(/\[\d*\]/g)[0]; + acc[cleanAttrName(cleanedChunk)] = cleanedChunk; + }); + return acc; + }, + {} as Record, + ); + + try { + const response = await this.clients.dynamo.update( + this.tableName, + { key, namespace }, + `REMOVE ${removeStatements.join(', ')}`, + undefined, + { ...valueAttributeNames, '#value': 'value' }, + ); + return { + res: response.Attributes?.value, + usage: writeUsage( + (response.ConsumedCapacity?.CapacityUnits as + | number + | undefined) ?? 1, + ), + }; + } catch (e) { + const err = e as Error; + if ( + err?.name === 'ValidationException' && + /document path|invalid updateexpression/i.test(err.message) + ) { + // Path didn't exist — treat as no-op, return current value + const fallback = await this.get({ key }, opts); + return { + res: fallback.res, + usage: addUsage(fallback.usage, writeUsage(1)), + }; + } + throw e; + } + } + + async update( + { + key, + pathAndValueMap, + ttl, + }: { + key: string; + pathAndValueMap: Record; + ttl?: number; + }, + opts?: KVOpts, + ): Promise> { + assertKey(key); + if (!pathAndValueMap || Object.keys(pathAndValueMap).length === 0) { + throw new Error('kv: update requires pathAndValueMap'); + } + + const actor = ensureActor(opts); + const namespace = getNamespace(actor, opts?.appUuid); + + const createPathsUsage = await this.createPaths( + namespace, + key, + Object.keys(pathAndValueMap), + ); + + const setStatements = Object.entries(pathAndValueMap).map( + ([valPath], idx) => { + const attrName = ['value', ...valPath.split('.')] + .filter(Boolean) + .map(cleanAttrName) + .join('.'); + return `${attrName} = :value${idx}`; + }, + ); + const valueAttributeValues = Object.entries(pathAndValueMap).reduce( + (acc, [_path, val], idx) => { + acc[`:value${idx}`] = val; + return acc; + }, + {} as Record, + ); + const valueAttributeNames = Object.entries(pathAndValueMap).reduce( + (acc, [valPath]) => { + ['value', ...valPath.split('.')] + .filter(Boolean) + .forEach((chunk) => { + const cleanedChunk = chunk.split(/\[\d*\]/g)[0]; + acc[cleanAttrName(cleanedChunk)] = cleanedChunk; + }); + return acc; + }, + {} as Record, + ); + + if (ttl !== undefined) { + const ttlSeconds = Number(ttl); + if (Number.isNaN(ttlSeconds)) + throw new Error('kv: ttl must be a number'); + const timestamp = Math.floor(Date.now() / 1000) + ttlSeconds; + setStatements.push('#ttl = :ttl'); + valueAttributeValues[':ttl'] = timestamp; + valueAttributeNames['#ttl'] = 'ttl'; + } + + const response = await this.clients.dynamo.update( + this.tableName, + { key, namespace }, + `SET ${setStatements.join(', ')}`, + valueAttributeValues, + { ...valueAttributeNames, '#value': 'value' }, + ); + + const usage = writeUsage( + Number(response.ConsumedCapacity?.CapacityUnits ?? 0) + + createPathsUsage, + ); + + return { res: response.Attributes?.value, usage }; + } + + // ── Internals ──────────────────────────────────────────────────── + + private async getBatches( + namespace: string, + allKeys: string[], + ): Promise<{ + entries: Array<{ key: string; value?: unknown; ttl?: number }>; + usage: KVUsage; + }> { + const batches: string[][] = []; + for (let i = 0; i < allKeys.length; i += BATCH_GET_CHUNK) { + batches.push(allKeys.slice(i, i + BATCH_GET_CHUNK)); + } + + const results = await Promise.all( + batches.map(async (keys) => { + const requests = [...new Set(keys)].map((k) => ({ + table: this.tableName, + items: { namespace, key: k }, + })); + const response = await this.clients.dynamo.batchGet(requests); + const entries = (response.Responses?.[this.tableName] ?? + []) as Array<{ + key: string; + value?: unknown; + ttl?: number; + }>; + const units = + response.ConsumedCapacity?.reduce( + (acc, curr) => acc + Number(curr.CapacityUnits ?? 0), + 0, + ) ?? 0; + return { entries, units }; + }), + ); + + return results.reduce( + (acc, curr) => { + acc.entries.push(...curr.entries); + acc.usage.read += curr.units; + return acc; + }, + { + entries: [] as Array<{ + key: string; + value?: unknown; + ttl?: number; + }>, + usage: emptyUsage(), + }, + ); + } + + private async rawExpireAt( + namespace: string, + key: string, + timestamp: number, + ): Promise { + const response = await this.clients.dynamo.update( + this.tableName, + { key, namespace }, + 'SET #ttl = :ttl, #value = if_not_exists(#value, :defaultValue)', + { ':ttl': timestamp, ':defaultValue': null }, + { '#ttl': 'ttl', '#value': 'value' }, + ); + return writeUsage( + (response.ConsumedCapacity?.CapacityUnits as number | undefined) ?? + 1, + ); + } + + /** + * Ensure each intermediate map layer exists for a set of nested paths. + * Returns write units consumed. DDB can't set nested paths on missing + * parents in one expression, so we walk the layers and + * `SET ... if_not_exists(..., {})` each one. + */ + private async createPaths( + namespace: string, + key: string, + pathList: string[], + ): Promise { + const nestedMapValue = (() => { + const valueRoot: Record = {}; + let hasPaths = false; + pathList.forEach((valPath) => { + if (!valPath) return; + hasPaths = true; + const chunks = valPath.split('.').filter(Boolean); + let cursor: Record = valueRoot; + for (let i = 0; i < chunks.length - 1; i++) { + const chunk = chunks[i]; + const existing = cursor[chunk]; + if ( + !existing || + typeof existing !== 'object' || + Array.isArray(existing) + ) { + cursor[chunk] = {}; + } + cursor = cursor[chunk] as Record; + } + }); + return hasPaths ? valueRoot : null; + })(); + + if (!nestedMapValue) return 0; + + const allIntermediatePaths = new Set(); + pathList.forEach((valPath) => { + const chunks = ['value', ...valPath.split('.')].filter(Boolean); + for (let i = 1; i < chunks.length; i++) { + allIntermediatePaths.add(chunks.slice(0, i).join('.')); + } + }); + + let writeUnits = 0; + const orderedPaths = [...allIntermediatePaths].sort( + (left, right) => left.split('.').length - right.split('.').length, + ); + + for (const layerPath of orderedPaths) { + const chunks = layerPath.split('.'); + const attrName = chunks.map(cleanAttrName).join('.'); + const expressionNames: Record = {}; + chunks.forEach((chunk) => { + const cleanedChunk = chunk.split(/\[\d*\]/g)[0]; + expressionNames[cleanAttrName(cleanedChunk)] = cleanedChunk; + }); + const isRootLayer = layerPath === 'value'; + const expressionValues = isRootLayer + ? { ':nestedMap': nestedMapValue } + : { ':emptyMap': {} }; + const valueToken = isRootLayer ? ':nestedMap' : ':emptyMap'; + + const response = await this.clients.dynamo.update( + this.tableName, + { key, namespace }, + `SET ${attrName} = if_not_exists(${attrName}, ${valueToken})`, + expressionValues, + expressionNames, + ); + writeUnits += Number(response.ConsumedCapacity?.CapacityUnits ?? 0); + + if ( + isRootLayer && + objectsEqual(response.Attributes?.value, nestedMapValue) + ) { + return writeUnits; + } + } + return writeUnits; + } +} diff --git a/src/backend/src/services/DynamoKVStore/tableDefinition.ts b/src/backend/stores/systemKv/tableDefinition.ts similarity index 91% rename from src/backend/src/services/DynamoKVStore/tableDefinition.ts rename to src/backend/stores/systemKv/tableDefinition.ts index b10dca9e9..1add3dd41 100644 --- a/src/backend/src/services/DynamoKVStore/tableDefinition.ts +++ b/src/backend/stores/systemKv/tableDefinition.ts @@ -1,4 +1,4 @@ -import { CreateTableCommandInput } from '@aws-sdk/client-dynamodb'; +import type { CreateTableCommandInput } from '@aws-sdk/client-dynamodb'; export const PUTER_KV_STORE_TABLE_DEFINITION: CreateTableCommandInput = { TableName: 'store-kv-v1', diff --git a/src/backend/stores/types.ts b/src/backend/stores/types.ts new file mode 100644 index 000000000..6104090d8 --- /dev/null +++ b/src/backend/stores/types.ts @@ -0,0 +1,93 @@ +import type { puterClients } from '../clients'; +import type { IConfig, LayerInstances, WithLifecycle } from '../types'; + +/** + * Stores may depend on clients and on *prior* stores (those declared earlier + * in the registry). The `stores` argument is the accumulating registry — it + * only contains peers constructed before this one. + */ +export type IPuterStore = new ( + config: IConfig, + clients: LayerInstances, + stores: Partial>, +) => T; + +const DEFAULT_BROADCAST_REFRESH_TTL_SECONDS = 15 * 60; + +export const PuterStore = class PuterStore implements WithLifecycle { + constructor( + protected config: IConfig, + protected clients: LayerInstances, + protected stores: Partial> = {}, + ) {} + public onServerStart() { + return; + } + public onServerPrepareShutdown() { + return; + } + public onServerShutdown() { + return; + } + + /** + * Refresh (pass `serializedData`) or invalidate (omit it) cache keys + * locally. Pass `broadcast: true` to also send the same mutation to + * peer nodes via `outer.cacheUpdate`. Pipelined for cluster-mode safety + * (no multi-key DEL/MSET that would CROSSSLOT on Valkey). + */ + protected async publishCacheKeys(params: { + keys: string[]; + serializedData?: string; + ttlSeconds?: number; + broadcast?: boolean; + }): Promise { + const { keys, serializedData } = params; + if (keys.length === 0) return; + + const ttl = Math.max( + 1, + Math.floor( + params.ttlSeconds ?? DEFAULT_BROADCAST_REFRESH_TTL_SECONDS, + ), + ); + + try { + const pipeline = this.clients.redis.pipeline(); + if (serializedData === undefined) { + for (const key of keys) pipeline.del(key); + } else { + for (const key of keys) { + pipeline.set(key, serializedData, 'EX', ttl); + } + } + await pipeline.exec(); + } catch { + console.warn( + '[PuterStore] publishCacheKeys failed to update local cache:', + keys, + ); + } + + if (!params.broadcast) return; + + try { + const payload = + serializedData === undefined + ? { cacheKey: keys } + : { cacheKey: keys, data: serializedData, ttlSeconds: ttl }; + this.clients.event.emit('outer.cacheUpdate', payload, {}); + } catch { + console.warn( + '[PuterStore] publishCacheKeys failed to broadcast cache update:', + keys, + ); + } + } +} satisfies IPuterStore; + +export type IPuterStoreRegistry = Record< + string, + | IPuterStore + | (InstanceType> & Record) +>; diff --git a/src/backend/stores/user/UserStore.ts b/src/backend/stores/user/UserStore.ts new file mode 100644 index 000000000..c32004fff --- /dev/null +++ b/src/backend/stores/user/UserStore.ts @@ -0,0 +1,468 @@ +import { PuterStore } from '../types'; + +// ── Types ──────────────────────────────────────────────────────────── + +/** + * Canonical user row. Typed fields cover everything auth/acl/quota code + * actually reads; `[k: string]: unknown` keeps the escape hatch for + * lesser-used columns the store doesn't surface yet. + * + * Note: `suspended` / `email_confirmed` / `requires_email_confirmation` come + * off the DB as MySQL TINYINT or SQLite INTEGER — 0 or 1. We coerce to + * booleans in `#normalizeRow` so downstream code gets consistent types. + */ +export interface UserRow { + id: number; + uuid: string; + username: string; + email?: string | null; + /** True when an admin has suspended the account. */ + suspended?: boolean; + /** True when the user has confirmed the email currently on file. */ + email_confirmed?: boolean; + /** True for accounts that must confirm email before taking most actions. */ + requires_email_confirmation?: boolean; + /** Metadata JSON blob; decoded on read when the DB returns it as a string. */ + metadata?: Record; + [k: string]: unknown; +} + +/** + * Identifying properties the store will look users up by. Adding a new + * property is as simple as adding a key here — lookups + cache fan-out + * follow automatically. + */ +export const USER_ID_PROPERTIES = ['id', 'uuid', 'username', 'email'] as const; +export type UserIdProperty = (typeof USER_ID_PROPERTIES)[number]; + +// ── Constants ──────────────────────────────────────────────────────── + +const CACHE_KEY_PREFIX = 'users'; +const CACHE_TTL_SECONDS = 15 * 60; +// Cap on placeholders per `IN (?, ?, …)` query. SQLite's default parameter +// limit is 999; staying well under that keeps `getByIds` portable across +// backends without splitting the cap by driver. +const BULK_QUERY_CHUNK_SIZE = 200; + +// ── UserStore ──────────────────────────────────────────────────────── + +/** + * Persistence + cache for the `user` table. Provides a multi-key Redis cache + * over property-indexed lookups and thin `user`-table accessors. + * + * Intentionally NOT folded in: + * - `generate_default_fsentries` (filesystem concern; belongs with FS) + * - `whoami.get_details` enrichment (service-level, not store) + * - runtime identifying-property registration — the identifying properties + * are declared inline in `USER_ID_PROPERTIES`; callers that need more + * add the property to that tuple. + */ +export class UserStore extends PuterStore { + // ── Reads ──────────────────────────────────────────────────────── + + async getById( + id: number, + opts: { cached?: boolean; force?: boolean } = {}, + ): Promise { + return this.getByProperty('id', id, opts); + } + + async getByUuid( + uuid: string, + opts: { cached?: boolean; force?: boolean } = {}, + ): Promise { + return this.getByProperty('uuid', uuid, opts); + } + + async getByUsername( + username: string, + opts: { cached?: boolean; force?: boolean } = {}, + ): Promise { + return this.getByProperty('username', username, opts); + } + + async getByEmail( + email: string, + opts: { cached?: boolean; force?: boolean } = {}, + ): Promise { + return this.getByProperty('email', email, opts); + } + + /** + * Batched lookup by id. Dedupes input ids, reads cache via a pipelined + * MGET, and resolves remaining misses with a single + * `SELECT … WHERE id IN (…)` per chunk. Use this in place of + * `Promise.all(ids.map(getById))` to avoid one connection per row on + * large id sets. + * + * Missing ids (no DB row) are simply absent from the returned map. + */ + async getByIds(ids: number[]): Promise> { + const result = new Map(); + const uniqueIds = [ + ...new Set( + (Array.isArray(ids) ? ids : []).filter( + (id): id is number => typeof id === 'number', + ), + ), + ]; + if (uniqueIds.length === 0) return result; + + const missingIds: number[] = []; + try { + const pipeline = this.clients.redis.pipeline(); + for (const id of uniqueIds) { + pipeline.get(this.#cacheKey('id', id)); + } + const cacheResults = (await pipeline.exec()) ?? []; + for (let i = 0; i < uniqueIds.length; i++) { + const id = uniqueIds[i]; + const raw = cacheResults[i]?.[1]; + if (typeof raw === 'string') { + try { + result.set(id, JSON.parse(raw) as UserRow); + continue; + } catch { + // Fall through to DB on any parse failure. + } + } + missingIds.push(id); + } + } catch { + missingIds.push(...uniqueIds); + } + + for ( + let offset = 0; + offset < missingIds.length; + offset += BULK_QUERY_CHUNK_SIZE + ) { + const chunk = missingIds.slice( + offset, + offset + BULK_QUERY_CHUNK_SIZE, + ); + const placeholders = chunk.map(() => '?').join(', '); + const rows = (await this.clients.db.tryHardRead( + `SELECT * FROM \`user\` WHERE \`id\` IN (${placeholders})`, + chunk, + )) as Array>; + for (const row of rows) { + const user = this.#normalizeRow(row); + result.set(user.id, user); + this.#writeCache(user).catch(() => { + // Best-effort cache backfill. + }); + } + } + + return result; + } + + /** + * Look up a user by the canonical `clean_email` column. Used by signup + * and OIDC link flows to collapse gmail-style aliases (`foo.bar+tag@…`) + * to the same account. + * + * Not cached — `clean_email` isn't an identifying property and callers + * use this for duplicate detection at write time, which needs fresh + * reads. Rehydrates through `getById` so the caller gets a normalized + * row (and warms the id-keyed cache for subsequent reads). + */ + async getByCleanEmail(cleanEmailValue: string): Promise { + if (!cleanEmailValue) return null; + const rows = (await this.clients.db.tryHardRead( + 'SELECT `id` FROM `user` WHERE `clean_email` = ? LIMIT 1', + [cleanEmailValue], + )) as Array<{ id: number }>; + const row = rows[0]; + if (!row) return null; + return this.getById(row.id as number); + } + + /** + * Generic property lookup. Fast-path reads redis first (cache is + * multi-key — every identifying property points at the same serialized + * row). On miss, falls back to DB and backfills the cache. + * + * `force: true` bypasses cache both on read and on replication. + */ + async getByProperty( + prop: UserIdProperty, + value: unknown, + options: { cached?: boolean; force?: boolean } = {}, + ): Promise { + const cached = options.cached ?? true; + const force = options.force ?? false; + + if (cached && !force) { + const hit = await this.#readCache(prop, value); + if (hit) return hit; + } + + // Replication-aware read: on `force`, go straight to the primary + // (`pread`) to bypass replica lag for hot reads (e.g., immediately + // after a signup). Otherwise `tryHardRead` parallels primary + + // replica and prefers whichever returns rows. + const sql = `SELECT * FROM \`user\` WHERE \`${prop}\` = ? LIMIT 1`; + const rows = force + ? await this.clients.db.pread(sql, [value]) + : await this.clients.db.tryHardRead(sql, [value]); + const row = rows[0]; + if (!row) return null; + + const user = this.#normalizeRow(row); + // Fire-and-forget cache write — don't block the caller on redis. + this.#writeCache(user).catch(() => { + // Best-effort cache; swallow errors. + }); + return user; + } + + // ── Writes ─────────────────────────────────────────────────────── + + /** + * Merge-update a user's `metadata` JSON blob. Reads current value, + * applies `Object.assign` semantics, and writes back. Invalidates + * every cache key pointing at this user on success. + */ + /** + * Create a new user row. + * + * Returns the created user (by id). Password must already be hashed. + * Pass `null` for temporary users (no email, no password). + */ + async create(fields: { + username: string; + uuid: string; + password: string | null; + email: string | null; + clean_email?: string | null; + free_storage?: number | null; + requires_email_confirmation?: boolean; + email_confirm_code?: string | null; + email_confirm_token?: string | null; + audit_metadata?: Record | null; + signup_ip?: string | null; + signup_ip_forwarded?: string | null; + signup_user_agent?: string | null; + signup_origin?: string | null; + signup_server?: string | null; + referrer?: string | null; + last_activity_ts?: string | null; + }): Promise { + const result = await this.clients.db.write( + `INSERT INTO \`user\` + (username, + email, + clean_email, + password, + uuid, + free_storage, + requires_email_confirmation, + email_confirm_code, + email_confirm_token, + audit_metadata, + signup_ip, + signup_ip_forwarded, + signup_user_agent, + signup_origin, + signup_server, + referrer, + last_activity_ts) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + fields.username, + fields.email, + fields.clean_email ?? null, + fields.password, + fields.uuid, + fields.free_storage ?? null, + fields.requires_email_confirmation ? 1 : 0, + fields.email_confirm_code ?? null, + fields.email_confirm_token ?? null, + fields.audit_metadata + ? JSON.stringify(fields.audit_metadata) + : null, + fields.signup_ip ?? null, + fields.signup_ip_forwarded ?? null, + fields.signup_user_agent ?? null, + fields.signup_origin ?? null, + fields.signup_server ?? null, + fields.referrer ?? null, + fields.last_activity_ts ?? null, + ], + ); + + const insertId = (result as unknown as { insertId?: number }).insertId; + if (!insertId) + throw new Error('Failed to create user — no insertId returned'); + + const user = await this.getById(insertId, { force: true }); + if (!user) throw new Error('Failed to fetch created user'); + return user; + } + + /** + * Update arbitrary user fields by id. Invalidates cache on write. + * + * Only pass whitelisted columns — this uses string interpolation for + * column names for ergonomic call sites. Never take column names from + * request bodies. + */ + async update( + userId: number, + patch: Record, + ): Promise { + const keys = Object.keys(patch); + if (keys.length === 0) return; + + const setClause = keys.map((k) => `\`${k}\` = ?`).join(', '); + const values = keys.map((k) => patch[k]); + + await this.clients.db.write( + `UPDATE \`user\` SET ${setClause} WHERE \`id\` = ?`, + [...values, userId], + ); + + const fresh = await this.getByProperty('id', userId, { force: true }); + if (fresh) { + await this.#refreshCache({ ...fresh, ...patch }); + } else { + await this.invalidateById(userId); + } + } + + async updateMetadata( + userId: number, + patch: Record, + ): Promise { + const user = await this.getById(userId); + const current: Record = user?.metadata ?? {}; + const merged = { ...current, ...patch }; + + await this.clients.db.write( + 'UPDATE `user` SET `metadata` = ? WHERE `id` = ?', + [JSON.stringify(merged), userId], + ); + if (user) { + const refreshed: UserRow = { ...user, metadata: merged }; + await this.#refreshCache(refreshed); + } + } + + async invalidate(user: UserRow): Promise { + const keys = this.#cacheKeysForUser(user); + await this.publishCacheKeys({ keys, broadcast: true }); + } + + /** Invalidate by id — fetches the cached row first so we know all its keys. */ + async invalidateById(id: number): Promise { + const cached = await this.#readCache('id', id); + if (cached) await this.invalidate(cached); + } + + // ── Internals ──────────────────────────────────────────────────── + + #cacheKey(prop: UserIdProperty, value: unknown): string { + return `${CACHE_KEY_PREFIX}:${prop}:${String(value)}`; + } + + #cacheKeysForUser(user: UserRow): string[] { + const keys: string[] = []; + for (const prop of USER_ID_PROPERTIES) { + const value = user[prop]; + if (value === undefined || value === null || value === '') continue; + keys.push(this.#cacheKey(prop, value)); + } + return keys; + } + + async #readCache( + prop: UserIdProperty, + value: unknown, + ): Promise { + try { + const raw = await this.clients.redis.get( + this.#cacheKey(prop, value), + ); + if (!raw) return null; + const parsed = JSON.parse(raw) as UserRow; + // Cached rows were normalized on the write path, so booleans are booleans. + return parsed; + } catch { + return null; + } + } + + async #writeCache(user: UserRow): Promise { + const keys = this.#cacheKeysForUser(user); + if (keys.length === 0) return; + const serialized = JSON.stringify(user); + await Promise.all( + keys.map((key) => + this.clients.redis.set( + key, + serialized, + 'EX', + CACHE_TTL_SECONDS, + ), + ), + ); + } + + async #refreshCache(user: UserRow): Promise { + const keys = this.#cacheKeysForUser(user); + if (keys.length === 0) return; + await this.publishCacheKeys({ + keys, + serializedData: JSON.stringify(user), + ttlSeconds: CACHE_TTL_SECONDS, + broadcast: true, + }); + } + + /** + * Coerce raw DB row values into consistent JS types. MySQL returns + * BOOLEAN/TINYINT as 0|1; SQLite returns INTEGER. JSON columns come as + * strings on SQLite, parsed objects on MySQL. + */ + #normalizeRow(row: Record): UserRow { + const { referral_code: _referralCode, ...rest } = row; + const asBool = (v: unknown): boolean | undefined => { + if (v === null || v === undefined) return undefined; + if (typeof v === 'boolean') return v; + if (typeof v === 'number') return v !== 0; + if (typeof v === 'string') + return v !== '0' && v.toLowerCase() !== 'false' && v !== ''; + return Boolean(v); + }; + + const metadata = this.clients.db.case<() => Record>({ + mysql: () => (row.metadata as Record) ?? {}, + otherwise: () => { + if (row.metadata == null) return {}; + if (typeof row.metadata === 'object') + return row.metadata as Record; + try { + return JSON.parse(String(row.metadata)); + } catch { + return {}; + } + }, + })(); + + return { + ...rest, + id: Number(rest.id), + uuid: String(rest.uuid), + username: String(rest.username), + email: rest.email == null ? null : String(rest.email), + suspended: asBool(rest.suspended), + email_confirmed: asBool(rest.email_confirmed), + requires_email_confirmation: asBool( + rest.requires_email_confirmation, + ), + metadata, + }; + } +} diff --git a/src/backend/telemetry.ts b/src/backend/telemetry.ts new file mode 100644 index 000000000..e15d1f37f --- /dev/null +++ b/src/backend/telemetry.ts @@ -0,0 +1,53 @@ +import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'; +import { Resource } from '@opentelemetry/resources'; +import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; +import { NodeSDK } from '@opentelemetry/sdk-node'; +import { + ParentBasedSampler, + TraceIdRatioBasedSampler, +} from '@opentelemetry/sdk-trace-base'; +import { + ATTR_SERVICE_NAME, + ATTR_SERVICE_VERSION, +} from '@opentelemetry/semantic-conventions'; + +const endpoint = + process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? 'http://localhost:4317'; +const sampleRatio = Number(process.env.OTEL_TRACE_SAMPLE_RATIO ?? 0.05); + +const sdk = new NodeSDK({ + resource: new Resource({ + [ATTR_SERVICE_NAME]: process.env.OTEL_SERVICE_NAME ?? 'puter-backend', + [ATTR_SERVICE_VERSION]: process.env.npm_package_version ?? '0.0.0', + 'deployment.environment': process.env.NODE_ENV ?? 'development', + }), + // Honour upstream sampling decisions; for root spans, keep ~5% of traces. + sampler: new ParentBasedSampler({ + root: new TraceIdRatioBasedSampler(sampleRatio), + }), + traceExporter: new OTLPTraceExporter({ url: endpoint }), + metricReader: new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter({ url: endpoint }), + exportIntervalMillis: 60_000, + }), + instrumentations: [ + getNodeAutoInstrumentations({ + // Too noisy — every file read / dns lookup becomes a span. + '@opentelemetry/instrumentation-fs': { enabled: false }, + '@opentelemetry/instrumentation-dns': { enabled: false }, + '@opentelemetry/instrumentation-net': { enabled: false }, + }), + ], +}); + +sdk.start(); + +const shutdown = () => { + sdk.shutdown() + .catch((err) => console.error('[telemetry] shutdown error', err)) + .finally(() => process.exit(0)); +}; +process.on('SIGTERM', shutdown); +process.on('SIGINT', shutdown); diff --git a/src/backend/test/modules/captcha/integration/extension-integration.test.js b/src/backend/test/modules/captcha/integration/extension-integration.test.js deleted file mode 100644 index 0dba99056..000000000 --- a/src/backend/test/modules/captcha/integration/extension-integration.test.js +++ /dev/null @@ -1,270 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { describe, it, expect, beforeEach, vi } from 'vitest'; - -// Mock the Context and services -const Context = { - get: vi.fn(), -}; - -// Mock the extension service -class ExtensionService { - constructor () { - this.extensions = new Map(); - this.eventHandlers = new Map(); - } - - registerExtension (name, extension) { - this.extensions.set(name, extension); - } - - on (event, handler) { - if ( ! this.eventHandlers.has(event) ) { - this.eventHandlers.set(event, []); - } - this.eventHandlers.get(event).push(handler); - } - - async emit (event, data) { - const handlers = this.eventHandlers.get(event) || []; - for ( const handler of handlers ) { - await handler(data); - } - } -} - -describe('Extension Integration with Captcha', () => { - let extensionService, captchaService, services; - - beforeEach(() => { - // Reset mocks - vi.clearAllMocks(); - - // Create fresh instances - extensionService = new ExtensionService(); - captchaService = { - enabled: true, - verifyCaptcha: vi.fn(), - }; - - services = { - get: vi.fn(), - }; - - // Configure service mocks - services.get.mockImplementation((serviceName) => { - if ( serviceName === 'extension' ) return extensionService; - if ( serviceName === 'captcha' ) return captchaService; - }); - - // Configure Context mock - Context.get.mockImplementation((key) => { - if ( key === 'services' ) return services; - }); - }); - - describe('Extension Event Handling', () => { - it('should allow extensions to require captcha via event handler', async () => { - // Setup - create a test extension that requires captcha - const testExtension = { - name: 'test-extension', - onCaptchaValidate: async (event) => { - if ( event.type === 'login' && event.ip === '1.2.3.4' ) { - event.require = true; - } - }, - }; - - // Register extension and event handler - extensionService.registerExtension(testExtension.name, testExtension); - extensionService.on('captcha.validate', testExtension.onCaptchaValidate); - - // Test event emission - const eventData = { - type: 'login', - ip: '1.2.3.4', - require: false, - }; - - await extensionService.emit('captcha.validate', eventData); - - // Assert - expect(eventData.require).toBe(true); - }); - - it('should allow extensions to disable captcha requirement', async () => { - // Setup - create a test extension that disables captcha - const testExtension = { - name: 'test-extension', - onCaptchaValidate: async (event) => { - if ( event.type === 'login' && event.ip === 'trusted-ip' ) { - event.require = false; - } - }, - }; - - // Register extension and event handler - extensionService.registerExtension(testExtension.name, testExtension); - extensionService.on('captcha.validate', testExtension.onCaptchaValidate); - - // Test event emission - const eventData = { - type: 'login', - ip: 'trusted-ip', - require: true, - }; - - await extensionService.emit('captcha.validate', eventData); - - // Assert - expect(eventData.require).toBe(false); - }); - - it('should handle multiple extensions modifying captcha requirement', async () => { - // Setup - create two test extensions with different rules - const extension1 = { - name: 'extension-1', - onCaptchaValidate: async (event) => { - if ( event.type === 'login' ) { - event.require = true; - } - }, - }; - - const extension2 = { - name: 'extension-2', - onCaptchaValidate: async (event) => { - if ( event.ip === 'trusted-ip' ) { - event.require = false; - } - }, - }; - - // Register extensions and event handlers - extensionService.registerExtension(extension1.name, extension1); - extensionService.registerExtension(extension2.name, extension2); - extensionService.on('captcha.validate', extension1.onCaptchaValidate); - extensionService.on('captcha.validate', extension2.onCaptchaValidate); - - // Test event emission - extension2 should override extension1 - const eventData = { - type: 'login', - ip: 'trusted-ip', - require: false, - }; - - await extensionService.emit('captcha.validate', eventData); - - // Assert - expect(eventData.require).toBe(false); - }); - - // TODO: Why was this behavior changed? - // it('should handle extension errors gracefully', async () => { - // // Setup - create a test extension that throws an error - // const testExtension = { - // name: 'test-extension', - // onCaptchaValidate: async () => { - // throw new Error('Extension error'); - // } - // }; - - // // Register extension and event handler - // extensionService.registerExtension(testExtension.name, testExtension); - // extensionService.on('captcha.validate', testExtension.onCaptchaValidate); - - // // Test event emission - // const eventData = { - // type: 'login', - // ip: '1.2.3.4', - // require: false - // }; - - // // The emit should not throw - // await extensionService.emit('captcha.validate', eventData); - - // // Assert - the original value should be preserved - // expect(eventData.require).toBe(false); - // }); - }); - - describe('Backward Compatibility', () => { - it('should maintain backward compatibility with older extension APIs', async () => { - // Setup - create a test extension using the old API format - const legacyExtension = { - name: 'legacy-extension', - handleCaptcha: async (event) => { - event.require = true; - }, - }; - - // Register legacy extension with old event name - extensionService.registerExtension(legacyExtension.name, legacyExtension); - extensionService.on('captcha.check', legacyExtension.handleCaptcha); - - // Test both old and new event names - const eventData = { - type: 'login', - ip: '1.2.3.4', - require: false, - }; - - // Should work with both old and new event names - await extensionService.emit('captcha.check', eventData); - await extensionService.emit('captcha.validate', eventData); - - // Assert - the requirement should be set by the legacy extension - expect(eventData.require).toBe(true); - }); - - it('should support legacy extension configuration formats', async () => { - // Setup - create a test extension with legacy configuration - const legacyExtension = { - name: 'legacy-extension', - config: { - captcha: { - always: true, - types: ['login', 'signup'], - }, - }, - onCaptchaValidate: async (event) => { - if ( legacyExtension.config.captcha.types.includes(event.type) ) { - event.require = legacyExtension.config.captcha.always; - } - }, - }; - - // Register extension and event handler - extensionService.registerExtension(legacyExtension.name, legacyExtension); - extensionService.on('captcha.validate', legacyExtension.onCaptchaValidate); - - // Test event emission - const eventData = { - type: 'login', - ip: '1.2.3.4', - require: false, - }; - - await extensionService.emit('captcha.validate', eventData); - - // Assert - expect(eventData.require).toBe(true); - }); - }); -}); \ No newline at end of file diff --git a/src/backend/tools/.test-webhook-config.json b/src/backend/tools/.test-webhook-config.json deleted file mode 100644 index b3adc8fd6..000000000 --- a/src/backend/tools/.test-webhook-config.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "key": "test-webhook-66999928605bc47b", - "webhook_secret": "9c193c4e111780a42b3d27661779adebba03c132078847490eb61639ce73288c", - "nonce": 13, - "instance_url": "http://api.puter.localhost:4100" -} \ No newline at end of file diff --git a/src/backend/tools/README.md b/src/backend/tools/README.md deleted file mode 100644 index cd45ab88e..000000000 --- a/src/backend/tools/README.md +++ /dev/null @@ -1,113 +0,0 @@ -# Backend Tools Directory - -## Manual Test for Broadcast Webhook Support - -`test-webhook.js` can be used for manual testing the `/broadcast/webhook` endpoint. -It prints a one-off peer config (peer id and `webhook_secret`) for you to add to your instance’s broadcast config, -then prompts for the instance base URL and sends an event with key `"test"`. - -**Usage** (from repo root): - -```bash -node src/backend/tools/test-webhook.js -``` - -Add the printed peer to your config under `broadcast.peers`, restart the instance, then run the script and enter the instance URL -(your Puter API URL, such as `http://api.puter.localhost:4100`) when prompted. - -## Test Kernel - -The **Test Kernel** is a drop-in replacement for Puter's main kernel. Instead of -actually initializing and running services, it only registers them and then invokes -a test iterator through all the services. - -The Test Kernel is ideal for running unit and integration tests against individual services, ensuring they behave correctly. - -### Usage - -``` -node src/backend/tools/test` -``` - -### Testing Services - -Implement the method `_test` on any service. When `_test` is called the "construct" -phase has already completed (meaning `_construct` on your service has been called -by now if you've implemented it), but the "init" phase will never happen (so _init -is never called). - -> **TODO:** I want to add support for mocking `_init` for deeper testing. - -For example, it should look similar to this snippet: - -```javascript -class ExampleService extends BaseService { - // ... - async _test ({ assert }) { - assert.equal('actual', 'expected', 'rule should have a description'); - } -} -``` - -Notice the parameter `assert` - this holds the TestKernel's testing API. The -reason this is a named parameter is to leave room for future support of -multiple testing APIs if this is ever desired or we decide to migrate -incrementally. - -The last parameter to `assert.equal` is a message describing the test rule. -The message should always be a short statement with minimal punctuation. - -#### TestKernel's testing API - -The `assert` value here is a function that also has other methods defined -in its properties. When called directly, `assert` will run the callback you -provide as an assertion. When no `name` parameter is specified, the callback -itself will be printed as the name of the assertion - this is useful for -very short expressions which are self-descriptive. - -```javascript -class ExampleService extends BaseService { - // ... - async _test ({ assert }) { - assert(() => 1 === 2, 'one should equal two'); - assert.equal(1, 2, 'one should equal two') - assert(() => 3 === 4); // prints out as: `() => 3 === 4` - } -} -``` - -| Method | Parameters | Types | Description | -|----------------|---------------------------------|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------| -| `assert` | `callback`, `name?` | `callback: () => boolean`, `name?: string` | Runs the callback as an assertion. If `name` is omitted, the callback's source text is used as the printed name of the assertion. | -| `assert.equal` | `actual`, `expected`, `message` | `actual: any`, `expected: any`, `message: string` | Asserts that `actual === expected`. The final parameter is a short descriptive message for the test rule. | - - - -### Test Kernel Notes - -1. **Logging**: - A custom `TestLogger` is provided for simplified logging output during tests. - Since LogService is never initialized, this is never replaced. - -2. **Context Management**: - The Test Kernel uses the same `Context` system as the main Kernel. This gives test environments a consistent way to access global state, configuration, and service containers. - -3. **Assertion & Results Tracking**: - The Test Kernel includes a simple testing structure that: - - Tracks passed and failed assertions. - - Repeats assertion outputs at the end of test runs for clarity. - - Allows specifying which services to test via command-line arguments. - -### Typical Workflow - -1. **Initialization**: - Instantiate the Test Kernel, and add any modules you want to test. - -2. **Module Installation**: - The Test Kernel installs these modules (via `_install_modules()`), making their services available in the `Container`. - -3. **Service Testing**: - After modules are installed, each service can be constructed and tested. Tests are implemented as `_test()` methods on services, using simple assertion helpers (`testapi.assert` and `testapi.assert.equal`). - -4. **Result Summarization**: - Once all tests run, the Test Kernel prints a summary of passed and failed assertions, aiding quick evaluation of test outcomes. diff --git a/src/backend/tools/test-webhook.js b/src/backend/tools/test-webhook.js deleted file mode 100644 index afd09af13..000000000 --- a/src/backend/tools/test-webhook.js +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const crypto = require('crypto'); -const fs = require('fs'); -const path = require('path'); -const readline = require('readline'); - -const CONFIG_PATH = path.join(__dirname, '.test-webhook-config.json'); - -function randomHex (bytes) { - return crypto.randomBytes(bytes).toString('hex'); -} - -function loadConfig () { - try { - const raw = fs.readFileSync(CONFIG_PATH, 'utf8'); - const data = JSON.parse(raw); - if ( data && typeof data.key === 'string' && typeof data.webhook_secret === 'string' ) { - const out = { - key: data.key, - webhook_secret: data.webhook_secret, - nonce: typeof data.nonce === 'number' ? data.nonce : 0, - }; - if ( typeof data.instance_url === 'string' && data.instance_url.trim() !== '' ) { - out.instance_url = data.instance_url.trim().replace(/\/+$/, ''); - } - return out; - } - } catch (e) { - const is_not_found = e.code === 'ENOENT'; - if ( ! is_not_found ) { - console.error('Saved config exists but could not be read:', e); - } - } - return null; -} - -/** - * Saves a dotfile beside the script so new configuration doesn't need to be - * re-entered into Puter every time this script is used. - * @param {*} peerId - The peer ID to save. - * @param {*} webhookSecret - The webhook secret to save. - * @param {*} nonce - The nonce to save. - * @param {*} instanceUrl - The instance URL to save. - */ -function saveConfig (peerId, webhookSecret, nonce, instanceUrl) { - const payload = { - key: peerId, - webhook_secret: webhookSecret, - nonce, - }; - if ( typeof instanceUrl === 'string' && instanceUrl.trim() !== '' ) { - payload.instance_url = instanceUrl.trim().replace(/\/+$/, ''); - } - fs.writeFileSync(CONFIG_PATH, JSON.stringify(payload, null, 2), 'utf8'); -} - -/** - * This wrapper around readline.question is used to promisify the interface - * and remove whitespace from the input. - * - * @param {*} rl - * @param {*} question - * @param {*} defaultAnswer - * @returns {Promise} - The trimmed answer. - */ -function ask (rl, question, defaultAnswer = '') { - const prompt = defaultAnswer ? `${question} [${defaultAnswer}]: ` : `${question} `; - return new Promise((resolve) => { - rl.question(prompt, (answer) => { - const trimmed = answer.trim(); - resolve(trimmed !== '' ? trimmed : defaultAnswer); - }); - }); -} - -async function main () { - const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); - - let peerId; - let webhookSecret; - let nonce; - const existing = loadConfig(); - - if ( existing ) { - const useExisting = await ask(rl, 'Existing key found. Use it? (y/n)', 'y'); - const noAnswers = ['n', 'no']; - if ( noAnswers.includes(useExisting.toLowerCase()) ) { - peerId = `test-webhook-${randomHex(8)}`; - webhookSecret = randomHex(32); - nonce = 0; - saveConfig(peerId, webhookSecret, nonce, existing.instance_url); - console.log(''); - console.log('New key generated.'); - console.log(''); - console.log('Add the following peer to your Puter instance config so it can accept'); - console.log('webhooks from this test script. In your config file (e.g. config.json),'); - console.log('under the "broadcast" section, add a "peers" array (if missing) and'); - console.log('include this entry:'); - console.log(''); - console.log(JSON.stringify({ - key: peerId, - webhook_secret: webhookSecret, - }, null, 2)); - console.log(''); - console.log('Example config structure:'); - console.log(' "broadcast": {'); - console.log(' "peers": ['); - console.log(' { "key": "", "webhook_secret": "" }'); - console.log(' ]'); - console.log(' }'); - console.log(''); - console.log('Restart your Puter instance after updating the config.'); - console.log(''); - } else { - peerId = existing.key; - webhookSecret = existing.webhook_secret; - nonce = existing.nonce; - console.log(''); - console.log('Using existing key:', peerId); - console.log(''); - } - } else { - peerId = `test-webhook-${randomHex(8)}`; - webhookSecret = randomHex(32); - nonce = 0; - saveConfig(peerId, webhookSecret, nonce, undefined); - console.log(''); - console.log('Add the following peer to your Puter instance config so it can accept'); - console.log('webhooks from this test script. In your config file (e.g. config.json),'); - console.log('under the "broadcast" section, add a "peers" array (if missing) and'); - console.log('include this entry:'); - console.log(''); - console.log(JSON.stringify({ - key: peerId, - webhook_secret: webhookSecret, - }, null, 2)); - console.log(''); - console.log('Example config structure:'); - console.log(' "broadcast": {'); - console.log(' "peers": ['); - console.log(' { "key": "", "webhook_secret": "" }'); - console.log(' ]'); - console.log(' }'); - console.log(''); - console.log('Restart your Puter instance after updating the config.'); - console.log(''); - } - - const defaultUrl = existing && existing.instance_url ? existing.instance_url : ''; - const baseUrl = await ask(rl, 'Instance base URL (e.g. http://api.puter.localhost:4100)', defaultUrl); - const url = baseUrl.trim().replace(/\/+$/, ''); - if ( ! url ) { - console.error('Please provide a URL.'); - rl.close(); - process.exit(1); - } - - const webhookUrl = `${url}/broadcast/webhook`; - const timestamp = Math.floor(Date.now() / 1000); - const body = { - key: 'test', - data: { contents: 'I am a test message from test-webhook.js' }, - meta: {}, - }; - const rawBody = JSON.stringify(body); - const payloadToSign = `${timestamp}.${nonce}.${rawBody}`; - const signature = crypto.createHmac('sha256', webhookSecret).update(payloadToSign).digest('hex'); - - try { - const res = await fetch(webhookUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Broadcast-Peer-Id': peerId, - 'X-Broadcast-Timestamp': String(timestamp), - 'X-Broadcast-Nonce': String(nonce), - 'X-Broadcast-Signature': signature, - }, - body: rawBody, - }); - - rl.close(); - - if ( res.ok ) { - saveConfig(peerId, webhookSecret, nonce + 1, url); - console.log(''); - console.log('Test event sent successfully. Status:', res.status); - const text = await res.text(); - if ( text ) console.log('Response:', text); - process.exit(0); - } else { - const text = await res.text(); - console.error(''); - console.error('Request failed. Status:', res.status, res.statusText); - if ( text ) console.error('Response:', text); - process.exit(1); - } - } catch ( err ) { - rl.close(); - console.error(''); - console.error('Request failed:', err.message); - process.exit(1); - } -} - -main(); diff --git a/src/backend/tools/test.mjs b/src/backend/tools/test.mjs deleted file mode 100644 index e4f0b1485..000000000 --- a/src/backend/tools/test.mjs +++ /dev/null @@ -1,364 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { AdvancedBase } from '@heyputer/putility'; -import useapi from 'useapi'; -import why from '../exports.js'; -import { RuntimeModuleRegistry } from '../src/extension/RuntimeModuleRegistry.js'; -import { Kernel } from '../src/Kernel.js'; -import { Core2Module } from '../src/modules/core/Core2Module.js'; -import { Container } from '../src/services/Container.js'; -import { consoleLogManager } from '../src/util/consolelog.js'; -import { Context } from '../src/util/context.js'; -import { TestCoreModule } from '../src/modules/test-core/TestCoreModule.js'; -import { config } from '../src/loadTestConfig.js'; -import { initializeS3Config } from '../src/clients/s3/s3ClientProvider.js'; -const { BaseService, EssentialModules } = why; - -/** - * A simple implementation of the log interface for the test kernel. - */ -class TestLogger { - constructor () { - console.log('\x1B[36;1mBoot logger started :)\x1B[0m'); - } - info (...args) { - console.log( - '\x1B[36;1m[TESTKERNEL/INFO]\x1B[0m', - ...args, - ); - } - error (...args) { - console.log( - '\x1B[31;1m[TESTKERNEL/ERROR]\x1B[0m', - ...args, - ); - } -} - -/** -* TestKernel class extends AdvancedBase to provide a testing environment for Puter services -* Implements a simplified version of the main Kernel for testing purposes, including: -* - Module management and installation -* - Service container initialization -* - Custom logging functionality -* - Context creation and management -* Does not include full service initialization or legacy service support -*/ -export class TestKernel extends AdvancedBase { - - /**@type {Context} */ - root_context; - constructor () { - super(); - - this.modules = []; - this.useapi = useapi(); - - /** - * Initializes the useapi instance for the test kernel. - * Defines base Module and Service classes in the useapi context. - * @returns {void} - */ - this.useapi.withuse(() => { - // eslint-disable-next-line no-undef - def('Module', AdvancedBase); - // eslint-disable-next-line no-undef - def('Service', BaseService); - }); - - this.logfn_ = (...a) => a; - - this.runtimeModuleRegistry = new RuntimeModuleRegistry(); - } - - add_module (module) { - this.modules.push(module); - } - - /** - * Adds a module to the test kernel's module list - * @param {Module} module - The module instance to add - * @description Stores the provided module in the kernel's internal modules array for later installation - */ - boot () { - consoleLogManager.initialize_proxy_methods(); - - consoleLogManager.decorate_all(({ _manager, replace }, ...a) => { - replace(...this.logfn_(...a)); - }); - - this.testLogger = new TestLogger(); - - const services = new Container({ logger: this.testLogger }); - this.services = services; - // app.set('services', services); - - const root_context = Context.create({ - services, - useapi: this.useapi, - 'runtime-modules': this.runtimeModuleRegistry, - args: {}, - }, 'app'); - this.root_context = root_context; - globalThis.root_context = root_context; - - root_context.arun(async () => { - await this._install_modules(); - // await this._boot_services(); - }); - - // Error.stackTraceLimit = Infinity; - Error.stackTraceLimit = 200; - } - - /** - * Installs modules into the test kernel environment - */ - async _install_modules () { - const { services } = this; - - const mod_install_root_context = Context.get(); - - for ( const module of this.modules ) { - try { - const mod_context = this._create_mod_context( - mod_install_root_context, - { - name: module.constructor.name, - 'module': module, - external: false, - }, - ); - await this.root_context.arun(async () => { - await module.install(mod_context); - }); - } catch (e) { - console.log(e); - throw e; - } - } - - // Real kernel initializes services here, but in this test kernel - // we don't initialize any services. - - // Real kernel adds legacy services here but these will break - // the test kernel. - - services.ready.resolve(); - - // provide services to helpers - // const { tmp_provide_services } = require('../src/helpers'); - // tmp_provide_services(services); - } -} - -TestKernel.prototype._create_mod_context = - Kernel.prototype._create_mod_context; - -const do_after_tests_ = []; - -/** -* Executes a function immediately and adds it to the list of functions to be executed after tests -* -* This is used to log things inline with console output from tests, and then -* again later without those console outputs. -* -* @param {Function} fn - The function to execute and store for later -*/ -const repeat_after = (fn) => { - fn(); - do_after_tests_.push(fn); -}; - -let total_passed = 0; -let total_failed = 0; - -/** -* Tracks test results across all services -* @type {number} total_passed - Count of all passed assertions -* @type {number} total_failed - Count of all failed assertions -*/ -const main = async () => { - const k = new TestKernel(); - for ( const mod of EssentialModules ) { - k.add_module(new mod()); - } - k.boot(); - console.log('awaiting services ready'); - await k.services.ready; - console.log('services have become ready'); - - const service_names = process.argv.length > 2 - ? process.argv.slice(2) - : Object.keys(k.services.instances_); - - for ( const name of service_names ) { - if ( ! k.services.instances_[name] ) { - console.log(`\x1B[31;1mService not found: ${name}\x1B[0m`); - process.exit(1); - } - - const ins = k.services.instances_[name]; - ins.construct(); - if ( !ins._test || typeof ins._test !== 'function' ) { - continue; - } - ins.log = k.testLogger; - let passed = 0; - let failed = 0; - - repeat_after(() => { - console.log(`\x1B[33;1m=== [ Service :: ${name} ] ===\x1B[0m`); - }); - - const testapi = { - assert: (condition, name) => { - name = name || condition.toString(); - if ( condition() ) { - passed++; - repeat_after(() => console.log(`\x1B[32;1m ✔ ${name}\x1B[0m`)); - } else { - failed++; - repeat_after(() => console.log(`\x1B[31;1m ✘ ${name}\x1B[0m`)); - } - }, - }; - - testapi.assert.equal = (a, b, name) => { - name = name || `${a} === ${b}`; - if ( a === b ) { - passed++; - repeat_after(() => console.log(`\x1B[32;1m ✔ ${name}\x1B[0m`)); - } else { - failed++; - repeat_after(() => { - console.log(`\x1B[31;1m ✘ ${name}\x1B[0m`); - console.log(`\x1B[31;1m Expected: ${b}\x1B[0m`); - console.log(`\x1B[31;1m Got: ${a}\x1B[0m`); - }); - } - }; - - await ins._test(testapi); - - total_passed += passed; - total_failed += failed; - } - - console.log('\x1B[36;1m<===\x1B[0m ' + - 'ASSERTION OUTPUTS ARE REPEATED BELOW' + - ' \x1B[36;1m===>\x1B[0m'); - - for ( const fn of do_after_tests_ ) { - fn(); - } - - console.log('\x1B[36;1m=== [ Summary ] ===\x1B[0m'); - console.log(`Passed: ${total_passed}`); - console.log(`Failed: ${total_failed}`); - - process.exit(total_failed ? 1 : 0); -}; - -if ( import.meta.main ) { - main(); -} - -export const createTestKernel = async ({ - serviceMap = {}, - initLevelString = 'construct', - extraSteps = true, - testCore = false, - serviceConfigOverrideMap = {}, - globalConfigOverrideMap = {}, - serviceMapArgs = {}, -}) => { - - const initLevelMap = { CONSTRUCT: 1, INIT: 2 }; - const initLevel = initLevelMap[(`${initLevelString}`).toUpperCase()]; - config.load_config({ - 'services': { - database: { - path: ':memory:', - }, - dynamo: { - path: ':memory:', - }, - }, - }); - - const testKernel = new TestKernel(); - testKernel.add_module(new Core2Module()); - if ( testCore ) testKernel.add_module(new TestCoreModule()); - for ( const [name, service] of Object.entries(serviceMap) ) { - testKernel.add_module({ - install: context => { - const services = context.get('services'); - services.registerService(name, service, serviceMapArgs[name] || undefined); - }, - }); - } - await initializeS3Config(true); - testKernel.boot(); - await testKernel.services.ready; - const service_names = Object.keys(testKernel.services.instances_); - - for ( const name of service_names ) { - - const serviceConfigOverride = serviceConfigOverrideMap[name]; - const globalConfigOverride = globalConfigOverrideMap[name]; - - if ( serviceConfigOverride ) { - const ins = testKernel.services.instances_[name]; - // Apply service config overrides - ins.config = { - ...ins.config, - ...serviceConfigOverride, - }; - } - - if ( globalConfigOverride ) { - const ins = testKernel.services.instances_[name]; - // Apply global config overrides - ins.global_config = { - ...ins.global_config, - ...globalConfigOverride, - }; - } - } - - for ( const name of service_names ) { - const ins = testKernel.services.instances_[name]; - // Fix context - ins.context = testKernel.root_context; - if ( initLevel >= initLevelMap.CONSTRUCT ) { - await ins.construct(); - } - } - for ( const name of service_names ) { - const ins = testKernel.services.instances_[name]; - if ( initLevel >= initLevelMap.INIT ) { - await ins.init(); - } - } - if ( extraSteps && testCore && initLevel >= initLevelMap.INIT ) { - await testKernel.services?.get('su').__on('boot.consolidation', []); - } - return testKernel; -}; diff --git a/src/backend/types.ts b/src/backend/types.ts new file mode 100644 index 000000000..dd5b4edd8 --- /dev/null +++ b/src/backend/types.ts @@ -0,0 +1,510 @@ +import type { PuterRouter } from './core/http/PuterRouter'; + +export interface IAWSCredentials { + access_key?: string; + secret_key?: string; + region?: string; +} + +export interface IDynamoConfig { + aws?: IAWSCredentials; + endpoint?: string; + path?: string; +} + +export interface IRedisConfig { + startupNodes?: Array<{ + host: string; + port: number; + }>; + useMock?: boolean; +} + +export interface IPagerConfig { + pagerduty?: { + enabled?: boolean; + routingKey?: string; + }; +} + +export interface ICfFileCacheConfig { + /** POST endpoint that accepts batched `{ site, path }[]` invalidation payloads. */ + endpoint: string; + /** Flush cadence in ms. Default 500. */ + throttle_ms?: number; +} + +export interface IClickhouseConfig { + url: string; + username?: string; + password?: string; + /** Milliseconds. Default 15000. */ + request_timeout?: number; + /** Max pending rows before backpressure drops oldest. Default 100000. */ + max_buffer_size?: number; + /** Rows per flush. Default 500. */ + batch_size?: number; + /** Flush cadence in ms. Default 5000. */ + flush_interval_ms?: number; +} + +export interface IEmailConfig { + /** "From" address used when callers don't override. */ + from?: string; + // nodemailer transport options (passed through as-is) + host?: string; + port?: number; + secure?: boolean; + auth?: { + user?: string; + pass?: string; + }; + service?: string; + [key: string]: unknown; +} + +/** + * S3-compatible bucket the thumbnails extension uses for storing generated + * thumbnails. When unset, the extension falls back to the main `S3Client` + * (fauxqs locally, real S3 in prod) and writes into the default bucket. + */ +export interface IThumbnailStoreConfig { + /** Bucket name. Default: `puter-local`. */ + name?: string; + /** Endpoint URL — unset forces the fallback. */ + endpoint?: string; + credentials?: { + accessKeyId: string; + secretAccessKey: string; + }; +} + +/** + * Shape of an entry under `config.providers.*` — each AI / integration driver + * reads a slightly different subset of these keys. Kept permissive so new + * providers don't have to touch the root type. + */ +export interface IAIProviderConfig { + /** API key. Sole canonical name — drivers no longer accept `secret_key`/`api_key`/`key` aliases. */ + apiKey?: string; + /** Cloudflare API token (semantically distinct from a regular key). Cloudflare-only. */ + apiToken?: string; + /** Override the provider's HTTP base URL (OpenRouter, Cloudflare, ElevenLabs, Ollama). */ + apiBaseUrl?: string; + /** Cloudflare account id. */ + accountId?: string; + /** ElevenLabs default voice id. */ + defaultVoiceId?: string; + /** ElevenLabs speech-to-speech model id. */ + speechToSpeechModelId?: string; + /** Ollama toggle — defaults true; set `false` to disable. */ + enabled?: boolean; + /** AWS credentials for AWS-backed providers (Polly, Textract). */ + aws?: IAWSCredentials; + /** Escape hatch — providers often expose additional tuning knobs. */ + [key: string]: unknown; +} + +/** + * OIDC provider sub-config (google, custom, …). `google` uses discovery, so + * only `client_id` + `client_secret` are required; custom providers must + * also supply the three endpoint URLs explicitly. + */ +export interface IOIDCProviderConfig { + client_id?: string; + client_secret?: string; + authorization_endpoint?: string; + token_endpoint?: string; + userinfo_endpoint?: string; + /** Space-separated OAuth scopes. Default depends on provider. */ + scopes?: string; + [key: string]: unknown; +} + +export interface IOIDCConfig { + providers?: Record; +} + +export interface IPeersConfig { + /** WebRTC signaller URL returned to clients. */ + signaller_url?: string; + /** Fallback ICE server list when TURN credential generation fails. */ + fallback_ice?: unknown[]; + /** TURN credential generation config (Cloudflare-backed). */ + turn?: { + cloudflare_turn_service_id?: string; + cloudflare_turn_api_token?: string; + /** Credential TTL in seconds. Default 86400. */ + ttl?: number; + }; + /** Shared secret for the internal `/turn/ingest-usage` endpoint. */ + internal_auth_secret?: string; +} + +export interface IBroadcastPeerConfig { + /** Stable id of the peer (also sent as `X-Broadcast-Peer-Id`). */ + peerId?: string; + /** Whether this peer should receive webhooks. Non-webhook peers are skipped. */ + webhook?: boolean; + /** HTTPS endpoint to POST broadcast events to. */ + webhook_url?: string; + /** HMAC-SHA256 secret shared with the peer for signing. */ + webhook_secret?: string; +} + +export interface IBroadcastConfig { + peers?: IBroadcastPeerConfig[]; + webhook?: { + /** This server's peerId, sent in outbound POSTs as `X-Broadcast-Peer-Id`. */ + peerId?: string; + /** Secret used to sign OUTBOUND POSTs. */ + secret?: string; + }; + /** Reject webhooks whose timestamp is more than this many seconds in the past. Default 300. */ + webhook_replay_window_seconds?: number; + /** Time to wait coalescing outbound events into a single peer POST. Default 2000ms. */ + outbound_flush_ms?: number; +} + +/** + * Cloudflare Workers deployment config used by `WorkerDriver`. + */ +export interface IWorkersConfig { + XAUTHKEY?: string; + ACCOUNTID?: string; + /** Optional dispatch namespace — when set, scripts deploy under `/dispatch/namespaces/`. */ + namespace?: string; + /** Base URL included as the `puter_endpoint` binding. Default `https://api.puter.com`. */ + internetExposedUrl?: string; + /** URL returned by `getLoggingUrl()` — surfaced to clients that render worker logs. */ + loggingUrl?: string; + [key: string]: string | undefined; +} + +/** + * Optional outbound-fetch proxy used by `secureFetch()` when the backend has + * to fetch a user-supplied URL (e.g. image-gen `input_image`). Requests get + * prefixed with `url` and sent through the Worker with `x-cors-proxy-auth- + * secret: `; the Worker authenticates the secret, fetches the real + * URL, and strips CORS on the response. Unset → fetches go direct (still + * guarded by the URL/redirect/DNS checks in secureFetch). + */ +export interface ISecureCorsProxyConfig { + url: string; + secret: string; +} + +export interface IWispConfig { + /** WISP relay server address returned to clients on token create. */ + server?: string; + [key: string]: unknown; +} + +export interface IServerHealthConfig { + /** DB liveness latency threshold (ms). Default 1500. */ + db_liveness_latency_fail_ms?: number; + /** Staleness threshold for the health-check loop itself (ms). */ + stale_health_loop_fail_ms?: number; +} + +export interface IS3LocalConfig { + inMemory?: boolean; + host?: string; + port?: number; + dataDir?: string; + s3StorageDir?: string; +} + +export interface IS3RemoteConfig { + useCredentialChain?: boolean; + endpoint: string; + accessKeyId: string; + secretAccessKey: string; + region?: string; +} + +export interface IS3Config { + localConfig?: IS3LocalConfig; + s3Config?: IS3RemoteConfig; +} + +export interface IDatabaseConfig { + engine: 'sqlite' | 'mysql'; + // sqlite + path?: string; + targetVersion?: number; + // mysql + host?: string; + port?: number; + user?: string; + password?: string; + database?: string; + replica?: { + host?: string; + port?: number; + user?: string; + password?: string; + database?: string; + }; +} + +/** + * Bucket of pass-through values surfaced to the client-side `gui()` boot + * function. Known fields are declared for lookup hygiene; unknown keys are + * still tolerated so product teams can add one-off flags without churn. + */ +export interface IGuiParams { + title?: string; + short_description?: string; + social_media_image?: string; + [key: string]: unknown; +} + +/** + * Complete shape of Puter's root config. Everything is optional here — + * mandatory fields (only `port` + `extensions`) are pulled out of the + * `Partial<...>` below and listed after it. + * + * When adding a new config field, declare it here with a doc comment so + * there's a single discoverable reference for every config-driven switch. + * + * One value, one location: each setting lives at exactly one key. There are + * no legacy aliases or fallback paths — older configs that relied on them + * need to migrate. + */ +interface IConfigOptional { + // ── Environment / identity ────────────────────────────────────── + + /** Environment marker. `dev` disables blocked-email checks, opens auto-browser, etc. */ + env: 'dev' | 'prod'; + /** Free-form name of the config profile (e.g. `oss-default`). Surfaced in logs. */ + config_name: string; + /** Server version. Falls back to `npm_package_version`. */ + version: string; + /** Stable identity for this server node. Enables pager alerts + graceful shutdown delay. */ + serverId: string; + + // ── Networking / URLs ─────────────────────────────────────────── + + /** Protocol used for the externally-visible origin ('http' or 'https'). Default: 'http'. */ + protocol: string; + /** Primary domain for Puter (e.g., `puter.localhost`, `puter.com`). */ + domain: string; + /** Externally-visible port. Defaults to `port`. Behind a reverse proxy, set this to the public port. */ + pub_port: number; + /** Fully-qualified externally-visible URL (protocol + domain + port). Computed from `protocol`/`domain`/`pub_port` if unset. */ + origin: string; + /** Public base URL for the API subdomain, e.g. `https://api.puter.com`. Used to build signed URLs. */ + api_base_url: string; + /** Static hosting domain for user sites (e.g., `puter.site`). */ + static_hosting_domain: string; + /** Alt static hosting domain. */ + static_hosting_domain_alt: string; + /** Private app hosting domain (e.g., `app.puter.localhost`). */ + private_app_hosting_domain: string; + /** Alt private app hosting domain. */ + private_app_hosting_domain_alt: string; + /** When true, accept any Host header value. Dev/testing only. */ + allow_all_host_values: boolean; + /** When true, accept requests without a Host header. */ + allow_no_host_header: boolean; + /** When true, allow nip.io wildcard domains. */ + allow_nipio_domains: boolean; + /** When true, support custom domain resolution for hosted sites. */ + custom_domains_enabled: boolean; + /** When true, enable IP validation via event bus. */ + enable_ip_validation: boolean; + /** + * Express `trust proxy` setting — controls how `req.ip` is derived from + * `X-Forwarded-For`. Set to the number of reverse-proxy hops in front of + * the server (e.g. `1` for a single Cloudflare or nginx hop, `2` for + * Cloudflare → ALB → app), or to a CIDR / IP / list of trusted proxy + * addresses. `false` (default) disables XFF parsing — `req.ip` returns + * the direct socket peer, which is the safe choice when no proxy is in + * front. Never set to `true` in production: it trusts *every* hop and + * makes XFF forgeable. See https://expressjs.com/en/guide/behind-proxies.html. + */ + trust_proxy: boolean | number | string | string[]; + /** Don't launch browser when starting. */ + no_browser_launch: boolean; + + // ── Auth / session ────────────────────────────────────────────── + + /** HMAC secret used to sign auth JWTs. */ + jwt_secret: string; + /** HMAC secret for signed file URLs (/file, /writeFile, /sign). */ + url_signature_secret: string; + /** Name of the session cookie the auth probe reads. */ + cookie_name: string; + /** Minimum password length for login/signup validation. */ + min_pass_length: number; + /** When true, allow the 'system' user to log in. */ + allow_system_login: boolean; + /** Reject auth-gated routes unless the user has confirmed their email. */ + strict_email_verification_required: boolean; + /** Captcha configuration. */ + captcha: { enabled: boolean; difficulty?: 'easy' | 'medium' | 'hard' }; + /** OIDC / OAuth2 providers (google + custom). */ + oidc: IOIDCConfig; + + // ── Groups / provisioning ─────────────────────────────────────── + + /** UID of the persistent group that non-temp users are enrolled in at signup. */ + default_user_group: string; + /** UID of the persistent group that temporary users are enrolled in at signup. */ + default_temp_group: string; + /** When true, ACL grants read/list/see on `//Public` to any actor. */ + enable_public_folders: boolean; + + // ── Storage / S3 ──────────────────────────────────────────────── + + /** S3 storage config (local fauxqs or remote). */ + s3: IS3Config; + /** Default S3 bucket for file storage. */ + s3_bucket: string; + /** Default S3 region. */ + s3_region: string; + /** Fallback AWS region. */ + region: string; + /** Default storage capacity per user (bytes). */ + storage_capacity: number; + /** When false, storage is effectively unlimited (bounded by device space). */ + is_storage_limited: boolean; + /** Bytes of device storage available (used when is_storage_limited=false). */ + available_device_storage: number; + /** Optional dedicated S3-compatible bucket used by the thumbnails extension. */ + thumbnailStore: IThumbnailStoreConfig; + + // ── Database ──────────────────────────────────────────────────── + + database: IDatabaseConfig; + + // ── Clients / infra ───────────────────────────────────────────── + + dynamo: IDynamoConfig; + redis: IRedisConfig; + pager: IPagerConfig; + email: IEmailConfig; + clickhouse: IClickhouseConfig; + cf_file_cache: ICfFileCacheConfig; + + // ── Rate limiting ─────────────────────────────────────────────── + + rate_limit: { + /** + * Rate limiter backend selection. + * - `memory`: per-node in-memory counters. + * - `redis`: sorted-sets in Redis — shared state across nodes (default). + * - `kv`: per-hit rows in the system KV store (DynamoDB), with TTL. + */ + backend?: 'memory' | 'redis' | 'kv'; + }; + + // ── AI / integration providers ────────────────────────────────── + // + // All AI providers — chat, image, video, TTS, OCR, speech-to-text, + // speech-to-speech — are configured under `providers[]`. + // Provider ids match the driver-side identifier (e.g. `claude`, + // `openai-image-generation`, `aws-textract`). There is no `services` + // bag and no top-level `openai`/`gemini`/`mistral`/`elevenlabs`/`aws` + // shortcut. + providers: Record; + + // ── Cross-node / external integrations ────────────────────────── + + /** Cross-node event replication config. */ + broadcast: IBroadcastConfig; + /** WebRTC signalling + TURN. */ + peers: IPeersConfig; + /** WISP relay proxy. */ + wisp: IWispConfig; + /** Cloudflare Workers driver config. */ + workers: IWorkersConfig; + /** Optional CORS-stripping signed-Worker proxy used by `secureFetch`. */ + secureCorsProxy: ISecureCorsProxyConfig; + /** Legacy Stripe billing extension. */ + + // ── GUI / static mounts ───────────────────────────────────────── + + /** Absolute path to the GUI assets root. */ + gui_assets_root: string; + /** Which profile in `puter-gui.json` to load. Default: `development`. */ + gui_profile: string; + /** + * Map of built-in app name → local directory served at `/builtin/`. + */ + builtin_apps: Record; + /** Force the bundled GUI even in dev. Default: false. */ + use_bundled_gui: boolean; + /** Override the GUI bundle JS path. Default: `/dist/bundle.min.js`. */ + gui_bundle: string; + /** Override the GUI CSS path when bundled. Default: `/dist/bundle.min.css`. */ + gui_css: string; + /** Override the puter.js preload URL when bundled. Default: `https://js.puter.com/v2/`. */ + gui_puterjs_bundle: string; + /** Free-form bag of values passed through to the client-side `gui()` function. */ + gui_params: IGuiParams; + /** + * Absolute path to the directory holding native app bundles, each in a + * subdirectory matching its subdomain (e.g. `/editor/`). + */ + native_apps_root: string; + /** Absolute path to a directory holding `puter.js`/`putility.js` version bundles. */ + client_libs_root: string; + /** Path to the puter-js SDK root (serves `/sdk/*` and `/puter.js/v{1,2}`). */ + puterjs_root: string; + + // ── Extension-specific ────────────────────────────────────────── + + /** + * Flat `{ flag_name: boolean }` bag of feature toggles. Non-boolean values + * are coerced before use. + * + * Server-only by default. Flags are surfaced to clients via `/whoami` only + * if their key is on the allowlist in `extensions/whoami.ts` + * (`CLIENT_VISIBLE_FEATURE_FLAGS`). New flags should be assumed internal — + * add them to the allowlist explicitly if (and only if) the client needs + * to read them. + */ + feature_flags: Record; + /** Blocked email TLDs / domains — checked in `prod` only. */ + blockedEmailDomains: string[]; + /** Contact-form recipient. Default `support@puter.com`. */ + support_email: string; + /** Worker / subdomain names that cannot be allocated by users. */ + reserved_words: string[]; + /** Max subdomains a single user may own. Default 10. */ + max_subdomains_per_user: number; + /** Health-check tuning. */ + server_health: IServerHealthConfig; +} + +export type IConfig = Partial & { + extensions: string[]; + port: number; +}; + +// eslint-disable-next-line @typescript-eslint/no-wrapper-object-types +export interface WithLifecycle extends Object { + onServerStart?: () => Promise | void; + onServerShutdown?: () => Promise | void; + onServerPrepareShutdown?: () => Promise | void; +} + +export interface WithCostsReporting extends WithLifecycle { + getReportedCosts?: () => Promise[]>; +} + +export interface WithControllerRegistration extends WithCostsReporting { + registerRoutes: (router: PuterRouter) => void; +} + +export type LayerInstances< + // eslint-disable-next-line @typescript-eslint/no-explicit-any + T extends Record any) | any>, +> = { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + [K in keyof T]: T[K] extends new (...args: any[]) => any + ? InstanceType + : T[K]; +}; diff --git a/src/backend/util/appIcon.ts b/src/backend/util/appIcon.ts new file mode 100644 index 000000000..6afe9c69d --- /dev/null +++ b/src/backend/util/appIcon.ts @@ -0,0 +1,201 @@ +// Always routes through the backend `/app-icon//` endpoint rather +// than the `puter-app-icons` subdomain directly. Some apps (especially those +// imported with a URL icon column that predates the sharp pipeline) only have +// the original PNG on the subdomain and no sized variants — a direct subdomain +// URL like `-256.png` 404s in that case. The backend endpoint self-heals: +// it can fall back to the original, decode data URLs inline, or serve the +// default placeholder. Mirrors v1's `getAppIconPath`. + +export const DEFAULT_APP_ICON_SIZE = 256; + +// Subdomain where AppIconService publishes generated icons. Mirrors the +// constant in AppIconService; duplicated here to avoid a dependency cycle +// between the util layer and the service layer. +const APP_ICONS_SUBDOMAIN = 'puter-app-icons'; + +// MIME types accepted on the write path for `data:` icon URLs. Anything +// outside this allowlist is rejected so a malicious caller can't stash, +// e.g., `data:text/html` or `data:application/javascript` in the icon +// column and get it echoed back by the server. +export const ICON_DATA_URL_MIME_ALLOWLIST = [ + 'image/png', + 'image/jpeg', + 'image/jpg', + 'image/gif', + 'image/webp', + 'image/svg+xml', +] as const; + +interface AppIconDeps { + apiBaseUrl?: string; +} + +interface TrustedIconHostConfig { + static_hosting_domain?: string; + static_hosting_domain_alt?: string; + api_base_url?: string; +} + +const RAW_BASE64_REGEX = /^[A-Za-z0-9+/]+={0,2}$/; +const APP_ICON_ENDPOINT_PATH_REGEX = /^\/app-icon\/[^/?#]+(?:\/\d+)?\/?$/; +// Direct subdomain file shape written by AppIconService: +// /app-.png (original) +// /app--.png (sized variant) +// Allowed on trusted hosts only — see isAppIconEndpointUrl. +const APP_ICON_SUBDOMAIN_PATH_REGEX = /^\/app-[A-Za-z0-9_-]+(?:-\d+)?\.png$/; + +/** + * v1-compatible raw-base64 detector. Legacy puter-js callers pass the + * base64 payload without a `data:` prefix; v1 accepted it and normalized + * to `data:image/png;base64,` before storage. We mirror that here + * so clients that worked on v1 keep working. + * + * Rejects anything shorter than 16 chars, not aligned to base64 length, + * or that doesn't round-trip through Buffer — catches random strings + * that happen to match the charset. + */ +export function isRawBase64ImageString(value: unknown): value is string { + if (typeof value !== 'string') return false; + const trimmed = value.trim(); + if (trimmed.length < 16) return false; + if (!RAW_BASE64_REGEX.test(trimmed)) return false; + if (trimmed.length % 4 !== 0) return false; + try { + const decoded = Buffer.from(trimmed, 'base64'); + if (decoded.length === 0) return false; + const stripped = trimmed.replace(/=+$/, ''); + const reencoded = decoded.toString('base64').replace(/=+$/, ''); + return stripped === reencoded; + } catch { + return false; + } +} + +/** Wrap raw base64 in a `data:image/png;base64,…` URL; pass other values through. */ +export function normalizeRawBase64ImageString(value: string): string { + const trimmed = value.trim(); + if (!isRawBase64ImageString(trimmed)) return value; + return `data:image/png;base64,${trimmed}`; +} + +/** + * Whether `value` is a reference we own — accepts two shapes: + * - `/app-icon/(/)?` : the AppController endpoint + * - `/app-(-)?.png` : the file written by + * AppIconService onto the + * `puter-app-icons` subdomain + * + * Relative paths must use the endpoint shape (the subdomain-file shape + * is only meaningful when paired with a trusted host). Absolute URLs + * accept either shape but only on a trusted host — without the host + * check an authenticated user could set `icon` to an attacker URL and + * turn `/app-icon/:uid` into a Puter-branded open redirector. + */ +export function isAppIconEndpointUrl( + value: string, + config: TrustedIconHostConfig, +): boolean { + const trimmed = value.trim(); + if (!trimmed) return false; + let parsed: URL; + try { + parsed = new URL(trimmed, 'http://localhost'); + } catch { + return false; + } + const isEndpointPath = APP_ICON_ENDPOINT_PATH_REGEX.test(parsed.pathname); + const isSubdomainPath = APP_ICON_SUBDOMAIN_PATH_REGEX.test(parsed.pathname); + if (!isEndpointPath && !isSubdomainPath) return false; + + // Relative paths (our placeholder base won't survive if the input + // was absolute). Detect absolute-vs-relative by scheme presence. + if (!/^[a-z][a-z0-9+\-.]*:/i.test(trimmed) && !trimmed.startsWith('//')) { + return isEndpointPath; + } + return isTrustedIconHost(trimmed, config); +} + +/** + * Whether `url` points at a host we control for app-icon hosting. + * + * Used to gate both the legacy redirect fallback in `/app-icon/:uid` and + * the write-path validator in AppDriver — without this check an + * authenticated user can set `icon` to an arbitrary attacker URL and + * turn the unauthenticated `/app-icon/:uid` route into a Puter-branded + * open redirector (cached publicly for 15 minutes). + * + * Accepts: + * - `puter-app-icons.` (and …_alt) + * - The configured `api_base_url` host (AppIconService rewrites icon + * columns to `${api_base_url}/app-icon/`, so it must be trusted + * or round-tripped writes would fail validation). + */ +export function isTrustedIconHost( + url: string, + config: TrustedIconHostConfig, +): boolean { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return false; + } + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + return false; + } + const hostname = parsed.hostname.toLowerCase(); + if (!hostname) return false; + + const trustedBases = [ + config.static_hosting_domain, + config.static_hosting_domain_alt, + ].filter((d): d is string => typeof d === 'string' && d.length > 0); + + for (const base of trustedBases) { + if (hostname === `${APP_ICONS_SUBDOMAIN}.${base.toLowerCase()}`) { + return true; + } + } + + if (config.api_base_url) { + try { + const apiBaseHost = new URL( + config.api_base_url, + ).hostname.toLowerCase(); + if (apiBaseHost && hostname === apiBaseHost) return true; + } catch { + // malformed config — treat as no match rather than throwing + } + } + + return false; +} + +export function getAppIconUrl( + app: Record, + deps: AppIconDeps, + size?: number, +): string | null { + const appUid = (app.uid ?? app.uuid) as string | undefined; + if (!appUid) return null; + + const normalizedUid = appUid.startsWith('app-') ? appUid : `app-${appUid}`; + const iconSize = Number.isFinite(Number(size)) + ? Number(size) + : DEFAULT_APP_ICON_SIZE; + + const normalizedApiBaseUrl = String(deps.apiBaseUrl ?? '').replace( + /\/+$/, + '', + ); + if (!normalizedApiBaseUrl) { + // No API base URL configured — fall back to the raw `icon` column so + // something still renders (even if it's the unsized original). + const appIcon = app.icon; + if (typeof appIcon === 'string' && /^https?:\/\//i.test(appIcon)) { + return appIcon; + } + return null; + } + return `${normalizedApiBaseUrl}/app-icon/${normalizedUid}/${iconSize}`; +} diff --git a/extensions/fsv2/src/utils/concurrency.ts b/src/backend/util/concurrency.ts similarity index 83% rename from extensions/fsv2/src/utils/concurrency.ts rename to src/backend/util/concurrency.ts index fa67445ad..47f276d54 100644 --- a/extensions/fsv2/src/utils/concurrency.ts +++ b/src/backend/util/concurrency.ts @@ -1,9 +1,9 @@ -export async function runWithConcurrencyLimit ( +export async function runWithConcurrencyLimit( values: TInput[], concurrency: number, worker: (value: TInput, index: number) => Promise, ): Promise { - if ( values.length === 0 ) { + if (values.length === 0) { return []; } @@ -12,14 +12,14 @@ export async function runWithConcurrencyLimit ( let nextIndex = 0; const runWorker = async () => { - while ( true ) { + while (true) { const index = nextIndex; - if ( index >= values.length ) { + if (index >= values.length) { return; } nextIndex++; const value = values[index]; - if ( value === undefined ) { + if (value === undefined) { throw new Error(`Missing value at index ${index}`); } results[index] = await worker(value, index); @@ -31,12 +31,12 @@ export async function runWithConcurrencyLimit ( return results; } -export async function runWithConcurrencyLimitSettled ( +export async function runWithConcurrencyLimitSettled( values: TInput[], concurrency: number, worker: (value: TInput, index: number) => Promise, ): Promise[]> { - if ( values.length === 0 ) { + if (values.length === 0) { return []; } @@ -45,14 +45,14 @@ export async function runWithConcurrencyLimitSettled ( let nextIndex = 0; const runWorker = async () => { - while ( true ) { + while (true) { const index = nextIndex; - if ( index >= values.length ) { + if (index >= values.length) { return; } nextIndex++; const value = values[index]; - if ( value === undefined ) { + if (value === undefined) { throw new Error(`Missing value at index ${index}`); } @@ -62,7 +62,7 @@ export async function runWithConcurrencyLimitSettled ( status: 'fulfilled', value: output, }; - } catch ( error ) { + } catch (error) { results[index] = { status: 'rejected', reason: error, diff --git a/src/backend/util/email.ts b/src/backend/util/email.ts new file mode 100644 index 000000000..438b69b83 --- /dev/null +++ b/src/backend/util/email.ts @@ -0,0 +1,91 @@ +/** + * Email normalization + block-list check. Stateless — used by + * AuthController (signup / change-email / save-account). + * + * cleanEmail('foo.bar+tag@gmail.com') === 'foobar@gmail.com' + * isBlockedEmail('temp@mailinator.com', ['mailinator.com']) === true + */ + +type RuleName = 'dots_dont_matter' | 'remove_subaddressing'; + +interface Parts { + local: string; + domain: string; +} + +const RULES: Record void> = { + dots_dont_matter: (p) => { + p.local = p.local.replace(/\./g, ''); + }, + remove_subaddressing: (p) => { + p.local = p.local.split('+')[0]; + }, +}; + +/** + * Providers whose addresses should be canonicalized before comparison. + * `rules` are added on top of the default `remove_subaddressing`; `rmrules` + * are subtracted (Yahoo permits `+` in local parts). + */ +const PROVIDERS: Record = + { + gmail: { rules: ['dots_dont_matter'] }, + icloud: { rules: ['dots_dont_matter'] }, + yahoo: { rmrules: ['remove_subaddressing'] }, + }; + +const DOMAIN_TO_PROVIDER: Record = { + 'gmail.com': 'gmail', + 'googlemail.com': 'gmail', + 'yahoo.com': 'yahoo', + 'yahoo.co.uk': 'yahoo', + 'yahoo.ca': 'yahoo', + 'yahoo.com.au': 'yahoo', + 'icloud.com': 'icloud', + 'me.com': 'icloud', + 'mac.com': 'icloud', +}; + +/** Aliases that resolve to the same inbox on the provider side. */ +const DOMAIN_NONDISTINCT: Record = { + 'googlemail.com': 'gmail.com', +}; + +/** + * Canonical form used for the `user.clean_email` column and for duplicate + * detection. Lowercases, collapses nondistinct domains, strips provider- + * insignificant characters. + */ +export function cleanEmail(email: string): string { + const lower = email.toLowerCase(); + const [localRaw, domainRaw] = lower.split('@'); + if (!domainRaw) return lower; + + const parts: Parts = { + local: localRaw, + domain: DOMAIN_NONDISTINCT[domainRaw] ?? domainRaw, + }; + + const applied = new Set(['remove_subaddressing']); + const provider = PROVIDERS[DOMAIN_TO_PROVIDER[parts.domain] ?? '']; + if (provider) { + for (const r of provider.rules ?? []) applied.add(r); + for (const r of provider.rmrules ?? []) applied.delete(r); + } + for (const rule of applied) RULES[rule](parts); + + return `${parts.local}@${parts.domain}`; +} + +/** + * Returns true when the (cleaned) email matches any of the blocked domain + * suffixes. Suffix-match so `mailinator.com` blocks `foo@bar.mailinator.com`. + */ +export function isBlockedEmail( + email: string, + blockedDomains: readonly string[] | undefined, +): boolean { + if (!blockedDomains || blockedDomains.length === 0) return false; + const clean = cleanEmail(email); + return blockedDomains.some((suffix) => clean.endsWith(suffix)); +} diff --git a/src/backend/util/fileSigning.ts b/src/backend/util/fileSigning.ts new file mode 100644 index 000000000..fe2e3fcce --- /dev/null +++ b/src/backend/util/fileSigning.ts @@ -0,0 +1,199 @@ +import { createHash } from 'node:crypto'; +import { HttpError } from '../core/http/HttpError.js'; +import type { FSEntry } from '../stores/fs/FSEntry.js'; + +/** + * HMAC-like file URL signing. The on-wire contract is fixed by existing + * clients. + * + * Signature scheme: `sha256(///)`. A `write` + * signature is treated as a superset (it also satisfies `read`). + */ + +export type SignAction = 'read' | 'write'; + +export interface SigningConfig { + secret: string; + apiBaseUrl: string; +} + +export interface SignedFile { + uid: string; + expires: number; + signature: string; + url: string; + read_url: string; + write_url: string; + metadata_url: string; + fsentry_type: string | null; + fsentry_is_dir: boolean; + fsentry_name: string; + fsentry_size: number | null; + fsentry_accessed: number | null; + fsentry_modified: number; + fsentry_created: number | null; +} + +function sha256(input: string): string { + return createHash('sha256').update(input).digest('hex'); +} + +function computeSignature( + uid: string, + action: SignAction, + secret: string, + expires: number, +): string { + return sha256(`${uid}/${action}/${secret}/${expires}`); +} + +/** + * Produce a signed-URL object. The default `expires` timestamp uses a + * ~317k-year TTL (effectively permanent) — existing clients depend on that + * default; callers that want shorter-lived signatures can pass their own + * `ttlSeconds`. + */ +export function signFile( + entry: FSEntry, + config: SigningConfig, + options: { ttlSeconds?: number } = {}, +): SignedFile { + const ttl = options.ttlSeconds ?? 9_999_999_999_999; + const expires = Math.ceil(Date.now() / 1000) + ttl; + const signature = computeSignature( + entry.uuid, + 'read', + config.secret, + expires, + ); + const writeSignature = computeSignature( + entry.uuid, + 'write', + config.secret, + expires, + ); + + const sigParams = `uid=${entry.uuid}&expires=${expires}&signature=${signature}`; + const writeParams = `uid=${entry.uuid}&expires=${expires}&signature=${writeSignature}`; + const base = config.apiBaseUrl.replace(/\/$/, ''); + + return { + uid: entry.uuid, + expires, + signature, + url: `${base}/file?${sigParams}`, + read_url: `${base}/file?${sigParams}`, + write_url: `${base}/writeFile?${writeParams}`, + metadata_url: `${base}/itemMetadata?${sigParams}`, + fsentry_type: mimeFromName(entry.name), + fsentry_is_dir: entry.isDir, + fsentry_name: entry.name, + fsentry_size: entry.size, + fsentry_accessed: entry.accessed, + fsentry_modified: entry.modified, + fsentry_created: entry.created, + }; +} + +/** + * Verify a request's URL signature for a given action. A valid `write` + * signature also authorises `read`. Throws HttpError(403) on mismatch, + * expired signatures, or missing params. + */ +export function verifySignature( + query: { uid?: string; expires?: string | number; signature?: string }, + action: SignAction, + config: SigningConfig, +): void { + const uid = typeof query.uid === 'string' ? query.uid : ''; + const signature = + typeof query.signature === 'string' ? query.signature : ''; + const expires = Number(query.expires); + if (!uid) + throw new HttpError(403, '`uid` is required for signature-based auth'); + if (!signature) + throw new HttpError( + 403, + '`signature` is required for signature-based auth', + ); + if (!Number.isFinite(expires)) + throw new HttpError( + 403, + '`expires` is required for signature-based auth', + ); + + if (expires < Date.now() / 1000) { + throw new HttpError(403, 'Authentication failed. Signature expired.'); + } + + // Write signature satisfies any action. + if (signature === computeSignature(uid, 'write', config.secret, expires)) + return; + if (signature === computeSignature(uid, action, config.secret, expires)) + return; + + throw new HttpError(403, 'Authentication failed'); +} + +/** + * Non-throwing variant that returns whether the signature is valid for the + * given action. Useful when callers want to attempt `write` auth and fall + * back to `read` without triggering error propagation. + */ +export function isSignatureValid( + query: { uid?: string; expires?: string | number; signature?: string }, + action: SignAction, + config: SigningConfig, +): boolean { + try { + verifySignature(query, action, config); + return true; + } catch { + return false; + } +} + +// Minimal MIME type inference from file extension. Uses a small inline map +// to avoid pulling in `mime-types`. Callers that need complete coverage +// should import `mime-types` directly. +const MIME_BY_EXT: Record = { + txt: 'text/plain', + html: 'text/html', + htm: 'text/html', + css: 'text/css', + js: 'application/javascript', + mjs: 'application/javascript', + json: 'application/json', + xml: 'application/xml', + pdf: 'application/pdf', + zip: 'application/zip', + gz: 'application/gzip', + tar: 'application/x-tar', + png: 'image/png', + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + gif: 'image/gif', + webp: 'image/webp', + svg: 'image/svg+xml', + ico: 'image/x-icon', + bmp: 'image/bmp', + tiff: 'image/tiff', + tif: 'image/tiff', + mp3: 'audio/mpeg', + wav: 'audio/wav', + ogg: 'audio/ogg', + m4a: 'audio/mp4', + mp4: 'video/mp4', + webm: 'video/webm', + mov: 'video/quicktime', + md: 'text/markdown', + markdown: 'text/markdown', + csv: 'text/csv', +}; + +export function mimeFromName(name: string): string | null { + const dot = name.lastIndexOf('.'); + if (dot <= 0) return null; + const ext = name.slice(dot + 1).toLowerCase(); + return MIME_BY_EXT[ext] ?? null; +} diff --git a/src/backend/util/identifier.js b/src/backend/util/identifier.js new file mode 100644 index 000000000..c6308645d --- /dev/null +++ b/src/backend/util/identifier.js @@ -0,0 +1,275 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +const adjectives = [ + 'amazing', + 'ambitious', + 'articulate', + 'cool', + 'bubbly', + 'mindful', + 'noble', + 'savvy', + 'serene', + 'sincere', + 'sleek', + 'sparkling', + 'spectacular', + 'splendid', + 'spotless', + 'stunning', + 'awesome', + 'beaming', + 'bold', + 'brilliant', + 'cheerful', + 'modest', + 'motivated', + 'friendly', + 'fun', + 'funny', + 'generous', + 'gifted', + 'graceful', + 'grateful', + 'passionate', + 'patient', + 'peaceful', + 'perceptive', + 'persistent', + 'helpful', + 'sensible', + 'loyal', + 'honest', + 'clever', + 'capable', + 'calm', + 'smart', + 'genius', + 'bright', + 'charming', + 'creative', + 'diligent', + 'elegant', + 'fancy', + 'colorful', + 'avid', + 'active', + 'gentle', + 'happy', + 'intelligent', + 'jolly', + 'kind', + 'lively', + 'merry', + 'nice', + 'optimistic', + 'polite', + 'quiet', + 'relaxed', + 'silly', + 'witty', + 'young', + 'strong', + 'brave', + 'agile', + 'bold', + 'confident', + 'daring', + 'fearless', + 'heroic', + 'mighty', + 'powerful', + 'valiant', + 'wise', + 'wonderful', + 'zealous', + 'warm', + 'swift', + 'neat', + 'tidy', + 'nifty', + 'lucky', + 'keen', + 'blue', + 'red', + 'aqua', + 'green', + 'orange', + 'pink', + 'purple', + 'cyan', + 'magenta', + 'lime', + 'teal', + 'lavender', + 'beige', + 'maroon', + 'navy', + 'olive', + 'silver', + 'gold', + 'ivory', +]; + +const nouns = [ + 'street', + 'roof', + 'floor', + 'tv', + 'idea', + 'morning', + 'game', + 'wheel', + 'bag', + 'clock', + 'pencil', + 'pen', + 'magnet', + 'chair', + 'table', + 'house', + 'room', + 'book', + 'car', + 'tree', + 'candle', + 'light', + 'planet', + 'flower', + 'bird', + 'fish', + 'sun', + 'moon', + 'star', + 'cloud', + 'rain', + 'snow', + 'wind', + 'mountain', + 'river', + 'lake', + 'sea', + 'ocean', + 'island', + 'bridge', + 'road', + 'train', + 'plane', + 'ship', + 'bicycle', + 'circle', + 'square', + 'garden', + 'harp', + 'grass', + 'forest', + 'rock', + 'cake', + 'pie', + 'cookie', + 'candy', + 'butterfly', + 'computer', + 'phone', + 'keyboard', + 'mouse', + 'cup', + 'plate', + 'glass', + 'door', + 'window', + 'key', + 'wallet', + 'pillow', + 'bed', + 'blanket', + 'soap', + 'towel', + 'lamp', + 'mirror', + 'camera', + 'hat', + 'shirt', + 'pants', + 'shoes', + 'watch', + 'ring', + 'necklace', + 'ball', + 'toy', + 'doll', + 'kite', + 'balloon', + 'guitar', + 'violin', + 'piano', + 'drum', + 'trumpet', + 'flute', + 'viola', + 'cello', + 'harp', + 'banjo', + 'tuba', +]; + +const randomItem = (arr, random) => + arr[Math.floor((random ?? Math.random)() * arr.length)]; + +/** + * A function that generates a unique identifier by combining a random adjective, a random noun, and a random number (between 0 and 9999). + * The result is returned as a string with components separated by the specified separator. + * It is useful when you need to create unique identifiers that are also human-friendly. + * + * @param {string} [separator='_'] - The character used to separate the adjective, noun, and number. Defaults to '_' if not provided. + * @returns {string} A unique, human-friendly identifier. + * + * @example + * + * let identifier = window.generate_identifier(); + * // identifier would be something like 'clever-idea-123' + * + */ +function generate_identifier(separator = '_', rng = Math.random) { + // return a random combination of first_adj + noun + number (between 0 and 9999) + // e.g. clever-idea-123 + return [ + randomItem(adjectives, rng), + randomItem(nouns, rng), + Math.floor(rng() * 10000), + ].join(separator); +} + +const HUMAN_READABLE_CASE_INSENSITIVE = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; + +function generate_random_code( + n, + { rng = Math.random, chars = HUMAN_READABLE_CASE_INSENSITIVE } = {}, +) { + let code = ''; + for (let i = 0; i < n; i++) { + code += randomItem(chars, rng); + } + return code; +} + +module.exports = { + generate_identifier, + generate_random_code, +}; diff --git a/src/backend/src/util/kvSingleton.js b/src/backend/util/kvSingleton.ts similarity index 55% rename from src/backend/src/util/kvSingleton.js rename to src/backend/util/kvSingleton.ts index ae9eb5b26..ce381696a 100644 --- a/src/backend/src/util/kvSingleton.js +++ b/src/backend/util/kvSingleton.ts @@ -1,3 +1,3 @@ import kvjs from '@heyputer/kv.js'; -export const kv = new kvjs(); \ No newline at end of file +export const kv = new kvjs(); diff --git a/src/backend/util/nativeImport.ts b/src/backend/util/nativeImport.ts new file mode 100644 index 000000000..a8041b452 --- /dev/null +++ b/src/backend/util/nativeImport.ts @@ -0,0 +1,4 @@ +export const nativeImport = new Function( + 'specifier', + 'return import(specifier)', +) as (specifier: string) => Promise; diff --git a/src/backend/util/privateLaunchAccess.ts b/src/backend/util/privateLaunchAccess.ts new file mode 100644 index 000000000..06a28d93d --- /dev/null +++ b/src/backend/util/privateLaunchAccess.ts @@ -0,0 +1,130 @@ +import type { EventClient } from '../clients/EventClient'; + +/** + * Emits `app.privateAccess.resolveLaunch` so the marketplace extension can + * decide whether a private app may launch for the current actor; returns the + * normalised decision with a fallback to `app-center` on denial. + * + * Public apps short-circuit to `hasAccess: true` without an emit. + */ + +const DEFAULT_FALLBACK_APP_NAME = 'app-center'; + +export interface PrivateLaunchDecision { + hasAccess: boolean; + fallbackAppName?: string; + fallbackArgs?: { path: string }; + reason?: string; + checkedBy?: string; +} + +interface AppLike { + uid?: string; + name?: string; + is_private?: boolean | number | null; +} + +function buildFallbackPath(appName: string | undefined): string { + if (typeof appName !== 'string' || !appName.trim()) return '/app'; + return `/app/${encodeURIComponent(appName.trim())}`; +} + +function buildDefaultDenied( + appName: string | undefined, + reason: string, +): PrivateLaunchDecision { + return { + hasAccess: false, + fallbackAppName: DEFAULT_FALLBACK_APP_NAME, + fallbackArgs: { path: buildFallbackPath(appName) }, + reason, + checkedBy: 'core/private-launch-access', + }; +} + +function normalize( + decision: PrivateLaunchDecision | undefined, + appName: string | undefined, +): PrivateLaunchDecision { + if (!decision || typeof decision !== 'object') { + return buildDefaultDenied(appName, 'invalid-private-access-result'); + } + if (decision.hasAccess) { + return { + hasAccess: true, + reason: + typeof decision.reason === 'string' + ? decision.reason + : undefined, + checkedBy: + typeof decision.checkedBy === 'string' + ? decision.checkedBy + : undefined, + }; + } + const fallbackAppName = + typeof decision.fallbackAppName === 'string' && + decision.fallbackAppName.trim() + ? decision.fallbackAppName.trim() + : DEFAULT_FALLBACK_APP_NAME; + const fallbackPath = decision.fallbackArgs?.path; + return { + hasAccess: false, + fallbackAppName, + fallbackArgs: + typeof fallbackPath === 'string' && fallbackPath.trim() + ? { path: fallbackPath.trim() } + : { path: buildFallbackPath(appName) }, + reason: + typeof decision.reason === 'string' ? decision.reason : undefined, + checkedBy: + typeof decision.checkedBy === 'string' + ? decision.checkedBy + : undefined, + }; +} + +export async function resolvePrivateLaunchAccess({ + app, + eventClient, + userUid, + source, + args, +}: { + app: AppLike | null | undefined; + eventClient: EventClient | undefined; + userUid: string | null; + source: string; + args: unknown; +}): Promise { + if (!app?.is_private) { + return { hasAccess: true, checkedBy: 'core/public-app' }; + } + if (!eventClient) { + return buildDefaultDenied( + app.name, + 'private-access-event-service-unavailable', + ); + } + + const payload = { + appUid: app.uid, + appName: app.name, + userUid, + source, + args, + result: buildDefaultDenied(app.name, 'private-access-required'), + }; + + try { + await eventClient.emitAndWait( + 'app.privateAccess.resolveLaunch', + payload, + {}, + ); + } catch { + return buildDefaultDenied(app.name, 'private-access-check-error'); + } + + return normalize(payload.result, app.name); +} diff --git a/src/backend/util/secureHttp.test.ts b/src/backend/util/secureHttp.test.ts new file mode 100644 index 000000000..bbaa9ea0c --- /dev/null +++ b/src/backend/util/secureHttp.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest'; +import { isPublicResolvedAddress, validateUrlNoIP } from './secureHttp.js'; + +describe('secureHttp URL validation', () => { + it('rejects raw IP and localhost URLs before fetching', () => { + expect(() => validateUrlNoIP('http://127.0.0.1/')).toThrow(); + expect(() => validateUrlNoIP('http://[::1]/')).toThrow(); + expect(() => validateUrlNoIP('http://localhost/')).toThrow(); + expect(() => + validateUrlNoIP('https://example.com/image.png'), + ).not.toThrow(); + }); +}); + +describe('secureHttp resolved address validation', () => { + it('allows public resolved addresses', () => { + for (const address of [ + '1.1.1.1', + '8.8.8.8', + '2001:4860:4860::8888', + '2606:4700:4700::1111', + ]) { + expect(isPublicResolvedAddress(address)).toBe(true); + } + }); + + it('rejects private, link-local, loopback, mapped, and reserved addresses', () => { + for (const address of [ + '0.0.0.0', + '10.0.0.1', + '100.64.0.1', + '127.0.0.1', + '169.254.169.254', + '172.16.0.1', + '192.168.0.1', + '198.18.0.1', + '224.0.0.1', + '255.255.255.255', + '::', + '::1', + '::ffff:8.8.8.8', + '0:0:0:0:0:ffff:8.8.8.8', + '::ffff:808:808', + 'fc00::1', + 'fe80::1', + 'ff02::1', + ]) { + expect(isPublicResolvedAddress(address)).toBe(false); + } + }); +}); diff --git a/src/backend/util/secureHttp.ts b/src/backend/util/secureHttp.ts new file mode 100644 index 000000000..65b2b99d5 --- /dev/null +++ b/src/backend/util/secureHttp.ts @@ -0,0 +1,219 @@ +import { Resolver } from 'node:dns'; +import net, { BlockList } from 'node:net'; +import { Agent as UndiciAgent } from 'undici'; +import type { LookupFunction } from 'node:net'; +import { HttpError } from '../core/http/HttpError.js'; +import { configContainer } from '../exports.js'; +import type { ISecureCorsProxyConfig } from '../types.js'; + +// Cloudflare's malware-blocking resolver. Used for all outbound fetches we +// make on behalf of user-provided URLs, so if a user points us at something +// on a CF block-list we resolve to the sinkhole rather than the real IP. +const SECURE_DNS_SERVER = '1.1.1.3'; +const BLOCKED_RESOLVED_IPS = new BlockList(); +const BLOCKED_IPV4_MAPPED_IPS = new BlockList(); + +for (const [address, prefix] of [ + ['0.0.0.0', 8], + ['10.0.0.0', 8], + ['100.64.0.0', 10], + ['127.0.0.0', 8], + ['169.254.0.0', 16], + ['172.16.0.0', 12], + ['192.0.0.0', 24], + ['192.0.2.0', 24], + ['192.88.99.0', 24], + ['192.168.0.0', 16], + ['198.18.0.0', 15], + ['198.51.100.0', 24], + ['203.0.113.0', 24], + ['224.0.0.0', 4], + ['240.0.0.0', 4], +] as const) { + BLOCKED_RESOLVED_IPS.addSubnet(address, prefix, 'ipv4'); +} + +for (const [address, prefix] of [ + ['::', 128], + ['::1', 128], + ['64:ff9b::', 96], + ['64:ff9b:1::', 48], + ['100::', 64], + ['2001::', 32], + ['2001:2::', 48], + ['2001:10::', 28], + ['2001:20::', 28], + ['2001:db8::', 32], + ['2002::', 16], + ['fc00::', 7], + ['fe80::', 10], + ['ff00::', 8], +] as const) { + BLOCKED_RESOLVED_IPS.addSubnet(address, prefix, 'ipv6'); +} +BLOCKED_IPV4_MAPPED_IPS.addSubnet('::ffff:0.0.0.0', 96, 'ipv6'); + +/** + * Reject URLs whose host is a raw IP or `localhost`. The goal is to make + * SSRF harder by stopping trivial `http://169.254.169.254/...` probes before + * DNS. Pair with the custom resolver below, which rejects private/reserved + * resolved addresses after DNS. + */ +export function validateUrlNoIP(url: string): void { + const { hostname } = new URL(url); + const bare = + hostname.startsWith('[') && hostname.endsWith(']') + ? hostname.slice(1, -1) + : hostname; + if (net.isIP(bare) !== 0) { + throw new HttpError(400, 'IP-addressed URLs are not allowed'); + } + if (bare === 'localhost') { + throw new HttpError(400, 'localhost URLs are not allowed'); + } +} + +export function isPublicResolvedAddress(address: string): boolean { + const family = net.isIP(address); + if (family === 0) return false; + if (family === 6 && BLOCKED_IPV4_MAPPED_IPS.check(address, 'ipv6')) { + return false; + } + return !BLOCKED_RESOLVED_IPS.check(address, family === 6 ? 'ipv6' : 'ipv4'); +} + +function selectPublicAddress(addresses: string[] | undefined): string | null { + return addresses?.find(isPublicResolvedAddress) ?? null; +} + +function blockedResolvedAddressError(hostname: string): HttpError { + return new HttpError( + 400, + `Resolved address for ${hostname} is not allowed`, + { + code: 'resolved_address_not_allowed', + }, + ); +} + +const secureLookup: LookupFunction = (hostname, options, cb) => { + // Normalise options (same shape as dns.lookup overloads). + const optsObj = + typeof options === 'number' ? { family: options } : (options ?? {}); + const family = optsObj.family ?? 0; + + const resolver = new Resolver(); + resolver.setServers([SECURE_DNS_SERVER]); + + const done4 = (err: Error | null, addrs?: string[]) => { + const publicAddress = selectPublicAddress(addrs); + if (!err && publicAddress) return cb(null, publicAddress, 4); + if (family === 4) { + if (addrs?.length) { + return cb(blockedResolvedAddressError(hostname), '', 4); + } + return cb(err ?? new Error('no IPv4 addresses'), '', 4); + } + resolver.resolve6(hostname, (e6, a6) => { + const publicIpv6Address = selectPublicAddress(a6); + if (!e6 && publicIpv6Address) { + return cb(null, publicIpv6Address, 6); + } + if (addrs?.length || a6?.length) { + return cb(blockedResolvedAddressError(hostname), '', 4); + } + return cb(e6 ?? err ?? new Error('no addresses'), '', 4); + }); + }; + + if (family === 6) { + resolver.resolve6(hostname, (e, a) => { + const publicAddress = selectPublicAddress(a); + if (!e && publicAddress) return cb(null, publicAddress, 6); + if (a?.length) { + return cb(blockedResolvedAddressError(hostname), '', 6); + } + return cb(e ?? new Error('no IPv6 addresses'), '', 6); + }); + return; + } + resolver.resolve4(hostname, done4); +}; + +// Shared dispatcher — built once so we're not re-creating the DNS resolver +// on every request. `keepAlive: false` matches v1's behaviour (short-lived +// connections; no risk of a stale DNS cache across requests). +const secureDispatcher = new UndiciAgent({ + connect: { lookup: secureLookup }, + keepAliveTimeout: 0, + keepAliveMaxTimeout: 0, +}); + +function proxyConfig(): ISecureCorsProxyConfig | undefined { + const cfg = configContainer.secureCorsProxy; + if (cfg?.url && cfg?.secret) return cfg; + return undefined; +} + +interface SecureFetchInit extends Omit { + /** Bypass the CORS proxy even if one is configured. Internal-only calls. */ + skipProxy?: boolean; +} + +/** + * Fetch `url` with SSRF guards: + * • Rejects raw-IP / localhost hosts via {@link validateUrlNoIP}. + * • Forces `redirect: 'manual'` and rejects any 3xx response so a permissive + * target can't bounce us onto an internal endpoint. + * • Resolves DNS through Cloudflare's 1.1.1.3 (malware-filtered) resolver + * and rejects private/reserved answers before connecting. + * • If `config.secureCorsProxy` is set AND the URL isn't a `data:` URI, + * rewrites the request through the configured signed Cloudflare Worker + * proxy (adds `x-cors-proxy-auth-secret`). + * + * Intended for any outbound fetch whose URL originates from user input. + * Internal-only endpoints (a provider's own API URL, etc.) should keep + * using plain `fetch`. + */ +export async function secureFetch( + url: string, + init: SecureFetchInit = {}, +): Promise { + // Data URIs bypass everything — nothing to resolve, nothing to proxy. + if (url.startsWith('data:')) { + return fetch(url, { ...init, redirect: 'manual' }); + } + + validateUrlNoIP(url); + + let finalUrl = url; + const headers = new Headers(init.headers ?? {}); + + if (!init.skipProxy) { + const proxy = proxyConfig(); + if (proxy) { + finalUrl = proxy.url + url; + headers.set('x-cors-proxy-auth-secret', proxy.secret); + } + } + + const { skipProxy: _skip, ...rest } = init; + const response = await fetch(finalUrl, { + ...rest, + headers, + redirect: 'manual', + // undici-specific; tsc's lib.dom.d.ts doesn't know about it but + // Node's fetch forwards it through. Cast-through Record to avoid + // the type error without opening an `any`. + ...({ dispatcher: secureDispatcher } as Record), + }); + + if (response.status >= 300 && response.status < 400) { + throw new HttpError( + 400, + `redirects are not allowed (target: ${response.headers.get('location') ?? 'unknown'})`, + ); + } + + return response; +} diff --git a/src/backend/util/span.ts b/src/backend/util/span.ts new file mode 100644 index 000000000..5402451f9 --- /dev/null +++ b/src/backend/util/span.ts @@ -0,0 +1,78 @@ +import { type Attributes, SpanStatusCode, trace } from '@opentelemetry/api'; + +const tracer = trace.getTracer('puter-backend'); + +type AttrsOrFactory = Attributes | (() => Attributes); + +/** + * Run `fn` inside an active span. Handles sync + async transparently, + * records exceptions, and always closes the span. + */ +export function withSpan( + name: string, + attrs: AttrsOrFactory, + fn: () => T, +): T { + return tracer.startActiveSpan(name, (span) => { + try { + const a = typeof attrs === 'function' ? attrs() : attrs; + if (a) span.setAttributes(a); + const result = fn(); + if (result instanceof Promise) { + return result + .then( + (v) => { + span.setStatus({ code: SpanStatusCode.OK }); + return v; + }, + (err: unknown) => { + recordError(span, err); + throw err; + }, + ) + .finally(() => span.end()) as T; + } + span.setStatus({ code: SpanStatusCode.OK }); + span.end(); + return result; + } catch (err) { + recordError(span, err); + span.end(); + throw err; + } + }); +} + +function recordError(span: ReturnType, err: unknown) { + const e = err instanceof Error ? err : new Error(String(err)); + span.recordException(e); + span.setStatus({ code: SpanStatusCode.ERROR, message: e.message }); +} + +/** + * Stage-3 method decorator: wrap the decorated method in a span. + * Usage: + * class Foo { + * @Span() // span name = "Foo.bar" + * async bar() { ... } + * + * @Span('custom.name') // span name = "custom.name" + * async baz() { ... } + * } + */ +export function Span(name?: string) { + return function ( + target: (this: This, ...args: Args) => Return, + ctx: ClassMethodDecoratorContext< + This, + (this: This, ...args: Args) => Return + >, + ) { + return function (this: This, ...args: Args): Return { + const spanName = + name ?? + `${(this as { constructor?: { name?: string } })?.constructor?.name ?? 'fn'}.${String(ctx.name)}`; + return withSpan(spanName, {}, () => target.apply(this, args)); + }; + }; +} diff --git a/src/backend/util/taskbarItems.ts b/src/backend/util/taskbarItems.ts new file mode 100644 index 000000000..1b820edd3 --- /dev/null +++ b/src/backend/util/taskbarItems.ts @@ -0,0 +1,105 @@ +import { getAppIconUrl } from './appIcon.js'; + +interface TaskbarEntry { + name?: string; + id?: number; + uid?: string; + type?: string; +} + +interface TaskbarOptions { + iconSize?: number; + noIcons?: boolean; +} + +interface TaskbarDeps { + apiBaseUrl?: string; + clients: { + db: { + write: (query: string, params?: unknown[]) => Promise; + }; + }; + stores: { + app: { + getByName: ( + name: string, + ) => Promise | null>; + getByUid: (uid: string) => Promise | null>; + getById: (id: number) => Promise | null>; + }; + user: { + invalidateById: (id: number) => Promise; + }; + }; +} + +const DEFAULT_TASKBAR_ITEMS: TaskbarEntry[] = [ + { name: 'app-center', type: 'app' }, + { name: 'dev-center', type: 'app' }, + { name: 'editor', type: 'app' }, + { name: 'code', type: 'app' }, + { name: 'camera', type: 'app' }, + { name: 'recorder', type: 'app' }, +]; + +export async function getTaskbarItems( + user: Record, + deps: TaskbarDeps, + options: TaskbarOptions = {}, +): Promise>> { + let raw: TaskbarEntry[]; + + if (!user.taskbar_items) { + raw = DEFAULT_TASKBAR_ITEMS; + await deps.clients.db.write( + 'UPDATE `user` SET `taskbar_items` = ? WHERE `id` = ?', + [JSON.stringify(raw), user.id], + ); + await deps.stores.user.invalidateById(user.id as number); + } else { + try { + raw = + typeof user.taskbar_items === 'string' + ? JSON.parse(user.taskbar_items as string) + : (user.taskbar_items as TaskbarEntry[]); + } catch { + raw = []; + } + } + + const items: Array> = []; + + for (const entry of raw) { + if (entry.type !== 'app') continue; + if (entry.name === 'explorer') continue; + + let app: Record | null = null; + if (entry.name) app = await deps.stores.app.getByName(entry.name); + else if (entry.uid) app = await deps.stores.app.getByUid(entry.uid); + else if (entry.id) app = await deps.stores.app.getById(entry.id); + if (!app) continue; + + const item: Record = { + uid: app.uid, + uuid: app.uid, + name: app.name, + title: app.title, + icon: app.icon ?? null, + godmode: Boolean(app.godmode), + maximize_on_start: Boolean(app.maximize_on_start), + index_url: app.index_url, + description: app.description, + }; + + if (options.noIcons) { + delete item.icon; + } else { + item.icon = + getAppIconUrl(app, deps, options.iconSize) ?? app.icon ?? null; + } + + items.push(item); + } + + return items; +} diff --git a/src/backend/util/userProvisioning.ts b/src/backend/util/userProvisioning.ts new file mode 100644 index 000000000..0977d9866 --- /dev/null +++ b/src/backend/util/userProvisioning.ts @@ -0,0 +1,139 @@ +import { v4 as uuidv4 } from 'uuid'; +import type { AbstractDatabaseClient } from '../clients/database/DatabaseClient'; +import type { GroupStore } from '../stores/group/GroupStore'; +import type { UserRow, UserStore } from '../stores/user/UserStore'; +import type { IConfig } from '../types'; + +const DEFAULT_FOLDERS = [ + 'Trash', + 'AppData', + 'Desktop', + 'Documents', + 'Pictures', + 'Videos', + 'Public', +] as const; +type FolderName = (typeof DEFAULT_FOLDERS)[number]; + +/** + * Creates a user's default FS tree: `/` (home) and the seven + * standard children (Trash, AppData, Desktop, Documents, Pictures, Videos, + * Public). Records each folder's `uuid` + `id` on the `user` row so callers + * can look them up without an extra SELECT. + * + * Safe to call once per user (e.g. after signup or during admin bootstrap). + * Callers should check `user.trash_uuid` / similar up-front if they need + * idempotency. + * + * Folder IDs are resolved by re-SELECTing by UUID rather than inferring them + * from a multi-row `INSERT`'s single `insertId` return — that approach is + * engine-dependent (MySQL returns the first inserted id; SQLite returns the + * last). One extra round-trip, zero ambiguity. + */ +export async function generateDefaultFsentries( + db: AbstractDatabaseClient, + userStore: UserStore, + user: UserRow, +): Promise { + // Idempotency guard: if trash_uuid is already set, the tree exists. + // Cheap check vs. a redundant INSERT + UPDATE on retries / re-runs. + if (user.trash_uuid) return; + + const home_uuid = uuidv4(); + const folderUuids: Record = { + Trash: uuidv4(), + AppData: uuidv4(), + Desktop: uuidv4(), + Documents: uuidv4(), + Pictures: uuidv4(), + Videos: uuidv4(), + Public: uuidv4(), + }; + const ts = Math.floor(Date.now() / 1000); + + // Rows: [uuid, parent_uid, name, path] + const rows: Array<[string, string | null, string, string]> = [ + [home_uuid, null, user.username, `/${user.username}`], + ...DEFAULT_FOLDERS.map((name): [string, string, string, string] => [ + folderUuids[name], + home_uuid, + name, + `/${user.username}/${name}`, + ]), + ]; + + // Each row: uuid, parent_uid, user_id, name, path, created, modified + // is_dir and immutable are hardcoded to 1. + const placeholders = rows + .map(() => '(?, ?, ?, ?, ?, 1, ?, ?, 1)') + .join(', '); + const params: unknown[] = []; + for (const [uuid, parent, name, path] of rows) { + params.push(uuid, parent, user.id, name, path, ts, ts); + } + + await db.write( + `INSERT INTO fsentries + (uuid, parent_uid, user_id, name, path, is_dir, created, modified, immutable) + VALUES ${placeholders}`, + params, + ); + + // Resolve auto-increment IDs by UUID so we can pin them on the user row. + const folderUuidList = Object.values(folderUuids); + const idPlaceholders = folderUuidList.map(() => '?').join(', '); + const idRows = (await db.pread( + `SELECT id, uuid FROM fsentries WHERE user_id = ? AND uuid IN (${idPlaceholders})`, + [user.id, ...folderUuidList], + )) as Array<{ id: number; uuid: string }>; + const idByUuid = new Map(idRows.map((r) => [String(r.uuid), Number(r.id)])); + + await userStore.update(user.id, { + trash_uuid: folderUuids.Trash, + appdata_uuid: folderUuids.AppData, + desktop_uuid: folderUuids.Desktop, + documents_uuid: folderUuids.Documents, + pictures_uuid: folderUuids.Pictures, + videos_uuid: folderUuids.Videos, + public_uuid: folderUuids.Public, + trash_id: idByUuid.get(folderUuids.Trash) ?? null, + appdata_id: idByUuid.get(folderUuids.AppData) ?? null, + desktop_id: idByUuid.get(folderUuids.Desktop) ?? null, + documents_id: idByUuid.get(folderUuids.Documents) ?? null, + pictures_id: idByUuid.get(folderUuids.Pictures) ?? null, + videos_id: idByUuid.get(folderUuids.Videos) ?? null, + public_id: idByUuid.get(folderUuids.Public) ?? null, + }); +} + +/** + * Moves a user from the default *temp* group to the default *user* group. + * Call after a user's `email_confirmed` flips to 1. + * + * Best-effort on both sides: missing temp membership is common (e.g. + * OIDC signups that come in already-verified), and a failing user-group + * add shouldn't fail the response — we just log it. + */ +export async function promoteToVerifiedGroup( + groupStore: GroupStore, + config: IConfig, + user: UserRow, +): Promise { + const tempGroup = config.default_temp_group; + const userGroup = config.default_user_group; + + if (tempGroup) { + try { + await groupStore.removeUsers(tempGroup, [user.username]); + } catch { + // Expected when the user was never in the temp group. + } + } + if (userGroup) { + try { + await groupStore.addUsers(userGroup, [user.username]); + } catch (e) { + console.warn('[verified-group] add to user group failed:', e); + } + } +} diff --git a/src/backend/util/validation.js b/src/backend/util/validation.js new file mode 100644 index 000000000..280a315f5 --- /dev/null +++ b/src/backend/util/validation.js @@ -0,0 +1,109 @@ +import { HttpError } from '../core/http/HttpError.js'; + +/** + * Small input validation utilities for driver methods. + * Throws HttpError(400, ...) on failure. Returns the value on success. + */ + +export function validateString( + value, + { key, maxLen, regex, required = true, allowEmpty = false } = {}, +) { + if (value === undefined || value === null) { + if (required) throw new HttpError(400, `Missing \`${key}\``); + return value; + } + if (typeof value !== 'string') { + throw new HttpError(400, `\`${key}\` must be a string`); + } + if (!allowEmpty && value.length === 0) { + throw new HttpError(400, `\`${key}\` must not be empty`); + } + if (maxLen && value.length > maxLen) { + throw new HttpError( + 400, + `\`${key}\` must be at most ${maxLen} characters`, + ); + } + if (regex && !regex.test(value)) { + throw new HttpError(400, `\`${key}\` has an invalid format`); + } + return value; +} + +export function validateUrl( + value, + { + key, + maxLen = 3000, + required = true, + // Default allowlist is http(s) only — anything else is an XSS/SSRF + // primitive when the value is later consumed as `iframe.src`, + // `window.location`, a server-side fetch, etc. `new URL()` alone + // happily parses `javascript:alert(1)`, `data:text/html,…`, + // `file:///etc/passwd`, and `vbscript:`; callers that need + // something exotic must opt in explicitly. + protocols = ['http:', 'https:'], + } = {}, +) { + if (value === undefined || value === null) { + if (required) throw new HttpError(400, `Missing \`${key}\``); + return value; + } + validateString(value, { key, maxLen, required }); + let parsed; + try { + parsed = new URL(value); + } catch { + throw new HttpError(400, `\`${key}\` must be a valid URL`); + } + if (!protocols.includes(parsed.protocol)) { + throw new HttpError( + 400, + `\`${key}\` must use one of the following protocols: ${protocols.join(', ')}`, + ); + } + return value; +} + +export function validateBool(value, { key, required = false } = {}) { + if (value === undefined || value === null) { + if (required) throw new HttpError(400, `Missing \`${key}\``); + return value; + } + return Boolean(value); +} + +export function validateJsonObject(value, { key, required = false } = {}) { + if (value === undefined || value === null) { + if (required) throw new HttpError(400, `Missing \`${key}\``); + return value; + } + if (typeof value === 'string') { + try { + value = JSON.parse(value); + } catch { + throw new HttpError(400, `\`${key}\` must be valid JSON`); + } + } + if (typeof value !== 'object' || Array.isArray(value)) { + throw new HttpError(400, `\`${key}\` must be an object`); + } + return value; +} + +export function validateArrayOfStrings(value, { key, required = false } = {}) { + if (value === undefined || value === null) { + if (required) throw new HttpError(400, `Missing \`${key}\``); + return value; + } + if (!Array.isArray(value)) { + throw new HttpError(400, `\`${key}\` must be an array`); + } + for (let i = 0; i < value.length; i++) { + if (typeof value[i] !== 'string') { + throw new HttpError(400, `\`${key}[${i}]\` must be a string`); + } + } + return value; +} diff --git a/src/backend/vitest.bench.config.js b/src/backend/vitest.bench.config.js index b2f2bb73e..e0b1a9c3f 100644 --- a/src/backend/vitest.bench.config.js +++ b/src/backend/vitest.bench.config.js @@ -1,5 +1,8 @@ -import { defineConfig } from 'vitest/config'; -export default defineConfig({ +'use strict'; +Object.defineProperty(exports, '__esModule', { value: true }); +// vitest.bench.config.ts - Vitest benchmark configuration for Puter backend +var config_1 = require('vitest/config'); +exports.default = (0, config_1.defineConfig)({ test: { benchmark: { include: ['src/**/*.bench.{js,ts}'], @@ -8,4 +11,3 @@ export default defineConfig({ root: __dirname, }, }); -//# sourceMappingURL=vitest.bench.config.js.map \ No newline at end of file diff --git a/src/backend/vitest.bench.config.ts b/src/backend/vitest.bench.config.ts index b1a44569c..603d050f3 100644 --- a/src/backend/vitest.bench.config.ts +++ b/src/backend/vitest.bench.config.ts @@ -10,4 +10,3 @@ export default defineConfig({ root: __dirname, }, }); - diff --git a/src/backend/vitest.config.ts b/src/backend/vitest.config.ts index a27d1c7cc..07594245f 100644 --- a/src/backend/vitest.config.ts +++ b/src/backend/vitest.config.ts @@ -30,7 +30,7 @@ export default defineConfig(({ mode }) => ({ ], }, env: loadEnv(mode, '', 'PUTER_'), - include: ['src/**/*.{test,spec}.{ts,js}'], + include: ['**/*.{test,spec}.{ts,js}'], root: __dirname, // Ensures paths are relative to backend/ }, })); diff --git a/src/gui/package.json b/src/gui/package.json index 3c1212b3c..381d54266 100644 --- a/src/gui/package.json +++ b/src/gui/package.json @@ -17,7 +17,7 @@ "clean-css": "^5.3.2", "dotenv": "^16.4.5", "eslint": "^9.1.1", - "express": "^4.18.2", + "express": "^5.0.0", "globals": "^15.0.0", "html-entities": "^2.3.3", "jsdom": "^29.0.0", diff --git a/src/gui/src/UI/Dashboard/TabHome.js b/src/gui/src/UI/Dashboard/TabHome.js index f9177bf84..0ec151006 100644 --- a/src/gui/src/UI/Dashboard/TabHome.js +++ b/src/gui/src/UI/Dashboard/TabHome.js @@ -273,11 +273,16 @@ const TabHome = { if ( hasSubscription ) { $el_window.find('.bento-plan-badge').text('Active subscription').addClass('active'); - $el_window.find('.bento-plan-upgrade').hide(); + $el_window.find('.bento-plan-upgrade').text('Manage →').show(); } else { $el_window.find('.bento-plan-badge').text('Upgrade for more features').addClass('free'); $el_window.find('.bento-plan-upgrade').show(); } + + $el_window.find('.bento-plan-upgrade').off('click.billing').on('click.billing', (e) => { + e.preventDefault(); + window.puterLegacyBilling?.openSubscriptionsDialog?.(); + }); } catch (e) { console.error('Failed to load plan data:', e); } diff --git a/src/gui/src/UI/UIDesktop.js b/src/gui/src/UI/UIDesktop.js index ad6b5f395..1f3e8605e 100644 --- a/src/gui/src/UI/UIDesktop.js +++ b/src/gui/src/UI/UIDesktop.js @@ -1253,13 +1253,18 @@ async function UIDesktop (options) { globalThis.services.emit('gui:ready'); //-------------------------------------------------------- - // Open the AI app + // Open the AI app (best-effort — the `ai` app isn't seeded + // in OSS, so swallow 404s / token failures silently here + // instead of surfacing a "Couldn't open" alert at boot). //-------------------------------------------------------- launch_app({ name: 'ai', + silent_on_failure: true, window_options: { is_panel: true, }, + }).catch(err => { + console.debug('auto-launch of ai panel skipped:', err?.message ?? err); }); //-------------------------------------------------------------------------------------- diff --git a/src/gui/src/UI/UIWindowNewPassword.js b/src/gui/src/UI/UIWindowNewPassword.js index 729c32a97..3e6e0fe38 100644 --- a/src/gui/src/UI/UIWindowNewPassword.js +++ b/src/gui/src/UI/UIWindowNewPassword.js @@ -191,7 +191,13 @@ async function UIWindowNewPassword (options) { }); }, error: function (err) { - $(el_window).find('.form-error-msg').html(html_encode(err.responseText)); + const errorText = err.responseText || ''; + let msg = errorText; + try { + const errorJson = JSON.parse(errorText); + msg = errorJson.message || errorJson.error || errorText; + } catch (_) { /* not JSON, use responseText */ } + $(el_window).find('.form-error-msg').html(html_encode(msg)); $(el_window).find('.form-error-msg').fadeIn(); }, }); diff --git a/src/gui/src/UI/UIWindowRecoverPassword.js b/src/gui/src/UI/UIWindowRecoverPassword.js index fe1a86bbf..7e3cd83c6 100644 --- a/src/gui/src/UI/UIWindowRecoverPassword.js +++ b/src/gui/src/UI/UIWindowRecoverPassword.js @@ -119,7 +119,13 @@ function UIWindowRecoverPassword (options) { }); }, error: function (err) { - $(el_window).find('.error').html(html_encode(err.responseText)); + const errorText = err.responseText || ''; + let msg = errorText; + try { + const errorJson = JSON.parse(errorText); + msg = errorJson.message || errorJson.error || errorText; + } catch (_) { /* not JSON, use responseText */ } + $(el_window).find('.error').html(html_encode(msg)); $(el_window).find('.error').fadeIn(); }, complete: function () { diff --git a/src/gui/src/UI/UIWindowSaveAccount.js b/src/gui/src/UI/UIWindowSaveAccount.js index 16ce05cf2..0abdc2b59 100644 --- a/src/gui/src/UI/UIWindowSaveAccount.js +++ b/src/gui/src/UI/UIWindowSaveAccount.js @@ -178,7 +178,13 @@ async function UIWindowSaveAccount (options) { $(el_window).find('input').prop('disabled', false); }, error: function (err) { - $(el_window).find('.signup-error-msg').html(html_encode(err.responseText)); + const errorText = err.responseText || ''; + let msg = errorText; + try { + const errorJson = JSON.parse(errorText); + msg = errorJson.message || errorJson.error || errorText; + } catch (_) { /* not JSON, use responseText */ } + $(el_window).find('.signup-error-msg').html(html_encode(msg)); $(el_window).find('.signup-error-msg').fadeIn(); // re-enable 'Create Account' button $(el_window).find('.signup-btn').prop('disabled', false); diff --git a/src/gui/src/helpers.js b/src/gui/src/helpers.js index 3e30602be..be9983ccc 100644 --- a/src/gui/src/helpers.js +++ b/src/gui/src/helpers.js @@ -77,9 +77,11 @@ window.suggest_apps_for_fsentry = async (options) => { * @returns */ window.byte_format = (bytes) => { - const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']; + const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB']; if ( bytes === 0 ) return '0 Byte'; - const i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024))); + let i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024))); + if ( i < 0 ) i = 0; + if ( i >= sizes.length ) i = sizes.length - 1; return `${(bytes / Math.pow(1024, i)).toFixed(2) } ${ sizes[i]}`; }; diff --git a/src/gui/src/helpers/launch_app.js b/src/gui/src/helpers/launch_app.js index 3bd1a909b..23d7ec793 100644 --- a/src/gui/src/helpers/launch_app.js +++ b/src/gui/src/helpers/launch_app.js @@ -195,14 +195,16 @@ const launch_app = async (options) => { return fallbackLaunchOutcome ?? { launchResult: redirectedLaunchResult }; } - const deniedAppTitle = app_info.title ?? app_info.name ?? options.name ?? 'this app'; - const safeDeniedAppTitle = window.html_encode - ? window.html_encode(deniedAppTitle) - : deniedAppTitle; - if ( typeof window.UIAlert === 'function' ) { - await window.UIAlert(`You don't have access to ${safeDeniedAppTitle}.`); - } else { - window.alert(`You don't have access to ${deniedAppTitle}.`); + if ( ! options?.silent_on_failure ) { + const deniedAppTitle = app_info.title ?? app_info.name ?? options.name ?? 'this app'; + const safeDeniedAppTitle = window.html_encode + ? window.html_encode(deniedAppTitle) + : deniedAppTitle; + if ( typeof window.UIAlert === 'function' ) { + await window.UIAlert(`You don't have access to ${safeDeniedAppTitle}.`); + } else { + window.alert(`You don't have access to ${deniedAppTitle}.`); + } } const deniedLaunchResult = { @@ -453,14 +455,18 @@ const launch_app = async (options) => { tokenResult, }); - const tokenErrorAppTitle = app_info?.title ?? app_info?.name ?? options?.name ?? 'this app'; - const safeTokenErrorAppTitle = window.html_encode - ? window.html_encode(tokenErrorAppTitle) - : tokenErrorAppTitle; - if ( typeof window.UIAlert === 'function' ) { - await window.UIAlert(`Couldn't open ${safeTokenErrorAppTitle}. Please try again.`); - } else { - window.alert(`Couldn't open ${tokenErrorAppTitle}. Please try again.`); + // `silent_on_failure` callers (e.g. best-effort auto-launches + // like the AI panel on desktop boot) skip the blocking alert. + if ( ! options?.silent_on_failure ) { + const tokenErrorAppTitle = app_info?.title ?? app_info?.name ?? options?.name ?? 'this app'; + const safeTokenErrorAppTitle = window.html_encode + ? window.html_encode(tokenErrorAppTitle) + : tokenErrorAppTitle; + if ( typeof window.UIAlert === 'function' ) { + await window.UIAlert(`Couldn't open ${safeTokenErrorAppTitle}. Please try again.`); + } else { + window.alert(`Couldn't open ${tokenErrorAppTitle}. Please try again.`); + } } const tokenFailureLaunchResult = { diff --git a/src/puter-js/src/modules/FileSystem/index.js b/src/puter-js/src/modules/FileSystem/index.js index 219db3be8..d58e81c9b 100644 --- a/src/puter-js/src/modules/FileSystem/index.js +++ b/src/puter-js/src/modules/FileSystem/index.js @@ -23,7 +23,6 @@ import revokeReadURL from './operations/revokeReadUrl.js'; import sign from './operations/sign.js'; import space from './operations/space.js'; import stat from './operations/stat.js'; -import symlink from './operations/symlink.js'; import upload from './operations/upload.js'; import write from './operations/write.js'; @@ -41,7 +40,6 @@ export class PuterJSFileSystemModule { move = move; write = write; sign = sign; - symlink = symlink; getReadURL = getReadURL; revokeReadURL = revokeReadURL; readdir = readdir; diff --git a/src/puter-js/src/modules/FileSystem/operations/symlink.js b/src/puter-js/src/modules/FileSystem/operations/symlink.js deleted file mode 100644 index d4578a0a0..000000000 --- a/src/puter-js/src/modules/FileSystem/operations/symlink.js +++ /dev/null @@ -1,53 +0,0 @@ -import getAbsolutePathForApp from '../utils/getAbsolutePathForApp.js'; -import pathLib from '../../../lib/path.js'; - -// This only works for absolute symlinks for now -const symlink = async function (target, linkPath) { - - // If auth token is not provided and we are in the web environment, - // try to authenticate with Puter - if ( !puter.authToken && puter.env === 'web' ) { - try { - await puter.ui.authenticateWithPuter(); - } catch (e) { - // if authentication fails, throw an error - throw 'Authentication failed.'; - } - } - - // convert path to absolute path - linkPath = getAbsolutePathForApp(linkPath); - target = getAbsolutePathForApp(target); - const name = pathLib.basename(linkPath); - const linkDir = pathLib.dirname(linkPath); - - const op = - { - op: 'symlink', - path: linkDir, - name: name, - target: target, - }; - - const formData = new FormData(); - formData.append('operation', JSON.stringify(op)); - - try { - const response = await fetch(`${this.APIOrigin }/batch`, { - method: 'POST', - headers: { 'Authorization': `Bearer ${puter.authToken}` }, - body: formData, - }); - if ( response.status !== 200 ) { - const error = await response.text(); - console.error('[symlink] fetch error: ', error); - throw error; - } - } catch (e) { - console.error('[symlink] fetch error: ', e); - throw e; - } - -}; - -export default symlink; \ No newline at end of file diff --git a/src/puter-js/src/modules/networking/PSocket.js b/src/puter-js/src/modules/networking/PSocket.js index fe03461d0..636a3ebf7 100644 --- a/src/puter-js/src/modules/networking/PSocket.js +++ b/src/puter-js/src/modules/networking/PSocket.js @@ -2,7 +2,7 @@ import EventListener from '../../lib/EventListener.js'; import { errors } from './parsers.js'; import { PWispHandler } from './PWispHandler.js'; const texten = new TextEncoder(); -const requireAuth = false; // for initial launch +const requireAuth = true; export let wispInfo = { server: 'wss://puter.cafe/', // Unused currently diff --git a/src/puter-js/test/ai.test.js b/src/puter-js/test/ai.test.js index eab1d306e..d491b76e7 100644 --- a/src/puter-js/test/ai.test.js +++ b/src/puter-js/test/ai.test.js @@ -25,7 +25,6 @@ const testChatBasicPromptCore = async function(model) { // Check response structure assert(typeof result.message === 'object', "result should have message object"); assert(typeof result.finish_reason === 'string', "result should have finish_reason string"); - assert(typeof result.via_ai_chat_service === 'boolean', "result should have via_ai_chat_service boolean"); // Check message structure assert(typeof result.message.role === 'string', "message should have role string"); @@ -69,8 +68,7 @@ const testChatWithParametersCore = async function(model) { assert(validFinishReasons.includes(result.finish_reason), `finish_reason should be one of: ${validFinishReasons.join(', ')}`); - // Check that via_ai_chat_service is true - assert(result.via_ai_chat_service === true, "via_ai_chat_service should be true"); + }; const testChatWithMessageArrayCore = async function(model) { diff --git a/src/puter-js/types/modules/filesystem.d.ts b/src/puter-js/types/modules/filesystem.d.ts index 202eaf0ba..5114a1002 100644 --- a/src/puter-js/types/modules/filesystem.d.ts +++ b/src/puter-js/types/modules/filesystem.d.ts @@ -154,7 +154,5 @@ export class FS { sign (appUid: string, items: unknown | unknown[], success?: (result: SignResult) => void, error?: (reason: unknown) => void): Promise; - symlink (target: string, linkPath: string): Promise; - getReadURL (path: string, expiresIn?: string): Promise; } diff --git a/src/putility/README.md b/src/putility/README.md deleted file mode 100644 index 08e10ce5f..000000000 --- a/src/putility/README.md +++ /dev/null @@ -1,125 +0,0 @@ -# Puter - Common Javascript Module - -This is a small module for javascript which you might call a -"language tool"; it adds some behavior to make javascript classes -more flexible, with an aim to avoid any significant complexity. - -Each class in this module is best described as an _idea_: - -## Libraries - -Putility contains general purpose library functions. - -### `putility.libs.context` - -This library exports class **Context**. This provides a context object -that works both in node and the browser. - -> **Note:** A lot of Puter's backend code uses a _different_ implementation -> for Context that uses AsyncLocalStorage (only available in node) - -When creating a context you pass it an object with values that the context -will hold: - -```javascript -const ctx = new Context({ - some_key: 'some value', -}); - -ctx.some_key; // works just like a regular object -``` - -You can create sub-contexts using Context**.sub()**: - -```javascript -const a = new Context({ - some_key: 'some value' -}); -const b = a.sub({ - another_key: 'another value' -}); - -b.another_key; // "another value" -b.some_key; // "some value" - -a.some_key = 'changed'; -b.some_key; // "changed" -``` - -### `putility.libs.string` - -#### `quote(text)` - -Wraps a string in backticks, escaping any present backticks as needed to -disambiguate. Note that this is meant for human-readable text, so the exact -solution to disambiguating backticks is allowed to change in the future. - -### `putility.libs.promise` - -Utilities for working with promises. - -#### **TeePromise** - -Possibily the most useful utility, TeePromise is a Promise that implements -externally-available `resolve()` and `reject()` methods. This is useful -when using async/await syntax as it avoids unnecessary callback handling. - -```javascript -const tp = new TeePromise(); - -new bb = Busboy({ /* ... */ }); - -// imagine you have lots of code here, that you don't want to -// indent in a `new Promise((resolve, reject) => { ...` block - -bb.on('error', err => { - tp.reject(err); -}); -bb.on('close', () => { - tp.resolve(); -}) - -return { - // Imagine you have other values here that don't require waiting - // for the promise to resolve; handling this when a large portion - // of the code is wrapped in a Promise constructor is error-prone. - promise: tp, -}; -``` - -## Basees - -Putility implements a chain of base classes for general purpose use. -Simply extend the **AdvancedBase** class to add functionality to your -class such as traits and inheritance-merged static objects. - -If a class must extend some class outside of putility, then putility is -not meant to support it. This is instead considered "utility code" - i.e. -not part of the application structure that adheres to the design -principles of putility. - -### BasicBase - -**BasicBase** is the idea that there should be a common way to -see the inheritance chain of the current instance, and obtain -merged objects and arrays from static members of these classes. - -### TraitBase - -**TraitBase** is the idea that there should be a common way to -"install" behavior into objects of a particular class, as -dictated by the class definition. A trait might install a common -set of methods ("mixins"), decorate all or a specified set of -methods in the class (performance monitors, sanitization, etc), -or anything else. - -### AdvancedBase - -**AdvancedBase** is the idea that, in a node.js environment, -you always want the ability to add traits to a class and there -are some default traits you want in all classes, which are: - -- `PropertiesTrait` - add lazy factories for instance members - instead of always populating them in the constructor. -- `NodeModuleDITrait` - require node modules in a way that - allows unit tests to inject mocks easily. diff --git a/src/putility/index.js b/src/putility/index.js deleted file mode 100644 index 6bedacc5d..000000000 --- a/src/putility/index.js +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -const { AdvancedBase } = require('./src/AdvancedBase'); -const { Service } = require('./src/concepts/Service'); -const { ServiceManager } = require('./src/system/ServiceManager'); -const traits = require('./src/traits/traits'); - -module.exports = { - AdvancedBase, - system: { - ServiceManager, - }, - libs: { - promise: require('./src/libs/promise'), - context: require('./src/libs/context'), - listener: require('./src/libs/listener'), - log: require('./src/libs/log'), - string: require('./src/libs/string'), - event: require('./src/libs/event'), - }, - features: { - EmitterFeature: require('./src/features/EmitterFeature'), - }, - concepts: { - Service, - }, - traits, -}; diff --git a/src/putility/package.json b/src/putility/package.json deleted file mode 100644 index accf062e7..000000000 --- a/src/putility/package.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "@heyputer/putility", - "version": "1.1.1", - "description": "", - "main": "index.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1", - "start-webpack": "webpack ./index.js --output-filename putility.js --output-library putility && webpack ./index.js --output-filename putility.dev.js --output-library putility --watch --devtool source-map", - "build": "webpack ./index.js --output-filename putility.js --output-library putility && { echo \"// Copyright 2024-present Puter Technologies Inc. All rights reserved.\"; echo \"// Generated on $(date '+%Y-%m-%d %H:%M')\n\"; cat ./dist/putility.js; echo \"\"; } > temp && mv temp ./dist/putility.js" - }, - "author": "Puter Technologies Inc.", - "license": "MIT" -} diff --git a/src/putility/src/AdvancedBase.js b/src/putility/src/AdvancedBase.js deleted file mode 100644 index 25319e33c..000000000 --- a/src/putility/src/AdvancedBase.js +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -// This doesn't go in ./bases because it logically depends on -// both ./bases and ./traits, and ./traits depends on ./bases. - -const { FeatureBase } = require('./bases/FeatureBase'); - -class AdvancedBase extends FeatureBase { - static FEATURES = [ - require('./features/NodeModuleDIFeature'), - require('./features/PropertiesFeature'), - require('./features/TraitsFeature'), - require('./features/TopicsFeature'), - ]; -} - -module.exports = { - AdvancedBase, -}; diff --git a/src/putility/src/bases/BasicBase.js b/src/putility/src/bases/BasicBase.js deleted file mode 100644 index 2d6d9f603..000000000 --- a/src/putility/src/bases/BasicBase.js +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -/** - * Base class that provides utilities for working with inheritance chains and static properties. - */ -class BasicBase { - /** - * Gets the inheritance chain for the current instance, starting from the most derived class - * and working up to BasicBase (excluded). - * @returns {Array} Array of constructor functions in inheritance order - */ - _get_inheritance_chain () { - const chain = []; - let cls = this.constructor; - while ( cls && cls !== BasicBase ) { - chain.push(cls); - cls = cls.__proto__; - } - return chain.reverse(); - } - - /** - * Merges static array properties from all classes in the inheritance chain. - * Avoids duplicating the same array reference from contiguous members - * of the inheritance chain (useful when using the decorator pattern with - * multiple classes sharing a common base) - * @param {string} key - The name of the static property to merge - * @returns {Array} Combined array containing all values from the inheritance chain - */ - _get_merged_static_array (key) { - const chain = this._get_inheritance_chain(); - const values = []; - let last = null; - for ( const cls of chain ) { - if ( cls[key] && cls[key] !== last ) { - last = cls[key]; - values.push(...cls[key]); - } - } - return values; - } - - /** - * Merges static object properties from all classes in the inheritance chain. - * Properties from derived classes override those from base classes. - * @param {string} key - The name of the static property to merge - * @returns {Object} Combined object containing all properties from the inheritance chain - */ - _get_merged_static_object (key) { - // TODO: check objects by reference - same object in a subclass shouldn't count - const chain = this._get_inheritance_chain(); - const values = {}; - for ( const cls of chain ) { - if ( cls[key] ) { - Object.assign(values, cls[key]); - } - } - return values; - } -} - -module.exports = { - BasicBase, -}; \ No newline at end of file diff --git a/src/putility/src/bases/FeatureBase.js b/src/putility/src/bases/FeatureBase.js deleted file mode 100644 index e65f8237a..000000000 --- a/src/putility/src/bases/FeatureBase.js +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -const { BasicBase } = require('./BasicBase'); - -class FeatureBase extends BasicBase { - constructor (parameters, ...a) { - super(parameters, ...a); - - this._ = { - features: this._get_merged_static_array('FEATURES'), - }; - - for ( const feature of this._.features ) { - feature.install_in_instance(this, - { - parameters: parameters || {}, - }); - } - } -} - -module.exports = { - FeatureBase, -}; diff --git a/src/putility/src/concepts/Service.js b/src/putility/src/concepts/Service.js deleted file mode 100644 index 1846372c1..000000000 --- a/src/putility/src/concepts/Service.js +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -const { AdvancedBase } = require('../AdvancedBase'); -const ServiceFeature = require('../features/ServiceFeature'); - -/** @type {Function} No-operation async function */ -const NOOP = async () => { -}; - -/** @type {Symbol} Service trait symbol */ -const TService = Symbol('TService'); - -/** - * Service class that will be incrementally updated to consolidate - * BaseService in Puter's backend with Service in Puter's frontend, - * becoming the common base for both and a useful utility in general. - * - * @class Service - * @extends AdvancedBase - */ -class Service extends AdvancedBase { - /** @type {Array} Array of features this service supports */ - static FEATURES = [ - ServiceFeature, - ]; - - /** - * Handles events by calling the appropriate event handler - * - * @param {string} id - The event identifier - * @param {Array} args - Arguments to pass to the event handler - * @returns {Promise<*>} The result of the event handler - */ - async __on (id, args) { - const handler = this.__get_event_handler(id); - - return await handler(id, ...args); - } - - /** - * Retrieves the event handler for a given event ID - * - * @param {string} id - The event identifier - * @returns {Function} The event handler function or NOOP if not found - */ - __get_event_handler (id) { - return this[`__on_${id}`]?.bind?.(this) - || this.constructor[`__on_${id}`]?.bind?.(this.constructor) - || NOOP; - } - - /** - * Factory method to create a new service instance - * - * @param {Object} config - Configuration object - * @param {Object} config.parameters - Parameters for service construction - * @param {Object} config.context - Context for the service - * @returns {Service} A new service instance - */ - static create ({ parameters, context }) { - const ins = new this(); - ins._.context = context; - ins.as(TService).construct(parameters); - return ins; - } - - static IMPLEMENTS = { - /** @type {Object} Implementation of the TService trait */ - [TService]: { - /** - * Initializes the service by running init hooks and calling _init if present - * - * @param {...*} a - Arguments to pass to _init method - * @returns {*} Result of _init method if it exists - */ - init (...a) { - if ( this._.init_hooks ) { - for ( const hook of this._.init_hooks ) { - hook.call(this); - } - } - if ( ! this._init ) return; - return this._init(...a); - }, - /** - * Constructs the service with given parameters - * - * @param {Object} o - Parameters object - * @returns {*} Result of _construct method if it exists - */ - construct (o) { - this.$parameters = {}; - for ( const k in o ) this.$parameters[k] = o[k]; - if ( ! this._construct ) return; - return this._construct(o); - }, - /** - * Gets the dependencies for this service - * - * @returns {Array} Array of dependencies - */ - get_depends () { - return [ - ...(this.constructor.DEPENDS ?? []), - ...(this.get_depends?.() ?? []), - ]; - }, - }, - }; -} - -module.exports = { - TService, - Service, -}; diff --git a/src/putility/src/features/EmitterFeature.js b/src/putility/src/features/EmitterFeature.js deleted file mode 100644 index 5f9b439c8..000000000 --- a/src/putility/src/features/EmitterFeature.js +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -/** - * A simpler alternative to TopicsFeature. This is an opt-in and not included - * in AdvancedBase. - * - * Adds methods `.on` and `emit`. Unlike TopicsFeature, this does not implement - * a trait. Usage is similar to node's built-in EventEmitter, but because it's - * installed as a mixin it can be used with other class features. - * - * When listeners return a promise, they will block the promise returned by the - * corresponding `emit()` call. Listeners are invoked concurrently, so - * listeners of the same event do not block each other. - */ -module.exports = ({ decorators } = {}) => ({ - install_in_instance (instance, { parameters }) { - // install the internal state - const state = instance._.emitterFeature = {}; - state.listeners_ = {}; - state.global_listeners_ = []; - state.callbackDecorators = decorators || []; - - instance.emit = async (key, data, meta) => { - meta = meta ?? {}; - const parts = key.split('.'); - - const promises = []; - - for ( let i = 0 ; i < state.global_listeners_.length ; i++ ) { - let callback = state.global_listeners_[i]; - for ( const decorator of state.callbackDecorators ) { - callback = decorator(callback); - } - - promises.push(callback(key, data, { ...meta, key })); - } - - for ( let i = 0; i < parts.length; i++ ) { - const part = i === parts.length - 1 - ? parts.join('.') - : `${parts.slice(0, i + 1).join('.') }.*`; - - // actual emit - const listeners = state.listeners_[part]; - if ( ! listeners ) continue; - for ( let i = 0; i < listeners.length; i++ ) { - let callback = listeners[i]; - for ( const decorator of state.callbackDecorators ) { - callback = decorator(callback); - } - - promises.push(callback(data, { - ...meta, - key, - })); - } - } - - return await Promise.all(promises); - }; - - instance.on = (selector, callback) => { - const listeners = state.listeners_[selector] || - (state.listeners_[selector] = []); - - listeners.push(callback); - - const det = { - detach: () => { - const idx = listeners.indexOf(callback); - if ( idx !== -1 ) { - listeners.splice(idx, 1); - } - }, - }; - - return det; - }; - - instance.on_all = (callback) => { - state.global_listeners_.push(callback); - }; - }, -}); diff --git a/src/putility/src/features/NodeModuleDIFeature.js b/src/putility/src/features/NodeModuleDIFeature.js deleted file mode 100644 index b00089566..000000000 --- a/src/putility/src/features/NodeModuleDIFeature.js +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -/** - * This trait allows dependency injection of node modules. - * This is incredibly useful for passing mock implementations - * of modules for unit testing. - * - * @example - * class MyClass extends AdvancedBase { - * static MODULES = { - * axios, - * }; - * } - * - * const my_class = new MyClass({ - * modules: { - * axios: MY_AXIOS_MOCK, - * } - * }); - */ -module.exports = { - install_in_instance: (instance, { parameters }) => { - const modules = instance._get_merged_static_object('MODULES'); - - if ( parameters.modules ) { - for ( const k in parameters.modules ) { - modules[k] = parameters.modules[k]; - } - } - - instance.modules = modules; - - // This "require" function can shadow the real one so - // that editor tools are aware of the modules that - // are being used. - instance.require = (name) => { - if ( instance.modules[name] ) { - return instance.modules[name]; - } - return require(name); - }; - }, -}; diff --git a/src/putility/src/features/PropertiesFeature.js b/src/putility/src/features/PropertiesFeature.js deleted file mode 100644 index 04608c2bb..000000000 --- a/src/putility/src/features/PropertiesFeature.js +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -module.exports = { - name: 'Properties', - depends: ['Listeners'], - install_in_instance: (instance, { parameters }) => { - const properties = instance._get_merged_static_object('PROPERTIES'); - - instance.onchange = (name, callback) => { - instance._.properties[name].listeners.push(callback); - }; - - instance._.properties = {}; - - for ( const k in properties ) { - const state = { - definition: properties[k], - listeners: [], - value: undefined, - }; - instance._.properties[k] = state; - - let spec = null; - if ( typeof properties[k] === 'object' ) { - spec = properties[k]; - if ( spec.factory ) { - spec.value = spec.factory({ parameters }); - } - } else if ( typeof properties[k] === 'function' ) { - spec = {}; - spec.value = properties[k](); - } - - if ( spec === null ) { - throw new Error('this will never happen'); - } - - Object.defineProperty(instance, k, { - get: () => { - return state.value; - }, - set: (value) => { - for ( const listener of instance._.properties[k].listeners ) { - listener(value, { - old_value: instance[k], - }); - } - const old_value = instance[k]; - const intermediate_value = value; - if ( spec.adapt ) { - value = spec.adapt(value); - } - state.value = value; - if ( spec.post_set ) { - spec.post_set.call(instance, value, { - intermediate_value, - old_value, - }); - } - }, - }); - - state.value = spec.value; - - if ( properties[k].construct ) { - const k_cons = typeof properties[k].construct === 'string' - ? properties[k].construct - : k; - instance[k] = parameters[k_cons]; - } - } - }, -}; diff --git a/src/putility/src/features/ServiceFeature.js b/src/putility/src/features/ServiceFeature.js deleted file mode 100644 index 13ef45fd0..000000000 --- a/src/putility/src/features/ServiceFeature.js +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -const { TTopics } = require('../traits/traits'); - -module.exports = { - install_in_instance: (instance, { parameters }) => { - // Convenient definition of listeners between services, - // which also makes these connections able to be understood as data - // without processing any code. - const hooks = instance._get_merged_static_array('HOOKS'); - instance._.init_hooks = instance._.init_hooks ?? []; - - for ( const spec of hooks ) { - - // We need to wait for the service to be initialized, because - // that's when the dependency services have already been - // initialized and are ready to accept listeners. - instance._.init_hooks.push(() => { - const service_entry = - instance._.context.services.info(spec.service); - const service_instance = service_entry.instance; - - service_instance.as(TTopics).sub( - spec.event, - spec.do.bind(instance)); - }); - } - }, -}; diff --git a/src/putility/src/features/TopicsFeature.js b/src/putility/src/features/TopicsFeature.js deleted file mode 100644 index e396680f3..000000000 --- a/src/putility/src/features/TopicsFeature.js +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -const { RemoveFromArrayDetachable } = require('../libs/listener'); -const { TTopics } = require('../traits/traits'); -const { install_in_instance } = require('./NodeModuleDIFeature'); - -module.exports = { - install_in_instance: (instance, { parameters }) => { - const topics = instance._get_merged_static_array('TOPICS'); - - instance._.topics = {}; - - for ( const name of topics ) { - instance._.topics[name] = { - listeners_: [], - }; - } - - instance.mixin(TTopics, { - pub: (k, v) => { - if ( k.includes('!') ) { - throw new Error('"!" in event name reserved for future use'); - } - const topic = instance._.topics[k]; - if ( ! topic ) { - console.warn(`missing topic: ${ topic}`); - return; - } - for ( const lis of topic.listeners_ ) { - lis(); - } - }, - sub: (k, fn) => { - const topic = instance._.topics[k]; - if ( ! topic ) { - console.warn(`missing topic: ${ topic}`); - return; - } - topic.listeners_.push(fn); - return new RemoveFromArrayDetachable(topic.listeners_, fn); - }, - }); - - }, -}; diff --git a/src/putility/src/features/TraitsFeature.js b/src/putility/src/features/TraitsFeature.js deleted file mode 100644 index 390b6fa9e..000000000 --- a/src/putility/src/features/TraitsFeature.js +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -module.exports = { - // old implementation - install_in_instance_: (instance, { parameters }) => { - const impls = instance._get_merged_static_object('IMPLEMENTS'); - - instance._.impls = {}; - - for ( const impl_name in impls ) { - const impl = impls[impl_name]; - const bound_impl = {}; - for ( const method_name in impl ) { - const fn = impl[method_name]; - bound_impl[method_name] = fn.bind(instance); - } - instance._.impls[impl_name] = bound_impl; - } - - instance.as = trait_name => instance._.impls[trait_name]; - instance.list_traits = () => Object.keys(instance._.impls); - }, - - // new implementation - install_in_instance: (instance, { parameters }) => { - const chain = instance._get_inheritance_chain(); - instance._.impls = {}; - - instance.as = trait_name => instance._.impls[trait_name]; - instance.list_traits = () => Object.keys(instance._.impls); - instance.mixin = (name, impl) => instance._.impls[name] = impl; - - for ( const cls of chain ) { - const cls_traits = cls.IMPLEMENTS; - if ( ! cls_traits ) continue; - const trait_keys = [ - ...Object.getOwnPropertySymbols(cls_traits), - ...Object.keys(cls_traits), - ]; - for ( const trait_name of trait_keys ) { - const impl = instance._.impls[trait_name] ?? - (instance._.impls[trait_name] = {}); - const cls_impl = cls_traits[trait_name]; - - for ( const method_name in cls_impl ) { - const fn = cls_impl[method_name]; - impl[method_name] = fn.bind(instance); - } - } - } - }, -}; diff --git a/src/putility/src/libs/context.js b/src/putility/src/libs/context.js deleted file mode 100644 index 7bac22f82..000000000 --- a/src/putility/src/libs/context.js +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -/** - * A context object that manages hierarchical property inheritance and sub-context creation. - * Properties are copied with their descriptors to maintain getter/setter behavior. - */ -class Context { - /** - * Creates a new Context instance with the provided values. - * @param {Object} [values={}] - Initial values to set on the context, with their property descriptors preserved - */ - constructor (values = {}) { - const descs = Object.getOwnPropertyDescriptors(values); - for ( const k in descs ) { - Object.defineProperty(this, k, descs[k]); - } - } - /** - * Creates a sub-context that follows specific properties from a source object. - * The returned context will have getters that reference the source object's properties. - * @param {Object} source - The source object to follow properties from - * @param {string[]} keys - Array of property names to follow from the source - * @returns {Context} A new sub-context with getters pointing to the source properties - */ - follow (source, keys) { - const values = {}; - for ( const k of keys ) { - Object.defineProperty(values, k, { - get: () => source[k], - }); - } - return this.sub(values); - } - /** - * Creates a sub-context that inherits from the current context with additional or overridden values. - * Nested Context instances are recursively sub-contexted with corresponding new values. - * @param {Object} [newValues={}] - New values to add or override in the sub-context - * @returns {Context} A new context that inherits from this context with the new values applied - */ - sub (newValues) { - if ( newValues === undefined ) newValues = {}; - const sub = Object.create(this); - - const alreadyApplied = {}; - for ( const k in sub ) { - if ( sub[k] instanceof Context ) { - const newValuesForK = - newValues.hasOwnProperty(k) - ? newValues[k] : undefined; - sub[k] = sub[k].sub(newValuesForK); - alreadyApplied[k] = true; - } - } - - const descs = Object.getOwnPropertyDescriptors(newValues); - for ( const k in descs ) { - if ( alreadyApplied[k] ) continue; - Object.defineProperty(sub, k, descs[k]); - } - - return sub; - } -} - -module.exports = { - Context, -}; diff --git a/src/putility/src/libs/event.js b/src/putility/src/libs/event.js deleted file mode 100644 index 88a3eb6ee..000000000 --- a/src/putility/src/libs/event.js +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -const { AdvancedBase } = require('../AdvancedBase'); -const EmitterFeature = require('../features/EmitterFeature'); - -class Emitter extends AdvancedBase { - static FEATURES = [ - EmitterFeature(), - ]; -} - -module.exports = { Emitter }; diff --git a/src/putility/src/libs/invoker.js b/src/putility/src/libs/invoker.js deleted file mode 100644 index f23724d92..000000000 --- a/src/putility/src/libs/invoker.js +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -const { AdvancedBase } = require('../..'); - -class Invoker extends AdvancedBase { - static create ({ - decorators, - delegate, - }) { - const invoker = new Invoker(); - invoker.decorators = decorators; - invoker.delegate = delegate; - return invoker; - } - async run (args) { - let fn = this.delegate; - const decorators = this.decorators; - for ( let i = decorators.length - 1 ; i >= 0 ; i-- ) { - const dec = decorators[i]; - fn = this.add_dec_(dec, fn); - } - return await fn(args); - } - add_dec_ (dec, fn) { - return async (args) => { - try { - if ( dec.on_call ) { - args = await dec.on_call(args); - } - let result = await fn(args); - if ( dec.on_return ) { - result = await dec.on_return(result); - } - return result; - } catch (e) { - if ( ! dec.on_error ) throw e; - - let cancel = false; - const a = { - error () { - return e; - }, - cancel_error () { - cancel = true; - }, - }; - const result = await dec.on_error(a); - if ( cancel ) { - return result; - } - throw result ?? e; - } - }; - } -} - -module.exports = { - Invoker, -}; diff --git a/src/putility/src/libs/listener.js b/src/putility/src/libs/listener.js deleted file mode 100644 index 979f4297d..000000000 --- a/src/putility/src/libs/listener.js +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -const { FeatureBase } = require('../bases/FeatureBase'); -const { TDetachable } = require('../traits/traits'); - -// NOTE: copied from src/backend/src/util/listenerutil.js, -// which is now deprecated. - -class MultiDetachable extends FeatureBase { - static FEATURES = [ - require('../features/TraitsFeature'), - ]; - - constructor () { - super(); - this.delegates = []; - this.detached_ = false; - } - - add (delegate) { - if ( this.detached_ ) { - delegate.detach(); - return; - } - - this.delegates.push(delegate); - } - - static IMPLEMENTS = { - [TDetachable]: { - detach () { - this.detached_ = true; - for ( const delegate of this.delegates ) { - delegate.detach(); - } - }, - }, - }; -} - -class AlsoDetachable extends FeatureBase { - static FEATURES = [ - require('../features/TraitsFeature'), - ]; - - constructor () { - super(); - this.also = () => { - }; - } - - also (also) { - this.also = also; - return this; - } - - static IMPLEMENTS = { - [TDetachable]: { - detach () { - this.detach_(); - this.also(); - }, - }, - }; -} - -// TODO: this doesn't work, but I don't know why yet. -class RemoveFromArrayDetachable extends AlsoDetachable { - constructor (array, element) { - super(); - this.array = new WeakRef(array); - this.element = element; - } - - detach_ () { - const array = this.array.deref(); - if ( ! array ) return; - const index = array.indexOf(this.element); - if ( index !== -1 ) { - array.splice(index, 1); - } - } -} - -module.exports = { - MultiDetachable, - RemoveFromArrayDetachable, -}; diff --git a/src/putility/src/libs/log.js b/src/putility/src/libs/log.js deleted file mode 100644 index f208eafb8..000000000 --- a/src/putility/src/libs/log.js +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -const { AdvancedBase } = require('../AdvancedBase'); -const { TLogger, AS } = require('../traits/traits'); - -/** - * Logger implementation that stores log entries in an internal array buffer. - * Useful for testing or collecting log entries for later processing. - */ -class ArrayLogger extends AdvancedBase { - static PROPERTIES = { - buffer: { - factory: () => [], - }, - }; - static IMPLEMENTS = { - [TLogger]: { - /** - * Logs a message by storing it in the internal buffer array. - * @param {string} level - The log level (e.g., 'info', 'warn', 'error') - * @param {string} message - The log message - * @param {Object} fields - Additional fields to include with the log entry - * @param {Array} values - Additional values to log - */ - log (level, message, fields, values) { - this.buffer.push({ level, message, fields, values }); - }, - }, - }; -} - -/** - * Logger that filters log entries based on enabled categories. - * Only logs messages for categories that have been explicitly enabled. - */ -class CategorizedToggleLogger extends AdvancedBase { - static PROPERTIES = { - categories: { - description: 'categories that are enabled', - factory: () => ({}), - }, - delegate: { - construct: true, - value: null, - adapt: v => AS(v, TLogger), - }, - }; - static IMPLEMENTS = { - [TLogger]: { - /** - * Logs a message only if the category specified in fields is enabled. - * @param {string} level - The log level - * @param {string} message - The log message - * @param {Object} fields - Fields object that should contain a 'category' property - * @param {Array} values - Additional values to log - * @returns {*} Result from delegate logger if category is enabled, undefined otherwise - */ - log (level, message, fields, values) { - const category = fields.category; - if ( ! this.categories[category] ) return; - return this.delegate.log(level, message, fields, values); - }, - }, - }; - /** - * Enables logging for the specified category. - * @param {string} category - The category to enable - */ - on (category) { - this.categories[category] = true; - } - /** - * Disables logging for the specified category. - * @param {string} category - The category to disable - */ - off (category) { - delete this.categories[category]; - } -} - -/** - * Logger that can be enabled or disabled globally. - * When disabled, all log messages are ignored. - */ -class ToggleLogger extends AdvancedBase { - static PROPERTIES = { - enabled: { - construct: true, - value: true, - }, - delegate: { - construct: true, - value: null, - adapt: v => AS(v, TLogger), - }, - }; - static IMPLEMENTS = { - [TLogger]: { - /** - * Logs a message only if the logger is enabled. - * @param {string} level - The log level - * @param {string} message - The log message - * @param {Object} fields - Additional fields to include - * @param {Array} values - Additional values to log - * @returns {*} Result from delegate logger if enabled, undefined otherwise - */ - log (level, message, fields, values) { - if ( ! this.enabled ) return; - return this.delegate.log(level, message, fields, values); - }, - }, - }; -} - -/** - * Logger that outputs formatted messages to the console. - * Supports colored output using ANSI escape codes and different log levels. - */ -class ConsoleLogger extends AdvancedBase { - static MODULES = { - // This would be cool, if it worked in a browser. - // util: require('util'), - - util: { - inspect: v => v, - // inspect: v => { - // if (typeof v === 'string') return v; - // try { - // return JSON.stringify(v); - // } catch (e) {} - // return '' + v; - // } - }, - }; - static PROPERTIES = { - console: { - construct: true, - factory: () => console, - }, - format: () => ({ - info: { - ansii: '\x1b[32;1m', - }, - warn: { - ansii: '\x1b[33;1m', - }, - error: { - ansii: '\x1b[31;1m', - err: true, - }, - debug: { - ansii: '\x1b[34;1m', - }, - }), - }; - static IMPLEMENTS = { - [TLogger]: { - /** - * Logs a formatted message to the console with color coding based on log level. - * @param {string} level - The log level (info, warn, error, debug) - * @param {string} message - The main log message - * @param {Object} fields - Additional fields to display - * @param {Array} values - Additional values to pass to console - */ - log (level, message, fields, values) { - const require = this.require; - const util = require('util'); - const l = this.format[level]; - let str = ''; - str += `${l.ansii}[${level.toUpperCase()}]\x1b[0m `; - str += message; - - // fields - if ( Object.keys(fields).length ) { - str += ' '; - str += `${Object.entries(fields) - .map(([k, v]) => `\n ${k}=${util.inspect(v)}`) - .join(' ') }\n`; - } - - (this.console ?? console)[l.err ? 'error' : 'log'](str, ...values); - }, - }, - }; -} - -/** - * Logger that adds a prefix to all log messages before delegating to another logger. - */ -class PrefixLogger extends AdvancedBase { - static PROPERTIES = { - prefix: { - construct: true, - value: '', - }, - delegate: { - construct: true, - value: null, - adapt: v => AS(v, TLogger), - }, - }; - static IMPLEMENTS = { - [TLogger]: { - /** - * Logs a message with the configured prefix prepended to the message. - * @param {string} level - The log level - * @param {string} message - The original message - * @param {Object} fields - Additional fields to include - * @param {Array} values - Additional values to log - * @returns {*} Result from the delegate logger - */ - log (level, message, fields, values) { - return this.delegate.log(level, this.prefix + message, fields, values); - }, - }, - }; -} - -/** - * Logger that adds default fields to all log entries before delegating to another logger. - */ -class FieldsLogger extends AdvancedBase { - static PROPERTIES = { - fields: { - construct: true, - factory: () => ({}), - }, - delegate: { - construct: true, - value: null, - adapt: v => AS(v, TLogger), - }, - }; - - static IMPLEMENTS = { - [TLogger]: { - /** - * Logs a message with the configured default fields merged with provided fields. - * @param {string} level - The log level - * @param {string} message - The log message - * @param {Object} fields - Additional fields that will be merged with default fields - * @param {Array} values - Additional values to log - * @returns {*} Result from the delegate logger - */ - log (level, message, fields, values) { - return this.delegate.log(level, message, Object.assign({}, this.fields, fields), values); - }, - }, - }; -} - -/** - * Facade that provides a convenient interface for logging operations. - * Supports method chaining and category management. - */ -class LoggerFacade extends AdvancedBase { - static PROPERTIES = { - impl: { - value: () => { - return new ConsoleLogger(); - }, - adapt: v => AS(v, TLogger), - construct: true, - }, - cat: { - construct: true, - }, - }; - - static IMPLEMENTS = { - [TLogger]: { - /** - * Basic log implementation (currently just outputs to console). - * @param {string} level - The log level - * @param {string} message - The log message - * @param {Object} fields - Additional fields - * @param {Array} values - Additional values - */ - log (level, message, fields, values) { - console.log(); - }, - }, - }; - - /** - * Creates a new logger facade with additional default fields. - * @param {Object} fields - Default fields to add to all log entries - * @returns {LoggerFacade} New logger facade instance with the specified fields - */ - fields (fields) { - const new_delegate = new FieldsLogger({ - fields, - delegate: this.impl, - }); - return new LoggerFacade({ - impl: new_delegate, - }); - } - - /** - * Logs an info-level message. - * @param {string} message - The message to log - * @param {...*} values - Additional values to include in the log - */ - info (message, ...values) { - this.impl.log('info', message, {}, values); - } - - /** - * Enables logging for a specific category. - * @param {string} category - The category to enable - */ - on (category) { - this.cat.on(category); - } - /** - * Disables logging for a specific category. - * @param {string} category - The category to disable - */ - off (category) { - this.cat.off(category); - } -} - -module.exports = { - ArrayLogger, - CategorizedToggleLogger, - ToggleLogger, - ConsoleLogger, - PrefixLogger, - FieldsLogger, - LoggerFacade, -}; diff --git a/src/putility/src/libs/promise.js b/src/putility/src/libs/promise.js deleted file mode 100644 index f32342c4e..000000000 --- a/src/putility/src/libs/promise.js +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -class TeePromise { - static STATUS_PENDING = Symbol('pending'); - static STATUS_RUNNING = {}; - static STATUS_DONE = Symbol('done'); - constructor () { - this.status_ = this.constructor.STATUS_PENDING; - this.donePromise = new Promise((resolve, reject) => { - this.doneResolve = resolve; - this.doneReject = reject; - }); - } - get status () { - return this.status_; - } - set status (status) { - this.status_ = status; - if ( status === this.constructor.STATUS_DONE ) { - this.doneResolve(); - } - } - resolve (value) { - this.status_ = this.constructor.STATUS_DONE; - this.doneResolve(value); - } - awaitDone () { - return this.donePromise; - } - then (fn, ...a) { - return this.donePromise.then(fn, ...a); - } - - reject (err) { - this.status_ = this.constructor.STATUS_DONE; - this.doneReject(err); - } - - /** - * @deprecated use then() instead - */ - onComplete (fn) { - return this.then(fn); - } -} - -class Lock { - constructor () { - this._locked = false; - this._waiting = []; - } - - async acquire (callback) { - await new Promise(resolve => { - if ( ! this._locked ) { - this._locked = true; - resolve(); - } else { - this._waiting.push({ - resolve, - }); - } - }); - if ( callback ) { - let retval; - try { - retval = await callback(); - } finally { - this.release(); - } - return retval; - } - } - - release () { - if ( this._waiting.length > 0 ) { - const { resolve } = this._waiting.shift(); - resolve(); - } else { - this._locked = false; - } - } -} - -class RWLock { - static TYPE_READ = Symbol('read'); - static TYPE_WRITE = Symbol('write'); - - constructor () { - this.queue = []; - - this.readers_ = 0; - this.writer_ = false; - - this.on_empty_ = () => { - }; - - this.mode = this.constructor.TYPE_READ; - } - get effective_mode () { - if ( this.readers_ > 0 ) return this.constructor.TYPE_READ; - if ( this.writer_ ) return this.constructor.TYPE_WRITE; - return undefined; - } - push_ (item) { - if ( this.readers_ === 0 && !this.writer_ ) { - this.mode = item.type; - } - this.queue.push(item); - this.check_queue_(); - } - check_queue_ () { - // console.log('check_queue_', { - // readers_: this.readers_, - // writer_: this.writer_, - // queue: this.queue.map(item => item.type), - // }); - if ( this.queue.length === 0 ) { - if ( this.readers_ === 0 && !this.writer_ ) { - this.on_empty_(); - } - return; - } - - const peek = () => this.queue[0]; - - if ( this.readers_ === 0 && !this.writer_ ) { - this.mode = peek().type; - } - - if ( this.mode === this.constructor.TYPE_READ ) { - while ( peek()?.type === this.constructor.TYPE_READ ) { - const item = this.queue.shift(); - this.readers_++; - (async () => { - await item.p_unlock; - this.readers_--; - this.check_queue_(); - })(); - item.p_operation.resolve(); - } - return; - } - - if ( this.writer_ ) return; - - const item = this.queue.shift(); - this.writer_ = true; - (async () => { - await item.p_unlock; - this.writer_ = false; - this.check_queue_(); - })(); - item.p_operation.resolve(); - } - async rlock () { - const p_read = new TeePromise(); - const p_unlock = new TeePromise(); - const handle = { - unlock: () => { - p_unlock.resolve(); - }, - }; - - this.push_({ - type: this.constructor.TYPE_READ, - p_operation: p_read, - p_unlock, - }); - await p_read; - - return handle; - } - - async wlock () { - const p_write = new TeePromise(); - const p_unlock = new TeePromise(); - const handle = { - unlock: () => { - p_unlock.resolve(); - }, - }; - - this.push_({ - type: this.constructor.TYPE_WRITE, - p_operation: p_write, - p_unlock, - }); - await p_write; - - return handle; - } - -} - -/** - * @callback behindScheduleCallback - * @param {number} drift - The number of milliseconds that the callback was - * called behind schedule. - * @returns {boolean} - If the callback returns true, the timer will be - * cancelled. - */ - -/** - * When passing an async callback to setInterval, it's possible for the - * callback to be called again before the previous invocation has finished. - * - * This function wraps setInterval and ensures that the callback is not - * called again until the previous invocation has finished. - * - * @param {Function} callback - The function to call when the timer elapses. - * @param {number} delay - The minimum number of milliseconds between invocations. - * @param {?Array} args - Additional arguments to pass to setInterval. - * @param {?Object} options - Additional options. - * @param {behindScheduleCallback} options.onBehindSchedule - A callback to call when the callback is called behind schedule. - */ -const asyncSafeSetInterval = async (callback, delay, args, options) => { - args = args ?? []; - options = options ?? {}; - const { onBehindSchedule } = options; - - const sleep = (ms) => new Promise(rslv => setTimeout(rslv, ms)); - - for ( ;; ) { - await sleep(delay); - - const ts_start = Date.now(); - await callback(...args); - const ts_end = Date.now(); - - const runtime = ts_end - ts_start; - const sleep_time = delay - runtime; - - if ( sleep_time < 0 ) { - if ( onBehindSchedule ) { - const cancel = await onBehindSchedule(-sleep_time); - if ( cancel ) { - return; - } - } - } else { - await sleep(sleep_time); - } - } -}; - -/** - * raceCase is like Promise.race except it takes an object instead of - * an array, and returns the key of the promise that resolves first - * as well as the value that it resolved to. - * - * @param {Object.} promise_map - * - * @returns {Promise.<[string, any]>} - */ -const raceCase = async (promise_map) => { - return Promise.race(Object.entries(promise_map).map( - ([key, promise]) => promise.then(value => [key, value]))); -}; - -module.exports = { - TeePromise, - Lock, - RWLock, - asyncSafeSetInterval, - raceCase, -}; diff --git a/src/putility/src/libs/string.js b/src/putility/src/libs/string.js deleted file mode 100644 index aad902dd4..000000000 --- a/src/putility/src/libs/string.js +++ /dev/null @@ -1,32 +0,0 @@ -// METADATA // {"def":"core.util.strutil","ai-params":{"service":"claude"},"":{"service":"claude"}} - -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -/*eslint no-control-regex: 'off'*/ - -/** -* Quotes a string value, handling special cases for undefined, null, functions, objects and numbers. -* Escapes quotes and returns a JSON-stringified version with quote character normalization. -* @param {*} str - The value to quote -* @returns {string} The quoted string representation -*/ -const quot = (str) => { - if ( str === undefined ) return '[undefined]'; - if ( str === null ) return '[null]'; - if ( typeof str === 'function' ) return '[function]'; - if ( typeof str === 'object' ) return '[object]'; - if ( typeof str === 'number' ) return `(${ str })`; - - str = `${ str}`; - - str = str.replace(/["`]/g, m => m === '"' ? '`' : '"'); - str = JSON.stringify(`${ str}`); - str = str.replace(/["`]/g, m => m === '"' ? '`' : '"'); - return str; -}; - -module.exports = { - quot, -}; diff --git a/src/putility/src/system/ServiceManager.js b/src/putility/src/system/ServiceManager.js deleted file mode 100644 index 1137065fb..000000000 --- a/src/putility/src/system/ServiceManager.js +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -const { AdvancedBase } = require('../AdvancedBase'); -const { TService } = require('../concepts/Service'); - -const StatusEnum = { - Registering: 'registering', - Pending: 'pending', - Initializing: 'initializing', - Running: 'running', -}; -class ServiceManager extends AdvancedBase { - constructor ({ context } = {}) { - super(); - - this.context = context; - - this.services_l_ = []; - this.services_m_ = {}; - this.service_infos_ = {}; - - this.init_listeners_ = []; - // services which are waiting for dependency servicces to be - // initialized; mapped like: waiting_[dependency] = Set(dependents) - this.waiting_ = {}; - } - async register (name, factory, options = {}) { - await new Promise(rslv => setTimeout(rslv, 0)); - - const ins = factory.create({ - parameters: options.parameters ?? {}, - context: this.context, - }); - const entry = { - name, - instance: ins, - status: StatusEnum.Registering, - }; - this.services_l_.push(entry); - this.services_m_[name] = entry; - - await this.maybe_init_(name); - } - info (name) { - return this.services_m_[name]; - } - get (name) { - const info = this.services_m_[name]; - if ( ! info ) throw new Error(`Service not registered: ${name}`); - if ( info.status !== StatusEnum.Running ) { - return undefined; - } - return info.instance; - } - async aget (name) { - await this.wait_for_init([name]); - return this.get(name); - } - - /** - * Wait for the specified list of services to be initialized. - * @param {*} depends - list of services to wait for - */ - async wait_for_init (depends) { - let check; - - await new Promise(rslv => { - check = () => { - // Get the list of required services that are not - // yet initialized - const waiting_for = this.get_waiting_for_(depends); - - // If there's nothing to wait for, remove the listener - // on service initializations and resolve - if ( waiting_for.length === 0 ) { - const i = this.init_listeners_.indexOf(check); - if ( i !== -1 ) { - this.init_listeners_.splice(i, 1); - } - rslv(); - - return true; - } - }; - - // Services might already be registered - if ( check() ) return; - - this.init_listeners_.push(check); - }); - }; - - get_waiting_for_ (depends) { - const waiting_for = []; - for ( const depend of depends ) { - const depend_entry = this.services_m_[depend]; - if ( ! depend_entry ) { - waiting_for.push(depend); - continue; - } - if ( ( depend_entry.status !== StatusEnum.Running ) ) { - waiting_for.push(depend); - } - } - return waiting_for; - } - - async maybe_init_ (name) { - const entry = this.services_m_[name]; - const depends = entry.instance.as(TService).get_depends(); - const waiting_for = this.get_waiting_for_(depends); - - if ( waiting_for.length === 0 ) { - await this.init_service_(name); - return; - } - - for ( const dependency of waiting_for ) { - if ( ! this.waiting_[dependency] ) { - this.waiting_[dependency] = new Set(); - } - this.waiting_[dependency].add(name); - } - - entry.status = StatusEnum.Pending; - entry.statusWaitingFor = waiting_for; - } - - // called when a service has all of its dependencies initialized - // and is ready to be initialized itself - async init_service_ (name, modifiers = {}) { - const entry = this.services_m_[name]; - entry.status = StatusEnum.Initializing; - - const service_impl = entry.instance.as(TService); - await service_impl.init(); - entry.status = StatusEnum.Running; - entry.statusStartTS = new Date(); - /** @type Set */ - const maybe_ready_set = this.waiting_[name]; - const promises = []; - if ( maybe_ready_set ) { - for ( const dependent of maybe_ready_set.values() ) { - promises.push(this.maybe_init_(dependent, { - no_init_listeners: true, - })); - } - } - await Promise.all(promises); - - if ( ! modifiers.no_init_listeners ) { - for ( const lis of this.init_listeners_ ) { - await lis(); - } - } - } -} - -module.exports = { - ServiceManager, -}; diff --git a/src/putility/src/traits/traits.js b/src/putility/src/traits/traits.js deleted file mode 100644 index 62b89818f..000000000 --- a/src/putility/src/traits/traits.js +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - */ - -module.exports = { - TTopics: Symbol('TTopics'), - TDetachable: Symbol('TDetachable'), - TLogger: Symbol('TLogger'), - - AS: (obj, trait) => { - if ( obj.constructor && obj.constructor.IMPLEMENTS && obj.constructor.IMPLEMENTS[trait] ) { - return obj.as(trait); - } - return obj; - }, -}; diff --git a/src/putility/test/ServiceManager.test.js b/src/putility/test/ServiceManager.test.js deleted file mode 100644 index 7d73fc284..000000000 --- a/src/putility/test/ServiceManager.test.js +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { expect } = require('chai'); -const { Service } = require('../src/concepts/Service.js'); -const { ServiceManager } = require('../src/system/ServiceManager.js'); - -class TestService extends Service { - _construct ({ name, depends }) { - this.name_ = name; - this.depends_ = depends; - this.initialized_ = false; - } - get_depends () { - return this.depends_; - } - async _init () { - // to ensure init is correctly awaited in tests - await new Promise(rslv => setTimeout(rslv, 0)); - - this.initialized_ = true; - } -} - -describe('ServiceManager', () => { - it('handles dependencies', async () => { - const serviceMgr = new ServiceManager(); - - // register a service with two depends; it will start last - await serviceMgr.register('a', TestService, { - parameters: { - name: 'a', - depends: ['b', 'c'], - }, - }); - - let a_info = serviceMgr.info('a'); - expect(a_info.status.describe()).to.equal('waiting for: b, c'); - - // register a service with no depends; should start right away - await serviceMgr.register('b', TestService, { - parameters: { - name: 'b', - depends: [], - }, - }); - - let b_info = serviceMgr.info('b'); - expect(b_info.status.label).to.equal('running'); - - a_info = serviceMgr.info('a'); - expect(a_info.status.describe()).to.equal('waiting for: c'); - - await serviceMgr.register('c', TestService, { - parameters: { - name: 'c', - depends: ['b'], - }, - }); - - let c_info = serviceMgr.info('c'); - expect(c_info.status.label).to.equal('running'); - a_info = serviceMgr.info('a'); - expect(a_info.status.label).to.equal('running'); - b_info = serviceMgr.info('b'); - expect(b_info.status.label).to.equal('running'); - }); -}); diff --git a/src/putility/test/context.test.js b/src/putility/test/context.test.js deleted file mode 100644 index 57bfac573..000000000 --- a/src/putility/test/context.test.js +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { Context } = require('../src/libs/context'); -const { expect } = require('chai'); - -describe('context', () => { - it('works', () => { - const c0 = new Context({ - a: 1, b: 2, - }); - const c1 = c0.sub({ - b: 3, - }); - - expect(c0.a).to.equal(1); - expect(c0.b).to.equal(2); - expect(c1.a).to.equal(1); - expect(c1.b).to.equal(3); - }); -}); diff --git a/src/putility/test/event.test.js b/src/putility/test/event.test.js deleted file mode 100644 index 2c938a2b8..000000000 --- a/src/putility/test/event.test.js +++ /dev/null @@ -1,14 +0,0 @@ -const { Emitter } = require('../src/libs/event'); -const { expect } = require('chai'); - -describe('Emitter', () => { - it('has EmitterFeature installed', async () => { - const em = new Emitter(); - let value = false; - em.on('test', () => { - value = true; - }); - await em.emit('test'); - expect(value).to.equal(true); - }); -}); diff --git a/src/putility/test/listener.test.js b/src/putility/test/listener.test.js deleted file mode 100644 index b00c00b54..000000000 --- a/src/putility/test/listener.test.js +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { RemoveFromArrayDetachable } = require('../src/libs/listener'); -const { expect } = require('chai'); -const { TDetachable } = require('../src/traits/traits'); - -describe('RemoveFromArrayDetachable', () => { - it ('does the thing', () => { - const someArray = []; - - const add_listener = (key, lis) => { - someArray.push(lis); - return new RemoveFromArrayDetachable(someArray, lis); - }; - - const det = add_listener('test', () => { - console.log('i am test func'); - }); - - expect(someArray.length).to.equal(1); - - det.as(TDetachable).detach(); - - expect(someArray.length).to.equal(0); - }); -}); diff --git a/src/putility/test/log.test.js b/src/putility/test/log.test.js deleted file mode 100644 index 14e3cf030..000000000 --- a/src/putility/test/log.test.js +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { LoggerFacade, ArrayLogger, ConsoleLogger } = require('../src/libs/log'); -const { expect } = require('chai'); - -describe('log', () => { - it('facade logger', () => { - const array_logger = new ArrayLogger(); - - let logger = new LoggerFacade({ - impl: array_logger, - }); - - logger.info('test message only'); - logger.info('test message and values', 1, 2); - logger = logger.fields({ a: 1 }); - logger.info('test fields', 3, 4); - - const logs = array_logger.buffer; - expect(logs).to.have.length(3); - - expect(logs[0].level).to.equal('info'); - expect(logs[0].message).to.equal('test message only'); - expect(logs[0].fields).to.eql({}); - expect(logs[0].values).to.eql([]); - - expect(logs[1].level).to.equal('info'); - expect(logs[1].message).to.equal('test message and values'); - expect(logs[1].fields).to.eql({}); - expect(logs[1].values).to.eql([1, 2]); - - expect(logs[2].level).to.equal('info'); - expect(logs[2].message).to.equal('test fields'); - expect(logs[2].fields).to.eql({ a: 1 }); - expect(logs[2].values).to.eql([3, 4]); - }); - it('console logger', () => { - let logger = new ConsoleLogger({ - console: console, - }); - logger = new LoggerFacade({ - impl: logger, - }); - - logger.fields({ - token: 'asdf', - user: 'joe', - }).info('Hello, world!', 'v1', 'v2', { a: 1 }); - }); -}); diff --git a/src/putility/test/test.js b/src/putility/test/test.js deleted file mode 100644 index 40dc88c89..000000000 --- a/src/putility/test/test.js +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { expect } = require('chai'); -const { BasicBase } = require('../src/bases/BasicBase'); -const { AdvancedBase } = require('../src/AdvancedBase'); -const { Invoker } = require('../src/libs/invoker'); - -class ClassA extends BasicBase { - static STATIC_OBJ = { - a: 1, - b: 2, - }; - static STATIC_ARR = ['a', 'b']; -} - -class ClassB extends ClassA { - static STATIC_OBJ = { - c: 3, - d: 4, - }; - static STATIC_ARR = ['c', 'd']; -} - -describe('testing', () => { - it('does a thing', () => { - const b = new ClassB(); - - console.log(b._get_inheritance_chain()); - console.log([ClassA, ClassB]); - expect(b._get_inheritance_chain()).deep.equal([ClassA, ClassB]); - expect(b._get_merged_static_array('STATIC_ARR')) - .deep.equal(['a', 'b', 'c', 'd']); - expect(b._get_merged_static_object('STATIC_OBJ')) - .deep.equal({ a: 1, b: 2, c: 3, d: 4 }); - }); -}); - -class ClassWithModule extends AdvancedBase { - static MODULES = { - axios: 'axios', - }; -} - -describe('AdvancedBase', () => { - it('passes DI modules to instance', () => { - const c1 = new ClassWithModule(); - expect(c1.modules.axios).to.equal('axios'); - - const c2 = new ClassWithModule({ - modules: { - axios: 'my-axios', - }, - }); - expect(c2.modules.axios).to.equal('my-axios'); - }); -}); - -describe('lib:invoker', () => { - it('works', async () => { - const invoker = Invoker.create({ - decorators: [ - { - name: 'uphill both ways', - on_call: (args) => { - return { - ...args, - n: args.n + 1, - }; - }, - on_return: (result) => { - return { - n: result.n + 1, - }; - }, - }, - { - name: 'error number five', - on_error: a => { - a.cancel_error(); - return { n: 5 }; - }, - }, - ], - async delegate (args) { - const { n } = args; - if ( n === 3 ) { - throw new Error('test error'); - } - return { n: 'oops' }; - }, - }); - expect(await invoker.run({ n: 2 })).to.deep.equal({ n: 6 }); - }); -}); diff --git a/src/putility/test/topics.test.js b/src/putility/test/topics.test.js deleted file mode 100644 index b6990331e..000000000 --- a/src/putility/test/topics.test.js +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { expect } = require('chai'); -const { AdvancedBase } = require('../src/AdvancedBase'); -const { TTopics, TDetachable } = require('../src/traits/traits'); - -describe('topics', () => { - it ('works', () => { - // A trait for something that's "punchable" - const TPunchable = Symbol('punchable'); - - class SomeClassWithTopics extends AdvancedBase { - // We can "listen on punched" - static TOPICS = ['punched']; - - // Punchable trait implementation - static IMPLEMENTS = { - [TPunchable]: { - punch () { - this.as(TTopics).pub('punched', { - information: 'about the punch', - in_whatever: 'format you desire', - }); - }, - }, - }; - } - - const thingy = new SomeClassWithTopics(); - - // Register the first listener, which we expect to be called both times - let first_listener_called = false; - thingy.as(TTopics).sub('punched', () => { - first_listener_called = true; - }); - - // Register the second listener, which we expect to be called once, - // and then we're gonna detach it and make sure detach works - let second_listener_call_count = 0; - const det = thingy.as(TTopics).sub('punched', () => { - second_listener_call_count++; - }); - - thingy.as(TPunchable).punch(); - det.as(TDetachable).detach(); - thingy.as(TPunchable).punch(); - - expect(first_listener_called).to.equal(true); - expect(second_listener_call_count).to.equal(1); - }); -}); diff --git a/src/putility/test/traits.test.js b/src/putility/test/traits.test.js deleted file mode 100644 index 6e88deda2..000000000 --- a/src/putility/test/traits.test.js +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { expect } = require('chai'); -const { AdvancedBase } = require('../src/AdvancedBase'); - -class TestClass extends AdvancedBase { - static IMPLEMENTS = { - test_trait: { - test_method: () => 'A', - }, - override_trait: { - preserved_method: () => 'B', - override_method: () => 'C', - }, - }; -} - -class TestSubClass extends TestClass { - static IMPLEMENTS = { - override_trait: { - override_method: () => 'D', - }, - }; -} - -describe('traits', () => { - it('instance.as', () => { - const o = new TestClass(); - expect(o.as).to.be.a('function'); - const ot = o.as('test_trait'); - expect(ot.test_method).to.be.a('function'); - expect(ot.test_method()).to.equal('A'); - }); - it('traits of parent', () => { - const o = new TestSubClass(); - console.log(o._get_merged_static_object('IMPLEMENTS')); - expect(o.as).to.be.a('function'); - const ot = o.as('test_trait'); - expect(ot.test_method).to.be.a('function'); - expect(ot.test_method()).to.equal('A'); - }); - it('trait method overrides', () => { - const o = new TestSubClass(); - expect(o.as).to.be.a('function'); - const ot = o.as('override_trait'); - expect(ot.preserved_method).to.be.a('function'); - expect(ot.override_method).to.be.a('function'); - expect (ot.preserved_method()).to.equal('B'); - expect (ot.override_method()).to.equal('D'); - }); -}); \ No newline at end of file diff --git a/src/worker/.gitignore b/src/worker/.gitignore new file mode 100644 index 000000000..30b950720 --- /dev/null +++ b/src/worker/.gitignore @@ -0,0 +1,2 @@ +dist/ + diff --git a/src/worker/package.json b/src/worker/package.json new file mode 100644 index 000000000..a62cf5f70 --- /dev/null +++ b/src/worker/package.json @@ -0,0 +1,17 @@ +{ + "name": "@heyputer/worker", + "type": "module", + "version": "1.0.0", + "description": "Worker preamble builder for Puter", + "main": "src/index.js", + "scripts": { + "build": "webpack --config webpack.config.cjs --mode production && node ./scripts/buildPreamble.mjs" + }, + "devDependencies": { + "terser-webpack-plugin": "^5.3.14", + "webpack": "^5.88.2", + "webpack-cli": "^5.1.1" + }, + "author": "Puter Technologies Inc.", + "license": "AGPL-3.0-only" +} diff --git a/src/worker/scripts/buildPreamble.mjs b/src/worker/scripts/buildPreamble.mjs new file mode 100644 index 000000000..addf5d9e1 --- /dev/null +++ b/src/worker/scripts/buildPreamble.mjs @@ -0,0 +1,40 @@ +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)); +const workerDir = path.resolve(scriptDir, '..'); +const templatePath = path.join(workerDir, 'template', 'puter-portable.template'); +const outputDir = path.join(workerDir, 'dist'); +const outputPath = path.join(outputDir, 'workerPreamble.js'); + +const inlineIncludes = async (filePath) => { + const fileContents = await readFile(filePath, 'utf-8'); + const lines = fileContents.split('\n'); + const expandedLines = []; + + for (const line of lines) { + const includeMatch = /^([ \t]*)#include "([^"]+)"$/.exec(line); + if (!includeMatch) { + expandedLines.push(line); + continue; + } + + const [, indent, relativePath] = includeMatch; + const includedPath = path.resolve(path.dirname(filePath), relativePath); + const includedContents = await inlineIncludes(includedPath); + for (const includedLine of includedContents.split('\n')) { + expandedLines.push( + includedLine.length > 0 + ? `${indent}${includedLine}` + : includedLine, + ); + } + } + + return expandedLines.join('\n'); +}; + +await mkdir(outputDir, { recursive: true }); +const preambleSource = await inlineIncludes(templatePath); +await writeFile(outputPath, preambleSource); diff --git a/src/worker/src/index.js b/src/worker/src/index.js new file mode 100644 index 000000000..e06dde569 --- /dev/null +++ b/src/worker/src/index.js @@ -0,0 +1,4 @@ +import initS2w from './s2w-router.js'; + +initS2w(); + diff --git a/src/worker/src/s2w-router.js b/src/worker/src/s2w-router.js new file mode 100644 index 000000000..2309f163f --- /dev/null +++ b/src/worker/src/s2w-router.js @@ -0,0 +1,209 @@ +const escapeRegex = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +const buildRouteMatcher = (route) => { + let pattern = '^'; + const paramNames = []; + + for (let index = 0; index < route.length; index += 1) { + const char = route[index]; + if (char === ':' || char === '*') { + let name = ''; + let offset = index + 1; + while (offset < route.length) { + const nextChar = route[offset]; + if (!/[A-Za-z0-9_]/.test(nextChar)) { + break; + } + name += nextChar; + offset += 1; + } + + if (name.length === 0) { + pattern += escapeRegex(char); + continue; + } + + paramNames.push(name); + pattern += char === ':' ? '([^/]+)' : '(.*)'; + index = offset - 1; + continue; + } + + pattern += escapeRegex(char); + } + + pattern += '$'; + const regex = new RegExp(pattern); + return (pathname) => { + const matches = regex.exec(pathname); + if (!matches) return false; + + const params = {}; + for (let index = 0; index < paramNames.length; index += 1) { + params[paramNames[index]] = matches[index + 1]; + } + return { params }; + }; +}; + +function initS2w () { + const router = { + routing: true, + handleCors: true, + map: new Map(), + custom(eventName, route, eventListener) { + const matchExp = buildRouteMatcher(route); + if (!this.map.has(eventName)) { + this.map.set(eventName, [[matchExp, eventListener]]); + return; + } + this.map.get(eventName).push([matchExp, eventListener]); + }, + get(...args) { + this.custom('GET', ...args); + }, + post(...args) { + this.custom('POST', ...args); + }, + options(...args) { + this.custom('OPTIONS', ...args); + }, + put(...args) { + this.custom('PUT', ...args); + }, + delete(...args) { + this.custom('DELETE', ...args); + }, + async handleOptions(request) { + const corsHeaders = { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'GET,HEAD,POST,OPTIONS', + 'Access-Control-Max-Age': '86400', + }; + if ( + request.headers.get('Origin') !== null && + request.headers.get('Access-Control-Request-Method') !== null && + request.headers.get('Access-Control-Request-Headers') !== null + ) { + return new Response(null, { + headers: { + ...corsHeaders, + 'Access-Control-Allow-Headers': + request.headers.get( + 'Access-Control-Request-Headers', + ), + }, + }); + } + return new Response(null, { + headers: { + Allow: 'GET, HEAD, POST, OPTIONS', + }, + }); + }, + async route(event) { + if (!globalThis.me) { + globalThis.me = { + puter: init_puter_portable( + globalThis.puter_auth, + globalThis.puter_endpoint || 'https://api.puter.com', + 'userPuter', + ), + }; + globalThis.my = me; + globalThis.myself = me; + } + if (event.request.headers.has('puter-auth')) { + event.requestor = { + puter: init_puter_portable( + event.request.headers.get('puter-auth'), + globalThis.puter_endpoint || 'https://api.puter.com', + 'userPuter', + ), + }; + event.user = event.requestor; + } + + const mappings = this.map.get(event.request.method); + if (this.handleCors && event.request.method === 'OPTIONS' && !mappings) { + return this.handleOptions(event.request); + } + if (!mappings) { + return new Response( + `No routes for given request type ${event.request.method}`, + { status: 404 }, + ); + } + + const url = new URL(event.request.url); + try { + for (const mapping of mappings) { + const results = mapping[0](url.pathname); + if (!results) continue; + + event.params = results.params; + let response = await mapping[1](event); + if (!(response instanceof Response)) { + try { + if ( + response instanceof Blob || + response instanceof ArrayBuffer || + response instanceof Uint8Array.__proto__ || + response instanceof ReadableStream || + response instanceof URLSearchParams || + typeof response === 'string' + ) { + response = new Response(response); + } else { + response = new Response(JSON.stringify(response), { + headers: { + 'content-type': 'application/json', + }, + }); + } + } catch { + throw new Error( + 'Returned response by handler was neither a Response object nor an object which can implicitly be converted into a Response object', + ); + } + } + if ( + this.handleCors && + !response.headers.has('access-control-allow-origin') + ) { + response.headers.set('Access-Control-Allow-Origin', '*'); + } + return response; + } + } catch (error) { + const response = new Response(error, { + status: 500, + statusText: 'Server Error', + }); + if ( + this.handleCors && + !response.headers.has('access-control-allow-origin') + ) { + response.headers.set('Access-Control-Allow-Origin', '*'); + } + return response; + } + + return new Response('Path not found', { + status: 404, + statusText: 'Not found', + }); + }, + }; + + globalThis.router = router; + self.addEventListener('fetch', (event) => { + if (!router.routing) { + return false; + } + event.respondWith(router.route(event)); + return true; + }); +} + +export default initS2w; diff --git a/src/worker/template/puter-portable.template b/src/worker/template/puter-portable.template new file mode 100644 index 000000000..832c97555 --- /dev/null +++ b/src/worker/template/puter-portable.template @@ -0,0 +1,47 @@ +// This file is not actually in the webpack project, it is handled separately. + +if (globalThis.Cloudflare) { + // Cloudflare Workers has a faulty EventTarget implementation which doesn't + // bind "this" to the event handler. + // https://github.com/cloudflare/workerd/issues/4453 + const CfEventTarget = EventTarget; + globalThis.EventTarget = class EventTarget extends CfEventTarget { + constructor(...args) { + super(...args); + } + + addEventListener(type, listener, options) { + super.addEventListener(type, listener.bind(this), options); + } + }; +} + +globalThis.init_puter_portable = (auth, apiOrigin, type) => { + if (type === 'userPuter') { + const goodContext = {}; + Object.getOwnPropertyNames(globalThis).forEach((name) => { + try { + goodContext[name] = globalThis[name]; + } catch {} + }); + goodContext.globalThis = goodContext; + goodContext.WorkerGlobalScope = WorkerGlobalScope; + goodContext.ServiceWorkerGlobalScope = ServiceWorkerGlobalScope; + goodContext.location = new URL('https://puter.work'); + goodContext.addEventListener = () => {}; + // @ts-ignore + with (goodContext) { + #include "../../puter-js/dist/puter.js" + } + goodContext.puter.setAPIOrigin(apiOrigin); + goodContext.puter.setAuthToken(auth); + return goodContext.puter; + } + + #include "../../puter-js/dist/puter.js" + + puter.setAPIOrigin(apiOrigin); + puter.setAuthToken(auth); +}; + +#include "../dist/webpackPreamplePart.js" diff --git a/src/backend/src/services/worker/webpack.config.js b/src/worker/webpack.config.cjs similarity index 93% rename from src/backend/src/services/worker/webpack.config.js rename to src/worker/webpack.config.cjs index 0cb6a8511..45b59dc73 100644 --- a/src/backend/src/services/worker/webpack.config.js +++ b/src/worker/webpack.config.cjs @@ -1,5 +1,6 @@ const path = require('path'); const webpack = require('webpack'); +const TerserPlugin = require('terser-webpack-plugin'); module.exports = { entry: './src/index.js', @@ -23,7 +24,7 @@ module.exports = { optimization: { minimize: true, minimizer: [ - new (require('terser-webpack-plugin'))({ + new TerserPlugin({ terserOptions: { keep_fnames: true, mangle: { @@ -54,4 +55,5 @@ module.exports = { entryOnly: false, }), ], -}; \ No newline at end of file +}; + diff --git a/tests/playwright/tests/file-system/fixtures.ts b/tests/playwright/tests/file-system/fixtures.ts index a621fd98c..67b0d52eb 100644 --- a/tests/playwright/tests/file-system/fixtures.ts +++ b/tests/playwright/tests/file-system/fixtures.ts @@ -1,6 +1,6 @@ import { test as base, expect, Page } from '@playwright/test'; import { validate as isValidUUID } from 'uuid'; -import { FSEntry } from '../../../../src/backend/src/filesystem/definitions/ts/fsentry'; +import { FSEntry } from '../../../../src/backend/stores/fs/FSEntry'; import { testConfig } from '../../config/test-config'; // The maximum time needed for file-system change to be propagated from diff --git a/tools/extensionSetup.sh b/tools/extensionSetup.sh new file mode 100755 index 000000000..97ec321c5 --- /dev/null +++ b/tools/extensionSetup.sh @@ -0,0 +1,8 @@ +#~!/bin/bash +# iterate through each folder in extensions/ if they contain a package.json, run npm install +for d in ./extensions/*/ ; do + if [ -f "$d/package.json" ]; then + echo "Installing dependencies for $d" + (cd "$d" && npm install) + fi +done \ No newline at end of file diff --git a/tools/keygen/gen-peer-keys.js b/tools/keygen/gen-peer-keys.js deleted file mode 100644 index 670a4bcb3..000000000 --- a/tools/keygen/gen-peer-keys.js +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const nacl = require('tweetnacl'); - -const pair = nacl.box.keyPair(); - -const format_key = key => { - const version = new Uint8Array([0x31]); - const buffer = Buffer.concat([ - Buffer.from(version), - Buffer.from(key), - ]); - return buffer.toString('base64'); -}; - -console.log(JSON.stringify({ - keys: { - public: format_key(pair.publicKey), - secret: format_key(pair.secretKey), - }, -}, undefined, ' ')); diff --git a/tools/keygen/package.json b/tools/keygen/package.json deleted file mode 100644 index d44fccaad..000000000 --- a/tools/keygen/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "keygen", - "version": "1.0.0", - "main": "gen-peer-keys.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "keywords": [], - "author": "", - "license": "AGPL-3.0-only", - "description": "" -} diff --git a/tools/lib/configMigration.mjs b/tools/lib/configMigration.mjs new file mode 100644 index 000000000..9ecf79a8b --- /dev/null +++ b/tools/lib/configMigration.mjs @@ -0,0 +1,479 @@ +// Shared helpers for migrateConfig.mjs / migrateServers.mjs. + +// ── JSON loading ────────────────────────────────────────────────────────── +// Old files may be multiple JSON objects glued together with `//` comments. +// Strip line-comments + trailing commas, then walk brace depth to split. + +const stripLineComments = (src) => src.replace(/^\s*\/\/.*$/gm, ''); +const stripTrailingCommas = (src) => src.replace(/,(\s*[}\]])/g, '$1'); + +const splitJsonDocs = (src) => { + const docs = []; + let depth = 0; + let start = -1; + let inStr = false; + let esc = false; + for ( let i = 0; i < src.length; i++ ) { + const c = src[i]; + if ( inStr ) { + if ( esc ) esc = false; + else if ( c === '\\' ) esc = true; + else if ( c === '"' ) inStr = false; + continue; + } + if ( c === '"' ) { inStr = true; continue; } + if ( c === '{' ) { + if ( depth === 0 ) start = i; + depth++; + } else if ( c === '}' ) { + depth--; + if ( depth === 0 && start !== -1 ) { + docs.push(src.slice(start, i + 1)); + start = -1; + } + } + } + return docs; +}; + +// Strip v1 JSON-extension conventions at every depth: +// • `$`-prefixed keys → v1 directives (`$preserve`, `$requires`, `$version`) +// • `__`-prefixed keys → v1 comment-out convention (disabled entries) +// • empty-string keys → trailing-comma placeholder (`"": null,`) +const stripDollarKeys = (value) => { + if ( Array.isArray(value) ) return value.map(stripDollarKeys); + if ( value && typeof value === 'object' ) { + const out = {}; + for ( const [k, v] of Object.entries(value) ) { + if ( k === '' || k.startsWith('$') || k.startsWith('__') ) continue; + out[k] = stripDollarKeys(v); + } + return out; + } + return value; +}; + +export const loadDocs = (raw) => { + const cleaned = stripTrailingCommas(stripLineComments(raw)); + const texts = splitJsonDocs(cleaned); + return texts.map((text, idx) => { + try { return stripDollarKeys(JSON.parse(text)); } + catch ( e ) { + throw new Error(`Failed to parse JSON document #${idx + 1}: ${e.message}`); + } + }); +}; + +// ── Doc classification ─────────────────────────────────────────────────── + +export const pickServersDoc = (docs) => docs.find(d => Array.isArray(d?.servers)); + +export const pickBaseDoc = (docs) => { + // Prefer prod base (has `services` and no `servers`), then OSS default, + // then any non-servers doc. + const prodBase = docs.find(d => d && !Array.isArray(d.servers) && d.services && d.config_name && d.env !== 'dev'); + if ( prodBase ) return prodBase; + const ossDefault = docs.find(d => d && !Array.isArray(d.servers) && (d.nginx_mode || d.env === 'dev')); + if ( ossDefault ) return ossDefault; + return docs.find(d => d && !Array.isArray(d.servers)) ?? null; +}; + +// ── Deep merge ─────────────────────────────────────────────────────────── +// Plain deep merge: objects recurse, arrays + primitives replace. + +export const deepMerge = (base, override) => { + if ( override === undefined ) return base; + if ( base === undefined ) return override; + if ( base === null || override === null ) return override; + if ( typeof base !== 'object' || typeof override !== 'object' ) return override; + if ( Array.isArray(base) || Array.isArray(override) ) return override; + const out = { ...base }; + for ( const [k, v] of Object.entries(override) ) { + out[k] = deepMerge(base[k], v); + } + return out; +}; + +// ── v1 → v2 transformation ─────────────────────────────────────────────── + +const copyIfSet = (src, sk, dst, dk = sk) => { + if ( src[sk] !== undefined ) dst[dk] = src[sk]; +}; + +export const transformToV2 = (source) => { + const out = {}; + + // Scalar + renamed top-level keys. + copyIfSet(source, 'config_name', out); + copyIfSet(source, 'env', out); + copyIfSet(source, 'server_id', out, 'serverId'); + copyIfSet(source, 'id', out, 'serverId'); + copyIfSet(source, 'region', out); + copyIfSet(source, 'domain', out); + copyIfSet(source, 'protocol', out); + copyIfSet(source, 'pub_port', out); + copyIfSet(source, 'cookie_name', out); + copyIfSet(source, 'jwt_secret', out); + copyIfSet(source, 'url_signature_secret', out); + copyIfSet(source, 'blocked_email_domains', out, 'blockedEmailDomains'); + copyIfSet(source, 'enable_public_folders', out); + copyIfSet(source, 'is_storage_limited', out); + copyIfSet(source, 'storage_capacity', out); + copyIfSet(source, 'static_hosting_domain', out); + copyIfSet(source, 'static_hosting_domain_alt', out); + copyIfSet(source, 'private_app_hosting_domain', out); + copyIfSet(source, 'private_app_hosting_domain_alt', out); + copyIfSet(source, 'min_pass_length', out); + copyIfSet(source, 'allow_system_login', out); + copyIfSet(source, 'allow_all_host_values', out); + copyIfSet(source, 'allow_no_host_header', out); + copyIfSet(source, 'allow_nipio_domains', out); + copyIfSet(source, 'custom_domains_enabled', out); + copyIfSet(source, 'enable_ip_validation', out); + copyIfSet(source, 'default_user_group', out); + copyIfSet(source, 'default_temp_group', out); + copyIfSet(source, 'api_base_url', out); + copyIfSet(source, 'origin', out); + copyIfSet(source, 'contact_email', out, 'support_email'); + + // `extensions` in v1 was overloaded — array form = scan dirs, object form + // = per-extension config bag. JSON.parse keeps only the last declaration + // per key, so in prod files where both appear we typically only see the + // object. Promote object-shape entries onto top-level keys (scoped npm + // names → camelCase: `@heyputer/app-store-and-purchases` → `appStoreAndPurchases`). + // v1 also had `mod_directories: string[]` with `{repo}/...` placeholders. + // v2 uses `extensions: string[]` of plain directory paths; post-cutover the + // only dir that survives is the repo-root `./extensions`, so synthesize that + // when only the config-bag (object) or mod_directories form is present. + // Some v1 configs used `extension` (singular) as the per-extension config + // bag alongside (or instead of) `extensions`. Accept either. + const extBag = (source.extensions && typeof source.extensions === 'object' && !Array.isArray(source.extensions)) + ? source.extensions + : (source.extension && typeof source.extension === 'object') + ? source.extension + : null; + if ( Array.isArray(source.extensions) ) { + out.extensions = source.extensions; + } else if ( extBag ) { + for ( const [k, v] of Object.entries(extBag) ) { + const bare = k.split('/').pop() ?? k; + const camel = bare.replace(/-([a-z])/g, (_, ch) => ch.toUpperCase()); + if ( out[camel] === undefined ) out[camel] = v; + } + } + if ( out.extensions === undefined && ( + Array.isArray(source.mod_directories) || extBag + ) ) { + out.extensions = ['./extensions']; + } + + // Port: http_port → port. Drop string "auto" (v2 requires numeric). + if ( source.http_port !== undefined && source.http_port !== 'auto' ) { + out.port = source.http_port; + } else if ( source.port !== undefined ) { + out.port = source.port; + } + + // S3: old flat keys → `s3.s3Config`. + if ( source.s3_access_key || source.s3_secret_key ) { + out.s3 = { + s3Config: { + endpoint: source.s3_endpoint ?? '', + accessKeyId: source.s3_access_key, + secretAccessKey: source.s3_secret_key, + ...(source.s3_region ? { region: source.s3_region } : {}), + }, + }; + } + copyIfSet(source, 's3_bucket', out); + copyIfSet(source, 's3_region', out); + + // Database: prefer services.database.{primary, engine}; else db_* flat. + const svc = source.services ?? {}; + if ( svc.database ) { + const db = {}; + if ( svc.database.engine ) db.engine = svc.database.engine; + if ( svc.database.primary ) { + for ( const k of ['host', 'port', 'user', 'password', 'database'] ) { + if ( svc.database.primary[k] !== undefined ) db[k] = svc.database.primary[k]; + } + } + if ( svc.database.path ) db.path = svc.database.path; + if ( Object.keys(db).length ) out.database = db; + } + if ( ! out.database && source.db_host ) { + out.database = { + engine: 'mysql', + host: source.db_host, + port: source.db_port, + user: source.db_user, + password: source.db_password, + database: source.db_database, + }; + } + if ( source.read_replica_db ) { + out.database = out.database ?? { engine: 'mysql' }; + const r = source.read_replica_db; + out.database.replica = { + host: r.host, port: r.port, user: r.user, password: r.password, database: r.database, + }; + } + + // Dynamo (services.dynamo → top-level) + if ( svc.dynamo ) out.dynamo = svc.dynamo; + + // Email (services.email → email; drop `engine` adapter switch). Fallback + // to old flat smtp_* fields. + if ( svc.email ) { + const { engine: _engine, ...rest } = svc.email; + out.email = rest; + } else if ( source.smtp_server || source.smtp_host ) { + out.email = { + host: source.smtp_server ?? source.smtp_host, + port: source.smtp_port ?? source.smpt_port, + secure: true, + auth: { user: source.smtp_username, pass: source.smtp_password }, + }; + } + + // Pager: routing_key → routingKey. + if ( source.pager?.pagerduty ) { + const pd = source.pager.pagerduty; + out.pager = { + pagerduty: { + enabled: pd.enabled, + ...(pd.routing_key ? { routingKey: pd.routing_key } : {}), + }, + }; + } + + // Captcha + if ( svc.captcha ) out.captcha = svc.captcha; + + // Homepage GUI bundle promotion (services.puter-homepage.* → top-level). + if ( svc['puter-homepage'] ) { + const h = svc['puter-homepage']; + copyIfSet(h, 'gui_bundle', out); + copyIfSet(h, 'gui_puterjs_bundle', out); + copyIfSet(h, 'gui_css', out); + } + + // Legacy billing consolidation (stripe/offerings/__subs-serve → legacyBilling). + const legacyBilling = {}; + if ( svc.stripe ) { + if ( svc.stripe.api_secret ) legacyBilling.api_secret = svc.stripe.api_secret; + if ( svc.stripe.endpoint_secret ) legacyBilling.endpoint_secret = svc.stripe.endpoint_secret; + } + if ( svc['__subs-serve']?.stripe_publishable_key ) { + legacyBilling.stripe_publishable_key = svc['__subs-serve'].stripe_publishable_key; + } + if ( svc.offerings?.price_ids ) legacyBilling.price_ids = svc.offerings.price_ids; + if ( Object.keys(legacyBilling).length ) out.legacyBilling = legacyBilling; + + // Abuse / clickhouse / cf_file_cache pass through if already top-level. + if ( source.abuse ) out.abuse = source.abuse; + if ( source.clickhouse ) out.clickhouse = source.clickhouse; + if ( source.cf_file_cache ) out.cf_file_cache = source.cf_file_cache; + + // Redis: v1 shape was `redis.config: [{host,port},…]`; v2 IRedisConfig + // expects `redis.startupNodes: [{host,port},…]`. + if ( source.redis && typeof source.redis === 'object' ) { + const { config: nodes, ...rest } = source.redis; + out.redis = { ...rest }; + if ( Array.isArray(nodes) ) out.redis.startupNodes = nodes; + else if ( Array.isArray(source.redis.startupNodes) ) out.redis.startupNodes = source.redis.startupNodes; + } + + // v1 services that became top-level IConfig entries (some with renames). + if ( svc.oidc ) out.oidc = svc.oidc; + if ( svc.wisp ) out.wisp = svc.wisp; + if ( svc.peer ) out.peers = svc.peer; + if ( svc.broadcast ) out.broadcast = svc.broadcast; + if ( svc['worker-service'] ) out.workers = svc['worker-service']; + if ( svc['entri-service'] ) out.entri = svc['entri-service']; + // v1's services.thumbnails wrapped the bucket config in `.bucket` and also + // carried an unrelated `engine`/`host` pointer to the thumbnail HTTP + // service. v2's IThumbnailStoreConfig is strictly the bucket — unwrap. + if ( svc.thumbnails?.bucket ) out.thumbnailStore = svc.thumbnails.bucket; + + // v2 onlyoffice extension reads `config.onlyoffice` (same field names as + // v1's `services.onlyoffice-app`), so it's a straight rename. + if ( svc['onlyoffice-app'] ) out.onlyoffice = svc['onlyoffice-app']; + + // Cloudflare Turnstile: v2 GUI renders the challenge widget when + // `gui_params.turnstileSiteKey` is set (see initgui.js / UIWindowSignup). + // Preserve the full block at `turnstile` for whenever the backend verifier + // is ported, and surface the site key into gui_params so the widget works. + if ( svc['cloudflare-turnstile'] ) { + out.turnstile = svc['cloudflare-turnstile']; + if ( svc['cloudflare-turnstile'].site_key ) { + out.gui_params = out.gui_params ?? {}; + out.gui_params.turnstileSiteKey = svc['cloudflare-turnstile'].site_key; + } + } + + // AI / integration providers: v1 kept each under `services.` (plus a + // top-level `openai` shortcut in some configs). v2 unifies them under + // `providers[]` and accepts only the canonical camelCase field names + // on IAIProviderConfig, so we rename the common snake_case aliases here. + const PROVIDER_IDS = [ + 'openai', 'claude', 'gemini', 'mistral', 'groq', 'deepseek', + 'xai', 'openrouter', 'together-ai', 'ollama', + 'elevenlabs', 'aws-polly', 'aws-textract', 'mistral-ocr', 'cloudflare', + 'openai-completion', 'openai-responses', + 'openai-image-generation', 'openai-video-generation', + 'gemini-image-generation', 'gemini-video-generation', + 'together-image-generation', 'together-video-generation', + 'cloudflare-image-generation', 'xai-image-generation', + 'replicate-image-generation', + ]; + const PROVIDER_RENAMES = [ + ['api_key', 'apiKey'], ['secret_key', 'apiKey'], ['key', 'apiKey'], + ['api_token', 'apiToken'], + ['api_base_url', 'apiBaseUrl'], + ['account_id', 'accountId'], + ['default_voice_id', 'defaultVoiceId'], + ['speech_to_speech_model_id', 'speechToSpeechModelId'], + ]; + const normalizeProvider = (raw) => { + if ( ! raw || typeof raw !== 'object' ) return raw; + const p = { ...raw }; + for ( const [from, to] of PROVIDER_RENAMES ) { + if ( p[from] !== undefined && p[to] === undefined ) p[to] = p[from]; + delete p[from]; + } + return p; + }; + const providers = {}; + for ( const id of PROVIDER_IDS ) { + if ( svc[id] ) providers[id] = normalizeProvider(svc[id]); + } + // v1 AWS aliases: some configs shortened `aws-polly` → `polly`, + // `aws-textract` → `textract`. v2 provider ids keep the prefix. + if ( svc.polly && providers['aws-polly'] === undefined ) { + providers['aws-polly'] = normalizeProvider(svc.polly); + } + if ( svc.textract && providers['aws-textract'] === undefined ) { + providers['aws-textract'] = normalizeProvider(svc.textract); + } + if ( source.openai && providers.openai === undefined ) { + providers.openai = normalizeProvider(source.openai); + } + // v1 had a single `services.replicate` that the image driver keyed on; + // v2 splits providers by capability, so the image one lands at + // `providers['replicate-image-generation']`. + if ( svc.replicate && providers['replicate-image-generation'] === undefined ) { + providers['replicate-image-generation'] = normalizeProvider(svc.replicate); + } + // Backward-compat fan-out: v1 had a single `openai` / `gemini` / `together-ai` + // / `xai` entry used for chat + image + video. v2 drivers look up split ids + // (e.g. `openai-completion`, `openai-image-generation`), so seed each + // split id from the base id when the split key isn't already set. + const FAN_OUT = { + openai: ['openai-completion', 'openai-responses', 'openai-image-generation', 'openai-video-generation'], + gemini: ['gemini-image-generation', 'gemini-video-generation'], + 'together-ai': ['together-image-generation', 'together-video-generation'], + xai: ['xai-image-generation'], + }; + for ( const [base, splits] of Object.entries(FAN_OUT) ) { + if ( ! providers[base] ) continue; + for ( const split of splits ) { + if ( providers[split] === undefined ) providers[split] = providers[base]; + } + } + if ( Object.keys(providers).length ) out.providers = providers; + + // Anything left in `services` that we didn't claim above is promoted to + // top-level (v2's IConfig has no `services` bag). Known consumers that + // live outside `services` in v2 are listed in `consumedServiceKeys` so + // we don't double-emit; known-dead v1 services are listed in + // `droppedServiceKeys` so their data is intentionally discarded. + const consumedServiceKeys = new Set([ + 'database', 'dynamo', 'email', 'captcha', 'puter-homepage', + 'stripe', 'offerings', '__subs-serve', + 'oidc', 'wisp', 'peer', 'broadcast', 'worker-service', 'entri-service', + 'thumbnails', 'onlyoffice-app', 'cloudflare-turnstile', 'replicate', + 'polly', 'textract', + ...PROVIDER_IDS, + ]); + const droppedServiceKeys = new Set([ + // v1 services with no v2 equivalent — data intentionally discarded. + 'heap-monitor', 'file-cache', 'telemetry', 'monitor', 'spending', + 'judge0', 'convert-api', + // `auth.uuid_fpe_key` — v2 AuthService uses plain session UUIDs + // (see services/auth/types.ts: "not FPE-encrypted"). + 'auth', + // SNS bounce handler not ported to v2 — no v2 SNSService. + 'sns', + // v1 config-only orphans: never actually read by v1 backend + // (confirmed via git grep on puter@main), no v2 consumers. + 'ipgeo', 'newsdata', 'weather', 'user-send-mail', + // `ai-chat.concurrentRequests` — v1 concurrency limiter not ported + // yet; see TODO in v2 drivers/ai-chat/ChatCompletionDriver.ts. Config + // shape should move under `rate_limit.*` in IConfig when reintroduced, + // so dropping avoids migrating a soon-to-be-renamed shape. + 'ai-chat', + // v1-only services with no v2 analogue. + 'web-server', // `disable_ip_validate_event` — flag gone + 'puter-kvstore', // v2 uses `dynamo` for system KV + ]); + for ( const [k, v] of Object.entries(svc) ) { + if ( consumedServiceKeys.has(k) || droppedServiceKeys.has(k) ) continue; + if ( out[k] === undefined ) out[k] = v; + } + + // Top-level v1 keys that v2 OSS doesn't read and no extension claims. + // Silently dropped — everything else falls through to the preservation + // loop below in case it belongs to a prod extension we're not aware of. + const droppedTopKeys = new Set([ + // `puter_hosted_data.puter_versions` — set in v1 config.js but never + // read. Planted for a version-check feature that never shipped. + 'puter_hosted_data', + // v1 dev-only toggles with no v2 equivalent. Dev vs prod behaviour + // in v2 branches off `env === 'dev'` and `config.abuse.enabled`. + 'disable_abuse_checks', + 'undefined_origin_allowed', + ]); + for ( const k of droppedTopKeys ) delete out[k]; + + // Preserve any other top-level keys we haven't explicitly translated + // (custom extension configs etc.). + const handledTop = new Set([ + 'config_name', 'env', 'http_port', 'port', 'pub_port', 'domain', 'protocol', + 'blocked_email_domains', 'toConsole', 'is_storage_limited', + 'legacy_token_migrate', 'forwarded', 'cross_origin_isolation', + 'enable_public_folders', 'cookie_name', 'jwt_secret', 'url_signature_secret', + 'extensions', 'mod_directories', + 'db_host', 'db_port', 'db_user', 'db_password', 'db_database', + 'db_waitForConnections', 'db_connectionLimit', 'db_enableKeepAlive', + 'db_queueLimit', 'db_read_replica_wait', 'read_replica_db', + 's3_access_key', 's3_secret_key', 's3_bucket', 's3_region', 's3_endpoint', + 'mailchimp', 'cloudwatch', 'monitor', + 'smtp_server', 'smtp_host', 'smtp_port', 'smpt_port', 'smtp_username', 'smtp_password', + 'max_subdomains_per_user', + 'storage_capacity', + 'static_hosting_domain', 'static_hosting_domain_alt', + 'private_app_hosting_domain', 'private_app_hosting_domain_alt', + 'openai', + 'pager', + 'defaultjs_asset_path', + 'services', + 'server_id', 'id', 'region', 'host', + 'nginx_mode', 'contact_email', + 'api_base_url', 'origin', + 'min_pass_length', 'allow_system_login', 'allow_all_host_values', + 'allow_no_host_header', 'allow_nipio_domains', 'custom_domains_enabled', + 'enable_ip_validation', + 'default_user_group', 'default_temp_group', + 'abuse', 'clickhouse', 'cf_file_cache', 'legacyBilling', + 'providers', 'thumbnailStore', + 'redis', 'extension', + ...droppedTopKeys, + ]); + for ( const [k, v] of Object.entries(source) ) { + if ( handledTop.has(k) || k === '' ) continue; + out[k] = v; + } + + return out; +}; diff --git a/tools/license-headers/main.js b/tools/license-headers/main.js deleted file mode 100644 index 7879624d9..000000000 --- a/tools/license-headers/main.js +++ /dev/null @@ -1,639 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const levenshtein = require('js-levenshtein'); -const DiffMatchPatch = require('diff-match-patch'); -const enq = require('enquirer'); -const dmp = new DiffMatchPatch(); -const dedent = require('dedent'); - -const { walk, EXCLUDE_LISTS } = require('file-walker'); -const { CommentParser } = require('../comment-parser/main'); - -const fs = require('fs'); -const path_ = require('path'); - - -/** -* Compares two license headers and returns their Levenshtein distance and formatted diff -* @param {Object} params - The parameters object -* @param {string} params.header1 - First header text to compare -* @param {string} params.header2 - Second header text to compare -* @param {boolean} [params.distance_only=false] - If true, only return distance without diff -* @returns {Object} Object containing distance and formatted terminal diff -*/ -const CompareFn = ({ header1, header2, distance_only = false }) => { - - // Calculate Levenshtein distance - const distance = levenshtein(header1, header2); - // console.log(`Levenshtein distance: ${distance}`); - - if ( distance_only ) return { distance }; - - // Generate diffs using diff-match-patch - const diffs = dmp.diff_main(header1, header2); - dmp.diff_cleanupSemantic(diffs); - - let term_diff = ''; - - // Manually format diffs for terminal display - diffs.forEach(([type, text]) => { - switch (type) { - case DiffMatchPatch.DIFF_INSERT: - term_diff += `\x1b[32m${text}\x1b[0m`; // Green for insertions - break; - case DiffMatchPatch.DIFF_DELETE: - term_diff += `\x1b[31m${text}\x1b[0m`; // Red for deletions - break; - case DiffMatchPatch.DIFF_EQUAL: - term_diff += text; // No color for equal parts - break; - } - }); - - return { - distance, - term_diff, - }; -} - -/** -* Creates a license checker instance that can compare and validate license headers -* @param {Object} params - Configuration parameters -* @param {Object} params.comment_parser - Comment parser instance to use -* @param {string} params.desired_header - The expected license header text -* @returns {Object} License checker instance with compare and supports methods -*/ -const LicenseChecker = ({ - comment_parser, - desired_header, -}) => { - const supports = ({ filename }) => { - return comment_parser.supports({ filename }); - }; - const compare = async ({ filename, source }) => { - const headers = await comment_parser.extract_top_comments( - { filename, source }); - const headers_lines = headers.map(h => h.lines); - - if ( headers.length < 1 ) { - return { - has_header: false, - }; - } - - // console.log('headers', headers); - - let top = 0; - let bottom = 0; - let current_distance = Number.MAX_SAFE_INTEGER; - - // "wah" - for ( let i=1 ; i <= headers.length ; i++ ) { - const combined = headers_lines.slice(top, i).flat(); - const combined_txt = combined.join('\n'); - const { distance } = - CompareFn({ - header1: desired_header, - header2: combined_txt, - distance_only: true, - }); - if ( distance < current_distance ) { - current_distance = distance; - bottom = i; - } else { - break; - } - } - // "woop" - for ( let i=1 ; i < headers.length ; i++ ) { - const combined = headers_lines.slice(i, bottom).flat(); - const combined_txt = combined.join('\n'); - const { distance } = - CompareFn({ - header1: desired_header, - header2: combined_txt, - distance_only: true, - }); - if ( distance < current_distance ) { - current_distance = distance; - top = i; - } else { - break; - } - } - - // console.log('headers', headers); - - const combined = headers_lines.slice(top, bottom).flat(); - const combined_txt = combined.join('\n'); - - const diff_info = CompareFn({ - header1: desired_header, - header2: combined_txt, - }) - - if ( diff_info.distance > 0.7*desired_header.length ) { - return { - has_header: false, - }; - } - - diff_info.range = [ - headers[top].range[0], - headers[bottom-1].range[1], - ]; - - diff_info.has_header = true; - - return diff_info; - }; - return { - compare, - supports, - }; -}; - -const license_check_test = async ({ options }) => { - const comment_parser = CommentParser(); - const license_checker = LicenseChecker({ - comment_parser, - desired_header: fs.readFileSync( - path_.join(__dirname, '../../doc/license_header.txt'), - 'utf-8', - ), - }); - - const walk_iterator = walk({ - excludes: EXCLUDE_LISTS.NOT_AGPL, - }, path_.join(__dirname, '../..')); - for await ( const value of walk_iterator ) { - if ( value.is_dir ) continue; - if ( options?.filename && value.name !== options.filename ) continue; - console.log(value.path); - const source = fs.readFileSync(value.path, 'utf-8'); - const diff_info = await license_checker.compare({ - filename: value.name, - source, - }) - if ( diff_info ) { - process.stdout.write('\x1B[36;1m=======\x1B[0m\n'); - process.stdout.write(diff_info.term_diff); - process.stdout.write('\n\x1B[36;1m=======\x1B[0m\n'); - // console.log('headers', headers); - } else { - console.log('NO COMMENT'); - } - - console.log('RANGE', diff_info.range) - - const new_comment = comment_parser.output_comment({ - filename: value.name, - style: 'block', - text: 'some text\nto display' - }); - - console.log('NEW COMMENT?', new_comment); - } -}; - - -/** -* Executes the main command line interface for the license header tool. -* Sets up Commander.js program with commands for checking and syncing license headers. -* Handles configuration file loading and command execution. -* -* @async -* @returns {Promise} Resolves when command execution is complete -*/ -const cmd_check_fn = async () => { - const comment_parser = CommentParser(); - const license_checker = LicenseChecker({ - comment_parser, - desired_header: fs.readFileSync( - path_.join(__dirname, '../../doc/license_header.txt'), - 'utf-8', - ), - }); - - const counts = { - ok: 0, - missing: 0, - conflict: 0, - error: 0, - unsupported: 0, - }; - - const walk_iterator = walk({ - excludes: EXCLUDE_LISTS.NOT_AGPL, - }, path_.join(__dirname, '../..')); - for await ( const value of walk_iterator ) { - if ( value.is_dir ) continue; - - process.stdout.write(value.path + ' ... '); - - if ( ! license_checker.supports({ filename: value.name }) ) { - process.stdout.write(`\x1B[37;1mUNSUPPORTED\x1B[0m\n`); - counts.unsupported++; - continue; - } - - const source = fs.readFileSync(value.path, 'utf-8'); - const diff_info = await license_checker.compare({ - filename: value.name, - source, - }) - if ( ! diff_info ) { - counts.error++; - continue; - } - if ( ! diff_info.has_header ) { - counts.missing++; - process.stdout.write(`\x1B[33;1mMISSING\x1B[0m\n`); - continue; - } - if ( diff_info ) { - if ( diff_info.distance !== 0 ) { - counts.conflict++; - process.stdout.write(`\x1B[31;1mCONFLICT\x1B[0m\n`); - } else { - counts.ok++; - process.stdout.write(`\x1B[32;1mOK\x1B[0m\n`); - } - } else { - console.log('NO COMMENT'); - } - } - - const { Table } = require('console-table-printer'); - const t = new Table({ - columns: [ - { - title: 'License Header', - name: 'situation', alignment: 'left', color: 'white_bold' }, - { - title: 'Number of Files', - name: 'count', alignment: 'right' }, - ], - colorMap: { - green: '\x1B[32;1m', - yellow: '\x1B[33;1m', - red: '\x1B[31;1m', - } - }); - - console.log(''); - - if ( counts.error > 0 ) { - console.log(`\x1B[31;1mTHERE WERE SOME ERRORS!\x1B[0m`); - console.log('check the log above for the stack trace'); - console.log(''); - t.addRow({ situation: 'error', count: counts.error }, - { color: 'red' }); - } - - console.log(dedent(` - \x1B[31;1mAny text below is mostly lies!\x1B[0m - This tool is still being developed and most of what's - described is "the plan" rather than a thing that will - actually happen. - \x1B[31;1m^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\x1B[0m - `)); - - if ( counts.conflict ) { - console.log(dedent(` - \x1B[37;1mIt looks like you have some conflicts!\x1B[0m - Run the following command to update license headers: - - \x1B[36;1maddlicense sync\x1B[0m - - This will begin an interactive license update. - Any time the license doesn't quite match you will - be given the option to replace it or skip the file. - \x1B[90mSee \`addlicense help sync\` for other options.\x1B[0m - - You will also be able to choose - "remember for headers matching this one" - if you know the same issue will come up later. - `)); - } else if ( counts.missing ) { - console.log(dedent(` - \x1B[37;1mSome missing license headers!\x1B[0m - Run the following command to add the missing license headers: - - \x1B[36;1maddlicense sync\x1B[0m - `)); - } else { - console.log(dedent(` - \x1B[37;1mNo action to perform!\x1B[0m - Run the following command to do absolutely nothing: - - \x1B[36;1maddlicense sync\x1B[0m - `)); - } - - console.log(''); - - t.addRow({ situation: 'ok', count: counts.ok }, - { color: 'green' }); - t.addRow({ situation: 'missing', count: counts.missing }, - { color: 'yellow' }); - t.addRow({ situation: 'conflict', count: counts.conflict }, - { color: 'red' }); - t.addRow({ situation: 'unsupported', count: counts.unsupported }); - t.printTable(); -}; - - -/** -* Synchronizes license headers in source files by adding missing headers and handling conflicts -* -* Walks through files, checks for license headers, and: -* - Adds headers to files missing them -* - Prompts user to resolve conflicts when headers don't match -* - Handles duplicate headers by allowing removal -* - Tracks counts of different header statuses (ok, missing, conflict, etc) -* -* @returns {Promise} Resolves when synchronization is complete -*/ -const cmd_sync_fn = async () => { - const comment_parser = CommentParser(); - const desired_header = fs.readFileSync( - path_.join(__dirname, '../../doc/license_header.txt'), - 'utf-8', - ); - const license_checker = LicenseChecker({ - comment_parser, - desired_header, - }); - - const counts = { - ok: 0, - missing: 0, - conflict: 0, - error: 0, - unsupported: 0, - }; - - const walk_iterator = walk({ - excludes: EXCLUDE_LISTS.NOT_AGPL, - }, '.'); - for await ( const value of walk_iterator ) { - if ( value.is_dir ) continue; - - process.stdout.write(value.path + ' ... '); - - if ( ! license_checker.supports({ filename: value.name }) ) { - process.stdout.write(`\x1B[37;1mUNSUPPORTED\x1B[0m\n`); - counts.unsupported++; - continue; - } - - const source = fs.readFileSync(value.path, 'utf-8'); - const diff_info = await license_checker.compare({ - filename: value.name, - source, - }) - if ( ! diff_info ) { - counts.error++; - continue; - } - if ( ! diff_info.has_header ) { - fs.writeFileSync( - value.path, - comment_parser.output_comment({ - style: 'block', - filename: value.name, - text: desired_header, - }) + - '\n' + - source - ); - continue; - } - if ( diff_info ) { - if ( diff_info.distance !== 0 ) { - counts.conflict++; - process.stdout.write(`\x1B[31;1mCONFLICT\x1B[0m\n`); - process.stdout.write('\x1B[36;1m=======\x1B[0m\n'); - process.stdout.write(diff_info.term_diff); - process.stdout.write('\n\x1B[36;1m=======\x1B[0m\n'); - const prompt = new enq.Select({ - message: 'Select Action', - choices: [ - { name: 'skip', message: 'Skip' }, - { name: 'replace', message: 'Replace' }, - ] - }) - const action = await prompt.run(); - if ( action === 'skip' ) continue; - const before = source.slice(0, diff_info.range[0]); - const after = source.slice(diff_info.range[1]); - const new_source = before + - comment_parser.output_comment({ - style: 'block', - filename: value.name, - text: desired_header, - }) + - after; - fs.writeFileSync(value.path, new_source); - } else { - let cut_diff_info = diff_info; - let cut_source = source; - const cut_header = async () => { - cut_source = cut_source.slice(cut_diff_info.range[1]); - cut_diff_info = await license_checker.compare({ - filename: value.name, - source: cut_source, - }); - }; - await cut_header(); - const cut_range = [ - diff_info.range[1], - diff_info.range[1], - ]; - const cut_diff_infos = []; - while ( cut_diff_info.has_header ) { - cut_diff_infos.push(cut_diff_info); - cut_range[1] += cut_diff_info.range[1]; - await cut_header(); - } - if ( cut_range[0] !== cut_range[1] ) { - process.stdout.write(`\x1B[31;1mDUPLICATE\x1B[0m\n`); - process.stdout.write('\x1B[36;1m==== KEEP ====\x1B[0m\n'); - process.stdout.write(diff_info.term_diff + '\n'); - process.stdout.write('\x1B[36;1m==== REMOVE ====\x1B[0m\n'); - for ( const diff_info of cut_diff_infos ) { - process.stdout.write(diff_info.term_diff); - } - process.stdout.write('\n\x1B[36;1m=======\x1B[0m\n'); - const prompt = new enq.Select({ - message: 'Select Action', - choices: [ - { name: 'skip', message: 'Skip' }, - { name: 'remove', message: 'Remove' }, - ] - }) - const action = await prompt.run(); - if ( action === 'skip' ) continue; - const new_source = - source.slice(0, cut_range[0]) + - source.slice(cut_range[1]); - fs.writeFileSync(value.path, new_source); - } - counts.ok++; - process.stdout.write(`\x1B[32;1mOK\x1B[0m\n`); - } - } else { - console.log('NO COMMENT'); - } - } - - const { Table } = require('console-table-printer'); - const t = new Table({ - columns: [ - { - title: 'License Header', - name: 'situation', alignment: 'left', color: 'white_bold' }, - { - title: 'Number of Files', - name: 'count', alignment: 'right' }, - ], - colorMap: { - green: '\x1B[32;1m', - yellow: '\x1B[33;1m', - red: '\x1B[31;1m', - } - }); - - console.log(''); - - if ( counts.error > 0 ) { - console.log(`\x1B[31;1mTHERE WERE SOME ERRORS!\x1B[0m`); - console.log('check the log above for the stack trace'); - console.log(''); - t.addRow({ situation: 'error', count: counts.error }, - { color: 'red' }); - } - - console.log(dedent(` - \x1B[31;1mAny text below is mostly lies!\x1B[0m - This tool is still being developed and most of what's - described is "the plan" rather than a thing that will - actually happen. - \x1B[31;1m^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\x1B[0m - `)); - - if ( counts.conflict ) { - console.log(dedent(` - \x1B[37;1mIt looks like you have some conflicts!\x1B[0m - Run the following command to update license headers: - - \x1B[36;1maddlicense sync\x1B[0m - - This will begin an interactive license update. - Any time the license doesn't quite match you will - be given the option to replace it or skip the file. - \x1B[90mSee \`addlicense help sync\` for other options.\x1B[0m - - You will also be able to choose - "remember for headers matching this one" - if you know the same issue will come up later. - `)); - } else if ( counts.missing ) { - console.log(dedent(` - \x1B[37;1mSome missing license headers!\x1B[0m - Run the following command to add the missing license headers: - - \x1B[36;1maddlicense sync\x1B[0m - `)); - } else { - console.log(dedent(` - \x1B[37;1mNo action to perform!\x1B[0m - Run the following command to do absolutely nothing: - - \x1B[36;1maddlicense sync\x1B[0m - `)); - } - - console.log(''); - - t.addRow({ situation: 'ok', count: counts.ok }, - { color: 'green' }); - t.addRow({ situation: 'missing', count: counts.missing }, - { color: 'yellow' }); - t.addRow({ situation: 'conflict', count: counts.conflict }, - { color: 'red' }); - t.addRow({ situation: 'unsupported', count: counts.unsupported }); - t.printTable(); -}; - - -/** -* Main entry point for the license header tool. -* Sets up command line interface using Commander and processes commands. -* Handles 'check' and 'sync' commands for managing license headers in files. -* -* @returns {Promise} Resolves when command processing is complete -*/ -const main = async () => { - const { program } = require('commander'); - const helptext = dedent(` - Usage: usage text - `); - - const run_command = async ({ cmd, cmd_fn }) => { - const options = { - program: program.opts(), - command: cmd.opts(), - }; - console.log('options', options); - - if ( ! fs.existsSync(options.program.config) ) { - // TODO: configuration wizard - fs.writeFileSync(options.program.config, ''); - } - - await cmd_fn({ options }); - }; - - program - .name('addlicense') - .option('-c, --config', 'configuration file', 'addlicense.yml') - .addHelpText('before', helptext) - ; - const cmd_check = program.command('check') - .description('check license headers') - .option('-n, --non-interactive', 'disable prompting') - .action(() => { - run_command({ cmd: cmd_check, cmd_fn: cmd_check_fn }); - }) - const cmd_sync = program.command('sync') - .description('synchronize files with license header rules') - .option('-n, --non-interactive', 'disable prompting') - .action(() => { - run_command({ cmd: cmd_sync, cmd_fn: cmd_sync_fn }) - }) - program.parse(process.argv); - -}; - -if ( require.main === module ) { - main(); -} diff --git a/tools/license-headers/package.json b/tools/license-headers/package.json deleted file mode 100644 index 0bf0b49e3..000000000 --- a/tools/license-headers/package.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "license-headers", - "version": "1.0.0", - "main": "main.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "keywords": [], - "author": "", - "license": "AGPL-3.0-only", - "description": "", - "dependencies": { - "console-table-printer": "^2.12.1", - "dedent": "^1.5.3", - "diff-match-patch": "^1.0.5", - "enquirer": "^2.4.1", - "js-levenshtein": "^1.1.6", - "yaml": "^2.4.5" - } -} diff --git a/tools/migrateConfig.mjs b/tools/migrateConfig.mjs new file mode 100644 index 000000000..c2555cfd2 --- /dev/null +++ b/tools/migrateConfig.mjs @@ -0,0 +1,52 @@ +#!/usr/bin/env node +/* + * Migrate a v1 kernel config into a v2 config.json. + * + * Usage: + * node tools/migrateConfig.mjs [--input ] [--output ] + * + * Defaults: + * --input /volatile/config/config.json + * --output /packages/puter/config.json + * + * The input may be either a single JSON document or several docs + * concatenated (the old prod layout: base + servers + oss-default). + * In the multi-doc case the "base" doc is chosen — servers overrides + * are handled by migrateServers.mjs. + */ +import { readFileSync, writeFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; + +import { loadDocs, pickBaseDoc, transformToV2 } from './lib/configMigration.mjs'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +const DEFAULT_INPUT = '/volatile/config/config.json'; +const DEFAULT_OUTPUT = resolve(__dirname, '../config.json'); + +const { values: args } = parseArgs({ + options: { + input: { type: 'string', short: 'i', default: DEFAULT_INPUT }, + output: { type: 'string', short: 'o', default: DEFAULT_OUTPUT }, + }, +}); + +const raw = readFileSync(args.input, 'utf8'); +const docs = loadDocs(raw); +if ( docs.length === 0 ) { + console.error('No JSON documents found in input.'); + process.exit(1); +} + +const base = pickBaseDoc(docs); +if ( ! base ) { + console.error('Could not identify a base config document in input.'); + process.exit(1); +} + +const migrated = transformToV2(base); +writeFileSync(args.output, JSON.stringify(migrated, null, 2) + '\n'); +console.log(`Migrated base config → ${args.output}`); diff --git a/tools/migrateServers.mjs b/tools/migrateServers.mjs new file mode 100644 index 000000000..89ad0a725 --- /dev/null +++ b/tools/migrateServers.mjs @@ -0,0 +1,65 @@ +#!/usr/bin/env node +/* + * Migrate v1 prod per-server overrides into a v2 servers.json. + * + * Usage: + * node tools/migrateServers.mjs [--input ] [--output ] + * + * Defaults: + * --input /volatile/config/config.json + * --output /packages/puter/servers.json + * + * Input file must contain a `{ servers: [...] }` doc (either on its own or + * concatenated with the base; the base is ignored — only per-server kernel + * overrides are transformed). + * + * Output: an array of per-server *deltas* in v2 shape. The runtime deep-merges + * the matching entry onto `config.json` at boot — so this file holds only the + * values that differ between nodes (s3 bucket, region, replica DB, broadcast + * peer list, etc). + */ +import { readFileSync, writeFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; + +import { loadDocs, pickServersDoc, transformToV2 } from './lib/configMigration.mjs'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +const DEFAULT_INPUT = '/volatile/config/config.json'; +const DEFAULT_OUTPUT = resolve(__dirname, '../servers.json'); + +const { values: args } = parseArgs({ + options: { + input: { type: 'string', short: 'i', default: DEFAULT_INPUT }, + output: { type: 'string', short: 'o', default: DEFAULT_OUTPUT }, + }, +}); + +const raw = readFileSync(args.input, 'utf8'); +const docs = loadDocs(raw); + +const serversDoc = pickServersDoc(docs); +if ( ! serversDoc ) { + console.error('No `{ servers: [...] }` doc found in input.'); + process.exit(1); +} + +const migrated = serversDoc.servers.map(server => { + const { kernel, ...serverMeta } = server; + // Start from just the kernel override so the output is a *delta*, not a + // full merged config. Server-level metadata (id, region) gets hoisted in + // first so transformToV2 maps `id` → `serverId` and keeps `region`. + const src = { ...(kernel ?? {}) }; + for ( const k of ['id', 'region'] ) { + if ( serverMeta[k] !== undefined && src[k] === undefined ) { + src[k] = serverMeta[k]; + } + } + return transformToV2(src); +}); + +writeFileSync(args.output, JSON.stringify(migrated, null, 2) + '\n'); +console.log(`Migrated ${migrated.length} server config(s) → ${args.output}`); diff --git a/tools/module-docgen/defs.js b/tools/module-docgen/defs.js deleted file mode 100644 index 90e2c8819..000000000 --- a/tools/module-docgen/defs.js +++ /dev/null @@ -1,363 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const dedent = require('dedent'); -const doctrine = require('doctrine'); - - -/** -* Out class - A utility class for generating formatted text output -* Provides methods for creating headings, line feeds, and text output -* -* ~~with a fluent interface.~~ -* ^ Nope, AI got this wrong but maybe it's a good idea to -* make this a fluent interface -* -* The constructor returns a bound function that -* maintains the output state and provides access to helper methods. -*/ -class Out { - constructor () { - this.str = ''; - const fn = this.out.bind(this); - fn.h = this.h.bind(this); - fn.lf = this.lf.bind(this); - fn.text = () => this.str; - return fn; - } - - h (n, text) { - this.str += '#'.repeat(n) + ' ' + text + '\n\n'; - } - - - /** - * Adds a line feed (newline) to the output string - * @returns {void} - */ - lf () { this.str += '\n'; } - - /** - * Append to the string - * @param {string} str - */ - out (str) { - this.str += str; - } -} - - -/** -* Doc class serves as a base class for documentation generation. -* Provides core functionality for parsing and storing documentation comments -* using the doctrine parser. Contains methods for handling JSDoc-style -* comments and maintaining documentation state. -*/ -class Doc { - constructor () { - this._construct(); - } - provide_comment (comment) { - const parsed_comment = doctrine.parse(comment.value, { unwrap: true }); - this.comment = parsed_comment.description; - } -} - - -/** -* ModuleDoc class extends Doc to represent documentation for a module. -* Handles module-level documentation including services, libraries, and requirements. -* Provides methods for adding services/libraries and generating markdown documentation. -* Tracks external imports and generates notes about module dependencies. -*/ -class ModuleDoc extends Doc { - /** - * Initializes the base properties for a ModuleDoc instance - * Sets up empty arrays for services, requires, and libs collections - * @private - */ - _construct () { - this.services = []; - this.requires = []; - this.libs = []; - } - - - /** - * Creates and adds a new service to this module's services array - * @returns {ServiceDoc} The newly created service document instance - */ - add_service () { - const service = new ServiceDoc(); - this.services.push(service); - return service; - } - - - /** - * Creates and adds a new LibDoc instance to the module's libs array - * @returns {LibDoc} The newly created LibDoc instance - */ - add_lib () { - const lib = new LibDoc(); - this.libs.push(lib); - return lib; - } - - - /** - * Populates a "notes" array for the module documentation - * based on findings about imports. - */ - ready () { - this.notes = []; - const rel_requires = this.requires.filter(r => r.startsWith('../')); - if ( rel_requires.length > 0 ) { - this.notes.push({ - title: 'Outside Imports', - desc: dedent(` - This module has external relative imports. When these are - removed it may become possible to move this module to an - extension. - - **Imports:** - ${rel_requires.map(r => { - let maybe_aside = ''; - if ( r.endsWith('BaseService') ) { - maybe_aside = ' (use.BaseService)'; - } - return `- \`${r}\`` + maybe_aside; - }).join('\n')} - `) - }); - } - } - - toMarkdown ({ hl, out } = { hl: 1 }) { - this.ready(); - - out = out ?? new Out(); - - out.h(hl, this.name); - - out(this.comment + '\n\n'); - - if ( this.services.length > 0 ) { - out.h(hl + 1, 'Services'); - - for ( const service of this.services ) { - service.toMarkdown({ out, hl: hl + 2 }); - } - } - - if ( this.libs.length > 0 ) { - out.h(hl + 1, 'Libraries'); - - for ( const lib of this.libs ) { - lib.toMarkdown({ out, hl: hl + 2 }); - } - } - - if ( this.notes.length > 0 ) { - out.h(hl + 1, 'Notes'); - for ( const note of this.notes ) { - out.h(hl + 2, note.title); - out(note.desc); - out.lf(); - } - } - - - return out.text(); - } -} - - -/** -* ServiceDoc class represents documentation for a service module. -* Handles parsing and formatting of service-related documentation including -* listeners, methods, and their associated parameters. Extends the base Doc class -* to provide specialized documentation capabilities for service components. -*/ -class ServiceDoc extends Doc { - /** - * Represents documentation for a service - * Handles parsing and storing service documentation including listeners and methods - * Initializes with empty arrays for listeners and methods - */ - _construct () { - this.listeners = []; - this.methods = []; - } - - provide_comment (comment) { - const parsed_comment = doctrine.parse(comment.value, { unwrap: true }); - this.comment = parsed_comment.description; - } - - provide_listener (listener) { - const parsed_comment = doctrine.parse(listener.comment, { unwrap: true }); - - const params = []; - for ( const tag of parsed_comment.tags ) { - if ( tag.title !== 'evtparam' ) continue; - const name = tag.description.slice(0, tag.description.indexOf(' ')); - const desc = tag.description.slice(tag.description.indexOf(' ')); - params.push({ name, desc }) - } - - this.listeners.push({ - ...listener, - comment: parsed_comment.description, - params, - }); - } - - provide_method (method) { - const parsed_comment = doctrine.parse(method.comment, { unwrap: true }); - - const params = []; - for ( const tag of parsed_comment.tags ) { - if ( tag.title !== 'param' ) continue; - const name = tag.name; - const desc = tag.description; - params.push({ name, desc }) - } - - this.methods.push({ - ...method, - comment: parsed_comment.description, - params, - }); - } - - toMarkdown ({ hl, out } = { hl: 1 }) { - out = out ?? new Out(); - - out.h(hl, this.name); - - out(this.comment + '\n\n'); - - if ( this.listeners.length > 0 ) { - out.h(hl + 1, 'Listeners'); - - for ( const listener of this.listeners ) { - out.h(hl + 2, '`' + listener.key + '`'); - out (listener.comment + '\n\n'); - - if ( listener.params.length > 0 ) { - out.h(hl + 3, 'Parameters'); - for ( const param of listener.params ) { - out(`- **${param.name}:** ${param.desc}\n`); - } - out.lf(); - } - } - } - - if ( this.methods.length > 0 ) { - out.h(hl + 1, 'Methods'); - - for ( const method of this.methods ) { - out.h(hl + 2, '`' + method.key + '`'); - out (method.comment + '\n\n'); - - if ( method.params.length > 0 ) { - out.h(hl + 3, 'Parameters'); - for ( const param of method.params ) { - out(`- **${param.name}:** ${param.desc}\n`); - } - out.lf(); - } - } - } - - return out.text(); - } -} - - -/** -* LibDoc class for documenting library modules -* Handles documentation for library functions including their descriptions, -* parameters, and markdown generation. Extends the base Doc class to provide -* specialized documentation capabilities for library components. -*/ -class LibDoc extends Doc { - /** - * Represents documentation for a library module - * - * Handles parsing and formatting documentation for library functions. - * Stores function definitions with their comments, parameters and descriptions. - * Can output formatted markdown documentation. - */ - _construct () { - this.functions = []; - } - - provide_function ({ key, comment, params }) { - const parsed_comment = doctrine.parse(comment, { unwrap: true }); - - const parsed_params = []; - for ( const tag of parsed_comment.tags ) { - if ( tag.title !== 'param' ) continue; - const name = tag.name; - const desc = tag.description; - parsed_params.push({ name, desc }); - } - - this.functions.push({ - key, - comment: parsed_comment.description, - params: parsed_params, - }); - } - - toMarkdown ({ hl, out } = { hl: 1 }) { - out = out ?? new Out(); - - out.h(hl, this.name); - - console.log('functions?', this.functions); - - if ( this.functions.length > 0 ) { - out.h(hl + 1, 'Functions'); - - for ( const func of this.functions ) { - out.h(hl + 2, '`' + func.key + '`'); - out(func.comment + '\n\n'); - - if ( func.params.length > 0 ) { - out.h(hl + 3, 'Parameters'); - for ( const param of func.params ) { - out(`- **${param.name}:** ${param.desc}\n`); - } - out.lf(); - } - } - } - - return out.text(); - } -} - -module.exports = { - ModuleDoc, - ServiceDoc, -}; diff --git a/tools/module-docgen/main.js b/tools/module-docgen/main.js deleted file mode 100644 index 31d266649..000000000 --- a/tools/module-docgen/main.js +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const fs = require("fs"); -const path_ = require("path"); - -const rootdir = path_.resolve(process.argv[2] ?? '.'); - -const parser = require('@babel/parser'); -const traverse = require('@babel/traverse').default; -const { ModuleDoc } = require("./defs"); -const processors = require("./processors"); - -const doc_module = new ModuleDoc(); - -const handle_file = (code, context) => { - const ast = parser.parse(code); - - const traverse_callbacks = {}; - for ( const processor of processors ) { - if ( processor.match(context) ) { - for ( const key in processor.traverse ) { - if ( ! traverse_callbacks[key] ) { - traverse_callbacks[key] = []; - } - traverse_callbacks[key].push(processor.traverse[key]); - } - } - } - for ( const key in traverse_callbacks ) { - traverse(ast, { - [key] (path) { - context.skip = false; - for ( const callback of traverse_callbacks[key] ) { - callback(path, context); - if ( context.skip ) return; - } - } - }); - } -} - -// Module and class files -{ - const files = fs.readdirSync(rootdir); - for ( const file of files ) { - const stat = fs.statSync(path_.join(rootdir, file)); - if ( stat.isDirectory() ) { - continue; - } - if ( ! file.endsWith('.js') ) continue; - - const type = - file.endsWith('Service.js') ? 'service' : - file.endsWith('Module.js') ? 'module' : - null; - - if ( type === null ) continue; - - console.log('file', file); - const code = fs.readFileSync(path_.join(rootdir, file), 'utf8'); - - const firstLine = code.slice(0, code.indexOf('\n')); - let metadata = {}; - const METADATA_PREFIX = '// METADATA // '; - if ( firstLine.startsWith(METADATA_PREFIX) ) { - metadata = JSON.parse(firstLine.slice(METADATA_PREFIX.length)); - } - - const context = { - metadata, - type, - doc_module, - filename: file, - }; - - handle_file(code, context); - } -} - -// Library files -if ( fs.existsSync(path_.join(rootdir, 'lib')) ) { - const files = fs.readdirSync(path_.join(rootdir, 'lib')); - for ( const file of files ) { - if ( file.startsWith('_') ) continue; - - const code = fs.readFileSync(path_.join(rootdir, 'lib', file), 'utf8'); - - const firstLine = code.slice(0, code.indexOf('\n')); - let metadata = {}; - const METADATA_PREFIX = '// METADATA // '; - if ( firstLine.startsWith(METADATA_PREFIX) ) { - metadata = JSON.parse(firstLine.slice(METADATA_PREFIX.length)); - } - - const doc_item = doc_module.add_lib(); - doc_item.name = metadata.def ?? file.slice(0, -3); - - const context = { - metadata, - type: 'lib', - doc_module, - doc_item, - filename: file, - }; - - handle_file(code, context); - } -} - -const outfile = path_.join(rootdir, 'README.md'); - -const out = doc_module.toMarkdown(); - -fs.writeFileSync(outfile, out); \ No newline at end of file diff --git a/tools/module-docgen/package.json b/tools/module-docgen/package.json deleted file mode 100644 index 4aa2bdd5b..000000000 --- a/tools/module-docgen/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "module-docgen", - "version": "1.0.0", - "main": "main.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "keywords": [], - "author": "", - "license": "AGPL-3.0-only", - "description": "", - "dependencies": { - "@babel/parser": "^7.26.2", - "@babel/traverse": "^7.25.9", - "dedent": "^1.5.3", - "doctrine": "^3.0.0" - } -} diff --git a/tools/module-docgen/processors.js b/tools/module-docgen/processors.js deleted file mode 100644 index a9a374872..000000000 --- a/tools/module-docgen/processors.js +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const processors = []; - -processors.push({ - title: 'track all require calls', - match () { return true; }, - traverse: { - CallExpression (path, context) { - const callee = path.get('callee'); - if ( ! callee.isIdentifier() ) return; - - if ( callee.node.name === 'require' ) { - context.doc_module.requires.push(path.node.arguments[0].value); - } - } - } -}); - -processors.push({ - title: 'get leading comment', - match () { return true; }, - traverse: { - ClassDeclaration (path, context) { - const node = path.node; - const comment = (node.leadingComments && ( - node.leadingComments.length < 1 ? '' : - node.leadingComments[node.leadingComments.length - 1] - )) ?? ''; - context.comment = comment; - } - } -}); - -processors.push({ - title: 'provide name and comment for modules and services', - match (context) { - return context.type === 'module' || context.type === 'service'; - }, - traverse: { - ClassDeclaration (path, context) { - context.doc_item = context.doc_module; - if ( context.type === 'service' ) { - // Skip if class name doesn't end with 'Service' - if ( ! path.node.id.name.endsWith('Service') ) { - context.skip = true; - return; - } - context.doc_item = context.doc_module.add_service(); - } - context.doc_item.name = path.node.id.name; - if ( context.comment === '' ) return; - context.doc_item.provide_comment(context.comment); - } - } -}); - -processors.push({ - title: 'provide methods and listeners for services', - match (context) { - return context.type === 'service'; - }, - traverse: { - ClassDeclaration (path, context) { - path.node.body.body.forEach(member => { - if ( member.type !== 'ClassMethod' ) return; - - const key = member.key.name ?? member.key.value; - - const comment = member.leadingComments?.[0]?.value ?? ''; - - if ( key.startsWith('__on_') ) { - // 2nd argument is always an object destructuring; - // we want the list of keys in the object: - const params = member.params?.[1]?.properties ?? []; - - context.doc_item.provide_listener({ - key: key.slice(5), - comment, - params, - }); - } else { - // Method overrides - if ( key.startsWith('_') ) return; - - // Private methods - if ( key.endsWith('_') ) return; - - const params = member.params ?? []; - - context.doc_item.provide_method({ - key, - comment, - params, - }); - } - }); - } - } -}); - -processors.push({ - title: 'provide library function documentation', - match (context) { - return context.type === 'lib'; - }, - traverse: { - VariableDeclaration (path, context) { - // skip non-const declarations - if ( path.node.kind !== 'const' ) return; - - // skip declarations with multiple declarators - if ( path.node.declarations.length !== 1 ) return; - - // skip declarations without an initializer - if ( ! path.node.declarations[0].init ) return; - - // skip declarations that aren't in the root scope - if ( path.scope.parent ) return; - - console.log('path.node', path.node.declarations); - - // is it a function? - if ( ! ['FunctionExpression', 'ArrowFunctionExpression'].includes( - path.node.declarations[0].init.type - ) ) return; - - // get the name of the function - const name = path.node.declarations[0].id.name; - - // get the comment - const comment = path.node.leadingComments?.[0]?.value ?? ''; - - // get the parameters - const params = path.node.declarations[0].init.params ?? []; - - context.doc_item.provide_function({ - key: name, - comment, - params, - }); - } - } -}); - -module.exports = processors; diff --git a/tools/run-selfhosted.js b/tools/run-selfhosted.js deleted file mode 100644 index 21289c7ec..000000000 --- a/tools/run-selfhosted.js +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import LocalDiskStorageModule from '@heyputer/backend/src/LocalDiskStorageModule.js'; -import process from 'node:process'; - -try { - await import('dotenv/config'); -} catch (e) { - // dotenv is optional -} - -// Annoying polyfill for inconsistency in different node versions -if ( ! import.meta.filename ) { - Object.defineProperty(import.meta, 'filename', { - get: () => import.meta.url.slice('file://'.length), - }); -} - -const main = async () => { - const { - Kernel, - EssentialModules, - DatabaseModule, - SelfHostedModule, - BroadcastModule, - TestDriversModule, - TestConfigModule, - InternetModule, - } = (await import('@heyputer/backend')).default; - - const k = new Kernel({ - entry_path: import.meta.filename, - }); - for ( const mod of EssentialModules ) { - k.add_module(new mod()); - } - k.add_module(new DatabaseModule()); - k.add_module(new LocalDiskStorageModule()); - k.add_module(new SelfHostedModule()); - k.add_module(new BroadcastModule()); - k.add_module(new TestDriversModule()); - k.add_module(new TestConfigModule()); - k.add_module(new InternetModule()); - k.boot(); -}; - -(async () => { - await main(); -})(); diff --git a/tools/token-count-accuracy/package.json b/tools/token-count-accuracy/package.json deleted file mode 100644 index 136b48688..000000000 --- a/tools/token-count-accuracy/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "token-count-accuracy", - "version": "1.0.0", - "main": "test.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "keywords": [], - "author": "", - "license": "AGPL-3.0-only", - "description": "" -} diff --git a/tools/token-count-accuracy/test.js b/tools/token-count-accuracy/test.js deleted file mode 100644 index 4af24a272..000000000 --- a/tools/token-count-accuracy/test.js +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const claude_examples = [ - { - type: 'output', // probably doesn't matter - text: "I am Claude, an AI assistant created by Anthropic. I'm running on Puter, an open-source platform, through a driver interface called puter-chat-completion. I aim to be direct and honest about my identity and capabilities.", - tokens: 55, - }, - { - type: 'output', - text: `Here's a list of fascinating numbers and their significance: - -1. 1.618033988749895 (φ, Phi, Golden Ratio) -- Found throughout nature and art -- Considered aesthetically pleasing -- Appears in the proportions of the Parthenon, nautilus shells, and spiral galaxies - -2. 2.71828... (e, Euler's Number) -- Base of natural logarithms -- Fundamental to exponential growth and decay -- Essential in compound interest calculations - -3. 3.14159... (π, Pi) -- Ratio of circle's circumference to diameter -- Transcendental number -- Appears across mathematics and physics - -4. 6.02214076 × 10²³ (Avogadro's Number) -- Number of particles in one mole of substance -- Fundamental to chemistry -- Named after Italian scientist Amedeo Avogadro - -5. 137 (Fine Structure Constant) -- Describes strength of electromagnetic interaction -- Mysterious number in physics -- No theoretical explanation for its value - -6. 1729 (Hardy-Ramanujan Number) -- Smallest number expressible as sum of two cubes in two different ways -- Famous from conversation between mathematicians Hardy and Ramanujan - -7. 42 -- Answer to "life, the universe, and everything" in Hitchhiker's Guide to the Galaxy -- Has gained cultural significance -- Interestingly appears in various mathematical contexts - -8. 0.577215... (γ, Euler-Mascheroni Constant) -- Appears in number theory and calculus -- Still unknown if it's irrational -- Connected to harmonic series - -9. 23! (Factorial of 23) -- Approximately equal to all seconds since Big Bang -- Shows how quickly factorial numbers grow - -10. 666 (Number of the Beast) -- Cultural and religious significance -- Appears in Book of Revelation -- Subject of numerous mathematical curiosities - -These numbers demonstrate how mathematics intersects with nature, science, culture, and human understanding of the universe.`, - tokens: 481, - }, - { - type: 'output', - text: `Here are some digits of Pi: -3.14159265358979323846264338327950288419716939937510... - -Some digits of the square root of 2: -1.41421356237309504880168872420969807856967187537694... - -And now for some random unpredictable things: - -Purple elephants juggling quantum calculators in zero gravity -The sound of silence tastes like neon butterflies -Yesterday's tomorrow forgot to attend its own birthday party -Clouds made of crystallized laughter are floating upside down -The number 7 decided to identify as a letter instead -Spinning teacups full of liquid starlight and abstract concepts -Time decided to flow sideways through a Klein bottle -Philosophical zombies debating the existence of consciousness while eating imaginary cookies -The color blue went on strike and was temporarily replaced by the smell of nostalgia -Dancing fractals wearing mismatched socks made of pure mathematics -A parade of impossible objects marching through an Escher painting -The concept of Tuesday learned to yodel in binary code -Metaphysical hiccups causing temporary glitches in the fabric of reason -Square circles plotting a revolution against euclidean geometry -The letter Q eloped with an ampersand and they had punctuation mark babies`, - tokens: 284, - } -]; - -// Measure each with tiktoken - -class TikTokenCounter { - constructor (model_to_try) { - this.model_to_try = model_to_try; - } - - get title () { - return `TikToken ${this.model_to_try}`; - } - - count (text) { - const tiktoken = require('tiktoken'); - const enc = tiktoken.encoding_for_model(this.model_to_try); - const tokens = enc.encode(text); - return tokens.length; - } -} - -class DivideCounter { - constructor (by) { - this.by = by; - } - - get title () { - return `Divide by ${this.by}`; - } - - count (text) { - return text.length / this.by; - } -} - -const counters_to_try = [ - new TikTokenCounter('gpt-3.5-turbo'), - new TikTokenCounter('gpt-4'), - new TikTokenCounter('gpt-4o'), - new TikTokenCounter('gpt-4o-mini'), - new DivideCounter(4), - new DivideCounter(5), -]; - -const scores = {}; - -const results = []; -for (const example of claude_examples) { - const result = { - example, - counts: {}, - diffs: {}, - }; - for (const counter of counters_to_try) { - result.counts[counter.title] = counter.count(example.text); - } - results.push(result); - - // Which one is the most accurate? - const real_amount = example.tokens; - for ( const count_name in result.counts ) { - const count = result.counts[count_name]; - const diff = Math.abs(count - real_amount); - result.diffs[count_name] = diff; - } - // Report the most accurate one - const most_accurate = - Object.keys(result.diffs) - .reduce((a, b) => result.diffs[a] < result.diffs[b] ? a : b); - result.most_accurate = most_accurate; - - scores[most_accurate] = (scores[most_accurate] || 0) + 1; -} - - -console.log(results); - -console.log(scores); \ No newline at end of file diff --git a/tools/validate-eslint.js b/tools/validate-eslint.js deleted file mode 100644 index 45e86a5a4..000000000 --- a/tools/validate-eslint.js +++ /dev/null @@ -1,44 +0,0 @@ -// This script does not validate that eslint rules are followed; it only -// ensures that the eslint configuration is valid. When there are errors -// present in the eslint configuration, vscode pretends everything is -// fine and that there are no linter errors in any files. - -import { ESLint } from 'eslint'; - -async function validateConfig() { - let exitWithError = false; - - try { - const eslint = new ESLint(); - await eslint.lintText('', { filePath: 'src/gui/**/*.js' }); - } catch (error) { - console.error('❌ ESLint configuration error (general):', error.message); - exitWithError = true; - } - - try { - const eslint = new ESLint(); - await eslint.lintText('', { filePath: 'src/backend/**/*.js' }); - } catch (error) { - console.error('❌ ESLint configuration error (backend):', error.message); - exitWithError = true; - } - - try { - const eslint = new ESLint(); - await eslint.lintText('', { filePath: 'extensions/**/*.js' }); - } catch (error) { - console.error('❌ ESLint configuration error (extensions):', error.message); - exitWithError = true; - } - - if ( exitWithError ) { - console.log('\x1B[36;1mYou should edit eslint.config.js to resolve this issue.\x1B[0m'); - console.log('\x1B[31;1mIf this is an emergency, use `git commit --no-verify`.\x1B[0m'); - process.exit(1); - } - - console.log('✅ ESLint configuration is valid'); -} - -validateConfig(); diff --git a/tools/write-dist-package-json.mjs b/tools/write-dist-package-json.mjs new file mode 100644 index 000000000..1ecd45a0a --- /dev/null +++ b/tools/write-dist-package-json.mjs @@ -0,0 +1,48 @@ +import { cpSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'; + +mkdirSync('dist', { recursive: true }); +writeFileSync( + 'dist/package.json', + `${JSON.stringify({ + name: '@heyputer/backend', + type: 'commonjs', + exports: { + // Post-flatten: `src/backend/` is the backend root. The `src/` + // in the compiled path (`dist/src/backend/...`) is an artifact + // of tsc's rootDir being the package root, not a meaningful + // subfolder. Canonical imports drop the `/src/` prefix — e.g. + // `@heyputer/backend/controllers/types`. Named shortcuts for + // the common entry points come first (Node picks the most + // specific pattern match). + './core': './src/backend/core/index.js', + './core/http': './src/backend/core/http/index.js', + './extensions': './src/backend/extensions.js', + // Dual-form patterns so both extensionless (TS-compiled + // requires drop `.js`) and extensioned (hand-written JS) + // subpaths resolve. + './*.js': './src/backend/*.js', + './*': './src/backend/*.js', + // Back-compat for the older `@heyputer/backend/src/*` style. + // Safe to keep indefinitely; remove once every extension has + // been rewritten. + './src/core': './src/backend/core/index.js', + './src/core/http': './src/backend/core/http/index.js', + './src/extensions': './src/backend/extensions.js', + './src/*.js': './src/backend/*.js', + './src/*': './src/backend/*.js', + }, + }, null, 2)}\n`, +); + +// tsc only emits .js for .ts inputs. Non-source files the runtime needs +// (SQL migrations + .dbmig.js scripts resolved relative to __dirname) have +// to be copied over by hand — otherwise `SqliteDatabaseClient.runMigrations` +// throws ENOENT on first boot. +const COPY_DIRS = [ + ['src/backend/clients/database/migrations', 'dist/src/backend/clients/database/migrations'], +]; +for ( const [from, to] of COPY_DIRS ) { + if ( ! existsSync(from) ) continue; + mkdirSync(to, { recursive: true }); + cpSync(from, to, { recursive: true }); +} diff --git a/tsconfig.base.json b/tsconfig.base.json deleted file mode 100644 index 8e4dc99dd..000000000 --- a/tsconfig.base.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "compilerOptions": { - "target": "esnext", - "module": "esnext", - "moduleResolution": "bundler", - "rootDir": "./src/backend", - "experimentalDecorators": true, - "strict": true, - "forceConsistentCasingInFileNames": true, - "skipLibCheck": true, - "sourceMap": true, - "removeComments": true, - "noEmitOnError": true, - "noImplicitAny": false - } -} diff --git a/tsconfig.build.json b/tsconfig.build.json deleted file mode 100644 index df1db7fd2..000000000 --- a/tsconfig.build.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "extends": "./tsconfig.base.json", - "compilerOptions": { - "allowJs": false, - "checkJs": false, - "noEmit": false - }, - "include": ["./src/backend"], - "exclude": [ - "**/*.test.ts", - "**/*.test.mts", - "**/vitest.config.ts", - "**/vitest.config.mts", - "**/*.spec.ts", - "**/*.spec.mts", - "**/tests/**", - "node_modules", - "dist", - "volatile", - "extensions", - "src/backend/src/services/worker/template/puter-portable.js" - ] -} diff --git a/tsconfig.json b/tsconfig.json index eaf9d6ef3..5e4408ec5 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,23 +1,60 @@ { - "extends": "./tsconfig.base.json", - "compilerOptions": { - "allowJs": true, - "checkJs": false, - "noEmit": true + "compilerOptions": { + "target": "ES2022", + "module": "CommonJS", + "moduleResolution": "Node", + "rootDir": ".", + "outDir": "./dist", + "baseUrl": ".", + "paths": { + "@heyputer/backend": [ + "src/backend/exports.ts" + ], + "@heyputer/backend/src/*": [ + "src/backend/*" + ], + "@heyputer/backend/*": [ + "src/backend/*" + ] }, - "include": ["./src/backend", "src/backend/src/deprecated/filesystem/PuterS3StorageStrategy.js"], - "exclude": [ - "**/*.test.ts", - "**/*.test.mts", - "**/vitest.config.ts", - "**/vitest.config.mts", - "**/*.spec.ts", - "**/*.spec.mts", - "**/tests/**", - "node_modules", - "dist", - "volatile", - "extensions", - "src/backend/src/services/worker/template/puter-portable.js" - ] + "allowJs": true, + "checkJs": false, + "noCheck": true, + "strict": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true, + "sourceMap": true, + "inlineSources": true, + "removeComments": true, + "noEmit": false, + "noEmitOnError": false, + "noImplicitAny": false + }, + "include": [ + "src/backend/**/*", + "extensions/**/*" + ], + "exclude": [ + "**/*.test.ts", + "**/*.test.mts", + "**/*.spec.ts", + "**/*.spec.mts", + "**/tests/**", + "**/node_modules/**", + "dist/**", + "volatile/**", + "extensions/**/node_modules/**", + "src/backend/test/**", + "src/backend/tools/**", + "src/backend/vitest.config.ts", + "src/backend/vitest.bench.config.ts", + "src/backend/vitest.bench.config.js", + "src/backend/services/worker/template/puter-portable.js", + "src/backend/services/DynamoKVStore/DynamoKVStore.ts", + "src/backend/clients/s3/S3Client.js", + "src/backend/clients/s3/s3ClientProvider.js", + "src/backend/clients/redis/RedisClient.js", + "src/backend/clients/dynamodb/DDBClient.js" + ] } diff --git a/volatile/config/.gitignore b/volatile/config/.gitignore deleted file mode 100644 index d6b7ef32c..000000000 --- a/volatile/config/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/ws-debug.mjs b/ws-debug.mjs deleted file mode 100644 index c530a81c2..000000000 --- a/ws-debug.mjs +++ /dev/null @@ -1,53 +0,0 @@ -import { createTestKernel } from './src/backend/tools/test.mjs'; -import { WorkerService } from '@heyputer/backend/src/services/worker/WorkerService.js'; -import { EntityStoreService } from './src/backend/src/services/EntityStoreService.js'; -import { AppLimitedES } from './src/backend/src/om/entitystorage/AppLimitedES.js'; -import { ESBuilder } from './src/backend/src/om/entitystorage/ESBuilder.js'; -import MaxLimitES from './src/backend/src/om/entitystorage/MaxLimitES.js'; -import SQLES from './src/backend/src/om/entitystorage/SQLES.js'; -import { SetOwnerES } from './src/backend/src/om/entitystorage/SetOwnerES.js'; -import SubdomainES from './src/backend/src/om/entitystorage/SubdomainES.js'; -import ValidationES from './src/backend/src/om/entitystorage/ValidationES.js'; -import WriteByOwnerOnlyES from './src/backend/src/om/entitystorage/WriteByOwnerOnlyES.js'; -import { Actor, UserActorType } from './src/backend/src/services/auth/Actor.js'; - -trying(); - -async function trying() { - const tk = await createTestKernel({ - serviceMap: { worker: WorkerService, 'es:subdomain': EntityStoreService }, - serviceMapArgs: { - 'es:subdomain': { - entity: 'subdomain', - upstream: ESBuilder.create([ - SQLES, - { table: 'subdomains', debug: true }, - SubdomainES, - AppLimitedES, - WriteByOwnerOnlyES, - ValidationES, - SetOwnerES, - MaxLimitES, { max: 5000 }, - ]), - }, - }, - serviceConfigOverrideMap: { - worker: { loggingUrl: 'x' }, - database: { path: ':memory:' }, - }, - initLevelString: 'init', - testCore: true, - globalConfigOverrideMap: { - worker: { reserved_words: [] }, - }, - }); - - const su = tk.services.get('su'); - const ws = tk.services.get('worker'); - const actor = new Actor({ type: new UserActorType({ user: { id: 1, uuid: 'u1', username: 'test' } }) }); - - globalThis.services = tk.services; - - const res = await tk.root_context.arun(() => su.sudo(actor, () => ws.create({ filePath: '/worker.js', workerName: 'MyWorker', authorization: 'auth' }))); - console.log('result', res); -}