From 50679e6a01bf1f56e26b6566cce67800dc1719fb Mon Sep 17 00:00:00 2001 From: Daniel Salazar Date: Mon, 4 May 2026 18:14:23 -0700 Subject: [PATCH] selfhosting: p2 fixed default configs and guide + some other errors (#2903) * fix: ollama support * fix: more self host cleanup * fix: bad auth * fix: more session cookie stuff * allow emailclient Co-authored-by: Copilot * fix: broken migration * fix: full-stack comment * fix: hardcoded perms * new docker build steps --------- Co-authored-by: Copilot --- .github/workflows/docker-image.yaml | 58 +-- .github/workflows/tag-alias.yaml | 72 ++++ doc/api/README.md | 5 - doc/api/drivers.md | 60 --- doc/api/group.md | 219 ----------- doc/api/notifications.md | 112 ------ doc/api/share.md | 367 ------------------ doc/self-hosters/config.md | 81 ---- doc/self-hosters/config_values.md | 72 ---- doc/self-hosters/domains.md | 85 ---- doc/self-hosters/instructions.md | 59 --- docker-compose.full.yml | 68 ++++ nginx/nginx.conf | 52 ++- selfhosted/docker.md | 2 + selfhosted/full-stack.md | 72 +++- .../clients/database/DatabaseClient.ts | 1 + .../clients/database/MySQLDatabaseClient.ts | 4 +- .../clients/database/SqliteDatabaseClient.ts | 1 + .../database/migrations/mysql/mysql_mig_2.sql | 5 +- .../database/migrations/mysql/mysql_mig_3.sql | 58 +++ src/backend/clients/email/EmailClient.ts | 6 +- src/backend/clients/s3/S3Client.ts | 56 +++ .../controllers/auth/AuthController.js | 7 +- .../controllers/oidc/OIDCController.ts | 8 +- src/backend/services/auth/AuthService.ts | 4 +- .../services/permission/PermissionService.ts | 124 +++--- src/backend/stores/fs/S3ObjectStore.ts | 16 +- src/backend/types.ts | 15 + src/backend/util/cookieFlags.ts | 45 +++ 29 files changed, 538 insertions(+), 1196 deletions(-) create mode 100644 .github/workflows/tag-alias.yaml delete mode 100644 doc/api/README.md delete mode 100644 doc/api/drivers.md delete mode 100644 doc/api/group.md delete mode 100644 doc/api/notifications.md delete mode 100644 doc/api/share.md delete mode 100644 doc/self-hosters/config.md delete mode 100644 doc/self-hosters/config_values.md delete mode 100644 doc/self-hosters/domains.md delete mode 100644 doc/self-hosters/instructions.md create mode 100644 src/backend/clients/database/migrations/mysql/mysql_mig_3.sql create mode 100644 src/backend/util/cookieFlags.ts diff --git a/.github/workflows/docker-image.yaml b/.github/workflows/docker-image.yaml index e61655017..3e1c636e2 100644 --- a/.github/workflows/docker-image.yaml +++ b/.github/workflows/docker-image.yaml @@ -1,41 +1,38 @@ -# name: Docker Image CI -# Configures this workflow to run every time a change is pushed to the -# branch called `main`. +# Builds only on calver tag pushes: YY.MM or YY.MM.p (all numeric). +# Each build publishes three tags: , latest, main. on: push: tags: - - '*.*.*' - branches: - - 'main' + - '[0-9][0-9].[0-9][0-9]' + - '[0-9][0-9].[0-9][0-9].[0-9]*' -# Defines two custom environment variables for the workflow. These are used -# for the Container registry domain, and a name for the Docker image that -# this workflow builds. env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} -# There is a single job in this workflow. It's configured to run on the -# latest available version of Ubuntu. jobs: build-and-push-image: runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions - # in this job. permissions: contents: read packages: write steps: + - name: Validate calver tag + env: + REF_NAME: ${{ github.ref_name }} + run: | + if ! [[ "$REF_NAME" =~ ^[0-9]{2}\.[0-9]{2}(\.[0-9]+)?$ ]]; then + echo "Tag '$REF_NAME' does not match YY.MM or YY.MM.p calver format." + exit 1 + fi + - name: Checkout repository uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container - # registry using the account and password that will publish the packages. - # Once published, the packages are scoped to the account defined here. - name: Log in to GitHub Package Container registry uses: docker/login-action@v3 with: @@ -49,43 +46,22 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) - # to extract tags and labels that will be applied to the specified image. - # The `id` "meta" allows the output of this step to be referenced in - # a subsequent step. The `images` value provides the base name for the - # tags and labels. - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v5 with: images: "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}" - # Tag plan: - # * version tag (vX.Y.Z) push β†’ 1.2.3, 1.2, latest - # * branch push (main) β†’ main - # selfhosted/docker.md tells users to pull `:latest`, which only - # resolves for tag pushes β€” never main, so unstable code can't - # claim `:latest`. tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=ref,event=branch - type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') }} + type=ref,event=tag + type=raw,value=latest + type=raw,value=main - # This step uses the `docker/build-push-action` action to build the - # image, based on your repository's `Dockerfile`. If the build succeeds, - # it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the - # set of files located in the specified path. For more information, see - # "[Usage](https://github.com/docker/build-push-action#usage)" in the - # README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image - # with the output from the "meta" step. - name: Build and push Docker image uses: docker/build-push-action@v5 with: platforms: linux/amd64,linux/arm64 context: . - push: ${{ github.event_name != 'pull_request' }} + push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha diff --git a/.github/workflows/tag-alias.yaml b/.github/workflows/tag-alias.yaml new file mode 100644 index 000000000..39f16475e --- /dev/null +++ b/.github/workflows/tag-alias.yaml @@ -0,0 +1,72 @@ +name: Tag Alias + +# Retags an existing image in GHCR under a custom name, without rebuilding. +# Uses `docker buildx imagetools create` so the multi-arch manifest is preserved. +# Re-running with the same name updates the alias to point at a different source tag. +on: + workflow_dispatch: + inputs: + name: + description: "Alias name (e.g., potato)" + required: true + source_tag: + description: "Source tag to alias from" + required: false + default: "latest" + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + alias: + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + steps: + - name: Validate alias name + env: + NAME: ${{ inputs.name }} + run: | + if ! [[ "$NAME" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then + echo "Invalid alias name '$NAME' (allowed: [A-Za-z0-9_.-], must start with [A-Za-z0-9_], max 128 chars)." + exit 1 + fi + if [[ "$NAME" == "latest" || "$NAME" == "main" ]]; then + echo "Alias name '$NAME' is reserved by the build workflow." + exit 1 + fi + if [[ "$NAME" =~ ^[0-9]{2}\.[0-9]{2}(\.[0-9]+)?$ ]]; then + echo "Alias name '$NAME' looks like a calver tag β€” pick a non-version name." + exit 1 + fi + + - name: Log in to GitHub Package Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Create alias tag + env: + IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + NAME: ${{ inputs.name }} + SOURCE_TAG: ${{ inputs.source_tag }} + run: | + SRC="${IMAGE}:${SOURCE_TAG}" + DST="${IMAGE}:${NAME}" + echo "Aliasing $DST -> $SRC" + docker buildx imagetools create -t "$DST" "$SRC" + { + echo "### 🏷️ Tag Alias" + echo "" + echo "- Source: \`$SRC\`" + echo "- Alias: \`$DST\`" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/doc/api/README.md b/doc/api/README.md deleted file mode 100644 index b93bea63a..000000000 --- a/doc/api/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# API Documentation - -Note that this documentation is different from the [puter.js docs](https://docs.puter.com). -The scope of the documentation in this directory includes both stable API endpoints that -are used by **puter.js**, as well as API endpoints that may be subject to future changes. diff --git a/doc/api/drivers.md b/doc/api/drivers.md deleted file mode 100644 index f38c656e7..000000000 --- a/doc/api/drivers.md +++ /dev/null @@ -1,60 +0,0 @@ -## Puter Drivers - -### **POST** `/drivers/call` - -#### Notes - -- **HTTP response status** - - A successful driver response, even if the response is an error message, will always have HTTP status `200`. Note that sometimes this will include rate limit and usage limit errors as well. - -This endpoint allows you to call a Puter driver. Whether or not the -driver call fails, this endpoint will respond with HTTP 200 OK. -When a driver call fails, you will get a JSON response from the driver -with - -#### Parameters - -Parameters are provided in the request body. The content type of the -request should be `application/json`. - -- **interface:** `string` - - **description:** The type of driver to call. For example, - LLMs use the interface called `puter-chat-completion`. -- **service:** `string` - - **description:** The name of the service to use. For example, the `claude` service might be used for `puter-chat-completion`. -- **method:** `string` - - **description:** The name of the method to call. For example, LLMs implement `complete` which does a chat completion, and `list` which lists models. -- **args:** `object` - - **description:** Parametized arguments for the driver call. For example, `puter-chat-completion`'s `complete` method supports the arguments `messages` and `temperature` (and others), so you might set this to `{ "messages": [...], "temperature": 1.2 }` - -#### Example -```json -{ - "interface": "", - "service": "", - "method": "", - "args": { "parametized": "arguments" } -} -``` - -#### Response - -- **Error Response** - Driver error responses will always have **status 200**, content type `application/json`, and a response body in this format: - ```json - { - "success": false, - "error": { - "code": "string identifier for the error", - "message": "some message about the error", - } - } - ``` -- **Success Response** - The success response is either a JSON response - wrapped in `{ "success": true, "result": ___ }`, or a response with a - `Content-Type` that is **not** `application/json`. - ```json - { - "success": true, - "result": {} - } - ``` \ No newline at end of file diff --git a/doc/api/group.md b/doc/api/group.md deleted file mode 100644 index f31ad4184..000000000 --- a/doc/api/group.md +++ /dev/null @@ -1,219 +0,0 @@ -# Group Endpoints - -## POST `/group/create` (auth required) - -### Description - -Creates a group and returns a UID (UUID formatted). -Groups do not have names, or any other descriptive attributes. -Instead they are always identified with a UUID, and they have -a `metadata` property. - -The `metadata` property will always be given back to the client -in the same way it was provided. The `extra` property, also an -object, may be changed by the backend. The behavior of setting -any property on `extra` is currently undefined as all properties -are reserved for future use. - -### Parameters - -- **metadata:** _- optional_ - - **accepts:** `object` - - **description:** arbitrary metadata to describe the group -- **extra:** _- optional_ - - **accepts:** `object` - - **description:** extra parameters (server may change these) - -### Request Example - -```javascript -await fetch(`${window.api_origin}/group/create`, { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - metadata: { title: 'Some Title' } - }), - "method": "POST", -}); - -// { uid: '9c644a1c-3e43-4df4-ab67-de5b68b235b6' } -``` - -### Response Example - -```json -{ - "uid": "9c644a1c-3e43-4df4-ab67-de5b68b235b6" -} -``` - -## POST `/group/add-users` - -### Description - -Adds one or more users to a group - -### Parameters - -- **uid:** _- required_ - - **accepts:** `string` - UUID of an existing group -- **users:** `Array` - usernames of users to add to the group - -### Request Example - -```javascript -await fetch(`${window.api_origin}/group/add-users`, { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - uid: '9c644a1c-3e43-4df4-ab67-de5b68b235b6', - users: ['first_user', 'second_user'], - }), - "method": "POST", -}); -``` - -## POST `/group/remove-users` - -### Description - -Remove one or more users from a group - -### Parameters - -- **uid:** _- required_ - - **accepts:** `string` - UUID of an existing group -- **users:** `Array` - usernames of users to remove from the group - -### Request Example - -```javascript -await fetch(`${window.api_origin}/group/add-users`, { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - uid: '9c644a1c-3e43-4df4-ab67-de5b68b235b6', - users: ['first_user', 'second_user'], - }), - "method": "POST", -}); -``` - -## GET `/group/list` - -### Description - -List groups associated with the current user - -### Parameters - -_none_ - -### Response Example - -```json -{ - "owned_groups": [ - { - "uid": "c3bd4047-fc65-4da8-9363-e52195890de4", - "metadata": {}, - "members": [ - "default_user" - ] - } - ], - "in_groups": [ - { - "uid": "c3bd4047-fc65-4da8-9363-e52195890de4", - "metadata": {}, - "members": [ - "default_user" - ] - } - ] -} -``` - -# Group Permission Endpoints - -## POST `/grant-user-group` - -Grant permission from the current user to a group. -This creates an association between the user and the -group for this permission; the group will only have -the permission effectively while the user who granted -permission has the permission. - -### Parameters - -- **group_uid:** _- required_ - - **accepts:** `string` - UUID of an existing group -- **permission:** _- required_ - - **accepts:** `string` - A permission string - -### Request Example - -```javascript -await fetch("http://puter.localhost:4100/auth/grant-user-group", { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - group_uid: '9c644a1c-3e43-4df4-ab67-de5b68b235b6', - permission: 'fs:/someuser/somedir/somefile:read' - }), - "method": "POST", -}); -``` - -## POST `/revoke-user-group` - -Revoke permission granted from the current user -to a group. - -### Parameters - -- **group_uid:** _- required_ - - **accepts:** `string` - UUID of an existing group -- **permission:** _- required_ - - **accepts:** `string` - A permission string - -### Request Example - -```javascript -await fetch("http://puter.localhost:4100/auth/grant-user-group", { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - group_uid: '9c644a1c-3e43-4df4-ab67-de5b68b235b6', - permission: 'fs:/someuser/somedir/somefile:read' - }), - "method": "POST", -}); -``` - -- > **TODO** figure out how to manage documentation that could - reasonably show up in two files. For example: this is a group - endpoint as well as a permission system endpoint. - (architecturally it's a permission system endpoint, and - the permissions feature depends on the groups feature; - at least until a time when PermissionService is refactored - so a service like GroupService can mutate the permission - check sequences) diff --git a/doc/api/notifications.md b/doc/api/notifications.md deleted file mode 100644 index ec002fc06..000000000 --- a/doc/api/notifications.md +++ /dev/null @@ -1,112 +0,0 @@ -# Notification Endpoints - -Endpoints for managing notifications. - -## POST `/notif/mark-ack` (auth required) - -### Description - -The `/notif/mark-ack` endpoint marks the specified notification -as "acknowledged". This indicates that the user has chosen to either -dismiss or act on this notification. - -### Parameters - -| Name | Description | Default Value | -| ---- | ----------- | -------- | -| uid | UUID associated with the notification | **required** | - -### Response - -This endpoint responds with an empty object (`{}`). - - -## POST `/notif/mark-read` (auth required) - -### Description - -The `/notif/mark-read` endpoint marks that the specified notification -has been shown to the user. It will not "pop up" as a new notification -if they load the gui again. - -### Parameters - -| Name | Description | Default Value | -| ---- | ----------- | -------- | -| uid | UUID associated with the notification | **required** | - -### Response - -This endpoint responds with an empty object (`{}`). - -### Request Example - -```javascript -await fetch("https://api.puter.local/notif/mark-read", { - headers: { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - body: JSON.stringify({ - uid: 'a14ea3d5-828b-42f9-9613-35f43b0a3cb8', - }), - method: "POST", -}); -``` -## ENTITY STORAGE `puter-notifications` - -The `puter-notifications` driver is an Entity Storage driver. -It is read-only. - -### Request Examples - -#### Select Unread Notifications - -```javascript -await fetch("http://api.puter.localhost:4100/drivers/call", { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - interface: 'puter-notifications', - method: 'select', - args: { predicate: ['unread'] } - }), - "method": "POST", -}); -``` - -#### Select First 200 Notifications - -```javascript -await fetch("http://api.puter.localhost:4100/drivers/call", { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - interface: 'puter-notifications', - method: 'select', - args: {} - }), - "method": "POST", -}); -``` - -#### Select Next 200 Notifications - -```javascript -await fetch("http://api.puter.localhost:4100/drivers/call", { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - interface: 'puter-notifications', - method: 'select', - args: { offset: 200 } - }), - "method": "POST", -}); -``` diff --git a/doc/api/share.md b/doc/api/share.md deleted file mode 100644 index 8b2a81910..000000000 --- a/doc/api/share.md +++ /dev/null @@ -1,367 +0,0 @@ -# Share Endpoints - -Share endpoints allow sharing files with other users. - -## POST `/share` (auth required) - -### Description - -The `/share` endpoint shares 1 or more filesystem items -with one or more recipients. The recipients will receive -some notification about the shared item, making this -different from calling `/grant-user-user` with a permission. - -When users are **specified by email** they will receive -a [share link](./concepts/share-link.md). - -Each item specified in the `shares` property is a tag-typed -object of type `fs-share` or `app-share`. - -#### File Shares (`fs-share`) - -File shares grant permission to a file or directory. By default -this is read permission. If `access` is specified as `"write"`, -then write permission will be granted. - -#### App Shares (`app-share`) - -App shares grant permission to read a protected app. - -##### subdomain permission -If there is a subdomain associated with the app, and the owner -of the subdomain is the same as the owner of the app, then -permission to access the subdomain will be granted. -Note that the subdomain is only associated if the subdomain -entry has `associated_app_id` set according to the app's id, -and will not be considered "associated" if only the index_url -happens to match the subdomain url. - -##### appdata permission -If the app has `shared_appdata` set to `true` in its metadata -object, the recipient of the share will also get write permission -to the app owner's corresponding appdata directory. The appdata -directory must exist for this to work as expected -(otherwise the permission rewrite rule fails since the uuid -can't be determined). - -### Example - -```json -{ - "recipients": [ - "user_that_gets_shared_to", - "another@example.com" - ], - "shares": [ - { - "$": "app-share", - "name": "some-app-name" - }, - { - "$": "app-share", - "uid": "app-SOME-APP-UID" - }, - { - "$": "fs-share", - "path": "/some/file/or/directory" - }, - { - "$": "fs-share", - "path": "SOME-FILE-UUID" - } - ] -} -``` - -### Parameters - -- **recipients** _- required_ - - **accepts:** `string | Array` - - **description:** - recipients for the filesystem entries being shared. - - **notes:** - - validation on `string`: email or username - - requirement of at least one value -- **shares:** _- required_ - - **accepts:** `object | Array` - - object is [type-tagged](./type-tagged.md) - - type is either [file-share](./types/file-share.md) - or [app-share](./types/app-share.md) - - **notes:** - - requirement that file/directory or app exists - - requirement of at least one entry -- **dry_run:** _- optional_ - - **accepts:** `bool` - - **description:** - when true, only validation will occur - -### Response - -- **$:** `api:share` -- **$version:** `v0.0.0` -- **status:** one of: `"success"`, `"mixed"`, `"aborted"` -- **recipients:** array of: `api:status-report` or - `heyputer:api/APIError` -- **paths:** array of: `api:status-report` or - `heyputer:api/APIError` -- **dry_run:** `true` if present - -### Request Example - -```javascript -await fetch("http://puter.localhost:4100/share", { - headers: { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - body: JSON.stringify({ - recipients: [ - "user_that_gets_shared_to", - "another@example.com" - ], - shares: [ - { - $: "app-share", - name: "some-app-name" - }, - { - $: "app-share", - uid: "app-SOME-APP-UID" - }, - { - $: "fs-share", - path: "/some/file/or/directory" - }, - { - $: "fs-share", - path: "SOME-FILE-UUID" - } - ] - }), - method: "POST", -}); -``` - -### Success Response - -```json -{ - "$": "api:share", - "$version": "v0.0.0", - "status": "success", - "recipients": [ - { - "$": "api:status-report", - "status": "success" - } - ], - "paths": [ - { - "$": "api:status-report", - "status": "success" - } - ], - "dry_run": true -} -``` - -### Error response (missing file) - -```json -{ - "$": "api:share", - "$version": "v0.0.0", - "status": "mixed", - "recipients": [ - { - "$": "api:status-report", - "status": "success" - } - ], - "paths": [ - { - "$": "heyputer:api/APIError", - "code": "subject_does_not_exist", - "message": "File or directory not found.", - "status": 404 - } - ], - "dry_run": true -} -``` - -### Error response (missing user) - -```json -{ - "$": "api:share", - "$version": "v0.0.0", - "status": "mixed", - "recipients": [ - { - "$": "heyputer:api/APIError", - "code": "user_does_not_exist", - "message": "The user `non_existing_user` does not exist.", - "username": "non_existing_user", - "status": 422 - } - ], - "paths": [ - { - "$": "api:status-report", - "status": "success" - } - ], - "dry_run": true -} -``` - -## POST `/sharelink/check` (no auth) - -### Description - -The `/sharelink/check` endpoint verifies that a token provided -by a share link is valid. - -### Example - -```javascript -await fetch(`${config.api_origin}/sharelink/check`, { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - token: '...', - }), - "method": "POST", -}); -``` - -### Parameters - -- **token:** _- required_ - - **accepts:** `string` - The token from the querystring parameter - -### Response - -A type-tagged object, either of type `api:share` or `api:error` - -### Success Response - -```json -{ - "$": "api:share", - "uid": "836671d4-ac5d-4bd3-bc0a-ec357e0d8f02", - "email": "asdf@example.com" -} -``` - -### Error Response - -```json -{ - "$": "api:error", - "message":"Field `token` is required.", - "key":"token", - "code":"field_missing" -} -``` - -## POST `/sharelink/apply` (no auth) - -### Description - -The `/sharelink/apply` endpoint applies a share to the current -user **if and only if** that user's email is confirmed and matches -the email associated with the share. - -### Example - -```javascript -await fetch(`${config.api_origin}/sharelink/apply`, { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - uid: '836671d4-ac5d-4bd3-bc0a-ec357e0d8f02', - }), - "method": "POST", -}); -``` - -### Parameters - -- **uid:** _- required_ - - **accepts:** `string` - The uid of an existing share, received using `/sharelink/check` - -### Response - -A type-tagged object, either of type `api:status-report` or `api:error` - -### Success Response - -```json -{"$":"api:status-report","status":"success"} -``` - -### Error Response - -```json -{ - "message": "This share can not be applied to this user.", - "code": "can_not_apply_to_this_user" -} -``` - -## POST `/sharelink/request` (no auth) - -### Description - -The `/sharelink/request` endpoint requests the permissions associated -with a share link to the issuer of the share (user that sent the share). -This can be used when a user is logged in, but that user's email does -not match the email associated with the share. - -### Example - -```javascript -await fetch(`${config.api_origin}/sharelink/request`, { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - uid: '836671d4-ac5d-4bd3-bc0a-ec357e0d8f02', - }), - "method": "POST", -}); -``` - -### Parameters - -- **uid:** _- required_ - - **accepts:** `string` - The uid of an existing share, received using `/sharelink/check` - -### Response - -A type-tagged object, either of type `api:status-report` or `api:error` - -### Success Response - -```json -{"$":"api:status-report","status":"success"} -``` - -### Error Response - -```json -{ - "message": "This share is already valid for this user; POST to /apply for access", - "code": "no_need_to_request" -} -``` diff --git a/doc/self-hosters/config.md b/doc/self-hosters/config.md deleted file mode 100644 index 6ed403908..000000000 --- a/doc/self-hosters/config.md +++ /dev/null @@ -1,81 +0,0 @@ -# Configuring Puter - -## Terminology - -- **root** - the "top level" of configuration; if a key-value pair is in/at "the root" - that means it is **not in a nested object** - (ex: values under "services" are **not** at the root). - -## Config Locations - -Running the server will generate a configuration file in one of these locations: -- `config/config.json` when [Using Docker](#using-docker) -- `volatile/config/config.json` in [Local Development](#local-development) -- `/etc/puter/config.json` on a server (or within a Docker container) - -## Editing Configuration - -For a list of all possible config values, see [config_values.md](./config_values.md) - -Instead of editing the generated `config.json`, you can make a config file -that references it. This makes it easier to maintain if you frequently update -Puter, since you can then just delete `config.json` to get new defaults. - -For example, a `local.json` might look like this: - -```json -{ - // Always include this header - "$version": "v1.1.0", - "$requires": [ - "config.json" - ], - "config_name": "local", - - // Your custom configuration - "domain": "my-puter.example.com" -} -``` - -To use `local.json` instead of `config.json` you will need to set the -environment variable `PUTER_CONFIG_PROFILE=local` in the context where -you are running Puter. - -## Sample Configuration - -The default configuration generated by Puter will look -something like the following (updated 2025-02-26): - -```json -{ - "config_name": "generated default config", - "mod_directories": [ - "{source}/../extensions" - ], - "env": "dev", - "nginx_mode": true, - "server_id": "localhost", - "http_port": "auto", - "domain": "puter.localhost", - "protocol": "http", - "contact_email": "hey@example.com", - "services": { - "database": { - "engine": "sqlite", - "path": "puter-database.sqlite" - }, - "dynamo" :{"path":"./puter-ddb"} - }, - "cookie_name": "...", - "jwt_secret": "...", - "url_signature_secret": "...", - "private_uid_secret": "...", - "private_uid_namespace": "...", - "": null -} -``` - -## Root-Level Parameters - -- **domain** - origin for Puter. Do **not** include URL schema (the 'http(s)://' portion) -- diff --git a/doc/self-hosters/config_values.md b/doc/self-hosters/config_values.md deleted file mode 100644 index 2e8bf4e09..000000000 --- a/doc/self-hosters/config_values.md +++ /dev/null @@ -1,72 +0,0 @@ -### `domain` - -Domain name of the Puter instance. This may be used to generate URLs -in the UI. If "allow_all_host_values" is false or undefined, the domain -will be used to validate the host header of incoming requests. - -#### Examples - -- `"domain": "example.com"` -- `"domain": "subdomain.example.com"` - -### `protocol` - -The protocol to use for URLs. This should be either "http" or "https". - -#### Examples - -- `"protocol": "http"` -- `"protocol": "https"` - -### `static_hosting_domain` - -This domain name will be used for public site URLs. For example: when -you right-click a directory and choose "Publish as Website". -This domain should point to the same server. If you have a LAN configuration -you could set this to something like -`site.192.168.555.12.nip.io`, replacing -`192.168.555.12` with a valid IP address belonging to the server. - - -### `allow_all_host_values` - -If true, Puter will accept any host header value in incoming requests. -This is useful for development, but should be disabled in production. - - -### `allow_nipio_domains` - -If true, Puter will allow requests with host headers that end in nip.io. -This is useful for development, LAN, and VPN configurations. - - -### `http_port` - -The port to listen on for HTTP requests. - - -### `enable_public_folders` - -If true, any /username/Public directory will be available to all -users, including anonymous users. - - -### `disable_temp_users` - -If true, new users will see the login/signup page instead of being -automatically logged in as a temporary user. - - -### `disable_user_signup` - -If true, the signup page will be disabled and the backend will not -accept new user registrations. - - -### `disable_fallback_mechanisms` - -A general setting to prevent any fallback behavior that might -"hide" errors. It is recommended to set this to true when -debugging, testing, or developing new features. - - diff --git a/doc/self-hosters/domains.md b/doc/self-hosters/domains.md deleted file mode 100644 index ec1edd2e9..000000000 --- a/doc/self-hosters/domains.md +++ /dev/null @@ -1,85 +0,0 @@ -# Configuring Domains for Self-Hosted Puter - -## Local Network Configuration - -### Prerequisite Conditions - -Ensure the hosting device has a static IP address to prevent potential connectivity issues due to IP changes. This setup will enable seamless access to Puter and its services across your local network. - -### Using `nip.io` - -We recommend this configuration for LAN setups. All you need to do is set the following -at root level in your configuration file: - -```json - "allow_nipio_domains": true -``` - -Puter requires multiple origins to work correctly. `nip.io` is a wildcard DNS for IP addresses, -so Puter can still have multiple subdomains and you don't need to configure your own DNS or -hosts file. - -### Using Hosts Files - -The hosts file is a straightforward way to map domain names to IP addresses on individual devices. It's simple to set up but requires manual changes on each device that needs access to the domains. - -#### Windows -1. Open Notepad as an administrator. -2. Open the file located at `C:\Windows\System32\drivers\etc\hosts`. -3. Add lines for your domain and subdomain with the server's IP address, in the - following format: - ``` - 192.168.1.10 puter.local - 192.168.1.10 api.puter.local - ``` - -#### For macOS and Linux: -1. Open a terminal. -2. Edit the hosts file with a text editor, e.g., `sudo nano /etc/hosts`. -3. Add lines for your domain and subdomain with the server's IP address, in the - following format: - ``` - 192.168.1.10 puter.local - 192.168.1.10 api.puter.local - ``` -4. Save and exit the editor. - - -### Using Router Configuration - -Some routers allow you to add custom DNS rules, letting you configure domain names network-wide without touching each device. - -1. Access your router’s admin interface (usually through a web browser). -2. Look for DNS or DHCP settings. -3. Add custom DNS mappings for `puter.local` and `api.puter.local` to the hosting device's IP address. -4. Save the changes and reboot the router if necessary. - -This method's availability and steps may vary depending on your router's model and firmware. - -### Using Local DNS - -Setting up a local DNS server on your network allows for flexible and scalable domain name resolution. This method works across all devices automatically once they're configured to use the DNS server. - -#### Options for DNS Software: - -- **Pi-hole**: Acts as both an ad-blocker and a DNS server. Ideal for easy setup and maintenance. -- **BIND9**: Offers comprehensive DNS server capabilities for complex setups. -- **dnsmasq**: Lightweight and suitable for smaller networks or those new to running a DNS server. - -**contributors note:** feel free to add any software you're aware of -which might help with this to the list. Also, feel free to add instructions here for specific software; our goal is for Puter to be easy to setup with tools you're already familiar with. - -#### General Steps: - -1. Choose and install DNS server software on a device within your network. -2. Configure the DNS server to resolve `puter.local` and `api.puter.local` to the IP address of your Puter hosting device. -3. Update your router's DHCP settings to distribute the DNS server's IP address to all devices on the network. - -By setting up a local DNS server, you gain the most flexibility and control over your network's domain name resolution, ensuring that all devices can access Puter and its API without manual configuration. - -## Production Configuration - -Please note the self-hosting feature is still in alpha and a public production -deployment is not recommended at this time. However, if you wish to host -publicly you can do so following the same steps you normally would to configure -a domain name and ensuring the `api` subdomain points to the server as well. diff --git a/doc/self-hosters/instructions.md b/doc/self-hosters/instructions.md deleted file mode 100644 index 4be684576..000000000 --- a/doc/self-hosters/instructions.md +++ /dev/null @@ -1,59 +0,0 @@ -# Self-Hosting Puter - -> [!WARNING] -> The self-hosted version of Puter is currently in alpha stage and should not be used in production yet. It is under active development and may contain bugs, other issues. Please exercise caution and use it for testing and evaluation purposes only. - -### Self-Hosting Differences -Currently, the self-hosted version of Puter is different in a few ways from [Puter.com](https://puter.com): -- There is no built-in way to access apps from puter.com (see below) -- Several "core" apps are missing, such as **Code** or **Draw** -- Some assets are different - -Work is ongoing to improve the **App Center** and make it available on self-hosted. -Until then, it is still possible to add apps using the **Dev Center** app. - -
- -## Configuration - -Running the server will generate a [configuration file](./config.md) in one of these locations: -- `config/config.json` when [Using Docker](#using-docker) -- `volatile/config/config.json` in [Local Development](#local-development) -- `/etc/puter/config.json` on a server (or within a Docker container) - -### Domain Name - -To access Puter on your device, you can simply go to the address printed in -the server console (usually `puter.localhost:4100`). - -To access Puter from another device on LAN, enable the following configuration: -```json -"allow_nipio_domains": true -``` - -To access Puter from another device, a domain name must be configured, as well as -an `api` subdomain. For example, `example.local` might be the domain name pointing -to the IP address of the server running puter, and `api.example.com` must point to -this address as well. This domain must be specified in the configuration file -(usually `volatile/config/config.json`) as well. - -See [domain configuration](./domains.md) for more information. - -### Configure the Port - -- You can specify a custom port by setting `http_port` to a desired value -- If you're using a reverse-proxy such as nginx or cloudflare, you should - also set `pub_port` to the public (external) port (usually `443`) -- If you have HTTPS enabled on your reverse-proxy, ensure that - `protocol` in config.json is set accordingly - -### Default User - -By default, Puter will create a user called `default_user`. -This user will have a randomly generated password, which will be printed -in the development console. -A warning will persist in the dev console until this user's -password is changed. Please login to this user and change the password as -your first step. - -
diff --git a/docker-compose.full.yml b/docker-compose.full.yml index 34d365f34..9134aa98e 100644 --- a/docker-compose.full.yml +++ b/docker-compose.full.yml @@ -110,6 +110,12 @@ services: RUSTFS_SECRET_KEY: ${S3_SECRET_KEY:-puter-secret-change-me} volumes: - ./puter/data/s3:/data + # Internal-only β€” browsers reach RustFS via nginx (`s3.`), + # which preserves the Host header for S3 signature validation and + # rides the same TLS termination as Puter. Uncomment to also expose + # 9000 directly on the host for `aws-cli` / debugging. + # ports: + # - "9000:9000" healthcheck: # RustFS exposes /health on the S3 port. Use wget (curl is not in # the slim image). @@ -150,6 +156,68 @@ services: fi restart: "no" + # ── Optional: local LLM ─────────────────────────────────────────── + # Behind the `ai` compose profile β€” only starts when explicitly opted + # into. Bring up with: + # docker compose -f docker-compose.full.yml --profile ai up -d + # When enabled, also set in your `puter/config/config.json`: + # "providers": { "ollama": { "apiBaseUrl": "http://ollama:11434" } } + # When NOT enabled, set: + # "providers": { "ollama": { "enabled": false } } + # otherwise Puter spams `ECONNREFUSED 127.0.0.1:11434` on startup. + ollama: + profiles: ["ai"] + # CPU-only out of the box; uncomment the GPU `deploy:` block below + # if you've got nvidia-docker for much faster inference. Disk + RAM + # scale with the model β€” `tinyllama` (1.1B, ~640 MB on disk, ~700 + # MB RAM) is the cheapest sane default. Swap via OLLAMA_DEFAULT_MODEL. + image: ollama/ollama:latest + container_name: puter-ollama + restart: unless-stopped + volumes: + - ./puter/data/ollama:/root/.ollama + # Uncomment to expose Ollama directly on the host (`localhost:11434`) + # for `ollama` CLI / OpenAI-API compatible tools. Internal-only by default. + # ports: + # - "11434:11434" + healthcheck: + test: + ["CMD-SHELL", "ollama list >/dev/null 2>&1 || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 15s + # GPU passthrough (NVIDIA). Requires nvidia-container-toolkit on host. + # deploy: + # resources: + # reservations: + # devices: + # - driver: nvidia + # count: all + # capabilities: [gpu] + + ollama-init: + profiles: ["ai"] + # One-shot β€” ensures the default model is present. `ollama pull` is + # idempotent: present-and-up-to-date β†’ fast no-op; missing β†’ downloads. + image: ollama/ollama:latest + container_name: puter-ollama-init + depends_on: + ollama: + condition: service_healthy + environment: + OLLAMA_HOST: http://ollama:11434 + OLLAMA_DEFAULT_MODEL: ${OLLAMA_DEFAULT_MODEL:-tinyllama} + entrypoint: + - /bin/sh + - -c + - | + set -e + echo "[ollama-init] ensuring $${OLLAMA_DEFAULT_MODEL}" + ollama pull "$${OLLAMA_DEFAULT_MODEL}" + echo "[ollama-init] done" + restart: "no" + puter: # image: ghcr.io/heyputer/puter:latest pull_policy: always diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 90de1d473..cc4837a0e 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -33,7 +33,34 @@ http { keepalive 32; } - # ── HTTP (port 80) β€” catches all hostnames ───────────────────── + upstream s3_backend { + # RustFS β€” see `s3` service in docker-compose.full.yml. Browsers + # PUT/GET here for presigned-URL uploads / downloads. Routed via + # the `s3.` subdomain so signature verification works + # (Host header preserved end-to-end) and so HTTPS stays clean + # (no mixed-content from a port-9000 host publish). + server s3:9000; + keepalive 32; + } + + # ── HTTP (port 80) ───────────────────────────────────────────── + server { + listen 80; + listen [::]:80; + server_name ~^s3\.; + + location / { + proxy_pass http://s3_backend; + proxy_http_version 1.1; + # Critical: preserve the original Host so RustFS validates + # the request against the same host the URL was signed for. + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + } + server { listen 80 default_server; listen [::]:80 default_server; @@ -57,6 +84,29 @@ http { # ── HTTPS (port 443) β€” uncomment after dropping certs in ./puter/tls/ ─ # server { + # listen 443 ssl; + # listen [::]:443 ssl; + # http2 on; + # server_name ~^s3\.; + # + # ssl_certificate /etc/nginx/tls/fullchain.pem; + # ssl_certificate_key /etc/nginx/tls/privkey.pem; + # ssl_protocols TLSv1.2 TLSv1.3; + # ssl_ciphers HIGH:!aNULL:!MD5; + # ssl_session_cache shared:SSL:10m; + # ssl_session_timeout 10m; + # + # location / { + # proxy_pass http://s3_backend; + # proxy_http_version 1.1; + # proxy_set_header Host $host; + # proxy_set_header X-Real-IP $remote_addr; + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # proxy_set_header X-Forwarded-Proto $scheme; + # } + # } + # + # server { # listen 443 ssl default_server; # listen [::]:443 ssl default_server; # http2 on; diff --git a/selfhosted/docker.md b/selfhosted/docker.md index 840a095c1..6407a517c 100644 --- a/selfhosted/docker.md +++ b/selfhosted/docker.md @@ -93,6 +93,8 @@ Two files run in order: `mysql_mig_1.sql` (tables) and `mysql_mig_2.sql` (defaul The bucket must exist already β€” Puter doesn't create it. +For real AWS S3 the example above works as-is β€” virtual-hosted DNS form (`.s3.amazonaws.com`) is the AWS SDK default. For S3-compatible servers (RustFS, MinIO, fauxqs), add `"forcePathStyle": true` inside the `s3Config` block β€” virtual-hosted DNS doesn't resolve there. + > ⚠️ **S3 uses camelCase keys** (`accessKeyId` / `secretAccessKey`). DynamoDB below uses snake_case. They're not the same. ### DynamoDB (real AWS) diff --git a/selfhosted/full-stack.md b/selfhosted/full-stack.md index 10266ad59..4ebcd4da3 100644 --- a/selfhosted/full-stack.md +++ b/selfhosted/full-stack.md @@ -20,6 +20,13 @@ | `puter-s3` | `rustfs/rustfs` | S3-compatible object storage (MinIO drop-in noted in file) | | `puter-s3-init` | `amazon/aws-cli` | One-shot β€” creates the bucket on first boot, then exits | +Optional services (compose profile `ai`, opt-in): + +| Container | Image | Role | +| ------------------- | --------------- | -------------------------------------------------------------- | +| `puter-ollama` | `ollama/ollama` | Local LLM provider (CPU; GPU passthrough opt-in) | +| `puter-ollama-init` | `ollama/ollama` | One-shot β€” pulls the default model (`tinyllama`) on first boot | + State lives under `./puter/data//`. --- @@ -93,13 +100,19 @@ cat > puter/config/config.json </`). Real AWS S3 wants virtual-hosted (`.`) β€” drop this flag (or set `false`) when you swap to real S3. +- `s3.s3Config.publicEndpoint` β€” `endpoint` (`http://s3:9000`) only resolves inside the docker network; presigned upload/download URLs handed to the browser need a host-reachable URL. nginx routes the `s3.` subdomain to RustFS internally and preserves the Host header end-to-end (required for S3 signature validation), so the browser hits the same port/protocol as the rest of the app β€” no separate published port, no mixed-content surprises when you turn on TLS. Switch to `https://s3.` once you enable TLS in Step 3. Real AWS S3 doesn't need this β€” its endpoint is already public; drop the field entirely. > If you ever change `MARIADB_PASSWORD` after first boot, `.env` alone won't update MariaDB β€” its credentials are baked into `./puter/data/mariadb/` on first init. Either rotate the password inside MariaDB by hand or `docker compose down && rm -rf ./puter/data/mariadb` to start fresh. ## Step 2 β€” Point DNS at the server \[Optional\] -In your DNS provider, add **two records**: +In your DNS provider, add records for the main domain plus the subdomains Puter and nginx route on (`api.*`, `site.*`, `app.*`, `s3.*`): ``` A puter.local β†’ +A *.puter.local β†’ A puter.sitelocal β†’ A *.puter.sitelocal β†’ A puter.hostlocal β†’ @@ -131,12 +148,12 @@ A puter.devlocal β†’ A *.puter.devlocal β†’ ``` -The wildcard is required β€” Puter routes via subdomains. +The wildcards are required β€” Puter routes via subdomains (`api.*`, `app.*`, etc.) and nginx routes browser S3 traffic via `s3.*` to RustFS. -If you only need these to resolve these locally to test you can add this (any any other needed subdomain) to your hosts file +For local-only testing, add this, and any specific subdomains, your hosts file (`/etc/hosts` on macOS/Linux, `C:\Windows\System32\drivers\etc\hosts` on Windows): ``` -127.0.0.1 puter.local +127.0.0.1 puter.local s3.puter.local api.puter.local puter-app-icons.puter.sitelocal ``` ## Step 3 β€” TLS (recommended for public installs) \[Optional\] @@ -147,21 +164,31 @@ Skip this for a quick local demo. Don't skip it for users typing passwords. ```bash sudo certbot certonly --manual --preferred-challenges dns \ - -d puter.local -d puter.sitelocal -d "*.puter.sitelocal" -d puter.hostlocal -d "*.puter.hostlocal" -d puter.applocal -d "*.puter.applocal" -d puter.devlocal -d "*.puter.devlocal" + -d puter.local -d "*.puter.local" \ + -d puter.sitelocal -d "*.puter.sitelocal" \ + -d puter.hostlocal -d "*.puter.hostlocal" \ + -d puter.applocal -d "*.puter.applocal" \ + -d puter.devlocal -d "*.puter.devlocal" ``` +The cert needs to cover `*.puter.local` so that `s3.puter.local` (browser S3 endpoint), plus Puter's own `api.*` / `app.*` subdomains, all validate. + Drop the resulting `fullchain.pem` and `privkey.pem` into `./puter/tls/`. **Wire nginx to use them:** -1. Open [nginx/nginx.conf](../nginx/nginx.conf), uncomment the entire `# server { listen 443 ssl … }` block. -2. (Optional) Replace the body of the port-80 block with `return 301 https://$host$request_uri;` to force HTTPS. +1. Open [nginx/nginx.conf](../nginx/nginx.conf), uncomment **both** `# server { listen 443 ssl … }` blocks (one for `s3.*`, one for the catch-all). +2. (Optional) Replace the body of the port-80 blocks with `return 301 https://$host$request_uri;` to force HTTPS everywhere. 3. In [docker-compose.full.yml](../docker-compose.full.yml), uncomment the `443:443` port mapping under the `nginx` service. 4. In `.env`, uncomment `HTTPS_PORT=443`. 5. In `config.json`, switch: ```json { "protocol": "https", "pub_port": 443 } ``` + …and update the S3 public endpoint: + ```json + "s3": { "s3Config": { "publicEndpoint": "https://s3.puter.local", ... } } + ``` ## Step 4 β€” Bring it up @@ -192,6 +219,33 @@ docker compose -f docker-compose.full.yml logs puter | grep tmp_password Change it in Settings after first login. +## Optional: local LLM (Ollama) + +The `ollama` and `ollama-init` services live behind a compose profile so they don't run unless you ask for them. By default, `puter/config/config.json` has `"ollama": { "enabled": false }` β€” Puter skips the auto-probe entirely. To run a local model: + +1. Flip the config: + ```json + "providers": { + "ollama": { "apiBaseUrl": "http://ollama:11434" } + } + ``` +2. (Optional) Pick a model in `.env`: + ```bash + OLLAMA_DEFAULT_MODEL=tinyllama # default β€” 1.1B, ~640 MB on disk, ~700 MB RAM + # Other tiny picks: qwen2.5:0.5b, llama3.2:1b + # Larger / better: phi3.5, llama3.2, mistral + ``` +3. Bring up with the `ai` profile: + ```bash + docker compose -f docker-compose.full.yml --profile ai up -d + docker compose -f docker-compose.full.yml logs -f ollama-init + ``` + `ollama-init` exits 0 once the model is pulled. Subsequent boots find the model already on disk and the pull is a fast no-op. + +Without `--profile ai`, the `ollama` containers stay down and Puter (with `enabled: false`) doesn't try to reach them β€” the rest of the stack runs identically. + +For GPU acceleration (NVIDIA), uncomment the `deploy:` block under the `ollama` service in [docker-compose.full.yml](../docker-compose.full.yml). Requires `nvidia-container-toolkit` on the host. + ## Building from source instead of pulling If you want to test local Dockerfile changes against the full stack, uncomment the `build:` block in [docker-compose.full.yml](../docker-compose.full.yml) under the `puter` service, change `pull_policy: always` β†’ `pull_policy: never`, then: diff --git a/src/backend/clients/database/DatabaseClient.ts b/src/backend/clients/database/DatabaseClient.ts index 41ba4ea34..df78efea1 100644 --- a/src/backend/clients/database/DatabaseClient.ts +++ b/src/backend/clients/database/DatabaseClient.ts @@ -22,6 +22,7 @@ import { PuterClient } from '../types'; export interface WriteResult { insertId: number | bigint; + affectedRows: number; anyRowsAffected: boolean; } diff --git a/src/backend/clients/database/MySQLDatabaseClient.ts b/src/backend/clients/database/MySQLDatabaseClient.ts index 7ac54c798..cbdf1f495 100644 --- a/src/backend/clients/database/MySQLDatabaseClient.ts +++ b/src/backend/clients/database/MySQLDatabaseClient.ts @@ -159,9 +159,11 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient { insertId?: number; affectedRows?: number; }; + const affectedRows = header.affectedRows ?? 0; return { insertId: header.insertId ?? 0, - anyRowsAffected: (header.affectedRows ?? 0) > 0, + affectedRows, + anyRowsAffected: affectedRows > 0, }; } diff --git a/src/backend/clients/database/SqliteDatabaseClient.ts b/src/backend/clients/database/SqliteDatabaseClient.ts index 4e31d9f77..e5edb56e9 100644 --- a/src/backend/clients/database/SqliteDatabaseClient.ts +++ b/src/backend/clients/database/SqliteDatabaseClient.ts @@ -143,6 +143,7 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient { return { insertId: info.lastInsertRowid, + affectedRows: info.changes, anyRowsAffected: info.changes > 0, }; } diff --git a/src/backend/clients/database/migrations/mysql/mysql_mig_2.sql b/src/backend/clients/database/migrations/mysql/mysql_mig_2.sql index 922215098..0f930cf78 100644 --- a/src/backend/clients/database/migrations/mysql/mysql_mig_2.sql +++ b/src/backend/clients/database/migrations/mysql/mysql_mig_2.sql @@ -8,8 +8,9 @@ -- INSERT IGNORE makes it safe to re-run; uid has a UNIQUE constraint. -- -- FK temporarily disabled because apps.owner_user_id references user.id, --- and the admin (id=1) is created by DefaultUserService AFTER this --- migration runs. Once the admin row exists, the references resolve. +-- and the `system` user (id=1) is created by mysql_mig_3.sql which +-- runs after this one. Once mig 3 inserts that row, the references +-- resolve. Matches the SQLite ordering: 0002 (apps) β†’ 0025 (system user). /*!40014 SET @OLD_FK = @@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; diff --git a/src/backend/clients/database/migrations/mysql/mysql_mig_3.sql b/src/backend/clients/database/migrations/mysql/mysql_mig_3.sql new file mode 100644 index 000000000..3ad184ed1 --- /dev/null +++ b/src/backend/clients/database/migrations/mysql/mysql_mig_3.sql @@ -0,0 +1,58 @@ +-- Copyright (C) 2024-present Puter Technologies Inc. +-- +-- Default groups + the `system` user that issues the hardcoded driver +-- permission grants in `data/hardcoded-permissions.js`. Mirrors what the +-- SQLite migrations 0024_default-groups.sql + 0025_system-user.dbmig.js +-- do for the source-tree dev path; without these rows, MySQL self-host +-- signups land in groups that don't exist and the hc-user-group +-- permission scanner has no `system` user to resolve as the issuer β‡’ +-- every `/drivers/call` 403s. +-- +-- Order matters: +-- 1. system user inserted first β†’ gets id=1, owns the default apps +-- that mysql_mig_2.sql already inserted with owner_user_id=1. +-- 2. groups inserted next, all owned by system (owner_user_id=1). +-- 3. DefaultUserService later creates the admin user (id=2) and adds +-- them to the admin group, which now exists. +-- +-- INSERT IGNORE keeps it idempotent across re-runs. +-- +-- FK temporarily disabled because owner_user_id columns reference +-- user.id, and we're inserting both sides in the same transaction. + +/*!40014 SET @OLD_FK = @@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; + +INSERT IGNORE INTO `user` (`uuid`, `username`) +VALUES ('5d4adce0-a381-4982-9c02-6e2540026238', 'system'); + +INSERT IGNORE INTO `group` (`uid`, `owner_user_id`, `extra`, `metadata`) VALUES + ('26bfb1fb-421f-45bc-9aa4-d81ea569e7a5', 1, + '{"critical": true, "type": "default", "name": "system"}', + '{"title": "System", "color": "#000000"}'), + ('ca342a5e-b13d-4dee-9048-58b11a57cc55', 1, + '{"critical": true, "type": "default", "name": "admin"}', + '{"title": "Admin", "color": "#a83232"}'), + ('78b1b1dd-c959-44d2-b02c-8735671f9997', 1, + '{"critical": true, "type": "default", "name": "user"}', + '{"title": "User", "color": "#3254a8"}'), + ('b7220104-7905-4985-b996-649fdcdb3c8f', 1, + '{"critical": true, "type": "default", "name": "temp"}', + '{"title": "Temp", "color": "#888888"}'), + ('3c2dfff7-d22a-41aa-a193-59a61dac4b64', 1, + '{"type": "default", "name": "moderator"}', + '{"title": "Moderator", "color": "#a432a8"}'), + ('5e8f251d-3382-4b0d-932c-7bb82f48652f', 1, + '{"type": "default", "name": "developer"}', + '{"title": "Developer", "color": "#32a852"}'); + +-- Mirrors 0025_system-user.dbmig.js: system grants the admin group +-- unrestricted `driver` access. Hardcoded permission rules in +-- data/hardcoded-permissions.js layer additional per-group grants on +-- top, but this row is the canonical "admin can drive everything" link. +INSERT IGNORE INTO `user_to_group_permissions` (`user_id`, `group_id`, `permission`, `extra`) +SELECT u.id, g.id, 'driver', '{}' +FROM `user` u, `group` g +WHERE u.username = 'system' + AND g.uid = 'ca342a5e-b13d-4dee-9048-58b11a57cc55'; + +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FK */; diff --git a/src/backend/clients/email/EmailClient.ts b/src/backend/clients/email/EmailClient.ts index cd3437eae..cd924d778 100644 --- a/src/backend/clients/email/EmailClient.ts +++ b/src/backend/clients/email/EmailClient.ts @@ -162,7 +162,11 @@ export class EmailClient extends PuterClient { */ async sendRaw(options: SendMailOptions): Promise { if (!this.transport) { - throw new Error('EmailClient transport is not configured'); + console.warn( + '[email] attempted to send email without transport. If you need to send email, configure an SMTP transport in your config file (see docs for details). Email content:', + options, + ); + return; } await this.transport.sendMail({ from: options.from ?? this.defaultFrom(), diff --git a/src/backend/clients/s3/S3Client.ts b/src/backend/clients/s3/S3Client.ts index b98ed02a7..98bc71a2a 100644 --- a/src/backend/clients/s3/S3Client.ts +++ b/src/backend/clients/s3/S3Client.ts @@ -46,7 +46,9 @@ type S3CommandSender = Pick; export class S3Client extends PuterClient { private clientMap = new Map(); + private presignClientMap = new Map(); private awsConfig: Partial = {}; + private presignAwsConfig: Partial | null = null; private fauxqsServer: FauxqsServer | null = null; private useProviderChain = false; @@ -70,10 +72,12 @@ export class S3Client extends PuterClient { // Real S3 / S3-compatible endpoint const { endpoint, + publicEndpoint, accessKeyId, secretAccessKey, region, useCredentialChain, + forcePathStyle, } = s3Conf.s3Config; if (useCredentialChain) { @@ -86,7 +90,22 @@ export class S3Client extends PuterClient { endpoint, credentials: { accessKeyId, secretAccessKey }, ...(region ? { region } : {}), + // Defaults to virtual-hosted style (real-AWS S3 native). + // S3-compatible servers (RustFS, MinIO, fauxqs) need + // `forcePathStyle: true` β€” see `IS3RemoteConfig`. + ...(forcePathStyle === undefined ? {} : { forcePathStyle }), }; + // Separate config for clients that mint browser-facing + // presigned URLs. Defaults to the same endpoint when + // unset, so prod (single public S3 endpoint) needs no + // change. Self-hosters with a docker-internal endpoint + // override this to a host-reachable URL. + if (publicEndpoint && publicEndpoint !== endpoint) { + this.presignAwsConfig = { + ...this.awsConfig, + endpoint: publicEndpoint, + }; + } } console.log('[s3] configured with remote endpoint'); @@ -153,7 +172,11 @@ export class S3Client extends PuterClient { for (const client of this.clientMap.values()) { client.destroy(); } + for (const client of this.presignClientMap.values()) { + client.destroy(); + } this.clientMap.clear(); + this.presignClientMap.clear(); } // ------------------------------------------------------------------ @@ -188,6 +211,39 @@ export class S3Client extends PuterClient { return client; } + /** + * Client used to generate browser-facing presigned URLs. When + * `s3Config.publicEndpoint` is set, this returns a client bound to + * that endpoint β€” its signatures resolve against the public host + * the browser will actually hit. When unset, falls back to the + * regular client (prod behavior: one public endpoint everywhere). + */ + getForPresign( + region = this.config.s3_region || this.config.region || 'us-west-2', + ): AwsS3Client { + if (!this.presignAwsConfig) return this.get(region); + + const existing = this.presignClientMap.get(region); + if (existing) return existing; + + const client = new AwsS3Client({ + region, + requestStreamBufferSize: 32 * 1024, + requestHandler: new NodeHttpHandler({ + socketTimeout: 5000, + httpsAgent: new HttpsAgent({ + maxSockets: 500, + keepAlive: true, + keepAliveMsecs: 1000, + }), + }), + ...this.presignAwsConfig, + }); + + this.presignClientMap.set(region, client); + return client; + } + // ------------------------------------------------------------------ // Legacy storage migration // ------------------------------------------------------------------ diff --git a/src/backend/controllers/auth/AuthController.js b/src/backend/controllers/auth/AuthController.js index e70420698..9ea5dbf02 100644 --- a/src/backend/controllers/auth/AuthController.js +++ b/src/backend/controllers/auth/AuthController.js @@ -32,6 +32,7 @@ import { verify as verifyOtp, } from '../../services/auth/OTPUtil.js'; import { cleanEmail, isBlockedEmail } from '../../util/email.js'; +import { sessionCookieFlags } from '../../util/cookieFlags.js'; import { generate_identifier } from '../../util/identifier.js'; import { getTaskbarItems } from '../../util/taskbarItems.js'; import { @@ -2273,8 +2274,7 @@ export class AuthController extends PuterController { req.actor.session.uid, ); res.cookie(this.config.cookie_name, sessionToken, { - sameSite: 'none', - secure: true, + ...sessionCookieFlags(this.config), httpOnly: true, }); res.status(204).end(); @@ -2392,8 +2392,7 @@ export class AuthController extends PuterController { // HTTP-only cookie gets the session token res.cookie(this.config.cookie_name, sessionToken, { - sameSite: 'none', - secure: true, + ...sessionCookieFlags(this.config), httpOnly: true, }); diff --git a/src/backend/controllers/oidc/OIDCController.ts b/src/backend/controllers/oidc/OIDCController.ts index 52d995160..94cb30602 100644 --- a/src/backend/controllers/oidc/OIDCController.ts +++ b/src/backend/controllers/oidc/OIDCController.ts @@ -21,6 +21,7 @@ import type { Request, Response } from 'express'; import { HttpError } from '../../core/http/HttpError.js'; import type { PuterRouter } from '../../core/http/PuterRouter.js'; import { PuterController } from '../types.js'; +import { sessionCookieFlags } from '../../util/cookieFlags.js'; const REVALIDATION_COOKIE_NAME = 'puter_revalidation'; const REVALIDATION_EXPIRY_SEC = 300; @@ -374,8 +375,8 @@ export class OIDCController extends PuterController { const token = this.services.oidc.signRevalidation(user.uuid); res.cookie(REVALIDATION_COOKIE_NAME, token, { - sameSite: 'lax', - secure: true, + // Revalidation flow is same-site only β€” `lax` even on HTTPS. + ...sessionCookieFlags(this.config, { crossSite: false }), httpOnly: true, maxAge: REVALIDATION_EXPIRY_SEC * 1000, path: '/', @@ -556,8 +557,7 @@ if (window.opener) { const cookieName = this.config.cookie_name ?? 'puter_token'; res.cookie(cookieName, sessionToken, { - sameSite: 'none', - secure: true, + ...sessionCookieFlags(this.config), httpOnly: true, }); diff --git a/src/backend/services/auth/AuthService.ts b/src/backend/services/auth/AuthService.ts index 55fad8446..e8ad3e1af 100644 --- a/src/backend/services/auth/AuthService.ts +++ b/src/backend/services/auth/AuthService.ts @@ -24,6 +24,7 @@ import type { UserRow } from '../../stores/user/UserStore'; import type { LayerInstances } from '../../types'; import type { puterServices } from '../index'; import { PuterService } from '../types'; +import { sessionCookieFlags } from '../../util/cookieFlags.js'; import type { AccessTokenPayload, AnyTokenPayload, @@ -677,8 +678,7 @@ export class AuthService extends PuterService { // subdomains β€” each app sees only its own cookie. const options: Record = { httpOnly: true, - secure: true, - sameSite: 'none', + ...sessionCookieFlags(this.config), maxAge: 7 * 24 * 60 * 60 * 1000, // 7 days path: '/', }; diff --git a/src/backend/services/permission/PermissionService.ts b/src/backend/services/permission/PermissionService.ts index 58a946559..e989151d3 100644 --- a/src/backend/services/permission/PermissionService.ts +++ b/src/backend/services/permission/PermissionService.ts @@ -469,6 +469,22 @@ export class PermissionService extends PuterService { * `hardcoded-permissions.js`, merged with any runtime grants registered * through `registerSystemGrantForEveryone` / `registerSystemGrantForUsers`. */ + /** + * Hardcoded user-group permissions, granted by `system` (the only + * issuer). DB-free: the default groups are static fixtures we never + * assign at runtime via DB, so we infer membership from the actor: + * - `username === 'admin'` β†’ admin group + * - `email_confirmed === true` β†’ `default_user_group` + * - otherwise β†’ `default_temp_group` + * + * The admin username + group UID are matched verbatim against + * `DefaultUserService` and the seed migration; if either is renamed, + * update both ends. + * + * Permissions for non-default groups (custom operator-managed + * groups) still flow through `#scanUserGroup`, which reads + * `user_to_group_permissions` directly. + */ async #scanHcUserGroupUser( actor: Actor, options: string[], @@ -477,78 +493,50 @@ export class PermissionService extends PuterService { if (actor.app || actor.accessToken) return; if (!actor.user?.id) return; - const memberGroups = await this.stores.group.listGroupsWithMember( - actor.user.id, - ); - if (memberGroups.length === 0) return; + const userGroupUid = this.config.default_user_group; + const tempGroupUid = this.config.default_temp_group; + const isAdmin = actor.user.username === 'admin'; + const inferredGroupUid = isAdmin + ? 'ca342a5e-b13d-4dee-9048-58b11a57cc55' // admin group + : actor.user.email_confirmed + ? userGroupUid + : tempGroupUid; + if (!inferredGroupUid) return; - const groupByUid: Record = {}; - for (const g of memberGroups) { - groupByUid[g.uid] = { id: g.id, uid: g.uid }; - } + const hcSystem = + ( + hardcoded_user_group_permissions as Record< + string, + Record> + > + ).system ?? {}; - // Compose the effective issuer β†’ group β†’ permission β†’ data map by - // merging the imported hardcoded data with runtime-registered system - // grants. Runtime grants are always attributed to the `system` issuer. - const hcMap = hardcoded_user_group_permissions as Record< - string, - Record> - >; - const hasRuntimeGrants = - Object.keys(this.systemGrantsByGroupUid).length > 0; - const byIssuer: Record< - string, - Record> - > = hasRuntimeGrants - ? { ...hcMap, system: { ...(hcMap.system ?? {}) } } - : hcMap; - if (hasRuntimeGrants) { - for (const [gUid, perms] of Object.entries( - this.systemGrantsByGroupUid, - )) { - byIssuer.system[gUid] = { - ...(byIssuer.system[gUid] ?? {}), - ...perms, - }; - } - } + // Hardcoded grants + runtime `registerSystemGrant*` additions. + // Runtime grants always shadow the static map for the same key. + const groupPerms: Record = { + ...(hcSystem[inferredGroupUid] ?? {}), + ...(this.systemGrantsByGroupUid[inferredGroupUid] ?? {}), + }; + if (Object.keys(groupPerms).length === 0) return; - for (const issuerUsername of Object.keys(byIssuer)) { - const issuerUser = - await this.stores.user.getByUsername(issuerUsername); - if (!issuerUser) continue; - const issuerActor = this.#userToActor(issuerUser); - const issuerGroups = byIssuer[issuerUsername]; - - for (const groupUid of Object.keys(issuerGroups)) { - if (!groupByUid[groupUid]) continue; - const issuerGroupPerms = issuerGroups[groupUid]; - - for (const permission of options) { - if ( - !Object.prototype.hasOwnProperty.call( - issuerGroupPerms, - permission, - ) - ) - continue; - const issuerReading = await this.scan( - issuerActor, - permission, - ); - reading.push({ - $: 'path', - via: 'hc-user-group', - has_terminal: readingHasTerminal(issuerReading), - permission, - data: issuerGroupPerms[permission], - holder_username: actor.user.username, - issuer_username: issuerUsername, - reading: issuerReading, - group_id: groupByUid[groupUid].id, - }); - } + for (const permission of options) { + if (!Object.prototype.hasOwnProperty.call(groupPerms, permission)) { + continue; } + reading.push({ + $: 'path', + via: 'hc-user-group', + // `system` is the issuer; `isSystemActor` short-circuits + // any scan to grant, so the chain is terminal by + // definition β€” no recursive verify needed. + has_terminal: true, + permission, + data: groupPerms[permission], + holder_username: actor.user.username, + issuer_username: 'system', + reading: null, + vgroup_id: inferredGroupUid, + }); } } diff --git a/src/backend/stores/fs/S3ObjectStore.ts b/src/backend/stores/fs/S3ObjectStore.ts index 1c9a0aeec..080181fd0 100644 --- a/src/backend/stores/fs/S3ObjectStore.ts +++ b/src/backend/stores/fs/S3ObjectStore.ts @@ -55,6 +55,15 @@ export class S3ObjectStore extends PuterStore { return this.clients.s3.get(region); } + /** + * Client to use when minting presigned URLs handed back to the + * browser. Same instance as `#getClientForRegion` unless + * `s3Config.publicEndpoint` is set β€” see `clients/s3/S3Client.ts`. + */ + #getPresignClientForRegion(region: string): S3Client { + return this.clients.s3.getForPresign(region); + } + // Older entries (migrated from v1) can have a null bucketRegion; callers // use this to fall back to the configured default instead of erroring. resolveRegion(region?: string | null): string { @@ -103,6 +112,7 @@ export class S3ObjectStore extends PuterStore { region: string, ): Promise { const client = this.#getClientForRegion(region); + const presignClient = this.#getPresignClientForRegion(region); const now = Date.now(); const settledResults = await Promise.allSettled( filesMetadata.map(async (fileMetadata) => { @@ -122,7 +132,7 @@ export class S3ObjectStore extends PuterStore { Key: fileMetadata.objectKey, ContentType: fileMetadata.contentType, }); - const url = await getSignedUrl(client, command, { + const url = await getSignedUrl(presignClient, command, { expiresIn: expiresInSeconds, }); return { @@ -244,7 +254,7 @@ export class S3ObjectStore extends PuterStore { input: SignedMultipartPartUrlsInput, region: string, ): Promise { - const client = this.#getClientForRegion(region); + const presignClient = this.#getPresignClientForRegion(region); const expiresInSeconds = Math.max( 60, Math.min(60 * 60, input.expiresInSeconds), @@ -258,7 +268,7 @@ export class S3ObjectStore extends PuterStore { UploadId: input.multipartUploadId, PartNumber: partNumber, }); - const url = await getSignedUrl(client, command, { + const url = await getSignedUrl(presignClient, command, { expiresIn: expiresInSeconds, }); return { diff --git a/src/backend/types.ts b/src/backend/types.ts index 53464f155..99120a859 100644 --- a/src/backend/types.ts +++ b/src/backend/types.ts @@ -249,9 +249,24 @@ export interface IS3LocalConfig { export interface IS3RemoteConfig { useCredentialChain?: boolean; endpoint: string; + /** + * Endpoint used when generating presigned URLs handed to clients + * (browser uploads/downloads). Defaults to `endpoint`. Set this when + * the server-side S3 endpoint isn't reachable from the browser β€” e.g. + * self-host with `endpoint: http://s3:9000` (docker-internal) and + * `publicEndpoint: http://localhost:9000` (host-published port). + */ + publicEndpoint?: string; accessKeyId: string; secretAccessKey: string; region?: string; + /** + * Use path-style URLs (`/`) instead of virtual-hosted + * style (`.`). Defaults to AWS SDK's default (virtual- + * hosted, which only works on real AWS S3). Set `true` for S3-compatible + * servers (RustFS, MinIO, fauxqs) where DNS-style addressing fails. + */ + forcePathStyle?: boolean; } export interface IS3Config { diff --git a/src/backend/util/cookieFlags.ts b/src/backend/util/cookieFlags.ts new file mode 100644 index 000000000..67f697116 --- /dev/null +++ b/src/backend/util/cookieFlags.ts @@ -0,0 +1,45 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import type { IConfig } from '../types'; + +/** + * `sameSite: 'none'` requires `secure: true`, and a `secure` cookie is + * silently dropped by browsers over plain HTTP. Self-host without TLS + * (`protocol: http`) needs the matched `secure: false` + `sameSite: + * 'lax'` pair, otherwise the cookie never lands and every authenticated + * request 401s. + * + * Pass `crossSite: true` for cookies that need to be sent on cross-site + * navigation (the default β€” matches the original prod-on-HTTPS behavior + * which used `sameSite: 'none'`). Pass `crossSite: false` for + * same-site-only cookies (e.g. revalidation flow that never crosses + * origins) β€” those can stay `lax` even on HTTPS. + */ +export function sessionCookieFlags( + config: IConfig, + opts: { crossSite?: boolean } = {}, +): { sameSite: 'none' | 'lax'; secure: boolean } { + const isHttps = config.protocol === 'https'; + const crossSite = opts.crossSite ?? true; + return { + sameSite: isHttps && crossSite ? 'none' : 'lax', + secure: isHttps, + }; +}