App telemetry user iteration (#2188)

* add app-telemetry interface, and user-iteration feature

* Rid app-user-count.ts from workspace imports

* Remove semicolon at the end of query :(

extensions/api.d.ts

@@ -11,6 +11,11 @@ import type { RequestHandler } from 'express';
 import type FSNodeContext from '../src/backend/src/filesystem/FSNodeContext.js';
 import type helpers from '../src/backend/src/helpers.js';
 import type * as ExtensionControllerExports from './ExtensionController/src/ExtensionController.ts';
+import { Context } from '@heyputer/backend/src/util/context.js';
+import config from '../volatile/config/config.json'
+import APIError from '@heyputer/backend/src/api/APIError.js';
+import query from '@heyputer/backend/src/om/query/query';
+
 declare global {
     namespace Express {
         interface Request {
@@ -34,6 +39,24 @@ interface EndpointOptions {
     }
 }
 
+// Driver interface types
+type ParameterDefinition = {
+    type: 'string' | 'number' | 'boolean' | 'object' | 'array';
+    optional: boolean;
+};
+type MethodDefinition = {
+    description: string;
+    parameters: Record<string, ParameterDefinition>;
+};
+type DriverInterface = {
+    description: string;
+    methods: Record<string, MethodDefinition>;
+};
+
+
+
+
+
 type HttpMethod = 'get' | 'post' | 'put' | 'delete' | 'patch';
 
 export type AddRouteFunction = (path: string, options: EndpointOptions, handler: RequestHandler) => void;
@@ -49,6 +72,8 @@ interface CoreRuntimeModule {
     util: {
         helpers: typeof helpers,
     }
+    Context: typeof Context,
+    APIError: typeof APIError
 }
 
 interface FilesystemModule {
@@ -72,10 +97,15 @@ interface Extension extends RouterMethods {
         run<T>(label: string, fn: () => T): T;
         run<T>(fn: () => T): T;
     },
+    config:  Record<string | number | symbol, any>,
     on<T extends unknown[]>(event: string, listener: (...args: T) => void): void, // TODO DS: type events better
-    import(module: 'data'): { db: BaseDatabaseAccessService, kv: DBKVStore, cache: unknown }// TODO DS: type cache better
+    on(event: 'create.drivers', listener: (event: {createDriver: (interface: string, service: string, executors: any)=>any}) => void),
+    on(event: 'create.permissions', listener: (event: {grant_to_everyone: (permission: string) => void, grant_to_users: (permission: string) => void})=>void)
+    on(event: 'create.interfaces', listener: (event: {createInterface: (interface: string, interfaces: DriverInterface) => void}) => void)
+    import(module: 'data'): { db: BaseDatabaseAccessService, kv: DBKVStore & {get: (string) => void, set: (string, string) => void}, cache: unknown }// TODO DS: type cache better
     import(module: 'core'): CoreRuntimeModule,
     import(module: 'fs'): FilesystemModule,
+    import(module: 'query'): typeof query,
     import(module: 'extensionController'): typeof ExtensionControllerExports
     import<T extends `service:${keyof ServiceNameMap}` | (string & {})>(module: T): T extends `service:${infer R extends keyof ServiceNameMap}`
         ? ServiceNameMap[R]
@@ -85,6 +115,6 @@ interface Extension extends RouterMethods {
 declare global {
     // Declare the extension variable
     const extension: Extension;
-    const config: Record<string | number | symbol, unknown>;
+    const config: Record<string | number | symbol, any>;
     const global_config: Record<string | number | symbol, unknown>;
 }

extensions/app-telemetry/app-user-count.ts

@@ -0,0 +1,81 @@
+const { Eq } = extension.import('query')
+const { kv } = extension.import('data');
+const span = extension.span;
+const { db } = extension.import('data');
+const { Context, APIError } = extension.import('core');
+const app_es: any = extension.import('service:es:app');
+
+extension.on('create.interfaces', (event) => {
+    event.createInterface('app-telemetry', {
+        description: 'Provides methods for getting app telemetry',
+        methods: {
+            get_users: {
+                description: 'Returns users who have used your app',
+                parameters: {
+                    app_uuid: {
+                        type: 'string',
+                        optional: false,
+                    },
+                    limit: {
+                        type: 'number',
+                        optional: true,
+                    },
+                    offset: {
+                        type: 'number',
+                        optional: true
+                    }
+                },
+            },
+            user_count: {
+                description: 'Returns number of users who have used your app',
+                parameters: {
+                    app_uuid: {
+                        type: 'string',
+                        optional: false,
+                    }
+                },
+            }
+        },
+    });
+});
+
+extension.on('create.drivers', event => {
+    event.createDriver('app-telemetry', 'app-telemetry', {
+        async get_users({ app_uuid, limit = 100, offset = 0 }: {app_uuid: string, limit: number, offset: number}) {
+            // first lets make sure executor owns this app
+            const [result] = (await app_es.select({ predicate: new Eq({ key: 'uid', value: app_uuid }) }));
+            if (!result) {
+                throw APIError.create('permission_denied');
+            }
+
+            // Fetch and return users
+            const users: Array<{username: string, uuid: string}> = await db.read(
+                `SELECT user.username, user.uuid FROM user_to_app_permissions 
+                    INNER JOIN user ON user_to_app_permissions.user_id = user.id  
+                    WHERE permission = 'flag:app-is-authenticated' AND app_id=? ORDER BY (dt IS NOT NULL), dt, user_id LIMIT ? OFFSET ?`,
+                [result.private_meta.mysql_id, limit, offset],
+            );
+            return users.map(e=>{return {user: e.username, user_uuid: e.uuid}});
+        },
+        async user_count({ app_uuid }: {app_uuid: string}) {
+            // first lets make sure executor owns this app
+            const [result] = (await app_es.select({ predicate: new Eq({ key: 'uid', value: app_uuid }) }));
+            if (!result) {
+                throw APIError.create('permission_denied');
+            }
+
+            // Fetch and return authenticated user count
+            const [data] = await db.read(
+                `SELECT count(*) FROM user_to_app_permissions 
+                    WHERE permission = 'flag:app-is-authenticated' AND app_id=?;`,
+                [result.private_meta.mysql_id],
+            );
+            const count = data['count(*)'];
+            return count;
+        }
+    });
+});
+
+extension.on('create.permissions', (event) => {
+    event.grant_to_everyone('service:app-telemetry:ii:app-telemetry');
+});

extensions/app-telemetry/package.json

@@ -0,0 +1,5 @@
+{
+  "name": "@heyputer/app-telemetry",
+  "main": "app-user-count.ts",
+  "type": "module"
+}

src/backend/src/CoreModule.js

@@ -28,6 +28,7 @@ const { TDetachable } = require('@heyputer/putility/src/traits/traits.js');
 const { MultiDetachable } = require('@heyputer/putility/src/libs/listener.js');
 const { OperationFrame } = require('./services/OperationTraceService');
 const opentelemetry = require('@opentelemetry/api');
+const query = require('./om/query/query');
 
 /**
  * @footgun - real install method is defined above
@@ -86,6 +87,11 @@ const install = async ({ context, services, app, useapi, modapi }) => {
             context.get('runtime-modules').register(runtimeModule);
             runtimeModule.exports = useapi.use('core');
         }
+        {
+            const runtimeModule = new RuntimeModule({ name: 'query' });
+            context.get('runtime-modules').register(runtimeModule);
+            runtimeModule.exports = query;
+        }
 
         // Extension module: 'tel'
         {

src/backend/src/services/User.d.ts

@@ -5,6 +5,6 @@ export interface IUser {
     uuid: string,
     username: string,
     email?: string,
-    subscription?: (typeof SUB_POLICIES)[number]['id'],
+    subscription?: (typeof SUB_POLICIES)[number]['id'] & {active: boolean, tier: string},
     metadata?: Record<string, unknown> & { hasDevAccountAccess?: boolean }
 }

src/backend/src/services/database/SqliteDatabaseAccessService.js

@@ -161,6 +161,12 @@ class SqliteDatabaseAccessService extends BaseDatabaseAccessService {
             [37, [
                 '0041_add_unique_constraint_user_uuid.sql',
             ]],
+            [38, [
+                '0042_add_cloudflare_d1.sql',
+            ]],
+            [39, [
+                '0043_add_dt.sql',
+            ]],
         ];
 
         // Database upgrade logic

src/backend/src/services/database/sqlite_setup/0042_add_cloudflare_d1.sql

@@ -0,0 +1 @@
+ALTER TABLE `subdomains` ADD COLUMN `database_id` varchar(40) DEFAULT NULL;

src/backend/src/services/database/sqlite_setup/0043_add_dt.sql

@@ -0,0 +1,22 @@
+PRAGMA foreign_keys = OFF;
+
+CREATE TABLE user_to_app_permissions_new (
+  user_id     INTEGER NOT NULL,
+  app_id      INTEGER NOT NULL,
+  permission  VARCHAR(255) NOT NULL,
+  extra       JSON DEFAULT NULL,
+  dt          DATETIME DEFAULT CURRENT_TIMESTAMP,
+
+  FOREIGN KEY (user_id) REFERENCES user(id) ON DELETE CASCADE ON UPDATE CASCADE,
+  FOREIGN KEY (app_id)  REFERENCES apps(id) ON DELETE CASCADE ON UPDATE CASCADE,
+  PRIMARY KEY (user_id, app_id, permission)
+);
+
+INSERT INTO user_to_app_permissions_new (user_id, app_id, permission, extra, dt)
+SELECT user_id, app_id, permission, extra, NULL
+FROM user_to_app_permissions;
+
+DROP TABLE user_to_app_permissions;
+ALTER TABLE user_to_app_permissions_new RENAME TO user_to_app_permissions;
+
+PRAGMA foreign_keys = ON;

src/puter-js/src/modules/Apps.js

@@ -159,8 +159,29 @@ class Apps {
         if ( typeof args[0] === 'object' && args[0] !== null ) {
             options.params = args[0];
         }
+        const app = await utils.make_driver_method(['uid'], 'puter-apps', undefined, 'read').call(this, options);
+        app.getUsers = async (params) => {
+            params = params ?? {};
+            return (await puter.drivers.call('app-telemetry', 'app-telemetry', 'get_users', { app_uuid: app.uid, limit: params.limit, offset: params.offset })).result;
+        }
+        app.users = async function* (pageSize = 100) {
+            let offset = 0;
+
+            while (true) {
+                const users = await app.getUsers({ limit: pageSize, offset });
+
+                if (!users || users.length === 0) return;
+
+                for (const user of users) {
+                    yield user;
+                }
+
+                offset += users.length;
+                if (users.length < pageSize) return;
+            }
+        }
+        return app;
 
-        return utils.make_driver_method(['uid'], 'puter-apps', undefined, 'read').call(this, options);
     };
 
     delete = async (...args) => {