diff --git a/src/backend/src/api/APIError.js b/src/backend/src/api/APIError.js index a0b958055..3f23dc404 100644 --- a/src/backend/src/api/APIError.js +++ b/src/backend/src/api/APIError.js @@ -509,6 +509,12 @@ module.exports = class APIError { status: 400, message: ({ engine, valid_engines }) => `Invalid engine: ${quot(engine)}. Valid engines are: ${valid_engines.map(quot).join(', ')}.`, }, + + // Abuse prevention + 'moderation_failed': { + status: 422, + message: `Content moderation failed`, + }, }; /** diff --git a/src/backend/src/modules/puterai/AIChatService.js b/src/backend/src/modules/puterai/AIChatService.js index 9edc00c8f..5ed9db4cd 100644 --- a/src/backend/src/modules/puterai/AIChatService.js +++ b/src/backend/src/modules/puterai/AIChatService.js @@ -324,6 +324,7 @@ class AIChatService extends BaseService { if ( ! test_mode && ! await this.moderate(parameters) ) { test_mode = true; + throw APIError.create('moderation_failed'); } if ( ! test_mode ) { @@ -622,6 +623,7 @@ class AIChatService extends BaseService { * Returns true if OpenAI service is unavailable or all messages pass moderation. */ async moderate({ messages }) { + if ( process.env.TEST_MODERATION_FAILURE ) return false; for ( const msg of messages ) { const texts = [];