diff --git a/src/backend/src/modules/puterai/AIChatService.js b/src/backend/src/modules/puterai/AIChatService.js index 4529e869..823bb6e3 100644 --- a/src/backend/src/modules/puterai/AIChatService.js +++ b/src/backend/src/modules/puterai/AIChatService.js @@ -92,10 +92,13 @@ class AIChatService extends BaseService { intended_service, parameters }; - await svc_event.emit('ai.prompt.validate', event); if ( ! event.allow ) { test_mode = true; } + + if ( ! test_mode && ! await this.moderate(parameters) ) { + test_mode = true; + } if ( test_mode ) { intended_service = 'fake-chat'; @@ -104,7 +107,7 @@ class AIChatService extends BaseService { if ( intended_service === this.service_name ) { throw new Error('Calling ai-chat directly is not yet supported'); } - + const svc_driver = this.services.get('driver'); const ret = await svc_driver.call_new_({ actor: Context.get('actor'), @@ -114,10 +117,47 @@ class AIChatService extends BaseService { args: parameters, }); ret.result.via_ai_chat_service = true; + + const username = Context.get('actor').type?.user?.username; + + console.log('emitting ai.prompt.complete'); + await svc_event.emit('ai.prompt.complete', { + username, + intended_service, + parameters, + result: ret.result, + }); + return ret.result; } } } + + async moderate ({ messages }) { + const svc_openai = this.services.get('openai-completion'); + + // We can't use moderation of openai service isn't available + if ( ! svc_openai ) return true; + + for ( const msg of messages ) { + const texts = []; + if ( typeof msg.content === 'string' ) texts.push(msg.content); + else if ( typeof msg.content === 'object' ) { + if ( Array.isArray(msg.content) ) { + texts.push(...msg.content.filter(o => ( + ( ! o.type && o.hasOwnProperty('text') ) || + o.type === 'text')).map(o => o.text)); + } + else texts.push(msg.content.text); + } + + const fulltext = texts.join('\n'); + + const mod_result = await svc_openai.check_moderation(fulltext); + if ( mod_result.flagged ) return false; + } + return true; + } async models_ () { return this.detail_model_list;