7f56cd92f8
There were customer reported issues with FAQs which were generated in a different langauge than what they were expecting. The reason behind this was that the language of the account was not considered in the prompt provided. If the language of the content was say Spanish, and the account locale was english. The output was not predicable. The output depends on the model and the execution time. This PR would update the prompt to behave consistently with the account locale. Even though the content provided is in a different language, it would generate FAQs in the account locale. Changes: - Updated the prompt to include a detailed expectation of the FAQs quality along with the language - Added specs for the services where the prompt generator is called. Tested the prompt using Phoenix playground across GPT 5, GPT 4.1, GPT 4.0. The reasoning setting for GPT 5 needs to be low so that it doesn't generate random questions like "What was this updated?"
69 lines
2.5 KiB
Ruby
69 lines
2.5 KiB
Ruby
require 'rails_helper'
|
|
|
|
RSpec.describe Captain::Documents::ResponseBuilderJob, type: :job do
|
|
let(:assistant) { create(:captain_assistant) }
|
|
let(:document) { create(:captain_document, assistant: assistant) }
|
|
let(:faq_generator) { instance_double(Captain::Llm::FaqGeneratorService) }
|
|
let(:faqs) do
|
|
[
|
|
{ 'question' => 'What is Ruby?', 'answer' => 'A programming language' },
|
|
{ 'question' => 'What is Rails?', 'answer' => 'A web framework' }
|
|
]
|
|
end
|
|
|
|
before do
|
|
allow(Captain::Llm::FaqGeneratorService).to receive(:new)
|
|
.with(document.content, document.account.locale_english_name)
|
|
.and_return(faq_generator)
|
|
allow(faq_generator).to receive(:generate).and_return(faqs)
|
|
end
|
|
|
|
describe '#perform' do
|
|
context 'when processing a document' do
|
|
it 'deletes previous responses' do
|
|
existing_response = create(:captain_assistant_response, documentable: document)
|
|
|
|
described_class.new.perform(document)
|
|
|
|
expect { existing_response.reload }.to raise_error(ActiveRecord::RecordNotFound)
|
|
end
|
|
|
|
it 'creates new responses for each FAQ' do
|
|
expect do
|
|
described_class.new.perform(document)
|
|
end.to change(Captain::AssistantResponse, :count).by(2)
|
|
|
|
responses = document.responses.reload
|
|
expect(responses.count).to eq(2)
|
|
|
|
first_response = responses.first
|
|
expect(first_response.question).to eq('What is Ruby?')
|
|
expect(first_response.answer).to eq('A programming language')
|
|
expect(first_response.assistant).to eq(assistant)
|
|
expect(first_response.documentable).to eq(document)
|
|
end
|
|
end
|
|
|
|
context 'with different locales' do
|
|
let(:spanish_account) { create(:account, locale: 'pt') }
|
|
let(:spanish_assistant) { create(:captain_assistant, account: spanish_account) }
|
|
let(:spanish_document) { create(:captain_document, assistant: spanish_assistant, account: spanish_account) }
|
|
let(:spanish_faq_generator) { instance_double(Captain::Llm::FaqGeneratorService) }
|
|
|
|
before do
|
|
allow(Captain::Llm::FaqGeneratorService).to receive(:new)
|
|
.with(spanish_document.content, 'portuguese')
|
|
.and_return(spanish_faq_generator)
|
|
allow(spanish_faq_generator).to receive(:generate).and_return(faqs)
|
|
end
|
|
|
|
it 'passes the correct locale to FAQ generator' do
|
|
described_class.new.perform(spanish_document)
|
|
|
|
expect(Captain::Llm::FaqGeneratorService).to have_received(:new)
|
|
.with(spanish_document.content, 'portuguese')
|
|
end
|
|
end
|
|
end
|
|
end
|