From 406b85026c0e15f1c9850d3eee88a7200c52bc6b Mon Sep 17 00:00:00 2001 From: fluxgravity <44489080+fluxgravity@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:29:32 -0400 Subject: [PATCH 1/7] feat(rubyllm): add Phase 1 scaffolding and fix preferences merge bug Add ruby_llm ~> 1.16.0 gem, feature flag (use_ruby_llm, off by default), empty AIBackend::RubyLLM shell with supports_driver? returning false, and TestClient::RubyLLM double. The APIService#ai_backend guard remains dead code until Phase 2 flips supports_driver? to true. Also fix a latent bug where preference writes clobbered unrelated keys -- both UsersController and Settings::PeopleController now deep-merge incoming preferences instead of replacing the entire column. Permit feature: [:use_ruby_llm] in both preference write paths so the flag is per-user writable (no UI yet). --- Gemfile | 1 + Gemfile.lock | 14 +++ app/controllers/settings/people_controller.rb | 9 +- app/controllers/users_controller.rb | 8 +- app/models/api_service.rb | 8 +- app/services/ai_backend/ruby_llm.rb | 54 +++++++++++ config/initializers/inflections.rb | 1 + config/initializers/ruby_llm.rb | 7 ++ config/options.yml | 1 + .../settings/people_controller_test.rb | 15 ++++ test/controllers/users_controller_test.rb | 15 ++++ test/models/feature_test.rb | 29 +++++- test/services/ai_backend/ruby_llm_test.rb | 41 +++++++++ test/support/test_client/ruby_llm.rb | 89 +++++++++++++++++++ 14 files changed, 280 insertions(+), 12 deletions(-) create mode 100644 app/services/ai_backend/ruby_llm.rb create mode 100644 config/initializers/ruby_llm.rb create mode 100644 test/services/ai_backend/ruby_llm_test.rb create mode 100644 test/support/test_client/ruby_llm.rb diff --git a/Gemfile b/Gemfile index 3360b5644..b20553c1d 100644 --- a/Gemfile +++ b/Gemfile @@ -44,6 +44,7 @@ gem "rails_heroicon", "~> 2.2.0" gem "ruby-openai", "~> 7.0.1" gem "ruby-anthropic", "~> 0.4.0" gem "gemini-ai", "~> 4.2.0" +gem "ruby_llm", "~> 1.16.0" gem "solid_queue", "~> 1.0.0" gem "name_of_person" gem "actioncable-enhanced-postgresql-adapter" # longer paylaods w/ postgresql actioncable diff --git a/Gemfile.lock b/Gemfile.lock index afbc8e6bf..e97016504 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -158,6 +158,8 @@ GEM multipart-post (~> 2.0) faraday-net_http (3.4.4) net-http (~> 0.5) + faraday-retry (2.4.0) + faraday (~> 2.0) faraday-typhoeus (1.1.0) faraday (~> 2.0) typhoeus (~> 1.4) @@ -453,6 +455,17 @@ GEM ruby-vips (2.3.0) ffi (~> 1.12) logger + ruby_llm (1.16.0) + base64 + event_stream_parser (~> 1) + faraday (>= 1.10.0) + faraday-multipart (>= 1) + faraday-net_http (>= 1) + faraday-retry (>= 1) + marcel (~> 1) + ruby_llm-schema (~> 0) + zeitwerk (~> 2) + ruby_llm-schema (0.4.0) rubyzip (3.4.1) securerandom (0.4.1) selenium-webdriver (4.47.0) @@ -608,6 +621,7 @@ DEPENDENCIES rubocop-rails ruby-anthropic (~> 0.4.0) ruby-openai (~> 7.0.1) + ruby_llm (~> 1.16.0) selenium-webdriver solid_queue (~> 1.0.0) sprockets-rails diff --git a/app/controllers/settings/people_controller.rb b/app/controllers/settings/people_controller.rb index b21f27c1d..15725608e 100644 --- a/app/controllers/settings/people_controller.rb +++ b/app/controllers/settings/people_controller.rb @@ -19,9 +19,14 @@ def update def person_params h = params.require(:person).permit(:email, personable_attributes: [ :id, :first_name, :last_name, :password, :profile_picture, :remove_profile_picture, - :dark_mode, + :dark_mode, preferences: [feature: [:use_ruby_llm]], credentials_attributes: [ :id, :type, :password ] ]).to_h + + if (prefs = h.dig("personable_attributes", "preferences")).present? && (user = Current.person.user) + h["personable_attributes"]["preferences"] = user.preferences.deep_merge(prefs.deep_symbolize_keys) + end + format_and_strip_all_but_first_valid_credential(h) end @@ -37,7 +42,7 @@ def apply_backend_choices! def check_personable_id personable_id = params[:person].try(:[], :personable_attributes).try(:[], :id) if personable_id.present? && personable_id.to_i != Current.person.personable_id - return render :edit, status: :unauthorized + render :edit, status: :unauthorized end end diff --git a/app/controllers/users_controller.rb b/app/controllers/users_controller.rb index 5111419b1..d624fb174 100644 --- a/app/controllers/users_controller.rb +++ b/app/controllers/users_controller.rb @@ -27,7 +27,9 @@ def create end def update - if @user.update(user_params) + incoming = user_params[:preferences]&.to_h&.deep_symbolize_keys || {} + @user.preferences = @user.preferences.deep_merge(incoming) + if @user.save Current.user.reload redirect_back fallback_location: root_path, status: :see_other else @@ -53,12 +55,12 @@ def prettify_flash_error_messages def person_params h = params.require(:person).permit(:email, :personable_type, personable_attributes: [ - :name, credentials_attributes: [ :type, :password ] + :name, credentials_attributes: [:type, :password] ]).to_h format_and_strip_all_but_first_valid_credential(h) end def user_params - params.require(:user).permit(:nav_closed) + params.require(:user).permit(preferences: [:nav_closed, :dark_mode, feature: [:use_ruby_llm]]) end end diff --git a/app/models/api_service.rb b/app/models/api_service.rb index 1baef36ef..8c51e37e8 100644 --- a/app/models/api_service.rb +++ b/app/models/api_service.rb @@ -14,16 +14,18 @@ class APIService < ApplicationRecord validates :url, format: URI::DEFAULT_PARSER.make_regexp(%w[http https]), if: -> { url.present? } validates :name, :url, presence: true - normalizes :url, with: -> url { url.strip } + normalizes :url, with: ->(url) { url.strip } encrypts :token - normalizes :token, with: -> token { token.strip } + normalizes :token, with: ->(token) { token.strip } before_save :soft_delete_language_models, if: -> { deleted_at && deleted_at_changed? && deleted_at_was.nil? } scope :ordered, -> { order(:name) } def ai_backend + return AIBackend::RubyLLM if Feature.use_ruby_llm? && AIBackend::RubyLLM.supports_driver?(driver) + if driver == "openai" && url == URL_GROQ AIBackend::Groq elsif driver == "anthropic" @@ -67,7 +69,7 @@ def default_llm_key return nil unless Feature.default_llm_keys? return Setting.default_openai_key if url == URL_OPEN_AI return Setting.default_anthropic_key if url == URL_ANTHROPIC - return Setting.default_groq_key if url == URL_GROQ + Setting.default_groq_key if url == URL_GROQ end def soft_delete_language_models diff --git a/app/services/ai_backend/ruby_llm.rb b/app/services/ai_backend/ruby_llm.rb new file mode 100644 index 000000000..7cef7f580 --- /dev/null +++ b/app/services/ai_backend/ruby_llm.rb @@ -0,0 +1,54 @@ +class AIBackend::RubyLLM < AIBackend + class ConfigurationError < StandardError; end + class RateLimitError < StandardError; end + + def self.supports_driver?(_driver) + false + end + + def self.client + Rails.env.test? ? ::TestClient::RubyLLM : ::RubyLLM + end + + def get_oneoff_message(*) + raise NotImplementedError + end + + def stream_next_conversation_message(*) + raise NotImplementedError + end + + def self.test_execute(*) + raise NotImplementedError + end + + private + + def client_method_name + raise NotImplementedError + end + + def configuration_error + raise NotImplementedError + end + + def set_client_config(*) + raise NotImplementedError + end + + def preceding_messages(*) + raise NotImplementedError + end + + def preceding_conversation_messages + raise NotImplementedError + end + + def stream_handler(*) + raise NotImplementedError + end + + def format_parallel_tool_calls(*) + raise NotImplementedError + end +end diff --git a/config/initializers/inflections.rb b/config/initializers/inflections.rb index affd3176e..fb4b8b908 100644 --- a/config/initializers/inflections.rb +++ b/config/initializers/inflections.rb @@ -15,6 +15,7 @@ # inflect.acronym "RESTful" inflect.acronym "AI" inflect.acronym "API" + inflect.acronym "LLM" inflect.acronym "SDK" inflect.acronym "URL" inflect.acronym "XML" diff --git a/config/initializers/ruby_llm.rb b/config/initializers/ruby_llm.rb new file mode 100644 index 000000000..37c7fa501 --- /dev/null +++ b/config/initializers/ruby_llm.rb @@ -0,0 +1,7 @@ +RubyLLM.configure do |config| + config.openai_api_key = ENV["DEFAULT_OPENAI_KEY"] || "dummy-key-for-test" + config.anthropic_api_key = ENV["DEFAULT_ANTHROPIC_KEY"] || "dummy-key-for-test" + config.gemini_api_key = ENV["DEFAULT_GEMINI_KEY"] || "dummy-key-for-test" + config.logger = Rails.logger + config.request_timeout = Rails.env.production? ? 120 : 30 +end diff --git a/config/options.yml b/config/options.yml index aaf012137..5fcc627a6 100644 --- a/config/options.yml +++ b/config/options.yml @@ -46,6 +46,7 @@ shared: default_to_voice: <%= ENV["DEFAULT_TO_VOICE_FEATURE"] || false %> email: <%= ENV["EMAIL_FEATURE"] || default_to(false, except_env_test: true) %> password_reset_email: <%= ENV["PASSWORD_RESET_EMAIL_FEATURE"] || default_to(false, except_env_test: true) %> + use_ruby_llm: <%= ENV["USE_RUBY_LLM_FEATURE"] || false %> assistants_page: <%= ENV["ASSISTANTS_PAGE_FEATURE"] || true %> use_ruby_llm: <%= ENV["USE_RUBY_LLM_FEATURE"] || false %> settings: diff --git a/test/controllers/settings/people_controller_test.rb b/test/controllers/settings/people_controller_test.rb index e7b5c1204..fde263646 100644 --- a/test/controllers/settings/people_controller_test.rb +++ b/test/controllers/settings/people_controller_test.rb @@ -7,6 +7,21 @@ class Settings::PeopleControllerTest < ActionDispatch::IntegrationTest login_as @person end + test "preferences merge preserves unrelated keys when updating dark_mode" do + @user.preferences = { dark_mode: "light", feature: { use_ruby_llm: true } } + @user.save! + + params = person_params + params["personable_attributes"]["preferences"] = { dark_mode: "dark" } + + patch settings_person_url, params: { person: params } + assert_redirected_to edit_settings_person_url + @user.reload + + assert_equal "dark", @user.preferences[:dark_mode] + assert_equal({ use_ruby_llm: true }, @user.preferences[:feature]) + end + test "should get edit with password field VISIBLE" do assert @user.password_credential.present? get edit_settings_person_url diff --git a/test/controllers/users_controller_test.rb b/test/controllers/users_controller_test.rb index 89b0b6489..8658ef3d0 100644 --- a/test/controllers/users_controller_test.rb +++ b/test/controllers/users_controller_test.rb @@ -86,6 +86,21 @@ class UsersControllerTest < ActionDispatch::IntegrationTest end end + test "preferences merge preserves unrelated keys when setting nav_closed" do + user = users(:keith) + user.preferences = { dark_mode: "light", feature: { use_ruby_llm: true } } + user.save! + login_as user + + patch user_url(user), params: { user: { preferences: { nav_closed: true } } } + assert_response :redirect + user.reload + + assert user.preferences[:nav_closed] + assert_equal "light", user.preferences[:dark_mode] + assert_equal({ use_ruby_llm: true }, user.preferences[:feature]) + end + test "updates nav_closed preference without touching other preferences" do user = users(:keith) user.update!(dark_mode: "dark", nav_closed: false) diff --git a/test/models/feature_test.rb b/test/models/feature_test.rb index b0cca2c33..2b6095121 100644 --- a/test/models/feature_test.rb +++ b/test/models/feature_test.rb @@ -1,7 +1,6 @@ require "test_helper" class FeatureTest < ActiveSupport::TestCase - test "should return value of feature" do stub_features(my_feature: true) do assert Feature.enabled?(:my_feature) @@ -33,7 +32,7 @@ class FeatureTest < ActiveSupport::TestCase test "a user's preferences can ENABLE a feature which is globally DISABLED" do user = users(:keith) - user.preferences = user.preferences.merge(feature: { my_feature: true }) + user.preferences = user.preferences.merge(feature: {my_feature: true}) user.save! stub_features(my_feature: false) do @@ -45,7 +44,7 @@ class FeatureTest < ActiveSupport::TestCase test "a user's preferences can DISABLE a feature which is globally ENABLED" do user = users(:keith) - user.preferences = user.preferences.merge(feature: { my_feature: false }) + user.preferences = user.preferences.merge(feature: {my_feature: false}) user.save! stub_features(my_feature: true) do @@ -74,7 +73,6 @@ class FeatureTest < ActiveSupport::TestCase end test "password and google auth are ALLOWED if HTTP header auth is DISABLED" do - stub_features( http_header_authentication: false, password_authentication: true, @@ -92,6 +90,29 @@ class FeatureTest < ActiveSupport::TestCase end end + test "use_ruby_llm? reads options.yml default and can be overridden by user preference" do + user = users(:keith) + + stub_features(use_ruby_llm: false) do + refute Feature.use_ruby_llm? + end + + stub_features(use_ruby_llm: false) do + Current.set(user: user) do + refute Feature.use_ruby_llm? + end + end + + user.preferences = user.preferences.merge(feature: {use_ruby_llm: true}) + user.save! + + stub_features(use_ruby_llm: false) do + Current.set(user: user) do + assert Feature.use_ruby_llm? + end + end + end + test "referencing a feature that does not exist raises an exception" do assert_raises do Feature.foobar? diff --git a/test/services/ai_backend/ruby_llm_test.rb b/test/services/ai_backend/ruby_llm_test.rb new file mode 100644 index 000000000..7c8828c55 --- /dev/null +++ b/test/services/ai_backend/ruby_llm_test.rb @@ -0,0 +1,41 @@ +require "test_helper" + +class AIBackend::RubyLLMTest < ActiveSupport::TestCase + setup do + @conversation = conversations(:attachments) + @assistant = assistants(:keith_gpt4) + @openai_service = api_services(:keith_openai_service) + @anthropic_service = api_services(:keith_anthropic_service) + @gemini_service = api_services(:keith_gemini_service) + end + + test "supports_driver? returns false for all drivers in Phase 1" do + refute AIBackend::RubyLLM.supports_driver?("openai") + refute AIBackend::RubyLLM.supports_driver?("anthropic") + refute AIBackend::RubyLLM.supports_driver?("gemini") + end + + test "APIService#ai_backend returns old OpenAI class when feature flag is off" do + assert_equal AIBackend::OpenAI, @openai_service.ai_backend + end + + test "APIService#ai_backend returns old Anthropic class when feature flag is off" do + assert_equal AIBackend::Anthropic, @anthropic_service.ai_backend + end + + test "APIService#ai_backend returns old Gemini class when feature flag is off" do + assert_equal AIBackend::Gemini, @gemini_service.ai_backend + end + + test "APIService#ai_backend still returns old classes even with flag on since supports_driver? is false" do + stub_features(use_ruby_llm: true) do + assert_equal AIBackend::OpenAI, @openai_service.ai_backend + assert_equal AIBackend::Anthropic, @anthropic_service.ai_backend + assert_equal AIBackend::Gemini, @gemini_service.ai_backend + end + end + + test "client returns TestClient::RubyLLM in test environment" do + assert_equal TestClient::RubyLLM, AIBackend::RubyLLM.client + end +end diff --git a/test/support/test_client/ruby_llm.rb b/test/support/test_client/ruby_llm.rb new file mode 100644 index 000000000..cc3146009 --- /dev/null +++ b/test/support/test_client/ruby_llm.rb @@ -0,0 +1,89 @@ +module TestClient + class RubyLLM + class Chat + attr_reader :messages + + def initialize(model:, provider: nil, assume_model_exists: nil, context: nil) + @@model = model + @context = context + @messages = [] + end + + def with_instructions(instructions) + @@instructions = instructions + self + end + + def with_params(**params) + @@params = params + self + end + + def with_tools(*tools) + @@tools = tools + self + end + + def add_message(msg) + @messages << msg + self + end + + def complete(&block) + if block + block.call(self.class.api_streaming_response) + else + self.class.api_oneoff_response.dig("choices", 0, "message", "content") + end + end + + def ask(message = nil, with: nil, &block) + add_message({role: "user", content: message}) if message + complete(&block) + end + + def self.api_oneoff_response + { + "choices" => [ + { + "message" => { + "content" => text || default_text + } + } + ] + } + end + + def self.api_streaming_response + OpenStruct.new( + content: text || default_text, + input_tokens: 8, + output_tokens: 9 + ) + end + + def self.text + raise "Attempting to return a text response but .text method is not stubbed. Stub this to nil if you want to return default text." + end + + def self.default_text + "Hello this is model #{@@model}! How can I assist you today?" + end + + class << self + attr_reader :instructions, :params + end + end + + class ContextDouble + attr_accessor :openai_api_key, :anthropic_api_key, :gemini_api_key, + :openai_api_base, :anthropic_api_base, :gemini_api_base + end + + def self.context(&block) + ctx = ContextDouble.new + block.call(ctx) if block + ctx + end + end +end From a04415cb221a8b87479a18e0a69e293a3c4e8be8 Mon Sep 17 00:00:00 2001 From: fluxgravity <44489080+fluxgravity@users.noreply.github.com> Date: Wed, 19 Aug 2026 03:01:38 -0400 Subject: [PATCH 2/7] =?UTF-8?q?feat(rubyllm):=20implement=20Phase=202=20?= =?UTF-8?q?=E2=80=94=20OpenAI=20text-only=20chat=20backend?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements AIBackend::RubyLLM with streaming, token counting, oneoff messages, and error mapping for the openai driver. RubyLLM gem's builder-pattern Chat is used with per-request context keys. - supports_driver? returns true for "openai" only - get_oneoff_message and stream_next_conversation_message are full overrides using RubyLLM's Chat API - stream_handler proc yields chunk content, captures input/output tokens, and maps RubyLLM errors to ConfigurationError/RateLimitError - preceding_conversation_messages returns text-only {role, content} hashes, skipping tool messages (tool support deferred to Phase 5) - test_execute mirrors the existing OpenAI pattern with context key - TestClient::RubyLLM::Chat upgraded with complete/content/ask, class-level stubbables (text, blank_response, error_to_raise, tokens) - 24 backend tests + 8 job integration tests --- app/services/ai_backend/ruby_llm.rb | 118 ++++++-- .../get_next_ai_message_job_ruby_llm_test.rb | 82 +++++ test/services/ai_backend/ruby_llm_test.rb | 281 +++++++++++++++++- test/support/test_client/ruby_llm.rb | 53 +++- 4 files changed, 501 insertions(+), 33 deletions(-) create mode 100644 test/jobs/get_next_ai_message_job_ruby_llm_test.rb diff --git a/app/services/ai_backend/ruby_llm.rb b/app/services/ai_backend/ruby_llm.rb index 7cef7f580..d5becf9ed 100644 --- a/app/services/ai_backend/ruby_llm.rb +++ b/app/services/ai_backend/ruby_llm.rb @@ -2,49 +2,133 @@ class AIBackend::RubyLLM < AIBackend class ConfigurationError < StandardError; end class RateLimitError < StandardError; end - def self.supports_driver?(_driver) - false + CONFIGURATION_ERRORS = [ + ::RubyLLM::UnauthorizedError, ::RubyLLM::ConfigurationError, + ::RubyLLM::BadRequestError, ::RubyLLM::ForbiddenError, + ::RubyLLM::ContextLengthExceededError, + ].freeze + RATE_LIMIT_ERRORS = [ + ::RubyLLM::RateLimitError, ::RubyLLM::PaymentRequiredError, + ::RubyLLM::OverloadedError, ::RubyLLM::ServiceUnavailableError, + ].freeze + + def self.supports_driver?(driver) + ["openai"].include?(driver) end def self.client Rails.env.test? ? ::TestClient::RubyLLM : ::RubyLLM end - def get_oneoff_message(*) - raise NotImplementedError + def self.gem_class + Rails.env.test? ? ::TestClient::RubyLLM::Chat : ::RubyLLM::Chat end - def stream_next_conversation_message(*) - raise NotImplementedError + def self.test_execute(url, token, api_name) + if Rails.env.test? + chat = TestClient::RubyLLM::Chat.new(model: api_name, provider: :openai, assume_model_exists: true) + chat.add_message({ role: "user", content: "Hello!" }) + chat.complete.content + else + Rails.logger.info "Connecting to OpenAI API server at #{url} with access token of length #{token.to_s.length}" + Rails.logger.info "Testing using model #{api_name}" + context = RubyLLM.context { |c| c.openai_api_key = token; c.openai_api_base = url if url != APIService::URL_OPEN_AI } + chat = RubyLLM::Chat.new(model: api_name, provider: :openai, assume_model_exists: true, context: context) + chat.add_message({ role: "user", content: "Hello!" }) + chat.complete.content + end + rescue ::Faraday::Error => e + "Error: #{e.message}" end - def self.test_execute(*) - raise NotImplementedError + def initialize(user, assistant, conversation = nil, message = nil) + super + @api_service = assistant.api_service + @token = @api_service.effective_token + @api_name = assistant.language_model.api_name + + raise ConfigurationError if @api_service.requires_token? && @token.blank? + end + + def get_oneoff_message(instructions, messages, params = {}) + chat = build_chat + chat.with_instructions(instructions) + preceding_messages(messages).each { |msg| chat.add_message(msg) } + chat.with_params(**params) if params.present? + chat.complete.content + end + + def stream_next_conversation_message(&chunk_handler) + @stream_response_text = "" + + chat = build_chat + chat.with_instructions(full_instructions) + preceding_conversation_messages.each { |msg| chat.add_message(msg) } + chat.complete { |chunk| stream_handler.call(chunk, chunk_handler) } + + raise ::Faraday::ParsingError if @stream_response_text.blank? + nil end private - def client_method_name - raise NotImplementedError + def build_chat + self.class.gem_class.new(model: @api_name, provider: :openai, assume_model_exists: true, context: ruby_llm_context) end - def configuration_error - raise NotImplementedError + def ruby_llm_context + self.class.client.context do |c| + c.openai_api_key = @token + c.openai_api_base = @api_service.url if @api_service.url != APIService::URL_OPEN_AI + end end - def set_client_config(*) - raise NotImplementedError + def stream_handler + proc do |chunk, chunk_handler| + input_tokens = chunk.respond_to?(:input_tokens) ? chunk.input_tokens : nil + output_tokens = chunk.respond_to?(:output_tokens) ? chunk.output_tokens : nil + + if input_tokens && output_tokens + @message.input_token_count = input_tokens + @message.output_token_count = output_tokens + end + + if chunk.respond_to?(:content) && chunk.content.present? + @stream_response_text += chunk.content + chunk_handler.call(chunk.content) + end + rescue ::GetNextAIMessageJob::ResponseCancelled => e + raise e + rescue *CONFIGURATION_ERRORS => e + raise ConfigurationError, e.message + rescue *RATE_LIMIT_ERRORS => e + raise RateLimitError, e.message + rescue => e + Rails.logger.info "\nUnhandled error in AIBackend::RubyLLM response handler: #{e.message}" + Rails.logger.info e.backtrace.join("\n") + end end - def preceding_messages(*) + def preceding_conversation_messages + @conversation.messages.for_conversation_version(@message.version).where("messages.index < ?", @message.index).collect do |message| + next if message.tool? + + { + role: message.role, + content: message.content_text, + } + end.compact + end + + def client_method_name raise NotImplementedError end - def preceding_conversation_messages + def configuration_error raise NotImplementedError end - def stream_handler(*) + def set_client_config(*) raise NotImplementedError end diff --git a/test/jobs/get_next_ai_message_job_ruby_llm_test.rb b/test/jobs/get_next_ai_message_job_ruby_llm_test.rb new file mode 100644 index 000000000..1c0bcfc65 --- /dev/null +++ b/test/jobs/get_next_ai_message_job_ruby_llm_test.rb @@ -0,0 +1,82 @@ +require "test_helper" + +class GetNextAIMessageJobRubyLLMTest < ActiveJob::TestCase + setup do + @conversation = conversations(:greeting) + @user = @conversation.user + @assistant = @conversation.assistant + @conversation.messages.create! role: :user, content_text: "Still there?", assistant: @assistant + @assistant.language_model.update!(supports_tools: false) + @message = @conversation.latest_message_for_version(:latest) + end + + test "populates the latest message from the assistant via RubyLLM" do + stub_features(use_ruby_llm: true) do + assert_no_difference "@conversation.messages.reload.length" do + TestClient::RubyLLM::Chat.stub :text, "Hello from RubyLLM" do + assert GetNextAIMessageJob.perform_now(@user.id, @message.id, @assistant.id) + end + end + end + + assert_equal "Hello from RubyLLM", @conversation.latest_message_for_version(:latest).content_text + end + + test "returns early if the message id was invalid" do + stub_features(use_ruby_llm: true) do + refute GetNextAIMessageJob.perform_now(@user.id, 0, @assistant.id) + end + end + + test "returns early if the assistant id was invalid" do + stub_features(use_ruby_llm: true) do + refute GetNextAIMessageJob.perform_now(@user.id, @message.id, 0) + end + end + + test "returns early if the message was already generated" do + @message.update!(content_text: "Hello") + stub_features(use_ruby_llm: true) do + refute GetNextAIMessageJob.perform_now(@user.id, @message.id, @assistant.id) + end + end + + test "returns early if the user has replied after this" do + @conversation.messages.create! role: :user, content_text: "Ignore that, new question:", assistant: @assistant + stub_features(use_ruby_llm: true) do + refute GetNextAIMessageJob.perform_now(@user.id, @message.id, @assistant.id) + end + end + + test "when API response is empty, a nice error message is displayed and the message is marked failed" do + stub_features(use_ruby_llm: true) do + TestClient::RubyLLM::Chat.stub :text, "" do + assert GetNextAIMessageJob.perform_now(@user.id, @message.id, @assistant.id) + end + end + + assert_includes @conversation.latest_message_for_version(:latest).content_text, "a blank response" + assert @message.reload.failed?, "The message should have been marked failed so a Retry button is offered" + end + + test "when the connection drops mid-stream, the message is marked failed" do + stub_features(use_ruby_llm: true) do + TestClient::RubyLLM::Chat.stub_any_instance :complete, proc { |*| raise Faraday::ConnectionFailed, "connection reset by peer" } do + assert GetNextAIMessageJob.perform_now(@user.id, @message.id, @assistant.id) + end + end + + assert_includes @message.reload.content_text, "connection error" + assert @message.failed?, "The message should have been marked failed so a Retry button is offered" + end + + test "a message which generated successfully is not marked failed" do + stub_features(use_ruby_llm: true) do + TestClient::RubyLLM::Chat.stub :text, "Hello" do + assert GetNextAIMessageJob.perform_now(@user.id, @message.id, @assistant.id) + end + end + + assert @message.reload.not_failed? + end +end diff --git a/test/services/ai_backend/ruby_llm_test.rb b/test/services/ai_backend/ruby_llm_test.rb index 7c8828c55..253c7d326 100644 --- a/test/services/ai_backend/ruby_llm_test.rb +++ b/test/services/ai_backend/ruby_llm_test.rb @@ -4,32 +4,46 @@ class AIBackend::RubyLLMTest < ActiveSupport::TestCase setup do @conversation = conversations(:attachments) @assistant = assistants(:keith_gpt4) + @user = @conversation.user @openai_service = api_services(:keith_openai_service) @anthropic_service = api_services(:keith_anthropic_service) @gemini_service = api_services(:keith_gemini_service) end - test "supports_driver? returns false for all drivers in Phase 1" do - refute AIBackend::RubyLLM.supports_driver?("openai") + # Phase 1 — updated for Phase 2 driver support + + test "supports_driver? returns true for openai, false for others in Phase 2" do + assert AIBackend::RubyLLM.supports_driver?("openai") refute AIBackend::RubyLLM.supports_driver?("anthropic") refute AIBackend::RubyLLM.supports_driver?("gemini") end test "APIService#ai_backend returns old OpenAI class when feature flag is off" do - assert_equal AIBackend::OpenAI, @openai_service.ai_backend + stub_features(use_ruby_llm: false) do + assert_equal AIBackend::OpenAI, @openai_service.ai_backend + end end test "APIService#ai_backend returns old Anthropic class when feature flag is off" do - assert_equal AIBackend::Anthropic, @anthropic_service.ai_backend + stub_features(use_ruby_llm: false) do + assert_equal AIBackend::Anthropic, @anthropic_service.ai_backend + end end test "APIService#ai_backend returns old Gemini class when feature flag is off" do - assert_equal AIBackend::Gemini, @gemini_service.ai_backend + stub_features(use_ruby_llm: false) do + assert_equal AIBackend::Gemini, @gemini_service.ai_backend + end end - test "APIService#ai_backend still returns old classes even with flag on since supports_driver? is false" do + test "APIService#ai_backend returns RubyLLM when flag on and driver is openai" do + stub_features(use_ruby_llm: true) do + assert_equal AIBackend::RubyLLM, @openai_service.ai_backend + end + end + + test "APIService#ai_backend falls back to old classes for unsupported drivers even with flag on" do stub_features(use_ruby_llm: true) do - assert_equal AIBackend::OpenAI, @openai_service.ai_backend assert_equal AIBackend::Anthropic, @anthropic_service.ai_backend assert_equal AIBackend::Gemini, @gemini_service.ai_backend end @@ -38,4 +52,257 @@ class AIBackend::RubyLLMTest < ActiveSupport::TestCase test "client returns TestClient::RubyLLM in test environment" do assert_equal TestClient::RubyLLM, AIBackend::RubyLLM.client end + + # Phase 2 — plain text chat, OpenAI only + + test "get_oneoff_message returns text content" do + backend = AIBackend::RubyLLM.new(@user, @assistant) + TestClient::RubyLLM::Chat.stub :text, "Hello, world!" do + assert_equal "Hello, world!", backend.get_oneoff_message("You are helpful", ["Hi"]) + end + end + + test "get_oneoff_message passes instructions to chat" do + backend = AIBackend::RubyLLM.new(@user, @assistant) + TestClient::RubyLLM::Chat.stub :text, "Test" do + backend.get_oneoff_message("Be a poet", ["Hello"]) + end + assert_equal "Be a poet", TestClient::RubyLLM::Chat.instructions + end + + test "get_oneoff_message passes params to chat" do + backend = AIBackend::RubyLLM.new(@user, @assistant) + TestClient::RubyLLM::Chat.stub :text, "Test" do + backend.get_oneoff_message("You are helpful", ["Hi"], { temperature: 0.5 }) + end + assert_equal({ temperature: 0.5 }, TestClient::RubyLLM::Chat.params) + end + + test "get_oneoff_message adds preceding messages" do + backend = AIBackend::RubyLLM.new(@user, @assistant) + chat = TestClient::RubyLLM::Chat.new(model: "gpt-4o", provider: :openai, assume_model_exists: true) + TestClient::RubyLLM::Chat.stub :new, chat do + TestClient::RubyLLM::Chat.stub :text, "Test" do + backend.get_oneoff_message("You are helpful", ["First", "Second"]) + end + end + assert_equal 2, chat.messages.length + assert_equal "user", chat.messages.first[:role] + assert_equal "First", chat.messages.first[:content] + assert_equal "assistant", chat.messages.last[:role] + assert_equal "Second", chat.messages.last[:content] + end + + test "stream_next_conversation_message yields chunk content" do + @assistant.language_model.update!(supports_tools: false) + message = @conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: @assistant, + index: @conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, @conversation, message) + TestClient::RubyLLM::Chat.stub :text, "Streaming response" do + chunks = [] + result = backend.stream_next_conversation_message { |c| chunks << c } + assert_equal ["Streaming response"], chunks + assert_nil result + end + end + + test "stream_handler accumulates multiple chunks and captures tokens" do + @assistant.language_model.update!(supports_tools: false) + message = @conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: @assistant, + index: @conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, @conversation, message) + backend.instance_variable_set(:@stream_response_text, "") + handler = backend.send(:stream_handler) + + chunk1 = OpenStruct.new(content: "Hello ", input_tokens: nil, output_tokens: nil) + chunk2 = OpenStruct.new(content: "world!", input_tokens: 10, output_tokens: 3) + + chunks = [] + handler.call(chunk1, ->(c) { chunks << c }) + handler.call(chunk2, ->(c) { chunks << c }) + + assert_equal ["Hello ", "world!"], chunks + assert_equal 10, message.input_token_count + assert_equal 3, message.output_token_count + end + + test "stream_handler captures token counts on first occurrence" do + @assistant.language_model.update!(supports_tools: false) + message = @conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: @assistant, + index: @conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, @conversation, message) + backend.instance_variable_set(:@stream_response_text, "") + handler = backend.send(:stream_handler) + + token_chunk = OpenStruct.new(content: "Hi", input_tokens: 100, output_tokens: 50) + handler.call(token_chunk, ->(c) { }) + + assert_equal 100, message.input_token_count + assert_equal 50, message.output_token_count + end + + test "stream_next_conversation_message raises Faraday::ParsingError on blank response" do + @assistant.language_model.update!(supports_tools: false) + message = @conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: @assistant, + index: @conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, @conversation, message) + TestClient::RubyLLM::Chat.stub :text, "" do + assert_raises(Faraday::ParsingError) do + backend.stream_next_conversation_message { |c| } + end + end + end + + test "stream_handler raises ConfigurationError on unauthorized" do + @assistant.language_model.update!(supports_tools: false) + message = @conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: @assistant, + index: @conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, @conversation, message) + handler = backend.send(:stream_handler) + + error_chunk = OpenStruct.new + error_chunk.define_singleton_method(:content) { raise ::RubyLLM::UnauthorizedError, "Unauthorized" } + error_chunk.define_singleton_method(:input_tokens) { nil } + error_chunk.define_singleton_method(:output_tokens) { nil } + + assert_raises(AIBackend::RubyLLM::ConfigurationError) do + handler.call(error_chunk, ->(c) { }) + end + end + + test "stream_handler raises RateLimitError on rate limit" do + @assistant.language_model.update!(supports_tools: false) + message = @conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: @assistant, + index: @conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, @conversation, message) + handler = backend.send(:stream_handler) + + error_chunk = OpenStruct.new + error_chunk.define_singleton_method(:content) { raise ::RubyLLM::RateLimitError, "Rate limited" } + error_chunk.define_singleton_method(:input_tokens) { nil } + error_chunk.define_singleton_method(:output_tokens) { nil } + + assert_raises(AIBackend::RubyLLM::RateLimitError) do + handler.call(error_chunk, ->(c) { }) + end + end + + test "stream_handler re-raises ResponseCancelled" do + @assistant.language_model.update!(supports_tools: false) + message = @conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: @assistant, + index: @conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, @conversation, message) + handler = backend.send(:stream_handler) + + error_chunk = OpenStruct.new + error_chunk.define_singleton_method(:content) { raise GetNextAIMessageJob::ResponseCancelled } + error_chunk.define_singleton_method(:input_tokens) { nil } + error_chunk.define_singleton_method(:output_tokens) { nil } + + assert_raises(GetNextAIMessageJob::ResponseCancelled) do + handler.call(error_chunk, ->(c) { }) + end + end + + test "preceding_conversation_messages returns messages with role and content" do + @assistant.language_model.update!(supports_tools: false) + message = @conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: @assistant, + index: @conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, @conversation, message) + msgs = backend.send(:preceding_conversation_messages) + + assert msgs.present? + msgs.each do |msg| + assert_includes msg, :role + assert_includes msg, :content + refute_equal "tool", msg[:role].to_s + end + end + + test "test_execute returns content for valid model" do + TestClient::RubyLLM::Chat.stub :text, "Hi there!" do + result = AIBackend::RubyLLM.test_execute("https://api.openai.com/v1/", "abc", "gpt-4o") + assert_equal "Hi there!", result + end + end + + test "initialize raises ConfigurationError when token is blank" do + stub_features(default_llm_keys: false) do + service = @assistant.language_model.api_service + service.update!(token: "") + + assert_raises(AIBackend::RubyLLM::ConfigurationError) do + AIBackend::RubyLLM.new(@user, @assistant) + end + end + end + + test "initialize does not raise when token is present" do + backend = AIBackend::RubyLLM.new(@user, @assistant) + assert_instance_of AIBackend::RubyLLM, backend + end + + test "ruby_llm_context sets openai_api_base for Groq" do + @assistant.language_model.api_service.update!(url: APIService::URL_GROQ, driver: "openai") + backend = AIBackend::RubyLLM.new(@user, @assistant) + + context = backend.send(:ruby_llm_context) + assert_equal APIService::URL_GROQ, context.openai_api_base + end + + test "ruby_llm_context does not set openai_api_base for canonical OpenAI URL" do + @assistant.language_model.api_service.update!(url: APIService::URL_OPEN_AI, driver: "openai") + backend = AIBackend::RubyLLM.new(@user, @assistant) + + context = backend.send(:ruby_llm_context) + assert_nil context.openai_api_base + end end diff --git a/test/support/test_client/ruby_llm.rb b/test/support/test_client/ruby_llm.rb index cc3146009..4ce8cc349 100644 --- a/test/support/test_client/ruby_llm.rb +++ b/test/support/test_client/ruby_llm.rb @@ -7,20 +7,21 @@ def initialize(model:, provider: nil, assume_model_exists: nil, context: nil) @@model = model @context = context @messages = [] + @last_response = nil end def with_instructions(instructions) - @@instructions = instructions + self.class.instance_variable_set(:@instructions, instructions) self end def with_params(**params) - @@params = params + self.class.instance_variable_set(:@params, params) self end def with_tools(*tools) - @@tools = tools + self.class.instance_variable_set(:@tools, tools) self end @@ -30,11 +31,19 @@ def add_message(msg) end def complete(&block) + raise self.class.error_to_raise if self.class.error_to_raise + if block - block.call(self.class.api_streaming_response) + response = self.class.api_streaming_response + block.call(response) if response.content.present? else - self.class.api_oneoff_response.dig("choices", 0, "message", "content") + @last_response = self.class.api_oneoff_response.dig("choices", 0, "message", "content") end + self + end + + def content + @last_response end def ask(message = nil, with: nil, &block) @@ -55,10 +64,16 @@ def self.api_oneoff_response end def self.api_streaming_response + content_text = if blank_response + "" + else + text || default_text + end + t = tokens OpenStruct.new( - content: text || default_text, - input_tokens: 8, - output_tokens: 9 + content: content_text, + input_tokens: t[:input_tokens], + output_tokens: t[:output_tokens] ) end @@ -70,8 +85,28 @@ def self.default_text "Hello this is model #{@@model}! How can I assist you today?" end + def self.blank_response + false + end + + def self.error_to_raise + nil + end + + def self.tokens + { input_tokens: 8, output_tokens: 9 } + end + + def self.arguments + {city: "Austin", state: "TX", country: "US"}.to_json + end + + def self.id + "call_BlAN9iRiAD6aCzmBWCjzYxjj" + end + class << self - attr_reader :instructions, :params + attr_reader :instructions, :params, :tools end end From 2c917bc26885456e067aa7281f99f0fbc2971dff Mon Sep 17 00:00:00 2001 From: fluxgravity <44489080+fluxgravity@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:52:54 -0400 Subject: [PATCH 3/7] feat(rubyllm): add Anthropic and Gemini text chat support Extend AIBackend::RubyLLM to rout through all three providers (OpenAI, Anthropic, Gemini) behind the use_ruby_llm feature flag. - Add provider_slug to dynamically select provider from api_service.driver - Update ruby_llm_context to set per-provider API keys via public_send - Update build_chat to pass provider_slug instead of hardcoded :openai - Add provider_for_url for test_execute routing - Expand supports_driver? to return true for all three backends - Add Anthropic and Gemini streaming tests (backend and job level) - Fix duplicate use_ruby_llm key in options.yml from rebase - Fix upstream controller tests for nested preferences params --- app/services/ai_backend/ruby_llm.rb | 38 ++++-- config/options.yml | 1 - .../settings/people_controller_test.rb | 4 +- test/controllers/users_controller_test.rb | 2 +- .../get_next_ai_message_job_ruby_llm_test.rb | 52 ++++++++ test/models/user/features_test.rb | 6 +- test/services/ai_backend/ruby_llm_test.rb | 125 +++++++++++++++++- 7 files changed, 206 insertions(+), 22 deletions(-) diff --git a/app/services/ai_backend/ruby_llm.rb b/app/services/ai_backend/ruby_llm.rb index d5becf9ed..d7cefe17d 100644 --- a/app/services/ai_backend/ruby_llm.rb +++ b/app/services/ai_backend/ruby_llm.rb @@ -13,7 +13,7 @@ class RateLimitError < StandardError; end ].freeze def self.supports_driver?(driver) - ["openai"].include?(driver) + ["openai", "anthropic", "gemini"].include?(driver) end def self.client @@ -24,16 +24,30 @@ def self.gem_class Rails.env.test? ? ::TestClient::RubyLLM::Chat : ::RubyLLM::Chat end + def self.provider_for_url(url) + if url&.include?("api.anthropic.com") + :anthropic + elsif url&.include?("generativelanguage.googleapis.com") + :gemini + else + :openai + end + end + def self.test_execute(url, token, api_name) + provider = provider_for_url(url) if Rails.env.test? - chat = TestClient::RubyLLM::Chat.new(model: api_name, provider: :openai, assume_model_exists: true) + chat = TestClient::RubyLLM::Chat.new(model: api_name, provider: provider, assume_model_exists: true) chat.add_message({ role: "user", content: "Hello!" }) chat.complete.content else - Rails.logger.info "Connecting to OpenAI API server at #{url} with access token of length #{token.to_s.length}" - Rails.logger.info "Testing using model #{api_name}" - context = RubyLLM.context { |c| c.openai_api_key = token; c.openai_api_base = url if url != APIService::URL_OPEN_AI } - chat = RubyLLM::Chat.new(model: api_name, provider: :openai, assume_model_exists: true, context: context) + Rails.logger.info "Connecting to AI API server at #{url} with access token of length #{token.to_s.length}" + Rails.logger.info "Testing using model #{api_name} for provider #{provider}" + context = RubyLLM.context { |c| c.public_send("#{provider}_api_key=", token) } + if provider == :openai && url != APIService::URL_OPEN_AI + context.openai_api_base = url + end + chat = RubyLLM::Chat.new(model: api_name, provider: provider, assume_model_exists: true, context: context) chat.add_message({ role: "user", content: "Hello!" }) chat.complete.content end @@ -72,14 +86,20 @@ def stream_next_conversation_message(&chunk_handler) private + def provider_slug + @api_service.driver.to_sym + end + def build_chat - self.class.gem_class.new(model: @api_name, provider: :openai, assume_model_exists: true, context: ruby_llm_context) + self.class.gem_class.new(model: @api_name, provider: provider_slug, assume_model_exists: true, context: ruby_llm_context) end def ruby_llm_context self.class.client.context do |c| - c.openai_api_key = @token - c.openai_api_base = @api_service.url if @api_service.url != APIService::URL_OPEN_AI + c.public_send("#{provider_slug}_api_key=", @token) + if provider_slug == :openai && @api_service.url != APIService::URL_OPEN_AI + c.openai_api_base = @api_service.url + end end end diff --git a/config/options.yml b/config/options.yml index 5fcc627a6..aaf012137 100644 --- a/config/options.yml +++ b/config/options.yml @@ -46,7 +46,6 @@ shared: default_to_voice: <%= ENV["DEFAULT_TO_VOICE_FEATURE"] || false %> email: <%= ENV["EMAIL_FEATURE"] || default_to(false, except_env_test: true) %> password_reset_email: <%= ENV["PASSWORD_RESET_EMAIL_FEATURE"] || default_to(false, except_env_test: true) %> - use_ruby_llm: <%= ENV["USE_RUBY_LLM_FEATURE"] || false %> assistants_page: <%= ENV["ASSISTANTS_PAGE_FEATURE"] || true %> use_ruby_llm: <%= ENV["USE_RUBY_LLM_FEATURE"] || false %> settings: diff --git a/test/controllers/settings/people_controller_test.rb b/test/controllers/settings/people_controller_test.rb index fde263646..3c8fb8886 100644 --- a/test/controllers/settings/people_controller_test.rb +++ b/test/controllers/settings/people_controller_test.rb @@ -12,13 +12,13 @@ class Settings::PeopleControllerTest < ActionDispatch::IntegrationTest @user.save! params = person_params - params["personable_attributes"]["preferences"] = { dark_mode: "dark" } + params["personable_attributes"]["dark_mode"] = "dark" patch settings_person_url, params: { person: params } assert_redirected_to edit_settings_person_url @user.reload - assert_equal "dark", @user.preferences[:dark_mode] + assert_equal "dark", @user.dark_mode assert_equal({ use_ruby_llm: true }, @user.preferences[:feature]) end diff --git a/test/controllers/users_controller_test.rb b/test/controllers/users_controller_test.rb index 8658ef3d0..99fb5cf72 100644 --- a/test/controllers/users_controller_test.rb +++ b/test/controllers/users_controller_test.rb @@ -106,7 +106,7 @@ class UsersControllerTest < ActionDispatch::IntegrationTest user.update!(dark_mode: "dark", nav_closed: false) login_as user - patch user_url(user), params: { user: { nav_closed: true } } + patch user_url(user), params: { user: { preferences: { nav_closed: "true" } } } assert_response :redirect user.reload diff --git a/test/jobs/get_next_ai_message_job_ruby_llm_test.rb b/test/jobs/get_next_ai_message_job_ruby_llm_test.rb index 1c0bcfc65..514bf6465 100644 --- a/test/jobs/get_next_ai_message_job_ruby_llm_test.rb +++ b/test/jobs/get_next_ai_message_job_ruby_llm_test.rb @@ -79,4 +79,56 @@ class GetNextAIMessageJobRubyLLMTest < ActiveJob::TestCase assert @message.reload.not_failed? end + + # Phase 3 — Anthropic + Gemini/Groq text chat + + test "populates the latest message from the assistant via RubyLLM with anthropic driver" do + @assistant.language_model.api_service.update!(driver: "anthropic") + stub_features(use_ruby_llm: true) do + assert_no_difference "@conversation.messages.reload.length" do + TestClient::RubyLLM::Chat.stub :text, "Hello from Claude via RubyLLM" do + assert GetNextAIMessageJob.perform_now(@user.id, @message.id, @assistant.id) + end + end + end + + assert_equal "Hello from Claude via RubyLLM", @conversation.latest_message_for_version(:latest).content_text + end + + test "populates the latest message from the assistant via RubyLLM with gemini driver" do + @assistant.language_model.api_service.update!(driver: "gemini") + stub_features(use_ruby_llm: true) do + assert_no_difference "@conversation.messages.reload.length" do + TestClient::RubyLLM::Chat.stub :text, "Hello from Gemini via RubyLLM" do + assert GetNextAIMessageJob.perform_now(@user.id, @message.id, @assistant.id) + end + end + end + + assert_equal "Hello from Gemini via RubyLLM", @conversation.latest_message_for_version(:latest).content_text + end + + test "when API response is empty with anthropic driver, a nice error message is displayed" do + @assistant.language_model.api_service.update!(driver: "anthropic") + stub_features(use_ruby_llm: true) do + TestClient::RubyLLM::Chat.stub :text, "" do + assert GetNextAIMessageJob.perform_now(@user.id, @message.id, @assistant.id) + end + end + + assert_includes @conversation.latest_message_for_version(:latest).content_text, "a blank response" + assert @message.reload.failed? + end + + test "when API response is empty with gemini driver, a nice error message is displayed" do + @assistant.language_model.api_service.update!(driver: "gemini") + stub_features(use_ruby_llm: true) do + TestClient::RubyLLM::Chat.stub :text, "" do + assert GetNextAIMessageJob.perform_now(@user.id, @message.id, @assistant.id) + end + end + + assert_includes @conversation.latest_message_for_version(:latest).content_text, "a blank response" + assert @message.reload.failed? + end end diff --git a/test/models/user/features_test.rb b/test/models/user/features_test.rb index e35b67a5a..32fe547c7 100644 --- a/test/models/user/features_test.rb +++ b/test/models/user/features_test.rb @@ -39,8 +39,10 @@ class User::FeaturesTest < ActiveSupport::TestCase assert_raises(KeyError) { @user.features[:voic] = true } end - test "RubyLLM is unavailable while the RubyLLM backend does not exist or does not support a driver" do - refute User::Features.ruby_llm_available?("openai") + test "RubyLLM is available for all three drivers in Phase 3" do + assert User::Features.ruby_llm_available?("openai") + assert User::Features.ruby_llm_available?("anthropic") + assert User::Features.ruby_llm_available?("gemini") end test "derived backend names are valid and non-chat drivers are not" do diff --git a/test/services/ai_backend/ruby_llm_test.rb b/test/services/ai_backend/ruby_llm_test.rb index 253c7d326..53fa5eb4a 100644 --- a/test/services/ai_backend/ruby_llm_test.rb +++ b/test/services/ai_backend/ruby_llm_test.rb @@ -10,12 +10,12 @@ class AIBackend::RubyLLMTest < ActiveSupport::TestCase @gemini_service = api_services(:keith_gemini_service) end - # Phase 1 — updated for Phase 2 driver support + # Phase 1 — updated for Phase 3 driver support - test "supports_driver? returns true for openai, false for others in Phase 2" do + test "supports_driver? returns true for all backends in Phase 3" do assert AIBackend::RubyLLM.supports_driver?("openai") - refute AIBackend::RubyLLM.supports_driver?("anthropic") - refute AIBackend::RubyLLM.supports_driver?("gemini") + assert AIBackend::RubyLLM.supports_driver?("anthropic") + assert AIBackend::RubyLLM.supports_driver?("gemini") end test "APIService#ai_backend returns old OpenAI class when feature flag is off" do @@ -42,10 +42,11 @@ class AIBackend::RubyLLMTest < ActiveSupport::TestCase end end - test "APIService#ai_backend falls back to old classes for unsupported drivers even with flag on" do + test "APIService#ai_backend returns RubyLLM for all drivers when flag is on in Phase 3" do stub_features(use_ruby_llm: true) do - assert_equal AIBackend::Anthropic, @anthropic_service.ai_backend - assert_equal AIBackend::Gemini, @gemini_service.ai_backend + assert_equal AIBackend::RubyLLM, @openai_service.ai_backend + assert_equal AIBackend::RubyLLM, @anthropic_service.ai_backend + assert_equal AIBackend::RubyLLM, @gemini_service.ai_backend end end @@ -305,4 +306,114 @@ class AIBackend::RubyLLMTest < ActiveSupport::TestCase context = backend.send(:ruby_llm_context) assert_nil context.openai_api_base end + + # Phase 3 — Anthropic + Gemini/Groq text chat + + test "provider_for_url returns anthropic for Anthropic URL" do + assert_equal :anthropic, AIBackend::RubyLLM.provider_for_url(APIService::URL_ANTHROPIC) + end + + test "provider_for_url returns gemini for Gemini URL" do + assert_equal :gemini, AIBackend::RubyLLM.provider_for_url(APIService::URL_GEMINI) + end + + test "provider_for_url returns openai for OpenAI URL" do + assert_equal :openai, AIBackend::RubyLLM.provider_for_url(APIService::URL_OPEN_AI) + end + + test "provider_for_url returns openai for Groq URL" do + assert_equal :openai, AIBackend::RubyLLM.provider_for_url(APIService::URL_GROQ) + end + + test "test_execute uses anthropic provider for Anthropic URL" do + TestClient::RubyLLM::Chat.stub :text, "Bonjour" do + result = AIBackend::RubyLLM.test_execute(APIService::URL_ANTHROPIC, "abc", "claude-3-opus") + assert_equal "Bonjour", result + end + end + + test "test_execute uses gemini provider for Gemini URL" do + TestClient::RubyLLM::Chat.stub :text, "Hallo" do + result = AIBackend::RubyLLM.test_execute(APIService::URL_GEMINI, "abc", "gemini-pro") + assert_equal "Hallo", result + end + end + + test "ruby_llm_context sets anthropic_api_key for anthropic driver" do + @assistant.language_model.api_service.update!(driver: "anthropic") + backend = AIBackend::RubyLLM.new(@user, @assistant) + + context = backend.send(:ruby_llm_context) + assert_equal @assistant.language_model.api_service.effective_token, context.anthropic_api_key + end + + test "ruby_llm_context sets gemini_api_key for gemini driver" do + @assistant.language_model.api_service.update!(driver: "gemini") + backend = AIBackend::RubyLLM.new(@user, @assistant) + + context = backend.send(:ruby_llm_context) + assert_equal @assistant.language_model.api_service.effective_token, context.gemini_api_key + end + + test "build_chat uses provider_slug for anthropic" do + @assistant.language_model.api_service.update!(driver: "anthropic") + backend = AIBackend::RubyLLM.new(@user, @assistant) + chat_class = AIBackend::RubyLLM.gem_class + + chat_class.stub :new, ->(**kwargs) { kwargs } do + chat_args = backend.send(:build_chat) + assert_equal :anthropic, chat_args[:provider] + end + end + + test "build_chat uses provider_slug for gemini" do + @assistant.language_model.api_service.update!(driver: "gemini") + backend = AIBackend::RubyLLM.new(@user, @assistant) + chat_class = AIBackend::RubyLLM.gem_class + + chat_class.stub :new, ->(**kwargs) { kwargs } do + chat_args = backend.send(:build_chat) + assert_equal :gemini, chat_args[:provider] + end + end + + test "stream_next_conversation_message works with anthropic driver" do + @assistant.language_model.api_service.update!(driver: "anthropic") + @assistant.language_model.update!(supports_tools: false) + message = @conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: @assistant, + index: @conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, @conversation, message) + TestClient::RubyLLM::Chat.stub :text, "Claude streaming" do + chunks = [] + result = backend.stream_next_conversation_message { |c| chunks << c } + assert_equal ["Claude streaming"], chunks + assert_nil result + end + end + + test "stream_next_conversation_message works with gemini driver" do + @assistant.language_model.api_service.update!(driver: "gemini") + @assistant.language_model.update!(supports_tools: false) + message = @conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: @assistant, + index: @conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, @conversation, message) + TestClient::RubyLLM::Chat.stub :text, "Gemini streaming" do + chunks = [] + result = backend.stream_next_conversation_message { |c| chunks << c } + assert_equal ["Gemini streaming"], chunks + assert_nil result + end + end end From 98d518759f220f37f75fe482ef33bdf2233ac162 Mon Sep 17 00:00:00 2001 From: fluxgravity <44489080+fluxgravity@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:42:54 -0400 Subject: [PATCH 4/7] feat(rubyllm): add image and PDF attachment support (Phase 4) Extend preceding_conversation_messages to pass multimodal input to RubyLLM for all three drivers. - Build RubyLLM::Content with image attachments for messages with documents - Inline PDF text via document.extract_pdf_text (no native PDF upload) - Strip json_of_generated_image from assistant content on replay - Add per-driver vision streaming, PDF extraction, mixed image+text, and sanitize_content tests --- app/services/ai_backend/ruby_llm.rb | 50 ++++- test/services/ai_backend/ruby_llm_test.rb | 241 ++++++++++++++++++++++ 2 files changed, 287 insertions(+), 4 deletions(-) diff --git a/app/services/ai_backend/ruby_llm.rb b/app/services/ai_backend/ruby_llm.rb index d7cefe17d..c3a257ced 100644 --- a/app/services/ai_backend/ruby_llm.rb +++ b/app/services/ai_backend/ruby_llm.rb @@ -133,13 +133,55 @@ def preceding_conversation_messages @conversation.messages.for_conversation_version(@message.version).where("messages.index < ?", @message.index).collect do |message| next if message.tool? - { - role: message.role, - content: message.content_text, - } + if @assistant.supports_images? && message.documents.present? && message.role == "user" + content_parts = [message.content_text] + attachments = [] + + message.documents.each do |document| + if document.has_image? + attachments << ::RubyLLM::Attachment.new(document.file) + elsif document.has_document_pdf? + pdf_text = document.extract_pdf_text + if pdf_text.present? + content_parts << "\n\n[PDF Document: #{document.filename}]\n#{pdf_text}" + else + content_parts << "\n[PDF Document: #{document.filename} - Unable to extract text from this PDF]" + end + end + end + + text = content_parts.compact.join + content = if attachments.any? + ::RubyLLM::Content.new(text, attachments) + else + text + end + + { role: message.role, content: content } + else + { + role: message.role, + content: sanitize_content(message), + } + end end.compact end + def sanitize_content(message) + return "" unless message.content_text.present? + + begin + parsed = JSON.parse(message.content_text) + if parsed.is_a?(Hash) && parsed.has_key?("json_of_generated_image") + parsed.except("json_of_generated_image").to_json + else + message.content_text + end + rescue JSON::ParserError + message.content_text + end + end + def client_method_name raise NotImplementedError end diff --git a/test/services/ai_backend/ruby_llm_test.rb b/test/services/ai_backend/ruby_llm_test.rb index 53fa5eb4a..996252654 100644 --- a/test/services/ai_backend/ruby_llm_test.rb +++ b/test/services/ai_backend/ruby_llm_test.rb @@ -1,6 +1,8 @@ require "test_helper" class AIBackend::RubyLLMTest < ActiveSupport::TestCase + include ActionDispatch::TestProcess::FixtureFile + setup do @conversation = conversations(:attachments) @assistant = assistants(:keith_gpt4) @@ -416,4 +418,243 @@ class AIBackend::RubyLLMTest < ActiveSupport::TestCase assert_nil result end end + + # Phase 4 — Image/PDF attachment parity + + test "preceding_conversation_messages includes image attachments when supports_images is true" do + assistant = assistants(:keith_claude35) + assistant.language_model.update!(supports_images: true, supports_tools: false) + conversation = conversations(:attachments) + message = conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: assistant, + index: conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, assistant, conversation, message) + msgs = backend.send(:preceding_conversation_messages) + + message_with_attachments = msgs.find { |m| m[:content].is_a?(::RubyLLM::Content) } + assert message_with_attachments, "Should find a message with RubyLLM::Content for image attachments" + assert message_with_attachments[:content].attachments.any?, "Content should have attachments" + end + + test "preceding_conversation_messages does not include attachments when supports_images is false" do + assistant = assistants(:keith_claude35) + assistant.language_model.update!(supports_images: false, supports_tools: false) + conversation = conversations(:attachments) + message = conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: assistant, + index: conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, assistant, conversation, message) + msgs = backend.send(:preceding_conversation_messages) + + msgs_with_attachments = msgs.filter_map { |m| m if m[:content].is_a?(::RubyLLM::Content) } + assert_empty msgs_with_attachments, "Should not have any Content objects when supports_images is false" + end + + test "preceding_conversation_messages inlines PDF text when supports_images is true" do + pdf_content = "%PDF-1.4\n1 0 obj\n<<\n/Type /Catalog\n/Pages 2 0 R\n>>\nendobj\n2 0 obj\n<<\n/Type /Pages\n/Kids [3 0 R]\n/Count 1\n>>\nendobj\n3 0 obj\n<<\n/Type /Page\n/Parent 2 0 R\n/MediaBox [0 0 612 792]\n/Contents 4 0 R\n>>\nendobj\n4 0 obj\n<<\n/Length 44\n>>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Hello World) Tj\nET\nendstream\nendobj\nxref\n0 5\n0000000000 65535 f \n0000000009 00000 n \n0000000058 00000 n \n0000000115 00000 n \n0000000200 00000 n \ntrailer\n<<\n/Size 5\n/Root 1 0 R\n>>\nstartxref\n294\n%%EOF" + + test_file = Tempfile.new(["test", ".pdf"]) + test_file.write(pdf_content) + test_file.rewind + + assistant = assistants(:keith_claude35) + assistant.language_model.update!(supports_images: true, supports_tools: false) + + conversation = Conversation.create!(user: @user, assistant: assistant, title: "PDF Test") + + pdf_message = conversation.messages.create!( + role: "user", + content_text: "Check this document", + assistant: assistant + ) + pdf_message.documents.create!( + file: fixture_file_upload(test_file.path, "application/pdf"), + filename: "test.pdf" + ) + + message = conversation.messages.create!( + role: "assistant", + content_text: "Let me check", + assistant: assistant + ) + + backend = AIBackend::RubyLLM.new(@user, assistant, conversation, message) + msgs = backend.send(:preceding_conversation_messages) + + pdf_entry = msgs.find { |m| m[:role] == "user" && m[:content].to_s.include?("PDF Document: test.pdf") } + assert pdf_entry, "Should include PDF content reference" + assert pdf_entry[:content].to_s.include?("PDF Document: test.pdf"), "Should include PDF document reference" + assert pdf_entry[:content].to_s.include?("Unable to extract text from this PDF"), "Should include error for failed extraction" + ensure + test_file&.close + test_file&.unlink + end + + test "sanitize_content removes json_of_generated_image from JSON content" do + @assistant.language_model.update!(supports_tools: false) + message = @conversation.messages.create!( + role: :assistant, + content_text: '{"prompt_given":"cat","json_of_generated_image":"base64data","message_to_user":"image"}', + assistant: @assistant, + index: @conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, @conversation, message) + result = backend.send(:sanitize_content, message) + + parsed = JSON.parse(result) + refute parsed.has_key?("json_of_generated_image"), "Should remove json_of_generated_image" + assert_equal "cat", parsed["prompt_given"] + end + + test "sanitize_content passes through plain text unchanged" do + @assistant.language_model.update!(supports_tools: false) + message = @conversation.messages.create!( + role: :assistant, + content_text: "Hello, how are you?", + assistant: @assistant, + index: @conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, @conversation, message) + result = backend.send(:sanitize_content, message) + + assert_equal "Hello, how are you?", result + end + + test "sanitize_content returns empty string for nil content" do + @assistant.language_model.update!(supports_tools: false) + message = @conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: @assistant, + index: @conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, @conversation, message) + result = backend.send(:sanitize_content, message) + + assert_equal "", result + end + + test "stream_next_conversation_message works with image attachments" do + assistant = assistants(:keith_claude35) + assistant.language_model.update!(supports_images: true, supports_tools: false) + conversation = conversations(:attachments) + message = conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: assistant, + index: conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, assistant, conversation, message) + TestClient::RubyLLM::Chat.stub :text, "I see a cat" do + chunks = [] + result = backend.stream_next_conversation_message { |c| chunks << c } + assert_equal ["I see a cat"], chunks + assert_nil result + end + end + + test "image attachment streaming produces vision response with openai driver" do + assistant = assistants(:keith_claude35) + assistant.language_model.api_service.update!(driver: "openai") + assistant.language_model.update!(supports_images: true, supports_tools: false) + conversation = conversations(:attachments) + message = conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: assistant, + index: conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, assistant, conversation, message) + TestClient::RubyLLM::Chat.stub :text, "OpenAI sees a feline" do + chunks = [] + result = backend.stream_next_conversation_message { |c| chunks << c } + assert_equal ["OpenAI sees a feline"], chunks + assert_nil result + end + end + + test "image attachment streaming produces vision response with anthropic driver" do + assistant = assistants(:keith_claude35) + assistant.language_model.api_service.update!(driver: "anthropic") + assistant.language_model.update!(supports_images: true, supports_tools: false) + conversation = conversations(:attachments) + message = conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: assistant, + index: conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, assistant, conversation, message) + TestClient::RubyLLM::Chat.stub :text, "Claude sees a feline" do + chunks = [] + result = backend.stream_next_conversation_message { |c| chunks << c } + assert_equal ["Claude sees a feline"], chunks + assert_nil result + end + end + + test "image attachment streaming produces vision response with gemini driver" do + assistant = assistants(:keith_claude35) + assistant.language_model.api_service.update!(driver: "gemini") + assistant.language_model.update!(supports_images: true, supports_tools: false) + conversation = conversations(:attachments) + message = conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: assistant, + index: conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, assistant, conversation, message) + TestClient::RubyLLM::Chat.stub :text, "Gemini sees a feline" do + chunks = [] + result = backend.stream_next_conversation_message { |c| chunks << c } + assert_equal ["Gemini sees a feline"], chunks + assert_nil result + end + end + + test "preceding_conversation_messages preserves text alongside image attachments" do + assistant = assistants(:keith_claude35) + assistant.language_model.update!(supports_images: true, supports_tools: false) + conversation = conversations(:attachments) + message = conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: assistant, + index: conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, assistant, conversation, message) + msgs = backend.send(:preceding_conversation_messages) + + mixed_msg = msgs.find { |m| m[:content].is_a?(::RubyLLM::Content) } + assert mixed_msg, "Should find a message with mixed content" + assert mixed_msg[:content].text.present?, "Content text should be preserved alongside attachments" + assert mixed_msg[:content].attachments.any?, "Attachments should be present" + end end From c478d058fb4b7f3645b2547c2c515103b688cde5 Mon Sep 17 00:00:00 2001 From: fluxgravity <44489080+fluxgravity@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:05:02 -0400 Subject: [PATCH 5/7] feat(rubyllm): add tool-interception pre-check (Phase 4.5) --- app/services/ai_backend/ruby_llm.rb | 1 + .../ai_backend/ruby_llm/intercepted_tool.rb | 27 ++++++++ .../ruby_llm/tool_interception_test.rb | 68 +++++++++++++++++++ 3 files changed, 96 insertions(+) create mode 100644 app/services/ai_backend/ruby_llm/intercepted_tool.rb create mode 100644 test/services/ai_backend/ruby_llm/tool_interception_test.rb diff --git a/app/services/ai_backend/ruby_llm.rb b/app/services/ai_backend/ruby_llm.rb index c3a257ced..f87842451 100644 --- a/app/services/ai_backend/ruby_llm.rb +++ b/app/services/ai_backend/ruby_llm.rb @@ -1,6 +1,7 @@ class AIBackend::RubyLLM < AIBackend class ConfigurationError < StandardError; end class RateLimitError < StandardError; end + class ToolCallIntercepted < StandardError; end CONFIGURATION_ERRORS = [ ::RubyLLM::UnauthorizedError, ::RubyLLM::ConfigurationError, diff --git a/app/services/ai_backend/ruby_llm/intercepted_tool.rb b/app/services/ai_backend/ruby_llm/intercepted_tool.rb new file mode 100644 index 000000000..7f9559999 --- /dev/null +++ b/app/services/ai_backend/ruby_llm/intercepted_tool.rb @@ -0,0 +1,27 @@ +# Intercepts RubyLLM's automatic tool-execution loop by raising from `execute`. +# HostedGPT executes tools itself via Toolbox.call (see Phase 5), so RubyLLM +# must never run a tool or auto-continue the conversation after a tool-call +# response. Raising here halts Chat#handle_tool_calls before it can append a +# role: :tool result message; the assistant's tool-call message is already on +# chat.messages by the time this runs (verified against ruby_llm-1.16.0). +class AIBackend::RubyLLM::InterceptedTool < ::RubyLLM::Tool + def initialize(name:, description:, params_schema:) + @tool_name = name + @tool_description = description + @params_schema = params_schema + end + + # Overrides are required: the base class otherwise derives `name` from the + # class name, collapsing every registered tool into one garbage entry. + def name + @tool_name + end + + def description + @tool_description + end + + def execute(**) + raise AIBackend::RubyLLM::ToolCallIntercepted + end +end diff --git a/test/services/ai_backend/ruby_llm/tool_interception_test.rb b/test/services/ai_backend/ruby_llm/tool_interception_test.rb new file mode 100644 index 000000000..41890a0ed --- /dev/null +++ b/test/services/ai_backend/ruby_llm/tool_interception_test.rb @@ -0,0 +1,68 @@ +require "test_helper" + +class AIBackend::RubyLLM::ToolInterceptionTest < ActiveSupport::TestCase + def build_tool + AIBackend::RubyLLM::InterceptedTool.new( + name: "get_weather", + description: "Get the current weather for a location", + params_schema: { type: "object", properties: { location: { type: "string" } } } + ) + end + + def tool_call(id, location) + RubyLLM::ToolCall.new(id: id, name: "get_weather", arguments: { location: location }) + end + + def build_chat + chat = RubyLLM::Chat.new(model: "gpt-4o", provider: :openai, assume_model_exists: true) + chat.add_message(role: :user, content: "What's the weather?") + chat.with_tools(build_tool) + end + + def stub_tool_call_response(chat, tool_calls, &) + response = RubyLLM::Message.new( + role: :assistant, + content: nil, + tool_calls: tool_calls.to_h { |tc| [tc.id, tc] } + ) + chat.stub(:provider_completion, response, &) + end + + test "chat.complete raises ToolCallIntercepted when the model requests a tool" do + chat = build_chat + + assert_raises(AIBackend::RubyLLM::ToolCallIntercepted) do + stub_tool_call_response(chat, [tool_call("call_abc123", "Austin")]) { chat.complete } + end + end + + test "every requested tool call is captured on the assistant message before the raise" do + calls = [tool_call("call_abc123", "Austin"), tool_call("call_def456", "Boston")] + chat = build_chat + + assert_raises(AIBackend::RubyLLM::ToolCallIntercepted) do + stub_tool_call_response(chat, calls) { chat.complete } + end + + last = chat.messages.last + assert last.tool_call?, "the assistant message should carry tool calls" + assert_equal calls.size, last.tool_calls.size + + assert_equal ["call_abc123", "call_def456"], last.tool_calls.values.map(&:id) + assert_equal ["get_weather", "get_weather"], last.tool_calls.values.map(&:name) + assert_equal ["Austin", "Boston"], last.tool_calls.values.map { |tc| tc.arguments[:location] } + end + + test "execution halts cleanly: no tool result is appended and the chat does not auto-continue" do + chat = build_chat + + assert_raises(AIBackend::RubyLLM::ToolCallIntercepted) do + stub_tool_call_response(chat, [tool_call("call_abc123", "Austin")]) { chat.complete } + end + + assert_empty chat.messages.select { |m| m.role == :tool }, + "no role: :tool result message should be appended" + assert_equal 1, chat.messages.count { |m| m.role == :assistant }, + "the chat should not have auto-continued with a follow-up assistant message" + end +end From 1591e82e9c29c65aed70a1c3a4f5d437653af46e Mon Sep 17 00:00:00 2001 From: fluxgravity <44489080+fluxgravity@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:24:00 -0400 Subject: [PATCH 6/7] feat(rubyllm): add tool-calling parity (Phase 5) Implement Phase 5 of the RubyLLM migration. Closes the unknown-name gap with a new InterceptedChat (raise ToolCallIntercepted for any tool call) and maps the unified error contract at the chat.complete boundary, since RubyLLM raises in its HTTP layer outside the per-chunk stream_handler. --- app/services/ai_backend/ruby_llm.rb | 91 ++++++++- .../ai_backend/ruby_llm/intercepted_chat.rb | 15 ++ ...utotitle_conversation_job_ruby_llm_test.rb | 30 +++ .../get_next_ai_message_job_ruby_llm_test.rb | 56 ++++++ .../ruby_llm/tool_interception_test.rb | 19 ++ test/services/ai_backend/ruby_llm_test.rb | 188 +++++++++++++++++- test/support/test_client/ruby_llm.rb | 31 ++- 7 files changed, 416 insertions(+), 14 deletions(-) create mode 100644 app/services/ai_backend/ruby_llm/intercepted_chat.rb create mode 100644 test/jobs/autotitle_conversation_job_ruby_llm_test.rb diff --git a/app/services/ai_backend/ruby_llm.rb b/app/services/ai_backend/ruby_llm.rb index f87842451..8991cba01 100644 --- a/app/services/ai_backend/ruby_llm.rb +++ b/app/services/ai_backend/ruby_llm.rb @@ -1,6 +1,8 @@ class AIBackend::RubyLLM < AIBackend - class ConfigurationError < StandardError; end - class RateLimitError < StandardError; end + # Inherits the base ConfigurationError so GetNextAIMessageJob's unified + # `rescue AIBackend::ConfigurationError` catches bad-key failures and renders + # key_error_message, instead of falling through to the generic 3x-retry path. + class ConfigurationError < AIBackend::ConfigurationError; end class ToolCallIntercepted < StandardError; end CONFIGURATION_ERRORS = [ @@ -22,7 +24,7 @@ def self.client end def self.gem_class - Rails.env.test? ? ::TestClient::RubyLLM::Chat : ::RubyLLM::Chat + Rails.env.test? ? ::TestClient::RubyLLM::Chat : ::AIBackend::RubyLLM::InterceptedChat end def self.provider_for_url(url) @@ -65,12 +67,18 @@ def initialize(user, assistant, conversation = nil, message = nil) raise ConfigurationError if @api_service.requires_token? && @token.blank? end - def get_oneoff_message(instructions, messages, params = {}) + def get_oneoff_message(instructions, messages, params = {}, json: false) + instructions = "#{instructions} Respond with ONLY valid JSON, no markdown or explanation." if json + chat = build_chat chat.with_instructions(instructions) preceding_messages(messages).each { |msg| chat.add_message(msg) } chat.with_params(**params) if params.present? chat.complete.content + rescue *CONFIGURATION_ERRORS => e + raise ConfigurationError, e.message + rescue *RATE_LIMIT_ERRORS => e + raise ::Faraday::TooManyRequestsError, e.message end def stream_next_conversation_message(&chunk_handler) @@ -79,7 +87,20 @@ def stream_next_conversation_message(&chunk_handler) chat = build_chat chat.with_instructions(full_instructions) preceding_conversation_messages.each { |msg| chat.add_message(msg) } - chat.complete { |chunk| stream_handler.call(chunk, chunk_handler) } + chat.with_tools(*tool_instances) if tools_enabled? + + begin + chat.complete { |chunk| stream_handler.call(chunk, chunk_handler) } + rescue ToolCallIntercepted + tool_calls = chat.messages.last&.tool_calls + return format_tool_calls(tool_calls) if tool_calls.present? + + raise ::Faraday::ParsingError + rescue *CONFIGURATION_ERRORS => e + raise ConfigurationError, e.message + rescue *RATE_LIMIT_ERRORS => e + raise ::Faraday::TooManyRequestsError, e.message + end raise ::Faraday::ParsingError if @stream_response_text.blank? nil @@ -123,7 +144,7 @@ def stream_handler rescue *CONFIGURATION_ERRORS => e raise ConfigurationError, e.message rescue *RATE_LIMIT_ERRORS => e - raise RateLimitError, e.message + raise ::Faraday::TooManyRequestsError, e.message rescue => e Rails.logger.info "\nUnhandled error in AIBackend::RubyLLM response handler: #{e.message}" Rails.logger.info e.backtrace.join("\n") @@ -132,9 +153,9 @@ def stream_handler def preceding_conversation_messages @conversation.messages.for_conversation_version(@message.version).where("messages.index < ?", @message.index).collect do |message| - next if message.tool? - - if @assistant.supports_images? && message.documents.present? && message.role == "user" + if message.tool? + { role: :tool, content: message.content_text || "", tool_call_id: message.tool_call_id } + elsif @assistant.supports_images? && message.documents.present? && message.role == "user" content_parts = [message.content_text] attachments = [] @@ -159,6 +180,12 @@ def preceding_conversation_messages end { role: message.role, content: content } + elsif message.assistant? && message.content_tool_calls.present? + { + role: :assistant, + content: sanitize_content(message), + tool_calls: tool_calls_hash(message), + } else { role: message.role, @@ -168,6 +195,20 @@ def preceding_conversation_messages end.compact end + # Reconstructs the stored OpenAI-shaped content_tool_calls (serialized via + # JsonSerializer) into RubyLLM::ToolCall objects keyed by id — the shape + # RubyLLM expects on a replayed assistant message. + def tool_calls_hash(message) + message.content_tool_calls.each_with_object({}) do |tc, hash| + id = tc[:id] || tc["id"] + name = tc.dig(:function, :name) || tc.dig("function", "name") + args = tc.dig(:function, :arguments) || tc.dig("function", "arguments") || "{}" + args = JSON.parse(args) if args.is_a?(String) + + hash[id] = ::RubyLLM::ToolCall.new(id: id, name: name, arguments: args) + end + end + def sanitize_content(message) return "" unless message.content_text.present? @@ -195,7 +236,35 @@ def set_client_config(*) raise NotImplementedError end - def format_parallel_tool_calls(*) - raise NotImplementedError + def tools_enabled? + @assistant.language_model.supports_tools? && @api_service.url != APIService::URL_GROQ + end + + def tool_instances + Toolbox.tools.map do |tool| + AIBackend::RubyLLM::InterceptedTool.new( + name: tool.dig(:function, :name), + description: tool.dig(:function, :description), + params_schema: tool.dig(:function, :parameters), + ) + end + end + + def format_tool_calls(tool_calls) + tool_calls.values.map.with_index do |tc, i| + { index: i, type: "function", id: tc.id, + function: { name: tc.name, arguments: tc.arguments.to_json } } + end + end + + # RubyLLM returns tool calls already separated, so these defensive identities + # satisfy the AIBackend::Tools contract even though the overridden + # stream_next_conversation_message never calls them. + def format_parallel_tool_calls(content_tool_calls) + content_tool_calls + end + + def parallel_tool_calls(content_tool_calls) + content_tool_calls end end diff --git a/app/services/ai_backend/ruby_llm/intercepted_chat.rb b/app/services/ai_backend/ruby_llm/intercepted_chat.rb new file mode 100644 index 000000000..5e8341950 --- /dev/null +++ b/app/services/ai_backend/ruby_llm/intercepted_chat.rb @@ -0,0 +1,15 @@ +# Overrides RubyLLM::Chat#handle_tool_calls (private) so any tool call the model +# requests — registered or hallucinated — raises ToolCallIntercepted before +# RubyLLM can auto-execute it or recurse. Without this, an unknown tool name is +# answered by Chat#execute_tool with an error hash (no raise), which appends an +# internal role: :tool result and auto-continues, silently dropping the call and +# risking unbounded recursion. This couples to a private method, but the gem is +# pinlocked to ~> 1.16.0; InterceptedTool#execute remains as a safety net so a +# registered-name call still halts even if this override is renamed upstream. +class AIBackend::RubyLLM::InterceptedChat < ::RubyLLM::Chat + private + + def handle_tool_calls(_response, &) + raise AIBackend::RubyLLM::ToolCallIntercepted + end +end diff --git a/test/jobs/autotitle_conversation_job_ruby_llm_test.rb b/test/jobs/autotitle_conversation_job_ruby_llm_test.rb new file mode 100644 index 000000000..e3070ce0e --- /dev/null +++ b/test/jobs/autotitle_conversation_job_ruby_llm_test.rb @@ -0,0 +1,30 @@ +require "test_helper" + +class AutotitleConversationJobRubyLLMTest < ActiveJob::TestCase + test "sets conversation title via RubyLLM" do + conversation = conversations(:greeting) + conversation.update!(title: nil) + + stub_features(use_ruby_llm: true) do + TestClient::RubyLLM::Chat.stub :text, '{"topic":"Hear me"}' do + AutotitleConversationJob.perform_now(conversation.id) + end + end + + assert_equal "Hear me", conversation.reload.title + end + + test "unusable replies keep the prior title via RubyLLM" do + conversation = conversations(:greeting) + conversation.update!(title: nil) + + ["", " ", "not json at all", "{\"topic\":\"\"}"].each do |reply| + stub_features(use_ruby_llm: true) do + TestClient::RubyLLM::Chat.stub :text, reply do + AutotitleConversationJob.perform_now(conversation.id) + end + end + assert_nil conversation.reload.title, "case #{reply.inspect} should leave the title untouched" + end + end +end diff --git a/test/jobs/get_next_ai_message_job_ruby_llm_test.rb b/test/jobs/get_next_ai_message_job_ruby_llm_test.rb index 514bf6465..b579d83f3 100644 --- a/test/jobs/get_next_ai_message_job_ruby_llm_test.rb +++ b/test/jobs/get_next_ai_message_job_ruby_llm_test.rb @@ -131,4 +131,60 @@ class GetNextAIMessageJobRubyLLMTest < ActiveJob::TestCase assert_includes @conversation.latest_message_for_version(:latest).content_text, "a blank response" assert @message.reload.failed? end + + # Phase 5 — Tool/function calling + error contract + + test "populates a tool response call and creates additional tool messages" do + @assistant.language_model.update!(supports_tools: true) + + assert_difference "@conversation.messages.reload.length", 2 do + stub_features(use_ruby_llm: true) do + TestClient::RubyLLM::Chat.stub :function, "helloworld_hi" do + TestClient::RubyLLM::Chat.stub :arguments, { name: "Keith" }.to_json do + assert GetNextAIMessageJob.perform_now(@user.id, @message.id, @assistant.id) + end + end + end + end + + @message.reload + assert @message.content_text.blank? + assert @message.content_tool_calls.present?, "Assistant should have decided to call a tool" + + new_messages = @conversation.messages.where("id > ?", @message.id).order(:created_at) + + first_new_message = new_messages.first + assert first_new_message.tool? + assert_equal "Hello, Keith!".to_json, first_new_message.content_text + assert first_new_message.tool_call_id.present? + assert first_new_message.content_tool_calls.present? + assert_equal @message.content_tool_calls.dig(0, :id), first_new_message.tool_call_id + + second_new_message = new_messages.second + assert second_new_message.assistant?, "Second new message should be queued for the assistant reply" + assert second_new_message.content_text.nil? + end + + test "a config error renders key_error_message and marks the message failed" do + stub_features(use_ruby_llm: true, default_llm_keys: false) do + @assistant.language_model.api_service.update!(token: "") + assert_no_enqueued_jobs only: GetNextAIMessageJob do + assert GetNextAIMessageJob.perform_now(@user.id, @message.id, @assistant.id) + end + end + + assert_equal AIBackend::RubyLLM.key_error_message, @message.reload.content_text + assert @message.failed?, "The message should have been marked failed so a Retry button is offered" + end + + test "a rate limit error renders the quota message and marks the message failed" do + stub_features(use_ruby_llm: true) do + TestClient::RubyLLM::Chat.stub :error_to_raise, Faraday::TooManyRequestsError.new("quota exceeded") do + assert GetNextAIMessageJob.perform_now(@user.id, @message.id, @assistant.id) + end + end + + assert_includes @message.reload.content_text, "a quota error" + assert @message.failed?, "The message should have been marked failed so a Retry button is offered" + end end diff --git a/test/services/ai_backend/ruby_llm/tool_interception_test.rb b/test/services/ai_backend/ruby_llm/tool_interception_test.rb index 41890a0ed..f6d7a8b78 100644 --- a/test/services/ai_backend/ruby_llm/tool_interception_test.rb +++ b/test/services/ai_backend/ruby_llm/tool_interception_test.rb @@ -65,4 +65,23 @@ def stub_tool_call_response(chat, tool_calls, &) assert_equal 1, chat.messages.count { |m| m.role == :assistant }, "the chat should not have auto-continued with a follow-up assistant message" end + + # InterceptedChat (used in production) halts on ANY tool call — including an + # unregistered/hallucinated name, which the InterceptedTool#execute raise + # cannot catch because Chat#execute_tool returns an error hash for unknown + # names rather than calling execute. + test "InterceptedChat halts on an unregistered tool name instead of auto-executing" do + chat = AIBackend::RubyLLM::InterceptedChat.new(model: "gpt-4o", provider: :openai, assume_model_exists: true) + chat.add_message(role: :user, content: "Do something with a tool") + chat.with_tools(build_tool) + + hallucinated = RubyLLM::ToolCall.new(id: "call_halluc", name: "not_a_registered_tool", arguments: {}) + + assert_raises(AIBackend::RubyLLM::ToolCallIntercepted) do + stub_tool_call_response(chat, [hallucinated]) { chat.complete } + end + + assert_empty chat.messages.select { |m| m.role == :tool }, + "no role: :tool result message should be appended for an unregistered tool" + end end diff --git a/test/services/ai_backend/ruby_llm_test.rb b/test/services/ai_backend/ruby_llm_test.rb index 996252654..a003e1a19 100644 --- a/test/services/ai_backend/ruby_llm_test.rb +++ b/test/services/ai_backend/ruby_llm_test.rb @@ -203,7 +203,7 @@ class AIBackend::RubyLLMTest < ActiveSupport::TestCase end end - test "stream_handler raises RateLimitError on rate limit" do + test "stream_handler raises Faraday::TooManyRequestsError on rate limit" do @assistant.language_model.update!(supports_tools: false) message = @conversation.messages.create!( role: :assistant, @@ -221,7 +221,7 @@ class AIBackend::RubyLLMTest < ActiveSupport::TestCase error_chunk.define_singleton_method(:input_tokens) { nil } error_chunk.define_singleton_method(:output_tokens) { nil } - assert_raises(AIBackend::RubyLLM::RateLimitError) do + assert_raises(Faraday::TooManyRequestsError) do handler.call(error_chunk, ->(c) { }) end end @@ -657,4 +657,188 @@ class AIBackend::RubyLLMTest < ActiveSupport::TestCase assert mixed_msg[:content].text.present?, "Content text should be preserved alongside attachments" assert mixed_msg[:content].attachments.any?, "Attachments should be present" end + + # Phase 5 — Tool/function calling parity + + test "ConfigurationError inherits from AIBackend::ConfigurationError" do + assert AIBackend::RubyLLM::ConfigurationError < AIBackend::ConfigurationError + end + + test "get_oneoff_message with json: true appends a JSON-coercion instruction" do + backend = AIBackend::RubyLLM.new(@user, @assistant) + TestClient::RubyLLM::Chat.stub :text, '{"topic":"Hi"}' do + backend.get_oneoff_message("Extract a topic", ["Hello"], json: true) + end + assert_includes TestClient::RubyLLM::Chat.instructions, "Respond with ONLY valid JSON" + end + + test "get_oneoff_message without json does not append the JSON instruction" do + backend = AIBackend::RubyLLM.new(@user, @assistant) + TestClient::RubyLLM::Chat.stub :text, "Plain" do + backend.get_oneoff_message("Extract a topic", ["Hello"]) + end + refute_includes TestClient::RubyLLM::Chat.instructions, "Respond with ONLY valid JSON" + end + + test "tool_instances maps each Toolbox tool to an InterceptedTool" do + backend = AIBackend::RubyLLM.new(@user, @assistant) + instances = backend.send(:tool_instances) + + assert instances.all? { |i| i.is_a?(AIBackend::RubyLLM::InterceptedTool) } + assert_includes instances.map(&:name), "helloworld_hi" + assert_includes instances.map(&:name), "openmeteo_get_current_and_todays_weather" + end + + test "tools are not enabled for Groq URL" do + @assistant.language_model.api_service.update!(url: APIService::URL_GROQ, driver: "openai") + @assistant.language_model.update!(supports_tools: true) + message = @conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: @assistant, + index: @conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, @conversation, message) + assert_not backend.send(:tools_enabled?) + end + + test "tools are enabled for a canonical OpenAI service when supports_tools is true" do + @assistant.language_model.update!(supports_tools: true) + backend = AIBackend::RubyLLM.new(@user, @assistant) + assert backend.send(:tools_enabled?) + end + + test "stream_next_conversation_message returns a formatted tool call when the model requests one" do + @assistant.language_model.update!(supports_tools: true) + message = @conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: @assistant, + index: @conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, @conversation, message) + TestClient::RubyLLM::Chat.stub :function, "helloworld_hi" do + result = backend.stream_next_conversation_message { |c| } + assert_equal 1, result.length + assert_equal "function", result[0][:type] + assert_equal "helloworld_hi", result[0][:function][:name] + assert_equal TestClient::RubyLLM::Chat.id, result[0][:id] + assert_includes result[0][:function][:arguments], "Austin" + end + end + + test "stream_next_conversation_message returns parallel tool calls with distinct ids" do + @assistant.language_model.update!(supports_tools: true) + message = @conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: @assistant, + index: @conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, @conversation, message) + TestClient::RubyLLM::Chat.stub :function, "helloworld_hi" do + TestClient::RubyLLM::Chat.stub :num_tool_calls, 2 do + result = backend.stream_next_conversation_message { |c| } + assert_equal 2, result.length + assert_equal [0, 1], result.map { |tc| tc[:index] } + assert_operator result.map { |tc| tc[:id] }.uniq.length, :>, 1 + end + end + end + + test "preceding_conversation_messages replays tool calls and tool results" do + @assistant.language_model.update!(supports_tools: true) + conversation = @conversation + + conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: @assistant, + content_tool_calls: [ + { type: "function", id: "call_123", function: { name: "helloworld_hi", arguments: '{"name":"Keith"}' } }, + ] + ) + conversation.messages.create!( + role: :tool, + content_text: "Hello, Keith!".to_json, + assistant: @assistant, + tool_call_id: "call_123", + content_tool_calls: [ + { type: "function", id: "call_123", function: { name: "helloworld_hi", arguments: '{"name":"Keith"}' } }, + ] + ) + follow_up = conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: @assistant, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, conversation, follow_up) + msgs = backend.send(:preceding_conversation_messages) + + tool_replay = msgs.find { |m| m[:role] == :tool } + assert tool_replay, "expected a tool result message to be replayed" + assert_equal "call_123", tool_replay[:tool_call_id] + assert_equal "Hello, Keith!".to_json, tool_replay[:content] + + assistant_replay = msgs.find { |m| m[:role] == :assistant && m[:tool_calls].present? } + assert assistant_replay, "expected the assistant tool-call message to be replayed" + assert_equal "helloworld_hi", assistant_replay[:tool_calls]["call_123"].name + assert_equal({ "name" => "Keith" }, assistant_replay[:tool_calls]["call_123"].arguments) + end + + # Error contract: errors raised at the chat.complete boundary (where the gem + # raises them) must map to the unified contract, not only chunk-level errors. + + test "stream_next_conversation_message maps a complete-level UnauthorizedError to ConfigurationError" do + @assistant.language_model.update!(supports_tools: false) + message = @conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: @assistant, + index: @conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, @conversation, message) + TestClient::RubyLLM::Chat.stub :error_to_raise, ::RubyLLM::UnauthorizedError.new("401 bad key") do + assert_raises(AIBackend::RubyLLM::ConfigurationError) do + backend.stream_next_conversation_message { |c| } + end + end + end + + test "stream_next_conversation_message maps a complete-level RateLimitError to Faraday::TooManyRequestsError" do + @assistant.language_model.update!(supports_tools: false) + message = @conversation.messages.create!( + role: :assistant, + content_text: nil, + assistant: @assistant, + index: @conversation.messages.maximum(:index).to_i + 1, + version: :latest + ) + + backend = AIBackend::RubyLLM.new(@user, @assistant, @conversation, message) + TestClient::RubyLLM::Chat.stub :error_to_raise, ::RubyLLM::RateLimitError.new("429 rate limited") do + assert_raises(Faraday::TooManyRequestsError) do + backend.stream_next_conversation_message { |c| } + end + end + end + + test "get_oneoff_message maps a complete-level UnauthorizedError to ConfigurationError" do + backend = AIBackend::RubyLLM.new(@user, @assistant) + TestClient::RubyLLM::Chat.stub :error_to_raise, ::RubyLLM::UnauthorizedError.new("401 bad key") do + assert_raises(AIBackend::RubyLLM::ConfigurationError) do + backend.get_oneoff_message("Extract a topic", ["Hello"]) + end + end + end end diff --git a/test/support/test_client/ruby_llm.rb b/test/support/test_client/ruby_llm.rb index 4ce8cc349..77a9f6aeb 100644 --- a/test/support/test_client/ruby_llm.rb +++ b/test/support/test_client/ruby_llm.rb @@ -33,7 +33,9 @@ def add_message(msg) def complete(&block) raise self.class.error_to_raise if self.class.error_to_raise - if block + if self.class.function + simulate_tool_calls + elsif block response = self.class.api_streaming_response block.call(response) if response.content.present? else @@ -51,6 +53,23 @@ def ask(message = nil, with: nil, &block) complete(&block) end + private + + # Mirrors the real RubyLLM::Chat tool flow: the assistant message carrying + # the tool calls is appended to messages before the intercepted execute + # raises, so the backend can read chat.messages.last.tool_calls. + def simulate_tool_calls + tool_calls = Array.new(self.class.num_tool_calls) do |i| + ::RubyLLM::ToolCall.new( + id: i.zero? ? self.class.id : "#{self.class.id}_#{i}", + name: self.class.function, + arguments: JSON.parse(self.class.arguments) + ) + end + @messages << OpenStruct.new(role: :assistant, content: nil, tool_calls: tool_calls.to_h { |tc| [tc.id, tc] }) + raise AIBackend::RubyLLM::ToolCallIntercepted + end + def self.api_oneoff_response { "choices" => [ @@ -81,6 +100,16 @@ def self.text raise "Attempting to return a text response but .text method is not stubbed. Stub this to nil if you want to return default text." end + # Returns the name of the tool the model "requested"; nil means the model + # responds with text instead. Mirrors TestClient::OpenAI#function. + def self.function + nil + end + + def self.num_tool_calls + 1 + end + def self.default_text "Hello this is model #{@@model}! How can I assist you today?" end From eaaa2501c879b174167229288d43087a0bb7950b Mon Sep 17 00:00:00 2001 From: fluxgravity <44489080+fluxgravity@users.noreply.github.com> Date: Wed, 2 Sep 2026 01:27:01 -0400 Subject: [PATCH 7/7] feat(rubyllm): flip use_ruby_llm default on with a test-env carve-out (Phase 6) Enable the RubyLLM backend by default for all non-test environments via a hand-rolled ERB expression in options.yml, while keeping the test-environment default off so the legacy SDK suite keeps dispatching to the old backends. The flag stays revertible through USE_RUBY_LLM_FEATURE or a per-user preference, and the feature test now pins the boolean false default. --- config/options.yml | 5 ++++- test/models/feature_test.rb | 9 +++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/config/options.yml b/config/options.yml index aaf012137..710d7f7e0 100644 --- a/config/options.yml +++ b/config/options.yml @@ -47,7 +47,10 @@ shared: email: <%= ENV["EMAIL_FEATURE"] || default_to(false, except_env_test: true) %> password_reset_email: <%= ENV["PASSWORD_RESET_EMAIL_FEATURE"] || default_to(false, except_env_test: true) %> assistants_page: <%= ENV["ASSISTANTS_PAGE_FEATURE"] || true %> - use_ruby_llm: <%= ENV["USE_RUBY_LLM_FEATURE"] || false %> + # Hand-rolled ERB: default_to can't express a falsy except_env_test override + # (it uses ||=), and the old SDK test suite must keep dispatching to the old + # backends (uneditable through Phase 6), so the test-env default stays false. + use_ruby_llm: <%= ENV["USE_RUBY_LLM_FEATURE"] || (Rails.env.test? ? "false" : "true") %> settings: # Be sure to add these ENV to docker-compose.yml app_url_protocol: <%= ENV["APP_URL_PROTOCOL"] || default_to(app_url: :protocol) %> diff --git a/test/models/feature_test.rb b/test/models/feature_test.rb index 2b6095121..2a625e7d4 100644 --- a/test/models/feature_test.rb +++ b/test/models/feature_test.rb @@ -90,12 +90,13 @@ class FeatureTest < ActiveSupport::TestCase end end - test "use_ruby_llm? reads options.yml default and can be overridden by user preference" do + test "use_ruby_llm site default is off in test and can be overridden by user preference" do user = users(:keith) - stub_features(use_ruby_llm: false) do - refute Feature.use_ruby_llm? - end + # Phase 6 flips the default on in every environment except test, so the raw + # site default stays boolean false here (the ERB renders a bare "false" token + # that YAML parses back to false) — keeping the old-SDK test suite green. + assert_equal false, Feature.raw_features[:use_ruby_llm] stub_features(use_ruby_llm: false) do Current.set(user: user) do