diff --git a/lib/ruby_llm.rb b/lib/ruby_llm.rb index 06d22287e..2d8e3da24 100644 --- a/lib/ruby_llm.rb +++ b/lib/ruby_llm.rb @@ -23,6 +23,7 @@ 'deepseek' => 'DeepSeek', 'gpustack' => 'GPUStack', 'llm' => 'LLM', + 'llm_gateway' => 'LLMGateway', 'mistral' => 'Mistral', 'openai' => 'OpenAI', 'openrouter' => 'OpenRouter', @@ -127,6 +128,7 @@ def logger RubyLLM::Provider.register :deepseek, RubyLLM::Providers::DeepSeek RubyLLM::Provider.register :gemini, RubyLLM::Providers::Gemini RubyLLM::Provider.register :gpustack, RubyLLM::Providers::GPUStack +RubyLLM::Provider.register :llm_gateway, RubyLLM::Providers::LLMGateway RubyLLM::Provider.register :mistral, RubyLLM::Providers::Mistral RubyLLM::Provider.register :ollama, RubyLLM::Providers::Ollama RubyLLM::Provider.register :openai, RubyLLM::Providers::OpenAI diff --git a/lib/ruby_llm/models.rb b/lib/ruby_llm/models.rb index 85087addf..81c5aa5f1 100644 --- a/lib/ruby_llm/models.rb +++ b/lib/ruby_llm/models.rb @@ -35,6 +35,7 @@ class Models vertexai bedrock openrouter + llm_gateway azure ollama gpustack diff --git a/lib/ruby_llm/providers/llm_gateway.rb b/lib/ruby_llm/providers/llm_gateway.rb new file mode 100644 index 000000000..6b31c344e --- /dev/null +++ b/lib/ruby_llm/providers/llm_gateway.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + # LLMGateway.io API integration. + class LLMGateway < Provider + # LLMGateway's dialect of the Chat Completions API. + class ChatCompletions < Protocols::ChatCompletions + include LLMGateway::Chat + include LLMGateway::Images + include LLMGateway::Models + include LLMGateway::Streaming + end + + protocol :chat_completions, ChatCompletions + files LLMGateway::Files + + def api_base + @config.llm_gateway_api_base || 'https://api.llmgateway.io/v1' + end + + def headers + { + 'Authorization' => "Bearer #{@config.llm_gateway_api_key}" + } + end + + def parse_error(response) + return if response.body.empty? + + body = try_parse_json(response.body) + case body + when Hash + parse_error_part_message body + when Array + body.map do |part| + parse_error_part_message part + end.join('. ') + else + body + end + end + + class << self + def configuration_options + %i[llm_gateway_api_key llm_gateway_api_base] + end + + def configuration_requirements + %i[llm_gateway_api_key] + end + end + + private + + def parse_error_part_message(part) + message = part.dig('error', 'message') + raw = try_parse_json(part.dig('error', 'metadata', 'raw')) + return message unless raw.is_a?(Hash) + + raw_message = raw.dig('error', 'message') + return [message, raw_message].compact.join(' - ') if raw_message + + message + end + end + end +end diff --git a/lib/ruby_llm/providers/llm_gateway/chat.rb b/lib/ruby_llm/providers/llm_gateway/chat.rb new file mode 100644 index 000000000..c63a8e201 --- /dev/null +++ b/lib/ruby_llm/providers/llm_gateway/chat.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + class LLMGateway + # Chat methods of the LLMGateway API integration + module Chat + LLM_GATEWAY_INLINE_FILE_THRESHOLD = 50 * 1024 * 1024 + LLM_GATEWAY_FILE_UPLOAD_LIMIT = 100 * 1024 * 1024 + + module_function + + # rubocop:disable Metrics/ParameterLists + def render_payload(messages, tools:, temperature:, model:, stream: false, schema: nil, + thinking: nil, citations: false, tool_prefs: nil) + payload = super + payload.delete(:reasoning_effort) + strip_schema_strict(payload) + + reasoning = build_reasoning(thinking) + payload[:reasoning] = reasoning if reasoning + payload + end + # rubocop:enable Metrics/ParameterLists + + def strip_schema_strict(payload) + schema_def = payload.dig(:response_format, :json_schema, :schema) + return unless schema_def.is_a?(Hash) + + schema_def = RubyLLM::Utils.deep_dup(schema_def) + schema_def.delete(:strict) + schema_def.delete('strict') + payload[:response_format][:json_schema][:schema] = schema_def + end + + def build_reasoning(thinking) + return nil unless thinking&.enabled? + + reasoning = {} + reasoning[:effort] = thinking.effort if thinking.respond_to?(:effort) && thinking.effort + reasoning[:max_tokens] = thinking.budget if thinking.respond_to?(:budget) && thinking.budget + reasoning[:enabled] = true if reasoning.empty? + reasoning + end + + def format_thinking(msg) + thinking = msg.thinking + return {} unless thinking && msg.role == :assistant + + details = [] + if thinking.text + details << { + type: 'reasoning.text', + text: thinking.text, + signature: thinking.signature + }.compact + elsif thinking.signature + details << { + type: 'reasoning.encrypted', + data: thinking.signature + } + end + + details.empty? ? {} : { reasoning_details: details } + end + + def supports_provider_file_references? + true + end + + def default_large_file_upload_threshold + LLM_GATEWAY_INLINE_FILE_THRESHOLD + end + + def provider_file_upload_limit + LLM_GATEWAY_FILE_UPLOAD_LIMIT + end + + def provider_file_attachable?(attachment) + attachment.pdf? + end + + def extract_thinking_text(message_data) + candidate = message_data['reasoning'] + return candidate if candidate.is_a?(String) + + details = message_data['reasoning_details'] + return nil unless details.is_a?(Array) + + text = details.filter_map do |detail| + case detail['type'] + when 'reasoning.text' + detail['text'] + when 'reasoning.summary' + detail['summary'] + end + end.join + + text.empty? ? nil : text + end + + def extract_thinking_signature(message_data) + details = message_data['reasoning_details'] + return nil unless details.is_a?(Array) + + signature = details.filter_map do |detail| + detail['signature'] if detail['signature'].is_a?(String) + end.first + return signature if signature + + encrypted = details.find { |detail| detail['type'] == 'reasoning.encrypted' && detail['data'].is_a?(String) } + encrypted&.dig('data') + end + end + end + end +end diff --git a/lib/ruby_llm/providers/llm_gateway/files.rb b/lib/ruby_llm/providers/llm_gateway/files.rb new file mode 100644 index 000000000..937315fc9 --- /dev/null +++ b/lib/ruby_llm/providers/llm_gateway/files.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + class LLMGateway + # LLMGateway Files API. + class Files < UploadedFile::Protocol + private + + def parse_file_response(data) + uploaded_file( + data, + id: data['id'], + filename: data['filename'], + byte_size: data['size_bytes'], + created_at: timestamp(data['created_at']), + mime_type: data['mime_type'], + downloadable: data['downloadable'] + ) + end + end + end + end +end diff --git a/lib/ruby_llm/providers/llm_gateway/images.rb b/lib/ruby_llm/providers/llm_gateway/images.rb new file mode 100644 index 000000000..60f966012 --- /dev/null +++ b/lib/ruby_llm/providers/llm_gateway/images.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + class LLMGateway + # Image generation methods for the LLMGateway API integration. + # LLMGateway uses the chat completions endpoint for image generation + # instead of a dedicated images endpoint. + module Images + module_function + + def images_url(with: nil, mask: nil) # rubocop:disable Lint/UnusedMethodArgument + 'chat/completions' + end + + def render_image_payload(prompt, model:, size:, with: nil, mask: nil, params: {}) # rubocop:disable Lint/UnusedMethodArgument,Metrics/ParameterLists + RubyLLM.logger.debug { "Ignoring size #{size}. LLMGateway image generation does not support size parameter." } + { + model: model, + messages: [ + { + role: 'user', + content: prompt + } + ], + modalities: %w[image text] + } + end + + def parse_image_response(response, model:) + data = response.body + message = data.dig('choices', 0, 'message') + + unless message&.key?('images') && message['images']&.any? + raise Error.new(nil, 'Unexpected response format from LLMGateway image generation API') + end + + image_data = message['images'].first + image_url = image_data.dig('image_url', 'url') || image_data['url'] + + raise Error.new(nil, 'No image URL found in LLMGateway response') unless image_url + + build_image_from_url(image_url, model) + end + + def build_image_from_url(image_url, model) + if image_url.start_with?('data:') + # Parse data URL format: data:image/png;base64, + match = image_url.match(/^data:([^;]+);base64,(.+)$/) + raise Error.new(nil, 'Invalid data URL format from LLMGateway') unless match + + Image.new( + data: match[2], + mime_type: match[1], + model_id: model + ) + else + # Regular URL + Image.new( + url: image_url, + mime_type: 'image/png', + model_id: model + ) + end + end + end + end + end +end diff --git a/lib/ruby_llm/providers/llm_gateway/models.rb b/lib/ruby_llm/providers/llm_gateway/models.rb new file mode 100644 index 000000000..f73099f44 --- /dev/null +++ b/lib/ruby_llm/providers/llm_gateway/models.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + class LLMGateway + # Models methods of the LLMGateway API integration + module Models + module_function + + def models_url + 'models' + end + + def parse_list_models_response(response, slug, _capabilities) + Array(response.body['data']).map do |model_data| # rubocop:disable Metrics/BlockLength + modalities = { + input: Array(model_data.dig('architecture', 'input_modalities')), + output: Array(model_data.dig('architecture', 'output_modalities')) + } + + pricing = { text_tokens: { standard: {} } } + + pricing_types = { + prompt: :input_per_million, + completion: :output_per_million, + input_cache_read: :cache_read_input_per_million, + internal_reasoning: :reasoning_output_per_million + } + + pricing_types.each do |source_key, target_key| + value = model_data.dig('pricing', source_key.to_s).to_f + pricing[:text_tokens][:standard][target_key] = value * 1_000_000 if value.positive? + end + + capabilities = supported_parameters_to_capabilities(model_data['supported_parameters']) + + Model::Info.new( + id: model_data['id'], + name: model_data['name'], + provider: slug, + family: model_data['family'] || model_data['id'].to_s.split('/').first, + created_at: model_data['created'] ? Time.at(model_data['created']) : nil, + context_window: model_data['context_length'], + max_output_tokens: model_data.dig('top_provider', 'max_completion_tokens'), + modalities: modalities, + capabilities: capabilities, + pricing: pricing, + metadata: { + description: model_data['description'], + architecture: model_data['architecture'], + top_provider: model_data['top_provider'], + per_request_limits: model_data['per_request_limits'], + supported_parameters: model_data['supported_parameters'] + } + ) + end + end + + def supported_parameters_to_capabilities(params) + return [] unless params + + capabilities = [] + capabilities << 'streaming' + capabilities << 'function_calling' if params.include?('tools') || params.include?('tool_choice') + capabilities << 'structured_output' if params.include?('response_format') + capabilities << 'batch' if params.include?('batch') + capabilities << 'predicted_outputs' if params.include?('logit_bias') && params.include?('top_k') + capabilities + end + end + end + end +end diff --git a/lib/ruby_llm/providers/llm_gateway/streaming.rb b/lib/ruby_llm/providers/llm_gateway/streaming.rb new file mode 100644 index 000000000..c5956193d --- /dev/null +++ b/lib/ruby_llm/providers/llm_gateway/streaming.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + class LLMGateway + # Streaming methods of the LLMGateway API integration + module Streaming + module_function + + def build_chunk(data) + usage = data['usage'] || {} + delta = data.dig('choices', 0, 'delta') || {} + + Chunk.new( + role: :assistant, + model_id: data['model'], + content: delta['content'], + thinking: Thinking.build( + text: extract_thinking_text(delta), + signature: extract_thinking_signature(delta) + ), + tool_calls: parse_tool_calls(delta['tool_calls'], parse_arguments: false), + input_tokens: input_tokens(usage), + output_tokens: output_tokens(usage), + cached_tokens: cache_read_tokens(usage), + cache_creation_tokens: cache_write_tokens(usage), + thinking_tokens: thinking_tokens(usage), + finish_reason: data.dig('choices', 0, 'finish_reason') + ) + end + end + end + end +end diff --git a/spec/ruby_llm/configuration_spec.rb b/spec/ruby_llm/configuration_spec.rb index 2bc492cf1..59c4b57a2 100644 --- a/spec/ruby_llm/configuration_spec.rb +++ b/spec/ruby_llm/configuration_spec.rb @@ -27,6 +27,7 @@ :default_speech_model, :model_registry_file, :openai_api_key, + :llm_gateway_api_base, :openrouter_api_base ) end diff --git a/spec/ruby_llm/provider_spec.rb b/spec/ruby_llm/provider_spec.rb index 898a7dca2..9f5bb7576 100644 --- a/spec/ruby_llm/provider_spec.rb +++ b/spec/ruby_llm/provider_spec.rb @@ -56,6 +56,12 @@ def api_base_cases custom: 'https://openai-proxy.example.com/v1', default: 'https://api.openai.com/v1' }, + llm_gateway: { + provider: RubyLLM::Providers::LLMGateway, + key: :llm_gateway_api_base, + custom: 'https://llm-gateway-proxy.example.com/v1', + default: 'https://api.llmgateway.io/v1' + }, openrouter: { provider: RubyLLM::Providers::OpenRouter, key: :openrouter_api_base, @@ -102,6 +108,8 @@ def config_for(slug) when :gpustack config.gpustack_api_base = 'https://gpustack.example.com/v1' config.gpustack_api_key = 'gpustack-key' + when :llm_gateway + config.llm_gateway_api_key = 'llm-gateway-key' when :mistral config.mistral_api_key = 'mistral-key' when :ollama @@ -313,7 +321,7 @@ def configuration_requirements describe 'file protocol resolution' do it 'exposes provider-managed files only where implemented' do - file_providers = %i[anthropic azure bedrock gemini mistral openai openrouter vertexai xai] + file_providers = %i[anthropic azure bedrock gemini llm_gateway mistral openai openrouter vertexai xai] described_class.providers.each do |slug, provider_class| expect(provider_class.new(config_for(slug)).files?).to eq(file_providers.include?(slug)) diff --git a/spec/ruby_llm/providers/llm_gateway/parse_error_spec.rb b/spec/ruby_llm/providers/llm_gateway/parse_error_spec.rb new file mode 100644 index 000000000..4f23bd689 --- /dev/null +++ b/spec/ruby_llm/providers/llm_gateway/parse_error_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe RubyLLM::Providers::LLMGateway do + let(:provider) do + config = RubyLLM::Configuration.new + config.llm_gateway_api_key = 'test' + described_class.new(config) + end + + describe '#parse_error' do + it 'appends nested provider message from metadata.raw when present' do + response = instance_double( + Faraday::Response, + body: { + error: { + message: 'Provider returned error', + code: 403, + metadata: { + raw: { + error: { + code: 'unsupported_country_region_territory', + message: 'Country, region, or territory not supported', + type: 'request_forbidden' + } + }.to_json, + provider_name: 'OpenAI' + } + }, + user_id: 'user_2' + }.to_json + ) + + expect(provider.parse_error(response)) + .to eq('Provider returned error - Country, region, or territory not supported') + end + + it 'returns the top-level message when metadata.raw is missing' do + response = instance_double( + Faraday::Response, + body: { + error: { + message: 'Provider returned error', + code: 403 + } + }.to_json + ) + + expect(provider.parse_error(response)).to eq('Provider returned error') + end + end +end diff --git a/spec/support/rubyllm_configuration.rb b/spec/support/rubyllm_configuration.rb index b31b4aa52..9d965435b 100644 --- a/spec/support/rubyllm_configuration.rb +++ b/spec/support/rubyllm_configuration.rb @@ -31,6 +31,7 @@ config.model_registry_class = 'Model' config.ollama_api_base = ENV.fetch('OLLAMA_API_BASE', 'http://localhost:11434/v1') config.ollama_api_key = ENV.fetch('OLLAMA_API_KEY', nil) + config.llm_gateway_api_key = ENV.fetch('LLM_GATEWAY_API_KEY', 'test') config.openai_api_key = ENV.fetch('OPENAI_API_KEY', 'test') config.openrouter_api_key = ENV.fetch('OPENROUTER_API_KEY', 'test') config.perplexity_api_key = ENV.fetch('PERPLEXITY_API_KEY', 'test')