Changes
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e08fb5a..0cf4c37 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -73,6 +73,7 @@ add_library(agent_smith ${AGENT_SMITH_LIBRARY_TYPE}
src/agent.cpp
src/detail/blocking_pool.cpp
src/detail/timer_service.cpp
+ src/llm_client.cpp
src/message.cpp
src/runtime.cpp
src/sse_parser.cpp
diff --git a/include/agent_smith/agent_smith.hpp b/include/agent_smith/agent_smith.hpp
index 16391a6..152b11b 100644
--- a/include/agent_smith/agent_smith.hpp
+++ b/include/agent_smith/agent_smith.hpp
@@ -2,6 +2,7 @@
#include "agent.hpp"
#include "error.hpp"
+#include "llm_client.hpp"
#include "message.hpp"
#include "runtime.hpp"
#include "version.hpp"
diff --git a/include/agent_smith/error.hpp b/include/agent_smith/error.hpp
index 3493ed4..6cea54f 100644
--- a/include/agent_smith/error.hpp
+++ b/include/agent_smith/error.hpp
@@ -19,6 +19,14 @@ struct ParseError
std::size_t offset = 0;
};
+/// Describes a non-success response returned by an LLM HTTP endpoint.
+struct LlmApiError
+{
+ std::string msg;
+ int http_status = 0;
+ std::string body_excerpt;
+};
+
namespace error
{
@@ -31,6 +39,16 @@ inline mw::Error parse(std::string_view message, std::size_t offset = 0)
};
}
+/// Create a typed error for a non-success response from an LLM endpoint.
+inline mw::Error llmApi(int http_status, std::string_view body_excerpt)
+{
+ return LlmApiError{
+ .msg = "LLM endpoint returned HTTP " + std::to_string(http_status),
+ .http_status = http_status,
+ .body_excerpt = std::string(body_excerpt),
+ };
+}
+
} // namespace error
} // namespace agent_smith
diff --git a/include/agent_smith/llm_client.hpp b/include/agent_smith/llm_client.hpp
new file mode 100644
index 0000000..996b5b9
--- /dev/null
+++ b/include/agent_smith/llm_client.hpp
@@ -0,0 +1,100 @@
+#pragma once
+
+#include <chrono>
+#include <functional>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include <mw/http_client_async.hpp>
+#include <mw/url.hpp>
+#include <nlohmann/json.hpp>
+
+#include "message.hpp"
+#include "runtime.hpp"
+
+namespace agent_smith
+{
+
+/// Sampling controls sent to an OpenAI-compatible chat-completions endpoint.
+struct SamplingParams
+{
+ /// Sampling temperature, when the provider should receive one.
+ std::optional<double> temperature;
+
+ /// Nucleus-sampling threshold, when the provider should receive one.
+ std::optional<double> top_p;
+
+ /// Maximum number of tokens the provider may generate.
+ std::optional<int> max_tokens;
+
+ /// Sequences that cause generation to stop.
+ std::vector<std::string> stop;
+
+ /// Opaque structured-output or JSON-mode configuration.
+ nlohmann::json response_format = nullptr;
+
+ /// Provider-specific request fields outside the standard API surface.
+ nlohmann::json extra = nullptr;
+};
+
+/// Connection and request defaults for an LLM client.
+struct LlmClientConfig
+{
+ /// Base URL of the OpenAI-compatible endpoint.
+ mw::URL endpoint;
+
+ /// Provider model identifier.
+ std::string model;
+
+ /// Optional bearer token for the endpoint.
+ std::optional<std::string> api_key;
+
+ /// Requested maximum duration of one HTTP transfer.
+ std::chrono::seconds request_timeout{60};
+
+ /// Number of retry attempts after the initial request.
+ int max_retries = 3;
+
+ /// Sampling values used when a request does not override them.
+ SamplingParams default_sampling;
+};
+
+/// One non-streaming LLM completion request.
+struct LlmRequest
+{
+ /// Ordered conversation history sent to the model.
+ std::vector<Message> messages;
+
+ /// Sampling values that override the client's defaults.
+ SamplingParams sampling;
+};
+
+/// The final assistant message returned by an LLM completion request.
+struct LlmResponse
+{
+ /// Parsed assistant response, including usage when supplied by the server.
+ AssistantMessage message;
+};
+
+/// Calls an OpenAI-compatible chat-completions endpoint.
+class LlmClient
+{
+public:
+ /// Construct a client that uses the runtime's shared HTTP session.
+ LlmClient(Runtime& runtime, LlmClientConfig config);
+
+ /// Construct a client with an HTTP session override, primarily for tests.
+ LlmClient(Runtime& runtime, mw::HTTPSessionAsyncInterface& http,
+ LlmClientConfig config);
+
+ /// Send a non-streaming request to `/v1/chat/completions`.
+ mw::Task<mw::E<LlmResponse>> complete(LlmRequest request);
+
+private:
+ Runtime* runtime_;
+ mw::HTTPSessionAsyncInterface* http_;
+ LlmClientConfig config_;
+};
+
+} // namespace agent_smith
diff --git a/src/llm_client.cpp b/src/llm_client.cpp
new file mode 100644
index 0000000..0c78892
--- /dev/null
+++ b/src/llm_client.cpp
@@ -0,0 +1,318 @@
+#include "agent_smith/llm_client.hpp"
+
+#include <algorithm>
+#include <array>
+#include <cctype>
+#include <charconv>
+#include <chrono>
+#include <cstdint>
+#include <random>
+#include <ranges>
+#include <string_view>
+#include <utility>
+
+namespace agent_smith
+{
+namespace
+{
+
+constexpr std::size_t MAX_ERROR_BODY_EXCERPT = 4096;
+constexpr std::chrono::milliseconds RETRY_BASE_DELAY{500};
+constexpr std::chrono::milliseconds RETRY_MAX_DELAY{8000};
+
+bool isReservedRequestKey(std::string_view key)
+{
+ constexpr std::array<std::string_view, 5> RESERVED_KEYS{
+ "model", "messages", "tools", "tool_choice", "stream",
+ };
+ return std::ranges::find(RESERVED_KEYS, key) != RESERVED_KEYS.end();
+}
+
+void applySamplingOverrides(SamplingParams& target,
+ const SamplingParams& overrides)
+{
+ if(overrides.temperature)
+ {
+ target.temperature = overrides.temperature;
+ }
+ if(overrides.top_p)
+ {
+ target.top_p = overrides.top_p;
+ }
+ if(overrides.max_tokens)
+ {
+ target.max_tokens = overrides.max_tokens;
+ }
+ if(!overrides.stop.empty())
+ {
+ target.stop = overrides.stop;
+ }
+ if(!overrides.response_format.is_null())
+ {
+ target.response_format = overrides.response_format;
+ }
+}
+
+mw::E<void> mergeExtra(nlohmann::json& target, const nlohmann::json& extra)
+{
+ if(extra.is_null())
+ {
+ return {};
+ }
+ if(!extra.is_object())
+ {
+ return std::unexpected(error::parse("expected sampling extra object"));
+ }
+
+ for(auto it = extra.begin(); it != extra.end(); ++it)
+ {
+ if(isReservedRequestKey(it.key()))
+ {
+ return std::unexpected(error::parse(
+ "sampling extra overrides reserved key: " + it.key()));
+ }
+ target[it.key()] = it.value();
+ }
+ return {};
+}
+
+mw::E<nlohmann::json> buildRequestBody(const LlmRequest& request,
+ const LlmClientConfig& config)
+{
+ SamplingParams sampling = config.default_sampling;
+ applySamplingOverrides(sampling, request.sampling);
+
+ nlohmann::json body{
+ {"model", config.model},
+ {"messages", nlohmann::json::array()},
+ {"stream", false},
+ };
+ for(const Message& message : request.messages)
+ {
+ body["messages"].push_back(toOpenAIJson(message));
+ }
+ if(sampling.temperature)
+ {
+ body["temperature"] = *sampling.temperature;
+ }
+ if(sampling.top_p)
+ {
+ body["top_p"] = *sampling.top_p;
+ }
+ if(sampling.max_tokens)
+ {
+ body["max_tokens"] = *sampling.max_tokens;
+ }
+ if(!sampling.stop.empty())
+ {
+ body["stop"] = sampling.stop;
+ }
+ if(!sampling.response_format.is_null())
+ {
+ body["response_format"] = sampling.response_format;
+ }
+
+ auto default_extra = mergeExtra(body, config.default_sampling.extra);
+ if(!default_extra)
+ {
+ return std::unexpected(default_extra.error());
+ }
+ auto request_extra = mergeExtra(body, request.sampling.extra);
+ if(!request_extra)
+ {
+ return std::unexpected(request_extra.error());
+ }
+ return body;
+}
+
+std::chrono::milliseconds retryDelay(int retry_number)
+{
+ const int exponent = std::clamp(retry_number - 1, 0, 4);
+ const auto base =
+ std::min(RETRY_BASE_DELAY * (1 << exponent), RETRY_MAX_DELAY);
+ thread_local std::mt19937 generator(std::random_device{}());
+ std::uniform_int_distribution<int> jitter(0, 250);
+ return std::min(base + std::chrono::milliseconds(jitter(generator)),
+ RETRY_MAX_DELAY);
+}
+
+std::optional<std::chrono::milliseconds>
+retryAfterDelay(const mw::HTTPResponse& response)
+{
+ if(response.status != 429)
+ {
+ return std::nullopt;
+ }
+
+ for(const auto& [name, value] : response.header)
+ {
+ const bool is_retry_after = std::ranges::equal(
+ name, std::string_view("retry-after"),
+ [](char left, char right)
+ {
+ return std::tolower(static_cast<unsigned char>(left)) == right;
+ });
+ if(!is_retry_after)
+ {
+ continue;
+ }
+
+ int seconds = 0;
+ const auto [end, error_code] =
+ std::from_chars(value.data(), value.data() + value.size(), seconds);
+ if(error_code == std::errc{} && end == value.data() + value.size() &&
+ seconds >= 0)
+ {
+ return std::min(std::chrono::seconds(seconds),
+ std::chrono::duration_cast<std::chrono::seconds>(
+ RETRY_MAX_DELAY));
+ }
+ }
+ return std::nullopt;
+}
+
+bool isRetryableStatus(int status)
+{
+ return status == 429 || (status >= 500 && status <= 599);
+}
+
+mw::E<LlmResponse> responseFromOpenAIJson(const nlohmann::json& json)
+{
+ if(!json.is_object())
+ {
+ return std::unexpected(error::parse("expected LLM response object"));
+ }
+
+ const auto choices = json.find("choices");
+ if(choices == json.end() || !choices->is_array() || choices->empty())
+ {
+ return std::unexpected(
+ error::parse("expected non-empty LLM response choices"));
+ }
+ const nlohmann::json& choice = choices->front();
+ if(!choice.is_object())
+ {
+ return std::unexpected(error::parse("expected LLM response choice"));
+ }
+ const auto message = choice.find("message");
+ if(message == choice.end() || !message->is_object())
+ {
+ return std::unexpected(
+ error::parse("expected assistant message in LLM response"));
+ }
+
+ nlohmann::json assistant_json = *message;
+ const auto usage = json.find("usage");
+ if(usage != json.end() && !usage->is_null())
+ {
+ assistant_json["usage"] = *usage;
+ }
+ auto parsed_message = assistantMessageFromOpenAIJson(assistant_json);
+ if(!parsed_message)
+ {
+ return std::unexpected(parsed_message.error());
+ }
+ const auto* assistant = std::get_if<AssistantMessage>(&*parsed_message);
+ if(assistant == nullptr)
+ {
+ return std::unexpected(
+ error::parse("expected parsed assistant message"));
+ }
+ return LlmResponse{
+ .message = std::move(*assistant),
+ };
+}
+
+mw::E<LlmResponse> responseFromHttp(const mw::HTTPResponse& response)
+{
+ const nlohmann::json json =
+ nlohmann::json::parse(response.payloadAsStr(), nullptr, false);
+ if(json.is_discarded())
+ {
+ return std::unexpected(error::parse("expected JSON LLM response"));
+ }
+ return responseFromOpenAIJson(json);
+}
+
+mw::HTTPRequest buildHttpRequest(const nlohmann::json& body,
+ const LlmClientConfig& config)
+{
+ mw::HTTPRequest request(config.endpoint.str());
+ request.setPayload(body.dump());
+ request.setContentType("application/json");
+ if(config.api_key)
+ {
+ request.addHeader("Authorization", "Bearer " + *config.api_key);
+ }
+ return request;
+}
+
+} // namespace
+
+LlmClient::LlmClient(Runtime& runtime, LlmClientConfig config)
+ : LlmClient(runtime, runtime.http(), std::move(config))
+{
+}
+
+LlmClient::LlmClient(Runtime& runtime, mw::HTTPSessionAsyncInterface& http,
+ LlmClientConfig config)
+ : runtime_(&runtime), http_(&http), config_(std::move(config))
+{
+}
+
+mw::Task<mw::E<LlmResponse>> LlmClient::complete(LlmRequest request)
+{
+ auto body = buildRequestBody(request, config_);
+ if(!body)
+ {
+ co_return std::unexpected(body.error());
+ }
+ if(!config_.endpoint.valid())
+ {
+ co_return std::unexpected(error::parse("expected valid LLM endpoint"));
+ }
+
+ auto endpoint = config_.endpoint.resolve("/v1/chat/completions");
+ if(!endpoint)
+ {
+ co_return std::unexpected(endpoint.error());
+ }
+ LlmClientConfig request_config = config_;
+ request_config.endpoint = std::move(*endpoint);
+
+ const int max_retries = std::max(config_.max_retries, 0);
+ for(int attempt = 0;; ++attempt)
+ {
+ auto response =
+ co_await http_->post(buildHttpRequest(*body, request_config));
+ if(!response)
+ {
+ if(attempt >= max_retries)
+ {
+ co_return std::unexpected(response.error());
+ }
+ }
+ else if(response->status >= 200 && response->status <= 299)
+ {
+ co_return responseFromHttp(*response);
+ }
+ else if(!isRetryableStatus(response->status) || attempt >= max_retries)
+ {
+ const std::string_view payload = response->payloadAsStr();
+ co_return std::unexpected(error::llmApi(
+ response->status, payload.substr(0, MAX_ERROR_BODY_EXCERPT)));
+ }
+
+ const auto delay = response
+ ? retryAfterDelay(*response)
+ : std::optional<std::chrono::milliseconds>{};
+ auto slept = co_await runtime_->sleepFor(
+ delay.value_or(retryDelay(attempt + 1)));
+ if(!slept)
+ {
+ co_return std::unexpected(slept.error());
+ }
+ }
+}
+
+} // namespace agent_smith
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 50ce20a..9afb236 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -1,4 +1,5 @@
add_executable(agent_smith_test_skeleton
+ test_llm_client.cpp
test_message.cpp
test_sse_parser.cpp
test_skeleton.cpp
@@ -6,6 +7,7 @@ add_executable(agent_smith_test_skeleton
target_link_libraries(agent_smith_test_skeleton
PRIVATE
agent_smith::agent_smith
+ GTest::gmock
GTest::gtest_main
)
agent_smith_enable_test_sanitizer(agent_smith_test_skeleton)
diff --git a/tests/test_llm_client.cpp b/tests/test_llm_client.cpp
new file mode 100644
index 0000000..7715592
--- /dev/null
+++ b/tests/test_llm_client.cpp
@@ -0,0 +1,299 @@
+#include <condition_variable>
+#include <coroutine>
+#include <exception>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <utility>
+
+#include <agent_smith/llm_client.hpp>
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include <mw/http_client_async_mock.hpp>
+
+namespace agent_smith
+{
+namespace
+{
+
+using ::testing::_;
+using ::testing::InSequence;
+using ::testing::Invoke;
+
+template <typename T> class SyncWaitTask
+{
+public:
+ struct promise_type
+ {
+ SyncWaitTask get_return_object()
+ {
+ return SyncWaitTask(
+ std::coroutine_handle<promise_type>::from_promise(*this));
+ }
+
+ std::suspend_always initial_suspend() noexcept
+ {
+ return {};
+ }
+
+ auto final_suspend() noexcept
+ {
+ struct FinalAwaiter
+ {
+ bool await_ready() noexcept
+ {
+ return false;
+ }
+
+ void await_suspend(
+ std::coroutine_handle<promise_type> handle) noexcept
+ {
+ auto& promise = handle.promise();
+ std::lock_guard lock(promise.mutex);
+ promise.done = true;
+ promise.condition.notify_one();
+ }
+
+ void await_resume() noexcept {}
+ };
+
+ return FinalAwaiter{};
+ }
+
+ template <typename U> void return_value(U&& value)
+ {
+ value_.emplace(std::forward<U>(value));
+ }
+
+ void unhandled_exception()
+ {
+ exception_ = std::current_exception();
+ }
+
+ std::mutex mutex;
+ std::condition_variable condition;
+ bool done = false;
+ std::optional<T> value_;
+ std::exception_ptr exception_;
+ };
+
+ explicit SyncWaitTask(std::coroutine_handle<promise_type> handle)
+ : handle_(handle)
+ {
+ }
+
+ SyncWaitTask(SyncWaitTask&& other) noexcept
+ : handle_(std::exchange(other.handle_, nullptr))
+ {
+ }
+
+ SyncWaitTask(const SyncWaitTask&) = delete;
+ SyncWaitTask& operator=(const SyncWaitTask&) = delete;
+
+ ~SyncWaitTask()
+ {
+ if(handle_ != nullptr)
+ {
+ handle_.destroy();
+ }
+ }
+
+ void start()
+ {
+ handle_.resume();
+ }
+
+ T get()
+ {
+ auto& promise = handle_.promise();
+ std::unique_lock lock(promise.mutex);
+ promise.condition.wait(lock, [&]() { return promise.done; });
+ lock.unlock();
+
+ if(promise.exception_ != nullptr)
+ {
+ std::rethrow_exception(promise.exception_);
+ }
+ return std::move(*promise.value_);
+ }
+
+private:
+ std::coroutine_handle<promise_type> handle_ = nullptr;
+};
+
+template <typename T> SyncWaitTask<T> makeWaiter(mw::Task<T> task)
+{
+ co_return co_await std::move(task);
+}
+
+template <typename T> T syncWait(mw::Task<T> task)
+{
+ auto waiter = makeWaiter(std::move(task));
+ waiter.start();
+ return waiter.get();
+}
+
+mw::URL testEndpoint()
+{
+ auto endpoint = mw::URL::fromStr("https://llm.example/api");
+ EXPECT_TRUE(endpoint.has_value());
+ return std::move(*endpoint);
+}
+
+LlmClientConfig testConfig()
+{
+ return LlmClientConfig{
+ .endpoint = testEndpoint(),
+ .model = "test-model",
+ .max_retries = 0,
+ };
+}
+
+TEST(LlmClientTest, SendsOpenAIRequestAndParsesAssistantResponse)
+{
+ Runtime runtime;
+ mw::HTTPSessionAsyncMock http;
+ LlmClientConfig config = testConfig();
+ config.api_key = "secret";
+ config.default_sampling.temperature = 0.2;
+ config.default_sampling.extra = {{"provider_option", true}};
+ LlmClient client(runtime, http, std::move(config));
+
+ mw::HTTPRequest captured;
+ EXPECT_CALL(http, post(_))
+ .WillOnce(Invoke(
+ [&](mw::HTTPRequest request)
+ {
+ captured = std::move(request);
+ return mw::HTTPSessionAsyncMock::complete(mw::HTTPResponse(
+ 200,
+ R"({"choices":[{"message":{"role":"assistant","content":"hello"}}],"usage":{"prompt_tokens":3,"completion_tokens":2,"total_tokens":5}})"));
+ }));
+
+ LlmRequest request{
+ .messages = {SystemMessage{"be concise"}, UserMessage{"hi"}},
+ .sampling =
+ SamplingParams{
+ .top_p = 0.9,
+ .max_tokens = 32,
+ },
+ };
+ auto result = syncWait(client.complete(std::move(request)));
+
+ ASSERT_TRUE(result.has_value());
+ EXPECT_EQ(result->message.content, "hello");
+ ASSERT_TRUE(result->message.usage.has_value());
+ EXPECT_EQ(result->message.usage->total, 5U);
+
+ EXPECT_EQ(captured.url, "https://llm.example/v1/chat/completions");
+ EXPECT_EQ(captured.header.at("Content-Type"), "application/json");
+ EXPECT_EQ(captured.header.at("Authorization"), "Bearer secret");
+ const nlohmann::json body = nlohmann::json::parse(captured.request_data);
+ EXPECT_EQ(body.at("model"), "test-model");
+ EXPECT_EQ(body.at("messages"),
+ nlohmann::json({{{"role", "system"}, {"content", "be concise"}},
+ {{"role", "user"}, {"content", "hi"}}}));
+ EXPECT_FALSE(body.at("stream"));
+ EXPECT_EQ(body.at("temperature"), 0.2);
+ EXPECT_EQ(body.at("top_p"), 0.9);
+ EXPECT_EQ(body.at("max_tokens"), 32);
+ EXPECT_TRUE(body.at("provider_option"));
+}
+
+TEST(LlmClientTest, RejectsReservedExtraBeforeSendingRequest)
+{
+ Runtime runtime;
+ mw::HTTPSessionAsyncMock http;
+ LlmClient client(runtime, http, testConfig());
+
+ EXPECT_CALL(http, post(_)).Times(0);
+ LlmRequest request{
+ .sampling =
+ SamplingParams{
+ .extra = {{"stream", true}},
+ },
+ };
+ auto result = syncWait(client.complete(std::move(request)));
+
+ ASSERT_FALSE(result.has_value());
+ EXPECT_NE(result.error().as<ParseError>(), nullptr);
+}
+
+TEST(LlmClientTest, ReturnsTypedErrorForHttpFailure)
+{
+ Runtime runtime;
+ mw::HTTPSessionAsyncMock http;
+ LlmClient client(runtime, http, testConfig());
+
+ EXPECT_CALL(http, post(_))
+ .WillOnce(Invoke(
+ [](mw::HTTPRequest)
+ {
+ return mw::HTTPSessionAsyncMock::complete(
+ mw::HTTPResponse(400, "bad request"));
+ }));
+
+ auto result = syncWait(client.complete({}));
+
+ ASSERT_FALSE(result.has_value());
+ const LlmApiError* error = result.error().as<LlmApiError>();
+ ASSERT_NE(error, nullptr);
+ EXPECT_EQ(error->http_status, 400);
+ EXPECT_EQ(error->body_excerpt, "bad request");
+}
+
+TEST(LlmClientTest, ReturnsTypedErrorForMalformedResponse)
+{
+ Runtime runtime;
+ mw::HTTPSessionAsyncMock http;
+ LlmClient client(runtime, http, testConfig());
+
+ EXPECT_CALL(http, post(_))
+ .WillOnce(Invoke(
+ [](mw::HTTPRequest)
+ {
+ return mw::HTTPSessionAsyncMock::complete(
+ mw::HTTPResponse(200, "not json"));
+ }));
+
+ auto result = syncWait(client.complete({}));
+
+ ASSERT_FALSE(result.has_value());
+ EXPECT_NE(result.error().as<ParseError>(), nullptr);
+}
+
+TEST(LlmClientTest, RetriesRetryableHttpFailures)
+{
+ Runtime runtime;
+ mw::HTTPSessionAsyncMock http;
+ LlmClientConfig config = testConfig();
+ config.max_retries = 1;
+ LlmClient client(runtime, http, std::move(config));
+
+ {
+ InSequence sequence;
+ EXPECT_CALL(http, post(_))
+ .WillOnce(Invoke(
+ [](mw::HTTPRequest)
+ {
+ return mw::HTTPSessionAsyncMock::complete(
+ mw::HTTPResponse(503, "temporarily unavailable"));
+ }));
+ EXPECT_CALL(http, post(_))
+ .WillOnce(Invoke(
+ [](mw::HTTPRequest)
+ {
+ return mw::HTTPSessionAsyncMock::complete(mw::HTTPResponse(
+ 200,
+ R"({"choices":[{"message":{"role":"assistant","content":"recovered"}}]})"));
+ }));
+ }
+
+ auto result = syncWait(client.complete({}));
+
+ ASSERT_TRUE(result.has_value());
+ EXPECT_EQ(result->message.content, "recovered");
+}
+
+} // namespace
+} // namespace agent_smith