From 45800d6da972b02d0225b1a9b20fc56e93d2d35c Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Tue, 6 Oct 2026 18:11:22 -0300 Subject: [PATCH 1/3] Improve timeout errors on the `llm` command Signed-off-by: Juan Cruz Viotti --- docs/llm.markdown | 3 -- src/command_llm.cc | 16 ++++++++- src/error.h | 25 ++++++++++++++ test/CMakeLists.txt | 1 + test/ci/fail_llm_timeout.sh | 65 +++++++++++++++++++++++++++++++++++++ 5 files changed, 106 insertions(+), 4 deletions(-) create mode 100755 test/ci/fail_llm_timeout.sh diff --git a/docs/llm.markdown b/docs/llm.markdown index c4e1e999a..158e0566b 100644 --- a/docs/llm.markdown +++ b/docs/llm.markdown @@ -73,9 +73,6 @@ jsonschema llm path/to/schema.json \ --header "Authorization: Bearer $DOUBLEWORD_API_KEY" ``` -Doubleword can sometimes be slow to serve a request, independently of the -model, which is what the generous default `--timeout/-T` leaves room for. - ### OpenAI ```sh diff --git a/src/command_llm.cc b/src/command_llm.cc index 071619af3..63a47a080 100644 --- a/src/command_llm.cc +++ b/src/command_llm.cc @@ -777,7 +777,21 @@ auto sourcemeta::jsonschema::llm(const sourcemeta::core::Options &options) // Sent once and never retried. A completion is not idempotent and it costs // money, so a request that fails is reported rather than repeated - const auto response{request.send()}; + const auto started{std::chrono::steady_clock::now()}; + sourcemeta::core::HTTPResponse response; + try { + response = request.send(); + } catch (const sourcemeta::core::HTTPError &) { + // What the backend says about a failure is its own business, so whether the + // time ran out is read off the clock rather than off the message + if (std::chrono::steady_clock::now() - started >= timeout) { + throw LLMTimeoutError{"The completion request did not answer within " + + std::to_string(timeout.count()) + " seconds", + std::string{url}}; + } + + throw; + } LOG_VERBOSE(options) << "Received HTTP " << response.status.code << "\n"; diff --git a/src/error.h b/src/error.h index 9d9f061d9..e69564df8 100644 --- a/src/error.h +++ b/src/error.h @@ -418,6 +418,22 @@ class MissingOptionError : public std::runtime_error { std::string example_; }; +// A request that ran out of time before an answer arrived. Waiting longer is +// the one thing the caller can do about it, so what it took is part of what is +// said +class LLMTimeoutError : public std::runtime_error { +public: + LLMTimeoutError(const std::string &message, std::string url) + : std::runtime_error{message}, url_{std::move(url)} {} + + [[nodiscard]] auto url() const noexcept -> const std::string & { + return this->url_; + } + +private: + std::string url_; +}; + class OptionConflictError : public std::runtime_error { public: OptionConflictError(const std::string &message) @@ -2305,6 +2321,15 @@ inline auto try_catch(const sourcemeta::core::Options &options, const auto is_json{options.contains("json")}; print_exception(is_json, error); return EXIT_OTHER_INPUT_ERROR; + } catch (const LLMTimeoutError &error) { + const auto is_json{options.contains("json")}; + print_exception(is_json, error); + if (!is_json) { + std::cerr << "\nPass --timeout/-T with a larger number of seconds to " + "wait longer\n"; + } + + return EXIT_UNEXPECTED_ERROR; } catch (const sourcemeta::core::HTTPStatusError &error) { const auto is_json{options.contains("json")}; print_exception(is_json, error); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 17e37c09a..1b331f6e9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1643,6 +1643,7 @@ add_jsonschema_test_unix_ci(pass_llm_request_response) add_jsonschema_test_unix_ci(fail_llm_response_invalid) add_jsonschema_test_unix_ci(fail_llm_response_rejected) add_jsonschema_test_unix_ci(fail_llm_response_unreadable) +add_jsonschema_test_unix_ci(fail_llm_timeout) add_jsonschema_test_unix_ci(precommit_lint) if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND NOT JSONSCHEMA_USE_SYSTEM_CURL) diff --git a/test/ci/fail_llm_timeout.sh b/test/ci/fail_llm_timeout.sh new file mode 100755 index 000000000..202f485d3 --- /dev/null +++ b/test/ci/fail_llm_timeout.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +PORT=5899 + +# A server that accepts the connection and never answers, which is the one +# failure the caller can do something about by waiting longer +cat << 'EOF' > "$TMP/server.js" +const http = require('http'); +const server = http.createServer(() => {}); +server.listen(parseInt(process.argv[2], 10)); +EOF + +node "$TMP/server.js" "$PORT" & +SERVER_PID="$!" + +clean() { + kill "$SERVER_PID" 2>/dev/null || true + rm -rf "$TMP" +} +trap clean EXIT + +sleep 2 + +cat << 'EOF' > "$TMP/schema.json" +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string" +} +EOF + +"$1" llm "$TMP/schema.json" --ask hello \ + --url "http://localhost:${PORT}/v1/chat/completions" --model my-model \ + --timeout 2 \ + > "$TMP/output.txt" 2>&1 && EXIT_CODE="$?" || EXIT_CODE="$?" +# Unexpected error +test "$EXIT_CODE" = "1" + +cat << EOF > "$TMP/expected.txt" +error: The completion request did not answer within 2 seconds + at url http://localhost:${PORT}/v1/chat/completions + +Pass --timeout/-T with a larger number of seconds to wait longer +EOF + +diff "$TMP/output.txt" "$TMP/expected.txt" + +"$1" llm "$TMP/schema.json" --ask hello \ + --url "http://localhost:${PORT}/v1/chat/completions" --model my-model \ + --timeout 2 --json \ + > "$TMP/output_json.txt" 2>&1 && EXIT_CODE="$?" || EXIT_CODE="$?" +# Unexpected error +test "$EXIT_CODE" = "1" + +cat << EOF > "$TMP/expected_json.txt" +{ + "error": "The completion request did not answer within 2 seconds", + "url": "http://localhost:${PORT}/v1/chat/completions" +} +EOF + +diff "$TMP/output_json.txt" "$TMP/expected_json.txt" From f553cb5fa3b8c44645bafaf6ea98b10d126a63d1 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Tue, 6 Oct 2026 18:19:51 -0300 Subject: [PATCH 2/3] More Signed-off-by: Juan Cruz Viotti --- src/command_llm.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/command_llm.cc b/src/command_llm.cc index 63a47a080..a18df16b5 100644 --- a/src/command_llm.cc +++ b/src/command_llm.cc @@ -784,6 +784,9 @@ auto sourcemeta::jsonschema::llm(const sourcemeta::core::Options &options) } catch (const sourcemeta::core::HTTPError &) { // What the backend says about a failure is its own business, so whether the // time ran out is read off the clock rather than off the message + // TODO: Have Core raise an exception of its own for a request that ran out + // of time, so that this is detected from the failure rather than inferred + // from how long it took if (std::chrono::steady_clock::now() - started >= timeout) { throw LLMTimeoutError{"The completion request did not answer within " + std::to_string(timeout.count()) + " seconds", From ee35e84308c27de22772a5d979d886c7acd6789a Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Sun, 25 Oct 2026 16:30:36 -0300 Subject: [PATCH 3/3] Simpler Signed-off-by: Juan Cruz Viotti --- src/command_llm.cc | 19 +------------------ src/error.h | 36 +++++++++++++++++++----------------- test/ci/fail_llm_timeout.sh | 8 ++++++-- 3 files changed, 26 insertions(+), 37 deletions(-) diff --git a/src/command_llm.cc b/src/command_llm.cc index a18df16b5..071619af3 100644 --- a/src/command_llm.cc +++ b/src/command_llm.cc @@ -777,24 +777,7 @@ auto sourcemeta::jsonschema::llm(const sourcemeta::core::Options &options) // Sent once and never retried. A completion is not idempotent and it costs // money, so a request that fails is reported rather than repeated - const auto started{std::chrono::steady_clock::now()}; - sourcemeta::core::HTTPResponse response; - try { - response = request.send(); - } catch (const sourcemeta::core::HTTPError &) { - // What the backend says about a failure is its own business, so whether the - // time ran out is read off the clock rather than off the message - // TODO: Have Core raise an exception of its own for a request that ran out - // of time, so that this is detected from the failure rather than inferred - // from how long it took - if (std::chrono::steady_clock::now() - started >= timeout) { - throw LLMTimeoutError{"The completion request did not answer within " + - std::to_string(timeout.count()) + " seconds", - std::string{url}}; - } - - throw; - } + const auto response{request.send()}; LOG_VERBOSE(options) << "Received HTTP " << response.status.code << "\n"; diff --git a/src/error.h b/src/error.h index e69564df8..737786d6d 100644 --- a/src/error.h +++ b/src/error.h @@ -18,6 +18,7 @@ #include #include +#include // std::chrono::milliseconds #include // std::same_as, std::convertible_to #include // std::size_t #include // std::uint64_t @@ -418,22 +419,6 @@ class MissingOptionError : public std::runtime_error { std::string example_; }; -// A request that ran out of time before an answer arrived. Waiting longer is -// the one thing the caller can do about it, so what it took is part of what is -// said -class LLMTimeoutError : public std::runtime_error { -public: - LLMTimeoutError(const std::string &message, std::string url) - : std::runtime_error{message}, url_{std::move(url)} {} - - [[nodiscard]] auto url() const noexcept -> const std::string & { - return this->url_; - } - -private: - std::string url_; -}; - class OptionConflictError : public std::runtime_error { public: OptionConflictError(const std::string &message) @@ -1342,6 +1327,23 @@ inline auto print_exception(const bool is_json, const Exception &exception) } } + // A backend does not report which of its bounds elapsed, so this is the + // total that was configured rather than what the request actually took + if constexpr (requires(const Exception ¤t) { + { + current.timeout() + } -> std::same_as; + }) { + const auto timeout{exception.timeout()}; + if (is_json) { + error_json.assign( + "timeoutMilliseconds", + sourcemeta::core::JSON{static_cast(timeout.count())}); + } else { + std::cerr << " after " << timeout.count() << " milliseconds\n"; + } + } + if constexpr (requires(const Exception ¤t) { { current.method() @@ -2321,7 +2323,7 @@ inline auto try_catch(const sourcemeta::core::Options &options, const auto is_json{options.contains("json")}; print_exception(is_json, error); return EXIT_OTHER_INPUT_ERROR; - } catch (const LLMTimeoutError &error) { + } catch (const sourcemeta::core::HTTPTimeoutError &error) { const auto is_json{options.contains("json")}; print_exception(is_json, error); if (!is_json) { diff --git a/test/ci/fail_llm_timeout.sh b/test/ci/fail_llm_timeout.sh index 202f485d3..8b2c38b1c 100755 --- a/test/ci/fail_llm_timeout.sh +++ b/test/ci/fail_llm_timeout.sh @@ -40,7 +40,9 @@ EOF test "$EXIT_CODE" = "1" cat << EOF > "$TMP/expected.txt" -error: The completion request did not answer within 2 seconds +error: The HTTP request timed out + after 2000 milliseconds + with method POST at url http://localhost:${PORT}/v1/chat/completions Pass --timeout/-T with a larger number of seconds to wait longer @@ -57,7 +59,9 @@ test "$EXIT_CODE" = "1" cat << EOF > "$TMP/expected_json.txt" { - "error": "The completion request did not answer within 2 seconds", + "error": "The HTTP request timed out", + "timeoutMilliseconds": 2000, + "method": "POST", "url": "http://localhost:${PORT}/v1/chat/completions" } EOF