Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions docs/llm.markdown
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,6 @@ jsonschema llm path/to/schema.json \
--header "Authorization: Bearer $DOUBLEWORD_API_KEY"
```

Doubleword can sometimes be slow to serve a request, independently of the
model, which is what the generous default `--timeout/-T` leaves room for.

### OpenAI

```sh
Expand Down
27 changes: 27 additions & 0 deletions src/error.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include <sourcemeta/core/options.h>
#include <sourcemeta/core/yaml.h>

#include <chrono> // std::chrono::milliseconds
#include <concepts> // std::same_as, std::convertible_to
#include <cstddef> // std::size_t
#include <cstdint> // std::uint64_t
Expand Down Expand Up @@ -1326,6 +1327,23 @@ inline auto print_exception(const bool is_json, const Exception &exception)
}
}

// A backend does not report which of its bounds elapsed, so this is the
// total that was configured rather than what the request actually took
if constexpr (requires(const Exception &current) {
{
current.timeout()
} -> std::same_as<std::chrono::milliseconds>;
}) {
const auto timeout{exception.timeout()};
if (is_json) {
error_json.assign(
"timeoutMilliseconds",
sourcemeta::core::JSON{static_cast<std::size_t>(timeout.count())});
} else {
std::cerr << " after " << timeout.count() << " milliseconds\n";
}
}

if constexpr (requires(const Exception &current) {
{
current.method()
Expand Down Expand Up @@ -2305,6 +2323,15 @@ inline auto try_catch(const sourcemeta::core::Options &options,
const auto is_json{options.contains("json")};
print_exception(is_json, error);
return EXIT_OTHER_INPUT_ERROR;
} catch (const sourcemeta::core::HTTPTimeoutError &error) {
const auto is_json{options.contains("json")};
print_exception(is_json, error);
if (!is_json) {
std::cerr << "\nPass --timeout/-T with a larger number of seconds to "
"wait longer\n";
}

return EXIT_UNEXPECTED_ERROR;
} catch (const sourcemeta::core::HTTPStatusError &error) {
const auto is_json{options.contains("json")};
print_exception(is_json, error);
Expand Down
1 change: 1 addition & 0 deletions test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1643,6 +1643,7 @@ add_jsonschema_test_unix_ci(pass_llm_request_response)
add_jsonschema_test_unix_ci(fail_llm_response_invalid)
add_jsonschema_test_unix_ci(fail_llm_response_rejected)
add_jsonschema_test_unix_ci(fail_llm_response_unreadable)
add_jsonschema_test_unix_ci(fail_llm_timeout)
add_jsonschema_test_unix_ci(precommit_lint)

if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND NOT JSONSCHEMA_USE_SYSTEM_CURL)
Expand Down
69 changes: 69 additions & 0 deletions test/ci/fail_llm_timeout.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
#!/bin/sh

set -o errexit
set -o nounset

TMP="$(mktemp -d)"
PORT=5899

# A server that accepts the connection and never answers, which is the one
# failure the caller can do something about by waiting longer
cat << 'EOF' > "$TMP/server.js"
const http = require('http');
const server = http.createServer(() => {});
server.listen(parseInt(process.argv[2], 10));
EOF

node "$TMP/server.js" "$PORT" &
SERVER_PID="$!"

clean() {
kill "$SERVER_PID" 2>/dev/null || true
rm -rf "$TMP"
}
trap clean EXIT

sleep 2

cat << 'EOF' > "$TMP/schema.json"
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "string"
}
EOF

"$1" llm "$TMP/schema.json" --ask hello \
--url "http://localhost:${PORT}/v1/chat/completions" --model my-model \
--timeout 2 \
> "$TMP/output.txt" 2>&1 && EXIT_CODE="$?" || EXIT_CODE="$?"
# Unexpected error
test "$EXIT_CODE" = "1"

cat << EOF > "$TMP/expected.txt"
error: The HTTP request timed out
after 2000 milliseconds
with method POST
at url http://localhost:${PORT}/v1/chat/completions

Pass --timeout/-T with a larger number of seconds to wait longer
EOF

diff "$TMP/output.txt" "$TMP/expected.txt"

"$1" llm "$TMP/schema.json" --ask hello \
--url "http://localhost:${PORT}/v1/chat/completions" --model my-model \
--timeout 2 --json \
> "$TMP/output_json.txt" 2>&1 && EXIT_CODE="$?" || EXIT_CODE="$?"
# Unexpected error
test "$EXIT_CODE" = "1"

cat << EOF > "$TMP/expected_json.txt"
{
"error": "The HTTP request timed out",
"timeoutMilliseconds": 2000,
"method": "POST",
"url": "http://localhost:${PORT}/v1/chat/completions"
}
EOF

diff "$TMP/output_json.txt" "$TMP/expected_json.txt"
Loading