feat(upstream): support request-scoped custom HTTP retry statuses

README.md

@@ -627,8 +627,8 @@ resty.kong.upstream.set\_next\_upstream
 
 **subsystems:** *http*
 
-Set upstream next enablement of current request to the given string of table
-argument . Global setting set by [`proxy_next_upstream`](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream) will be overwritten.
+Set the retry criteria for the current request. This overrides the settings
+from [`proxy_next_upstream`](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream).
 
 The `set_next_upstream` function supports variable length of arguments, and each argument must be one of the following strings (also defined in [`proxy_next_upstream`](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream)):
 - `error`
@@ -644,8 +644,22 @@ The `set_next_upstream` function supports variable length of arguments, and each
 - `non_idempotent`
 - `off`
 
-On success, this function returns `nil`. Otherwise throw a string
-describing the error will be returned.
+In addition to the options above, `http_<status>` accepts any three-digit HTTP
+error status from `400` through `599`. Custom statuses are stored per request;
+no reload or per-status nginx constant is needed. Calling this function again
+replaces both the previous options and the custom status set. `off` disables all
+retries, including when combined with other options.
+
+Native statuses retain nginx's failure accounting. Custom 4xx statuses advance
+to the next peer without marking the peer failed; custom 5xx statuses mark it
+failed. The real upstream status is retained in retry history. Retry limits,
+timeouts, request-body buffering and `non_idempotent` restrictions still apply.
+Custom status retries require the companion nginx dynamic next-upstream status
+patch. The module can build without it, but setting custom statuses alone does
+not enable those retries in an unpatched nginx.
+
+On success, this function returns `nil`. On failure, it returns a string
+describing the error and leaves the previous criteria unchanged.
 
 This function can be called multiple times in the same request. Later calls override
 previous ones.

lualib/resty/kong/upstream.lua

@@ -21,7 +21,8 @@ base.allows_subsystem("http")
 
 ffi.cdef([[
 int
-ngx_http_lua_ffi_set_next_upstream(ngx_http_request_t *r, uint32_t next_upstream, char **err);
+ngx_http_lua_ffi_set_next_upstream_statuses(ngx_http_request_t *r, uint32_t next_upstream,
+    const uint16_t *statuses, size_t count, char **err);
 const uint32_t ngx_http_lua_kong_next_upstream_mask_error;
 const uint32_t ngx_http_lua_kong_next_upstream_mask_timeout;
 const uint32_t ngx_http_lua_kong_next_upstream_mask_invalid_header;
@@ -71,22 +72,30 @@ function _M.set_next_upstream(...)
 
     local arg_table = { ... }
     local next_upstream = 0
+    local statuses = ffi.new("uint16_t[?]", nargs)
+    local count = 0
     for i = 1, nargs do
         local v = arg_table[i]
         if type(v) ~= "string" then
             return "argument #" .. i .. " is not a string"
         end
 
         local next_upstream_value = next_upstream_table[v]
-        if not next_upstream_value then
-            return "argument #" .. i .. " is not a valid argument"
+        if next_upstream_value then
+            next_upstream = bit.bor(next_upstream, next_upstream_value)
+        else
+            local status = tonumber(v:match("^http_([45]%d%d)$"))
+            if not status then
+                return "argument #" .. i .. " is not a valid argument"
+            end
+
+            statuses[count] = status
+            count = count + 1
         end
-
-        next_upstream = bit.bor(next_upstream, next_upstream_value)
     end
 
     local err = ffi.new("char *[1]")
-    local rc = C.ngx_http_lua_ffi_set_next_upstream(r, next_upstream, err)
+    local rc = C.ngx_http_lua_ffi_set_next_upstream_statuses(r, next_upstream, statuses, count, err)
 
     if rc ~= NGX_OK then
         return "failed to set upstream next: " .. ffi_str(err[0])

src/ngx_http_lua_kong_common.h

@@ -24,11 +24,14 @@
 #include <ngx_http.h>
 #include "ssl/ngx_lua_kong_ssl.h"
 
+#define NGX_HTTP_LUA_KONG_NEXT_UPSTREAM_BITMAP_SIZE  25
+
 typedef struct {
     ngx_lua_kong_ssl_ctx_t   ssl_ctx;
     ngx_str_t                grpc_authority;
     ngx_http_log_handler_pt  orig_log_handler;
     ngx_uint_t               next_upstream;
+    u_char                  *next_upstream_statuses; /* lazy bitmap: HTTP 400..599 */
 } ngx_http_lua_kong_ctx_t;
 
 

src/ngx_http_lua_kong_module.c

@@ -18,6 +18,11 @@
 #include "ngx_http_lua_kong_directive.h"
 #include "ngx_http_upstream.h"
 
+/* Keep in sync with the companion nginx dynamic next-upstream status patch. */
+#ifndef NGX_HTTP_UPSTREAM_FT_HTTP_CUSTOM
+#define NGX_HTTP_UPSTREAM_FT_HTTP_CUSTOM  0x00008000
+#endif
+
 static ngx_int_t ngx_http_lua_kong_init(ngx_conf_t *cf);
 static void* ngx_http_lua_kong_create_loc_conf(ngx_conf_t* cf);
 static char* ngx_http_lua_kong_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child);
@@ -198,6 +203,82 @@ ngx_http_lua_ffi_set_next_upstream(ngx_http_request_t *r, ngx_uint_t next_upstre
         return NGX_ERROR;
     }
 
+    ctx->next_upstream = next_upstream;
+    if (ctx->next_upstream_statuses != NULL) {
+        ngx_memzero(ctx->next_upstream_statuses,
+                    NGX_HTTP_LUA_KONG_NEXT_UPSTREAM_BITMAP_SIZE);
+    }
+    return NGX_OK;
+}
+
+
+/* Read-only: do not allocate a module context on the normal proxy path. */
+ngx_flag_t
+ngx_http_lua_kong_next_upstream_status(ngx_http_request_t *r, ngx_uint_t status)
+{
+    ngx_http_lua_kong_ctx_t  *ctx;
+
+    if (status < 400 || status > 599) {
+        return 0;
+    }
+
+    ctx = ngx_http_get_module_ctx(r, ngx_http_lua_kong_module);
+    if (ctx == NULL || ctx->next_upstream_statuses == NULL
+        || (ctx->next_upstream & NGX_HTTP_UPSTREAM_FT_OFF))
+    {
+        return 0;
+    }
+
+    status -= 400;
+    return (ctx->next_upstream_statuses[status / 8] >> (status % 8)) & 1;
+}
+
+
+int
+ngx_http_lua_ffi_set_next_upstream_statuses(ngx_http_request_t *r,
+    uint32_t next_upstream, const uint16_t *statuses, size_t count, char **err)
+{
+    ngx_http_lua_kong_ctx_t  *ctx;
+    u_char                  bitmap[NGX_HTTP_LUA_KONG_NEXT_UPSTREAM_BITMAP_SIZE];
+    size_t                  i;
+    ngx_uint_t              status;
+
+    ngx_memzero(bitmap, sizeof(bitmap));
+    for (i = 0; i < count; i++) {
+        if (statuses[i] < 400 || statuses[i] > 599) {
+            *err = "HTTP status must be between 400 and 599";
+            return NGX_ERROR;
+        }
+
+        status = statuses[i] - 400;
+        bitmap[status / 8] |= (u_char) (1 << (status % 8));
+    }
+
+    ctx = ngx_http_lua_kong_get_module_ctx(r);
+    if (ctx == NULL) {
+        *err = "failed to allocate request context";
+        return NGX_ERROR;
+    }
+
+    if (next_upstream & NGX_HTTP_UPSTREAM_FT_OFF) {
+        next_upstream = NGX_HTTP_UPSTREAM_FT_OFF;
+        ngx_memzero(bitmap, sizeof(bitmap));
+
+    } else if (count) {
+        if (ctx->next_upstream_statuses == NULL) {
+            ctx->next_upstream_statuses = ngx_palloc(r->pool, sizeof(bitmap));
+            if (ctx->next_upstream_statuses == NULL) {
+                *err = "failed to allocate HTTP status bitmap";
+                return NGX_ERROR;
+            }
+        }
+
+        next_upstream |= NGX_HTTP_UPSTREAM_FT_HTTP_CUSTOM;
+    }
+
+    if (ctx->next_upstream_statuses != NULL) {
+        ngx_memcpy(ctx->next_upstream_statuses, bitmap, sizeof(bitmap));
+    }
     ctx->next_upstream = next_upstream;
     return NGX_OK;
 }

src/ngx_http_lua_kong_module.h

@@ -40,6 +40,9 @@ ngx_flag_t
 ngx_http_lua_kong_get_next_upstream_mask(ngx_http_request_t *r,
     ngx_flag_t upstream_next);
 
+ngx_flag_t
+ngx_http_lua_kong_next_upstream_status(ngx_http_request_t *r, ngx_uint_t status);
+
 ngx_str_t *
 ngx_http_lua_kong_ssl_get_upstream_ssl_sans_dnsnames(ngx_http_request_t *r);