bugfix: respect max retry after using balancer pool.

In the balancer phase, when obtaining a connection from the upstream
connection pool, the `cached` attribute of the peer connection is set
to 1(`pc->cached = 1;`), indicating that the connection is obtained
from the cache.

If an error occurs during the use of this connection, such as
"upstream prematurely closed connection" the system will increase the
`tries` attribute of the peer connection by executing
`u->peer.tries++`.

This PR restores tries by callbacks to the balancer when
`u->peer.tries++` is unexpectedly set.

Signed-off-by: tzssangglass <[email protected]>

Author: tzssangglass

src/ngx_http_lua_balancer.c

@@ -87,6 +87,8 @@ static ngx_int_t ngx_http_lua_balancer_by_chunk(lua_State *L,
     ngx_http_request_t *r);
 static void ngx_http_lua_balancer_free_peer(ngx_peer_connection_t *pc,
     void *data, ngx_uint_t state);
+static void ngx_http_lua_balancer_notify_peer(ngx_peer_connection_t *pc,
+    void *data, ngx_uint_t type);
 static void ngx_http_lua_balancer_close(ngx_connection_t *c);
 static void ngx_http_lua_balancer_dummy_handler(ngx_event_t *ev);
 static void ngx_http_lua_balancer_close_handler(ngx_event_t *ev);
@@ -365,6 +367,7 @@ ngx_http_lua_balancer_init_peer(ngx_http_request_t *r,
     r->upstream->peer.data = bp;
     r->upstream->peer.get = ngx_http_lua_balancer_get_peer;
     r->upstream->peer.free = ngx_http_lua_balancer_free_peer;
+    r->upstream->peer.notify = ngx_http_lua_balancer_notify_peer;
 
 #if (NGX_HTTP_SSL)
     bp->original_set_session = r->upstream->peer.set_session;
@@ -737,6 +740,18 @@ ngx_http_lua_balancer_free_peer(ngx_peer_connection_t *pc, void *data,
 }
 
 
+static void
+ngx_http_lua_balancer_notify_peer(ngx_peer_connection_t *pc, void *data,
+    ngx_uint_t type)
+{
+#ifdef NGX_HTTP_UPSTREAM_NOTIFY_CACHED_CONNECTION_ERROR
+    if (type == NGX_HTTP_UPSTREAM_NOTIFY_CACHED_CONNECTION_ERROR) {
+        pc->tries--;
+    }
+#endif
+}
+
+
 static void
 ngx_http_lua_balancer_close(ngx_connection_t *c)
 {

t/188-balancer_keepalive_pool_max_retry.t

@@ -0,0 +1,89 @@
+# vim:set ft= ts=4 sw=4 et:
+
+use Test::Nginx::Socket::Lua;
+use Cwd qw(cwd);
+
+log_level('info');
+repeat_each(1);
+
+plan tests => repeat_each() * (blocks() * 6);
+
+my $pwd = cwd();
+
+no_long_string();
+
+check_accum_error_log();
+run_tests();
+
+__DATA__
+
+=== TEST 1: sanity
+--- http_config
+    lua_shared_dict request_counter 1m;
+    upstream my_upstream {
+        server 127.0.0.1;
+        balancer_by_lua_block {
+            local balancer = require "ngx.balancer"
+
+            if not ngx.ctx.tries then
+                ngx.ctx.tries = 0
+            end
+
+            ngx.ctx.tries = ngx.ctx.tries + 1
+            ngx.log(ngx.INFO, "tries ", ngx.ctx.tries)
+
+            if ngx.ctx.tries == 1 then
+                balancer.set_more_tries(5)
+            end
+            
+            local host = "127.0.0.1"
+            local port = 8090;
+
+            local ok, err = balancer.set_current_peer(host, port)
+            if not ok then
+                ngx.log(ngx.ERR, "failed to set the current peer: ", err)
+                return ngx.exit(500)
+            end
+
+            balancer.set_timeouts(60000, 60000, 60000)
+
+            local ok, err = balancer.enable_keepalive(60, 100)
+            if not ok then
+                ngx.log(ngx.ERR, "failed to enable keepalive: ", err)
+                return ngx.exit(500)
+            end
+        }
+    }
+
+    server {
+        listen 0.0.0.0:8090;
+        location /hello {
+            content_by_lua_block{                
+                local request_counter = ngx.shared.request_counter
+                local first_request = request_counter:get("first_request")
+                if first_request == nil then
+                    request_counter:set("first_request", "yes")
+                    ngx.print("hello")
+                else
+                    ngx.exit(ngx.HTTP_CLOSE)
+                end
+            }
+        }
+    }
+--- config
+    location = /t {
+        proxy_pass http://my_upstream;
+        proxy_set_header Connection "keep-alive";
+        
+        rewrite_by_lua_block {
+           ngx.req.set_uri("/hello")
+        }
+    }
+--- pipelined_requests eval
+["GET /t HTTP/1.1" , "GET /t HTTP/1.1"]
+--- response_body eval
+["hello", qr/502/]
+--- error_code eval
+[200, 502]
+--- no_error_log eval
+qr/tries 7/