fastcgi performance at 10K

Maxim Dounin mdounin at mdounin.ru
Wed Apr 15 22:23:11 MSD 2009


Hello!

On Wed, Apr 15, 2009 at 01:09:30PM +0100, Valery Kholodkov wrote:

> 
> ----- "Maxim Dounin" <mdounin at mdounin.ru> wrote:
> 
> > У меня есть работающий прототип поддержки keepalive для fastcgi.  
> > Если очень хочется потестировать - могу поделиться патчами.
> > 
> > Но надо понимать что на сколько-нибудь тяжёлых fastcgi запросах 
> > это не приведёт к заметному ускорению, и описанные проблемы скорее 
> > всего не вылечит (а может быть и усугубит).
> 
> Вне зависимости от того, кто что про это думает, код в студию!

Патчи прилагаются, накладывать по очереди:

patch-nginx-proxy-flush-2.txt
patch-nginx-proxy-length-2.txt
patch-nginx-keepalive.txt

Всё должно нормально накладываться на 0.7.51.  Первые два патча 
уже публиковались ранее, и вообще сами по себе полезные.  Третий 
грязненький, но я честно предупреждал что это лишь прототип.  :)

Использовать вместе со свежей версией ngx_http_upstream_keepalive, 
берут тут:

http://mdounin.ru/hg/ngx_http_upstream_keepalive/

При этом при сборке nginx'а надо определить 
NGX_UPSTREAM_KEEPALIVE_PATCHED, иначе работать будет как раньше 
(т.е. применимо только для memcached'а).  Как-то так:

./configure --with-cc-opt="-D NGX_UPSTREAM_KEEPALIVE_PATCHED" \
    --add-module=/path/to/ngx_http_upstream_keepalive

Maxim Dounin
-------------- next part --------------
# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1239806853 -14400
# Node ID 8fdcdee25947ea149445c1de2561605b3c4c7190
# Parent  eb4fdebda67325ad8c7b431b04b4b73ba0b40752
Upstream: flush partially filled buffers on upstream timeout.

Fix typo in comment while here.

diff --git a/src/event/ngx_event_pipe.c b/src/event/ngx_event_pipe.c
--- a/src/event/ngx_event_pipe.c
+++ b/src/event/ngx_event_pipe.c
@@ -10,7 +10,8 @@
 #include <ngx_event_pipe.h>
 
 
-static ngx_int_t ngx_event_pipe_read_upstream(ngx_event_pipe_t *p);
+static ngx_int_t ngx_event_pipe_read_upstream(ngx_event_pipe_t *p,
+                                              ngx_int_t do_flush);
 static ngx_int_t ngx_event_pipe_write_to_downstream(ngx_event_pipe_t *p);
 
 static ngx_int_t ngx_event_pipe_write_chain_to_temp_file(ngx_event_pipe_t *p);
@@ -40,7 +41,7 @@ ngx_event_pipe(ngx_event_pipe_t *p, ngx_
 
         p->log->action = "reading upstream";
 
-        if (ngx_event_pipe_read_upstream(p) == NGX_ABORT) {
+        if (ngx_event_pipe_read_upstream(p, 0) == NGX_ABORT) {
             return NGX_ABORT;
         }
 
@@ -88,8 +89,27 @@ ngx_event_pipe(ngx_event_pipe_t *p, ngx_
 }
 
 
+ngx_int_t
+ngx_event_pipe_flush(ngx_event_pipe_t *p)
+{
+    p->log->action = "reading upstream";
+
+    if (ngx_event_pipe_read_upstream(p, 1) == NGX_ABORT) {
+        return NGX_ABORT;
+    }
+
+    p->log->action = "sending to client";
+
+    if (ngx_event_pipe_write_to_downstream(p) == NGX_ABORT) {
+        return NGX_ABORT;
+    }
+
+    return NGX_OK;
+}
+
+
 static ngx_int_t
-ngx_event_pipe_read_upstream(ngx_event_pipe_t *p)
+ngx_event_pipe_read_upstream(ngx_event_pipe_t *p, ngx_int_t do_flush)
 {
     ssize_t       n, size;
     ngx_int_t     rc;
@@ -97,6 +117,9 @@ ngx_event_pipe_read_upstream(ngx_event_p
     ngx_chain_t  *chain, *cl, *ln;
 
     if (p->upstream_eof || p->upstream_error || p->upstream_done) {
+        if (do_flush) {
+            goto flush;
+        }
         return NGX_OK;
     }
 
@@ -331,6 +354,8 @@ ngx_event_pipe_read_upstream(ngx_event_p
         }
     }
 
+flush:
+
 #if (NGX_DEBUG)
 
     for (cl = p->busy; cl; cl = cl->next) {
@@ -940,7 +965,7 @@ ngx_event_pipe_add_free_buf(ngx_event_pi
         return NGX_OK;
     }
 
-    /* the first free buf is partialy filled, thus add the free buf after it */
+    /* the first free buf is partially filled, thus add the free buf after it */
 
     cl->next = p->free_raw_bufs->next;
     p->free_raw_bufs->next = cl;
diff --git a/src/event/ngx_event_pipe.h b/src/event/ngx_event_pipe.h
--- a/src/event/ngx_event_pipe.h
+++ b/src/event/ngx_event_pipe.h
@@ -87,6 +87,7 @@ struct ngx_event_pipe_s {
 
 
 ngx_int_t ngx_event_pipe(ngx_event_pipe_t *p, ngx_int_t do_write);
+ngx_int_t ngx_event_pipe_flush(ngx_event_pipe_t *p);
 ngx_int_t ngx_event_pipe_copy_input_filter(ngx_event_pipe_t *p, ngx_buf_t *buf);
 ngx_int_t ngx_event_pipe_add_free_buf(ngx_event_pipe_t *p, ngx_buf_t *b);
 
diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c
+++ b/src/http/ngx_http_upstream.c
@@ -2366,6 +2366,18 @@ ngx_http_upstream_process_upstream(ngx_h
         u->pipe->upstream_error = 1;
         ngx_connection_error(c, NGX_ETIMEDOUT, "upstream timed out");
 
+        c = r->connection;
+
+        if (ngx_event_pipe_flush(u->pipe) == NGX_ABORT) {
+
+            if (c->destroyed) {
+                return;
+            }
+
+            ngx_http_upstream_finalize_request(r, u, 0);
+            return;
+        }
+
     } else {
         c = r->connection;
 
-------------- next part --------------
# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1239806853 -14400
# Node ID 92becfe381d252f2f9bf562469e5562aefcf2b62
# Parent  8fdcdee25947ea149445c1de2561605b3c4c7190
Upstream: don't wait for upstream close if we know length.

This change basically deligates right to control how many bytes may be
buffered by ngx_event_pipe to input filter by means of changing p->length.

Currently p->length is set to NGX_MAX_OFF_T_VALUE if content length isn't
known (and thus input must be terminated by upstream connection close), or
to content length if it's known.

Futher improvement may involve correct parsing of chunked transfer encoding -
by setting p->length in input filter as approprate.  E.g. set it to 2 if you
just parsed chunk end and expect other chunk to start (chunk takes at least
3 bytes, "0" CRLF, but RFC 2616 recomends to be tolerant and recognize bare
LF too).

diff --git a/src/event/ngx_event_pipe.c b/src/event/ngx_event_pipe.c
--- a/src/event/ngx_event_pipe.c
+++ b/src/event/ngx_event_pipe.c
@@ -328,22 +328,26 @@ ngx_event_pipe_read_upstream(ngx_event_p
 
             if (n >= size) {
                 cl->buf->last = cl->buf->end;
-
-                /* STUB */ cl->buf->num = p->num++;
-
-                if (p->input_filter(p, cl->buf) == NGX_ERROR) {
-                    return NGX_ABORT;
-                }
-
                 n -= size;
-                ln = cl;
-                cl = cl->next;
-                ngx_free_chain(p->pool, ln);
 
             } else {
                 cl->buf->last += n;
                 n = 0;
+
+                if (cl->buf->last - cl->buf->pos < p->length) {
+                    continue;
+                }
             }
+
+            /* STUB */ cl->buf->num = p->num++;
+
+            if (p->input_filter(p, cl->buf) == NGX_ERROR) {
+                return NGX_ABORT;
+            }
+
+            ln = cl;
+            cl = cl->next;
+            ngx_free_chain(p->pool, ln);
         }
 
         if (cl) {
@@ -410,6 +414,9 @@ flush:
                        cl->buf->file_last - cl->buf->file_pos);
     }
 
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, p->log, 0,
+                   "pipe length: %O", p->length);
+
 #endif
 
     if ((p->upstream_eof || p->upstream_error) && p->free_raw_bufs) {
@@ -868,6 +875,16 @@ ngx_event_pipe_copy_input_filter(ngx_eve
     }
     p->last_in = &cl->next;
 
+    if (p->length == NGX_MAX_OFF_T_VALUE) {
+        return NGX_OK;
+    }
+
+    p->length -= b->last - b->pos;
+
+    if (p->length <= 0) {
+        p->upstream_done = 1;
+    }
+
     return NGX_OK;
 }
 
diff --git a/src/event/ngx_event_pipe.h b/src/event/ngx_event_pipe.h
--- a/src/event/ngx_event_pipe.h
+++ b/src/event/ngx_event_pipe.h
@@ -65,6 +65,7 @@ struct ngx_event_pipe_s {
     ssize_t            busy_size;
 
     off_t              read_length;
+    off_t              length;
 
     off_t              max_temp_file_size;
     ssize_t            temp_file_write_size;
diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c
+++ b/src/http/ngx_http_upstream.c
@@ -1962,6 +1962,13 @@ ngx_http_upstream_send_response(ngx_http
     p->pool = r->pool;
     p->log = c->log;
 
+    if (r->headers_out.content_length_n != -1) {
+        p->length = r->headers_out.content_length_n;
+
+    } else {
+        p->length = NGX_MAX_OFF_T_VALUE;
+    }
+
     p->cacheable = u->cacheable || u->store;
 
     p->temp_file = ngx_pcalloc(r->pool, sizeof(ngx_temp_file_t));
-------------- next part --------------
# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1239806853 -14400
# Node ID 57491e6ae0a92157c6ddb2808f37037fc7e04977
# Parent  92becfe381d252f2f9bf562469e5562aefcf2b62
Upstream: keepalive support in memcached and fastcgi.

Currently dirty, but works (and passes all tests).

diff --git a/src/http/modules/ngx_http_fastcgi_module.c b/src/http/modules/ngx_http_fastcgi_module.c
--- a/src/http/modules/ngx_http_fastcgi_module.c
+++ b/src/http/modules/ngx_http_fastcgi_module.c
@@ -73,6 +73,8 @@ typedef struct {
 
 #define NGX_HTTP_FASTCGI_RESPONDER      1
 
+#define NGX_HTTP_FASTCGI_KEEP_CONN      1
+
 #define NGX_HTTP_FASTCGI_BEGIN_REQUEST  1
 #define NGX_HTTP_FASTCGI_ABORT_REQUEST  2
 #define NGX_HTTP_FASTCGI_END_REQUEST    3
@@ -463,7 +465,7 @@ static ngx_http_fastcgi_request_start_t 
 
     { 0,                                               /* role_hi */
       NGX_HTTP_FASTCGI_RESPONDER,                      /* role_lo */
-      0, /* NGX_HTTP_FASTCGI_KEEP_CONN */              /* flags */
+      NGX_HTTP_FASTCGI_KEEP_CONN,                      /* flags */
       { 0, 0, 0, 0, 0 } },                             /* reserved[5] */
 
     { 1,                                               /* version */
@@ -1397,6 +1399,7 @@ ngx_http_fastcgi_process_header(ngx_http
         }
 
         if (rc == NGX_HTTP_PARSE_HEADER_DONE) {
+            u->pipe->length = sizeof(ngx_http_fastcgi_header_t);
             return NGX_OK;
         }
 
@@ -1465,7 +1468,10 @@ ngx_http_fastcgi_input_filter(ngx_event_
 
             if (f->type == NGX_HTTP_FASTCGI_STDOUT && f->length == 0) {
                 f->state = ngx_http_fastcgi_st_version;
+                /* XXX */
+#if 0
                 p->upstream_done = 1;
+#endif
 
                 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, p->log, 0,
                                "http fastcgi closed stdout");
@@ -1476,6 +1482,7 @@ ngx_http_fastcgi_input_filter(ngx_event_
             if (f->type == NGX_HTTP_FASTCGI_END_REQUEST) {
                 f->state = ngx_http_fastcgi_st_version;
                 p->upstream_done = 1;
+                r->upstream->keepalive = 1;
 
                 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, p->log, 0,
                                "http fastcgi sent end request");
@@ -1627,6 +1634,8 @@ ngx_http_fastcgi_input_filter(ngx_event_
 
             b->last = f->last;
 
+            p->length = sizeof(ngx_http_fastcgi_header_t);
+
             break;
         }
 
@@ -1634,6 +1643,9 @@ ngx_http_fastcgi_input_filter(ngx_event_
 
         b->last = f->last;
 
+        /* XXX */
+        p->length = f->length;
+
         break;
 
     }
diff --git a/src/http/modules/ngx_http_memcached_module.c b/src/http/modules/ngx_http_memcached_module.c
--- a/src/http/modules/ngx_http_memcached_module.c
+++ b/src/http/modules/ngx_http_memcached_module.c
@@ -385,6 +385,7 @@ found:
 
         u->headers_in.status_n = 404;
         u->state->status = 404;
+        u->keepalive = 1;
 
         return NGX_OK;
     }
@@ -435,6 +436,7 @@ ngx_http_memcached_filter(void *data, ss
         {
             ngx_log_error(NGX_LOG_ERR, ctx->request->connection->log, 0,
                           "memcached sent invalid trailer");
+            u->keepalive = 0;
         }
 
         u->length = 0;
@@ -477,6 +479,9 @@ ngx_http_memcached_filter(void *data, ss
     if (ngx_strncmp(last, ngx_http_memcached_end, b->last - last) != 0) {
         ngx_log_error(NGX_LOG_ERR, ctx->request->connection->log, 0,
                       "memcached sent invalid trailer");
+
+    } else {
+        u->keepalive = 1;
     }
 
     ctx->rest -= b->last - last;
diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c
--- a/src/http/modules/ngx_http_proxy_module.c
+++ b/src/http/modules/ngx_http_proxy_module.c
@@ -1527,6 +1527,15 @@ ngx_http_proxy_process_header(ngx_http_r
                 h->lowcase_key = (u_char *) "date";
             }
 
+            /* XXX */
+
+            if (r->headers_out.content_length_n != -1) {
+                r->upstream->pipe->length = r->headers_out.content_length_n;
+
+            } else {
+                r->upstream->pipe->length = NGX_MAX_OFF_T_VALUE;
+            }
+
             return NGX_OK;
         }
 
diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c
+++ b/src/http/ngx_http_upstream.c
@@ -1371,6 +1371,8 @@ ngx_http_upstream_process_header(ngx_htt
 
         rc = u->process_header(r);
 
+        /* XXX */
+
         if (rc == NGX_AGAIN) {
 
             if (u->buffer.pos == u->buffer.end) {
@@ -1726,6 +1728,8 @@ ngx_http_upstream_process_headers(ngx_ht
 
     u->headers_in.content_length_n = r->headers_out.content_length_n;
 
+    /* XXX */
+
     if (r->headers_out.content_length_n != -1) {
         u->length = (size_t) r->headers_out.content_length_n;
 
@@ -1839,6 +1843,8 @@ ngx_http_upstream_send_response(ngx_http
 
     if (!u->buffering) {
 
+        /* XXX */
+
         if (u->input_filter == NULL) {
             u->input_filter_init = ngx_http_upstream_non_buffered_filter_init;
             u->input_filter = ngx_http_upstream_non_buffered_filter;
@@ -1962,13 +1968,6 @@ ngx_http_upstream_send_response(ngx_http
     p->pool = r->pool;
     p->log = c->log;
 
-    if (r->headers_out.content_length_n != -1) {
-        p->length = r->headers_out.content_length_n;
-
-    } else {
-        p->length = NGX_MAX_OFF_T_VALUE;
-    }
-
     p->cacheable = u->cacheable || u->store;
 
     p->temp_file = ngx_pcalloc(r->pool, sizeof(ngx_temp_file_t));
diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h
--- a/src/http/ngx_http_upstream.h
+++ b/src/http/ngx_http_upstream.h
@@ -293,6 +293,7 @@ struct ngx_http_upstream_s {
 #endif
 
     unsigned                         buffering:1;
+    unsigned                         keepalive:1;
 
     unsigned                         request_sent:1;
     unsigned                         header_sent:1;


More information about the nginx-ru mailing list