# HG changeset patch # User Maxim Dounin # Date 1352393278 -14400 # Node ID e5f163a85ec1830c36a7e34acb5b8a4371706f05 # Parent a2138093701b82286c6f25cb58b9fb41ab159e90 Dav: fixed segfault on PUT if body was already read (ticket #238). If request body reading happens with different options it's possible that there will be no r->request_body->temp_file available (or even no r->request_body available if body was discarded). Return internal server error in this case instead of committing suicide by dereferencing a null pointer. diff --git a/src/http/modules/ngx_http_dav_module.c b/src/http/modules/ngx_http_dav_module.c --- a/src/http/modules/ngx_http_dav_module.c +++ b/src/http/modules/ngx_http_dav_module.c @@ -209,6 +209,11 @@ ngx_http_dav_put_handler(ngx_http_reques ngx_ext_rename_file_t ext; ngx_http_dav_loc_conf_t *dlcf; + if (r->request_body == NULL || r->request_body->temp_file == NULL) { + ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); + return; + } + ngx_http_map_uri_to_path(r, &path, &root, 0); path.len--; # HG changeset patch # User Maxim Dounin # Date 1352393278 -14400 # Node ID 3951448c8fcd0592e12e47a0197cf3b0d72a9c86 # Parent e5f163a85ec1830c36a7e34acb5b8a4371706f05 Core: added debug logging of writev() in ngx_write_chain_to_file(). diff --git a/src/os/unix/ngx_files.c b/src/os/unix/ngx_files.c --- a/src/os/unix/ngx_files.c +++ b/src/os/unix/ngx_files.c @@ -241,6 +241,9 @@ ngx_write_chain_to_file(ngx_file_t *file return NGX_ERROR; } + ngx_log_debug2(NGX_LOG_DEBUG_CORE, file->log, 0, + "writev: %d, %z", file->fd, n); + file->sys_offset += n; file->offset += n; total += n; # HG changeset patch # User Maxim Dounin # Date 1352393278 -14400 # Node ID 6d665fc11097eb32265ee498467c2af80fe9d182 # Parent 3951448c8fcd0592e12e47a0197cf3b0d72a9c86 Request body: fixed "501 Not Implemented" error handling. It is not about "Method" but a generic message, and is expected to be used e.g. if specified Transfer-Encoding is not supported. Fixed message to match RFC 2616. Additionally, disable keepalive on such errors as we won't be able to read request body correctly if we don't understand Transfer-Encoding used. diff --git a/src/http/ngx_http_header_filter_module.c b/src/http/ngx_http_header_filter_module.c --- a/src/http/ngx_http_header_filter_module.c +++ b/src/http/ngx_http_header_filter_module.c @@ -112,7 +112,7 @@ static ngx_str_t ngx_http_status_lines[] #define NGX_HTTP_OFF_5XX (NGX_HTTP_LAST_4XX - 400 + NGX_HTTP_OFF_4XX) ngx_string("500 Internal Server Error"), - ngx_string("501 Method Not Implemented"), + ngx_string("501 Not Implemented"), ngx_string("502 Bad Gateway"), ngx_string("503 Service Temporarily Unavailable"), ngx_string("504 Gateway Time-out"), diff --git a/src/http/ngx_http_special_response.c b/src/http/ngx_http_special_response.c --- a/src/http/ngx_http_special_response.c +++ b/src/http/ngx_http_special_response.c @@ -260,9 +260,9 @@ static char ngx_http_error_500_page[] = static char ngx_http_error_501_page[] = "" CRLF -"501 Method Not Implemented" CRLF +"501 Not Implemented" CRLF "" CRLF -"

501 Method Not Implemented

" CRLF +"

501 Not Implemented

" CRLF ; @@ -384,6 +384,7 @@ ngx_http_special_response_handler(ngx_ht case NGX_HTTPS_CERT_ERROR: case NGX_HTTPS_NO_CERT: case NGX_HTTP_INTERNAL_SERVER_ERROR: + case NGX_HTTP_NOT_IMPLEMENTED: r->keepalive = 0; } } # HG changeset patch # User Maxim Dounin # Date 1352393278 -14400 # Node ID 042fe7d8401e3858e2b98b05f56bbe6f2381c0e0 # Parent 6d665fc11097eb32265ee498467c2af80fe9d182 Request body: $request_body variable generalization. The $request_body variable was assuming there can't be more than two buffers. While this is currently true due to request body reading implementation details, this is not a good thing to depend on and may change in the future. diff --git a/src/http/ngx_http_variables.c b/src/http/ngx_http_variables.c --- a/src/http/ngx_http_variables.c +++ b/src/http/ngx_http_variables.c @@ -1757,7 +1757,7 @@ ngx_http_variable_request_body(ngx_http_ { u_char *p; size_t len; - ngx_buf_t *buf, *next; + ngx_buf_t *buf; ngx_chain_t *cl; if (r->request_body == NULL @@ -1782,8 +1782,13 @@ ngx_http_variable_request_body(ngx_http_ return NGX_OK; } - next = cl->next->buf; - len = (buf->last - buf->pos) + (next->last - next->pos); + len = buf->last - buf->pos; + cl = cl->next; + + for ( /* void */ ; cl; cl = cl->next) { + buf = cl->buf; + len += buf->last - buf->pos; + } p = ngx_pnalloc(r->pool, len); if (p == NULL) { @@ -1791,9 +1796,12 @@ ngx_http_variable_request_body(ngx_http_ } v->data = p; - - p = ngx_cpymem(p, buf->pos, buf->last - buf->pos); - ngx_memcpy(p, next->pos, next->last - next->pos); + cl = r->request_body->bufs; + + for ( /* void */ ; cl; cl = cl->next) { + buf = cl->buf; + p = ngx_cpymem(p, buf->pos, buf->last - buf->pos); + } v->len = len; v->valid = 1; # HG changeset patch # User Maxim Dounin # Date 1352393278 -14400 # Node ID 5881f77000a52426e0b997004efa614673e98a0c # Parent 042fe7d8401e3858e2b98b05f56bbe6f2381c0e0 Request body: code duplication reduced, no functional changes. The r->request_body_in_file_only with empty body case is now handled in ngx_http_write_request_body(). diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c --- a/src/http/ngx_http_request_body.c +++ b/src/http/ngx_http_request_body.c @@ -33,7 +33,6 @@ ngx_http_read_client_request_body(ngx_ht ssize_t size; ngx_buf_t *b; ngx_chain_t *cl, **next; - ngx_temp_file_t *tf; ngx_http_request_body_t *rb; ngx_http_core_loc_conf_t *clcf; @@ -65,30 +64,7 @@ ngx_http_read_client_request_body(ngx_ht if (r->headers_in.content_length_n == 0) { if (r->request_body_in_file_only) { - tf = ngx_pcalloc(r->pool, sizeof(ngx_temp_file_t)); - if (tf == NULL) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; - } - - tf->file.fd = NGX_INVALID_FILE; - tf->file.log = r->connection->log; - tf->path = clcf->client_body_temp_path; - tf->pool = r->pool; - tf->warn = "a client request body is buffered to a temporary file"; - tf->log_level = r->request_body_file_log_level; - tf->persistent = r->request_body_in_persistent_file; - tf->clean = r->request_body_in_clean_file; - - if (r->request_body_file_group_access) { - tf->access = 0660; - } - - rb->temp_file = tf; - - if (ngx_create_temp_file(&tf->file, tf->path, tf->pool, - tf->persistent, tf->clean, tf->access) - != NGX_OK) - { + if (ngx_http_write_request_body(r, NULL) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } } @@ -419,6 +395,19 @@ ngx_http_write_request_body(ngx_http_req } rb->temp_file = tf; + + if (body == NULL) { + /* empty body with r->request_body_in_file_only */ + + if (ngx_create_temp_file(&tf->file, tf->path, tf->pool, + tf->persistent, tf->clean, tf->access) + != NGX_OK) + { + return NGX_ERROR; + } + + return NGX_OK; + } } n = ngx_write_chain_to_temp_file(rb->temp_file, body); # HG changeset patch # User Maxim Dounin # Date 1352393422 -14400 # Node ID b1cad9fc4bc57d1b9c801b694670725dd05a5e5a # Parent 5881f77000a52426e0b997004efa614673e98a0c Request body: fixed socket leak on errors. The r->main->count reference counter was always incremented in ngx_http_read_client_request_body(), while it is only needs to be incremented on positive returns. diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c --- a/src/http/ngx_http_request_body.c +++ b/src/http/ngx_http_request_body.c @@ -31,6 +31,7 @@ ngx_http_read_client_request_body(ngx_ht { size_t preread; ssize_t size; + ngx_int_t rc; ngx_buf_t *b; ngx_chain_t *cl, **next; ngx_http_request_body_t *rb; @@ -44,12 +45,14 @@ ngx_http_read_client_request_body(ngx_ht } if (ngx_http_test_expect(r) != NGX_OK) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; + goto done; } rb = ngx_pcalloc(r->pool, sizeof(ngx_http_request_body_t)); if (rb == NULL) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; + goto done; } r->request_body = rb; @@ -65,7 +68,8 @@ ngx_http_read_client_request_body(ngx_ht if (r->request_body_in_file_only) { if (ngx_http_write_request_body(r, NULL) != NGX_OK) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; + goto done; } } @@ -95,7 +99,8 @@ ngx_http_read_client_request_body(ngx_ht b = ngx_calloc_buf(r->pool); if (b == NULL) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; + goto done; } b->temporary = 1; @@ -106,7 +111,8 @@ ngx_http_read_client_request_body(ngx_ht rb->bufs = ngx_alloc_chain_link(r->pool); if (rb->bufs == NULL) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; + goto done; } rb->bufs->buf = b; @@ -124,7 +130,8 @@ ngx_http_read_client_request_body(ngx_ht if (r->request_body_in_file_only) { if (ngx_http_write_request_body(r, rb->bufs) != NGX_OK) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; + goto done; } } @@ -151,7 +158,8 @@ ngx_http_read_client_request_body(ngx_ht r->read_event_handler = ngx_http_read_client_request_body_handler; - return ngx_http_do_read_client_request_body(r); + rc = ngx_http_do_read_client_request_body(r); + goto done; } next = &rb->bufs->next; @@ -181,12 +189,14 @@ ngx_http_read_client_request_body(ngx_ht rb->buf = ngx_create_temp_buf(r->pool, size); if (rb->buf == NULL) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; + goto done; } cl = ngx_alloc_chain_link(r->pool); if (cl == NULL) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; + goto done; } cl->buf = rb->buf; @@ -211,7 +221,15 @@ ngx_http_read_client_request_body(ngx_ht r->read_event_handler = ngx_http_read_client_request_body_handler; - return ngx_http_do_read_client_request_body(r); + rc = ngx_http_do_read_client_request_body(r); + +done: + + if (rc >= NGX_HTTP_SPECIAL_RESPONSE) { + r->main->count--; + } + + return rc; } # HG changeset patch # User Maxim Dounin # Date 1352393278 -14400 # Node ID e4ebfc6fb2a81453408dff5d108d54756a268276 # Parent b1cad9fc4bc57d1b9c801b694670725dd05a5e5a Request body: chunked parsing moved to ngx_http_parse.c from proxy. No functional changes. diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -81,13 +81,10 @@ typedef struct { typedef struct { ngx_http_status_t status; + ngx_http_chunked_t chunked; ngx_http_proxy_vars_t vars; size_t internal_body_length; - ngx_uint_t state; - off_t size; - off_t length; - ngx_uint_t head; /* unsigned head:1 */ } ngx_http_proxy_ctx_t; @@ -1252,7 +1249,7 @@ ngx_http_proxy_reinit_request(ngx_http_r ctx->status.count = 0; ctx->status.start = NULL; ctx->status.end = NULL; - ctx->state = 0; + ctx->chunked.state = 0; r->upstream->process_header = ngx_http_proxy_process_status_line; r->upstream->pipe->input_filter = ngx_http_proxy_copy_filter; @@ -1617,265 +1614,6 @@ ngx_http_proxy_copy_filter(ngx_event_pip } -static ngx_inline ngx_int_t -ngx_http_proxy_parse_chunked(ngx_http_request_t *r, ngx_buf_t *buf) -{ - u_char *pos, ch, c; - ngx_int_t rc; - ngx_http_proxy_ctx_t *ctx; - enum { - sw_chunk_start = 0, - sw_chunk_size, - sw_chunk_extension, - sw_chunk_extension_almost_done, - sw_chunk_data, - sw_after_data, - sw_after_data_almost_done, - sw_last_chunk_extension, - sw_last_chunk_extension_almost_done, - sw_trailer, - sw_trailer_almost_done, - sw_trailer_header, - sw_trailer_header_almost_done - } state; - - ctx = ngx_http_get_module_ctx(r, ngx_http_proxy_module); - - if (ctx == NULL) { - return NGX_ERROR; - } - - state = ctx->state; - - if (state == sw_chunk_data && ctx->size == 0) { - state = sw_after_data; - } - - rc = NGX_AGAIN; - - for (pos = buf->pos; pos < buf->last; pos++) { - - ch = *pos; - - ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "http proxy chunked byte: %02Xd s:%d", ch, state); - - switch (state) { - - case sw_chunk_start: - if (ch >= '0' && ch <= '9') { - state = sw_chunk_size; - ctx->size = ch - '0'; - break; - } - - c = (u_char) (ch | 0x20); - - if (c >= 'a' && c <= 'f') { - state = sw_chunk_size; - ctx->size = c - 'a' + 10; - break; - } - - goto invalid; - - case sw_chunk_size: - if (ch >= '0' && ch <= '9') { - ctx->size = ctx->size * 16 + (ch - '0'); - break; - } - - c = (u_char) (ch | 0x20); - - if (c >= 'a' && c <= 'f') { - ctx->size = ctx->size * 16 + (c - 'a' + 10); - break; - } - - if (ctx->size == 0) { - - switch (ch) { - case CR: - state = sw_last_chunk_extension_almost_done; - break; - case LF: - state = sw_trailer; - break; - case ';': - case ' ': - case '\t': - state = sw_last_chunk_extension; - break; - default: - goto invalid; - } - - break; - } - - switch (ch) { - case CR: - state = sw_chunk_extension_almost_done; - break; - case LF: - state = sw_chunk_data; - break; - case ';': - case ' ': - case '\t': - state = sw_chunk_extension; - break; - default: - goto invalid; - } - - break; - - case sw_chunk_extension: - switch (ch) { - case CR: - state = sw_chunk_extension_almost_done; - break; - case LF: - state = sw_chunk_data; - } - break; - - case sw_chunk_extension_almost_done: - if (ch == LF) { - state = sw_chunk_data; - break; - } - goto invalid; - - case sw_chunk_data: - rc = NGX_OK; - goto data; - - case sw_after_data: - switch (ch) { - case CR: - state = sw_after_data_almost_done; - break; - case LF: - state = sw_chunk_start; - } - break; - - case sw_after_data_almost_done: - if (ch == LF) { - state = sw_chunk_start; - break; - } - goto invalid; - - case sw_last_chunk_extension: - switch (ch) { - case CR: - state = sw_last_chunk_extension_almost_done; - break; - case LF: - state = sw_trailer; - } - break; - - case sw_last_chunk_extension_almost_done: - if (ch == LF) { - state = sw_trailer; - break; - } - goto invalid; - - case sw_trailer: - switch (ch) { - case CR: - state = sw_trailer_almost_done; - break; - case LF: - goto done; - default: - state = sw_trailer_header; - } - break; - - case sw_trailer_almost_done: - if (ch == LF) { - goto done; - } - goto invalid; - - case sw_trailer_header: - switch (ch) { - case CR: - state = sw_trailer_header_almost_done; - break; - case LF: - state = sw_trailer; - } - break; - - case sw_trailer_header_almost_done: - if (ch == LF) { - state = sw_trailer; - break; - } - goto invalid; - - } - } - -data: - - ctx->state = state; - buf->pos = pos; - - switch (state) { - - case sw_chunk_start: - ctx->length = 3 /* "0" LF LF */; - break; - case sw_chunk_size: - ctx->length = 2 /* LF LF */ - + (ctx->size ? ctx->size + 4 /* LF "0" LF LF */ : 0); - break; - case sw_chunk_extension: - case sw_chunk_extension_almost_done: - ctx->length = 1 /* LF */ + ctx->size + 4 /* LF "0" LF LF */; - break; - case sw_chunk_data: - ctx->length = ctx->size + 4 /* LF "0" LF LF */; - break; - case sw_after_data: - case sw_after_data_almost_done: - ctx->length = 4 /* LF "0" LF LF */; - break; - case sw_last_chunk_extension: - case sw_last_chunk_extension_almost_done: - ctx->length = 2 /* LF LF */; - break; - case sw_trailer: - case sw_trailer_almost_done: - ctx->length = 1 /* LF */; - break; - case sw_trailer_header: - case sw_trailer_header_almost_done: - ctx->length = 2 /* LF LF */; - break; - - } - - return rc; - -done: - - return NGX_DONE; - -invalid: - - return NGX_ERROR; -} - - static ngx_int_t ngx_http_proxy_chunked_filter(ngx_event_pipe_t *p, ngx_buf_t *buf) { @@ -1901,7 +1639,7 @@ ngx_http_proxy_chunked_filter(ngx_event_ for ( ;; ) { - rc = ngx_http_proxy_parse_chunked(r, buf); + rc = ngx_http_parse_chunked(r, buf, &ctx->chunked); if (rc == NGX_OK) { @@ -1952,16 +1690,16 @@ ngx_http_proxy_chunked_filter(ngx_event_ ngx_log_debug2(NGX_LOG_DEBUG_EVENT, p->log, 0, "input buf #%d %p", b->num, b->pos); - if (buf->last - buf->pos >= ctx->size) { - - buf->pos += ctx->size; + if (buf->last - buf->pos >= ctx->chunked.size) { + + buf->pos += ctx->chunked.size; b->last = buf->pos; - ctx->size = 0; + ctx->chunked.size = 0; continue; } - ctx->size -= buf->last - buf->pos; + ctx->chunked.size -= buf->last - buf->pos; buf->pos = buf->last; b->last = buf->last; @@ -1982,7 +1720,7 @@ ngx_http_proxy_chunked_filter(ngx_event_ /* set p->length, minimal amount of data we want to see */ - p->length = ctx->length; + p->length = ctx->chunked.length; break; } @@ -1997,7 +1735,7 @@ ngx_http_proxy_chunked_filter(ngx_event_ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http proxy chunked state %d, length %d", - ctx->state, p->length); + ctx->chunked.state, p->length); if (b) { b->shadow = buf; @@ -2094,7 +1832,7 @@ ngx_http_proxy_non_buffered_chunked_filt for ( ;; ) { - rc = ngx_http_proxy_parse_chunked(r, buf); + rc = ngx_http_parse_chunked(r, buf, &ctx->chunked); if (rc == NGX_OK) { @@ -2116,13 +1854,13 @@ ngx_http_proxy_non_buffered_chunked_filt b->pos = buf->pos; b->tag = u->output.tag; - if (buf->last - buf->pos >= ctx->size) { - buf->pos += ctx->size; + if (buf->last - buf->pos >= ctx->chunked.size) { + buf->pos += ctx->chunked.size; b->last = buf->pos; - ctx->size = 0; + ctx->chunked.size = 0; } else { - ctx->size -= buf->last - buf->pos; + ctx->chunked.size -= buf->last - buf->pos; buf->pos = buf->last; b->last = buf->last; } diff --git a/src/http/ngx_http.h b/src/http/ngx_http.h --- a/src/http/ngx_http.h +++ b/src/http/ngx_http.h @@ -18,6 +18,7 @@ typedef struct ngx_http_upstream_s ng typedef struct ngx_http_cache_s ngx_http_cache_t; typedef struct ngx_http_file_cache_s ngx_http_file_cache_t; typedef struct ngx_http_log_ctx_s ngx_http_log_ctx_t; +typedef struct ngx_http_chunked_s ngx_http_chunked_t; typedef ngx_int_t (*ngx_http_header_handler_pt)(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset); @@ -52,6 +53,13 @@ struct ngx_http_log_ctx_s { }; +struct ngx_http_chunked_s { + ngx_uint_t state; + off_t size; + off_t length; +}; + + typedef struct { ngx_uint_t http_version; ngx_uint_t code; @@ -92,6 +100,8 @@ ngx_int_t ngx_http_arg(ngx_http_request_ ngx_str_t *value); void ngx_http_split_args(ngx_http_request_t *r, ngx_str_t *uri, ngx_str_t *args); +ngx_int_t ngx_http_parse_chunked(ngx_http_request_t *r, ngx_buf_t *b, + ngx_http_chunked_t *ctx); ngx_int_t ngx_http_find_server_conf(ngx_http_request_t *r); diff --git a/src/http/ngx_http_parse.c b/src/http/ngx_http_parse.c --- a/src/http/ngx_http_parse.c +++ b/src/http/ngx_http_parse.c @@ -1818,3 +1818,256 @@ ngx_http_split_args(ngx_http_request_t * args->len = 0; } } + + +ngx_int_t +ngx_http_parse_chunked(ngx_http_request_t *r, ngx_buf_t *b, + ngx_http_chunked_t *ctx) +{ + u_char *pos, ch, c; + ngx_int_t rc; + enum { + sw_chunk_start = 0, + sw_chunk_size, + sw_chunk_extension, + sw_chunk_extension_almost_done, + sw_chunk_data, + sw_after_data, + sw_after_data_almost_done, + sw_last_chunk_extension, + sw_last_chunk_extension_almost_done, + sw_trailer, + sw_trailer_almost_done, + sw_trailer_header, + sw_trailer_header_almost_done + } state; + + state = ctx->state; + + if (state == sw_chunk_data && ctx->size == 0) { + state = sw_after_data; + } + + rc = NGX_AGAIN; + + for (pos = b->pos; pos < b->last; pos++) { + + ch = *pos; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http chunked byte: %02Xd s:%d", ch, state); + + switch (state) { + + case sw_chunk_start: + if (ch >= '0' && ch <= '9') { + state = sw_chunk_size; + ctx->size = ch - '0'; + break; + } + + c = (u_char) (ch | 0x20); + + if (c >= 'a' && c <= 'f') { + state = sw_chunk_size; + ctx->size = c - 'a' + 10; + break; + } + + goto invalid; + + case sw_chunk_size: + if (ch >= '0' && ch <= '9') { + ctx->size = ctx->size * 16 + (ch - '0'); + break; + } + + c = (u_char) (ch | 0x20); + + if (c >= 'a' && c <= 'f') { + ctx->size = ctx->size * 16 + (c - 'a' + 10); + break; + } + + if (ctx->size == 0) { + + switch (ch) { + case CR: + state = sw_last_chunk_extension_almost_done; + break; + case LF: + state = sw_trailer; + break; + case ';': + case ' ': + case '\t': + state = sw_last_chunk_extension; + break; + default: + goto invalid; + } + + break; + } + + switch (ch) { + case CR: + state = sw_chunk_extension_almost_done; + break; + case LF: + state = sw_chunk_data; + break; + case ';': + case ' ': + case '\t': + state = sw_chunk_extension; + break; + default: + goto invalid; + } + + break; + + case sw_chunk_extension: + switch (ch) { + case CR: + state = sw_chunk_extension_almost_done; + break; + case LF: + state = sw_chunk_data; + } + break; + + case sw_chunk_extension_almost_done: + if (ch == LF) { + state = sw_chunk_data; + break; + } + goto invalid; + + case sw_chunk_data: + rc = NGX_OK; + goto data; + + case sw_after_data: + switch (ch) { + case CR: + state = sw_after_data_almost_done; + break; + case LF: + state = sw_chunk_start; + } + break; + + case sw_after_data_almost_done: + if (ch == LF) { + state = sw_chunk_start; + break; + } + goto invalid; + + case sw_last_chunk_extension: + switch (ch) { + case CR: + state = sw_last_chunk_extension_almost_done; + break; + case LF: + state = sw_trailer; + } + break; + + case sw_last_chunk_extension_almost_done: + if (ch == LF) { + state = sw_trailer; + break; + } + goto invalid; + + case sw_trailer: + switch (ch) { + case CR: + state = sw_trailer_almost_done; + break; + case LF: + goto done; + default: + state = sw_trailer_header; + } + break; + + case sw_trailer_almost_done: + if (ch == LF) { + goto done; + } + goto invalid; + + case sw_trailer_header: + switch (ch) { + case CR: + state = sw_trailer_header_almost_done; + break; + case LF: + state = sw_trailer; + } + break; + + case sw_trailer_header_almost_done: + if (ch == LF) { + state = sw_trailer; + break; + } + goto invalid; + + } + } + +data: + + ctx->state = state; + b->pos = pos; + + switch (state) { + + case sw_chunk_start: + ctx->length = 3 /* "0" LF LF */; + break; + case sw_chunk_size: + ctx->length = 2 /* LF LF */ + + (ctx->size ? ctx->size + 4 /* LF "0" LF LF */ : 0); + break; + case sw_chunk_extension: + case sw_chunk_extension_almost_done: + ctx->length = 1 /* LF */ + ctx->size + 4 /* LF "0" LF LF */; + break; + case sw_chunk_data: + ctx->length = ctx->size + 4 /* LF "0" LF LF */; + break; + case sw_after_data: + case sw_after_data_almost_done: + ctx->length = 4 /* LF "0" LF LF */; + break; + case sw_last_chunk_extension: + case sw_last_chunk_extension_almost_done: + ctx->length = 2 /* LF LF */; + break; + case sw_trailer: + case sw_trailer_almost_done: + ctx->length = 1 /* LF */; + break; + case sw_trailer_header: + case sw_trailer_header_almost_done: + ctx->length = 2 /* LF LF */; + break; + + } + + return rc; + +done: + + return NGX_DONE; + +invalid: + + return NGX_ERROR; +} # HG changeset patch # User Maxim Dounin # Date 1352393278 -14400 # Node ID 72862d71e8871c0eaee7c6c559742adfb555f375 # Parent e4ebfc6fb2a81453408dff5d108d54756a268276 Request body: adjust b->pos when chunked parsing done. This is a nop for the current code, though will allow to correctly parse pipelined requests. diff --git a/src/http/ngx_http_parse.c b/src/http/ngx_http_parse.c --- a/src/http/ngx_http_parse.c +++ b/src/http/ngx_http_parse.c @@ -2065,6 +2065,9 @@ data: done: + ctx->state = 0; + b->pos = pos + 1; + return NGX_DONE; invalid: # HG changeset patch # User Maxim Dounin # Date 1352393278 -14400 # Node ID 6d99d0dbead8da638dd2a37f23cd5964058ad8b1 # Parent 72862d71e8871c0eaee7c6c559742adfb555f375 Request body: always use calculated size of a request body in proxy. This allows to handle requests with chunked body, and also simplifies handling of various request body modifications. diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -83,7 +83,7 @@ typedef struct { ngx_http_status_t status; ngx_http_chunked_t chunked; ngx_http_proxy_vars_t vars; - size_t internal_body_length; + off_t internal_body_length; ngx_uint_t head; /* unsigned head:1 */ } ngx_http_proxy_ctx_t; @@ -555,6 +555,8 @@ static char ngx_http_proxy_version_11[] static ngx_keyval_t ngx_http_proxy_headers[] = { { ngx_string("Host"), ngx_string("$proxy_host") }, { ngx_string("Connection"), ngx_string("close") }, + { ngx_string("Content-Length"), ngx_string("$proxy_internal_body_length") }, + { ngx_string("Transfer-Encoding"), ngx_string("") }, { ngx_string("Keep-Alive"), ngx_string("") }, { ngx_string("Expect"), ngx_string("") }, { ngx_string("Upgrade"), ngx_string("") }, @@ -580,6 +582,8 @@ static ngx_str_t ngx_http_proxy_hide_he static ngx_keyval_t ngx_http_proxy_cache_headers[] = { { ngx_string("Host"), ngx_string("$proxy_host") }, { ngx_string("Connection"), ngx_string("close") }, + { ngx_string("Content-Length"), ngx_string("$proxy_internal_body_length") }, + { ngx_string("Transfer-Encoding"), ngx_string("") }, { ngx_string("Keep-Alive"), ngx_string("") }, { ngx_string("Expect"), ngx_string("") }, { ngx_string("Upgrade"), ngx_string("") }, @@ -1003,6 +1007,9 @@ ngx_http_proxy_create_request(ngx_http_r ctx->internal_body_length = body_len; len += body_len; + + } else { + ctx->internal_body_length = r->headers_in.content_length_n; } le.ip = plcf->headers_set_len->elts; @@ -2039,7 +2046,7 @@ ngx_http_proxy_internal_body_length_vari ctx = ngx_http_get_module_ctx(r, ngx_http_proxy_module); - if (ctx == NULL) { + if (ctx == NULL || ctx->internal_body_length < 0) { v->not_found = 1; return NGX_OK; } @@ -2048,13 +2055,13 @@ ngx_http_proxy_internal_body_length_vari v->no_cacheable = 0; v->not_found = 0; - v->data = ngx_pnalloc(r->connection->pool, NGX_SIZE_T_LEN); + v->data = ngx_pnalloc(r->connection->pool, NGX_OFF_T_LEN); if (v->data == NULL) { return NGX_ERROR; } - v->len = ngx_sprintf(v->data, "%uz", ctx->internal_body_length) - v->data; + v->len = ngx_sprintf(v->data, "%O", ctx->internal_body_length) - v->data; return NGX_OK; } @@ -2822,8 +2829,6 @@ ngx_http_proxy_merge_headers(ngx_conf_t } if (conf->headers_set_hash.buckets - && ((conf->body_source.data == NULL) - == (prev->body_source.data == NULL)) #if (NGX_HTTP_CACHE) && ((conf->upstream.cache == NULL) == (prev->upstream.cache == NULL)) #endif @@ -2906,16 +2911,6 @@ ngx_http_proxy_merge_headers(ngx_conf_t h++; } - if (conf->body_source.data) { - s = ngx_array_push(&headers_merged); - if (s == NULL) { - return NGX_ERROR; - } - - ngx_str_set(&s->key, "Content-Length"); - ngx_str_set(&s->value, "$proxy_internal_body_length"); - } - src = headers_merged.elts; for (i = 0; i < headers_merged.nelts; i++) { # HG changeset patch # User Maxim Dounin # Date 1352393278 -14400 # Node ID 0e61d0ae8e76dc77cd22c7c00ef2b3639c39b9e1 # Parent 6d99d0dbead8da638dd2a37f23cd5964058ad8b1 Request body: $content_length variable to honor real body size. This allows to handle requests with chunked body by fastcgi and uwsgi modules, and also simplifies handling of various request body modifications. diff --git a/src/http/ngx_http_variables.c b/src/http/ngx_http_variables.c --- a/src/http/ngx_http_variables.c +++ b/src/http/ngx_http_variables.c @@ -39,6 +39,8 @@ static ngx_int_t ngx_http_variable_tcpin ngx_http_variable_value_t *v, uintptr_t data); #endif +static ngx_int_t ngx_http_variable_content_length(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_http_variable_host(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_http_variable_binary_remote_addr(ngx_http_request_t *r, @@ -149,8 +151,8 @@ static ngx_http_variable_t ngx_http_cor { ngx_string("http_cookie"), NULL, ngx_http_variable_headers, offsetof(ngx_http_request_t, headers_in.cookies), 0, 0 }, - { ngx_string("content_length"), NULL, ngx_http_variable_header, - offsetof(ngx_http_request_t, headers_in.content_length), 0, 0 }, + { ngx_string("content_length"), NULL, ngx_http_variable_content_length, + 0, 0, 0 }, { ngx_string("content_type"), NULL, ngx_http_variable_header, offsetof(ngx_http_request_t, headers_in.content_type), 0, 0 }, @@ -980,6 +982,39 @@ ngx_http_variable_tcpinfo(ngx_http_reque static ngx_int_t +ngx_http_variable_content_length(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data) +{ + u_char *p; + + if (r->headers_in.content_length) { + v->len = r->headers_in.content_length->value.len; + v->data = r->headers_in.content_length->value.data; + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + + } else if (r->headers_in.content_length_n >= 0) { + p = ngx_pnalloc(r->pool, NGX_OFF_T_LEN); + if (p == NULL) { + return NGX_ERROR; + } + + v->len = ngx_sprintf(p, "%O", r->headers_in.content_length_n) - p; + v->data = p; + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + + } else { + v->not_found = 1; + } + + return NGX_OK; +} + + +static ngx_int_t ngx_http_variable_host(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) { # HG changeset patch # User Maxim Dounin # Date 1352393278 -14400 # Node ID 64edd6c168a11d150f11425f98482ec66d2e2805 # Parent 0e61d0ae8e76dc77cd22c7c00ef2b3639c39b9e1 Request body: recalculate size of a request body in scgi module. This allows to handle requests with chunked body by scgi module, and also simplifies handling of various request body modifications. diff --git a/src/http/modules/ngx_http_scgi_module.c b/src/http/modules/ngx_http_scgi_module.c --- a/src/http/modules/ngx_http_scgi_module.c +++ b/src/http/modules/ngx_http_scgi_module.c @@ -533,10 +533,11 @@ ngx_http_scgi_create_key(ngx_http_reques static ngx_int_t ngx_http_scgi_create_request(ngx_http_request_t *r) { + off_t content_length_n; u_char ch, *key, *val, *lowcase_key; size_t len, key_len, val_len, allocated; ngx_buf_t *b; - ngx_str_t *content_length; + ngx_str_t content_length; ngx_uint_t i, n, hash, skip_empty, header_params; ngx_chain_t *cl, *body; ngx_list_part_t *part; @@ -545,12 +546,20 @@ ngx_http_scgi_create_request(ngx_http_re ngx_http_script_engine_t e, le; ngx_http_scgi_loc_conf_t *scf; ngx_http_script_len_code_pt lcode; - static ngx_str_t zero = ngx_string("0"); + u_char buffer[NGX_OFF_T_LEN]; - content_length = r->headers_in.content_length ? - &r->headers_in.content_length->value : &zero; + content_length_n = 0; + body = r->upstream->request_bufs; - len = sizeof("CONTENT_LENGTH") + content_length->len + 1; + while (body) { + content_length_n += ngx_buf_size(body->buf); + body = body->next; + } + + content_length.data = buffer; + content_length.len = ngx_sprintf(buffer, "%O", content_length_n) - buffer; + + len = sizeof("CONTENT_LENGTH") + content_length.len + 1; header_params = 0; ignored = NULL; @@ -672,11 +681,8 @@ ngx_http_scgi_create_request(ngx_http_re cl->buf = b; - b->last = ngx_snprintf(b->last, - NGX_SIZE_T_LEN + 1 + sizeof("CONTENT_LENGTH") - + NGX_OFF_T_LEN + 1, - "%ui:CONTENT_LENGTH%Z%V%Z", - len, content_length); + b->last = ngx_sprintf(b->last, "%ui:CONTENT_LENGTH%Z%V%Z", + len, &content_length); if (scf->params_len) { ngx_memzero(&e, sizeof(ngx_http_script_engine_t)); # HG changeset patch # User Maxim Dounin # Date 1352393278 -14400 # Node ID 789afefe02c68a1abad5cafc9b15d7f3def4e15f # Parent 64edd6c168a11d150f11425f98482ec66d2e2805 Request body: chunked transfer encoding support. diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -848,7 +848,8 @@ ngx_http_handler(ngx_http_request_t *r) break; } - r->lingering_close = (r->headers_in.content_length_n > 0); + r->lingering_close = (r->headers_in.content_length_n > 0 + || r->headers_in.chunked); r->phase_handler = 0; } else { diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -1574,19 +1574,11 @@ ngx_http_process_request_header(ngx_http if (r->headers_in.content_length_n == NGX_ERROR) { ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, "client sent invalid \"Content-Length\" header"); - ngx_http_finalize_request(r, NGX_HTTP_LENGTH_REQUIRED); + ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); return NGX_ERROR; } } - if (r->method & NGX_HTTP_PUT && r->headers_in.content_length_n == -1) { - ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, - "client sent %V method without \"Content-Length\" header", - &r->method_name); - ngx_http_finalize_request(r, NGX_HTTP_LENGTH_REQUIRED); - return NGX_ERROR; - } - if (r->method & NGX_HTTP_TRACE) { ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, "client sent TRACE method"); @@ -1594,14 +1586,25 @@ ngx_http_process_request_header(ngx_http return NGX_ERROR; } - if (r->headers_in.transfer_encoding - && ngx_strcasestrn(r->headers_in.transfer_encoding->value.data, - "chunked", 7 - 1)) - { - ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, - "client sent \"Transfer-Encoding: chunked\" header"); - ngx_http_finalize_request(r, NGX_HTTP_LENGTH_REQUIRED); - return NGX_ERROR; + if (r->headers_in.transfer_encoding) { + if (r->headers_in.transfer_encoding->value.len == 7 + && ngx_strncasecmp(r->headers_in.transfer_encoding->value.data, + (u_char *) "chunked", 7) == 0) + { + r->headers_in.content_length = NULL; + r->headers_in.content_length_n = -1; + r->headers_in.chunked = 1; + + } else if (r->headers_in.transfer_encoding->value.len != 8 + || ngx_strncasecmp(r->headers_in.transfer_encoding->value.data, + (u_char *) "identity", 8) != 0) + { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent unknown \"Transfer-Encoding\": \"%V\"", + &r->headers_in.transfer_encoding->value); + ngx_http_finalize_request(r, NGX_HTTP_NOT_IMPLEMENTED); + return NGX_ERROR; + } } if (r->headers_in.connection_type == NGX_HTTP_CONNECTION_KEEP_ALIVE) { diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h --- a/src/http/ngx_http_request.h +++ b/src/http/ngx_http_request.h @@ -224,6 +224,7 @@ typedef struct { time_t keep_alive_n; unsigned connection_type:2; + unsigned chunked:1; unsigned msie:1; unsigned msie6:1; unsigned opera:1; @@ -276,7 +277,9 @@ typedef struct { ngx_chain_t *bufs; ngx_buf_t *buf; off_t rest; - ngx_chain_t *to_write; + ngx_chain_t *free; + ngx_chain_t *busy; + ngx_http_chunked_t *chunked; ngx_http_client_body_handler_pt post_handler; } ngx_http_request_body_t; diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c --- a/src/http/ngx_http_request_body.c +++ b/src/http/ngx_http_request_body.c @@ -12,18 +12,21 @@ static void ngx_http_read_client_request_body_handler(ngx_http_request_t *r); static ngx_int_t ngx_http_do_read_client_request_body(ngx_http_request_t *r); -static ngx_int_t ngx_http_write_request_body(ngx_http_request_t *r, - ngx_chain_t *body); +static ngx_int_t ngx_http_write_request_body(ngx_http_request_t *r); static ngx_int_t ngx_http_read_discarded_request_body(ngx_http_request_t *r); +static ngx_int_t ngx_http_discard_request_body_filter(ngx_http_request_t *r, + ngx_buf_t *b); static ngx_int_t ngx_http_test_expect(ngx_http_request_t *r); +static ngx_int_t ngx_http_request_body_filter(ngx_http_request_t *r, + ngx_chain_t *in); +static ngx_int_t ngx_http_request_body_length_filter(ngx_http_request_t *r, + ngx_chain_t *in); +static ngx_int_t ngx_http_request_body_save_filter(ngx_http_request_t *r, + ngx_chain_t *in); +static ngx_int_t ngx_http_request_body_chunked_filter(ngx_http_request_t *r, + ngx_chain_t *in); -/* - * on completion ngx_http_read_client_request_body() adds to - * r->request_body->bufs one or two bufs: - * *) one memory buf that was preread in r->header_in; - * *) one memory or file buf that contains the rest of the body - */ ngx_int_t ngx_http_read_client_request_body(ngx_http_request_t *r, @@ -32,8 +35,7 @@ ngx_http_read_client_request_body(ngx_ht size_t preread; ssize_t size; ngx_int_t rc; - ngx_buf_t *b; - ngx_chain_t *cl, **next; + ngx_chain_t out; ngx_http_request_body_t *rb; ngx_http_core_loc_conf_t *clcf; @@ -55,19 +57,73 @@ ngx_http_read_client_request_body(ngx_ht goto done; } + /* + * set by ngx_pcalloc(): + * + * rb->bufs = NULL; + * rb->buf = NULL; + * rb->free = NULL; + * rb->busy = NULL; + * rb->chunked = NULL; + */ + + rb->rest = -1; + rb->post_handler = post_handler; + r->request_body = rb; - if (r->headers_in.content_length_n < 0) { + if (r->headers_in.content_length_n < 0 && !r->headers_in.chunked) { post_handler(r); return NGX_OK; } - clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + preread = r->header_in->last - r->header_in->pos; - if (r->headers_in.content_length_n == 0) { + if (preread) { + + /* there is the pre-read part of the request body */ + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http client request body preread %uz", preread); + + out.buf = r->header_in; + out.next = NULL; + + rc = ngx_http_request_body_filter(r, &out); + + if (rc != NGX_OK) { + goto done; + } + + r->request_length += preread - (r->header_in->last - r->header_in->pos); + + if (!r->headers_in.chunked + && rb->rest > 0 + && rb->rest <= (off_t) (r->header_in->end - r->header_in->last)) + { + /* the whole request body may be placed in r->header_in */ + + rb->buf = r->header_in; + r->read_event_handler = ngx_http_read_client_request_body_handler; + + rc = ngx_http_do_read_client_request_body(r); + goto done; + } + + } else { + /* set rb->rest */ + + if (ngx_http_request_body_filter(r, NULL) != NGX_OK) { + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; + goto done; + } + } + + if (rb->rest == 0) { + /* the whole request body was pre-read */ if (r->request_body_in_file_only) { - if (ngx_http_write_request_body(r, NULL) != NGX_OK) { + if (ngx_http_write_request_body(r) != NGX_OK) { rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } @@ -78,102 +134,14 @@ ngx_http_read_client_request_body(ngx_ht return NGX_OK; } - rb->post_handler = post_handler; - - /* - * set by ngx_pcalloc(): - * - * rb->bufs = NULL; - * rb->buf = NULL; - * rb->rest = 0; - */ - - preread = r->header_in->last - r->header_in->pos; - - if (preread) { - - /* there is the pre-read part of the request body */ - - ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "http client request body preread %uz", preread); - - b = ngx_calloc_buf(r->pool); - if (b == NULL) { - rc = NGX_HTTP_INTERNAL_SERVER_ERROR; - goto done; - } - - b->temporary = 1; - b->start = r->header_in->pos; - b->pos = r->header_in->pos; - b->last = r->header_in->last; - b->end = r->header_in->end; - - rb->bufs = ngx_alloc_chain_link(r->pool); - if (rb->bufs == NULL) { - rc = NGX_HTTP_INTERNAL_SERVER_ERROR; - goto done; - } - - rb->bufs->buf = b; - rb->bufs->next = NULL; - - rb->buf = b; - - if ((off_t) preread >= r->headers_in.content_length_n) { - - /* the whole request body was pre-read */ - - r->header_in->pos += (size_t) r->headers_in.content_length_n; - r->request_length += r->headers_in.content_length_n; - b->last = r->header_in->pos; - - if (r->request_body_in_file_only) { - if (ngx_http_write_request_body(r, rb->bufs) != NGX_OK) { - rc = NGX_HTTP_INTERNAL_SERVER_ERROR; - goto done; - } - } - - post_handler(r); - - return NGX_OK; - } - - /* - * to not consider the body as pipelined request in - * ngx_http_set_keepalive() - */ - r->header_in->pos = r->header_in->last; - - r->request_length += preread; - - rb->rest = r->headers_in.content_length_n - preread; - - if (rb->rest <= (off_t) (b->end - b->last)) { - - /* the whole request body may be placed in r->header_in */ - - rb->to_write = rb->bufs; - - r->read_event_handler = ngx_http_read_client_request_body_handler; - - rc = ngx_http_do_read_client_request_body(r); - goto done; - } - - next = &rb->bufs->next; - - } else { - b = NULL; - rb->rest = r->headers_in.content_length_n; - next = &rb->bufs; - } + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); size = clcf->client_body_buffer_size; size += size >> 2; - if (rb->rest < size) { + /* TODO: honor r->request_body_in_single_buf */ + + if (!r->headers_in.chunked && rb->rest < size) { size = (ssize_t) rb->rest; if (r->request_body_in_single_buf) { @@ -182,9 +150,6 @@ ngx_http_read_client_request_body(ngx_ht } else { size = clcf->client_body_buffer_size; - - /* disable copying buffer for r->request_body_in_single_buf */ - b = NULL; } rb->buf = ngx_create_temp_buf(r->pool, size); @@ -193,32 +158,6 @@ ngx_http_read_client_request_body(ngx_ht goto done; } - cl = ngx_alloc_chain_link(r->pool); - if (cl == NULL) { - rc = NGX_HTTP_INTERNAL_SERVER_ERROR; - goto done; - } - - cl->buf = rb->buf; - cl->next = NULL; - - if (b && r->request_body_in_single_buf) { - size = b->last - b->pos; - ngx_memcpy(rb->buf->pos, b->pos, size); - rb->buf->last += size; - - next = &rb->bufs; - } - - *next = cl; - - if (r->request_body_in_file_only || r->request_body_in_single_buf) { - rb->to_write = rb->bufs; - - } else { - rb->to_write = rb->bufs->next ? rb->bufs->next : rb->bufs; - } - r->read_event_handler = ngx_http_read_client_request_body_handler; rc = ngx_http_do_read_client_request_body(r); @@ -255,9 +194,12 @@ ngx_http_read_client_request_body_handle static ngx_int_t ngx_http_do_read_client_request_body(ngx_http_request_t *r) { + off_t rest; size_t size; ssize_t n; + ngx_int_t rc; ngx_buf_t *b; + ngx_chain_t *cl, out; ngx_connection_t *c; ngx_http_request_body_t *rb; ngx_http_core_loc_conf_t *clcf; @@ -272,18 +214,44 @@ ngx_http_do_read_client_request_body(ngx for ( ;; ) { if (rb->buf->last == rb->buf->end) { - if (ngx_http_write_request_body(r, rb->to_write) != NGX_OK) { + /* pass buffer to request body filter chain */ + + out.buf = rb->buf; + out.next = NULL; + + rc = ngx_http_request_body_filter(r, &out); + + if (rc != NGX_OK) { + return rc; + } + + /* write to file */ + + if (ngx_http_write_request_body(r) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } - rb->to_write = rb->bufs->next ? rb->bufs->next : rb->bufs; + /* update chains */ + + rc = ngx_http_request_body_filter(r, NULL); + + if (rc != NGX_OK) { + return rc; + } + + if (rb->busy != NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + rb->buf->pos = rb->buf->start; rb->buf->last = rb->buf->start; } size = rb->buf->end - rb->buf->last; + rest = rb->rest - (rb->buf->last - rb->buf->pos); - if ((off_t) size > rb->rest) { - size = (size_t) rb->rest; + if ((off_t) size > rest) { + size = (size_t) rest; } n = c->recv(c, rb->buf->last, size); @@ -306,9 +274,21 @@ ngx_http_do_read_client_request_body(ngx } rb->buf->last += n; - rb->rest -= n; r->request_length += n; + if (n == rest) { + /* pass buffer to request body filter chain */ + + out.buf = rb->buf; + out.next = NULL; + + rc = ngx_http_request_body_filter(r, &out); + + if (rc != NGX_OK) { + return rc; + } + } + if (rb->rest == 0) { break; } @@ -345,32 +325,24 @@ ngx_http_do_read_client_request_body(ngx /* save the last part */ - if (ngx_http_write_request_body(r, rb->to_write) != NGX_OK) { + if (ngx_http_write_request_body(r) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } - b = ngx_calloc_buf(r->pool); - if (b == NULL) { + cl = ngx_chain_get_free_buf(r->pool, &rb->free); + if (cl == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } + b = cl->buf; + + ngx_memzero(b, sizeof(ngx_buf_t)); + b->in_file = 1; - b->file_pos = 0; b->file_last = rb->temp_file->file.offset; b->file = &rb->temp_file->file; - if (rb->bufs->next) { - rb->bufs->next->buf = b; - - } else { - rb->bufs->buf = b; - } - } - - if (rb->bufs->next - && (r->request_body_in_file_only || r->request_body_in_single_buf)) - { - rb->bufs = rb->bufs->next; + rb->bufs = cl; } r->read_event_handler = ngx_http_block_reading; @@ -382,15 +354,19 @@ ngx_http_do_read_client_request_body(ngx static ngx_int_t -ngx_http_write_request_body(ngx_http_request_t *r, ngx_chain_t *body) +ngx_http_write_request_body(ngx_http_request_t *r) { ssize_t n; + ngx_chain_t *cl; ngx_temp_file_t *tf; ngx_http_request_body_t *rb; ngx_http_core_loc_conf_t *clcf; rb = r->request_body; + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http write client request body, bufs %p", rb->bufs); + if (rb->temp_file == NULL) { tf = ngx_pcalloc(r->pool, sizeof(ngx_temp_file_t)); if (tf == NULL) { @@ -414,7 +390,7 @@ ngx_http_write_request_body(ngx_http_req rb->temp_file = tf; - if (body == NULL) { + if (rb->bufs == NULL) { /* empty body with r->request_body_in_file_only */ if (ngx_create_temp_file(&tf->file, tf->path, tf->pool, @@ -428,7 +404,11 @@ ngx_http_write_request_body(ngx_http_req } } - n = ngx_write_chain_to_temp_file(rb->temp_file, body); + if (rb->bufs == NULL) { + return NGX_OK; + } + + n = ngx_write_chain_to_temp_file(rb->temp_file, rb->bufs); /* TODO: n == 0 or not complete and level event */ @@ -438,6 +418,14 @@ ngx_http_write_request_body(ngx_http_req rb->temp_file->offset += n; + /* mark all buffers as written */ + + for (cl = rb->bufs; cl; cl = cl->next) { + cl->buf->pos = cl->buf->last; + } + + rb->bufs = NULL; + return NGX_OK; } @@ -446,9 +434,10 @@ ngx_int_t ngx_http_discard_request_body(ngx_http_request_t *r) { ssize_t size; + ngx_int_t rc; ngx_event_t *rev; - if (r != r->main || r->discard_body) { + if (r != r->main || r->discard_body || r->request_body) { return NGX_OK; } @@ -464,20 +453,20 @@ ngx_http_discard_request_body(ngx_http_r ngx_del_timer(rev); } - if (r->headers_in.content_length_n <= 0 || r->request_body) { + if (r->headers_in.content_length_n <= 0 && !r->headers_in.chunked) { return NGX_OK; } size = r->header_in->last - r->header_in->pos; if (size) { - if (r->headers_in.content_length_n > size) { - r->header_in->pos += size; - r->headers_in.content_length_n -= size; + rc = ngx_http_discard_request_body_filter(r, r->header_in); - } else { - r->header_in->pos += (size_t) r->headers_in.content_length_n; - r->headers_in.content_length_n = 0; + if (rc != NGX_OK) { + return rc; + } + + if (r->headers_in.content_length_n == 0) { return NGX_OK; } } @@ -568,13 +557,19 @@ ngx_http_discarded_request_body_handler( static ngx_int_t ngx_http_read_discarded_request_body(ngx_http_request_t *r) { - size_t size; - ssize_t n; - u_char buffer[NGX_HTTP_DISCARD_BUFFER_SIZE]; + size_t size; + ssize_t n; + ngx_int_t rc; + ngx_buf_t b; + u_char buffer[NGX_HTTP_DISCARD_BUFFER_SIZE]; ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http read discarded body"); + ngx_memzero(&b, sizeof(ngx_buf_t)); + + b.temporary = 1; + for ( ;; ) { if (r->headers_in.content_length_n == 0) { r->read_event_handler = ngx_http_block_reading; @@ -585,9 +580,8 @@ ngx_http_read_discarded_request_body(ngx return NGX_AGAIN; } - size = (r->headers_in.content_length_n > NGX_HTTP_DISCARD_BUFFER_SIZE) ? - NGX_HTTP_DISCARD_BUFFER_SIZE: - (size_t) r->headers_in.content_length_n; + size = (size_t) ngx_min(r->headers_in.content_length_n, + NGX_HTTP_DISCARD_BUFFER_SIZE); n = r->connection->recv(r->connection, buffer, size); @@ -604,12 +598,109 @@ ngx_http_read_discarded_request_body(ngx return NGX_OK; } - r->headers_in.content_length_n -= n; + b.pos = buffer; + b.last = buffer + n; + + rc = ngx_http_discard_request_body_filter(r, &b); + + if (rc != NGX_OK) { + r->connection->error = 1; + return NGX_OK; + } } } static ngx_int_t +ngx_http_discard_request_body_filter(ngx_http_request_t *r, ngx_buf_t *b) +{ + size_t size; + ngx_int_t rc; + ngx_http_request_body_t *rb; + + if (r->headers_in.chunked) { + + rb = r->request_body; + + if (rb == NULL) { + + rb = ngx_pcalloc(r->pool, sizeof(ngx_http_request_body_t)); + if (rb == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + rb->chunked = ngx_pcalloc(r->pool, sizeof(ngx_http_chunked_t)); + if (rb == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + r->request_body = rb; + } + + for ( ;; ) { + + rc = ngx_http_parse_chunked(r, b, rb->chunked); + + if (rc == NGX_OK) { + + /* a chunk has been parsed successfully */ + + size = b->last - b->pos; + + if ((off_t) size > rb->chunked->size) { + b->pos += rb->chunked->size; + rb->chunked->size = 0; + + } else { + rb->chunked->size -= size; + b->pos = b->last; + } + + continue; + } + + if (rc == NGX_DONE) { + + /* a whole response has been parsed successfully */ + + r->headers_in.content_length_n = 0; + break; + } + + if (rc == NGX_AGAIN) { + + /* set amount of data we want to see next time */ + + r->headers_in.content_length_n = rb->chunked->length; + break; + } + + /* invalid */ + + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "client sent invalid chunked body"); + + return NGX_HTTP_BAD_REQUEST; + } + + } else { + size = b->last - b->pos; + + if ((off_t) size > r->headers_in.content_length_n) { + b->pos += r->headers_in.content_length_n; + r->headers_in.content_length_n = 0; + + } else { + b->pos = b->last; + r->headers_in.content_length_n -= size; + } + } + + return NGX_OK; +} + + +static ngx_int_t ngx_http_test_expect(ngx_http_request_t *r) { ngx_int_t n; @@ -649,3 +740,274 @@ ngx_http_test_expect(ngx_http_request_t return NGX_ERROR; } + + +static ngx_int_t +ngx_http_request_body_filter(ngx_http_request_t *r, ngx_chain_t *in) +{ + if (r->headers_in.chunked) { + return ngx_http_request_body_chunked_filter(r, in); + + } else { + return ngx_http_request_body_length_filter(r, in); + } +} + + +static ngx_int_t +ngx_http_request_body_length_filter(ngx_http_request_t *r, ngx_chain_t *in) +{ + size_t size; + ngx_int_t rc; + ngx_buf_t *b; + ngx_chain_t *cl, *tl, *out, **ll; + ngx_http_request_body_t *rb; + + rb = r->request_body; + + if (rb->rest == -1) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http request body content length filter"); + + rb->rest = r->headers_in.content_length_n; + } + + out = NULL; + ll = &out; + + for (cl = in; cl; cl = cl->next) { + + tl = ngx_chain_get_free_buf(r->pool, &rb->free); + if (tl == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + b = tl->buf; + + ngx_memzero(b, sizeof(ngx_buf_t)); + + b->temporary = 1; + b->tag = (ngx_buf_tag_t) &ngx_http_read_client_request_body; + b->start = cl->buf->start; + b->pos = cl->buf->pos; + b->last = cl->buf->last; + b->end = cl->buf->end; + + size = cl->buf->last - cl->buf->pos; + + if ((off_t) size < rb->rest) { + cl->buf->pos = cl->buf->last; + rb->rest -= size; + + } else { + cl->buf->pos += rb->rest; + rb->rest = 0; + b->last = cl->buf->pos; + b->last_buf = 1; + } + + *ll = tl; + ll = &tl->next; + } + + rc = ngx_http_request_body_save_filter(r, out); + + ngx_chain_update_chains(r->pool, &rb->free, &rb->busy, &out, + (ngx_buf_tag_t) &ngx_http_read_client_request_body); + + return rc; +} + + +static ngx_int_t +ngx_http_request_body_save_filter(ngx_http_request_t *r, ngx_chain_t *in) +{ + ngx_chain_t *cl; + ngx_http_request_body_t *rb; + + rb = r->request_body; + +#if (NGX_DEBUG) + + for (cl = rb->bufs; cl; cl = cl->next) { + ngx_log_debug7(NGX_LOG_DEBUG_EVENT, r->connection->log, 0, + "http body old buf t:%d f:%d %p, pos %p, size: %z " + "file: %O, size: %z", + cl->buf->temporary, cl->buf->in_file, + cl->buf->start, cl->buf->pos, + cl->buf->last - cl->buf->pos, + cl->buf->file_pos, + cl->buf->file_last - cl->buf->file_pos); + } + + for (cl = in; cl; cl = cl->next) { + ngx_log_debug7(NGX_LOG_DEBUG_EVENT, r->connection->log, 0, + "http body new buf t:%d f:%d %p, pos %p, size: %z " + "file: %O, size: %z", + cl->buf->temporary, cl->buf->in_file, + cl->buf->start, cl->buf->pos, + cl->buf->last - cl->buf->pos, + cl->buf->file_pos, + cl->buf->file_last - cl->buf->file_pos); + } + +#endif + + /* TODO: coalesce neighbouring buffers */ + + ngx_chain_add_copy(r->pool, &rb->bufs, in); + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_request_body_chunked_filter(ngx_http_request_t *r, ngx_chain_t *in) +{ + size_t size; + ngx_int_t rc; + ngx_buf_t *b; + ngx_chain_t *cl, *out, *tl, **ll; + ngx_http_request_body_t *rb; + ngx_http_core_loc_conf_t *clcf; + + rb = r->request_body; + + if (rb->rest == -1) { + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http request body chunked filter"); + + rb->chunked = ngx_pcalloc(r->pool, sizeof(ngx_http_chunked_t)); + if (rb->chunked == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + r->headers_in.content_length_n = 0; + rb->rest = 3; + } + + out = NULL; + ll = &out; + + for (cl = in; cl; cl = cl->next) { + + for ( ;; ) { + + ngx_log_debug7(NGX_LOG_DEBUG_EVENT, r->connection->log, 0, + "http body chunked buf " + "t:%d f:%d %p, pos %p, size: %z file: %O, size: %z", + cl->buf->temporary, cl->buf->in_file, + cl->buf->start, cl->buf->pos, + cl->buf->last - cl->buf->pos, + cl->buf->file_pos, + cl->buf->file_last - cl->buf->file_pos); + + rc = ngx_http_parse_chunked(r, cl->buf, rb->chunked); + + if (rc == NGX_OK) { + + /* a chunk has been parsed successfully */ + + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + + if (clcf->client_max_body_size + && clcf->client_max_body_size + < r->headers_in.content_length_n + rb->chunked->size) + { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "client intended to send too large chunked " + "body: %O bytes", + r->headers_in.content_length_n + + rb->chunked->size); + + r->lingering_close = 1; + + return NGX_HTTP_REQUEST_ENTITY_TOO_LARGE; + } + + tl = ngx_chain_get_free_buf(r->pool, &rb->free); + if (tl == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + b = tl->buf; + + ngx_memzero(b, sizeof(ngx_buf_t)); + + b->temporary = 1; + b->tag = (ngx_buf_tag_t) &ngx_http_read_client_request_body; + b->start = cl->buf->start; + b->pos = cl->buf->pos; + b->last = cl->buf->last; + b->end = cl->buf->end; + + *ll = tl; + ll = &tl->next; + + size = cl->buf->last - cl->buf->pos; + + if ((off_t) size > rb->chunked->size) { + cl->buf->pos += rb->chunked->size; + r->headers_in.content_length_n += rb->chunked->size; + rb->chunked->size = 0; + + } else { + rb->chunked->size -= size; + r->headers_in.content_length_n += size; + cl->buf->pos = cl->buf->last; + } + + b->last = cl->buf->pos; + + continue; + } + + if (rc == NGX_DONE) { + + /* a whole response has been parsed successfully */ + + rb->rest = 0; + + tl = ngx_chain_get_free_buf(r->pool, &rb->free); + if (tl == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + b = tl->buf; + + ngx_memzero(b, sizeof(ngx_buf_t)); + + b->last_buf = 1; + + *ll = tl; + ll = &tl->next; + + break; + } + + if (rc == NGX_AGAIN) { + + /* set rb->rest, amount of data we want to see next time */ + + rb->rest = rb->chunked->length; + + break; + } + + /* invalid */ + + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "client sent invalid chunked body"); + + return NGX_HTTP_BAD_REQUEST; + } + } + + rc = ngx_http_request_body_save_filter(r, out); + + ngx_chain_update_chains(r->pool, &rb->free, &rb->busy, &out, + (ngx_buf_tag_t) &ngx_http_read_client_request_body); + + return rc; +}