From f3915c33e756f958d9f20641350febf202c0c24d Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 23 Nov 2021 12:52:43 +0300 Subject: [PATCH 1/6] Version bump. --- src/core/nginx.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/nginx.h b/src/core/nginx.h index bc4af23af..5ad6aff45 100644 --- a/src/core/nginx.h +++ b/src/core/nginx.h @@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1021004 -#define NGINX_VERSION "1.21.4" +#define nginx_version 1021005 +#define NGINX_VERSION "1.21.5" #define NGINX_VER "nginx/" NGINX_VERSION #ifdef NGX_BUILD From 5c3249964403356601e64ab701f2e563a1f98630 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 1 Nov 2021 18:09:34 +0300 Subject: [PATCH 2/6] SSL: $ssl_curve (ticket #2135). The variable contains a negotiated curve used for the handshake key exchange process. Known curves are listed by their names, unknown ones are shown in hex. Note that for resumed sessions in TLSv1.2 and older protocols, $ssl_curve contains the curve used during the initial handshake, while in TLSv1.3 it contains the curve used during the session resumption (see the SSL_get_negotiated_group manual page for details). The variable is only meaningful when using OpenSSL 3.0 and above. With older versions the variable is empty. --- src/event/ngx_event_openssl.c | 36 ++++++++++++++++++++++++++ src/event/ngx_event_openssl.h | 2 ++ src/http/modules/ngx_http_ssl_module.c | 3 +++ src/stream/ngx_stream_ssl_module.c | 3 +++ 4 files changed, 44 insertions(+) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 84afecd0b..b03ca59ae 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -4733,6 +4733,42 @@ ngx_ssl_get_ciphers(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s) } +ngx_int_t +ngx_ssl_get_curve(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s) +{ +#ifdef SSL_get_negotiated_group + + int nid; + + nid = SSL_get_negotiated_group(c->ssl->connection); + + if (nid != NID_undef) { + + if ((nid & TLSEXT_nid_unknown) == 0) { + s->len = ngx_strlen(OBJ_nid2sn(nid)); + s->data = (u_char *) OBJ_nid2sn(nid); + return NGX_OK; + } + + s->len = sizeof("0x0000") - 1; + + s->data = ngx_pnalloc(pool, s->len); + if (s->data == NULL) { + return NGX_ERROR; + } + + ngx_sprintf(s->data, "0x%04xd", nid & 0xffff); + + return NGX_OK; + } + +#endif + + s->len = 0; + return NGX_OK; +} + + ngx_int_t ngx_ssl_get_curves(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s) { diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index 4afdfad48..c9e86d9c0 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -256,6 +256,8 @@ ngx_int_t ngx_ssl_get_cipher_name(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s); ngx_int_t ngx_ssl_get_ciphers(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s); +ngx_int_t ngx_ssl_get_curve(ngx_connection_t *c, ngx_pool_t *pool, + ngx_str_t *s); ngx_int_t ngx_ssl_get_curves(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s); ngx_int_t ngx_ssl_get_session_id(ngx_connection_t *c, ngx_pool_t *pool, diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c index c633f3464..d74d46094 100644 --- a/src/http/modules/ngx_http_ssl_module.c +++ b/src/http/modules/ngx_http_ssl_module.c @@ -342,6 +342,9 @@ static ngx_http_variable_t ngx_http_ssl_vars[] = { { ngx_string("ssl_ciphers"), NULL, ngx_http_ssl_variable, (uintptr_t) ngx_ssl_get_ciphers, NGX_HTTP_VAR_CHANGEABLE, 0 }, + { ngx_string("ssl_curve"), NULL, ngx_http_ssl_variable, + (uintptr_t) ngx_ssl_get_curve, NGX_HTTP_VAR_CHANGEABLE, 0 }, + { ngx_string("ssl_curves"), NULL, ngx_http_ssl_variable, (uintptr_t) ngx_ssl_get_curves, NGX_HTTP_VAR_CHANGEABLE, 0 }, diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c index 530fe8b3d..c5308322a 100644 --- a/src/stream/ngx_stream_ssl_module.c +++ b/src/stream/ngx_stream_ssl_module.c @@ -269,6 +269,9 @@ static ngx_stream_variable_t ngx_stream_ssl_vars[] = { { ngx_string("ssl_ciphers"), NULL, ngx_stream_ssl_variable, (uintptr_t) ngx_ssl_get_ciphers, NGX_STREAM_VAR_CHANGEABLE, 0 }, + { ngx_string("ssl_curve"), NULL, ngx_stream_ssl_variable, + (uintptr_t) ngx_ssl_get_curve, NGX_STREAM_VAR_CHANGEABLE, 0 }, + { ngx_string("ssl_curves"), NULL, ngx_stream_ssl_variable, (uintptr_t) ngx_ssl_get_curves, NGX_STREAM_VAR_CHANGEABLE, 0 }, From 2361e98a34ca3b66c45fbb4e906401b430e41013 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Thu, 25 Nov 2021 22:02:05 +0300 Subject: [PATCH 3/6] HTTP/2: fixed "task already active" with sendfile in threads. With sendfile in threads, "task already active" alerts might appear in logs if a write event happens on the main HTTP/2 connection, triggering a sendfile in threads while another thread operation is already running. Observed with "aio threads; aio_write on; sendfile on;" and with thread event handlers modified to post a write event to the main HTTP/2 connection (though can happen without any modifications). Similarly, sendfile() with AIO preloading on FreeBSD can trigger duplicate aio operation, resulting in "second aio post" alerts. This is, however, harder to reproduce, especially on modern FreeBSD systems, since sendfile() usually does not return EBUSY. Fix is to avoid starting a sendfile operation if other thread operation is active by checking r->aio in the thread handler (and, similarly, in aio preload handler). The added check also makes duplicate calls protection redundant, so it is removed. --- src/http/ngx_http_copy_filter_module.c | 38 ++++++++++++++++++++++-- src/http/ngx_http_upstream.c | 22 ++++++++++++++ src/os/unix/ngx_freebsd_sendfile_chain.c | 13 -------- src/os/unix/ngx_linux_sendfile_chain.c | 9 ------ 4 files changed, 58 insertions(+), 24 deletions(-) diff --git a/src/http/ngx_http_copy_filter_module.c b/src/http/ngx_http_copy_filter_module.c index c8ad5daee..6c93a3bd9 100644 --- a/src/http/ngx_http_copy_filter_module.c +++ b/src/http/ngx_http_copy_filter_module.c @@ -219,13 +219,25 @@ ngx_http_copy_aio_sendfile_preload(ngx_buf_t *file) ngx_http_request_t *r; ngx_output_chain_ctx_t *ctx; + aio = file->file->aio; + r = aio->data; + + if (r->aio) { + /* + * tolerate sendfile() calls if another operation is already + * running; this can happen due to subrequests, multiple calls + * of the next body filter from a filter, or in HTTP/2 due to + * a write event on the main connection + */ + + return NGX_AGAIN; + } + n = ngx_file_aio_read(file->file, buf, 1, file->file_pos, NULL); if (n == NGX_AGAIN) { - aio = file->file->aio; aio->handler = ngx_http_copy_aio_sendfile_event_handler; - r = aio->data; r->main->blocked++; r->aio = 1; @@ -263,6 +275,7 @@ static ngx_int_t ngx_http_copy_thread_handler(ngx_thread_task_t *task, ngx_file_t *file) { ngx_str_t name; + ngx_connection_t *c; ngx_thread_pool_t *tp; ngx_http_request_t *r; ngx_output_chain_ctx_t *ctx; @@ -270,6 +283,27 @@ ngx_http_copy_thread_handler(ngx_thread_task_t *task, ngx_file_t *file) r = file->thread_ctx; + if (r->aio) { + /* + * tolerate sendfile() calls if another operation is already + * running; this can happen due to subrequests, multiple calls + * of the next body filter from a filter, or in HTTP/2 due to + * a write event on the main connection + */ + + c = r->connection; + +#if (NGX_HTTP_V2) + if (r->stream) { + c = r->stream->connection->connection; + } +#endif + + if (task == c->sendfile_task) { + return NGX_OK; + } + } + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); tp = clcf->thread_pool; diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index 002b3e65d..e22849342 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -3847,6 +3847,7 @@ ngx_http_upstream_thread_handler(ngx_thread_task_t *task, ngx_file_t *file) { ngx_str_t name; ngx_event_pipe_t *p; + ngx_connection_t *c; ngx_thread_pool_t *tp; ngx_http_request_t *r; ngx_http_core_loc_conf_t *clcf; @@ -3854,6 +3855,27 @@ ngx_http_upstream_thread_handler(ngx_thread_task_t *task, ngx_file_t *file) r = file->thread_ctx; p = r->upstream->pipe; + if (r->aio) { + /* + * tolerate sendfile() calls if another operation is already + * running; this can happen due to subrequests, multiple calls + * of the next body filter from a filter, or in HTTP/2 due to + * a write event on the main connection + */ + + c = r->connection; + +#if (NGX_HTTP_V2) + if (r->stream) { + c = r->stream->connection->connection; + } +#endif + + if (task == c->sendfile_task) { + return NGX_OK; + } + } + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); tp = clcf->thread_pool; diff --git a/src/os/unix/ngx_freebsd_sendfile_chain.c b/src/os/unix/ngx_freebsd_sendfile_chain.c index 3d415bd2c..750c12fc3 100644 --- a/src/os/unix/ngx_freebsd_sendfile_chain.c +++ b/src/os/unix/ngx_freebsd_sendfile_chain.c @@ -255,19 +255,6 @@ ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) #if (NGX_HAVE_AIO_SENDFILE) if (ebusy) { - if (aio->event.active) { - /* - * tolerate duplicate calls; they can happen due to subrequests - * or multiple calls of the next body filter from a filter - */ - - if (sent) { - c->busy_count = 0; - } - - return in; - } - if (sent == 0) { c->busy_count++; diff --git a/src/os/unix/ngx_linux_sendfile_chain.c b/src/os/unix/ngx_linux_sendfile_chain.c index 91e7f1d93..101d91a9a 100644 --- a/src/os/unix/ngx_linux_sendfile_chain.c +++ b/src/os/unix/ngx_linux_sendfile_chain.c @@ -379,15 +379,6 @@ ngx_linux_sendfile_thread(ngx_connection_t *c, ngx_buf_t *file, size_t size) return ctx->sent; } - if (task->event.active && ctx->file == file) { - /* - * tolerate duplicate calls; they can happen due to subrequests - * or multiple calls of the next body filter from a filter - */ - - return NGX_DONE; - } - ctx->file = file; ctx->socket = c->fd; ctx->size = size; From 83e92a2edd6bf7c6867b653284ac44962c4e33c9 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Thu, 25 Nov 2021 22:02:10 +0300 Subject: [PATCH 4/6] HTTP/2: fixed sendfile() aio handling. With sendfile() in threads ("aio threads; sendfile on;"), client connection can block on writing, waiting for sendfile() to complete. In HTTP/2 this might result in the request hang, since an attempt to continue processing in thread event handler will call request's write event handler, which is usually stopped by ngx_http_v2_send_chain(): it does nothing if there are no additional data and stream->queued is set. Further, HTTP/2 resets stream's c->write->ready to 0 if writing blocks, so just fixing ngx_http_v2_send_chain() is not enough. Can be reproduced with test suite on Linux with: TEST_NGINX_GLOBALS_HTTP="aio threads; sendfile on;" prove h2*.t The following tests currently fail: h2_keepalive.t, h2_priority.t, h2_proxy_max_temp_file_size.t, h2.t, h2_trailers.t. Similarly, sendfile() with AIO preloading on FreeBSD can block as well, with similar results. This is, however, harder to reproduce, especially on modern FreeBSD systems, since sendfile() usually does not return EBUSY. Fix is to modify ngx_http_v2_send_chain() so it actually tries to send data to the main connection when called, and to make sure that c->write->ready is set by the relevant event handlers. --- src/http/ngx_http_copy_filter_module.c | 32 ++++++++++++++++++++++++- src/http/ngx_http_upstream.c | 14 +++++++++++ src/http/v2/ngx_http_v2_filter_module.c | 29 +++++++++++++++------- 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/src/http/ngx_http_copy_filter_module.c b/src/http/ngx_http_copy_filter_module.c index 6c93a3bd9..b47e4afa7 100644 --- a/src/http/ngx_http_copy_filter_module.c +++ b/src/http/ngx_http_copy_filter_module.c @@ -253,16 +253,32 @@ static void ngx_http_copy_aio_sendfile_event_handler(ngx_event_t *ev) { ngx_event_aio_t *aio; + ngx_connection_t *c; ngx_http_request_t *r; aio = ev->data; r = aio->data; + c = r->connection; r->main->blocked--; r->aio = 0; ev->complete = 0; - r->connection->write->handler(r->connection->write); +#if (NGX_HTTP_V2) + + if (r->stream) { + /* + * for HTTP/2, update write event to make sure processing will + * reach the main connection to handle sendfile() preload + */ + + c->write->ready = 1; + c->write->active = 0; + } + +#endif + + c->write->handler(c->write); } #endif @@ -357,6 +373,20 @@ ngx_http_copy_thread_event_handler(ngx_event_t *ev) r->main->blocked--; r->aio = 0; +#if (NGX_HTTP_V2) + + if (r->stream) { + /* + * for HTTP/2, update write event to make sure processing will + * reach the main connection to handle sendfile() in threads + */ + + c->write->ready = 1; + c->write->active = 0; + } + +#endif + if (r->done) { /* * trigger connection event handler if the subrequest was diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index e22849342..ded833c41 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -3927,6 +3927,20 @@ ngx_http_upstream_thread_event_handler(ngx_event_t *ev) r->main->blocked--; r->aio = 0; +#if (NGX_HTTP_V2) + + if (r->stream) { + /* + * for HTTP/2, update write event to make sure processing will + * reach the main connection to handle sendfile() in threads + */ + + c->write->ready = 1; + c->write->active = 0; + } + +#endif + if (r->done) { /* * trigger connection event handler if the subrequest was diff --git a/src/http/v2/ngx_http_v2_filter_module.c b/src/http/v2/ngx_http_v2_filter_module.c index a6e5e7d4f..9ffb155df 100644 --- a/src/http/v2/ngx_http_v2_filter_module.c +++ b/src/http/v2/ngx_http_v2_filter_module.c @@ -1432,6 +1432,9 @@ ngx_http_v2_send_chain(ngx_connection_t *fc, ngx_chain_t *in, off_t limit) size = 0; #endif + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, + "http2 send chain: %p", in); + while (in) { size = ngx_buf_size(in->buf); @@ -1450,12 +1453,8 @@ ngx_http_v2_send_chain(ngx_connection_t *fc, ngx_chain_t *in, off_t limit) return NGX_CHAIN_ERROR; } - if (stream->queued) { - fc->write->active = 1; - fc->write->ready = 0; - - } else { - fc->buffered &= ~NGX_HTTP_V2_BUFFERED; + if (ngx_http_v2_filter_send(fc, stream) == NGX_ERROR) { + return NGX_CHAIN_ERROR; } return NULL; @@ -1464,9 +1463,16 @@ ngx_http_v2_send_chain(ngx_connection_t *fc, ngx_chain_t *in, off_t limit) h2c = stream->connection; if (size && ngx_http_v2_flow_control(h2c, stream) == NGX_DECLINED) { - fc->write->active = 1; - fc->write->ready = 0; - return in; + + if (ngx_http_v2_filter_send(fc, stream) == NGX_ERROR) { + return NGX_CHAIN_ERROR; + } + + if (ngx_http_v2_flow_control(h2c, stream) == NGX_DECLINED) { + fc->write->active = 1; + fc->write->ready = 0; + return in; + } } if (in->buf->tag == (ngx_buf_tag_t) &ngx_http_v2_filter_get_shadow) { @@ -1809,6 +1815,11 @@ ngx_http_v2_waiting_queue(ngx_http_v2_connection_t *h2c, static ngx_inline ngx_int_t ngx_http_v2_filter_send(ngx_connection_t *fc, ngx_http_v2_stream_t *stream) { + if (stream->queued == 0) { + fc->buffered &= ~NGX_HTTP_V2_BUFFERED; + return NGX_OK; + } + stream->blocked = 1; if (ngx_http_v2_send_output_queue(stream->connection) == NGX_ERROR) { From b5d022e79756daf0ca2a5221a9e1dddb910d13da Mon Sep 17 00:00:00 2001 From: Gena Makhomed Date: Mon, 20 Dec 2021 20:02:48 +0200 Subject: [PATCH 5/6] Contrib: vim syntax, update core and 3rd party module directives. --- contrib/vim/syntax/nginx.vim | 40 +++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/contrib/vim/syntax/nginx.vim b/contrib/vim/syntax/nginx.vim index 4907e2a54..1b52d4869 100644 --- a/contrib/vim/syntax/nginx.vim +++ b/contrib/vim/syntax/nginx.vim @@ -152,6 +152,7 @@ syn keyword ngxDirective contained auth_jwt_header_set syn keyword ngxDirective contained auth_jwt_key_file syn keyword ngxDirective contained auth_jwt_key_request syn keyword ngxDirective contained auth_jwt_leeway +syn keyword ngxDirective contained auth_jwt_require syn keyword ngxDirective contained auth_jwt_type syn keyword ngxDirective contained auth_request syn keyword ngxDirective contained auth_request_set @@ -335,6 +336,10 @@ syn keyword ngxDirective contained ip_hash syn keyword ngxDirective contained js_access syn keyword ngxDirective contained js_body_filter syn keyword ngxDirective contained js_content +syn keyword ngxDirective contained js_fetch_ciphers +syn keyword ngxDirective contained js_fetch_protocols +syn keyword ngxDirective contained js_fetch_trusted_certificate +syn keyword ngxDirective contained js_fetch_verify_depth syn keyword ngxDirective contained js_filter syn keyword ngxDirective contained js_header_filter syn keyword ngxDirective contained js_import @@ -402,6 +407,7 @@ syn keyword ngxDirective contained mp4_buffer_size syn keyword ngxDirective contained mp4_limit_rate syn keyword ngxDirective contained mp4_limit_rate_after syn keyword ngxDirective contained mp4_max_buffer_size +syn keyword ngxDirective contained mp4_start_key_frame syn keyword ngxDirective contained msie_padding syn keyword ngxDirective contained msie_refresh syn keyword ngxDirective contained multi_accept @@ -458,6 +464,7 @@ syn keyword ngxDirective contained proxy_cookie_flags syn keyword ngxDirective contained proxy_cookie_path syn keyword ngxDirective contained proxy_download_rate syn keyword ngxDirective contained proxy_force_ranges +syn keyword ngxDirective contained proxy_half_close syn keyword ngxDirective contained proxy_headers_hash_bucket_size syn keyword ngxDirective contained proxy_headers_hash_max_size syn keyword ngxDirective contained proxy_hide_header @@ -597,6 +604,7 @@ syn keyword ngxDirective contained ssi_min_file_chunk syn keyword ngxDirective contained ssi_silent_errors syn keyword ngxDirective contained ssi_types syn keyword ngxDirective contained ssi_value_length +syn keyword ngxDirective contained ssl_alpn syn keyword ngxDirective contained ssl_buffer_size syn keyword ngxDirective contained ssl_certificate syn keyword ngxDirective contained ssl_certificate_key @@ -788,11 +796,15 @@ syn keyword ngxDirectiveThirdParty contained auth_digest_user_file syn keyword ngxDirectiveThirdParty contained auth_gss syn keyword ngxDirectiveThirdParty contained auth_gss_allow_basic_fallback syn keyword ngxDirectiveThirdParty contained auth_gss_authorized_principal +syn keyword ngxDirectiveThirdParty contained auth_gss_authorized_principal_regex +syn keyword ngxDirectiveThirdParty contained auth_gss_constrained_delegation +syn keyword ngxDirectiveThirdParty contained auth_gss_delegate_credentials syn keyword ngxDirectiveThirdParty contained auth_gss_force_realm syn keyword ngxDirectiveThirdParty contained auth_gss_format_full syn keyword ngxDirectiveThirdParty contained auth_gss_keytab syn keyword ngxDirectiveThirdParty contained auth_gss_map_to_local syn keyword ngxDirectiveThirdParty contained auth_gss_realm +syn keyword ngxDirectiveThirdParty contained auth_gss_service_ccache syn keyword ngxDirectiveThirdParty contained auth_gss_service_name " LDAP Authentication @@ -969,7 +981,6 @@ syn keyword ngxDirectiveThirdParty contained fancyindex_hide_parent_dir syn keyword ngxDirectiveThirdParty contained fancyindex_hide_symlinks syn keyword ngxDirectiveThirdParty contained fancyindex_ignore syn keyword ngxDirectiveThirdParty contained fancyindex_localtime -syn keyword ngxDirectiveThirdParty contained fancyindex_name_length syn keyword ngxDirectiveThirdParty contained fancyindex_show_dotfiles syn keyword ngxDirectiveThirdParty contained fancyindex_show_path syn keyword ngxDirectiveThirdParty contained fancyindex_time_format @@ -1059,7 +1070,9 @@ syn keyword ngxDirectiveThirdParty contained nchan_publisher_upstream_request syn keyword ngxDirectiveThirdParty contained nchan_pubsub syn keyword ngxDirectiveThirdParty contained nchan_pubsub_channel_id syn keyword ngxDirectiveThirdParty contained nchan_pubsub_location +syn keyword ngxDirectiveThirdParty contained nchan_redis_cluster_check_interval syn keyword ngxDirectiveThirdParty contained nchan_redis_connect_timeout +syn keyword ngxDirectiveThirdParty contained nchan_redis_discovered_ip_range_blacklist syn keyword ngxDirectiveThirdParty contained nchan_redis_fakesub_timer_interval syn keyword ngxDirectiveThirdParty contained nchan_redis_idle_channel_cache_timeout syn keyword ngxDirectiveThirdParty contained nchan_redis_namespace @@ -1067,12 +1080,29 @@ syn keyword ngxDirectiveThirdParty contained nchan_redis_nostore_fastpublish syn keyword ngxDirectiveThirdParty contained nchan_redis_optimize_target syn keyword ngxDirectiveThirdParty contained nchan_redis_pass syn keyword ngxDirectiveThirdParty contained nchan_redis_pass_inheritable +syn keyword ngxDirectiveThirdParty contained nchan_redis_password syn keyword ngxDirectiveThirdParty contained nchan_redis_ping_interval syn keyword ngxDirectiveThirdParty contained nchan_redis_publish_msgpacked_max_size syn keyword ngxDirectiveThirdParty contained nchan_redis_server +syn keyword ngxDirectiveThirdParty contained nchan_redis_ssl +syn keyword ngxDirectiveThirdParty contained nchan_redis_ssl_ciphers +syn keyword ngxDirectiveThirdParty contained nchan_redis_ssl_client_certificate +syn keyword ngxDirectiveThirdParty contained nchan_redis_ssl_client_certificate_key +syn keyword ngxDirectiveThirdParty contained nchan_redis_ssl_server_name +syn keyword ngxDirectiveThirdParty contained nchan_redis_ssl_trusted_certificate +syn keyword ngxDirectiveThirdParty contained nchan_redis_ssl_trusted_certificate_path +syn keyword ngxDirectiveThirdParty contained nchan_redis_ssl_verify_certificate syn keyword ngxDirectiveThirdParty contained nchan_redis_storage_mode syn keyword ngxDirectiveThirdParty contained nchan_redis_subscribe_weights +syn keyword ngxDirectiveThirdParty contained nchan_redis_tls +syn keyword ngxDirectiveThirdParty contained nchan_redis_tls_ciphers +syn keyword ngxDirectiveThirdParty contained nchan_redis_tls_client_certificate +syn keyword ngxDirectiveThirdParty contained nchan_redis_tls_server_name +syn keyword ngxDirectiveThirdParty contained nchan_redis_tls_trusted_certificate +syn keyword ngxDirectiveThirdParty contained nchan_redis_tls_trusted_certificate_path +syn keyword ngxDirectiveThirdParty contained nchan_redis_tls_verify_certificate syn keyword ngxDirectiveThirdParty contained nchan_redis_url +syn keyword ngxDirectiveThirdParty contained nchan_redis_username syn keyword ngxDirectiveThirdParty contained nchan_redis_wait_after_connecting syn keyword ngxDirectiveThirdParty contained nchan_shared_memory_size syn keyword ngxDirectiveThirdParty contained nchan_storage_engine @@ -1385,6 +1415,7 @@ syn keyword ngxDirectiveThirdParty contained lua_socket_read_timeout syn keyword ngxDirectiveThirdParty contained lua_socket_send_lowat syn keyword ngxDirectiveThirdParty contained lua_socket_send_timeout syn keyword ngxDirectiveThirdParty contained lua_ssl_ciphers +syn keyword ngxDirectiveThirdParty contained lua_ssl_conf_command syn keyword ngxDirectiveThirdParty contained lua_ssl_crl syn keyword ngxDirectiveThirdParty contained lua_ssl_protocols syn keyword ngxDirectiveThirdParty contained lua_ssl_trusted_certificate @@ -1392,6 +1423,7 @@ syn keyword ngxDirectiveThirdParty contained lua_ssl_verify_depth syn keyword ngxDirectiveThirdParty contained lua_thread_cache_max_entries syn keyword ngxDirectiveThirdParty contained lua_transform_underscores_in_response_headers syn keyword ngxDirectiveThirdParty contained lua_use_default_type +syn keyword ngxDirectiveThirdParty contained lua_worker_thread_vm_pool_size syn keyword ngxDirectiveThirdParty contained rewrite_by_lua syn keyword ngxDirectiveThirdParty contained rewrite_by_lua_block syn keyword ngxDirectiveThirdParty contained rewrite_by_lua_file @@ -1401,6 +1433,8 @@ syn keyword ngxDirectiveThirdParty contained set_by_lua_block syn keyword ngxDirectiveThirdParty contained set_by_lua_file syn keyword ngxDirectiveThirdParty contained ssl_certificate_by_lua_block syn keyword ngxDirectiveThirdParty contained ssl_certificate_by_lua_file +syn keyword ngxDirectiveThirdParty contained ssl_client_hello_by_lua_block +syn keyword ngxDirectiveThirdParty contained ssl_client_hello_by_lua_file syn keyword ngxDirectiveThirdParty contained ssl_session_fetch_by_lua_block syn keyword ngxDirectiveThirdParty contained ssl_session_fetch_by_lua_file syn keyword ngxDirectiveThirdParty contained ssl_session_store_by_lua_block @@ -1719,15 +1753,18 @@ syn keyword ngxDirectiveThirdParty contained set_base32_alphabet syn keyword ngxDirectiveThirdParty contained set_base32_padding syn keyword ngxDirectiveThirdParty contained set_decode_base32 syn keyword ngxDirectiveThirdParty contained set_decode_base64 +syn keyword ngxDirectiveThirdParty contained set_decode_base64url syn keyword ngxDirectiveThirdParty contained set_decode_hex syn keyword ngxDirectiveThirdParty contained set_encode_base32 syn keyword ngxDirectiveThirdParty contained set_encode_base64 +syn keyword ngxDirectiveThirdParty contained set_encode_base64url syn keyword ngxDirectiveThirdParty contained set_encode_hex syn keyword ngxDirectiveThirdParty contained set_escape_uri syn keyword ngxDirectiveThirdParty contained set_formatted_gmt_time syn keyword ngxDirectiveThirdParty contained set_formatted_local_time syn keyword ngxDirectiveThirdParty contained set_hashed_upstream syn keyword ngxDirectiveThirdParty contained set_hmac_sha1 +syn keyword ngxDirectiveThirdParty contained set_hmac_sha256 syn keyword ngxDirectiveThirdParty contained set_if_empty syn keyword ngxDirectiveThirdParty contained set_local_today syn keyword ngxDirectiveThirdParty contained set_misc_base32_padding @@ -1849,6 +1886,7 @@ syn keyword ngxDirectiveThirdParty contained vod_notification_uri syn keyword ngxDirectiveThirdParty contained vod_open_file_thread_pool syn keyword ngxDirectiveThirdParty contained vod_output_buffer_pool syn keyword ngxDirectiveThirdParty contained vod_parse_hdlr_name +syn keyword ngxDirectiveThirdParty contained vod_parse_udta_name syn keyword ngxDirectiveThirdParty contained vod_path_response_postfix syn keyword ngxDirectiveThirdParty contained vod_path_response_prefix syn keyword ngxDirectiveThirdParty contained vod_performance_counters From 363505e806feebb7ceb1f9edb0e3f75c1253384f Mon Sep 17 00:00:00 2001 From: Ruslan Ermilov Date: Tue, 21 Dec 2021 07:54:16 +0300 Subject: [PATCH 6/6] Moved Huffman coding out of HTTP/2. ngx_http_v2_huff_decode.c and ngx_http_v2_huff_encode.c are renamed to ngx_http_huff_decode.c and ngx_http_huff_encode.c. --- auto/modules | 7 +++-- auto/sources | 3 ++ src/http/modules/ngx_http_grpc_module.c | 16 +++++----- src/http/ngx_http.h | 8 +++++ ...2_huff_decode.c => ngx_http_huff_decode.c} | 18 +++++------ ...2_huff_encode.c => ngx_http_huff_encode.c} | 30 +++++++++---------- src/http/v2/ngx_http_v2.c | 8 ++--- src/http/v2/ngx_http_v2.h | 6 ---- src/http/v2/ngx_http_v2_encode.c | 2 +- 9 files changed, 53 insertions(+), 45 deletions(-) rename src/http/{v2/ngx_http_v2_huff_decode.c => ngx_http_huff_decode.c} (99%) rename src/http/{v2/ngx_http_v2_huff_encode.c => ngx_http_huff_encode.c} (93%) diff --git a/auto/modules b/auto/modules index f5a459783..94867bfc0 100644 --- a/auto/modules +++ b/auto/modules @@ -102,6 +102,11 @@ if [ $HTTP = YES ]; then fi + if [ $HTTP_V2 = YES ]; then + HTTP_SRCS="$HTTP_SRCS $HTTP_HUFF_SRCS" + fi + + # the module order is important # ngx_http_static_module # ngx_http_gzip_static_module @@ -414,8 +419,6 @@ if [ $HTTP = YES ]; then ngx_module_srcs="src/http/v2/ngx_http_v2.c \ src/http/v2/ngx_http_v2_table.c \ src/http/v2/ngx_http_v2_encode.c \ - src/http/v2/ngx_http_v2_huff_decode.c \ - src/http/v2/ngx_http_v2_huff_encode.c \ src/http/v2/ngx_http_v2_module.c" ngx_module_libs= ngx_module_link=$HTTP_V2 diff --git a/auto/sources b/auto/sources index 3dad11132..156f7979e 100644 --- a/auto/sources +++ b/auto/sources @@ -255,3 +255,6 @@ NGX_WIN32_RC="src/os/win32/nginx.rc" HTTP_FILE_CACHE_SRCS=src/http/ngx_http_file_cache.c + +HTTP_HUFF_SRCS="src/http/ngx_http_huff_decode.c + src/http/ngx_http_huff_encode.c" diff --git a/src/http/modules/ngx_http_grpc_module.c b/src/http/modules/ngx_http_grpc_module.c index 6842b7c6e..864fc4fda 100644 --- a/src/http/modules/ngx_http_grpc_module.c +++ b/src/http/modules/ngx_http_grpc_module.c @@ -3180,10 +3180,10 @@ ngx_http_grpc_parse_fragment(ngx_http_request_t *r, ngx_http_grpc_ctx_t *ctx, ctx->field_rest -= size; if (ctx->field_huffman) { - if (ngx_http_v2_huff_decode(&ctx->field_state, p, size, - &ctx->field_end, - ctx->field_rest == 0, - r->connection->log) + if (ngx_http_huff_decode(&ctx->field_state, p, size, + &ctx->field_end, + ctx->field_rest == 0, + r->connection->log) != NGX_OK) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, @@ -3289,10 +3289,10 @@ ngx_http_grpc_parse_fragment(ngx_http_request_t *r, ngx_http_grpc_ctx_t *ctx, ctx->field_rest -= size; if (ctx->field_huffman) { - if (ngx_http_v2_huff_decode(&ctx->field_state, p, size, - &ctx->field_end, - ctx->field_rest == 0, - r->connection->log) + if (ngx_http_huff_decode(&ctx->field_state, p, size, + &ctx->field_end, + ctx->field_rest == 0, + r->connection->log) != NGX_OK) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, diff --git a/src/http/ngx_http.h b/src/http/ngx_http.h index 8b43857ee..be8b7cd71 100644 --- a/src/http/ngx_http.h +++ b/src/http/ngx_http.h @@ -167,6 +167,14 @@ ngx_uint_t ngx_http_degraded(ngx_http_request_t *); #endif +#if (NGX_HTTP_V2) +ngx_int_t ngx_http_huff_decode(u_char *state, u_char *src, size_t len, + u_char **dst, ngx_uint_t last, ngx_log_t *log); +size_t ngx_http_huff_encode(u_char *src, size_t len, u_char *dst, + ngx_uint_t lower); +#endif + + extern ngx_module_t ngx_http_module; extern ngx_str_t ngx_http_html_default_types[]; diff --git a/src/http/v2/ngx_http_v2_huff_decode.c b/src/http/ngx_http_huff_decode.c similarity index 99% rename from src/http/v2/ngx_http_v2_huff_decode.c rename to src/http/ngx_http_huff_decode.c index 49ca576f7..14b7b7896 100644 --- a/src/http/v2/ngx_http_v2_huff_decode.c +++ b/src/http/ngx_http_huff_decode.c @@ -15,14 +15,14 @@ typedef struct { u_char emit; u_char sym; u_char ending; -} ngx_http_v2_huff_decode_code_t; +} ngx_http_huff_decode_code_t; -static ngx_inline ngx_int_t ngx_http_v2_huff_decode_bits(u_char *state, +static ngx_inline ngx_int_t ngx_http_huff_decode_bits(u_char *state, u_char *ending, ngx_uint_t bits, u_char **dst); -static ngx_http_v2_huff_decode_code_t ngx_http_v2_huff_decode_codes[256][16] = +static ngx_http_huff_decode_code_t ngx_http_huff_decode_codes[256][16] = { /* 0 */ { @@ -2640,7 +2640,7 @@ static ngx_http_v2_huff_decode_code_t ngx_http_v2_huff_decode_codes[256][16] = ngx_int_t -ngx_http_v2_huff_decode(u_char *state, u_char *src, size_t len, u_char **dst, +ngx_http_huff_decode(u_char *state, u_char *src, size_t len, u_char **dst, ngx_uint_t last, ngx_log_t *log) { u_char *end, ch, ending; @@ -2653,7 +2653,7 @@ ngx_http_v2_huff_decode(u_char *state, u_char *src, size_t len, u_char **dst, while (src != end) { ch = *src++; - if (ngx_http_v2_huff_decode_bits(state, &ending, ch >> 4, dst) + if (ngx_http_huff_decode_bits(state, &ending, ch >> 4, dst) != NGX_OK) { ngx_log_debug2(NGX_LOG_DEBUG_HTTP, log, 0, @@ -2663,7 +2663,7 @@ ngx_http_v2_huff_decode(u_char *state, u_char *src, size_t len, u_char **dst, return NGX_ERROR; } - if (ngx_http_v2_huff_decode_bits(state, &ending, ch & 0xf, dst) + if (ngx_http_huff_decode_bits(state, &ending, ch & 0xf, dst) != NGX_OK) { ngx_log_debug2(NGX_LOG_DEBUG_HTTP, log, 0, @@ -2692,12 +2692,12 @@ ngx_http_v2_huff_decode(u_char *state, u_char *src, size_t len, u_char **dst, static ngx_inline ngx_int_t -ngx_http_v2_huff_decode_bits(u_char *state, u_char *ending, ngx_uint_t bits, +ngx_http_huff_decode_bits(u_char *state, u_char *ending, ngx_uint_t bits, u_char **dst) { - ngx_http_v2_huff_decode_code_t code; + ngx_http_huff_decode_code_t code; - code = ngx_http_v2_huff_decode_codes[*state][bits]; + code = ngx_http_huff_decode_codes[*state][bits]; if (code.next == *state) { return NGX_ERROR; diff --git a/src/http/v2/ngx_http_v2_huff_encode.c b/src/http/ngx_http_huff_encode.c similarity index 93% rename from src/http/v2/ngx_http_v2_huff_encode.c rename to src/http/ngx_http_huff_encode.c index 3f822cd0b..c03b153da 100644 --- a/src/http/v2/ngx_http_v2_huff_encode.c +++ b/src/http/ngx_http_huff_encode.c @@ -14,10 +14,10 @@ typedef struct { uint32_t code; uint32_t len; -} ngx_http_v2_huff_encode_code_t; +} ngx_http_huff_encode_code_t; -static ngx_http_v2_huff_encode_code_t ngx_http_v2_huff_encode_table[256] = +static ngx_http_huff_encode_code_t ngx_http_huff_encode_table[256] = { {0x00001ff8, 13}, {0x007fffd8, 23}, {0x0fffffe2, 28}, {0x0fffffe3, 28}, {0x0fffffe4, 28}, {0x0fffffe5, 28}, {0x0fffffe6, 28}, {0x0fffffe7, 28}, @@ -87,7 +87,7 @@ static ngx_http_v2_huff_encode_code_t ngx_http_v2_huff_encode_table[256] = /* same as above, but embeds lowercase transformation */ -static ngx_http_v2_huff_encode_code_t ngx_http_v2_huff_encode_table_lc[256] = +static ngx_http_huff_encode_code_t ngx_http_huff_encode_table_lc[256] = { {0x00001ff8, 13}, {0x007fffd8, 23}, {0x0fffffe2, 28}, {0x0fffffe3, 28}, {0x0fffffe4, 28}, {0x0fffffe5, 28}, {0x0fffffe6, 28}, {0x0fffffe7, 28}, @@ -161,10 +161,10 @@ static ngx_http_v2_huff_encode_code_t ngx_http_v2_huff_encode_table_lc[256] = #if (NGX_HAVE_LITTLE_ENDIAN) #if (NGX_HAVE_GCC_BSWAP64) -#define ngx_http_v2_huff_encode_buf(dst, buf) \ +#define ngx_http_huff_encode_buf(dst, buf) \ (*(uint64_t *) (dst) = __builtin_bswap64(buf)) #else -#define ngx_http_v2_huff_encode_buf(dst, buf) \ +#define ngx_http_huff_encode_buf(dst, buf) \ ((dst)[0] = (u_char) ((buf) >> 56), \ (dst)[1] = (u_char) ((buf) >> 48), \ (dst)[2] = (u_char) ((buf) >> 40), \ @@ -176,28 +176,28 @@ static ngx_http_v2_huff_encode_code_t ngx_http_v2_huff_encode_table_lc[256] = #endif #else /* !NGX_HAVE_LITTLE_ENDIAN */ -#define ngx_http_v2_huff_encode_buf(dst, buf) \ +#define ngx_http_huff_encode_buf(dst, buf) \ (*(uint64_t *) (dst) = (buf)) #endif #else /* NGX_PTR_SIZE == 4 */ -#define ngx_http_v2_huff_encode_buf(dst, buf) \ +#define ngx_http_huff_encode_buf(dst, buf) \ (*(uint32_t *) (dst) = htonl(buf)) #endif size_t -ngx_http_v2_huff_encode(u_char *src, size_t len, u_char *dst, ngx_uint_t lower) +ngx_http_huff_encode(u_char *src, size_t len, u_char *dst, ngx_uint_t lower) { - u_char *end; - size_t hlen; - ngx_uint_t buf, pending, code; - ngx_http_v2_huff_encode_code_t *table, *next; + u_char *end; + size_t hlen; + ngx_uint_t buf, pending, code; + ngx_http_huff_encode_code_t *table, *next; - table = lower ? ngx_http_v2_huff_encode_table_lc - : ngx_http_v2_huff_encode_table; + table = lower ? ngx_http_huff_encode_table_lc + : ngx_http_huff_encode_table; hlen = 0; buf = 0; pending = 0; @@ -224,7 +224,7 @@ ngx_http_v2_huff_encode(u_char *src, size_t len, u_char *dst, ngx_uint_t lower) buf |= code >> pending; - ngx_http_v2_huff_encode_buf(&dst[hlen], buf); + ngx_http_huff_encode_buf(&dst[hlen], buf); hlen += sizeof(buf); diff --git a/src/http/v2/ngx_http_v2.c b/src/http/v2/ngx_http_v2.c index 3afa8b638..0e45a7b27 100644 --- a/src/http/v2/ngx_http_v2.c +++ b/src/http/v2/ngx_http_v2.c @@ -1600,10 +1600,10 @@ ngx_http_v2_state_field_huff(ngx_http_v2_connection_t *h2c, u_char *pos, h2c->state.length -= size; h2c->state.field_rest -= size; - if (ngx_http_v2_huff_decode(&h2c->state.field_state, pos, size, - &h2c->state.field_end, - h2c->state.field_rest == 0, - h2c->connection->log) + if (ngx_http_huff_decode(&h2c->state.field_state, pos, size, + &h2c->state.field_end, + h2c->state.field_rest == 0, + h2c->connection->log) != NGX_OK) { ngx_log_error(NGX_LOG_INFO, h2c->connection->log, 0, diff --git a/src/http/v2/ngx_http_v2.h b/src/http/v2/ngx_http_v2.h index 0eceae3d5..70ee287ae 100644 --- a/src/http/v2/ngx_http_v2.h +++ b/src/http/v2/ngx_http_v2.h @@ -311,12 +311,6 @@ ngx_int_t ngx_http_v2_add_header(ngx_http_v2_connection_t *h2c, ngx_int_t ngx_http_v2_table_size(ngx_http_v2_connection_t *h2c, size_t size); -ngx_int_t ngx_http_v2_huff_decode(u_char *state, u_char *src, size_t len, - u_char **dst, ngx_uint_t last, ngx_log_t *log); -size_t ngx_http_v2_huff_encode(u_char *src, size_t len, u_char *dst, - ngx_uint_t lower); - - #define ngx_http_v2_prefix(bits) ((1 << (bits)) - 1) diff --git a/src/http/v2/ngx_http_v2_encode.c b/src/http/v2/ngx_http_v2_encode.c index ac792084e..8798aa9ae 100644 --- a/src/http/v2/ngx_http_v2_encode.c +++ b/src/http/v2/ngx_http_v2_encode.c @@ -20,7 +20,7 @@ ngx_http_v2_string_encode(u_char *dst, u_char *src, size_t len, u_char *tmp, { size_t hlen; - hlen = ngx_http_v2_huff_encode(src, len, tmp, lower); + hlen = ngx_http_huff_encode(src, len, tmp, lower); if (hlen > 0) { *dst = NGX_HTTP_V2_ENCODE_HUFF;