SNI: using ClientHello callback.

The change introduces a SNI based virtual server selection with
the ClientHello callback used for early server-side ClientHello
processing.  It reverts 46b9f5d38.

The ClientHello callback is available in OpenSSL version 1.1.1+,
where the TLSv1.3 protocol support was introduced.  For other
versions, the servername callback is preserved.

Notably, this follows BoringSSL behaviour to process server name
before SSL session resumption.

src/http/modules/ngx_http_ssl_module.c

@@ -737,7 +737,13 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child)
     cln->handler = ngx_ssl_cleanup_ctx;
     cln->data = &conf->ssl;
 
-#ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME
+#ifdef SSL_CLIENT_HELLO_SUCCESS
+
+    /* OpenSSL 1.1.1+ */
+
+    SSL_CTX_set_client_hello_cb(conf->ssl.ctx, ngx_http_ssl_servername, NULL);
+
+#elif defined SSL_CTRL_SET_TLSEXT_HOSTNAME
 
     if (SSL_CTX_set_tlsext_servername_callback(conf->ssl.ctx,
                                                ngx_http_ssl_servername)

src/http/ngx_http.h

@@ -90,7 +90,8 @@ ngx_int_t ngx_http_add_listen(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf,
 void ngx_http_init_connection(ngx_connection_t *c);
 void ngx_http_close_connection(ngx_connection_t *c);
 
-#if (NGX_HTTP_SSL && defined SSL_CTRL_SET_TLSEXT_HOSTNAME)
+#if (NGX_HTTP_SSL && (defined SSL_CLIENT_HELLO_SUCCESS                        \
+                      || defined SSL_CTRL_SET_TLSEXT_HOSTNAME))
 int ngx_http_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg);
 #endif
 #if (NGX_HTTP_SSL && defined SSL_R_CERT_CB_ERROR)

src/http/ngx_http_request.c

@@ -869,48 +869,92 @@ ngx_http_ssl_handshake_handler(ngx_connection_t *c)
 }
 
 
-#ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME
+#if (defined SSL_CLIENT_HELLO_SUCCESS || defined SSL_CTRL_SET_TLSEXT_HOSTNAME)
 
 int
 ngx_http_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg)
 {
-    ngx_int_t                  rc;
+    ngx_int_t                  rc, success, error;
     ngx_str_t                  host;
-    const char                *servername;
+    const char                *p;
     ngx_connection_t          *c;
     ngx_http_connection_t     *hc;
     ngx_http_ssl_srv_conf_t   *sscf;
     ngx_http_core_loc_conf_t  *clcf;
     ngx_http_core_srv_conf_t  *cscf;
 
+#ifdef SSL_CLIENT_HELLO_SUCCESS
+    success = SSL_CLIENT_HELLO_SUCCESS;
+    error = SSL_CLIENT_HELLO_ERROR;
+#else
+    success = SSL_TLSEXT_ERR_OK;
+    error = SSL_TLSEXT_ERR_ALERT_FATAL;
+#endif
+
     c = ngx_ssl_get_connection(ssl_conn);
 
     if (c->ssl->handshaked) {
         *ad = SSL_AD_NO_RENEGOTIATION;
-        return SSL_TLSEXT_ERR_ALERT_FATAL;
+        return error;
     }
 
     hc = c->data;
 
-    servername = SSL_get_servername(ssl_conn, TLSEXT_NAMETYPE_host_name);
+#ifdef SSL_CLIENT_HELLO_SUCCESS
 
-    if (servername == NULL) {
+    if (SSL_client_hello_get0_ext(ssl_conn, TLSEXT_TYPE_server_name,
+                                  (const unsigned char **) &p, &host.len)
+        == 0)
+    {
         ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
                        "SSL server name: null");
         goto done;
     }
 
-    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0,
+    /* RFC 6066 mandates non-zero HostName length, we follow OpenSSL */
-                   "SSL server name: \"%s\"", servername);
 
-    host.len = ngx_strlen(servername);
+    if (host.len < 5
+        || (size_t) (p[0] << 8) + p[1] + 2 != host.len
+        || p[2] != TLSEXT_NAMETYPE_host_name
+        || (size_t) (p[3] << 8) + p[4] + 2 + 3 != host.len)
+    {
+        *ad = SSL_AD_DECODE_ERROR;
+        return error;
+    }
+
+    host.len -= 5;
+    host.data = (u_char *) p + 5;
+
+    if (host.len > TLSEXT_MAXLEN_host_name
+        || ngx_strlchr(host.data, host.data + host.len, '\0'))
+    {
+        c->ssl->handshake_rejected = 1;
+        *ad = SSL_AD_UNRECOGNIZED_NAME;
+        return error;
+    }
+
+#else
+
+    p = SSL_get_servername(ssl_conn, TLSEXT_NAMETYPE_host_name);
+
+    if (p == NULL) {
+        ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
+                       "SSL server name: null");
+        goto done;
+    }
+
+    host.len = ngx_strlen(p);
+    host.data = (u_char *) p;
+
+#endif
+
+    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0,
+                   "SSL server name: \"%V\"", &host);
 
     if (host.len == 0) {
         goto done;
     }
 
-    host.data = (u_char *) servername;
-
     rc = ngx_http_validate_host(&host, c->pool, 1);
 
     if (rc == NGX_ERROR) {
@@ -932,31 +976,6 @@ ngx_http_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg)
         goto done;
     }
 
-    sscf = ngx_http_get_module_srv_conf(cscf->ctx, ngx_http_ssl_module);
-
-#if (defined TLS1_3_VERSION                                                   \
-     && !defined LIBRESSL_VERSION_NUMBER && !defined OPENSSL_IS_BORINGSSL)
-
-    /*
-     * SSL_SESSION_get0_hostname() is only available in OpenSSL 1.1.1+,
-     * but servername being negotiated in every TLSv1.3 handshake
-     * is only returned in OpenSSL 1.1.1+ as well
-     */
-
-    if (sscf->verify) {
-        const char  *hostname;
-
-        hostname = SSL_SESSION_get0_hostname(SSL_get0_session(ssl_conn));
-
-        if (hostname != NULL && ngx_strcmp(hostname, servername) != 0) {
-            c->ssl->handshake_rejected = 1;
-            *ad = SSL_AD_ACCESS_DENIED;
-            return SSL_TLSEXT_ERR_ALERT_FATAL;
-        }
-    }
-
-#endif
-
     hc->ssl_servername = ngx_palloc(c->pool, sizeof(ngx_str_t));
     if (hc->ssl_servername == NULL) {
         goto error;
@@ -970,6 +989,8 @@ ngx_http_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg)
 
     ngx_set_connection_log(c, clcf->error_log);
 
+    sscf = ngx_http_get_module_srv_conf(cscf->ctx, ngx_http_ssl_module);
+
     c->ssl->buffer_size = sscf->buffer_size;
 
     if (sscf->ssl.ctx) {
@@ -1015,15 +1036,15 @@ done:
     if (sscf->reject_handshake) {
         c->ssl->handshake_rejected = 1;
         *ad = SSL_AD_UNRECOGNIZED_NAME;
-        return SSL_TLSEXT_ERR_ALERT_FATAL;
+        return error;
     }
 
-    return SSL_TLSEXT_ERR_OK;
+    return success;
 
 error:
 
     *ad = SSL_AD_INTERNAL_ERROR;
-    return SSL_TLSEXT_ERR_ALERT_FATAL;
+    return error;
 }
 
 #endif

src/stream/ngx_stream_ssl_module.c

@@ -22,7 +22,7 @@ static ngx_int_t ngx_stream_ssl_handler(ngx_stream_session_t *s);
 static ngx_int_t ngx_stream_ssl_init_connection(ngx_ssl_t *ssl,
     ngx_connection_t *c);
 static void ngx_stream_ssl_handshake_handler(ngx_connection_t *c);
-#ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME
+#if (defined SSL_CLIENT_HELLO_SUCCESS || defined SSL_CTRL_SET_TLSEXT_HOSTNAME)
 static int ngx_stream_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad,
     void *arg);
 #endif
@@ -528,47 +528,91 @@ ngx_stream_ssl_handshake_handler(ngx_connection_t *c)
 }
 
 
-#ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME
+#if (defined SSL_CLIENT_HELLO_SUCCESS || defined SSL_CTRL_SET_TLSEXT_HOSTNAME)
 
 static int
 ngx_stream_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg)
 {
-    ngx_int_t                    rc;
+    ngx_int_t                    rc, success, error;
     ngx_str_t                    host;
-    const char                  *servername;
+    const char                  *p;
     ngx_connection_t            *c;
     ngx_stream_session_t        *s;
     ngx_stream_ssl_srv_conf_t   *sscf;
     ngx_stream_core_srv_conf_t  *cscf;
 
+#ifdef SSL_CLIENT_HELLO_SUCCESS
+    success = SSL_CLIENT_HELLO_SUCCESS;
+    error = SSL_CLIENT_HELLO_ERROR;
+#else
+    success = SSL_TLSEXT_ERR_OK;
+    error = SSL_TLSEXT_ERR_ALERT_FATAL;
+#endif
+
     c = ngx_ssl_get_connection(ssl_conn);
 
     if (c->ssl->handshaked) {
         *ad = SSL_AD_NO_RENEGOTIATION;
-        return SSL_TLSEXT_ERR_ALERT_FATAL;
+        return error;
     }
 
     s = c->data;
 
-    servername = SSL_get_servername(ssl_conn, TLSEXT_NAMETYPE_host_name);
+#ifdef SSL_CLIENT_HELLO_SUCCESS
 
-    if (servername == NULL) {
+    if (SSL_client_hello_get0_ext(ssl_conn, TLSEXT_TYPE_server_name,
+                                  (const unsigned char **) &p, &host.len)
+        == 0)
+    {
         ngx_log_debug0(NGX_LOG_DEBUG_STREAM, c->log, 0,
                        "SSL server name: null");
         goto done;
     }
 
-    ngx_log_debug1(NGX_LOG_DEBUG_STREAM, c->log, 0,
+    /* RFC 6066 mandates non-zero HostName length, we follow OpenSSL */
-                   "SSL server name: \"%s\"", servername);
 
-    host.len = ngx_strlen(servername);
+    if (host.len < 5
+        || (size_t) (p[0] << 8) + p[1] + 2 != host.len
+        || p[2] != TLSEXT_NAMETYPE_host_name
+        || (size_t) (p[3] << 8) + p[4] + 2 + 3 != host.len)
+    {
+        *ad = SSL_AD_DECODE_ERROR;
+        return error;
+    }
+
+    host.len -= 5;
+    host.data = (u_char *) p + 5;
+
+    if (host.len > TLSEXT_MAXLEN_host_name
+        || ngx_strlchr(host.data, host.data + host.len, '\0'))
+    {
+        c->ssl->handshake_rejected = 1;
+        *ad = SSL_AD_UNRECOGNIZED_NAME;
+        return error;
+    }
+
+#else
+
+    p = SSL_get_servername(ssl_conn, TLSEXT_NAMETYPE_host_name);
+
+    if (p == NULL) {
+        ngx_log_debug0(NGX_LOG_DEBUG_STREAM, c->log, 0,
+                       "SSL server name: null");
+        goto done;
+    }
+
+    host.len = ngx_strlen(p);
+    host.data = (u_char *) p;
+
+#endif
+
+    ngx_log_debug1(NGX_LOG_DEBUG_STREAM, c->log, 0,
+                   "SSL server name: \"%V\"", &host);
 
     if (host.len == 0) {
         goto done;
     }
 
-    host.data = (u_char *) servername;
-
     rc = ngx_stream_validate_host(&host, c->pool, 1);
 
     if (rc == NGX_ERROR) {
@@ -589,35 +633,12 @@ ngx_stream_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg)
         goto done;
     }
 
-    sscf = ngx_stream_get_module_srv_conf(cscf->ctx, ngx_stream_ssl_module);
-
-#if (defined TLS1_3_VERSION                                                   \
-     && !defined LIBRESSL_VERSION_NUMBER && !defined OPENSSL_IS_BORINGSSL)
-
-    /*
-     * SSL_SESSION_get0_hostname() is only available in OpenSSL 1.1.1+,
-     * but servername being negotiated in every TLSv1.3 handshake
-     * is only returned in OpenSSL 1.1.1+ as well
-     */
-
-    if (sscf->verify) {
-        const char  *hostname;
-
-        hostname = SSL_SESSION_get0_hostname(SSL_get0_session(ssl_conn));
-
-        if (hostname != NULL && ngx_strcmp(hostname, servername) != 0) {
-            c->ssl->handshake_rejected = 1;
-            *ad = SSL_AD_ACCESS_DENIED;
-            return SSL_TLSEXT_ERR_ALERT_FATAL;
-        }
-    }
-
-#endif
-
     s->srv_conf = cscf->ctx->srv_conf;
 
     ngx_set_connection_log(c, cscf->error_log);
 
+    sscf = ngx_stream_get_module_srv_conf(cscf->ctx, ngx_stream_ssl_module);
+
     if (sscf->ssl.ctx) {
         if (SSL_set_SSL_CTX(ssl_conn, sscf->ssl.ctx) == NULL) {
             goto error;
@@ -653,15 +674,15 @@ done:
     if (sscf->reject_handshake) {
         c->ssl->handshake_rejected = 1;
         *ad = SSL_AD_UNRECOGNIZED_NAME;
-        return SSL_TLSEXT_ERR_ALERT_FATAL;
+        return error;
     }
 
-    return SSL_TLSEXT_ERR_OK;
+    return success;
 
 error:
 
     *ad = SSL_AD_INTERNAL_ERROR;
-    return SSL_TLSEXT_ERR_ALERT_FATAL;
+    return error;
 }
 
 #endif
@@ -990,7 +1011,13 @@ ngx_stream_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child)
     cln->handler = ngx_ssl_cleanup_ctx;
     cln->data = &conf->ssl;
 
-#ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME
+#ifdef SSL_CLIENT_HELLO_SUCCESS
+
+    /* OpenSSL 1.1.1+ */
+
+    SSL_CTX_set_client_hello_cb(conf->ssl.ctx, ngx_stream_ssl_servername, NULL);
+
+#elif defined SSL_CTRL_SET_TLSEXT_HOSTNAME
     SSL_CTX_set_tlsext_servername_callback(conf->ssl.ctx,
                                            ngx_stream_ssl_servername);
 #endif