SSL: support for TLSv1.3 early data with OpenSSL.

In collaboration with Maxim Dounin.

src/event/ngx_event_openssl.c

@@ -26,9 +26,23 @@ static void ngx_ssl_info_callback(const ngx_ssl_conn_t *ssl_conn, int where,
 static void ngx_ssl_passwords_cleanup(void *data);
 static int ngx_ssl_new_client_session(ngx_ssl_conn_t *ssl_conn,
     ngx_ssl_session_t *sess);
+#ifdef SSL_READ_EARLY_DATA_SUCCESS
+static ngx_int_t ngx_ssl_try_early_data(ngx_connection_t *c);
+#endif
+#if (NGX_DEBUG)
+static void ngx_ssl_handshake_log(ngx_connection_t *c);
+#endif
 static void ngx_ssl_handshake_handler(ngx_event_t *ev);
+#ifdef SSL_READ_EARLY_DATA_SUCCESS
+static ssize_t ngx_ssl_recv_early(ngx_connection_t *c, u_char *buf,
+    size_t size);
+#endif
 static ngx_int_t ngx_ssl_handle_recv(ngx_connection_t *c, int n);
 static void ngx_ssl_write_handler(ngx_event_t *wev);
+#ifdef SSL_READ_EARLY_DATA_SUCCESS
+static ssize_t ngx_ssl_write_early(ngx_connection_t *c, u_char *data,
+    size_t size);
+#endif
 static void ngx_ssl_read_handler(ngx_event_t *rev);
 static void ngx_ssl_shutdown_handler(ngx_event_t *ev);
 static void ngx_ssl_connection_error(ngx_connection_t *c, int sslerr,
@@ -340,6 +354,10 @@ ngx_ssl_create(ngx_ssl_t *ssl, ngx_uint_t protocols, void *data)
     SSL_CTX_set_options(ssl->ctx, SSL_OP_NO_COMPRESSION);
 #endif
 
+#ifdef SSL_OP_NO_ANTI_REPLAY
+    SSL_CTX_set_options(ssl->ctx, SSL_OP_NO_ANTI_REPLAY);
+#endif
+
 #ifdef SSL_MODE_RELEASE_BUFFERS
     SSL_CTX_set_mode(ssl->ctx, SSL_MODE_RELEASE_BUFFERS);
 #endif
@@ -1185,6 +1203,12 @@ ngx_ssl_early_data(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_uint_t enable)
 
     SSL_CTX_set_early_data_enabled(ssl->ctx, 1);
 
+#elif defined SSL_READ_EARLY_DATA_SUCCESS
+
+    /* OpenSSL */
+
+    SSL_CTX_set_max_early_data(ssl->ctx, NGX_SSL_BUFSIZE);
+
 #else
     ngx_log_error(NGX_LOG_WARN, ssl->log, 0,
                   "\"ssl_early_data\" is not supported on this platform, "
@@ -1246,6 +1270,12 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags)
 
     sc->session_ctx = ssl->ctx;
 
+#ifdef SSL_READ_EARLY_DATA_SUCCESS
+    if (SSL_CTX_get_max_early_data(ssl->ctx)) {
+        sc->try_early_data = 1;
+    }
+#endif
+
     sc->connection = SSL_new(ssl->ctx);
 
     if (sc->connection == NULL) {
@@ -1325,6 +1355,12 @@ ngx_ssl_handshake(ngx_connection_t *c)
     int        n, sslerr;
     ngx_err_t  err;
 
+#ifdef SSL_READ_EARLY_DATA_SUCCESS
+    if (c->ssl->try_early_data) {
+        return ngx_ssl_try_early_data(c);
+    }
+#endif
+
     ngx_ssl_clear_error(c->log);
 
     n = SSL_do_handshake(c->ssl->connection);
@@ -1342,50 +1378,7 @@ ngx_ssl_handshake(ngx_connection_t *c)
         }
 
 #if (NGX_DEBUG)
-        {
+        ngx_ssl_handshake_log(c);
-        char         buf[129], *s, *d;
-#if OPENSSL_VERSION_NUMBER >= 0x10000000L
-        const
-#endif
-        SSL_CIPHER  *cipher;
-
-        cipher = SSL_get_current_cipher(c->ssl->connection);
-
-        if (cipher) {
-            SSL_CIPHER_description(cipher, &buf[1], 128);
-
-            for (s = &buf[1], d = buf; *s; s++) {
-                if (*s == ' ' && *d == ' ') {
-                    continue;
-                }
-
-                if (*s == LF || *s == CR) {
-                    continue;
-                }
-
-                *++d = *s;
-            }
-
-            if (*d != ' ') {
-                d++;
-            }
-
-            *d = '\0';
-
-            ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
-                           "SSL: %s, cipher: \"%s\"",
-                           SSL_get_version(c->ssl->connection), &buf[1]);
-
-            if (SSL_session_reused(c->ssl->connection)) {
-                ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
-                               "SSL reused session");
-            }
-
-        } else {
-            ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
-                           "SSL no shared ciphers");
-        }
-        }
 #endif
 
         c->ssl->handshaked = 1;
@@ -1468,6 +1461,173 @@ ngx_ssl_handshake(ngx_connection_t *c)
 }
 
 
+#ifdef SSL_READ_EARLY_DATA_SUCCESS
+
+static ngx_int_t
+ngx_ssl_try_early_data(ngx_connection_t *c)
+{
+    int        n, sslerr;
+    u_char     buf;
+    size_t     readbytes;
+    ngx_err_t  err;
+
+    ngx_ssl_clear_error(c->log);
+
+    readbytes = 0;
+
+    n = SSL_read_early_data(c->ssl->connection, &buf, 1, &readbytes);
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "SSL_read_early_data: %d, %uz", n, readbytes);
+
+    if (n == SSL_READ_EARLY_DATA_FINISH) {
+        c->ssl->try_early_data = 0;
+        return ngx_ssl_handshake(c);
+    }
+
+    if (n == SSL_READ_EARLY_DATA_SUCCESS) {
+
+        if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+            return NGX_ERROR;
+        }
+
+        if (ngx_handle_write_event(c->write, 0) != NGX_OK) {
+            return NGX_ERROR;
+        }
+
+#if (NGX_DEBUG)
+        ngx_ssl_handshake_log(c);
+#endif
+
+        c->ssl->try_early_data = 0;
+
+        c->ssl->early_buf = buf;
+        c->ssl->early_preread = 1;
+
+        c->ssl->handshaked = 1;
+        c->ssl->in_early = 1;
+
+        c->recv = ngx_ssl_recv;
+        c->send = ngx_ssl_write;
+        c->recv_chain = ngx_ssl_recv_chain;
+        c->send_chain = ngx_ssl_send_chain;
+
+        return NGX_OK;
+    }
+
+    /* SSL_READ_EARLY_DATA_ERROR */
+
+    sslerr = SSL_get_error(c->ssl->connection, n);
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_get_error: %d", sslerr);
+
+    if (sslerr == SSL_ERROR_WANT_READ) {
+        c->read->ready = 0;
+        c->read->handler = ngx_ssl_handshake_handler;
+        c->write->handler = ngx_ssl_handshake_handler;
+
+        if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+            return NGX_ERROR;
+        }
+
+        if (ngx_handle_write_event(c->write, 0) != NGX_OK) {
+            return NGX_ERROR;
+        }
+
+        return NGX_AGAIN;
+    }
+
+    if (sslerr == SSL_ERROR_WANT_WRITE) {
+        c->write->ready = 0;
+        c->read->handler = ngx_ssl_handshake_handler;
+        c->write->handler = ngx_ssl_handshake_handler;
+
+        if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+            return NGX_ERROR;
+        }
+
+        if (ngx_handle_write_event(c->write, 0) != NGX_OK) {
+            return NGX_ERROR;
+        }
+
+        return NGX_AGAIN;
+    }
+
+    err = (sslerr == SSL_ERROR_SYSCALL) ? ngx_errno : 0;
+
+    c->ssl->no_wait_shutdown = 1;
+    c->ssl->no_send_shutdown = 1;
+    c->read->eof = 1;
+
+    if (sslerr == SSL_ERROR_ZERO_RETURN || ERR_peek_error() == 0) {
+        ngx_connection_error(c, err,
+                             "peer closed connection in SSL handshake");
+
+        return NGX_ERROR;
+    }
+
+    c->read->error = 1;
+
+    ngx_ssl_connection_error(c, sslerr, err, "SSL_read_early_data() failed");
+
+    return NGX_ERROR;
+}
+
+#endif
+
+
+#if (NGX_DEBUG)
+
+static void
+ngx_ssl_handshake_log(ngx_connection_t *c)
+{
+    char         buf[129], *s, *d;
+#if OPENSSL_VERSION_NUMBER >= 0x10000000L
+    const
+#endif
+    SSL_CIPHER  *cipher;
+
+    cipher = SSL_get_current_cipher(c->ssl->connection);
+
+    if (cipher) {
+        SSL_CIPHER_description(cipher, &buf[1], 128);
+
+        for (s = &buf[1], d = buf; *s; s++) {
+            if (*s == ' ' && *d == ' ') {
+                continue;
+            }
+
+            if (*s == LF || *s == CR) {
+                continue;
+            }
+
+            *++d = *s;
+        }
+
+        if (*d != ' ') {
+            d++;
+        }
+
+        *d = '\0';
+
+        ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                       "SSL: %s, cipher: \"%s\"",
+                       SSL_get_version(c->ssl->connection), &buf[1]);
+
+        if (SSL_session_reused(c->ssl->connection)) {
+            ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                           "SSL reused session");
+        }
+
+    } else {
+        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                       "SSL no shared ciphers");
+    }
+}
+
+#endif
+
+
 static void
 ngx_ssl_handshake_handler(ngx_event_t *ev)
 {
@@ -1555,6 +1715,12 @@ ngx_ssl_recv(ngx_connection_t *c, u_char *buf, size_t size)
 {
     int  n, bytes;
 
+#ifdef SSL_READ_EARLY_DATA_SUCCESS
+    if (c->ssl->in_early) {
+        return ngx_ssl_recv_early(c, buf, size);
+    }
+#endif
+
     if (c->ssl->last == NGX_ERROR) {
         c->read->error = 1;
         return NGX_ERROR;
@@ -1628,6 +1794,123 @@ ngx_ssl_recv(ngx_connection_t *c, u_char *buf, size_t size)
 }
 
 
+#ifdef SSL_READ_EARLY_DATA_SUCCESS
+
+static ssize_t
+ngx_ssl_recv_early(ngx_connection_t *c, u_char *buf, size_t size)
+{
+    int        n, bytes;
+    size_t     readbytes;
+
+    if (c->ssl->last == NGX_ERROR) {
+        c->read->error = 1;
+        return NGX_ERROR;
+    }
+
+    if (c->ssl->last == NGX_DONE) {
+        c->read->ready = 0;
+        c->read->eof = 1;
+        return 0;
+    }
+
+    bytes = 0;
+
+    ngx_ssl_clear_error(c->log);
+
+    if (c->ssl->early_preread) {
+
+        if (size == 0) {
+            c->read->ready = 0;
+            c->read->eof = 1;
+            return 0;
+        }
+
+        *buf = c->ssl->early_buf;
+
+        c->ssl->early_preread = 0;
+
+        bytes = 1;
+        size -= 1;
+        buf += 1;
+    }
+
+    /*
+     * SSL_read_early_data() may return data in parts, so try to read
+     * until SSL_read_early_data() would return no data
+     */
+
+    for ( ;; ) {
+
+        readbytes = 0;
+
+        n = SSL_read_early_data(c->ssl->connection, buf, size, &readbytes);
+
+        ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                       "SSL_read_early_data: %d, %uz", n, readbytes);
+
+        if (n == SSL_READ_EARLY_DATA_SUCCESS) {
+
+            c->ssl->last = ngx_ssl_handle_recv(c, 1);
+
+            bytes += readbytes;
+            size -= readbytes;
+
+            if (size == 0) {
+                c->read->ready = 1;
+                return bytes;
+            }
+
+            buf += readbytes;
+
+            continue;
+        }
+
+        if (n == SSL_READ_EARLY_DATA_FINISH) {
+
+            c->ssl->last = ngx_ssl_handle_recv(c, 1);
+            c->ssl->in_early = 0;
+
+            if (bytes) {
+                c->read->ready = 1;
+                return bytes;
+            }
+
+            return ngx_ssl_recv(c, buf, size);
+        }
+
+        /* SSL_READ_EARLY_DATA_ERROR */
+
+        c->ssl->last = ngx_ssl_handle_recv(c, 0);
+
+        if (bytes) {
+            if (c->ssl->last != NGX_AGAIN) {
+                c->read->ready = 1;
+            }
+
+            return bytes;
+        }
+
+        switch (c->ssl->last) {
+
+        case NGX_DONE:
+            c->read->ready = 0;
+            c->read->eof = 1;
+            return 0;
+
+        case NGX_ERROR:
+            c->read->error = 1;
+
+            /* fall through */
+
+        case NGX_AGAIN:
+            return c->ssl->last;
+        }
+    }
+}
+
+#endif
+
+
 static ngx_int_t
 ngx_ssl_handle_recv(ngx_connection_t *c, int n)
 {
@@ -1923,6 +2206,12 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size)
     int        n, sslerr;
     ngx_err_t  err;
 
+#ifdef SSL_READ_EARLY_DATA_SUCCESS
+    if (c->ssl->in_early) {
+        return ngx_ssl_write_early(c, data, size);
+    }
+#endif
+
     ngx_ssl_clear_error(c->log);
 
     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL to write: %uz", size);
@@ -2010,6 +2299,107 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size)
 }
 
 
+#ifdef SSL_READ_EARLY_DATA_SUCCESS
+
+ssize_t
+ngx_ssl_write_early(ngx_connection_t *c, u_char *data, size_t size)
+{
+    int        n, sslerr;
+    size_t     written;
+    ngx_err_t  err;
+
+    ngx_ssl_clear_error(c->log);
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL to write: %uz", size);
+
+    written = 0;
+
+    n = SSL_write_early_data(c->ssl->connection, data, size, &written);
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "SSL_write_early_data: %d, %uz", n, written);
+
+    if (n > 0) {
+
+        if (c->ssl->saved_read_handler) {
+
+            c->read->handler = c->ssl->saved_read_handler;
+            c->ssl->saved_read_handler = NULL;
+            c->read->ready = 1;
+
+            if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+                return NGX_ERROR;
+            }
+
+            ngx_post_event(c->read, &ngx_posted_events);
+        }
+
+        c->sent += written;
+
+        return written;
+    }
+
+    sslerr = SSL_get_error(c->ssl->connection, n);
+
+    err = (sslerr == SSL_ERROR_SYSCALL) ? ngx_errno : 0;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_get_error: %d", sslerr);
+
+    if (sslerr == SSL_ERROR_WANT_WRITE) {
+
+        if (c->ssl->saved_read_handler) {
+
+            c->read->handler = c->ssl->saved_read_handler;
+            c->ssl->saved_read_handler = NULL;
+            c->read->ready = 1;
+
+            if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+                return NGX_ERROR;
+            }
+
+            ngx_post_event(c->read, &ngx_posted_events);
+        }
+
+        c->write->ready = 0;
+        return NGX_AGAIN;
+    }
+
+    if (sslerr == SSL_ERROR_WANT_READ) {
+
+        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                       "SSL_write_early_data: want read");
+
+        c->read->ready = 0;
+
+        if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+            return NGX_ERROR;
+        }
+
+        /*
+         * we do not set the timer because there is already
+         * the write event timer
+         */
+
+        if (c->ssl->saved_read_handler == NULL) {
+            c->ssl->saved_read_handler = c->read->handler;
+            c->read->handler = ngx_ssl_read_handler;
+        }
+
+        return NGX_AGAIN;
+    }
+
+    c->ssl->no_wait_shutdown = 1;
+    c->ssl->no_send_shutdown = 1;
+    c->write->error = 1;
+
+    ngx_ssl_connection_error(c, sslerr, err, "SSL_write_early_data() failed");
+
+    return NGX_ERROR;
+}
+
+#endif
+
+
 static void
 ngx_ssl_read_handler(ngx_event_t *rev)
 {
@@ -3694,9 +4084,21 @@ ngx_ssl_get_early_data(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s)
     s->len = 0;
 
 #ifdef SSL_ERROR_EARLY_DATA_REJECTED
+
+    /* BoringSSL */
+
     if (SSL_in_early_data(c->ssl->connection)) {
         ngx_str_set(s, "1");
     }
+
+#elif defined SSL_READ_EARLY_DATA_SUCCESS
+
+    /* OpenSSL */
+
+    if (!SSL_is_init_finished(c->ssl->connection)) {
+        ngx_str_set(s, "1");
+    }
+
 #endif
 
     return NGX_OK;

src/event/ngx_event_openssl.h

@@ -87,12 +87,17 @@ struct ngx_ssl_connection_s {
     ngx_event_handler_pt        saved_read_handler;
     ngx_event_handler_pt        saved_write_handler;
 
+    u_char                      early_buf;
+
     unsigned                    handshaked:1;
     unsigned                    renegotiation:1;
     unsigned                    buffer:1;
     unsigned                    no_wait_shutdown:1;
     unsigned                    no_send_shutdown:1;
     unsigned                    handshake_buffer_set:1;
+    unsigned                    try_early_data:1;
+    unsigned                    in_early:1;
+    unsigned                    early_preread:1;
 };