From 78ed123e71aab17ffecfa0b2b27a349cfb4b2502 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 9 Sep 2024 19:03:52 +0400 Subject: [PATCH] SSL: caching certificates. Certificate chains are now loaded once. The certificate cache provides each chain as a unique stack of reference counted elements. This shallow copy is required because OpenSSL stacks aren't reference counted. Based on previous work by Mini Hawthorne. --- src/event/ngx_event_openssl.c | 105 ++--------------- src/event/ngx_event_openssl.h | 3 + src/event/ngx_event_openssl_cache.c | 167 +++++++++++++++++++++++++++- 3 files changed, 173 insertions(+), 102 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 3ed003062..018d03016 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -18,8 +18,6 @@ typedef struct { } ngx_openssl_conf_t; -static X509 *ngx_ssl_load_certificate(ngx_pool_t *pool, char **err, - ngx_str_t *cert, STACK_OF(X509) **chain); static EVP_PKEY *ngx_ssl_load_certificate_key(ngx_pool_t *pool, char **err, ngx_str_t *key, ngx_array_t *passwords); static int ngx_ssl_password_callback(char *buf, int size, int rwflag, @@ -449,8 +447,8 @@ ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert, EVP_PKEY *pkey; STACK_OF(X509) *chain; - x509 = ngx_ssl_load_certificate(cf->pool, &err, cert, &chain); - if (x509 == NULL) { + chain = ngx_ssl_cache_fetch(cf, NGX_SSL_CACHE_CERT, &err, cert, NULL); + if (chain == NULL) { if (err != NULL) { ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, "cannot load certificate \"%s\": %s", @@ -460,6 +458,8 @@ ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert, return NGX_ERROR; } + x509 = sk_X509_shift(chain); + if (SSL_CTX_use_certificate(ssl->ctx, x509) == 0) { ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, "SSL_CTX_use_certificate(\"%s\") failed", cert->data); @@ -570,8 +570,9 @@ ngx_ssl_connection_certificate(ngx_connection_t *c, ngx_pool_t *pool, EVP_PKEY *pkey; STACK_OF(X509) *chain; - x509 = ngx_ssl_load_certificate(pool, &err, cert, &chain); - if (x509 == NULL) { + chain = ngx_ssl_cache_connection_fetch(pool, NGX_SSL_CACHE_CERT, &err, + cert, NULL); + if (chain == NULL) { if (err != NULL) { ngx_ssl_error(NGX_LOG_ERR, c->log, 0, "cannot load certificate \"%s\": %s", @@ -581,6 +582,8 @@ ngx_ssl_connection_certificate(ngx_connection_t *c, ngx_pool_t *pool, return NGX_ERROR; } + x509 = sk_X509_shift(chain); + if (SSL_use_certificate(c->ssl->connection, x509) == 0) { ngx_ssl_error(NGX_LOG_ERR, c->log, 0, "SSL_use_certificate(\"%s\") failed", cert->data); @@ -632,96 +635,6 @@ ngx_ssl_connection_certificate(ngx_connection_t *c, ngx_pool_t *pool, } -static X509 * -ngx_ssl_load_certificate(ngx_pool_t *pool, char **err, ngx_str_t *cert, - STACK_OF(X509) **chain) -{ - BIO *bio; - X509 *x509, *temp; - u_long n; - - if (ngx_strncmp(cert->data, "data:", sizeof("data:") - 1) == 0) { - - bio = BIO_new_mem_buf(cert->data + sizeof("data:") - 1, - cert->len - (sizeof("data:") - 1)); - if (bio == NULL) { - *err = "BIO_new_mem_buf() failed"; - return NULL; - } - - } else { - - if (ngx_get_full_name(pool, (ngx_str_t *) &ngx_cycle->conf_prefix, cert) - != NGX_OK) - { - *err = NULL; - return NULL; - } - - bio = BIO_new_file((char *) cert->data, "r"); - if (bio == NULL) { - *err = "BIO_new_file() failed"; - return NULL; - } - } - - /* certificate itself */ - - x509 = PEM_read_bio_X509_AUX(bio, NULL, NULL, NULL); - if (x509 == NULL) { - *err = "PEM_read_bio_X509_AUX() failed"; - BIO_free(bio); - return NULL; - } - - /* rest of the chain */ - - *chain = sk_X509_new_null(); - if (*chain == NULL) { - *err = "sk_X509_new_null() failed"; - BIO_free(bio); - X509_free(x509); - return NULL; - } - - for ( ;; ) { - - temp = PEM_read_bio_X509(bio, NULL, NULL, NULL); - if (temp == NULL) { - n = ERR_peek_last_error(); - - if (ERR_GET_LIB(n) == ERR_LIB_PEM - && ERR_GET_REASON(n) == PEM_R_NO_START_LINE) - { - /* end of file */ - ERR_clear_error(); - break; - } - - /* some real error */ - - *err = "PEM_read_bio_X509() failed"; - BIO_free(bio); - X509_free(x509); - sk_X509_pop_free(*chain, X509_free); - return NULL; - } - - if (sk_X509_push(*chain, temp) == 0) { - *err = "sk_X509_push() failed"; - BIO_free(bio); - X509_free(x509); - sk_X509_pop_free(*chain, X509_free); - return NULL; - } - } - - BIO_free(bio); - - return x509; -} - - static EVP_PKEY * ngx_ssl_load_certificate_key(ngx_pool_t *pool, char **err, ngx_str_t *key, ngx_array_t *passwords) diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index fa2282ce8..addc7b4d3 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -193,6 +193,9 @@ typedef struct { #define NGX_SSL_BUFSIZE 16384 +#define NGX_SSL_CACHE_CERT 0 + + ngx_int_t ngx_ssl_init(ngx_log_t *log); ngx_int_t ngx_ssl_create(ngx_ssl_t *ssl, ngx_uint_t protocols, void *data); diff --git a/src/event/ngx_event_openssl_cache.c b/src/event/ngx_event_openssl_cache.c index 98867de50..20b87b79a 100644 --- a/src/event/ngx_event_openssl_cache.c +++ b/src/event/ngx_event_openssl_cache.c @@ -10,6 +10,7 @@ #define NGX_SSL_CACHE_PATH 0 +#define NGX_SSL_CACHE_DATA 1 typedef struct { @@ -51,6 +52,13 @@ static ngx_int_t ngx_ssl_cache_init_key(ngx_pool_t *pool, ngx_uint_t index, static ngx_ssl_cache_node_t *ngx_ssl_cache_lookup(ngx_ssl_cache_t *cache, ngx_ssl_cache_type_t *type, ngx_ssl_cache_key_t *id, uint32_t hash); +static void *ngx_ssl_cache_cert_create(ngx_ssl_cache_key_t *id, char **err, + void *data); +static void ngx_ssl_cache_cert_free(void *data); +static void *ngx_ssl_cache_cert_ref(char **err, void *data); + +static BIO *ngx_ssl_cache_create_bio(ngx_ssl_cache_key_t *id, char **err); + static void *ngx_openssl_cache_create_conf(ngx_cycle_t *cycle); static void ngx_ssl_cache_cleanup(void *data); static void ngx_ssl_cache_node_insert(ngx_rbtree_node_t *temp, @@ -82,6 +90,10 @@ ngx_module_t ngx_openssl_cache_module = { static ngx_ssl_cache_type_t ngx_ssl_cache_types[] = { + /* NGX_SSL_CACHE_CERT */ + { ngx_ssl_cache_cert_create, + ngx_ssl_cache_cert_free, + ngx_ssl_cache_cert_ref }, }; @@ -152,13 +164,18 @@ static ngx_int_t ngx_ssl_cache_init_key(ngx_pool_t *pool, ngx_uint_t index, ngx_str_t *path, ngx_ssl_cache_key_t *id) { - if (ngx_get_full_name(pool, (ngx_str_t *) &ngx_cycle->conf_prefix, path) - != NGX_OK) - { - return NGX_ERROR; - } + if (ngx_strncmp(path->data, "data:", sizeof("data:") - 1) == 0) { + id->type = NGX_SSL_CACHE_DATA; - id->type = NGX_SSL_CACHE_PATH; + } else { + if (ngx_get_full_name(pool, (ngx_str_t *) &ngx_cycle->conf_prefix, path) + != NGX_OK) + { + return NGX_ERROR; + } + + id->type = NGX_SSL_CACHE_PATH; + } id->len = path->len; id->data = path->data; @@ -219,6 +236,144 @@ ngx_ssl_cache_lookup(ngx_ssl_cache_t *cache, ngx_ssl_cache_type_t *type, } +static void * +ngx_ssl_cache_cert_create(ngx_ssl_cache_key_t *id, char **err, void *data) +{ + BIO *bio; + X509 *x509; + u_long n; + STACK_OF(X509) *chain; + + chain = sk_X509_new_null(); + if (chain == NULL) { + *err = "sk_X509_new_null() failed"; + return NULL; + } + + bio = ngx_ssl_cache_create_bio(id, err); + if (bio == NULL) { + sk_X509_pop_free(chain, X509_free); + return NULL; + } + + /* certificate itself */ + + x509 = PEM_read_bio_X509_AUX(bio, NULL, NULL, NULL); + if (x509 == NULL) { + *err = "PEM_read_bio_X509_AUX() failed"; + BIO_free(bio); + sk_X509_pop_free(chain, X509_free); + return NULL; + } + + if (sk_X509_push(chain, x509) == 0) { + *err = "sk_X509_push() failed"; + BIO_free(bio); + X509_free(x509); + sk_X509_pop_free(chain, X509_free); + return NULL; + } + + /* rest of the chain */ + + for ( ;; ) { + + x509 = PEM_read_bio_X509(bio, NULL, NULL, NULL); + if (x509 == NULL) { + n = ERR_peek_last_error(); + + if (ERR_GET_LIB(n) == ERR_LIB_PEM + && ERR_GET_REASON(n) == PEM_R_NO_START_LINE) + { + /* end of file */ + ERR_clear_error(); + break; + } + + /* some real error */ + + *err = "PEM_read_bio_X509() failed"; + BIO_free(bio); + sk_X509_pop_free(chain, X509_free); + return NULL; + } + + if (sk_X509_push(chain, x509) == 0) { + *err = "sk_X509_push() failed"; + BIO_free(bio); + X509_free(x509); + sk_X509_pop_free(chain, X509_free); + return NULL; + } + } + + BIO_free(bio); + + return chain; +} + + +static void +ngx_ssl_cache_cert_free(void *data) +{ + sk_X509_pop_free(data, X509_free); +} + + +static void * +ngx_ssl_cache_cert_ref(char **err, void *data) +{ + int n, i; + X509 *x509; + STACK_OF(X509) *chain; + + chain = sk_X509_dup(data); + if (chain == NULL) { + *err = "sk_X509_dup() failed"; + return NULL; + } + + n = sk_X509_num(chain); + + for (i = 0; i < n; i++) { + x509 = sk_X509_value(chain, i); + +#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) + X509_up_ref(x509); +#else + CRYPTO_add(&x509->references, 1, CRYPTO_LOCK_X509); +#endif + } + + return chain; +} + + +static BIO * +ngx_ssl_cache_create_bio(ngx_ssl_cache_key_t *id, char **err) +{ + BIO *bio; + + if (id->type == NGX_SSL_CACHE_DATA) { + + bio = BIO_new_mem_buf(id->data + sizeof("data:") - 1, + id->len - (sizeof("data:") - 1)); + if (bio == NULL) { + *err = "BIO_new_mem_buf() failed"; + } + + return bio; + } + + bio = BIO_new_file((char *) id->data, "r"); + if (bio == NULL) { + *err = "BIO_new_file() failed"; + } + + return bio; +} + + static void * ngx_openssl_cache_create_conf(ngx_cycle_t *cycle) {