mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-06-07 09:25:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feature: added ssl support for HCFS (#6699) (#6775)

Browse Source
This commit is contained in:
orthoxerox 2025-05-09 19:09:13 +03:00 committed by GitHub
parent 848d1f7c34
commit d8cc269294
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 243 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
other/java/client/src/main/java/seaweedfs/client/FilerClient.java
View File

@ -14,11 +14,13 @@ public class FilerClient extends FilerGrpcClient {
private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class);
public FilerClient(String filerHost, int filerGrpcPort) {
super(filerHost, filerGrpcPort-10000, filerGrpcPort);
this(filerHost, filerGrpcPort-10000, filerGrpcPort, "");
}
public FilerClient(String filerHost, int filerGrpcPort, String cn) { this(filerHost, filerGrpcPort-10000, filerGrpcPort, cn); }
public FilerClient(String filerHost, int filerPort, int filerGrpcPort) { this(filerHost, filerPort, filerGrpcPort, ""); }
public FilerClient(String filerHost, int filerPort, int filerGrpcPort) {
super(filerHost, filerPort, filerGrpcPort);
public FilerClient(String filerHost, int filerPort, int filerGrpcPort, String cn) {
super(filerHost, filerPort, filerGrpcPort, cn);
}
public static String toFileId(FilerProto.FileId fid) {

42
other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java
View File

@ -8,7 +8,6 @@ import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLException;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
@ -17,14 +16,12 @@ import java.util.concurrent.TimeUnit;
public class FilerGrpcClient {
private static final Logger logger = LoggerFactory.getLogger(FilerGrpcClient.class);
static SslContext sslContext;
private static final SslContext sslContext;
private static final String protocol;
static {
try {
sslContext = FilerSslContext.loadSslContext();
} catch (SSLException e) {
logger.warn("failed to load ssl context", e);
}
sslContext = FilerSecurityContext.getGrpcSslContext();
protocol = FilerSecurityContext.isHttpSecurityEnabled() ? "https" : "http";
}
public final int VOLUME_SERVER_ACCESS_DIRECT = 0;
@ -42,19 +39,27 @@ public class FilerGrpcClient {
private int volumeServerAccess = VOLUME_SERVER_ACCESS_DIRECT;
private String filerAddress;
public FilerGrpcClient(String host, int port, int grpcPort) {
this(host, port, grpcPort, sslContext);
public FilerGrpcClient(String host, int port, int grpcPort, String cn) {
this(host, port, grpcPort, cn, sslContext);
}
public FilerGrpcClient(String host, int port, int grpcPort, SslContext sslContext) {
public FilerGrpcClient(String host, int port, int grpcPort, String cn, SslContext sslContext) {
this(sslContext == null ?
ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext()
ManagedChannelBuilder.forAddress(host, grpcPort)
.usePlaintext()
.maxInboundMessageSize(1024 * 1024 * 1024) :
NettyChannelBuilder.forAddress(host, grpcPort)
.maxInboundMessageSize(1024 * 1024 * 1024)
.negotiationType(NegotiationType.TLS)
.sslContext(sslContext));
cn.isEmpty() ?
NettyChannelBuilder.forAddress(host, grpcPort)
.maxInboundMessageSize(1024 * 1024 * 1024)
.negotiationType(NegotiationType.TLS)
.sslContext(sslContext) :
NettyChannelBuilder.forAddress(host, grpcPort)
.maxInboundMessageSize(1024 * 1024 * 1024)
.negotiationType(NegotiationType.TLS)
.overrideAuthority(cn) //will not check hostname of the filer server
.sslContext(sslContext)
);
filerAddress = SeaweedUtil.joinHostPort(host, port);
@ -130,12 +135,11 @@ public class FilerGrpcClient {
public String getChunkUrl(String chunkId, String url, String publicUrl) {
switch (this.volumeServerAccess) {
case VOLUME_SERVER_ACCESS_PUBLIC_URL:
return String.format("http://%s/%s", publicUrl, chunkId);
return String.format("%s://%s/%s", protocol, publicUrl, chunkId);
case VOLUME_SERVER_ACCESS_FILER_PROXY:
return String.format("http://%s/?proxyChunkId=%s", this.filerAddress, chunkId);
return String.format("%s://%s/?proxyChunkId=%s", protocol, this.filerAddress, chunkId);
default:
return String.format("http://%s/%s", url, chunkId);
return String.format("%s://%s/%s", protocol, url, chunkId);
}
}
}

164
other/java/client/src/main/java/seaweedfs/client/FilerSecurityContext.java Normal file
View File

@ -0,0 +1,164 @@
package seaweedfs.client;
import com.google.common.base.Strings;
import com.moandjiezana.toml.Toml;
import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;
import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.http.ssl.SSLContexts;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.*;
import java.io.File;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.security.PrivateKey;
import java.security.cert.X509Certificate;
public abstract class FilerSecurityContext extends SslContext {
//extends Netty SslContext to access its protected static utility methods in
//buildHttpSslContext()
private static final Logger logger = LoggerFactory.getLogger(FilerSecurityContext.class);
private static boolean grpcSecurityEnabled;
private static boolean httpSecurityEnabled;
private static SslContext grpcSslContext;
private static SSLContext httpSslContext;
private static String grpcTrustCertCollectionFilePath;
private static String grpcClientCertChainFilePath;
private static String grpcClientPrivateKeyFilePath;
private static String httpTrustCertCollectionFilePath;
private static String httpClientCertChainFilePath;
private static String httpClientPrivateKeyFilePath;
static {
String securityFileName = "security.toml";
String home = System.getProperty("user.home");
File f1 = new File("./"+securityFileName);
File f2 = new File(home + "/.seaweedfs/"+securityFileName);
File f3 = new File("/etc/seaweedfs/"+securityFileName);
File securityFile = f1.exists()? f1 : f2.exists() ? f2 : f3.exists()? f3 : null;
if (securityFile==null){
logger.debug("Security file not found");
grpcSecurityEnabled = false;
httpSecurityEnabled = false;
} else {
Toml toml = new Toml().read(securityFile);
logger.debug("reading ssl setup from {}", securityFile);
grpcTrustCertCollectionFilePath = toml.getString("grpc.ca");
logger.debug("loading gRPC ca from {}", grpcTrustCertCollectionFilePath);
grpcClientCertChainFilePath = toml.getString("grpc.client.cert");
logger.debug("loading gRPC client ca from {}", grpcClientCertChainFilePath);
grpcClientPrivateKeyFilePath = toml.getString("grpc.client.key");
logger.debug("loading gRPC client key from {}", grpcClientPrivateKeyFilePath);
if (Strings.isNullOrEmpty(grpcClientCertChainFilePath) && Strings.isNullOrEmpty(grpcClientPrivateKeyFilePath)) {
logger.debug("gRPC private key file locations not set");
grpcSecurityEnabled = false;
} else {
try {
grpcSslContext = buildGrpcSslContext();
grpcSecurityEnabled = true;
} catch (Exception e) {
logger.warn("Couldn't initialize gRPC security context, filer operations are likely to fail!", e);
grpcSslContext = null;
grpcSecurityEnabled = false;
}
}
if (toml.getBoolean("https.client.enabled")) {
httpTrustCertCollectionFilePath = toml.getString("https.client.ca");
logger.debug("loading HTTP ca from {}", httpTrustCertCollectionFilePath);
httpClientCertChainFilePath = toml.getString("https.client.cert");
logger.debug("loading HTTP client ca from {}", httpClientCertChainFilePath);
httpClientPrivateKeyFilePath = toml.getString("https.client.key");
logger.debug("loading HTTP client key from {}", httpClientPrivateKeyFilePath);
if (Strings.isNullOrEmpty(httpClientCertChainFilePath) && Strings.isNullOrEmpty(httpClientPrivateKeyFilePath)) {
logger.debug("HTTP private key file locations not set");
httpSecurityEnabled = false;
} else {
try {
httpSslContext = buildHttpSslContext();
httpSecurityEnabled = true;
} catch (Exception e) {
logger.warn("Couldn't initialize HTTP security context, volume operations are likely to fail!", e);
httpSslContext = null;
httpSecurityEnabled = false;
}
}
} else {
httpSecurityEnabled = false;
}
}
// possibly fix the format https://netty.io/wiki/sslcontextbuilder-and-private-key.html
}
public static boolean isGrpcSecurityEnabled() {
return grpcSecurityEnabled;
}
public static boolean isHttpSecurityEnabled() {
return httpSecurityEnabled;
}
public static SslContext getGrpcSslContext() {
return grpcSslContext;
}
public static SSLContext getHttpSslContext() {
return httpSslContext;
}
private static SslContext buildGrpcSslContext() throws SSLException {
SslContextBuilder builder = GrpcSslContexts.forClient();
if (grpcTrustCertCollectionFilePath != null) {
builder.trustManager(new File(grpcTrustCertCollectionFilePath));
}
if (grpcClientCertChainFilePath != null && grpcClientPrivateKeyFilePath != null) {
builder.keyManager(new File(grpcClientCertChainFilePath), new File(grpcClientPrivateKeyFilePath));
}
return builder.build();
}
private static SSLContext buildHttpSslContext() throws GeneralSecurityException, IOException {
SSLContextBuilder builder = SSLContexts.custom();
if (httpTrustCertCollectionFilePath != null) {
final X509Certificate[] trustCerts = toX509Certificates(new File(httpTrustCertCollectionFilePath));
final KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
ks.load(null, null);
int i = 0;
for (X509Certificate cert: trustCerts) {
String alias = Integer.toString(++i);
ks.setCertificateEntry(alias, cert);
}
builder.loadTrustMaterial(ks, null);
}
if (httpClientCertChainFilePath != null && httpClientPrivateKeyFilePath != null) {
final X509Certificate[] keyCerts = toX509Certificates(new File(httpClientCertChainFilePath));
final PrivateKey key = toPrivateKey(new File(httpClientPrivateKeyFilePath), null);
char[] emptyPassword = new char[0];
final KeyStore ks = buildKeyStore(keyCerts, key, emptyPassword, null);
logger.debug("Loaded {} key certificates", ks.size());
builder.loadKeyMaterial(ks, emptyPassword);
}
return builder.build();
}
}

64
other/java/client/src/main/java/seaweedfs/client/FilerSslContext.java
View File

@ -1,64 +0,0 @@
package seaweedfs.client;
import com.google.common.base.Strings;
import com.moandjiezana.toml.Toml;
import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;
import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLException;
import java.io.File;
public class FilerSslContext {
private static final Logger logger = LoggerFactory.getLogger(FilerSslContext.class);
public static SslContext loadSslContext() throws SSLException {
String securityFileName = "security.toml";
String home = System.getProperty("user.home");
File f1 = new File("./"+securityFileName);
File f2 = new File(home + "/.seaweedfs/"+securityFileName);
File f3 = new File("/etc/seaweedfs/"+securityFileName);
File securityFile = f1.exists()? f1 : f2.exists() ? f2 : f3.exists()? f3 : null;
if (securityFile==null){
return null;
}
Toml toml = new Toml().read(securityFile);
logger.debug("reading ssl setup from {}", securityFile);
String trustCertCollectionFilePath = toml.getString("grpc.ca");
logger.debug("loading ca from {}", trustCertCollectionFilePath);
String clientCertChainFilePath = toml.getString("grpc.client.cert");
logger.debug("loading client ca from {}", clientCertChainFilePath);
String clientPrivateKeyFilePath = toml.getString("grpc.client.key");
logger.debug("loading client key from {}", clientPrivateKeyFilePath);
if (Strings.isNullOrEmpty(clientPrivateKeyFilePath) && Strings.isNullOrEmpty(clientPrivateKeyFilePath)){
return null;
}
// possibly fix the format https://netty.io/wiki/sslcontextbuilder-and-private-key.html
return buildSslContext(trustCertCollectionFilePath, clientCertChainFilePath, clientPrivateKeyFilePath);
}
private static SslContext buildSslContext(String trustCertCollectionFilePath,
String clientCertChainFilePath,
String clientPrivateKeyFilePath) throws SSLException {
SslContextBuilder builder = GrpcSslContexts.forClient();
if (trustCertCollectionFilePath != null) {
builder.trustManager(new File(trustCertCollectionFilePath));
}
if (clientCertChainFilePath != null && clientPrivateKeyFilePath != null) {
builder.keyManager(new File(clientCertChainFilePath), new File(clientPrivateKeyFilePath));
}
return builder.trustManager(InsecureTrustManagerFactory.INSTANCE).build();
}
}

45
other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java
View File

@ -1,27 +1,64 @@
package seaweedfs.client;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.DefaultHostnameVerifier;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.DefaultConnectionReuseStrategy;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
public class SeaweedUtil {
static PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
private static final Logger logger = LoggerFactory.getLogger(SeaweedUtil.class);
static PoolingHttpClientConnectionManager cm;
static CloseableHttpClient httpClient;
static {
//Apache HTTP client has a terrible API that makes you configure everything twice
//NoopHostnameVerifier is required because SeaweedFS doesn't verify hostnames
//and the servers are likely to have TLS certificates that do not match their hosts
if (FilerSecurityContext.isHttpSecurityEnabled()) {
SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(
FilerSecurityContext.getHttpSslContext(),
NoopHostnameVerifier.INSTANCE);
Registry<ConnectionSocketFactory> socketFactoryRegistry =
RegistryBuilder.<ConnectionSocketFactory>create()
.register("https", sslSocketFactory)
.register("http", new PlainConnectionSocketFactory())
.build();
cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
} else {
cm = new PoolingHttpClientConnectionManager();
}
// Increase max total connection to 200
cm.setMaxTotal(200);
// Increase default max connection per route to 20
cm.setDefaultMaxPerRoute(20);
httpClient = HttpClientBuilder.create()
HttpClientBuilder builder = HttpClientBuilder.create()
.setConnectionManager(cm)
.setConnectionReuseStrategy(DefaultConnectionReuseStrategy.INSTANCE)
.setKeepAliveStrategy(DefaultConnectionKeepAliveStrategy.INSTANCE)
.build();
.setKeepAliveStrategy(DefaultConnectionKeepAliveStrategy.INSTANCE);
if (FilerSecurityContext.isHttpSecurityEnabled()) {
builder.setSSLContext(FilerSecurityContext.getHttpSslContext());
builder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE);
}
httpClient = builder.build();
}
public static CloseableHttpClient getClosableHttpClient() {

4
other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
View File

@ -29,6 +29,7 @@ public class SeaweedFileSystem extends FileSystem {
public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication";
public static final String FS_SEAWEED_VOLUME_SERVER_ACCESS = "fs.seaweed.volume.server.access";
public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024;
public static final String FS_SEAWEED_FILER_CN = "fs.seaweed.filer.cn";
private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
@ -63,8 +64,9 @@ public class SeaweedFileSystem extends FileSystem {
setConf(conf);
this.uri = uri;
seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, conf);
String cn = conf.get(FS_SEAWEED_FILER_CN, "");
seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, cn, conf);
}
@Override

5
other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
View File

@ -27,8 +27,8 @@ public class SeaweedFileSystemStore {
private FilerClient filerClient;
private Configuration conf;
public SeaweedFileSystemStore(String host, int port, int grpcPort, Configuration conf) {
filerClient = new FilerClient(host, port, grpcPort);
public SeaweedFileSystemStore(String host, int port, int grpcPort, String cn, Configuration conf) {
filerClient = new FilerClient(host, port, grpcPort, cn);
this.conf = conf;
String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct");
if (volumeServerAccessMode.equals("publicUrl")) {
@ -36,7 +36,6 @@ public class SeaweedFileSystemStore {
} else if (volumeServerAccessMode.equals("filerProxy")) {
filerClient.setAccessVolumeServerByFilerProxy();
}
}
public void close() {

4
other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
View File

@ -29,6 +29,7 @@ public class SeaweedFileSystem extends FileSystem {
public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication";
public static final String FS_SEAWEED_VOLUME_SERVER_ACCESS = "fs.seaweed.volume.server.access";
public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024;
public static final String FS_SEAWEED_FILER_CN = "fs.seaweed.filer.cn";
private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
@ -63,8 +64,9 @@ public class SeaweedFileSystem extends FileSystem {
setConf(conf);
this.uri = uri;
seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, conf);
String cn = conf.get(FS_SEAWEED_FILER_CN, "");
seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, cn, conf);
}
@Override

5
other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
View File

@ -27,8 +27,8 @@ public class SeaweedFileSystemStore {
private FilerClient filerClient;
private Configuration conf;
public SeaweedFileSystemStore(String host, int port, int grpcPort, Configuration conf) {
filerClient = new FilerClient(host, port, grpcPort);
public SeaweedFileSystemStore(String host, int port, int grpcPort, String cn, Configuration conf) {
filerClient = new FilerClient(host, port, grpcPort, cn);
this.conf = conf;
String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct");
if (volumeServerAccessMode.equals("publicUrl")) {
@ -36,7 +36,6 @@ public class SeaweedFileSystemStore {
} else if (volumeServerAccessMode.equals("filerProxy")) {
filerClient.setAccessVolumeServerByFilerProxy();
}
}
public void close() {