grpcproxy: make grpc keep alive related options configurable (#11711)

Currently grpc-proxy doesn't config keep alive related options, so it
will use the default values provided by the underlay gprc library. If
clients uses a keep alive ping interval smaller than server's default
minTime, connections between server and clients will be closed and
reopened frequently.
release-3.5
mlmhl 2020-04-18 12:20:03 +08:00 committed by GitHub
parent 0908a8bd10
commit 0461b3fa51
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 55 additions and 16 deletions

View File

@ -33,6 +33,7 @@ import (
"go.etcd.io/etcd/clientv3/leasing"
"go.etcd.io/etcd/clientv3/namespace"
"go.etcd.io/etcd/clientv3/ordering"
"go.etcd.io/etcd/embed"
"go.etcd.io/etcd/etcdserver/api/v3election/v3electionpb"
"go.etcd.io/etcd/etcdserver/api/v3lock/v3lockpb"
pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
@ -47,6 +48,7 @@ import (
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/grpclog"
"google.golang.org/grpc/keepalive"
)
var (
@ -86,6 +88,11 @@ var (
grpcProxyEnableOrdering bool
grpcProxyDebug bool
// GRPC keep alive related options.
grpcKeepAliveMinTime time.Duration
grpcKeepAliveTimeout time.Duration
grpcKeepAliveInterval time.Duration
)
const defaultGRPCMaxCallSendMsgSize = 1.5 * 1024 * 1024
@ -126,6 +133,9 @@ func newGRPCProxyStartCommand() *cobra.Command {
cmd.Flags().StringVar(&grpcProxyDataDir, "data-dir", "default.proxy", "Data directory for persistent data")
cmd.Flags().IntVar(&grpcMaxCallSendMsgSize, "max-send-bytes", defaultGRPCMaxCallSendMsgSize, "message send limits in bytes (default value is 1.5 MiB)")
cmd.Flags().IntVar(&grpcMaxCallRecvMsgSize, "max-recv-bytes", math.MaxInt32, "message receive limits in bytes (default value is math.MaxInt32)")
cmd.Flags().DurationVar(&grpcKeepAliveMinTime, "grpc-keepalive-min-time", embed.DefaultGRPCKeepAliveMinTime, "Minimum interval duration that a client should wait before pinging proxy.")
cmd.Flags().DurationVar(&grpcKeepAliveInterval, "grpc-keepalive-interval", embed.DefaultGRPCKeepAliveInterval, "Frequency duration of server-to-client ping to check if a connection is alive (0 to disable).")
cmd.Flags().DurationVar(&grpcKeepAliveTimeout, "grpc-keepalive-timeout", embed.DefaultGRPCKeepAliveTimeout, "Additional duration of wait before closing a non-responsive connection (0 to disable).")
// client TLS for connecting to server
cmd.Flags().StringVar(&grpcProxyCert, "cert", "", "identify secure connections with etcd servers using this TLS certificate file")
@ -358,11 +368,26 @@ func newGRPCProxyServer(lg *zap.Logger, client *clientv3.Client) *grpc.Server {
electionp := grpcproxy.NewElectionProxy(client)
lockp := grpcproxy.NewLockProxy(client)
server := grpc.NewServer(
gopts := []grpc.ServerOption{
grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor),
grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor),
grpc.MaxConcurrentStreams(math.MaxUint32),
)
}
if grpcKeepAliveMinTime > time.Duration(0) {
gopts = append(gopts, grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
MinTime: grpcKeepAliveMinTime,
PermitWithoutStream: false,
}))
}
if grpcKeepAliveInterval > time.Duration(0) ||
grpcKeepAliveTimeout > time.Duration(0) {
gopts = append(gopts, grpc.KeepaliveParams(keepalive.ServerParameters{
Time: grpcKeepAliveInterval,
Timeout: grpcKeepAliveTimeout,
}))
}
server := grpc.NewServer(gopts...)
pb.RegisterKVServer(server, kvp)
pb.RegisterWatchServer(server, watchp)

View File

@ -3,12 +3,12 @@
CFSSL = @env PATH=$(GOPATH)/bin:$(PATH) cfssl
JSON = env PATH=$(GOPATH)/bin:$(PATH) cfssljson
all: cfssl ca req
all: ca req
cfssl:
go get -u -tags nopkcs11 github.com/cloudflare/cfssl/cmd/cfssl
go get -u github.com/cloudflare/cfssl/cmd/cfssljson
go get -u github.com/mattn/goreman
HTTPS_PROXY=127.0.0.1:12639 go get -u -tags nopkcs11 github.com/cloudflare/cfssl/cmd/cfssl
HTTPS_PROXY=127.0.0.1:12639 go get -u github.com/cloudflare/cfssl/cmd/cfssljson
HTTPS_PROXY=127.0.0.1:12639 go get -u github.com/mattn/goreman
ca:
mkdir -p certs
@ -19,22 +19,32 @@ req:
-ca certs/ca.pem \
-ca-key certs/ca-key.pem \
-config config/ca-config.json \
config/req-csr.json | $(JSON) -bare certs/etcd1
config/req-csr.json | $(JSON) -bare certs/9.145.89.120
$(CFSSL) gencert \
-ca certs/ca.pem \
-ca-key certs/ca-key.pem \
-config config/ca-config.json \
config/req-csr.json | $(JSON) -bare certs/etcd2
config/req-csr.json | $(JSON) -bare certs/9.145.89.173
$(CFSSL) gencert \
-ca certs/ca.pem \
-ca-key certs/ca-key.pem \
-config config/ca-config.json \
config/req-csr.json | $(JSON) -bare certs/etcd3
config/req-csr.json | $(JSON) -bare certs/9.145.89.225
$(CFSSL) gencert \
-ca certs/ca.pem \
-ca-key certs/ca-key.pem \
-config config/ca-config.json \
config/req-csr.json | $(JSON) -bare certs/proxy1
config/req-csr.json | $(JSON) -bare certs/peer-9.145.89.120
$(CFSSL) gencert \
-ca certs/ca.pem \
-ca-key certs/ca-key.pem \
-config config/ca-config.json \
config/req-csr.json | $(JSON) -bare certs/peer-9.145.89.173
$(CFSSL) gencert \
-ca certs/ca.pem \
-ca-key certs/ca-key.pem \
-config config/ca-config.json \
config/req-csr.json | $(JSON) -bare certs/peer-9.145.89.225
clean:
rm -rf certs

View File

@ -7,7 +7,7 @@
"server auth",
"client auth"
],
"expiry": "8760h"
"expiry": "876000h"
}
}
}

View File

@ -1,8 +1,8 @@
{
"CN": "Autogenerated CA",
"key": {
"algo": "ecdsa",
"size": 384
"algo": "rsa",
"size": 2048
},
"names": [
{

View File

@ -1,11 +1,15 @@
{
"CN": "etcd",
"hosts": [
"localhost"
"localhost",
"127.0.0.1",
"9.145.89.120",
"9.145.89.173",
"9.145.89.225"
],
"key": {
"algo": "ecdsa",
"size": 384
"algo": "rsa",
"size": 2048
},
"names": [
{