grpcproxy: make grpc keep alive related options configurable (#11711)
Currently grpc-proxy doesn't config keep alive related options, so it will use the default values provided by the underlay gprc library. If clients uses a keep alive ping interval smaller than server's default minTime, connections between server and clients will be closed and reopened frequently.release-3.5
parent
0908a8bd10
commit
0461b3fa51
|
@ -33,6 +33,7 @@ import (
|
||||||
"go.etcd.io/etcd/clientv3/leasing"
|
"go.etcd.io/etcd/clientv3/leasing"
|
||||||
"go.etcd.io/etcd/clientv3/namespace"
|
"go.etcd.io/etcd/clientv3/namespace"
|
||||||
"go.etcd.io/etcd/clientv3/ordering"
|
"go.etcd.io/etcd/clientv3/ordering"
|
||||||
|
"go.etcd.io/etcd/embed"
|
||||||
"go.etcd.io/etcd/etcdserver/api/v3election/v3electionpb"
|
"go.etcd.io/etcd/etcdserver/api/v3election/v3electionpb"
|
||||||
"go.etcd.io/etcd/etcdserver/api/v3lock/v3lockpb"
|
"go.etcd.io/etcd/etcdserver/api/v3lock/v3lockpb"
|
||||||
pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
|
pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
|
||||||
|
@ -47,6 +48,7 @@ import (
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
"google.golang.org/grpc/grpclog"
|
"google.golang.org/grpc/grpclog"
|
||||||
|
"google.golang.org/grpc/keepalive"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
@ -86,6 +88,11 @@ var (
|
||||||
grpcProxyEnableOrdering bool
|
grpcProxyEnableOrdering bool
|
||||||
|
|
||||||
grpcProxyDebug bool
|
grpcProxyDebug bool
|
||||||
|
|
||||||
|
// GRPC keep alive related options.
|
||||||
|
grpcKeepAliveMinTime time.Duration
|
||||||
|
grpcKeepAliveTimeout time.Duration
|
||||||
|
grpcKeepAliveInterval time.Duration
|
||||||
)
|
)
|
||||||
|
|
||||||
const defaultGRPCMaxCallSendMsgSize = 1.5 * 1024 * 1024
|
const defaultGRPCMaxCallSendMsgSize = 1.5 * 1024 * 1024
|
||||||
|
@ -126,6 +133,9 @@ func newGRPCProxyStartCommand() *cobra.Command {
|
||||||
cmd.Flags().StringVar(&grpcProxyDataDir, "data-dir", "default.proxy", "Data directory for persistent data")
|
cmd.Flags().StringVar(&grpcProxyDataDir, "data-dir", "default.proxy", "Data directory for persistent data")
|
||||||
cmd.Flags().IntVar(&grpcMaxCallSendMsgSize, "max-send-bytes", defaultGRPCMaxCallSendMsgSize, "message send limits in bytes (default value is 1.5 MiB)")
|
cmd.Flags().IntVar(&grpcMaxCallSendMsgSize, "max-send-bytes", defaultGRPCMaxCallSendMsgSize, "message send limits in bytes (default value is 1.5 MiB)")
|
||||||
cmd.Flags().IntVar(&grpcMaxCallRecvMsgSize, "max-recv-bytes", math.MaxInt32, "message receive limits in bytes (default value is math.MaxInt32)")
|
cmd.Flags().IntVar(&grpcMaxCallRecvMsgSize, "max-recv-bytes", math.MaxInt32, "message receive limits in bytes (default value is math.MaxInt32)")
|
||||||
|
cmd.Flags().DurationVar(&grpcKeepAliveMinTime, "grpc-keepalive-min-time", embed.DefaultGRPCKeepAliveMinTime, "Minimum interval duration that a client should wait before pinging proxy.")
|
||||||
|
cmd.Flags().DurationVar(&grpcKeepAliveInterval, "grpc-keepalive-interval", embed.DefaultGRPCKeepAliveInterval, "Frequency duration of server-to-client ping to check if a connection is alive (0 to disable).")
|
||||||
|
cmd.Flags().DurationVar(&grpcKeepAliveTimeout, "grpc-keepalive-timeout", embed.DefaultGRPCKeepAliveTimeout, "Additional duration of wait before closing a non-responsive connection (0 to disable).")
|
||||||
|
|
||||||
// client TLS for connecting to server
|
// client TLS for connecting to server
|
||||||
cmd.Flags().StringVar(&grpcProxyCert, "cert", "", "identify secure connections with etcd servers using this TLS certificate file")
|
cmd.Flags().StringVar(&grpcProxyCert, "cert", "", "identify secure connections with etcd servers using this TLS certificate file")
|
||||||
|
@ -358,11 +368,26 @@ func newGRPCProxyServer(lg *zap.Logger, client *clientv3.Client) *grpc.Server {
|
||||||
electionp := grpcproxy.NewElectionProxy(client)
|
electionp := grpcproxy.NewElectionProxy(client)
|
||||||
lockp := grpcproxy.NewLockProxy(client)
|
lockp := grpcproxy.NewLockProxy(client)
|
||||||
|
|
||||||
server := grpc.NewServer(
|
gopts := []grpc.ServerOption{
|
||||||
grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor),
|
grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor),
|
||||||
grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor),
|
grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor),
|
||||||
grpc.MaxConcurrentStreams(math.MaxUint32),
|
grpc.MaxConcurrentStreams(math.MaxUint32),
|
||||||
)
|
}
|
||||||
|
if grpcKeepAliveMinTime > time.Duration(0) {
|
||||||
|
gopts = append(gopts, grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
|
||||||
|
MinTime: grpcKeepAliveMinTime,
|
||||||
|
PermitWithoutStream: false,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
if grpcKeepAliveInterval > time.Duration(0) ||
|
||||||
|
grpcKeepAliveTimeout > time.Duration(0) {
|
||||||
|
gopts = append(gopts, grpc.KeepaliveParams(keepalive.ServerParameters{
|
||||||
|
Time: grpcKeepAliveInterval,
|
||||||
|
Timeout: grpcKeepAliveTimeout,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
server := grpc.NewServer(gopts...)
|
||||||
|
|
||||||
pb.RegisterKVServer(server, kvp)
|
pb.RegisterKVServer(server, kvp)
|
||||||
pb.RegisterWatchServer(server, watchp)
|
pb.RegisterWatchServer(server, watchp)
|
||||||
|
|
|
@ -3,12 +3,12 @@
|
||||||
CFSSL = @env PATH=$(GOPATH)/bin:$(PATH) cfssl
|
CFSSL = @env PATH=$(GOPATH)/bin:$(PATH) cfssl
|
||||||
JSON = env PATH=$(GOPATH)/bin:$(PATH) cfssljson
|
JSON = env PATH=$(GOPATH)/bin:$(PATH) cfssljson
|
||||||
|
|
||||||
all: cfssl ca req
|
all: ca req
|
||||||
|
|
||||||
cfssl:
|
cfssl:
|
||||||
go get -u -tags nopkcs11 github.com/cloudflare/cfssl/cmd/cfssl
|
HTTPS_PROXY=127.0.0.1:12639 go get -u -tags nopkcs11 github.com/cloudflare/cfssl/cmd/cfssl
|
||||||
go get -u github.com/cloudflare/cfssl/cmd/cfssljson
|
HTTPS_PROXY=127.0.0.1:12639 go get -u github.com/cloudflare/cfssl/cmd/cfssljson
|
||||||
go get -u github.com/mattn/goreman
|
HTTPS_PROXY=127.0.0.1:12639 go get -u github.com/mattn/goreman
|
||||||
|
|
||||||
ca:
|
ca:
|
||||||
mkdir -p certs
|
mkdir -p certs
|
||||||
|
@ -19,22 +19,32 @@ req:
|
||||||
-ca certs/ca.pem \
|
-ca certs/ca.pem \
|
||||||
-ca-key certs/ca-key.pem \
|
-ca-key certs/ca-key.pem \
|
||||||
-config config/ca-config.json \
|
-config config/ca-config.json \
|
||||||
config/req-csr.json | $(JSON) -bare certs/etcd1
|
config/req-csr.json | $(JSON) -bare certs/9.145.89.120
|
||||||
$(CFSSL) gencert \
|
$(CFSSL) gencert \
|
||||||
-ca certs/ca.pem \
|
-ca certs/ca.pem \
|
||||||
-ca-key certs/ca-key.pem \
|
-ca-key certs/ca-key.pem \
|
||||||
-config config/ca-config.json \
|
-config config/ca-config.json \
|
||||||
config/req-csr.json | $(JSON) -bare certs/etcd2
|
config/req-csr.json | $(JSON) -bare certs/9.145.89.173
|
||||||
$(CFSSL) gencert \
|
$(CFSSL) gencert \
|
||||||
-ca certs/ca.pem \
|
-ca certs/ca.pem \
|
||||||
-ca-key certs/ca-key.pem \
|
-ca-key certs/ca-key.pem \
|
||||||
-config config/ca-config.json \
|
-config config/ca-config.json \
|
||||||
config/req-csr.json | $(JSON) -bare certs/etcd3
|
config/req-csr.json | $(JSON) -bare certs/9.145.89.225
|
||||||
$(CFSSL) gencert \
|
$(CFSSL) gencert \
|
||||||
-ca certs/ca.pem \
|
-ca certs/ca.pem \
|
||||||
-ca-key certs/ca-key.pem \
|
-ca-key certs/ca-key.pem \
|
||||||
-config config/ca-config.json \
|
-config config/ca-config.json \
|
||||||
config/req-csr.json | $(JSON) -bare certs/proxy1
|
config/req-csr.json | $(JSON) -bare certs/peer-9.145.89.120
|
||||||
|
$(CFSSL) gencert \
|
||||||
|
-ca certs/ca.pem \
|
||||||
|
-ca-key certs/ca-key.pem \
|
||||||
|
-config config/ca-config.json \
|
||||||
|
config/req-csr.json | $(JSON) -bare certs/peer-9.145.89.173
|
||||||
|
$(CFSSL) gencert \
|
||||||
|
-ca certs/ca.pem \
|
||||||
|
-ca-key certs/ca-key.pem \
|
||||||
|
-config config/ca-config.json \
|
||||||
|
config/req-csr.json | $(JSON) -bare certs/peer-9.145.89.225
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
rm -rf certs
|
rm -rf certs
|
||||||
|
|
|
@ -7,7 +7,7 @@
|
||||||
"server auth",
|
"server auth",
|
||||||
"client auth"
|
"client auth"
|
||||||
],
|
],
|
||||||
"expiry": "8760h"
|
"expiry": "876000h"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,8 +1,8 @@
|
||||||
{
|
{
|
||||||
"CN": "Autogenerated CA",
|
"CN": "Autogenerated CA",
|
||||||
"key": {
|
"key": {
|
||||||
"algo": "ecdsa",
|
"algo": "rsa",
|
||||||
"size": 384
|
"size": 2048
|
||||||
},
|
},
|
||||||
"names": [
|
"names": [
|
||||||
{
|
{
|
||||||
|
|
|
@ -1,11 +1,15 @@
|
||||||
{
|
{
|
||||||
"CN": "etcd",
|
"CN": "etcd",
|
||||||
"hosts": [
|
"hosts": [
|
||||||
"localhost"
|
"localhost",
|
||||||
|
"127.0.0.1",
|
||||||
|
"9.145.89.120",
|
||||||
|
"9.145.89.173",
|
||||||
|
"9.145.89.225"
|
||||||
],
|
],
|
||||||
"key": {
|
"key": {
|
||||||
"algo": "ecdsa",
|
"algo": "rsa",
|
||||||
"size": 384
|
"size": 2048
|
||||||
},
|
},
|
||||||
"names": [
|
"names": [
|
||||||
{
|
{
|
||||||
|
|
Loading…
Reference in New Issue