grpcproxy: make grpc keep alive related options configurable (#11711)

Currently grpc-proxy doesn't config keep alive related options, so it
will use the default values provided by the underlay gprc library. If
clients uses a keep alive ping interval smaller than server's default
minTime, connections between server and clients will be closed and
reopened frequently.

etcdmain/grpc_proxy.go

@@ -33,6 +33,7 @@ import (
 	"go.etcd.io/etcd/clientv3/leasing"
 	"go.etcd.io/etcd/clientv3/namespace"
 	"go.etcd.io/etcd/clientv3/ordering"
+	"go.etcd.io/etcd/embed"
 	"go.etcd.io/etcd/etcdserver/api/v3election/v3electionpb"
 	"go.etcd.io/etcd/etcdserver/api/v3lock/v3lockpb"
 	pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
@@ -47,6 +48,7 @@ import (
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/keepalive"
 )
 
 var (
@@ -86,6 +88,11 @@ var (
 	grpcProxyEnableOrdering bool
 
 	grpcProxyDebug bool
+
+	// GRPC keep alive related options.
+	grpcKeepAliveMinTime  time.Duration
+	grpcKeepAliveTimeout  time.Duration
+	grpcKeepAliveInterval time.Duration
 )
 
 const defaultGRPCMaxCallSendMsgSize = 1.5 * 1024 * 1024
@@ -126,6 +133,9 @@ func newGRPCProxyStartCommand() *cobra.Command {
 	cmd.Flags().StringVar(&grpcProxyDataDir, "data-dir", "default.proxy", "Data directory for persistent data")
 	cmd.Flags().IntVar(&grpcMaxCallSendMsgSize, "max-send-bytes", defaultGRPCMaxCallSendMsgSize, "message send limits in bytes (default value is 1.5 MiB)")
 	cmd.Flags().IntVar(&grpcMaxCallRecvMsgSize, "max-recv-bytes", math.MaxInt32, "message receive limits in bytes (default value is math.MaxInt32)")
+	cmd.Flags().DurationVar(&grpcKeepAliveMinTime, "grpc-keepalive-min-time", embed.DefaultGRPCKeepAliveMinTime, "Minimum interval duration that a client should wait before pinging proxy.")
+	cmd.Flags().DurationVar(&grpcKeepAliveInterval, "grpc-keepalive-interval", embed.DefaultGRPCKeepAliveInterval, "Frequency duration of server-to-client ping to check if a connection is alive (0 to disable).")
+	cmd.Flags().DurationVar(&grpcKeepAliveTimeout, "grpc-keepalive-timeout", embed.DefaultGRPCKeepAliveTimeout, "Additional duration of wait before closing a non-responsive connection (0 to disable).")
 
 	// client TLS for connecting to server
 	cmd.Flags().StringVar(&grpcProxyCert, "cert", "", "identify secure connections with etcd servers using this TLS certificate file")
@@ -358,11 +368,26 @@ func newGRPCProxyServer(lg *zap.Logger, client *clientv3.Client) *grpc.Server {
 	electionp := grpcproxy.NewElectionProxy(client)
 	lockp := grpcproxy.NewLockProxy(client)
 
-	server := grpc.NewServer(
+	gopts := []grpc.ServerOption{
 		grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor),
 		grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor),
 		grpc.MaxConcurrentStreams(math.MaxUint32),
-	)
+	}
+	if grpcKeepAliveMinTime > time.Duration(0) {
+		gopts = append(gopts, grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
+			MinTime:             grpcKeepAliveMinTime,
+			PermitWithoutStream: false,
+		}))
+	}
+	if grpcKeepAliveInterval > time.Duration(0) ||
+		grpcKeepAliveTimeout > time.Duration(0) {
+		gopts = append(gopts, grpc.KeepaliveParams(keepalive.ServerParameters{
+			Time:    grpcKeepAliveInterval,
+			Timeout: grpcKeepAliveTimeout,
+		}))
+	}
+
+	server := grpc.NewServer(gopts...)
 
 	pb.RegisterKVServer(server, kvp)
 	pb.RegisterWatchServer(server, watchp)

hack/tls-setup/Makefile

@@ -3,12 +3,12 @@
 CFSSL	= @env PATH=$(GOPATH)/bin:$(PATH) cfssl
 JSON	= env PATH=$(GOPATH)/bin:$(PATH) cfssljson
 
-all: cfssl ca req
+all:  ca req
 
 cfssl:
-	go get -u -tags nopkcs11 github.com/cloudflare/cfssl/cmd/cfssl
+	HTTPS_PROXY=127.0.0.1:12639 go get -u -tags nopkcs11 github.com/cloudflare/cfssl/cmd/cfssl
-	go get -u github.com/cloudflare/cfssl/cmd/cfssljson
+	HTTPS_PROXY=127.0.0.1:12639 go get -u github.com/cloudflare/cfssl/cmd/cfssljson
-	go get -u github.com/mattn/goreman
+	HTTPS_PROXY=127.0.0.1:12639 go get -u github.com/mattn/goreman
 
 ca:
 	mkdir -p certs
@@ -19,22 +19,32 @@ req:
 	  -ca certs/ca.pem \
 	  -ca-key certs/ca-key.pem \
 	  -config config/ca-config.json \
-	  config/req-csr.json | $(JSON) -bare certs/etcd1
+	  config/req-csr.json | $(JSON) -bare certs/9.145.89.120
 	$(CFSSL) gencert \
 	  -ca certs/ca.pem \
 	  -ca-key certs/ca-key.pem \
 	  -config config/ca-config.json \
-	  config/req-csr.json | $(JSON) -bare certs/etcd2
+	  config/req-csr.json | $(JSON) -bare certs/9.145.89.173
 	$(CFSSL) gencert \
 	  -ca certs/ca.pem \
 	  -ca-key certs/ca-key.pem \
 	  -config config/ca-config.json \
-	  config/req-csr.json | $(JSON) -bare certs/etcd3
+	  config/req-csr.json | $(JSON) -bare certs/9.145.89.225
 	$(CFSSL) gencert \
 	  -ca certs/ca.pem \
 	  -ca-key certs/ca-key.pem \
 	  -config config/ca-config.json \
-	  config/req-csr.json | $(JSON) -bare certs/proxy1
+	  config/req-csr.json | $(JSON) -bare certs/peer-9.145.89.120
+	$(CFSSL) gencert \
+	  -ca certs/ca.pem \
+	  -ca-key certs/ca-key.pem \
+	  -config config/ca-config.json \
+	  config/req-csr.json | $(JSON) -bare certs/peer-9.145.89.173
+	$(CFSSL) gencert \
+	  -ca certs/ca.pem \
+	  -ca-key certs/ca-key.pem \
+	  -config config/ca-config.json \
+	  config/req-csr.json | $(JSON) -bare certs/peer-9.145.89.225
 
 clean:
 	rm -rf certs

hack/tls-setup/config/ca-config.json

@@ -7,7 +7,7 @@
           "server auth",
           "client auth"
         ],
-        "expiry": "8760h"
+        "expiry": "876000h"
     }
   }
 }

hack/tls-setup/config/ca-csr.json

@@ -1,8 +1,8 @@
 {
   "CN": "Autogenerated CA",
   "key": {
-    "algo": "ecdsa",
+    "algo": "rsa",
-    "size": 384
+    "size": 2048
   },
   "names": [
     {

hack/tls-setup/config/req-csr.json

@@ -1,11 +1,15 @@
 {
   "CN": "etcd",
   "hosts": [
-    "localhost"
+    "localhost",
+    "127.0.0.1",
+    "9.145.89.120",
+    "9.145.89.173",
+    "9.145.89.225"
   ],
   "key": {
-    "algo": "ecdsa",
+    "algo": "rsa",
-    "size": 384
+    "size": 2048
   },
   "names": [
     {