Merge pull request #7730 from guggero/grpc-keepalive

config+grpc: allow configuring gRPC keepalive params

config.go

@@ -209,6 +209,21 @@ const (
 	// defaultKeepFailedPaymentAttempts is the default setting for whether
 	// to keep failed payments in the database.
 	defaultKeepFailedPaymentAttempts = false
+
+	// defaultGrpcServerPingTime is the default duration for the amount of
+	// time of no activity after which the server pings the client to see if
+	// the transport is still alive. If set below 1s, a minimum value of 1s
+	// will be used instead.
+	defaultGrpcServerPingTime = time.Minute
+
+	// defaultGrpcServerPingTimeout is the default duration the server waits
+	// after having pinged for keepalive check, and if no activity is seen
+	// even after that the connection is closed.
+	defaultGrpcServerPingTimeout = 20 * time.Second
+
+	// defaultGrpcClientPingMinWait is the default minimum amount of time a
+	// client should wait before sending a keepalive ping.
+	defaultGrpcClientPingMinWait = 5 * time.Second
 )
 
 var (
@@ -454,6 +469,8 @@ type Config struct {
 
 	Htlcswitch *lncfg.Htlcswitch `group:"htlcswitch" namespace:"htlcswitch"`
 
+	GRPC *GRPCConfig `group:"grpc" namespace:"grpc"`
+
 	// LogWriter is the root logger that all of the daemon's subloggers are
 	// hooked up to.
 	LogWriter *build.RotatingLogWriter
@@ -474,6 +491,35 @@ type Config struct {
 	Estimator routing.Estimator
 }
 
+// GRPCConfig holds the configuration options for the gRPC server.
+// See https://github.com/grpc/grpc-go/blob/v1.41.0/keepalive/keepalive.go#L50
+// for more details. Any value of 0 means we use the gRPC internal default
+// values.
+//
+//nolint:lll
+type GRPCConfig struct {
+	// ServerPingTime is a duration for the amount of time of no activity
+	// after which the server pings the client to see if the transport is
+	// still alive. If set below 1s, a minimum value of 1s will be used
+	// instead.
+	ServerPingTime time.Duration `long:"server-ping-time" description:"How long the server waits on a gRPC stream with no activity before pinging the client."`
+
+	// ServerPingTimeout is the duration the server waits after having
+	// pinged for keepalive check, and if no activity is seen even after
+	// that the connection is closed.
+	ServerPingTimeout time.Duration `long:"server-ping-timeout" description:"How long the server waits for the response from the client for the keepalive ping response."`
+
+	// ClientPingMinWait is the minimum amount of time a client should wait
+	// before sending a keepalive ping.
+	ClientPingMinWait time.Duration `long:"client-ping-min-wait" description:"The minimum amount of time the client should wait before sending a keepalive ping."`
+
+	// ClientAllowPingWithoutStream specifies whether pings from the client
+	// are allowed even if there are no active gRPC streams. This might be
+	// useful to keep the underlying HTTP/2 connection open for future
+	// requests.
+	ClientAllowPingWithoutStream bool `long:"client-allow-ping-without-stream" description:"If true, the server allows keepalive pings from the client even when there are no active gRPC streams. This might be useful to keep the underlying HTTP/2 connection open for future requests."`
+}
+
 // DefaultConfig returns all default values for the Config struct.
 //
 //nolint:lll
@@ -662,6 +708,11 @@ func DefaultConfig() Config {
 		Htlcswitch: &lncfg.Htlcswitch{
 			MailboxDeliveryTimeout: htlcswitch.DefaultMailboxDeliveryTimeout,
 		},
+		GRPC: &GRPCConfig{
+			ServerPingTime:    defaultGrpcServerPingTime,
+			ServerPingTimeout: defaultGrpcServerPingTimeout,
+			ClientPingMinWait: defaultGrpcClientPingMinWait,
+		},
 	}
 }
 

docs/release-notes/release-notes-0.17.0.md

@@ -53,6 +53,25 @@ package](https://github.com/lightningnetwork/lnd/pull/7356)
   a helpful note-to-self containing arbitrary useful information about the
   channel.
 
+* [gRPC keepalive parameters can now be set in the
+  configuration](https://github.com/lightningnetwork/lnd/pull/7730). The `lnd`
+  configuration settings `grpc.server-ping-time` and `grpc.server-ping-timeout`
+  configure how often `lnd` pings its clients and how long a pong response is
+  allowed to take. The default values for there settings are improved over the
+  gRPC protocol internal default values, so most users won't need to change
+  those. The `grpc.client-ping-min-wait` setting defines how often a client is
+  allowed to ping `lnd` to check for connection healthiness. The `lnd` default
+  value of 5 seconds is much lower than the previously used protocol internal
+  value, which means clients can now check connection health more often. For
+  this to be activated on the client side, gRPC clients are encouraged to set
+  the keepalive setting on their end as well (using the `grpc.keepalive_time_ms`
+  option in JavaScript or Python, or the equivalent setting in the gRPC library
+  they are using, might be an environment variable or a different syntax
+  depending on the programming language used) when creating long open streams
+  over a network topology that might silently fail connections. A value of
+  `grpc.keepalive_time_ms=5100` is recommended on the client side (adding 100ms
+  to account for slightly different clock speeds).
+
 ## Misc
 
 * [Generate default macaroons

lnd.go

@@ -37,6 +37,7 @@ import (
 	"github.com/lightningnetwork/lnd/watchtower"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/keepalive"
 	"gopkg.in/macaroon-bakery.v2/bakery"
 	"gopkg.in/macaroon.v2"
 )
@@ -303,10 +304,23 @@ func Main(cfg *Config, lisCfg ListenerCfg, implCfg *ImplementationCfg,
 		}
 	}()
 
+	// Allow the user to overwrite some defaults of the gRPC library related
+	// to connection keepalive (server side and client side pings).
+	serverKeepalive := keepalive.ServerParameters{
+		Time:    cfg.GRPC.ServerPingTime,
+		Timeout: cfg.GRPC.ServerPingTimeout,
+	}
+	clientKeepalive := keepalive.EnforcementPolicy{
+		MinTime:             cfg.GRPC.ClientPingMinWait,
+		PermitWithoutStream: cfg.GRPC.ClientAllowPingWithoutStream,
+	}
+
 	rpcServerOpts := interceptorChain.CreateServerOpts()
 	serverOpts = append(serverOpts, rpcServerOpts...)
 	serverOpts = append(
 		serverOpts, grpc.MaxRecvMsgSize(lnrpc.MaxGrpcMsgSize),
+		grpc.KeepaliveParams(serverKeepalive),
+		grpc.KeepaliveEnforcementPolicy(clientKeepalive),
 	)
 
 	grpcServer := grpc.NewServer(serverOpts...)

sample-lnd.conf

@@ -1501,3 +1501,21 @@ litecoin.node=ltcd
 ; are sent over a short period.
 ; htlcswitch.mailboxdeliverytimeout=60s
 
+[grpc]
+
+; How long the server waits on a gRPC stream with no activity before pinging the
+; client. Valid time units are {s, m, h}. Default: 1m
+; grpc.server-ping-time=1m
+
+; How long the server waits for the response from the client for the keepalive
+; ping response. Valid time units are {s, m, h}. Default: 20s
+; grpc.server-ping-timeout=20s
+
+; The minimum amount of time the client should wait before sending a keepalive
+; ping. Valid time units are {s, m, h}. Default: 5s
+; grpc.client-ping-min-wait=5s
+
+; If true, the server allows keepalive pings from the client even when there are
+; no active gRPC streams. This might be useful to keep the underlying HTTP/2
+; connection open for future requests.
+; grpc.client-allow-ping-without-stream=false