From f9f8f6406b110168553d4e4b6aae7a857ef9fe3c Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Thu, 25 May 2023 14:09:14 +0200 Subject: [PATCH] config+lnd: allow configuring gRPC keepalive settings This change allows users to customize the gRPC keepalive settings. The server ping values configure when the server sends out a ping by itself and how quickly the client is expected to respond. The client ping min wait configures the minimum time a client has to wait before sending another ping. A connection might be closed by the server if a client pings more frequently than the allowed min wait time. We choose lower default values than the gRPC library used before: - ServerPingTime: 1 minute instead of 2 hours - ClientPingMinWait: 5 seconds instead of 5 minutes This should by itself already make long-standing gRPC streams more stable but also allow clients to set a custom client ping time of 5 seconds or more (a value of slightly more than 5000 milliseconds should be chosen to avoid the server disconnecting due to slight timing skews). --- config.go | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ lnd.go | 14 ++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/config.go b/config.go index 0398c4b27..5018994a7 100644 --- a/config.go +++ b/config.go @@ -209,6 +209,21 @@ const ( // defaultKeepFailedPaymentAttempts is the default setting for whether // to keep failed payments in the database. defaultKeepFailedPaymentAttempts = false + + // defaultGrpcServerPingTime is the default duration for the amount of + // time of no activity after which the server pings the client to see if + // the transport is still alive. If set below 1s, a minimum value of 1s + // will be used instead. + defaultGrpcServerPingTime = time.Minute + + // defaultGrpcServerPingTimeout is the default duration the server waits + // after having pinged for keepalive check, and if no activity is seen + // even after that the connection is closed. + defaultGrpcServerPingTimeout = 20 * time.Second + + // defaultGrpcClientPingMinWait is the default minimum amount of time a + // client should wait before sending a keepalive ping. + defaultGrpcClientPingMinWait = 5 * time.Second ) var ( @@ -454,6 +469,8 @@ type Config struct { Htlcswitch *lncfg.Htlcswitch `group:"htlcswitch" namespace:"htlcswitch"` + GRPC *GRPCConfig `group:"grpc" namespace:"grpc"` + // LogWriter is the root logger that all of the daemon's subloggers are // hooked up to. LogWriter *build.RotatingLogWriter @@ -474,6 +491,35 @@ type Config struct { Estimator routing.Estimator } +// GRPCConfig holds the configuration options for the gRPC server. +// See https://github.com/grpc/grpc-go/blob/v1.41.0/keepalive/keepalive.go#L50 +// for more details. Any value of 0 means we use the gRPC internal default +// values. +// +//nolint:lll +type GRPCConfig struct { + // ServerPingTime is a duration for the amount of time of no activity + // after which the server pings the client to see if the transport is + // still alive. If set below 1s, a minimum value of 1s will be used + // instead. + ServerPingTime time.Duration `long:"server-ping-time" description:"How long the server waits on a gRPC stream with no activity before pinging the client."` + + // ServerPingTimeout is the duration the server waits after having + // pinged for keepalive check, and if no activity is seen even after + // that the connection is closed. + ServerPingTimeout time.Duration `long:"server-ping-timeout" description:"How long the server waits for the response from the client for the keepalive ping response."` + + // ClientPingMinWait is the minimum amount of time a client should wait + // before sending a keepalive ping. + ClientPingMinWait time.Duration `long:"client-ping-min-wait" description:"The minimum amount of time the client should wait before sending a keepalive ping."` + + // ClientAllowPingWithoutStream specifies whether pings from the client + // are allowed even if there are no active gRPC streams. This might be + // useful to keep the underlying HTTP/2 connection open for future + // requests. + ClientAllowPingWithoutStream bool `long:"client-allow-ping-without-stream" description:"If true, the server allows keepalive pings from the client even when there are no active gRPC streams. This might be useful to keep the underlying HTTP/2 connection open for future requests."` +} + // DefaultConfig returns all default values for the Config struct. // //nolint:lll @@ -662,6 +708,11 @@ func DefaultConfig() Config { Htlcswitch: &lncfg.Htlcswitch{ MailboxDeliveryTimeout: htlcswitch.DefaultMailboxDeliveryTimeout, }, + GRPC: &GRPCConfig{ + ServerPingTime: defaultGrpcServerPingTime, + ServerPingTimeout: defaultGrpcServerPingTimeout, + ClientPingMinWait: defaultGrpcClientPingMinWait, + }, } } diff --git a/lnd.go b/lnd.go index 7d52d63da..57d417983 100644 --- a/lnd.go +++ b/lnd.go @@ -37,6 +37,7 @@ import ( "github.com/lightningnetwork/lnd/watchtower" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/keepalive" "gopkg.in/macaroon-bakery.v2/bakery" "gopkg.in/macaroon.v2" ) @@ -303,10 +304,23 @@ func Main(cfg *Config, lisCfg ListenerCfg, implCfg *ImplementationCfg, } }() + // Allow the user to overwrite some defaults of the gRPC library related + // to connection keepalive (server side and client side pings). + serverKeepalive := keepalive.ServerParameters{ + Time: cfg.GRPC.ServerPingTime, + Timeout: cfg.GRPC.ServerPingTimeout, + } + clientKeepalive := keepalive.EnforcementPolicy{ + MinTime: cfg.GRPC.ClientPingMinWait, + PermitWithoutStream: cfg.GRPC.ClientAllowPingWithoutStream, + } + rpcServerOpts := interceptorChain.CreateServerOpts() serverOpts = append(serverOpts, rpcServerOpts...) serverOpts = append( serverOpts, grpc.MaxRecvMsgSize(lnrpc.MaxGrpcMsgSize), + grpc.KeepaliveParams(serverKeepalive), + grpc.KeepaliveEnforcementPolicy(clientKeepalive), ) grpcServer := grpc.NewServer(serverOpts...)