From 40af029413bb8ef5d21467972deaf842221de7b8 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 5 Jan 2022 17:11:30 +0100 Subject: [PATCH 1/5] etcd: extend kvdb_etcd with a real ForAll implementation --- go.mod | 2 +- kvdb/etcd/readwrite_bucket.go | 39 +++++++++++++++++++++------ kvdb/etcd/stm.go | 51 +++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 63da824ce..52b6e9aab 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/lightningnetwork/lnd/cert v1.1.0 github.com/lightningnetwork/lnd/clock v1.1.0 github.com/lightningnetwork/lnd/healthcheck v1.2.0 - github.com/lightningnetwork/lnd/kvdb v1.3.0 + github.com/lightningnetwork/lnd/kvdb v1.3.1 github.com/lightningnetwork/lnd/queue v1.1.0 github.com/lightningnetwork/lnd/ticker v1.1.0 github.com/lightningnetwork/lnd/tlv v1.0.1 diff --git a/kvdb/etcd/readwrite_bucket.go b/kvdb/etcd/readwrite_bucket.go index 0b7782326..16fc892a7 100644 --- a/kvdb/etcd/readwrite_bucket.go +++ b/kvdb/etcd/readwrite_bucket.go @@ -17,6 +17,9 @@ type readWriteBucket struct { // appropriate prefix to prefix the key. id []byte + // key is the bucket key. + key []byte + // tx holds the parent transaction. tx *readWriteTx } @@ -25,8 +28,9 @@ type readWriteBucket struct { // and bucket id. func newReadWriteBucket(tx *readWriteTx, key, id []byte) *readWriteBucket { return &readWriteBucket{ - id: id, - tx: tx, + id: id, + key: key, + tx: tx, } } @@ -66,6 +70,31 @@ func (b *readWriteBucket) ForEach(cb func(k, v []byte) error) error { return nil } +// ForAll is an optimized version of ForEach for the case when we know we will +// fetch all (or almost all) items. +// +// NOTE: ForAll differs from ForEach in that no additional queries can +// be executed within the callback. +func (b *readWriteBucket) ForAll(cb func(k, v []byte) error) error { + // When we opened this bucket, we fetched the bucket key using the STM + // which put a revision "lock" in the read set. We can leverage this + // by incrementing the revision on the bucket, making any transaction + // retry that'd touch this same bucket. This way we can safely read all + // keys from the bucket and not cache them in the STM. + // To increment the bucket's revision, we simply put in the bucket key + // value again (which is idempotent if the bucket has just been created). + b.tx.stm.Put(string(b.key), string(b.id)) + + // TODO(bhandras): page size should be configurable in ForAll. + return b.tx.stm.FetchRangePaginatedRaw( + string(b.id), 1000, + func(kv KV) error { + key, val := getKeyVal(&kv) + return cb(key, val) + }, + ) +} + // Get returns the value for the given key. Returns nil if the key does // not exist in this bucket. func (b *readWriteBucket) Get(key []byte) []byte { @@ -406,9 +435,3 @@ func (b *readWriteBucket) Prefetch(paths ...[]string) { b.tx.stm.Prefetch(flattenMap(keys), flattenMap(ranges)) } - -// ForAll is an optimized version of ForEach with the limitation that no -// additional queries can be executed within the callback. -func (b *readWriteBucket) ForAll(cb func(k, v []byte) error) error { - return b.ForEach(cb) -} diff --git a/kvdb/etcd/stm.go b/kvdb/etcd/stm.go index 3a3d20233..94a401f4b 100644 --- a/kvdb/etcd/stm.go +++ b/kvdb/etcd/stm.go @@ -81,6 +81,11 @@ type STM interface { // Prefetch prefetches the passed keys and prefixes. For prefixes it'll // fetch the whole range. Prefetch(keys []string, prefix []string) + + // FetchRangePaginatedRaw will fetch the range with the passed prefix up + // to the passed limit per page. + FetchRangePaginatedRaw(prefix string, limit int64, + cb func(kv KV) error) error } // CommitError is used to check if there was an error @@ -589,6 +594,52 @@ func (ws writeSet) puts() []v3.Op { return puts } +// FetchRangePaginatedRaw will fetch the range with the passed prefix up to the +// passed limit per page. +func (s *stm) FetchRangePaginatedRaw(prefix string, limit int64, + cb func(kv KV) error) error { + + s.callCount++ + + opts := []v3.OpOption{ + v3.WithSort(v3.SortByKey, v3.SortAscend), + v3.WithRange(v3.GetPrefixRangeEnd(prefix)), + v3.WithLimit(limit), + } + + key := prefix + for { + resp, err := s.client.Get( + s.options.ctx, key, append(opts, s.getOpts...)..., + ) + if err != nil { + return DatabaseError{ + msg: "stm.fetch() failed", + err: err, + } + } + + // Fill the read set with key/values returned. + for _, kv := range resp.Kvs { + err := cb(KV{string(kv.Key), string(kv.Value)}) + + if err != nil { + return err + } + } + + // We've reached the range end. + if !resp.More { + break + } + + // Continue from the page end + "\x00". + key = string(resp.Kvs[len(resp.Kvs)-1].Key) + "\x00" + } + + return nil +} + // fetch is a helper to fetch key/value given options. If a value is returned // then fetch will try to fix the STM's snapshot revision (if not already set). // We'll also cache the returned key/value in the read set. From 5e9019318039a5737e6921b0a3f138ec97cf6a21 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Fri, 21 Jan 2022 17:23:04 +0100 Subject: [PATCH 2/5] etcd: fix config clone --- kvdb/etcd/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kvdb/etcd/config.go b/kvdb/etcd/config.go index 6d060e895..03cdb489f 100644 --- a/kvdb/etcd/config.go +++ b/kvdb/etcd/config.go @@ -61,6 +61,7 @@ func (c *Config) CloneWithSubNamespace(subNamespace string) *Config { KeyFile: c.KeyFile, InsecureSkipVerify: c.InsecureSkipVerify, CollectStats: c.CollectStats, + MaxMsgSize: c.MaxMsgSize, SingleWriter: c.SingleWriter, } } @@ -81,6 +82,7 @@ func (c *Config) CloneWithSingleWriter() *Config { KeyFile: c.KeyFile, InsecureSkipVerify: c.InsecureSkipVerify, CollectStats: c.CollectStats, + MaxMsgSize: c.MaxMsgSize, SingleWriter: true, } } From 1093683b4761160682abbcf55b80b9da29ada1a1 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Fri, 21 Jan 2022 17:24:55 +0100 Subject: [PATCH 3/5] etcd: set embedded etcd default max msg size and up txn limit --- kvdb/etcd/embed.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kvdb/etcd/embed.go b/kvdb/etcd/embed.go index 0df2f804f..107fca1d3 100644 --- a/kvdb/etcd/embed.go +++ b/kvdb/etcd/embed.go @@ -68,7 +68,7 @@ func NewEmbeddedEtcdInstance(path string, clientPort, peerPort uint16, cfg.Dir = path // To ensure that we can submit large transactions. - cfg.MaxTxnOps = 8192 + cfg.MaxTxnOps = 16384 cfg.MaxRequestBytes = 16384 * 1024 cfg.Logger = "zap" if logFile != "" { @@ -109,6 +109,7 @@ func NewEmbeddedEtcdInstance(path string, clientPort, peerPort uint16, Host: "http://" + clientURL, InsecureSkipVerify: true, Namespace: defaultNamespace, + MaxMsgSize: int(cfg.MaxRequestBytes), } return connConfig, func() { From bee5e7b76188328343edb64aaff637e680e60b81 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Mon, 31 Jan 2022 16:10:52 +0100 Subject: [PATCH 4/5] docs: update etcd docs --- docs/etcd.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/etcd.md b/docs/etcd.md index 15cf4e299..60a9dbe0c 100644 --- a/docs/etcd.md +++ b/docs/etcd.md @@ -39,10 +39,8 @@ To start your local etcd instance for testing run: ``` The large `max-txn-ops` and `max-request-bytes` values are currently required in -case of running LND with the full graph in etcd. Upcoming versions will split -the database to local and replicated parts and only essential parts will remain -in the replicated database, removing the requirement for these additional -settings. These parameters have been tested to work with testnet LND. +case of running LND with the full graph in etcd. These parameters have been +tested to work with testnet LND. ## Configuring LND to run on etcd From d28c9000c547522152e47b9f4696cd36fa9962b8 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Mon, 28 Feb 2022 12:01:42 +0100 Subject: [PATCH 5/5] docs: update release notes 0.15.0 --- docs/release-notes/release-notes-0.15.0.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/release-notes/release-notes-0.15.0.md b/docs/release-notes/release-notes-0.15.0.md index 301f41e02..3606b0f27 100644 --- a/docs/release-notes/release-notes-0.15.0.md +++ b/docs/release-notes/release-notes-0.15.0.md @@ -84,6 +84,11 @@ change, it allows encrypted failure messages to be returned to the sender. Additionally it is possible to signal a malformed htlc. +## Database + +* [Add ForAll implementation for etcd to speed up + graph cache at startup](https://github.com/lightningnetwork/lnd/pull/6136) + ## Documentation * Improved instructions on [how to build lnd for mobile](https://github.com/lightningnetwork/lnd/pull/6085). @@ -129,6 +134,7 @@ gRPC performance metrics (latency to process `GetInfo`, etc)](https://github.com # Contributors (Alphabetical Order) * 3nprob +* Andras Banki-Horvath * Andreas Schjønhaug * asvdf * BTCparadigm