diff --git a/config_builder.go b/config_builder.go index d3ca5e2b2..8e2ba1776 100644 --- a/config_builder.go +++ b/config_builder.go @@ -36,6 +36,7 @@ import ( "github.com/lightningnetwork/lnd/fn/v2" "github.com/lightningnetwork/lnd/funding" graphdb "github.com/lightningnetwork/lnd/graph/db" + graphdbmig1 "github.com/lightningnetwork/lnd/graph/db/migration1" "github.com/lightningnetwork/lnd/htlcswitch" "github.com/lightningnetwork/lnd/invoices" "github.com/lightningnetwork/lnd/keychain" @@ -1134,12 +1135,12 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( } graphMig := func(tx *sqlc.Queries) error { - cfg := &graphdb.SQLStoreConfig{ + cfg := &graphdbmig1.SQLStoreConfig{ //nolint:ll ChainHash: *d.cfg.ActiveNetParams.GenesisHash, QueryCfg: queryCfg, } - err := graphdb.MigrateGraphToSQL( + err := graphdbmig1.MigrateGraphToSQL( ctx, cfg, dbs.ChanStateDB.Backend, tx, ) if err != nil { diff --git a/graph/db/migration1/addr.go b/graph/db/migration1/addr.go new file mode 100644 index 000000000..4a2ed6e46 --- /dev/null +++ b/graph/db/migration1/addr.go @@ -0,0 +1,324 @@ +package migration1 + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "net" + + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/tor" +) + +// addressType specifies the network protocol and version that should be used +// when connecting to a node at a particular address. +type addressType uint8 + +const ( + // tcp4Addr denotes an IPv4 TCP address. + tcp4Addr addressType = 0 + + // tcp6Addr denotes an IPv6 TCP address. + tcp6Addr addressType = 1 + + // v2OnionAddr denotes a version 2 Tor onion service address. + v2OnionAddr addressType = 2 + + // v3OnionAddr denotes a version 3 Tor (prop224) onion service address. + v3OnionAddr addressType = 3 + + // opaqueAddrs denotes an address (or a set of addresses) that LND was + // not able to parse since LND is not yet aware of the address type. + opaqueAddrs addressType = 4 + + // dnsAddr denotes a DNS address type. + dnsAddr addressType = 5 +) + +// encodeDNSAddr encodes a DNS address. +func encodeDNSAddr(w io.Writer, addr *lnwire.DNSAddress) error { + if _, err := w.Write([]byte{byte(dnsAddr)}); err != nil { + return err + } + + // Write the length of the hostname. + hostLen := len(addr.Hostname) + if _, err := w.Write([]byte{byte(hostLen)}); err != nil { + return err + } + + if _, err := w.Write([]byte(addr.Hostname)); err != nil { + return err + } + + var port [2]byte + byteOrder.PutUint16(port[:], addr.Port) + if _, err := w.Write(port[:]); err != nil { + return err + } + + return nil +} + +// encodeTCPAddr serializes a TCP address into its compact raw bytes +// representation. +func encodeTCPAddr(w io.Writer, addr *net.TCPAddr) error { + var ( + addrType byte + ip []byte + ) + + if addr.IP.To4() != nil { + addrType = byte(tcp4Addr) + ip = addr.IP.To4() + } else { + addrType = byte(tcp6Addr) + ip = addr.IP.To16() + } + + if ip == nil { + return fmt.Errorf("unable to encode IP %v", addr.IP) + } + + if _, err := w.Write([]byte{addrType}); err != nil { + return err + } + + if _, err := w.Write(ip); err != nil { + return err + } + + var port [2]byte + byteOrder.PutUint16(port[:], uint16(addr.Port)) + if _, err := w.Write(port[:]); err != nil { + return err + } + + return nil +} + +// encodeOnionAddr serializes an onion address into its compact raw bytes +// representation. +func encodeOnionAddr(w io.Writer, addr *tor.OnionAddr) error { + var suffixIndex int + hostLen := len(addr.OnionService) + switch hostLen { + case tor.V2Len: + if _, err := w.Write([]byte{byte(v2OnionAddr)}); err != nil { + return err + } + suffixIndex = tor.V2Len - tor.OnionSuffixLen + case tor.V3Len: + if _, err := w.Write([]byte{byte(v3OnionAddr)}); err != nil { + return err + } + suffixIndex = tor.V3Len - tor.OnionSuffixLen + default: + return errors.New("unknown onion service length") + } + + suffix := addr.OnionService[suffixIndex:] + if suffix != tor.OnionSuffix { + return fmt.Errorf("invalid suffix \"%v\"", suffix) + } + + host, err := tor.Base32Encoding.DecodeString( + addr.OnionService[:suffixIndex], + ) + if err != nil { + return err + } + + // Sanity check the decoded length. + switch { + case hostLen == tor.V2Len && len(host) != tor.V2DecodedLen: + return fmt.Errorf("onion service %v decoded to invalid host %x", + addr.OnionService, host) + + case hostLen == tor.V3Len && len(host) != tor.V3DecodedLen: + return fmt.Errorf("onion service %v decoded to invalid host %x", + addr.OnionService, host) + } + + if _, err := w.Write(host); err != nil { + return err + } + + var port [2]byte + byteOrder.PutUint16(port[:], uint16(addr.Port)) + if _, err := w.Write(port[:]); err != nil { + return err + } + + return nil +} + +// encodeOpaqueAddrs serializes the lnwire.OpaqueAddrs type to a raw set of +// bytes that we will persist. +func encodeOpaqueAddrs(w io.Writer, addr *lnwire.OpaqueAddrs) error { + // Write the type byte. + if _, err := w.Write([]byte{byte(opaqueAddrs)}); err != nil { + return err + } + + // Write the length of the payload. + var l [2]byte + binary.BigEndian.PutUint16(l[:], uint16(len(addr.Payload))) + if _, err := w.Write(l[:]); err != nil { + return err + } + + // Write the payload. + _, err := w.Write(addr.Payload) + + return err +} + +// DeserializeAddr reads the serialized raw representation of an address and +// deserializes it into the actual address. This allows us to avoid address +// resolution within the channeldb package. +func DeserializeAddr(r io.Reader) (net.Addr, error) { + var addrType [1]byte + if _, err := r.Read(addrType[:]); err != nil { + return nil, err + } + + var address net.Addr + switch addressType(addrType[0]) { + case tcp4Addr: + var ip [4]byte + if _, err := r.Read(ip[:]); err != nil { + return nil, err + } + + var port [2]byte + if _, err := r.Read(port[:]); err != nil { + return nil, err + } + + address = &net.TCPAddr{ + IP: net.IP(ip[:]), + Port: int(binary.BigEndian.Uint16(port[:])), + } + + case tcp6Addr: + var ip [16]byte + if _, err := r.Read(ip[:]); err != nil { + return nil, err + } + + var port [2]byte + if _, err := r.Read(port[:]); err != nil { + return nil, err + } + + address = &net.TCPAddr{ + IP: net.IP(ip[:]), + Port: int(binary.BigEndian.Uint16(port[:])), + } + + case v2OnionAddr: + var h [tor.V2DecodedLen]byte + if _, err := r.Read(h[:]); err != nil { + return nil, err + } + + var p [2]byte + if _, err := r.Read(p[:]); err != nil { + return nil, err + } + + onionService := tor.Base32Encoding.EncodeToString(h[:]) + onionService += tor.OnionSuffix + port := int(binary.BigEndian.Uint16(p[:])) + + address = &tor.OnionAddr{ + OnionService: onionService, + Port: port, + } + + case v3OnionAddr: + var h [tor.V3DecodedLen]byte + if _, err := r.Read(h[:]); err != nil { + return nil, err + } + + var p [2]byte + if _, err := r.Read(p[:]); err != nil { + return nil, err + } + + onionService := tor.Base32Encoding.EncodeToString(h[:]) + onionService += tor.OnionSuffix + port := int(binary.BigEndian.Uint16(p[:])) + + address = &tor.OnionAddr{ + OnionService: onionService, + Port: port, + } + + case dnsAddr: + // Read the length of the hostname. + var hostLen [1]byte + if _, err := r.Read(hostLen[:]); err != nil { + return nil, err + } + + // Read the hostname. + hostname := make([]byte, hostLen[0]) + if _, err := r.Read(hostname); err != nil { + return nil, err + } + + // Read the port. + var port [2]byte + if _, err := r.Read(port[:]); err != nil { + return nil, err + } + + address = &lnwire.DNSAddress{ + Hostname: string(hostname), + Port: binary.BigEndian.Uint16(port[:]), + } + + case opaqueAddrs: + // Read the length of the payload. + var l [2]byte + if _, err := r.Read(l[:]); err != nil { + return nil, err + } + + // Read the payload. + payload := make([]byte, binary.BigEndian.Uint16(l[:])) + if _, err := r.Read(payload); err != nil { + return nil, err + } + + address = &lnwire.OpaqueAddrs{ + Payload: payload, + } + + default: + return nil, ErrUnknownAddressType + } + + return address, nil +} + +// SerializeAddr serializes an address into its raw bytes representation so that +// it can be deserialized without requiring address resolution. +func SerializeAddr(w io.Writer, address net.Addr) error { + switch addr := address.(type) { + case *net.TCPAddr: + return encodeTCPAddr(w, addr) + case *tor.OnionAddr: + return encodeOnionAddr(w, addr) + case *lnwire.OpaqueAddrs: + return encodeOpaqueAddrs(w, addr) + case *lnwire.DNSAddress: + return encodeDNSAddr(w, addr) + default: + return ErrUnknownAddressType + } +} diff --git a/graph/db/migration1/codec.go b/graph/db/migration1/codec.go new file mode 100644 index 000000000..243308f6c --- /dev/null +++ b/graph/db/migration1/codec.go @@ -0,0 +1,80 @@ +package migration1 + +import ( + "encoding/binary" + "fmt" + "image/color" + "io" + "strconv" + + "github.com/btcsuite/btcd/wire" +) + +var ( + // byteOrder defines the preferred byte order, which is Big Endian. + byteOrder = binary.BigEndian +) + +// WriteOutpoint writes an outpoint to the passed writer using the minimal +// amount of bytes possible. +func WriteOutpoint(w io.Writer, o *wire.OutPoint) error { + if _, err := w.Write(o.Hash[:]); err != nil { + return err + } + if err := binary.Write(w, byteOrder, o.Index); err != nil { + return err + } + + return nil +} + +// ReadOutpoint reads an outpoint from the passed reader that was previously +// written using the WriteOutpoint struct. +func ReadOutpoint(r io.Reader, o *wire.OutPoint) error { + if _, err := io.ReadFull(r, o.Hash[:]); err != nil { + return err + } + if err := binary.Read(r, byteOrder, &o.Index); err != nil { + return err + } + + return nil +} + +// EncodeHexColor takes a color and returns it in hex code format. +func EncodeHexColor(color color.RGBA) string { + return fmt.Sprintf("#%02x%02x%02x", color.R, color.G, color.B) +} + +// DecodeHexColor takes a hex color string like "#rrggbb" and returns a +// color.RGBA. +func DecodeHexColor(hex string) (color.RGBA, error) { + if len(hex) != 7 || hex[0] != '#' { + return color.RGBA{}, fmt.Errorf("invalid hex color string: %s", + hex) + } + + r, err := strconv.ParseUint(hex[1:3], 16, 8) + if err != nil { + return color.RGBA{}, fmt.Errorf("invalid red component: %w", + err) + } + + g, err := strconv.ParseUint(hex[3:5], 16, 8) + if err != nil { + return color.RGBA{}, fmt.Errorf("invalid green component: %w", + err) + } + + b, err := strconv.ParseUint(hex[5:7], 16, 8) + if err != nil { + return color.RGBA{}, fmt.Errorf("invalid blue component: %w", + err) + } + + return color.RGBA{ + R: uint8(r), + G: uint8(g), + B: uint8(b), + }, nil +} diff --git a/graph/db/migration1/errors.go b/graph/db/migration1/errors.go new file mode 100644 index 000000000..745119775 --- /dev/null +++ b/graph/db/migration1/errors.go @@ -0,0 +1,84 @@ +package migration1 + +import ( + "errors" + "fmt" +) + +var ( + // ErrEdgePolicyOptionalFieldNotFound is an error returned if a channel + // policy field is not found in the db even though its message flags + // indicate it should be. + ErrEdgePolicyOptionalFieldNotFound = fmt.Errorf("optional field not " + + "present") + + // ErrParsingExtraTLVBytes is returned when we attempt to parse + // extra opaque bytes as a TLV stream, but the parsing fails. + ErrParsingExtraTLVBytes = fmt.Errorf("error parsing extra TLV bytes") + + // ErrGraphNotFound is returned when at least one of the components of + // graph doesn't exist. + ErrGraphNotFound = fmt.Errorf("graph bucket not initialized") + + // ErrGraphNeverPruned is returned when graph was never pruned. + ErrGraphNeverPruned = fmt.Errorf("graph never pruned") + + // ErrSourceNodeNotSet is returned if the source node of the graph + // hasn't been added The source node is the center node within a + // star-graph. + ErrSourceNodeNotSet = fmt.Errorf("source node does not exist") + + // ErrGraphNodesNotFound is returned in case none of the nodes has + // been added in graph node bucket. + ErrGraphNodesNotFound = fmt.Errorf("no graph nodes exist") + + // ErrGraphNoEdgesFound is returned in case of none of the channel/edges + // has been added in graph edge bucket. + ErrGraphNoEdgesFound = fmt.Errorf("no graph edges exist") + + // ErrGraphNodeNotFound is returned when we're unable to find the target + // node. + ErrGraphNodeNotFound = fmt.Errorf("unable to find node") + + // ErrZombieEdge is an error returned when we attempt to look up an edge + // but it is marked as a zombie within the zombie index. + ErrZombieEdge = errors.New("edge marked as zombie") + + // ErrEdgeNotFound is returned when an edge for the target chanID + // can't be found. + ErrEdgeNotFound = fmt.Errorf("edge not found") + + // ErrEdgeAlreadyExist is returned when edge with specific + // channel id can't be added because it already exist. + ErrEdgeAlreadyExist = fmt.Errorf("edge already exist") + + // ErrNodeAliasNotFound is returned when alias for node can't be found. + ErrNodeAliasNotFound = fmt.Errorf("alias for node not found") + + // ErrClosedScidsNotFound is returned when the closed scid bucket + // hasn't been created. + ErrClosedScidsNotFound = fmt.Errorf("closed scid bucket doesn't exist") + + // ErrZombieEdgeNotFound is an error returned when we attempt to find an + // edge in the zombie index which is not there. + ErrZombieEdgeNotFound = errors.New("edge not found in zombie index") + + // ErrUnknownAddressType is returned when a node's addressType is not + // an expected value. + ErrUnknownAddressType = fmt.Errorf("address type cannot be resolved") + + // ErrCantCheckIfZombieEdgeStr is an error returned when we + // attempt to check if an edge is a zombie but encounter an error. + ErrCantCheckIfZombieEdgeStr = fmt.Errorf("unable to check if edge " + + "is a zombie") +) + +// ErrTooManyExtraOpaqueBytes creates an error which should be returned if the +// caller attempts to write an announcement message which bares too many extra +// opaque bytes. We limit this value in order to ensure that we don't waste +// disk space due to nodes unnecessarily padding out their announcements with +// garbage data. +func ErrTooManyExtraOpaqueBytes(numBytes int) error { + return fmt.Errorf("max allowed number of opaque bytes is %v, received "+ + "%v bytes", MaxAllowedExtraOpaqueBytes, numBytes) +} diff --git a/graph/db/migration1/interfaces.go b/graph/db/migration1/interfaces.go new file mode 100644 index 000000000..da3f2f258 --- /dev/null +++ b/graph/db/migration1/interfaces.go @@ -0,0 +1,37 @@ +package migration1 + +import ( + "context" + + "github.com/lightningnetwork/lnd/graph/db/migration1/models" +) + +// V1Store represents the main interface for the channel graph database for all +// channels and nodes gossiped via the V1 gossip protocol as defined in BOLT 7. +type V1Store interface { + // ForEachNode iterates through all the stored vertices/nodes in the + // graph, executing the passed callback with each node encountered. If + // the callback returns an error, then the transaction is aborted and + // the iteration stops early. + ForEachNode(ctx context.Context, cb func(*models.Node) error, + reset func()) error + + // ForEachChannel iterates through all the channel edges stored within + // the graph and invokes the passed callback for each edge. The callback + // takes two edges as since this is a directed graph, both the in/out + // edges are visited. If the callback returns an error, then the + // transaction is aborted and the iteration stops early. + // + // NOTE: If an edge can't be found, or wasn't advertised, then a nil + // pointer for that particular channel edge routing policy will be + // passed into the callback. + ForEachChannel(ctx context.Context, cb func(*models.ChannelEdgeInfo, + *models.ChannelEdgePolicy, *models.ChannelEdgePolicy) error, + reset func()) error + + // SourceNode returns the source node of the graph. The source node is + // treated as the center node within a star-graph. This method may be + // used to kick off a path finding algorithm in order to explore the + // reachability of another node based off the source node. + SourceNode(ctx context.Context) (*models.Node, error) +} diff --git a/graph/db/migration1/kv_store.go b/graph/db/migration1/kv_store.go new file mode 100644 index 000000000..1146ec758 --- /dev/null +++ b/graph/db/migration1/kv_store.go @@ -0,0 +1,2154 @@ +package migration1 + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "image/color" + "io" + "net" + "time" + + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/wire" + "github.com/lightningnetwork/lnd/batch" + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/lightningnetwork/lnd/graph/db/migration1/models" + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/routing/route" +) + +var ( + // nodeBucket is a bucket which houses all the vertices or nodes within + // the channel graph. This bucket has a single-sub bucket which adds an + // additional index from pubkey -> alias. Within the top-level of this + // bucket, the key space maps a node's compressed public key to the + // serialized information for that node. Additionally, there's a + // special key "source" which stores the pubkey of the source node. The + // source node is used as the starting point for all graph/queries and + // traversals. The graph is formed as a star-graph with the source node + // at the center. + // + // maps: pubKey -> nodeInfo + // maps: source -> selfPubKey + nodeBucket = []byte("graph-node") + + // nodeUpdateIndexBucket is a sub-bucket of the nodeBucket. This bucket + // will be used to quickly look up the "freshness" of a node's last + // update to the network. The bucket only contains keys, and no values, + // it's mapping: + // + // maps: updateTime || nodeID -> nil + nodeUpdateIndexBucket = []byte("graph-node-update-index") + + // sourceKey is a special key that resides within the nodeBucket. The + // sourceKey maps a key to the public key of the "self node". + sourceKey = []byte("source") + + // aliasIndexBucket is a sub-bucket that's nested within the main + // nodeBucket. This bucket maps the public key of a node to its + // current alias. This bucket is provided as it can be used within a + // future UI layer to add an additional degree of confirmation. + aliasIndexBucket = []byte("alias") + + // edgeBucket is a bucket which houses all of the edge or channel + // information within the channel graph. This bucket essentially acts + // as an adjacency list, which in conjunction with a range scan, can be + // used to iterate over all the incoming and outgoing edges for a + // particular node. Key in the bucket use a prefix scheme which leads + // with the node's public key and sends with the compact edge ID. + // For each chanID, there will be two entries within the bucket, as the + // graph is directed: nodes may have different policies w.r.t to fees + // for their respective directions. + // + // maps: pubKey || chanID -> channel edge policy for node + edgeBucket = []byte("graph-edge") + + // unknownPolicy is represented as an empty slice. It is + // used as the value in edgeBucket for unknown channel edge policies. + // Unknown policies are still stored in the database to enable efficient + // lookup of incoming channel edges. + unknownPolicy = []byte{} + + // edgeIndexBucket is an index which can be used to iterate all edges + // in the bucket, grouping them according to their in/out nodes. + // Additionally, the items in this bucket also contain the complete + // edge information for a channel. The edge information includes the + // capacity of the channel, the nodes that made the channel, etc. This + // bucket resides within the edgeBucket above. Creation of an edge + // proceeds in two phases: first the edge is added to the edge index, + // afterwards the edgeBucket can be updated with the latest details of + // the edge as they are announced on the network. + // + // maps: chanID -> pubKey1 || pubKey2 || restofEdgeInfo + edgeIndexBucket = []byte("edge-index") + + // edgeUpdateIndexBucket is a sub-bucket of the main edgeBucket. This + // bucket contains an index which allows us to gauge the "freshness" of + // a channel's last updates. + // + // maps: updateTime || chanID -> nil + edgeUpdateIndexBucket = []byte("edge-update-index") + + // channelPointBucket maps a channel's full outpoint (txid:index) to + // its short 8-byte channel ID. This bucket resides within the + // edgeBucket above, and can be used to quickly remove an edge due to + // the outpoint being spent, or to query for existence of a channel. + // + // maps: outPoint -> chanID + channelPointBucket = []byte("chan-index") + + // zombieBucket is a sub-bucket of the main edgeBucket bucket + // responsible for maintaining an index of zombie channels. Each entry + // exists within the bucket as follows: + // + // maps: chanID -> pubKey1 || pubKey2 + // + // The chanID represents the channel ID of the edge that is marked as a + // zombie and is used as the key, which maps to the public keys of the + // edge's participants. + zombieBucket = []byte("zombie-index") + + // disabledEdgePolicyBucket is a sub-bucket of the main edgeBucket + // bucket responsible for maintaining an index of disabled edge + // policies. Each entry exists within the bucket as follows: + // + // maps: -> []byte{} + // + // The chanID represents the channel ID of the edge and the direction is + // one byte representing the direction of the edge. The main purpose of + // this index is to allow pruning disabled channels in a fast way + // without the need to iterate all over the graph. + disabledEdgePolicyBucket = []byte("disabled-edge-policy-index") + + // graphMetaBucket is a top-level bucket which stores various meta-deta + // related to the on-disk channel graph. Data stored in this bucket + // includes the block to which the graph has been synced to, the total + // number of channels, etc. + graphMetaBucket = []byte("graph-meta") + + // pruneLogBucket is a bucket within the graphMetaBucket that stores + // a mapping from the block height to the hash for the blocks used to + // prune the graph. + // Once a new block is discovered, any channels that have been closed + // (by spending the outpoint) can safely be removed from the graph, and + // the block is added to the prune log. We need to keep such a log for + // the case where a reorg happens, and we must "rewind" the state of the + // graph by removing channels that were previously confirmed. In such a + // case we'll remove all entries from the prune log with a block height + // that no longer exists. + pruneLogBucket = []byte("prune-log") + + // closedScidBucket is a top-level bucket that stores scids for + // channels that we know to be closed. This is used so that we don't + // need to perform expensive validation checks if we receive a channel + // announcement for the channel again. + // + // maps: scid -> []byte{} + closedScidBucket = []byte("closed-scid") +) + +const ( + // MaxAllowedExtraOpaqueBytes is the largest amount of opaque bytes that + // we'll permit to be written to disk. We limit this as otherwise, it + // would be possible for a node to create a ton of updates and slowly + // fill our disk, and also waste bandwidth due to relaying. + MaxAllowedExtraOpaqueBytes = 10000 +) + +// KVStore is a persistent, on-disk graph representation of the Lightning +// Network. This struct can be used to implement path finding algorithms on top +// of, and also to update a node's view based on information received from the +// p2p network. Internally, the graph is stored using a modified adjacency list +// representation with some added object interaction possible with each +// serialized edge/node. The graph is stored is directed, meaning that are two +// edges stored for each channel: an inbound/outbound edge for each node pair. +// Nodes, edges, and edge information can all be added to the graph +// independently. Edge removal results in the deletion of all edge information +// for that edge. +type KVStore struct { + db kvdb.Backend +} + +// A compile-time assertion to ensure that the KVStore struct implements the +// V1Store interface. +var _ V1Store = (*KVStore)(nil) + +// NewKVStore allocates a new KVStore backed by a DB instance. The +// returned instance has its own unique reject cache and channel cache. +func NewKVStore(db kvdb.Backend) (*KVStore, error) { + if err := initKVStore(db); err != nil { + return nil, err + } + + g := &KVStore{ + db: db, + } + + return g, nil +} + +// channelMapKey is the key structure used for storing channel edge policies. +type channelMapKey struct { + nodeKey route.Vertex + chanID [8]byte +} + +// String returns a human-readable representation of the key. +func (c channelMapKey) String() string { + return fmt.Sprintf("node=%v, chanID=%x", c.nodeKey, c.chanID) +} + +// getChannelMap loads all channel edge policies from the database and stores +// them in a map. +func getChannelMap(edges kvdb.RBucket) ( + map[channelMapKey]*models.ChannelEdgePolicy, error) { + + // Create a map to store all channel edge policies. + channelMap := make(map[channelMapKey]*models.ChannelEdgePolicy) + + err := kvdb.ForAll(edges, func(k, edgeBytes []byte) error { + // Skip embedded buckets. + if bytes.Equal(k, edgeIndexBucket) || + bytes.Equal(k, edgeUpdateIndexBucket) || + bytes.Equal(k, zombieBucket) || + bytes.Equal(k, disabledEdgePolicyBucket) || + bytes.Equal(k, channelPointBucket) { + + return nil + } + + // Validate key length. + if len(k) != 33+8 { + return fmt.Errorf("invalid edge key %x encountered", k) + } + + var key channelMapKey + copy(key.nodeKey[:], k[:33]) + copy(key.chanID[:], k[33:]) + + // No need to deserialize unknown policy. + if bytes.Equal(edgeBytes, unknownPolicy) { + return nil + } + + edgeReader := bytes.NewReader(edgeBytes) + edge, err := deserializeChanEdgePolicyRaw( + edgeReader, + ) + + switch { + // If the db policy was missing an expected optional field, we + // return nil as if the policy was unknown. + case errors.Is(err, ErrEdgePolicyOptionalFieldNotFound): + return nil + + // We don't want a single policy with bad TLV data to stop us + // from loading the rest of the data, so we just skip this + // policy. This is for backwards compatibility since we did not + // use to validate TLV data in the past before persisting it. + case errors.Is(err, ErrParsingExtraTLVBytes): + return nil + + case err != nil: + return err + } + + channelMap[key] = edge + + return nil + }) + if err != nil { + return nil, err + } + + return channelMap, nil +} + +var graphTopLevelBuckets = [][]byte{ + nodeBucket, + edgeBucket, + graphMetaBucket, + closedScidBucket, +} + +// createChannelDB creates and initializes a fresh version of In +// the case that the target path has not yet been created or doesn't yet exist, +// then the path is created. Additionally, all required top-level buckets used +// within the database are created. +func initKVStore(db kvdb.Backend) error { + err := kvdb.Update(db, func(tx kvdb.RwTx) error { + for _, tlb := range graphTopLevelBuckets { + if _, err := tx.CreateTopLevelBucket(tlb); err != nil { + return err + } + } + + nodes := tx.ReadWriteBucket(nodeBucket) + _, err := nodes.CreateBucketIfNotExists(aliasIndexBucket) + if err != nil { + return err + } + _, err = nodes.CreateBucketIfNotExists(nodeUpdateIndexBucket) + if err != nil { + return err + } + + edges := tx.ReadWriteBucket(edgeBucket) + _, err = edges.CreateBucketIfNotExists(edgeIndexBucket) + if err != nil { + return err + } + _, err = edges.CreateBucketIfNotExists(edgeUpdateIndexBucket) + if err != nil { + return err + } + _, err = edges.CreateBucketIfNotExists(channelPointBucket) + if err != nil { + return err + } + _, err = edges.CreateBucketIfNotExists(zombieBucket) + if err != nil { + return err + } + + graphMeta := tx.ReadWriteBucket(graphMetaBucket) + _, err = graphMeta.CreateBucketIfNotExists(pruneLogBucket) + + return err + }, func() {}) + if err != nil { + return fmt.Errorf("unable to create new channel graph: %w", err) + } + + return nil +} + +// SourceNode returns the source node of the graph. The source node is treated +// as the center node within a star-graph. This method may be used to kick off +// a path finding algorithm in order to explore the reachability of another +// node based off the source node. +func (c *KVStore) SourceNode(_ context.Context) (*models.Node, error) { + return sourceNode(c.db) +} + +// ForEachChannel iterates through all the channel edges stored within the +// graph and invokes the passed callback for each edge. The callback takes two +// edges as since this is a directed graph, both the in/out edges are visited. +// If the callback returns an error, then the transaction is aborted and the +// iteration stops early. +// +// NOTE: If an edge can't be found, or wasn't advertised, then a nil pointer +// for that particular channel edge routing policy will be passed into the +// callback. +func (c *KVStore) ForEachChannel(_ context.Context, + cb func(*models.ChannelEdgeInfo, *models.ChannelEdgePolicy, + *models.ChannelEdgePolicy) error, reset func()) error { + + return forEachChannel(c.db, cb, reset) +} + +// forEachChannel iterates through all the channel edges stored within the +// graph and invokes the passed callback for each edge. The callback takes two +// edges as since this is a directed graph, both the in/out edges are visited. +// If the callback returns an error, then the transaction is aborted and the +// iteration stops early. +// +// NOTE: If an edge can't be found, or wasn't advertised, then a nil pointer +// for that particular channel edge routing policy will be passed into the +// callback. +func forEachChannel(db kvdb.Backend, cb func(*models.ChannelEdgeInfo, + *models.ChannelEdgePolicy, *models.ChannelEdgePolicy) error, + reset func()) error { + + return db.View(func(tx kvdb.RTx) error { + edges := tx.ReadBucket(edgeBucket) + if edges == nil { + return ErrGraphNoEdgesFound + } + + // First, load all edges in memory indexed by node and channel + // id. + channelMap, err := getChannelMap(edges) + if err != nil { + return err + } + + edgeIndex := edges.NestedReadBucket(edgeIndexBucket) + if edgeIndex == nil { + return ErrGraphNoEdgesFound + } + + // Load edge index, recombine each channel with the policies + // loaded above and invoke the callback. + return kvdb.ForAll( + edgeIndex, func(k, edgeInfoBytes []byte) error { + var chanID [8]byte + copy(chanID[:], k) + + edgeInfoReader := bytes.NewReader(edgeInfoBytes) + info, err := deserializeChanEdgeInfo( + edgeInfoReader, + ) + if err != nil { + return err + } + + policy1 := channelMap[channelMapKey{ + nodeKey: info.NodeKey1Bytes, + chanID: chanID, + }] + + policy2 := channelMap[channelMapKey{ + nodeKey: info.NodeKey2Bytes, + chanID: chanID, + }] + + return cb(info, policy1, policy2) + }, + ) + }, reset) +} + +// ForEachNode iterates through all the stored vertices/nodes in the graph, +// executing the passed callback with each node encountered. If the callback +// returns an error, then the transaction is aborted and the iteration stops +// early. +// +// NOTE: this is part of the V1Store interface. +func (c *KVStore) ForEachNode(_ context.Context, + cb func(*models.Node) error, reset func()) error { + + return forEachNode(c.db, func(tx kvdb.RTx, + node *models.Node) error { + + return cb(node) + }, reset) +} + +// forEachNode iterates through all the stored vertices/nodes in the graph, +// executing the passed callback with each node encountered. If the callback +// returns an error, then the transaction is aborted and the iteration stops +// early. +// +// TODO(roasbeef): add iterator interface to allow for memory efficient graph +// traversal when graph gets mega. +func forEachNode(db kvdb.Backend, + cb func(kvdb.RTx, *models.Node) error, reset func()) error { + + traversal := func(tx kvdb.RTx) error { + // First grab the nodes bucket which stores the mapping from + // pubKey to node information. + nodes := tx.ReadBucket(nodeBucket) + if nodes == nil { + return ErrGraphNotFound + } + + return nodes.ForEach(func(pubKey, nodeBytes []byte) error { + // If this is the source key, then we skip this + // iteration as the value for this key is a pubKey + // rather than raw node information. + if bytes.Equal(pubKey, sourceKey) || len(pubKey) != 33 { + return nil + } + + nodeReader := bytes.NewReader(nodeBytes) + node, err := deserializeLightningNode(nodeReader) + if err != nil { + return err + } + + // Execute the callback, the transaction will abort if + // this returns an error. + return cb(tx, node) + }) + } + + return kvdb.View(db, traversal, reset) +} + +// sourceNode fetches the source node of the graph. The source node is treated +// as the center node within a star-graph. +func sourceNode(db kvdb.Backend) (*models.Node, error) { + var source *models.Node + err := kvdb.View(db, func(tx kvdb.RTx) error { + // First grab the nodes bucket which stores the mapping from + // pubKey to node information. + nodes := tx.ReadBucket(nodeBucket) + if nodes == nil { + return ErrGraphNotFound + } + + node, err := sourceNodeWithTx(nodes) + if err != nil { + return err + } + source = node + + return nil + }, func() { + source = nil + }) + if err != nil { + return nil, err + } + + return source, nil +} + +// sourceNodeWithTx uses an existing database transaction and returns the source +// node of the graph. The source node is treated as the center node within a +// star-graph. This method may be used to kick off a path finding algorithm in +// order to explore the reachability of another node based off the source node. +func sourceNodeWithTx(nodes kvdb.RBucket) (*models.Node, error) { + selfPub := nodes.Get(sourceKey) + if selfPub == nil { + return nil, ErrSourceNodeNotSet + } + + // With the pubKey of the source node retrieved, we're able to + // fetch the full node information. + return fetchLightningNode(nodes, selfPub) +} + +// SetSourceNode sets the source node within the graph database. The source +// node is to be used as the center of a star-graph within path finding +// algorithms. +func (c *KVStore) SetSourceNode(_ context.Context, + node *models.Node) error { + + nodePubBytes := node.PubKeyBytes[:] + + return kvdb.Update(c.db, func(tx kvdb.RwTx) error { + // First grab the nodes bucket which stores the mapping from + // pubKey to node information. + nodes, err := tx.CreateTopLevelBucket(nodeBucket) + if err != nil { + return err + } + + // Next we create the mapping from source to the targeted + // public key. + if err := nodes.Put(sourceKey, nodePubBytes); err != nil { + return err + } + + // Finally, we commit the information of the lightning node + // itself. + return addLightningNode(tx, node) + }, func() {}) +} + +// AddNode adds a vertex/node to the graph database. If the node is not +// in the database from before, this will add a new, unconnected one to the +// graph. If it is present from before, this will update that node's +// information. Note that this method is expected to only be called to update an +// already present node from a node announcement, or to insert a node found in a +// channel update. +// +// TODO(roasbeef): also need sig of announcement. +func (c *KVStore) AddNode(_ context.Context, + node *models.Node, _ ...batch.SchedulerOption) error { + + return kvdb.Update(c.db, func(tx kvdb.RwTx) error { + return addLightningNode(tx, node) + }, func() {}) +} + +func addLightningNode(tx kvdb.RwTx, node *models.Node) error { + nodes, err := tx.CreateTopLevelBucket(nodeBucket) + if err != nil { + return err + } + + aliases, err := nodes.CreateBucketIfNotExists(aliasIndexBucket) + if err != nil { + return err + } + + updateIndex, err := nodes.CreateBucketIfNotExists( + nodeUpdateIndexBucket, + ) + if err != nil { + return err + } + + return putLightningNode(nodes, aliases, updateIndex, node) +} + +// deleteLightningNode uses an existing database transaction to remove a +// vertex/node from the database according to the node's public key. +func (c *KVStore) deleteLightningNode(nodes kvdb.RwBucket, + compressedPubKey []byte) error { + + aliases := nodes.NestedReadWriteBucket(aliasIndexBucket) + if aliases == nil { + return ErrGraphNodesNotFound + } + + if err := aliases.Delete(compressedPubKey); err != nil { + return err + } + + // Before we delete the node, we'll fetch its current state so we can + // determine when its last update was to clear out the node update + // index. + node, err := fetchLightningNode(nodes, compressedPubKey) + if err != nil { + return err + } + + if err := nodes.Delete(compressedPubKey); err != nil { + return err + } + + // Finally, we'll delete the index entry for the node within the + // nodeUpdateIndexBucket as this node is no longer active, so we don't + // need to track its last update. + nodeUpdateIndex := nodes.NestedReadWriteBucket(nodeUpdateIndexBucket) + if nodeUpdateIndex == nil { + return ErrGraphNodesNotFound + } + + // In order to delete the entry, we'll need to reconstruct the key for + // its last update. + updateUnix := uint64(node.LastUpdate.Unix()) + var indexKey [8 + 33]byte + byteOrder.PutUint64(indexKey[:8], updateUnix) + copy(indexKey[8:], compressedPubKey) + + return nodeUpdateIndex.Delete(indexKey[:]) +} + +// AddChannelEdge adds a new (undirected, blank) edge to the graph database. An +// undirected edge from the two target nodes are created. The information stored +// denotes the static attributes of the channel, such as the channelID, the keys +// involved in creation of the channel, and the set of features that the channel +// supports. The chanPoint and chanID are used to uniquely identify the edge +// globally within the database. +func (c *KVStore) AddChannelEdge(_ context.Context, + edge *models.ChannelEdgeInfo, _ ...batch.SchedulerOption) error { + + return kvdb.Update(c.db, func(tx kvdb.RwTx) error { + return c.addChannelEdge(tx, edge) + }, func() {}) +} + +// addChannelEdge is the private form of AddChannelEdge that allows callers to +// utilize an existing db transaction. +func (c *KVStore) addChannelEdge(tx kvdb.RwTx, + edge *models.ChannelEdgeInfo) error { + + // Construct the channel's primary key which is the 8-byte channel ID. + var chanKey [8]byte + binary.BigEndian.PutUint64(chanKey[:], edge.ChannelID) + + nodes, err := tx.CreateTopLevelBucket(nodeBucket) + if err != nil { + return err + } + edges, err := tx.CreateTopLevelBucket(edgeBucket) + if err != nil { + return err + } + edgeIndex, err := edges.CreateBucketIfNotExists(edgeIndexBucket) + if err != nil { + return err + } + chanIndex, err := edges.CreateBucketIfNotExists(channelPointBucket) + if err != nil { + return err + } + + // First, attempt to check if this edge has already been created. If + // so, then we can exit early as this method is meant to be idempotent. + if edgeInfo := edgeIndex.Get(chanKey[:]); edgeInfo != nil { + return ErrEdgeAlreadyExist + } + + // Before we insert the channel into the database, we'll ensure that + // both nodes already exist in the channel graph. If either node + // doesn't, then we'll insert a "shell" node that just includes its + // public key, so subsequent validation and queries can work properly. + _, node1Err := fetchLightningNode(nodes, edge.NodeKey1Bytes[:]) + switch { + case errors.Is(node1Err, ErrGraphNodeNotFound): + err := addLightningNode( + tx, models.NewV1ShellNode(edge.NodeKey1Bytes), + ) + if err != nil { + return fmt.Errorf("unable to create shell node "+ + "for: %x: %w", edge.NodeKey1Bytes, err) + } + case node1Err != nil: + return node1Err + } + + _, node2Err := fetchLightningNode(nodes, edge.NodeKey2Bytes[:]) + switch { + case errors.Is(node2Err, ErrGraphNodeNotFound): + err := addLightningNode( + tx, models.NewV1ShellNode(edge.NodeKey2Bytes), + ) + if err != nil { + return fmt.Errorf("unable to create shell node "+ + "for: %x: %w", edge.NodeKey2Bytes, err) + } + case node2Err != nil: + return node2Err + } + + // If the edge hasn't been created yet, then we'll first add it to the + // edge index in order to associate the edge between two nodes and also + // store the static components of the channel. + if err := putChanEdgeInfo(edgeIndex, edge, chanKey); err != nil { + return err + } + + // Mark edge policies for both sides as unknown. This is to enable + // efficient incoming channel lookup for a node. + keys := []*[33]byte{ + &edge.NodeKey1Bytes, + &edge.NodeKey2Bytes, + } + for _, key := range keys { + err := putChanEdgePolicyUnknown(edges, edge.ChannelID, key[:]) + if err != nil { + return err + } + } + + // Finally we add it to the channel index which maps channel points + // (outpoints) to the shorter channel ID's. + var b bytes.Buffer + if err := WriteOutpoint(&b, &edge.ChannelPoint); err != nil { + return err + } + + return chanIndex.Put(b.Bytes(), chanKey[:]) +} + +const ( + // pruneTipBytes is the total size of the value which stores a prune + // entry of the graph in the prune log. The "prune tip" is the last + // entry in the prune log, and indicates if the channel graph is in + // sync with the current UTXO state. The structure of the value + // is: blockHash, taking 32 bytes total. + pruneTipBytes = 32 +) + +// PruneGraph prunes newly closed channels from the channel graph in response +// to a new block being solved on the network. Any transactions which spend the +// funding output of any known channels within he graph will be deleted. +// Additionally, the "prune tip", or the last block which has been used to +// prune the graph is stored so callers can ensure the graph is fully in sync +// with the current UTXO state. A slice of channels that have been closed by +// the target block along with any pruned nodes are returned if the function +// succeeds without error. +func (c *KVStore) PruneGraph(spentOutputs []*wire.OutPoint, + blockHash *chainhash.Hash, blockHeight uint32) ( + []*models.ChannelEdgeInfo, []route.Vertex, error) { + + var ( + chansClosed []*models.ChannelEdgeInfo + prunedNodes []route.Vertex + ) + + err := kvdb.Update(c.db, func(tx kvdb.RwTx) error { + // First grab the edges bucket which houses the information + // we'd like to delete + edges, err := tx.CreateTopLevelBucket(edgeBucket) + if err != nil { + return err + } + + // Next grab the two edge indexes which will also need to be + // updated. + edgeIndex, err := edges.CreateBucketIfNotExists(edgeIndexBucket) + if err != nil { + return err + } + chanIndex, err := edges.CreateBucketIfNotExists( + channelPointBucket, + ) + if err != nil { + return err + } + nodes := tx.ReadWriteBucket(nodeBucket) + if nodes == nil { + return ErrSourceNodeNotSet + } + zombieIndex, err := edges.CreateBucketIfNotExists(zombieBucket) + if err != nil { + return err + } + + // For each of the outpoints that have been spent within the + // block, we attempt to delete them from the graph as if that + // outpoint was a channel, then it has now been closed. + for _, chanPoint := range spentOutputs { + // TODO(roasbeef): load channel bloom filter, continue + // if NOT if filter + + var opBytes bytes.Buffer + err := WriteOutpoint(&opBytes, chanPoint) + if err != nil { + return err + } + + // First attempt to see if the channel exists within + // the database, if not, then we can exit early. + chanID := chanIndex.Get(opBytes.Bytes()) + if chanID == nil { + continue + } + + // Attempt to delete the channel, an ErrEdgeNotFound + // will be returned if that outpoint isn't known to be + // a channel. If no error is returned, then a channel + // was successfully pruned. + edgeInfo, err := c.delChannelEdgeUnsafe( + edges, edgeIndex, chanIndex, zombieIndex, + chanID, false, false, + ) + if err != nil && !errors.Is(err, ErrEdgeNotFound) { + return err + } + + chansClosed = append(chansClosed, edgeInfo) + } + + metaBucket, err := tx.CreateTopLevelBucket(graphMetaBucket) + if err != nil { + return err + } + + pruneBucket, err := metaBucket.CreateBucketIfNotExists( + pruneLogBucket, + ) + if err != nil { + return err + } + + // With the graph pruned, add a new entry to the prune log, + // which can be used to check if the graph is fully synced with + // the current UTXO state. + var blockHeightBytes [4]byte + byteOrder.PutUint32(blockHeightBytes[:], blockHeight) + + var newTip [pruneTipBytes]byte + copy(newTip[:], blockHash[:]) + + err = pruneBucket.Put(blockHeightBytes[:], newTip[:]) + if err != nil { + return err + } + + // Now that the graph has been pruned, we'll also attempt to + // prune any nodes that have had a channel closed within the + // latest block. + prunedNodes, err = c.pruneGraphNodes(nodes, edgeIndex) + + return err + }, func() { + chansClosed = nil + prunedNodes = nil + }) + if err != nil { + return nil, nil, err + } + + return chansClosed, prunedNodes, nil +} + +// pruneGraphNodes attempts to remove any nodes from the graph who have had a +// channel closed within the current block. If the node still has existing +// channels in the graph, this will act as a no-op. +func (c *KVStore) pruneGraphNodes(nodes kvdb.RwBucket, + edgeIndex kvdb.RwBucket) ([]route.Vertex, error) { + + log.Trace("Pruning nodes from graph with no open channels") + + // We'll retrieve the graph's source node to ensure we don't remove it + // even if it no longer has any open channels. + sourceNode, err := sourceNodeWithTx(nodes) + if err != nil { + return nil, err + } + + // We'll use this map to keep count the number of references to a node + // in the graph. A node should only be removed once it has no more + // references in the graph. + nodeRefCounts := make(map[[33]byte]int) + err = nodes.ForEach(func(pubKey, nodeBytes []byte) error { + // If this is the source key, then we skip this + // iteration as the value for this key is a pubKey + // rather than raw node information. + if bytes.Equal(pubKey, sourceKey) || len(pubKey) != 33 { + return nil + } + + var nodePub [33]byte + copy(nodePub[:], pubKey) + nodeRefCounts[nodePub] = 0 + + return nil + }) + if err != nil { + return nil, err + } + + // To ensure we never delete the source node, we'll start off by + // bumping its ref count to 1. + nodeRefCounts[sourceNode.PubKeyBytes] = 1 + + // Next, we'll run through the edgeIndex which maps a channel ID to the + // edge info. We'll use this scan to populate our reference count map + // above. + err = edgeIndex.ForEach(func(chanID, edgeInfoBytes []byte) error { + // The first 66 bytes of the edge info contain the pubkeys of + // the nodes that this edge attaches. We'll extract them, and + // add them to the ref count map. + var node1, node2 [33]byte + copy(node1[:], edgeInfoBytes[:33]) + copy(node2[:], edgeInfoBytes[33:]) + + // With the nodes extracted, we'll increase the ref count of + // each of the nodes. + nodeRefCounts[node1]++ + nodeRefCounts[node2]++ + + return nil + }) + if err != nil { + return nil, err + } + + // Finally, we'll make a second pass over the set of nodes, and delete + // any nodes that have a ref count of zero. + var pruned []route.Vertex + for nodePubKey, refCount := range nodeRefCounts { + // If the ref count of the node isn't zero, then we can safely + // skip it as it still has edges to or from it within the + // graph. + if refCount != 0 { + continue + } + + // If we reach this point, then there are no longer any edges + // that connect this node, so we can delete it. + err := c.deleteLightningNode(nodes, nodePubKey[:]) + if err != nil { + if errors.Is(err, ErrGraphNodeNotFound) || + errors.Is(err, ErrGraphNodesNotFound) { + + log.Warnf("Unable to prune node %x from the "+ + "graph: %v", nodePubKey, err) + continue + } + + return nil, err + } + + log.Infof("Pruned unconnected node %x from channel graph", + nodePubKey[:]) + + pruned = append(pruned, nodePubKey) + } + + if len(pruned) > 0 { + log.Infof("Pruned %v unconnected nodes from the channel graph", + len(pruned)) + } + + return pruned, err +} + +// PruneTip returns the block height and hash of the latest block that has been +// used to prune channels in the graph. Knowing the "prune tip" allows callers +// to tell if the graph is currently in sync with the current best known UTXO +// state. +func (c *KVStore) PruneTip() (*chainhash.Hash, uint32, error) { + var ( + tipHash chainhash.Hash + tipHeight uint32 + ) + + err := kvdb.View(c.db, func(tx kvdb.RTx) error { + graphMeta := tx.ReadBucket(graphMetaBucket) + if graphMeta == nil { + return ErrGraphNotFound + } + pruneBucket := graphMeta.NestedReadBucket(pruneLogBucket) + if pruneBucket == nil { + return ErrGraphNeverPruned + } + + pruneCursor := pruneBucket.ReadCursor() + + // The prune key with the largest block height will be our + // prune tip. + k, v := pruneCursor.Last() + if k == nil { + return ErrGraphNeverPruned + } + + // Once we have the prune tip, the value will be the block hash, + // and the key the block height. + copy(tipHash[:], v) + tipHeight = byteOrder.Uint32(k) + + return nil + }, func() {}) + if err != nil { + return nil, 0, err + } + + return &tipHash, tipHeight, nil +} + +func delEdgeUpdateIndexEntry(edgesBucket kvdb.RwBucket, chanID uint64, + edge1, edge2 *models.ChannelEdgePolicy) error { + + // First, we'll fetch the edge update index bucket which currently + // stores an entry for the channel we're about to delete. + updateIndex := edgesBucket.NestedReadWriteBucket(edgeUpdateIndexBucket) + if updateIndex == nil { + // No edges in bucket, return early. + return nil + } + + // Now that we have the bucket, we'll attempt to construct a template + // for the index key: updateTime || chanid. + var indexKey [8 + 8]byte + byteOrder.PutUint64(indexKey[8:], chanID) + + // With the template constructed, we'll attempt to delete an entry that + // would have been created by both edges: we'll alternate the update + // times, as one may had overridden the other. + if edge1 != nil { + byteOrder.PutUint64( + indexKey[:8], uint64(edge1.LastUpdate.Unix()), + ) + if err := updateIndex.Delete(indexKey[:]); err != nil { + return err + } + } + + // We'll also attempt to delete the entry that may have been created by + // the second edge. + if edge2 != nil { + byteOrder.PutUint64( + indexKey[:8], uint64(edge2.LastUpdate.Unix()), + ) + if err := updateIndex.Delete(indexKey[:]); err != nil { + return err + } + } + + return nil +} + +// delChannelEdgeUnsafe deletes the edge with the given chanID from the graph +// cache. It then goes on to delete any policy info and edge info for this +// channel from the DB and finally, if isZombie is true, it will add an entry +// for this channel in the zombie index. +// +// NOTE: this method MUST only be called if the cacheMu has already been +// acquired. +func (c *KVStore) delChannelEdgeUnsafe(edges, edgeIndex, chanIndex, + zombieIndex kvdb.RwBucket, chanID []byte, isZombie, + strictZombie bool) (*models.ChannelEdgeInfo, error) { + + edgeInfo, err := fetchChanEdgeInfo(edgeIndex, chanID) + if err != nil { + return nil, err + } + + // We'll also remove the entry in the edge update index bucket before + // we delete the edges themselves so we can access their last update + // times. + cid := byteOrder.Uint64(chanID) + edge1, edge2, err := fetchChanEdgePolicies(edgeIndex, edges, chanID) + if err != nil { + return nil, err + } + err = delEdgeUpdateIndexEntry(edges, cid, edge1, edge2) + if err != nil { + return nil, err + } + + // The edge key is of the format pubKey || chanID. First we construct + // the latter half, populating the channel ID. + var edgeKey [33 + 8]byte + copy(edgeKey[33:], chanID) + + // With the latter half constructed, copy over the first public key to + // delete the edge in this direction, then the second to delete the + // edge in the opposite direction. + copy(edgeKey[:33], edgeInfo.NodeKey1Bytes[:]) + if edges.Get(edgeKey[:]) != nil { + if err := edges.Delete(edgeKey[:]); err != nil { + return nil, err + } + } + copy(edgeKey[:33], edgeInfo.NodeKey2Bytes[:]) + if edges.Get(edgeKey[:]) != nil { + if err := edges.Delete(edgeKey[:]); err != nil { + return nil, err + } + } + + // As part of deleting the edge we also remove all disabled entries + // from the edgePolicyDisabledIndex bucket. We do that for both + // directions. + err = updateEdgePolicyDisabledIndex(edges, cid, false, false) + if err != nil { + return nil, err + } + err = updateEdgePolicyDisabledIndex(edges, cid, true, false) + if err != nil { + return nil, err + } + + // With the edge data deleted, we can purge the information from the two + // edge indexes. + if err := edgeIndex.Delete(chanID); err != nil { + return nil, err + } + var b bytes.Buffer + if err := WriteOutpoint(&b, &edgeInfo.ChannelPoint); err != nil { + return nil, err + } + if err := chanIndex.Delete(b.Bytes()); err != nil { + return nil, err + } + + // Finally, we'll mark the edge as a zombie within our index if it's + // being removed due to the channel becoming a zombie. We do this to + // ensure we don't store unnecessary data for spent channels. + if !isZombie { + return edgeInfo, nil + } + + nodeKey1, nodeKey2 := edgeInfo.NodeKey1Bytes, edgeInfo.NodeKey2Bytes + if strictZombie { + var e1UpdateTime, e2UpdateTime *time.Time + if edge1 != nil { + e1UpdateTime = &edge1.LastUpdate + } + if edge2 != nil { + e2UpdateTime = &edge2.LastUpdate + } + + nodeKey1, nodeKey2 = makeZombiePubkeys( + edgeInfo.NodeKey1Bytes, edgeInfo.NodeKey2Bytes, + e1UpdateTime, e2UpdateTime, + ) + } + + return edgeInfo, markEdgeZombie( + zombieIndex, byteOrder.Uint64(chanID), nodeKey1, nodeKey2, + ) +} + +// makeZombiePubkeys derives the node pubkeys to store in the zombie index for a +// particular pair of channel policies. The return values are one of: +// 1. (pubkey1, pubkey2) +// 2. (pubkey1, blank) +// 3. (blank, pubkey2) +// +// A blank pubkey means that corresponding node will be unable to resurrect a +// channel on its own. For example, node1 may continue to publish recent +// updates, but node2 has fallen way behind. After marking an edge as a zombie, +// we don't want another fresh update from node1 to resurrect, as the edge can +// only become live once node2 finally sends something recent. +// +// In the case where we have neither update, we allow either party to resurrect +// the channel. If the channel were to be marked zombie again, it would be +// marked with the correct lagging channel since we received an update from only +// one side. +func makeZombiePubkeys(node1, node2 [33]byte, e1, e2 *time.Time) ([33]byte, + [33]byte) { + + switch { + // If we don't have either edge policy, we'll return both pubkeys so + // that the channel can be resurrected by either party. + case e1 == nil && e2 == nil: + return node1, node2 + + // If we're missing edge1, or if both edges are present but edge1 is + // older, we'll return edge1's pubkey and a blank pubkey for edge2. This + // means that only an update from edge1 will be able to resurrect the + // channel. + case e1 == nil || (e2 != nil && e1.Before(*e2)): + return node1, [33]byte{} + + // Otherwise, we're missing edge2 or edge2 is the older side, so we + // return a blank pubkey for edge1. In this case, only an update from + // edge2 can resurect the channel. + default: + return [33]byte{}, node1 + } +} + +// UpdateEdgePolicy updates the edge routing policy for a single directed edge +// within the database for the referenced channel. The `flags` attribute within +// the ChannelEdgePolicy determines which of the directed edges are being +// updated. If the flag is 1, then the first node's information is being +// updated, otherwise it's the second node's information. The node ordering is +// determined by the lexicographical ordering of the identity public keys of the +// nodes on either side of the channel. +func (c *KVStore) UpdateEdgePolicy(_ context.Context, + edge *models.ChannelEdgePolicy, + _ ...batch.SchedulerOption) (route.Vertex, route.Vertex, error) { + + var from, to route.Vertex + err := kvdb.Update(c.db, func(tx kvdb.RwTx) error { + // Validate that the ExtraOpaqueData is in fact a valid + // TLV stream. This is done here instead of within + // updateEdgePolicy so that updateEdgePolicy can be used + // by unit tests to recreate the case where we already + // have nodes persisted with invalid TLV data. + err := edge.ExtraOpaqueData.ValidateTLV() + if err != nil { + return fmt.Errorf("%w: %w", + ErrParsingExtraTLVBytes, err) + } + + from, to, _, err = updateEdgePolicy(tx, edge) + + return err + }, func() {}) + + return from, to, err +} + +// updateEdgePolicy attempts to update an edge's policy within the relevant +// buckets using an existing database transaction. The returned boolean will be +// true if the updated policy belongs to node1, and false if the policy belonged +// to node2. +func updateEdgePolicy(tx kvdb.RwTx, edge *models.ChannelEdgePolicy) ( + route.Vertex, route.Vertex, bool, error) { + + var noVertex route.Vertex + + edges := tx.ReadWriteBucket(edgeBucket) + if edges == nil { + return noVertex, noVertex, false, ErrEdgeNotFound + } + edgeIndex := edges.NestedReadWriteBucket(edgeIndexBucket) + if edgeIndex == nil { + return noVertex, noVertex, false, ErrEdgeNotFound + } + + // Create the channelID key be converting the channel ID + // integer into a byte slice. + var chanID [8]byte + byteOrder.PutUint64(chanID[:], edge.ChannelID) + + // With the channel ID, we then fetch the value storing the two + // nodes which connect this channel edge. + nodeInfo := edgeIndex.Get(chanID[:]) + if nodeInfo == nil { + return noVertex, noVertex, false, ErrEdgeNotFound + } + + // Depending on the flags value passed above, either the first + // or second edge policy is being updated. + var fromNode, toNode []byte + var isUpdate1 bool + if edge.ChannelFlags&lnwire.ChanUpdateDirection == 0 { + fromNode = nodeInfo[:33] + toNode = nodeInfo[33:66] + isUpdate1 = true + } else { + fromNode = nodeInfo[33:66] + toNode = nodeInfo[:33] + isUpdate1 = false + } + + // Finally, with the direction of the edge being updated + // identified, we update the on-disk edge representation. + err := putChanEdgePolicy(edges, edge, fromNode, toNode) + if err != nil { + return noVertex, noVertex, false, err + } + + var ( + fromNodePubKey route.Vertex + toNodePubKey route.Vertex + ) + copy(fromNodePubKey[:], fromNode) + copy(toNodePubKey[:], toNode) + + return fromNodePubKey, toNodePubKey, isUpdate1, nil +} + +// MarkEdgeZombie attempts to mark a channel identified by its channel ID as a +// zombie. This method is used on an ad-hoc basis, when channels need to be +// marked as zombies outside the normal pruning cycle. +func (c *KVStore) MarkEdgeZombie(chanID uint64, + pubKey1, pubKey2 [33]byte) error { + + err := kvdb.Batch(c.db, func(tx kvdb.RwTx) error { + edges := tx.ReadWriteBucket(edgeBucket) + if edges == nil { + return ErrGraphNoEdgesFound + } + zombieIndex, err := edges.CreateBucketIfNotExists(zombieBucket) + if err != nil { + return fmt.Errorf("unable to create zombie "+ + "bucket: %w", err) + } + + return markEdgeZombie(zombieIndex, chanID, pubKey1, pubKey2) + }) + if err != nil { + return err + } + + return nil +} + +// markEdgeZombie marks an edge as a zombie within our zombie index. The public +// keys should represent the node public keys of the two parties involved in the +// edge. +func markEdgeZombie(zombieIndex kvdb.RwBucket, chanID uint64, pubKey1, + pubKey2 [33]byte) error { + + var k [8]byte + byteOrder.PutUint64(k[:], chanID) + + var v [66]byte + copy(v[:33], pubKey1[:]) + copy(v[33:], pubKey2[:]) + + return zombieIndex.Put(k[:], v[:]) +} + +// PutClosedScid stores a SCID for a closed channel in the database. This is so +// that we can ignore channel announcements that we know to be closed without +// having to validate them and fetch a block. +func (c *KVStore) PutClosedScid(scid lnwire.ShortChannelID) error { + return kvdb.Update(c.db, func(tx kvdb.RwTx) error { + closedScids, err := tx.CreateTopLevelBucket(closedScidBucket) + if err != nil { + return err + } + + var k [8]byte + byteOrder.PutUint64(k[:], scid.ToUint64()) + + return closedScids.Put(k[:], []byte{}) + }, func() {}) +} + +func putLightningNode(nodeBucket, aliasBucket, updateIndex kvdb.RwBucket, + node *models.Node) error { + + var ( + scratch [16]byte + b bytes.Buffer + ) + + pub, err := node.PubKey() + if err != nil { + return err + } + nodePub := pub.SerializeCompressed() + + // If the node has the update time set, write it, else write 0. + updateUnix := uint64(0) + if node.LastUpdate.Unix() > 0 { + updateUnix = uint64(node.LastUpdate.Unix()) + } + + byteOrder.PutUint64(scratch[:8], updateUnix) + if _, err := b.Write(scratch[:8]); err != nil { + return err + } + + if _, err := b.Write(nodePub); err != nil { + return err + } + + // If we got a node announcement for this node, we will have the rest + // of the data available. If not we don't have more data to write. + if !node.HaveAnnouncement() { + // Write HaveNodeAnnouncement=0. + byteOrder.PutUint16(scratch[:2], 0) + if _, err := b.Write(scratch[:2]); err != nil { + return err + } + + return nodeBucket.Put(nodePub, b.Bytes()) + } + + // Write HaveNodeAnnouncement=1. + byteOrder.PutUint16(scratch[:2], 1) + if _, err := b.Write(scratch[:2]); err != nil { + return err + } + + nodeColor := node.Color.UnwrapOr(color.RGBA{}) + + if err := binary.Write(&b, byteOrder, nodeColor.R); err != nil { + return err + } + if err := binary.Write(&b, byteOrder, nodeColor.G); err != nil { + return err + } + if err := binary.Write(&b, byteOrder, nodeColor.B); err != nil { + return err + } + + err = wire.WriteVarString(&b, 0, node.Alias.UnwrapOr("")) + if err != nil { + return err + } + + if err := node.Features.Encode(&b); err != nil { + return err + } + + numAddresses := uint16(len(node.Addresses)) + byteOrder.PutUint16(scratch[:2], numAddresses) + if _, err := b.Write(scratch[:2]); err != nil { + return err + } + + for _, address := range node.Addresses { + if err := SerializeAddr(&b, address); err != nil { + return err + } + } + + sigLen := len(node.AuthSigBytes) + if sigLen > 80 { + return fmt.Errorf("max sig len allowed is 80, had %v", + sigLen) + } + + err = wire.WriteVarBytes(&b, 0, node.AuthSigBytes) + if err != nil { + return err + } + + if len(node.ExtraOpaqueData) > MaxAllowedExtraOpaqueBytes { + return ErrTooManyExtraOpaqueBytes(len(node.ExtraOpaqueData)) + } + err = wire.WriteVarBytes(&b, 0, node.ExtraOpaqueData) + if err != nil { + return err + } + + err = aliasBucket.Put(nodePub, []byte(node.Alias.UnwrapOr(""))) + if err != nil { + return err + } + + // With the alias bucket updated, we'll now update the index that + // tracks the time series of node updates. + var indexKey [8 + 33]byte + byteOrder.PutUint64(indexKey[:8], updateUnix) + copy(indexKey[8:], nodePub) + + // If there was already an old index entry for this node, then we'll + // delete the old one before we write the new entry. + if nodeBytes := nodeBucket.Get(nodePub); nodeBytes != nil { + // Extract out the old update time to we can reconstruct the + // prior index key to delete it from the index. + oldUpdateTime := nodeBytes[:8] + + var oldIndexKey [8 + 33]byte + copy(oldIndexKey[:8], oldUpdateTime) + copy(oldIndexKey[8:], nodePub) + + if err := updateIndex.Delete(oldIndexKey[:]); err != nil { + return err + } + } + + if err := updateIndex.Put(indexKey[:], nil); err != nil { + return err + } + + return nodeBucket.Put(nodePub, b.Bytes()) +} + +func fetchLightningNode(nodeBucket kvdb.RBucket, + nodePub []byte) (*models.Node, error) { + + nodeBytes := nodeBucket.Get(nodePub) + if nodeBytes == nil { + return nil, ErrGraphNodeNotFound + } + + nodeReader := bytes.NewReader(nodeBytes) + + return deserializeLightningNode(nodeReader) +} + +func deserializeLightningNode(r io.Reader) (*models.Node, error) { + var ( + scratch [8]byte + err error + pubKey [33]byte + ) + + if _, err := r.Read(scratch[:]); err != nil { + return nil, err + } + + unix := int64(byteOrder.Uint64(scratch[:])) + lastUpdate := time.Unix(unix, 0) + + if _, err := io.ReadFull(r, pubKey[:]); err != nil { + return nil, err + } + + node := models.NewV1ShellNode(pubKey) + node.LastUpdate = lastUpdate + + if _, err := r.Read(scratch[:2]); err != nil { + return nil, err + } + + hasNodeAnn := byteOrder.Uint16(scratch[:2]) + // The rest of the data is optional, and will only be there if we got a + // node announcement for this node. + if hasNodeAnn == 0 { + return node, nil + } + + // We did get a node announcement for this node, so we'll have the rest + // of the data available. + var nodeColor color.RGBA + if err := binary.Read(r, byteOrder, &nodeColor.R); err != nil { + return nil, err + } + if err := binary.Read(r, byteOrder, &nodeColor.G); err != nil { + return nil, err + } + if err := binary.Read(r, byteOrder, &nodeColor.B); err != nil { + return nil, err + } + node.Color = fn.Some(nodeColor) + + alias, err := wire.ReadVarString(r, 0) + if err != nil { + return nil, err + } + node.Alias = fn.Some(alias) + + err = node.Features.Decode(r) + if err != nil { + return nil, err + } + + if _, err := r.Read(scratch[:2]); err != nil { + return nil, err + } + numAddresses := int(byteOrder.Uint16(scratch[:2])) + + var addresses []net.Addr + for i := 0; i < numAddresses; i++ { + address, err := DeserializeAddr(r) + if err != nil { + return nil, err + } + addresses = append(addresses, address) + } + node.Addresses = addresses + + node.AuthSigBytes, err = wire.ReadVarBytes(r, 0, 80, "sig") + if err != nil { + return nil, err + } + + // We'll try and see if there are any opaque bytes left, if not, then + // we'll ignore the EOF error and return the node as is. + extraBytes, err := wire.ReadVarBytes( + r, 0, MaxAllowedExtraOpaqueBytes, "blob", + ) + switch { + case errors.Is(err, io.ErrUnexpectedEOF): + case errors.Is(err, io.EOF): + case err != nil: + return nil, err + } + + if len(extraBytes) > 0 { + node.ExtraOpaqueData = extraBytes + } + + return node, nil +} + +func putChanEdgeInfo(edgeIndex kvdb.RwBucket, + edgeInfo *models.ChannelEdgeInfo, chanID [8]byte) error { + + var b bytes.Buffer + + if _, err := b.Write(edgeInfo.NodeKey1Bytes[:]); err != nil { + return err + } + if _, err := b.Write(edgeInfo.NodeKey2Bytes[:]); err != nil { + return err + } + if _, err := b.Write(edgeInfo.BitcoinKey1Bytes[:]); err != nil { + return err + } + if _, err := b.Write(edgeInfo.BitcoinKey2Bytes[:]); err != nil { + return err + } + + var featureBuf bytes.Buffer + if err := edgeInfo.Features.Encode(&featureBuf); err != nil { + return fmt.Errorf("unable to encode features: %w", err) + } + + if err := wire.WriteVarBytes(&b, 0, featureBuf.Bytes()); err != nil { + return err + } + + authProof := edgeInfo.AuthProof + var nodeSig1, nodeSig2, bitcoinSig1, bitcoinSig2 []byte + if authProof != nil { + nodeSig1 = authProof.NodeSig1Bytes + nodeSig2 = authProof.NodeSig2Bytes + bitcoinSig1 = authProof.BitcoinSig1Bytes + bitcoinSig2 = authProof.BitcoinSig2Bytes + } + + if err := wire.WriteVarBytes(&b, 0, nodeSig1); err != nil { + return err + } + if err := wire.WriteVarBytes(&b, 0, nodeSig2); err != nil { + return err + } + if err := wire.WriteVarBytes(&b, 0, bitcoinSig1); err != nil { + return err + } + if err := wire.WriteVarBytes(&b, 0, bitcoinSig2); err != nil { + return err + } + + if err := WriteOutpoint(&b, &edgeInfo.ChannelPoint); err != nil { + return err + } + err := binary.Write(&b, byteOrder, uint64(edgeInfo.Capacity)) + if err != nil { + return err + } + if _, err := b.Write(chanID[:]); err != nil { + return err + } + if _, err := b.Write(edgeInfo.ChainHash[:]); err != nil { + return err + } + + if len(edgeInfo.ExtraOpaqueData) > MaxAllowedExtraOpaqueBytes { + return ErrTooManyExtraOpaqueBytes(len(edgeInfo.ExtraOpaqueData)) + } + err = wire.WriteVarBytes(&b, 0, edgeInfo.ExtraOpaqueData) + if err != nil { + return err + } + + return edgeIndex.Put(chanID[:], b.Bytes()) +} + +func fetchChanEdgeInfo(edgeIndex kvdb.RBucket, + chanID []byte) (*models.ChannelEdgeInfo, error) { + + edgeInfoBytes := edgeIndex.Get(chanID) + if edgeInfoBytes == nil { + return nil, ErrEdgeNotFound + } + + edgeInfoReader := bytes.NewReader(edgeInfoBytes) + + return deserializeChanEdgeInfo(edgeInfoReader) +} + +func deserializeChanEdgeInfo(r io.Reader) (*models.ChannelEdgeInfo, error) { + var ( + err error + edgeInfo models.ChannelEdgeInfo + ) + + if _, err := io.ReadFull(r, edgeInfo.NodeKey1Bytes[:]); err != nil { + return nil, err + } + if _, err := io.ReadFull(r, edgeInfo.NodeKey2Bytes[:]); err != nil { + return nil, err + } + if _, err := io.ReadFull(r, edgeInfo.BitcoinKey1Bytes[:]); err != nil { + return nil, err + } + if _, err := io.ReadFull(r, edgeInfo.BitcoinKey2Bytes[:]); err != nil { + return nil, err + } + + featureBytes, err := wire.ReadVarBytes(r, 0, 900, "features") + if err != nil { + return nil, err + } + + features := lnwire.NewRawFeatureVector() + err = features.Decode(bytes.NewReader(featureBytes)) + if err != nil { + return nil, fmt.Errorf("unable to decode "+ + "features: %w", err) + } + edgeInfo.Features = lnwire.NewFeatureVector(features, lnwire.Features) + + proof := &models.ChannelAuthProof{} + + proof.NodeSig1Bytes, err = wire.ReadVarBytes(r, 0, 80, "sigs") + if err != nil { + return nil, err + } + proof.NodeSig2Bytes, err = wire.ReadVarBytes(r, 0, 80, "sigs") + if err != nil { + return nil, err + } + proof.BitcoinSig1Bytes, err = wire.ReadVarBytes(r, 0, 80, "sigs") + if err != nil { + return nil, err + } + proof.BitcoinSig2Bytes, err = wire.ReadVarBytes(r, 0, 80, "sigs") + if err != nil { + return nil, err + } + + if !proof.IsEmpty() { + edgeInfo.AuthProof = proof + } + + edgeInfo.ChannelPoint = wire.OutPoint{} + if err := ReadOutpoint(r, &edgeInfo.ChannelPoint); err != nil { + return nil, err + } + if err := binary.Read(r, byteOrder, &edgeInfo.Capacity); err != nil { + return nil, err + } + if err := binary.Read(r, byteOrder, &edgeInfo.ChannelID); err != nil { + return nil, err + } + + if _, err := io.ReadFull(r, edgeInfo.ChainHash[:]); err != nil { + return nil, err + } + + // We'll try and see if there are any opaque bytes left, if not, then + // we'll ignore the EOF error and return the edge as is. + edgeInfo.ExtraOpaqueData, err = wire.ReadVarBytes( + r, 0, MaxAllowedExtraOpaqueBytes, "blob", + ) + switch { + case errors.Is(err, io.ErrUnexpectedEOF): + case errors.Is(err, io.EOF): + case err != nil: + return nil, err + } + + return &edgeInfo, nil +} + +func putChanEdgePolicy(edges kvdb.RwBucket, edge *models.ChannelEdgePolicy, + from, to []byte) error { + + var edgeKey [33 + 8]byte + copy(edgeKey[:], from) + byteOrder.PutUint64(edgeKey[33:], edge.ChannelID) + + var b bytes.Buffer + if err := serializeChanEdgePolicy(&b, edge, to); err != nil { + return err + } + + // Before we write out the new edge, we'll create a new entry in the + // update index in order to keep it fresh. + updateUnix := uint64(edge.LastUpdate.Unix()) + var indexKey [8 + 8]byte + byteOrder.PutUint64(indexKey[:8], updateUnix) + byteOrder.PutUint64(indexKey[8:], edge.ChannelID) + + updateIndex, err := edges.CreateBucketIfNotExists(edgeUpdateIndexBucket) + if err != nil { + return err + } + + // If there was already an entry for this edge, then we'll need to + // delete the old one to ensure we don't leave around any after-images. + // An unknown policy value does not have a update time recorded, so + // it also does not need to be removed. + if edgeBytes := edges.Get(edgeKey[:]); edgeBytes != nil && + !bytes.Equal(edgeBytes, unknownPolicy) { + + // In order to delete the old entry, we'll need to obtain the + // *prior* update time in order to delete it. To do this, we'll + // need to deserialize the existing policy within the database + // (now outdated by the new one), and delete its corresponding + // entry within the update index. We'll ignore any + // ErrEdgePolicyOptionalFieldNotFound or ErrParsingExtraTLVBytes + // errors, as we only need the channel ID and update time to + // delete the entry. + // + // TODO(halseth): get rid of these invalid policies in a + // migration. + // + // NOTE: the above TODO was completed in the SQL migration and + // so such edge cases no longer need to be handled there. + oldEdgePolicy, err := deserializeChanEdgePolicy( + bytes.NewReader(edgeBytes), + ) + if err != nil && + !errors.Is(err, ErrEdgePolicyOptionalFieldNotFound) && + !errors.Is(err, ErrParsingExtraTLVBytes) { + + return err + } + + oldUpdateTime := uint64(oldEdgePolicy.LastUpdate.Unix()) + + var oldIndexKey [8 + 8]byte + byteOrder.PutUint64(oldIndexKey[:8], oldUpdateTime) + byteOrder.PutUint64(oldIndexKey[8:], edge.ChannelID) + + if err := updateIndex.Delete(oldIndexKey[:]); err != nil { + return err + } + } + + if err := updateIndex.Put(indexKey[:], nil); err != nil { + return err + } + + err = updateEdgePolicyDisabledIndex( + edges, edge.ChannelID, + edge.ChannelFlags&lnwire.ChanUpdateDirection > 0, + edge.IsDisabled(), + ) + if err != nil { + return err + } + + return edges.Put(edgeKey[:], b.Bytes()) +} + +// updateEdgePolicyDisabledIndex is used to update the disabledEdgePolicyIndex +// bucket by either add a new disabled ChannelEdgePolicy or remove an existing +// one. +// The direction represents the direction of the edge and disabled is used for +// deciding whether to remove or add an entry to the bucket. +// In general a channel is disabled if two entries for the same chanID exist +// in this bucket. +// Maintaining the bucket this way allows a fast retrieval of disabled +// channels, for example when prune is needed. +func updateEdgePolicyDisabledIndex(edges kvdb.RwBucket, chanID uint64, + direction bool, disabled bool) error { + + var disabledEdgeKey [8 + 1]byte + byteOrder.PutUint64(disabledEdgeKey[0:], chanID) + if direction { + disabledEdgeKey[8] = 1 + } + + disabledEdgePolicyIndex, err := edges.CreateBucketIfNotExists( + disabledEdgePolicyBucket, + ) + if err != nil { + return err + } + + if disabled { + return disabledEdgePolicyIndex.Put(disabledEdgeKey[:], []byte{}) + } + + return disabledEdgePolicyIndex.Delete(disabledEdgeKey[:]) +} + +// putChanEdgePolicyUnknown marks the edge policy as unknown +// in the edges bucket. +func putChanEdgePolicyUnknown(edges kvdb.RwBucket, channelID uint64, + from []byte) error { + + var edgeKey [33 + 8]byte + copy(edgeKey[:], from) + byteOrder.PutUint64(edgeKey[33:], channelID) + + if edges.Get(edgeKey[:]) != nil { + return fmt.Errorf("cannot write unknown policy for channel %v "+ + " when there is already a policy present", channelID) + } + + return edges.Put(edgeKey[:], unknownPolicy) +} + +func fetchChanEdgePolicy(edges kvdb.RBucket, chanID []byte, + nodePub []byte) (*models.ChannelEdgePolicy, error) { + + var edgeKey [33 + 8]byte + copy(edgeKey[:], nodePub) + copy(edgeKey[33:], chanID) + + edgeBytes := edges.Get(edgeKey[:]) + if edgeBytes == nil { + return nil, ErrEdgeNotFound + } + + // No need to deserialize unknown policy. + if bytes.Equal(edgeBytes, unknownPolicy) { + return nil, nil + } + + edgeReader := bytes.NewReader(edgeBytes) + + ep, err := deserializeChanEdgePolicy(edgeReader) + switch { + // If the db policy was missing an expected optional field, we return + // nil as if the policy was unknown. + case errors.Is(err, ErrEdgePolicyOptionalFieldNotFound): + return nil, nil + + // If the policy contains invalid TLV bytes, we return nil as if + // the policy was unknown. + case errors.Is(err, ErrParsingExtraTLVBytes): + return nil, nil + + case err != nil: + return nil, err + } + + return ep, nil +} + +func fetchChanEdgePolicies(edgeIndex kvdb.RBucket, edges kvdb.RBucket, + chanID []byte) (*models.ChannelEdgePolicy, *models.ChannelEdgePolicy, + error) { + + edgeInfo := edgeIndex.Get(chanID) + if edgeInfo == nil { + return nil, nil, fmt.Errorf("%w: chanID=%x", ErrEdgeNotFound, + chanID) + } + + // The first node is contained within the first half of the edge + // information. We only propagate the error here and below if it's + // something other than edge non-existence. + node1Pub := edgeInfo[:33] + edge1, err := fetchChanEdgePolicy(edges, chanID, node1Pub) + if err != nil { + return nil, nil, fmt.Errorf("%w: node1Pub=%x", ErrEdgeNotFound, + node1Pub) + } + + // Similarly, the second node is contained within the latter + // half of the edge information. + node2Pub := edgeInfo[33:66] + edge2, err := fetchChanEdgePolicy(edges, chanID, node2Pub) + if err != nil { + return nil, nil, fmt.Errorf("%w: node2Pub=%x", ErrEdgeNotFound, + node2Pub) + } + + return edge1, edge2, nil +} + +func serializeChanEdgePolicy(w io.Writer, edge *models.ChannelEdgePolicy, + to []byte) error { + + err := wire.WriteVarBytes(w, 0, edge.SigBytes) + if err != nil { + return err + } + + if err := binary.Write(w, byteOrder, edge.ChannelID); err != nil { + return err + } + + var scratch [8]byte + updateUnix := uint64(edge.LastUpdate.Unix()) + byteOrder.PutUint64(scratch[:], updateUnix) + if _, err := w.Write(scratch[:]); err != nil { + return err + } + + if err := binary.Write(w, byteOrder, edge.MessageFlags); err != nil { + return err + } + if err := binary.Write(w, byteOrder, edge.ChannelFlags); err != nil { + return err + } + if err := binary.Write(w, byteOrder, edge.TimeLockDelta); err != nil { + return err + } + if err := binary.Write(w, byteOrder, uint64(edge.MinHTLC)); err != nil { + return err + } + err = binary.Write(w, byteOrder, uint64(edge.FeeBaseMSat)) + if err != nil { + return err + } + err = binary.Write( + w, byteOrder, uint64(edge.FeeProportionalMillionths), + ) + if err != nil { + return err + } + + if _, err := w.Write(to); err != nil { + return err + } + + // If the max_htlc field is present, we write it. To be compatible with + // older versions that wasn't aware of this field, we write it as part + // of the opaque data. + // TODO(halseth): clean up when moving to TLV. + var opaqueBuf bytes.Buffer + if edge.MessageFlags.HasMaxHtlc() { + err := binary.Write(&opaqueBuf, byteOrder, uint64(edge.MaxHTLC)) + if err != nil { + return err + } + } + + if len(edge.ExtraOpaqueData) > MaxAllowedExtraOpaqueBytes { + return ErrTooManyExtraOpaqueBytes(len(edge.ExtraOpaqueData)) + } + if _, err := opaqueBuf.Write(edge.ExtraOpaqueData); err != nil { + return err + } + + if err := wire.WriteVarBytes(w, 0, opaqueBuf.Bytes()); err != nil { + return err + } + + return nil +} + +func deserializeChanEdgePolicy(r io.Reader) (*models.ChannelEdgePolicy, error) { + // Deserialize the policy. Note that in case an optional field is not + // found or if the edge has invalid TLV data, then both an error and a + // populated policy object are returned so that the caller can decide + // if it still wants to use the edge or not. + edge, err := deserializeChanEdgePolicyRaw(r) + if err != nil && + !errors.Is(err, ErrEdgePolicyOptionalFieldNotFound) && + !errors.Is(err, ErrParsingExtraTLVBytes) { + + return nil, err + } + + return edge, err +} + +func deserializeChanEdgePolicyRaw(r io.Reader) (*models.ChannelEdgePolicy, + error) { + + edge := &models.ChannelEdgePolicy{} + + var err error + edge.SigBytes, err = wire.ReadVarBytes(r, 0, 80, "sig") + if err != nil { + return nil, err + } + + if err := binary.Read(r, byteOrder, &edge.ChannelID); err != nil { + return nil, err + } + + var scratch [8]byte + if _, err := r.Read(scratch[:]); err != nil { + return nil, err + } + unix := int64(byteOrder.Uint64(scratch[:])) + edge.LastUpdate = time.Unix(unix, 0) + + if err := binary.Read(r, byteOrder, &edge.MessageFlags); err != nil { + return nil, err + } + if err := binary.Read(r, byteOrder, &edge.ChannelFlags); err != nil { + return nil, err + } + if err := binary.Read(r, byteOrder, &edge.TimeLockDelta); err != nil { + return nil, err + } + + var n uint64 + if err := binary.Read(r, byteOrder, &n); err != nil { + return nil, err + } + edge.MinHTLC = lnwire.MilliSatoshi(n) + + if err := binary.Read(r, byteOrder, &n); err != nil { + return nil, err + } + edge.FeeBaseMSat = lnwire.MilliSatoshi(n) + + if err := binary.Read(r, byteOrder, &n); err != nil { + return nil, err + } + edge.FeeProportionalMillionths = lnwire.MilliSatoshi(n) + + if _, err := r.Read(edge.ToNode[:]); err != nil { + return nil, err + } + + // We'll try and see if there are any opaque bytes left, if not, then + // we'll ignore the EOF error and return the edge as is. + edge.ExtraOpaqueData, err = wire.ReadVarBytes( + r, 0, MaxAllowedExtraOpaqueBytes, "blob", + ) + switch { + case errors.Is(err, io.ErrUnexpectedEOF): + case errors.Is(err, io.EOF): + case err != nil: + return nil, err + } + + // See if optional fields are present. + if edge.MessageFlags.HasMaxHtlc() { + // The max_htlc field should be at the beginning of the opaque + // bytes. + opq := edge.ExtraOpaqueData + + // If the max_htlc field is not present, it might be old data + // stored before this field was validated. We'll return the + // edge along with an error. + if len(opq) < 8 { + return edge, ErrEdgePolicyOptionalFieldNotFound + } + + maxHtlc := byteOrder.Uint64(opq[:8]) + edge.MaxHTLC = lnwire.MilliSatoshi(maxHtlc) + + // Exclude the parsed field from the rest of the opaque data. + edge.ExtraOpaqueData = opq[8:] + } + + // Attempt to extract the inbound fee from the opaque data. If we fail + // to parse the TLV here, we return an error we also return the edge + // so that the caller can still use it. This is for backwards + // compatibility in case we have already persisted some policies that + // have invalid TLV data. + var inboundFee lnwire.Fee + typeMap, err := edge.ExtraOpaqueData.ExtractRecords(&inboundFee) + if err != nil { + return edge, fmt.Errorf("%w: %w", ErrParsingExtraTLVBytes, err) + } + + val, ok := typeMap[lnwire.FeeRecordType] + if ok && val == nil { + edge.InboundFee = fn.Some(inboundFee) + } + + return edge, nil +} diff --git a/graph/db/migration1/log.go b/graph/db/migration1/log.go new file mode 100644 index 000000000..d0814e154 --- /dev/null +++ b/graph/db/migration1/log.go @@ -0,0 +1,31 @@ +package migration1 + +import ( + "github.com/btcsuite/btclog/v2" + "github.com/lightningnetwork/lnd/build" +) + +// Subsystem defines the logging code for this subsystem. +const Subsystem = "GRDB" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log btclog.Logger + +func init() { + UseLogger(build.NewSubLogger(Subsystem, nil)) +} + +// DisableLog disables all library log output. Logging output is disabled +// by default until UseLogger is called. +func DisableLog() { + UseLogger(btclog.Disabled) +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using btclog. +func UseLogger(logger btclog.Logger) { + log = logger +} diff --git a/graph/db/migration1/models/channel_auth_proof.go b/graph/db/migration1/models/channel_auth_proof.go new file mode 100644 index 000000000..daf120b10 --- /dev/null +++ b/graph/db/migration1/models/channel_auth_proof.go @@ -0,0 +1,35 @@ +package models + +// ChannelAuthProof is the authentication proof (the signature portion) for a +// channel. Using the four signatures contained in the struct, and some +// auxiliary knowledge (the funding script, node identities, and outpoint) nodes +// on the network are able to validate the authenticity and existence of a +// channel. Each of these signatures signs the following digest: chanID || +// nodeID1 || nodeID2 || bitcoinKey1|| bitcoinKey2 || 2-byte-feature-len || +// features. +type ChannelAuthProof struct { + // NodeSig1Bytes are the raw bytes of the first node signature encoded + // in DER format. + NodeSig1Bytes []byte + + // NodeSig2Bytes are the raw bytes of the second node signature + // encoded in DER format. + NodeSig2Bytes []byte + + // BitcoinSig1Bytes are the raw bytes of the first bitcoin signature + // encoded in DER format. + BitcoinSig1Bytes []byte + + // BitcoinSig2Bytes are the raw bytes of the second bitcoin signature + // encoded in DER format. + BitcoinSig2Bytes []byte +} + +// IsEmpty check is the authentication proof is empty Proof is empty if at +// least one of the signatures are equal to nil. +func (c *ChannelAuthProof) IsEmpty() bool { + return len(c.NodeSig1Bytes) == 0 || + len(c.NodeSig2Bytes) == 0 || + len(c.BitcoinSig1Bytes) == 0 || + len(c.BitcoinSig2Bytes) == 0 +} diff --git a/graph/db/migration1/models/channel_edge_info.go b/graph/db/migration1/models/channel_edge_info.go new file mode 100644 index 000000000..c99dfa215 --- /dev/null +++ b/graph/db/migration1/models/channel_edge_info.go @@ -0,0 +1,69 @@ +package models + +import ( + "github.com/btcsuite/btcd/btcutil" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/wire" + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/lightningnetwork/lnd/lnwire" +) + +// ChannelEdgeInfo represents a fully authenticated channel along with all its +// unique attributes. Once an authenticated channel announcement has been +// processed on the network, then an instance of ChannelEdgeInfo encapsulating +// the channels attributes is stored. The other portions relevant to routing +// policy of a channel are stored within a ChannelEdgePolicy for each direction +// of the channel. +type ChannelEdgeInfo struct { + // ChannelID is the unique channel ID for the channel. The first 3 + // bytes are the block height, the next 3 the index within the block, + // and the last 2 bytes are the output index for the channel. + ChannelID uint64 + + // ChainHash is the hash that uniquely identifies the chain that this + // channel was opened within. + ChainHash chainhash.Hash + + // NodeKey1Bytes is the raw public key of the first node. + NodeKey1Bytes [33]byte + + // NodeKey2Bytes is the raw public key of the first node. + NodeKey2Bytes [33]byte + + // BitcoinKey1Bytes is the raw public key of the first node. + BitcoinKey1Bytes [33]byte + + // BitcoinKey2Bytes is the raw public key of the first node. + BitcoinKey2Bytes [33]byte + + // Features is the list of protocol features supported by this channel + // edge. + Features *lnwire.FeatureVector + + // AuthProof is the authentication proof for this channel. This proof + // contains a set of signatures binding four identities, which attests + // to the legitimacy of the advertised channel. + AuthProof *ChannelAuthProof + + // ChannelPoint is the funding outpoint of the channel. This can be + // used to uniquely identify the channel within the channel graph. + ChannelPoint wire.OutPoint + + // Capacity is the total capacity of the channel, this is determined by + // the value output in the outpoint that created this channel. + Capacity btcutil.Amount + + // FundingScript holds the script of the channel's funding transaction. + // + // NOTE: this is not currently persisted and so will not be present if + // the edge object is loaded from the database. + FundingScript fn.Option[[]byte] + + // ExtraOpaqueData is the set of data that was appended to this + // message, some of which we may not actually know how to iterate or + // parse. By holding onto this data, we ensure that we're able to + // properly validate the set of signatures that cover these new fields, + // and ensure we're able to make upgrades to the network in a forwards + // compatible manner. + ExtraOpaqueData []byte +} diff --git a/graph/db/migration1/models/channel_edge_policy.go b/graph/db/migration1/models/channel_edge_policy.go new file mode 100644 index 000000000..1469602aa --- /dev/null +++ b/graph/db/migration1/models/channel_edge_policy.go @@ -0,0 +1,85 @@ +package models + +import ( + "time" + + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/lightningnetwork/lnd/lnwire" +) + +// ChannelEdgePolicy represents a *directed* edge within the channel graph. For +// each channel in the database, there are two distinct edges: one for each +// possible direction of travel along the channel. The edges themselves hold +// information concerning fees, and minimum time-lock information which is +// utilized during path finding. +type ChannelEdgePolicy struct { + // SigBytes is the raw bytes of the signature of the channel edge + // policy. We'll only parse these if the caller needs to access the + // signature for validation purposes. Do not set SigBytes directly, but + // use SetSigBytes instead to make sure that the cache is invalidated. + SigBytes []byte + + // ChannelID is the unique channel ID for the channel. The first 3 + // bytes are the block height, the next 3 the index within the block, + // and the last 2 bytes are the output index for the channel. + ChannelID uint64 + + // LastUpdate is the last time an authenticated edge for this channel + // was received. + LastUpdate time.Time + + // MessageFlags is a bitfield which indicates the presence of optional + // fields (like max_htlc) in the policy. + MessageFlags lnwire.ChanUpdateMsgFlags + + // ChannelFlags is a bitfield which signals the capabilities of the + // channel as well as the directed edge this update applies to. + ChannelFlags lnwire.ChanUpdateChanFlags + + // TimeLockDelta is the number of blocks this node will subtract from + // the expiry of an incoming HTLC. This value expresses the time buffer + // the node would like to HTLC exchanges. + TimeLockDelta uint16 + + // MinHTLC is the smallest value HTLC this node will forward, expressed + // in millisatoshi. + MinHTLC lnwire.MilliSatoshi + + // MaxHTLC is the largest value HTLC this node will forward, expressed + // in millisatoshi. + MaxHTLC lnwire.MilliSatoshi + + // FeeBaseMSat is the base HTLC fee that will be charged for forwarding + // ANY HTLC, expressed in mSAT's. + FeeBaseMSat lnwire.MilliSatoshi + + // FeeProportionalMillionths is the rate that the node will charge for + // HTLCs for each millionth of a satoshi forwarded. + FeeProportionalMillionths lnwire.MilliSatoshi + + // ToNode is the public key of the node that this directed edge leads + // to. Using this pub key, the channel graph can further be traversed. + ToNode [33]byte + + // InboundFee is the fee that must be paid for incoming HTLCs. + // + // NOTE: for our kvdb implementation of the graph store, inbound fees + // are still only persisted as part of extra opaque data and so this + // field is not explicitly stored but is rather populated from the + // ExtraOpaqueData field on deserialization. For our SQL implementation, + // this field will be explicitly persisted in the database. + InboundFee fn.Option[lnwire.Fee] + + // ExtraOpaqueData is the set of data that was appended to this + // message, some of which we may not actually know how to iterate or + // parse. By holding onto this data, we ensure that we're able to + // properly validate the set of signatures that cover these new fields, + // and ensure we're able to make upgrades to the network in a forwards + // compatible manner. + ExtraOpaqueData lnwire.ExtraOpaqueData +} + +// IsDisabled determines whether the edge has the disabled bit set. +func (c *ChannelEdgePolicy) IsDisabled() bool { + return c.ChannelFlags.IsDisabled() +} diff --git a/graph/db/migration1/models/node.go b/graph/db/migration1/models/node.go new file mode 100644 index 000000000..0b0223833 --- /dev/null +++ b/graph/db/migration1/models/node.go @@ -0,0 +1,146 @@ +package models + +import ( + "image/color" + "net" + "time" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/routing/route" +) + +// Node represents an individual vertex/node within the channel graph. +// A node is connected to other nodes by one or more channel edges emanating +// from it. As the graph is directed, a node will also have an incoming edge +// attached to it for each outgoing edge. +type Node struct { + // Version is the gossip version that this node was advertised on. + Version lnwire.GossipVersion + + // PubKeyBytes is the raw bytes of the public key of the target node. + PubKeyBytes [33]byte + pubKey *btcec.PublicKey + + // LastUpdate is the last time the vertex information for this node has + // been updated. + LastUpdate time.Time + + // Address is the TCP address this node is reachable over. + Addresses []net.Addr + + // Color is the selected color for the node. + Color fn.Option[color.RGBA] + + // Alias is a nick-name for the node. The alias can be used to confirm + // a node's identity or to serve as a short ID for an address book. + Alias fn.Option[string] + + // AuthSigBytes is the raw signature under the advertised public key + // which serves to authenticate the attributes announced by this node. + AuthSigBytes []byte + + // Features is the list of protocol features supported by this node. + Features *lnwire.FeatureVector + + // ExtraOpaqueData is the set of data that was appended to this + // message, some of which we may not actually know how to iterate or + // parse. By holding onto this data, we ensure that we're able to + // properly validate the set of signatures that cover these new fields, + // and ensure we're able to make upgrades to the network in a forwards + // compatible manner. + ExtraOpaqueData []byte +} + +// NodeV1Fields houses the fields that are specific to a version 1 node +// announcement. +type NodeV1Fields struct { + // Address is the TCP address this node is reachable over. + Addresses []net.Addr + + // AuthSigBytes is the raw signature under the advertised public key + // which serves to authenticate the attributes announced by this node. + AuthSigBytes []byte + + // Features is the list of protocol features supported by this node. + Features *lnwire.RawFeatureVector + + // Color is the selected color for the node. + Color color.RGBA + + // Alias is a nick-name for the node. The alias can be used to confirm + // a node's identity or to serve as a short ID for an address book. + Alias string + + // LastUpdate is the last time the vertex information for this node has + // been updated. + LastUpdate time.Time + + // ExtraOpaqueData is the set of data that was appended to this + // message, some of which we may not actually know how to iterate or + // parse. By holding onto this data, we ensure that we're able to + // properly validate the set of signatures that cover these new fields, + // and ensure we're able to make upgrades to the network in a forwards + // compatible manner. + ExtraOpaqueData []byte +} + +// NewV1Node creates a new version 1 node from the passed fields. +func NewV1Node(pub route.Vertex, n *NodeV1Fields) *Node { + return &Node{ + Version: lnwire.GossipVersion1, + PubKeyBytes: pub, + Addresses: n.Addresses, + AuthSigBytes: n.AuthSigBytes, + Features: lnwire.NewFeatureVector( + n.Features, lnwire.Features, + ), + Color: fn.Some(n.Color), + Alias: fn.Some(n.Alias), + LastUpdate: n.LastUpdate, + ExtraOpaqueData: n.ExtraOpaqueData, + } +} + +// NewV1ShellNode creates a new shell version 1 node. +func NewV1ShellNode(pubKey route.Vertex) *Node { + return NewShellNode(lnwire.GossipVersion1, pubKey) +} + +// NewShellNode creates a new shell node with the given gossip version and +// public key. +func NewShellNode(v lnwire.GossipVersion, pubKey route.Vertex) *Node { + return &Node{ + Version: v, + PubKeyBytes: pubKey, + Features: lnwire.EmptyFeatureVector(), + LastUpdate: time.Unix(0, 0), + } +} + +// HaveAnnouncement returns true if we have received a node announcement for +// this node. We determine this by checking if we have a signature for the +// announcement. +func (n *Node) HaveAnnouncement() bool { + return len(n.AuthSigBytes) > 0 +} + +// PubKey is the node's long-term identity public key. This key will be used to +// authenticated any advertisements/updates sent by the node. +// +// NOTE: By having this method to access an attribute, we ensure we only need +// to fully deserialize the pubkey if absolutely necessary. +func (n *Node) PubKey() (*btcec.PublicKey, error) { + if n.pubKey != nil { + return n.pubKey, nil + } + + key, err := btcec.ParsePubKey(n.PubKeyBytes[:]) + if err != nil { + return nil, err + } + n.pubKey = key + + return key, nil +} diff --git a/graph/db/sql_migration.go b/graph/db/migration1/sql_migration.go similarity index 99% rename from graph/db/sql_migration.go rename to graph/db/migration1/sql_migration.go index 5b5f6bf40..dd9b19e5d 100644 --- a/graph/db/sql_migration.go +++ b/graph/db/migration1/sql_migration.go @@ -1,4 +1,4 @@ -package graphdb +package migration1 import ( "bytes" @@ -13,7 +13,7 @@ import ( "time" "github.com/btcsuite/btcd/chaincfg/chainhash" - "github.com/lightningnetwork/lnd/graph/db/models" + "github.com/lightningnetwork/lnd/graph/db/migration1/models" "github.com/lightningnetwork/lnd/kvdb" "github.com/lightningnetwork/lnd/lnwire" "github.com/lightningnetwork/lnd/routing/route" diff --git a/graph/db/sql_migration_test.go b/graph/db/migration1/sql_migration_test.go similarity index 94% rename from graph/db/sql_migration_test.go rename to graph/db/migration1/sql_migration_test.go index 9819cf7f9..d1c38683f 100644 --- a/graph/db/sql_migration_test.go +++ b/graph/db/migration1/sql_migration_test.go @@ -1,11 +1,12 @@ //go:build test_db_postgres || test_db_sqlite -package graphdb +package migration1 import ( "bytes" "cmp" "crypto/rand" + "encoding/hex" "errors" "fmt" "image/color" @@ -16,27 +17,46 @@ import ( "path" "slices" "strings" + "sync" "testing" "time" "github.com/btcsuite/btcd/btcec/v2" + "github.com/btcsuite/btcd/btcec/v2/ecdsa" "github.com/btcsuite/btcd/btcutil" "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btclog/v2" - "github.com/lightningnetwork/lnd/graph/db/models" + "github.com/lightningnetwork/lnd/graph/db/migration1/models" "github.com/lightningnetwork/lnd/kvdb" "github.com/lightningnetwork/lnd/kvdb/sqlbase" "github.com/lightningnetwork/lnd/kvdb/sqlite" "github.com/lightningnetwork/lnd/lnwire" "github.com/lightningnetwork/lnd/routing/route" "github.com/lightningnetwork/lnd/sqldb" + "github.com/lightningnetwork/lnd/tor" "github.com/stretchr/testify/require" "pgregory.net/rapid" ) var ( + testPub = route.Vertex{2, 202, 4} + + testRBytes, _ = hex.DecodeString( + "8ce2bc69281ce27da07e6683571319d18e949ddfa2965fb6caa1bf03" + + "14f882d7", + ) + testSBytes, _ = hex.DecodeString( + "299105481d63e0f4bc2a88121167221b6700d72a0ead154c03be696a2" + + "92d24ae", + ) + testRScalar = new(btcec.ModNScalar) + testSScalar = new(btcec.ModNScalar) + _ = testRScalar.SetByteSlice(testRBytes) + _ = testSScalar.SetByteSlice(testSBytes) + testSig = ecdsa.NewSignature(testRScalar, testSScalar) + testChain = *chaincfg.MainNetParams.GenesisHash testColor = color.RGBA{R: 1, G: 2, B: 3} testTime = time.Unix(11111, 0) @@ -121,6 +141,58 @@ var ( 0x1F, 0x90, }, } + + testIP4 = net.ParseIP("192.168.1.1").To4() + testIP6 = net.ParseIP("2001:0db8:0000:0000:0000:ff00:0042:8329") + + testIPV4Addr = &net.TCPAddr{ + IP: testIP4, + Port: 12345, + } + + testIPV6Addr = &net.TCPAddr{ + IP: testIP6, + Port: 65535, + } + + testOnionV2Addr = &tor.OnionAddr{ + OnionService: "3g2upl4pq6kufc4m.onion", + Port: 9735, + } + + testOnionV3Addr = &tor.OnionAddr{ + OnionService: "vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd.onion", //nolint:ll + Port: 80, + } + + testOpaqueAddr = &lnwire.OpaqueAddrs{ + // NOTE: the first byte is a protocol level address type. So + // for we set it to 0xff to guarantee that we do not know this + // type yet. + Payload: []byte{0xff, 0x02, 0x03, 0x04, 0x05, 0x06}, + } + + testDNSAddr = &lnwire.DNSAddress{ + Hostname: "example.com", + Port: 8080, + } + + testAddr = &net.TCPAddr{IP: (net.IP)([]byte{0xA, 0x0, 0x0, 0x1}), + Port: 9000} + anotherAddr, _ = net.ResolveTCPAddr("tcp", + "[2001:db8:85a3:0:0:8a2e:370:7334]:80") + testAddrs = []net.Addr{testAddr, anotherAddr} + + testFeatures = lnwire.NewFeatureVector( + lnwire.NewRawFeatureVector(lnwire.GossipQueriesRequired), + lnwire.Features, + ) + + rev = [chainhash.HashSize]byte{ + 0x51, 0xb6, 0x37, 0xd8, 0xfc, 0xd2, 0xc6, 0xda, + 0x48, 0x59, 0xe6, 0x96, 0x31, 0x13, 0xa1, 0x17, + 0x2d, 0xe7, 0x93, 0xe4, + } ) // TestMigrateGraphToSQL tests various deterministic cases that we want to test @@ -821,6 +893,20 @@ func makeTestChannel(t *testing.T, // attributes of a models.ChannelEdgePolicy created by makeTestPolicy. type testPolicyOpt func(*models.ChannelEdgePolicy) +var ( + updateTime = prand.Int63() + updateTimeMu sync.Mutex +) + +func nextUpdateTime() time.Time { + updateTimeMu.Lock() + defer updateTimeMu.Unlock() + + updateTime++ + + return time.Unix(updateTime, 0) +} + // makeTestPolicy creates a test models.ChannelEdgePolicy. The functional // options can be used to modify the policy's attributes. func makeTestPolicy(chanID uint64, toNode route.Vertex, isNode1 bool, @@ -1823,3 +1909,35 @@ func genRandomNode(t *rapid.T) *models.Node { return node } + +// putSerializedPolicy is a helper function that writes a serialized +// ChannelEdgePolicy to the edge bucket in the database. +func putSerializedPolicy(t *testing.T, db kvdb.Backend, from []byte, + chanID uint64, b []byte) { + + err := kvdb.Update(db, func(tx kvdb.RwTx) error { + edges := tx.ReadWriteBucket(edgeBucket) + require.NotNil(t, edges) + + edgeIndex := edges.NestedReadWriteBucket(edgeIndexBucket) + require.NotNil(t, edgeIndex) + + var edgeKey [33 + 8]byte + copy(edgeKey[:], from) + byteOrder.PutUint64(edgeKey[33:], chanID) + + var scratch [8]byte + var indexKey [8 + 8]byte + copy(indexKey[:], scratch[:]) + byteOrder.PutUint64(indexKey[8:], chanID) + + updateIndex, err := edges.CreateBucketIfNotExists( + edgeUpdateIndexBucket, + ) + require.NoError(t, err) + require.NoError(t, updateIndex.Put(indexKey[:], nil)) + + return edges.Put(edgeKey[:], b) + }, func() {}) + require.NoError(t, err, "error writing db") +} diff --git a/graph/db/migration1/sql_store.go b/graph/db/migration1/sql_store.go new file mode 100644 index 000000000..6ac10db4b --- /dev/null +++ b/graph/db/migration1/sql_store.go @@ -0,0 +1,1808 @@ +package migration1 + +import ( + "bytes" + "context" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "math" + "net" + "strconv" + "time" + + "github.com/btcsuite/btcd/btcutil" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/wire" + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/lightningnetwork/lnd/graph/db/migration1/models" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/routing/route" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/lightningnetwork/lnd/sqldb/sqlc" + "github.com/lightningnetwork/lnd/tlv" + "github.com/lightningnetwork/lnd/tor" +) + +// SQLQueries is a subset of the sqlc.Querier interface that can be used to +// execute queries against the SQL graph tables. +// +//nolint:ll,interfacebloat +type SQLQueries interface { + /* + Node queries. + */ + UpsertNode(ctx context.Context, arg sqlc.UpsertNodeParams) (int64, error) + GetNodeByPubKey(ctx context.Context, arg sqlc.GetNodeByPubKeyParams) (sqlc.GraphNode, error) + GetNodesByIDs(ctx context.Context, ids []int64) ([]sqlc.GraphNode, error) + GetNodeIDByPubKey(ctx context.Context, arg sqlc.GetNodeIDByPubKeyParams) (int64, error) + GetNodesByLastUpdateRange(ctx context.Context, arg sqlc.GetNodesByLastUpdateRangeParams) ([]sqlc.GraphNode, error) + ListNodesPaginated(ctx context.Context, arg sqlc.ListNodesPaginatedParams) ([]sqlc.GraphNode, error) + ListNodeIDsAndPubKeys(ctx context.Context, arg sqlc.ListNodeIDsAndPubKeysParams) ([]sqlc.ListNodeIDsAndPubKeysRow, error) + DeleteUnconnectedNodes(ctx context.Context) ([][]byte, error) + DeleteNodeByPubKey(ctx context.Context, arg sqlc.DeleteNodeByPubKeyParams) (sql.Result, error) + DeleteNode(ctx context.Context, id int64) error + + GetExtraNodeTypes(ctx context.Context, nodeID int64) ([]sqlc.GraphNodeExtraType, error) + GetNodeExtraTypesBatch(ctx context.Context, ids []int64) ([]sqlc.GraphNodeExtraType, error) + UpsertNodeExtraType(ctx context.Context, arg sqlc.UpsertNodeExtraTypeParams) error + DeleteExtraNodeType(ctx context.Context, arg sqlc.DeleteExtraNodeTypeParams) error + + UpsertNodeAddress(ctx context.Context, arg sqlc.UpsertNodeAddressParams) error + GetNodeAddresses(ctx context.Context, nodeID int64) ([]sqlc.GetNodeAddressesRow, error) + GetNodeAddressesBatch(ctx context.Context, ids []int64) ([]sqlc.GraphNodeAddress, error) + DeleteNodeAddresses(ctx context.Context, nodeID int64) error + + InsertNodeFeature(ctx context.Context, arg sqlc.InsertNodeFeatureParams) error + GetNodeFeaturesBatch(ctx context.Context, ids []int64) ([]sqlc.GraphNodeFeature, error) + GetNodeFeaturesByPubKey(ctx context.Context, arg sqlc.GetNodeFeaturesByPubKeyParams) ([]int32, error) + DeleteNodeFeature(ctx context.Context, arg sqlc.DeleteNodeFeatureParams) error + + /* + Source node queries. + */ + AddSourceNode(ctx context.Context, nodeID int64) error + GetSourceNodesByVersion(ctx context.Context, version int16) ([]sqlc.GetSourceNodesByVersionRow, error) + + /* + Channel queries. + */ + CreateChannel(ctx context.Context, arg sqlc.CreateChannelParams) (int64, error) + AddV1ChannelProof(ctx context.Context, arg sqlc.AddV1ChannelProofParams) (sql.Result, error) + GetChannelBySCID(ctx context.Context, arg sqlc.GetChannelBySCIDParams) (sqlc.GraphChannel, error) + GetChannelsBySCIDs(ctx context.Context, arg sqlc.GetChannelsBySCIDsParams) ([]sqlc.GraphChannel, error) + GetChannelsByOutpoints(ctx context.Context, outpoints []string) ([]sqlc.GetChannelsByOutpointsRow, error) + GetChannelsBySCIDRange(ctx context.Context, arg sqlc.GetChannelsBySCIDRangeParams) ([]sqlc.GetChannelsBySCIDRangeRow, error) + GetChannelBySCIDWithPolicies(ctx context.Context, arg sqlc.GetChannelBySCIDWithPoliciesParams) (sqlc.GetChannelBySCIDWithPoliciesRow, error) + GetChannelsBySCIDWithPolicies(ctx context.Context, arg sqlc.GetChannelsBySCIDWithPoliciesParams) ([]sqlc.GetChannelsBySCIDWithPoliciesRow, error) + GetChannelsByIDs(ctx context.Context, ids []int64) ([]sqlc.GetChannelsByIDsRow, error) + GetChannelAndNodesBySCID(ctx context.Context, arg sqlc.GetChannelAndNodesBySCIDParams) (sqlc.GetChannelAndNodesBySCIDRow, error) + HighestSCID(ctx context.Context, version int16) ([]byte, error) + ListChannelsByNodeID(ctx context.Context, arg sqlc.ListChannelsByNodeIDParams) ([]sqlc.ListChannelsByNodeIDRow, error) + ListChannelsForNodeIDs(ctx context.Context, arg sqlc.ListChannelsForNodeIDsParams) ([]sqlc.ListChannelsForNodeIDsRow, error) + ListChannelsWithPoliciesPaginated(ctx context.Context, arg sqlc.ListChannelsWithPoliciesPaginatedParams) ([]sqlc.ListChannelsWithPoliciesPaginatedRow, error) + ListChannelsPaginated(ctx context.Context, arg sqlc.ListChannelsPaginatedParams) ([]sqlc.ListChannelsPaginatedRow, error) + GetChannelsByPolicyLastUpdateRange(ctx context.Context, arg sqlc.GetChannelsByPolicyLastUpdateRangeParams) ([]sqlc.GetChannelsByPolicyLastUpdateRangeRow, error) + GetChannelByOutpointWithPolicies(ctx context.Context, arg sqlc.GetChannelByOutpointWithPoliciesParams) (sqlc.GetChannelByOutpointWithPoliciesRow, error) + GetPublicV1ChannelsBySCID(ctx context.Context, arg sqlc.GetPublicV1ChannelsBySCIDParams) ([]sqlc.GraphChannel, error) + GetSCIDByOutpoint(ctx context.Context, arg sqlc.GetSCIDByOutpointParams) ([]byte, error) + DeleteChannels(ctx context.Context, ids []int64) error + + UpsertChannelExtraType(ctx context.Context, arg sqlc.UpsertChannelExtraTypeParams) error + GetChannelExtrasBatch(ctx context.Context, chanIds []int64) ([]sqlc.GraphChannelExtraType, error) + InsertChannelFeature(ctx context.Context, arg sqlc.InsertChannelFeatureParams) error + GetChannelFeaturesBatch(ctx context.Context, chanIds []int64) ([]sqlc.GraphChannelFeature, error) + + /* + Channel Policy table queries. + */ + UpsertEdgePolicy(ctx context.Context, arg sqlc.UpsertEdgePolicyParams) (int64, error) + GetChannelPolicyByChannelAndNode(ctx context.Context, arg sqlc.GetChannelPolicyByChannelAndNodeParams) (sqlc.GraphChannelPolicy, error) + GetV1DisabledSCIDs(ctx context.Context) ([][]byte, error) + + UpsertChanPolicyExtraType(ctx context.Context, arg sqlc.UpsertChanPolicyExtraTypeParams) error + GetChannelPolicyExtraTypesBatch(ctx context.Context, policyIds []int64) ([]sqlc.GetChannelPolicyExtraTypesBatchRow, error) + DeleteChannelPolicyExtraTypes(ctx context.Context, channelPolicyID int64) error + + /* + Zombie index queries. + */ + UpsertZombieChannel(ctx context.Context, arg sqlc.UpsertZombieChannelParams) error + GetZombieChannel(ctx context.Context, arg sqlc.GetZombieChannelParams) (sqlc.GraphZombieChannel, error) + GetZombieChannelsSCIDs(ctx context.Context, arg sqlc.GetZombieChannelsSCIDsParams) ([]sqlc.GraphZombieChannel, error) + CountZombieChannels(ctx context.Context, version int16) (int64, error) + DeleteZombieChannel(ctx context.Context, arg sqlc.DeleteZombieChannelParams) (sql.Result, error) + IsZombieChannel(ctx context.Context, arg sqlc.IsZombieChannelParams) (bool, error) + + /* + Prune log table queries. + */ + GetPruneTip(ctx context.Context) (sqlc.GraphPruneLog, error) + GetPruneHashByHeight(ctx context.Context, blockHeight int64) ([]byte, error) + GetPruneEntriesForHeights(ctx context.Context, heights []int64) ([]sqlc.GraphPruneLog, error) + UpsertPruneLogEntry(ctx context.Context, arg sqlc.UpsertPruneLogEntryParams) error + DeletePruneLogEntriesInRange(ctx context.Context, arg sqlc.DeletePruneLogEntriesInRangeParams) error + + /* + Closed SCID table queries. + */ + InsertClosedChannel(ctx context.Context, scid []byte) error + IsClosedChannel(ctx context.Context, scid []byte) (bool, error) + GetClosedChannelsSCIDs(ctx context.Context, scids [][]byte) ([][]byte, error) + + /* + Migration specific queries. + + NOTE: these should not be used in code other than migrations. + Once sqldbv2 is in place, these can be removed from this struct + as then migrations will have their own dedicated queries + structs. + */ + InsertNodeMig(ctx context.Context, arg sqlc.InsertNodeMigParams) (int64, error) + InsertChannelMig(ctx context.Context, arg sqlc.InsertChannelMigParams) (int64, error) + InsertEdgePolicyMig(ctx context.Context, arg sqlc.InsertEdgePolicyMigParams) (int64, error) +} + +// BatchedSQLQueries is a version of SQLQueries that's capable of batched +// database operations. +type BatchedSQLQueries interface { + SQLQueries + sqldb.BatchedTx[SQLQueries] +} + +// SQLStore is an implementation of the V1Store interface that uses a SQL +// database as the backend. +type SQLStore struct { + cfg *SQLStoreConfig + db BatchedSQLQueries +} + +// A compile-time assertion to ensure that SQLStore implements the V1Store +// interface. +var _ V1Store = (*SQLStore)(nil) + +// SQLStoreConfig holds the configuration for the SQLStore. +type SQLStoreConfig struct { + // ChainHash is the genesis hash for the chain that all the gossip + // messages in this store are aimed at. + ChainHash chainhash.Hash + + // QueryConfig holds configuration values for SQL queries. + QueryCfg *sqldb.QueryConfig +} + +// NewSQLStore creates a new SQLStore instance given an open BatchedSQLQueries +// storage backend. +func NewSQLStore(cfg *SQLStoreConfig, db BatchedSQLQueries) (*SQLStore, error) { + s := &SQLStore{ + cfg: cfg, + db: db, + } + + return s, nil +} + +// SourceNode returns the source node of the graph. The source node is treated +// as the center node within a star-graph. This method may be used to kick off +// a path finding algorithm in order to explore the reachability of another +// node based off the source node. +// +// NOTE: part of the V1Store interface. +func (s *SQLStore) SourceNode(ctx context.Context) (*models.Node, + error) { + + var node *models.Node + err := s.db.ExecTx(ctx, sqldb.ReadTxOpt(), func(db SQLQueries) error { + _, nodePub, err := s.getSourceNode( + ctx, db, lnwire.GossipVersion1, + ) + if err != nil { + return fmt.Errorf("unable to fetch V1 source node: %w", + err) + } + + _, node, err = getNodeByPubKey(ctx, s.cfg.QueryCfg, db, nodePub) + + return err + }, sqldb.NoOpReset) + if err != nil { + return nil, fmt.Errorf("unable to fetch source node: %w", err) + } + + return node, nil +} + +// ForEachNode iterates through all the stored vertices/nodes in the graph, +// executing the passed callback with each node encountered. If the callback +// returns an error, then the transaction is aborted and the iteration stops +// early. +// +// NOTE: part of the V1Store interface. +func (s *SQLStore) ForEachNode(ctx context.Context, + cb func(node *models.Node) error, reset func()) error { + + return s.db.ExecTx(ctx, sqldb.ReadTxOpt(), func(db SQLQueries) error { + return forEachNodePaginated( + ctx, s.cfg.QueryCfg, db, + lnwire.GossipVersion1, func(_ context.Context, _ int64, + node *models.Node) error { + + return cb(node) + }, + ) + }, reset) +} + +// ForEachChannel iterates through all the channel edges stored within the +// graph and invokes the passed callback for each edge. The callback takes two +// edges as since this is a directed graph, both the in/out edges are visited. +// If the callback returns an error, then the transaction is aborted and the +// iteration stops early. +// +// NOTE: If an edge can't be found, or wasn't advertised, then a nil pointer +// for that particular channel edge routing policy will be passed into the +// callback. +// +// NOTE: part of the V1Store interface. +func (s *SQLStore) ForEachChannel(ctx context.Context, + cb func(*models.ChannelEdgeInfo, *models.ChannelEdgePolicy, + *models.ChannelEdgePolicy) error, reset func()) error { + + return s.db.ExecTx(ctx, sqldb.ReadTxOpt(), func(db SQLQueries) error { + return forEachChannelWithPolicies(ctx, db, s.cfg, cb) + }, reset) +} + +// IsZombieEdge returns whether the edge is considered zombie. If it is a +// zombie, then the two node public keys corresponding to this edge are also +// returned. +// +// NOTE: part of the V1Store interface. +func (s *SQLStore) IsZombieEdge(chanID uint64) (bool, [33]byte, [33]byte, + error) { + + var ( + ctx = context.TODO() + isZombie bool + pubKey1, pubKey2 route.Vertex + chanIDB = channelIDToBytes(chanID) + ) + + err := s.db.ExecTx(ctx, sqldb.ReadTxOpt(), func(db SQLQueries) error { + zombie, err := db.GetZombieChannel( + ctx, sqlc.GetZombieChannelParams{ + Scid: chanIDB, + Version: int16(lnwire.GossipVersion1), + }, + ) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + return fmt.Errorf("unable to fetch zombie channel: %w", + err) + } + + copy(pubKey1[:], zombie.NodeKey1) + copy(pubKey2[:], zombie.NodeKey2) + isZombie = true + + return nil + }, sqldb.NoOpReset) + if err != nil { + return false, route.Vertex{}, route.Vertex{}, + fmt.Errorf("%w: %w (chanID=%d)", + ErrCantCheckIfZombieEdgeStr, err, chanID) + } + + return isZombie, pubKey1, pubKey2, nil +} + +// PruneTip returns the block height and hash of the latest block that has been +// used to prune channels in the graph. Knowing the "prune tip" allows callers +// to tell if the graph is currently in sync with the current best known UTXO +// state. +// +// NOTE: part of the V1Store interface. +func (s *SQLStore) PruneTip() (*chainhash.Hash, uint32, error) { + var ( + ctx = context.TODO() + tipHash chainhash.Hash + tipHeight uint32 + ) + err := s.db.ExecTx(ctx, sqldb.WriteTxOpt(), func(db SQLQueries) error { + pruneTip, err := db.GetPruneTip(ctx) + if errors.Is(err, sql.ErrNoRows) { + return ErrGraphNeverPruned + } else if err != nil { + return fmt.Errorf("unable to fetch prune tip: %w", err) + } + + tipHash = chainhash.Hash(pruneTip.BlockHash) + tipHeight = uint32(pruneTip.BlockHeight) + + return nil + }, sqldb.NoOpReset) + if err != nil { + return nil, 0, err + } + + return &tipHash, tipHeight, nil +} + +// IsClosedScid checks whether a channel identified by the passed in scid is +// closed. This helps avoid having to perform expensive validation checks. +// +// NOTE: part of the V1Store interface. +func (s *SQLStore) IsClosedScid(scid lnwire.ShortChannelID) (bool, error) { + var ( + ctx = context.TODO() + isClosed bool + chanIDB = channelIDToBytes(scid.ToUint64()) + ) + err := s.db.ExecTx(ctx, sqldb.ReadTxOpt(), func(db SQLQueries) error { + var err error + isClosed, err = db.IsClosedChannel(ctx, chanIDB) + if err != nil { + return fmt.Errorf("unable to fetch closed channel: %w", + err) + } + + return nil + }, sqldb.NoOpReset) + if err != nil { + return false, fmt.Errorf("unable to fetch closed channel: %w", + err) + } + + return isClosed, nil +} + +// getNodeByPubKey attempts to look up a target node by its public key. +func getNodeByPubKey(ctx context.Context, cfg *sqldb.QueryConfig, db SQLQueries, + pubKey route.Vertex) (int64, *models.Node, error) { + + dbNode, err := db.GetNodeByPubKey( + ctx, sqlc.GetNodeByPubKeyParams{ + Version: int16(lnwire.GossipVersion1), + PubKey: pubKey[:], + }, + ) + if errors.Is(err, sql.ErrNoRows) { + return 0, nil, ErrGraphNodeNotFound + } else if err != nil { + return 0, nil, fmt.Errorf("unable to fetch node: %w", err) + } + + node, err := buildNode(ctx, cfg, db, dbNode) + if err != nil { + return 0, nil, fmt.Errorf("unable to build node: %w", err) + } + + return dbNode.ID, node, nil +} + +// buildNode constructs a Node instance from the given database node +// record. The node's features, addresses and extra signed fields are also +// fetched from the database and set on the node. +func buildNode(ctx context.Context, cfg *sqldb.QueryConfig, db SQLQueries, + dbNode sqlc.GraphNode) (*models.Node, error) { + + data, err := batchLoadNodeData(ctx, cfg, db, []int64{dbNode.ID}) + if err != nil { + return nil, fmt.Errorf("unable to batch load node data: %w", + err) + } + + return buildNodeWithBatchData(dbNode, data) +} + +// buildNodeWithBatchData builds a models.Node instance +// from the provided sqlc.GraphNode and batchNodeData. If the node does have +// features/addresses/extra fields, then the corresponding fields are expected +// to be present in the batchNodeData. +func buildNodeWithBatchData(dbNode sqlc.GraphNode, + batchData *batchNodeData) (*models.Node, error) { + + if dbNode.Version != int16(lnwire.GossipVersion1) { + return nil, fmt.Errorf("unsupported node version: %d", + dbNode.Version) + } + + var pub [33]byte + copy(pub[:], dbNode.PubKey) + + node := models.NewV1ShellNode(pub) + + if len(dbNode.Signature) == 0 { + return node, nil + } + + node.AuthSigBytes = dbNode.Signature + + if dbNode.Alias.Valid { + node.Alias = fn.Some(dbNode.Alias.String) + } + if dbNode.LastUpdate.Valid { + node.LastUpdate = time.Unix(dbNode.LastUpdate.Int64, 0) + } + + var err error + if dbNode.Color.Valid { + nodeColor, err := DecodeHexColor(dbNode.Color.String) + if err != nil { + return nil, fmt.Errorf("unable to decode color: %w", + err) + } + + node.Color = fn.Some(nodeColor) + } + + // Use preloaded features. + if features, exists := batchData.features[dbNode.ID]; exists { + fv := lnwire.EmptyFeatureVector() + for _, bit := range features { + fv.Set(lnwire.FeatureBit(bit)) + } + node.Features = fv + } + + // Use preloaded addresses. + addresses, exists := batchData.addresses[dbNode.ID] + if exists && len(addresses) > 0 { + node.Addresses, err = buildNodeAddresses(addresses) + if err != nil { + return nil, fmt.Errorf("unable to build addresses "+ + "for node(%d): %w", dbNode.ID, err) + } + } + + // Use preloaded extra fields. + if extraFields, exists := batchData.extraFields[dbNode.ID]; exists { + recs, err := lnwire.CustomRecords(extraFields).Serialize() + if err != nil { + return nil, fmt.Errorf("unable to serialize extra "+ + "signed fields: %w", err) + } + if len(recs) != 0 { + node.ExtraOpaqueData = recs + } + } + + return node, nil +} + +// dbAddressType is an enum type that represents the different address types +// that we store in the node_addresses table. The address type determines how +// the address is to be serialised/deserialize. +type dbAddressType uint8 + +const ( + addressTypeIPv4 dbAddressType = 1 + addressTypeIPv6 dbAddressType = 2 + addressTypeTorV2 dbAddressType = 3 + addressTypeTorV3 dbAddressType = 4 + addressTypeDNS dbAddressType = 5 + addressTypeOpaque dbAddressType = math.MaxInt8 +) + +// collectAddressRecords collects the addresses from the provided +// net.Addr slice and returns a map of dbAddressType to a slice of address +// strings. +func collectAddressRecords(addresses []net.Addr) (map[dbAddressType][]string, + error) { + + // Copy the nodes latest set of addresses. + newAddresses := map[dbAddressType][]string{ + addressTypeIPv4: {}, + addressTypeIPv6: {}, + addressTypeTorV2: {}, + addressTypeTorV3: {}, + addressTypeDNS: {}, + addressTypeOpaque: {}, + } + addAddr := func(t dbAddressType, addr net.Addr) { + newAddresses[t] = append(newAddresses[t], addr.String()) + } + + for _, address := range addresses { + switch addr := address.(type) { + case *net.TCPAddr: + if ip4 := addr.IP.To4(); ip4 != nil { + addAddr(addressTypeIPv4, addr) + } else if ip6 := addr.IP.To16(); ip6 != nil { + addAddr(addressTypeIPv6, addr) + } else { + return nil, fmt.Errorf("unhandled IP "+ + "address: %v", addr) + } + + case *tor.OnionAddr: + switch len(addr.OnionService) { + case tor.V2Len: + addAddr(addressTypeTorV2, addr) + case tor.V3Len: + addAddr(addressTypeTorV3, addr) + default: + return nil, fmt.Errorf("invalid length for " + + "a tor address") + } + + case *lnwire.DNSAddress: + addAddr(addressTypeDNS, addr) + + case *lnwire.OpaqueAddrs: + addAddr(addressTypeOpaque, addr) + + default: + return nil, fmt.Errorf("unhandled address type: %T", + addr) + } + } + + return newAddresses, nil +} + +// sourceNode returns the DB node ID and pub key of the source node for the +// specified protocol version. +func (s *SQLStore) getSourceNode(ctx context.Context, db SQLQueries, + version lnwire.GossipVersion) (int64, route.Vertex, error) { + + var pubKey route.Vertex + + nodes, err := db.GetSourceNodesByVersion(ctx, int16(version)) + if err != nil { + return 0, pubKey, fmt.Errorf("unable to fetch source node: %w", + err) + } + + if len(nodes) == 0 { + return 0, pubKey, ErrSourceNodeNotSet + } else if len(nodes) > 1 { + return 0, pubKey, fmt.Errorf("multiple source nodes for "+ + "protocol %s found", version) + } + + copy(pubKey[:], nodes[0].PubKey) + + return nodes[0].NodeID, pubKey, nil +} + +// marshalExtraOpaqueData takes a flat byte slice parses it as a TLV stream. +// This then produces a map from TLV type to value. If the input is not a +// valid TLV stream, then an error is returned. +func marshalExtraOpaqueData(data []byte) (map[uint64][]byte, error) { + r := bytes.NewReader(data) + + tlvStream, err := tlv.NewStream() + if err != nil { + return nil, err + } + + // Since ExtraOpaqueData is provided by a potentially malicious peer, + // pass it into the P2P decoding variant. + parsedTypes, err := tlvStream.DecodeWithParsedTypesP2P(r) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrParsingExtraTLVBytes, err) + } + if len(parsedTypes) == 0 { + return nil, nil + } + + records := make(map[uint64][]byte) + for k, v := range parsedTypes { + records[uint64(k)] = v + } + + return records, nil +} + +// maybeCreateShellNode checks if a shell node entry exists for the +// given public key. If it does not exist, then a new shell node entry is +// created. The ID of the node is returned. A shell node only has a protocol +// version and public key persisted. +func maybeCreateShellNode(ctx context.Context, db SQLQueries, + pubKey route.Vertex) (int64, error) { + + dbNode, err := db.GetNodeByPubKey( + ctx, sqlc.GetNodeByPubKeyParams{ + PubKey: pubKey[:], + Version: int16(lnwire.GossipVersion1), + }, + ) + // The node exists. Return the ID. + if err == nil { + return dbNode.ID, nil + } else if !errors.Is(err, sql.ErrNoRows) { + return 0, err + } + + // Otherwise, the node does not exist, so we create a shell entry for + // it. + id, err := db.UpsertNode(ctx, sqlc.UpsertNodeParams{ + Version: int16(lnwire.GossipVersion1), + PubKey: pubKey[:], + }) + if err != nil { + return 0, fmt.Errorf("unable to create shell node: %w", err) + } + + return id, nil +} + +// buildEdgeInfoWithBatchData builds edge info using pre-loaded batch data. +func buildEdgeInfoWithBatchData(chain chainhash.Hash, + dbChan sqlc.GraphChannel, node1, node2 route.Vertex, + batchData *batchChannelData) (*models.ChannelEdgeInfo, error) { + + if dbChan.Version != int16(lnwire.GossipVersion1) { + return nil, fmt.Errorf("unsupported channel version: %d", + dbChan.Version) + } + + // Use pre-loaded features and extras types. + fv := lnwire.EmptyFeatureVector() + if features, exists := batchData.chanfeatures[dbChan.ID]; exists { + for _, bit := range features { + fv.Set(lnwire.FeatureBit(bit)) + } + } + + var extras map[uint64][]byte + channelExtras, exists := batchData.chanExtraTypes[dbChan.ID] + if exists { + extras = channelExtras + } else { + extras = make(map[uint64][]byte) + } + + op, err := wire.NewOutPointFromString(dbChan.Outpoint) + if err != nil { + return nil, err + } + + recs, err := lnwire.CustomRecords(extras).Serialize() + if err != nil { + return nil, fmt.Errorf("unable to serialize extra signed "+ + "fields: %w", err) + } + if recs == nil { + recs = make([]byte, 0) + } + + var btcKey1, btcKey2 route.Vertex + copy(btcKey1[:], dbChan.BitcoinKey1) + copy(btcKey2[:], dbChan.BitcoinKey2) + + channel := &models.ChannelEdgeInfo{ + ChainHash: chain, + ChannelID: byteOrder.Uint64(dbChan.Scid), + NodeKey1Bytes: node1, + NodeKey2Bytes: node2, + BitcoinKey1Bytes: btcKey1, + BitcoinKey2Bytes: btcKey2, + ChannelPoint: *op, + Capacity: btcutil.Amount(dbChan.Capacity.Int64), + Features: fv, + ExtraOpaqueData: recs, + } + + // We always set all the signatures at the same time, so we can + // safely check if one signature is present to determine if we have the + // rest of the signatures for the auth proof. + if len(dbChan.Bitcoin1Signature) > 0 { + channel.AuthProof = &models.ChannelAuthProof{ + NodeSig1Bytes: dbChan.Node1Signature, + NodeSig2Bytes: dbChan.Node2Signature, + BitcoinSig1Bytes: dbChan.Bitcoin1Signature, + BitcoinSig2Bytes: dbChan.Bitcoin2Signature, + } + } + + return channel, nil +} + +// buildNodeVertices is a helper that converts raw node public keys +// into route.Vertex instances. +func buildNodeVertices(node1Pub, node2Pub []byte) (route.Vertex, + route.Vertex, error) { + + node1Vertex, err := route.NewVertexFromBytes(node1Pub) + if err != nil { + return route.Vertex{}, route.Vertex{}, fmt.Errorf("unable to "+ + "create vertex from node1 pubkey: %w", err) + } + + node2Vertex, err := route.NewVertexFromBytes(node2Pub) + if err != nil { + return route.Vertex{}, route.Vertex{}, fmt.Errorf("unable to "+ + "create vertex from node2 pubkey: %w", err) + } + + return node1Vertex, node2Vertex, nil +} + +// buildChanPolicy builds a models.ChannelEdgePolicy instance from the +// provided sqlc.GraphChannelPolicy and other required information. +func buildChanPolicy(dbPolicy sqlc.GraphChannelPolicy, channelID uint64, + extras map[uint64][]byte, + toNode route.Vertex) (*models.ChannelEdgePolicy, error) { + + recs, err := lnwire.CustomRecords(extras).Serialize() + if err != nil { + return nil, fmt.Errorf("unable to serialize extra signed "+ + "fields: %w", err) + } + + var inboundFee fn.Option[lnwire.Fee] + if dbPolicy.InboundFeeRateMilliMsat.Valid || + dbPolicy.InboundBaseFeeMsat.Valid { + + inboundFee = fn.Some(lnwire.Fee{ + BaseFee: int32(dbPolicy.InboundBaseFeeMsat.Int64), + FeeRate: int32(dbPolicy.InboundFeeRateMilliMsat.Int64), + }) + } + + return &models.ChannelEdgePolicy{ + SigBytes: dbPolicy.Signature, + ChannelID: channelID, + LastUpdate: time.Unix( + dbPolicy.LastUpdate.Int64, 0, + ), + MessageFlags: sqldb.ExtractSqlInt16[lnwire.ChanUpdateMsgFlags]( + dbPolicy.MessageFlags, + ), + ChannelFlags: sqldb.ExtractSqlInt16[lnwire.ChanUpdateChanFlags]( + dbPolicy.ChannelFlags, + ), + TimeLockDelta: uint16(dbPolicy.Timelock), + MinHTLC: lnwire.MilliSatoshi( + dbPolicy.MinHtlcMsat, + ), + MaxHTLC: lnwire.MilliSatoshi( + dbPolicy.MaxHtlcMsat.Int64, + ), + FeeBaseMSat: lnwire.MilliSatoshi( + dbPolicy.BaseFeeMsat, + ), + FeeProportionalMillionths: lnwire.MilliSatoshi(dbPolicy.FeePpm), + ToNode: toNode, + InboundFee: inboundFee, + ExtraOpaqueData: recs, + }, nil +} + +// extractChannelPolicies extracts the sqlc.GraphChannelPolicy records from the give +// row which is expected to be a sqlc type that contains channel policy +// information. It returns two policies, which may be nil if the policy +// information is not present in the row. +// +//nolint:ll,dupl,funlen +func extractChannelPolicies(row any) (*sqlc.GraphChannelPolicy, + *sqlc.GraphChannelPolicy, error) { + + var policy1, policy2 *sqlc.GraphChannelPolicy + switch r := row.(type) { + case sqlc.ListChannelsWithPoliciesForCachePaginatedRow: + if r.Policy1Timelock.Valid { + policy1 = &sqlc.GraphChannelPolicy{ + Timelock: r.Policy1Timelock.Int32, + FeePpm: r.Policy1FeePpm.Int64, + BaseFeeMsat: r.Policy1BaseFeeMsat.Int64, + MinHtlcMsat: r.Policy1MinHtlcMsat.Int64, + MaxHtlcMsat: r.Policy1MaxHtlcMsat, + InboundBaseFeeMsat: r.Policy1InboundBaseFeeMsat, + InboundFeeRateMilliMsat: r.Policy1InboundFeeRateMilliMsat, + Disabled: r.Policy1Disabled, + MessageFlags: r.Policy1MessageFlags, + ChannelFlags: r.Policy1ChannelFlags, + } + } + if r.Policy2Timelock.Valid { + policy2 = &sqlc.GraphChannelPolicy{ + Timelock: r.Policy2Timelock.Int32, + FeePpm: r.Policy2FeePpm.Int64, + BaseFeeMsat: r.Policy2BaseFeeMsat.Int64, + MinHtlcMsat: r.Policy2MinHtlcMsat.Int64, + MaxHtlcMsat: r.Policy2MaxHtlcMsat, + InboundBaseFeeMsat: r.Policy2InboundBaseFeeMsat, + InboundFeeRateMilliMsat: r.Policy2InboundFeeRateMilliMsat, + Disabled: r.Policy2Disabled, + MessageFlags: r.Policy2MessageFlags, + ChannelFlags: r.Policy2ChannelFlags, + } + } + + return policy1, policy2, nil + + case sqlc.GetChannelsBySCIDWithPoliciesRow: + if r.Policy1ID.Valid { + policy1 = &sqlc.GraphChannelPolicy{ + ID: r.Policy1ID.Int64, + Version: r.Policy1Version.Int16, + ChannelID: r.GraphChannel.ID, + NodeID: r.Policy1NodeID.Int64, + Timelock: r.Policy1Timelock.Int32, + FeePpm: r.Policy1FeePpm.Int64, + BaseFeeMsat: r.Policy1BaseFeeMsat.Int64, + MinHtlcMsat: r.Policy1MinHtlcMsat.Int64, + MaxHtlcMsat: r.Policy1MaxHtlcMsat, + LastUpdate: r.Policy1LastUpdate, + InboundBaseFeeMsat: r.Policy1InboundBaseFeeMsat, + InboundFeeRateMilliMsat: r.Policy1InboundFeeRateMilliMsat, + Disabled: r.Policy1Disabled, + MessageFlags: r.Policy1MessageFlags, + ChannelFlags: r.Policy1ChannelFlags, + Signature: r.Policy1Signature, + } + } + if r.Policy2ID.Valid { + policy2 = &sqlc.GraphChannelPolicy{ + ID: r.Policy2ID.Int64, + Version: r.Policy2Version.Int16, + ChannelID: r.GraphChannel.ID, + NodeID: r.Policy2NodeID.Int64, + Timelock: r.Policy2Timelock.Int32, + FeePpm: r.Policy2FeePpm.Int64, + BaseFeeMsat: r.Policy2BaseFeeMsat.Int64, + MinHtlcMsat: r.Policy2MinHtlcMsat.Int64, + MaxHtlcMsat: r.Policy2MaxHtlcMsat, + LastUpdate: r.Policy2LastUpdate, + InboundBaseFeeMsat: r.Policy2InboundBaseFeeMsat, + InboundFeeRateMilliMsat: r.Policy2InboundFeeRateMilliMsat, + Disabled: r.Policy2Disabled, + MessageFlags: r.Policy2MessageFlags, + ChannelFlags: r.Policy2ChannelFlags, + Signature: r.Policy2Signature, + } + } + + return policy1, policy2, nil + + case sqlc.GetChannelByOutpointWithPoliciesRow: + if r.Policy1ID.Valid { + policy1 = &sqlc.GraphChannelPolicy{ + ID: r.Policy1ID.Int64, + Version: r.Policy1Version.Int16, + ChannelID: r.GraphChannel.ID, + NodeID: r.Policy1NodeID.Int64, + Timelock: r.Policy1Timelock.Int32, + FeePpm: r.Policy1FeePpm.Int64, + BaseFeeMsat: r.Policy1BaseFeeMsat.Int64, + MinHtlcMsat: r.Policy1MinHtlcMsat.Int64, + MaxHtlcMsat: r.Policy1MaxHtlcMsat, + LastUpdate: r.Policy1LastUpdate, + InboundBaseFeeMsat: r.Policy1InboundBaseFeeMsat, + InboundFeeRateMilliMsat: r.Policy1InboundFeeRateMilliMsat, + Disabled: r.Policy1Disabled, + MessageFlags: r.Policy1MessageFlags, + ChannelFlags: r.Policy1ChannelFlags, + Signature: r.Policy1Signature, + } + } + if r.Policy2ID.Valid { + policy2 = &sqlc.GraphChannelPolicy{ + ID: r.Policy2ID.Int64, + Version: r.Policy2Version.Int16, + ChannelID: r.GraphChannel.ID, + NodeID: r.Policy2NodeID.Int64, + Timelock: r.Policy2Timelock.Int32, + FeePpm: r.Policy2FeePpm.Int64, + BaseFeeMsat: r.Policy2BaseFeeMsat.Int64, + MinHtlcMsat: r.Policy2MinHtlcMsat.Int64, + MaxHtlcMsat: r.Policy2MaxHtlcMsat, + LastUpdate: r.Policy2LastUpdate, + InboundBaseFeeMsat: r.Policy2InboundBaseFeeMsat, + InboundFeeRateMilliMsat: r.Policy2InboundFeeRateMilliMsat, + Disabled: r.Policy2Disabled, + MessageFlags: r.Policy2MessageFlags, + ChannelFlags: r.Policy2ChannelFlags, + Signature: r.Policy2Signature, + } + } + + return policy1, policy2, nil + + case sqlc.GetChannelBySCIDWithPoliciesRow: + if r.Policy1ID.Valid { + policy1 = &sqlc.GraphChannelPolicy{ + ID: r.Policy1ID.Int64, + Version: r.Policy1Version.Int16, + ChannelID: r.GraphChannel.ID, + NodeID: r.Policy1NodeID.Int64, + Timelock: r.Policy1Timelock.Int32, + FeePpm: r.Policy1FeePpm.Int64, + BaseFeeMsat: r.Policy1BaseFeeMsat.Int64, + MinHtlcMsat: r.Policy1MinHtlcMsat.Int64, + MaxHtlcMsat: r.Policy1MaxHtlcMsat, + LastUpdate: r.Policy1LastUpdate, + InboundBaseFeeMsat: r.Policy1InboundBaseFeeMsat, + InboundFeeRateMilliMsat: r.Policy1InboundFeeRateMilliMsat, + Disabled: r.Policy1Disabled, + MessageFlags: r.Policy1MessageFlags, + ChannelFlags: r.Policy1ChannelFlags, + Signature: r.Policy1Signature, + } + } + if r.Policy2ID.Valid { + policy2 = &sqlc.GraphChannelPolicy{ + ID: r.Policy2ID.Int64, + Version: r.Policy2Version.Int16, + ChannelID: r.GraphChannel.ID, + NodeID: r.Policy2NodeID.Int64, + Timelock: r.Policy2Timelock.Int32, + FeePpm: r.Policy2FeePpm.Int64, + BaseFeeMsat: r.Policy2BaseFeeMsat.Int64, + MinHtlcMsat: r.Policy2MinHtlcMsat.Int64, + MaxHtlcMsat: r.Policy2MaxHtlcMsat, + LastUpdate: r.Policy2LastUpdate, + InboundBaseFeeMsat: r.Policy2InboundBaseFeeMsat, + InboundFeeRateMilliMsat: r.Policy2InboundFeeRateMilliMsat, + Disabled: r.Policy2Disabled, + MessageFlags: r.Policy2MessageFlags, + ChannelFlags: r.Policy2ChannelFlags, + Signature: r.Policy2Signature, + } + } + + return policy1, policy2, nil + + case sqlc.GetChannelsByPolicyLastUpdateRangeRow: + if r.Policy1ID.Valid { + policy1 = &sqlc.GraphChannelPolicy{ + ID: r.Policy1ID.Int64, + Version: r.Policy1Version.Int16, + ChannelID: r.GraphChannel.ID, + NodeID: r.Policy1NodeID.Int64, + Timelock: r.Policy1Timelock.Int32, + FeePpm: r.Policy1FeePpm.Int64, + BaseFeeMsat: r.Policy1BaseFeeMsat.Int64, + MinHtlcMsat: r.Policy1MinHtlcMsat.Int64, + MaxHtlcMsat: r.Policy1MaxHtlcMsat, + LastUpdate: r.Policy1LastUpdate, + InboundBaseFeeMsat: r.Policy1InboundBaseFeeMsat, + InboundFeeRateMilliMsat: r.Policy1InboundFeeRateMilliMsat, + Disabled: r.Policy1Disabled, + MessageFlags: r.Policy1MessageFlags, + ChannelFlags: r.Policy1ChannelFlags, + Signature: r.Policy1Signature, + } + } + if r.Policy2ID.Valid { + policy2 = &sqlc.GraphChannelPolicy{ + ID: r.Policy2ID.Int64, + Version: r.Policy2Version.Int16, + ChannelID: r.GraphChannel.ID, + NodeID: r.Policy2NodeID.Int64, + Timelock: r.Policy2Timelock.Int32, + FeePpm: r.Policy2FeePpm.Int64, + BaseFeeMsat: r.Policy2BaseFeeMsat.Int64, + MinHtlcMsat: r.Policy2MinHtlcMsat.Int64, + MaxHtlcMsat: r.Policy2MaxHtlcMsat, + LastUpdate: r.Policy2LastUpdate, + InboundBaseFeeMsat: r.Policy2InboundBaseFeeMsat, + InboundFeeRateMilliMsat: r.Policy2InboundFeeRateMilliMsat, + Disabled: r.Policy2Disabled, + MessageFlags: r.Policy2MessageFlags, + ChannelFlags: r.Policy2ChannelFlags, + Signature: r.Policy2Signature, + } + } + + return policy1, policy2, nil + + case sqlc.ListChannelsForNodeIDsRow: + if r.Policy1ID.Valid { + policy1 = &sqlc.GraphChannelPolicy{ + ID: r.Policy1ID.Int64, + Version: r.Policy1Version.Int16, + ChannelID: r.GraphChannel.ID, + NodeID: r.Policy1NodeID.Int64, + Timelock: r.Policy1Timelock.Int32, + FeePpm: r.Policy1FeePpm.Int64, + BaseFeeMsat: r.Policy1BaseFeeMsat.Int64, + MinHtlcMsat: r.Policy1MinHtlcMsat.Int64, + MaxHtlcMsat: r.Policy1MaxHtlcMsat, + LastUpdate: r.Policy1LastUpdate, + InboundBaseFeeMsat: r.Policy1InboundBaseFeeMsat, + InboundFeeRateMilliMsat: r.Policy1InboundFeeRateMilliMsat, + Disabled: r.Policy1Disabled, + MessageFlags: r.Policy1MessageFlags, + ChannelFlags: r.Policy1ChannelFlags, + Signature: r.Policy1Signature, + } + } + if r.Policy2ID.Valid { + policy2 = &sqlc.GraphChannelPolicy{ + ID: r.Policy2ID.Int64, + Version: r.Policy2Version.Int16, + ChannelID: r.GraphChannel.ID, + NodeID: r.Policy2NodeID.Int64, + Timelock: r.Policy2Timelock.Int32, + FeePpm: r.Policy2FeePpm.Int64, + BaseFeeMsat: r.Policy2BaseFeeMsat.Int64, + MinHtlcMsat: r.Policy2MinHtlcMsat.Int64, + MaxHtlcMsat: r.Policy2MaxHtlcMsat, + LastUpdate: r.Policy2LastUpdate, + InboundBaseFeeMsat: r.Policy2InboundBaseFeeMsat, + InboundFeeRateMilliMsat: r.Policy2InboundFeeRateMilliMsat, + Disabled: r.Policy2Disabled, + MessageFlags: r.Policy2MessageFlags, + ChannelFlags: r.Policy2ChannelFlags, + Signature: r.Policy2Signature, + } + } + + return policy1, policy2, nil + + case sqlc.ListChannelsByNodeIDRow: + if r.Policy1ID.Valid { + policy1 = &sqlc.GraphChannelPolicy{ + ID: r.Policy1ID.Int64, + Version: r.Policy1Version.Int16, + ChannelID: r.GraphChannel.ID, + NodeID: r.Policy1NodeID.Int64, + Timelock: r.Policy1Timelock.Int32, + FeePpm: r.Policy1FeePpm.Int64, + BaseFeeMsat: r.Policy1BaseFeeMsat.Int64, + MinHtlcMsat: r.Policy1MinHtlcMsat.Int64, + MaxHtlcMsat: r.Policy1MaxHtlcMsat, + LastUpdate: r.Policy1LastUpdate, + InboundBaseFeeMsat: r.Policy1InboundBaseFeeMsat, + InboundFeeRateMilliMsat: r.Policy1InboundFeeRateMilliMsat, + Disabled: r.Policy1Disabled, + MessageFlags: r.Policy1MessageFlags, + ChannelFlags: r.Policy1ChannelFlags, + Signature: r.Policy1Signature, + } + } + if r.Policy2ID.Valid { + policy2 = &sqlc.GraphChannelPolicy{ + ID: r.Policy2ID.Int64, + Version: r.Policy2Version.Int16, + ChannelID: r.GraphChannel.ID, + NodeID: r.Policy2NodeID.Int64, + Timelock: r.Policy2Timelock.Int32, + FeePpm: r.Policy2FeePpm.Int64, + BaseFeeMsat: r.Policy2BaseFeeMsat.Int64, + MinHtlcMsat: r.Policy2MinHtlcMsat.Int64, + MaxHtlcMsat: r.Policy2MaxHtlcMsat, + LastUpdate: r.Policy2LastUpdate, + InboundBaseFeeMsat: r.Policy2InboundBaseFeeMsat, + InboundFeeRateMilliMsat: r.Policy2InboundFeeRateMilliMsat, + Disabled: r.Policy2Disabled, + MessageFlags: r.Policy2MessageFlags, + ChannelFlags: r.Policy2ChannelFlags, + Signature: r.Policy2Signature, + } + } + + return policy1, policy2, nil + + case sqlc.ListChannelsWithPoliciesPaginatedRow: + if r.Policy1ID.Valid { + policy1 = &sqlc.GraphChannelPolicy{ + ID: r.Policy1ID.Int64, + Version: r.Policy1Version.Int16, + ChannelID: r.GraphChannel.ID, + NodeID: r.Policy1NodeID.Int64, + Timelock: r.Policy1Timelock.Int32, + FeePpm: r.Policy1FeePpm.Int64, + BaseFeeMsat: r.Policy1BaseFeeMsat.Int64, + MinHtlcMsat: r.Policy1MinHtlcMsat.Int64, + MaxHtlcMsat: r.Policy1MaxHtlcMsat, + LastUpdate: r.Policy1LastUpdate, + InboundBaseFeeMsat: r.Policy1InboundBaseFeeMsat, + InboundFeeRateMilliMsat: r.Policy1InboundFeeRateMilliMsat, + Disabled: r.Policy1Disabled, + MessageFlags: r.Policy1MessageFlags, + ChannelFlags: r.Policy1ChannelFlags, + Signature: r.Policy1Signature, + } + } + if r.Policy2ID.Valid { + policy2 = &sqlc.GraphChannelPolicy{ + ID: r.Policy2ID.Int64, + Version: r.Policy2Version.Int16, + ChannelID: r.GraphChannel.ID, + NodeID: r.Policy2NodeID.Int64, + Timelock: r.Policy2Timelock.Int32, + FeePpm: r.Policy2FeePpm.Int64, + BaseFeeMsat: r.Policy2BaseFeeMsat.Int64, + MinHtlcMsat: r.Policy2MinHtlcMsat.Int64, + MaxHtlcMsat: r.Policy2MaxHtlcMsat, + LastUpdate: r.Policy2LastUpdate, + InboundBaseFeeMsat: r.Policy2InboundBaseFeeMsat, + InboundFeeRateMilliMsat: r.Policy2InboundFeeRateMilliMsat, + Disabled: r.Policy2Disabled, + MessageFlags: r.Policy2MessageFlags, + ChannelFlags: r.Policy2ChannelFlags, + Signature: r.Policy2Signature, + } + } + + return policy1, policy2, nil + + case sqlc.GetChannelsByIDsRow: + if r.Policy1ID.Valid { + policy1 = &sqlc.GraphChannelPolicy{ + ID: r.Policy1ID.Int64, + Version: r.Policy1Version.Int16, + ChannelID: r.GraphChannel.ID, + NodeID: r.Policy1NodeID.Int64, + Timelock: r.Policy1Timelock.Int32, + FeePpm: r.Policy1FeePpm.Int64, + BaseFeeMsat: r.Policy1BaseFeeMsat.Int64, + MinHtlcMsat: r.Policy1MinHtlcMsat.Int64, + MaxHtlcMsat: r.Policy1MaxHtlcMsat, + LastUpdate: r.Policy1LastUpdate, + InboundBaseFeeMsat: r.Policy1InboundBaseFeeMsat, + InboundFeeRateMilliMsat: r.Policy1InboundFeeRateMilliMsat, + Disabled: r.Policy1Disabled, + MessageFlags: r.Policy1MessageFlags, + ChannelFlags: r.Policy1ChannelFlags, + Signature: r.Policy1Signature, + } + } + if r.Policy2ID.Valid { + policy2 = &sqlc.GraphChannelPolicy{ + ID: r.Policy2ID.Int64, + Version: r.Policy2Version.Int16, + ChannelID: r.GraphChannel.ID, + NodeID: r.Policy2NodeID.Int64, + Timelock: r.Policy2Timelock.Int32, + FeePpm: r.Policy2FeePpm.Int64, + BaseFeeMsat: r.Policy2BaseFeeMsat.Int64, + MinHtlcMsat: r.Policy2MinHtlcMsat.Int64, + MaxHtlcMsat: r.Policy2MaxHtlcMsat, + LastUpdate: r.Policy2LastUpdate, + InboundBaseFeeMsat: r.Policy2InboundBaseFeeMsat, + InboundFeeRateMilliMsat: r.Policy2InboundFeeRateMilliMsat, + Disabled: r.Policy2Disabled, + MessageFlags: r.Policy2MessageFlags, + ChannelFlags: r.Policy2ChannelFlags, + Signature: r.Policy2Signature, + } + } + + return policy1, policy2, nil + + default: + return nil, nil, fmt.Errorf("unexpected row type in "+ + "extractChannelPolicies: %T", r) + } +} + +// channelIDToBytes converts a channel ID (SCID) to a byte array +// representation. +func channelIDToBytes(channelID uint64) []byte { + var chanIDB [8]byte + byteOrder.PutUint64(chanIDB[:], channelID) + + return chanIDB[:] +} + +// buildNodeAddresses converts a slice of nodeAddress into a slice of net.Addr. +func buildNodeAddresses(addresses []nodeAddress) ([]net.Addr, error) { + if len(addresses) == 0 { + return nil, nil + } + + result := make([]net.Addr, 0, len(addresses)) + for _, addr := range addresses { + netAddr, err := parseAddress(addr.addrType, addr.address) + if err != nil { + return nil, fmt.Errorf("unable to parse address %s "+ + "of type %d: %w", addr.address, addr.addrType, + err) + } + if netAddr != nil { + result = append(result, netAddr) + } + } + + // If we have no valid addresses, return nil instead of empty slice. + if len(result) == 0 { + return nil, nil + } + + return result, nil +} + +// parseAddress parses the given address string based on the address type +// and returns a net.Addr instance. It supports IPv4, IPv6, Tor v2, Tor v3, +// and opaque addresses. +func parseAddress(addrType dbAddressType, address string) (net.Addr, error) { + switch addrType { + case addressTypeIPv4: + tcp, err := net.ResolveTCPAddr("tcp4", address) + if err != nil { + return nil, err + } + + tcp.IP = tcp.IP.To4() + + return tcp, nil + + case addressTypeIPv6: + tcp, err := net.ResolveTCPAddr("tcp6", address) + if err != nil { + return nil, err + } + + return tcp, nil + + case addressTypeTorV3, addressTypeTorV2: + service, portStr, err := net.SplitHostPort(address) + if err != nil { + return nil, fmt.Errorf("unable to split tor "+ + "address: %v", address) + } + + port, err := strconv.Atoi(portStr) + if err != nil { + return nil, err + } + + return &tor.OnionAddr{ + OnionService: service, + Port: port, + }, nil + + case addressTypeDNS: + hostname, portStr, err := net.SplitHostPort(address) + if err != nil { + return nil, fmt.Errorf("unable to split DNS "+ + "address: %v", address) + } + + port, err := strconv.Atoi(portStr) + if err != nil { + return nil, err + } + + return &lnwire.DNSAddress{ + Hostname: hostname, + Port: uint16(port), + }, nil + + case addressTypeOpaque: + opaque, err := hex.DecodeString(address) + if err != nil { + return nil, fmt.Errorf("unable to decode opaque "+ + "address: %v", address) + } + + return &lnwire.OpaqueAddrs{ + Payload: opaque, + }, nil + + default: + return nil, fmt.Errorf("unknown address type: %v", addrType) + } +} + +// batchNodeData holds all the related data for a batch of nodes. +type batchNodeData struct { + // features is a map from a DB node ID to the feature bits for that + // node. + features map[int64][]int + + // addresses is a map from a DB node ID to the node's addresses. + addresses map[int64][]nodeAddress + + // extraFields is a map from a DB node ID to the extra signed fields + // for that node. + extraFields map[int64]map[uint64][]byte +} + +// nodeAddress holds the address type, position and address string for a +// node. This is used to batch the fetching of node addresses. +type nodeAddress struct { + addrType dbAddressType + position int32 + address string +} + +// batchLoadNodeData loads all related data for a batch of node IDs using the +// provided SQLQueries interface. It returns a batchNodeData instance containing +// the node features, addresses and extra signed fields. +func batchLoadNodeData(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, nodeIDs []int64) (*batchNodeData, error) { + + // Batch load the node features. + features, err := batchLoadNodeFeaturesHelper(ctx, cfg, db, nodeIDs) + if err != nil { + return nil, fmt.Errorf("unable to batch load node "+ + "features: %w", err) + } + + // Batch load the node addresses. + addrs, err := batchLoadNodeAddressesHelper(ctx, cfg, db, nodeIDs) + if err != nil { + return nil, fmt.Errorf("unable to batch load node "+ + "addresses: %w", err) + } + + // Batch load the node extra signed fields. + extraTypes, err := batchLoadNodeExtraTypesHelper(ctx, cfg, db, nodeIDs) + if err != nil { + return nil, fmt.Errorf("unable to batch load node extra "+ + "signed fields: %w", err) + } + + return &batchNodeData{ + features: features, + addresses: addrs, + extraFields: extraTypes, + }, nil +} + +// batchLoadNodeFeaturesHelper loads node features for a batch of node IDs +// using ExecuteBatchQuery wrapper around the GetNodeFeaturesBatch query. +func batchLoadNodeFeaturesHelper(ctx context.Context, + cfg *sqldb.QueryConfig, db SQLQueries, + nodeIDs []int64) (map[int64][]int, error) { + + features := make(map[int64][]int) + + return features, sqldb.ExecuteBatchQuery( + ctx, cfg, nodeIDs, + func(id int64) int64 { + return id + }, + func(ctx context.Context, ids []int64) ([]sqlc.GraphNodeFeature, + error) { + + return db.GetNodeFeaturesBatch(ctx, ids) + }, + func(ctx context.Context, feature sqlc.GraphNodeFeature) error { + features[feature.NodeID] = append( + features[feature.NodeID], + int(feature.FeatureBit), + ) + + return nil + }, + ) +} + +// batchLoadNodeAddressesHelper loads node addresses using ExecuteBatchQuery +// wrapper around the GetNodeAddressesBatch query. It returns a map from +// node ID to a slice of nodeAddress structs. +func batchLoadNodeAddressesHelper(ctx context.Context, + cfg *sqldb.QueryConfig, db SQLQueries, + nodeIDs []int64) (map[int64][]nodeAddress, error) { + + addrs := make(map[int64][]nodeAddress) + + return addrs, sqldb.ExecuteBatchQuery( + ctx, cfg, nodeIDs, + func(id int64) int64 { + return id + }, + func(ctx context.Context, ids []int64) ([]sqlc.GraphNodeAddress, + error) { + + return db.GetNodeAddressesBatch(ctx, ids) + }, + func(ctx context.Context, addr sqlc.GraphNodeAddress) error { + addrs[addr.NodeID] = append( + addrs[addr.NodeID], nodeAddress{ + addrType: dbAddressType(addr.Type), + position: addr.Position, + address: addr.Address, + }, + ) + + return nil + }, + ) +} + +// batchLoadNodeExtraTypesHelper loads node extra type bytes for a batch of +// node IDs using ExecuteBatchQuery wrapper around the GetNodeExtraTypesBatch +// query. +func batchLoadNodeExtraTypesHelper(ctx context.Context, + cfg *sqldb.QueryConfig, db SQLQueries, + nodeIDs []int64) (map[int64]map[uint64][]byte, error) { + + extraFields := make(map[int64]map[uint64][]byte) + + callback := func(ctx context.Context, + field sqlc.GraphNodeExtraType) error { + + if extraFields[field.NodeID] == nil { + extraFields[field.NodeID] = make(map[uint64][]byte) + } + extraFields[field.NodeID][uint64(field.Type)] = field.Value + + return nil + } + + return extraFields, sqldb.ExecuteBatchQuery( + ctx, cfg, nodeIDs, + func(id int64) int64 { + return id + }, + func(ctx context.Context, ids []int64) ( + []sqlc.GraphNodeExtraType, error) { + + return db.GetNodeExtraTypesBatch(ctx, ids) + }, + callback, + ) +} + +// buildChanPoliciesWithBatchData builds two models.ChannelEdgePolicy instances +// from the provided sqlc.GraphChannelPolicy records and the +// provided batchChannelData. +func buildChanPoliciesWithBatchData(dbPol1, dbPol2 *sqlc.GraphChannelPolicy, + channelID uint64, node1, node2 route.Vertex, + batchData *batchChannelData) (*models.ChannelEdgePolicy, + *models.ChannelEdgePolicy, error) { + + pol1, err := buildChanPolicyWithBatchData( + dbPol1, channelID, node2, batchData, + ) + if err != nil { + return nil, nil, fmt.Errorf("unable to build policy1: %w", err) + } + + pol2, err := buildChanPolicyWithBatchData( + dbPol2, channelID, node1, batchData, + ) + if err != nil { + return nil, nil, fmt.Errorf("unable to build policy2: %w", err) + } + + return pol1, pol2, nil +} + +// buildChanPolicyWithBatchData builds a models.ChannelEdgePolicy instance from +// the provided sqlc.GraphChannelPolicy and the provided batchChannelData. +func buildChanPolicyWithBatchData(dbPol *sqlc.GraphChannelPolicy, + channelID uint64, toNode route.Vertex, + batchData *batchChannelData) (*models.ChannelEdgePolicy, error) { + + if dbPol == nil { + return nil, nil + } + + var dbPol1Extras map[uint64][]byte + if extras, exists := batchData.policyExtras[dbPol.ID]; exists { + dbPol1Extras = extras + } else { + dbPol1Extras = make(map[uint64][]byte) + } + + return buildChanPolicy(*dbPol, channelID, dbPol1Extras, toNode) +} + +// batchChannelData holds all the related data for a batch of channels. +type batchChannelData struct { + // chanFeatures is a map from DB channel ID to a slice of feature bits. + chanfeatures map[int64][]int + + // chanExtras is a map from DB channel ID to a map of TLV type to + // extra signed field bytes. + chanExtraTypes map[int64]map[uint64][]byte + + // policyExtras is a map from DB channel policy ID to a map of TLV type + // to extra signed field bytes. + policyExtras map[int64]map[uint64][]byte +} + +// batchLoadChannelData loads all related data for batches of channels and +// policies. +func batchLoadChannelData(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, channelIDs []int64, + policyIDs []int64) (*batchChannelData, error) { + + batchData := &batchChannelData{ + chanfeatures: make(map[int64][]int), + chanExtraTypes: make(map[int64]map[uint64][]byte), + policyExtras: make(map[int64]map[uint64][]byte), + } + + // Batch load channel features and extras + var err error + if len(channelIDs) > 0 { + batchData.chanfeatures, err = batchLoadChannelFeaturesHelper( + ctx, cfg, db, channelIDs, + ) + if err != nil { + return nil, fmt.Errorf("unable to batch load "+ + "channel features: %w", err) + } + + batchData.chanExtraTypes, err = batchLoadChannelExtrasHelper( + ctx, cfg, db, channelIDs, + ) + if err != nil { + return nil, fmt.Errorf("unable to batch load "+ + "channel extras: %w", err) + } + } + + if len(policyIDs) > 0 { + policyExtras, err := batchLoadChannelPolicyExtrasHelper( + ctx, cfg, db, policyIDs, + ) + if err != nil { + return nil, fmt.Errorf("unable to batch load "+ + "policy extras: %w", err) + } + batchData.policyExtras = policyExtras + } + + return batchData, nil +} + +// batchLoadChannelFeaturesHelper loads channel features for a batch of +// channel IDs using ExecuteBatchQuery wrapper around the +// GetChannelFeaturesBatch query. It returns a map from DB channel ID to a +// slice of feature bits. +func batchLoadChannelFeaturesHelper(ctx context.Context, + cfg *sqldb.QueryConfig, db SQLQueries, + channelIDs []int64) (map[int64][]int, error) { + + features := make(map[int64][]int) + + return features, sqldb.ExecuteBatchQuery( + ctx, cfg, channelIDs, + func(id int64) int64 { + return id + }, + func(ctx context.Context, + ids []int64) ([]sqlc.GraphChannelFeature, error) { + + return db.GetChannelFeaturesBatch(ctx, ids) + }, + func(ctx context.Context, + feature sqlc.GraphChannelFeature) error { + + features[feature.ChannelID] = append( + features[feature.ChannelID], + int(feature.FeatureBit), + ) + + return nil + }, + ) +} + +// batchLoadChannelExtrasHelper loads channel extra types for a batch of +// channel IDs using ExecuteBatchQuery wrapper around the GetChannelExtrasBatch +// query. It returns a map from DB channel ID to a map of TLV type to extra +// signed field bytes. +func batchLoadChannelExtrasHelper(ctx context.Context, + cfg *sqldb.QueryConfig, db SQLQueries, + channelIDs []int64) (map[int64]map[uint64][]byte, error) { + + extras := make(map[int64]map[uint64][]byte) + + cb := func(ctx context.Context, + extra sqlc.GraphChannelExtraType) error { + + if extras[extra.ChannelID] == nil { + extras[extra.ChannelID] = make(map[uint64][]byte) + } + extras[extra.ChannelID][uint64(extra.Type)] = extra.Value + + return nil + } + + return extras, sqldb.ExecuteBatchQuery( + ctx, cfg, channelIDs, + func(id int64) int64 { + return id + }, + func(ctx context.Context, + ids []int64) ([]sqlc.GraphChannelExtraType, error) { + + return db.GetChannelExtrasBatch(ctx, ids) + }, cb, + ) +} + +// batchLoadChannelPolicyExtrasHelper loads channel policy extra types for a +// batch of policy IDs using ExecuteBatchQuery wrapper around the +// GetChannelPolicyExtraTypesBatch query. It returns a map from DB policy ID to +// a map of TLV type to extra signed field bytes. +func batchLoadChannelPolicyExtrasHelper(ctx context.Context, + cfg *sqldb.QueryConfig, db SQLQueries, + policyIDs []int64) (map[int64]map[uint64][]byte, error) { + + extras := make(map[int64]map[uint64][]byte) + + return extras, sqldb.ExecuteBatchQuery( + ctx, cfg, policyIDs, + func(id int64) int64 { + return id + }, + func(ctx context.Context, ids []int64) ( + []sqlc.GetChannelPolicyExtraTypesBatchRow, error) { + + return db.GetChannelPolicyExtraTypesBatch(ctx, ids) + }, + func(ctx context.Context, + row sqlc.GetChannelPolicyExtraTypesBatchRow) error { + + if extras[row.PolicyID] == nil { + extras[row.PolicyID] = make(map[uint64][]byte) + } + extras[row.PolicyID][uint64(row.Type)] = row.Value + + return nil + }, + ) +} + +// forEachNodePaginated executes a paginated query to process each node in the +// graph. It uses the provided SQLQueries interface to fetch nodes in batches +// and applies the provided processNode function to each node. +func forEachNodePaginated(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, protocol lnwire.GossipVersion, + processNode func(context.Context, int64, + *models.Node) error) error { + + pageQueryFunc := func(ctx context.Context, lastID int64, + limit int32) ([]sqlc.GraphNode, error) { + + return db.ListNodesPaginated( + ctx, sqlc.ListNodesPaginatedParams{ + Version: int16(protocol), + ID: lastID, + Limit: limit, + }, + ) + } + + extractPageCursor := func(node sqlc.GraphNode) int64 { + return node.ID + } + + collectFunc := func(node sqlc.GraphNode) (int64, error) { + return node.ID, nil + } + + batchQueryFunc := func(ctx context.Context, + nodeIDs []int64) (*batchNodeData, error) { + + return batchLoadNodeData(ctx, cfg, db, nodeIDs) + } + + processItem := func(ctx context.Context, dbNode sqlc.GraphNode, + batchData *batchNodeData) error { + + node, err := buildNodeWithBatchData(dbNode, batchData) + if err != nil { + return fmt.Errorf("unable to build "+ + "node(id=%d): %w", dbNode.ID, err) + } + + return processNode(ctx, dbNode.ID, node) + } + + return sqldb.ExecuteCollectAndBatchWithSharedDataQuery( + ctx, cfg, int64(-1), pageQueryFunc, extractPageCursor, + collectFunc, batchQueryFunc, processItem, + ) +} + +// forEachChannelWithPolicies executes a paginated query to process each channel +// with policies in the graph. +func forEachChannelWithPolicies(ctx context.Context, db SQLQueries, + cfg *SQLStoreConfig, processChannel func(*models.ChannelEdgeInfo, + *models.ChannelEdgePolicy, + *models.ChannelEdgePolicy) error) error { + + type channelBatchIDs struct { + channelID int64 + policyIDs []int64 + } + + pageQueryFunc := func(ctx context.Context, lastID int64, + limit int32) ([]sqlc.ListChannelsWithPoliciesPaginatedRow, + error) { + + return db.ListChannelsWithPoliciesPaginated( + ctx, sqlc.ListChannelsWithPoliciesPaginatedParams{ + Version: int16(lnwire.GossipVersion1), + ID: lastID, + Limit: limit, + }, + ) + } + + extractPageCursor := func( + row sqlc.ListChannelsWithPoliciesPaginatedRow) int64 { + + return row.GraphChannel.ID + } + + collectFunc := func(row sqlc.ListChannelsWithPoliciesPaginatedRow) ( + channelBatchIDs, error) { + + ids := channelBatchIDs{ + channelID: row.GraphChannel.ID, + } + + // Extract policy IDs from the row. + dbPol1, dbPol2, err := extractChannelPolicies(row) + if err != nil { + return ids, err + } + + if dbPol1 != nil { + ids.policyIDs = append(ids.policyIDs, dbPol1.ID) + } + if dbPol2 != nil { + ids.policyIDs = append(ids.policyIDs, dbPol2.ID) + } + + return ids, nil + } + + batchDataFunc := func(ctx context.Context, + allIDs []channelBatchIDs) (*batchChannelData, error) { + + // Separate channel IDs from policy IDs. + var ( + channelIDs = make([]int64, len(allIDs)) + policyIDs = make([]int64, 0, len(allIDs)*2) + ) + + for i, ids := range allIDs { + channelIDs[i] = ids.channelID + policyIDs = append(policyIDs, ids.policyIDs...) + } + + return batchLoadChannelData( + ctx, cfg.QueryCfg, db, channelIDs, policyIDs, + ) + } + + processItem := func(ctx context.Context, + row sqlc.ListChannelsWithPoliciesPaginatedRow, + batchData *batchChannelData) error { + + node1, node2, err := buildNodeVertices( + row.Node1Pubkey, row.Node2Pubkey, + ) + if err != nil { + return err + } + + edge, err := buildEdgeInfoWithBatchData( + cfg.ChainHash, row.GraphChannel, node1, node2, + batchData, + ) + if err != nil { + return fmt.Errorf("unable to build channel info: %w", + err) + } + + dbPol1, dbPol2, err := extractChannelPolicies(row) + if err != nil { + return err + } + + p1, p2, err := buildChanPoliciesWithBatchData( + dbPol1, dbPol2, edge.ChannelID, node1, node2, batchData, + ) + if err != nil { + return err + } + + return processChannel(edge, p1, p2) + } + + return sqldb.ExecuteCollectAndBatchWithSharedDataQuery( + ctx, cfg.QueryCfg, int64(-1), pageQueryFunc, extractPageCursor, + collectFunc, batchDataFunc, processItem, + ) +} diff --git a/graph/db/migration1/test_kvdb.go b/graph/db/migration1/test_kvdb.go new file mode 100644 index 000000000..4d2f0dbb7 --- /dev/null +++ b/graph/db/migration1/test_kvdb.go @@ -0,0 +1,23 @@ +//go:build !test_db_sqlite && !test_db_postgres + +package migration1 + +import ( + "testing" + + "github.com/lightningnetwork/lnd/kvdb" + "github.com/stretchr/testify/require" +) + +// NewTestDB is a helper function that creates an BBolt database for testing. +func NewTestDB(t testing.TB) V1Store { + backend, backendCleanup, err := kvdb.GetTestBackend(t.TempDir(), "cgr") + require.NoError(t, err) + + t.Cleanup(backendCleanup) + + graphStore, err := NewKVStore(backend) + require.NoError(t, err) + + return graphStore +} diff --git a/graph/db/migration1/test_postgres.go b/graph/db/migration1/test_postgres.go new file mode 100644 index 000000000..666e6fb34 --- /dev/null +++ b/graph/db/migration1/test_postgres.go @@ -0,0 +1,79 @@ +//go:build test_db_postgres && !test_db_sqlite + +package migration1 + +import ( + "database/sql" + "testing" + + "github.com/btcsuite/btcd/chaincfg" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/stretchr/testify/require" +) + +// NewTestDB is a helper function that creates a SQLStore backed by a SQL +// database for testing. +func NewTestDB(t testing.TB) V1Store { + return NewTestDBWithFixture(t, nil) +} + +// NewTestDBFixture creates a new sqldb.TestPgFixture for testing purposes. +func NewTestDBFixture(t *testing.T) *sqldb.TestPgFixture { + pgFixture := sqldb.NewTestPgFixture( + t, sqldb.DefaultPostgresFixtureLifetime, + ) + t.Cleanup(func() { + pgFixture.TearDown(t) + }) + return pgFixture +} + +// NewTestDBWithFixture is a helper function that creates a SQLStore backed by a +// SQL database for testing. +func NewTestDBWithFixture(t testing.TB, + pgFixture *sqldb.TestPgFixture) V1Store { + + var querier BatchedSQLQueries + if pgFixture == nil { + querier = newBatchQuerier(t) + } else { + querier = newBatchQuerierWithFixture(t, pgFixture) + } + + store, err := NewSQLStore( + &SQLStoreConfig{ + ChainHash: *chaincfg.MainNetParams.GenesisHash, + QueryCfg: sqldb.DefaultPostgresConfig(), + }, querier, + ) + require.NoError(t, err) + + return store +} + +// newBatchQuerier creates a new BatchedSQLQueries instance for testing +// using a PostgreSQL database fixture. +func newBatchQuerier(t testing.TB) BatchedSQLQueries { + pgFixture := sqldb.NewTestPgFixture( + t, sqldb.DefaultPostgresFixtureLifetime, + ) + t.Cleanup(func() { + pgFixture.TearDown(t) + }) + + return newBatchQuerierWithFixture(t, pgFixture) +} + +// newBatchQuerierWithFixture creates a new BatchedSQLQueries instance for +// testing using a PostgreSQL database fixture. +func newBatchQuerierWithFixture(t testing.TB, + pgFixture *sqldb.TestPgFixture) BatchedSQLQueries { + + db := sqldb.NewTestPostgresDB(t, pgFixture).BaseDB + + return sqldb.NewTransactionExecutor( + db, func(tx *sql.Tx) SQLQueries { + return db.WithTx(tx) + }, + ) +} diff --git a/graph/db/migration1/test_sqlite.go b/graph/db/migration1/test_sqlite.go new file mode 100644 index 000000000..3b7d80fe1 --- /dev/null +++ b/graph/db/migration1/test_sqlite.go @@ -0,0 +1,56 @@ +//go:build !test_db_postgres && test_db_sqlite + +package migration1 + +import ( + "database/sql" + "testing" + + "github.com/btcsuite/btcd/chaincfg" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/stretchr/testify/require" +) + +// NewTestDB is a helper function that creates a SQLStore backed by a SQL +// database for testing. +func NewTestDB(t testing.TB) V1Store { + return NewTestDBWithFixture(t, nil) +} + +// NewTestDBFixture is a no-op for the sqlite build. +func NewTestDBFixture(_ *testing.T) *sqldb.TestPgFixture { + return nil +} + +// NewTestDBWithFixture is a helper function that creates a SQLStore backed by a +// SQL database for testing. +func NewTestDBWithFixture(t testing.TB, _ *sqldb.TestPgFixture) V1Store { + store, err := NewSQLStore( + &SQLStoreConfig{ + ChainHash: *chaincfg.MainNetParams.GenesisHash, + QueryCfg: sqldb.DefaultSQLiteConfig(), + }, newBatchQuerier(t), + ) + require.NoError(t, err) + return store +} + +// newBatchQuerier creates a new BatchedSQLQueries instance for testing +// using a SQLite database. +func newBatchQuerier(t testing.TB) BatchedSQLQueries { + return newBatchQuerierWithFixture(t, nil) +} + +// newBatchQuerierWithFixture creates a new BatchedSQLQueries instance for +// testing using a SQLite database. +func newBatchQuerierWithFixture(t testing.TB, + _ *sqldb.TestPgFixture) BatchedSQLQueries { + + db := sqldb.NewTestSqliteDB(t).BaseDB + + return sqldb.NewTransactionExecutor( + db, func(tx *sql.Tx) SQLQueries { + return db.WithTx(tx) + }, + ) +}