channeldb: make copy for migrations

This commit is a direct copy of the complete channeldb package. It only
changes the package declaration at the top of every file. We make this
full copy so that review can be focused on the actual changes made.
Otherwise changes may drown in all the file moves.

Linting for the new package is disabled, as it contains lots of
pre-existing issues.

.golangci.yml

@@ -10,6 +10,9 @@ run:
   skip-files:
     - "mobile\\/.*generated\\.go"
 
+  skip-dirs:
+    - channeldb/migration_01_to_11
+
   build-tags:
     - autopilotrpc
     - chainrpc

channeldb/migration_01_to_11/README.md

@@ -0,0 +1,24 @@
+channeldb
+==========
+
+[![Build Status](http://img.shields.io/travis/lightningnetwork/lnd.svg)](https://travis-ci.org/lightningnetwork/lnd) 
+[![MIT licensed](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/lightningnetwork/lnd/blob/master/LICENSE)
+[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/lightningnetwork/lnd/channeldb)
+
+The channeldb implements the persistent storage engine for `lnd` and
+generically a data storage layer for the required state within the Lightning
+Network. The backing storage engine is
+[boltdb](https://github.com/coreos/bbolt), an embedded pure-go key-value store
+based off of LMDB.
+
+The package implements an object-oriented storage model with queries and
+mutations flowing through a particular object instance rather than the database
+itself. The storage implemented by the objects includes: open channels, past
+commitment revocation states, the channel graph which includes authenticated
+node and channel announcements, outgoing payments, and invoices
+
+## Installation and Updating
+
+```bash
+$ go get -u github.com/lightningnetwork/lnd/channeldb
+```

channeldb/migration_01_to_11/addr.go

@@ -0,0 +1,221 @@
+package migration_01_to_11
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+
+	"github.com/lightningnetwork/lnd/tor"
+)
+
+// addressType specifies the network protocol and version that should be used
+// when connecting to a node at a particular address.
+type addressType uint8
+
+const (
+	// tcp4Addr denotes an IPv4 TCP address.
+	tcp4Addr addressType = 0
+
+	// tcp6Addr denotes an IPv6 TCP address.
+	tcp6Addr addressType = 1
+
+	// v2OnionAddr denotes a version 2 Tor onion service address.
+	v2OnionAddr addressType = 2
+
+	// v3OnionAddr denotes a version 3 Tor (prop224) onion service address.
+	v3OnionAddr addressType = 3
+)
+
+// encodeTCPAddr serializes a TCP address into its compact raw bytes
+// representation.
+func encodeTCPAddr(w io.Writer, addr *net.TCPAddr) error {
+	var (
+		addrType byte
+		ip       []byte
+	)
+
+	if addr.IP.To4() != nil {
+		addrType = byte(tcp4Addr)
+		ip = addr.IP.To4()
+	} else {
+		addrType = byte(tcp6Addr)
+		ip = addr.IP.To16()
+	}
+
+	if ip == nil {
+		return fmt.Errorf("unable to encode IP %v", addr.IP)
+	}
+
+	if _, err := w.Write([]byte{addrType}); err != nil {
+		return err
+	}
+
+	if _, err := w.Write(ip); err != nil {
+		return err
+	}
+
+	var port [2]byte
+	byteOrder.PutUint16(port[:], uint16(addr.Port))
+	if _, err := w.Write(port[:]); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// encodeOnionAddr serializes an onion address into its compact raw bytes
+// representation.
+func encodeOnionAddr(w io.Writer, addr *tor.OnionAddr) error {
+	var suffixIndex int
+	hostLen := len(addr.OnionService)
+	switch hostLen {
+	case tor.V2Len:
+		if _, err := w.Write([]byte{byte(v2OnionAddr)}); err != nil {
+			return err
+		}
+		suffixIndex = tor.V2Len - tor.OnionSuffixLen
+	case tor.V3Len:
+		if _, err := w.Write([]byte{byte(v3OnionAddr)}); err != nil {
+			return err
+		}
+		suffixIndex = tor.V3Len - tor.OnionSuffixLen
+	default:
+		return errors.New("unknown onion service length")
+	}
+
+	suffix := addr.OnionService[suffixIndex:]
+	if suffix != tor.OnionSuffix {
+		return fmt.Errorf("invalid suffix \"%v\"", suffix)
+	}
+
+	host, err := tor.Base32Encoding.DecodeString(
+		addr.OnionService[:suffixIndex],
+	)
+	if err != nil {
+		return err
+	}
+
+	// Sanity check the decoded length.
+	switch {
+	case hostLen == tor.V2Len && len(host) != tor.V2DecodedLen:
+		return fmt.Errorf("onion service %v decoded to invalid host %x",
+			addr.OnionService, host)
+
+	case hostLen == tor.V3Len && len(host) != tor.V3DecodedLen:
+		return fmt.Errorf("onion service %v decoded to invalid host %x",
+			addr.OnionService, host)
+	}
+
+	if _, err := w.Write(host); err != nil {
+		return err
+	}
+
+	var port [2]byte
+	byteOrder.PutUint16(port[:], uint16(addr.Port))
+	if _, err := w.Write(port[:]); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// deserializeAddr reads the serialized raw representation of an address and
+// deserializes it into the actual address. This allows us to avoid address
+// resolution within the channeldb package.
+func deserializeAddr(r io.Reader) (net.Addr, error) {
+	var addrType [1]byte
+	if _, err := r.Read(addrType[:]); err != nil {
+		return nil, err
+	}
+
+	var address net.Addr
+	switch addressType(addrType[0]) {
+	case tcp4Addr:
+		var ip [4]byte
+		if _, err := r.Read(ip[:]); err != nil {
+			return nil, err
+		}
+
+		var port [2]byte
+		if _, err := r.Read(port[:]); err != nil {
+			return nil, err
+		}
+
+		address = &net.TCPAddr{
+			IP:   net.IP(ip[:]),
+			Port: int(binary.BigEndian.Uint16(port[:])),
+		}
+	case tcp6Addr:
+		var ip [16]byte
+		if _, err := r.Read(ip[:]); err != nil {
+			return nil, err
+		}
+
+		var port [2]byte
+		if _, err := r.Read(port[:]); err != nil {
+			return nil, err
+		}
+
+		address = &net.TCPAddr{
+			IP:   net.IP(ip[:]),
+			Port: int(binary.BigEndian.Uint16(port[:])),
+		}
+	case v2OnionAddr:
+		var h [tor.V2DecodedLen]byte
+		if _, err := r.Read(h[:]); err != nil {
+			return nil, err
+		}
+
+		var p [2]byte
+		if _, err := r.Read(p[:]); err != nil {
+			return nil, err
+		}
+
+		onionService := tor.Base32Encoding.EncodeToString(h[:])
+		onionService += tor.OnionSuffix
+		port := int(binary.BigEndian.Uint16(p[:]))
+
+		address = &tor.OnionAddr{
+			OnionService: onionService,
+			Port:         port,
+		}
+	case v3OnionAddr:
+		var h [tor.V3DecodedLen]byte
+		if _, err := r.Read(h[:]); err != nil {
+			return nil, err
+		}
+
+		var p [2]byte
+		if _, err := r.Read(p[:]); err != nil {
+			return nil, err
+		}
+
+		onionService := tor.Base32Encoding.EncodeToString(h[:])
+		onionService += tor.OnionSuffix
+		port := int(binary.BigEndian.Uint16(p[:]))
+
+		address = &tor.OnionAddr{
+			OnionService: onionService,
+			Port:         port,
+		}
+	default:
+		return nil, ErrUnknownAddressType
+	}
+
+	return address, nil
+}
+
+// serializeAddr serializes an address into its raw bytes representation so that
+// it can be deserialized without requiring address resolution.
+func serializeAddr(w io.Writer, address net.Addr) error {
+	switch addr := address.(type) {
+	case *net.TCPAddr:
+		return encodeTCPAddr(w, addr)
+	case *tor.OnionAddr:
+		return encodeOnionAddr(w, addr)
+	default:
+		return ErrUnknownAddressType
+	}
+}

channeldb/migration_01_to_11/addr_test.go

@@ -0,0 +1,149 @@
+package migration_01_to_11
+
+import (
+	"bytes"
+	"net"
+	"strings"
+	"testing"
+
+	"github.com/lightningnetwork/lnd/tor"
+)
+
+type unknownAddrType struct{}
+
+func (t unknownAddrType) Network() string { return "unknown" }
+func (t unknownAddrType) String() string  { return "unknown" }
+
+var testIP4 = net.ParseIP("192.168.1.1")
+var testIP6 = net.ParseIP("2001:0db8:0000:0000:0000:ff00:0042:8329")
+
+var addrTests = []struct {
+	expAddr net.Addr
+	serErr  string
+}{
+	// Valid addresses.
+	{
+		expAddr: &net.TCPAddr{
+			IP:   testIP4,
+			Port: 12345,
+		},
+	},
+	{
+		expAddr: &net.TCPAddr{
+			IP:   testIP6,
+			Port: 65535,
+		},
+	},
+	{
+		expAddr: &tor.OnionAddr{
+			OnionService: "3g2upl4pq6kufc4m.onion",
+			Port:         9735,
+		},
+	},
+	{
+		expAddr: &tor.OnionAddr{
+			OnionService: "vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd.onion",
+			Port:         80,
+		},
+	},
+
+	// Invalid addresses.
+	{
+		expAddr: unknownAddrType{},
+		serErr:  ErrUnknownAddressType.Error(),
+	},
+	{
+		expAddr: &net.TCPAddr{
+			// Remove last byte of IPv4 address.
+			IP:   testIP4[:len(testIP4)-1],
+			Port: 12345,
+		},
+		serErr: "unable to encode",
+	},
+	{
+		expAddr: &net.TCPAddr{
+			// Add an extra byte of IPv4 address.
+			IP:   append(testIP4, 0xff),
+			Port: 12345,
+		},
+		serErr: "unable to encode",
+	},
+	{
+		expAddr: &net.TCPAddr{
+			// Remove last byte of IPv6 address.
+			IP:   testIP6[:len(testIP6)-1],
+			Port: 65535,
+		},
+		serErr: "unable to encode",
+	},
+	{
+		expAddr: &net.TCPAddr{
+			// Add an extra byte to the IPv6 address.
+			IP:   append(testIP6, 0xff),
+			Port: 65535,
+		},
+		serErr: "unable to encode",
+	},
+	{
+		expAddr: &tor.OnionAddr{
+			// Invalid suffix.
+			OnionService: "vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd.inion",
+			Port:         80,
+		},
+		serErr: "invalid suffix",
+	},
+	{
+		expAddr: &tor.OnionAddr{
+			// Invalid length.
+			OnionService: "vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyy.onion",
+			Port:         80,
+		},
+		serErr: "unknown onion service length",
+	},
+	{
+		expAddr: &tor.OnionAddr{
+			// Invalid encoding.
+			OnionService: "vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyA.onion",
+			Port:         80,
+		},
+		serErr: "illegal base32",
+	},
+}
+
+// TestAddrSerialization tests that the serialization method used by channeldb
+// for net.Addr's works as intended.
+func TestAddrSerialization(t *testing.T) {
+	t.Parallel()
+
+	var b bytes.Buffer
+	for _, test := range addrTests {
+		err := serializeAddr(&b, test.expAddr)
+		switch {
+		case err == nil && test.serErr != "":
+			t.Fatalf("expected serialization err for addr %v",
+				test.expAddr)
+
+		case err != nil && test.serErr == "":
+			t.Fatalf("unexpected serialization err for addr %v: %v",
+				test.expAddr, err)
+
+		case err != nil && !strings.Contains(err.Error(), test.serErr):
+			t.Fatalf("unexpected serialization err for addr %v, "+
+				"want: %v, got %v", test.expAddr, test.serErr,
+				err)
+
+		case err != nil:
+			continue
+		}
+
+		addr, err := deserializeAddr(&b)
+		if err != nil {
+			t.Fatalf("unable to deserialize address: %v", err)
+		}
+
+		if addr.String() != test.expAddr.String() {
+			t.Fatalf("expected address %v after serialization, "+
+				"got %v", addr, test.expAddr)
+		}
+	}
+}

channeldb/migration_01_to_11/channel_cache.go

@@ -0,0 +1,50 @@
+package migration_01_to_11
+
+// channelCache is an in-memory cache used to improve the performance of
+// ChanUpdatesInHorizon. It caches the chan info and edge policies for a
+// particular channel.
+type channelCache struct {
+	n        int
+	channels map[uint64]ChannelEdge
+}
+
+// newChannelCache creates a new channelCache with maximum capacity of n
+// channels.
+func newChannelCache(n int) *channelCache {
+	return &channelCache{
+		n:        n,
+		channels: make(map[uint64]ChannelEdge),
+	}
+}
+
+// get returns the channel from the cache, if it exists.
+func (c *channelCache) get(chanid uint64) (ChannelEdge, bool) {
+	channel, ok := c.channels[chanid]
+	return channel, ok
+}
+
+// insert adds the entry to the channel cache. If an entry for chanid already
+// exists, it will be replaced with the new entry. If the entry doesn't exist,
+// it will be inserted to the cache, performing a random eviction if the cache
+// is at capacity.
+func (c *channelCache) insert(chanid uint64, channel ChannelEdge) {
+	// If entry exists, replace it.
+	if _, ok := c.channels[chanid]; ok {
+		c.channels[chanid] = channel
+		return
+	}
+
+	// Otherwise, evict an entry at random and insert.
+	if len(c.channels) == c.n {
+		for id := range c.channels {
+			delete(c.channels, id)
+			break
+		}
+	}
+	c.channels[chanid] = channel
+}
+
+// remove deletes an edge for chanid from the cache, if it exists.
+func (c *channelCache) remove(chanid uint64) {
+	delete(c.channels, chanid)
+}

channeldb/migration_01_to_11/channel_cache_test.go

@@ -0,0 +1,105 @@
+package migration_01_to_11
+
+import (
+	"reflect"
+	"testing"
+)
+
+// TestChannelCache checks the behavior of the channelCache with respect to
+// insertion, eviction, and removal of cache entries.
+func TestChannelCache(t *testing.T) {
+	const cacheSize = 100
+
+	// Create a new channel cache with the configured max size.
+	c := newChannelCache(cacheSize)
+
+	// As a sanity check, assert that querying the empty cache does not
+	// return an entry.
+	_, ok := c.get(0)
+	if ok {
+		t.Fatalf("channel cache should be empty")
+	}
+
+	// Now, fill up the cache entirely.
+	for i := uint64(0); i < cacheSize; i++ {
+		c.insert(i, channelForInt(i))
+	}
+
+	// Assert that the cache has all of the entries just inserted, since no
+	// eviction should occur until we try to surpass the max size.
+	assertHasChanEntries(t, c, 0, cacheSize)
+
+	// Now, insert a new element that causes the cache to evict an element.
+	c.insert(cacheSize, channelForInt(cacheSize))
+
+	// Assert that the cache has this last entry, as the cache should evict
+	// some prior element and not the newly inserted one.
+	assertHasChanEntries(t, c, cacheSize, cacheSize)
+
+	// Iterate over all inserted elements and construct a set of the evicted
+	// elements.
+	evicted := make(map[uint64]struct{})
+	for i := uint64(0); i < cacheSize+1; i++ {
+		_, ok := c.get(i)
+		if !ok {
+			evicted[i] = struct{}{}
+		}
+	}
+
+	// Assert that exactly one element has been evicted.
+	numEvicted := len(evicted)
+	if numEvicted != 1 {
+		t.Fatalf("expected one evicted entry, got: %d", numEvicted)
+	}
+
+	// Remove the highest item which initially caused the eviction and
+	// reinsert the element that was evicted prior.
+	c.remove(cacheSize)
+	for i := range evicted {
+		c.insert(i, channelForInt(i))
+	}
+
+	// Since the removal created an extra slot, the last insertion should
+	// not have caused an eviction and the entries for all channels in the
+	// original set that filled the cache should be present.
+	assertHasChanEntries(t, c, 0, cacheSize)
+
+	// Finally, reinsert the existing set back into the cache and test that
+	// the cache still has all the entries. If the randomized eviction were
+	// happening on inserts for existing cache items, we expect this to fail
+	// with high probability.
+	for i := uint64(0); i < cacheSize; i++ {
+		c.insert(i, channelForInt(i))
+	}
+	assertHasChanEntries(t, c, 0, cacheSize)
+
+}
+
+// assertHasEntries queries the edge cache for all channels in the range [start,
+// end), asserting that they exist and their value matches the entry produced by
+// entryForInt.
+func assertHasChanEntries(t *testing.T, c *channelCache, start, end uint64) {
+	t.Helper()
+
+	for i := start; i < end; i++ {
+		entry, ok := c.get(i)
+		if !ok {
+			t.Fatalf("channel cache should contain chan %d", i)
+		}
+
+		expEntry := channelForInt(i)
+		if !reflect.DeepEqual(entry, expEntry) {
+			t.Fatalf("entry mismatch, want: %v, got: %v",
+				expEntry, entry)
+		}
+	}
+}
+
+// channelForInt generates a unique ChannelEdge given an integer.
+func channelForInt(i uint64) ChannelEdge {
+	return ChannelEdge{
+		Info: &ChannelEdgeInfo{
+			ChannelID: i,
+		},
+	}
+}

channeldb/migration_01_to_11/codec.go

@@ -0,0 +1,454 @@
+package migration_01_to_11
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net"
+
+	"github.com/btcsuite/btcd/btcec"
+	"github.com/btcsuite/btcd/chaincfg/chainhash"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+	"github.com/lightningnetwork/lnd/keychain"
+	"github.com/lightningnetwork/lnd/lnwire"
+	"github.com/lightningnetwork/lnd/shachain"
+)
+
+// writeOutpoint writes an outpoint to the passed writer using the minimal
+// amount of bytes possible.
+func writeOutpoint(w io.Writer, o *wire.OutPoint) error {
+	if _, err := w.Write(o.Hash[:]); err != nil {
+		return err
+	}
+	if err := binary.Write(w, byteOrder, o.Index); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// readOutpoint reads an outpoint from the passed reader that was previously
+// written using the writeOutpoint struct.
+func readOutpoint(r io.Reader, o *wire.OutPoint) error {
+	if _, err := io.ReadFull(r, o.Hash[:]); err != nil {
+		return err
+	}
+	if err := binary.Read(r, byteOrder, &o.Index); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// UnknownElementType is an error returned when the codec is unable to encode or
+// decode a particular type.
+type UnknownElementType struct {
+	method  string
+	element interface{}
+}
+
+// NewUnknownElementType creates a new UnknownElementType error from the passed
+// method name and element.
+func NewUnknownElementType(method string, el interface{}) UnknownElementType {
+	return UnknownElementType{method: method, element: el}
+}
+
+// Error returns the name of the method that encountered the error, as well as
+// the type that was unsupported.
+func (e UnknownElementType) Error() string {
+	return fmt.Sprintf("Unknown type in %s: %T", e.method, e.element)
+}
+
+// WriteElement is a one-stop shop to write the big endian representation of
+// any element which is to be serialized for storage on disk. The passed
+// io.Writer should be backed by an appropriately sized byte slice, or be able
+// to dynamically expand to accommodate additional data.
+func WriteElement(w io.Writer, element interface{}) error {
+	switch e := element.(type) {
+	case keychain.KeyDescriptor:
+		if err := binary.Write(w, byteOrder, e.Family); err != nil {
+			return err
+		}
+		if err := binary.Write(w, byteOrder, e.Index); err != nil {
+			return err
+		}
+
+		if e.PubKey != nil {
+			if err := binary.Write(w, byteOrder, true); err != nil {
+				return fmt.Errorf("error writing serialized element: %s", err)
+			}
+
+			return WriteElement(w, e.PubKey)
+		}
+
+		return binary.Write(w, byteOrder, false)
+	case ChannelType:
+		if err := binary.Write(w, byteOrder, e); err != nil {
+			return err
+		}
+
+	case chainhash.Hash:
+		if _, err := w.Write(e[:]); err != nil {
+			return err
+		}
+
+	case wire.OutPoint:
+		return writeOutpoint(w, &e)
+
+	case lnwire.ShortChannelID:
+		if err := binary.Write(w, byteOrder, e.ToUint64()); err != nil {
+			return err
+		}
+
+	case lnwire.ChannelID:
+		if _, err := w.Write(e[:]); err != nil {
+			return err
+		}
+
+	case int64, uint64:
+		if err := binary.Write(w, byteOrder, e); err != nil {
+			return err
+		}
+
+	case uint32:
+		if err := binary.Write(w, byteOrder, e); err != nil {
+			return err
+		}
+
+	case int32:
+		if err := binary.Write(w, byteOrder, e); err != nil {
+			return err
+		}
+
+	case uint16:
+		if err := binary.Write(w, byteOrder, e); err != nil {
+			return err
+		}
+
+	case uint8:
+		if err := binary.Write(w, byteOrder, e); err != nil {
+			return err
+		}
+
+	case bool:
+		if err := binary.Write(w, byteOrder, e); err != nil {
+			return err
+		}
+
+	case btcutil.Amount:
+		if err := binary.Write(w, byteOrder, uint64(e)); err != nil {
+			return err
+		}
+
+	case lnwire.MilliSatoshi:
+		if err := binary.Write(w, byteOrder, uint64(e)); err != nil {
+			return err
+		}
+
+	case *btcec.PrivateKey:
+		b := e.Serialize()
+		if _, err := w.Write(b); err != nil {
+			return err
+		}
+
+	case *btcec.PublicKey:
+		b := e.SerializeCompressed()
+		if _, err := w.Write(b); err != nil {
+			return err
+		}
+
+	case shachain.Producer:
+		return e.Encode(w)
+
+	case shachain.Store:
+		return e.Encode(w)
+
+	case *wire.MsgTx:
+		return e.Serialize(w)
+
+	case [32]byte:
+		if _, err := w.Write(e[:]); err != nil {
+			return err
+		}
+
+	case []byte:
+		if err := wire.WriteVarBytes(w, 0, e); err != nil {
+			return err
+		}
+
+	case lnwire.Message:
+		if _, err := lnwire.WriteMessage(w, e, 0); err != nil {
+			return err
+		}
+
+	case ChannelStatus:
+		if err := binary.Write(w, byteOrder, e); err != nil {
+			return err
+		}
+
+	case ClosureType:
+		if err := binary.Write(w, byteOrder, e); err != nil {
+			return err
+		}
+
+	case lnwire.FundingFlag:
+		if err := binary.Write(w, byteOrder, e); err != nil {
+			return err
+		}
+
+	case net.Addr:
+		if err := serializeAddr(w, e); err != nil {
+			return err
+		}
+
+	case []net.Addr:
+		if err := WriteElement(w, uint32(len(e))); err != nil {
+			return err
+		}
+
+		for _, addr := range e {
+			if err := serializeAddr(w, addr); err != nil {
+				return err
+			}
+		}
+
+	default:
+		return UnknownElementType{"WriteElement", e}
+	}
+
+	return nil
+}
+
+// WriteElements is writes each element in the elements slice to the passed
+// io.Writer using WriteElement.
+func WriteElements(w io.Writer, elements ...interface{}) error {
+	for _, element := range elements {
+		err := WriteElement(w, element)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ReadElement is a one-stop utility function to deserialize any datastructure
+// encoded using the serialization format of the database.
+func ReadElement(r io.Reader, element interface{}) error {
+	switch e := element.(type) {
+	case *keychain.KeyDescriptor:
+		if err := binary.Read(r, byteOrder, &e.Family); err != nil {
+			return err
+		}
+		if err := binary.Read(r, byteOrder, &e.Index); err != nil {
+			return err
+		}
+
+		var hasPubKey bool
+		if err := binary.Read(r, byteOrder, &hasPubKey); err != nil {
+			return err
+		}
+
+		if hasPubKey {
+			return ReadElement(r, &e.PubKey)
+		}
+
+	case *ChannelType:
+		if err := binary.Read(r, byteOrder, e); err != nil {
+			return err
+		}
+
+	case *chainhash.Hash:
+		if _, err := io.ReadFull(r, e[:]); err != nil {
+			return err
+		}
+
+	case *wire.OutPoint:
+		return readOutpoint(r, e)
+
+	case *lnwire.ShortChannelID:
+		var a uint64
+		if err := binary.Read(r, byteOrder, &a); err != nil {
+			return err
+		}
+		*e = lnwire.NewShortChanIDFromInt(a)
+
+	case *lnwire.ChannelID:
+		if _, err := io.ReadFull(r, e[:]); err != nil {
+			return err
+		}
+
+	case *int64, *uint64:
+		if err := binary.Read(r, byteOrder, e); err != nil {
+			return err
+		}
+
+	case *uint32:
+		if err := binary.Read(r, byteOrder, e); err != nil {
+			return err
+		}
+
+	case *int32:
+		if err := binary.Read(r, byteOrder, e); err != nil {
+			return err
+		}
+
+	case *uint16:
+		if err := binary.Read(r, byteOrder, e); err != nil {
+			return err
+		}
+
+	case *uint8:
+		if err := binary.Read(r, byteOrder, e); err != nil {
+			return err
+		}
+
+	case *bool:
+		if err := binary.Read(r, byteOrder, e); err != nil {
+			return err
+		}
+
+	case *btcutil.Amount:
+		var a uint64
+		if err := binary.Read(r, byteOrder, &a); err != nil {
+			return err
+		}
+
+		*e = btcutil.Amount(a)
+
+	case *lnwire.MilliSatoshi:
+		var a uint64
+		if err := binary.Read(r, byteOrder, &a); err != nil {
+			return err
+		}
+
+		*e = lnwire.MilliSatoshi(a)
+
+	case **btcec.PrivateKey:
+		var b [btcec.PrivKeyBytesLen]byte
+		if _, err := io.ReadFull(r, b[:]); err != nil {
+			return err
+		}
+
+		priv, _ := btcec.PrivKeyFromBytes(btcec.S256(), b[:])
+		*e = priv
+
+	case **btcec.PublicKey:
+		var b [btcec.PubKeyBytesLenCompressed]byte
+		if _, err := io.ReadFull(r, b[:]); err != nil {
+			return err
+		}
+
+		pubKey, err := btcec.ParsePubKey(b[:], btcec.S256())
+		if err != nil {
+			return err
+		}
+		*e = pubKey
+
+	case *shachain.Producer:
+		var root [32]byte
+		if _, err := io.ReadFull(r, root[:]); err != nil {
+			return err
+		}
+
+		// TODO(roasbeef): remove
+		producer, err := shachain.NewRevocationProducerFromBytes(root[:])
+		if err != nil {
+			return err
+		}
+
+		*e = producer
+
+	case *shachain.Store:
+		store, err := shachain.NewRevocationStoreFromBytes(r)
+		if err != nil {
+			return err
+		}
+
+		*e = store
+
+	case **wire.MsgTx:
+		tx := wire.NewMsgTx(2)
+		if err := tx.Deserialize(r); err != nil {
+			return err
+		}
+
+		*e = tx
+
+	case *[32]byte:
+		if _, err := io.ReadFull(r, e[:]); err != nil {
+			return err
+		}
+
+	case *[]byte:
+		bytes, err := wire.ReadVarBytes(r, 0, 66000, "[]byte")
+		if err != nil {
+			return err
+		}
+
+		*e = bytes
+
+	case *lnwire.Message:
+		msg, err := lnwire.ReadMessage(r, 0)
+		if err != nil {
+			return err
+		}
+
+		*e = msg
+
+	case *ChannelStatus:
+		if err := binary.Read(r, byteOrder, e); err != nil {
+			return err
+		}
+
+	case *ClosureType:
+		if err := binary.Read(r, byteOrder, e); err != nil {
+			return err
+		}
+
+	case *lnwire.FundingFlag:
+		if err := binary.Read(r, byteOrder, e); err != nil {
+			return err
+		}
+
+	case *net.Addr:
+		addr, err := deserializeAddr(r)
+		if err != nil {
+			return err
+		}
+		*e = addr
+
+	case *[]net.Addr:
+		var numAddrs uint32
+		if err := ReadElement(r, &numAddrs); err != nil {
+			return err
+		}
+
+		*e = make([]net.Addr, numAddrs)
+		for i := uint32(0); i < numAddrs; i++ {
+			addr, err := deserializeAddr(r)
+			if err != nil {
+				return err
+			}
+			(*e)[i] = addr
+		}
+
+	default:
+		return UnknownElementType{"ReadElement", e}
+	}
+
+	return nil
+}
+
+// ReadElements deserializes a variable number of elements into the passed
+// io.Reader, with each element being deserialized according to the ReadElement
+// function.
+func ReadElements(r io.Reader, elements ...interface{}) error {
+	for _, element := range elements {
+		err := ReadElement(r, element)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

channeldb/migration_01_to_11/db_test.go

@@ -0,0 +1,471 @@
+package migration_01_to_11
+
+import (
+	"io/ioutil"
+	"math"
+	"math/rand"
+	"net"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+
+	"github.com/btcsuite/btcd/btcec"
+	"github.com/btcsuite/btcd/chaincfg/chainhash"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/lightningnetwork/lnd/keychain"
+	"github.com/lightningnetwork/lnd/lnwire"
+	"github.com/lightningnetwork/lnd/shachain"
+)
+
+func TestOpenWithCreate(t *testing.T) {
+	t.Parallel()
+
+	// First, create a temporary directory to be used for the duration of
+	// this test.
+	tempDirName, err := ioutil.TempDir("", "channeldb")
+	if err != nil {
+		t.Fatalf("unable to create temp dir: %v", err)
+	}
+	defer os.RemoveAll(tempDirName)
+
+	// Next, open thereby creating channeldb for the first time.
+	dbPath := filepath.Join(tempDirName, "cdb")
+	cdb, err := Open(dbPath)
+	if err != nil {
+		t.Fatalf("unable to create channeldb: %v", err)
+	}
+	if err := cdb.Close(); err != nil {
+		t.Fatalf("unable to close channeldb: %v", err)
+	}
+
+	// The path should have been successfully created.
+	if !fileExists(dbPath) {
+		t.Fatalf("channeldb failed to create data directory")
+	}
+}
+
+// TestWipe tests that the database wipe operation completes successfully
+// and that the buckets are deleted. It also checks that attempts to fetch
+// information while the buckets are not set return the correct errors.
+func TestWipe(t *testing.T) {
+	t.Parallel()
+
+	// First, create a temporary directory to be used for the duration of
+	// this test.
+	tempDirName, err := ioutil.TempDir("", "channeldb")
+	if err != nil {
+		t.Fatalf("unable to create temp dir: %v", err)
+	}
+	defer os.RemoveAll(tempDirName)
+
+	// Next, open thereby creating channeldb for the first time.
+	dbPath := filepath.Join(tempDirName, "cdb")
+	cdb, err := Open(dbPath)
+	if err != nil {
+		t.Fatalf("unable to create channeldb: %v", err)
+	}
+	defer cdb.Close()
+
+	if err := cdb.Wipe(); err != nil {
+		t.Fatalf("unable to wipe channeldb: %v", err)
+	}
+	// Check correct errors are returned
+	_, err = cdb.FetchAllOpenChannels()
+	if err != ErrNoActiveChannels {
+		t.Fatalf("fetching open channels: expected '%v' instead got '%v'",
+			ErrNoActiveChannels, err)
+	}
+	_, err = cdb.FetchClosedChannels(false)
+	if err != ErrNoClosedChannels {
+		t.Fatalf("fetching closed channels: expected '%v' instead got '%v'",
+			ErrNoClosedChannels, err)
+	}
+}
+
+// TestFetchClosedChannelForID tests that we are able to properly retrieve a
+// ChannelCloseSummary from the DB given a ChannelID.
+func TestFetchClosedChannelForID(t *testing.T) {
+	t.Parallel()
+
+	const numChans = 101
+
+	cdb, cleanUp, err := makeTestDB()
+	if err != nil {
+		t.Fatalf("unable to make test database: %v", err)
+	}
+	defer cleanUp()
+
+	// Create the test channel state, that we will mutate the index of the
+	// funding point.
+	state, err := createTestChannelState(cdb)
+	if err != nil {
+		t.Fatalf("unable to create channel state: %v", err)
+	}
+
+	// Now run through the number of channels, and modify the outpoint index
+	// to create new channel IDs.
+	for i := uint32(0); i < numChans; i++ {
+		// Save the open channel to disk.
+		state.FundingOutpoint.Index = i
+
+		addr := &net.TCPAddr{
+			IP:   net.ParseIP("127.0.0.1"),
+			Port: 18556,
+		}
+		if err := state.SyncPending(addr, 101); err != nil {
+			t.Fatalf("unable to save and serialize channel "+
+				"state: %v", err)
+		}
+
+		// Close the channel. To make sure we retrieve the correct
+		// summary later, we make them differ in the SettledBalance.
+		closeSummary := &ChannelCloseSummary{
+			ChanPoint:      state.FundingOutpoint,
+			RemotePub:      state.IdentityPub,
+			SettledBalance: btcutil.Amount(500 + i),
+		}
+		if err := state.CloseChannel(closeSummary); err != nil {
+			t.Fatalf("unable to close channel: %v", err)
+		}
+	}
+
+	// Now run though them all again and make sure we are able to retrieve
+	// summaries from the DB.
+	for i := uint32(0); i < numChans; i++ {
+		state.FundingOutpoint.Index = i
+
+		// We calculate the ChannelID and use it to fetch the summary.
+		cid := lnwire.NewChanIDFromOutPoint(&state.FundingOutpoint)
+		fetchedSummary, err := cdb.FetchClosedChannelForID(cid)
+		if err != nil {
+			t.Fatalf("unable to fetch close summary: %v", err)
+		}
+
+		// Make sure we retrieved the correct one by checking the
+		// SettledBalance.
+		if fetchedSummary.SettledBalance != btcutil.Amount(500+i) {
+			t.Fatalf("summaries don't match: expected %v got %v",
+				btcutil.Amount(500+i),
+				fetchedSummary.SettledBalance)
+		}
+	}
+
+	// As a final test we make sure that we get ErrClosedChannelNotFound
+	// for a ChannelID we didn't add to the DB.
+	state.FundingOutpoint.Index++
+	cid := lnwire.NewChanIDFromOutPoint(&state.FundingOutpoint)
+	_, err = cdb.FetchClosedChannelForID(cid)
+	if err != ErrClosedChannelNotFound {
+		t.Fatalf("expected ErrClosedChannelNotFound, instead got: %v", err)
+	}
+}
+
+// TestAddrsForNode tests the we're able to properly obtain all the addresses
+// for a target node.
+func TestAddrsForNode(t *testing.T) {
+	t.Parallel()
+
+	cdb, cleanUp, err := makeTestDB()
+	if err != nil {
+		t.Fatalf("unable to make test database: %v", err)
+	}
+	defer cleanUp()
+
+	graph := cdb.ChannelGraph()
+
+	// We'll make a test vertex to insert into the database, as the source
+	// node, but this node will only have half the number of addresses it
+	// usually does.
+	testNode, err := createTestVertex(cdb)
+	if err != nil {
+		t.Fatalf("unable to create test node: %v", err)
+	}
+	testNode.Addresses = []net.Addr{testAddr}
+	if err := graph.SetSourceNode(testNode); err != nil {
+		t.Fatalf("unable to set source node: %v", err)
+	}
+
+	// Next, we'll make a link node with the same pubkey, but with an
+	// additional address.
+	nodePub, err := testNode.PubKey()
+	if err != nil {
+		t.Fatalf("unable to recv node pub: %v", err)
+	}
+	linkNode := cdb.NewLinkNode(
+		wire.MainNet, nodePub, anotherAddr,
+	)
+	if err := linkNode.Sync(); err != nil {
+		t.Fatalf("unable to sync link node: %v", err)
+	}
+
+	// Now that we've created a link node, as well as a vertex for the
+	// node, we'll query for all its addresses.
+	nodeAddrs, err := cdb.AddrsForNode(nodePub)
+	if err != nil {
+		t.Fatalf("unable to obtain node addrs: %v", err)
+	}
+
+	expectedAddrs := make(map[string]struct{})
+	expectedAddrs[testAddr.String()] = struct{}{}
+	expectedAddrs[anotherAddr.String()] = struct{}{}
+
+	// Finally, ensure that all the expected addresses are found.
+	if len(nodeAddrs) != len(expectedAddrs) {
+		t.Fatalf("expected %v addrs, got %v",
+			len(expectedAddrs), len(nodeAddrs))
+	}
+	for _, addr := range nodeAddrs {
+		if _, ok := expectedAddrs[addr.String()]; !ok {
+			t.Fatalf("unexpected addr: %v", addr)
+		}
+	}
+}
+
+// TestFetchChannel tests that we're able to fetch an arbitrary channel from
+// disk.
+func TestFetchChannel(t *testing.T) {
+	t.Parallel()
+
+	cdb, cleanUp, err := makeTestDB()
+	if err != nil {
+		t.Fatalf("unable to make test database: %v", err)
+	}
+	defer cleanUp()
+
+	// Create the test channel state that we'll sync to the database
+	// shortly.
+	channelState, err := createTestChannelState(cdb)
+	if err != nil {
+		t.Fatalf("unable to create channel state: %v", err)
+	}
+
+	// Mark the channel as pending, then immediately mark it as open to it
+	// can be fully visible.
+	addr := &net.TCPAddr{
+		IP:   net.ParseIP("127.0.0.1"),
+		Port: 18555,
+	}
+	if err := channelState.SyncPending(addr, 9); err != nil {
+		t.Fatalf("unable to save and serialize channel state: %v", err)
+	}
+	err = channelState.MarkAsOpen(lnwire.NewShortChanIDFromInt(99))
+	if err != nil {
+		t.Fatalf("unable to mark channel open: %v", err)
+	}
+
+	// Next, attempt to fetch the channel by its chan point.
+	dbChannel, err := cdb.FetchChannel(channelState.FundingOutpoint)
+	if err != nil {
+		t.Fatalf("unable to fetch channel: %v", err)
+	}
+
+	// The decoded channel state should be identical to what we stored
+	// above.
+	if !reflect.DeepEqual(channelState, dbChannel) {
+		t.Fatalf("channel state doesn't match:: %v vs %v",
+			spew.Sdump(channelState), spew.Sdump(dbChannel))
+	}
+
+	// If we attempt to query for a non-exist ante channel, then we should
+	// get an error.
+	channelState2, err := createTestChannelState(cdb)
+	if err != nil {
+		t.Fatalf("unable to create channel state: %v", err)
+	}
+	channelState2.FundingOutpoint.Index ^= 1
+
+	_, err = cdb.FetchChannel(channelState2.FundingOutpoint)
+	if err == nil {
+		t.Fatalf("expected query to fail")
+	}
+}
+
+func genRandomChannelShell() (*ChannelShell, error) {
+	var testPriv [32]byte
+	if _, err := rand.Read(testPriv[:]); err != nil {
+		return nil, err
+	}
+
+	_, pub := btcec.PrivKeyFromBytes(btcec.S256(), testPriv[:])
+
+	var chanPoint wire.OutPoint
+	if _, err := rand.Read(chanPoint.Hash[:]); err != nil {
+		return nil, err
+	}
+
+	pub.Curve = nil
+
+	chanPoint.Index = uint32(rand.Intn(math.MaxUint16))
+
+	chanStatus := ChanStatusDefault | ChanStatusRestored
+
+	var shaChainPriv [32]byte
+	if _, err := rand.Read(testPriv[:]); err != nil {
+		return nil, err
+	}
+	revRoot, err := chainhash.NewHash(shaChainPriv[:])
+	if err != nil {
+		return nil, err
+	}
+	shaChainProducer := shachain.NewRevocationProducer(*revRoot)
+
+	return &ChannelShell{
+		NodeAddrs: []net.Addr{&net.TCPAddr{
+			IP:   net.ParseIP("127.0.0.1"),
+			Port: 18555,
+		}},
+		Chan: &OpenChannel{
+			chanStatus:      chanStatus,
+			ChainHash:       rev,
+			FundingOutpoint: chanPoint,
+			ShortChannelID: lnwire.NewShortChanIDFromInt(
+				uint64(rand.Int63()),
+			),
+			IdentityPub: pub,
+			LocalChanCfg: ChannelConfig{
+				ChannelConstraints: ChannelConstraints{
+					CsvDelay: uint16(rand.Int63()),
+				},
+				PaymentBasePoint: keychain.KeyDescriptor{
+					KeyLocator: keychain.KeyLocator{
+						Family: keychain.KeyFamily(rand.Int63()),
+						Index:  uint32(rand.Int63()),
+					},
+				},
+			},
+			RemoteCurrentRevocation: pub,
+			IsPending:               false,
+			RevocationStore:         shachain.NewRevocationStore(),
+			RevocationProducer:      shaChainProducer,
+		},
+	}, nil
+}
+
+// TestRestoreChannelShells tests that we're able to insert a partially channel
+// populated to disk. This is useful for channel recovery purposes. We should
+// find the new channel shell on disk, and also the db should be populated with
+// an edge for that channel.
+func TestRestoreChannelShells(t *testing.T) {
+	t.Parallel()
+
+	cdb, cleanUp, err := makeTestDB()
+	if err != nil {
+		t.Fatalf("unable to make test database: %v", err)
+	}
+	defer cleanUp()
+
+	// First, we'll make our channel shell, it will only have the minimal
+	// amount of information required for us to initiate the data loss
+	// protection feature.
+	channelShell, err := genRandomChannelShell()
+	if err != nil {
+		t.Fatalf("unable to gen channel shell: %v", err)
+	}
+
+	graph := cdb.ChannelGraph()
+
+	// Before we can restore the channel, we'll need to make a source node
+	// in the graph as the channel edge we create will need to have a
+	// origin.
+	testNode, err := createTestVertex(cdb)
+	if err != nil {
+		t.Fatalf("unable to create test node: %v", err)
+	}
+	if err := graph.SetSourceNode(testNode); err != nil {
+		t.Fatalf("unable to set source node: %v", err)
+	}
+
+	// With the channel shell constructed, we'll now insert it into the
+	// database with the restoration method.
+	if err := cdb.RestoreChannelShells(channelShell); err != nil {
+		t.Fatalf("unable to restore channel shell: %v", err)
+	}
+
+	// Now that the channel has been inserted, we'll attempt to query for
+	// it to ensure we can properly locate it via various means.
+	//
+	// First, we'll attempt to query for all channels that we have with the
+	// node public key that was restored.
+	nodeChans, err := cdb.FetchOpenChannels(channelShell.Chan.IdentityPub)
+	if err != nil {
+		t.Fatalf("unable find channel: %v", err)
+	}
+
+	// We should now find a single channel from the database.
+	if len(nodeChans) != 1 {
+		t.Fatalf("unable to find restored channel by node "+
+			"pubkey: %v", err)
+	}
+
+	// Ensure that it isn't possible to modify the commitment state machine
+	// of this restored channel.
+	channel := nodeChans[0]
+	err = channel.UpdateCommitment(nil)
+	if err != ErrNoRestoredChannelMutation {
+		t.Fatalf("able to mutate restored channel")
+	}
+	err = channel.AppendRemoteCommitChain(nil)
+	if err != ErrNoRestoredChannelMutation {
+		t.Fatalf("able to mutate restored channel")
+	}
+	err = channel.AdvanceCommitChainTail(nil)
+	if err != ErrNoRestoredChannelMutation {
+		t.Fatalf("able to mutate restored channel")
+	}
+
+	// That single channel should have the proper channel point, and also
+	// the expected set of flags to indicate that it was a restored
+	// channel.
+	if nodeChans[0].FundingOutpoint != channelShell.Chan.FundingOutpoint {
+		t.Fatalf("wrong funding outpoint: expected %v, got %v",
+			nodeChans[0].FundingOutpoint,
+			channelShell.Chan.FundingOutpoint)
+	}
+	if !nodeChans[0].HasChanStatus(ChanStatusRestored) {
+		t.Fatalf("node has wrong status flags: %v",
+			nodeChans[0].chanStatus)
+	}
+
+	// We should also be able to find the channel if we query for it
+	// directly.
+	_, err = cdb.FetchChannel(channelShell.Chan.FundingOutpoint)
+	if err != nil {
+		t.Fatalf("unable to fetch channel: %v", err)
+	}
+
+	// We should also be able to find the link node that was inserted by
+	// its public key.
+	linkNode, err := cdb.FetchLinkNode(channelShell.Chan.IdentityPub)
+	if err != nil {
+		t.Fatalf("unable to fetch link node: %v", err)
+	}
+
+	// The node should have the same address, as specified in the channel
+	// shell.
+	if reflect.DeepEqual(linkNode.Addresses, channelShell.NodeAddrs) {
+		t.Fatalf("addr mismach: expected %v, got %v",
+			linkNode.Addresses, channelShell.NodeAddrs)
+	}
+
+	// Finally, we'll ensure that the edge for the channel was properly
+	// inserted.
+	chanInfos, err := graph.FetchChanInfos(
+		[]uint64{channelShell.Chan.ShortChannelID.ToUint64()},
+	)
+	if err != nil {
+		t.Fatalf("unable to find edges: %v", err)
+	}
+
+	if len(chanInfos) != 1 {
+		t.Fatalf("wrong amount of chan infos: expected %v got %v",
+			len(chanInfos), 1)
+	}
+
+	// We should only find a single edge.
+	if chanInfos[0].Policy1 != nil && chanInfos[0].Policy2 != nil {
+		t.Fatalf("only a single edge should be inserted: %v", err)
+	}
+}

channeldb/migration_01_to_11/doc.go

@@ -0,0 +1 @@
+package migration_01_to_11

channeldb/migration_01_to_11/error.go

@@ -0,0 +1,128 @@
+package migration_01_to_11
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	// ErrNoChanDBExists is returned when a channel bucket hasn't been
+	// created.
+	ErrNoChanDBExists = fmt.Errorf("channel db has not yet been created")
+
+	// ErrDBReversion is returned when detecting an attempt to revert to a
+	// prior database version.
+	ErrDBReversion = fmt.Errorf("channel db cannot revert to prior version")
+
+	// ErrLinkNodesNotFound is returned when node info bucket hasn't been
+	// created.
+	ErrLinkNodesNotFound = fmt.Errorf("no link nodes exist")
+
+	// ErrNoActiveChannels  is returned when there is no active (open)
+	// channels within the database.
+	ErrNoActiveChannels = fmt.Errorf("no active channels exist")
+
+	// ErrNoPastDeltas is returned when the channel delta bucket hasn't been
+	// created.
+	ErrNoPastDeltas = fmt.Errorf("channel has no recorded deltas")
+
+	// ErrInvoiceNotFound is returned when a targeted invoice can't be
+	// found.
+	ErrInvoiceNotFound = fmt.Errorf("unable to locate invoice")
+
+	// ErrNoInvoicesCreated is returned when we don't have invoices in
+	// our database to return.
+	ErrNoInvoicesCreated = fmt.Errorf("there are no existing invoices")
+
+	// ErrDuplicateInvoice is returned when an invoice with the target
+	// payment hash already exists.
+	ErrDuplicateInvoice = fmt.Errorf("invoice with payment hash already exists")
+
+	// ErrNoPaymentsCreated is returned when bucket of payments hasn't been
+	// created.
+	ErrNoPaymentsCreated = fmt.Errorf("there are no existing payments")
+
+	// ErrNodeNotFound is returned when node bucket exists, but node with
+	// specific identity can't be found.
+	ErrNodeNotFound = fmt.Errorf("link node with target identity not found")
+
+	// ErrChannelNotFound is returned when we attempt to locate a channel
+	// for a specific chain, but it is not found.
+	ErrChannelNotFound = fmt.Errorf("channel not found")
+
+	// ErrMetaNotFound is returned when meta bucket hasn't been
+	// created.
+	ErrMetaNotFound = fmt.Errorf("unable to locate meta information")
+
+	// ErrGraphNotFound is returned when at least one of the components of
+	// graph doesn't exist.
+	ErrGraphNotFound = fmt.Errorf("graph bucket not initialized")
+
+	// ErrGraphNeverPruned is returned when graph was never pruned.
+	ErrGraphNeverPruned = fmt.Errorf("graph never pruned")
+
+	// ErrSourceNodeNotSet is returned if the source node of the graph
+	// hasn't been added The source node is the center node within a
+	// star-graph.
+	ErrSourceNodeNotSet = fmt.Errorf("source node does not exist")
+
+	// ErrGraphNodesNotFound is returned in case none of the nodes has
+	// been added in graph node bucket.
+	ErrGraphNodesNotFound = fmt.Errorf("no graph nodes exist")
+
+	// ErrGraphNoEdgesFound is returned in case of none of the channel/edges
+	// has been added in graph edge bucket.
+	ErrGraphNoEdgesFound = fmt.Errorf("no graph edges exist")
+
+	// ErrGraphNodeNotFound is returned when we're unable to find the target
+	// node.
+	ErrGraphNodeNotFound = fmt.Errorf("unable to find node")
+
+	// ErrEdgeNotFound is returned when an edge for the target chanID
+	// can't be found.
+	ErrEdgeNotFound = fmt.Errorf("edge not found")
+
+	// ErrZombieEdge is an error returned when we attempt to look up an edge
+	// but it is marked as a zombie within the zombie index.
+	ErrZombieEdge = errors.New("edge marked as zombie")
+
+	// ErrEdgeAlreadyExist is returned when edge with specific
+	// channel id can't be added because it already exist.
+	ErrEdgeAlreadyExist = fmt.Errorf("edge already exist")
+
+	// ErrNodeAliasNotFound is returned when alias for node can't be found.
+	ErrNodeAliasNotFound = fmt.Errorf("alias for node not found")
+
+	// ErrUnknownAddressType is returned when a node's addressType is not
+	// an expected value.
+	ErrUnknownAddressType = fmt.Errorf("address type cannot be resolved")
+
+	// ErrNoClosedChannels is returned when a node is queries for all the
+	// channels it has closed, but it hasn't yet closed any channels.
+	ErrNoClosedChannels = fmt.Errorf("no channel have been closed yet")
+
+	// ErrNoForwardingEvents is returned in the case that a query fails due
+	// to the log not having any recorded events.
+	ErrNoForwardingEvents = fmt.Errorf("no recorded forwarding events")
+
+	// ErrEdgePolicyOptionalFieldNotFound is an error returned if a channel
+	// policy field is not found in the db even though its message flags
+	// indicate it should be.
+	ErrEdgePolicyOptionalFieldNotFound = fmt.Errorf("optional field not " +
+		"present")
+
+	// ErrChanAlreadyExists is return when the caller attempts to create a
+	// channel with a channel point that is already present in the
+	// database.
+	ErrChanAlreadyExists = fmt.Errorf("channel already exists")
+)
+
+// ErrTooManyExtraOpaqueBytes creates an error which should be returned if the
+// caller attempts to write an announcement message which bares too many extra
+// opaque bytes. We limit this value in order to ensure that we don't waste
+// disk space due to nodes unnecessarily padding out their announcements with
+// garbage data.
+func ErrTooManyExtraOpaqueBytes(numBytes int) error {
+	return fmt.Errorf("max allowed number of opaque bytes is %v, received "+
+		"%v bytes", MaxAllowedExtraOpaqueBytes, numBytes)
+}

channeldb/migration_01_to_11/fees.go

@@ -0,0 +1 @@
+package migration_01_to_11

channeldb/migration_01_to_11/forwarding_log.go

@@ -0,0 +1,274 @@
+package migration_01_to_11
+
+import (
+	"bytes"
+	"io"
+	"sort"
+	"time"
+
+	"github.com/coreos/bbolt"
+	"github.com/lightningnetwork/lnd/lnwire"
+)
+
+var (
+	// forwardingLogBucket is the bucket that we'll use to store the
+	// forwarding log. The forwarding log contains a time series database
+	// of the forwarding history of a lightning daemon. Each key within the
+	// bucket is a timestamp (in nano seconds since the unix epoch), and
+	// the value a slice of a forwarding event for that timestamp.
+	forwardingLogBucket = []byte("circuit-fwd-log")
+)
+
+const (
+	// forwardingEventSize is the size of a forwarding event. The breakdown
+	// is as follows:
+	//
+	//  * 8 byte incoming chan ID || 8 byte outgoing chan ID || 8 byte value in
+	//    || 8 byte value out
+	//
+	// From the value in and value out, callers can easily compute the
+	// total fee extract from a forwarding event.
+	forwardingEventSize = 32
+
+	// MaxResponseEvents is the max number of forwarding events that will
+	// be returned by a single query response. This size was selected to
+	// safely remain under gRPC's 4MiB message size response limit. As each
+	// full forwarding event (including the timestamp) is 40 bytes, we can
+	// safely return 50k entries in a single response.
+	MaxResponseEvents = 50000
+)
+
+// ForwardingLog returns an instance of the ForwardingLog object backed by the
+// target database instance.
+func (d *DB) ForwardingLog() *ForwardingLog {
+	return &ForwardingLog{
+		db: d,
+	}
+}
+
+// ForwardingLog is a time series database that logs the fulfilment of payment
+// circuits by a lightning network daemon. The log contains a series of
+// forwarding events which map a timestamp to a forwarding event. A forwarding
+// event describes which channels were used to create+settle a circuit, and the
+// amount involved. Subtracting the outgoing amount from the incoming amount
+// reveals the fee charged for the forwarding service.
+type ForwardingLog struct {
+	db *DB
+}
+
+// ForwardingEvent is an event in the forwarding log's time series. Each
+// forwarding event logs the creation and tear-down of a payment circuit. A
+// circuit is created once an incoming HTLC has been fully forwarded, and
+// destroyed once the payment has been settled.
+type ForwardingEvent struct {
+	// Timestamp is the settlement time of this payment circuit.
+	Timestamp time.Time
+
+	// IncomingChanID is the incoming channel ID of the payment circuit.
+	IncomingChanID lnwire.ShortChannelID
+
+	// OutgoingChanID is the outgoing channel ID of the payment circuit.
+	OutgoingChanID lnwire.ShortChannelID
+
+	// AmtIn is the amount of the incoming HTLC. Subtracting this from the
+	// outgoing amount gives the total fees of this payment circuit.
+	AmtIn lnwire.MilliSatoshi
+
+	// AmtOut is the amount of the outgoing HTLC. Subtracting the incoming
+	// amount from this gives the total fees for this payment circuit.
+	AmtOut lnwire.MilliSatoshi
+}
+
+// encodeForwardingEvent writes out the target forwarding event to the passed
+// io.Writer, using the expected DB format. Note that the timestamp isn't
+// serialized as this will be the key value within the bucket.
+func encodeForwardingEvent(w io.Writer, f *ForwardingEvent) error {
+	return WriteElements(
+		w, f.IncomingChanID, f.OutgoingChanID, f.AmtIn, f.AmtOut,
+	)
+}
+
+// decodeForwardingEvent attempts to decode the raw bytes of a serialized
+// forwarding event into the target ForwardingEvent. Note that the timestamp
+// won't be decoded, as the caller is expected to set this due to the bucket
+// structure of the forwarding log.
+func decodeForwardingEvent(r io.Reader, f *ForwardingEvent) error {
+	return ReadElements(
+		r, &f.IncomingChanID, &f.OutgoingChanID, &f.AmtIn, &f.AmtOut,
+	)
+}
+
+// AddForwardingEvents adds a series of forwarding events to the database.
+// Before inserting, the set of events will be sorted according to their
+// timestamp. This ensures that all writes to disk are sequential.
+func (f *ForwardingLog) AddForwardingEvents(events []ForwardingEvent) error {
+	// Before we create the database transaction, we'll ensure that the set
+	// of forwarding events are properly sorted according to their
+	// timestamp.
+	sort.Slice(events, func(i, j int) bool {
+		return events[i].Timestamp.Before(events[j].Timestamp)
+	})
+
+	var timestamp [8]byte
+
+	return f.db.Batch(func(tx *bbolt.Tx) error {
+		// First, we'll fetch the bucket that stores our time series
+		// log.
+		logBucket, err := tx.CreateBucketIfNotExists(
+			forwardingLogBucket,
+		)
+		if err != nil {
+			return err
+		}
+
+		// With the bucket obtained, we can now begin to write out the
+		// series of events.
+		for _, event := range events {
+			var eventBytes [forwardingEventSize]byte
+			eventBuf := bytes.NewBuffer(eventBytes[0:0:forwardingEventSize])
+
+			// First, we'll serialize this timestamp into our
+			// timestamp buffer.
+			byteOrder.PutUint64(
+				timestamp[:], uint64(event.Timestamp.UnixNano()),
+			)
+
+			// With the key encoded, we'll then encode the event
+			// into our buffer, then write it out to disk.
+			err := encodeForwardingEvent(eventBuf, &event)
+			if err != nil {
+				return err
+			}
+			err = logBucket.Put(timestamp[:], eventBuf.Bytes())
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+// ForwardingEventQuery represents a query to the forwarding log payment
+// circuit time series database. The query allows a caller to retrieve all
+// records for a particular time slice, offset in that time slice, limiting the
+// total number of responses returned.
+type ForwardingEventQuery struct {
+	// StartTime is the start time of the time slice.
+	StartTime time.Time
+
+	// EndTime is the end time of the time slice.
+	EndTime time.Time
+
+	// IndexOffset is the offset within the time slice to start at. This
+	// can be used to start the response at a particular record.
+	IndexOffset uint32
+
+	// NumMaxEvents is the max number of events to return.
+	NumMaxEvents uint32
+}
+
+// ForwardingLogTimeSlice is the response to a forwarding query. It includes
+// the original query, the set  events that match the query, and an integer
+// which represents the offset index of the last item in the set of retuned
+// events. This integer allows callers to resume their query using this offset
+// in the event that the query's response exceeds the max number of returnable
+// events.
+type ForwardingLogTimeSlice struct {
+	ForwardingEventQuery
+
+	// ForwardingEvents is the set of events in our time series that answer
+	// the query embedded above.
+	ForwardingEvents []ForwardingEvent
+
+	// LastIndexOffset is the index of the last element in the set of
+	// returned ForwardingEvents above. Callers can use this to resume
+	// their query in the event that the time slice has too many events to
+	// fit into a single response.
+	LastIndexOffset uint32
+}
+
+// Query allows a caller to query the forwarding event time series for a
+// particular time slice. The caller can control the precise time as well as
+// the number of events to be returned.
+//
+// TODO(roasbeef): rename?
+func (f *ForwardingLog) Query(q ForwardingEventQuery) (ForwardingLogTimeSlice, error) {
+	resp := ForwardingLogTimeSlice{
+		ForwardingEventQuery: q,
+	}
+
+	// If the user provided an index offset, then we'll not know how many
+	// records we need to skip. We'll also keep track of the record offset
+	// as that's part of the final return value.
+	recordsToSkip := q.IndexOffset
+	recordOffset := q.IndexOffset
+
+	err := f.db.View(func(tx *bbolt.Tx) error {
+		// If the bucket wasn't found, then there aren't any events to
+		// be returned.
+		logBucket := tx.Bucket(forwardingLogBucket)
+		if logBucket == nil {
+			return ErrNoForwardingEvents
+		}
+
+		// We'll be using a cursor to seek into the database, so we'll
+		// populate byte slices that represent the start of the key
+		// space we're interested in, and the end.
+		var startTime, endTime [8]byte
+		byteOrder.PutUint64(startTime[:], uint64(q.StartTime.UnixNano()))
+		byteOrder.PutUint64(endTime[:], uint64(q.EndTime.UnixNano()))
+
+		// If we know that a set of log events exists, then we'll begin
+		// our seek through the log in order to satisfy the query.
+		// We'll continue until either we reach the end of the range,
+		// or reach our max number of events.
+		logCursor := logBucket.Cursor()
+		timestamp, events := logCursor.Seek(startTime[:])
+		for ; timestamp != nil && bytes.Compare(timestamp, endTime[:]) <= 0; timestamp, events = logCursor.Next() {
+			// If our current return payload exceeds the max number
+			// of events, then we'll exit now.
+			if uint32(len(resp.ForwardingEvents)) >= q.NumMaxEvents {
+				return nil
+			}
+
+			// If we're not yet past the user defined offset, then
+			// we'll continue to seek forward.
+			if recordsToSkip > 0 {
+				recordsToSkip--
+				continue
+			}
+
+			currentTime := time.Unix(
+				0, int64(byteOrder.Uint64(timestamp)),
+			)
+
+			// At this point, we've skipped enough records to start
+			// to collate our query. For each record, we'll
+			// increment the final record offset so the querier can
+			// utilize pagination to seek further.
+			readBuf := bytes.NewReader(events)
+			for readBuf.Len() != 0 {
+				var event ForwardingEvent
+				err := decodeForwardingEvent(readBuf, &event)
+				if err != nil {
+					return err
+				}
+
+				event.Timestamp = currentTime
+				resp.ForwardingEvents = append(resp.ForwardingEvents, event)
+
+				recordOffset++
+			}
+		}
+
+		return nil
+	})
+	if err != nil && err != ErrNoForwardingEvents {
+		return ForwardingLogTimeSlice{}, err
+	}
+
+	resp.LastIndexOffset = recordOffset
+
+	return resp, nil
+}

channeldb/migration_01_to_11/forwarding_log_test.go

@@ -0,0 +1,265 @@
+package migration_01_to_11
+
+import (
+	"math/rand"
+	"reflect"
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/lightningnetwork/lnd/lnwire"
+
+	"time"
+)
+
+// TestForwardingLogBasicStorageAndQuery tests that we're able to store and
+// then query for items that have previously been added to the event log.
+func TestForwardingLogBasicStorageAndQuery(t *testing.T) {
+	t.Parallel()
+
+	// First, we'll set up a test database, and use that to instantiate the
+	// forwarding event log that we'll be using for the duration of the
+	// test.
+	db, cleanUp, err := makeTestDB()
+	defer cleanUp()
+	if err != nil {
+		t.Fatalf("unable to make test db: %v", err)
+	}
+	log := ForwardingLog{
+		db: db,
+	}
+
+	initialTime := time.Unix(1234, 0)
+	timestamp := time.Unix(1234, 0)
+
+	// We'll create 100 random events, which each event being spaced 10
+	// minutes after the prior event.
+	numEvents := 100
+	events := make([]ForwardingEvent, numEvents)
+	for i := 0; i < numEvents; i++ {
+		events[i] = ForwardingEvent{
+			Timestamp:      timestamp,
+			IncomingChanID: lnwire.NewShortChanIDFromInt(uint64(rand.Int63())),
+			OutgoingChanID: lnwire.NewShortChanIDFromInt(uint64(rand.Int63())),
+			AmtIn:          lnwire.MilliSatoshi(rand.Int63()),
+			AmtOut:         lnwire.MilliSatoshi(rand.Int63()),
+		}
+
+		timestamp = timestamp.Add(time.Minute * 10)
+	}
+
+	// Now that all of our set of events constructed, we'll add them to the
+	// database in a batch manner.
+	if err := log.AddForwardingEvents(events); err != nil {
+		t.Fatalf("unable to add events: %v", err)
+	}
+
+	// With our events added we'll now construct a basic query to retrieve
+	// all of the events.
+	eventQuery := ForwardingEventQuery{
+		StartTime:    initialTime,
+		EndTime:      timestamp,
+		IndexOffset:  0,
+		NumMaxEvents: 1000,
+	}
+	timeSlice, err := log.Query(eventQuery)
+	if err != nil {
+		t.Fatalf("unable to query for events: %v", err)
+	}
+
+	// The set of returned events should match identically, as they should
+	// be returned in sorted order.
+	if !reflect.DeepEqual(events, timeSlice.ForwardingEvents) {
+		t.Fatalf("event mismatch: expected %v vs %v",
+			spew.Sdump(events), spew.Sdump(timeSlice.ForwardingEvents))
+	}
+
+	// The offset index of the final entry should be numEvents, so the
+	// number of total events we've written.
+	if timeSlice.LastIndexOffset != uint32(numEvents) {
+		t.Fatalf("wrong final offset: expected %v, got %v",
+			timeSlice.LastIndexOffset, numEvents)
+	}
+}
+
+// TestForwardingLogQueryOptions tests that the query offset works properly. So
+// if we add a series of events, then we should be able to seek within the
+// timeslice accordingly. This exercises the index offset and num max event
+// field in the query, and also the last index offset field int he response.
+func TestForwardingLogQueryOptions(t *testing.T) {
+	t.Parallel()
+
+	// First, we'll set up a test database, and use that to instantiate the
+	// forwarding event log that we'll be using for the duration of the
+	// test.
+	db, cleanUp, err := makeTestDB()
+	defer cleanUp()
+	if err != nil {
+		t.Fatalf("unable to make test db: %v", err)
+	}
+	log := ForwardingLog{
+		db: db,
+	}
+
+	initialTime := time.Unix(1234, 0)
+	endTime := time.Unix(1234, 0)
+
+	// We'll create 20 random events, which each event being spaced 10
+	// minutes after the prior event.
+	numEvents := 20
+	events := make([]ForwardingEvent, numEvents)
+	for i := 0; i < numEvents; i++ {
+		events[i] = ForwardingEvent{
+			Timestamp:      endTime,
+			IncomingChanID: lnwire.NewShortChanIDFromInt(uint64(rand.Int63())),
+			OutgoingChanID: lnwire.NewShortChanIDFromInt(uint64(rand.Int63())),
+			AmtIn:          lnwire.MilliSatoshi(rand.Int63()),
+			AmtOut:         lnwire.MilliSatoshi(rand.Int63()),
+		}
+
+		endTime = endTime.Add(time.Minute * 10)
+	}
+
+	// Now that all of our set of events constructed, we'll add them to the
+	// database in a batch manner.
+	if err := log.AddForwardingEvents(events); err != nil {
+		t.Fatalf("unable to add events: %v", err)
+	}
+
+	// With all of our events added, we should be able to query for the
+	// first 10 events using the max event query field.
+	eventQuery := ForwardingEventQuery{
+		StartTime:    initialTime,
+		EndTime:      endTime,
+		IndexOffset:  0,
+		NumMaxEvents: 10,
+	}
+	timeSlice, err := log.Query(eventQuery)
+	if err != nil {
+		t.Fatalf("unable to query for events: %v", err)
+	}
+
+	// We should get exactly 10 events back.
+	if len(timeSlice.ForwardingEvents) != 10 {
+		t.Fatalf("wrong number of events: expected %v, got %v", 10,
+			len(timeSlice.ForwardingEvents))
+	}
+
+	// The set of events returned should be the first 10 events that we
+	// added.
+	if !reflect.DeepEqual(events[:10], timeSlice.ForwardingEvents) {
+		t.Fatalf("wrong response: expected %v, got %v",
+			spew.Sdump(events[:10]),
+			spew.Sdump(timeSlice.ForwardingEvents))
+	}
+
+	// The final offset should be the exact number of events returned.
+	if timeSlice.LastIndexOffset != 10 {
+		t.Fatalf("wrong index offset: expected %v, got %v", 10,
+			timeSlice.LastIndexOffset)
+	}
+
+	// If we use the final offset to query again, then we should get 10
+	// more events, that are the last 10 events we wrote.
+	eventQuery.IndexOffset = 10
+	timeSlice, err = log.Query(eventQuery)
+	if err != nil {
+		t.Fatalf("unable to query for events: %v", err)
+	}
+
+	// We should get exactly 10 events back once again.
+	if len(timeSlice.ForwardingEvents) != 10 {
+		t.Fatalf("wrong number of events: expected %v, got %v", 10,
+			len(timeSlice.ForwardingEvents))
+	}
+
+	// The events that we got back should be the last 10 events that we
+	// wrote out.
+	if !reflect.DeepEqual(events[10:], timeSlice.ForwardingEvents) {
+		t.Fatalf("wrong response: expected %v, got %v",
+			spew.Sdump(events[10:]),
+			spew.Sdump(timeSlice.ForwardingEvents))
+	}
+
+	// Finally, the last index offset should be 20, or the number of
+	// records we've written out.
+	if timeSlice.LastIndexOffset != 20 {
+		t.Fatalf("wrong index offset: expected %v, got %v", 20,
+			timeSlice.LastIndexOffset)
+	}
+}
+
+// TestForwardingLogQueryLimit tests that we're able to properly limit the
+// number of events that are returned as part of a query.
+func TestForwardingLogQueryLimit(t *testing.T) {
+	t.Parallel()
+
+	// First, we'll set up a test database, and use that to instantiate the
+	// forwarding event log that we'll be using for the duration of the
+	// test.
+	db, cleanUp, err := makeTestDB()
+	defer cleanUp()
+	if err != nil {
+		t.Fatalf("unable to make test db: %v", err)
+	}
+	log := ForwardingLog{
+		db: db,
+	}
+
+	initialTime := time.Unix(1234, 0)
+	endTime := time.Unix(1234, 0)
+
+	// We'll create 200 random events, which each event being spaced 10
+	// minutes after the prior event.
+	numEvents := 200
+	events := make([]ForwardingEvent, numEvents)
+	for i := 0; i < numEvents; i++ {
+		events[i] = ForwardingEvent{
+			Timestamp:      endTime,
+			IncomingChanID: lnwire.NewShortChanIDFromInt(uint64(rand.Int63())),
+			OutgoingChanID: lnwire.NewShortChanIDFromInt(uint64(rand.Int63())),
+			AmtIn:          lnwire.MilliSatoshi(rand.Int63()),
+			AmtOut:         lnwire.MilliSatoshi(rand.Int63()),
+		}
+
+		endTime = endTime.Add(time.Minute * 10)
+	}
+
+	// Now that all of our set of events constructed, we'll add them to the
+	// database in a batch manner.
+	if err := log.AddForwardingEvents(events); err != nil {
+		t.Fatalf("unable to add events: %v", err)
+	}
+
+	// Once the events have been written out, we'll issue a query over the
+	// entire range, but restrict the number of events to the first 100.
+	eventQuery := ForwardingEventQuery{
+		StartTime:    initialTime,
+		EndTime:      endTime,
+		IndexOffset:  0,
+		NumMaxEvents: 100,
+	}
+	timeSlice, err := log.Query(eventQuery)
+	if err != nil {
+		t.Fatalf("unable to query for events: %v", err)
+	}
+
+	// We should get exactly 100 events back.
+	if len(timeSlice.ForwardingEvents) != 100 {
+		t.Fatalf("wrong number of events: expected %v, got %v", 10,
+			len(timeSlice.ForwardingEvents))
+	}
+
+	// The set of events returned should be the first 100 events that we
+	// added.
+	if !reflect.DeepEqual(events[:100], timeSlice.ForwardingEvents) {
+		t.Fatalf("wrong response: expected %v, got %v",
+			spew.Sdump(events[:100]),
+			spew.Sdump(timeSlice.ForwardingEvents))
+	}
+
+	// The final offset should be the exact number of events returned.
+	if timeSlice.LastIndexOffset != 100 {
+		t.Fatalf("wrong index offset: expected %v, got %v", 100,
+			timeSlice.LastIndexOffset)
+	}
+}

channeldb/migration_01_to_11/forwarding_package.go

@@ -0,0 +1,928 @@
+package migration_01_to_11
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/coreos/bbolt"
+	"github.com/lightningnetwork/lnd/lnwire"
+)
+
+// ErrCorruptedFwdPkg signals that the on-disk structure of the forwarding
+// package has potentially been mangled.
+var ErrCorruptedFwdPkg = errors.New("fwding package db has been corrupted")
+
+// FwdState is an enum used to describe the lifecycle of a FwdPkg.
+type FwdState byte
+
+const (
+	// FwdStateLockedIn is the starting state for all forwarding packages.
+	// Packages in this state have not yet committed to the exact set of
+	// Adds to forward to the switch.
+	FwdStateLockedIn FwdState = iota
+
+	// FwdStateProcessed marks the state in which all Adds have been
+	// locally processed and the forwarding decision to the switch has been
+	// persisted.
+	FwdStateProcessed
+
+	// FwdStateCompleted signals that all Adds have been acked, and that all
+	// settles and fails have been delivered to their sources. Packages in
+	// this state can be removed permanently.
+	FwdStateCompleted
+)
+
+var (
+	// fwdPackagesKey is the root-level bucket that all forwarding packages
+	// are written. This bucket is further subdivided based on the short
+	// channel ID of each channel.
+	fwdPackagesKey = []byte("fwd-packages")
+
+	// addBucketKey is the bucket to which all Add log updates are written.
+	addBucketKey = []byte("add-updates")
+
+	// failSettleBucketKey is the bucket to which all Settle/Fail log
+	// updates are written.
+	failSettleBucketKey = []byte("fail-settle-updates")
+
+	// fwdFilterKey is a key used to write the set of Adds that passed
+	// validation and are to be forwarded to the switch.
+	// NOTE: The presence of this key within a forwarding package indicates
+	// that the package has reached FwdStateProcessed.
+	fwdFilterKey = []byte("fwd-filter-key")
+
+	// ackFilterKey is a key used to access the PkgFilter indicating which
+	// Adds have received a Settle/Fail. This response may come from a
+	// number of sources, including: exitHop settle/fails, switch failures,
+	// chain arbiter interjections, as well as settle/fails from the
+	// next hop in the route.
+	ackFilterKey = []byte("ack-filter-key")
+
+	// settleFailFilterKey is a key used to access the PkgFilter indicating
+	// which Settles/Fails in have been received and processed by the link
+	// that originally received the Add.
+	settleFailFilterKey = []byte("settle-fail-filter-key")
+)
+
+// PkgFilter is used to compactly represent a particular subset of the Adds in a
+// forwarding package. Each filter is represented as a simple, statically-sized
+// bitvector, where the elements are intended to be the indices of the Adds as
+// they are written in the FwdPkg.
+type PkgFilter struct {
+	count  uint16
+	filter []byte
+}
+
+// NewPkgFilter initializes an empty PkgFilter supporting `count` elements.
+func NewPkgFilter(count uint16) *PkgFilter {
+	// We add 7 to ensure that the integer division yields properly rounded
+	// values.
+	filterLen := (count + 7) / 8
+
+	return &PkgFilter{
+		count:  count,
+		filter: make([]byte, filterLen),
+	}
+}
+
+// Count returns the number of elements represented by this PkgFilter.
+func (f *PkgFilter) Count() uint16 {
+	return f.count
+}
+
+// Set marks the `i`-th element as included by this filter.
+// NOTE: It is assumed that i is always less than count.
+func (f *PkgFilter) Set(i uint16) {
+	byt := i / 8
+	bit := i % 8
+
+	// Set the i-th bit in the filter.
+	// TODO(conner): ignore if > count to prevent panic?
+	f.filter[byt] |= byte(1 << (7 - bit))
+}
+
+// Contains queries the filter for membership of index `i`.
+// NOTE: It is assumed that i is always less than count.
+func (f *PkgFilter) Contains(i uint16) bool {
+	byt := i / 8
+	bit := i % 8
+
+	// Read the i-th bit in the filter.
+	// TODO(conner): ignore if > count to prevent panic?
+	return f.filter[byt]&(1<<(7-bit)) != 0
+}
+
+// Equal checks two PkgFilters for equality.
+func (f *PkgFilter) Equal(f2 *PkgFilter) bool {
+	if f == f2 {
+		return true
+	}
+	if f.count != f2.count {
+		return false
+	}
+
+	return bytes.Equal(f.filter, f2.filter)
+}
+
+// IsFull returns true if every element in the filter has been Set, and false
+// otherwise.
+func (f *PkgFilter) IsFull() bool {
+	// Batch validate bytes that are fully used.
+	for i := uint16(0); i < f.count/8; i++ {
+		if f.filter[i] != 0xFF {
+			return false
+		}
+	}
+
+	// If the count is not a multiple of 8, check that the filter contains
+	// all remaining bits.
+	rem := f.count % 8
+	for idx := f.count - rem; idx < f.count; idx++ {
+		if !f.Contains(idx) {
+			return false
+		}
+	}
+
+	return true
+}
+
+// Size returns number of bytes produced when the PkgFilter is serialized.
+func (f *PkgFilter) Size() uint16 {
+	// 2 bytes for uint16 `count`, then round up number of bytes required to
+	// represent `count` bits.
+	return 2 + (f.count+7)/8
+}
+
+// Encode writes the filter to the provided io.Writer.
+func (f *PkgFilter) Encode(w io.Writer) error {
+	if err := binary.Write(w, binary.BigEndian, f.count); err != nil {
+		return err
+	}
+
+	_, err := w.Write(f.filter)
+
+	return err
+}
+
+// Decode reads the filter from the provided io.Reader.
+func (f *PkgFilter) Decode(r io.Reader) error {
+	if err := binary.Read(r, binary.BigEndian, &f.count); err != nil {
+		return err
+	}
+
+	f.filter = make([]byte, f.Size()-2)
+	_, err := io.ReadFull(r, f.filter)
+
+	return err
+}
+
+// FwdPkg records all adds, settles, and fails that were locked in as a result
+// of the remote peer sending us a revocation. Each package is identified by
+// the short chanid and remote commitment height corresponding to the revocation
+// that locked in the HTLCs. For everything except a locally initiated payment,
+// settles and fails in a forwarding package must have a corresponding Add in
+// another package, and can be removed individually once the source link has
+// received the fail/settle.
+//
+// Adds cannot be removed, as we need to present the same batch of Adds to
+// properly handle replay protection. Instead, we use a PkgFilter to mark that
+// we have finished processing a particular Add. A FwdPkg should only be deleted
+// after the AckFilter is full and all settles and fails have been persistently
+// removed.
+type FwdPkg struct {
+	// Source identifies the channel that wrote this forwarding package.
+	Source lnwire.ShortChannelID
+
+	// Height is the height of the remote commitment chain that locked in
+	// this forwarding package.
+	Height uint64
+
+	// State signals the persistent condition of the package and directs how
+	// to reprocess the package in the event of failures.
+	State FwdState
+
+	// Adds contains all add messages which need to be processed and
+	// forwarded to the switch. Adds does not change over the life of a
+	// forwarding package.
+	Adds []LogUpdate
+
+	// FwdFilter is a filter containing the indices of all Adds that were
+	// forwarded to the switch.
+	FwdFilter *PkgFilter
+
+	// AckFilter is a filter containing the indices of all Adds for which
+	// the source has received a settle or fail and is reflected in the next
+	// commitment txn. A package should not be removed until IsFull()
+	// returns true.
+	AckFilter *PkgFilter
+
+	// SettleFails contains all settle and fail messages that should be
+	// forwarded to the switch.
+	SettleFails []LogUpdate
+
+	// SettleFailFilter is a filter containing the indices of all Settle or
+	// Fails originating in this package that have been received and locked
+	// into the incoming link's commitment state.
+	SettleFailFilter *PkgFilter
+}
+
+// NewFwdPkg initializes a new forwarding package in FwdStateLockedIn. This
+// should be used to create a package at the time we receive a revocation.
+func NewFwdPkg(source lnwire.ShortChannelID, height uint64,
+	addUpdates, settleFailUpdates []LogUpdate) *FwdPkg {
+
+	nAddUpdates := uint16(len(addUpdates))
+	nSettleFailUpdates := uint16(len(settleFailUpdates))
+
+	return &FwdPkg{
+		Source:           source,
+		Height:           height,
+		State:            FwdStateLockedIn,
+		Adds:             addUpdates,
+		FwdFilter:        NewPkgFilter(nAddUpdates),
+		AckFilter:        NewPkgFilter(nAddUpdates),
+		SettleFails:      settleFailUpdates,
+		SettleFailFilter: NewPkgFilter(nSettleFailUpdates),
+	}
+}
+
+// ID returns an unique identifier for this package, used to ensure that sphinx
+// replay processing of this batch is idempotent.
+func (f *FwdPkg) ID() []byte {
+	var id = make([]byte, 16)
+	byteOrder.PutUint64(id[:8], f.Source.ToUint64())
+	byteOrder.PutUint64(id[8:], f.Height)
+	return id
+}
+
+// String returns a human-readable description of the forwarding package.
+func (f *FwdPkg) String() string {
+	return fmt.Sprintf("%T(src=%v, height=%v, nadds=%v, nfailsettles=%v)",
+		f, f.Source, f.Height, len(f.Adds), len(f.SettleFails))
+}
+
+// AddRef is used to identify a particular Add in a FwdPkg. The short channel ID
+// is assumed to be that of the packager.
+type AddRef struct {
+	// Height is the remote commitment height that locked in the Add.
+	Height uint64
+
+	// Index is the index of the Add within the fwd pkg's Adds.
+	//
+	// NOTE: This index is static over the lifetime of a forwarding package.
+	Index uint16
+}
+
+// Encode serializes the AddRef to the given io.Writer.
+func (a *AddRef) Encode(w io.Writer) error {
+	if err := binary.Write(w, binary.BigEndian, a.Height); err != nil {
+		return err
+	}
+
+	return binary.Write(w, binary.BigEndian, a.Index)
+}
+
+// Decode deserializes the AddRef from the given io.Reader.
+func (a *AddRef) Decode(r io.Reader) error {
+	if err := binary.Read(r, binary.BigEndian, &a.Height); err != nil {
+		return err
+	}
+
+	return binary.Read(r, binary.BigEndian, &a.Index)
+}
+
+// SettleFailRef is used to locate a Settle/Fail in another channel's FwdPkg. A
+// channel does not remove its own Settle/Fail htlcs, so the source is provided
+// to locate a db bucket belonging to another channel.
+type SettleFailRef struct {
+	// Source identifies the outgoing link that locked in the settle or
+	// fail. This is then used by the *incoming* link to find the settle
+	// fail in another link's forwarding packages.
+	Source lnwire.ShortChannelID
+
+	// Height is the remote commitment height that locked in this
+	// Settle/Fail.
+	Height uint64
+
+	// Index is the index of the Add with the fwd pkg's SettleFails.
+	//
+	// NOTE: This index is static over the lifetime of a forwarding package.
+	Index uint16
+}
+
+// SettleFailAcker is a generic interface providing the ability to acknowledge
+// settle/fail HTLCs stored in forwarding packages.
+type SettleFailAcker interface {
+	// AckSettleFails atomically updates the settle-fail filters in *other*
+	// channels' forwarding packages.
+	AckSettleFails(tx *bbolt.Tx, settleFailRefs ...SettleFailRef) error
+}
+
+// GlobalFwdPkgReader is an interface used to retrieve the forwarding packages
+// of any active channel.
+type GlobalFwdPkgReader interface {
+	// LoadChannelFwdPkgs loads all known forwarding packages for the given
+	// channel.
+	LoadChannelFwdPkgs(tx *bbolt.Tx,
+		source lnwire.ShortChannelID) ([]*FwdPkg, error)
+}
+
+// FwdOperator defines the interfaces for managing forwarding packages that are
+// external to a particular channel. This interface is used by the switch to
+// read forwarding packages from arbitrary channels, and acknowledge settles and
+// fails for locally-sourced payments.
+type FwdOperator interface {
+	// GlobalFwdPkgReader provides read access to all known forwarding
+	// packages
+	GlobalFwdPkgReader
+
+	// SettleFailAcker grants the ability to acknowledge settles or fails
+	// residing in arbitrary forwarding packages.
+	SettleFailAcker
+}
+
+// SwitchPackager is a concrete implementation of the FwdOperator interface.
+// A SwitchPackager offers the ability to read any forwarding package, and ack
+// arbitrary settle and fail HTLCs.
+type SwitchPackager struct{}
+
+// NewSwitchPackager instantiates a new SwitchPackager.
+func NewSwitchPackager() *SwitchPackager {
+	return &SwitchPackager{}
+}
+
+// AckSettleFails atomically updates the settle-fail filters in *other*
+// channels' forwarding packages, to mark that the switch has received a settle
+// or fail residing in the forwarding package of a link.
+func (*SwitchPackager) AckSettleFails(tx *bbolt.Tx,
+	settleFailRefs ...SettleFailRef) error {
+
+	return ackSettleFails(tx, settleFailRefs)
+}
+
+// LoadChannelFwdPkgs loads all forwarding packages for a particular channel.
+func (*SwitchPackager) LoadChannelFwdPkgs(tx *bbolt.Tx,
+	source lnwire.ShortChannelID) ([]*FwdPkg, error) {
+
+	return loadChannelFwdPkgs(tx, source)
+}
+
+// FwdPackager supports all operations required to modify fwd packages, such as
+// creation, updates, reading, and removal. The interfaces are broken down in
+// this way to support future delegation of the subinterfaces.
+type FwdPackager interface {
+	// AddFwdPkg serializes and writes a FwdPkg for this channel at the
+	// remote commitment height included in the forwarding package.
+	AddFwdPkg(tx *bbolt.Tx, fwdPkg *FwdPkg) error
+
+	// SetFwdFilter looks up the forwarding package at the remote `height`
+	// and sets the `fwdFilter`, marking the Adds for which:
+	// 1) We are not the exit node
+	// 2) Passed all validation
+	// 3) Should be forwarded to the switch immediately after a failure
+	SetFwdFilter(tx *bbolt.Tx, height uint64, fwdFilter *PkgFilter) error
+
+	// AckAddHtlcs atomically updates the add filters in this channel's
+	// forwarding packages to mark the resolution of an Add that was
+	// received from the remote party.
+	AckAddHtlcs(tx *bbolt.Tx, addRefs ...AddRef) error
+
+	// SettleFailAcker allows a link to acknowledge settle/fail HTLCs
+	// belonging to other channels.
+	SettleFailAcker
+
+	// LoadFwdPkgs loads all known forwarding packages owned by this
+	// channel.
+	LoadFwdPkgs(tx *bbolt.Tx) ([]*FwdPkg, error)
+
+	// RemovePkg deletes a forwarding package owned by this channel at
+	// the provided remote `height`.
+	RemovePkg(tx *bbolt.Tx, height uint64) error
+}
+
+// ChannelPackager is used by a channel to manage the lifecycle of its forwarding
+// packages. The packager is tied to a particular source channel ID, allowing it
+// to create and edit its own packages. Each packager also has the ability to
+// remove fail/settle htlcs that correspond to an add contained in one of
+// source's packages.
+type ChannelPackager struct {
+	source lnwire.ShortChannelID
+}
+
+// NewChannelPackager creates a new packager for a single channel.
+func NewChannelPackager(source lnwire.ShortChannelID) *ChannelPackager {
+	return &ChannelPackager{
+		source: source,
+	}
+}
+
+// AddFwdPkg writes a newly locked in forwarding package to disk.
+func (*ChannelPackager) AddFwdPkg(tx *bbolt.Tx, fwdPkg *FwdPkg) error {
+	fwdPkgBkt, err := tx.CreateBucketIfNotExists(fwdPackagesKey)
+	if err != nil {
+		return err
+	}
+
+	source := makeLogKey(fwdPkg.Source.ToUint64())
+	sourceBkt, err := fwdPkgBkt.CreateBucketIfNotExists(source[:])
+	if err != nil {
+		return err
+	}
+
+	heightKey := makeLogKey(fwdPkg.Height)
+	heightBkt, err := sourceBkt.CreateBucketIfNotExists(heightKey[:])
+	if err != nil {
+		return err
+	}
+
+	// Write ADD updates we received at this commit height.
+	addBkt, err := heightBkt.CreateBucketIfNotExists(addBucketKey)
+	if err != nil {
+		return err
+	}
+
+	// Write SETTLE/FAIL updates we received at this commit height.
+	failSettleBkt, err := heightBkt.CreateBucketIfNotExists(failSettleBucketKey)
+	if err != nil {
+		return err
+	}
+
+	for i := range fwdPkg.Adds {
+		err = putLogUpdate(addBkt, uint16(i), &fwdPkg.Adds[i])
+		if err != nil {
+			return err
+		}
+	}
+
+	// Persist the initialized pkg filter, which will be used to determine
+	// when we can remove this forwarding package from disk.
+	var ackFilterBuf bytes.Buffer
+	if err := fwdPkg.AckFilter.Encode(&ackFilterBuf); err != nil {
+		return err
+	}
+
+	if err := heightBkt.Put(ackFilterKey, ackFilterBuf.Bytes()); err != nil {
+		return err
+	}
+
+	for i := range fwdPkg.SettleFails {
+		err = putLogUpdate(failSettleBkt, uint16(i), &fwdPkg.SettleFails[i])
+		if err != nil {
+			return err
+		}
+	}
+
+	var settleFailFilterBuf bytes.Buffer
+	err = fwdPkg.SettleFailFilter.Encode(&settleFailFilterBuf)
+	if err != nil {
+		return err
+	}
+
+	return heightBkt.Put(settleFailFilterKey, settleFailFilterBuf.Bytes())
+}
+
+// putLogUpdate writes an htlc to the provided `bkt`, using `index` as the key.
+func putLogUpdate(bkt *bbolt.Bucket, idx uint16, htlc *LogUpdate) error {
+	var b bytes.Buffer
+	if err := htlc.Encode(&b); err != nil {
+		return err
+	}
+
+	return bkt.Put(uint16Key(idx), b.Bytes())
+}
+
+// LoadFwdPkgs scans the forwarding log for any packages that haven't been
+// processed, and returns their deserialized log updates in a map indexed by the
+// remote commitment height at which the updates were locked in.
+func (p *ChannelPackager) LoadFwdPkgs(tx *bbolt.Tx) ([]*FwdPkg, error) {
+	return loadChannelFwdPkgs(tx, p.source)
+}
+
+// loadChannelFwdPkgs loads all forwarding packages owned by `source`.
+func loadChannelFwdPkgs(tx *bbolt.Tx, source lnwire.ShortChannelID) ([]*FwdPkg, error) {
+	fwdPkgBkt := tx.Bucket(fwdPackagesKey)
+	if fwdPkgBkt == nil {
+		return nil, nil
+	}
+
+	sourceKey := makeLogKey(source.ToUint64())
+	sourceBkt := fwdPkgBkt.Bucket(sourceKey[:])
+	if sourceBkt == nil {
+		return nil, nil
+	}
+
+	var heights []uint64
+	if err := sourceBkt.ForEach(func(k, _ []byte) error {
+		if len(k) != 8 {
+			return ErrCorruptedFwdPkg
+		}
+
+		heights = append(heights, byteOrder.Uint64(k))
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	// Load the forwarding package for each retrieved height.
+	fwdPkgs := make([]*FwdPkg, 0, len(heights))
+	for _, height := range heights {
+		fwdPkg, err := loadFwdPkg(fwdPkgBkt, source, height)
+		if err != nil {
+			return nil, err
+		}
+
+		fwdPkgs = append(fwdPkgs, fwdPkg)
+	}
+
+	return fwdPkgs, nil
+}
+
+// loadFwPkg reads the packager's fwd pkg at a given height, and determines the
+// appropriate FwdState.
+func loadFwdPkg(fwdPkgBkt *bbolt.Bucket, source lnwire.ShortChannelID,
+	height uint64) (*FwdPkg, error) {
+
+	sourceKey := makeLogKey(source.ToUint64())
+	sourceBkt := fwdPkgBkt.Bucket(sourceKey[:])
+	if sourceBkt == nil {
+		return nil, ErrCorruptedFwdPkg
+	}
+
+	heightKey := makeLogKey(height)
+	heightBkt := sourceBkt.Bucket(heightKey[:])
+	if heightBkt == nil {
+		return nil, ErrCorruptedFwdPkg
+	}
+
+	// Load ADDs from disk.
+	addBkt := heightBkt.Bucket(addBucketKey)
+	if addBkt == nil {
+		return nil, ErrCorruptedFwdPkg
+	}
+
+	adds, err := loadHtlcs(addBkt)
+	if err != nil {
+		return nil, err
+	}
+
+	// Load ack filter from disk.
+	ackFilterBytes := heightBkt.Get(ackFilterKey)
+	if ackFilterBytes == nil {
+		return nil, ErrCorruptedFwdPkg
+	}
+	ackFilterReader := bytes.NewReader(ackFilterBytes)
+
+	ackFilter := &PkgFilter{}
+	if err := ackFilter.Decode(ackFilterReader); err != nil {
+		return nil, err
+	}
+
+	// Load SETTLE/FAILs from disk.
+	failSettleBkt := heightBkt.Bucket(failSettleBucketKey)
+	if failSettleBkt == nil {
+		return nil, ErrCorruptedFwdPkg
+	}
+
+	failSettles, err := loadHtlcs(failSettleBkt)
+	if err != nil {
+		return nil, err
+	}
+
+	// Load settle fail filter from disk.
+	settleFailFilterBytes := heightBkt.Get(settleFailFilterKey)
+	if settleFailFilterBytes == nil {
+		return nil, ErrCorruptedFwdPkg
+	}
+	settleFailFilterReader := bytes.NewReader(settleFailFilterBytes)
+
+	settleFailFilter := &PkgFilter{}
+	if err := settleFailFilter.Decode(settleFailFilterReader); err != nil {
+		return nil, err
+	}
+
+	// Initialize the fwding package, which always starts in the
+	// FwdStateLockedIn. We can determine what state the package was left in
+	// by examining constraints on the information loaded from disk.
+	fwdPkg := &FwdPkg{
+		Source:           source,
+		State:            FwdStateLockedIn,
+		Height:           height,
+		Adds:             adds,
+		AckFilter:        ackFilter,
+		SettleFails:      failSettles,
+		SettleFailFilter: settleFailFilter,
+	}
+
+	// Check to see if we have written the set exported filter adds to
+	// disk. If we haven't, processing of this package was never started, or
+	// failed during the last attempt.
+	fwdFilterBytes := heightBkt.Get(fwdFilterKey)
+	if fwdFilterBytes == nil {
+		nAdds := uint16(len(adds))
+		fwdPkg.FwdFilter = NewPkgFilter(nAdds)
+		return fwdPkg, nil
+	}
+
+	fwdFilterReader := bytes.NewReader(fwdFilterBytes)
+	fwdPkg.FwdFilter = &PkgFilter{}
+	if err := fwdPkg.FwdFilter.Decode(fwdFilterReader); err != nil {
+		return nil, err
+	}
+
+	// Otherwise, a complete round of processing was completed, and we
+	// advance the package to FwdStateProcessed.
+	fwdPkg.State = FwdStateProcessed
+
+	// If every add, settle, and fail has been fully acknowledged, we can
+	// safely set the package's state to FwdStateCompleted, signalling that
+	// it can be garbage collected.
+	if fwdPkg.AckFilter.IsFull() && fwdPkg.SettleFailFilter.IsFull() {
+		fwdPkg.State = FwdStateCompleted
+	}
+
+	return fwdPkg, nil
+}
+
+// loadHtlcs retrieves all serialized htlcs in a bucket, returning
+// them in order of the indexes they were written under.
+func loadHtlcs(bkt *bbolt.Bucket) ([]LogUpdate, error) {
+	var htlcs []LogUpdate
+	if err := bkt.ForEach(func(_, v []byte) error {
+		var htlc LogUpdate
+		if err := htlc.Decode(bytes.NewReader(v)); err != nil {
+			return err
+		}
+
+		htlcs = append(htlcs, htlc)
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return htlcs, nil
+}
+
+// SetFwdFilter writes the set of indexes corresponding to Adds at the
+// `height` that are to be forwarded to the switch. Calling this method causes
+// the forwarding package at `height` to be in FwdStateProcessed. We write this
+// forwarding decision so that we always arrive at the same behavior for HTLCs
+// leaving this channel. After a restart, we skip validation of these Adds,
+// since they are assumed to have already been validated, and make the switch or
+// outgoing link responsible for handling replays.
+func (p *ChannelPackager) SetFwdFilter(tx *bbolt.Tx, height uint64,
+	fwdFilter *PkgFilter) error {
+
+	fwdPkgBkt := tx.Bucket(fwdPackagesKey)
+	if fwdPkgBkt == nil {
+		return ErrCorruptedFwdPkg
+	}
+
+	source := makeLogKey(p.source.ToUint64())
+	sourceBkt := fwdPkgBkt.Bucket(source[:])
+	if sourceBkt == nil {
+		return ErrCorruptedFwdPkg
+	}
+
+	heightKey := makeLogKey(height)
+	heightBkt := sourceBkt.Bucket(heightKey[:])
+	if heightBkt == nil {
+		return ErrCorruptedFwdPkg
+	}
+
+	// If the fwd filter has already been written, we return early to avoid
+	// modifying the persistent state.
+	forwardedAddsBytes := heightBkt.Get(fwdFilterKey)
+	if forwardedAddsBytes != nil {
+		return nil
+	}
+
+	// Otherwise we serialize and write the provided fwd filter.
+	var b bytes.Buffer
+	if err := fwdFilter.Encode(&b); err != nil {
+		return err
+	}
+
+	return heightBkt.Put(fwdFilterKey, b.Bytes())
+}
+
+// AckAddHtlcs accepts a list of references to add htlcs, and updates the
+// AckAddFilter of those forwarding packages to indicate that a settle or fail
+// has been received in response to the add.
+func (p *ChannelPackager) AckAddHtlcs(tx *bbolt.Tx, addRefs ...AddRef) error {
+	if len(addRefs) == 0 {
+		return nil
+	}
+
+	fwdPkgBkt := tx.Bucket(fwdPackagesKey)
+	if fwdPkgBkt == nil {
+		return ErrCorruptedFwdPkg
+	}
+
+	sourceKey := makeLogKey(p.source.ToUint64())
+	sourceBkt := fwdPkgBkt.Bucket(sourceKey[:])
+	if sourceBkt == nil {
+		return ErrCorruptedFwdPkg
+	}
+
+	// Organize the forward references such that we just get a single slice
+	// of indexes for each unique height.
+	heightDiffs := make(map[uint64][]uint16)
+	for _, addRef := range addRefs {
+		heightDiffs[addRef.Height] = append(
+			heightDiffs[addRef.Height],
+			addRef.Index,
+		)
+	}
+
+	// Load each height bucket once and remove all acked htlcs at that
+	// height.
+	for height, indexes := range heightDiffs {
+		err := ackAddHtlcsAtHeight(sourceBkt, height, indexes)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// ackAddHtlcsAtHeight updates the AddAckFilter of a single forwarding package
+// with a list of indexes, writing the resulting filter back in its place.
+func ackAddHtlcsAtHeight(sourceBkt *bbolt.Bucket, height uint64,
+	indexes []uint16) error {
+
+	heightKey := makeLogKey(height)
+	heightBkt := sourceBkt.Bucket(heightKey[:])
+	if heightBkt == nil {
+		// If the height bucket isn't found, this could be because the
+		// forwarding package was already removed. We'll return nil to
+		// signal that the operation is successful, as there is nothing
+		// to ack.
+		return nil
+	}
+
+	// Load ack filter from disk.
+	ackFilterBytes := heightBkt.Get(ackFilterKey)
+	if ackFilterBytes == nil {
+		return ErrCorruptedFwdPkg
+	}
+
+	ackFilter := &PkgFilter{}
+	ackFilterReader := bytes.NewReader(ackFilterBytes)
+	if err := ackFilter.Decode(ackFilterReader); err != nil {
+		return err
+	}
+
+	// Update the ack filter for this height.
+	for _, index := range indexes {
+		ackFilter.Set(index)
+	}
+
+	// Write the resulting filter to disk.
+	var ackFilterBuf bytes.Buffer
+	if err := ackFilter.Encode(&ackFilterBuf); err != nil {
+		return err
+	}
+
+	return heightBkt.Put(ackFilterKey, ackFilterBuf.Bytes())
+}
+
+// AckSettleFails persistently acknowledges settles or fails from a remote forwarding
+// package. This should only be called after the source of the Add has locked in
+// the settle/fail, or it becomes otherwise safe to forgo retransmitting the
+// settle/fail after a restart.
+func (p *ChannelPackager) AckSettleFails(tx *bbolt.Tx, settleFailRefs ...SettleFailRef) error {
+	return ackSettleFails(tx, settleFailRefs)
+}
+
+// ackSettleFails persistently acknowledges a batch of settle fail references.
+func ackSettleFails(tx *bbolt.Tx, settleFailRefs []SettleFailRef) error {
+	if len(settleFailRefs) == 0 {
+		return nil
+	}
+
+	fwdPkgBkt := tx.Bucket(fwdPackagesKey)
+	if fwdPkgBkt == nil {
+		return ErrCorruptedFwdPkg
+	}
+
+	// Organize the forward references such that we just get a single slice
+	// of indexes for each unique destination-height pair.
+	destHeightDiffs := make(map[lnwire.ShortChannelID]map[uint64][]uint16)
+	for _, settleFailRef := range settleFailRefs {
+		destHeights, ok := destHeightDiffs[settleFailRef.Source]
+		if !ok {
+			destHeights = make(map[uint64][]uint16)
+			destHeightDiffs[settleFailRef.Source] = destHeights
+		}
+
+		destHeights[settleFailRef.Height] = append(
+			destHeights[settleFailRef.Height],
+			settleFailRef.Index,
+		)
+	}
+
+	// With the references organized by destination and height, we now load
+	// each remote bucket, and update the settle fail filter for any
+	// settle/fail htlcs.
+	for dest, destHeights := range destHeightDiffs {
+		destKey := makeLogKey(dest.ToUint64())
+		destBkt := fwdPkgBkt.Bucket(destKey[:])
+		if destBkt == nil {
+			// If the destination bucket is not found, this is
+			// likely the result of the destination channel being
+			// closed and having it's forwarding packages wiped. We
+			// won't treat this as an error, because the response
+			// will no longer be retransmitted internally.
+			continue
+		}
+
+		for height, indexes := range destHeights {
+			err := ackSettleFailsAtHeight(destBkt, height, indexes)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// ackSettleFailsAtHeight given a destination bucket, acks the provided indexes
+// at particular a height by updating the settle fail filter.
+func ackSettleFailsAtHeight(destBkt *bbolt.Bucket, height uint64,
+	indexes []uint16) error {
+
+	heightKey := makeLogKey(height)
+	heightBkt := destBkt.Bucket(heightKey[:])
+	if heightBkt == nil {
+		// If the height bucket isn't found, this could be because the
+		// forwarding package was already removed. We'll return nil to
+		// signal that the operation is as there is nothing to ack.
+		return nil
+	}
+
+	// Load ack filter from disk.
+	settleFailFilterBytes := heightBkt.Get(settleFailFilterKey)
+	if settleFailFilterBytes == nil {
+		return ErrCorruptedFwdPkg
+	}
+
+	settleFailFilter := &PkgFilter{}
+	settleFailFilterReader := bytes.NewReader(settleFailFilterBytes)
+	if err := settleFailFilter.Decode(settleFailFilterReader); err != nil {
+		return err
+	}
+
+	// Update the ack filter for this height.
+	for _, index := range indexes {
+		settleFailFilter.Set(index)
+	}
+
+	// Write the resulting filter to disk.
+	var settleFailFilterBuf bytes.Buffer
+	if err := settleFailFilter.Encode(&settleFailFilterBuf); err != nil {
+		return err
+	}
+
+	return heightBkt.Put(settleFailFilterKey, settleFailFilterBuf.Bytes())
+}
+
+// RemovePkg deletes the forwarding package at the given height from the
+// packager's source bucket.
+func (p *ChannelPackager) RemovePkg(tx *bbolt.Tx, height uint64) error {
+	fwdPkgBkt := tx.Bucket(fwdPackagesKey)
+	if fwdPkgBkt == nil {
+		return nil
+	}
+
+	sourceBytes := makeLogKey(p.source.ToUint64())
+	sourceBkt := fwdPkgBkt.Bucket(sourceBytes[:])
+	if sourceBkt == nil {
+		return ErrCorruptedFwdPkg
+	}
+
+	heightKey := makeLogKey(height)
+
+	return sourceBkt.DeleteBucket(heightKey[:])
+}
+
+// uint16Key writes the provided 16-bit unsigned integer to a 2-byte slice.
+func uint16Key(i uint16) []byte {
+	key := make([]byte, 2)
+	byteOrder.PutUint16(key, i)
+	return key
+}
+
+// Compile-time constraint to ensure that ChannelPackager implements the public
+// FwdPackager interface.
+var _ FwdPackager = (*ChannelPackager)(nil)
+
+// Compile-time constraint to ensure that SwitchPackager implements the public
+// FwdOperator interface.
+var _ FwdOperator = (*SwitchPackager)(nil)

channeldb/migration_01_to_11/forwarding_package_test.go

@@ -0,0 +1,815 @@
+package migration_01_to_11_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/btcsuite/btcd/wire"
+	"github.com/coreos/bbolt"
+	"github.com/lightningnetwork/lnd/channeldb"
+	"github.com/lightningnetwork/lnd/lnwire"
+)
+
+// TestPkgFilterBruteForce tests the behavior of a pkg filter up to size 1000,
+// which is greater than the number of HTLCs we permit on a commitment txn.
+// This should encapsulate every potential filter used in practice.
+func TestPkgFilterBruteForce(t *testing.T) {
+	t.Parallel()
+
+	checkPkgFilterRange(t, 1000)
+}
+
+// checkPkgFilterRange verifies the behavior of a pkg filter when doing a linear
+// insertion of `high` elements. This is primarily to test that IsFull functions
+// properly for all relevant sizes of `high`.
+func checkPkgFilterRange(t *testing.T, high int) {
+	for i := uint16(0); i < uint16(high); i++ {
+		f := channeldb.NewPkgFilter(i)
+
+		if f.Count() != i {
+			t.Fatalf("pkg filter count=%d is actually %d",
+				i, f.Count())
+		}
+		checkPkgFilterEncodeDecode(t, i, f)
+
+		for j := uint16(0); j < i; j++ {
+			if f.Contains(j) {
+				t.Fatalf("pkg filter count=%d contains %d "+
+					"before being added", i, j)
+			}
+
+			f.Set(j)
+			checkPkgFilterEncodeDecode(t, i, f)
+
+			if !f.Contains(j) {
+				t.Fatalf("pkg filter count=%d missing %d "+
+					"after being added", i, j)
+			}
+
+			if j < i-1 && f.IsFull() {
+				t.Fatalf("pkg filter count=%d already full", i)
+			}
+		}
+
+		if !f.IsFull() {
+			t.Fatalf("pkg filter count=%d not full", i)
+		}
+		checkPkgFilterEncodeDecode(t, i, f)
+	}
+}
+
+// TestPkgFilterRand uses a random permutation to verify the proper behavior of
+// the pkg filter if the entries are not inserted in-order.
+func TestPkgFilterRand(t *testing.T) {
+	t.Parallel()
+
+	checkPkgFilterRand(t, 3, 17)
+}
+
+// checkPkgFilterRand checks the behavior of a pkg filter by randomly inserting
+// indices and asserting the invariants. The order in which indices are inserted
+// is parameterized by a base `b` coprime to `p`, and using modular
+// exponentiation to generate all elements in [1,p).
+func checkPkgFilterRand(t *testing.T, b, p uint16) {
+	f := channeldb.NewPkgFilter(p)
+	var j = b
+	for i := uint16(1); i < p; i++ {
+		if f.Contains(j) {
+			t.Fatalf("pkg filter contains %d-%d "+
+				"before being added", i, j)
+		}
+
+		f.Set(j)
+		checkPkgFilterEncodeDecode(t, i, f)
+
+		if !f.Contains(j) {
+			t.Fatalf("pkg filter missing %d-%d "+
+				"after being added", i, j)
+		}
+
+		if i < p-1 && f.IsFull() {
+			t.Fatalf("pkg filter %d already full", i)
+		}
+		checkPkgFilterEncodeDecode(t, i, f)
+
+		j = (b * j) % p
+	}
+
+	// Set 0 independently, since it will never be emitted by the generator.
+	f.Set(0)
+	checkPkgFilterEncodeDecode(t, p, f)
+
+	if !f.IsFull() {
+		t.Fatalf("pkg filter count=%d not full", p)
+	}
+	checkPkgFilterEncodeDecode(t, p, f)
+}
+
+// checkPkgFilterEncodeDecode tests the serialization of a pkg filter by:
+//   1) writing it to a buffer
+//   2) verifying the number of bytes written matches the filter's Size()
+//   3) reconstructing the filter decoding the bytes
+//   4) checking that the two filters are the same according to Equal
+func checkPkgFilterEncodeDecode(t *testing.T, i uint16, f *channeldb.PkgFilter) {
+	var b bytes.Buffer
+	if err := f.Encode(&b); err != nil {
+		t.Fatalf("unable to serialize pkg filter: %v", err)
+	}
+
+	// +2 for uint16 length
+	size := uint16(len(b.Bytes()))
+	if size != f.Size() {
+		t.Fatalf("pkg filter count=%d serialized size differs, "+
+			"Size(): %d, len(bytes): %v", i, f.Size(), size)
+	}
+
+	reader := bytes.NewReader(b.Bytes())
+
+	f2 := &channeldb.PkgFilter{}
+	if err := f2.Decode(reader); err != nil {
+		t.Fatalf("unable to deserialize pkg filter: %v", err)
+	}
+
+	if !f.Equal(f2) {
+		t.Fatalf("pkg filter count=%v does is not equal "+
+			"after deserialization, want: %v, got %v",
+			i, f, f2)
+	}
+}
+
+var (
+	chanID = lnwire.NewChanIDFromOutPoint(&wire.OutPoint{})
+
+	adds = []channeldb.LogUpdate{
+		{
+			LogIndex: 0,
+			UpdateMsg: &lnwire.UpdateAddHTLC{
+				ChanID:      chanID,
+				ID:          1,
+				Amount:      100,
+				Expiry:      1000,
+				PaymentHash: [32]byte{0},
+			},
+		},
+		{
+			LogIndex: 1,
+			UpdateMsg: &lnwire.UpdateAddHTLC{
+				ChanID:      chanID,
+				ID:          1,
+				Amount:      101,
+				Expiry:      1001,
+				PaymentHash: [32]byte{1},
+			},
+		},
+	}
+
+	settleFails = []channeldb.LogUpdate{
+		{
+			LogIndex: 2,
+			UpdateMsg: &lnwire.UpdateFulfillHTLC{
+				ChanID:          chanID,
+				ID:              0,
+				PaymentPreimage: [32]byte{0},
+			},
+		},
+		{
+			LogIndex: 3,
+			UpdateMsg: &lnwire.UpdateFailHTLC{
+				ChanID: chanID,
+				ID:     1,
+				Reason: []byte{},
+			},
+		},
+	}
+)
+
+// TestPackagerEmptyFwdPkg checks that the state transitions exhibited by a
+// forwarding package that contains no adds, fails or settles. We expect that
+// the fwdpkg reaches FwdStateCompleted immediately after writing the forwarding
+// decision via SetFwdFilter.
+func TestPackagerEmptyFwdPkg(t *testing.T) {
+	t.Parallel()
+
+	db := makeFwdPkgDB(t, "")
+
+	shortChanID := lnwire.NewShortChanIDFromInt(1)
+	packager := channeldb.NewChannelPackager(shortChanID)
+
+	// To begin, there should be no forwarding packages on disk.
+	fwdPkgs := loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 0 {
+		t.Fatalf("no forwarding packages should exist, found %d", len(fwdPkgs))
+	}
+
+	// Next, create and write a new forwarding package with no htlcs.
+	fwdPkg := channeldb.NewFwdPkg(shortChanID, 0, nil, nil)
+
+	if err := db.Update(func(tx *bbolt.Tx) error {
+		return packager.AddFwdPkg(tx, fwdPkg)
+	}); err != nil {
+		t.Fatalf("unable to add fwd pkg: %v", err)
+	}
+
+	// There should now be one fwdpkg on disk. Since no forwarding decision
+	// has been written, we expect it to be FwdStateLockedIn. With no HTLCs,
+	// the ack filter will have no elements, and should always return true.
+	fwdPkgs = loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 1 {
+		t.Fatalf("expected 1 fwdpkg, instead found %d", len(fwdPkgs))
+	}
+	assertFwdPkgState(t, fwdPkgs[0], channeldb.FwdStateLockedIn)
+	assertFwdPkgNumAddsSettleFails(t, fwdPkgs[0], 0, 0)
+	assertAckFilterIsFull(t, fwdPkgs[0], true)
+
+	// Now, write the forwarding decision. In this case, its just an empty
+	// fwd filter.
+	if err := db.Update(func(tx *bbolt.Tx) error {
+		return packager.SetFwdFilter(tx, fwdPkg.Height, fwdPkg.FwdFilter)
+	}); err != nil {
+		t.Fatalf("unable to set fwdfiter: %v", err)
+	}
+
+	// We should still have one package on disk. Since the forwarding
+	// decision has been written, it will minimally be in FwdStateProcessed.
+	// However with no htlcs, it should leap frog to FwdStateCompleted.
+	fwdPkgs = loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 1 {
+		t.Fatalf("expected 1 fwdpkg, instead found %d", len(fwdPkgs))
+	}
+	assertFwdPkgState(t, fwdPkgs[0], channeldb.FwdStateCompleted)
+	assertFwdPkgNumAddsSettleFails(t, fwdPkgs[0], 0, 0)
+	assertAckFilterIsFull(t, fwdPkgs[0], true)
+
+	// Lastly, remove the completed forwarding package from disk.
+	if err := db.Update(func(tx *bbolt.Tx) error {
+		return packager.RemovePkg(tx, fwdPkg.Height)
+	}); err != nil {
+		t.Fatalf("unable to remove fwdpkg: %v", err)
+	}
+
+	// Check that the fwd package was actually removed.
+	fwdPkgs = loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 0 {
+		t.Fatalf("no forwarding packages should exist, found %d", len(fwdPkgs))
+	}
+}
+
+// TestPackagerOnlyAdds checks that the fwdpkg does not reach FwdStateCompleted
+// as soon as all the adds in the package have been acked using AckAddHtlcs.
+func TestPackagerOnlyAdds(t *testing.T) {
+	t.Parallel()
+
+	db := makeFwdPkgDB(t, "")
+
+	shortChanID := lnwire.NewShortChanIDFromInt(1)
+	packager := channeldb.NewChannelPackager(shortChanID)
+
+	// To begin, there should be no forwarding packages on disk.
+	fwdPkgs := loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 0 {
+		t.Fatalf("no forwarding packages should exist, found %d", len(fwdPkgs))
+	}
+
+	// Next, create and write a new forwarding package that only has add
+	// htlcs.
+	fwdPkg := channeldb.NewFwdPkg(shortChanID, 0, adds, nil)
+
+	nAdds := len(adds)
+
+	if err := db.Update(func(tx *bbolt.Tx) error {
+		return packager.AddFwdPkg(tx, fwdPkg)
+	}); err != nil {
+		t.Fatalf("unable to add fwd pkg: %v", err)
+	}
+
+	// There should now be one fwdpkg on disk. Since no forwarding decision
+	// has been written, we expect it to be FwdStateLockedIn. The package
+	// has unacked add HTLCs, so the ack filter should not be full.
+	fwdPkgs = loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 1 {
+		t.Fatalf("expected 1 fwdpkg, instead found %d", len(fwdPkgs))
+	}
+	assertFwdPkgState(t, fwdPkgs[0], channeldb.FwdStateLockedIn)
+	assertFwdPkgNumAddsSettleFails(t, fwdPkgs[0], nAdds, 0)
+	assertAckFilterIsFull(t, fwdPkgs[0], false)
+
+	// Now, write the forwarding decision. Since we have not explicitly
+	// added any adds to the fwdfilter, this would indicate that all of the
+	// adds were 1) settled locally by this link (exit hop), or 2) the htlc
+	// was failed locally.
+	if err := db.Update(func(tx *bbolt.Tx) error {
+		return packager.SetFwdFilter(tx, fwdPkg.Height, fwdPkg.FwdFilter)
+	}); err != nil {
+		t.Fatalf("unable to set fwdfiter: %v", err)
+	}
+
+	for i := range adds {
+		// We should still have one package on disk. Since the forwarding
+		// decision has been written, it will minimally be in FwdStateProcessed.
+		// However not allf of the HTLCs have been acked, so should not
+		// have advanced further.
+		fwdPkgs = loadFwdPkgs(t, db, packager)
+		if len(fwdPkgs) != 1 {
+			t.Fatalf("expected 1 fwdpkg, instead found %d", len(fwdPkgs))
+		}
+		assertFwdPkgState(t, fwdPkgs[0], channeldb.FwdStateProcessed)
+		assertFwdPkgNumAddsSettleFails(t, fwdPkgs[0], nAdds, 0)
+		assertAckFilterIsFull(t, fwdPkgs[0], false)
+
+		addRef := channeldb.AddRef{
+			Height: fwdPkg.Height,
+			Index:  uint16(i),
+		}
+
+		if err := db.Update(func(tx *bbolt.Tx) error {
+			return packager.AckAddHtlcs(tx, addRef)
+		}); err != nil {
+			t.Fatalf("unable to ack add htlc: %v", err)
+		}
+	}
+
+	// We should still have one package on disk. Now that all adds have been
+	// acked, the ack filter should return true and the package should be
+	// FwdStateCompleted since there are no other settle/fail packets.
+	fwdPkgs = loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 1 {
+		t.Fatalf("expected 1 fwdpkg, instead found %d", len(fwdPkgs))
+	}
+	assertFwdPkgState(t, fwdPkgs[0], channeldb.FwdStateCompleted)
+	assertFwdPkgNumAddsSettleFails(t, fwdPkgs[0], nAdds, 0)
+	assertAckFilterIsFull(t, fwdPkgs[0], true)
+
+	// Lastly, remove the completed forwarding package from disk.
+	if err := db.Update(func(tx *bbolt.Tx) error {
+		return packager.RemovePkg(tx, fwdPkg.Height)
+	}); err != nil {
+		t.Fatalf("unable to remove fwdpkg: %v", err)
+	}
+
+	// Check that the fwd package was actually removed.
+	fwdPkgs = loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 0 {
+		t.Fatalf("no forwarding packages should exist, found %d", len(fwdPkgs))
+	}
+}
+
+// TestPackagerOnlySettleFails asserts that the fwdpkg remains in
+// FwdStateProcessed after writing the forwarding decision when there are no
+// adds in the fwdpkg. We expect this because an empty FwdFilter will always
+// return true, but we are still waiting for the remaining fails and settles to
+// be deleted.
+func TestPackagerOnlySettleFails(t *testing.T) {
+	t.Parallel()
+
+	db := makeFwdPkgDB(t, "")
+
+	shortChanID := lnwire.NewShortChanIDFromInt(1)
+	packager := channeldb.NewChannelPackager(shortChanID)
+
+	// To begin, there should be no forwarding packages on disk.
+	fwdPkgs := loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 0 {
+		t.Fatalf("no forwarding packages should exist, found %d", len(fwdPkgs))
+	}
+
+	// Next, create and write a new forwarding package that only has add
+	// htlcs.
+	fwdPkg := channeldb.NewFwdPkg(shortChanID, 0, nil, settleFails)
+
+	nSettleFails := len(settleFails)
+
+	if err := db.Update(func(tx *bbolt.Tx) error {
+		return packager.AddFwdPkg(tx, fwdPkg)
+	}); err != nil {
+		t.Fatalf("unable to add fwd pkg: %v", err)
+	}
+
+	// There should now be one fwdpkg on disk. Since no forwarding decision
+	// has been written, we expect it to be FwdStateLockedIn. The package
+	// has unacked add HTLCs, so the ack filter should not be full.
+	fwdPkgs = loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 1 {
+		t.Fatalf("expected 1 fwdpkg, instead found %d", len(fwdPkgs))
+	}
+	assertFwdPkgState(t, fwdPkgs[0], channeldb.FwdStateLockedIn)
+	assertFwdPkgNumAddsSettleFails(t, fwdPkgs[0], 0, nSettleFails)
+	assertAckFilterIsFull(t, fwdPkgs[0], true)
+
+	// Now, write the forwarding decision. Since we have not explicitly
+	// added any adds to the fwdfilter, this would indicate that all of the
+	// adds were 1) settled locally by this link (exit hop), or 2) the htlc
+	// was failed locally.
+	if err := db.Update(func(tx *bbolt.Tx) error {
+		return packager.SetFwdFilter(tx, fwdPkg.Height, fwdPkg.FwdFilter)
+	}); err != nil {
+		t.Fatalf("unable to set fwdfiter: %v", err)
+	}
+
+	for i := range settleFails {
+		// We should still have one package on disk. Since the
+		// forwarding decision has been written, it will minimally be in
+		// FwdStateProcessed. However, not all of the HTLCs have been
+		// acked, so should not have advanced further.
+		fwdPkgs = loadFwdPkgs(t, db, packager)
+		if len(fwdPkgs) != 1 {
+			t.Fatalf("expected 1 fwdpkg, instead found %d", len(fwdPkgs))
+		}
+		assertFwdPkgState(t, fwdPkgs[0], channeldb.FwdStateProcessed)
+		assertFwdPkgNumAddsSettleFails(t, fwdPkgs[0], 0, nSettleFails)
+		assertSettleFailFilterIsFull(t, fwdPkgs[0], false)
+		assertAckFilterIsFull(t, fwdPkgs[0], true)
+
+		failSettleRef := channeldb.SettleFailRef{
+			Source: shortChanID,
+			Height: fwdPkg.Height,
+			Index:  uint16(i),
+		}
+
+		if err := db.Update(func(tx *bbolt.Tx) error {
+			return packager.AckSettleFails(tx, failSettleRef)
+		}); err != nil {
+			t.Fatalf("unable to ack add htlc: %v", err)
+		}
+	}
+
+	// We should still have one package on disk. Now that all settles and
+	// fails have been removed, package should be FwdStateCompleted since
+	// there are no other add packets.
+	fwdPkgs = loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 1 {
+		t.Fatalf("expected 1 fwdpkg, instead found %d", len(fwdPkgs))
+	}
+	assertFwdPkgState(t, fwdPkgs[0], channeldb.FwdStateCompleted)
+	assertFwdPkgNumAddsSettleFails(t, fwdPkgs[0], 0, nSettleFails)
+	assertSettleFailFilterIsFull(t, fwdPkgs[0], true)
+	assertAckFilterIsFull(t, fwdPkgs[0], true)
+
+	// Lastly, remove the completed forwarding package from disk.
+	if err := db.Update(func(tx *bbolt.Tx) error {
+		return packager.RemovePkg(tx, fwdPkg.Height)
+	}); err != nil {
+		t.Fatalf("unable to remove fwdpkg: %v", err)
+	}
+
+	// Check that the fwd package was actually removed.
+	fwdPkgs = loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 0 {
+		t.Fatalf("no forwarding packages should exist, found %d", len(fwdPkgs))
+	}
+}
+
+// TestPackagerAddsThenSettleFails writes a fwdpkg containing both adds and
+// settle/fails, then checks the behavior when the adds are acked before any of
+// the settle fails. Here we expect pkg to remain in FwdStateProcessed while the
+// remainder of the fail/settles are being deleted.
+func TestPackagerAddsThenSettleFails(t *testing.T) {
+	t.Parallel()
+
+	db := makeFwdPkgDB(t, "")
+
+	shortChanID := lnwire.NewShortChanIDFromInt(1)
+	packager := channeldb.NewChannelPackager(shortChanID)
+
+	// To begin, there should be no forwarding packages on disk.
+	fwdPkgs := loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 0 {
+		t.Fatalf("no forwarding packages should exist, found %d", len(fwdPkgs))
+	}
+
+	// Next, create and write a new forwarding package that only has add
+	// htlcs.
+	fwdPkg := channeldb.NewFwdPkg(shortChanID, 0, adds, settleFails)
+
+	nAdds := len(adds)
+	nSettleFails := len(settleFails)
+
+	if err := db.Update(func(tx *bbolt.Tx) error {
+		return packager.AddFwdPkg(tx, fwdPkg)
+	}); err != nil {
+		t.Fatalf("unable to add fwd pkg: %v", err)
+	}
+
+	// There should now be one fwdpkg on disk. Since no forwarding decision
+	// has been written, we expect it to be FwdStateLockedIn. The package
+	// has unacked add HTLCs, so the ack filter should not be full.
+	fwdPkgs = loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 1 {
+		t.Fatalf("expected 1 fwdpkg, instead found %d", len(fwdPkgs))
+	}
+	assertFwdPkgState(t, fwdPkgs[0], channeldb.FwdStateLockedIn)
+	assertFwdPkgNumAddsSettleFails(t, fwdPkgs[0], nAdds, nSettleFails)
+	assertAckFilterIsFull(t, fwdPkgs[0], false)
+
+	// Now, write the forwarding decision. Since we have not explicitly
+	// added any adds to the fwdfilter, this would indicate that all of the
+	// adds were 1) settled locally by this link (exit hop), or 2) the htlc
+	// was failed locally.
+	if err := db.Update(func(tx *bbolt.Tx) error {
+		return packager.SetFwdFilter(tx, fwdPkg.Height, fwdPkg.FwdFilter)
+	}); err != nil {
+		t.Fatalf("unable to set fwdfiter: %v", err)
+	}
+
+	for i := range adds {
+		// We should still have one package on disk. Since the forwarding
+		// decision has been written, it will minimally be in FwdStateProcessed.
+		// However not allf of the HTLCs have been acked, so should not
+		// have advanced further.
+		fwdPkgs = loadFwdPkgs(t, db, packager)
+		if len(fwdPkgs) != 1 {
+			t.Fatalf("expected 1 fwdpkg, instead found %d", len(fwdPkgs))
+		}
+		assertFwdPkgState(t, fwdPkgs[0], channeldb.FwdStateProcessed)
+		assertFwdPkgNumAddsSettleFails(t, fwdPkgs[0], nAdds, nSettleFails)
+		assertSettleFailFilterIsFull(t, fwdPkgs[0], false)
+		assertAckFilterIsFull(t, fwdPkgs[0], false)
+
+		addRef := channeldb.AddRef{
+			Height: fwdPkg.Height,
+			Index:  uint16(i),
+		}
+
+		if err := db.Update(func(tx *bbolt.Tx) error {
+			return packager.AckAddHtlcs(tx, addRef)
+		}); err != nil {
+			t.Fatalf("unable to ack add htlc: %v", err)
+		}
+	}
+
+	for i := range settleFails {
+		// We should still have one package on disk. Since the
+		// forwarding decision has been written, it will minimally be in
+		// FwdStateProcessed.  However not allf of the HTLCs have been
+		// acked, so should not have advanced further.
+		fwdPkgs = loadFwdPkgs(t, db, packager)
+		if len(fwdPkgs) != 1 {
+			t.Fatalf("expected 1 fwdpkg, instead found %d", len(fwdPkgs))
+		}
+		assertFwdPkgState(t, fwdPkgs[0], channeldb.FwdStateProcessed)
+		assertFwdPkgNumAddsSettleFails(t, fwdPkgs[0], nAdds, nSettleFails)
+		assertSettleFailFilterIsFull(t, fwdPkgs[0], false)
+		assertAckFilterIsFull(t, fwdPkgs[0], true)
+
+		failSettleRef := channeldb.SettleFailRef{
+			Source: shortChanID,
+			Height: fwdPkg.Height,
+			Index:  uint16(i),
+		}
+
+		if err := db.Update(func(tx *bbolt.Tx) error {
+			return packager.AckSettleFails(tx, failSettleRef)
+		}); err != nil {
+			t.Fatalf("unable to remove settle/fail htlc: %v", err)
+		}
+	}
+
+	// We should still have one package on disk. Now that all settles and
+	// fails have been removed, package should be FwdStateCompleted since
+	// there are no other add packets.
+	fwdPkgs = loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 1 {
+		t.Fatalf("expected 1 fwdpkg, instead found %d", len(fwdPkgs))
+	}
+	assertFwdPkgState(t, fwdPkgs[0], channeldb.FwdStateCompleted)
+	assertFwdPkgNumAddsSettleFails(t, fwdPkgs[0], nAdds, nSettleFails)
+	assertSettleFailFilterIsFull(t, fwdPkgs[0], true)
+	assertAckFilterIsFull(t, fwdPkgs[0], true)
+
+	// Lastly, remove the completed forwarding package from disk.
+	if err := db.Update(func(tx *bbolt.Tx) error {
+		return packager.RemovePkg(tx, fwdPkg.Height)
+	}); err != nil {
+		t.Fatalf("unable to remove fwdpkg: %v", err)
+	}
+
+	// Check that the fwd package was actually removed.
+	fwdPkgs = loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 0 {
+		t.Fatalf("no forwarding packages should exist, found %d", len(fwdPkgs))
+	}
+}
+
+// TestPackagerSettleFailsThenAdds writes a fwdpkg with both adds and
+// settle/fails, then checks the behavior when the settle/fails are removed
+// before any of the adds have been acked. This should cause the fwdpkg to
+// remain in FwdStateProcessed until the final ack is recorded, at which point
+// it should be promoted directly to FwdStateCompleted.since all adds have been
+// removed.
+func TestPackagerSettleFailsThenAdds(t *testing.T) {
+	t.Parallel()
+
+	db := makeFwdPkgDB(t, "")
+
+	shortChanID := lnwire.NewShortChanIDFromInt(1)
+	packager := channeldb.NewChannelPackager(shortChanID)
+
+	// To begin, there should be no forwarding packages on disk.
+	fwdPkgs := loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 0 {
+		t.Fatalf("no forwarding packages should exist, found %d", len(fwdPkgs))
+	}
+
+	// Next, create and write a new forwarding package that has both add
+	// and settle/fail htlcs.
+	fwdPkg := channeldb.NewFwdPkg(shortChanID, 0, adds, settleFails)
+
+	nAdds := len(adds)
+	nSettleFails := len(settleFails)
+
+	if err := db.Update(func(tx *bbolt.Tx) error {
+		return packager.AddFwdPkg(tx, fwdPkg)
+	}); err != nil {
+		t.Fatalf("unable to add fwd pkg: %v", err)
+	}
+
+	// There should now be one fwdpkg on disk. Since no forwarding decision
+	// has been written, we expect it to be FwdStateLockedIn. The package
+	// has unacked add HTLCs, so the ack filter should not be full.
+	fwdPkgs = loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 1 {
+		t.Fatalf("expected 1 fwdpkg, instead found %d", len(fwdPkgs))
+	}
+	assertFwdPkgState(t, fwdPkgs[0], channeldb.FwdStateLockedIn)
+	assertFwdPkgNumAddsSettleFails(t, fwdPkgs[0], nAdds, nSettleFails)
+	assertAckFilterIsFull(t, fwdPkgs[0], false)
+
+	// Now, write the forwarding decision. Since we have not explicitly
+	// added any adds to the fwdfilter, this would indicate that all of the
+	// adds were 1) settled locally by this link (exit hop), or 2) the htlc
+	// was failed locally.
+	if err := db.Update(func(tx *bbolt.Tx) error {
+		return packager.SetFwdFilter(tx, fwdPkg.Height, fwdPkg.FwdFilter)
+	}); err != nil {
+		t.Fatalf("unable to set fwdfiter: %v", err)
+	}
+
+	// Simulate another channel deleting the settle/fails it received from
+	// the original fwd pkg.
+	// TODO(conner): use different packager/s?
+	for i := range settleFails {
+		// We should still have one package on disk. Since the
+		// forwarding decision has been written, it will minimally be in
+		// FwdStateProcessed.  However none all of the add HTLCs have
+		// been acked, so should not have advanced further.
+		fwdPkgs = loadFwdPkgs(t, db, packager)
+		if len(fwdPkgs) != 1 {
+			t.Fatalf("expected 1 fwdpkg, instead found %d", len(fwdPkgs))
+		}
+		assertFwdPkgState(t, fwdPkgs[0], channeldb.FwdStateProcessed)
+		assertFwdPkgNumAddsSettleFails(t, fwdPkgs[0], nAdds, nSettleFails)
+		assertSettleFailFilterIsFull(t, fwdPkgs[0], false)
+		assertAckFilterIsFull(t, fwdPkgs[0], false)
+
+		failSettleRef := channeldb.SettleFailRef{
+			Source: shortChanID,
+			Height: fwdPkg.Height,
+			Index:  uint16(i),
+		}
+
+		if err := db.Update(func(tx *bbolt.Tx) error {
+			return packager.AckSettleFails(tx, failSettleRef)
+		}); err != nil {
+			t.Fatalf("unable to remove settle/fail htlc: %v", err)
+		}
+	}
+
+	// Now simulate this channel receiving a fail/settle for the adds in the
+	// fwdpkg.
+	for i := range adds {
+		// Again, we should still have one package on disk and be in
+		// FwdStateProcessed. This should not change until all of the
+		// add htlcs have been acked.
+		fwdPkgs = loadFwdPkgs(t, db, packager)
+		if len(fwdPkgs) != 1 {
+			t.Fatalf("expected 1 fwdpkg, instead found %d", len(fwdPkgs))
+		}
+		assertFwdPkgState(t, fwdPkgs[0], channeldb.FwdStateProcessed)
+		assertFwdPkgNumAddsSettleFails(t, fwdPkgs[0], nAdds, nSettleFails)
+		assertSettleFailFilterIsFull(t, fwdPkgs[0], true)
+		assertAckFilterIsFull(t, fwdPkgs[0], false)
+
+		addRef := channeldb.AddRef{
+			Height: fwdPkg.Height,
+			Index:  uint16(i),
+		}
+
+		if err := db.Update(func(tx *bbolt.Tx) error {
+			return packager.AckAddHtlcs(tx, addRef)
+		}); err != nil {
+			t.Fatalf("unable to ack add htlc: %v", err)
+		}
+	}
+
+	// We should still have one package on disk. Now that all settles and
+	// fails have been removed, package should be FwdStateCompleted since
+	// there are no other add packets.
+	fwdPkgs = loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 1 {
+		t.Fatalf("expected 1 fwdpkg, instead found %d", len(fwdPkgs))
+	}
+	assertFwdPkgState(t, fwdPkgs[0], channeldb.FwdStateCompleted)
+	assertFwdPkgNumAddsSettleFails(t, fwdPkgs[0], nAdds, nSettleFails)
+	assertSettleFailFilterIsFull(t, fwdPkgs[0], true)
+	assertAckFilterIsFull(t, fwdPkgs[0], true)
+
+	// Lastly, remove the completed forwarding package from disk.
+	if err := db.Update(func(tx *bbolt.Tx) error {
+		return packager.RemovePkg(tx, fwdPkg.Height)
+	}); err != nil {
+		t.Fatalf("unable to remove fwdpkg: %v", err)
+	}
+
+	// Check that the fwd package was actually removed.
+	fwdPkgs = loadFwdPkgs(t, db, packager)
+	if len(fwdPkgs) != 0 {
+		t.Fatalf("no forwarding packages should exist, found %d", len(fwdPkgs))
+	}
+}
+
+// assertFwdPkgState checks the current state of a fwdpkg meets our
+// expectations.
+func assertFwdPkgState(t *testing.T, fwdPkg *channeldb.FwdPkg,
+	state channeldb.FwdState) {
+	_, _, line, _ := runtime.Caller(1)
+	if fwdPkg.State != state {
+		t.Fatalf("line %d: expected fwdpkg in state %v, found %v",
+			line, state, fwdPkg.State)
+	}
+}
+
+// assertFwdPkgNumAddsSettleFails checks that the number of adds and
+// settle/fail log updates are correct.
+func assertFwdPkgNumAddsSettleFails(t *testing.T, fwdPkg *channeldb.FwdPkg,
+	expectedNumAdds, expectedNumSettleFails int) {
+	_, _, line, _ := runtime.Caller(1)
+	if len(fwdPkg.Adds) != expectedNumAdds {
+		t.Fatalf("line %d: expected fwdpkg to have %d adds, found %d",
+			line, expectedNumAdds, len(fwdPkg.Adds))
+	}
+
+	if len(fwdPkg.SettleFails) != expectedNumSettleFails {
+		t.Fatalf("line %d: expected fwdpkg to have %d settle/fails, found %d",
+			line, expectedNumSettleFails, len(fwdPkg.SettleFails))
+	}
+}
+
+// assertAckFilterIsFull checks whether or not a fwdpkg's ack filter matches our
+// expected full-ness.
+func assertAckFilterIsFull(t *testing.T, fwdPkg *channeldb.FwdPkg, expected bool) {
+	_, _, line, _ := runtime.Caller(1)
+	if fwdPkg.AckFilter.IsFull() != expected {
+		t.Fatalf("line %d: expected fwdpkg ack filter IsFull to be %v, "+
+			"found %v", line, expected, fwdPkg.AckFilter.IsFull())
+	}
+}
+
+// assertSettleFailFilterIsFull checks whether or not a fwdpkg's settle fail
+// filter matches our expected full-ness.
+func assertSettleFailFilterIsFull(t *testing.T, fwdPkg *channeldb.FwdPkg, expected bool) {
+	_, _, line, _ := runtime.Caller(1)
+	if fwdPkg.SettleFailFilter.IsFull() != expected {
+		t.Fatalf("line %d: expected fwdpkg settle/fail filter IsFull to be %v, "+
+			"found %v", line, expected, fwdPkg.SettleFailFilter.IsFull())
+	}
+}
+
+// loadFwdPkgs is a helper method that reads all forwarding packages for a
+// particular packager.
+func loadFwdPkgs(t *testing.T, db *bbolt.DB,
+	packager channeldb.FwdPackager) []*channeldb.FwdPkg {
+
+	var fwdPkgs []*channeldb.FwdPkg
+	if err := db.View(func(tx *bbolt.Tx) error {
+		var err error
+		fwdPkgs, err = packager.LoadFwdPkgs(tx)
+		return err
+	}); err != nil {
+		t.Fatalf("unable to load fwd pkgs: %v", err)
+	}
+
+	return fwdPkgs
+}
+
+// makeFwdPkgDB initializes a test database for forwarding packages. If the
+// provided path is an empty, it will create a temp dir/file to use.
+func makeFwdPkgDB(t *testing.T, path string) *bbolt.DB {
+	if path == "" {
+		var err error
+		path, err = ioutil.TempDir("", "fwdpkgdb")
+		if err != nil {
+			t.Fatalf("unable to create temp path: %v", err)
+		}
+
+		path = filepath.Join(path, "fwdpkg.db")
+	}
+
+	db, err := bbolt.Open(path, 0600, nil)
+	if err != nil {
+		t.Fatalf("unable to open boltdb: %v", err)
+	}
+
+	return db
+}

channeldb/migration_01_to_11/invoice_test.go

@@ -0,0 +1,694 @@
+package migration_01_to_11
+
+import (
+	"crypto/rand"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/lightningnetwork/lnd/lnwire"
+)
+
+func randInvoice(value lnwire.MilliSatoshi) (*Invoice, error) {
+	var pre [32]byte
+	if _, err := rand.Read(pre[:]); err != nil {
+		return nil, err
+	}
+
+	i := &Invoice{
+		// Use single second precision to avoid false positive test
+		// failures due to the monotonic time component.
+		CreationDate: time.Unix(time.Now().Unix(), 0),
+		Terms: ContractTerm{
+			PaymentPreimage: pre,
+			Value:           value,
+		},
+		Htlcs:  map[CircuitKey]*InvoiceHTLC{},
+		Expiry: 4000,
+	}
+	i.Memo = []byte("memo")
+	i.Receipt = []byte("receipt")
+
+	// Create a random byte slice of MaxPaymentRequestSize bytes to be used
+	// as a dummy paymentrequest, and  determine if it should be set based
+	// on one of the random bytes.
+	var r [MaxPaymentRequestSize]byte
+	if _, err := rand.Read(r[:]); err != nil {
+		return nil, err
+	}
+	if r[0]&1 == 0 {
+		i.PaymentRequest = r[:]
+	} else {
+		i.PaymentRequest = []byte("")
+	}
+
+	return i, nil
+}
+
+func TestInvoiceWorkflow(t *testing.T) {
+	t.Parallel()
+
+	db, cleanUp, err := makeTestDB()
+	defer cleanUp()
+	if err != nil {
+		t.Fatalf("unable to make test db: %v", err)
+	}
+
+	// Create a fake invoice which we'll use several times in the tests
+	// below.
+	fakeInvoice := &Invoice{
+		// Use single second precision to avoid false positive test
+		// failures due to the monotonic time component.
+		CreationDate: time.Unix(time.Now().Unix(), 0),
+		Htlcs:        map[CircuitKey]*InvoiceHTLC{},
+	}
+	fakeInvoice.Memo = []byte("memo")
+	fakeInvoice.Receipt = []byte("receipt")
+	fakeInvoice.PaymentRequest = []byte("")
+	copy(fakeInvoice.Terms.PaymentPreimage[:], rev[:])
+	fakeInvoice.Terms.Value = lnwire.NewMSatFromSatoshis(10000)
+
+	paymentHash := fakeInvoice.Terms.PaymentPreimage.Hash()
+
+	// Add the invoice to the database, this should succeed as there aren't
+	// any existing invoices within the database with the same payment
+	// hash.
+	if _, err := db.AddInvoice(fakeInvoice, paymentHash); err != nil {
+		t.Fatalf("unable to find invoice: %v", err)
+	}
+
+	// Attempt to retrieve the invoice which was just added to the
+	// database. It should be found, and the invoice returned should be
+	// identical to the one created above.
+	dbInvoice, err := db.LookupInvoice(paymentHash)
+	if err != nil {
+		t.Fatalf("unable to find invoice: %v", err)
+	}
+	if !reflect.DeepEqual(*fakeInvoice, dbInvoice) {
+		t.Fatalf("invoice fetched from db doesn't match original %v vs %v",
+			spew.Sdump(fakeInvoice), spew.Sdump(dbInvoice))
+	}
+
+	// The add index of the invoice retrieved from the database should now
+	// be fully populated. As this is the first index written to the DB,
+	// the addIndex should be 1.
+	if dbInvoice.AddIndex != 1 {
+		t.Fatalf("wrong add index: expected %v, got %v", 1,
+			dbInvoice.AddIndex)
+	}
+
+	// Settle the invoice, the version retrieved from the database should
+	// now have the settled bit toggle to true and a non-default
+	// SettledDate
+	payAmt := fakeInvoice.Terms.Value * 2
+	_, err = db.UpdateInvoice(paymentHash, getUpdateInvoice(payAmt))
+	if err != nil {
+		t.Fatalf("unable to settle invoice: %v", err)
+	}
+	dbInvoice2, err := db.LookupInvoice(paymentHash)
+	if err != nil {
+		t.Fatalf("unable to fetch invoice: %v", err)
+	}
+	if dbInvoice2.Terms.State != ContractSettled {
+		t.Fatalf("invoice should now be settled but isn't")
+	}
+	if dbInvoice2.SettleDate.IsZero() {
+		t.Fatalf("invoice should have non-zero SettledDate but isn't")
+	}
+
+	// Our 2x payment should be reflected, and also the settle index of 1
+	// should also have been committed for this index.
+	if dbInvoice2.AmtPaid != payAmt {
+		t.Fatalf("wrong amt paid: expected %v, got %v", payAmt,
+			dbInvoice2.AmtPaid)
+	}
+	if dbInvoice2.SettleIndex != 1 {
+		t.Fatalf("wrong settle index: expected %v, got %v", 1,
+			dbInvoice2.SettleIndex)
+	}
+
+	// Attempt to insert generated above again, this should fail as
+	// duplicates are rejected by the processing logic.
+	if _, err := db.AddInvoice(fakeInvoice, paymentHash); err != ErrDuplicateInvoice {
+		t.Fatalf("invoice insertion should fail due to duplication, "+
+			"instead %v", err)
+	}
+
+	// Attempt to look up a non-existent invoice, this should also fail but
+	// with a "not found" error.
+	var fakeHash [32]byte
+	if _, err := db.LookupInvoice(fakeHash); err != ErrInvoiceNotFound {
+		t.Fatalf("lookup should have failed, instead %v", err)
+	}
+
+	// Add 10 random invoices.
+	const numInvoices = 10
+	amt := lnwire.NewMSatFromSatoshis(1000)
+	invoices := make([]*Invoice, numInvoices+1)
+	invoices[0] = &dbInvoice2
+	for i := 1; i < len(invoices)-1; i++ {
+		invoice, err := randInvoice(amt)
+		if err != nil {
+			t.Fatalf("unable to create invoice: %v", err)
+		}
+
+		hash := invoice.Terms.PaymentPreimage.Hash()
+		if _, err := db.AddInvoice(invoice, hash); err != nil {
+			t.Fatalf("unable to add invoice %v", err)
+		}
+
+		invoices[i] = invoice
+	}
+
+	// Perform a scan to collect all the active invoices.
+	dbInvoices, err := db.FetchAllInvoices(false)
+	if err != nil {
+		t.Fatalf("unable to fetch all invoices: %v", err)
+	}
+
+	// The retrieve list of invoices should be identical as since we're
+	// using big endian, the invoices should be retrieved in ascending
+	// order (and the primary key should be incremented with each
+	// insertion).
+	for i := 0; i < len(invoices)-1; i++ {
+		if !reflect.DeepEqual(*invoices[i], dbInvoices[i]) {
+			t.Fatalf("retrieved invoices don't match %v vs %v",
+				spew.Sdump(invoices[i]),
+				spew.Sdump(dbInvoices[i]))
+		}
+	}
+}
+
+// TestInvoiceTimeSeries tests that newly added invoices invoices, as well as
+// settled invoices are added to the database are properly placed in the add
+// add or settle index which serves as an event time series.
+func TestInvoiceAddTimeSeries(t *testing.T) {
+	t.Parallel()
+
+	db, cleanUp, err := makeTestDB()
+	defer cleanUp()
+	if err != nil {
+		t.Fatalf("unable to make test db: %v", err)
+	}
+
+	// We'll start off by creating 20 random invoices, and inserting them
+	// into the database.
+	const numInvoices = 20
+	amt := lnwire.NewMSatFromSatoshis(1000)
+	invoices := make([]Invoice, numInvoices)
+	for i := 0; i < len(invoices); i++ {
+		invoice, err := randInvoice(amt)
+		if err != nil {
+			t.Fatalf("unable to create invoice: %v", err)
+		}
+
+		paymentHash := invoice.Terms.PaymentPreimage.Hash()
+
+		if _, err := db.AddInvoice(invoice, paymentHash); err != nil {
+			t.Fatalf("unable to add invoice %v", err)
+		}
+
+		invoices[i] = *invoice
+	}
+
+	// With the invoices constructed, we'll now create a series of queries
+	// that we'll use to assert expected return values of
+	// InvoicesAddedSince.
+	addQueries := []struct {
+		sinceAddIndex uint64
+
+		resp []Invoice
+	}{
+		// If we specify a value of zero, we shouldn't get any invoices
+		// back.
+		{
+			sinceAddIndex: 0,
+		},
+
+		// If we specify a value well beyond the number of inserted
+		// invoices, we shouldn't get any invoices back.
+		{
+			sinceAddIndex: 99999999,
+		},
+
+		// Using an index of 1 should result in all values, but the
+		// first one being returned.
+		{
+			sinceAddIndex: 1,
+			resp:          invoices[1:],
+		},
+
+		// If we use an index of 10, then we should retrieve the
+		// reaming 10 invoices.
+		{
+			sinceAddIndex: 10,
+			resp:          invoices[10:],
+		},
+	}
+
+	for i, query := range addQueries {
+		resp, err := db.InvoicesAddedSince(query.sinceAddIndex)
+		if err != nil {
+			t.Fatalf("unable to query: %v", err)
+		}
+
+		if !reflect.DeepEqual(query.resp, resp) {
+			t.Fatalf("test #%v: expected %v, got %v", i,
+				spew.Sdump(query.resp), spew.Sdump(resp))
+		}
+	}
+
+	// We'll now only settle the latter half of each of those invoices.
+	for i := 10; i < len(invoices); i++ {
+		invoice := &invoices[i]
+
+		paymentHash := invoice.Terms.PaymentPreimage.Hash()
+
+		_, err := db.UpdateInvoice(
+			paymentHash, getUpdateInvoice(0),
+		)
+		if err != nil {
+			t.Fatalf("unable to settle invoice: %v", err)
+		}
+	}
+
+	invoices, err = db.FetchAllInvoices(false)
+	if err != nil {
+		t.Fatalf("unable to fetch invoices: %v", err)
+	}
+
+	// We'll slice off the first 10 invoices, as we only settled the last
+	// 10.
+	invoices = invoices[10:]
+
+	// We'll now prepare an additional set of queries to ensure the settle
+	// time series has properly been maintained in the database.
+	settleQueries := []struct {
+		sinceSettleIndex uint64
+
+		resp []Invoice
+	}{
+		// If we specify a value of zero, we shouldn't get any settled
+		// invoices back.
+		{
+			sinceSettleIndex: 0,
+		},
+
+		// If we specify a value well beyond the number of settled
+		// invoices, we shouldn't get any invoices back.
+		{
+			sinceSettleIndex: 99999999,
+		},
+
+		// Using an index of 1 should result in the final 10 invoices
+		// being returned, as we only settled those.
+		{
+			sinceSettleIndex: 1,
+			resp:             invoices[1:],
+		},
+	}
+
+	for i, query := range settleQueries {
+		resp, err := db.InvoicesSettledSince(query.sinceSettleIndex)
+		if err != nil {
+			t.Fatalf("unable to query: %v", err)
+		}
+
+		if !reflect.DeepEqual(query.resp, resp) {
+			t.Fatalf("test #%v: expected %v, got %v", i,
+				spew.Sdump(query.resp), spew.Sdump(resp))
+		}
+	}
+}
+
+// TestDuplicateSettleInvoice tests that if we add a new invoice and settle it
+// twice, then the second time we also receive the invoice that we settled as a
+// return argument.
+func TestDuplicateSettleInvoice(t *testing.T) {
+	t.Parallel()
+
+	db, cleanUp, err := makeTestDB()
+	defer cleanUp()
+	if err != nil {
+		t.Fatalf("unable to make test db: %v", err)
+	}
+	db.now = func() time.Time { return time.Unix(1, 0) }
+
+	// We'll start out by creating an invoice and writing it to the DB.
+	amt := lnwire.NewMSatFromSatoshis(1000)
+	invoice, err := randInvoice(amt)
+	if err != nil {
+		t.Fatalf("unable to create invoice: %v", err)
+	}
+
+	payHash := invoice.Terms.PaymentPreimage.Hash()
+
+	if _, err := db.AddInvoice(invoice, payHash); err != nil {
+		t.Fatalf("unable to add invoice %v", err)
+	}
+
+	// With the invoice in the DB, we'll now attempt to settle the invoice.
+	dbInvoice, err := db.UpdateInvoice(
+		payHash, getUpdateInvoice(amt),
+	)
+	if err != nil {
+		t.Fatalf("unable to settle invoice: %v", err)
+	}
+
+	// We'll update what we expect the settle invoice to be so that our
+	// comparison below has the correct assumption.
+	invoice.SettleIndex = 1
+	invoice.Terms.State = ContractSettled
+	invoice.AmtPaid = amt
+	invoice.SettleDate = dbInvoice.SettleDate
+	invoice.Htlcs = map[CircuitKey]*InvoiceHTLC{
+		{}: {
+			Amt:         amt,
+			AcceptTime:  time.Unix(1, 0),
+			ResolveTime: time.Unix(1, 0),
+			State:       HtlcStateSettled,
+		},
+	}
+
+	// We should get back the exact same invoice that we just inserted.
+	if !reflect.DeepEqual(dbInvoice, invoice) {
+		t.Fatalf("wrong invoice after settle, expected %v got %v",
+			spew.Sdump(invoice), spew.Sdump(dbInvoice))
+	}
+
+	// If we try to settle the invoice again, then we should get the very
+	// same invoice back, but with an error this time.
+	dbInvoice, err = db.UpdateInvoice(
+		payHash, getUpdateInvoice(amt),
+	)
+	if err != ErrInvoiceAlreadySettled {
+		t.Fatalf("expected ErrInvoiceAlreadySettled")
+	}
+
+	if dbInvoice == nil {
+		t.Fatalf("invoice from db is nil after settle!")
+	}
+
+	invoice.SettleDate = dbInvoice.SettleDate
+	if !reflect.DeepEqual(dbInvoice, invoice) {
+		t.Fatalf("wrong invoice after second settle, expected %v got %v",
+			spew.Sdump(invoice), spew.Sdump(dbInvoice))
+	}
+}
+
+// TestQueryInvoices ensures that we can properly query the invoice database for
+// invoices using different types of queries.
+func TestQueryInvoices(t *testing.T) {
+	t.Parallel()
+
+	db, cleanUp, err := makeTestDB()
+	defer cleanUp()
+	if err != nil {
+		t.Fatalf("unable to make test db: %v", err)
+	}
+
+	// To begin the test, we'll add 50 invoices to the database. We'll
+	// assume that the index of the invoice within the database is the same
+	// as the amount of the invoice itself.
+	const numInvoices = 50
+	for i := lnwire.MilliSatoshi(1); i <= numInvoices; i++ {
+		invoice, err := randInvoice(i)
+		if err != nil {
+			t.Fatalf("unable to create invoice: %v", err)
+		}
+
+		paymentHash := invoice.Terms.PaymentPreimage.Hash()
+
+		if _, err := db.AddInvoice(invoice, paymentHash); err != nil {
+			t.Fatalf("unable to add invoice: %v", err)
+		}
+
+		// We'll only settle half of all invoices created.
+		if i%2 == 0 {
+			_, err := db.UpdateInvoice(
+				paymentHash, getUpdateInvoice(i),
+			)
+			if err != nil {
+				t.Fatalf("unable to settle invoice: %v", err)
+			}
+		}
+	}
+
+	// We'll then retrieve the set of all invoices and pending invoices.
+	// This will serve useful when comparing the expected responses of the
+	// query with the actual ones.
+	invoices, err := db.FetchAllInvoices(false)
+	if err != nil {
+		t.Fatalf("unable to retrieve invoices: %v", err)
+	}
+	pendingInvoices, err := db.FetchAllInvoices(true)
+	if err != nil {
+		t.Fatalf("unable to retrieve pending invoices: %v", err)
+	}
+
+	// The test will consist of several queries along with their respective
+	// expected response. Each query response should match its expected one.
+	testCases := []struct {
+		query    InvoiceQuery
+		expected []Invoice
+	}{
+		// Fetch all invoices with a single query.
+		{
+			query: InvoiceQuery{
+				NumMaxInvoices: numInvoices,
+			},
+			expected: invoices,
+		},
+		// Fetch all invoices with a single query, reversed.
+		{
+			query: InvoiceQuery{
+				Reversed:       true,
+				NumMaxInvoices: numInvoices,
+			},
+			expected: invoices,
+		},
+		// Fetch the first 25 invoices.
+		{
+			query: InvoiceQuery{
+				NumMaxInvoices: numInvoices / 2,
+			},
+			expected: invoices[:numInvoices/2],
+		},
+		// Fetch the first 10 invoices, but this time iterating
+		// backwards.
+		{
+			query: InvoiceQuery{
+				IndexOffset:    11,
+				Reversed:       true,
+				NumMaxInvoices: numInvoices,
+			},
+			expected: invoices[:10],
+		},
+		// Fetch the last 40 invoices.
+		{
+			query: InvoiceQuery{
+				IndexOffset:    10,
+				NumMaxInvoices: numInvoices,
+			},
+			expected: invoices[10:],
+		},
+		// Fetch all but the first invoice.
+		{
+			query: InvoiceQuery{
+				IndexOffset:    1,
+				NumMaxInvoices: numInvoices,
+			},
+			expected: invoices[1:],
+		},
+		// Fetch one invoice, reversed, with index offset 3. This
+		// should give us the second invoice in the array.
+		{
+			query: InvoiceQuery{
+				IndexOffset:    3,
+				Reversed:       true,
+				NumMaxInvoices: 1,
+			},
+			expected: invoices[1:2],
+		},
+		// Same as above, at index 2.
+		{
+			query: InvoiceQuery{
+				IndexOffset:    2,
+				Reversed:       true,
+				NumMaxInvoices: 1,
+			},
+			expected: invoices[0:1],
+		},
+		// Fetch one invoice, at index 1, reversed. Since invoice#1 is
+		// the very first, there won't be any left in a reverse search,
+		// so we expect no invoices to be returned.
+		{
+			query: InvoiceQuery{
+				IndexOffset:    1,
+				Reversed:       true,
+				NumMaxInvoices: 1,
+			},
+			expected: nil,
+		},
+		// Same as above, but don't restrict the number of invoices to
+		// 1.
+		{
+			query: InvoiceQuery{
+				IndexOffset:    1,
+				Reversed:       true,
+				NumMaxInvoices: numInvoices,
+			},
+			expected: nil,
+		},
+		// Fetch one invoice, reversed, with no offset set. We expect
+		// the last invoice in the response.
+		{
+			query: InvoiceQuery{
+				Reversed:       true,
+				NumMaxInvoices: 1,
+			},
+			expected: invoices[numInvoices-1:],
+		},
+		// Fetch one invoice, reversed, the offset set at numInvoices+1.
+		// We expect this to return the last invoice.
+		{
+			query: InvoiceQuery{
+				IndexOffset:    numInvoices + 1,
+				Reversed:       true,
+				NumMaxInvoices: 1,
+			},
+			expected: invoices[numInvoices-1:],
+		},
+		// Same as above, at offset numInvoices.
+		{
+			query: InvoiceQuery{
+				IndexOffset:    numInvoices,
+				Reversed:       true,
+				NumMaxInvoices: 1,
+			},
+			expected: invoices[numInvoices-2 : numInvoices-1],
+		},
+		// Fetch one invoice, at no offset (same as offset 0). We
+		// expect the first invoice only in the response.
+		{
+			query: InvoiceQuery{
+				NumMaxInvoices: 1,
+			},
+			expected: invoices[:1],
+		},
+		// Same as above, at offset 1.
+		{
+			query: InvoiceQuery{
+				IndexOffset:    1,
+				NumMaxInvoices: 1,
+			},
+			expected: invoices[1:2],
+		},
+		// Same as above, at offset 2.
+		{
+			query: InvoiceQuery{
+				IndexOffset:    2,
+				NumMaxInvoices: 1,
+			},
+			expected: invoices[2:3],
+		},
+		// Same as above, at offset numInvoices-1. Expect the last
+		// invoice to be returned.
+		{
+			query: InvoiceQuery{
+				IndexOffset:    numInvoices - 1,
+				NumMaxInvoices: 1,
+			},
+			expected: invoices[numInvoices-1:],
+		},
+		// Same as above, at offset numInvoices. No invoices should be
+		// returned, as there are no invoices after this offset.
+		{
+			query: InvoiceQuery{
+				IndexOffset:    numInvoices,
+				NumMaxInvoices: 1,
+			},
+			expected: nil,
+		},
+		// Fetch all pending invoices with a single query.
+		{
+			query: InvoiceQuery{
+				PendingOnly:    true,
+				NumMaxInvoices: numInvoices,
+			},
+			expected: pendingInvoices,
+		},
+		// Fetch the first 12 pending invoices.
+		{
+			query: InvoiceQuery{
+				PendingOnly:    true,
+				NumMaxInvoices: numInvoices / 4,
+			},
+			expected: pendingInvoices[:len(pendingInvoices)/2],
+		},
+		// Fetch the first 5 pending invoices, but this time iterating
+		// backwards.
+		{
+			query: InvoiceQuery{
+				IndexOffset:    10,
+				PendingOnly:    true,
+				Reversed:       true,
+				NumMaxInvoices: numInvoices,
+			},
+			// Since we seek to the invoice with index 10 and
+			// iterate backwards, there should only be 5 pending
+			// invoices before it as every other invoice within the
+			// index is settled.
+			expected: pendingInvoices[:5],
+		},
+		// Fetch the last 15 invoices.
+		{
+			query: InvoiceQuery{
+				IndexOffset:    20,
+				PendingOnly:    true,
+				NumMaxInvoices: numInvoices,
+			},
+			// Since we seek to the invoice with index 20, there are
+			// 30 invoices left. From these 30, only 15 of them are
+			// still pending.
+			expected: pendingInvoices[len(pendingInvoices)-15:],
+		},
+	}
+
+	for i, testCase := range testCases {
+		response, err := db.QueryInvoices(testCase.query)
+		if err != nil {
+			t.Fatalf("unable to query invoice database: %v", err)
+		}
+
+		if !reflect.DeepEqual(response.Invoices, testCase.expected) {
+			t.Fatalf("test #%d: query returned incorrect set of "+
+				"invoices: expcted %v, got %v", i,
+				spew.Sdump(response.Invoices),
+				spew.Sdump(testCase.expected))
+		}
+	}
+}
+
+// getUpdateInvoice returns an invoice update callback that, when called,
+// settles the invoice with the given amount.
+func getUpdateInvoice(amt lnwire.MilliSatoshi) InvoiceUpdateCallback {
+	return func(invoice *Invoice) (*InvoiceUpdateDesc, error) {
+		if invoice.Terms.State == ContractSettled {
+			return nil, ErrInvoiceAlreadySettled
+		}
+
+		update := &InvoiceUpdateDesc{
+			Preimage: invoice.Terms.PaymentPreimage,
+			State:    ContractSettled,
+			Htlcs: map[CircuitKey]*HtlcAcceptDesc{
+				{}: {
+					Amt: amt,
+				},
+			},
+		}
+
+		return update, nil
+	}
+}

channeldb/migration_01_to_11/legacy_serialization.go

@@ -0,0 +1,55 @@
+package migration_01_to_11
+
+import (
+	"io"
+)
+
+// deserializeCloseChannelSummaryV6 reads the v6 database format for
+// ChannelCloseSummary.
+//
+// NOTE: deprecated, only for migration.
+func deserializeCloseChannelSummaryV6(r io.Reader) (*ChannelCloseSummary, error) {
+	c := &ChannelCloseSummary{}
+
+	err := ReadElements(r,
+		&c.ChanPoint, &c.ShortChanID, &c.ChainHash, &c.ClosingTXID,
+		&c.CloseHeight, &c.RemotePub, &c.Capacity, &c.SettledBalance,
+		&c.TimeLockedBalance, &c.CloseType, &c.IsPending,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	// We'll now check to see if the channel close summary was encoded with
+	// any of the additional optional fields.
+	err = ReadElements(r, &c.RemoteCurrentRevocation)
+	switch {
+	case err == io.EOF:
+		return c, nil
+
+	// If we got a non-eof error, then we know there's an actually issue.
+	// Otherwise, it may have been the case that this summary didn't have
+	// the set of optional fields.
+	case err != nil:
+		return nil, err
+	}
+
+	if err := readChanConfig(r, &c.LocalChanConfig); err != nil {
+		return nil, err
+	}
+
+	// Finally, we'll attempt to read the next unrevoked commitment point
+	// for the remote party. If we closed the channel before receiving a
+	// funding locked message, then this can be nil. As a result, we'll use
+	// the same technique to read the field, only if there's still data
+	// left in the buffer.
+	err = ReadElements(r, &c.RemoteNextRevocation)
+	if err != nil && err != io.EOF {
+		// If we got a non-eof error, then we know there's an actually
+		// issue. Otherwise, it may have been the case that this
+		// summary didn't have the set of optional fields.
+		return nil, err
+	}
+
+	return c, nil
+}

channeldb/migration_01_to_11/log.go

@@ -0,0 +1,28 @@
+package migration_01_to_11
+
+import (
+	"github.com/btcsuite/btclog"
+	"github.com/lightningnetwork/lnd/build"
+)
+
+// log is a logger that is initialized with no output filters.  This
+// means the package will not perform any logging by default until the caller
+// requests it.
+var log btclog.Logger
+
+func init() {
+	UseLogger(build.NewSubLogger("CHDB", nil))
+}
+
+// DisableLog disables all library log output.  Logging output is disabled
+// by default until UseLogger is called.
+func DisableLog() {
+	UseLogger(btclog.Disabled)
+}
+
+// UseLogger uses a specified Logger to output package logging info.
+// This should be used in preference to SetLogWriter if the caller is also
+// using btclog.
+func UseLogger(logger btclog.Logger) {
+	log = logger
+}

channeldb/migration_01_to_11/meta.go

@@ -0,0 +1,78 @@
+package migration_01_to_11
+
+import "github.com/coreos/bbolt"
+
+var (
+	// metaBucket stores all the meta information concerning the state of
+	// the database.
+	metaBucket = []byte("metadata")
+
+	// dbVersionKey is a boltdb key and it's used for storing/retrieving
+	// current database version.
+	dbVersionKey = []byte("dbp")
+)
+
+// Meta structure holds the database meta information.
+type Meta struct {
+	// DbVersionNumber is the current schema version of the database.
+	DbVersionNumber uint32
+}
+
+// FetchMeta fetches the meta data from boltdb and returns filled meta
+// structure.
+func (d *DB) FetchMeta(tx *bbolt.Tx) (*Meta, error) {
+	meta := &Meta{}
+
+	err := d.View(func(tx *bbolt.Tx) error {
+		return fetchMeta(meta, tx)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return meta, nil
+}
+
+// fetchMeta is an internal helper function used in order to allow callers to
+// re-use a database transaction. See the publicly exported FetchMeta method
+// for more information.
+func fetchMeta(meta *Meta, tx *bbolt.Tx) error {
+	metaBucket := tx.Bucket(metaBucket)
+	if metaBucket == nil {
+		return ErrMetaNotFound
+	}
+
+	data := metaBucket.Get(dbVersionKey)
+	if data == nil {
+		meta.DbVersionNumber = getLatestDBVersion(dbVersions)
+	} else {
+		meta.DbVersionNumber = byteOrder.Uint32(data)
+	}
+
+	return nil
+}
+
+// PutMeta writes the passed instance of the database met-data struct to disk.
+func (d *DB) PutMeta(meta *Meta) error {
+	return d.Update(func(tx *bbolt.Tx) error {
+		return putMeta(meta, tx)
+	})
+}
+
+// putMeta is an internal helper function used in order to allow callers to
+// re-use a database transaction. See the publicly exported PutMeta method for
+// more information.
+func putMeta(meta *Meta, tx *bbolt.Tx) error {
+	metaBucket, err := tx.CreateBucketIfNotExists(metaBucket)
+	if err != nil {
+		return err
+	}
+
+	return putDbVersion(metaBucket, meta)
+}
+
+func putDbVersion(metaBucket *bbolt.Bucket, meta *Meta) error {
+	scratch := make([]byte, 4)
+	byteOrder.PutUint32(scratch, meta.DbVersionNumber)
+	return metaBucket.Put(dbVersionKey, scratch)
+}

channeldb/migration_01_to_11/meta_test.go

@@ -0,0 +1,442 @@
+package migration_01_to_11
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	"github.com/coreos/bbolt"
+	"github.com/go-errors/errors"
+)
+
+// applyMigration is a helper test function that encapsulates the general steps
+// which are needed to properly check the result of applying migration function.
+func applyMigration(t *testing.T, beforeMigration, afterMigration func(d *DB),
+	migrationFunc migration, shouldFail bool) {
+
+	cdb, cleanUp, err := makeTestDB()
+	defer cleanUp()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a test node that will be our source node.
+	testNode, err := createTestVertex(cdb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	graph := cdb.ChannelGraph()
+	if err := graph.SetSourceNode(testNode); err != nil {
+		t.Fatal(err)
+	}
+
+	// beforeMigration usually used for populating the database
+	// with test data.
+	beforeMigration(cdb)
+
+	// Create test meta info with zero database version and put it on disk.
+	// Than creating the version list pretending that new version was added.
+	meta := &Meta{DbVersionNumber: 0}
+	if err := cdb.PutMeta(meta); err != nil {
+		t.Fatalf("unable to store meta data: %v", err)
+	}
+
+	versions := []version{
+		{
+			number:    0,
+			migration: nil,
+		},
+		{
+			number:    1,
+			migration: migrationFunc,
+		},
+	}
+
+	defer func() {
+		if r := recover(); r != nil {
+			err = errors.New(r)
+		}
+
+		if err == nil && shouldFail {
+			t.Fatal("error wasn't received on migration stage")
+		} else if err != nil && !shouldFail {
+			t.Fatalf("error was received on migration stage: %v", err)
+		}
+
+		// afterMigration usually used for checking the database state and
+		// throwing the error if something went wrong.
+		afterMigration(cdb)
+	}()
+
+	// Sync with the latest version - applying migration function.
+	err = cdb.syncVersions(versions)
+	if err != nil {
+		log.Error(err)
+	}
+}
+
+// TestVersionFetchPut checks the propernces of fetch/put methods
+// and also initialization of meta data in case if don't have any in
+// database.
+func TestVersionFetchPut(t *testing.T) {
+	t.Parallel()
+
+	db, cleanUp, err := makeTestDB()
+	defer cleanUp()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	meta, err := db.FetchMeta(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if meta.DbVersionNumber != getLatestDBVersion(dbVersions) {
+		t.Fatal("initialization of meta information wasn't performed")
+	}
+
+	newVersion := getLatestDBVersion(dbVersions) + 1
+	meta.DbVersionNumber = newVersion
+
+	if err := db.PutMeta(meta); err != nil {
+		t.Fatalf("update of meta failed %v", err)
+	}
+
+	meta, err = db.FetchMeta(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if meta.DbVersionNumber != newVersion {
+		t.Fatal("update of meta information wasn't performed")
+	}
+}
+
+// TestOrderOfMigrations checks that migrations are applied in proper order.
+func TestOrderOfMigrations(t *testing.T) {
+	t.Parallel()
+
+	appliedMigration := -1
+	versions := []version{
+		{0, nil},
+		{1, nil},
+		{2, func(tx *bbolt.Tx) error {
+			appliedMigration = 2
+			return nil
+		}},
+		{3, func(tx *bbolt.Tx) error {
+			appliedMigration = 3
+			return nil
+		}},
+	}
+
+	// Retrieve the migration that should be applied to db, as far as
+	// current version is 1, we skip zero and first versions.
+	migrations, _ := getMigrationsToApply(versions, 1)
+
+	if len(migrations) != 2 {
+		t.Fatal("incorrect number of migrations to apply")
+	}
+
+	// Apply first migration.
+	migrations[0](nil)
+
+	// Check that first migration corresponds to the second version.
+	if appliedMigration != 2 {
+		t.Fatal("incorrect order of applying migrations")
+	}
+
+	// Apply second migration.
+	migrations[1](nil)
+
+	// Check that second migration corresponds to the third version.
+	if appliedMigration != 3 {
+		t.Fatal("incorrect order of applying migrations")
+	}
+}
+
+// TestGlobalVersionList checks that there is no mistake in global version list
+// in terms of version ordering.
+func TestGlobalVersionList(t *testing.T) {
+	t.Parallel()
+
+	if dbVersions == nil {
+		t.Fatal("can't find versions list")
+	}
+
+	if len(dbVersions) == 0 {
+		t.Fatal("db versions list is empty")
+	}
+
+	prev := dbVersions[0].number
+	for i := 1; i < len(dbVersions); i++ {
+		version := dbVersions[i].number
+
+		if version == prev {
+			t.Fatal("duplicates db versions")
+		}
+		if version < prev {
+			t.Fatal("order of db versions is wrong")
+		}
+
+		prev = version
+	}
+}
+
+// TestMigrationWithPanic asserts that if migration logic panics, we will return
+// to the original state unaltered.
+func TestMigrationWithPanic(t *testing.T) {
+	t.Parallel()
+
+	bucketPrefix := []byte("somebucket")
+	keyPrefix := []byte("someprefix")
+	beforeMigration := []byte("beforemigration")
+	afterMigration := []byte("aftermigration")
+
+	beforeMigrationFunc := func(d *DB) {
+		// Insert data in database and in order then make sure that the
+		// key isn't changes in case of panic or fail.
+		d.Update(func(tx *bbolt.Tx) error {
+			bucket, err := tx.CreateBucketIfNotExists(bucketPrefix)
+			if err != nil {
+				return err
+			}
+
+			bucket.Put(keyPrefix, beforeMigration)
+			return nil
+		})
+	}
+
+	// Create migration function which changes the initially created data and
+	// throw the panic, in this case we pretending that something goes.
+	migrationWithPanic := func(tx *bbolt.Tx) error {
+		bucket, err := tx.CreateBucketIfNotExists(bucketPrefix)
+		if err != nil {
+			return err
+		}
+
+		bucket.Put(keyPrefix, afterMigration)
+		panic("panic!")
+	}
+
+	// Check that version of database and data wasn't changed.
+	afterMigrationFunc := func(d *DB) {
+		meta, err := d.FetchMeta(nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if meta.DbVersionNumber != 0 {
+			t.Fatal("migration panicked but version is changed")
+		}
+
+		err = d.Update(func(tx *bbolt.Tx) error {
+			bucket, err := tx.CreateBucketIfNotExists(bucketPrefix)
+			if err != nil {
+				return err
+			}
+
+			value := bucket.Get(keyPrefix)
+			if !bytes.Equal(value, beforeMigration) {
+				return errors.New("migration failed but data is " +
+					"changed")
+			}
+
+			return nil
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	applyMigration(t,
+		beforeMigrationFunc,
+		afterMigrationFunc,
+		migrationWithPanic,
+		true)
+}
+
+// TestMigrationWithFatal asserts that migrations which fail do not modify the
+// database.
+func TestMigrationWithFatal(t *testing.T) {
+	t.Parallel()
+
+	bucketPrefix := []byte("somebucket")
+	keyPrefix := []byte("someprefix")
+	beforeMigration := []byte("beforemigration")
+	afterMigration := []byte("aftermigration")
+
+	beforeMigrationFunc := func(d *DB) {
+		d.Update(func(tx *bbolt.Tx) error {
+			bucket, err := tx.CreateBucketIfNotExists(bucketPrefix)
+			if err != nil {
+				return err
+			}
+
+			bucket.Put(keyPrefix, beforeMigration)
+			return nil
+		})
+	}
+
+	// Create migration function which changes the initially created data and
+	// return the error, in this case we pretending that something goes
+	// wrong.
+	migrationWithFatal := func(tx *bbolt.Tx) error {
+		bucket, err := tx.CreateBucketIfNotExists(bucketPrefix)
+		if err != nil {
+			return err
+		}
+
+		bucket.Put(keyPrefix, afterMigration)
+		return errors.New("some error")
+	}
+
+	// Check that version of database and initial data wasn't changed.
+	afterMigrationFunc := func(d *DB) {
+		meta, err := d.FetchMeta(nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if meta.DbVersionNumber != 0 {
+			t.Fatal("migration failed but version is changed")
+		}
+
+		err = d.Update(func(tx *bbolt.Tx) error {
+			bucket, err := tx.CreateBucketIfNotExists(bucketPrefix)
+			if err != nil {
+				return err
+			}
+
+			value := bucket.Get(keyPrefix)
+			if !bytes.Equal(value, beforeMigration) {
+				return errors.New("migration failed but data is " +
+					"changed")
+			}
+
+			return nil
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	applyMigration(t,
+		beforeMigrationFunc,
+		afterMigrationFunc,
+		migrationWithFatal,
+		true)
+}
+
+// TestMigrationWithoutErrors asserts that a successful migration has its
+// changes applied to the database.
+func TestMigrationWithoutErrors(t *testing.T) {
+	t.Parallel()
+
+	bucketPrefix := []byte("somebucket")
+	keyPrefix := []byte("someprefix")
+	beforeMigration := []byte("beforemigration")
+	afterMigration := []byte("aftermigration")
+
+	// Populate database with initial data.
+	beforeMigrationFunc := func(d *DB) {
+		d.Update(func(tx *bbolt.Tx) error {
+			bucket, err := tx.CreateBucketIfNotExists(bucketPrefix)
+			if err != nil {
+				return err
+			}
+
+			bucket.Put(keyPrefix, beforeMigration)
+			return nil
+		})
+	}
+
+	// Create migration function which changes the initially created data.
+	migrationWithoutErrors := func(tx *bbolt.Tx) error {
+		bucket, err := tx.CreateBucketIfNotExists(bucketPrefix)
+		if err != nil {
+			return err
+		}
+
+		bucket.Put(keyPrefix, afterMigration)
+		return nil
+	}
+
+	// Check that version of database and data was properly changed.
+	afterMigrationFunc := func(d *DB) {
+		meta, err := d.FetchMeta(nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if meta.DbVersionNumber != 1 {
+			t.Fatal("version number isn't changed after " +
+				"successfully applied migration")
+		}
+
+		err = d.Update(func(tx *bbolt.Tx) error {
+			bucket, err := tx.CreateBucketIfNotExists(bucketPrefix)
+			if err != nil {
+				return err
+			}
+
+			value := bucket.Get(keyPrefix)
+			if !bytes.Equal(value, afterMigration) {
+				return errors.New("migration wasn't applied " +
+					"properly")
+			}
+
+			return nil
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	applyMigration(t,
+		beforeMigrationFunc,
+		afterMigrationFunc,
+		migrationWithoutErrors,
+		false)
+}
+
+// TestMigrationReversion tests after performing a migration to a higher
+// database version, opening the database with a lower latest db version returns
+// ErrDBReversion.
+func TestMigrationReversion(t *testing.T) {
+	t.Parallel()
+
+	tempDirName, err := ioutil.TempDir("", "channeldb")
+	if err != nil {
+		t.Fatalf("unable to create temp dir: %v", err)
+	}
+
+	cdb, err := Open(tempDirName)
+	if err != nil {
+		t.Fatalf("unable to open channeldb: %v", err)
+	}
+
+	// Update the database metadata to point to one more than the highest
+	// known version.
+	err = cdb.Update(func(tx *bbolt.Tx) error {
+		newMeta := &Meta{
+			DbVersionNumber: getLatestDBVersion(dbVersions) + 1,
+		}
+
+		return putMeta(newMeta, tx)
+	})
+
+	// Close the database. Even if we succeeded, our next step is to reopen.
+	cdb.Close()
+
+	if err != nil {
+		t.Fatalf("unable to increase db version: %v", err)
+	}
+
+	_, err = Open(tempDirName)
+	if err != ErrDBReversion {
+		t.Fatalf("unexpected error when opening channeldb, "+
+			"want: %v, got: %v", ErrDBReversion, err)
+	}
+}

channeldb/migration_01_to_11/migration_09_legacy_serialization.go

@@ -0,0 +1,497 @@
+package migration_01_to_11
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"sort"
+
+	"github.com/coreos/bbolt"
+	"github.com/lightningnetwork/lnd/lntypes"
+	"github.com/lightningnetwork/lnd/lnwire"
+	"github.com/lightningnetwork/lnd/routing/route"
+)
+
+var (
+	// paymentBucket is the name of the bucket within the database that
+	// stores all data related to payments.
+	//
+	// Within the payments bucket, each invoice is keyed by its invoice ID
+	// which is a monotonically increasing uint64.  BoltDB's sequence
+	// feature is used for generating monotonically increasing id.
+	//
+	// NOTE: Deprecated. Kept around for migration purposes.
+	paymentBucket = []byte("payments")
+
+	// paymentStatusBucket is the name of the bucket within the database
+	// that stores the status of a payment indexed by the payment's
+	// preimage.
+	//
+	// NOTE: Deprecated. Kept around for migration purposes.
+	paymentStatusBucket = []byte("payment-status")
+)
+
+// outgoingPayment represents a successful payment between the daemon and a
+// remote node. Details such as the total fee paid, and the time of the payment
+// are stored.
+//
+// NOTE: Deprecated. Kept around for migration purposes.
+type outgoingPayment struct {
+	Invoice
+
+	// Fee is the total fee paid for the payment in milli-satoshis.
+	Fee lnwire.MilliSatoshi
+
+	// TotalTimeLock is the total cumulative time-lock in the HTLC extended
+	// from the second-to-last hop to the destination.
+	TimeLockLength uint32
+
+	// Path encodes the path the payment took through the network. The path
+	// excludes the outgoing node and consists of the hex-encoded
+	// compressed public key of each of the nodes involved in the payment.
+	Path [][33]byte
+
+	// PaymentPreimage is the preImage of a successful payment. This is used
+	// to calculate the PaymentHash as well as serve as a proof of payment.
+	PaymentPreimage [32]byte
+}
+
+// addPayment saves a successful payment to the database. It is assumed that
+// all payment are sent using unique payment hashes.
+//
+// NOTE: Deprecated. Kept around for migration purposes.
+func (db *DB) addPayment(payment *outgoingPayment) error {
+	// Validate the field of the inner voice within the outgoing payment,
+	// these must also adhere to the same constraints as regular invoices.
+	if err := validateInvoice(&payment.Invoice); err != nil {
+		return err
+	}
+
+	// We first serialize the payment before starting the database
+	// transaction so we can avoid creating a DB payment in the case of a
+	// serialization error.
+	var b bytes.Buffer
+	if err := serializeOutgoingPayment(&b, payment); err != nil {
+		return err
+	}
+	paymentBytes := b.Bytes()
+
+	return db.Batch(func(tx *bbolt.Tx) error {
+		payments, err := tx.CreateBucketIfNotExists(paymentBucket)
+		if err != nil {
+			return err
+		}
+
+		// Obtain the new unique sequence number for this payment.
+		paymentID, err := payments.NextSequence()
+		if err != nil {
+			return err
+		}
+
+		// We use BigEndian for keys as it orders keys in
+		// ascending order. This allows bucket scans to order payments
+		// in the order in which they were created.
+		paymentIDBytes := make([]byte, 8)
+		binary.BigEndian.PutUint64(paymentIDBytes, paymentID)
+
+		return payments.Put(paymentIDBytes, paymentBytes)
+	})
+}
+
+// fetchAllPayments returns all outgoing payments in DB.
+//
+// NOTE: Deprecated. Kept around for migration purposes.
+func (db *DB) fetchAllPayments() ([]*outgoingPayment, error) {
+	var payments []*outgoingPayment
+
+	err := db.View(func(tx *bbolt.Tx) error {
+		bucket := tx.Bucket(paymentBucket)
+		if bucket == nil {
+			return ErrNoPaymentsCreated
+		}
+
+		return bucket.ForEach(func(k, v []byte) error {
+			// If the value is nil, then we ignore it as it may be
+			// a sub-bucket.
+			if v == nil {
+				return nil
+			}
+
+			r := bytes.NewReader(v)
+			payment, err := deserializeOutgoingPayment(r)
+			if err != nil {
+				return err
+			}
+
+			payments = append(payments, payment)
+			return nil
+		})
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return payments, nil
+}
+
+// fetchPaymentStatus returns the payment status for outgoing payment.
+// If status of the payment isn't found, it will default to "StatusUnknown".
+//
+// NOTE: Deprecated. Kept around for migration purposes.
+func (db *DB) fetchPaymentStatus(paymentHash [32]byte) (PaymentStatus, error) {
+	var paymentStatus = StatusUnknown
+	err := db.View(func(tx *bbolt.Tx) error {
+		var err error
+		paymentStatus, err = fetchPaymentStatusTx(tx, paymentHash)
+		return err
+	})
+	if err != nil {
+		return StatusUnknown, err
+	}
+
+	return paymentStatus, nil
+}
+
+// fetchPaymentStatusTx is a helper method that returns the payment status for
+// outgoing payment.  If status of the payment isn't found, it will default to
+// "StatusUnknown". It accepts the boltdb transactions such that this method
+// can be composed into other atomic operations.
+//
+// NOTE: Deprecated. Kept around for migration purposes.
+func fetchPaymentStatusTx(tx *bbolt.Tx, paymentHash [32]byte) (PaymentStatus, error) {
+	// The default status for all payments that aren't recorded in database.
+	var paymentStatus = StatusUnknown
+
+	bucket := tx.Bucket(paymentStatusBucket)
+	if bucket == nil {
+		return paymentStatus, nil
+	}
+
+	paymentStatusBytes := bucket.Get(paymentHash[:])
+	if paymentStatusBytes == nil {
+		return paymentStatus, nil
+	}
+
+	paymentStatus.FromBytes(paymentStatusBytes)
+
+	return paymentStatus, nil
+}
+
+func serializeOutgoingPayment(w io.Writer, p *outgoingPayment) error {
+	var scratch [8]byte
+
+	if err := serializeInvoiceLegacy(w, &p.Invoice); err != nil {
+		return err
+	}
+
+	byteOrder.PutUint64(scratch[:], uint64(p.Fee))
+	if _, err := w.Write(scratch[:]); err != nil {
+		return err
+	}
+
+	// First write out the length of the bytes to prefix the value.
+	pathLen := uint32(len(p.Path))
+	byteOrder.PutUint32(scratch[:4], pathLen)
+	if _, err := w.Write(scratch[:4]); err != nil {
+		return err
+	}
+
+	// Then with the path written, we write out the series of public keys
+	// involved in the path.
+	for _, hop := range p.Path {
+		if _, err := w.Write(hop[:]); err != nil {
+			return err
+		}
+	}
+
+	byteOrder.PutUint32(scratch[:4], p.TimeLockLength)
+	if _, err := w.Write(scratch[:4]); err != nil {
+		return err
+	}
+
+	if _, err := w.Write(p.PaymentPreimage[:]); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func deserializeOutgoingPayment(r io.Reader) (*outgoingPayment, error) {
+	var scratch [8]byte
+
+	p := &outgoingPayment{}
+
+	inv, err := deserializeInvoiceLegacy(r)
+	if err != nil {
+		return nil, err
+	}
+	p.Invoice = inv
+
+	if _, err := r.Read(scratch[:]); err != nil {
+		return nil, err
+	}
+	p.Fee = lnwire.MilliSatoshi(byteOrder.Uint64(scratch[:]))
+
+	if _, err = r.Read(scratch[:4]); err != nil {
+		return nil, err
+	}
+	pathLen := byteOrder.Uint32(scratch[:4])
+
+	path := make([][33]byte, pathLen)
+	for i := uint32(0); i < pathLen; i++ {
+		if _, err := r.Read(path[i][:]); err != nil {
+			return nil, err
+		}
+	}
+	p.Path = path
+
+	if _, err = r.Read(scratch[:4]); err != nil {
+		return nil, err
+	}
+	p.TimeLockLength = byteOrder.Uint32(scratch[:4])
+
+	if _, err := r.Read(p.PaymentPreimage[:]); err != nil {
+		return nil, err
+	}
+
+	return p, nil
+}
+
+// serializePaymentAttemptInfoMigration9 is the serializePaymentAttemptInfo
+// version as existed when migration #9 was created. We keep this around, along
+// with the methods below to ensure that clients that upgrade will use the
+// correct version of this method.
+func serializePaymentAttemptInfoMigration9(w io.Writer, a *PaymentAttemptInfo) error {
+	if err := WriteElements(w, a.PaymentID, a.SessionKey); err != nil {
+		return err
+	}
+
+	if err := serializeRouteMigration9(w, a.Route); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func serializeHopMigration9(w io.Writer, h *route.Hop) error {
+	if err := WriteElements(w,
+		h.PubKeyBytes[:], h.ChannelID, h.OutgoingTimeLock,
+		h.AmtToForward,
+	); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func serializeRouteMigration9(w io.Writer, r route.Route) error {
+	if err := WriteElements(w,
+		r.TotalTimeLock, r.TotalAmount, r.SourcePubKey[:],
+	); err != nil {
+		return err
+	}
+
+	if err := WriteElements(w, uint32(len(r.Hops))); err != nil {
+		return err
+	}
+
+	for _, h := range r.Hops {
+		if err := serializeHopMigration9(w, h); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func deserializePaymentAttemptInfoMigration9(r io.Reader) (*PaymentAttemptInfo, error) {
+	a := &PaymentAttemptInfo{}
+	err := ReadElements(r, &a.PaymentID, &a.SessionKey)
+	if err != nil {
+		return nil, err
+	}
+	a.Route, err = deserializeRouteMigration9(r)
+	if err != nil {
+		return nil, err
+	}
+	return a, nil
+}
+
+func deserializeRouteMigration9(r io.Reader) (route.Route, error) {
+	rt := route.Route{}
+	if err := ReadElements(r,
+		&rt.TotalTimeLock, &rt.TotalAmount,
+	); err != nil {
+		return rt, err
+	}
+
+	var pub []byte
+	if err := ReadElements(r, &pub); err != nil {
+		return rt, err
+	}
+	copy(rt.SourcePubKey[:], pub)
+
+	var numHops uint32
+	if err := ReadElements(r, &numHops); err != nil {
+		return rt, err
+	}
+
+	var hops []*route.Hop
+	for i := uint32(0); i < numHops; i++ {
+		hop, err := deserializeHopMigration9(r)
+		if err != nil {
+			return rt, err
+		}
+		hops = append(hops, hop)
+	}
+	rt.Hops = hops
+
+	return rt, nil
+}
+
+func deserializeHopMigration9(r io.Reader) (*route.Hop, error) {
+	h := &route.Hop{}
+
+	var pub []byte
+	if err := ReadElements(r, &pub); err != nil {
+		return nil, err
+	}
+	copy(h.PubKeyBytes[:], pub)
+
+	if err := ReadElements(r,
+		&h.ChannelID, &h.OutgoingTimeLock, &h.AmtToForward,
+	); err != nil {
+		return nil, err
+	}
+
+	return h, nil
+}
+
+// fetchPaymentsMigration9 returns all sent payments found in the DB using the
+// payment attempt info format that was present as of migration #9. We need
+// this as otherwise, the current FetchPayments version will use the latest
+// decoding format. Note that we only need this for the
+// TestOutgoingPaymentsMigration migration test case.
+func (db *DB) fetchPaymentsMigration9() ([]*Payment, error) {
+	var payments []*Payment
+
+	err := db.View(func(tx *bbolt.Tx) error {
+		paymentsBucket := tx.Bucket(paymentsRootBucket)
+		if paymentsBucket == nil {
+			return nil
+		}
+
+		return paymentsBucket.ForEach(func(k, v []byte) error {
+			bucket := paymentsBucket.Bucket(k)
+			if bucket == nil {
+				// We only expect sub-buckets to be found in
+				// this top-level bucket.
+				return fmt.Errorf("non bucket element in " +
+					"payments bucket")
+			}
+
+			p, err := fetchPaymentMigration9(bucket)
+			if err != nil {
+				return err
+			}
+
+			payments = append(payments, p)
+
+			// For older versions of lnd, duplicate payments to a
+			// payment has was possible. These will be found in a
+			// sub-bucket indexed by their sequence number if
+			// available.
+			dup := bucket.Bucket(paymentDuplicateBucket)
+			if dup == nil {
+				return nil
+			}
+
+			return dup.ForEach(func(k, v []byte) error {
+				subBucket := dup.Bucket(k)
+				if subBucket == nil {
+					// We one bucket for each duplicate to
+					// be found.
+					return fmt.Errorf("non bucket element" +
+						"in duplicate bucket")
+				}
+
+				p, err := fetchPaymentMigration9(subBucket)
+				if err != nil {
+					return err
+				}
+
+				payments = append(payments, p)
+				return nil
+			})
+		})
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// Before returning, sort the payments by their sequence number.
+	sort.Slice(payments, func(i, j int) bool {
+		return payments[i].sequenceNum < payments[j].sequenceNum
+	})
+
+	return payments, nil
+}
+
+func fetchPaymentMigration9(bucket *bbolt.Bucket) (*Payment, error) {
+	var (
+		err error
+		p   = &Payment{}
+	)
+
+	seqBytes := bucket.Get(paymentSequenceKey)
+	if seqBytes == nil {
+		return nil, fmt.Errorf("sequence number not found")
+	}
+
+	p.sequenceNum = binary.BigEndian.Uint64(seqBytes)
+
+	// Get the payment status.
+	p.Status = fetchPaymentStatus(bucket)
+
+	// Get the PaymentCreationInfo.
+	b := bucket.Get(paymentCreationInfoKey)
+	if b == nil {
+		return nil, fmt.Errorf("creation info not found")
+	}
+
+	r := bytes.NewReader(b)
+	p.Info, err = deserializePaymentCreationInfo(r)
+	if err != nil {
+		return nil, err
+
+	}
+
+	// Get the PaymentAttemptInfo. This can be unset.
+	b = bucket.Get(paymentAttemptInfoKey)
+	if b != nil {
+		r = bytes.NewReader(b)
+		p.Attempt, err = deserializePaymentAttemptInfoMigration9(r)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Get the payment preimage. This is only found for
+	// completed payments.
+	b = bucket.Get(paymentSettleInfoKey)
+	if b != nil {
+		var preimg lntypes.Preimage
+		copy(preimg[:], b[:])
+		p.PaymentPreimage = &preimg
+	}
+
+	// Get failure reason if available.
+	b = bucket.Get(paymentFailInfoKey)
+	if b != nil {
+		reason := FailureReason(b[0])
+		p.Failure = &reason
+	}
+
+	return p, nil
+}

channeldb/migration_01_to_11/migration_10_route_tlv_records.go

@@ -0,0 +1,236 @@
+package migration_01_to_11
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/coreos/bbolt"
+	"github.com/lightningnetwork/lnd/routing/route"
+)
+
+// migrateRouteSerialization migrates the way we serialize routes across the
+// entire database. At the time of writing of this migration, this includes our
+// payment attempts, as well as the payment results in mission control.
+func migrateRouteSerialization(tx *bbolt.Tx) error {
+	// First, we'll do all the payment attempts.
+	rootPaymentBucket := tx.Bucket(paymentsRootBucket)
+	if rootPaymentBucket == nil {
+		return nil
+	}
+
+	// As we can't mutate a bucket while we're iterating over it with
+	// ForEach, we'll need to collect all the known payment hashes in
+	// memory first.
+	var payHashes [][]byte
+	err := rootPaymentBucket.ForEach(func(k, v []byte) error {
+		if v != nil {
+			return nil
+		}
+
+		payHashes = append(payHashes, k)
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	// Now that we have all the payment hashes, we can carry out the
+	// migration itself.
+	for _, payHash := range payHashes {
+		payHashBucket := rootPaymentBucket.Bucket(payHash)
+
+		// First, we'll migrate the main (non duplicate) payment to
+		// this hash.
+		err := migrateAttemptEncoding(tx, payHashBucket)
+		if err != nil {
+			return err
+		}
+
+		// Now that we've migrated the main payment, we'll also check
+		// for any duplicate payments to the same payment hash.
+		dupBucket := payHashBucket.Bucket(paymentDuplicateBucket)
+
+		// If there's no dup bucket, then we can move on to the next
+		// payment.
+		if dupBucket == nil {
+			continue
+		}
+
+		// Otherwise, we'll now iterate through all the duplicate pay
+		// hashes and migrate those.
+		var dupSeqNos [][]byte
+		err = dupBucket.ForEach(func(k, v []byte) error {
+			dupSeqNos = append(dupSeqNos, k)
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		// Now in this second pass, we'll re-serialize their duplicate
+		// payment attempts under the new encoding.
+		for _, seqNo := range dupSeqNos {
+			dupPayHashBucket := dupBucket.Bucket(seqNo)
+			err := migrateAttemptEncoding(tx, dupPayHashBucket)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	log.Infof("Migration of route/hop serialization complete!")
+
+	log.Infof("Migrating to new mission control store by clearing " +
+		"existing data")
+
+	resultsKey := []byte("missioncontrol-results")
+	err = tx.DeleteBucket(resultsKey)
+	if err != nil && err != bbolt.ErrBucketNotFound {
+		return err
+	}
+
+	log.Infof("Migration to new mission control completed!")
+
+	return nil
+}
+
+// migrateAttemptEncoding migrates payment attempts using the legacy format to
+// the new format.
+func migrateAttemptEncoding(tx *bbolt.Tx, payHashBucket *bbolt.Bucket) error {
+	payAttemptBytes := payHashBucket.Get(paymentAttemptInfoKey)
+	if payAttemptBytes == nil {
+		return nil
+	}
+
+	// For our migration, we'll first read out the existing payment attempt
+	// using the legacy serialization of the attempt.
+	payAttemptReader := bytes.NewReader(payAttemptBytes)
+	payAttempt, err := deserializePaymentAttemptInfoLegacy(
+		payAttemptReader,
+	)
+	if err != nil {
+		return err
+	}
+
+	// Now that we have the old attempts, we'll explicitly mark this as
+	// needing a legacy payload, since after this migration, the modern
+	// payload will be the default if signalled.
+	for _, hop := range payAttempt.Route.Hops {
+		hop.LegacyPayload = true
+	}
+
+	// Finally, we'll write out the payment attempt using the new encoding.
+	var b bytes.Buffer
+	err = serializePaymentAttemptInfo(&b, payAttempt)
+	if err != nil {
+		return err
+	}
+
+	return payHashBucket.Put(paymentAttemptInfoKey, b.Bytes())
+}
+
+func deserializePaymentAttemptInfoLegacy(r io.Reader) (*PaymentAttemptInfo, error) {
+	a := &PaymentAttemptInfo{}
+	err := ReadElements(r, &a.PaymentID, &a.SessionKey)
+	if err != nil {
+		return nil, err
+	}
+	a.Route, err = deserializeRouteLegacy(r)
+	if err != nil {
+		return nil, err
+	}
+	return a, nil
+}
+
+func serializePaymentAttemptInfoLegacy(w io.Writer, a *PaymentAttemptInfo) error {
+	if err := WriteElements(w, a.PaymentID, a.SessionKey); err != nil {
+		return err
+	}
+
+	if err := serializeRouteLegacy(w, a.Route); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func deserializeHopLegacy(r io.Reader) (*route.Hop, error) {
+	h := &route.Hop{}
+
+	var pub []byte
+	if err := ReadElements(r, &pub); err != nil {
+		return nil, err
+	}
+	copy(h.PubKeyBytes[:], pub)
+
+	if err := ReadElements(r,
+		&h.ChannelID, &h.OutgoingTimeLock, &h.AmtToForward,
+	); err != nil {
+		return nil, err
+	}
+
+	return h, nil
+}
+
+func serializeHopLegacy(w io.Writer, h *route.Hop) error {
+	if err := WriteElements(w,
+		h.PubKeyBytes[:], h.ChannelID, h.OutgoingTimeLock,
+		h.AmtToForward,
+	); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func deserializeRouteLegacy(r io.Reader) (route.Route, error) {
+	rt := route.Route{}
+	if err := ReadElements(r,
+		&rt.TotalTimeLock, &rt.TotalAmount,
+	); err != nil {
+		return rt, err
+	}
+
+	var pub []byte
+	if err := ReadElements(r, &pub); err != nil {
+		return rt, err
+	}
+	copy(rt.SourcePubKey[:], pub)
+
+	var numHops uint32
+	if err := ReadElements(r, &numHops); err != nil {
+		return rt, err
+	}
+
+	var hops []*route.Hop
+	for i := uint32(0); i < numHops; i++ {
+		hop, err := deserializeHopLegacy(r)
+		if err != nil {
+			return rt, err
+		}
+		hops = append(hops, hop)
+	}
+	rt.Hops = hops
+
+	return rt, nil
+}
+
+func serializeRouteLegacy(w io.Writer, r route.Route) error {
+	if err := WriteElements(w,
+		r.TotalTimeLock, r.TotalAmount, r.SourcePubKey[:],
+	); err != nil {
+		return err
+	}
+
+	if err := WriteElements(w, uint32(len(r.Hops))); err != nil {
+		return err
+	}
+
+	for _, h := range r.Hops {
+		if err := serializeHopLegacy(w, h); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

channeldb/migration_01_to_11/migration_11_invoices.go

@@ -0,0 +1,230 @@
+package migration_01_to_11
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"io"
+
+	bitcoinCfg "github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/coreos/bbolt"
+	"github.com/lightningnetwork/lnd/lnwire"
+	"github.com/lightningnetwork/lnd/zpay32"
+	litecoinCfg "github.com/ltcsuite/ltcd/chaincfg"
+)
+
+// migrateInvoices adds invoice htlcs and a separate cltv delta field to the
+// invoices.
+func migrateInvoices(tx *bbolt.Tx) error {
+	log.Infof("Migrating invoices to new invoice format")
+
+	invoiceB := tx.Bucket(invoiceBucket)
+	if invoiceB == nil {
+		return nil
+	}
+
+	// Iterate through the entire key space of the top-level invoice bucket.
+	// If key with a non-nil value stores the next invoice ID which maps to
+	// the corresponding invoice. Store those keys first, because it isn't
+	// safe to modify the bucket inside a ForEach loop.
+	var invoiceKeys [][]byte
+	err := invoiceB.ForEach(func(k, v []byte) error {
+		if v == nil {
+			return nil
+		}
+
+		invoiceKeys = append(invoiceKeys, k)
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	nets := []*bitcoinCfg.Params{
+		&bitcoinCfg.MainNetParams, &bitcoinCfg.SimNetParams,
+		&bitcoinCfg.RegressionNetParams, &bitcoinCfg.TestNet3Params,
+	}
+
+	ltcNets := []*litecoinCfg.Params{
+		&litecoinCfg.MainNetParams, &litecoinCfg.SimNetParams,
+		&litecoinCfg.RegressionNetParams, &litecoinCfg.TestNet4Params,
+	}
+	for _, net := range ltcNets {
+		var convertedNet bitcoinCfg.Params
+		convertedNet.Bech32HRPSegwit = net.Bech32HRPSegwit
+		nets = append(nets, &convertedNet)
+	}
+
+	// Iterate over all stored keys and migrate the invoices.
+	for _, k := range invoiceKeys {
+		v := invoiceB.Get(k)
+
+		// Deserialize the invoice with the deserializing function that
+		// was in use for this version of the database.
+		invoiceReader := bytes.NewReader(v)
+		invoice, err := deserializeInvoiceLegacy(invoiceReader)
+		if err != nil {
+			return err
+		}
+
+		if invoice.Terms.State == ContractAccepted {
+			return fmt.Errorf("cannot upgrade with invoice(s) " +
+				"in accepted state, see release notes")
+		}
+
+		// Try to decode the payment request for every possible net to
+		// avoid passing a the active network to channeldb. This would
+		// be a layering violation, while this migration is only running
+		// once and will likely be removed in the future.
+		var payReq *zpay32.Invoice
+		for _, net := range nets {
+			payReq, err = zpay32.Decode(
+				string(invoice.PaymentRequest), net,
+			)
+			if err == nil {
+				break
+			}
+		}
+		if payReq == nil {
+			return fmt.Errorf("cannot decode payreq")
+		}
+		invoice.FinalCltvDelta = int32(payReq.MinFinalCLTVExpiry())
+		invoice.Expiry = payReq.Expiry()
+
+		// Serialize the invoice in the new format and use it to replace
+		// the old invoice in the database.
+		var buf bytes.Buffer
+		if err := serializeInvoice(&buf, &invoice); err != nil {
+			return err
+		}
+
+		err = invoiceB.Put(k, buf.Bytes())
+		if err != nil {
+			return err
+		}
+	}
+
+	log.Infof("Migration of invoices completed!")
+	return nil
+}
+
+func deserializeInvoiceLegacy(r io.Reader) (Invoice, error) {
+	var err error
+	invoice := Invoice{}
+
+	// TODO(roasbeef): use read full everywhere
+	invoice.Memo, err = wire.ReadVarBytes(r, 0, MaxMemoSize, "")
+	if err != nil {
+		return invoice, err
+	}
+	invoice.Receipt, err = wire.ReadVarBytes(r, 0, MaxReceiptSize, "")
+	if err != nil {
+		return invoice, err
+	}
+
+	invoice.PaymentRequest, err = wire.ReadVarBytes(r, 0, MaxPaymentRequestSize, "")
+	if err != nil {
+		return invoice, err
+	}
+
+	birthBytes, err := wire.ReadVarBytes(r, 0, 300, "birth")
+	if err != nil {
+		return invoice, err
+	}
+	if err := invoice.CreationDate.UnmarshalBinary(birthBytes); err != nil {
+		return invoice, err
+	}
+
+	settledBytes, err := wire.ReadVarBytes(r, 0, 300, "settled")
+	if err != nil {
+		return invoice, err
+	}
+	if err := invoice.SettleDate.UnmarshalBinary(settledBytes); err != nil {
+		return invoice, err
+	}
+
+	if _, err := io.ReadFull(r, invoice.Terms.PaymentPreimage[:]); err != nil {
+		return invoice, err
+	}
+	var scratch [8]byte
+	if _, err := io.ReadFull(r, scratch[:]); err != nil {
+		return invoice, err
+	}
+	invoice.Terms.Value = lnwire.MilliSatoshi(byteOrder.Uint64(scratch[:]))
+
+	if err := binary.Read(r, byteOrder, &invoice.Terms.State); err != nil {
+		return invoice, err
+	}
+
+	if err := binary.Read(r, byteOrder, &invoice.AddIndex); err != nil {
+		return invoice, err
+	}
+	if err := binary.Read(r, byteOrder, &invoice.SettleIndex); err != nil {
+		return invoice, err
+	}
+	if err := binary.Read(r, byteOrder, &invoice.AmtPaid); err != nil {
+		return invoice, err
+	}
+
+	return invoice, nil
+}
+
+// serializeInvoiceLegacy serializes an invoice in the format of the previous db
+// version.
+func serializeInvoiceLegacy(w io.Writer, i *Invoice) error {
+	if err := wire.WriteVarBytes(w, 0, i.Memo[:]); err != nil {
+		return err
+	}
+	if err := wire.WriteVarBytes(w, 0, i.Receipt[:]); err != nil {
+		return err
+	}
+	if err := wire.WriteVarBytes(w, 0, i.PaymentRequest[:]); err != nil {
+		return err
+	}
+
+	birthBytes, err := i.CreationDate.MarshalBinary()
+	if err != nil {
+		return err
+	}
+
+	if err := wire.WriteVarBytes(w, 0, birthBytes); err != nil {
+		return err
+	}
+
+	settleBytes, err := i.SettleDate.MarshalBinary()
+	if err != nil {
+		return err
+	}
+
+	if err := wire.WriteVarBytes(w, 0, settleBytes); err != nil {
+		return err
+	}
+
+	if _, err := w.Write(i.Terms.PaymentPreimage[:]); err != nil {
+		return err
+	}
+
+	var scratch [8]byte
+	byteOrder.PutUint64(scratch[:], uint64(i.Terms.Value))
+	if _, err := w.Write(scratch[:]); err != nil {
+		return err
+	}
+
+	if err := binary.Write(w, byteOrder, i.Terms.State); err != nil {
+		return err
+	}
+
+	if err := binary.Write(w, byteOrder, i.AddIndex); err != nil {
+		return err
+	}
+	if err := binary.Write(w, byteOrder, i.SettleIndex); err != nil {
+		return err
+	}
+	if err := binary.Write(w, byteOrder, int64(i.AmtPaid)); err != nil {
+		return err
+	}
+
+	return nil
+}

channeldb/migration_01_to_11/migration_11_invoices_test.go

@@ -0,0 +1,193 @@
+package migration_01_to_11
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/btcsuite/btcd/btcec"
+	bitcoinCfg "github.com/btcsuite/btcd/chaincfg"
+	"github.com/coreos/bbolt"
+	"github.com/lightningnetwork/lnd/zpay32"
+	litecoinCfg "github.com/ltcsuite/ltcd/chaincfg"
+)
+
+var (
+	testPrivKeyBytes = []byte{
+		0x2b, 0xd8, 0x06, 0xc9, 0x7f, 0x0e, 0x00, 0xaf,
+		0x1a, 0x1f, 0xc3, 0x32, 0x8f, 0xa7, 0x63, 0xa9,
+		0x26, 0x97, 0x23, 0xc8, 0xdb, 0x8f, 0xac, 0x4f,
+		0x93, 0xaf, 0x71, 0xdb, 0x18, 0x6d, 0x6e, 0x90,
+	}
+
+	testCltvDelta = int32(50)
+)
+
+// beforeMigrationFuncV11 insert the test invoices in the database.
+func beforeMigrationFuncV11(t *testing.T, d *DB, invoices []Invoice) {
+	err := d.Update(func(tx *bbolt.Tx) error {
+		invoicesBucket, err := tx.CreateBucketIfNotExists(
+			invoiceBucket,
+		)
+		if err != nil {
+			return err
+		}
+
+		invoiceNum := uint32(1)
+		for _, invoice := range invoices {
+			var invoiceKey [4]byte
+			byteOrder.PutUint32(invoiceKey[:], invoiceNum)
+			invoiceNum++
+
+			var buf bytes.Buffer
+			err := serializeInvoiceLegacy(&buf, &invoice) // nolint:scopelint
+			if err != nil {
+				return err
+			}
+
+			err = invoicesBucket.Put(
+				invoiceKey[:], buf.Bytes(),
+			)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestMigrateInvoices checks that invoices are migrated correctly.
+func TestMigrateInvoices(t *testing.T) {
+	t.Parallel()
+
+	payReqBtc, err := getPayReq(&bitcoinCfg.MainNetParams)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var ltcNetParams bitcoinCfg.Params
+	ltcNetParams.Bech32HRPSegwit = litecoinCfg.MainNetParams.Bech32HRPSegwit
+	payReqLtc, err := getPayReq(&ltcNetParams)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	invoices := []Invoice{
+		{
+			PaymentRequest: []byte(payReqBtc),
+		},
+		{
+			PaymentRequest: []byte(payReqLtc),
+		},
+	}
+
+	// Verify that all invoices were migrated.
+	afterMigrationFunc := func(d *DB) {
+		meta, err := d.FetchMeta(nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if meta.DbVersionNumber != 1 {
+			t.Fatal("migration 'invoices' wasn't applied")
+		}
+
+		dbInvoices, err := d.FetchAllInvoices(false)
+		if err != nil {
+			t.Fatalf("unable to fetch invoices: %v", err)
+		}
+
+		if len(invoices) != len(dbInvoices) {
+			t.Fatalf("expected %d invoices, got %d", len(invoices),
+				len(dbInvoices))
+		}
+
+		for _, dbInvoice := range dbInvoices {
+			if dbInvoice.FinalCltvDelta != testCltvDelta {
+				t.Fatal("incorrect final cltv delta")
+			}
+			if dbInvoice.Expiry != 3600*time.Second {
+				t.Fatal("incorrect expiry")
+			}
+			if len(dbInvoice.Htlcs) != 0 {
+				t.Fatal("expected no htlcs after migration")
+			}
+		}
+	}
+
+	applyMigration(t,
+		func(d *DB) { beforeMigrationFuncV11(t, d, invoices) },
+		afterMigrationFunc,
+		migrateInvoices,
+		false)
+}
+
+// TestMigrateInvoicesHodl checks that a hodl invoice in the accepted state
+// fails the migration.
+func TestMigrateInvoicesHodl(t *testing.T) {
+	t.Parallel()
+
+	payReqBtc, err := getPayReq(&bitcoinCfg.MainNetParams)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	invoices := []Invoice{
+		{
+			PaymentRequest: []byte(payReqBtc),
+			Terms: ContractTerm{
+				State: ContractAccepted,
+			},
+		},
+	}
+
+	applyMigration(t,
+		func(d *DB) { beforeMigrationFuncV11(t, d, invoices) },
+		func(d *DB) {},
+		migrateInvoices,
+		true)
+}
+
+// signDigestCompact generates a test signature to be used in the generation of
+// test payment requests.
+func signDigestCompact(hash []byte) ([]byte, error) {
+	// Should the signature reference a compressed public key or not.
+	isCompressedKey := true
+
+	privKey, _ := btcec.PrivKeyFromBytes(btcec.S256(), testPrivKeyBytes)
+
+	// btcec.SignCompact returns a pubkey-recoverable signature
+	sig, err := btcec.SignCompact(
+		btcec.S256(), privKey, hash, isCompressedKey,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("can't sign the hash: %v", err)
+	}
+
+	return sig, nil
+}
+
+// getPayReq creates a payment request for the given net.
+func getPayReq(net *bitcoinCfg.Params) (string, error) {
+	options := []func(*zpay32.Invoice){
+		zpay32.CLTVExpiry(uint64(testCltvDelta)),
+		zpay32.Description("test"),
+	}
+
+	payReq, err := zpay32.NewInvoice(
+		net, [32]byte{}, time.Unix(1, 0), options...,
+	)
+	if err != nil {
+		return "", err
+	}
+	return payReq.Encode(
+		zpay32.MessageSigner{
+			SignCompact: signDigestCompact,
+		},
+	)
+}

channeldb/migration_01_to_11/migrations.go

@@ -0,0 +1,939 @@
+package migration_01_to_11
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/binary"
+	"fmt"
+
+	"github.com/btcsuite/btcd/btcec"
+	"github.com/coreos/bbolt"
+	"github.com/lightningnetwork/lnd/lnwire"
+	"github.com/lightningnetwork/lnd/routing/route"
+)
+
+// migrateNodeAndEdgeUpdateIndex is a migration function that will update the
+// database from version 0 to version 1. In version 1, we add two new indexes
+// (one for nodes and one for edges) to keep track of the last time a node or
+// edge was updated on the network. These new indexes allow us to implement the
+// new graph sync protocol added.
+func migrateNodeAndEdgeUpdateIndex(tx *bbolt.Tx) error {
+	// First, we'll populating the node portion of the new index. Before we
+	// can add new values to the index, we'll first create the new bucket
+	// where these items will be housed.
+	nodes, err := tx.CreateBucketIfNotExists(nodeBucket)
+	if err != nil {
+		return fmt.Errorf("unable to create node bucket: %v", err)
+	}
+	nodeUpdateIndex, err := nodes.CreateBucketIfNotExists(
+		nodeUpdateIndexBucket,
+	)
+	if err != nil {
+		return fmt.Errorf("unable to create node update index: %v", err)
+	}
+
+	log.Infof("Populating new node update index bucket")
+
+	// Now that we know the bucket has been created, we'll iterate over the
+	// entire node bucket so we can add the (updateTime || nodePub) key
+	// into the node update index.
+	err = nodes.ForEach(func(nodePub, nodeInfo []byte) error {
+		if len(nodePub) != 33 {
+			return nil
+		}
+
+		log.Tracef("Adding %x to node update index", nodePub)
+
+		// The first 8 bytes of a node's serialize data is the update
+		// time, so we can extract that without decoding the entire
+		// structure.
+		updateTime := nodeInfo[:8]
+
+		// Now that we have the update time, we can construct the key
+		// to insert into the index.
+		var indexKey [8 + 33]byte
+		copy(indexKey[:8], updateTime)
+		copy(indexKey[8:], nodePub)
+
+		return nodeUpdateIndex.Put(indexKey[:], nil)
+	})
+	if err != nil {
+		return fmt.Errorf("unable to update node indexes: %v", err)
+	}
+
+	log.Infof("Populating new edge update index bucket")
+
+	// With the set of nodes updated, we'll now update all edges to have a
+	// corresponding entry in the edge update index.
+	edges, err := tx.CreateBucketIfNotExists(edgeBucket)
+	if err != nil {
+		return fmt.Errorf("unable to create edge bucket: %v", err)
+	}
+	edgeUpdateIndex, err := edges.CreateBucketIfNotExists(
+		edgeUpdateIndexBucket,
+	)
+	if err != nil {
+		return fmt.Errorf("unable to create edge update index: %v", err)
+	}
+
+	// We'll now run through each edge policy in the database, and update
+	// the index to ensure each edge has the proper record.
+	err = edges.ForEach(func(edgeKey, edgePolicyBytes []byte) error {
+		if len(edgeKey) != 41 {
+			return nil
+		}
+
+		// Now that we know this is the proper record, we'll grab the
+		// channel ID (last 8 bytes of the key), and then decode the
+		// edge policy so we can access the update time.
+		chanID := edgeKey[33:]
+		edgePolicyReader := bytes.NewReader(edgePolicyBytes)
+
+		edgePolicy, err := deserializeChanEdgePolicy(
+			edgePolicyReader, nodes,
+		)
+		if err != nil {
+			return err
+		}
+
+		log.Tracef("Adding chan_id=%v to edge update index",
+			edgePolicy.ChannelID)
+
+		// We'll now construct the index key using the channel ID, and
+		// the last time it was updated: (updateTime || chanID).
+		var indexKey [8 + 8]byte
+		byteOrder.PutUint64(
+			indexKey[:], uint64(edgePolicy.LastUpdate.Unix()),
+		)
+		copy(indexKey[8:], chanID)
+
+		return edgeUpdateIndex.Put(indexKey[:], nil)
+	})
+	if err != nil {
+		return fmt.Errorf("unable to update edge indexes: %v", err)
+	}
+
+	log.Infof("Migration to node and edge update indexes complete!")
+
+	return nil
+}
+
+// migrateInvoiceTimeSeries is a database migration that assigns all existing
+// invoices an index in the add and/or the settle index. Additionally, all
+// existing invoices will have their bytes padded out in order to encode the
+// add+settle index as well as the amount paid.
+func migrateInvoiceTimeSeries(tx *bbolt.Tx) error {
+	invoices, err := tx.CreateBucketIfNotExists(invoiceBucket)
+	if err != nil {
+		return err
+	}
+
+	addIndex, err := invoices.CreateBucketIfNotExists(
+		addIndexBucket,
+	)
+	if err != nil {
+		return err
+	}
+	settleIndex, err := invoices.CreateBucketIfNotExists(
+		settleIndexBucket,
+	)
+	if err != nil {
+		return err
+	}
+
+	log.Infof("Migrating invoice database to new time series format")
+
+	// Now that we have all the buckets we need, we'll run through each
+	// invoice in the database, and update it to reflect the new format
+	// expected post migration.
+	// NOTE: we store the converted invoices and put them back into the
+	// database after the loop, since modifying the bucket within the
+	// ForEach loop is not safe.
+	var invoicesKeys [][]byte
+	var invoicesValues [][]byte
+	err = invoices.ForEach(func(invoiceNum, invoiceBytes []byte) error {
+		// If this is a sub bucket, then we'll skip it.
+		if invoiceBytes == nil {
+			return nil
+		}
+
+		// First, we'll make a copy of the encoded invoice bytes.
+		invoiceBytesCopy := make([]byte, len(invoiceBytes))
+		copy(invoiceBytesCopy, invoiceBytes)
+
+		// With the bytes copied over, we'll append 24 additional
+		// bytes. We do this so we can decode the invoice under the new
+		// serialization format.
+		padding := bytes.Repeat([]byte{0}, 24)
+		invoiceBytesCopy = append(invoiceBytesCopy, padding...)
+
+		invoiceReader := bytes.NewReader(invoiceBytesCopy)
+		invoice, err := deserializeInvoiceLegacy(invoiceReader)
+		if err != nil {
+			return fmt.Errorf("unable to decode invoice: %v", err)
+		}
+
+		// Now that we have the fully decoded invoice, we can update
+		// the various indexes that we're added, and finally the
+		// invoice itself before re-inserting it.
+
+		// First, we'll get the new sequence in the addIndex in order
+		// to create the proper mapping.
+		nextAddSeqNo, err := addIndex.NextSequence()
+		if err != nil {
+			return err
+		}
+		var seqNoBytes [8]byte
+		byteOrder.PutUint64(seqNoBytes[:], nextAddSeqNo)
+		err = addIndex.Put(seqNoBytes[:], invoiceNum[:])
+		if err != nil {
+			return err
+		}
+
+		log.Tracef("Adding invoice (preimage=%x, add_index=%v) to add "+
+			"time series", invoice.Terms.PaymentPreimage[:],
+			nextAddSeqNo)
+
+		// Next, we'll check if the invoice has been settled or not. If
+		// so, then we'll also add it to the settle index.
+		var nextSettleSeqNo uint64
+		if invoice.Terms.State == ContractSettled {
+			nextSettleSeqNo, err = settleIndex.NextSequence()
+			if err != nil {
+				return err
+			}
+
+			var seqNoBytes [8]byte
+			byteOrder.PutUint64(seqNoBytes[:], nextSettleSeqNo)
+			err := settleIndex.Put(seqNoBytes[:], invoiceNum)
+			if err != nil {
+				return err
+			}
+
+			invoice.AmtPaid = invoice.Terms.Value
+
+			log.Tracef("Adding invoice (preimage=%x, "+
+				"settle_index=%v) to add time series",
+				invoice.Terms.PaymentPreimage[:],
+				nextSettleSeqNo)
+		}
+
+		// Finally, we'll update the invoice itself with the new
+		// indexing information as well as the amount paid if it has
+		// been settled or not.
+		invoice.AddIndex = nextAddSeqNo
+		invoice.SettleIndex = nextSettleSeqNo
+
+		// We've fully migrated an invoice, so we'll now update the
+		// invoice in-place.
+		var b bytes.Buffer
+		if err := serializeInvoiceLegacy(&b, &invoice); err != nil {
+			return err
+		}
+
+		// Save the key and value pending update for after the ForEach
+		// is done.
+		invoicesKeys = append(invoicesKeys, invoiceNum)
+		invoicesValues = append(invoicesValues, b.Bytes())
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	// Now put the converted invoices into the DB.
+	for i := range invoicesKeys {
+		key := invoicesKeys[i]
+		value := invoicesValues[i]
+		if err := invoices.Put(key, value); err != nil {
+			return err
+		}
+	}
+
+	log.Infof("Migration to invoice time series index complete!")
+
+	return nil
+}
+
+// migrateInvoiceTimeSeriesOutgoingPayments is a follow up to the
+// migrateInvoiceTimeSeries migration. As at the time of writing, the
+// OutgoingPayment struct embeddeds an instance of the Invoice struct. As a
+// result, we also need to migrate the internal invoice to the new format.
+func migrateInvoiceTimeSeriesOutgoingPayments(tx *bbolt.Tx) error {
+	payBucket := tx.Bucket(paymentBucket)
+	if payBucket == nil {
+		return nil
+	}
+
+	log.Infof("Migrating invoice database to new outgoing payment format")
+
+	// We store the keys and values we want to modify since it is not safe
+	// to modify them directly within the ForEach loop.
+	var paymentKeys [][]byte
+	var paymentValues [][]byte
+	err := payBucket.ForEach(func(payID, paymentBytes []byte) error {
+		log.Tracef("Migrating payment %x", payID[:])
+
+		// The internal invoices for each payment only contain a
+		// populated contract term, and creation date, as a result,
+		// most of the bytes will be "empty".
+
+		// We'll calculate the end of the invoice index assuming a
+		// "minimal" index that's embedded within the greater
+		// OutgoingPayment. The breakdown is:
+		//  3 bytes empty var bytes, 16 bytes creation date, 16 bytes
+		//  settled date, 32 bytes payment pre-image, 8 bytes value, 1
+		//  byte settled.
+		endOfInvoiceIndex := 1 + 1 + 1 + 16 + 16 + 32 + 8 + 1
+
+		// We'll now extract the prefix of the pure invoice embedded
+		// within.
+		invoiceBytes := paymentBytes[:endOfInvoiceIndex]
+
+		// With the prefix extracted, we'll copy over the invoice, and
+		// also add padding for the new 24 bytes of fields, and finally
+		// append the remainder of the outgoing payment.
+		paymentCopy := make([]byte, len(invoiceBytes))
+		copy(paymentCopy[:], invoiceBytes)
+
+		padding := bytes.Repeat([]byte{0}, 24)
+		paymentCopy = append(paymentCopy, padding...)
+		paymentCopy = append(
+			paymentCopy, paymentBytes[endOfInvoiceIndex:]...,
+		)
+
+		// At this point, we now have the new format of the outgoing
+		// payments, we'll attempt to deserialize it to ensure the
+		// bytes are properly formatted.
+		paymentReader := bytes.NewReader(paymentCopy)
+		_, err := deserializeOutgoingPayment(paymentReader)
+		if err != nil {
+			return fmt.Errorf("unable to deserialize payment: %v", err)
+		}
+
+		// Now that we know the modifications was successful, we'll
+		// store it to our slice of keys and values, and write it back
+		// to disk in the new format after the ForEach loop is over.
+		paymentKeys = append(paymentKeys, payID)
+		paymentValues = append(paymentValues, paymentCopy)
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	// Finally store the updated payments to the bucket.
+	for i := range paymentKeys {
+		key := paymentKeys[i]
+		value := paymentValues[i]
+		if err := payBucket.Put(key, value); err != nil {
+			return err
+		}
+	}
+
+	log.Infof("Migration to outgoing payment invoices complete!")
+
+	return nil
+}
+
+// migrateEdgePolicies is a migration function that will update the edges
+// bucket. It ensure that edges with unknown policies will also have an entry
+// in the bucket. After the migration, there will be two edge entries for
+// every channel, regardless of whether the policies are known.
+func migrateEdgePolicies(tx *bbolt.Tx) error {
+	nodes := tx.Bucket(nodeBucket)
+	if nodes == nil {
+		return nil
+	}
+
+	edges := tx.Bucket(edgeBucket)
+	if edges == nil {
+		return nil
+	}
+
+	edgeIndex := edges.Bucket(edgeIndexBucket)
+	if edgeIndex == nil {
+		return nil
+	}
+
+	// checkKey gets the policy from the database with a low-level call
+	// so that it is still possible to distinguish between unknown and
+	// not present.
+	checkKey := func(channelId uint64, keyBytes []byte) error {
+		var channelID [8]byte
+		byteOrder.PutUint64(channelID[:], channelId)
+
+		_, err := fetchChanEdgePolicy(edges,
+			channelID[:], keyBytes, nodes)
+
+		if err == ErrEdgeNotFound {
+			log.Tracef("Adding unknown edge policy present for node %x, channel %v",
+				keyBytes, channelId)
+
+			err := putChanEdgePolicyUnknown(edges, channelId, keyBytes)
+			if err != nil {
+				return err
+			}
+
+			return nil
+		}
+
+		return err
+	}
+
+	// Iterate over all channels and check both edge policies.
+	err := edgeIndex.ForEach(func(chanID, edgeInfoBytes []byte) error {
+		infoReader := bytes.NewReader(edgeInfoBytes)
+		edgeInfo, err := deserializeChanEdgeInfo(infoReader)
+		if err != nil {
+			return err
+		}
+
+		for _, key := range [][]byte{edgeInfo.NodeKey1Bytes[:],
+			edgeInfo.NodeKey2Bytes[:]} {
+
+			if err := checkKey(edgeInfo.ChannelID, key); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("unable to update edge policies: %v", err)
+	}
+
+	log.Infof("Migration of edge policies complete!")
+
+	return nil
+}
+
+// paymentStatusesMigration is a database migration intended for adding payment
+// statuses for each existing payment entity in bucket to be able control
+// transitions of statuses and prevent cases such as double payment
+func paymentStatusesMigration(tx *bbolt.Tx) error {
+	// Get the bucket dedicated to storing statuses of payments,
+	// where a key is payment hash, value is payment status.
+	paymentStatuses, err := tx.CreateBucketIfNotExists(paymentStatusBucket)
+	if err != nil {
+		return err
+	}
+
+	log.Infof("Migrating database to support payment statuses")
+
+	circuitAddKey := []byte("circuit-adds")
+	circuits := tx.Bucket(circuitAddKey)
+	if circuits != nil {
+		log.Infof("Marking all known circuits with status InFlight")
+
+		err = circuits.ForEach(func(k, v []byte) error {
+			// Parse the first 8 bytes as the short chan ID for the
+			// circuit. We'll skip all short chan IDs are not
+			// locally initiated, which includes all non-zero short
+			// chan ids.
+			chanID := binary.BigEndian.Uint64(k[:8])
+			if chanID != 0 {
+				return nil
+			}
+
+			// The payment hash is the third item in the serialized
+			// payment circuit. The first two items are an AddRef
+			// (10 bytes) and the incoming circuit key (16 bytes).
+			const payHashOffset = 10 + 16
+
+			paymentHash := v[payHashOffset : payHashOffset+32]
+
+			return paymentStatuses.Put(
+				paymentHash[:], StatusInFlight.Bytes(),
+			)
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	log.Infof("Marking all existing payments with status Completed")
+
+	// Get the bucket dedicated to storing payments
+	bucket := tx.Bucket(paymentBucket)
+	if bucket == nil {
+		return nil
+	}
+
+	// For each payment in the bucket, deserialize the payment and mark it
+	// as completed.
+	err = bucket.ForEach(func(k, v []byte) error {
+		// Ignores if it is sub-bucket.
+		if v == nil {
+			return nil
+		}
+
+		r := bytes.NewReader(v)
+		payment, err := deserializeOutgoingPayment(r)
+		if err != nil {
+			return err
+		}
+
+		// Calculate payment hash for current payment.
+		paymentHash := sha256.Sum256(payment.PaymentPreimage[:])
+
+		// Update status for current payment to completed. If it fails,
+		// the migration is aborted and the payment bucket is returned
+		// to its previous state.
+		return paymentStatuses.Put(paymentHash[:], StatusSucceeded.Bytes())
+	})
+	if err != nil {
+		return err
+	}
+
+	log.Infof("Migration of payment statuses complete!")
+
+	return nil
+}
+
+// migratePruneEdgeUpdateIndex is a database migration that attempts to resolve
+// some lingering bugs with regards to edge policies and their update index.
+// Stale entries within the edge update index were not being properly pruned due
+// to a miscalculation on the offset of an edge's policy last update. This
+// migration also fixes the case where the public keys within edge policies were
+// being serialized with an extra byte, causing an even greater error when
+// attempting to perform the offset calculation described earlier.
+func migratePruneEdgeUpdateIndex(tx *bbolt.Tx) error {
+	// To begin the migration, we'll retrieve the update index bucket. If it
+	// does not exist, we have nothing left to do so we can simply exit.
+	edges := tx.Bucket(edgeBucket)
+	if edges == nil {
+		return nil
+	}
+	edgeUpdateIndex := edges.Bucket(edgeUpdateIndexBucket)
+	if edgeUpdateIndex == nil {
+		return nil
+	}
+
+	// Retrieve some buckets that will be needed later on. These should
+	// already exist given the assumption that the buckets above do as
+	// well.
+	edgeIndex, err := edges.CreateBucketIfNotExists(edgeIndexBucket)
+	if err != nil {
+		return fmt.Errorf("error creating edge index bucket: %s", err)
+	}
+	if edgeIndex == nil {
+		return fmt.Errorf("unable to create/fetch edge index " +
+			"bucket")
+	}
+	nodes, err := tx.CreateBucketIfNotExists(nodeBucket)
+	if err != nil {
+		return fmt.Errorf("unable to make node bucket")
+	}
+
+	log.Info("Migrating database to properly prune edge update index")
+
+	// We'll need to properly prune all the outdated entries within the edge
+	// update index. To do so, we'll gather all of the existing policies
+	// within the graph to re-populate them later on.
+	var edgeKeys [][]byte
+	err = edges.ForEach(func(edgeKey, edgePolicyBytes []byte) error {
+		// All valid entries are indexed by a public key (33 bytes)
+		// followed by a channel ID (8 bytes), so we'll skip any entries
+		// with keys that do not match this.
+		if len(edgeKey) != 33+8 {
+			return nil
+		}
+
+		edgeKeys = append(edgeKeys, edgeKey)
+
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("unable to gather existing edge policies: %v",
+			err)
+	}
+
+	log.Info("Constructing set of edge update entries to purge.")
+
+	// Build the set of keys that we will remove from the edge update index.
+	// This will include all keys contained within the bucket.
+	var updateKeysToRemove [][]byte
+	err = edgeUpdateIndex.ForEach(func(updKey, _ []byte) error {
+		updateKeysToRemove = append(updateKeysToRemove, updKey)
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("unable to gather existing edge updates: %v",
+			err)
+	}
+
+	log.Infof("Removing %d entries from edge update index.",
+		len(updateKeysToRemove))
+
+	// With the set of keys contained in the edge update index constructed,
+	// we'll proceed in purging all of them from the index.
+	for _, updKey := range updateKeysToRemove {
+		if err := edgeUpdateIndex.Delete(updKey); err != nil {
+			return err
+		}
+	}
+
+	log.Infof("Repopulating edge update index with %d valid entries.",
+		len(edgeKeys))
+
+	// For each edge key, we'll retrieve the policy, deserialize it, and
+	// re-add it to the different buckets. By doing so, we'll ensure that
+	// all existing edge policies are serialized correctly within their
+	// respective buckets and that the correct entries are populated within
+	// the edge update index.
+	for _, edgeKey := range edgeKeys {
+		edgePolicyBytes := edges.Get(edgeKey)
+
+		// Skip any entries with unknown policies as there will not be
+		// any entries for them in the edge update index.
+		if bytes.Equal(edgePolicyBytes[:], unknownPolicy) {
+			continue
+		}
+
+		edgePolicy, err := deserializeChanEdgePolicy(
+			bytes.NewReader(edgePolicyBytes), nodes,
+		)
+		if err != nil {
+			return err
+		}
+
+		_, err = updateEdgePolicy(tx, edgePolicy)
+		if err != nil {
+			return err
+		}
+	}
+
+	log.Info("Migration to properly prune edge update index complete!")
+
+	return nil
+}
+
+// migrateOptionalChannelCloseSummaryFields migrates the serialized format of
+// ChannelCloseSummary to a format where optional fields' presence is indicated
+// with boolean markers.
+func migrateOptionalChannelCloseSummaryFields(tx *bbolt.Tx) error {
+	closedChanBucket := tx.Bucket(closedChannelBucket)
+	if closedChanBucket == nil {
+		return nil
+	}
+
+	log.Info("Migrating to new closed channel format...")
+
+	// We store the converted keys and values and put them back into the
+	// database after the loop, since modifying the bucket within the
+	// ForEach loop is not safe.
+	var closedChansKeys [][]byte
+	var closedChansValues [][]byte
+	err := closedChanBucket.ForEach(func(chanID, summary []byte) error {
+		r := bytes.NewReader(summary)
+
+		// Read the old (v6) format from the database.
+		c, err := deserializeCloseChannelSummaryV6(r)
+		if err != nil {
+			return err
+		}
+
+		// Serialize using the new format, and put back into the
+		// bucket.
+		var b bytes.Buffer
+		if err := serializeChannelCloseSummary(&b, c); err != nil {
+			return err
+		}
+
+		// Now that we know the modifications was successful, we'll
+		// Store the key and value to our slices, and write it back to
+		// disk in the new format after the ForEach loop is over.
+		closedChansKeys = append(closedChansKeys, chanID)
+		closedChansValues = append(closedChansValues, b.Bytes())
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("unable to update closed channels: %v", err)
+	}
+
+	// Now put the new format back into the DB.
+	for i := range closedChansKeys {
+		key := closedChansKeys[i]
+		value := closedChansValues[i]
+		if err := closedChanBucket.Put(key, value); err != nil {
+			return err
+		}
+	}
+
+	log.Info("Migration to new closed channel format complete!")
+
+	return nil
+}
+
+var messageStoreBucket = []byte("message-store")
+
+// migrateGossipMessageStoreKeys migrates the key format for gossip messages
+// found in the message store to a new one that takes into consideration the of
+// the message being stored.
+func migrateGossipMessageStoreKeys(tx *bbolt.Tx) error {
+	// We'll start by retrieving the bucket in which these messages are
+	// stored within. If there isn't one, there's nothing left for us to do
+	// so we can avoid the migration.
+	messageStore := tx.Bucket(messageStoreBucket)
+	if messageStore == nil {
+		return nil
+	}
+
+	log.Info("Migrating to the gossip message store new key format")
+
+	// Otherwise we'll proceed with the migration. We'll start by coalescing
+	// all the current messages within the store, which are indexed by the
+	// public key of the peer which they should be sent to, followed by the
+	// short channel ID of the channel for which the message belongs to. We
+	// should only expect to find channel announcement signatures as that
+	// was the only support message type previously.
+	msgs := make(map[[33 + 8]byte]*lnwire.AnnounceSignatures)
+	err := messageStore.ForEach(func(k, v []byte) error {
+		var msgKey [33 + 8]byte
+		copy(msgKey[:], k)
+
+		msg := &lnwire.AnnounceSignatures{}
+		if err := msg.Decode(bytes.NewReader(v), 0); err != nil {
+			return err
+		}
+
+		msgs[msgKey] = msg
+
+		return nil
+
+	})
+	if err != nil {
+		return err
+	}
+
+	// Then, we'll go over all of our messages, remove their previous entry,
+	// and add another with the new key format. Once we've done this for
+	// every message, we can consider the migration complete.
+	for oldMsgKey, msg := range msgs {
+		if err := messageStore.Delete(oldMsgKey[:]); err != nil {
+			return err
+		}
+
+		// Construct the new key for which we'll find this message with
+		// in the store. It'll be the same as the old, but we'll also
+		// include the message type.
+		var msgType [2]byte
+		binary.BigEndian.PutUint16(msgType[:], uint16(msg.MsgType()))
+		newMsgKey := append(oldMsgKey[:], msgType[:]...)
+
+		// Serialize the message with its wire encoding.
+		var b bytes.Buffer
+		if _, err := lnwire.WriteMessage(&b, msg, 0); err != nil {
+			return err
+		}
+
+		if err := messageStore.Put(newMsgKey, b.Bytes()); err != nil {
+			return err
+		}
+	}
+
+	log.Info("Migration to the gossip message store new key format complete!")
+
+	return nil
+}
+
+// migrateOutgoingPayments moves the OutgoingPayments into a new bucket format
+// where they all reside in a top-level bucket indexed by the payment hash. In
+// this sub-bucket we store information relevant to this payment, such as the
+// payment status.
+//
+// Since the router cannot handle resumed payments that have the status
+// InFlight (we have no PaymentAttemptInfo available for pre-migration
+// payments) we delete those statuses, so only Completed payments remain in the
+// new bucket structure.
+func migrateOutgoingPayments(tx *bbolt.Tx) error {
+	log.Infof("Migrating outgoing payments to new bucket structure")
+
+	oldPayments := tx.Bucket(paymentBucket)
+
+	// Return early if there are no payments to migrate.
+	if oldPayments == nil {
+		log.Infof("No outgoing payments found, nothing to migrate.")
+		return nil
+	}
+
+	newPayments, err := tx.CreateBucket(paymentsRootBucket)
+	if err != nil {
+		return err
+	}
+
+	// Helper method to get the source pubkey. We define it such that we
+	// only attempt to fetch it if needed.
+	sourcePub := func() ([33]byte, error) {
+		var pub [33]byte
+		nodes := tx.Bucket(nodeBucket)
+		if nodes == nil {
+			return pub, ErrGraphNotFound
+		}
+
+		selfPub := nodes.Get(sourceKey)
+		if selfPub == nil {
+			return pub, ErrSourceNodeNotSet
+		}
+		copy(pub[:], selfPub[:])
+		return pub, nil
+	}
+
+	err = oldPayments.ForEach(func(k, v []byte) error {
+		// Ignores if it is sub-bucket.
+		if v == nil {
+			return nil
+		}
+
+		// Read the old payment format.
+		r := bytes.NewReader(v)
+		payment, err := deserializeOutgoingPayment(r)
+		if err != nil {
+			return err
+		}
+
+		// Calculate payment hash from the payment preimage.
+		paymentHash := sha256.Sum256(payment.PaymentPreimage[:])
+
+		// Now create and add a PaymentCreationInfo to the bucket.
+		c := &PaymentCreationInfo{
+			PaymentHash:    paymentHash,
+			Value:          payment.Terms.Value,
+			CreationDate:   payment.CreationDate,
+			PaymentRequest: payment.PaymentRequest,
+		}
+
+		var infoBuf bytes.Buffer
+		if err := serializePaymentCreationInfo(&infoBuf, c); err != nil {
+			return err
+		}
+
+		sourcePubKey, err := sourcePub()
+		if err != nil {
+			return err
+		}
+
+		// Do the same for the PaymentAttemptInfo.
+		totalAmt := payment.Terms.Value + payment.Fee
+		rt := route.Route{
+			TotalTimeLock: payment.TimeLockLength,
+			TotalAmount:   totalAmt,
+			SourcePubKey:  sourcePubKey,
+			Hops:          []*route.Hop{},
+		}
+		for _, hop := range payment.Path {
+			rt.Hops = append(rt.Hops, &route.Hop{
+				PubKeyBytes:  hop,
+				AmtToForward: totalAmt,
+			})
+		}
+
+		// Since the old format didn't store the fee for individual
+		// hops, we let the last hop eat the whole fee for the total to
+		// add up.
+		if len(rt.Hops) > 0 {
+			rt.Hops[len(rt.Hops)-1].AmtToForward = payment.Terms.Value
+		}
+
+		// Since we don't have the session key for old payments, we
+		// create a random one to be able to serialize the attempt
+		// info.
+		priv, _ := btcec.NewPrivateKey(btcec.S256())
+		s := &PaymentAttemptInfo{
+			PaymentID:  0,    // unknown.
+			SessionKey: priv, // unknown.
+			Route:      rt,
+		}
+
+		var attemptBuf bytes.Buffer
+		if err := serializePaymentAttemptInfoMigration9(&attemptBuf, s); err != nil {
+			return err
+		}
+
+		// Reuse the existing payment sequence number.
+		var seqNum [8]byte
+		copy(seqNum[:], k)
+
+		// Create a bucket indexed by the payment hash.
+		bucket, err := newPayments.CreateBucket(paymentHash[:])
+
+		// If the bucket already exists, it means that we are migrating
+		// from a database containing duplicate payments to a payment
+		// hash. To keep this information, we store such duplicate
+		// payments in a sub-bucket.
+		if err == bbolt.ErrBucketExists {
+			pHashBucket := newPayments.Bucket(paymentHash[:])
+
+			// Create a bucket for duplicate payments within this
+			// payment hash's bucket.
+			dup, err := pHashBucket.CreateBucketIfNotExists(
+				paymentDuplicateBucket,
+			)
+			if err != nil {
+				return err
+			}
+
+			// Each duplicate will get its own sub-bucket within
+			// this bucket, so use their sequence number to index
+			// them by.
+			bucket, err = dup.CreateBucket(seqNum[:])
+			if err != nil {
+				return err
+			}
+
+		} else if err != nil {
+			return err
+		}
+
+		// Store the payment's information to the bucket.
+		err = bucket.Put(paymentSequenceKey, seqNum[:])
+		if err != nil {
+			return err
+		}
+
+		err = bucket.Put(paymentCreationInfoKey, infoBuf.Bytes())
+		if err != nil {
+			return err
+		}
+
+		err = bucket.Put(paymentAttemptInfoKey, attemptBuf.Bytes())
+		if err != nil {
+			return err
+		}
+
+		err = bucket.Put(paymentSettleInfoKey, payment.PaymentPreimage[:])
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	// To continue producing unique sequence numbers, we set the sequence
+	// of the new bucket to that of the old one.
+	seq := oldPayments.Sequence()
+	if err := newPayments.SetSequence(seq); err != nil {
+		return err
+	}
+
+	// Now we delete the old buckets. Deleting the payment status buckets
+	// deletes all payment statuses other than Complete.
+	err = tx.DeleteBucket(paymentStatusBucket)
+	if err != nil && err != bbolt.ErrBucketNotFound {
+		return err
+	}
+
+	// Finally delete the old payment bucket.
+	err = tx.DeleteBucket(paymentBucket)
+	if err != nil && err != bbolt.ErrBucketNotFound {
+		return err
+	}
+
+	log.Infof("Migration of outgoing payment bucket structure completed!")
+	return nil
+}

channeldb/migration_01_to_11/migrations_test.go

@@ -0,0 +1,952 @@
+package migration_01_to_11
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/binary"
+	"fmt"
+	"math/rand"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/btcsuite/btcutil"
+	"github.com/coreos/bbolt"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/go-errors/errors"
+	"github.com/lightningnetwork/lnd/lntypes"
+	"github.com/lightningnetwork/lnd/lnwire"
+	"github.com/lightningnetwork/lnd/routing/route"
+)
+
+// TestPaymentStatusesMigration checks that already completed payments will have
+// their payment statuses set to Completed after the migration.
+func TestPaymentStatusesMigration(t *testing.T) {
+	t.Parallel()
+
+	fakePayment := makeFakePayment()
+	paymentHash := sha256.Sum256(fakePayment.PaymentPreimage[:])
+
+	// Add fake payment to test database, verifying that it was created,
+	// that we have only one payment, and its status is not "Completed".
+	beforeMigrationFunc := func(d *DB) {
+		if err := d.addPayment(fakePayment); err != nil {
+			t.Fatalf("unable to add payment: %v", err)
+		}
+
+		payments, err := d.fetchAllPayments()
+		if err != nil {
+			t.Fatalf("unable to fetch payments: %v", err)
+		}
+
+		if len(payments) != 1 {
+			t.Fatalf("wrong qty of paymets: expected 1, got %v",
+				len(payments))
+		}
+
+		paymentStatus, err := d.fetchPaymentStatus(paymentHash)
+		if err != nil {
+			t.Fatalf("unable to fetch payment status: %v", err)
+		}
+
+		// We should receive default status if we have any in database.
+		if paymentStatus != StatusUnknown {
+			t.Fatalf("wrong payment status: expected %v, got %v",
+				StatusUnknown.String(), paymentStatus.String())
+		}
+
+		// Lastly, we'll add a locally-sourced circuit and
+		// non-locally-sourced circuit to the circuit map. The
+		// locally-sourced payment should end up with an InFlight
+		// status, while the other should remain unchanged, which
+		// defaults to Grounded.
+		err = d.Update(func(tx *bbolt.Tx) error {
+			circuits, err := tx.CreateBucketIfNotExists(
+				[]byte("circuit-adds"),
+			)
+			if err != nil {
+				return err
+			}
+
+			groundedKey := make([]byte, 16)
+			binary.BigEndian.PutUint64(groundedKey[:8], 1)
+			binary.BigEndian.PutUint64(groundedKey[8:], 1)
+
+			// Generated using TestHalfCircuitSerialization with nil
+			// ErrorEncrypter, which is the case for locally-sourced
+			// payments. No payment status should end up being set
+			// for this circuit, since the short channel id of the
+			// key is non-zero (e.g., a forwarded circuit). This
+			// will default it to Grounded.
+			groundedCircuit := []byte{
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x01,
+				// start payment hash
+				0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				// end payment hash
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x0f,
+				0x42, 0x40, 0x00,
+			}
+
+			err = circuits.Put(groundedKey, groundedCircuit)
+			if err != nil {
+				return err
+			}
+
+			inFlightKey := make([]byte, 16)
+			binary.BigEndian.PutUint64(inFlightKey[:8], 0)
+			binary.BigEndian.PutUint64(inFlightKey[8:], 1)
+
+			// Generated using TestHalfCircuitSerialization with nil
+			// ErrorEncrypter, which is not the case for forwarded
+			// payments, but should have no impact on the
+			// correctness of the test. The payment status for this
+			// circuit should be set to InFlight, since the short
+			// channel id in the key is 0 (sourceHop).
+			inFlightCircuit := []byte{
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x01,
+				// start payment hash
+				0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				// end payment hash
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x0f,
+				0x42, 0x40, 0x00,
+			}
+
+			return circuits.Put(inFlightKey, inFlightCircuit)
+		})
+		if err != nil {
+			t.Fatalf("unable to add circuit map entry: %v", err)
+		}
+	}
+
+	// Verify that the created payment status is "Completed" for our one
+	// fake payment.
+	afterMigrationFunc := func(d *DB) {
+		meta, err := d.FetchMeta(nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if meta.DbVersionNumber != 1 {
+			t.Fatal("migration 'paymentStatusesMigration' wasn't applied")
+		}
+
+		// Check that our completed payments were migrated.
+		paymentStatus, err := d.fetchPaymentStatus(paymentHash)
+		if err != nil {
+			t.Fatalf("unable to fetch payment status: %v", err)
+		}
+
+		if paymentStatus != StatusSucceeded {
+			t.Fatalf("wrong payment status: expected %v, got %v",
+				StatusSucceeded.String(), paymentStatus.String())
+		}
+
+		inFlightHash := [32]byte{
+			0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		}
+
+		// Check that the locally sourced payment was transitioned to
+		// InFlight.
+		paymentStatus, err = d.fetchPaymentStatus(inFlightHash)
+		if err != nil {
+			t.Fatalf("unable to fetch payment status: %v", err)
+		}
+
+		if paymentStatus != StatusInFlight {
+			t.Fatalf("wrong payment status: expected %v, got %v",
+				StatusInFlight.String(), paymentStatus.String())
+		}
+
+		groundedHash := [32]byte{
+			0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		}
+
+		// Check that non-locally sourced payments remain in the default
+		// Grounded state.
+		paymentStatus, err = d.fetchPaymentStatus(groundedHash)
+		if err != nil {
+			t.Fatalf("unable to fetch payment status: %v", err)
+		}
+
+		if paymentStatus != StatusUnknown {
+			t.Fatalf("wrong payment status: expected %v, got %v",
+				StatusUnknown.String(), paymentStatus.String())
+		}
+	}
+
+	applyMigration(t,
+		beforeMigrationFunc,
+		afterMigrationFunc,
+		paymentStatusesMigration,
+		false)
+}
+
+// TestMigrateOptionalChannelCloseSummaryFields properly converts a
+// ChannelCloseSummary to the v7 format, where optional fields have their
+// presence indicated with boolean markers.
+func TestMigrateOptionalChannelCloseSummaryFields(t *testing.T) {
+	t.Parallel()
+
+	chanState, err := createTestChannelState(nil)
+	if err != nil {
+		t.Fatalf("unable to create channel state: %v", err)
+	}
+
+	var chanPointBuf bytes.Buffer
+	err = writeOutpoint(&chanPointBuf, &chanState.FundingOutpoint)
+	if err != nil {
+		t.Fatalf("unable to write outpoint: %v", err)
+	}
+
+	chanID := chanPointBuf.Bytes()
+
+	testCases := []struct {
+		closeSummary     *ChannelCloseSummary
+		oldSerialization func(c *ChannelCloseSummary) []byte
+	}{
+		{
+			// A close summary where none of the new fields are
+			// set.
+			closeSummary: &ChannelCloseSummary{
+				ChanPoint:      chanState.FundingOutpoint,
+				ShortChanID:    chanState.ShortChanID(),
+				ChainHash:      chanState.ChainHash,
+				ClosingTXID:    testTx.TxHash(),
+				CloseHeight:    100,
+				RemotePub:      chanState.IdentityPub,
+				Capacity:       chanState.Capacity,
+				SettledBalance: btcutil.Amount(50000),
+				CloseType:      RemoteForceClose,
+				IsPending:      true,
+
+				// The last fields will be unset.
+				RemoteCurrentRevocation: nil,
+				LocalChanConfig:         ChannelConfig{},
+				RemoteNextRevocation:    nil,
+			},
+
+			// In the old format the last field written is the
+			// IsPendingField. It should be converted by adding an
+			// extra boolean marker at the end to indicate that the
+			// remaining fields are not there.
+			oldSerialization: func(cs *ChannelCloseSummary) []byte {
+				var buf bytes.Buffer
+				err := WriteElements(&buf, cs.ChanPoint,
+					cs.ShortChanID, cs.ChainHash,
+					cs.ClosingTXID, cs.CloseHeight,
+					cs.RemotePub, cs.Capacity,
+					cs.SettledBalance, cs.TimeLockedBalance,
+					cs.CloseType, cs.IsPending,
+				)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				// For the old format, these are all the fields
+				// that are written.
+				return buf.Bytes()
+			},
+		},
+		{
+			// A close summary where the new fields are present,
+			// but the optional RemoteNextRevocation field is not
+			// set.
+			closeSummary: &ChannelCloseSummary{
+				ChanPoint:               chanState.FundingOutpoint,
+				ShortChanID:             chanState.ShortChanID(),
+				ChainHash:               chanState.ChainHash,
+				ClosingTXID:             testTx.TxHash(),
+				CloseHeight:             100,
+				RemotePub:               chanState.IdentityPub,
+				Capacity:                chanState.Capacity,
+				SettledBalance:          btcutil.Amount(50000),
+				CloseType:               RemoteForceClose,
+				IsPending:               true,
+				RemoteCurrentRevocation: chanState.RemoteCurrentRevocation,
+				LocalChanConfig:         chanState.LocalChanCfg,
+
+				// RemoteNextRevocation is optional, and here
+				// it is not set.
+				RemoteNextRevocation: nil,
+			},
+
+			// In the old format the last field written is the
+			// LocalChanConfig. This indicates that the optional
+			// RemoteNextRevocation field is not present. It should
+			// be converted by adding boolean markers for all these
+			// fields.
+			oldSerialization: func(cs *ChannelCloseSummary) []byte {
+				var buf bytes.Buffer
+				err := WriteElements(&buf, cs.ChanPoint,
+					cs.ShortChanID, cs.ChainHash,
+					cs.ClosingTXID, cs.CloseHeight,
+					cs.RemotePub, cs.Capacity,
+					cs.SettledBalance, cs.TimeLockedBalance,
+					cs.CloseType, cs.IsPending,
+				)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				err = WriteElements(&buf, cs.RemoteCurrentRevocation)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				err = writeChanConfig(&buf, &cs.LocalChanConfig)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				// RemoteNextRevocation is not written.
+				return buf.Bytes()
+			},
+		},
+		{
+			// A close summary where all fields are present.
+			closeSummary: &ChannelCloseSummary{
+				ChanPoint:               chanState.FundingOutpoint,
+				ShortChanID:             chanState.ShortChanID(),
+				ChainHash:               chanState.ChainHash,
+				ClosingTXID:             testTx.TxHash(),
+				CloseHeight:             100,
+				RemotePub:               chanState.IdentityPub,
+				Capacity:                chanState.Capacity,
+				SettledBalance:          btcutil.Amount(50000),
+				CloseType:               RemoteForceClose,
+				IsPending:               true,
+				RemoteCurrentRevocation: chanState.RemoteCurrentRevocation,
+				LocalChanConfig:         chanState.LocalChanCfg,
+
+				// RemoteNextRevocation is optional, and in
+				// this case we set it.
+				RemoteNextRevocation: chanState.RemoteNextRevocation,
+			},
+
+			// In the old format all the fields are written. It
+			// should be converted by adding boolean markers for
+			// all these fields.
+			oldSerialization: func(cs *ChannelCloseSummary) []byte {
+				var buf bytes.Buffer
+				err := WriteElements(&buf, cs.ChanPoint,
+					cs.ShortChanID, cs.ChainHash,
+					cs.ClosingTXID, cs.CloseHeight,
+					cs.RemotePub, cs.Capacity,
+					cs.SettledBalance, cs.TimeLockedBalance,
+					cs.CloseType, cs.IsPending,
+				)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				err = WriteElements(&buf, cs.RemoteCurrentRevocation)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				err = writeChanConfig(&buf, &cs.LocalChanConfig)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				err = WriteElements(&buf, cs.RemoteNextRevocation)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				return buf.Bytes()
+			},
+		},
+	}
+
+	for _, test := range testCases {
+
+		// Before the migration we must add the old format to the DB.
+		beforeMigrationFunc := func(d *DB) {
+
+			// Get the old serialization format for this test's
+			// close summary, and it to the closed channel bucket.
+			old := test.oldSerialization(test.closeSummary)
+			err = d.Update(func(tx *bbolt.Tx) error {
+				closedChanBucket, err := tx.CreateBucketIfNotExists(
+					closedChannelBucket,
+				)
+				if err != nil {
+					return err
+				}
+				return closedChanBucket.Put(chanID, old)
+			})
+			if err != nil {
+				t.Fatalf("unable to add old serialization: %v",
+					err)
+			}
+		}
+
+		// After the migration it should be found in the new format.
+		afterMigrationFunc := func(d *DB) {
+			meta, err := d.FetchMeta(nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if meta.DbVersionNumber != 1 {
+				t.Fatal("migration wasn't applied")
+			}
+
+			// We generate the new serialized version, to check
+			// against what is found in the DB.
+			var b bytes.Buffer
+			err = serializeChannelCloseSummary(&b, test.closeSummary)
+			if err != nil {
+				t.Fatalf("unable to serialize: %v", err)
+			}
+			newSerialization := b.Bytes()
+
+			var dbSummary []byte
+			err = d.View(func(tx *bbolt.Tx) error {
+				closedChanBucket := tx.Bucket(closedChannelBucket)
+				if closedChanBucket == nil {
+					return errors.New("unable to find bucket")
+				}
+
+				// Get the serialized verision from the DB and
+				// make sure it matches what we expected.
+				dbSummary = closedChanBucket.Get(chanID)
+				if !bytes.Equal(dbSummary, newSerialization) {
+					return fmt.Errorf("unexpected new " +
+						"serialization")
+				}
+				return nil
+			})
+			if err != nil {
+				t.Fatalf("unable to view DB: %v", err)
+			}
+
+			// Finally we fetch the deserialized summary from the
+			// DB and check that it is equal to our original one.
+			dbChannels, err := d.FetchClosedChannels(false)
+			if err != nil {
+				t.Fatalf("unable to fetch closed channels: %v",
+					err)
+			}
+
+			if len(dbChannels) != 1 {
+				t.Fatalf("expected 1 closed channels, found %v",
+					len(dbChannels))
+			}
+
+			dbChan := dbChannels[0]
+			if !reflect.DeepEqual(dbChan, test.closeSummary) {
+				dbChan.RemotePub.Curve = nil
+				test.closeSummary.RemotePub.Curve = nil
+				t.Fatalf("not equal: %v vs %v",
+					spew.Sdump(dbChan),
+					spew.Sdump(test.closeSummary))
+			}
+
+		}
+
+		applyMigration(t,
+			beforeMigrationFunc,
+			afterMigrationFunc,
+			migrateOptionalChannelCloseSummaryFields,
+			false)
+	}
+}
+
+// TestMigrateGossipMessageStoreKeys ensures that the migration to the new
+// gossip message store key format is successful/unsuccessful under various
+// scenarios.
+func TestMigrateGossipMessageStoreKeys(t *testing.T) {
+	t.Parallel()
+
+	// Construct the message which we'll use to test the migration, along
+	// with its old and new key formats.
+	shortChanID := lnwire.ShortChannelID{BlockHeight: 10}
+	msg := &lnwire.AnnounceSignatures{ShortChannelID: shortChanID}
+
+	var oldMsgKey [33 + 8]byte
+	copy(oldMsgKey[:33], pubKey.SerializeCompressed())
+	binary.BigEndian.PutUint64(oldMsgKey[33:41], shortChanID.ToUint64())
+
+	var newMsgKey [33 + 8 + 2]byte
+	copy(newMsgKey[:41], oldMsgKey[:])
+	binary.BigEndian.PutUint16(newMsgKey[41:43], uint16(msg.MsgType()))
+
+	// Before the migration, we'll create the bucket where the messages
+	// should live and insert them.
+	beforeMigration := func(db *DB) {
+		var b bytes.Buffer
+		if err := msg.Encode(&b, 0); err != nil {
+			t.Fatalf("unable to serialize message: %v", err)
+		}
+
+		err := db.Update(func(tx *bbolt.Tx) error {
+			messageStore, err := tx.CreateBucketIfNotExists(
+				messageStoreBucket,
+			)
+			if err != nil {
+				return err
+			}
+
+			return messageStore.Put(oldMsgKey[:], b.Bytes())
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// After the migration, we'll make sure that:
+	//   1. We cannot find the message under its old key.
+	//   2. We can find the message under its new key.
+	//   3. The message matches the original.
+	afterMigration := func(db *DB) {
+		meta, err := db.FetchMeta(nil)
+		if err != nil {
+			t.Fatalf("unable to fetch db version: %v", err)
+		}
+		if meta.DbVersionNumber != 1 {
+			t.Fatalf("migration should have succeeded but didn't")
+		}
+
+		var rawMsg []byte
+		err = db.View(func(tx *bbolt.Tx) error {
+			messageStore := tx.Bucket(messageStoreBucket)
+			if messageStore == nil {
+				return errors.New("message store bucket not " +
+					"found")
+			}
+			rawMsg = messageStore.Get(oldMsgKey[:])
+			if rawMsg != nil {
+				t.Fatal("expected to not find message under " +
+					"old key, but did")
+			}
+			rawMsg = messageStore.Get(newMsgKey[:])
+			if rawMsg == nil {
+				return fmt.Errorf("expected to find message " +
+					"under new key, but didn't")
+			}
+
+			return nil
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		gotMsg, err := lnwire.ReadMessage(bytes.NewReader(rawMsg), 0)
+		if err != nil {
+			t.Fatalf("unable to deserialize raw message: %v", err)
+		}
+		if !reflect.DeepEqual(msg, gotMsg) {
+			t.Fatalf("expected message: %v\ngot message: %v",
+				spew.Sdump(msg), spew.Sdump(gotMsg))
+		}
+	}
+
+	applyMigration(
+		t, beforeMigration, afterMigration,
+		migrateGossipMessageStoreKeys, false,
+	)
+}
+
+// TestOutgoingPaymentsMigration checks that OutgoingPayments are migrated to a
+// new bucket structure after the migration.
+func TestOutgoingPaymentsMigration(t *testing.T) {
+	t.Parallel()
+
+	const numPayments = 4
+	var oldPayments []*outgoingPayment
+
+	// Add fake payments to test database, verifying that it was created.
+	beforeMigrationFunc := func(d *DB) {
+		for i := 0; i < numPayments; i++ {
+			var p *outgoingPayment
+			var err error
+
+			// We fill the database with random payments. For the
+			// very last one we'll use a duplicate of the first, to
+			// ensure we are able to handle migration from a
+			// database that has copies.
+			if i < numPayments-1 {
+				p, err = makeRandomFakePayment()
+				if err != nil {
+					t.Fatalf("unable to create payment: %v",
+						err)
+				}
+			} else {
+				p = oldPayments[0]
+			}
+
+			if err := d.addPayment(p); err != nil {
+				t.Fatalf("unable to add payment: %v", err)
+			}
+
+			oldPayments = append(oldPayments, p)
+		}
+
+		payments, err := d.fetchAllPayments()
+		if err != nil {
+			t.Fatalf("unable to fetch payments: %v", err)
+		}
+
+		if len(payments) != numPayments {
+			t.Fatalf("wrong qty of paymets: expected %d got %v",
+				numPayments, len(payments))
+		}
+	}
+
+	// Verify that all payments were migrated.
+	afterMigrationFunc := func(d *DB) {
+		meta, err := d.FetchMeta(nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if meta.DbVersionNumber != 1 {
+			t.Fatal("migration 'paymentStatusesMigration' wasn't applied")
+		}
+
+		sentPayments, err := d.fetchPaymentsMigration9()
+		if err != nil {
+			t.Fatalf("unable to fetch sent payments: %v", err)
+		}
+
+		if len(sentPayments) != numPayments {
+			t.Fatalf("expected %d payments, got %d", numPayments,
+				len(sentPayments))
+		}
+
+		graph := d.ChannelGraph()
+		sourceNode, err := graph.SourceNode()
+		if err != nil {
+			t.Fatalf("unable to fetch source node: %v", err)
+		}
+
+		for i, p := range sentPayments {
+			// The payment status should be Completed.
+			if p.Status != StatusSucceeded {
+				t.Fatalf("expected Completed, got %v", p.Status)
+			}
+
+			// Check that the sequence number is preserved. They
+			// start counting at 1.
+			if p.sequenceNum != uint64(i+1) {
+				t.Fatalf("expected seqnum %d, got %d", i,
+					p.sequenceNum)
+			}
+
+			// Order of payments should be be preserved.
+			old := oldPayments[i]
+
+			// Check the individial fields.
+			if p.Info.Value != old.Terms.Value {
+				t.Fatalf("value mismatch")
+			}
+
+			if p.Info.CreationDate != old.CreationDate {
+				t.Fatalf("date mismatch")
+			}
+
+			if !bytes.Equal(p.Info.PaymentRequest, old.PaymentRequest) {
+				t.Fatalf("payreq mismatch")
+			}
+
+			if *p.PaymentPreimage != old.PaymentPreimage {
+				t.Fatalf("preimage mismatch")
+			}
+
+			if p.Attempt.Route.TotalFees() != old.Fee {
+				t.Fatalf("Fee mismatch")
+			}
+
+			if p.Attempt.Route.TotalAmount != old.Fee+old.Terms.Value {
+				t.Fatalf("Total amount mismatch")
+			}
+
+			if p.Attempt.Route.TotalTimeLock != old.TimeLockLength {
+				t.Fatalf("timelock mismatch")
+			}
+
+			if p.Attempt.Route.SourcePubKey != sourceNode.PubKeyBytes {
+				t.Fatalf("source mismatch: %x vs %x",
+					p.Attempt.Route.SourcePubKey[:],
+					sourceNode.PubKeyBytes[:])
+			}
+
+			for i, hop := range old.Path {
+				if hop != p.Attempt.Route.Hops[i].PubKeyBytes {
+					t.Fatalf("path mismatch")
+				}
+			}
+		}
+
+		// Finally, check that the payment sequence number is updated
+		// to reflect the migrated payments.
+		err = d.View(func(tx *bbolt.Tx) error {
+			payments := tx.Bucket(paymentsRootBucket)
+			if payments == nil {
+				return fmt.Errorf("payments bucket not found")
+			}
+
+			seq := payments.Sequence()
+			if seq != numPayments {
+				return fmt.Errorf("expected sequence to be "+
+					"%d, got %d", numPayments, seq)
+			}
+
+			return nil
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	applyMigration(t,
+		beforeMigrationFunc,
+		afterMigrationFunc,
+		migrateOutgoingPayments,
+		false)
+}
+
+func makeRandPaymentCreationInfo() (*PaymentCreationInfo, error) {
+	var payHash lntypes.Hash
+	if _, err := rand.Read(payHash[:]); err != nil {
+		return nil, err
+	}
+
+	return &PaymentCreationInfo{
+		PaymentHash:    payHash,
+		Value:          lnwire.MilliSatoshi(rand.Int63()),
+		CreationDate:   time.Now(),
+		PaymentRequest: []byte("test"),
+	}, nil
+}
+
+// TestPaymentRouteSerialization tests that we're able to properly migrate
+// existing payments on disk that contain the traversed routes to the new
+// routing format which supports the TLV payloads. We also test that the
+// migration is able to handle duplicate payment attempts.
+func TestPaymentRouteSerialization(t *testing.T) {
+	t.Parallel()
+
+	legacyHop1 := &route.Hop{
+		PubKeyBytes:      route.NewVertex(pub),
+		ChannelID:        12345,
+		OutgoingTimeLock: 111,
+		LegacyPayload:    true,
+		AmtToForward:     555,
+	}
+	legacyHop2 := &route.Hop{
+		PubKeyBytes:      route.NewVertex(pub),
+		ChannelID:        12345,
+		OutgoingTimeLock: 111,
+		LegacyPayload:    true,
+		AmtToForward:     555,
+	}
+	legacyRoute := route.Route{
+		TotalTimeLock: 123,
+		TotalAmount:   1234567,
+		SourcePubKey:  route.NewVertex(pub),
+		Hops:          []*route.Hop{legacyHop1, legacyHop2},
+	}
+
+	const numPayments = 4
+	var oldPayments []*Payment
+
+	sharedPayAttempt := PaymentAttemptInfo{
+		PaymentID:  1,
+		SessionKey: priv,
+		Route:      legacyRoute,
+	}
+
+	// We'll first add a series of fake payments, using the existing legacy
+	// serialization format.
+	beforeMigrationFunc := func(d *DB) {
+		err := d.Update(func(tx *bbolt.Tx) error {
+			paymentsBucket, err := tx.CreateBucket(
+				paymentsRootBucket,
+			)
+			if err != nil {
+				t.Fatalf("unable to create new payments "+
+					"bucket: %v", err)
+			}
+
+			for i := 0; i < numPayments; i++ {
+				var seqNum [8]byte
+				byteOrder.PutUint64(seqNum[:], uint64(i))
+
+				// All payments will be randomly generated,
+				// other than the final payment. We'll force
+				// the final payment to re-use an existing
+				// payment hash so we can insert it into the
+				// duplicate payment hash bucket.
+				var payInfo *PaymentCreationInfo
+				if i < numPayments-1 {
+					payInfo, err = makeRandPaymentCreationInfo()
+					if err != nil {
+						t.Fatalf("unable to create "+
+							"payment: %v", err)
+					}
+				} else {
+					payInfo = oldPayments[0].Info
+				}
+
+				// Next, legacy encoded when needed, we'll
+				// serialize the info and the attempt.
+				var payInfoBytes bytes.Buffer
+				err = serializePaymentCreationInfo(
+					&payInfoBytes, payInfo,
+				)
+				if err != nil {
+					t.Fatalf("unable to encode pay "+
+						"info: %v", err)
+				}
+				var payAttemptBytes bytes.Buffer
+				err = serializePaymentAttemptInfoLegacy(
+					&payAttemptBytes, &sharedPayAttempt,
+				)
+				if err != nil {
+					t.Fatalf("unable to encode payment attempt: "+
+						"%v", err)
+				}
+
+				// Before we write to disk, we'll need to fetch
+				// the proper bucket. If this is the duplicate
+				// payment, then we'll grab the dup bucket,
+				// otherwise, we'll use the top level bucket.
+				var payHashBucket *bbolt.Bucket
+				if i < numPayments-1 {
+					payHashBucket, err = paymentsBucket.CreateBucket(
+						payInfo.PaymentHash[:],
+					)
+					if err != nil {
+						t.Fatalf("unable to create payments bucket: %v", err)
+					}
+				} else {
+					payHashBucket = paymentsBucket.Bucket(
+						payInfo.PaymentHash[:],
+					)
+					dupPayBucket, err := payHashBucket.CreateBucket(
+						paymentDuplicateBucket,
+					)
+					if err != nil {
+						t.Fatalf("unable to create "+
+							"dup hash bucket: %v", err)
+					}
+
+					payHashBucket, err = dupPayBucket.CreateBucket(
+						seqNum[:],
+					)
+					if err != nil {
+						t.Fatalf("unable to make dup "+
+							"bucket: %v", err)
+					}
+				}
+
+				err = payHashBucket.Put(paymentSequenceKey, seqNum[:])
+				if err != nil {
+					t.Fatalf("unable to write seqno: %v", err)
+				}
+
+				err = payHashBucket.Put(
+					paymentCreationInfoKey, payInfoBytes.Bytes(),
+				)
+				if err != nil {
+					t.Fatalf("unable to write creation "+
+						"info: %v", err)
+				}
+
+				err = payHashBucket.Put(
+					paymentAttemptInfoKey, payAttemptBytes.Bytes(),
+				)
+				if err != nil {
+					t.Fatalf("unable to write attempt "+
+						"info: %v", err)
+				}
+
+				oldPayments = append(oldPayments, &Payment{
+					Info:    payInfo,
+					Attempt: &sharedPayAttempt,
+				})
+			}
+
+			return nil
+		})
+		if err != nil {
+			t.Fatalf("unable to create test payments: %v", err)
+		}
+	}
+
+	afterMigrationFunc := func(d *DB) {
+		newPayments, err := d.FetchPayments()
+		if err != nil {
+			t.Fatalf("unable to fetch new payments: %v", err)
+		}
+
+		if len(newPayments) != numPayments {
+			t.Fatalf("expected %d payments, got %d", numPayments,
+				len(newPayments))
+		}
+
+		for i, p := range newPayments {
+			// Order of payments should be be preserved.
+			old := oldPayments[i]
+
+			if p.Attempt.PaymentID != old.Attempt.PaymentID {
+				t.Fatalf("wrong pay ID: expected %v, got %v",
+					p.Attempt.PaymentID,
+					old.Attempt.PaymentID)
+			}
+
+			if p.Attempt.Route.TotalFees() != old.Attempt.Route.TotalFees() {
+				t.Fatalf("Fee mismatch")
+			}
+
+			if p.Attempt.Route.TotalAmount != old.Attempt.Route.TotalAmount {
+				t.Fatalf("Total amount mismatch")
+			}
+
+			if p.Attempt.Route.TotalTimeLock != old.Attempt.Route.TotalTimeLock {
+				t.Fatalf("timelock mismatch")
+			}
+
+			if p.Attempt.Route.SourcePubKey != old.Attempt.Route.SourcePubKey {
+				t.Fatalf("source mismatch: %x vs %x",
+					p.Attempt.Route.SourcePubKey[:],
+					old.Attempt.Route.SourcePubKey[:])
+			}
+
+			for i, hop := range p.Attempt.Route.Hops {
+				if !reflect.DeepEqual(hop, legacyRoute.Hops[i]) {
+					t.Fatalf("hop mismatch")
+				}
+			}
+		}
+	}
+
+	applyMigration(t,
+		beforeMigrationFunc,
+		afterMigrationFunc,
+		migrateRouteSerialization,
+		false)
+}

channeldb/migration_01_to_11/nodes.go

@@ -0,0 +1,316 @@
+package migration_01_to_11
+
+import (
+	"bytes"
+	"io"
+	"net"
+	"time"
+
+	"github.com/btcsuite/btcd/btcec"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/coreos/bbolt"
+)
+
+var (
+	// nodeInfoBucket stores metadata pertaining to nodes that we've had
+	// direct channel-based correspondence with. This bucket allows one to
+	// query for all open channels pertaining to the node by exploring each
+	// node's sub-bucket within the openChanBucket.
+	nodeInfoBucket = []byte("nib")
+)
+
+// LinkNode stores metadata related to node's that we have/had a direct
+// channel open with. Information such as the Bitcoin network the node
+// advertised, and its identity public key are also stored. Additionally, this
+// struct and the bucket its stored within have store data similar to that of
+// Bitcoin's addrmanager. The TCP address information stored within the struct
+// can be used to establish persistent connections will all channel
+// counterparties on daemon startup.
+//
+// TODO(roasbeef): also add current OnionKey plus rotation schedule?
+// TODO(roasbeef): add bitfield for supported services
+//  * possibly add a wire.NetAddress type, type
+type LinkNode struct {
+	// Network indicates the Bitcoin network that the LinkNode advertises
+	// for incoming channel creation.
+	Network wire.BitcoinNet
+
+	// IdentityPub is the node's current identity public key. Any
+	// channel/topology related information received by this node MUST be
+	// signed by this public key.
+	IdentityPub *btcec.PublicKey
+
+	// LastSeen tracks the last time this node was seen within the network.
+	// A node should be marked as seen if the daemon either is able to
+	// establish an outgoing connection to the node or receives a new
+	// incoming connection from the node. This timestamp (stored in unix
+	// epoch) may be used within a heuristic which aims to determine when a
+	// channel should be unilaterally closed due to inactivity.
+	//
+	// TODO(roasbeef): replace with block hash/height?
+	//  * possibly add a time-value metric into the heuristic?
+	LastSeen time.Time
+
+	// Addresses is a list of IP address in which either we were able to
+	// reach the node over in the past, OR we received an incoming
+	// authenticated connection for the stored identity public key.
+	Addresses []net.Addr
+
+	db *DB
+}
+
+// NewLinkNode creates a new LinkNode from the provided parameters, which is
+// backed by an instance of channeldb.
+func (db *DB) NewLinkNode(bitNet wire.BitcoinNet, pub *btcec.PublicKey,
+	addrs ...net.Addr) *LinkNode {
+
+	return &LinkNode{
+		Network:     bitNet,
+		IdentityPub: pub,
+		LastSeen:    time.Now(),
+		Addresses:   addrs,
+		db:          db,
+	}
+}
+
+// UpdateLastSeen updates the last time this node was directly encountered on
+// the Lightning Network.
+func (l *LinkNode) UpdateLastSeen(lastSeen time.Time) error {
+	l.LastSeen = lastSeen
+
+	return l.Sync()
+}
+
+// AddAddress appends the specified TCP address to the list of known addresses
+// this node is/was known to be reachable at.
+func (l *LinkNode) AddAddress(addr net.Addr) error {
+	for _, a := range l.Addresses {
+		if a.String() == addr.String() {
+			return nil
+		}
+	}
+
+	l.Addresses = append(l.Addresses, addr)
+
+	return l.Sync()
+}
+
+// Sync performs a full database sync which writes the current up-to-date data
+// within the struct to the database.
+func (l *LinkNode) Sync() error {
+
+	// Finally update the database by storing the link node and updating
+	// any relevant indexes.
+	return l.db.Update(func(tx *bbolt.Tx) error {
+		nodeMetaBucket := tx.Bucket(nodeInfoBucket)
+		if nodeMetaBucket == nil {
+			return ErrLinkNodesNotFound
+		}
+
+		return putLinkNode(nodeMetaBucket, l)
+	})
+}
+
+// putLinkNode serializes then writes the encoded version of the passed link
+// node into the nodeMetaBucket. This function is provided in order to allow
+// the ability to re-use a database transaction across many operations.
+func putLinkNode(nodeMetaBucket *bbolt.Bucket, l *LinkNode) error {
+	// First serialize the LinkNode into its raw-bytes encoding.
+	var b bytes.Buffer
+	if err := serializeLinkNode(&b, l); err != nil {
+		return err
+	}
+
+	// Finally insert the link-node into the node metadata bucket keyed
+	// according to the its pubkey serialized in compressed form.
+	nodePub := l.IdentityPub.SerializeCompressed()
+	return nodeMetaBucket.Put(nodePub, b.Bytes())
+}
+
+// DeleteLinkNode removes the link node with the given identity from the
+// database.
+func (db *DB) DeleteLinkNode(identity *btcec.PublicKey) error {
+	return db.Update(func(tx *bbolt.Tx) error {
+		return db.deleteLinkNode(tx, identity)
+	})
+}
+
+func (db *DB) deleteLinkNode(tx *bbolt.Tx, identity *btcec.PublicKey) error {
+	nodeMetaBucket := tx.Bucket(nodeInfoBucket)
+	if nodeMetaBucket == nil {
+		return ErrLinkNodesNotFound
+	}
+
+	pubKey := identity.SerializeCompressed()
+	return nodeMetaBucket.Delete(pubKey)
+}
+
+// FetchLinkNode attempts to lookup the data for a LinkNode based on a target
+// identity public key. If a particular LinkNode for the passed identity public
+// key cannot be found, then ErrNodeNotFound if returned.
+func (db *DB) FetchLinkNode(identity *btcec.PublicKey) (*LinkNode, error) {
+	var linkNode *LinkNode
+	err := db.View(func(tx *bbolt.Tx) error {
+		node, err := fetchLinkNode(tx, identity)
+		if err != nil {
+			return err
+		}
+
+		linkNode = node
+		return nil
+	})
+
+	return linkNode, err
+}
+
+func fetchLinkNode(tx *bbolt.Tx, targetPub *btcec.PublicKey) (*LinkNode, error) {
+	// First fetch the bucket for storing node metadata, bailing out early
+	// if it hasn't been created yet.
+	nodeMetaBucket := tx.Bucket(nodeInfoBucket)
+	if nodeMetaBucket == nil {
+		return nil, ErrLinkNodesNotFound
+	}
+
+	// If a link node for that particular public key cannot be located,
+	// then exit early with an ErrNodeNotFound.
+	pubKey := targetPub.SerializeCompressed()
+	nodeBytes := nodeMetaBucket.Get(pubKey)
+	if nodeBytes == nil {
+		return nil, ErrNodeNotFound
+	}
+
+	// Finally, decode and allocate a fresh LinkNode object to be returned
+	// to the caller.
+	nodeReader := bytes.NewReader(nodeBytes)
+	return deserializeLinkNode(nodeReader)
+}
+
+// TODO(roasbeef): update link node addrs in server upon connection
+
+// FetchAllLinkNodes starts a new database transaction to fetch all nodes with
+// whom we have active channels with.
+func (db *DB) FetchAllLinkNodes() ([]*LinkNode, error) {
+	var linkNodes []*LinkNode
+	err := db.View(func(tx *bbolt.Tx) error {
+		nodes, err := db.fetchAllLinkNodes(tx)
+		if err != nil {
+			return err
+		}
+
+		linkNodes = nodes
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return linkNodes, nil
+}
+
+// fetchAllLinkNodes uses an existing database transaction to fetch all nodes
+// with whom we have active channels with.
+func (db *DB) fetchAllLinkNodes(tx *bbolt.Tx) ([]*LinkNode, error) {
+	nodeMetaBucket := tx.Bucket(nodeInfoBucket)
+	if nodeMetaBucket == nil {
+		return nil, ErrLinkNodesNotFound
+	}
+
+	var linkNodes []*LinkNode
+	err := nodeMetaBucket.ForEach(func(k, v []byte) error {
+		if v == nil {
+			return nil
+		}
+
+		nodeReader := bytes.NewReader(v)
+		linkNode, err := deserializeLinkNode(nodeReader)
+		if err != nil {
+			return err
+		}
+
+		linkNodes = append(linkNodes, linkNode)
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return linkNodes, nil
+}
+
+func serializeLinkNode(w io.Writer, l *LinkNode) error {
+	var buf [8]byte
+
+	byteOrder.PutUint32(buf[:4], uint32(l.Network))
+	if _, err := w.Write(buf[:4]); err != nil {
+		return err
+	}
+
+	serializedID := l.IdentityPub.SerializeCompressed()
+	if _, err := w.Write(serializedID); err != nil {
+		return err
+	}
+
+	seenUnix := uint64(l.LastSeen.Unix())
+	byteOrder.PutUint64(buf[:], seenUnix)
+	if _, err := w.Write(buf[:]); err != nil {
+		return err
+	}
+
+	numAddrs := uint32(len(l.Addresses))
+	byteOrder.PutUint32(buf[:4], numAddrs)
+	if _, err := w.Write(buf[:4]); err != nil {
+		return err
+	}
+
+	for _, addr := range l.Addresses {
+		if err := serializeAddr(w, addr); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func deserializeLinkNode(r io.Reader) (*LinkNode, error) {
+	var (
+		err error
+		buf [8]byte
+	)
+
+	node := &LinkNode{}
+
+	if _, err := io.ReadFull(r, buf[:4]); err != nil {
+		return nil, err
+	}
+	node.Network = wire.BitcoinNet(byteOrder.Uint32(buf[:4]))
+
+	var pub [33]byte
+	if _, err := io.ReadFull(r, pub[:]); err != nil {
+		return nil, err
+	}
+	node.IdentityPub, err = btcec.ParsePubKey(pub[:], btcec.S256())
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err := io.ReadFull(r, buf[:]); err != nil {
+		return nil, err
+	}
+	node.LastSeen = time.Unix(int64(byteOrder.Uint64(buf[:])), 0)
+
+	if _, err := io.ReadFull(r, buf[:4]); err != nil {
+		return nil, err
+	}
+	numAddrs := byteOrder.Uint32(buf[:4])
+
+	node.Addresses = make([]net.Addr, numAddrs)
+	for i := uint32(0); i < numAddrs; i++ {
+		addr, err := deserializeAddr(r)
+		if err != nil {
+			return nil, err
+		}
+		node.Addresses[i] = addr
+	}
+
+	return node, nil
+}

channeldb/migration_01_to_11/nodes_test.go

@@ -0,0 +1,140 @@
+package migration_01_to_11
+
+import (
+	"bytes"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/btcsuite/btcd/btcec"
+	"github.com/btcsuite/btcd/wire"
+)
+
+func TestLinkNodeEncodeDecode(t *testing.T) {
+	t.Parallel()
+
+	cdb, cleanUp, err := makeTestDB()
+	if err != nil {
+		t.Fatalf("unable to make test database: %v", err)
+	}
+	defer cleanUp()
+
+	// First we'll create some initial data to use for populating our test
+	// LinkNode instances.
+	_, pub1 := btcec.PrivKeyFromBytes(btcec.S256(), key[:])
+	_, pub2 := btcec.PrivKeyFromBytes(btcec.S256(), rev[:])
+	addr1, err := net.ResolveTCPAddr("tcp", "10.0.0.1:9000")
+	if err != nil {
+		t.Fatalf("unable to create test addr: %v", err)
+	}
+	addr2, err := net.ResolveTCPAddr("tcp", "10.0.0.2:9000")
+	if err != nil {
+		t.Fatalf("unable to create test addr: %v", err)
+	}
+
+	// Create two fresh link node instances with the above dummy data, then
+	// fully sync both instances to disk.
+	node1 := cdb.NewLinkNode(wire.MainNet, pub1, addr1)
+	node2 := cdb.NewLinkNode(wire.TestNet3, pub2, addr2)
+	if err := node1.Sync(); err != nil {
+		t.Fatalf("unable to sync node: %v", err)
+	}
+	if err := node2.Sync(); err != nil {
+		t.Fatalf("unable to sync node: %v", err)
+	}
+
+	// Fetch all current link nodes from the database, they should exactly
+	// match the two created above.
+	originalNodes := []*LinkNode{node2, node1}
+	linkNodes, err := cdb.FetchAllLinkNodes()
+	if err != nil {
+		t.Fatalf("unable to fetch nodes: %v", err)
+	}
+	for i, node := range linkNodes {
+		if originalNodes[i].Network != node.Network {
+			t.Fatalf("node networks don't match: expected %v, got %v",
+				originalNodes[i].Network, node.Network)
+		}
+
+		originalPubkey := originalNodes[i].IdentityPub.SerializeCompressed()
+		dbPubkey := node.IdentityPub.SerializeCompressed()
+		if !bytes.Equal(originalPubkey, dbPubkey) {
+			t.Fatalf("node pubkeys don't match: expected %x, got %x",
+				originalPubkey, dbPubkey)
+		}
+		if originalNodes[i].LastSeen.Unix() != node.LastSeen.Unix() {
+			t.Fatalf("last seen timestamps don't match: expected %v got %v",
+				originalNodes[i].LastSeen.Unix(), node.LastSeen.Unix())
+		}
+		if originalNodes[i].Addresses[0].String() != node.Addresses[0].String() {
+			t.Fatalf("addresses don't match: expected %v, got %v",
+				originalNodes[i].Addresses, node.Addresses)
+		}
+	}
+
+	// Next, we'll exercise the methods to append additional IP
+	// addresses, and also to update the last seen time.
+	if err := node1.UpdateLastSeen(time.Now()); err != nil {
+		t.Fatalf("unable to update last seen: %v", err)
+	}
+	if err := node1.AddAddress(addr2); err != nil {
+		t.Fatalf("unable to update addr: %v", err)
+	}
+
+	// Fetch the same node from the database according to its public key.
+	node1DB, err := cdb.FetchLinkNode(pub1)
+	if err != nil {
+		t.Fatalf("unable to find node: %v", err)
+	}
+
+	// Both the last seen timestamp and the list of reachable addresses for
+	// the node should be updated.
+	if node1DB.LastSeen.Unix() != node1.LastSeen.Unix() {
+		t.Fatalf("last seen timestamps don't match: expected %v got %v",
+			node1.LastSeen.Unix(), node1DB.LastSeen.Unix())
+	}
+	if len(node1DB.Addresses) != 2 {
+		t.Fatalf("wrong length for node1 addresses: expected %v, got %v",
+			2, len(node1DB.Addresses))
+	}
+	if node1DB.Addresses[0].String() != addr1.String() {
+		t.Fatalf("wrong address for node: expected %v, got %v",
+			addr1.String(), node1DB.Addresses[0].String())
+	}
+	if node1DB.Addresses[1].String() != addr2.String() {
+		t.Fatalf("wrong address for node: expected %v, got %v",
+			addr2.String(), node1DB.Addresses[1].String())
+	}
+}
+
+func TestDeleteLinkNode(t *testing.T) {
+	t.Parallel()
+
+	cdb, cleanUp, err := makeTestDB()
+	if err != nil {
+		t.Fatalf("unable to make test database: %v", err)
+	}
+	defer cleanUp()
+
+	_, pubKey := btcec.PrivKeyFromBytes(btcec.S256(), key[:])
+	addr := &net.TCPAddr{
+		IP:   net.ParseIP("127.0.0.1"),
+		Port: 1337,
+	}
+	linkNode := cdb.NewLinkNode(wire.TestNet3, pubKey, addr)
+	if err := linkNode.Sync(); err != nil {
+		t.Fatalf("unable to write link node to db: %v", err)
+	}
+
+	if _, err := cdb.FetchLinkNode(pubKey); err != nil {
+		t.Fatalf("unable to find link node: %v", err)
+	}
+
+	if err := cdb.DeleteLinkNode(pubKey); err != nil {
+		t.Fatalf("unable to delete link node from db: %v", err)
+	}
+
+	if _, err := cdb.FetchLinkNode(pubKey); err == nil {
+		t.Fatal("should not have found link node in db, but did")
+	}
+}

channeldb/migration_01_to_11/options.go

@@ -0,0 +1,62 @@
+package migration_01_to_11
+
+const (
+	// DefaultRejectCacheSize is the default number of rejectCacheEntries to
+	// cache for use in the rejection cache of incoming gossip traffic. This
+	// produces a cache size of around 1MB.
+	DefaultRejectCacheSize = 50000
+
+	// DefaultChannelCacheSize is the default number of ChannelEdges cached
+	// in order to reply to gossip queries. This produces a cache size of
+	// around 40MB.
+	DefaultChannelCacheSize = 20000
+)
+
+// Options holds parameters for tuning and customizing a channeldb.DB.
+type Options struct {
+	// RejectCacheSize is the maximum number of rejectCacheEntries to hold
+	// in the rejection cache.
+	RejectCacheSize int
+
+	// ChannelCacheSize is the maximum number of ChannelEdges to hold in the
+	// channel cache.
+	ChannelCacheSize int
+
+	// NoFreelistSync, if true, prevents the database from syncing its
+	// freelist to disk, resulting in improved performance at the expense of
+	// increased startup time.
+	NoFreelistSync bool
+}
+
+// DefaultOptions returns an Options populated with default values.
+func DefaultOptions() Options {
+	return Options{
+		RejectCacheSize:  DefaultRejectCacheSize,
+		ChannelCacheSize: DefaultChannelCacheSize,
+		NoFreelistSync:   true,
+	}
+}
+
+// OptionModifier is a function signature for modifying the default Options.
+type OptionModifier func(*Options)
+
+// OptionSetRejectCacheSize sets the RejectCacheSize to n.
+func OptionSetRejectCacheSize(n int) OptionModifier {
+	return func(o *Options) {
+		o.RejectCacheSize = n
+	}
+}
+
+// OptionSetChannelCacheSize sets the ChannelCacheSize to n.
+func OptionSetChannelCacheSize(n int) OptionModifier {
+	return func(o *Options) {
+		o.ChannelCacheSize = n
+	}
+}
+
+// OptionSetSyncFreelist allows the database to sync its freelist.
+func OptionSetSyncFreelist(b bool) OptionModifier {
+	return func(o *Options) {
+		o.NoFreelistSync = !b
+	}
+}

channeldb/migration_01_to_11/payment_control.go

@@ -0,0 +1,497 @@
+package migration_01_to_11
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+
+	"github.com/coreos/bbolt"
+	"github.com/lightningnetwork/lnd/lntypes"
+	"github.com/lightningnetwork/lnd/routing/route"
+)
+
+var (
+	// ErrAlreadyPaid signals we have already paid this payment hash.
+	ErrAlreadyPaid = errors.New("invoice is already paid")
+
+	// ErrPaymentInFlight signals that payment for this payment hash is
+	// already "in flight" on the network.
+	ErrPaymentInFlight = errors.New("payment is in transition")
+
+	// ErrPaymentNotInitiated is returned  if payment wasn't initiated in
+	// switch.
+	ErrPaymentNotInitiated = errors.New("payment isn't initiated")
+
+	// ErrPaymentAlreadySucceeded is returned in the event we attempt to
+	// change the status of a payment already succeeded.
+	ErrPaymentAlreadySucceeded = errors.New("payment is already succeeded")
+
+	// ErrPaymentAlreadyFailed is returned in the event we attempt to
+	// re-fail a failed payment.
+	ErrPaymentAlreadyFailed = errors.New("payment has already failed")
+
+	// ErrUnknownPaymentStatus is returned when we do not recognize the
+	// existing state of a payment.
+	ErrUnknownPaymentStatus = errors.New("unknown payment status")
+
+	// errNoAttemptInfo is returned when no attempt info is stored yet.
+	errNoAttemptInfo = errors.New("unable to find attempt info for " +
+		"inflight payment")
+)
+
+// PaymentControl implements persistence for payments and payment attempts.
+type PaymentControl struct {
+	db *DB
+}
+
+// NewPaymentControl creates a new instance of the PaymentControl.
+func NewPaymentControl(db *DB) *PaymentControl {
+	return &PaymentControl{
+		db: db,
+	}
+}
+
+// InitPayment checks or records the given PaymentCreationInfo with the DB,
+// making sure it does not already exist as an in-flight payment. Then this
+// method returns successfully, the payment is guranteeed to be in the InFlight
+// state.
+func (p *PaymentControl) InitPayment(paymentHash lntypes.Hash,
+	info *PaymentCreationInfo) error {
+
+	var b bytes.Buffer
+	if err := serializePaymentCreationInfo(&b, info); err != nil {
+		return err
+	}
+	infoBytes := b.Bytes()
+
+	var updateErr error
+	err := p.db.Batch(func(tx *bbolt.Tx) error {
+		// Reset the update error, to avoid carrying over an error
+		// from a previous execution of the batched db transaction.
+		updateErr = nil
+
+		bucket, err := createPaymentBucket(tx, paymentHash)
+		if err != nil {
+			return err
+		}
+
+		// Get the existing status of this payment, if any.
+		paymentStatus := fetchPaymentStatus(bucket)
+
+		switch paymentStatus {
+
+		// We allow retrying failed payments.
+		case StatusFailed:
+
+		// This is a new payment that is being initialized for the
+		// first time.
+		case StatusUnknown:
+
+		// We already have an InFlight payment on the network. We will
+		// disallow any new payments.
+		case StatusInFlight:
+			updateErr = ErrPaymentInFlight
+			return nil
+
+		// We've already succeeded a payment to this payment hash,
+		// forbid the switch from sending another.
+		case StatusSucceeded:
+			updateErr = ErrAlreadyPaid
+			return nil
+
+		default:
+			updateErr = ErrUnknownPaymentStatus
+			return nil
+		}
+
+		// Obtain a new sequence number for this payment. This is used
+		// to sort the payments in order of creation, and also acts as
+		// a unique identifier for each payment.
+		sequenceNum, err := nextPaymentSequence(tx)
+		if err != nil {
+			return err
+		}
+
+		err = bucket.Put(paymentSequenceKey, sequenceNum)
+		if err != nil {
+			return err
+		}
+
+		// Add the payment info to the bucket, which contains the
+		// static information for this payment
+		err = bucket.Put(paymentCreationInfoKey, infoBytes)
+		if err != nil {
+			return err
+		}
+
+		// We'll delete any lingering attempt info to start with, in
+		// case we are initializing a payment that was attempted
+		// earlier, but left in a state where we could retry.
+		err = bucket.Delete(paymentAttemptInfoKey)
+		if err != nil {
+			return err
+		}
+
+		// Also delete any lingering failure info now that we are
+		// re-attempting.
+		return bucket.Delete(paymentFailInfoKey)
+	})
+	if err != nil {
+		return err
+	}
+
+	return updateErr
+}
+
+// RegisterAttempt atomically records the provided PaymentAttemptInfo to the
+// DB.
+func (p *PaymentControl) RegisterAttempt(paymentHash lntypes.Hash,
+	attempt *PaymentAttemptInfo) error {
+
+	// Serialize the information before opening the db transaction.
+	var a bytes.Buffer
+	if err := serializePaymentAttemptInfo(&a, attempt); err != nil {
+		return err
+	}
+	attemptBytes := a.Bytes()
+
+	var updateErr error
+	err := p.db.Batch(func(tx *bbolt.Tx) error {
+		// Reset the update error, to avoid carrying over an error
+		// from a previous execution of the batched db transaction.
+		updateErr = nil
+
+		bucket, err := fetchPaymentBucket(tx, paymentHash)
+		if err == ErrPaymentNotInitiated {
+			updateErr = ErrPaymentNotInitiated
+			return nil
+		} else if err != nil {
+			return err
+		}
+
+		// We can only register attempts for payments that are
+		// in-flight.
+		if err := ensureInFlight(bucket); err != nil {
+			updateErr = err
+			return nil
+		}
+
+		// Add the payment attempt to the payments bucket.
+		return bucket.Put(paymentAttemptInfoKey, attemptBytes)
+	})
+	if err != nil {
+		return err
+	}
+
+	return updateErr
+}
+
+// Success transitions a payment into the Succeeded state. After invoking this
+// method, InitPayment should always return an error to prevent us from making
+// duplicate payments to the same payment hash. The provided preimage is
+// atomically saved to the DB for record keeping.
+func (p *PaymentControl) Success(paymentHash lntypes.Hash,
+	preimage lntypes.Preimage) (*route.Route, error) {
+
+	var (
+		updateErr error
+		route     *route.Route
+	)
+	err := p.db.Batch(func(tx *bbolt.Tx) error {
+		// Reset the update error, to avoid carrying over an error
+		// from a previous execution of the batched db transaction.
+		updateErr = nil
+
+		bucket, err := fetchPaymentBucket(tx, paymentHash)
+		if err == ErrPaymentNotInitiated {
+			updateErr = ErrPaymentNotInitiated
+			return nil
+		} else if err != nil {
+			return err
+		}
+
+		// We can only mark in-flight payments as succeeded.
+		if err := ensureInFlight(bucket); err != nil {
+			updateErr = err
+			return nil
+		}
+
+		// Record the successful payment info atomically to the
+		// payments record.
+		err = bucket.Put(paymentSettleInfoKey, preimage[:])
+		if err != nil {
+			return err
+		}
+
+		// Retrieve attempt info for the notification.
+		attempt, err := fetchPaymentAttempt(bucket)
+		if err != nil {
+			return err
+		}
+
+		route = &attempt.Route
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return route, updateErr
+}
+
+// Fail transitions a payment into the Failed state, and records the reason the
+// payment failed. After invoking this method, InitPayment should return nil on
+// its next call for this payment hash, allowing the switch to make a
+// subsequent payment.
+func (p *PaymentControl) Fail(paymentHash lntypes.Hash,
+	reason FailureReason) (*route.Route, error) {
+
+	var (
+		updateErr error
+		route     *route.Route
+	)
+	err := p.db.Batch(func(tx *bbolt.Tx) error {
+		// Reset the update error, to avoid carrying over an error
+		// from a previous execution of the batched db transaction.
+		updateErr = nil
+
+		bucket, err := fetchPaymentBucket(tx, paymentHash)
+		if err == ErrPaymentNotInitiated {
+			updateErr = ErrPaymentNotInitiated
+			return nil
+		} else if err != nil {
+			return err
+		}
+
+		// We can only mark in-flight payments as failed.
+		if err := ensureInFlight(bucket); err != nil {
+			updateErr = err
+			return nil
+		}
+
+		// Put the failure reason in the bucket for record keeping.
+		v := []byte{byte(reason)}
+		err = bucket.Put(paymentFailInfoKey, v)
+		if err != nil {
+			return err
+		}
+
+		// Retrieve attempt info for the notification, if available.
+		attempt, err := fetchPaymentAttempt(bucket)
+		if err != nil && err != errNoAttemptInfo {
+			return err
+		}
+		if err != errNoAttemptInfo {
+			route = &attempt.Route
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return route, updateErr
+}
+
+// FetchPayment returns information about a payment from the database.
+func (p *PaymentControl) FetchPayment(paymentHash lntypes.Hash) (
+	*Payment, error) {
+
+	var payment *Payment
+	err := p.db.View(func(tx *bbolt.Tx) error {
+		bucket, err := fetchPaymentBucket(tx, paymentHash)
+		if err != nil {
+			return err
+		}
+
+		payment, err = fetchPayment(bucket)
+
+		return err
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return payment, nil
+}
+
+// createPaymentBucket creates or fetches the sub-bucket assigned to this
+// payment hash.
+func createPaymentBucket(tx *bbolt.Tx, paymentHash lntypes.Hash) (
+	*bbolt.Bucket, error) {
+
+	payments, err := tx.CreateBucketIfNotExists(paymentsRootBucket)
+	if err != nil {
+		return nil, err
+	}
+
+	return payments.CreateBucketIfNotExists(paymentHash[:])
+}
+
+// fetchPaymentBucket fetches the sub-bucket assigned to this payment hash. If
+// the bucket does not exist, it returns ErrPaymentNotInitiated.
+func fetchPaymentBucket(tx *bbolt.Tx, paymentHash lntypes.Hash) (
+	*bbolt.Bucket, error) {
+
+	payments := tx.Bucket(paymentsRootBucket)
+	if payments == nil {
+		return nil, ErrPaymentNotInitiated
+	}
+
+	bucket := payments.Bucket(paymentHash[:])
+	if bucket == nil {
+		return nil, ErrPaymentNotInitiated
+	}
+
+	return bucket, nil
+
+}
+
+// nextPaymentSequence returns the next sequence number to store for a new
+// payment.
+func nextPaymentSequence(tx *bbolt.Tx) ([]byte, error) {
+	payments, err := tx.CreateBucketIfNotExists(paymentsRootBucket)
+	if err != nil {
+		return nil, err
+	}
+
+	seq, err := payments.NextSequence()
+	if err != nil {
+		return nil, err
+	}
+
+	b := make([]byte, 8)
+	binary.BigEndian.PutUint64(b, seq)
+	return b, nil
+}
+
+// fetchPaymentStatus fetches the payment status of the payment. If the payment
+// isn't found, it will default to "StatusUnknown".
+func fetchPaymentStatus(bucket *bbolt.Bucket) PaymentStatus {
+	if bucket.Get(paymentSettleInfoKey) != nil {
+		return StatusSucceeded
+	}
+
+	if bucket.Get(paymentFailInfoKey) != nil {
+		return StatusFailed
+	}
+
+	if bucket.Get(paymentCreationInfoKey) != nil {
+		return StatusInFlight
+	}
+
+	return StatusUnknown
+}
+
+// ensureInFlight checks whether the payment found in the given bucket has
+// status InFlight, and returns an error otherwise. This should be used to
+// ensure we only mark in-flight payments as succeeded or failed.
+func ensureInFlight(bucket *bbolt.Bucket) error {
+	paymentStatus := fetchPaymentStatus(bucket)
+
+	switch {
+
+	// The payment was indeed InFlight, return.
+	case paymentStatus == StatusInFlight:
+		return nil
+
+	// Our records show the payment as unknown, meaning it never
+	// should have left the switch.
+	case paymentStatus == StatusUnknown:
+		return ErrPaymentNotInitiated
+
+	// The payment succeeded previously.
+	case paymentStatus == StatusSucceeded:
+		return ErrPaymentAlreadySucceeded
+
+	// The payment was already failed.
+	case paymentStatus == StatusFailed:
+		return ErrPaymentAlreadyFailed
+
+	default:
+		return ErrUnknownPaymentStatus
+	}
+}
+
+// fetchPaymentAttempt fetches the payment attempt from the bucket.
+func fetchPaymentAttempt(bucket *bbolt.Bucket) (*PaymentAttemptInfo, error) {
+	attemptData := bucket.Get(paymentAttemptInfoKey)
+	if attemptData == nil {
+		return nil, errNoAttemptInfo
+	}
+
+	r := bytes.NewReader(attemptData)
+	return deserializePaymentAttemptInfo(r)
+}
+
+// InFlightPayment is a wrapper around a payment that has status InFlight.
+type InFlightPayment struct {
+	// Info is the PaymentCreationInfo of the in-flight payment.
+	Info *PaymentCreationInfo
+
+	// Attempt contains information about the last payment attempt that was
+	// made to this payment hash.
+	//
+	// NOTE: Might be nil.
+	Attempt *PaymentAttemptInfo
+}
+
+// FetchInFlightPayments returns all payments with status InFlight.
+func (p *PaymentControl) FetchInFlightPayments() ([]*InFlightPayment, error) {
+	var inFlights []*InFlightPayment
+	err := p.db.View(func(tx *bbolt.Tx) error {
+		payments := tx.Bucket(paymentsRootBucket)
+		if payments == nil {
+			return nil
+		}
+
+		return payments.ForEach(func(k, _ []byte) error {
+			bucket := payments.Bucket(k)
+			if bucket == nil {
+				return fmt.Errorf("non bucket element")
+			}
+
+			// If the status is not InFlight, we can return early.
+			paymentStatus := fetchPaymentStatus(bucket)
+			if paymentStatus != StatusInFlight {
+				return nil
+			}
+
+			var (
+				inFlight = &InFlightPayment{}
+				err      error
+			)
+
+			// Get the CreationInfo.
+			b := bucket.Get(paymentCreationInfoKey)
+			if b == nil {
+				return fmt.Errorf("unable to find creation " +
+					"info for inflight payment")
+			}
+
+			r := bytes.NewReader(b)
+			inFlight.Info, err = deserializePaymentCreationInfo(r)
+			if err != nil {
+				return err
+			}
+
+			// Now get the attempt info. It could be that there is
+			// no attempt info yet.
+			inFlight.Attempt, err = fetchPaymentAttempt(bucket)
+			if err != nil && err != errNoAttemptInfo {
+				return err
+			}
+
+			inFlights = append(inFlights, inFlight)
+			return nil
+		})
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return inFlights, nil
+}

channeldb/migration_01_to_11/payment_control_test.go

@@ -0,0 +1,550 @@
+package migration_01_to_11
+
+import (
+	"bytes"
+	"crypto/rand"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/btcsuite/fastsha256"
+	"github.com/coreos/bbolt"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/lightningnetwork/lnd/lntypes"
+	"github.com/lightningnetwork/lnd/routing/route"
+)
+
+func initDB() (*DB, error) {
+	tempPath, err := ioutil.TempDir("", "switchdb")
+	if err != nil {
+		return nil, err
+	}
+
+	db, err := Open(tempPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return db, err
+}
+
+func genPreimage() ([32]byte, error) {
+	var preimage [32]byte
+	if _, err := io.ReadFull(rand.Reader, preimage[:]); err != nil {
+		return preimage, err
+	}
+	return preimage, nil
+}
+
+func genInfo() (*PaymentCreationInfo, *PaymentAttemptInfo,
+	lntypes.Preimage, error) {
+
+	preimage, err := genPreimage()
+	if err != nil {
+		return nil, nil, preimage, fmt.Errorf("unable to "+
+			"generate preimage: %v", err)
+	}
+
+	rhash := fastsha256.Sum256(preimage[:])
+	return &PaymentCreationInfo{
+			PaymentHash:    rhash,
+			Value:          1,
+			CreationDate:   time.Unix(time.Now().Unix(), 0),
+			PaymentRequest: []byte("hola"),
+		},
+		&PaymentAttemptInfo{
+			PaymentID:  1,
+			SessionKey: priv,
+			Route:      testRoute,
+		}, preimage, nil
+}
+
+// TestPaymentControlSwitchFail checks that payment status returns to Failed
+// status after failing, and that InitPayment allows another HTLC for the
+// same payment hash.
+func TestPaymentControlSwitchFail(t *testing.T) {
+	t.Parallel()
+
+	db, err := initDB()
+	if err != nil {
+		t.Fatalf("unable to init db: %v", err)
+	}
+
+	pControl := NewPaymentControl(db)
+
+	info, attempt, preimg, err := genInfo()
+	if err != nil {
+		t.Fatalf("unable to generate htlc message: %v", err)
+	}
+
+	// Sends base htlc message which initiate StatusInFlight.
+	err = pControl.InitPayment(info.PaymentHash, info)
+	if err != nil {
+		t.Fatalf("unable to send htlc message: %v", err)
+	}
+
+	assertPaymentStatus(t, db, info.PaymentHash, StatusInFlight)
+	assertPaymentInfo(
+		t, db, info.PaymentHash, info, nil, lntypes.Preimage{},
+		nil,
+	)
+
+	// Fail the payment, which should moved it to Failed.
+	failReason := FailureReasonNoRoute
+	_, err = pControl.Fail(info.PaymentHash, failReason)
+	if err != nil {
+		t.Fatalf("unable to fail payment hash: %v", err)
+	}
+
+	// Verify the status is indeed Failed.
+	assertPaymentStatus(t, db, info.PaymentHash, StatusFailed)
+	assertPaymentInfo(
+		t, db, info.PaymentHash, info, nil, lntypes.Preimage{},
+		&failReason,
+	)
+
+	// Sends the htlc again, which should succeed since the prior payment
+	// failed.
+	err = pControl.InitPayment(info.PaymentHash, info)
+	if err != nil {
+		t.Fatalf("unable to send htlc message: %v", err)
+	}
+
+	assertPaymentStatus(t, db, info.PaymentHash, StatusInFlight)
+	assertPaymentInfo(
+		t, db, info.PaymentHash, info, nil, lntypes.Preimage{},
+		nil,
+	)
+
+	// Record a new attempt.
+	attempt.PaymentID = 2
+	err = pControl.RegisterAttempt(info.PaymentHash, attempt)
+	if err != nil {
+		t.Fatalf("unable to send htlc message: %v", err)
+	}
+	assertPaymentStatus(t, db, info.PaymentHash, StatusInFlight)
+	assertPaymentInfo(
+		t, db, info.PaymentHash, info, attempt, lntypes.Preimage{},
+		nil,
+	)
+
+	// Verifies that status was changed to StatusSucceeded.
+	var route *route.Route
+	route, err = pControl.Success(info.PaymentHash, preimg)
+	if err != nil {
+		t.Fatalf("error shouldn't have been received, got: %v", err)
+	}
+
+	err = assertRouteEqual(route, &attempt.Route)
+	if err != nil {
+		t.Fatalf("unexpected route returned: %v vs %v: %v",
+			spew.Sdump(attempt.Route), spew.Sdump(*route), err)
+	}
+
+	assertPaymentStatus(t, db, info.PaymentHash, StatusSucceeded)
+	assertPaymentInfo(t, db, info.PaymentHash, info, attempt, preimg, nil)
+
+	// Attempt a final payment, which should now fail since the prior
+	// payment succeed.
+	err = pControl.InitPayment(info.PaymentHash, info)
+	if err != ErrAlreadyPaid {
+		t.Fatalf("unable to send htlc message: %v", err)
+	}
+}
+
+// TestPaymentControlSwitchDoubleSend checks the ability of payment control to
+// prevent double sending of htlc message, when message is in StatusInFlight.
+func TestPaymentControlSwitchDoubleSend(t *testing.T) {
+	t.Parallel()
+
+	db, err := initDB()
+	if err != nil {
+		t.Fatalf("unable to init db: %v", err)
+	}
+
+	pControl := NewPaymentControl(db)
+
+	info, attempt, preimg, err := genInfo()
+	if err != nil {
+		t.Fatalf("unable to generate htlc message: %v", err)
+	}
+
+	// Sends base htlc message which initiate base status and move it to
+	// StatusInFlight and verifies that it was changed.
+	err = pControl.InitPayment(info.PaymentHash, info)
+	if err != nil {
+		t.Fatalf("unable to send htlc message: %v", err)
+	}
+
+	assertPaymentStatus(t, db, info.PaymentHash, StatusInFlight)
+	assertPaymentInfo(
+		t, db, info.PaymentHash, info, nil, lntypes.Preimage{},
+		nil,
+	)
+
+	// Try to initiate double sending of htlc message with the same
+	// payment hash, should result in error indicating that payment has
+	// already been sent.
+	err = pControl.InitPayment(info.PaymentHash, info)
+	if err != ErrPaymentInFlight {
+		t.Fatalf("payment control wrong behaviour: " +
+			"double sending must trigger ErrPaymentInFlight error")
+	}
+
+	// Record an attempt.
+	err = pControl.RegisterAttempt(info.PaymentHash, attempt)
+	if err != nil {
+		t.Fatalf("unable to send htlc message: %v", err)
+	}
+	assertPaymentStatus(t, db, info.PaymentHash, StatusInFlight)
+	assertPaymentInfo(
+		t, db, info.PaymentHash, info, attempt, lntypes.Preimage{},
+		nil,
+	)
+
+	// Sends base htlc message which initiate StatusInFlight.
+	err = pControl.InitPayment(info.PaymentHash, info)
+	if err != ErrPaymentInFlight {
+		t.Fatalf("payment control wrong behaviour: " +
+			"double sending must trigger ErrPaymentInFlight error")
+	}
+
+	// After settling, the error should be ErrAlreadyPaid.
+	if _, err := pControl.Success(info.PaymentHash, preimg); err != nil {
+		t.Fatalf("error shouldn't have been received, got: %v", err)
+	}
+	assertPaymentStatus(t, db, info.PaymentHash, StatusSucceeded)
+	assertPaymentInfo(t, db, info.PaymentHash, info, attempt, preimg, nil)
+
+	err = pControl.InitPayment(info.PaymentHash, info)
+	if err != ErrAlreadyPaid {
+		t.Fatalf("unable to send htlc message: %v", err)
+	}
+}
+
+// TestPaymentControlSuccessesWithoutInFlight checks that the payment
+// control will disallow calls to Success when no payment is in flight.
+func TestPaymentControlSuccessesWithoutInFlight(t *testing.T) {
+	t.Parallel()
+
+	db, err := initDB()
+	if err != nil {
+		t.Fatalf("unable to init db: %v", err)
+	}
+
+	pControl := NewPaymentControl(db)
+
+	info, _, preimg, err := genInfo()
+	if err != nil {
+		t.Fatalf("unable to generate htlc message: %v", err)
+	}
+
+	// Attempt to complete the payment should fail.
+	_, err = pControl.Success(info.PaymentHash, preimg)
+	if err != ErrPaymentNotInitiated {
+		t.Fatalf("expected ErrPaymentNotInitiated, got %v", err)
+	}
+
+	assertPaymentStatus(t, db, info.PaymentHash, StatusUnknown)
+	assertPaymentInfo(
+		t, db, info.PaymentHash, nil, nil, lntypes.Preimage{},
+		nil,
+	)
+}
+
+// TestPaymentControlFailsWithoutInFlight checks that a strict payment
+// control will disallow calls to Fail when no payment is in flight.
+func TestPaymentControlFailsWithoutInFlight(t *testing.T) {
+	t.Parallel()
+
+	db, err := initDB()
+	if err != nil {
+		t.Fatalf("unable to init db: %v", err)
+	}
+
+	pControl := NewPaymentControl(db)
+
+	info, _, _, err := genInfo()
+	if err != nil {
+		t.Fatalf("unable to generate htlc message: %v", err)
+	}
+
+	// Calling Fail should return an error.
+	_, err = pControl.Fail(info.PaymentHash, FailureReasonNoRoute)
+	if err != ErrPaymentNotInitiated {
+		t.Fatalf("expected ErrPaymentNotInitiated, got %v", err)
+	}
+
+	assertPaymentStatus(t, db, info.PaymentHash, StatusUnknown)
+	assertPaymentInfo(
+		t, db, info.PaymentHash, nil, nil, lntypes.Preimage{}, nil,
+	)
+}
+
+// TestPaymentControlDeleteNonInFlight checks that calling DeletaPayments only
+// deletes payments from the database that are not in-flight.
+func TestPaymentControlDeleteNonInFligt(t *testing.T) {
+	t.Parallel()
+
+	db, err := initDB()
+	if err != nil {
+		t.Fatalf("unable to init db: %v", err)
+	}
+
+	pControl := NewPaymentControl(db)
+
+	payments := []struct {
+		failed  bool
+		success bool
+	}{
+		{
+			failed:  true,
+			success: false,
+		},
+		{
+			failed:  false,
+			success: true,
+		},
+		{
+			failed:  false,
+			success: false,
+		},
+	}
+
+	for _, p := range payments {
+		info, attempt, preimg, err := genInfo()
+		if err != nil {
+			t.Fatalf("unable to generate htlc message: %v", err)
+		}
+
+		// Sends base htlc message which initiate StatusInFlight.
+		err = pControl.InitPayment(info.PaymentHash, info)
+		if err != nil {
+			t.Fatalf("unable to send htlc message: %v", err)
+		}
+		err = pControl.RegisterAttempt(info.PaymentHash, attempt)
+		if err != nil {
+			t.Fatalf("unable to send htlc message: %v", err)
+		}
+
+		if p.failed {
+			// Fail the payment, which should moved it to Failed.
+			failReason := FailureReasonNoRoute
+			_, err = pControl.Fail(info.PaymentHash, failReason)
+			if err != nil {
+				t.Fatalf("unable to fail payment hash: %v", err)
+			}
+
+			// Verify the status is indeed Failed.
+			assertPaymentStatus(t, db, info.PaymentHash, StatusFailed)
+			assertPaymentInfo(
+				t, db, info.PaymentHash, info, attempt,
+				lntypes.Preimage{}, &failReason,
+			)
+		} else if p.success {
+			// Verifies that status was changed to StatusSucceeded.
+			_, err := pControl.Success(info.PaymentHash, preimg)
+			if err != nil {
+				t.Fatalf("error shouldn't have been received, got: %v", err)
+			}
+
+			assertPaymentStatus(t, db, info.PaymentHash, StatusSucceeded)
+			assertPaymentInfo(
+				t, db, info.PaymentHash, info, attempt, preimg, nil,
+			)
+		} else {
+			assertPaymentStatus(t, db, info.PaymentHash, StatusInFlight)
+			assertPaymentInfo(
+				t, db, info.PaymentHash, info, attempt,
+				lntypes.Preimage{}, nil,
+			)
+		}
+	}
+
+	// Delete payments.
+	if err := db.DeletePayments(); err != nil {
+		t.Fatal(err)
+	}
+
+	// This should leave the in-flight payment.
+	dbPayments, err := db.FetchPayments()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(dbPayments) != 1 {
+		t.Fatalf("expected one payment, got %d", len(dbPayments))
+	}
+
+	status := dbPayments[0].Status
+	if status != StatusInFlight {
+		t.Fatalf("expected in-fligth status, got %v", status)
+	}
+}
+
+func assertPaymentStatus(t *testing.T, db *DB,
+	hash [32]byte, expStatus PaymentStatus) {
+
+	t.Helper()
+
+	var paymentStatus = StatusUnknown
+	err := db.View(func(tx *bbolt.Tx) error {
+		payments := tx.Bucket(paymentsRootBucket)
+		if payments == nil {
+			return nil
+		}
+
+		bucket := payments.Bucket(hash[:])
+		if bucket == nil {
+			return nil
+		}
+
+		// Get the existing status of this payment, if any.
+		paymentStatus = fetchPaymentStatus(bucket)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("unable to fetch payment status: %v", err)
+	}
+
+	if paymentStatus != expStatus {
+		t.Fatalf("payment status mismatch: expected %v, got %v",
+			expStatus, paymentStatus)
+	}
+}
+
+func checkPaymentCreationInfo(bucket *bbolt.Bucket, c *PaymentCreationInfo) error {
+	b := bucket.Get(paymentCreationInfoKey)
+	switch {
+	case b == nil && c == nil:
+		return nil
+	case b == nil:
+		return fmt.Errorf("expected creation info not found")
+	case c == nil:
+		return fmt.Errorf("unexpected creation info found")
+	}
+
+	r := bytes.NewReader(b)
+	c2, err := deserializePaymentCreationInfo(r)
+	if err != nil {
+		return err
+	}
+	if !reflect.DeepEqual(c, c2) {
+		return fmt.Errorf("PaymentCreationInfos don't match: %v vs %v",
+			spew.Sdump(c), spew.Sdump(c2))
+	}
+
+	return nil
+}
+
+func checkPaymentAttemptInfo(bucket *bbolt.Bucket, a *PaymentAttemptInfo) error {
+	b := bucket.Get(paymentAttemptInfoKey)
+	switch {
+	case b == nil && a == nil:
+		return nil
+	case b == nil:
+		return fmt.Errorf("expected attempt info not found")
+	case a == nil:
+		return fmt.Errorf("unexpected attempt info found")
+	}
+
+	r := bytes.NewReader(b)
+	a2, err := deserializePaymentAttemptInfo(r)
+	if err != nil {
+		return err
+	}
+
+	return assertRouteEqual(&a.Route, &a2.Route)
+}
+
+func checkSettleInfo(bucket *bbolt.Bucket, preimg lntypes.Preimage) error {
+	zero := lntypes.Preimage{}
+	b := bucket.Get(paymentSettleInfoKey)
+	switch {
+	case b == nil && preimg == zero:
+		return nil
+	case b == nil:
+		return fmt.Errorf("expected preimage not found")
+	case preimg == zero:
+		return fmt.Errorf("unexpected preimage found")
+	}
+
+	var pre2 lntypes.Preimage
+	copy(pre2[:], b[:])
+	if preimg != pre2 {
+		return fmt.Errorf("Preimages don't match: %x vs %x",
+			preimg, pre2)
+	}
+
+	return nil
+}
+
+func checkFailInfo(bucket *bbolt.Bucket, failReason *FailureReason) error {
+	b := bucket.Get(paymentFailInfoKey)
+	switch {
+	case b == nil && failReason == nil:
+		return nil
+	case b == nil:
+		return fmt.Errorf("expected fail info not found")
+	case failReason == nil:
+		return fmt.Errorf("unexpected fail info found")
+	}
+
+	failReason2 := FailureReason(b[0])
+	if *failReason != failReason2 {
+		return fmt.Errorf("Failure infos don't match: %v vs %v",
+			*failReason, failReason2)
+	}
+
+	return nil
+}
+
+func assertPaymentInfo(t *testing.T, db *DB, hash lntypes.Hash,
+	c *PaymentCreationInfo, a *PaymentAttemptInfo, s lntypes.Preimage,
+	f *FailureReason) {
+
+	t.Helper()
+
+	err := db.View(func(tx *bbolt.Tx) error {
+		payments := tx.Bucket(paymentsRootBucket)
+		if payments == nil && c == nil {
+			return nil
+		}
+		if payments == nil {
+			return fmt.Errorf("sent payments not found")
+		}
+
+		bucket := payments.Bucket(hash[:])
+		if bucket == nil && c == nil {
+			return nil
+		}
+
+		if bucket == nil {
+			return fmt.Errorf("payment not found")
+		}
+
+		if err := checkPaymentCreationInfo(bucket, c); err != nil {
+			return err
+		}
+
+		if err := checkPaymentAttemptInfo(bucket, a); err != nil {
+			return err
+		}
+
+		if err := checkSettleInfo(bucket, s); err != nil {
+			return err
+		}
+
+		if err := checkFailInfo(bucket, f); err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("assert payment info failed: %v", err)
+	}
+
+}

channeldb/migration_01_to_11/payments.go

@@ -0,0 +1,669 @@
+package migration_01_to_11
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"sort"
+	"time"
+
+	"github.com/btcsuite/btcd/btcec"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/coreos/bbolt"
+	"github.com/lightningnetwork/lnd/lntypes"
+	"github.com/lightningnetwork/lnd/lnwire"
+	"github.com/lightningnetwork/lnd/routing/route"
+	"github.com/lightningnetwork/lnd/tlv"
+)
+
+var (
+	// paymentsRootBucket is the name of the top-level bucket within the
+	// database that stores all data related to payments. Within this
+	// bucket, each payment hash its own sub-bucket keyed by its payment
+	// hash.
+	//
+	// Bucket hierarchy:
+	//
+	// root-bucket
+	//      |
+	//      |-- <paymenthash>
+	//      |        |--sequence-key: <sequence number>
+	//      |        |--creation-info-key: <creation info>
+	//      |        |--attempt-info-key: <attempt info>
+	//      |        |--settle-info-key: <settle info>
+	//      |        |--fail-info-key: <fail info>
+	//      |        |
+	//      |        |--duplicate-bucket (only for old, completed payments)
+	//      |                 |
+	//      |                 |-- <seq-num>
+	//      |                 |       |--sequence-key: <sequence number>
+	//      |                 |       |--creation-info-key: <creation info>
+	//      |                 |       |--attempt-info-key: <attempt info>
+	//      |                 |       |--settle-info-key: <settle info>
+	//      |                 |       |--fail-info-key: <fail info>
+	//      |                 |
+	//      |                 |-- <seq-num>
+	//      |                 |       |
+	//      |                ...     ...
+	//      |
+	//      |-- <paymenthash>
+	//      |        |
+	//      |       ...
+	//     ...
+	//
+	paymentsRootBucket = []byte("payments-root-bucket")
+
+	// paymentDublicateBucket is the name of a optional sub-bucket within
+	// the payment hash bucket, that is used to hold duplicate payments to
+	// a payment hash. This is needed to support information from earlier
+	// versions of lnd, where it was possible to pay to a payment hash more
+	// than once.
+	paymentDuplicateBucket = []byte("payment-duplicate-bucket")
+
+	// paymentSequenceKey is a key used in the payment's sub-bucket to
+	// store the sequence number of the payment.
+	paymentSequenceKey = []byte("payment-sequence-key")
+
+	// paymentCreationInfoKey is a key used in the payment's sub-bucket to
+	// store the creation info of the payment.
+	paymentCreationInfoKey = []byte("payment-creation-info")
+
+	// paymentAttemptInfoKey is a key used in the payment's sub-bucket to
+	// store the info about the latest attempt that was done for the
+	// payment in question.
+	paymentAttemptInfoKey = []byte("payment-attempt-info")
+
+	// paymentSettleInfoKey is a key used in the payment's sub-bucket to
+	// store the settle info of the payment.
+	paymentSettleInfoKey = []byte("payment-settle-info")
+
+	// paymentFailInfoKey is a key used in the payment's sub-bucket to
+	// store information about the reason a payment failed.
+	paymentFailInfoKey = []byte("payment-fail-info")
+)
+
+// FailureReason encodes the reason a payment ultimately failed.
+type FailureReason byte
+
+const (
+	// FailureReasonTimeout indicates that the payment did timeout before a
+	// successful payment attempt was made.
+	FailureReasonTimeout FailureReason = 0
+
+	// FailureReasonNoRoute indicates no successful route to the
+	// destination was found during path finding.
+	FailureReasonNoRoute FailureReason = 1
+
+	// FailureReasonError indicates that an unexpected error happened during
+	// payment.
+	FailureReasonError FailureReason = 2
+
+	// FailureReasonIncorrectPaymentDetails indicates that either the hash
+	// is unknown or the final cltv delta or amount is incorrect.
+	FailureReasonIncorrectPaymentDetails FailureReason = 3
+
+	// TODO(halseth): cancel state.
+
+	// TODO(joostjager): Add failure reasons for:
+	// LocalLiquidityInsufficient, RemoteCapacityInsufficient.
+)
+
+// String returns a human readable FailureReason
+func (r FailureReason) String() string {
+	switch r {
+	case FailureReasonTimeout:
+		return "timeout"
+	case FailureReasonNoRoute:
+		return "no_route"
+	case FailureReasonError:
+		return "error"
+	case FailureReasonIncorrectPaymentDetails:
+		return "incorrect_payment_details"
+	}
+
+	return "unknown"
+}
+
+// PaymentStatus represent current status of payment
+type PaymentStatus byte
+
+const (
+	// StatusUnknown is the status where a payment has never been initiated
+	// and hence is unknown.
+	StatusUnknown PaymentStatus = 0
+
+	// StatusInFlight is the status where a payment has been initiated, but
+	// a response has not been received.
+	StatusInFlight PaymentStatus = 1
+
+	// StatusSucceeded is the status where a payment has been initiated and
+	// the payment was completed successfully.
+	StatusSucceeded PaymentStatus = 2
+
+	// StatusFailed is the status where a payment has been initiated and a
+	// failure result has come back.
+	StatusFailed PaymentStatus = 3
+)
+
+// Bytes returns status as slice of bytes.
+func (ps PaymentStatus) Bytes() []byte {
+	return []byte{byte(ps)}
+}
+
+// FromBytes sets status from slice of bytes.
+func (ps *PaymentStatus) FromBytes(status []byte) error {
+	if len(status) != 1 {
+		return errors.New("payment status is empty")
+	}
+
+	switch PaymentStatus(status[0]) {
+	case StatusUnknown, StatusInFlight, StatusSucceeded, StatusFailed:
+		*ps = PaymentStatus(status[0])
+	default:
+		return errors.New("unknown payment status")
+	}
+
+	return nil
+}
+
+// String returns readable representation of payment status.
+func (ps PaymentStatus) String() string {
+	switch ps {
+	case StatusUnknown:
+		return "Unknown"
+	case StatusInFlight:
+		return "In Flight"
+	case StatusSucceeded:
+		return "Succeeded"
+	case StatusFailed:
+		return "Failed"
+	default:
+		return "Unknown"
+	}
+}
+
+// PaymentCreationInfo is the information necessary to have ready when
+// initiating a payment, moving it into state InFlight.
+type PaymentCreationInfo struct {
+	// PaymentHash is the hash this payment is paying to.
+	PaymentHash lntypes.Hash
+
+	// Value is the amount we are paying.
+	Value lnwire.MilliSatoshi
+
+	// CreatingDate is the time when this payment was initiated.
+	CreationDate time.Time
+
+	// PaymentRequest is the full payment request, if any.
+	PaymentRequest []byte
+}
+
+// PaymentAttemptInfo contains information about a specific payment attempt for
+// a given payment. This information is used by the router to handle any errors
+// coming back after an attempt is made, and to query the switch about the
+// status of a payment. For settled payment this will be the information for
+// the succeeding payment attempt.
+type PaymentAttemptInfo struct {
+	// PaymentID is the unique ID used for this attempt.
+	PaymentID uint64
+
+	// SessionKey is the ephemeral key used for this payment attempt.
+	SessionKey *btcec.PrivateKey
+
+	// Route is the route attempted to send the HTLC.
+	Route route.Route
+}
+
+// Payment is a wrapper around a payment's PaymentCreationInfo,
+// PaymentAttemptInfo, and preimage. All payments will have the
+// PaymentCreationInfo set, the PaymentAttemptInfo will be set only if at least
+// one payment attempt has been made, while only completed payments will have a
+// non-zero payment preimage.
+type Payment struct {
+	// sequenceNum is a unique identifier used to sort the payments in
+	// order of creation.
+	sequenceNum uint64
+
+	// Status is the current PaymentStatus of this payment.
+	Status PaymentStatus
+
+	// Info holds all static information about this payment, and is
+	// populated when the payment is initiated.
+	Info *PaymentCreationInfo
+
+	// Attempt is the information about the last payment attempt made.
+	//
+	// NOTE: Can be nil if no attempt is yet made.
+	Attempt *PaymentAttemptInfo
+
+	// PaymentPreimage is the preimage of a successful payment. This serves
+	// as a proof of payment. It will only be non-nil for settled payments.
+	//
+	// NOTE: Can be nil if payment is not settled.
+	PaymentPreimage *lntypes.Preimage
+
+	// Failure is a failure reason code indicating the reason the payment
+	// failed. It is only non-nil for failed payments.
+	//
+	// NOTE: Can be nil if payment is not failed.
+	Failure *FailureReason
+}
+
+// FetchPayments returns all sent payments found in the DB.
+func (db *DB) FetchPayments() ([]*Payment, error) {
+	var payments []*Payment
+
+	err := db.View(func(tx *bbolt.Tx) error {
+		paymentsBucket := tx.Bucket(paymentsRootBucket)
+		if paymentsBucket == nil {
+			return nil
+		}
+
+		return paymentsBucket.ForEach(func(k, v []byte) error {
+			bucket := paymentsBucket.Bucket(k)
+			if bucket == nil {
+				// We only expect sub-buckets to be found in
+				// this top-level bucket.
+				return fmt.Errorf("non bucket element in " +
+					"payments bucket")
+			}
+
+			p, err := fetchPayment(bucket)
+			if err != nil {
+				return err
+			}
+
+			payments = append(payments, p)
+
+			// For older versions of lnd, duplicate payments to a
+			// payment has was possible. These will be found in a
+			// sub-bucket indexed by their sequence number if
+			// available.
+			dup := bucket.Bucket(paymentDuplicateBucket)
+			if dup == nil {
+				return nil
+			}
+
+			return dup.ForEach(func(k, v []byte) error {
+				subBucket := dup.Bucket(k)
+				if subBucket == nil {
+					// We one bucket for each duplicate to
+					// be found.
+					return fmt.Errorf("non bucket element" +
+						"in duplicate bucket")
+				}
+
+				p, err := fetchPayment(subBucket)
+				if err != nil {
+					return err
+				}
+
+				payments = append(payments, p)
+				return nil
+			})
+		})
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// Before returning, sort the payments by their sequence number.
+	sort.Slice(payments, func(i, j int) bool {
+		return payments[i].sequenceNum < payments[j].sequenceNum
+	})
+
+	return payments, nil
+}
+
+func fetchPayment(bucket *bbolt.Bucket) (*Payment, error) {
+	var (
+		err error
+		p   = &Payment{}
+	)
+
+	seqBytes := bucket.Get(paymentSequenceKey)
+	if seqBytes == nil {
+		return nil, fmt.Errorf("sequence number not found")
+	}
+
+	p.sequenceNum = binary.BigEndian.Uint64(seqBytes)
+
+	// Get the payment status.
+	p.Status = fetchPaymentStatus(bucket)
+
+	// Get the PaymentCreationInfo.
+	b := bucket.Get(paymentCreationInfoKey)
+	if b == nil {
+		return nil, fmt.Errorf("creation info not found")
+	}
+
+	r := bytes.NewReader(b)
+	p.Info, err = deserializePaymentCreationInfo(r)
+	if err != nil {
+		return nil, err
+
+	}
+
+	// Get the PaymentAttemptInfo. This can be unset.
+	b = bucket.Get(paymentAttemptInfoKey)
+	if b != nil {
+		r = bytes.NewReader(b)
+		p.Attempt, err = deserializePaymentAttemptInfo(r)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Get the payment preimage. This is only found for
+	// completed payments.
+	b = bucket.Get(paymentSettleInfoKey)
+	if b != nil {
+		var preimg lntypes.Preimage
+		copy(preimg[:], b[:])
+		p.PaymentPreimage = &preimg
+	}
+
+	// Get failure reason if available.
+	b = bucket.Get(paymentFailInfoKey)
+	if b != nil {
+		reason := FailureReason(b[0])
+		p.Failure = &reason
+	}
+
+	return p, nil
+}
+
+// DeletePayments deletes all completed and failed payments from the DB.
+func (db *DB) DeletePayments() error {
+	return db.Update(func(tx *bbolt.Tx) error {
+		payments := tx.Bucket(paymentsRootBucket)
+		if payments == nil {
+			return nil
+		}
+
+		var deleteBuckets [][]byte
+		err := payments.ForEach(func(k, _ []byte) error {
+			bucket := payments.Bucket(k)
+			if bucket == nil {
+				// We only expect sub-buckets to be found in
+				// this top-level bucket.
+				return fmt.Errorf("non bucket element in " +
+					"payments bucket")
+			}
+
+			// If the status is InFlight, we cannot safely delete
+			// the payment information, so we return early.
+			paymentStatus := fetchPaymentStatus(bucket)
+			if paymentStatus == StatusInFlight {
+				return nil
+			}
+
+			deleteBuckets = append(deleteBuckets, k)
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		for _, k := range deleteBuckets {
+			if err := payments.DeleteBucket(k); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+func serializePaymentCreationInfo(w io.Writer, c *PaymentCreationInfo) error {
+	var scratch [8]byte
+
+	if _, err := w.Write(c.PaymentHash[:]); err != nil {
+		return err
+	}
+
+	byteOrder.PutUint64(scratch[:], uint64(c.Value))
+	if _, err := w.Write(scratch[:]); err != nil {
+		return err
+	}
+
+	byteOrder.PutUint64(scratch[:], uint64(c.CreationDate.Unix()))
+	if _, err := w.Write(scratch[:]); err != nil {
+		return err
+	}
+
+	byteOrder.PutUint32(scratch[:4], uint32(len(c.PaymentRequest)))
+	if _, err := w.Write(scratch[:4]); err != nil {
+		return err
+	}
+
+	if _, err := w.Write(c.PaymentRequest[:]); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func deserializePaymentCreationInfo(r io.Reader) (*PaymentCreationInfo, error) {
+	var scratch [8]byte
+
+	c := &PaymentCreationInfo{}
+
+	if _, err := io.ReadFull(r, c.PaymentHash[:]); err != nil {
+		return nil, err
+	}
+
+	if _, err := io.ReadFull(r, scratch[:]); err != nil {
+		return nil, err
+	}
+	c.Value = lnwire.MilliSatoshi(byteOrder.Uint64(scratch[:]))
+
+	if _, err := io.ReadFull(r, scratch[:]); err != nil {
+		return nil, err
+	}
+	c.CreationDate = time.Unix(int64(byteOrder.Uint64(scratch[:])), 0)
+
+	if _, err := io.ReadFull(r, scratch[:4]); err != nil {
+		return nil, err
+	}
+
+	reqLen := uint32(byteOrder.Uint32(scratch[:4]))
+	payReq := make([]byte, reqLen)
+	if reqLen > 0 {
+		if _, err := io.ReadFull(r, payReq[:]); err != nil {
+			return nil, err
+		}
+	}
+	c.PaymentRequest = payReq
+
+	return c, nil
+}
+
+func serializePaymentAttemptInfo(w io.Writer, a *PaymentAttemptInfo) error {
+	if err := WriteElements(w, a.PaymentID, a.SessionKey); err != nil {
+		return err
+	}
+
+	if err := SerializeRoute(w, a.Route); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func deserializePaymentAttemptInfo(r io.Reader) (*PaymentAttemptInfo, error) {
+	a := &PaymentAttemptInfo{}
+	err := ReadElements(r, &a.PaymentID, &a.SessionKey)
+	if err != nil {
+		return nil, err
+	}
+	a.Route, err = DeserializeRoute(r)
+	if err != nil {
+		return nil, err
+	}
+	return a, nil
+}
+
+func serializeHop(w io.Writer, h *route.Hop) error {
+	if err := WriteElements(w,
+		h.PubKeyBytes[:], h.ChannelID, h.OutgoingTimeLock,
+		h.AmtToForward,
+	); err != nil {
+		return err
+	}
+
+	if err := binary.Write(w, byteOrder, h.LegacyPayload); err != nil {
+		return err
+	}
+
+	// For legacy payloads, we don't need to write any TLV records, so
+	// we'll write a zero indicating the our serialized TLV map has no
+	// records.
+	if h.LegacyPayload {
+		return WriteElements(w, uint32(0))
+	}
+
+	// Otherwise, we'll transform our slice of records into a map of the
+	// raw bytes, then serialize them in-line with a length (number of
+	// elements) prefix.
+	mapRecords, err := tlv.RecordsToMap(h.TLVRecords)
+	if err != nil {
+		return err
+	}
+
+	numRecords := uint32(len(mapRecords))
+	if err := WriteElements(w, numRecords); err != nil {
+		return err
+	}
+
+	for recordType, rawBytes := range mapRecords {
+		if err := WriteElements(w, recordType); err != nil {
+			return err
+		}
+
+		if err := wire.WriteVarBytes(w, 0, rawBytes); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// maxOnionPayloadSize is the largest Sphinx payload possible, so we don't need
+// to read/write a TLV stream larger than this.
+const maxOnionPayloadSize = 1300
+
+func deserializeHop(r io.Reader) (*route.Hop, error) {
+	h := &route.Hop{}
+
+	var pub []byte
+	if err := ReadElements(r, &pub); err != nil {
+		return nil, err
+	}
+	copy(h.PubKeyBytes[:], pub)
+
+	if err := ReadElements(r,
+		&h.ChannelID, &h.OutgoingTimeLock, &h.AmtToForward,
+	); err != nil {
+		return nil, err
+	}
+
+	// TODO(roasbeef): change field to allow LegacyPayload false to be the
+	// legacy default?
+	err := binary.Read(r, byteOrder, &h.LegacyPayload)
+	if err != nil {
+		return nil, err
+	}
+
+	var numElements uint32
+	if err := ReadElements(r, &numElements); err != nil {
+		return nil, err
+	}
+
+	// If there're no elements, then we can return early.
+	if numElements == 0 {
+		return h, nil
+	}
+
+	tlvMap := make(map[uint64][]byte)
+	for i := uint32(0); i < numElements; i++ {
+		var tlvType uint64
+		if err := ReadElements(r, &tlvType); err != nil {
+			return nil, err
+		}
+
+		rawRecordBytes, err := wire.ReadVarBytes(
+			r, 0, maxOnionPayloadSize, "tlv",
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		tlvMap[tlvType] = rawRecordBytes
+	}
+
+	tlvRecords, err := tlv.MapToRecords(tlvMap)
+	if err != nil {
+		return nil, err
+	}
+
+	h.TLVRecords = tlvRecords
+
+	return h, nil
+}
+
+// SerializeRoute serializes a route.
+func SerializeRoute(w io.Writer, r route.Route) error {
+	if err := WriteElements(w,
+		r.TotalTimeLock, r.TotalAmount, r.SourcePubKey[:],
+	); err != nil {
+		return err
+	}
+
+	if err := WriteElements(w, uint32(len(r.Hops))); err != nil {
+		return err
+	}
+
+	for _, h := range r.Hops {
+		if err := serializeHop(w, h); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// DeserializeRoute deserializes a route.
+func DeserializeRoute(r io.Reader) (route.Route, error) {
+	rt := route.Route{}
+	if err := ReadElements(r,
+		&rt.TotalTimeLock, &rt.TotalAmount,
+	); err != nil {
+		return rt, err
+	}
+
+	var pub []byte
+	if err := ReadElements(r, &pub); err != nil {
+		return rt, err
+	}
+	copy(rt.SourcePubKey[:], pub)
+
+	var numHops uint32
+	if err := ReadElements(r, &numHops); err != nil {
+		return rt, err
+	}
+
+	var hops []*route.Hop
+	for i := uint32(0); i < numHops; i++ {
+		hop, err := deserializeHop(r)
+		if err != nil {
+			return rt, err
+		}
+		hops = append(hops, hop)
+	}
+	rt.Hops = hops
+
+	return rt, nil
+}

channeldb/migration_01_to_11/payments_test.go

@@ -0,0 +1,324 @@
+package migration_01_to_11
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"math/rand"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/btcsuite/btcd/btcec"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/lightningnetwork/lnd/lntypes"
+	"github.com/lightningnetwork/lnd/lnwire"
+	"github.com/lightningnetwork/lnd/routing/route"
+	"github.com/lightningnetwork/lnd/tlv"
+)
+
+var (
+	priv, _ = btcec.NewPrivateKey(btcec.S256())
+	pub     = priv.PubKey()
+
+	tlvBytes   = []byte{1, 2, 3}
+	tlvEncoder = tlv.StubEncoder(tlvBytes)
+	testHop1   = &route.Hop{
+		PubKeyBytes:      route.NewVertex(pub),
+		ChannelID:        12345,
+		OutgoingTimeLock: 111,
+		AmtToForward:     555,
+		TLVRecords: []tlv.Record{
+			tlv.MakeStaticRecord(1, nil, 3, tlvEncoder, nil),
+			tlv.MakeStaticRecord(2, nil, 3, tlvEncoder, nil),
+		},
+	}
+
+	testHop2 = &route.Hop{
+		PubKeyBytes:      route.NewVertex(pub),
+		ChannelID:        12345,
+		OutgoingTimeLock: 111,
+		AmtToForward:     555,
+		LegacyPayload:    true,
+	}
+
+	testRoute = route.Route{
+		TotalTimeLock: 123,
+		TotalAmount:   1234567,
+		SourcePubKey:  route.NewVertex(pub),
+		Hops: []*route.Hop{
+			testHop1,
+			testHop2,
+		},
+	}
+)
+
+func makeFakePayment() *outgoingPayment {
+	fakeInvoice := &Invoice{
+		// Use single second precision to avoid false positive test
+		// failures due to the monotonic time component.
+		CreationDate:   time.Unix(time.Now().Unix(), 0),
+		Memo:           []byte("fake memo"),
+		Receipt:        []byte("fake receipt"),
+		PaymentRequest: []byte(""),
+	}
+
+	copy(fakeInvoice.Terms.PaymentPreimage[:], rev[:])
+	fakeInvoice.Terms.Value = lnwire.NewMSatFromSatoshis(10000)
+
+	fakePath := make([][33]byte, 3)
+	for i := 0; i < 3; i++ {
+		copy(fakePath[i][:], bytes.Repeat([]byte{byte(i)}, 33))
+	}
+
+	fakePayment := &outgoingPayment{
+		Invoice:        *fakeInvoice,
+		Fee:            101,
+		Path:           fakePath,
+		TimeLockLength: 1000,
+	}
+	copy(fakePayment.PaymentPreimage[:], rev[:])
+	return fakePayment
+}
+
+func makeFakeInfo() (*PaymentCreationInfo, *PaymentAttemptInfo) {
+	var preimg lntypes.Preimage
+	copy(preimg[:], rev[:])
+
+	c := &PaymentCreationInfo{
+		PaymentHash: preimg.Hash(),
+		Value:       1000,
+		// Use single second precision to avoid false positive test
+		// failures due to the monotonic time component.
+		CreationDate:   time.Unix(time.Now().Unix(), 0),
+		PaymentRequest: []byte(""),
+	}
+
+	a := &PaymentAttemptInfo{
+		PaymentID:  44,
+		SessionKey: priv,
+		Route:      testRoute,
+	}
+	return c, a
+}
+
+// randomBytes creates random []byte with length in range [minLen, maxLen)
+func randomBytes(minLen, maxLen int) ([]byte, error) {
+	randBuf := make([]byte, minLen+rand.Intn(maxLen-minLen))
+
+	if _, err := rand.Read(randBuf); err != nil {
+		return nil, fmt.Errorf("Internal error. "+
+			"Cannot generate random string: %v", err)
+	}
+
+	return randBuf, nil
+}
+
+func makeRandomFakePayment() (*outgoingPayment, error) {
+	var err error
+	fakeInvoice := &Invoice{
+		// Use single second precision to avoid false positive test
+		// failures due to the monotonic time component.
+		CreationDate: time.Unix(time.Now().Unix(), 0),
+	}
+
+	fakeInvoice.Memo, err = randomBytes(1, 50)
+	if err != nil {
+		return nil, err
+	}
+
+	fakeInvoice.Receipt, err = randomBytes(1, 50)
+	if err != nil {
+		return nil, err
+	}
+
+	fakeInvoice.PaymentRequest, err = randomBytes(1, 50)
+	if err != nil {
+		return nil, err
+	}
+
+	preImg, err := randomBytes(32, 33)
+	if err != nil {
+		return nil, err
+	}
+	copy(fakeInvoice.Terms.PaymentPreimage[:], preImg)
+
+	fakeInvoice.Terms.Value = lnwire.MilliSatoshi(rand.Intn(10000))
+
+	fakePathLen := 1 + rand.Intn(5)
+	fakePath := make([][33]byte, fakePathLen)
+	for i := 0; i < fakePathLen; i++ {
+		b, err := randomBytes(33, 34)
+		if err != nil {
+			return nil, err
+		}
+		copy(fakePath[i][:], b)
+	}
+
+	fakePayment := &outgoingPayment{
+		Invoice:        *fakeInvoice,
+		Fee:            lnwire.MilliSatoshi(rand.Intn(1001)),
+		Path:           fakePath,
+		TimeLockLength: uint32(rand.Intn(10000)),
+	}
+	copy(fakePayment.PaymentPreimage[:], fakeInvoice.Terms.PaymentPreimage[:])
+
+	return fakePayment, nil
+}
+
+func TestSentPaymentSerialization(t *testing.T) {
+	t.Parallel()
+
+	c, s := makeFakeInfo()
+
+	var b bytes.Buffer
+	if err := serializePaymentCreationInfo(&b, c); err != nil {
+		t.Fatalf("unable to serialize creation info: %v", err)
+	}
+
+	newCreationInfo, err := deserializePaymentCreationInfo(&b)
+	if err != nil {
+		t.Fatalf("unable to deserialize creation info: %v", err)
+	}
+
+	if !reflect.DeepEqual(c, newCreationInfo) {
+		t.Fatalf("Payments do not match after "+
+			"serialization/deserialization %v vs %v",
+			spew.Sdump(c), spew.Sdump(newCreationInfo),
+		)
+	}
+
+	b.Reset()
+	if err := serializePaymentAttemptInfo(&b, s); err != nil {
+		t.Fatalf("unable to serialize info: %v", err)
+	}
+
+	newAttemptInfo, err := deserializePaymentAttemptInfo(&b)
+	if err != nil {
+		t.Fatalf("unable to deserialize info: %v", err)
+	}
+
+	// First we verify all the records match up porperly, as they aren't
+	// able to be properly compared using reflect.DeepEqual.
+	err = assertRouteEqual(&s.Route, &newAttemptInfo.Route)
+	if err != nil {
+		t.Fatalf("Routes do not match after "+
+			"serialization/deserialization: %v", err)
+	}
+
+	// Clear routes to allow DeepEqual to compare the remaining fields.
+	newAttemptInfo.Route = route.Route{}
+	s.Route = route.Route{}
+
+	if !reflect.DeepEqual(s, newAttemptInfo) {
+		s.SessionKey.Curve = nil
+		newAttemptInfo.SessionKey.Curve = nil
+		t.Fatalf("Payments do not match after "+
+			"serialization/deserialization %v vs %v",
+			spew.Sdump(s), spew.Sdump(newAttemptInfo),
+		)
+	}
+}
+
+// assertRouteEquals compares to routes for equality and returns an error if
+// they are not equal.
+func assertRouteEqual(a, b *route.Route) error {
+	err := assertRouteHopRecordsEqual(a, b)
+	if err != nil {
+		return err
+	}
+
+	// TLV records have already been compared and need to be cleared to
+	// properly compare the remaining fields using DeepEqual.
+	copyRouteNoHops := func(r *route.Route) *route.Route {
+		copy := *r
+		copy.Hops = make([]*route.Hop, len(r.Hops))
+		for i, hop := range r.Hops {
+			hopCopy := *hop
+			hopCopy.TLVRecords = nil
+			copy.Hops[i] = &hopCopy
+		}
+		return &copy
+	}
+
+	if !reflect.DeepEqual(copyRouteNoHops(a), copyRouteNoHops(b)) {
+		return fmt.Errorf("PaymentAttemptInfos don't match: %v vs %v",
+			spew.Sdump(a), spew.Sdump(b))
+	}
+
+	return nil
+}
+
+func assertRouteHopRecordsEqual(r1, r2 *route.Route) error {
+	if len(r1.Hops) != len(r2.Hops) {
+		return errors.New("route hop count mismatch")
+	}
+
+	for i := 0; i < len(r1.Hops); i++ {
+		records1 := r1.Hops[i].TLVRecords
+		records2 := r2.Hops[i].TLVRecords
+		if len(records1) != len(records2) {
+			return fmt.Errorf("route record count for hop %v "+
+				"mismatch", i)
+		}
+
+		for j := 0; j < len(records1); j++ {
+			expectedRecord := records1[j]
+			newRecord := records2[j]
+
+			err := assertHopRecordsEqual(expectedRecord, newRecord)
+			if err != nil {
+				return fmt.Errorf("route record mismatch: %v", err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func assertHopRecordsEqual(h1, h2 tlv.Record) error {
+	if h1.Type() != h2.Type() {
+		return fmt.Errorf("wrong type: expected %v, got %v", h1.Type(),
+			h2.Type())
+	}
+
+	var b bytes.Buffer
+	if err := h2.Encode(&b); err != nil {
+		return fmt.Errorf("unable to encode record: %v", err)
+	}
+
+	if !bytes.Equal(b.Bytes(), tlvBytes) {
+		return fmt.Errorf("wrong raw record: expected %x, got %x",
+			tlvBytes, b.Bytes())
+	}
+
+	if h1.Size() != h2.Size() {
+		return fmt.Errorf("wrong size: expected %v, "+
+			"got %v", h1.Size(), h2.Size())
+	}
+
+	return nil
+}
+
+func TestRouteSerialization(t *testing.T) {
+	t.Parallel()
+
+	var b bytes.Buffer
+	if err := SerializeRoute(&b, testRoute); err != nil {
+		t.Fatal(err)
+	}
+
+	r := bytes.NewReader(b.Bytes())
+	route2, err := DeserializeRoute(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// First we verify all the records match up porperly, as they aren't
+	// able to be properly compared using reflect.DeepEqual.
+	err = assertRouteEqual(&testRoute, &route2)
+	if err != nil {
+		t.Fatalf("routes not equal: \n%v vs \n%v",
+			spew.Sdump(testRoute), spew.Sdump(route2))
+	}
+}

channeldb/migration_01_to_11/reject_cache.go

@@ -0,0 +1,95 @@
+package migration_01_to_11
+
+// rejectFlags is a compact representation of various metadata stored by the
+// reject cache about a particular channel.
+type rejectFlags uint8
+
+const (
+	// rejectFlagExists is a flag indicating whether the channel exists,
+	// i.e. the channel is open and has a recent channel update. If this
+	// flag is not set, the channel is either a zombie or unknown.
+	rejectFlagExists rejectFlags = 1 << iota
+
+	// rejectFlagZombie is a flag indicating whether the channel is a
+	// zombie, i.e. the channel is open but has no recent channel updates.
+	rejectFlagZombie
+)
+
+// packRejectFlags computes the rejectFlags corresponding to the passed boolean
+// values indicating whether the edge exists or is a zombie.
+func packRejectFlags(exists, isZombie bool) rejectFlags {
+	var flags rejectFlags
+	if exists {
+		flags |= rejectFlagExists
+	}
+	if isZombie {
+		flags |= rejectFlagZombie
+	}
+
+	return flags
+}
+
+// unpack returns the booleans packed into the rejectFlags. The first indicates
+// if the edge exists in our graph, the second indicates if the edge is a
+// zombie.
+func (f rejectFlags) unpack() (bool, bool) {
+	return f&rejectFlagExists == rejectFlagExists,
+		f&rejectFlagZombie == rejectFlagZombie
+}
+
+// rejectCacheEntry caches frequently accessed information about a channel,
+// including the timestamps of its latest edge policies and whether or not the
+// channel exists in the graph.
+type rejectCacheEntry struct {
+	upd1Time int64
+	upd2Time int64
+	flags    rejectFlags
+}
+
+// rejectCache is an in-memory cache used to improve the performance of
+// HasChannelEdge. It caches information about the whether or channel exists, as
+// well as the most recent timestamps for each policy (if they exists).
+type rejectCache struct {
+	n     int
+	edges map[uint64]rejectCacheEntry
+}
+
+// newRejectCache creates a new rejectCache with maximum capacity of n entries.
+func newRejectCache(n int) *rejectCache {
+	return &rejectCache{
+		n:     n,
+		edges: make(map[uint64]rejectCacheEntry, n),
+	}
+}
+
+// get returns the entry from the cache for chanid, if it exists.
+func (c *rejectCache) get(chanid uint64) (rejectCacheEntry, bool) {
+	entry, ok := c.edges[chanid]
+	return entry, ok
+}
+
+// insert adds the entry to the reject cache. If an entry for chanid already
+// exists, it will be replaced with the new entry. If the entry doesn't exists,
+// it will be inserted to the cache, performing a random eviction if the cache
+// is at capacity.
+func (c *rejectCache) insert(chanid uint64, entry rejectCacheEntry) {
+	// If entry exists, replace it.
+	if _, ok := c.edges[chanid]; ok {
+		c.edges[chanid] = entry
+		return
+	}
+
+	// Otherwise, evict an entry at random and insert.
+	if len(c.edges) == c.n {
+		for id := range c.edges {
+			delete(c.edges, id)
+			break
+		}
+	}
+	c.edges[chanid] = entry
+}
+
+// remove deletes an entry for chanid from the cache, if it exists.
+func (c *rejectCache) remove(chanid uint64) {
+	delete(c.edges, chanid)
+}

channeldb/migration_01_to_11/reject_cache_test.go

@@ -0,0 +1,107 @@
+package migration_01_to_11
+
+import (
+	"reflect"
+	"testing"
+)
+
+// TestRejectCache checks the behavior of the rejectCache with respect to insertion,
+// eviction, and removal of cache entries.
+func TestRejectCache(t *testing.T) {
+	const cacheSize = 100
+
+	// Create a new reject cache with the configured max size.
+	c := newRejectCache(cacheSize)
+
+	// As a sanity check, assert that querying the empty cache does not
+	// return an entry.
+	_, ok := c.get(0)
+	if ok {
+		t.Fatalf("reject cache should be empty")
+	}
+
+	// Now, fill up the cache entirely.
+	for i := uint64(0); i < cacheSize; i++ {
+		c.insert(i, entryForInt(i))
+	}
+
+	// Assert that the cache has all of the entries just inserted, since no
+	// eviction should occur until we try to surpass the max size.
+	assertHasEntries(t, c, 0, cacheSize)
+
+	// Now, insert a new element that causes the cache to evict an element.
+	c.insert(cacheSize, entryForInt(cacheSize))
+
+	// Assert that the cache has this last entry, as the cache should evict
+	// some prior element and not the newly inserted one.
+	assertHasEntries(t, c, cacheSize, cacheSize)
+
+	// Iterate over all inserted elements and construct a set of the evicted
+	// elements.
+	evicted := make(map[uint64]struct{})
+	for i := uint64(0); i < cacheSize+1; i++ {
+		_, ok := c.get(i)
+		if !ok {
+			evicted[i] = struct{}{}
+		}
+	}
+
+	// Assert that exactly one element has been evicted.
+	numEvicted := len(evicted)
+	if numEvicted != 1 {
+		t.Fatalf("expected one evicted entry, got: %d", numEvicted)
+	}
+
+	// Remove the highest item which initially caused the eviction and
+	// reinsert the element that was evicted prior.
+	c.remove(cacheSize)
+	for i := range evicted {
+		c.insert(i, entryForInt(i))
+	}
+
+	// Since the removal created an extra slot, the last insertion should
+	// not have caused an eviction and the entries for all channels in the
+	// original set that filled the cache should be present.
+	assertHasEntries(t, c, 0, cacheSize)
+
+	// Finally, reinsert the existing set back into the cache and test that
+	// the cache still has all the entries. If the randomized eviction were
+	// happening on inserts for existing cache items, we expect this to fail
+	// with high probability.
+	for i := uint64(0); i < cacheSize; i++ {
+		c.insert(i, entryForInt(i))
+	}
+	assertHasEntries(t, c, 0, cacheSize)
+
+}
+
+// assertHasEntries queries the reject cache for all channels in the range [start,
+// end), asserting that they exist and their value matches the entry produced by
+// entryForInt.
+func assertHasEntries(t *testing.T, c *rejectCache, start, end uint64) {
+	t.Helper()
+
+	for i := start; i < end; i++ {
+		entry, ok := c.get(i)
+		if !ok {
+			t.Fatalf("reject cache should contain chan %d", i)
+		}
+
+		expEntry := entryForInt(i)
+		if !reflect.DeepEqual(entry, expEntry) {
+			t.Fatalf("entry mismatch, want: %v, got: %v",
+				expEntry, entry)
+		}
+	}
+}
+
+// entryForInt generates a unique rejectCacheEntry given an integer.
+func entryForInt(i uint64) rejectCacheEntry {
+	exists := i%2 == 0
+	isZombie := i%3 == 0
+	return rejectCacheEntry{
+		upd1Time: int64(2 * i),
+		upd2Time: int64(2*i + 1),
+		flags:    packRejectFlags(exists, isZombie),
+	}
+}

channeldb/migration_01_to_11/waitingproof.go

@@ -0,0 +1,251 @@
+package migration_01_to_11
+
+import (
+	"encoding/binary"
+	"sync"
+
+	"io"
+
+	"bytes"
+
+	"github.com/coreos/bbolt"
+	"github.com/go-errors/errors"
+	"github.com/lightningnetwork/lnd/lnwire"
+)
+
+var (
+	// waitingProofsBucketKey byte string name of the waiting proofs store.
+	waitingProofsBucketKey = []byte("waitingproofs")
+
+	// ErrWaitingProofNotFound is returned if waiting proofs haven't been
+	// found by db.
+	ErrWaitingProofNotFound = errors.New("waiting proofs haven't been " +
+		"found")
+
+	// ErrWaitingProofAlreadyExist is returned if waiting proofs haven't been
+	// found by db.
+	ErrWaitingProofAlreadyExist = errors.New("waiting proof with such " +
+		"key already exist")
+)
+
+// WaitingProofStore is the bold db map-like storage for half announcement
+// signatures. The one responsibility of this storage is to be able to
+// retrieve waiting proofs after client restart.
+type WaitingProofStore struct {
+	// cache is used in order to reduce the number of redundant get
+	// calls, when object isn't stored in it.
+	cache map[WaitingProofKey]struct{}
+	db    *DB
+	mu    sync.RWMutex
+}
+
+// NewWaitingProofStore creates new instance of proofs storage.
+func NewWaitingProofStore(db *DB) (*WaitingProofStore, error) {
+	s := &WaitingProofStore{
+		db:    db,
+		cache: make(map[WaitingProofKey]struct{}),
+	}
+
+	if err := s.ForAll(func(proof *WaitingProof) error {
+		s.cache[proof.Key()] = struct{}{}
+		return nil
+	}); err != nil && err != ErrWaitingProofNotFound {
+		return nil, err
+	}
+
+	return s, nil
+}
+
+// Add adds new waiting proof in the storage.
+func (s *WaitingProofStore) Add(proof *WaitingProof) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	err := s.db.Update(func(tx *bbolt.Tx) error {
+		var err error
+		var b bytes.Buffer
+
+		// Get or create the bucket.
+		bucket, err := tx.CreateBucketIfNotExists(waitingProofsBucketKey)
+		if err != nil {
+			return err
+		}
+
+		// Encode the objects and place it in the bucket.
+		if err := proof.Encode(&b); err != nil {
+			return err
+		}
+
+		key := proof.Key()
+
+		return bucket.Put(key[:], b.Bytes())
+	})
+	if err != nil {
+		return err
+	}
+
+	// Knowing that the write succeeded, we can now update the in-memory
+	// cache with the proof's key.
+	s.cache[proof.Key()] = struct{}{}
+
+	return nil
+}
+
+// Remove removes the proof from storage by its key.
+func (s *WaitingProofStore) Remove(key WaitingProofKey) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if _, ok := s.cache[key]; !ok {
+		return ErrWaitingProofNotFound
+	}
+
+	err := s.db.Update(func(tx *bbolt.Tx) error {
+		// Get or create the top bucket.
+		bucket := tx.Bucket(waitingProofsBucketKey)
+		if bucket == nil {
+			return ErrWaitingProofNotFound
+		}
+
+		return bucket.Delete(key[:])
+	})
+	if err != nil {
+		return err
+	}
+
+	// Since the proof was successfully deleted from the store, we can now
+	// remove it from the in-memory cache.
+	delete(s.cache, key)
+
+	return nil
+}
+
+// ForAll iterates thought all waiting proofs and passing the waiting proof
+// in the given callback.
+func (s *WaitingProofStore) ForAll(cb func(*WaitingProof) error) error {
+	return s.db.View(func(tx *bbolt.Tx) error {
+		bucket := tx.Bucket(waitingProofsBucketKey)
+		if bucket == nil {
+			return ErrWaitingProofNotFound
+		}
+
+		// Iterate over objects buckets.
+		return bucket.ForEach(func(k, v []byte) error {
+			// Skip buckets fields.
+			if v == nil {
+				return nil
+			}
+
+			r := bytes.NewReader(v)
+			proof := &WaitingProof{}
+			if err := proof.Decode(r); err != nil {
+				return err
+			}
+
+			return cb(proof)
+		})
+	})
+}
+
+// Get returns the object which corresponds to the given index.
+func (s *WaitingProofStore) Get(key WaitingProofKey) (*WaitingProof, error) {
+	proof := &WaitingProof{}
+
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	if _, ok := s.cache[key]; !ok {
+		return nil, ErrWaitingProofNotFound
+	}
+
+	err := s.db.View(func(tx *bbolt.Tx) error {
+		bucket := tx.Bucket(waitingProofsBucketKey)
+		if bucket == nil {
+			return ErrWaitingProofNotFound
+		}
+
+		// Iterate over objects buckets.
+		v := bucket.Get(key[:])
+		if v == nil {
+			return ErrWaitingProofNotFound
+		}
+
+		r := bytes.NewReader(v)
+		return proof.Decode(r)
+	})
+
+	return proof, err
+}
+
+// WaitingProofKey is the proof key which uniquely identifies the waiting
+// proof object. The goal of this key is distinguish the local and remote
+// proof for the same channel id.
+type WaitingProofKey [9]byte
+
+// WaitingProof is the storable object, which encapsulate the half proof and
+// the information about from which side this proof came. This structure is
+// needed to make channel proof exchange persistent, so that after client
+// restart we may receive remote/local half proof and process it.
+type WaitingProof struct {
+	*lnwire.AnnounceSignatures
+	isRemote bool
+}
+
+// NewWaitingProof constructs a new waiting prof instance.
+func NewWaitingProof(isRemote bool, proof *lnwire.AnnounceSignatures) *WaitingProof {
+	return &WaitingProof{
+		AnnounceSignatures: proof,
+		isRemote:           isRemote,
+	}
+}
+
+// OppositeKey returns the key which uniquely identifies opposite waiting proof.
+func (p *WaitingProof) OppositeKey() WaitingProofKey {
+	var key [9]byte
+	binary.BigEndian.PutUint64(key[:8], p.ShortChannelID.ToUint64())
+
+	if !p.isRemote {
+		key[8] = 1
+	}
+	return key
+}
+
+// Key returns the key which uniquely identifies waiting proof.
+func (p *WaitingProof) Key() WaitingProofKey {
+	var key [9]byte
+	binary.BigEndian.PutUint64(key[:8], p.ShortChannelID.ToUint64())
+
+	if p.isRemote {
+		key[8] = 1
+	}
+	return key
+}
+
+// Encode writes the internal representation of waiting proof in byte stream.
+func (p *WaitingProof) Encode(w io.Writer) error {
+	if err := binary.Write(w, byteOrder, p.isRemote); err != nil {
+		return err
+	}
+
+	if err := p.AnnounceSignatures.Encode(w, 0); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Decode reads the data from the byte stream and initializes the
+// waiting proof object with it.
+func (p *WaitingProof) Decode(r io.Reader) error {
+	if err := binary.Read(r, byteOrder, &p.isRemote); err != nil {
+		return err
+	}
+
+	msg := &lnwire.AnnounceSignatures{}
+	if err := msg.Decode(r, 0); err != nil {
+		return err
+	}
+
+	(*p).AnnounceSignatures = msg
+	return nil
+}

channeldb/migration_01_to_11/waitingproof_test.go

@@ -0,0 +1,59 @@
+package migration_01_to_11
+
+import (
+	"testing"
+
+	"reflect"
+
+	"github.com/go-errors/errors"
+	"github.com/lightningnetwork/lnd/lnwire"
+)
+
+// TestWaitingProofStore tests add/get/remove functions of the waiting proof
+// storage.
+func TestWaitingProofStore(t *testing.T) {
+	t.Parallel()
+
+	db, cleanup, err := makeTestDB()
+	if err != nil {
+		t.Fatalf("failed to make test database: %s", err)
+	}
+	defer cleanup()
+
+	proof1 := NewWaitingProof(true, &lnwire.AnnounceSignatures{
+		NodeSignature:    wireSig,
+		BitcoinSignature: wireSig,
+	})
+
+	store, err := NewWaitingProofStore(db)
+	if err != nil {
+		t.Fatalf("unable to create the waiting proofs storage: %v",
+			err)
+	}
+
+	if err := store.Add(proof1); err != nil {
+		t.Fatalf("unable add proof to storage: %v", err)
+	}
+
+	proof2, err := store.Get(proof1.Key())
+	if err != nil {
+		t.Fatalf("unable retrieve proof from storage: %v", err)
+	}
+	if !reflect.DeepEqual(proof1, proof2) {
+		t.Fatal("wrong proof retrieved")
+	}
+
+	if _, err := store.Get(proof1.OppositeKey()); err != ErrWaitingProofNotFound {
+		t.Fatalf("proof shouldn't be found: %v", err)
+	}
+
+	if err := store.Remove(proof1.Key()); err != nil {
+		t.Fatalf("unable remove proof from storage: %v", err)
+	}
+
+	if err := store.ForAll(func(proof *WaitingProof) error {
+		return errors.New("storage should be empty")
+	}); err != nil && err != ErrWaitingProofNotFound {
+		t.Fatal(err)
+	}
+}

channeldb/migration_01_to_11/witness_cache.go

@@ -0,0 +1,229 @@
+package migration_01_to_11
+
+import (
+	"fmt"
+
+	"github.com/coreos/bbolt"
+	"github.com/lightningnetwork/lnd/lntypes"
+)
+
+var (
+	// ErrNoWitnesses is an error that's returned when no new witnesses have
+	// been added to the WitnessCache.
+	ErrNoWitnesses = fmt.Errorf("no witnesses")
+
+	// ErrUnknownWitnessType is returned if a caller attempts to
+	ErrUnknownWitnessType = fmt.Errorf("unknown witness type")
+)
+
+// WitnessType is enum that denotes what "type" of witness is being
+// stored/retrieved. As the WitnessCache itself is agnostic and doesn't enforce
+// any structure on added witnesses, we use this type to partition the
+// witnesses on disk, and also to know how to map a witness to its look up key.
+type WitnessType uint8
+
+var (
+	// Sha256HashWitness is a witness that is simply the pre image to a
+	// hash image. In order to map to its key, we'll use sha256.
+	Sha256HashWitness WitnessType = 1
+)
+
+// toDBKey is a helper method that maps a witness type to the key that we'll
+// use to store it within the database.
+func (w WitnessType) toDBKey() ([]byte, error) {
+	switch w {
+
+	case Sha256HashWitness:
+		return []byte{byte(w)}, nil
+
+	default:
+		return nil, ErrUnknownWitnessType
+	}
+}
+
+var (
+	// witnessBucketKey is the name of the bucket that we use to store all
+	// witnesses encountered. Within this bucket, we'll create a sub-bucket for
+	// each witness type.
+	witnessBucketKey = []byte("byte")
+)
+
+// WitnessCache is a persistent cache of all witnesses we've encountered on the
+// network. In the case of multi-hop, multi-step contracts, a cache of all
+// witnesses can be useful in the case of partial contract resolution. If
+// negotiations break down, we may be forced to locate the witness for a
+// portion of the contract on-chain. In this case, we'll then add that witness
+// to the cache so the incoming contract can fully resolve witness.
+// Additionally, as one MUST always use a unique witness on the network, we may
+// use this cache to detect duplicate witnesses.
+//
+// TODO(roasbeef): need expiry policy?
+//  * encrypt?
+type WitnessCache struct {
+	db *DB
+}
+
+// NewWitnessCache returns a new instance of the witness cache.
+func (d *DB) NewWitnessCache() *WitnessCache {
+	return &WitnessCache{
+		db: d,
+	}
+}
+
+// witnessEntry is a key-value struct that holds each key -> witness pair, used
+// when inserting records into the cache.
+type witnessEntry struct {
+	key     []byte
+	witness []byte
+}
+
+// AddSha256Witnesses adds a batch of new sha256 preimages into the witness
+// cache. This is an alias for AddWitnesses that uses Sha256HashWitness as the
+// preimages' witness type.
+func (w *WitnessCache) AddSha256Witnesses(preimages ...lntypes.Preimage) error {
+	// Optimistically compute the preimages' hashes before attempting to
+	// start the db transaction.
+	entries := make([]witnessEntry, 0, len(preimages))
+	for i := range preimages {
+		hash := preimages[i].Hash()
+		entries = append(entries, witnessEntry{
+			key:     hash[:],
+			witness: preimages[i][:],
+		})
+	}
+
+	return w.addWitnessEntries(Sha256HashWitness, entries)
+}
+
+// addWitnessEntries inserts the witnessEntry key-value pairs into the cache,
+// using the appropriate witness type to segment the namespace of possible
+// witness types.
+func (w *WitnessCache) addWitnessEntries(wType WitnessType,
+	entries []witnessEntry) error {
+
+	// Exit early if there are no witnesses to add.
+	if len(entries) == 0 {
+		return nil
+	}
+
+	return w.db.Batch(func(tx *bbolt.Tx) error {
+		witnessBucket, err := tx.CreateBucketIfNotExists(witnessBucketKey)
+		if err != nil {
+			return err
+		}
+
+		witnessTypeBucketKey, err := wType.toDBKey()
+		if err != nil {
+			return err
+		}
+		witnessTypeBucket, err := witnessBucket.CreateBucketIfNotExists(
+			witnessTypeBucketKey,
+		)
+		if err != nil {
+			return err
+		}
+
+		for _, entry := range entries {
+			err = witnessTypeBucket.Put(entry.key, entry.witness)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+// LookupSha256Witness attempts to lookup the preimage for a sha256 hash. If
+// the witness isn't found, ErrNoWitnesses will be returned.
+func (w *WitnessCache) LookupSha256Witness(hash lntypes.Hash) (lntypes.Preimage, error) {
+	witness, err := w.lookupWitness(Sha256HashWitness, hash[:])
+	if err != nil {
+		return lntypes.Preimage{}, err
+	}
+
+	return lntypes.MakePreimage(witness)
+}
+
+// lookupWitness attempts to lookup a witness according to its type and also
+// its witness key. In the case that the witness isn't found, ErrNoWitnesses
+// will be returned.
+func (w *WitnessCache) lookupWitness(wType WitnessType, witnessKey []byte) ([]byte, error) {
+	var witness []byte
+	err := w.db.View(func(tx *bbolt.Tx) error {
+		witnessBucket := tx.Bucket(witnessBucketKey)
+		if witnessBucket == nil {
+			return ErrNoWitnesses
+		}
+
+		witnessTypeBucketKey, err := wType.toDBKey()
+		if err != nil {
+			return err
+		}
+		witnessTypeBucket := witnessBucket.Bucket(witnessTypeBucketKey)
+		if witnessTypeBucket == nil {
+			return ErrNoWitnesses
+		}
+
+		dbWitness := witnessTypeBucket.Get(witnessKey)
+		if dbWitness == nil {
+			return ErrNoWitnesses
+		}
+
+		witness = make([]byte, len(dbWitness))
+		copy(witness[:], dbWitness)
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return witness, nil
+}
+
+// DeleteSha256Witness attempts to delete a sha256 preimage identified by hash.
+func (w *WitnessCache) DeleteSha256Witness(hash lntypes.Hash) error {
+	return w.deleteWitness(Sha256HashWitness, hash[:])
+}
+
+// deleteWitness attempts to delete a particular witness from the database.
+func (w *WitnessCache) deleteWitness(wType WitnessType, witnessKey []byte) error {
+	return w.db.Batch(func(tx *bbolt.Tx) error {
+		witnessBucket, err := tx.CreateBucketIfNotExists(witnessBucketKey)
+		if err != nil {
+			return err
+		}
+
+		witnessTypeBucketKey, err := wType.toDBKey()
+		if err != nil {
+			return err
+		}
+		witnessTypeBucket, err := witnessBucket.CreateBucketIfNotExists(
+			witnessTypeBucketKey,
+		)
+		if err != nil {
+			return err
+		}
+
+		return witnessTypeBucket.Delete(witnessKey)
+	})
+}
+
+// DeleteWitnessClass attempts to delete an *entire* class of witnesses. After
+// this function return with a non-nil error,
+func (w *WitnessCache) DeleteWitnessClass(wType WitnessType) error {
+	return w.db.Batch(func(tx *bbolt.Tx) error {
+		witnessBucket, err := tx.CreateBucketIfNotExists(witnessBucketKey)
+		if err != nil {
+			return err
+		}
+
+		witnessTypeBucketKey, err := wType.toDBKey()
+		if err != nil {
+			return err
+		}
+
+		return witnessBucket.DeleteBucket(witnessTypeBucketKey)
+	})
+}

channeldb/migration_01_to_11/witness_cache_test.go

@@ -0,0 +1,238 @@
+package migration_01_to_11
+
+import (
+	"crypto/sha256"
+	"testing"
+
+	"github.com/lightningnetwork/lnd/lntypes"
+)
+
+// TestWitnessCacheSha256Retrieval tests that we're able to add and lookup new
+// sha256 preimages to the witness cache.
+func TestWitnessCacheSha256Retrieval(t *testing.T) {
+	t.Parallel()
+
+	cdb, cleanUp, err := makeTestDB()
+	if err != nil {
+		t.Fatalf("unable to make test database: %v", err)
+	}
+	defer cleanUp()
+
+	wCache := cdb.NewWitnessCache()
+
+	// We'll be attempting to add then lookup two simple sha256 preimages
+	// within this test.
+	preimage1 := lntypes.Preimage(rev)
+	preimage2 := lntypes.Preimage(key)
+
+	preimages := []lntypes.Preimage{preimage1, preimage2}
+	hashes := []lntypes.Hash{preimage1.Hash(), preimage2.Hash()}
+
+	// First, we'll attempt to add the preimages to the database.
+	err = wCache.AddSha256Witnesses(preimages...)
+	if err != nil {
+		t.Fatalf("unable to add witness: %v", err)
+	}
+
+	// With the preimages stored, we'll now attempt to look them up.
+	for i, hash := range hashes {
+		preimage := preimages[i]
+
+		// We should get back the *exact* same preimage as we originally
+		// stored.
+		dbPreimage, err := wCache.LookupSha256Witness(hash)
+		if err != nil {
+			t.Fatalf("unable to look up witness: %v", err)
+		}
+
+		if preimage != dbPreimage {
+			t.Fatalf("witnesses don't match: expected %x, got %x",
+				preimage[:], dbPreimage[:])
+		}
+	}
+}
+
+// TestWitnessCacheSha256Deletion tests that we're able to delete a single
+// sha256 preimage, and also a class of witnesses from the cache.
+func TestWitnessCacheSha256Deletion(t *testing.T) {
+	t.Parallel()
+
+	cdb, cleanUp, err := makeTestDB()
+	if err != nil {
+		t.Fatalf("unable to make test database: %v", err)
+	}
+	defer cleanUp()
+
+	wCache := cdb.NewWitnessCache()
+
+	// We'll start by adding two preimages to the cache.
+	preimage1 := lntypes.Preimage(key)
+	hash1 := preimage1.Hash()
+
+	preimage2 := lntypes.Preimage(rev)
+	hash2 := preimage2.Hash()
+
+	if err := wCache.AddSha256Witnesses(preimage1); err != nil {
+		t.Fatalf("unable to add witness: %v", err)
+	}
+
+	if err := wCache.AddSha256Witnesses(preimage2); err != nil {
+		t.Fatalf("unable to add witness: %v", err)
+	}
+
+	// We'll now delete the first preimage. If we attempt to look it up, we
+	// should get ErrNoWitnesses.
+	err = wCache.DeleteSha256Witness(hash1)
+	if err != nil {
+		t.Fatalf("unable to delete witness: %v", err)
+	}
+	_, err = wCache.LookupSha256Witness(hash1)
+	if err != ErrNoWitnesses {
+		t.Fatalf("expected ErrNoWitnesses instead got: %v", err)
+	}
+
+	// Next, we'll attempt to delete the entire witness class itself. When
+	// we try to lookup the second preimage, we should again get
+	// ErrNoWitnesses.
+	if err := wCache.DeleteWitnessClass(Sha256HashWitness); err != nil {
+		t.Fatalf("unable to delete witness class: %v", err)
+	}
+	_, err = wCache.LookupSha256Witness(hash2)
+	if err != ErrNoWitnesses {
+		t.Fatalf("expected ErrNoWitnesses instead got: %v", err)
+	}
+}
+
+// TestWitnessCacheUnknownWitness tests that we get an error if we attempt to
+// query/add/delete an unknown witness.
+func TestWitnessCacheUnknownWitness(t *testing.T) {
+	t.Parallel()
+
+	cdb, cleanUp, err := makeTestDB()
+	if err != nil {
+		t.Fatalf("unable to make test database: %v", err)
+	}
+	defer cleanUp()
+
+	wCache := cdb.NewWitnessCache()
+
+	// We'll attempt to add a new, undefined witness type to the database.
+	// We should get an error.
+	err = wCache.legacyAddWitnesses(234, key[:])
+	if err != ErrUnknownWitnessType {
+		t.Fatalf("expected ErrUnknownWitnessType, got %v", err)
+	}
+}
+
+// TestAddSha256Witnesses tests that insertion using AddSha256Witnesses behaves
+// identically to the insertion via the generalized interface.
+func TestAddSha256Witnesses(t *testing.T) {
+	cdb, cleanUp, err := makeTestDB()
+	if err != nil {
+		t.Fatalf("unable to make test database: %v", err)
+	}
+	defer cleanUp()
+
+	wCache := cdb.NewWitnessCache()
+
+	// We'll start by adding a witnesses to the cache using the generic
+	// AddWitnesses method.
+	witness1 := rev[:]
+	preimage1 := lntypes.Preimage(rev)
+	hash1 := preimage1.Hash()
+
+	witness2 := key[:]
+	preimage2 := lntypes.Preimage(key)
+	hash2 := preimage2.Hash()
+
+	var (
+		witnesses = [][]byte{witness1, witness2}
+		preimages = []lntypes.Preimage{preimage1, preimage2}
+		hashes    = []lntypes.Hash{hash1, hash2}
+	)
+
+	err = wCache.legacyAddWitnesses(Sha256HashWitness, witnesses...)
+	if err != nil {
+		t.Fatalf("unable to add witness: %v", err)
+	}
+
+	for i, hash := range hashes {
+		preimage := preimages[i]
+
+		dbPreimage, err := wCache.LookupSha256Witness(hash)
+		if err != nil {
+			t.Fatalf("unable to lookup witness: %v", err)
+		}
+
+		// Assert that the retrieved witness matches the original.
+		if dbPreimage != preimage {
+			t.Fatalf("retrieved witness mismatch, want: %x, "+
+				"got: %x", preimage, dbPreimage)
+		}
+
+		// We'll now delete the witness, as we'll be reinserting it
+		// using the specialized AddSha256Witnesses method.
+		err = wCache.DeleteSha256Witness(hash)
+		if err != nil {
+			t.Fatalf("unable to delete witness: %v", err)
+		}
+	}
+
+	// Now, add the same witnesses using the type-safe interface for
+	// lntypes.Preimages..
+	err = wCache.AddSha256Witnesses(preimages...)
+	if err != nil {
+		t.Fatalf("unable to add sha256 preimage: %v", err)
+	}
+
+	// Finally, iterate over the keys and assert that the returned witnesses
+	// match the original witnesses. This asserts that the specialized
+	// insertion method behaves identically to the generalized interface.
+	for i, hash := range hashes {
+		preimage := preimages[i]
+
+		dbPreimage, err := wCache.LookupSha256Witness(hash)
+		if err != nil {
+			t.Fatalf("unable to lookup witness: %v", err)
+		}
+
+		// Assert that the retrieved witness matches the original.
+		if dbPreimage != preimage {
+			t.Fatalf("retrieved witness mismatch, want: %x, "+
+				"got: %x", preimage, dbPreimage)
+		}
+	}
+}
+
+// legacyAddWitnesses adds a batch of new witnesses of wType to the witness
+// cache. The type of the witness will be used to map each witness to the key
+// that will be used to look it up. All witnesses should be of the same
+// WitnessType.
+//
+// NOTE: Previously this method exposed a generic interface for adding
+// witnesses, which has since been deprecated in favor of a strongly typed
+// interface for each witness class. We keep this method around to assert the
+// correctness of specialized witness adding methods.
+func (w *WitnessCache) legacyAddWitnesses(wType WitnessType,
+	witnesses ...[]byte) error {
+
+	// Optimistically compute the witness keys before attempting to start
+	// the db transaction.
+	entries := make([]witnessEntry, 0, len(witnesses))
+	for _, witness := range witnesses {
+		// Map each witness to its key by applying the appropriate
+		// transformation for the given witness type.
+		switch wType {
+		case Sha256HashWitness:
+			key := sha256.Sum256(witness)
+			entries = append(entries, witnessEntry{
+				key:     key[:],
+				witness: witness,
+			})
+		default:
+			return ErrUnknownWitnessType
+		}
+	}
+
+	return w.addWitnessEntries(wType, entries)
+}