mirror/lnd
1
0 Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

channeldb: reference migrations in package

Browse Source 
This commit removes the migrations from channeldb and references those
in the migrations_01_to_11 package. This creates a one-way dependency on
the migrations. Future changes to channeldb won't be able to break
migrations anymore.
This commit is contained in:
Joost Jager committed 2019-10-24 12:49:40 +02:00
1 parent 6e463c1634
commit 6913cd64b6
16 files changed
+43 -3634
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

channeldb/migration_01_to_11/db.go
+1 -92
View File
@@ -32,91 +32,6 @@ type version struct {
}
var (
// dbVersions is storing all versions of database. If current version
// of database don't match with latest version this list will be used
// for retrieving all migration function that are need to apply to the
// current db.
dbVersions = []version{
{
// The base DB version requires no migration.
number: 0,
migration: nil,
},
{
// The version of the database where two new indexes
// for the update time of node and channel updates were
// added.
number: 1,
migration: migrateNodeAndEdgeUpdateIndex,
},
{
// The DB version that added the invoice event time
// series.
number: 2,
migration: migrateInvoiceTimeSeries,
},
{
// The DB version that updated the embedded invoice in
// outgoing payments to match the new format.
number: 3,
migration: migrateInvoiceTimeSeriesOutgoingPayments,
},
{
// The version of the database where every channel
// always has two entries in the edges bucket. If
// a policy is unknown, this will be represented
// by a special byte sequence.
number: 4,
migration: migrateEdgePolicies,
},
{
// The DB version where we persist each attempt to send
// an HTLC to a payment hash, and track whether the
// payment is in-flight, succeeded, or failed.
number: 5,
migration: paymentStatusesMigration,
},
{
// The DB version that properly prunes stale entries
// from the edge update index.
number: 6,
migration: migratePruneEdgeUpdateIndex,
},
{
// The DB version that migrates the ChannelCloseSummary
// to a format where optional fields are indicated with
// boolean flags.
number: 7,
migration: migrateOptionalChannelCloseSummaryFields,
},
{
// The DB version that changes the gossiper's message
// store keys to account for the message's type and
// ShortChannelID.
number: 8,
migration: migrateGossipMessageStoreKeys,
},
{
// The DB version where the payments and payment
// statuses are moved to being stored in a combined
// bucket.
number: 9,
migration: migrateOutgoingPayments,
},
{
// The DB version where we started to store legacy
// payload information for all routes, as well as the
// optional TLV records.
number: 10,
migration: migrateRouteSerialization,
},
{
// Add invoice htlc and cltv delta fields.
number: 11,
migration: migrateInvoices,
},
}
// Big endian is the preferred byte order, due to cursor scans over
// integer keys iterating in order.
byteOrder = binary.BigEndian
@@ -169,12 +84,6 @@ func Open(dbPath string, modifiers ...OptionModifier) (*DB, error) {
chanDB, opts.RejectCacheSize, opts.ChannelCacheSize,
)
// Synchronize the version of database and apply migrations if needed.
if err := chanDB.syncVersions(dbVersions); err != nil {
bdb.Close()
return nil, err
}
return chanDB, nil
}
@@ -318,7 +227,7 @@ func createChannelDB(dbPath string) error {
}
meta := &Meta{
DbVersionNumber: getLatestDBVersion(dbVersions),
DbVersionNumber: 0,
}
return putMeta(meta, tx)
})
channeldb/migration_01_to_11/meta.go
+1 -1
View File
@@ -44,7 +44,7 @@ func fetchMeta(meta *Meta, tx *bbolt.Tx) error {
data := metaBucket.Get(dbVersionKey)
if data == nil {
meta.DbVersionNumber = getLatestDBVersion(dbVersions)
meta.DbVersionNumber = 0
} else {
meta.DbVersionNumber = byteOrder.Uint32(data)
}
channeldb/migration_01_to_11/meta_test.go
-369
View File
@@ -1,11 +1,8 @@
package migration_01_to_11
import (
"bytes"
"io/ioutil"
"testing"
"github.com/coreos/bbolt"
"github.com/go-errors/errors"
)
@@ -74,369 +71,3 @@ func applyMigration(t *testing.T, beforeMigration, afterMigration func(d *DB),
log.Error(err)
}
}
// TestVersionFetchPut checks the propernces of fetch/put methods
// and also initialization of meta data in case if don't have any in
// database.
func TestVersionFetchPut(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
defer cleanUp()
if err != nil {
t.Fatal(err)
}
meta, err := db.FetchMeta(nil)
if err != nil {
t.Fatal(err)
}
if meta.DbVersionNumber != getLatestDBVersion(dbVersions) {
t.Fatal("initialization of meta information wasn't performed")
}
newVersion := getLatestDBVersion(dbVersions) + 1
meta.DbVersionNumber = newVersion
if err := db.PutMeta(meta); err != nil {
t.Fatalf("update of meta failed %v", err)
}
meta, err = db.FetchMeta(nil)
if err != nil {
t.Fatal(err)
}
if meta.DbVersionNumber != newVersion {
t.Fatal("update of meta information wasn't performed")
}
}
// TestOrderOfMigrations checks that migrations are applied in proper order.
func TestOrderOfMigrations(t *testing.T) {
t.Parallel()
appliedMigration := -1
versions := []version{
{0, nil},
{1, nil},
{2, func(tx *bbolt.Tx) error {
appliedMigration = 2
return nil
}},
{3, func(tx *bbolt.Tx) error {
appliedMigration = 3
return nil
}},
}
// Retrieve the migration that should be applied to db, as far as
// current version is 1, we skip zero and first versions.
migrations, _ := getMigrationsToApply(versions, 1)
if len(migrations) != 2 {
t.Fatal("incorrect number of migrations to apply")
}
// Apply first migration.
migrations[0](nil)
// Check that first migration corresponds to the second version.
if appliedMigration != 2 {
t.Fatal("incorrect order of applying migrations")
}
// Apply second migration.
migrations[1](nil)
// Check that second migration corresponds to the third version.
if appliedMigration != 3 {
t.Fatal("incorrect order of applying migrations")
}
}
// TestGlobalVersionList checks that there is no mistake in global version list
// in terms of version ordering.
func TestGlobalVersionList(t *testing.T) {
t.Parallel()
if dbVersions == nil {
t.Fatal("can't find versions list")
}
if len(dbVersions) == 0 {
t.Fatal("db versions list is empty")
}
prev := dbVersions[0].number
for i := 1; i < len(dbVersions); i++ {
version := dbVersions[i].number
if version == prev {
t.Fatal("duplicates db versions")
}
if version < prev {
t.Fatal("order of db versions is wrong")
}
prev = version
}
}
// TestMigrationWithPanic asserts that if migration logic panics, we will return
// to the original state unaltered.
func TestMigrationWithPanic(t *testing.T) {
t.Parallel()
bucketPrefix := []byte("somebucket")
keyPrefix := []byte("someprefix")
beforeMigration := []byte("beforemigration")
afterMigration := []byte("aftermigration")
beforeMigrationFunc := func(d *DB) {
// Insert data in database and in order then make sure that the
// key isn't changes in case of panic or fail.
d.Update(func(tx *bbolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists(bucketPrefix)
if err != nil {
return err
}
bucket.Put(keyPrefix, beforeMigration)
return nil
})
}
// Create migration function which changes the initially created data and
// throw the panic, in this case we pretending that something goes.
migrationWithPanic := func(tx *bbolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists(bucketPrefix)
if err != nil {
return err
}
bucket.Put(keyPrefix, afterMigration)
panic("panic!")
}
// Check that version of database and data wasn't changed.
afterMigrationFunc := func(d *DB) {
meta, err := d.FetchMeta(nil)
if err != nil {
t.Fatal(err)
}
if meta.DbVersionNumber != 0 {
t.Fatal("migration panicked but version is changed")
}
err = d.Update(func(tx *bbolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists(bucketPrefix)
if err != nil {
return err
}
value := bucket.Get(keyPrefix)
if !bytes.Equal(value, beforeMigration) {
return errors.New("migration failed but data is " +
"changed")
}
return nil
})
if err != nil {
t.Fatal(err)
}
}
applyMigration(t,
beforeMigrationFunc,
afterMigrationFunc,
migrationWithPanic,
true)
}
// TestMigrationWithFatal asserts that migrations which fail do not modify the
// database.
func TestMigrationWithFatal(t *testing.T) {
t.Parallel()
bucketPrefix := []byte("somebucket")
keyPrefix := []byte("someprefix")
beforeMigration := []byte("beforemigration")
afterMigration := []byte("aftermigration")
beforeMigrationFunc := func(d *DB) {
d.Update(func(tx *bbolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists(bucketPrefix)
if err != nil {
return err
}
bucket.Put(keyPrefix, beforeMigration)
return nil
})
}
// Create migration function which changes the initially created data and
// return the error, in this case we pretending that something goes
// wrong.
migrationWithFatal := func(tx *bbolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists(bucketPrefix)
if err != nil {
return err
}
bucket.Put(keyPrefix, afterMigration)
return errors.New("some error")
}
// Check that version of database and initial data wasn't changed.
afterMigrationFunc := func(d *DB) {
meta, err := d.FetchMeta(nil)
if err != nil {
t.Fatal(err)
}
if meta.DbVersionNumber != 0 {
t.Fatal("migration failed but version is changed")
}
err = d.Update(func(tx *bbolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists(bucketPrefix)
if err != nil {
return err
}
value := bucket.Get(keyPrefix)
if !bytes.Equal(value, beforeMigration) {
return errors.New("migration failed but data is " +
"changed")
}
return nil
})
if err != nil {
t.Fatal(err)
}
}
applyMigration(t,
beforeMigrationFunc,
afterMigrationFunc,
migrationWithFatal,
true)
}
// TestMigrationWithoutErrors asserts that a successful migration has its
// changes applied to the database.
func TestMigrationWithoutErrors(t *testing.T) {
t.Parallel()
bucketPrefix := []byte("somebucket")
keyPrefix := []byte("someprefix")
beforeMigration := []byte("beforemigration")
afterMigration := []byte("aftermigration")
// Populate database with initial data.
beforeMigrationFunc := func(d *DB) {
d.Update(func(tx *bbolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists(bucketPrefix)
if err != nil {
return err
}
bucket.Put(keyPrefix, beforeMigration)
return nil
})
}
// Create migration function which changes the initially created data.
migrationWithoutErrors := func(tx *bbolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists(bucketPrefix)
if err != nil {
return err
}
bucket.Put(keyPrefix, afterMigration)
return nil
}
// Check that version of database and data was properly changed.
afterMigrationFunc := func(d *DB) {
meta, err := d.FetchMeta(nil)
if err != nil {
t.Fatal(err)
}
if meta.DbVersionNumber != 1 {
t.Fatal("version number isn't changed after " +
"successfully applied migration")
}
err = d.Update(func(tx *bbolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists(bucketPrefix)
if err != nil {
return err
}
value := bucket.Get(keyPrefix)
if !bytes.Equal(value, afterMigration) {
return errors.New("migration wasn't applied " +
"properly")
}
return nil
})
if err != nil {
t.Fatal(err)
}
}
applyMigration(t,
beforeMigrationFunc,
afterMigrationFunc,
migrationWithoutErrors,
false)
}
// TestMigrationReversion tests after performing a migration to a higher
// database version, opening the database with a lower latest db version returns
// ErrDBReversion.
func TestMigrationReversion(t *testing.T) {
t.Parallel()
tempDirName, err := ioutil.TempDir("", "channeldb")
if err != nil {
t.Fatalf("unable to create temp dir: %v", err)
}
cdb, err := Open(tempDirName)
if err != nil {
t.Fatalf("unable to open channeldb: %v", err)
}
// Update the database metadata to point to one more than the highest
// known version.
err = cdb.Update(func(tx *bbolt.Tx) error {
newMeta := &Meta{
DbVersionNumber: getLatestDBVersion(dbVersions) + 1,
}
return putMeta(newMeta, tx)
})
// Close the database. Even if we succeeded, our next step is to reopen.
cdb.Close()
if err != nil {
t.Fatalf("unable to increase db version: %v", err)
}
_, err = Open(tempDirName)
if err != ErrDBReversion {
t.Fatalf("unexpected error when opening channeldb, "+
"want: %v, got: %v", ErrDBReversion, err)
}
}
channeldb/migration_01_to_11/migration_10_route_tlv_records.go
+2 -2
View File
@@ -8,10 +8,10 @@ import (
"github.com/lightningnetwork/lnd/routing/route"
)
// migrateRouteSerialization migrates the way we serialize routes across the
// MigrateRouteSerialization migrates the way we serialize routes across the
// entire database. At the time of writing of this migration, this includes our
// payment attempts, as well as the payment results in mission control.
func migrateRouteSerialization(tx *bbolt.Tx) error {
func MigrateRouteSerialization(tx *bbolt.Tx) error {
// First, we'll do all the payment attempts.
rootPaymentBucket := tx.Bucket(paymentsRootBucket)
if rootPaymentBucket == nil {
channeldb/migration_01_to_11/migration_11_invoices.go
+2 -2
View File
@@ -14,9 +14,9 @@ import (
litecoinCfg "github.com/ltcsuite/ltcd/chaincfg"
)
// migrateInvoices adds invoice htlcs and a separate cltv delta field to the
// MigrateInvoices adds invoice htlcs and a separate cltv delta field to the
// invoices.
func migrateInvoices(tx *bbolt.Tx) error {
func MigrateInvoices(tx *bbolt.Tx) error {
log.Infof("Migrating invoices to new invoice format")
invoiceB := tx.Bucket(invoiceBucket)
channeldb/migration_01_to_11/migration_11_invoices_test.go
+2 -2
View File
@@ -123,7 +123,7 @@ func TestMigrateInvoices(t *testing.T) {
applyMigration(t,
func(d *DB) { beforeMigrationFuncV11(t, d, invoices) },
afterMigrationFunc,
migrateInvoices,
MigrateInvoices,
false)
}
@@ -149,7 +149,7 @@ func TestMigrateInvoicesHodl(t *testing.T) {
applyMigration(t,
func(d *DB) { beforeMigrationFuncV11(t, d, invoices) },
func(d *DB) {},
migrateInvoices,
MigrateInvoices,
true)
}
channeldb/migration_01_to_11/migrations.go
+18 -18
View File
@@ -12,12 +12,12 @@ import (
"github.com/lightningnetwork/lnd/routing/route"
)
// migrateNodeAndEdgeUpdateIndex is a migration function that will update the
// MigrateNodeAndEdgeUpdateIndex is a migration function that will update the
// database from version 0 to version 1. In version 1, we add two new indexes
// (one for nodes and one for edges) to keep track of the last time a node or
// edge was updated on the network. These new indexes allow us to implement the
// new graph sync protocol added.
func migrateNodeAndEdgeUpdateIndex(tx *bbolt.Tx) error {
func MigrateNodeAndEdgeUpdateIndex(tx *bbolt.Tx) error {
// First, we'll populating the node portion of the new index. Before we
// can add new values to the index, we'll first create the new bucket
// where these items will be housed.
@@ -118,11 +118,11 @@ func migrateNodeAndEdgeUpdateIndex(tx *bbolt.Tx) error {
return nil
}
// migrateInvoiceTimeSeries is a database migration that assigns all existing
// MigrateInvoiceTimeSeries is a database migration that assigns all existing
// invoices an index in the add and/or the settle index. Additionally, all
// existing invoices will have their bytes padded out in order to encode the
// add+settle index as well as the amount paid.
func migrateInvoiceTimeSeries(tx *bbolt.Tx) error {
func MigrateInvoiceTimeSeries(tx *bbolt.Tx) error {
invoices, err := tx.CreateBucketIfNotExists(invoiceBucket)
if err != nil {
return err
@@ -255,11 +255,11 @@ func migrateInvoiceTimeSeries(tx *bbolt.Tx) error {
return nil
}
// migrateInvoiceTimeSeriesOutgoingPayments is a follow up to the
// MigrateInvoiceTimeSeriesOutgoingPayments is a follow up to the
// migrateInvoiceTimeSeries migration. As at the time of writing, the
// OutgoingPayment struct embeddeds an instance of the Invoice struct. As a
// result, we also need to migrate the internal invoice to the new format.
func migrateInvoiceTimeSeriesOutgoingPayments(tx *bbolt.Tx) error {
func MigrateInvoiceTimeSeriesOutgoingPayments(tx *bbolt.Tx) error {
payBucket := tx.Bucket(paymentBucket)
if payBucket == nil {
return nil
@@ -336,11 +336,11 @@ func migrateInvoiceTimeSeriesOutgoingPayments(tx *bbolt.Tx) error {
return nil
}
// migrateEdgePolicies is a migration function that will update the edges
// MigrateEdgePolicies is a migration function that will update the edges
// bucket. It ensure that edges with unknown policies will also have an entry
// in the bucket. After the migration, there will be two edge entries for
// every channel, regardless of whether the policies are known.
func migrateEdgePolicies(tx *bbolt.Tx) error {
func MigrateEdgePolicies(tx *bbolt.Tx) error {
nodes := tx.Bucket(nodeBucket)
if nodes == nil {
return nil
@@ -409,10 +409,10 @@ func migrateEdgePolicies(tx *bbolt.Tx) error {
return nil
}
// paymentStatusesMigration is a database migration intended for adding payment
// PaymentStatusesMigration is a database migration intended for adding payment
// statuses for each existing payment entity in bucket to be able control
// transitions of statuses and prevent cases such as double payment
func paymentStatusesMigration(tx *bbolt.Tx) error {
func PaymentStatusesMigration(tx *bbolt.Tx) error {
// Get the bucket dedicated to storing statuses of payments,
// where a key is payment hash, value is payment status.
paymentStatuses, err := tx.CreateBucketIfNotExists(paymentStatusBucket)
@@ -492,14 +492,14 @@ func paymentStatusesMigration(tx *bbolt.Tx) error {
return nil
}
// migratePruneEdgeUpdateIndex is a database migration that attempts to resolve
// MigratePruneEdgeUpdateIndex is a database migration that attempts to resolve
// some lingering bugs with regards to edge policies and their update index.
// Stale entries within the edge update index were not being properly pruned due
// to a miscalculation on the offset of an edge's policy last update. This
// migration also fixes the case where the public keys within edge policies were
// being serialized with an extra byte, causing an even greater error when
// attempting to perform the offset calculation described earlier.
func migratePruneEdgeUpdateIndex(tx *bbolt.Tx) error {
func MigratePruneEdgeUpdateIndex(tx *bbolt.Tx) error {
// To begin the migration, we'll retrieve the update index bucket. If it
// does not exist, we have nothing left to do so we can simply exit.
edges := tx.Bucket(edgeBucket)
@@ -610,10 +610,10 @@ func migratePruneEdgeUpdateIndex(tx *bbolt.Tx) error {
return nil
}
// migrateOptionalChannelCloseSummaryFields migrates the serialized format of
// MigrateOptionalChannelCloseSummaryFields migrates the serialized format of
// ChannelCloseSummary to a format where optional fields' presence is indicated
// with boolean markers.
func migrateOptionalChannelCloseSummaryFields(tx *bbolt.Tx) error {
func MigrateOptionalChannelCloseSummaryFields(tx *bbolt.Tx) error {
closedChanBucket := tx.Bucket(closedChannelBucket)
if closedChanBucket == nil {
return nil
@@ -669,10 +669,10 @@ func migrateOptionalChannelCloseSummaryFields(tx *bbolt.Tx) error {
var messageStoreBucket = []byte("message-store")
// migrateGossipMessageStoreKeys migrates the key format for gossip messages
// MigrateGossipMessageStoreKeys migrates the key format for gossip messages
// found in the message store to a new one that takes into consideration the of
// the message being stored.
func migrateGossipMessageStoreKeys(tx *bbolt.Tx) error {
func MigrateGossipMessageStoreKeys(tx *bbolt.Tx) error {
// We'll start by retrieving the bucket in which these messages are
// stored within. If there isn't one, there's nothing left for us to do
// so we can avoid the migration.
@@ -739,7 +739,7 @@ func migrateGossipMessageStoreKeys(tx *bbolt.Tx) error {
return nil
}
// migrateOutgoingPayments moves the OutgoingPayments into a new bucket format
// MigrateOutgoingPayments moves the OutgoingPayments into a new bucket format
// where they all reside in a top-level bucket indexed by the payment hash. In
// this sub-bucket we store information relevant to this payment, such as the
// payment status.
@@ -748,7 +748,7 @@ func migrateGossipMessageStoreKeys(tx *bbolt.Tx) error {
// InFlight (we have no PaymentAttemptInfo available for pre-migration
// payments) we delete those statuses, so only Completed payments remain in the
// new bucket structure.
func migrateOutgoingPayments(tx *bbolt.Tx) error {
func MigrateOutgoingPayments(tx *bbolt.Tx) error {
log.Infof("Migrating outgoing payments to new bucket structure")
oldPayments := tx.Bucket(paymentBucket)
channeldb/migration_01_to_11/migrations_test.go
+5 -5
View File
@@ -197,7 +197,7 @@ func TestPaymentStatusesMigration(t *testing.T) {
applyMigration(t,
beforeMigrationFunc,
afterMigrationFunc,
paymentStatusesMigration,
PaymentStatusesMigration,
false)
}
@@ -469,7 +469,7 @@ func TestMigrateOptionalChannelCloseSummaryFields(t *testing.T) {
applyMigration(t,
beforeMigrationFunc,
afterMigrationFunc,
migrateOptionalChannelCloseSummaryFields,
MigrateOptionalChannelCloseSummaryFields,
false)
}
}
@@ -565,7 +565,7 @@ func TestMigrateGossipMessageStoreKeys(t *testing.T) {
applyMigration(
t, beforeMigration, afterMigration,
migrateGossipMessageStoreKeys, false,
MigrateGossipMessageStoreKeys, false,
)
}
@@ -724,7 +724,7 @@ func TestOutgoingPaymentsMigration(t *testing.T) {
applyMigration(t,
beforeMigrationFunc,
afterMigrationFunc,
migrateOutgoingPayments,
MigrateOutgoingPayments,
false)
}
@@ -947,6 +947,6 @@ func TestPaymentRouteSerialization(t *testing.T) {
applyMigration(t,
beforeMigrationFunc,
afterMigrationFunc,
migrateRouteSerialization,
MigrateRouteSerialization,
false)
}