Merge pull request #5550 from bottlepay/postgres-run-channeldb

kvdb: run channeldb test suite on postgres

.github/workflows/main.yml

@@ -206,7 +206,8 @@ jobs:
       matrix:
         unit_type:
           - btcd unit-cover
-          - unit tags="kvdb_etcd kvdb_postgres"
+          - unit tags="kvdb_etcd"
+          - unit tags="kvdb_postgres"
           - travis-race
     steps:
       - name: git checkout

channeldb/db_test.go

@@ -25,6 +25,11 @@ import (
 func TestOpenWithCreate(t *testing.T) {
 	t.Parallel()
 
+	// Checking for db file existence is not possible with postgres.
+	if kvdb.PostgresBackend {
+		t.Skip()
+	}
+
 	// First, create a temporary directory to be used for the duration of
 	// this test.
 	tempDirName, err := ioutil.TempDir("", "channeldb")

channeldb/setup_test.go

@@ -0,0 +1,11 @@
+package channeldb
+
+import (
+	"testing"
+
+	"github.com/lightningnetwork/lnd/kvdb"
+)
+
+func TestMain(m *testing.M) {
+	kvdb.RunTests(m)
+}

contractcourt/setup_test.go

@@ -0,0 +1,11 @@
+package contractcourt
+
+import (
+	"testing"
+
+	"github.com/lightningnetwork/lnd/kvdb"
+)
+
+func TestMain(m *testing.M) {
+	kvdb.RunTests(m)
+}

docs/release-notes/release-notes-0.14.0.md

@@ -460,6 +460,8 @@ messages directly. There is no routing/path finding involved.
 
 * [Fixed flake that occurred with the switch dust forwarding test under the data race unit build.](https://github.com/lightningnetwork/lnd/pull/5828)
 
+* [Run channeldb tests on postgres](https://github.com/lightningnetwork/lnd/pull/5550)
+
 ## Database
 
 * [Ensure single writer for legacy

kvdb/backend.go

@@ -5,7 +5,9 @@ package kvdb
 
 import (
 	"context"
+	"crypto/sha256"
 	"encoding/binary"
+	"encoding/hex"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -245,7 +247,20 @@ func updateLastCompactionDate(dbFile string) error {
 func GetTestBackend(path, name string) (Backend, func(), error) {
 	empty := func() {}
 
-	if TestBackend == BoltBackendName {
+	switch {
+	case PostgresBackend:
+		key := filepath.Join(path, name)
+		keyHash := sha256.Sum256([]byte(key))
+
+		f, err := NewPostgresFixture("test_" + hex.EncodeToString(keyHash[:]))
+		if err != nil {
+			return nil, func() {}, err
+		}
+		return f.DB(), func() {
+			_ = f.DB().Close()
+		}, nil
+
+	case TestBackend == BoltBackendName:
 		db, err := GetBoltBackend(&BoltBackendConfig{
 			DBPath:         path,
 			DBFileName:     name,
@@ -256,7 +271,8 @@ func GetTestBackend(path, name string) (Backend, func(), error) {
 			return nil, nil, err
 		}
 		return db, empty, nil
-	} else if TestBackend == EtcdBackendName {
+
+	case TestBackend == EtcdBackendName:
 		etcdConfig, cancel, err := StartEtcdTestBackend(path, 0, 0, "")
 		if err != nil {
 			return nil, empty, err
@@ -265,6 +281,7 @@ func GetTestBackend(path, name string) (Backend, func(), error) {
 			EtcdBackendName, context.TODO(), etcdConfig,
 		)
 		return backend, cancel, err
+
 	}
 
 	return nil, nil, fmt.Errorf("unknown backend")

kvdb/kvdb_no_postgres.go

@@ -0,0 +1,19 @@
+// +build !kvdb_postgres
+
+package kvdb
+
+import (
+	"errors"
+
+	"github.com/lightningnetwork/lnd/kvdb/postgres"
+)
+
+const PostgresBackend = false
+
+func NewPostgresFixture(dbName string) (postgres.Fixture, error) {
+	return nil, errors.New("postgres backend not available")
+}
+
+func StartEmbeddedPostgres() (func() error, error) {
+	return nil, errors.New("postgres backend not available")
+}

kvdb/kvdb_postgres.go

@@ -0,0 +1,15 @@
+// +build kvdb_postgres
+
+package kvdb
+
+import "github.com/lightningnetwork/lnd/kvdb/postgres"
+
+const PostgresBackend = true
+
+func NewPostgresFixture(dbName string) (postgres.Fixture, error) {
+	return postgres.NewFixture(dbName)
+}
+
+func StartEmbeddedPostgres() (func() error, error) {
+	return postgres.StartEmbeddedPostgres()
+}

kvdb/postgres/db.go

@@ -151,8 +151,15 @@ func (db *db) getPrefixedTableName(table string) string {
 func catchPanic(f func() error) (err error) {
 	defer func() {
 		if r := recover(); r != nil {
-			err = r.(error)
+			log.Criticalf("Caught unhandled error: %v", r)
-			log.Criticalf("Caught unhandled error: %v", err)
+
+			switch data := r.(type) {
+			case error:
+				err = data
+
+			default:
+				err = errors.New(fmt.Sprintf("%v", data))
+			}
 		}
 	}()
 

kvdb/postgres/db_test.go

@@ -14,15 +14,19 @@ import (
 
 // TestInterface performs all interfaces tests for this database driver.
 func TestInterface(t *testing.T) {
-	f := NewFixture(t)
+	stop, err := StartEmbeddedPostgres()
-	defer f.Cleanup()
+	require.NoError(t, err)
+	defer stop()
+
+	f, err := NewFixture("")
+	require.NoError(t, err)
 
 	// dbType is the database type name for this driver.
 	const dbType = "postgres"
 
 	ctx := context.Background()
 	cfg := &Config{
-		Dsn: testDsn,
+		Dsn: f.Dsn,
 	}
 
 	walletdbtest.TestInterface(t, dbType, ctx, cfg, prefix)
@@ -30,17 +34,19 @@ func TestInterface(t *testing.T) {
 
 // TestPanic tests recovery from panic conditions.
 func TestPanic(t *testing.T) {
-	f := NewFixture(t)
+	stop, err := StartEmbeddedPostgres()
-	defer f.Cleanup()
+	require.NoError(t, err)
+	defer stop()
 
-	d := f.NewBackend()
+	f, err := NewFixture("")
+	require.NoError(t, err)
 
-	err := d.(*db).Update(func(tx walletdb.ReadWriteTx) error {
+	err = f.Db.(*db).Update(func(tx walletdb.ReadWriteTx) error {
 		bucket, err := tx.CreateTopLevelBucket([]byte("test"))
 		require.NoError(t, err)
 
 		// Stop database server.
-		f.Cleanup()
+		stop()
 
 		// Keep trying to get data until Get panics because the
 		// connection is lost.

kvdb/postgres/fixture.go

@@ -4,87 +4,122 @@ package postgres
 
 import (
 	"context"
+	"crypto/rand"
 	"database/sql"
-	"testing"
+	"encoding/hex"
+	"fmt"
+	"strings"
 
 	"github.com/btcsuite/btcwallet/walletdb"
 	embeddedpostgres "github.com/fergusstrange/embedded-postgres"
-	"github.com/stretchr/testify/require"
 )
 
 const (
-	testDsn = "postgres://postgres:postgres@localhost:9876/postgres?sslmode=disable"
+	testDsnTemplate = "postgres://postgres:postgres@localhost:9876/%v?sslmode=disable"
-	prefix  = "test"
+	prefix          = "test"
 )
 
-func clearTestDb(t *testing.T) {
+func getTestDsn(dbName string) string {
-	dbConn, err := sql.Open("pgx", testDsn)
+	return fmt.Sprintf(testDsnTemplate, dbName)
-	require.NoError(t, err)
-
-	_, err = dbConn.ExecContext(context.Background(), "DROP SCHEMA IF EXISTS public CASCADE;")
-	require.NoError(t, err)
 }
 
-func openTestDb(t *testing.T) *db {
+var testPostgres *embeddedpostgres.EmbeddedPostgres
-	clearTestDb(t)
 
-	db, err := newPostgresBackend(
+// StartEmbeddedPostgres starts an embedded postgres instance. This only needs
-		context.Background(),
+// to be done once, because NewFixture will create random new databases on every
-		&Config{
+// call. It returns a stop closure that stops the database if called.
-			Dsn: testDsn,
+func StartEmbeddedPostgres() (func() error, error) {
-		},
-		prefix,
-	)
-	require.NoError(t, err)
-
-	return db
-}
-
-type fixture struct {
-	t        *testing.T
-	tempDir  string
-	postgres *embeddedpostgres.EmbeddedPostgres
-}
-
-func NewFixture(t *testing.T) *fixture {
 	postgres := embeddedpostgres.NewDatabase(
 		embeddedpostgres.DefaultConfig().
 			Port(9876))
 
 	err := postgres.Start()
-	require.NoError(t, err)
+	if err != nil {
+		return nil, err
+	}
+
+	testPostgres = postgres
+
+	return testPostgres.Stop, nil
+}
+
+// NewFixture returns a new postgres test database. The database name is
+// randomly generated.
+func NewFixture(dbName string) (*fixture, error) {
+	if dbName == "" {
+		// Create random database name.
+		randBytes := make([]byte, 8)
+		_, err := rand.Read(randBytes)
+		if err != nil {
+			return nil, err
+		}
+
+		dbName = "test_" + hex.EncodeToString(randBytes)
+	}
+
+	// Create database if it doesn't exist yet.
+	dbConn, err := sql.Open("pgx", getTestDsn("postgres"))
+	if err != nil {
+		return nil, err
+	}
+	defer dbConn.Close()
+
+	_, err = dbConn.ExecContext(
+		context.Background(), "CREATE DATABASE "+dbName,
+	)
+	if err != nil && !strings.Contains(err.Error(), "already exists") {
+		return nil, err
+	}
+
+	// Open database
+	dsn := getTestDsn(dbName)
+	db, err := newPostgresBackend(
+		context.Background(),
+		&Config{
+			Dsn: dsn,
+		},
+		prefix,
+	)
+	if err != nil {
+		return nil, err
+	}
 
 	return &fixture{
-		t:        t,
+		Dsn: dsn,
-		postgres: postgres,
+		Db:  db,
+	}, nil
+}
+
+type fixture struct {
+	Dsn string
+	Db  walletdb.DB
+}
+
+func (b *fixture) DB() walletdb.DB {
+	return b.Db
+}
+
+// Dump returns the raw contents of the database.
+func (b *fixture) Dump() (map[string]interface{}, error) {
+	dbConn, err := sql.Open("pgx", b.Dsn)
+	if err != nil {
+		return nil, err
 	}
-}
-
-func (b *fixture) Cleanup() {
-	b.postgres.Stop()
-}
-
-func (b *fixture) NewBackend() walletdb.DB {
-	clearTestDb(b.t)
-	db := openTestDb(b.t)
-
-	return db
-}
-
-func (b *fixture) Dump() map[string]interface{} {
-	dbConn, err := sql.Open("pgx", testDsn)
-	require.NoError(b.t, err)
 
 	rows, err := dbConn.Query(
 		"SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname='public'",
 	)
-	require.NoError(b.t, err)
+	if err != nil {
+		return nil, err
+	}
 
 	var tables []string
 	for rows.Next() {
 		var table string
 		err := rows.Scan(&table)
-		require.NoError(b.t, err)
+		if err != nil {
+			return nil, err
+		}
 
 		tables = append(tables, table)
 	}
@@ -93,10 +128,14 @@ func (b *fixture) Dump() map[string]interface{} {
 
 	for _, table := range tables {
 		rows, err := dbConn.Query("SELECT * FROM " + table)
-		require.NoError(b.t, err)
+		if err != nil {
+			return nil, err
+		}
 
 		cols, err := rows.Columns()
-		require.NoError(b.t, err)
+		if err != nil {
+			return nil, err
+		}
 		colCount := len(cols)
 
 		var tableRows []map[string]interface{}
@@ -108,7 +147,9 @@ func (b *fixture) Dump() map[string]interface{} {
 			}
 
 			err := rows.Scan(valuePtrs...)
-			require.NoError(b.t, err)
+			if err != nil {
+				return nil, err
+			}
 
 			tableData := make(map[string]interface{})
 			for i, v := range values {
@@ -127,5 +168,5 @@ func (b *fixture) Dump() map[string]interface{} {
 		result[table] = tableRows
 	}
 
-	return result
+	return result, nil
 }

kvdb/postgres/fixture_interface.go

@@ -0,0 +1,8 @@
+package postgres
+
+import "github.com/btcsuite/btcwallet/walletdb"
+
+type Fixture interface {
+	DB() walletdb.DB
+	Dump() (map[string]interface{}, error)
+}

kvdb/postgres_test.go

@@ -13,8 +13,9 @@ import (
 type m = map[string]interface{}
 
 func TestPostgres(t *testing.T) {
-	f := postgres.NewFixture(t)
+	stop, err := postgres.StartEmbeddedPostgres()
-	defer f.Cleanup()
+	require.NoError(t, err)
+	defer stop()
 
 	tests := []struct {
 		name       string
@@ -173,10 +174,14 @@ func TestPostgres(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			test.test(t, f.NewBackend())
+			f, err := postgres.NewFixture("")
+			require.NoError(t, err)
+
+			test.test(t, f.Db)
 
 			if test.expectedDb != nil {
-				dump := f.Dump()
+				dump, err := f.Dump()
+				require.NoError(t, err)
 				require.Equal(t, test.expectedDb, dump)
 			}
 		})

kvdb/test_utils.go

@@ -0,0 +1,34 @@
+package kvdb
+
+import (
+	"fmt"
+	"os"
+	"testing"
+)
+
+// RunTests is a helper function to run the tests in a package with
+// initialization and tear-down of a test kvdb backend.
+func RunTests(m *testing.M) {
+	var close func() error
+	if PostgresBackend {
+		var err error
+		close, err = StartEmbeddedPostgres()
+		if err != nil {
+			fmt.Printf("Error: %v\n", err)
+			os.Exit(1)
+		}
+	}
+
+	// os.Exit() does not respect defer statements
+	code := m.Run()
+
+	if close != nil {
+		err := close()
+		if err != nil {
+			fmt.Printf("Error: %v\n", err)
+		}
+	}
+
+	os.Exit(code)
+
+}

routing/setup_test.go

@@ -0,0 +1,11 @@
+package routing
+
+import (
+	"testing"
+
+	"github.com/lightningnetwork/lnd/kvdb"
+)
+
+func TestMain(m *testing.M) {
+	kvdb.RunTests(m)
+}

sweep/setup_test.go

@@ -0,0 +1,11 @@
+package sweep
+
+import (
+	"testing"
+
+	"github.com/lightningnetwork/lnd/kvdb"
+)
+
+func TestMain(m *testing.M) {
+	kvdb.RunTests(m)
+}