From a8450875f693c81445965d593c4185fc2fb04c2f Mon Sep 17 00:00:00 2001 From: Conner Fromknecht Date: Wed, 27 Sep 2017 05:44:38 -0700 Subject: [PATCH 1/6] nursery_store: adds a new file for utxon database This commit introduces a new type called a NurseryStore which abstract many of the persistent operations required by the utxo nursery. The code in this commit is intentionally introduced as dead code, as the utxo nursery will later be modified to hook into the new database. It is designed to support incubation of commitment and htlc outputs, and includes full chain segmentation! --- nursery_store.go | 1361 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1361 insertions(+) create mode 100644 nursery_store.go diff --git a/nursery_store.go b/nursery_store.go new file mode 100644 index 000000000..d32a1cad7 --- /dev/null +++ b/nursery_store.go @@ -0,0 +1,1361 @@ +package main + +import ( + "bytes" + "errors" + + "github.com/boltdb/bolt" + "github.com/lightningnetwork/lnd/channeldb" + "github.com/roasbeef/btcd/chaincfg/chainhash" + "github.com/roasbeef/btcd/wire" +) + +// NurseryStore abstracts the persistent storage layer for the utxo nursery. +// Concretely, it stores commitment and htlc outputs that until any time-bounded +// constraints have fully matured. The store exposes methods for enumerating +// its contents, and persisting state transitions detected by the utxo nursery. +type NurseryStore interface { + + // Incubation Entry Points. + + // EnterCrib accepts a new htlc output that the nursery will incubate + // through its two-stage process of sweeping funds back to the user's + // wallet. These outputs are persisted in the nursery store's crib + // bucket, and will be revisited after the output's CLTV has expired. + EnterCrib(*babyOutput) error + + // EnterPreschool accepts a new commitment output that the nursery will + // incubate through a single stage before sweeping. Outputs are stored + // in the preschool bucket until the commitment transaction has been + // confirmed, at which point they will be moved to the kindergarten + // bucket. + EnterPreschool(*kidOutput) error + + // On-chain Driven State Transtitions. + + // CribToKinder atomically moves a babyOutput in the crib bucket to the + // kindergarten bucket. The now mature kidOutput contained in the + // babyOutput will be stored as it waits out the kidOutput's CSV delay. + CribToKinder(*babyOutput) error + + // PreschoolToKinder atomically moves a kidOutput from the preschool + // bucket to the kindergarten bucket. This transition should be executed + // after receiving confirmation of the preschool output's commitment + // transaction. + PreschoolToKinder(*kidOutput) error + + // AwardDiplomas accepts a variadic number of kidOutputs from the + // kindergarten bucket, and removes their corresponding entries from the + // height and channel indexes. If this method detects that all outputs + // for a particular contract have been incubated, it returns the channel + // points that are ready to be marked as fully closed. + // TODO: make this handle one output at a time? + AwardDiplomas(...kidOutput) ([]wire.OutPoint, error) + + // FinalizeClass accepts a block height as a parameter and purges its + // persistent state for all outputs at that height. During a restart, + // the utxo nursery will begin it's recovery procedure from the next + // height that has yet to be finalized. This block height should lag + // beyond the best height for this chain as a measure of reorg + // protection. + FinalizeClass(height uint32) error + + // State Bucket Enumeration. + + // FetchCribs returns a list of babyOutputs in the crib bucket whose + // CLTV delay expires at the provided block height. + FetchCribs(height uint32) ([]babyOutput, error) + + // FetchKindergartens returns a list of kidOutputs in the kindergarten + // bucket whose CSV delay expires at the provided block height. + FetchKindergartens(height uint32) ([]kidOutput, error) + + // FetchPreschools returns a list of all outputs currently stored in the + // preschool bucket. + FetchPreschools() ([]kidOutput, error) + + // Channel Output Enumeration. + + // ForChanOutputs iterates over all outputs being incubated for a + // particular channel point. This method accepts a callback that allows + // the caller to process each key-value pair. The key will be a prefixed + // outpoint, and the value will be the serialized bytes for an output, + // whose type should be inferred from the key's prefix. + ForChanOutputs(*wire.OutPoint, func([]byte, []byte) error) error + + // The Point of No Return. + + // LastFinalizedHeight returns the last block height for which the + // nursery store has purged all persistent state. + LastFinalizedHeight() (uint32, error) +} + +// prefixChainKey creates the root level keys for the nursery store. The keys +// are comprised of a nursery-specific prefix and the intended chain hash that +// this nursery store will be used for. This allows multiple nursery stores to +// isolate their state when operating on multiple chains or forks. +func prefixChainKey(sysPrefix []byte, hash *chainhash.Hash) ([]byte, error) { + // Create a buffer to which we will write the system prefix, e.g. + // "utxn", followed by the provided chain hash. + var pfxChainBuffer bytes.Buffer + if _, err := pfxChainBuffer.Write(sysPrefix); err != nil { + return nil, err + } + + if _, err := pfxChainBuffer.Write(hash[:]); err != nil { + return nil, err + } + + return pfxChainBuffer.Bytes(), nil +} + +// prefixOutputKey creates a serialized key that prefixes the serialized +// outpoint with the provided state prefix. The returned bytes will be of the +// form . +func prefixOutputKey(statePrefix []byte, + outpoint *wire.OutPoint) ([]byte, error) { + + // Create a buffer to which we will first write the state prefix, + // followed by the outpoint. + var pfxOutputBuffer bytes.Buffer + if _, err := pfxOutputBuffer.Write(statePrefix); err != nil { + return nil, err + } + + err := writeOutpoint(&pfxOutputBuffer, outpoint) + if err != nil { + return nil, err + } + + return pfxOutputBuffer.Bytes(), nil +} + +var ( + // utxnChainPrefix is used to prefix a particular chain hash and create + // the root-level, chain-segmented bucket for each nursery store. + utxnChainPrefix = []byte("utxn") + + // lastFinalizedHeightKey is a static key used to locate nursery store's + // last finalized height. + lastFinalizedHeightKey = []byte("last-finalized-height") + + // channelIndexKey is a static key used to lookup the bucket containing + // all of the nursery's active channels. + channelIndexKey = []byte("channel-index") + + // channelIndexKey is a static key used to retrieve a directory + // containing all heights for which the nursery will need to take + // action. + heightIndexKey = []byte("height-index") + + // cribPrefix is the state prefix given to htlc outputs waiting for + // their first-stage, absolute locktime to elapse. + cribPrefix = []byte("crib") + + // psclPrefix is the state prefix given to commitment outputs awaiting + // the // confirmation of the commitment transaction, as this solidifies + // the absolute height at which they can be spent. + psclPrefix = []byte("pscl") + + // kndrPrefix is the state prefix given to all CSV delayed outputs, + // either from the commitment transaction, or a stage-one htlc + // transaction, whose maturity height has solidified. Outputs marked in + // this state are in their final stage of incubation withn the nursery, + // and will be swept into the wallet after waiting out the relative + // timelock. + kndrPrefix = []byte("kndr") +) + +// Overview of Nursery Store Storage Hierarchy +// +// CHAIN SEGMENTATION +// +// The root directory of a nursery store is bucketed by the chain hash and +// the 'utxn' prefix. This allows multiple utxo nurseries for distinct chains +// to simultaneously use the same channel.DB instance. This is critical for +// providing replay protection and more to isolate chain-specific data in the +// multichain setting. +// +// utxn/ +// | +// | LAST FINALIZED HEIGHT +// | +// | Each nursery store tracks a "last finalized height", which records the +// | most recent block height for which the nursery store has purged all +// | state. This value lags behind the best block height for reorg safety, +// | and serves as a starting height for rescans after a restart. +// | +// ├── last-finalized-height-key: +// | +// | CHANNEL INDEX +// | +// | The channel index contains a directory for each channel that has a +// | non-zero number of outputs being tracked by the nursery store. +// | Inside each channel directory are files contains serialized spendable +// | outputs that are awaiting some state transition. The name of each file +// | contains the outpoint of the spendable output in the file, and is +// | prefixed with 4-byte state prefix, indicating whether the spendable +// | output is a crib, preschool, or kindergarten output. The nursery store +// | supports the ability to enumerate all outputs for a particular channel, +// | which is useful in constructing nursery reports. +// | +// ├── channel-index-key/ +// │   ├── / <- CHANNEL BUCKET +// | |   ├── : +// | |   └── : +// │   ├── / +// | |   └── : +// │   └── / +// |    ├── : +// |    └── : +// | +// | HEIGHT INDEX +// | +// | The height index contains a directory for each height at which the +// | nursery still has uncompleted actions. If an output is a crib or +// | kindergarten output, +// | it will have an associated entry in the height index. Inside a +// | particular height directory, the structure is similar to that of the +// | channel index, containing multiple channel directories, each of which +// | contains subdirectories named with a prefixed outpoint belonging to +// | the channel. Enumerating these combinations yields a relative file +// | path: +// | e.g. // +// | that can be queried in the channel index to retrieve the serialized +// | output. +// | +// └── height-index-key/ +//    ├── / <- HEIGHT BUCKET +// |   └── / <- HEIGHT-CHANNEL BUCKET +// | |    ├── / <- PREFIXED OUTPOINT +// | |    └── / +// |   └── / +// |    └── / +//    └── / +//    └── / +//    └── / +//    └── / + +// nurseryStore is a concrete instantiation of a NurseryStore that is backed by +// a channeldb.DB instance. +type nurseryStore struct { + chainHash chainhash.Hash + pfxChainKey []byte + + db *channeldb.DB +} + +// newNurseryStore accepts a chain hash and a channeldb.DB instance, returning +// an instance of nurseryStore who's database is properly segmented for the +// given chain. +func newNurseryStore(chainHash *chainhash.Hash, + db *channeldb.DB) (*nurseryStore, error) { + + // Prefix the provided chain hash with "utxn" to create the key for the + // nursery store's root bucket, ensuring each one has proper chain + // segmentation. + pfxChainKey, err := prefixChainKey(utxnChainPrefix, chainHash) + if err != nil { + return nil, err + } + + return &nurseryStore{ + chainHash: *chainHash, + pfxChainKey: pfxChainKey, + db: db, + }, nil +} + +// Incubation Entry Points. + +// EnterCrib accepts a new htlc output that the nursery will incubate through +// its two-stage process of sweeping funds back to the user's wallet. These +// outputs are persisted in the nursery store's crib bucket, and will be +// revisited after the output's CLTV has expired. +func (ns *nurseryStore) EnterCrib(bby *babyOutput) error { + return ns.db.Update(func(tx *bolt.Tx) error { + + // First, retrieve or create the channel bucket corresponding to + // the baby output's origin channel point. + chanPoint := bby.OriginChanPoint() + chanBucket, err := ns.createChannelBucket(tx, chanPoint) + if err != nil { + return err + } + + // Next, retrieve or create the height-channel bucket located in + // the height bucket corresponding to the baby output's CLTV + // expiry height. + hghtChanBucket, err := ns.createHeightChanBucket(tx, + bby.expiry, chanPoint) + if err != nil { + return err + } + + // Since we are inserting this output into the crib bucket, we + // create a key that prefixes the baby output's outpoint with + // the crib prefix. + pfxOutputKey, err := prefixOutputKey(cribPrefix, bby.OutPoint()) + if err != nil { + return err + } + + // Serialize the baby output so that it can be written to the + // underlying key-value store. + var babyBuffer bytes.Buffer + if err := bby.Encode(&babyBuffer); err != nil { + return err + } + babyBytes := babyBuffer.Bytes() + + // Now, insert the serialized output into its channel bucket + // under the prefixed key created above. + if err := chanBucket.Put(pfxOutputKey, babyBytes); err != nil { + return err + } + + // Finally, create a corresponding bucket in the height-channel + // bucket for this crib output. The existence of this bucket + // indicates that the serialized output can be retrieved from + // the channel bucket using the same prefix key. + _, err = hghtChanBucket.CreateBucketIfNotExists(pfxOutputKey) + + return err + }) +} + +// EnterPreschool accepts a new commitment output that the nursery will +// incubate through a single stage before sweeping. Outputs are stored in the +// preschool bucket until the commitment transaction has been confirmed, at +// which point they will be moved to the kindergarten bucket. +func (ns *nurseryStore) EnterPreschool(kid *kidOutput) error { + return ns.db.Update(func(tx *bolt.Tx) error { + + // First, retrieve or create the channel bucket corresponding to + // the baby output's origin channel point. + chanPoint := kid.OriginChanPoint() + chanBucket, err := ns.createChannelBucket(tx, chanPoint) + if err != nil { + return err + } + + // Since the babyOutput is being inserted into the preschool + // bucket, we create a key that prefixes its outpoint with the + // preschool prefix. + pfxOutputKey, err := prefixOutputKey(psclPrefix, kid.OutPoint()) + if err != nil { + return err + } + + // Serialize the kidOutput and insert it into the channel + // bucket. + var kidBuffer bytes.Buffer + if err := kid.Encode(&kidBuffer); err != nil { + return err + } + + return chanBucket.Put(pfxOutputKey, kidBuffer.Bytes()) + }) +} + +// On-chain Drive State Transitions. + +// CribToKinder atomically moves a babyOutput in the crib bucket to the +// kindergarten bucket. The now mature kidOutput contained in the babyOutput +// will be stored as it waits out the kidOutput's CSV delay. +func (ns *nurseryStore) CribToKinder(bby *babyOutput) error { + return ns.db.Update(func(tx *bolt.Tx) error { + + // First, retrieve or create the channel bucket corresponding to + // the baby output's origin channel point. + chanPoint := bby.OriginChanPoint() + chanBucket, err := ns.createChannelBucket(tx, chanPoint) + if err != nil { + return err + } + + // The babyOutput should currently be stored in the crib bucket. + // So, we create a key that prefixes the babyOutput's outpoint + // with the crib prefix, allowing us to reference it in the + // store. + pfxOutputKey, err := prefixOutputKey(cribPrefix, bby.OutPoint()) + if err != nil { + return err + } + + // Since the babyOutput is being moved to the kindergarten + // bucket, we remove the entry from the channel bucket under the + // crib-prefixed outpoint key. + if err := chanBucket.Delete(pfxOutputKey); err != nil { + return err + } + + // Next, retrieve the height-channel bucket located in the + // height bucket corresponding to the baby output's CLTV expiry + // height. This bucket should always exist, but if it doesn't + // then we have nothing to clean up. + hghtChanBucketCltv := ns.getHeightChanBucket(tx, bby.expiry, + chanPoint) + if hghtChanBucketCltv != nil { + // We successfully located an existing height chan + // bucket at this babyOutput's expiry height, proceed by + // removing it from the index. + err := hghtChanBucketCltv.DeleteBucket(pfxOutputKey) + if err != nil { + return err + } + } + + // Since we are moving this output from the crib bucket to the + // kindergarten bucket, we overwrite the existing prefix of this + // key with the kindergarten prefix. + copy(pfxOutputKey, kndrPrefix) + + // Now, serialize babyOutput's encapsulated kidOutput such that + // it can be written to the channel bucket under the new + // kindergarten-prefixed key. + var kidBuffer bytes.Buffer + if err := bby.kidOutput.Encode(&kidBuffer); err != nil { + return err + } + kidBytes := kidBuffer.Bytes() + + // Persist the serialized kidOutput under the + // kindergarten-prefixed outpoint key. + if err := chanBucket.Put(pfxOutputKey, kidBytes); err != nil { + return err + } + + // Now, compute the height at which this kidOutput's CSV delay + // will expire. This is done by adding the required delay to + // the block height at which the output was confirmed. + maturityHeight := bby.ConfHeight() + bby.BlocksToMaturity() + + // Retrive or create a height-channel bucket corresponding to + // the kidOutput's maturity height. + hghtChanBucketCsv, err := ns.createHeightChanBucket(tx, + maturityHeight, chanPoint) + if err != nil { + return err + } + + // Register the kindergarten output's prefixed output key in the + // height-channel bucket corresponding to its maturity height. + // This informs the utxo nursery that it should attempt to spend + // this output when the blockchain reaches the maturity height. + _, err = hghtChanBucketCsv.CreateBucketIfNotExists(pfxOutputKey) + if err != nil { + return err + } + + // Finally, since we removed a crib output from the height + // index, we opportunistically prune the height bucket + // corresponding to the babyOutput's CLTV delay. This allows us + // to clean up any persistent state as outputs are progressed + // through the incubation process. + err = ns.pruneHeight(tx, bby.expiry) + switch err { + case nil, ErrBucketDoesNotExist: + return nil + case ErrBucketNotEmpty: + return nil + default: + return err + } + }) +} + +// PreschoolToKinder atomically moves a kidOutput from the preschool bucket to +// the kindergarten bucket. This transition should be executed after receiving +// confirmation of the preschool output's commitment transaction. +func (ns *nurseryStore) PreschoolToKinder(kid *kidOutput) error { + return ns.db.Update(func(tx *bolt.Tx) error { + + // Create or retrieve the channel bucket corresponding to the + // kid output's origin channel point. + chanPoint := kid.OriginChanPoint() + chanBucket, err := ns.createChannelBucket(tx, chanPoint) + if err != nil { + return err + } + + // First, we will attempt to remove the existing serialized + // output from the channel bucket, where the kid's outpoint will + // be prefixed by a preschool prefix. + + // Generate the key of existing serialized kid output by + // prefixing its outpoint with the preschool prefix... + pfxOutputKey, err := prefixOutputKey(psclPrefix, kid.OutPoint()) + if err != nil { + return err + } + + // And remove the old serialized output from the database. + if err := chanBucket.Delete(pfxOutputKey); err != nil { + return err + } + + // Next, we will write the provided kid outpoint to the channel + // bucket, using a key prefixed by the kindergarten prefix. + + // Convert the preschool prefix key into a kindergarten key for + // the same outpoint. + copy(pfxOutputKey, kndrPrefix) + + // Reserialize the kid here to capture any differences in the + // new and old kid output, such as the confirmation height. + var kidBuffer bytes.Buffer + if err := kid.Encode(&kidBuffer); err != nil { + return err + } + kidBytes := kidBuffer.Bytes() + + // And store the kid output in its channel bucket using the + // kindergarten prefixed key. + if err := chanBucket.Put(pfxOutputKey, kidBytes); err != nil { + return err + } + + // Since the CSV delay on the kid output has now begun ticking, + // we must insert a record of in the height index to remind us + // to revisit this output once it has fully matured. + + // Compute the maturity height, by adding the output's CSV delay + // to its confirmation height. + maturityHeight := kid.ConfHeight() + kid.BlocksToMaturity() + + // Create or retrieve the height-channel bucket for this + // channel. This method will first create a height bucket for + // the given maturity height if none exists. + hghtChanBucket, err := ns.createHeightChanBucket(tx, + maturityHeight, chanPoint) + if err != nil { + return err + } + + // Finally, we touch a bucket in the height-channel created + // above. The bucket is named using a kindergarten prefixed + // key, signaling that this CSV delayed output will be ready to + // broadcast at the maturity height, after a brief period of + // incubation. + _, err = hghtChanBucket.CreateBucketIfNotExists(pfxOutputKey) + + return err + }) +} + +// AwardDiplomas accepts a list of kidOutputs in the kindergarten bucket, +// removing their corresponding entries from the height and channel indexes. +// If this method detects that all outputs for a particular contract have been +// incubated, it returns the channel points that are ready to be marked as +// fully closed. This method will iterate through the provided kidOutputs and do +// the following: +// 1) Prune the kid height bucket at the kid's confirmation height, if it is +// empty. +// 2) Prune the channel bucket belonging to the kid's origin channel point, if +// it is empty. +func (ns *nurseryStore) AwardDiplomas( + kids ...kidOutput) ([]wire.OutPoint, error) { + + // As we iterate over the kids, we will build a list of the channels + // which have been pruned entirely from the nursery store. We will + // return this list to the caller, the utxo nursery, so that it can + // proceed to mark the channels as closed. + // TODO(conner): write list of closed channels to separate bucket so + // that they can be replayed on restart? + var closedChannelSet = make(map[wire.OutPoint]struct{}) + if err := ns.db.Update(func(tx *bolt.Tx) error { + for _, kid := range kids { + // Attempt to prune the height bucket matching the kid + // output's confirmation height if it contains no active + // outputs. + err := ns.pruneHeight(tx, kid.ConfHeight()) + switch err { + case ErrBucketNotEmpty: + // Bucket still has active outputs, proceed to + // prune channel bucket. + + case ErrBucketDoesNotExist: + // Bucket was previously pruned by another + // graduating output. + + case nil: + // Bucket was pruned successfully and no errors + // were encounter. + utxnLog.Infof("Height bucket %d pruned", + kid.ConfHeight()) + + default: + // Unexpected database error. + return err + } + + outpoint := kid.OutPoint() + chanPoint := kid.OriginChanPoint() + + // Remove the outpoint belonging to the kid output from + // it's channel bucket, then attempt to prune the + // channel bucket if it is now empty. + err = ns.deleteAndPruneChannel(tx, chanPoint, outpoint) + switch err { + case ErrBucketNotEmpty: + // Bucket still has active outputs, continue to + // next kid to avoid adding this channel point + // to the set of channels to be closed. + continue + + case ErrBucketDoesNotExist: + // Bucket may have been removed previously, + // allow this to fall through and ensure the + // channel point is added to the set channels to + // be closed. + + case nil: + // Channel bucket was successfully pruned, + // proceed to add to set of channels to be + // closed. + utxnLog.Infof("Height bucket %d pruned", + kid.ConfHeight()) + + default: + // Uh oh, database error. + return err + } + + // If we've arrived here, we have encountered no + // database errors and a bucket was either successfully + // pruned or already has been. Thus it is safe to add it + // to our set of closed channels to be closed, since + // these may need to be replayed to ensure the channel + // database is aware that incubation has completed. + closedChannelSet[*chanPoint] = struct{}{} + } + + return nil + }); err != nil { + return nil, err + } + + // Convert our set of channels to be closed into a list. + channelsToBeClosed := make([]wire.OutPoint, 0, len(closedChannelSet)) + for chanPoint := range closedChannelSet { + channelsToBeClosed = append(channelsToBeClosed, chanPoint) + } + + utxnLog.Infof("Channels to be marked fully closed: %x", + channelsToBeClosed) + + return channelsToBeClosed, nil +} + +// FinalizeClass accepts a block height as a parameter and purges its +// persistent state for all outputs at that height. During a restart, the utxo +// nursery will begin it's recovery procedure from the next height that has +// yet to be finalized. +func (ns *nurseryStore) FinalizeClass(height uint32) error { + utxnLog.Infof("Finalizing class at height %v", height) + return ns.db.Update(func(tx *bolt.Tx) error { + return ns.putLastFinalizedHeight(tx, height) + }) +} + +// State Bucket Enumeration. + +// FetchCribs returns a list of babyOutputs in the crib bucket whose CLTV +// delay expires at the provided block height. +func (ns *nurseryStore) FetchCribs(height uint32) ([]babyOutput, error) { + // Construct a list of all babyOutputs that need TLC at the provided + // block height. + var babies []babyOutput + if err := ns.forEachHeightPrefix(cribPrefix, height, + func(buf []byte) error { + + // We will attempt to deserialize all outputs stored + // with the crib prefix into babyOutputs, since this is + // the expected type that would have been serialized + // previously. + var bby babyOutput + if err := bby.Decode(bytes.NewReader(buf)); err != nil { + return err + } + + // Append the deserialized object to our list of + // babyOutputs. + babies = append(babies, bby) + + return nil + + }); err != nil { + return nil, err + } + + return babies, nil +} + +// FetchKindergartens returns a list of kidOutputs in the kindergarten bucket +// whose CSV delay expires at the provided block height. +func (ns *nurseryStore) FetchKindergartens(height uint32) ([]kidOutput, error) { + // Construct a list of all kidOutputs that mature at the provided block + // height. + var kids []kidOutput + if err := ns.forEachHeightPrefix(kndrPrefix, height, + func(buf []byte) error { + + // We will attempt to deserialize all outputs stored + // with the kindergarten prefix into kidOutputs, since + // this is the expected type that would have been + // serialized previously. + var kid kidOutput + if err := kid.Decode(bytes.NewReader(buf)); err != nil { + return err + } + + // Append the deserialized object to our list of + // kidOutputs. + kids = append(kids, kid) + + return nil + + }); err != nil { + return nil, err + } + + return kids, nil +} + +// FetchPreschools returns a list of all outputs currently stored in the +// preschool bucket. +func (ns *nurseryStore) FetchPreschools() ([]kidOutput, error) { + var kids []kidOutput + if err := ns.db.View(func(tx *bolt.Tx) error { + + // Retrieve the existing chain bucket for this nursery store. + chainBucket := tx.Bucket(ns.pfxChainKey) + if chainBucket == nil { + return nil + } + + // Load the existing channel index from the chain bucket. + chanIndex := chainBucket.Bucket(channelIndexKey) + if chanIndex == nil { + return nil + } + + // Construct a list of all channels in the channel index that + // are currently being tracked by the nursery store. + var activeChannels [][]byte + if err := chanIndex.ForEach(func(chanBytes, _ []byte) error { + activeChannels = append(activeChannels, chanBytes) + return nil + }); err != nil { + return err + } + + // Iterate over all of the accumulated channels, and do a prefix + // scan inside of each channel bucket. Each output found that + // has a preschool prefix will be deserialized into a kidOutput, + // and added to our list of preschool outputs to return to the + // caller. + for _, chanBytes := range activeChannels { + // Retrieve the channel bucket associated with this + // channel. + chanBucket := chanIndex.Bucket(chanBytes) + if chanBucket == nil { + continue + } + + // All of the outputs of interest will start with the + // "pscl" prefix. So, we will perform a prefix scan of + // the channel bucket to efficiently enumerate all the + // desired outputs. + c := chanBucket.Cursor() + + // Seek and iterate over all outputs starting with the + // prefix "pscl". + pfxOutputKey, kidBytes := c.Seek(psclPrefix) + for bytes.HasPrefix(pfxOutputKey, psclPrefix) { + + // Deserialize each output as a kidOutput, since + // this should have been the type that was + // serialized when it was written to disk. + var psclOutput kidOutput + psclReader := bytes.NewReader(kidBytes) + err := psclOutput.Decode(psclReader) + if err != nil { + return err + } + + // Add the deserialized output to our list of + // preschool outputs. + kids = append(kids, psclOutput) + + // Advance to the subsequent key-value pair of + // the prefix scan. + pfxOutputKey, kidBytes = c.Next() + } + } + + return nil + }); err != nil { + return nil, err + } + + return kids, nil +} + +// Channel Output Enumberation. + +// ForChanOutputs iterates over all outputs being incubated for a particular +// channel point. This method accepts a callback that allows the caller to +// process each key-value pair. The key will be a prefixed outpoint, and the +// value will be the serialized bytes for an output, whose type should be +// inferred from the key's prefix. +// NOTE: The callback should be not modify the provided byte slices and is +// preferably non-blocking. +func (ns *nurseryStore) ForChanOutputs(chanPoint *wire.OutPoint, + callback func([]byte, []byte) error) error { + + return ns.db.View(func(tx *bolt.Tx) error { + chanBucket := ns.getChannelBucket(tx, chanPoint) + if chanBucket == nil { + return ErrContractNotFound + } + + return chanBucket.ForEach(callback) + }) +} + +// The Point of No Return. + +// LastFinalizedHeight returns the last block height for which the nursery +// store has purged all persistent state. This occurs after a fixed interval +// for reorg safety. +func (ns *nurseryStore) LastFinalizedHeight() (uint32, error) { + var lastFinalizedHeight uint32 + err := ns.db.View(func(tx *bolt.Tx) error { + lastHeight, err := ns.getLastFinalizedHeight(tx) + if err != nil { + return err + } + + lastFinalizedHeight = lastHeight + + return nil + }) + return lastFinalizedHeight, err +} + +// getLastFinalizedHeight is a helper method that retrieves the last height for +// which the database finalized its persistent state. +func (ns *nurseryStore) getLastFinalizedHeight(tx *bolt.Tx) (uint32, error) { + // Retrieve the chain bucket associated with the given nursery store. + chainBucket := tx.Bucket(ns.pfxChainKey) + if chainBucket == nil { + return 0, nil + } + + // Lookup the last finalized height in the top-level chain bucket. + heightBytes := chainBucket.Get(lastFinalizedHeightKey) + + // If the resulting bytes are not sized like a uint32, then we have + // never finalized, so we return 0. + if len(heightBytes) != 4 { + return 0, nil + } + + // Otherwise, parse the bytes and return the last finalized height. + return byteOrder.Uint32(heightBytes), nil +} + +// pubLastFinalizedHeight is a helper method that writes the provided height +// under the last finalized height key. +func (ns *nurseryStore) putLastFinalizedHeight(tx *bolt.Tx, + height uint32) error { + + // Ensure that the chain bucket for this nursery store exists. + chainBucket, err := tx.CreateBucketIfNotExists(ns.pfxChainKey) + if err != nil { + return err + } + + // Serialize the provided last-finalized height, and store it in the + // top-level chain bucket for this nursery store. + var lastHeightBytes [4]byte + byteOrder.PutUint32(lastHeightBytes[:], height) + + return chainBucket.Put(lastFinalizedHeightKey, lastHeightBytes[:]) +} + +// createChannelBucket creates or retrieves a channel bucket for the provided +// channel point. +func (ns *nurseryStore) createChannelBucket(tx *bolt.Tx, + chanPoint *wire.OutPoint) (*bolt.Bucket, error) { + + // Ensure that the chain bucket for this nursery store exists. + chainBucket, err := tx.CreateBucketIfNotExists(ns.pfxChainKey) + if err != nil { + return nil, err + } + + // Ensure that the channel index has been properly initialized for this + // chain. + chanIndex, err := chainBucket.CreateBucketIfNotExists(channelIndexKey) + if err != nil { + return nil, err + } + + // Serialize the provided channel point, as this provides the name of + // the channel bucket of interest. + var chanBuffer bytes.Buffer + if err := writeOutpoint(&chanBuffer, chanPoint); err != nil { + return nil, err + } + + // Finally, create or retrieve the channel bucket using the serialized + // key. + return chanIndex.CreateBucketIfNotExists(chanBuffer.Bytes()) +} + +// getChannelBucket retrieves an existing channel bucket from the nursery store, +// using the given channel point. If the bucket does not exist, or any bucket +// along its path does not exist, a nil value is returned. +func (ns *nurseryStore) getChannelBucket(tx *bolt.Tx, + chanPoint *wire.OutPoint) *bolt.Bucket { + + // Retrieve the existing chain bucket for this nursery store. + chainBucket := tx.Bucket(ns.pfxChainKey) + if chainBucket == nil { + return nil + } + + // Retrieve the existing channel index. + chanIndex := chainBucket.Bucket(channelIndexKey) + if chanIndex == nil { + return nil + } + + // Serialize the provided channel point and return the bucket matching + // the serialized key. + var chanBuffer bytes.Buffer + if err := writeOutpoint(&chanBuffer, chanPoint); err != nil { + return nil + } + + return chanIndex.Bucket(chanBuffer.Bytes()) +} + +// createHeightBucket creates or retrieves an existing bucket from the height +// index, corresponding to the provided height. +func (ns *nurseryStore) createHeightBucket(tx *bolt.Tx, + height uint32) (*bolt.Bucket, error) { + + // Ensure that the chain bucket for this nursery store exists. + chainBucket, err := tx.CreateBucketIfNotExists(ns.pfxChainKey) + if err != nil { + return nil, err + } + + // Ensure that the height index has been properly initialized for this + // chain. + hghtIndex, err := chainBucket.CreateBucketIfNotExists(heightIndexKey) + if err != nil { + return nil, err + } + + // Serialize the provided height, as this will form the name of the + // bucket. + var heightBytes [4]byte + byteOrder.PutUint32(heightBytes[:], height) + + // Finally, create or retrieve the bucket in question. + return hghtIndex.CreateBucketIfNotExists(heightBytes[:]) +} + +// getHeightBucket retrieves an existing height bucket from the nursery store, +// using the provided block height. If the bucket does not exist, or any bucket +// along its path does not exist, a nil value is returned. +func (ns *nurseryStore) getHeightBucket(tx *bolt.Tx, + height uint32) *bolt.Bucket { + + // Retrieve the existing chain bucket for this nursery store. + chainBucket := tx.Bucket(ns.pfxChainKey) + if chainBucket == nil { + return nil + } + + // Retrieve the existing channel index. + hghtIndex := chainBucket.Bucket(heightIndexKey) + if hghtIndex == nil { + return nil + } + + // Serialize the provided block height and return the bucket matching + // the serialized key. + var heightBytes [4]byte + byteOrder.PutUint32(heightBytes[:], height) + + return hghtIndex.Bucket(heightBytes[:]) +} + +// createHeightChanBucket creates or retrieves an existing height-channel bucket +// for the provided block height and channel point. This method will attempt to +// instantiate all buckets along the path if required. +func (ns *nurseryStore) createHeightChanBucket(tx *bolt.Tx, + height uint32, chanPoint *wire.OutPoint) (*bolt.Bucket, error) { + + // Ensure that the height bucket for this nursery store exists. + hghtBucket, err := ns.createHeightBucket(tx, height) + if err != nil { + return nil, err + } + + // Serialize the provided channel point, as this generates the name of + // the subdirectory corresponding to the channel of interest. + var chanBuffer bytes.Buffer + if err := writeOutpoint(&chanBuffer, chanPoint); err != nil { + return nil, err + } + chanBytes := chanBuffer.Bytes() + + // Finally, create or retrieve an existing height-channel bucket for + // this channel point. + return hghtBucket.CreateBucketIfNotExists(chanBytes) +} + +// getHeightChanBucket retrieves an existing height-channel bucket from the +// nursery store, using the provided block height and channel point. if the +// bucket does not exist, or any bucket along its path does not exist, a nil +// value is returned. +func (ns *nurseryStore) getHeightChanBucket(tx *bolt.Tx, + height uint32, chanPoint *wire.OutPoint) *bolt.Bucket { + + // Retrieve the existing height bucket from this nursery store. + hghtBucket := ns.getHeightBucket(tx, height) + if hghtBucket == nil { + return nil + } + + // Serialize the provided channel point, which generates the key for + // looking up the proper height-channel bucket inside the height bucket. + var chanBuffer bytes.Buffer + if err := writeOutpoint(&chanBuffer, chanPoint); err != nil { + return nil + } + chanBytes := chanBuffer.Bytes() + + // Finally, return the height bucket specified by the serialized channel + // point. + return hghtBucket.Bucket(chanBytes) +} + +// forEachHeightPrefix enumerates all outputs at the given height whose state +// prefix matches that which is provided. This is used as a subroutine to help +// enumerate crib and kindergarten outputs at a particular height. The callback +// is invoked with serialized bytes retrieved for each output of interest, +// allowing the caller to deserialize them into the appropriate type. +func (ns *nurseryStore) forEachHeightPrefix(prefix []byte, height uint32, + callback func([]byte) error) error { + + return ns.db.View(func(tx *bolt.Tx) error { + // Start by retrieving the height bucket corresponding to the + // provided block height. + hghtBucket := ns.getHeightBucket(tx, height) + if hghtBucket == nil { + return nil + } + + // Using the height bucket as a starting point, we will traverse + // its entire two-tier directory structure, and filter for + // outputs that have the provided prefix. The first layer of the + // height bucket contains buckets identified by a channel point, + // thus we first create list of channels contained in this + // height bucket. + var channelsAtHeight [][]byte + if err := hghtBucket.ForEach(func(chanBytes, _ []byte) error { + channelsAtHeight = append(channelsAtHeight, chanBytes) + return nil + }); err != nil { + return err + } + + // As we enumerate the outputs referenced in this height bucket, + // we will need to load the serialized value of each output, + // which is ultimately stored its respective channel bucket. To + // do so, we first load the top-level chain bucket, which should + // already be created if we are at this point. + chainBucket := tx.Bucket(ns.pfxChainKey) + if chainBucket == nil { + return nil + } + + // Additionally, grab the chain index, which we will facilitate + // queries for each of the channel buckets of each of the + // channels in the list we assembled above. + chanIndex := chainBucket.Bucket(channelIndexKey) + if chanIndex == nil { + return nil + } + + // Now, we are ready to enumerate all outputs with the desired + // prefix t this block height. We do so by iterating over our + // list of channels at this height, filtering for outputs in + // each height-channel bucket that begin with the given prefix, + // and then retrieving the serialized outputs from the + // appropriate channel bucket. + for _, chanBytes := range channelsAtHeight { + // Retrieve the height-channel bucket for this channel, + // which holds a sub-bucket for all outputs maturing at + // this height. + hghtChanBucket := hghtBucket.Bucket(chanBytes) + if hghtChanBucket == nil { + continue + } + + // Load the appropriate channel bucket from the channel + // index, this will allow us to retrieve the individual + // serialized outputs. + chanBucket := chanIndex.Bucket(chanBytes) + if chanBucket == nil { + continue + } + + // Since all of the outputs of interest will start with + // the same prefix, we will perform a prefix scan of the + // buckets contained in the height-channel bucket, + // efficiently enumerating the desired outputs. + c := hghtChanBucket.Cursor() + + // Seek to and iterate over all entries starting with + // the given prefix. + var pfxOutputKey, _ = c.Seek(prefix) + for bytes.HasPrefix(pfxOutputKey, prefix) { + + // Use the prefix output key emitted from our + // scan to load the serialized babyOutput from + // the appropriate channel bucket. + outputBytes := chanBucket.Get(pfxOutputKey) + if outputBytes == nil { + continue + } + + // Present the serialized bytes to our call back + // function, which is responsible for + // deserializing the bytes into the appropriate + // type. + if err := callback(outputBytes); err != nil { + return err + } + + // Lastly, advance our prefix output key for the + // next iteration. + pfxOutputKey, _ = c.Next() + } + } + + return nil + }) +} + +var ( + // ErrBucketDoesNotExist signals that a bucket has already been removed, + // or was never created. + ErrBucketDoesNotExist = errors.New("bucket does not exist") + + // ErrBucketNotEmpty signals that an attempt to prune a particular + // bucket failed because it still has active outputs. + ErrBucketNotEmpty = errors.New("bucket is not empty, cannot be pruned") +) + +// deleteAndPruneHeight removes an output from a channel bucket matching the +// provided channel point. The output is assumed top be in the kindergarten +// bucket, since pruning should never occur for any other type of output. If +// after deletion, the channel bucket is empty, this method will attempt to +// delete the bucket as well. +// NOTE: This method returns two concrete errors apart from those returned by +// the underlying database: ErrBucketDoesNotExist and ErrBucketNotEmpty. These +// should be handled in the context of the caller so as they may be benign +// depending on context. Errors returned other than these two should be +// interpreted as database errors. +func (ns *nurseryStore) deleteAndPruneChannel(tx *bolt.Tx, + chanPoint, outpoint *wire.OutPoint) error { + + // Retrieve the existing chain bucket for this nursery store. + chainBucket := tx.Bucket(ns.pfxChainKey) + if chainBucket == nil { + return nil + } + + // Retrieve the channel index stored in the chain bucket. + chanIndex := chainBucket.Bucket(channelIndexKey) + if chanIndex == nil { + return nil + } + + // Serialize the provided channel point, such that we can retrieve the + // desired channel bucket. + var chanBuffer bytes.Buffer + if err := writeOutpoint(&chanBuffer, chanPoint); err != nil { + return err + } + chanBytes := chanBuffer.Bytes() + + // Retrieve the existing channel bucket. If none exists, then our job is + // complete and it is safe to return nil. + chanBucket := chanIndex.Bucket(chanBytes) + if chanBucket == nil { + return nil + } + + // Otherwise, the bucket still exists. Serialize the outpoint that needs + // deletion, prefixing the key with kindergarten prefix. Since all + // outputs eventually make their way to becoming kindergarten outputs, + // we can safely assume that they will be stored with a kindergarten + // prefix. + pfxOutputBytes, err := prefixOutputKey(kndrPrefix, outpoint) + if err != nil { + return err + } + + // Remove the output in question using the kindergarten-prefixed key we + // generated above. + if err := chanBucket.Delete(pfxOutputBytes); err != nil { + return err + } + + // Finally, now that the outpoint has been removed from this channel + // bucket, try to remove the channel bucket altogether if it is now + // empty. + return ns.removeBucketIfEmpty(chanIndex, chanBytes) +} + +// pruneHeight +// NOTE: This method returns two concrete errors apart from those returned by +// the underlying database: ErrBucketDoesNotExist and ErrBucketNotEmpty. These +// should be handled in the context of the caller so as they may be benign +// depending on context. Errors returned other than these two should be +// interpreted as database errors. +func (ns *nurseryStore) pruneHeight(tx *bolt.Tx, height uint32) error { + + // Fetch the existing chain bucket for this nursery store. + chainBucket := tx.Bucket(ns.pfxChainKey) + if chainBucket == nil { + return nil + } + + // Load the existing height bucket for the height in question. + hghtIndex := chainBucket.Bucket(heightIndexKey) + if hghtIndex == nil { + return nil + } + + // Serialize the provided block height, such that it can be used as the + // key to locate the desired height bucket. + var heightBytes [4]byte + byteOrder.PutUint32(heightBytes[:], height) + + // Retrieve the height bucket using the serialized height as the bucket + // name. + hghtBucket := hghtIndex.Bucket(heightBytes[:]) + if hghtBucket == nil { + return ErrBucketDoesNotExist + } + + // Iterate over the contents of this height bucket, which is comprised + // of sub-buckets named after the channel points that need attention at + // this block height. We will attempt to remove each one if they are + // empty, keeping track of the number of height-channel buckets that + // still have active outputs. + var nActiveBuckets int + if err := hghtBucket.ForEach(func(chanBytes, _ []byte) error { + + // Attempt to each height-channel bucket from the height bucket + // located above. + err := ns.removeBucketIfEmpty(hghtBucket, chanBytes) + switch err { + case nil: + // The height-channel bucket was removed successfully! + return nil + + case ErrBucketDoesNotExist: + // The height-channel bucket could not be located--no + // harm, no foul. + return nil + + case ErrBucketNotEmpty: + // The bucket still has active outputs at this height, + // increment our number of still active height-channel + // buckets. + nActiveBuckets++ + return nil + + default: + // Database error! + return err + } + + }); err != nil { + return err + } + + // If we located any height-channel buckets that still have active + // outputs, it is unsafe to delete this height bucket. Signal this event + // to the caller so that they can determine the appropriate action. + if nActiveBuckets > 0 { + return ErrBucketNotEmpty + } + + // All of the height-channel buckets are empty or have been previously + // removed, proceed by attempting to remove the height bucket + // altogether. + return ns.removeBucketIfEmpty(hghtIndex, heightBytes[:]) +} + +// removeBucketIfEmpty attempts to delete a bucket specified by name from the +// provided parent bucket. +// NOTE: This method returns two concrete errors apart from those returned by +// the underlying database: ErrBucketDoesNotExist and ErrBucketNotEmpty. These +// should be handled in the context of the caller so as they may be benign +// depending on context. Errors returned other than these two should be +// interpreted as database errors. +func (ns *nurseryStore) removeBucketIfEmpty(parent *bolt.Bucket, + bktName []byte) error { + + // Attempt to fetch the named bucket from its parent. + bkt := parent.Bucket(bktName) + if bkt == nil { + // No bucket was found, signal this to the caller. + return ErrBucketDoesNotExist + } + + // The bucket exists, now compute how many children *it* has. + nChildren, err := ns.numChildrenInBucket(bkt) + if err != nil { + return err + } + + // If the number of children is non-zero, alert the caller that the + // named bucket is not being removed. + if nChildren > 0 { + return ErrBucketNotEmpty + } + + // Otherwise, remove the empty bucket from its parent. + return parent.DeleteBucket(bktName) +} + +// numChildrenInBucket computes the number of children contained in the given +// boltdb bucket. +func (ns *nurseryStore) numChildrenInBucket(parent *bolt.Bucket) (int, error) { + var nChildren int + if err := parent.ForEach(func(_, _ []byte) error { + nChildren++ + return nil + }); err != nil { + return 0, err + } + + return nChildren, nil +} + +// Compile-time constraint to ensure nurseryStore implements NurseryStore. +var _ NurseryStore = (*nurseryStore)(nil) From f02f1355e7be1e3aa8d7bd31ddf52a0c2d3d171d Mon Sep 17 00:00:00 2001 From: Conner Fromknecht Date: Wed, 11 Oct 2017 16:33:12 -0700 Subject: [PATCH 2/6] nursery_store: adds graduating state and proper finalization --- nursery_store.go | 257 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 193 insertions(+), 64 deletions(-) diff --git a/nursery_store.go b/nursery_store.go index d32a1cad7..f346a8319 100644 --- a/nursery_store.go +++ b/nursery_store.go @@ -52,13 +52,15 @@ type NurseryStore interface { // TODO: make this handle one output at a time? AwardDiplomas(...kidOutput) ([]wire.OutPoint, error) - // FinalizeClass accepts a block height as a parameter and purges its + IsMatureChannel(*wire.OutPoint) (bool, error) + + // TryFinalizeClass accepts a block height as a parameter and purges its // persistent state for all outputs at that height. During a restart, // the utxo nursery will begin it's recovery procedure from the next // height that has yet to be finalized. This block height should lag // beyond the best height for this chain as a measure of reorg // protection. - FinalizeClass(height uint32) error + TryFinalizeClass(height uint32) error // State Bucket Enumeration. @@ -83,6 +85,8 @@ type NurseryStore interface { // whose type should be inferred from the key's prefix. ForChanOutputs(*wire.OutPoint, func([]byte, []byte) error) error + RemoveChannel(*wire.OutPoint) error + // The Point of No Return. // LastFinalizedHeight returns the last block height for which the @@ -164,6 +168,8 @@ var ( // and will be swept into the wallet after waiting out the relative // timelock. kndrPrefix = []byte("kndr") + + gradPrefix = []byte("grad") ) // Overview of Nursery Store Storage Hierarchy @@ -563,62 +569,71 @@ func (ns *nurseryStore) AwardDiplomas( // proceed to mark the channels as closed. // TODO(conner): write list of closed channels to separate bucket so // that they can be replayed on restart? - var closedChannelSet = make(map[wire.OutPoint]struct{}) + var possibleCloseSet = make(map[wire.OutPoint]struct{}) if err := ns.db.Update(func(tx *bolt.Tx) error { for _, kid := range kids { - // Attempt to prune the height bucket matching the kid - // output's confirmation height if it contains no active - // outputs. - err := ns.pruneHeight(tx, kid.ConfHeight()) - switch err { - case ErrBucketNotEmpty: - // Bucket still has active outputs, proceed to - // prune channel bucket. - case ErrBucketDoesNotExist: - // Bucket was previously pruned by another - // graduating output. - - case nil: - // Bucket was pruned successfully and no errors - // were encounter. - utxnLog.Infof("Height bucket %d pruned", - kid.ConfHeight()) - - default: - // Unexpected database error. - return err - } + confHeight := kid.ConfHeight() outpoint := kid.OutPoint() chanPoint := kid.OriginChanPoint() - // Remove the outpoint belonging to the kid output from - // it's channel bucket, then attempt to prune the - // channel bucket if it is now empty. - err = ns.deleteAndPruneChannel(tx, chanPoint, outpoint) - switch err { - case ErrBucketNotEmpty: - // Bucket still has active outputs, continue to - // next kid to avoid adding this channel point - // to the set of channels to be closed. - continue + // Remove output from kindergarten bucket. - case ErrBucketDoesNotExist: - // Bucket may have been removed previously, - // allow this to fall through and ensure the - // channel point is added to the set channels to - // be closed. + pfxOutputKey, err := prefixOutputKey(kndrPrefix, outpoint) + if err != nil { + return err + } - case nil: - // Channel bucket was successfully pruned, - // proceed to add to set of channels to be - // closed. - utxnLog.Infof("Height bucket %d pruned", - kid.ConfHeight()) + hghtChanBucket := ns.getHeightChanBucket(tx, confHeight, chanPoint) + if hghtChanBucket != nil { + if err := hghtChanBucket.DeleteBucket(pfxOutputKey); err != nil { + return err + } - default: - // Uh oh, database error. + // Attempt to prune the height bucket matching the kid + // output's confirmation height if it contains no active + // outputs. + err := ns.pruneHeight(tx, confHeight) + switch err { + case ErrBucketNotEmpty: + // Bucket still has active outputs, proceed to + // prune channel bucket. + + case ErrBucketDoesNotExist: + // Bucket was previously pruned by another + // graduating output. + + case nil: + // Bucket was pruned successfully and no errors + // were encounter. + utxnLog.Infof("Height bucket %d pruned", confHeight) + + default: + // Unexpected database error. + return err + } + } + + chanBucket, err := ns.createChannelBucket(tx, chanPoint) + if err != nil { + return err + } + + if err := chanBucket.Delete(pfxOutputKey); err != nil { + return err + } + + // Convert kindergarten key to graduate key. + copy(pfxOutputKey, gradPrefix) + + var gradBuffer bytes.Buffer + if err := kid.Encode(&gradBuffer); err != nil { + return err + } + + err = chanBucket.Put(pfxOutputKey, gradBuffer.Bytes()) + if err != nil { return err } @@ -628,7 +643,7 @@ func (ns *nurseryStore) AwardDiplomas( // to our set of closed channels to be closed, since // these may need to be replayed to ensure the channel // database is aware that incubation has completed. - closedChannelSet[*chanPoint] = struct{}{} + possibleCloseSet[*chanPoint] = struct{}{} } return nil @@ -637,25 +652,31 @@ func (ns *nurseryStore) AwardDiplomas( } // Convert our set of channels to be closed into a list. - channelsToBeClosed := make([]wire.OutPoint, 0, len(closedChannelSet)) - for chanPoint := range closedChannelSet { - channelsToBeClosed = append(channelsToBeClosed, chanPoint) + possibleCloses := make([]wire.OutPoint, 0, len(possibleCloseSet)) + for chanPoint := range possibleCloseSet { + possibleCloses = append(possibleCloses, chanPoint) } - utxnLog.Infof("Channels to be marked fully closed: %x", - channelsToBeClosed) + utxnLog.Infof("Possible channels to be marked fully closed: %v", + possibleCloses) - return channelsToBeClosed, nil + return possibleCloses, nil } -// FinalizeClass accepts a block height as a parameter and purges its +// TryFinalizeClass accepts a block height as a parameter and purges its // persistent state for all outputs at that height. During a restart, the utxo // nursery will begin it's recovery procedure from the next height that has // yet to be finalized. -func (ns *nurseryStore) FinalizeClass(height uint32) error { - utxnLog.Infof("Finalizing class at height %v", height) +func (ns *nurseryStore) TryFinalizeClass(height uint32) error { return ns.db.Update(func(tx *bolt.Tx) error { - return ns.putLastFinalizedHeight(tx, height) + utxnLog.Infof("Attempting to finalize class at height %v", height) + lastHeight, err := ns.getNextLastFinalizedHeight(tx, height) + if err != nil { + return err + } + utxnLog.Infof("Finalizing class at height %v", lastHeight) + + return ns.putLastFinalizedHeight(tx, lastHeight) }) } @@ -697,9 +718,11 @@ func (ns *nurseryStore) FetchCribs(height uint32) ([]babyOutput, error) { func (ns *nurseryStore) FetchKindergartens(height uint32) ([]kidOutput, error) { // Construct a list of all kidOutputs that mature at the provided block // height. + utxnLog.Infof("Fetching kinders") var kids []kidOutput if err := ns.forEachHeightPrefix(kndrPrefix, height, func(buf []byte) error { + utxnLog.Infof("Inside kinder") // We will attempt to deserialize all outputs stored // with the kindergarten prefix into kidOutputs, since @@ -719,6 +742,7 @@ func (ns *nurseryStore) FetchKindergartens(height uint32) ([]kidOutput, error) { }); err != nil { return nil, err } + utxnLog.Infof("Returning kinders") return kids, nil } @@ -816,13 +840,49 @@ func (ns *nurseryStore) ForChanOutputs(chanPoint *wire.OutPoint, callback func([]byte, []byte) error) error { return ns.db.View(func(tx *bolt.Tx) error { - chanBucket := ns.getChannelBucket(tx, chanPoint) - if chanBucket == nil { - return ErrContractNotFound + return ns.forChanOutputs(tx, chanPoint, callback) + }) +} +func (ns *nurseryStore) forChanOutputs(tx *bolt.Tx, chanPoint *wire.OutPoint, + callback func([]byte, []byte) error) error { + + chanBucket := ns.getChannelBucket(tx, chanPoint) + if chanBucket == nil { + return ErrContractNotFound + } + + return chanBucket.ForEach(callback) +} + +func (ns *nurseryStore) IsMatureChannel(chanPoint *wire.OutPoint) (bool, error) { + var isMature bool + if err := ns.db.View(func(tx *bolt.Tx) error { + var nOutputs, nGrads int + if err := ns.forChanOutputs(tx, chanPoint, func(pfxKey, _ []byte) error { + // Count total and number of grad outputs + if string(pfxKey[:4]) == string(gradPrefix) { + nGrads++ + } + nOutputs++ + + return nil + + }); err != nil { + return err } - return chanBucket.ForEach(callback) - }) + utxnLog.Infof("Found %d graduated outputs out of %d", nGrads, nOutputs) + // Channel is mature if all outputs are graduated. + if nGrads == nOutputs { + isMature = true + } + + return nil + }); err != nil { + return false, err + } + + return isMature, nil } // The Point of No Return. @@ -867,6 +927,49 @@ func (ns *nurseryStore) getLastFinalizedHeight(tx *bolt.Tx) (uint32, error) { return byteOrder.Uint32(heightBytes), nil } +func (ns *nurseryStore) getNextLastFinalizedHeight(tx *bolt.Tx, height uint32) (uint32, error) { + // Retrieve the previous last finalized height. + lastHeight, err := ns.getLastFinalizedHeight(tx) + if err != nil { + return 0, err + } + + // TODO(conner): add lower bound after which all state is purged. + + // Retrieve the existing chain bucket for this nursery store. + chainBucket := tx.Bucket(ns.pfxChainKey) + if chainBucket == nil { + return lastHeight, nil + } + + // Retrieve the existing height index. + hghtIndex := chainBucket.Bucket(heightIndexKey) + if hghtIndex == nil { + return lastHeight, nil + } + + for curHeight := lastHeight + 1; curHeight <= height; curHeight++ { + var curHeightBytes [4]byte + byteOrder.PutUint32(curHeightBytes[:], curHeight) + + hghtBucket := hghtIndex.Bucket(curHeightBytes[:]) + if hghtBucket == nil { + continue + } + + nChildren, err := ns.numChildrenInBucket(hghtBucket) + if err != nil { + return 0, err + } + + if nChildren > 0 { + return curHeight - 1, nil + } + } + + return height, nil +} + // pubLastFinalizedHeight is a helper method that writes the provided height // under the last finalized height key. func (ns *nurseryStore) putLastFinalizedHeight(tx *bolt.Tx, @@ -1135,6 +1238,7 @@ func (ns *nurseryStore) forEachHeightPrefix(prefix []byte, height uint32, // the appropriate channel bucket. outputBytes := chanBucket.Get(pfxOutputKey) if outputBytes == nil { + pfxOutputKey, _ = c.Next() continue } @@ -1228,6 +1332,31 @@ func (ns *nurseryStore) deleteAndPruneChannel(tx *bolt.Tx, return ns.removeBucketIfEmpty(chanIndex, chanBytes) } +func (ns *nurseryStore) RemoveChannel(chanPoint *wire.OutPoint) error { + return ns.db.Update(func(tx *bolt.Tx) error { + // Retrieve the existing chain bucket for this nursery store. + chainBucket := tx.Bucket(ns.pfxChainKey) + if chainBucket == nil { + return nil + } + + // Retrieve the channel index stored in the chain bucket. + chanIndex := chainBucket.Bucket(channelIndexKey) + if chanIndex == nil { + return nil + } + + // Serialize the provided channel point, such that we can delete + // the mature channel bucket. + var chanBuffer bytes.Buffer + if err := writeOutpoint(&chanBuffer, chanPoint); err != nil { + return err + } + + return chanIndex.DeleteBucket(chanBuffer.Bytes()) + }) +} + // pruneHeight // NOTE: This method returns two concrete errors apart from those returned by // the underlying database: ErrBucketDoesNotExist and ErrBucketNotEmpty. These From 23e36a58f0ecc5da6d746a9f45782790b7d6b7fd Mon Sep 17 00:00:00 2001 From: Conner Fromknecht Date: Thu, 12 Oct 2017 05:03:25 -0700 Subject: [PATCH 3/6] nursery_store: refactors ifaces to be more atomic Also includes: * improved error handling when pruning channels * more aggressively signals errors when enumerating height outputs * removes use of variadic functions in Incubate, AwardDiplomas * renames AwardDiplomas to GraduateKinder * short circuits channel maturity test after finding 1 non-grad output * replaces prefixed outputs in height buckets with files, instead of empty buckets --- nursery_store.go | 1097 ++++++++++++++++++++-------------------------- 1 file changed, 465 insertions(+), 632 deletions(-) diff --git a/nursery_store.go b/nursery_store.go index f346a8319..3fdc55984 100644 --- a/nursery_store.go +++ b/nursery_store.go @@ -16,22 +16,11 @@ import ( // its contents, and persisting state transitions detected by the utxo nursery. type NurseryStore interface { - // Incubation Entry Points. - - // EnterCrib accepts a new htlc output that the nursery will incubate - // through its two-stage process of sweeping funds back to the user's - // wallet. These outputs are persisted in the nursery store's crib - // bucket, and will be revisited after the output's CLTV has expired. - EnterCrib(*babyOutput) error - - // EnterPreschool accepts a new commitment output that the nursery will - // incubate through a single stage before sweeping. Outputs are stored - // in the preschool bucket until the commitment transaction has been - // confirmed, at which point they will be moved to the kindergarten - // bucket. - EnterPreschool(*kidOutput) error - - // On-chain Driven State Transtitions. + // Incubate registers a commitment output and a slice of htlc outputs to + // be swept back into the user's wallet. The event is persisted to disk, + // such that the nursery can resume the incubation process after a + // potential crash. + Incubate(*kidOutput, []babyOutput) error // CribToKinder atomically moves a babyOutput in the crib bucket to the // kindergarten bucket. The now mature kidOutput contained in the @@ -44,40 +33,34 @@ type NurseryStore interface { // transaction. PreschoolToKinder(*kidOutput) error - // AwardDiplomas accepts a variadic number of kidOutputs from the - // kindergarten bucket, and removes their corresponding entries from the - // height and channel indexes. If this method detects that all outputs - // for a particular contract have been incubated, it returns the channel + // GraduateKinder accepts a slice of kidOutputs from the kindergarten + // bucket, and removes their corresponding entries from the height and + // channel indexes. If this method detects that all outputs for a + // particular contract have been incubated, it returns the channel // points that are ready to be marked as fully closed. // TODO: make this handle one output at a time? - AwardDiplomas(...kidOutput) ([]wire.OutPoint, error) + GraduateKinder([]kidOutput) error - IsMatureChannel(*wire.OutPoint) (bool, error) - - // TryFinalizeClass accepts a block height as a parameter and purges its + // FinalizeHeight accepts a block height as a parameter and purges its // persistent state for all outputs at that height. During a restart, // the utxo nursery will begin it's recovery procedure from the next // height that has yet to be finalized. This block height should lag // beyond the best height for this chain as a measure of reorg // protection. - TryFinalizeClass(height uint32) error + FinalizeHeight(height uint32) error - // State Bucket Enumeration. + // LastFinalizedHeight returns the last block height for which the + // nursery store has purged all persistent state. + LastFinalizedHeight() (uint32, error) - // FetchCribs returns a list of babyOutputs in the crib bucket whose + // FetchClass returns a list of babyOutputs in the crib bucket whose // CLTV delay expires at the provided block height. - FetchCribs(height uint32) ([]babyOutput, error) - - // FetchKindergartens returns a list of kidOutputs in the kindergarten - // bucket whose CSV delay expires at the provided block height. - FetchKindergartens(height uint32) ([]kidOutput, error) + FetchClass(height uint32) ([]kidOutput, []babyOutput, error) // FetchPreschools returns a list of all outputs currently stored in the // preschool bucket. FetchPreschools() ([]kidOutput, error) - // Channel Output Enumeration. - // ForChanOutputs iterates over all outputs being incubated for a // particular channel point. This method accepts a callback that allows // the caller to process each key-value pair. The key will be a prefixed @@ -85,13 +68,13 @@ type NurseryStore interface { // whose type should be inferred from the key's prefix. ForChanOutputs(*wire.OutPoint, func([]byte, []byte) error) error + // IsMatureChannel determines the whether or not all of the outputs in a + // particular channel bucket have been marked as graduated. + IsMatureChannel(*wire.OutPoint) (bool, error) + + // RemoveChannel channel erases all entries from the channel bucket for + // the provided channel point. RemoveChannel(*wire.OutPoint) error - - // The Point of No Return. - - // LastFinalizedHeight returns the last block height for which the - // nursery store has purged all persistent state. - LastFinalizedHeight() (uint32, error) } // prefixChainKey creates the root level keys for the nursery store. The keys @@ -169,6 +152,10 @@ var ( // timelock. kndrPrefix = []byte("kndr") + // gradPrefix is the state prefix given to all outputs that have been + // completely incubated. Once all outputs have been marked as graduated, + // this serves as a persistent marker that the nursery should mark the + // channel fully closed in the channeldb. gradPrefix = []byte("grad") ) @@ -197,7 +184,7 @@ var ( // | // | The channel index contains a directory for each channel that has a // | non-zero number of outputs being tracked by the nursery store. -// | Inside each channel directory are files contains serialized spendable +// | Inside each channel directory are files containing serialized spendable // | outputs that are awaiting some state transition. The name of each file // | contains the outpoint of the spendable output in the file, and is // | prefixed with 4-byte state prefix, indicating whether the spendable @@ -206,12 +193,12 @@ var ( // | which is useful in constructing nursery reports. // | // ├── channel-index-key/ -// │   ├── / <- CHANNEL BUCKET +// │   ├── / <- CHANNEL BUCKET // | |   ├── : // | |   └── : -// │   ├── / +// │   ├── / // | |   └── : -// │   └── / +// │   └── / // |    ├── : // |    └── : // | @@ -219,13 +206,12 @@ var ( // | // | The height index contains a directory for each height at which the // | nursery still has uncompleted actions. If an output is a crib or -// | kindergarten output, -// | it will have an associated entry in the height index. Inside a -// | particular height directory, the structure is similar to that of the -// | channel index, containing multiple channel directories, each of which -// | contains subdirectories named with a prefixed outpoint belonging to -// | the channel. Enumerating these combinations yields a relative file -// | path: +// | kindergarten output, it will have an associated entry in the height +// | index. Inside a particular height directory, the structure is similar +// | to that of the channel index, containing multiple channel directories, +// | each of which contains subdirectories named with a prefixed outpoint +// | belonging to the channel. Enumerating these combinations yields a +// | relative file path: // | e.g. // // | that can be queried in the channel index to retrieve the serialized // | output. @@ -233,22 +219,22 @@ var ( // └── height-index-key/ //    ├── / <- HEIGHT BUCKET // |   └── / <- HEIGHT-CHANNEL BUCKET -// | |    ├── / <- PREFIXED OUTPOINT -// | |    └── / +// | |    ├── : "" <- PREFIXED OUTPOINT +// | |    └── : "" // |   └── / -// |    └── / +// |    └── : "" //    └── / //    └── / -//    └── / -//    └── / +//    └── : "" +//    └── : "" // nurseryStore is a concrete instantiation of a NurseryStore that is backed by // a channeldb.DB instance. type nurseryStore struct { - chainHash chainhash.Hash - pfxChainKey []byte + chainHash chainhash.Hash + db *channeldb.DB - db *channeldb.DB + pfxChainKey []byte } // newNurseryStore accepts a chain hash and a channeldb.DB instance, returning @@ -267,105 +253,33 @@ func newNurseryStore(chainHash *chainhash.Hash, return &nurseryStore{ chainHash: *chainHash, - pfxChainKey: pfxChainKey, db: db, + pfxChainKey: pfxChainKey, }, nil } -// Incubation Entry Points. - -// EnterCrib accepts a new htlc output that the nursery will incubate through -// its two-stage process of sweeping funds back to the user's wallet. These -// outputs are persisted in the nursery store's crib bucket, and will be -// revisited after the output's CLTV has expired. -func (ns *nurseryStore) EnterCrib(bby *babyOutput) error { +// Incubate initiates the incubation process for the CSV-delayed commitment +// output and any number of CLTV-delayed htlc outputs. +func (ns *nurseryStore) Incubate(kid *kidOutput, babies []babyOutput) error { return ns.db.Update(func(tx *bolt.Tx) error { - - // First, retrieve or create the channel bucket corresponding to - // the baby output's origin channel point. - chanPoint := bby.OriginChanPoint() - chanBucket, err := ns.createChannelBucket(tx, chanPoint) - if err != nil { - return err + // Store commitment output in preschool bucket if not nil. + if kid != nil { + if err := ns.enterPreschool(tx, kid); err != nil { + return err + } } - // Next, retrieve or create the height-channel bucket located in - // the height bucket corresponding to the baby output's CLTV - // expiry height. - hghtChanBucket, err := ns.createHeightChanBucket(tx, - bby.expiry, chanPoint) - if err != nil { - return err + // Add all htlc outputs to the crib bucket. + for _, baby := range babies { + if err := ns.enterCrib(tx, &baby); err != nil { + return err + } } - // Since we are inserting this output into the crib bucket, we - // create a key that prefixes the baby output's outpoint with - // the crib prefix. - pfxOutputKey, err := prefixOutputKey(cribPrefix, bby.OutPoint()) - if err != nil { - return err - } - - // Serialize the baby output so that it can be written to the - // underlying key-value store. - var babyBuffer bytes.Buffer - if err := bby.Encode(&babyBuffer); err != nil { - return err - } - babyBytes := babyBuffer.Bytes() - - // Now, insert the serialized output into its channel bucket - // under the prefixed key created above. - if err := chanBucket.Put(pfxOutputKey, babyBytes); err != nil { - return err - } - - // Finally, create a corresponding bucket in the height-channel - // bucket for this crib output. The existence of this bucket - // indicates that the serialized output can be retrieved from - // the channel bucket using the same prefix key. - _, err = hghtChanBucket.CreateBucketIfNotExists(pfxOutputKey) - - return err + return nil }) } -// EnterPreschool accepts a new commitment output that the nursery will -// incubate through a single stage before sweeping. Outputs are stored in the -// preschool bucket until the commitment transaction has been confirmed, at -// which point they will be moved to the kindergarten bucket. -func (ns *nurseryStore) EnterPreschool(kid *kidOutput) error { - return ns.db.Update(func(tx *bolt.Tx) error { - - // First, retrieve or create the channel bucket corresponding to - // the baby output's origin channel point. - chanPoint := kid.OriginChanPoint() - chanBucket, err := ns.createChannelBucket(tx, chanPoint) - if err != nil { - return err - } - - // Since the babyOutput is being inserted into the preschool - // bucket, we create a key that prefixes its outpoint with the - // preschool prefix. - pfxOutputKey, err := prefixOutputKey(psclPrefix, kid.OutPoint()) - if err != nil { - return err - } - - // Serialize the kidOutput and insert it into the channel - // bucket. - var kidBuffer bytes.Buffer - if err := kid.Encode(&kidBuffer); err != nil { - return err - } - - return chanBucket.Put(pfxOutputKey, kidBuffer.Bytes()) - }) -} - -// On-chain Drive State Transitions. - // CribToKinder atomically moves a babyOutput in the crib bucket to the // kindergarten bucket. The now mature kidOutput contained in the babyOutput // will be stored as it waits out the kidOutput's CSV delay. @@ -406,10 +320,22 @@ func (ns *nurseryStore) CribToKinder(bby *babyOutput) error { // We successfully located an existing height chan // bucket at this babyOutput's expiry height, proceed by // removing it from the index. - err := hghtChanBucketCltv.DeleteBucket(pfxOutputKey) + err := hghtChanBucketCltv.Delete(pfxOutputKey) if err != nil { return err } + + // Since we removed a crib output from the height index, + // we opportunistically prune the height bucket + // corresponding to the babyOutput's CLTV delay. This + // allows us to clean up any persistent state as outputs + // are progressed through the incubation process. + pruned, err := ns.pruneHeight(tx, bby.expiry) + if err != nil && err != ErrBucketNotEmpty { + return err + } else if err == nil && pruned { + utxnLog.Infof("Height bucket %d pruned", bby.expiry) + } } // Since we are moving this output from the crib bucket to the @@ -449,25 +375,7 @@ func (ns *nurseryStore) CribToKinder(bby *babyOutput) error { // height-channel bucket corresponding to its maturity height. // This informs the utxo nursery that it should attempt to spend // this output when the blockchain reaches the maturity height. - _, err = hghtChanBucketCsv.CreateBucketIfNotExists(pfxOutputKey) - if err != nil { - return err - } - - // Finally, since we removed a crib output from the height - // index, we opportunistically prune the height bucket - // corresponding to the babyOutput's CLTV delay. This allows us - // to clean up any persistent state as outputs are progressed - // through the incubation process. - err = ns.pruneHeight(tx, bby.expiry) - switch err { - case nil, ErrBucketDoesNotExist: - return nil - case ErrBucketNotEmpty: - return nil - default: - return err - } + return hghtChanBucketCsv.Put(pfxOutputKey, []byte{}) }) } @@ -550,7 +458,7 @@ func (ns *nurseryStore) PreschoolToKinder(kid *kidOutput) error { }) } -// AwardDiplomas accepts a list of kidOutputs in the kindergarten bucket, +// GraduateKinder accepts a list of kidOutputs in the kindergarten bucket, // removing their corresponding entries from the height and channel indexes. // If this method detects that all outputs for a particular contract have been // incubated, it returns the channel points that are ready to be marked as @@ -560,16 +468,8 @@ func (ns *nurseryStore) PreschoolToKinder(kid *kidOutput) error { // empty. // 2) Prune the channel bucket belonging to the kid's origin channel point, if // it is empty. -func (ns *nurseryStore) AwardDiplomas( - kids ...kidOutput) ([]wire.OutPoint, error) { +func (ns *nurseryStore) GraduateKinder(kids []kidOutput) error { - // As we iterate over the kids, we will build a list of the channels - // which have been pruned entirely from the nursery store. We will - // return this list to the caller, the utxo nursery, so that it can - // proceed to mark the channels as closed. - // TODO(conner): write list of closed channels to separate bucket so - // that they can be replayed on restart? - var possibleCloseSet = make(map[wire.OutPoint]struct{}) if err := ns.db.Update(func(tx *bolt.Tx) error { for _, kid := range kids { @@ -578,40 +478,29 @@ func (ns *nurseryStore) AwardDiplomas( outpoint := kid.OutPoint() chanPoint := kid.OriginChanPoint() - // Remove output from kindergarten bucket. - + // Construct the key under which the output is currently + // stored height and channel indexes. pfxOutputKey, err := prefixOutputKey(kndrPrefix, outpoint) if err != nil { return err } + // Load the height-channel bucket, remove this output, + // and attempt to prune the bucket if it empty. hghtChanBucket := ns.getHeightChanBucket(tx, confHeight, chanPoint) if hghtChanBucket != nil { - if err := hghtChanBucket.DeleteBucket(pfxOutputKey); err != nil { + if err := hghtChanBucket.Delete(pfxOutputKey); err != nil { return err } // Attempt to prune the height bucket matching the kid // output's confirmation height if it contains no active // outputs. - err := ns.pruneHeight(tx, confHeight) - switch err { - case ErrBucketNotEmpty: - // Bucket still has active outputs, proceed to - // prune channel bucket. - - case ErrBucketDoesNotExist: - // Bucket was previously pruned by another - // graduating output. - - case nil: - // Bucket was pruned successfully and no errors - // were encounter. - utxnLog.Infof("Height bucket %d pruned", confHeight) - - default: - // Unexpected database error. + pruned, err := ns.pruneHeight(tx, confHeight) + if err != nil && err != ErrBucketNotEmpty { return err + } else if err == nil && pruned { + utxnLog.Infof("Height bucket %d pruned", confHeight) } } @@ -620,6 +509,7 @@ func (ns *nurseryStore) AwardDiplomas( return err } + // Remove previous output with kindergarten prefix. if err := chanBucket.Delete(pfxOutputKey); err != nil { return err } @@ -632,104 +522,75 @@ func (ns *nurseryStore) AwardDiplomas( return err } + // Insert serialized output into channel bucket using + // kindergarten-prefixed key. err = chanBucket.Put(pfxOutputKey, gradBuffer.Bytes()) if err != nil { return err } - // If we've arrived here, we have encountered no - // database errors and a bucket was either successfully - // pruned or already has been. Thus it is safe to add it - // to our set of closed channels to be closed, since - // these may need to be replayed to ensure the channel - // database is aware that incubation has completed. - possibleCloseSet[*chanPoint] = struct{}{} } return nil }); err != nil { - return nil, err + return err } - // Convert our set of channels to be closed into a list. - possibleCloses := make([]wire.OutPoint, 0, len(possibleCloseSet)) - for chanPoint := range possibleCloseSet { - possibleCloses = append(possibleCloses, chanPoint) - } - - utxnLog.Infof("Possible channels to be marked fully closed: %v", - possibleCloses) - - return possibleCloses, nil + return nil } -// TryFinalizeClass accepts a block height as a parameter and purges its +// FinalizeHeight accepts a block height as a parameter and purges its // persistent state for all outputs at that height. During a restart, the utxo // nursery will begin it's recovery procedure from the next height that has // yet to be finalized. -func (ns *nurseryStore) TryFinalizeClass(height uint32) error { +func (ns *nurseryStore) FinalizeHeight(height uint32) error { return ns.db.Update(func(tx *bolt.Tx) error { - utxnLog.Infof("Attempting to finalize class at height %v", height) - lastHeight, err := ns.getNextLastFinalizedHeight(tx, height) - if err != nil { + if err := ns.deleteHeightBucket(tx, height); err != nil { return err } - utxnLog.Infof("Finalizing class at height %v", lastHeight) - return ns.putLastFinalizedHeight(tx, lastHeight) + return ns.putLastFinalizedHeight(tx, height) }) } -// State Bucket Enumeration. - -// FetchCribs returns a list of babyOutputs in the crib bucket whose CLTV +// FetchClass returns a list of babyOutputs in the crib bucket whose CLTV // delay expires at the provided block height. -func (ns *nurseryStore) FetchCribs(height uint32) ([]babyOutput, error) { - // Construct a list of all babyOutputs that need TLC at the provided - // block height. +func (ns *nurseryStore) FetchClass(height uint32) ([]kidOutput, []babyOutput, error) { + // Construct list of all crib and kindergarten outputs that need TLC at + // the provided block height. + var kids []kidOutput var babies []babyOutput - if err := ns.forEachHeightPrefix(cribPrefix, height, - func(buf []byte) error { + if err := ns.db.View(func(tx *bolt.Tx) error { + if err := ns.forEachHeightPrefix(tx, cribPrefix, height, func(buf []byte) error { - // We will attempt to deserialize all outputs stored - // with the crib prefix into babyOutputs, since this is - // the expected type that would have been serialized - // previously. - var bby babyOutput - if err := bby.Decode(bytes.NewReader(buf)); err != nil { + // We will attempt to deserialize all outputs + // stored with the crib prefix into babyOutputs, + // since this is the expected type that would + // have been serialized previously. + var baby babyOutput + babyReader := bytes.NewReader(buf) + if err := baby.Decode(babyReader); err != nil { return err } // Append the deserialized object to our list of // babyOutputs. - babies = append(babies, bby) + babies = append(babies, baby) return nil }); err != nil { - return nil, err - } - - return babies, nil -} - -// FetchKindergartens returns a list of kidOutputs in the kindergarten bucket -// whose CSV delay expires at the provided block height. -func (ns *nurseryStore) FetchKindergartens(height uint32) ([]kidOutput, error) { - // Construct a list of all kidOutputs that mature at the provided block - // height. - utxnLog.Infof("Fetching kinders") - var kids []kidOutput - if err := ns.forEachHeightPrefix(kndrPrefix, height, - func(buf []byte) error { - utxnLog.Infof("Inside kinder") + return err + } + return ns.forEachHeightPrefix(tx, kndrPrefix, height, func(buf []byte) error { // We will attempt to deserialize all outputs stored // with the kindergarten prefix into kidOutputs, since // this is the expected type that would have been // serialized previously. var kid kidOutput - if err := kid.Decode(bytes.NewReader(buf)); err != nil { + kidReader := bytes.NewReader(buf) + if err := kid.Decode(kidReader); err != nil { return err } @@ -739,12 +600,13 @@ func (ns *nurseryStore) FetchKindergartens(height uint32) ([]kidOutput, error) { return nil - }); err != nil { - return nil, err - } - utxnLog.Infof("Returning kinders") + }) - return kids, nil + }); err != nil { + return nil, nil, err + } + + return kids, babies, nil } // FetchPreschools returns a list of all outputs currently stored in the @@ -827,8 +689,6 @@ func (ns *nurseryStore) FetchPreschools() ([]kidOutput, error) { return kids, nil } -// Channel Output Enumberation. - // ForChanOutputs iterates over all outputs being incubated for a particular // channel point. This method accepts a callback that allows the caller to // process each key-value pair. The key will be a prefixed outpoint, and the @@ -843,150 +703,158 @@ func (ns *nurseryStore) ForChanOutputs(chanPoint *wire.OutPoint, return ns.forChanOutputs(tx, chanPoint, callback) }) } -func (ns *nurseryStore) forChanOutputs(tx *bolt.Tx, chanPoint *wire.OutPoint, - callback func([]byte, []byte) error) error { - chanBucket := ns.getChannelBucket(tx, chanPoint) - if chanBucket == nil { - return ErrContractNotFound +// errImmatureChannel signals that not all outputs in a channel bucket have +// graduated. +var errImmatureChannel = errors.New("channel has non-graduated outputs") + +// IsMatureChannel determines the whether or not all of the outputs in a +// particular channel bucket have been marked as graduated. +func (ns *nurseryStore) IsMatureChannel(chanPoint *wire.OutPoint) (bool, error) { + if err := ns.db.View(func(tx *bolt.Tx) error { + // Iterate over the contents of the channel bucket, computing + // both total number of outputs, and those that have the grad + // prefix. + return ns.forChanOutputs(tx, chanPoint, func(pfxKey, _ []byte) error { + if string(pfxKey[:4]) != string(gradPrefix) { + utxnLog.Infof("Found non-graduated output: %x", pfxKey) + return errImmatureChannel + } + return nil + }) + + }); err != nil && err != errImmatureChannel { + return false, err + } else { + return err == nil, nil } - return chanBucket.ForEach(callback) } -func (ns *nurseryStore) IsMatureChannel(chanPoint *wire.OutPoint) (bool, error) { - var isMature bool - if err := ns.db.View(func(tx *bolt.Tx) error { - var nOutputs, nGrads int - if err := ns.forChanOutputs(tx, chanPoint, func(pfxKey, _ []byte) error { - // Count total and number of grad outputs - if string(pfxKey[:4]) == string(gradPrefix) { - nGrads++ - } - nOutputs++ - +// RemoveChannel channel erases all entries from the channel bucket for the +// provided channel point. +func (ns *nurseryStore) RemoveChannel(chanPoint *wire.OutPoint) error { + return ns.db.Update(func(tx *bolt.Tx) error { + // Retrieve the existing chain bucket for this nursery store. + chainBucket := tx.Bucket(ns.pfxChainKey) + if chainBucket == nil { return nil + } - }); err != nil { + // Retrieve the channel index stored in the chain bucket. + chanIndex := chainBucket.Bucket(channelIndexKey) + if chanIndex == nil { + return nil + } + + // Serialize the provided channel point, such that we can delete + // the mature channel bucket. + var chanBuffer bytes.Buffer + if err := writeOutpoint(&chanBuffer, chanPoint); err != nil { return err } - utxnLog.Infof("Found %d graduated outputs out of %d", nGrads, nOutputs) - // Channel is mature if all outputs are graduated. - if nGrads == nOutputs { - isMature = true + err := chanIndex.DeleteBucket(chanBuffer.Bytes()) + if err != nil { + return err } return nil - }); err != nil { - return false, err - } - - return isMature, nil + }) } -// The Point of No Return. - // LastFinalizedHeight returns the last block height for which the nursery // store has purged all persistent state. This occurs after a fixed interval // for reorg safety. func (ns *nurseryStore) LastFinalizedHeight() (uint32, error) { var lastFinalizedHeight uint32 err := ns.db.View(func(tx *bolt.Tx) error { - lastHeight, err := ns.getLastFinalizedHeight(tx) - if err != nil { - return err - } - - lastFinalizedHeight = lastHeight - - return nil + var err error + lastFinalizedHeight, err = ns.getLastFinalizedHeight(tx) + return err }) + return lastFinalizedHeight, err } -// getLastFinalizedHeight is a helper method that retrieves the last height for -// which the database finalized its persistent state. -func (ns *nurseryStore) getLastFinalizedHeight(tx *bolt.Tx) (uint32, error) { - // Retrieve the chain bucket associated with the given nursery store. - chainBucket := tx.Bucket(ns.pfxChainKey) - if chainBucket == nil { - return 0, nil - } +// Helper Methods - // Lookup the last finalized height in the top-level chain bucket. - heightBytes := chainBucket.Get(lastFinalizedHeightKey) - - // If the resulting bytes are not sized like a uint32, then we have - // never finalized, so we return 0. - if len(heightBytes) != 4 { - return 0, nil - } - - // Otherwise, parse the bytes and return the last finalized height. - return byteOrder.Uint32(heightBytes), nil -} - -func (ns *nurseryStore) getNextLastFinalizedHeight(tx *bolt.Tx, height uint32) (uint32, error) { - // Retrieve the previous last finalized height. - lastHeight, err := ns.getLastFinalizedHeight(tx) - if err != nil { - return 0, err - } - - // TODO(conner): add lower bound after which all state is purged. - - // Retrieve the existing chain bucket for this nursery store. - chainBucket := tx.Bucket(ns.pfxChainKey) - if chainBucket == nil { - return lastHeight, nil - } - - // Retrieve the existing height index. - hghtIndex := chainBucket.Bucket(heightIndexKey) - if hghtIndex == nil { - return lastHeight, nil - } - - for curHeight := lastHeight + 1; curHeight <= height; curHeight++ { - var curHeightBytes [4]byte - byteOrder.PutUint32(curHeightBytes[:], curHeight) - - hghtBucket := hghtIndex.Bucket(curHeightBytes[:]) - if hghtBucket == nil { - continue - } - - nChildren, err := ns.numChildrenInBucket(hghtBucket) - if err != nil { - return 0, err - } - - if nChildren > 0 { - return curHeight - 1, nil - } - } - - return height, nil -} - -// pubLastFinalizedHeight is a helper method that writes the provided height -// under the last finalized height key. -func (ns *nurseryStore) putLastFinalizedHeight(tx *bolt.Tx, - height uint32) error { - - // Ensure that the chain bucket for this nursery store exists. - chainBucket, err := tx.CreateBucketIfNotExists(ns.pfxChainKey) +// enterCrib accepts a new htlc output that the nursery will incubate through +// its two-stage process of sweeping funds back to the user's wallet. These +// outputs are persisted in the nursery store's crib bucket, and will be +// revisited after the output's CLTV has expired. +func (ns *nurseryStore) enterCrib(tx *bolt.Tx, baby *babyOutput) error { + // First, retrieve or create the channel bucket corresponding to the + // baby output's origin channel point. + chanPoint := baby.OriginChanPoint() + chanBucket, err := ns.createChannelBucket(tx, chanPoint) if err != nil { return err } - // Serialize the provided last-finalized height, and store it in the - // top-level chain bucket for this nursery store. - var lastHeightBytes [4]byte - byteOrder.PutUint32(lastHeightBytes[:], height) + // Next, retrieve or create the height-channel bucket located in the + // height bucket corresponding to the baby output's CLTV expiry height. + hghtChanBucket, err := ns.createHeightChanBucket(tx, + baby.expiry, chanPoint) + if err != nil { + return err + } - return chainBucket.Put(lastFinalizedHeightKey, lastHeightBytes[:]) + // Since we are inserting this output into the crib bucket, we create a + // key that prefixes the baby output's outpoint with the crib prefix. + pfxOutputKey, err := prefixOutputKey(cribPrefix, baby.OutPoint()) + if err != nil { + return err + } + + // Serialize the baby output so that it can be written to the underlying + // key-value store. + var babyBuffer bytes.Buffer + if err := baby.Encode(&babyBuffer); err != nil { + return err + } + babyBytes := babyBuffer.Bytes() + + // Now, insert the serialized output into its channel bucket under the + // prefixed key created above. + if err := chanBucket.Put(pfxOutputKey, babyBytes); err != nil { + return err + } + + // Finally, create a corresponding bucket in the height-channel bucket + // for this crib output. The existence of this bucket indicates that the + // serialized output can be retrieved from the channel bucket using the + // same prefix key. + return hghtChanBucket.Put(pfxOutputKey, []byte{}) +} + +// enterPreschool accepts a new commitment output that the nursery will incubate +// through a single stage before sweeping. Outputs are stored in the preschool +// bucket until the commitment transaction has been confirmed, at which point +// they will be moved to the kindergarten bucket. +func (ns *nurseryStore) enterPreschool(tx *bolt.Tx, kid *kidOutput) error { + // First, retrieve or create the channel bucket corresponding to the + // baby output's origin channel point. + chanPoint := kid.OriginChanPoint() + chanBucket, err := ns.createChannelBucket(tx, chanPoint) + if err != nil { + return err + } + + // Since the babyOutput is being inserted into the preschool bucket, we + // create a key that prefixes its outpoint with the preschool prefix. + pfxOutputKey, err := prefixOutputKey(psclPrefix, kid.OutPoint()) + if err != nil { + return err + } + + // Serialize the kidOutput and insert it into the channel bucket. + var kidBuffer bytes.Buffer + if err := kid.Encode(&kidBuffer); err != nil { + return err + } + + return chanBucket.Put(pfxOutputKey, kidBuffer.Bytes()) } // createChannelBucket creates or retrieves a channel bucket for the provided @@ -1074,22 +942,22 @@ func (ns *nurseryStore) createHeightBucket(tx *bolt.Tx, return hghtIndex.CreateBucketIfNotExists(heightBytes[:]) } -// getHeightBucket retrieves an existing height bucket from the nursery store, +// getHeightBucketPath retrieves an existing height bucket from the nursery store, // using the provided block height. If the bucket does not exist, or any bucket // along its path does not exist, a nil value is returned. -func (ns *nurseryStore) getHeightBucket(tx *bolt.Tx, - height uint32) *bolt.Bucket { +func (ns *nurseryStore) getHeightBucketPath(tx *bolt.Tx, + height uint32) (*bolt.Bucket, *bolt.Bucket, *bolt.Bucket) { // Retrieve the existing chain bucket for this nursery store. chainBucket := tx.Bucket(ns.pfxChainKey) if chainBucket == nil { - return nil + return nil, nil, nil } // Retrieve the existing channel index. hghtIndex := chainBucket.Bucket(heightIndexKey) if hghtIndex == nil { - return nil + return nil, nil, nil } // Serialize the provided block height and return the bucket matching @@ -1097,7 +965,47 @@ func (ns *nurseryStore) getHeightBucket(tx *bolt.Tx, var heightBytes [4]byte byteOrder.PutUint32(heightBytes[:], height) - return hghtIndex.Bucket(heightBytes[:]) + return chainBucket, hghtIndex, hghtIndex.Bucket(heightBytes[:]) +} + +// getHeightBucket retrieves an existing height bucket from the nursery store, +// using the provided block height. If the bucket does not exist, or any bucket +// along its path does not exist, a nil value is returned. +func (ns *nurseryStore) getHeightBucket(tx *bolt.Tx, + height uint32) *bolt.Bucket { + _, _, hghtBucket := ns.getHeightBucketPath(tx, height) + + return hghtBucket +} + +// deleteHeightBucket ensures that the height bucket at the provided index is +// purged from the nursery store. +func (ns *nurseryStore) deleteHeightBucket(tx *bolt.Tx, height uint32) error { + // Ensure that the chain bucket for this nursery store exists. + chainBucket := tx.Bucket(ns.pfxChainKey) + if chainBucket == nil { + return nil + } + + // Ensure that the height index has been properly initialized for this + // chain. + hghtIndex := chainBucket.Bucket(heightIndexKey) + if hghtIndex == nil { + return nil + } + + // Serialize the provided height, as this will form the name of the + // bucket. + var heightBytes [4]byte + byteOrder.PutUint32(heightBytes[:], height) + + // Finally, delete the bucket in question. + err := hghtIndex.DeleteBucket(heightBytes[:]) + if err != nil && err != bolt.ErrBucketNotFound { + return err + } + + return nil } // createHeightChanBucket creates or retrieves an existing height-channel bucket @@ -1156,108 +1064,149 @@ func (ns *nurseryStore) getHeightChanBucket(tx *bolt.Tx, // enumerate crib and kindergarten outputs at a particular height. The callback // is invoked with serialized bytes retrieved for each output of interest, // allowing the caller to deserialize them into the appropriate type. -func (ns *nurseryStore) forEachHeightPrefix(prefix []byte, height uint32, - callback func([]byte) error) error { - - return ns.db.View(func(tx *bolt.Tx) error { - // Start by retrieving the height bucket corresponding to the - // provided block height. - hghtBucket := ns.getHeightBucket(tx, height) - if hghtBucket == nil { - return nil - } - - // Using the height bucket as a starting point, we will traverse - // its entire two-tier directory structure, and filter for - // outputs that have the provided prefix. The first layer of the - // height bucket contains buckets identified by a channel point, - // thus we first create list of channels contained in this - // height bucket. - var channelsAtHeight [][]byte - if err := hghtBucket.ForEach(func(chanBytes, _ []byte) error { - channelsAtHeight = append(channelsAtHeight, chanBytes) - return nil - }); err != nil { - return err - } - - // As we enumerate the outputs referenced in this height bucket, - // we will need to load the serialized value of each output, - // which is ultimately stored its respective channel bucket. To - // do so, we first load the top-level chain bucket, which should - // already be created if we are at this point. - chainBucket := tx.Bucket(ns.pfxChainKey) - if chainBucket == nil { - return nil - } - - // Additionally, grab the chain index, which we will facilitate - // queries for each of the channel buckets of each of the - // channels in the list we assembled above. - chanIndex := chainBucket.Bucket(channelIndexKey) - if chanIndex == nil { - return nil - } - - // Now, we are ready to enumerate all outputs with the desired - // prefix t this block height. We do so by iterating over our - // list of channels at this height, filtering for outputs in - // each height-channel bucket that begin with the given prefix, - // and then retrieving the serialized outputs from the - // appropriate channel bucket. - for _, chanBytes := range channelsAtHeight { - // Retrieve the height-channel bucket for this channel, - // which holds a sub-bucket for all outputs maturing at - // this height. - hghtChanBucket := hghtBucket.Bucket(chanBytes) - if hghtChanBucket == nil { - continue - } - - // Load the appropriate channel bucket from the channel - // index, this will allow us to retrieve the individual - // serialized outputs. - chanBucket := chanIndex.Bucket(chanBytes) - if chanBucket == nil { - continue - } - - // Since all of the outputs of interest will start with - // the same prefix, we will perform a prefix scan of the - // buckets contained in the height-channel bucket, - // efficiently enumerating the desired outputs. - c := hghtChanBucket.Cursor() - - // Seek to and iterate over all entries starting with - // the given prefix. - var pfxOutputKey, _ = c.Seek(prefix) - for bytes.HasPrefix(pfxOutputKey, prefix) { - - // Use the prefix output key emitted from our - // scan to load the serialized babyOutput from - // the appropriate channel bucket. - outputBytes := chanBucket.Get(pfxOutputKey) - if outputBytes == nil { - pfxOutputKey, _ = c.Next() - continue - } - - // Present the serialized bytes to our call back - // function, which is responsible for - // deserializing the bytes into the appropriate - // type. - if err := callback(outputBytes); err != nil { - return err - } - - // Lastly, advance our prefix output key for the - // next iteration. - pfxOutputKey, _ = c.Next() - } - } +func (ns *nurseryStore) forEachHeightPrefix(tx *bolt.Tx, prefix []byte, + height uint32, callback func([]byte) error) error { + // Start by retrieving the height bucket corresponding to the provided + // block height. + chainBucket, _, hghtBucket := ns.getHeightBucketPath(tx, height) + if hghtBucket == nil { return nil - }) + } + + // Using the height bucket as a starting point, we will traverse its + // entire two-tier directory structure, and filter for outputs that have + // the provided prefix. The first layer of the height bucket contains + // buckets identified by a channel point, thus we first create list of + // channels contained in this height bucket. + var channelsAtHeight [][]byte + if err := hghtBucket.ForEach(func(chanBytes, _ []byte) error { + channelsAtHeight = append(channelsAtHeight, chanBytes) + return nil + }); err != nil { + return err + } + + // Additionally, grab the chain index, which we will facilitate queries + // for each of the channel buckets of each of the channels in the list + // we assembled above. + chanIndex := chainBucket.Bucket(channelIndexKey) + if chanIndex == nil { + return errors.New("unable to retrieve channel index") + } + + // Now, we are ready to enumerate all outputs with the desired prefix at + // this block height. We do so by iterating over our list of channels at + // this height, filtering for outputs in each height-channel bucket that + // begin with the given prefix, and then retrieving the serialized + // outputs from the appropriate channel bucket. + for _, chanBytes := range channelsAtHeight { + // Retrieve the height-channel bucket for this channel, which + // holds a sub-bucket for all outputs maturing at this height. + hghtChanBucket := hghtBucket.Bucket(chanBytes) + if hghtChanBucket == nil { + return errors.New("unable to retrieve height-channel bucket") + } + + // Load the appropriate channel bucket from the channel index, + // this will allow us to retrieve the individual serialized + // outputs. + chanBucket := chanIndex.Bucket(chanBytes) + if chanBucket == nil { + return errors.New("unable to retrieve channel bucket") + } + + // Since all of the outputs of interest will start with the same + // prefix, we will perform a prefix scan of the buckets + // contained in the height-channel bucket, efficiently + // enumerating the desired outputs. + c := hghtChanBucket.Cursor() + + // Seek to and iterate over all entries starting with the given + // prefix. + pfxOutputKey, _ := c.Seek(prefix) + for bytes.HasPrefix(pfxOutputKey, prefix) { + + // Use the prefix output key emitted from our scan to + // load the serialized babyOutput from the appropriate + // channel bucket. + outputBytes := chanBucket.Get(pfxOutputKey) + if outputBytes == nil { + return errors.New("unable to retrieve output") + } + + // Present the serialized bytes to our call back + // function, which is responsible for deserializing the + // bytes into the appropriate type. + if err := callback(outputBytes); err != nil { + return err + } + + // Lastly, advance our prefix output key for the next + // iteration. + pfxOutputKey, _ = c.Next() + } + } + + return nil +} + +// forChanOutputs enumerates the outputs contained in a channel bucket to the +// provided callback. The callback accepts a key-value pair of byte slices +// corresponding to the prefixed-output key and the serialized output, +// respectively. +func (ns *nurseryStore) forChanOutputs(tx *bolt.Tx, chanPoint *wire.OutPoint, + callback func([]byte, []byte) error) error { + + chanBucket := ns.getChannelBucket(tx, chanPoint) + if chanBucket == nil { + return ErrContractNotFound + } + + return chanBucket.ForEach(callback) +} + +// getLastFinalizedHeight is a helper method that retrieves the last height for +// which the database finalized its persistent state. +func (ns *nurseryStore) getLastFinalizedHeight(tx *bolt.Tx) (uint32, error) { + // Retrieve the chain bucket associated with the given nursery store. + chainBucket := tx.Bucket(ns.pfxChainKey) + if chainBucket == nil { + return 0, nil + } + + // Lookup the last finalized height in the top-level chain bucket. + heightBytes := chainBucket.Get(lastFinalizedHeightKey) + + // If the resulting bytes are not sized like a uint32, then we have + // never finalized, so we return 0. + if len(heightBytes) != 4 { + return 0, nil + } + + // Otherwise, parse the bytes and return the last finalized height. + return byteOrder.Uint32(heightBytes), nil +} + +// pubLastFinalizedHeight is a helper method that writes the provided height +// under the last finalized height key. +func (ns *nurseryStore) putLastFinalizedHeight(tx *bolt.Tx, + height uint32) error { + + // Ensure that the chain bucket for this nursery store exists. + chainBucket, err := tx.CreateBucketIfNotExists(ns.pfxChainKey) + if err != nil { + return err + } + + // TODO(conner): purge all state below reorg depth. + + // Serialize the provided last-finalized height, and store it in the + // top-level chain bucket for this nursery store. + var lastHeightBytes [4]byte + byteOrder.PutUint32(lastHeightBytes[:], height) + + return chainBucket.Put(lastFinalizedHeightKey, lastHeightBytes[:]) } var ( @@ -1270,206 +1219,90 @@ var ( ErrBucketNotEmpty = errors.New("bucket is not empty, cannot be pruned") ) -// deleteAndPruneHeight removes an output from a channel bucket matching the -// provided channel point. The output is assumed top be in the kindergarten -// bucket, since pruning should never occur for any other type of output. If -// after deletion, the channel bucket is empty, this method will attempt to -// delete the bucket as well. -// NOTE: This method returns two concrete errors apart from those returned by -// the underlying database: ErrBucketDoesNotExist and ErrBucketNotEmpty. These -// should be handled in the context of the caller so as they may be benign -// depending on context. Errors returned other than these two should be -// interpreted as database errors. -func (ns *nurseryStore) deleteAndPruneChannel(tx *bolt.Tx, - chanPoint, outpoint *wire.OutPoint) error { - - // Retrieve the existing chain bucket for this nursery store. - chainBucket := tx.Bucket(ns.pfxChainKey) - if chainBucket == nil { - return nil - } - - // Retrieve the channel index stored in the chain bucket. - chanIndex := chainBucket.Bucket(channelIndexKey) - if chanIndex == nil { - return nil - } - - // Serialize the provided channel point, such that we can retrieve the - // desired channel bucket. - var chanBuffer bytes.Buffer - if err := writeOutpoint(&chanBuffer, chanPoint); err != nil { - return err - } - chanBytes := chanBuffer.Bytes() - - // Retrieve the existing channel bucket. If none exists, then our job is - // complete and it is safe to return nil. - chanBucket := chanIndex.Bucket(chanBytes) - if chanBucket == nil { - return nil - } - - // Otherwise, the bucket still exists. Serialize the outpoint that needs - // deletion, prefixing the key with kindergarten prefix. Since all - // outputs eventually make their way to becoming kindergarten outputs, - // we can safely assume that they will be stored with a kindergarten - // prefix. - pfxOutputBytes, err := prefixOutputKey(kndrPrefix, outpoint) - if err != nil { - return err - } - - // Remove the output in question using the kindergarten-prefixed key we - // generated above. - if err := chanBucket.Delete(pfxOutputBytes); err != nil { - return err - } - - // Finally, now that the outpoint has been removed from this channel - // bucket, try to remove the channel bucket altogether if it is now - // empty. - return ns.removeBucketIfEmpty(chanIndex, chanBytes) -} - -func (ns *nurseryStore) RemoveChannel(chanPoint *wire.OutPoint) error { - return ns.db.Update(func(tx *bolt.Tx) error { - // Retrieve the existing chain bucket for this nursery store. - chainBucket := tx.Bucket(ns.pfxChainKey) - if chainBucket == nil { - return nil - } - - // Retrieve the channel index stored in the chain bucket. - chanIndex := chainBucket.Bucket(channelIndexKey) - if chanIndex == nil { - return nil - } - - // Serialize the provided channel point, such that we can delete - // the mature channel bucket. - var chanBuffer bytes.Buffer - if err := writeOutpoint(&chanBuffer, chanPoint); err != nil { - return err - } - - return chanIndex.DeleteBucket(chanBuffer.Bytes()) - }) -} - -// pruneHeight -// NOTE: This method returns two concrete errors apart from those returned by -// the underlying database: ErrBucketDoesNotExist and ErrBucketNotEmpty. These -// should be handled in the context of the caller so as they may be benign -// depending on context. Errors returned other than these two should be -// interpreted as database errors. -func (ns *nurseryStore) pruneHeight(tx *bolt.Tx, height uint32) error { - - // Fetch the existing chain bucket for this nursery store. - chainBucket := tx.Bucket(ns.pfxChainKey) - if chainBucket == nil { - return nil - } - - // Load the existing height bucket for the height in question. - hghtIndex := chainBucket.Bucket(heightIndexKey) - if hghtIndex == nil { - return nil - } - - // Serialize the provided block height, such that it can be used as the - // key to locate the desired height bucket. - var heightBytes [4]byte - byteOrder.PutUint32(heightBytes[:], height) - - // Retrieve the height bucket using the serialized height as the bucket - // name. - hghtBucket := hghtIndex.Bucket(heightBytes[:]) +// pruneHeight removes the height bucket at the provided height if and only if +// all active outputs at this height have been removed from their respective +// height-channel buckets. +func (ns *nurseryStore) pruneHeight(tx *bolt.Tx, height uint32) (bool, error) { + // Fetch the existing height index and height bucket. + _, hghtIndex, hghtBucket := ns.getHeightBucketPath(tx, height) if hghtBucket == nil { - return ErrBucketDoesNotExist + return false, nil } - // Iterate over the contents of this height bucket, which is comprised - // of sub-buckets named after the channel points that need attention at + // TODO(conner): fix this comment // this block height. We will attempt to remove each one if they are // empty, keeping track of the number of height-channel buckets that // still have active outputs. var nActiveBuckets int if err := hghtBucket.ForEach(func(chanBytes, _ []byte) error { - // Attempt to each height-channel bucket from the height bucket // located above. - err := ns.removeBucketIfEmpty(hghtBucket, chanBytes) - switch err { - case nil: - // The height-channel bucket was removed successfully! - return nil - - case ErrBucketDoesNotExist: - // The height-channel bucket could not be located--no - // harm, no foul. - return nil - - case ErrBucketNotEmpty: - // The bucket still has active outputs at this height, - // increment our number of still active height-channel - // buckets. - nActiveBuckets++ - return nil - - default: - // Database error! + _, err := ns.removeBucketIfEmpty(hghtBucket, chanBytes) + if err != nil && err != ErrBucketNotEmpty { return err + } else if err == ErrBucketNotEmpty { + nActiveBuckets++ } + return nil + }); err != nil { - return err + return false, err } // If we located any height-channel buckets that still have active // outputs, it is unsafe to delete this height bucket. Signal this event // to the caller so that they can determine the appropriate action. if nActiveBuckets > 0 { - return ErrBucketNotEmpty + return false, ErrBucketNotEmpty } + // Serialize the provided block height, such that it can be used as the + // key to delete desired height bucket. + var heightBytes [4]byte + byteOrder.PutUint32(heightBytes[:], height) + // All of the height-channel buckets are empty or have been previously - // removed, proceed by attempting to remove the height bucket + // removed, proceed by removing the height bucket // altogether. - return ns.removeBucketIfEmpty(hghtIndex, heightBytes[:]) + if err := hghtIndex.DeleteBucket(heightBytes[:]); err != nil { + return false, err + } + + return true, nil } // removeBucketIfEmpty attempts to delete a bucket specified by name from the // provided parent bucket. -// NOTE: This method returns two concrete errors apart from those returned by -// the underlying database: ErrBucketDoesNotExist and ErrBucketNotEmpty. These -// should be handled in the context of the caller so as they may be benign -// depending on context. Errors returned other than these two should be -// interpreted as database errors. func (ns *nurseryStore) removeBucketIfEmpty(parent *bolt.Bucket, - bktName []byte) error { + bktName []byte) (bool, error) { // Attempt to fetch the named bucket from its parent. bkt := parent.Bucket(bktName) if bkt == nil { // No bucket was found, signal this to the caller. - return ErrBucketDoesNotExist + return false, nil } // The bucket exists, now compute how many children *it* has. nChildren, err := ns.numChildrenInBucket(bkt) if err != nil { - return err + return false, nil } // If the number of children is non-zero, alert the caller that the // named bucket is not being removed. if nChildren > 0 { - return ErrBucketNotEmpty + return false, nil } // Otherwise, remove the empty bucket from its parent. - return parent.DeleteBucket(bktName) + err = parent.DeleteBucket(bktName) + if err != nil { + return false, err + } + + return true, nil } // numChildrenInBucket computes the number of children contained in the given From ef81f0064c15091283eceac50e383c42acce9c0e Mon Sep 17 00:00:00 2001 From: Conner Fromknecht Date: Tue, 17 Oct 2017 17:58:49 -0700 Subject: [PATCH 4/6] nursery_store: tightens up height index pruning and formatting --- nursery_store.go | 387 ++++++++++++++++++++++++----------------------- 1 file changed, 200 insertions(+), 187 deletions(-) diff --git a/nursery_store.go b/nursery_store.go index 3fdc55984..64f560698 100644 --- a/nursery_store.go +++ b/nursery_store.go @@ -3,6 +3,7 @@ package main import ( "bytes" "errors" + "fmt" "github.com/boltdb/bolt" "github.com/lightningnetwork/lnd/channeldb" @@ -38,7 +39,6 @@ type NurseryStore interface { // channel indexes. If this method detects that all outputs for a // particular contract have been incubated, it returns the channel // points that are ready to be marked as fully closed. - // TODO: make this handle one output at a time? GraduateKinder([]kidOutput) error // FinalizeHeight accepts a block height as a parameter and purges its @@ -310,32 +310,10 @@ func (ns *nurseryStore) CribToKinder(bby *babyOutput) error { return err } - // Next, retrieve the height-channel bucket located in the - // height bucket corresponding to the baby output's CLTV expiry - // height. This bucket should always exist, but if it doesn't - // then we have nothing to clean up. - hghtChanBucketCltv := ns.getHeightChanBucket(tx, bby.expiry, - chanPoint) - if hghtChanBucketCltv != nil { - // We successfully located an existing height chan - // bucket at this babyOutput's expiry height, proceed by - // removing it from the index. - err := hghtChanBucketCltv.Delete(pfxOutputKey) - if err != nil { - return err - } - - // Since we removed a crib output from the height index, - // we opportunistically prune the height bucket - // corresponding to the babyOutput's CLTV delay. This - // allows us to clean up any persistent state as outputs - // are progressed through the incubation process. - pruned, err := ns.pruneHeight(tx, bby.expiry) - if err != nil && err != ErrBucketNotEmpty { - return err - } else if err == nil && pruned { - utxnLog.Infof("Height bucket %d pruned", bby.expiry) - } + err = ns.removeOutputFromHeight(tx, bby.expiry, chanPoint, + pfxOutputKey) + if err != nil { + return err } // Since we are moving this output from the crib bucket to the @@ -447,68 +425,46 @@ func (ns *nurseryStore) PreschoolToKinder(kid *kidOutput) error { return err } - // Finally, we touch a bucket in the height-channel created - // above. The bucket is named using a kindergarten prefixed - // key, signaling that this CSV delayed output will be ready to - // broadcast at the maturity height, after a brief period of - // incubation. - _, err = hghtChanBucket.CreateBucketIfNotExists(pfxOutputKey) - - return err + // Finally, we touch a key in the height-channel created above. + // The key is named using a kindergarten prefixed key, signaling + // that this CSV delayed output will be ready to broadcast at + // the maturity height, after a brief period of incubation. + return hghtChanBucket.Put(pfxOutputKey, []byte{}) }) } -// GraduateKinder accepts a list of kidOutputs in the kindergarten bucket, -// removing their corresponding entries from the height and channel indexes. -// If this method detects that all outputs for a particular contract have been -// incubated, it returns the channel points that are ready to be marked as -// fully closed. This method will iterate through the provided kidOutputs and do -// the following: -// 1) Prune the kid height bucket at the kid's confirmation height, if it is -// empty. -// 2) Prune the channel bucket belonging to the kid's origin channel point, if -// it is empty. +// GraduateKinder accepts a list of kidOutputs in the kindergarten bucket and +// marks them as graduated. This method also removes their corresponding entries +// from the height and channel indexes corresponding to their kindergarten +// status. func (ns *nurseryStore) GraduateKinder(kids []kidOutput) error { - if err := ns.db.Update(func(tx *bolt.Tx) error { for _, kid := range kids { confHeight := kid.ConfHeight() - outpoint := kid.OutPoint() chanPoint := kid.OriginChanPoint() // Construct the key under which the output is currently // stored height and channel indexes. - pfxOutputKey, err := prefixOutputKey(kndrPrefix, outpoint) + pfxOutputKey, err := prefixOutputKey(kndrPrefix, + outpoint) if err != nil { return err } - // Load the height-channel bucket, remove this output, - // and attempt to prune the bucket if it empty. - hghtChanBucket := ns.getHeightChanBucket(tx, confHeight, chanPoint) - if hghtChanBucket != nil { - if err := hghtChanBucket.Delete(pfxOutputKey); err != nil { - return err - } - - // Attempt to prune the height bucket matching the kid - // output's confirmation height if it contains no active - // outputs. - pruned, err := ns.pruneHeight(tx, confHeight) - if err != nil && err != ErrBucketNotEmpty { - return err - } else if err == nil && pruned { - utxnLog.Infof("Height bucket %d pruned", confHeight) - } - } - - chanBucket, err := ns.createChannelBucket(tx, chanPoint) + // Remove the grad output's entry in the height index. + err = ns.removeOutputFromHeight(tx, confHeight, chanPoint, + pfxOutputKey) if err != nil { return err } + chanBucket := ns.getChannelBucket(tx, chanPoint) + if chanBucket == nil { + return ErrContractNotFound + } + // Remove previous output with kindergarten prefix. if err := chanBucket.Delete(pfxOutputKey); err != nil { return err @@ -528,7 +484,6 @@ func (ns *nurseryStore) GraduateKinder(kids []kidOutput) error { if err != nil { return err } - } return nil @@ -555,52 +510,54 @@ func (ns *nurseryStore) FinalizeHeight(height uint32) error { // FetchClass returns a list of babyOutputs in the crib bucket whose CLTV // delay expires at the provided block height. -func (ns *nurseryStore) FetchClass(height uint32) ([]kidOutput, []babyOutput, error) { +func (ns *nurseryStore) FetchClass( + height uint32) ([]kidOutput, []babyOutput, error) { + // Construct list of all crib and kindergarten outputs that need TLC at // the provided block height. var kids []kidOutput var babies []babyOutput if err := ns.db.View(func(tx *bolt.Tx) error { - if err := ns.forEachHeightPrefix(tx, cribPrefix, height, func(buf []byte) error { + // Append each crib output to our list of babyOutputs. + if err := ns.forEachHeightPrefix(tx, cribPrefix, height, + func(buf []byte) error { - // We will attempt to deserialize all outputs - // stored with the crib prefix into babyOutputs, - // since this is the expected type that would - // have been serialized previously. - var baby babyOutput - babyReader := bytes.NewReader(buf) - if err := baby.Decode(babyReader); err != nil { - return err - } + // We will attempt to deserialize all outputs + // stored with the crib prefix into babyOutputs, + // since this is the expected type that would + // have been serialized previously. + var baby babyOutput + babyReader := bytes.NewReader(buf) + if err := baby.Decode(babyReader); err != nil { + return err + } - // Append the deserialized object to our list of - // babyOutputs. - babies = append(babies, baby) + babies = append(babies, baby) - return nil + return nil - }); err != nil { + }); err != nil { return err } - return ns.forEachHeightPrefix(tx, kndrPrefix, height, func(buf []byte) error { - // We will attempt to deserialize all outputs stored - // with the kindergarten prefix into kidOutputs, since - // this is the expected type that would have been - // serialized previously. - var kid kidOutput - kidReader := bytes.NewReader(buf) - if err := kid.Decode(kidReader); err != nil { - return err - } + // Append each kindergarten output to our list of kidOutputs. + return ns.forEachHeightPrefix(tx, kndrPrefix, height, + func(buf []byte) error { + // We will attempt to deserialize all outputs + // stored with the kindergarten prefix into + // kidOutputs, since this is the expected type + // that would have been serialized previously. + var kid kidOutput + kidReader := bytes.NewReader(buf) + if err := kid.Decode(kidReader); err != nil { + return err + } - // Append the deserialized object to our list of - // kidOutputs. - kids = append(kids, kid) + kids = append(kids, kid) - return nil + return nil - }) + }) }); err != nil { return nil, nil, err @@ -675,8 +632,6 @@ func (ns *nurseryStore) FetchPreschools() ([]kidOutput, error) { // preschool outputs. kids = append(kids, psclOutput) - // Advance to the subsequent key-value pair of - // the prefix scan. pfxOutputKey, kidBytes = c.Next() } } @@ -711,28 +666,31 @@ var errImmatureChannel = errors.New("channel has non-graduated outputs") // IsMatureChannel determines the whether or not all of the outputs in a // particular channel bucket have been marked as graduated. func (ns *nurseryStore) IsMatureChannel(chanPoint *wire.OutPoint) (bool, error) { - if err := ns.db.View(func(tx *bolt.Tx) error { + err := ns.db.View(func(tx *bolt.Tx) error { // Iterate over the contents of the channel bucket, computing // both total number of outputs, and those that have the grad // prefix. - return ns.forChanOutputs(tx, chanPoint, func(pfxKey, _ []byte) error { - if string(pfxKey[:4]) != string(gradPrefix) { - utxnLog.Infof("Found non-graduated output: %x", pfxKey) - return errImmatureChannel - } - return nil - }) + return ns.forChanOutputs(tx, chanPoint, + func(pfxKey, _ []byte) error { + if string(pfxKey[:4]) != string(gradPrefix) { + utxnLog.Infof("Found non-graduated "+ + "output: %x", pfxKey) + return errImmatureChannel + } + return nil + }) - }); err != nil && err != errImmatureChannel { + }) + if err != nil && err != errImmatureChannel { return false, err - } else { - return err == nil, nil } + return err == nil, nil } // RemoveChannel channel erases all entries from the channel bucket for the // provided channel point. +// NOTE: The channel's entries in the height index are assumed to be removed. func (ns *nurseryStore) RemoveChannel(chanPoint *wire.OutPoint) error { return ns.db.Update(func(tx *bolt.Tx) error { // Retrieve the existing chain bucket for this nursery store. @@ -753,13 +711,42 @@ func (ns *nurseryStore) RemoveChannel(chanPoint *wire.OutPoint) error { if err := writeOutpoint(&chanBuffer, chanPoint); err != nil { return err } + chanBytes := chanBuffer.Bytes() - err := chanIndex.DeleteBucket(chanBuffer.Bytes()) + utxnLog.Infof("Pruning and removing channel: %v", chanPoint) + + err := ns.forChanOutputs(tx, chanPoint, func(k, v []byte) error { + if string(k[:4]) != string(gradPrefix) { + return errors.New("expected grad output") + } + + // Construct a kindergarten prefixed key, since this + // would have been the preceding state for a grad + // output. + kndrKey := make([]byte, len(k)) + copy(kndrKey, k) + copy(kndrKey[:4], kndrPrefix) + + // Decode each to retrieve the output's maturity height. + var kid kidOutput + if err := kid.Decode(bytes.NewReader(v)); err != nil { + return err + } + + maturityHeight := kid.ConfHeight() + kid.BlocksToMaturity() + + hghtBucket := ns.getHeightBucket(tx, maturityHeight) + if hghtBucket == nil { + return nil + } + + return removeBucketIfExists(hghtBucket, chanBytes) + }) if err != nil { return err } - return nil + return removeBucketIfExists(chanIndex, chanBytes) }) } @@ -942,9 +929,9 @@ func (ns *nurseryStore) createHeightBucket(tx *bolt.Tx, return hghtIndex.CreateBucketIfNotExists(heightBytes[:]) } -// getHeightBucketPath retrieves an existing height bucket from the nursery store, -// using the provided block height. If the bucket does not exist, or any bucket -// along its path does not exist, a nil value is returned. +// getHeightBucketPath retrieves an existing height bucket from the nursery +// store, using the provided block height. If the bucket does not exist, or any +// bucket along its path does not exist, a nil value is returned. func (ns *nurseryStore) getHeightBucketPath(tx *bolt.Tx, height uint32) (*bolt.Bucket, *bolt.Bucket, *bolt.Bucket) { @@ -981,18 +968,13 @@ func (ns *nurseryStore) getHeightBucket(tx *bolt.Tx, // deleteHeightBucket ensures that the height bucket at the provided index is // purged from the nursery store. func (ns *nurseryStore) deleteHeightBucket(tx *bolt.Tx, height uint32) error { - // Ensure that the chain bucket for this nursery store exists. - chainBucket := tx.Bucket(ns.pfxChainKey) - if chainBucket == nil { + // Ensure that the height bucket already exists. + _, hghtIndex, hghtBucket := ns.getHeightBucketPath(tx, height) + if hghtBucket == nil { return nil } - // Ensure that the height index has been properly initialized for this - // chain. - hghtIndex := chainBucket.Bucket(heightIndexKey) - if hghtIndex == nil { - return nil - } + utxnLog.Infof("Deleting height bucket %d", height) // Serialize the provided height, as this will form the name of the // bucket. @@ -1000,12 +982,7 @@ func (ns *nurseryStore) deleteHeightBucket(tx *bolt.Tx, height uint32) error { byteOrder.PutUint32(heightBytes[:], height) // Finally, delete the bucket in question. - err := hghtIndex.DeleteBucket(heightBytes[:]) - if err != nil && err != bolt.ErrBucketNotFound { - return err - } - - return nil + return removeBucketIfExists(hghtIndex, heightBytes[:]) } // createHeightChanBucket creates or retrieves an existing height-channel bucket @@ -1105,7 +1082,8 @@ func (ns *nurseryStore) forEachHeightPrefix(tx *bolt.Tx, prefix []byte, // holds a sub-bucket for all outputs maturing at this height. hghtChanBucket := hghtBucket.Bucket(chanBytes) if hghtChanBucket == nil { - return errors.New("unable to retrieve height-channel bucket") + return fmt.Errorf("unable to retrieve height-channel "+ + "bucket at height %d for %x", height, chanBytes) } // Load the appropriate channel bucket from the channel index, @@ -1113,7 +1091,7 @@ func (ns *nurseryStore) forEachHeightPrefix(tx *bolt.Tx, prefix []byte, // outputs. chanBucket := chanIndex.Bucket(chanBytes) if chanBucket == nil { - return errors.New("unable to retrieve channel bucket") + return fmt.Errorf("unable to retrieve channel bucket: '%x'", chanBytes) } // Since all of the outputs of interest will start with the same @@ -1199,8 +1177,6 @@ func (ns *nurseryStore) putLastFinalizedHeight(tx *bolt.Tx, return err } - // TODO(conner): purge all state below reorg depth. - // Serialize the provided last-finalized height, and store it in the // top-level chain bucket for this nursery store. var lastHeightBytes [4]byte @@ -1219,6 +1195,60 @@ var ( ErrBucketNotEmpty = errors.New("bucket is not empty, cannot be pruned") ) +// removeOutputFromHeight will delete the given output from the specified +// height-channel bucket, and attempt to prune the upstream directories if they +// are empty. +func (ns *nurseryStore) removeOutputFromHeight(tx *bolt.Tx, height uint32, + chanPoint *wire.OutPoint, pfxKey []byte) error { + + // Retrieve the height-channel bucket and delete the prefixed output. + hghtChanBucket := ns.getHeightChanBucket(tx, height, chanPoint) + if hghtChanBucket == nil { + // Height-channel bucket already removed. + return nil + } + + // Try to delete the prefixed output key if it still exists. The output + // may have already been removed after confirmation, but a final pass is + // done when removing a channel as well. + if hghtChanBucket.Get(pfxKey) != nil { + if err := hghtChanBucket.Delete(pfxKey); err != nil { + return err + } + } + + // Retrieve the height bucket that contains the height-channel bucket. + hghtBucket := ns.getHeightBucket(tx, height) + if hghtBucket == nil { + return errors.New("height bucket not found") + } + + var chanBuffer bytes.Buffer + if err := writeOutpoint(&chanBuffer, chanPoint); err != nil { + return err + } + + // Try to remove the channel-height bucket if it this was the last + // output in the bucket. + err := removeBucketIfEmpty(hghtBucket, chanBuffer.Bytes()) + if err != nil && err != ErrBucketNotEmpty { + return err + } else if err == ErrBucketNotEmpty { + return nil + } + + // Attempt to prune the height bucket matching the kid output's + // confirmation height in case that was the last height-chan bucket. + pruned, err := ns.pruneHeight(tx, height) + if err != nil && err != ErrBucketNotEmpty { + return err + } else if err == nil && pruned { + utxnLog.Infof("Height bucket %d pruned", height) + } + + return nil +} + // pruneHeight removes the height bucket at the provided height if and only if // all active outputs at this height have been removed from their respective // height-channel buckets. @@ -1228,35 +1258,26 @@ func (ns *nurseryStore) pruneHeight(tx *bolt.Tx, height uint32) (bool, error) { if hghtBucket == nil { return false, nil } + utxnLog.Infof("pruning height %d", height) - // TODO(conner): fix this comment - // this block height. We will attempt to remove each one if they are - // empty, keeping track of the number of height-channel buckets that - // still have active outputs. - var nActiveBuckets int + // Iterate over all channels stored at this block height. We will + // attempt to remove each one if they are empty, keeping track of the + // number of height-channel buckets that still have active outputs. if err := hghtBucket.ForEach(func(chanBytes, _ []byte) error { // Attempt to each height-channel bucket from the height bucket // located above. - _, err := ns.removeBucketIfEmpty(hghtBucket, chanBytes) - if err != nil && err != ErrBucketNotEmpty { - return err - } else if err == ErrBucketNotEmpty { - nActiveBuckets++ + + hghtChanBucket := hghtBucket.Bucket(chanBytes) + if hghtChanBucket == nil { + return errors.New("unable to find height-channel bucket") } - return nil + return isBucketEmpty(hghtChanBucket) }); err != nil { return false, err } - // If we located any height-channel buckets that still have active - // outputs, it is unsafe to delete this height bucket. Signal this event - // to the caller so that they can determine the appropriate action. - if nActiveBuckets > 0 { - return false, ErrBucketNotEmpty - } - // Serialize the provided block height, such that it can be used as the // key to delete desired height bucket. var heightBytes [4]byte @@ -1265,7 +1286,7 @@ func (ns *nurseryStore) pruneHeight(tx *bolt.Tx, height uint32) (bool, error) { // All of the height-channel buckets are empty or have been previously // removed, proceed by removing the height bucket // altogether. - if err := hghtIndex.DeleteBucket(heightBytes[:]); err != nil { + if err := removeBucketIfExists(hghtIndex, heightBytes[:]); err != nil { return false, err } @@ -1274,49 +1295,41 @@ func (ns *nurseryStore) pruneHeight(tx *bolt.Tx, height uint32) (bool, error) { // removeBucketIfEmpty attempts to delete a bucket specified by name from the // provided parent bucket. -func (ns *nurseryStore) removeBucketIfEmpty(parent *bolt.Bucket, - bktName []byte) (bool, error) { - +func removeBucketIfEmpty(parent *bolt.Bucket, bktName []byte) error { // Attempt to fetch the named bucket from its parent. bkt := parent.Bucket(bktName) if bkt == nil { - // No bucket was found, signal this to the caller. - return false, nil + // No bucket was found, already removed? + return nil } - // The bucket exists, now compute how many children *it* has. - nChildren, err := ns.numChildrenInBucket(bkt) - if err != nil { - return false, nil + // The bucket exists, fail if it still has children. + if err := isBucketEmpty(bkt); err != nil { + return err } - // If the number of children is non-zero, alert the caller that the - // named bucket is not being removed. - if nChildren > 0 { - return false, nil - } - - // Otherwise, remove the empty bucket from its parent. - err = parent.DeleteBucket(bktName) - if err != nil { - return false, err - } - - return true, nil + return parent.DeleteBucket(bktName) } -// numChildrenInBucket computes the number of children contained in the given -// boltdb bucket. -func (ns *nurseryStore) numChildrenInBucket(parent *bolt.Bucket) (int, error) { - var nChildren int - if err := parent.ForEach(func(_, _ []byte) error { - nChildren++ +// removeBucketIfExists safely deletes the named bucket by first checking +// that it exists in the parent bucket. +func removeBucketIfExists(parent *bolt.Bucket, bktName []byte) error { + // Attempt to fetch the named bucket from its parent. + bkt := parent.Bucket(bktName) + if bkt == nil { + // No bucket was found, already removed? return nil - }); err != nil { - return 0, err } - return nChildren, nil + return parent.DeleteBucket(bktName) +} + +// isBucketEmpty returns ErrBucketNotEmpty if the bucket has a non-zero number +// of children. +func isBucketEmpty(parent *bolt.Bucket) error { + return parent.ForEach(func(_, _ []byte) error { + return ErrBucketNotEmpty + }) } // Compile-time constraint to ensure nurseryStore implements NurseryStore. From 0ed5f83dced04d67b92b5edcf4f2b32e471157c1 Mon Sep 17 00:00:00 2001 From: Conner Fromknecht Date: Wed, 18 Oct 2017 06:58:24 -0700 Subject: [PATCH 5/6] nursery_store: adds kindergarten txn finalization --- nursery_store.go | 555 +++++++++++++++++++++++++++++++---------------- 1 file changed, 362 insertions(+), 193 deletions(-) diff --git a/nursery_store.go b/nursery_store.go index 64f560698..32aaaaf94 100644 --- a/nursery_store.go +++ b/nursery_store.go @@ -11,10 +11,90 @@ import ( "github.com/roasbeef/btcd/wire" ) +// Overview of Nursery Store Storage Hierarchy +// +// CHAIN SEGMENTATION +// +// The root directory of a nursery store is bucketed by the chain hash and +// the 'utxn' prefix. This allows multiple utxo nurseries for distinct chains +// to simultaneously use the same channel.DB instance. This is critical for +// providing replay protection and more to isolate chain-specific data in the +// multichain setting. +// +// utxn/ +// | +// | LAST PURGED + FINALIZED HEIGHTS +// | +// | Each nursery store tracks a "last purged height", which records the +// | most recent block height for which the nursery store has purged all +// | state. This value lags behind the best block height for reorg safety, +// | and serves as a starting height for rescans after a restart. It also +// | tracks a "last finalized height", which records the last block height +// | that the nursery attempted to graduate. If a finalized height has +// | kindergarten outputs, the sweep txn for these outputs will be stored in +// | the height bucket. This ensure that the same txid will be used after +// | restarts. Otherwise, the nursery will be unable to recover the txid +// | of kindergarten sweep transaction it has already broadcast. +// | +// ├── last-purged-height-key: +// ├── last-finalized-height-key: +// | +// | CHANNEL INDEX +// | +// | The channel index contains a directory for each channel that has a +// | non-zero number of outputs being tracked by the nursery store. +// | Inside each channel directory are files containing serialized spendable +// | outputs that are awaiting some state transition. The name of each file +// | contains the outpoint of the spendable output in the file, and is +// | prefixed with 4-byte state prefix, indicating whether the spendable +// | output is a crib, preschool, or kindergarten, or graduated output. The +// | nursery store supports the ability to enumerate all outputs for a +// | particular channel, which is useful in constructing nursery reports. +// | +// ├── channel-index-key/ +// │   ├── / <- CHANNEL BUCKET +// | |   ├── : +// | |   └── : +// │   ├── / +// | |   └── : +// │   └── / +// |    ├── : +// |    └── : +// | +// | HEIGHT INDEX +// | +// | The height index contains a directory for each height at which the +// | nursery still has scheduled actions. If an output is a crib or +// | kindergarten output, it will have an associated entry in the height +// | index. Inside a particular height directory, the structure is similar +// | to that of the channel index, containing multiple channel directories, +// | each of which contains subdirectories named with a prefixed outpoint +// | belonging to the channel. Enumerating these combinations yields a +// | relative file path: +// | e.g. // +// | that can be queried in the channel index to retrieve the serialized +// | output. If a height bucket is less than or equal to the current last +// | finalized height and has a non-zero number of kindergarten outputs, a +// | height bucket will also contain the finalized kindergarten sweep txn +// | under the "finalized-kndr-txn" key. +// | +// └── height-index-key/ +//    ├── / <- HEIGHT BUCKET +// |   ├── / <- HEIGHT-CHANNEL BUCKET +// | |    ├── : "" <- PREFIXED OUTPOINT +// | |    └── : "" +// |   ├── / +// | |    └── : "" +//    | └── finalized-kndr-txn: "" | +//    └── / +//    └── / +//    └── : "" +//    └── : "" + // NurseryStore abstracts the persistent storage layer for the utxo nursery. -// Concretely, it stores commitment and htlc outputs that until any time-bounded -// constraints have fully matured. The store exposes methods for enumerating -// its contents, and persisting state transitions detected by the utxo nursery. +// Concretely, it stores commitment and htlc outputs until any time-bounded +// constraints have fully matured. The store exposes methods for enumerating its +// contents, and persisting state transitions detected by the utxo nursery. type NurseryStore interface { // Incubate registers a commitment output and a slice of htlc outputs to @@ -35,32 +115,39 @@ type NurseryStore interface { PreschoolToKinder(*kidOutput) error // GraduateKinder accepts a slice of kidOutputs from the kindergarten - // bucket, and removes their corresponding entries from the height and - // channel indexes. If this method detects that all outputs for a - // particular contract have been incubated, it returns the channel - // points that are ready to be marked as fully closed. - GraduateKinder([]kidOutput) error - - // FinalizeHeight accepts a block height as a parameter and purges its - // persistent state for all outputs at that height. During a restart, - // the utxo nursery will begin it's recovery procedure from the next - // height that has yet to be finalized. This block height should lag - // beyond the best height for this chain as a measure of reorg - // protection. - FinalizeHeight(height uint32) error - - // LastFinalizedHeight returns the last block height for which the - // nursery store has purged all persistent state. - LastFinalizedHeight() (uint32, error) - - // FetchClass returns a list of babyOutputs in the crib bucket whose - // CLTV delay expires at the provided block height. - FetchClass(height uint32) ([]kidOutput, []babyOutput, error) + // bucket and marks them graduated. It also removes their corresponding + // entries from the height and channel indexes, pruning any intermediate + // buckets that become empty. + GraduateKinder(height uint32, kndrOutputs []kidOutput) error // FetchPreschools returns a list of all outputs currently stored in the // preschool bucket. FetchPreschools() ([]kidOutput, error) + // FetchClass returns a list of kindergarten and crib outputs whose + // timelocks expire at the given height. If the kindergarten class at + // this height hash been finalized previously, via FinalizeKinder, it + // will also returns the finalized kindergarten sweep txn. + FetchClass(height uint32) (*wire.MsgTx, []kidOutput, []babyOutput, error) + + // FinalizeKinder accepts a block height and the kindergarten sweep txn + // computed for this height. Upon startup, we will rebroadcast any + // finalized kindergarten txns instead of signing a new txn, as this + // result in a different txid from a preceding broadcast. + FinalizeKinder(height uint32, tx *wire.MsgTx) error + + // LastFinalizedHeight returns the last block height for which the + // nursery store finalized a kindergarten class. + LastFinalizedHeight() (uint32, error) + + // PurgeHeight deletes specified the height bucket if it exists, and + // records it as that last purged height. + PurgeHeight(height uint32) error + + // LastPurgedHeight returns the last block height for which the nursery + // store has purged all persistent state. + LastPurgedHeight() (uint32, error) + // ForChanOutputs iterates over all outputs being incubated for a // particular channel point. This method accepts a callback that allows // the caller to process each key-value pair. The key will be a prefixed @@ -73,10 +160,68 @@ type NurseryStore interface { IsMatureChannel(*wire.OutPoint) (bool, error) // RemoveChannel channel erases all entries from the channel bucket for - // the provided channel point. + // the provided channel point, this method should only be called if + // IsMatureChannel indicates the channel is ready for removal. RemoveChannel(*wire.OutPoint) error } +var ( + // utxnChainPrefix is used to prefix a particular chain hash and create + // the root-level, chain-segmented bucket for each nursery store. + utxnChainPrefix = []byte("utxn") + + // lastFinalizedHeightKey is a static key used to locate nursery store's + // last finalized height. + lastFinalizedHeightKey = []byte("last-finalized-height") + + // lastPurgedHeightKey is a static key used to retrieve the height of + // the last bucket that was purged. + lastPurgedHeightKey = []byte("last-purged-height") + + // channelIndexKey is a static key used to lookup the bucket containing + // all of the nursery's active channels. + channelIndexKey = []byte("channel-index") + + // channelIndexKey is a static key used to retrieve a directory + // containing all heights for which the nursery will need to take + // action. + heightIndexKey = []byte("height-index") + + // finalizedKndrTxnKey is a static key that can be used to locate a + // finalized kindergarten sweep txn. + finalizedKndrTxnKey = []byte("finalized-kndr-txn") +) + +// Defines the state prefixes that will be used to persistently track an +// output's progress through the nursery. +// NOTE: Each state prefix MUST be exactly 4 bytes in length, the nursery logic +// depends on the ability to create keys for a different state by overwriting +// an existing state prefix. +var ( + // cribPrefix is the state prefix given to htlc outputs waiting for + // their first-stage, absolute locktime to elapse. + cribPrefix = []byte("crib") + + // psclPrefix is the state prefix given to commitment outputs awaiting + // the confirmation of the commitment transaction, as this solidifies + // the absolute height at which they can be spent. + psclPrefix = []byte("pscl") + + // kndrPrefix is the state prefix given to all CSV delayed outputs, + // either from the commitment transaction, or a stage-one htlc + // transaction, whose maturity height has solidified. Outputs marked in + // this state are in their final stage of incubation withn the nursery, + // and will be swept into the wallet after waiting out the relative + // timelock. + kndrPrefix = []byte("kndr") + + // gradPrefix is the state prefix given to all outputs that have been + // completely incubated. Once all outputs have been marked as graduated, + // this serves as a persistent marker that the nursery should mark the + // channel fully closed in the channeldb. + gradPrefix = []byte("grad") +) + // prefixChainKey creates the root level keys for the nursery store. The keys // are comprised of a nursery-specific prefix and the intended chain hash that // this nursery store will be used for. This allows multiple nursery stores to @@ -117,117 +262,6 @@ func prefixOutputKey(statePrefix []byte, return pfxOutputBuffer.Bytes(), nil } -var ( - // utxnChainPrefix is used to prefix a particular chain hash and create - // the root-level, chain-segmented bucket for each nursery store. - utxnChainPrefix = []byte("utxn") - - // lastFinalizedHeightKey is a static key used to locate nursery store's - // last finalized height. - lastFinalizedHeightKey = []byte("last-finalized-height") - - // channelIndexKey is a static key used to lookup the bucket containing - // all of the nursery's active channels. - channelIndexKey = []byte("channel-index") - - // channelIndexKey is a static key used to retrieve a directory - // containing all heights for which the nursery will need to take - // action. - heightIndexKey = []byte("height-index") - - // cribPrefix is the state prefix given to htlc outputs waiting for - // their first-stage, absolute locktime to elapse. - cribPrefix = []byte("crib") - - // psclPrefix is the state prefix given to commitment outputs awaiting - // the // confirmation of the commitment transaction, as this solidifies - // the absolute height at which they can be spent. - psclPrefix = []byte("pscl") - - // kndrPrefix is the state prefix given to all CSV delayed outputs, - // either from the commitment transaction, or a stage-one htlc - // transaction, whose maturity height has solidified. Outputs marked in - // this state are in their final stage of incubation withn the nursery, - // and will be swept into the wallet after waiting out the relative - // timelock. - kndrPrefix = []byte("kndr") - - // gradPrefix is the state prefix given to all outputs that have been - // completely incubated. Once all outputs have been marked as graduated, - // this serves as a persistent marker that the nursery should mark the - // channel fully closed in the channeldb. - gradPrefix = []byte("grad") -) - -// Overview of Nursery Store Storage Hierarchy -// -// CHAIN SEGMENTATION -// -// The root directory of a nursery store is bucketed by the chain hash and -// the 'utxn' prefix. This allows multiple utxo nurseries for distinct chains -// to simultaneously use the same channel.DB instance. This is critical for -// providing replay protection and more to isolate chain-specific data in the -// multichain setting. -// -// utxn/ -// | -// | LAST FINALIZED HEIGHT -// | -// | Each nursery store tracks a "last finalized height", which records the -// | most recent block height for which the nursery store has purged all -// | state. This value lags behind the best block height for reorg safety, -// | and serves as a starting height for rescans after a restart. -// | -// ├── last-finalized-height-key: -// | -// | CHANNEL INDEX -// | -// | The channel index contains a directory for each channel that has a -// | non-zero number of outputs being tracked by the nursery store. -// | Inside each channel directory are files containing serialized spendable -// | outputs that are awaiting some state transition. The name of each file -// | contains the outpoint of the spendable output in the file, and is -// | prefixed with 4-byte state prefix, indicating whether the spendable -// | output is a crib, preschool, or kindergarten output. The nursery store -// | supports the ability to enumerate all outputs for a particular channel, -// | which is useful in constructing nursery reports. -// | -// ├── channel-index-key/ -// │   ├── / <- CHANNEL BUCKET -// | |   ├── : -// | |   └── : -// │   ├── / -// | |   └── : -// │   └── / -// |    ├── : -// |    └── : -// | -// | HEIGHT INDEX -// | -// | The height index contains a directory for each height at which the -// | nursery still has uncompleted actions. If an output is a crib or -// | kindergarten output, it will have an associated entry in the height -// | index. Inside a particular height directory, the structure is similar -// | to that of the channel index, containing multiple channel directories, -// | each of which contains subdirectories named with a prefixed outpoint -// | belonging to the channel. Enumerating these combinations yields a -// | relative file path: -// | e.g. // -// | that can be queried in the channel index to retrieve the serialized -// | output. -// | -// └── height-index-key/ -//    ├── / <- HEIGHT BUCKET -// |   └── / <- HEIGHT-CHANNEL BUCKET -// | |    ├── : "" <- PREFIXED OUTPOINT -// | |    └── : "" -// |   └── / -// |    └── : "" -//    └── / -//    └── / -//    └── : "" -//    └── : "" - // nurseryStore is a concrete instantiation of a NurseryStore that is backed by // a channeldb.DB instance. type nurseryStore struct { @@ -258,8 +292,8 @@ func newNurseryStore(chainHash *chainhash.Hash, }, nil } -// Incubate initiates the incubation process for the CSV-delayed commitment -// output and any number of CLTV-delayed htlc outputs. +// Incubate persists the beginning of the incubation process for the CSV-delayed +// commitment output and a list of two-stage htlc outputs. func (ns *nurseryStore) Incubate(kid *kidOutput, babies []babyOutput) error { return ns.db.Update(func(tx *bolt.Tx) error { // Store commitment output in preschool bucket if not nil. @@ -436,12 +470,12 @@ func (ns *nurseryStore) PreschoolToKinder(kid *kidOutput) error { // GraduateKinder accepts a list of kidOutputs in the kindergarten bucket and // marks them as graduated. This method also removes their corresponding entries // from the height and channel indexes corresponding to their kindergarten -// status. -func (ns *nurseryStore) GraduateKinder(kids []kidOutput) error { +// status. It also ensures that the finalized kindergarten sweep txn is removed +// for this height. +func (ns *nurseryStore) GraduateKinder(height uint32, kids []kidOutput) error { if err := ns.db.Update(func(tx *bolt.Tx) error { for _, kid := range kids { - confHeight := kid.ConfHeight() outpoint := kid.OutPoint() chanPoint := kid.OriginChanPoint() @@ -454,7 +488,7 @@ func (ns *nurseryStore) GraduateKinder(kids []kidOutput) error { } // Remove the grad output's entry in the height index. - err = ns.removeOutputFromHeight(tx, confHeight, chanPoint, + err = ns.removeOutputFromHeight(tx, height, chanPoint, pfxOutputKey) if err != nil { return err @@ -479,14 +513,23 @@ func (ns *nurseryStore) GraduateKinder(kids []kidOutput) error { } // Insert serialized output into channel bucket using - // kindergarten-prefixed key. + // graduate-prefixed key. err = chanBucket.Put(pfxOutputKey, gradBuffer.Bytes()) if err != nil { return err } } - return nil + hghtBucket := ns.getHeightBucket(tx, height) + if hghtBucket == nil { + return nil + } + + // Since all kindergarten outputs at a particular height are + // swept in a single txn, we can now safely delete the finalized + // txn, since it has already been broadcast and confirmed. + return hghtBucket.Delete(finalizedKndrTxnKey) + }); err != nil { return err } @@ -494,32 +537,54 @@ func (ns *nurseryStore) GraduateKinder(kids []kidOutput) error { return nil } -// FinalizeHeight accepts a block height as a parameter and purges its +// FinalizeKinder accepts a block height as a parameter and purges its // persistent state for all outputs at that height. During a restart, the utxo // nursery will begin it's recovery procedure from the next height that has // yet to be finalized. -func (ns *nurseryStore) FinalizeHeight(height uint32) error { +func (ns *nurseryStore) FinalizeKinder(height uint32, + finalTx *wire.MsgTx) error { + + return ns.db.Update(func(tx *bolt.Tx) error { + return ns.finalizeKinder(tx, height, finalTx) + }) +} + +// PurgeHeight accepts a block height as a parameter and purges its persistent +// state for all outputs at that height. During a restart, the utxo nursery will +// begin it's recovery procedure from the next height that has yet to be +// finalized. +func (ns *nurseryStore) PurgeHeight(height uint32) error { return ns.db.Update(func(tx *bolt.Tx) error { if err := ns.deleteHeightBucket(tx, height); err != nil { return err } - return ns.putLastFinalizedHeight(tx, height) + return ns.putLastPurgedHeight(tx, height) }) } // FetchClass returns a list of babyOutputs in the crib bucket whose CLTV // delay expires at the provided block height. +// FetchClass returns a list of the kindergarten and crib outputs whose timeouts +// are expiring func (ns *nurseryStore) FetchClass( - height uint32) ([]kidOutput, []babyOutput, error) { + height uint32) (*wire.MsgTx, []kidOutput, []babyOutput, error) { - // Construct list of all crib and kindergarten outputs that need TLC at - // the provided block height. + // Construct list of all crib and kindergarten outputs that need to be + // processed at the provided block height. + var finalTx *wire.MsgTx var kids []kidOutput var babies []babyOutput if err := ns.db.View(func(tx *bolt.Tx) error { + + var err error + finalTx, err = ns.getFinalizedTxn(tx, height) + if err != nil { + return err + } + // Append each crib output to our list of babyOutputs. - if err := ns.forEachHeightPrefix(tx, cribPrefix, height, + if err = ns.forEachHeightPrefix(tx, cribPrefix, height, func(buf []byte) error { // We will attempt to deserialize all outputs @@ -560,10 +625,10 @@ func (ns *nurseryStore) FetchClass( }) }); err != nil { - return nil, nil, err + return nil, nil, nil, err } - return kids, babies, nil + return finalTx, kids, babies, nil } // FetchPreschools returns a list of all outputs currently stored in the @@ -649,7 +714,7 @@ func (ns *nurseryStore) FetchPreschools() ([]kidOutput, error) { // process each key-value pair. The key will be a prefixed outpoint, and the // value will be the serialized bytes for an output, whose type should be // inferred from the key's prefix. -// NOTE: The callback should be not modify the provided byte slices and is +// NOTE: The callback should not modify the provided byte slices and is // preferably non-blocking. func (ns *nurseryStore) ForChanOutputs(chanPoint *wire.OutPoint, callback func([]byte, []byte) error) error { @@ -672,9 +737,7 @@ func (ns *nurseryStore) IsMatureChannel(chanPoint *wire.OutPoint) (bool, error) // prefix. return ns.forChanOutputs(tx, chanPoint, func(pfxKey, _ []byte) error { - if string(pfxKey[:4]) != string(gradPrefix) { - utxnLog.Infof("Found non-graduated "+ - "output: %x", pfxKey) + if !bytes.HasPrefix(pfxKey, gradPrefix) { return errImmatureChannel } return nil @@ -713,10 +776,8 @@ func (ns *nurseryStore) RemoveChannel(chanPoint *wire.OutPoint) error { } chanBytes := chanBuffer.Bytes() - utxnLog.Infof("Pruning and removing channel: %v", chanPoint) - err := ns.forChanOutputs(tx, chanPoint, func(k, v []byte) error { - if string(k[:4]) != string(gradPrefix) { + if !bytes.HasPrefix(k, gradPrefix) { return errors.New("expected grad output") } @@ -764,12 +825,26 @@ func (ns *nurseryStore) LastFinalizedHeight() (uint32, error) { return lastFinalizedHeight, err } +// LastPurgedHeight returns the last block height for which the nursery store +// has purged all persistent state. This occurs after a fixed interval for reorg +// safety. +func (ns *nurseryStore) LastPurgedHeight() (uint32, error) { + var lastPurgedHeight uint32 + err := ns.db.View(func(tx *bolt.Tx) error { + var err error + lastPurgedHeight, err = ns.getLastPurgedHeight(tx) + return err + }) + + return lastPurgedHeight, err +} + // Helper Methods // enterCrib accepts a new htlc output that the nursery will incubate through // its two-stage process of sweeping funds back to the user's wallet. These -// outputs are persisted in the nursery store's crib bucket, and will be -// revisited after the output's CLTV has expired. +// outputs are persisted in the nursery store in the crib state, and will be +// revisited after the first-stage output's CLTV has expired. func (ns *nurseryStore) enterCrib(tx *bolt.Tx, baby *babyOutput) error { // First, retrieve or create the channel bucket corresponding to the // baby output's origin channel point. @@ -828,7 +903,7 @@ func (ns *nurseryStore) enterPreschool(tx *bolt.Tx, kid *kidOutput) error { return err } - // Since the babyOutput is being inserted into the preschool bucket, we + // Since the kidOutput is being inserted into the preschool bucket, we // create a key that prefixes its outpoint with the preschool prefix. pfxOutputKey, err := prefixOutputKey(psclPrefix, kid.OutPoint()) if err != nil { @@ -965,17 +1040,15 @@ func (ns *nurseryStore) getHeightBucket(tx *bolt.Tx, return hghtBucket } -// deleteHeightBucket ensures that the height bucket at the provided index is +// purgeHeightBucket ensures that the height bucket at the provided index is // purged from the nursery store. -func (ns *nurseryStore) deleteHeightBucket(tx *bolt.Tx, height uint32) error { +func (ns *nurseryStore) purgeHeightBucket(tx *bolt.Tx, height uint32) error { // Ensure that the height bucket already exists. _, hghtIndex, hghtBucket := ns.getHeightBucketPath(tx, height) if hghtBucket == nil { return nil } - utxnLog.Infof("Deleting height bucket %d", height) - // Serialize the provided height, as this will form the name of the // bucket. var heightBytes [4]byte @@ -1057,8 +1130,10 @@ func (ns *nurseryStore) forEachHeightPrefix(tx *bolt.Tx, prefix []byte, // buckets identified by a channel point, thus we first create list of // channels contained in this height bucket. var channelsAtHeight [][]byte - if err := hghtBucket.ForEach(func(chanBytes, _ []byte) error { - channelsAtHeight = append(channelsAtHeight, chanBytes) + if err := hghtBucket.ForEach(func(chanBytes, v []byte) error { + if v == nil { + channelsAtHeight = append(channelsAtHeight, chanBytes) + } return nil }); err != nil { return err @@ -1166,10 +1241,19 @@ func (ns *nurseryStore) getLastFinalizedHeight(tx *bolt.Tx) (uint32, error) { return byteOrder.Uint32(heightBytes), nil } -// pubLastFinalizedHeight is a helper method that writes the provided height -// under the last finalized height key. -func (ns *nurseryStore) putLastFinalizedHeight(tx *bolt.Tx, - height uint32) error { +// finalizeKinder records a finalized kingergarten sweep txn to the given height +// bucket. It also updates the nursery store's last finalized height, so that we +// do not finalize the same height twice. If the finalized txn is nil, i.e. if +// the height has no kindergarten outputs, the height will be marked as +// finalized, and we skip the process of writing the txn. When the class is +// loaded, a nil value will be returned if no txn has been written to a +// finalized height bucket. +func (ns *nurseryStore) finalizeKinder(tx *bolt.Tx, height uint32, + finalTx *wire.MsgTx) error { + + // TODO(conner) ensure height is greater that current finalized height. + + // 1. Write the last finalized height to the chain bucket. // Ensure that the chain bucket for this nursery store exists. chainBucket, err := tx.CreateBucketIfNotExists(ns.pfxChainKey) @@ -1182,18 +1266,102 @@ func (ns *nurseryStore) putLastFinalizedHeight(tx *bolt.Tx, var lastHeightBytes [4]byte byteOrder.PutUint32(lastHeightBytes[:], height) - return chainBucket.Put(lastFinalizedHeightKey, lastHeightBytes[:]) + err = chainBucket.Put(lastFinalizedHeightKey, lastHeightBytes[:]) + if err != nil { + return err + } + + // 2. Write the finalized txn in the appropriate height bucket. + + // If there is no finalized txn, we have nothing to do. + if finalTx == nil { + return nil + } + + // Otherwise serialize the finalized txn and write it to the height + // bucket. + hghtBucket := ns.getHeightBucket(tx, height) + if hghtBucket == nil { + return nil + } + + var finalTxnBuf bytes.Buffer + if err := finalTx.Serialize(&finalTxnBuf); err != nil { + return err + } + + return hghtBucket.Put(finalizedKndrTxnKey, finalTxnBuf.Bytes()) } -var ( - // ErrBucketDoesNotExist signals that a bucket has already been removed, - // or was never created. - ErrBucketDoesNotExist = errors.New("bucket does not exist") +// getFinalizedTxn retrieves the finalized kindergarten sweep txn at the given +// height, returning nil if one was not found. +func (ns *nurseryStore) getFinalizedTxn(tx *bolt.Tx, + height uint32) (*wire.MsgTx, error) { - // ErrBucketNotEmpty signals that an attempt to prune a particular - // bucket failed because it still has active outputs. - ErrBucketNotEmpty = errors.New("bucket is not empty, cannot be pruned") -) + hghtBucket := ns.getHeightBucket(tx, height) + if hghtBucket == nil { + // No class to finalize. + return nil, nil + } + + finalTxBytes := hghtBucket.Get(finalizedKndrTxnKey) + if finalTxBytes == nil { + // No finalized txn for this height. + return nil, nil + } + + // Otherwise, deserialize and return the finalized transaction. + txn := &wire.MsgTx{} + if err := txn.Deserialize(bytes.NewReader(finalTxBytes)); err != nil { + return nil, err + } + + return txn, nil +} + +// getLastPurgedHeight is a helper method that retrieves the last height for +// which the database purged its persistent state. +func (ns *nurseryStore) getLastPurgedHeight(tx *bolt.Tx) (uint32, error) { + // Retrieve the chain bucket associated with the given nursery store. + chainBucket := tx.Bucket(ns.pfxChainKey) + if chainBucket == nil { + return 0, nil + } + + // Lookup the last finalized height in the top-level chain bucket. + heightBytes := chainBucket.Get(lastPurgedHeightKey) + + // If the resulting bytes are not sized like a uint32, then we have + // never finalized, so we return 0. + if len(heightBytes) != 4 { + return 0, nil + } + + // Otherwise, parse the bytes and return the last finalized height. + return byteOrder.Uint32(heightBytes), nil +} + +// pubLastPurgedHeight is a helper method that writes the provided height under +// the last purged height key. +func (ns *nurseryStore) putLastPurgedHeight(tx *bolt.Tx, height uint32) error { + + // Ensure that the chain bucket for this nursery store exists. + chainBucket, err := tx.CreateBucketIfNotExists(ns.pfxChainKey) + if err != nil { + return err + } + + // Serialize the provided last-finalized height, and store it in the + // top-level chain bucket for this nursery store. + var lastHeightBytes [4]byte + byteOrder.PutUint32(lastHeightBytes[:], height) + + return chainBucket.Put(lastPurgedHeightKey, lastHeightBytes[:]) +} + +// ErrBucketNotEmpty signals that an attempt to prune a particular +// bucket failed because it still has active outputs. +var ErrBucketNotEmpty = errors.New("bucket is not empty, cannot be pruned") // removeOutputFromHeight will delete the given output from the specified // height-channel bucket, and attempt to prune the upstream directories if they @@ -1208,13 +1376,10 @@ func (ns *nurseryStore) removeOutputFromHeight(tx *bolt.Tx, height uint32, return nil } - // Try to delete the prefixed output key if it still exists. The output - // may have already been removed after confirmation, but a final pass is - // done when removing a channel as well. - if hghtChanBucket.Get(pfxKey) != nil { - if err := hghtChanBucket.Delete(pfxKey); err != nil { - return err - } + // Try to delete the prefixed output from the target height-channel + // bucket. + if err := hghtChanBucket.Delete(pfxKey); err != nil { + return err } // Retrieve the height bucket that contains the height-channel bucket. @@ -1251,22 +1416,26 @@ func (ns *nurseryStore) removeOutputFromHeight(tx *bolt.Tx, height uint32, // pruneHeight removes the height bucket at the provided height if and only if // all active outputs at this height have been removed from their respective -// height-channel buckets. +// height-channel buckets. The returned boolean value indicated whether or not +// this invocation successfully pruned the height bucket. func (ns *nurseryStore) pruneHeight(tx *bolt.Tx, height uint32) (bool, error) { // Fetch the existing height index and height bucket. _, hghtIndex, hghtBucket := ns.getHeightBucketPath(tx, height) if hghtBucket == nil { return false, nil } - utxnLog.Infof("pruning height %d", height) // Iterate over all channels stored at this block height. We will // attempt to remove each one if they are empty, keeping track of the // number of height-channel buckets that still have active outputs. - if err := hghtBucket.ForEach(func(chanBytes, _ []byte) error { + if err := hghtBucket.ForEach(func(chanBytes, v []byte) error { + // Skip the finalized txn key. + if v != nil { + return nil + } + // Attempt to each height-channel bucket from the height bucket // located above. - hghtChanBucket := hghtBucket.Bucket(chanBytes) if hghtChanBucket == nil { return errors.New("unable to find height-channel bucket") From d0155f31283c77faf24d05b82f76c31c58c26247 Mon Sep 17 00:00:00 2001 From: Conner Fromknecht Date: Fri, 20 Oct 2017 00:47:01 -0700 Subject: [PATCH 6/6] nursery_store: make GraduateKinder only accept height + list channels --- nursery_store.go | 266 +++++++++++++++++++++++++++-------------------- 1 file changed, 155 insertions(+), 111 deletions(-) diff --git a/nursery_store.go b/nursery_store.go index 32aaaaf94..5d694abaf 100644 --- a/nursery_store.go +++ b/nursery_store.go @@ -114,11 +114,13 @@ type NurseryStore interface { // transaction. PreschoolToKinder(*kidOutput) error - // GraduateKinder accepts a slice of kidOutputs from the kindergarten - // bucket and marks them graduated. It also removes their corresponding - // entries from the height and channel indexes, pruning any intermediate - // buckets that become empty. - GraduateKinder(height uint32, kndrOutputs []kidOutput) error + // GraduateKinder atomically moves the kindergarten class at the + // provided height into the graduated status. This involves removing the + // kindergarten entries from both the height and channel indexes, and + // cleaning up the finalized kindergarten sweep txn. The height bucket + // will be opportunistically pruned from the height index as outputs are + // removed. + GraduateKinder(height uint32) error // FetchPreschools returns a list of all outputs currently stored in the // preschool bucket. @@ -155,6 +157,9 @@ type NurseryStore interface { // whose type should be inferred from the key's prefix. ForChanOutputs(*wire.OutPoint, func([]byte, []byte) error) error + // ListChannels returns all channels the nursery is currently tracking. + ListChannels() ([]wire.OutPoint, error) + // IsMatureChannel determines the whether or not all of the outputs in a // particular channel bucket have been marked as graduated. IsMatureChannel(*wire.OutPoint) (bool, error) @@ -344,6 +349,7 @@ func (ns *nurseryStore) CribToKinder(bby *babyOutput) error { return err } + // Remove the crib output's entry in the height index. err = ns.removeOutputFromHeight(tx, bby.expiry, chanPoint, pfxOutputKey) if err != nil { @@ -467,74 +473,88 @@ func (ns *nurseryStore) PreschoolToKinder(kid *kidOutput) error { }) } -// GraduateKinder accepts a list of kidOutputs in the kindergarten bucket and -// marks them as graduated. This method also removes their corresponding entries -// from the height and channel indexes corresponding to their kindergarten -// status. It also ensures that the finalized kindergarten sweep txn is removed -// for this height. -func (ns *nurseryStore) GraduateKinder(height uint32, kids []kidOutput) error { - if err := ns.db.Update(func(tx *bolt.Tx) error { - for _, kid := range kids { - - outpoint := kid.OutPoint() - chanPoint := kid.OriginChanPoint() - - // Construct the key under which the output is currently - // stored height and channel indexes. - pfxOutputKey, err := prefixOutputKey(kndrPrefix, - outpoint) - if err != nil { - return err - } - - // Remove the grad output's entry in the height index. - err = ns.removeOutputFromHeight(tx, height, chanPoint, - pfxOutputKey) - if err != nil { - return err - } - - chanBucket := ns.getChannelBucket(tx, chanPoint) - if chanBucket == nil { - return ErrContractNotFound - } - - // Remove previous output with kindergarten prefix. - if err := chanBucket.Delete(pfxOutputKey); err != nil { - return err - } - - // Convert kindergarten key to graduate key. - copy(pfxOutputKey, gradPrefix) - - var gradBuffer bytes.Buffer - if err := kid.Encode(&gradBuffer); err != nil { - return err - } - - // Insert serialized output into channel bucket using - // graduate-prefixed key. - err = chanBucket.Put(pfxOutputKey, gradBuffer.Bytes()) - if err != nil { - return err - } - } - - hghtBucket := ns.getHeightBucket(tx, height) - if hghtBucket == nil { - return nil - } +// GraduateKinder atomically moves the kindergarten class at the provided height +// into the graduated status. This involves removing the kindergarten entries +// from both the height and channel indexes, and cleaning up the finalized +// kindergarten sweep txn. The height bucket will be opportunistically pruned +// from the height index as outputs are removed. +func (ns *nurseryStore) GraduateKinder(height uint32) error { + return ns.db.Update(func(tx *bolt.Tx) error { // Since all kindergarten outputs at a particular height are // swept in a single txn, we can now safely delete the finalized // txn, since it has already been broadcast and confirmed. - return hghtBucket.Delete(finalizedKndrTxnKey) + hghtBucket := ns.getHeightBucket(tx, height) + if hghtBucket == nil { + // Nothing to delete, bucket has already been removed. + return nil + } - }); err != nil { - return err - } + // Remove the finalized kindergarten txn, we do this before + // removing the outputs so that the extra entry doesn't prevent + // the height bucket from being opportunistically pruned below. + if err := hghtBucket.Delete(finalizedKndrTxnKey); err != nil { + return err + } - return nil + // For each kindergarten found output, delete its entry from the + // height and channel index, and create a new grad output in the + // channel index. + return ns.forEachHeightPrefix(tx, kndrPrefix, height, + func(v []byte) error { + var kid kidOutput + err := kid.Decode(bytes.NewReader(v)) + if err != nil { + return err + } + + outpoint := kid.OutPoint() + chanPoint := kid.OriginChanPoint() + + // Construct the key under which the output is + // currently stored height and channel indexes. + pfxOutputKey, err := prefixOutputKey(kndrPrefix, + outpoint) + if err != nil { + return err + } + + // Remove the grad output's entry in the height + // index. + err = ns.removeOutputFromHeight(tx, height, + chanPoint, pfxOutputKey) + if err != nil { + return err + } + + chanBucket := ns.getChannelBucket(tx, + chanPoint) + if chanBucket == nil { + return ErrContractNotFound + } + + // Remove previous output with kindergarten + // prefix. + err = chanBucket.Delete(pfxOutputKey) + if err != nil { + return err + } + + // Convert kindergarten key to graduate key. + copy(pfxOutputKey, gradPrefix) + + var gradBuffer bytes.Buffer + if err := kid.Encode(&gradBuffer); err != nil { + return err + } + + // Insert serialized output into channel bucket + // using graduate-prefixed key. + return chanBucket.Put(pfxOutputKey, + gradBuffer.Bytes()) + }, + ) + }) } // FinalizeKinder accepts a block height as a parameter and purges its @@ -555,7 +575,7 @@ func (ns *nurseryStore) FinalizeKinder(height uint32, // finalized. func (ns *nurseryStore) PurgeHeight(height uint32) error { return ns.db.Update(func(tx *bolt.Tx) error { - if err := ns.deleteHeightBucket(tx, height); err != nil { + if err := ns.purgeHeightBucket(tx, height); err != nil { return err } @@ -601,7 +621,8 @@ func (ns *nurseryStore) FetchClass( return nil - }); err != nil { + }, + ); err != nil { return err } @@ -677,17 +698,14 @@ func (ns *nurseryStore) FetchPreschools() ([]kidOutput, error) { // the channel bucket to efficiently enumerate all the // desired outputs. c := chanBucket.Cursor() - - // Seek and iterate over all outputs starting with the - // prefix "pscl". - pfxOutputKey, kidBytes := c.Seek(psclPrefix) - for bytes.HasPrefix(pfxOutputKey, psclPrefix) { + for k, v := c.Seek(psclPrefix); bytes.HasPrefix( + k, psclPrefix); k, v = c.Next() { // Deserialize each output as a kidOutput, since // this should have been the type that was // serialized when it was written to disk. var psclOutput kidOutput - psclReader := bytes.NewReader(kidBytes) + psclReader := bytes.NewReader(v) err := psclOutput.Decode(psclReader) if err != nil { return err @@ -696,8 +714,6 @@ func (ns *nurseryStore) FetchPreschools() ([]kidOutput, error) { // Add the deserialized output to our list of // preschool outputs. kids = append(kids, psclOutput) - - pfxOutputKey, kidBytes = c.Next() } } @@ -724,9 +740,39 @@ func (ns *nurseryStore) ForChanOutputs(chanPoint *wire.OutPoint, }) } -// errImmatureChannel signals that not all outputs in a channel bucket have -// graduated. -var errImmatureChannel = errors.New("channel has non-graduated outputs") +// ListChannels returns all channels the nursery is currently tracking. +func (ns *nurseryStore) ListChannels() ([]wire.OutPoint, error) { + var activeChannels []wire.OutPoint + if err := ns.db.View(func(tx *bolt.Tx) error { + // Retrieve the existing chain bucket for this nursery store. + chainBucket := tx.Bucket(ns.pfxChainKey) + if chainBucket == nil { + return nil + } + + // Retrieve the existing channel index. + chanIndex := chainBucket.Bucket(channelIndexKey) + if chanIndex == nil { + return nil + } + + return chanIndex.ForEach(func(chanBytes, _ []byte) error { + var chanPoint wire.OutPoint + err := readOutpoint(bytes.NewReader(chanBytes), &chanPoint) + if err != nil { + return err + } + + activeChannels = append(activeChannels, chanPoint) + + return nil + }) + }); err != nil { + return nil, err + } + + return activeChannels, nil +} // IsMatureChannel determines the whether or not all of the outputs in a // particular channel bucket have been marked as graduated. @@ -738,19 +784,24 @@ func (ns *nurseryStore) IsMatureChannel(chanPoint *wire.OutPoint) (bool, error) return ns.forChanOutputs(tx, chanPoint, func(pfxKey, _ []byte) error { if !bytes.HasPrefix(pfxKey, gradPrefix) { - return errImmatureChannel + return ErrImmatureChannel } return nil }) }) - if err != nil && err != errImmatureChannel { + if err != nil && err != ErrImmatureChannel { return false, err } return err == nil, nil } +// ErrImmatureChannel signals a channel cannot be removed because not all of its +// outputs have graduated. +var ErrImmatureChannel = errors.New("cannot remove immature channel, " + + "still has ungraduated outputs") + // RemoveChannel channel erases all entries from the channel bucket for the // provided channel point. // NOTE: The channel's entries in the height index are assumed to be removed. @@ -778,7 +829,7 @@ func (ns *nurseryStore) RemoveChannel(chanPoint *wire.OutPoint) error { err := ns.forChanOutputs(tx, chanPoint, func(k, v []byte) error { if !bytes.HasPrefix(k, gradPrefix) { - return errors.New("expected grad output") + return ErrImmatureChannel } // Construct a kindergarten prefixed key, since this @@ -1166,7 +1217,8 @@ func (ns *nurseryStore) forEachHeightPrefix(tx *bolt.Tx, prefix []byte, // outputs. chanBucket := chanIndex.Bucket(chanBytes) if chanBucket == nil { - return fmt.Errorf("unable to retrieve channel bucket: '%x'", chanBytes) + return fmt.Errorf("unable to retrieve channel "+ + "bucket: '%x'", chanBytes) } // Since all of the outputs of interest will start with the same @@ -1174,16 +1226,13 @@ func (ns *nurseryStore) forEachHeightPrefix(tx *bolt.Tx, prefix []byte, // contained in the height-channel bucket, efficiently // enumerating the desired outputs. c := hghtChanBucket.Cursor() - - // Seek to and iterate over all entries starting with the given - // prefix. - pfxOutputKey, _ := c.Seek(prefix) - for bytes.HasPrefix(pfxOutputKey, prefix) { + for k, _ := c.Seek(prefix); bytes.HasPrefix( + k, prefix); k, _ = c.Next() { // Use the prefix output key emitted from our scan to // load the serialized babyOutput from the appropriate // channel bucket. - outputBytes := chanBucket.Get(pfxOutputKey) + outputBytes := chanBucket.Get(k) if outputBytes == nil { return errors.New("unable to retrieve output") } @@ -1194,10 +1243,6 @@ func (ns *nurseryStore) forEachHeightPrefix(tx *bolt.Tx, prefix []byte, if err := callback(outputBytes); err != nil { return err } - - // Lastly, advance our prefix output key for the next - // iteration. - pfxOutputKey, _ = c.Next() } } @@ -1230,12 +1275,13 @@ func (ns *nurseryStore) getLastFinalizedHeight(tx *bolt.Tx) (uint32, error) { // Lookup the last finalized height in the top-level chain bucket. heightBytes := chainBucket.Get(lastFinalizedHeightKey) + if heightBytes == nil { + // We have never finalized, return height 0. + return 0, nil + } // If the resulting bytes are not sized like a uint32, then we have // never finalized, so we return 0. - if len(heightBytes) != 4 { - return 0, nil - } // Otherwise, parse the bytes and return the last finalized height. return byteOrder.Uint32(heightBytes), nil @@ -1328,16 +1374,14 @@ func (ns *nurseryStore) getLastPurgedHeight(tx *bolt.Tx) (uint32, error) { return 0, nil } - // Lookup the last finalized height in the top-level chain bucket. + // Lookup the last purged height in the top-level chain bucket. heightBytes := chainBucket.Get(lastPurgedHeightKey) - - // If the resulting bytes are not sized like a uint32, then we have - // never finalized, so we return 0. - if len(heightBytes) != 4 { + if heightBytes == nil { + // We have never purged before, return height 0. return 0, nil } - // Otherwise, parse the bytes and return the last finalized height. + // Otherwise, parse the bytes and return the last purged height. return byteOrder.Uint32(heightBytes), nil } @@ -1351,7 +1395,7 @@ func (ns *nurseryStore) putLastPurgedHeight(tx *bolt.Tx, height uint32) error { return err } - // Serialize the provided last-finalized height, and store it in the + // Serialize the provided last-purged height, and store it in the // top-level chain bucket for this nursery store. var lastHeightBytes [4]byte byteOrder.PutUint32(lastHeightBytes[:], height) @@ -1359,9 +1403,9 @@ func (ns *nurseryStore) putLastPurgedHeight(tx *bolt.Tx, height uint32) error { return chainBucket.Put(lastPurgedHeightKey, lastHeightBytes[:]) } -// ErrBucketNotEmpty signals that an attempt to prune a particular +// errBucketNotEmpty signals that an attempt to prune a particular // bucket failed because it still has active outputs. -var ErrBucketNotEmpty = errors.New("bucket is not empty, cannot be pruned") +var errBucketNotEmpty = errors.New("bucket is not empty, cannot be pruned") // removeOutputFromHeight will delete the given output from the specified // height-channel bucket, and attempt to prune the upstream directories if they @@ -1396,16 +1440,16 @@ func (ns *nurseryStore) removeOutputFromHeight(tx *bolt.Tx, height uint32, // Try to remove the channel-height bucket if it this was the last // output in the bucket. err := removeBucketIfEmpty(hghtBucket, chanBuffer.Bytes()) - if err != nil && err != ErrBucketNotEmpty { + if err != nil && err != errBucketNotEmpty { return err - } else if err == ErrBucketNotEmpty { + } else if err == errBucketNotEmpty { return nil } // Attempt to prune the height bucket matching the kid output's // confirmation height in case that was the last height-chan bucket. pruned, err := ns.pruneHeight(tx, height) - if err != nil && err != ErrBucketNotEmpty { + if err != nil && err != errBucketNotEmpty { return err } else if err == nil && pruned { utxnLog.Infof("Height bucket %d pruned", height) @@ -1493,11 +1537,11 @@ func removeBucketIfExists(parent *bolt.Bucket, bktName []byte) error { return parent.DeleteBucket(bktName) } -// isBucketEmpty returns ErrBucketNotEmpty if the bucket has a non-zero number +// isBucketEmpty returns errBucketNotEmpty if the bucket has a non-zero number // of children. func isBucketEmpty(parent *bolt.Bucket) error { return parent.ForEach(func(_, _ []byte) error { - return ErrBucketNotEmpty + return errBucketNotEmpty }) }