From 7fdcf9bfbc9813677e748bb743312de82cd3499d Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Wed, 10 Feb 2016 13:22:15 -0800
Subject: [PATCH 01/22] start trying to connect to segnet

doesn't work or anything yet
---
 shell.go           | 10 +++++-----
 uspv/eight333.go   |  7 +++++--
 uspv/msghandler.go |  2 +-
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/shell.go b/shell.go
index 9438e7de3..3654cf728 100644
--- a/shell.go
+++ b/shell.go
@@ -25,15 +25,15 @@ testing.  It can send and receive coins.
 const (
 	keyFileName    = "testkey.hex"
 	headerFileName = "headers.bin"
-	dbFileName     = "/dev/shm/utxo.db"
+	dbFileName     = "utxo.db"
 	// this is my local testnet node, replace it with your own close by.
 	// Random internet testnet nodes usually work but sometimes don't, so
 	// maybe I should test against different versions out there.
-	SPVHostAdr = "127.0.0.1:18333"
+	SPVHostAdr = "127.0.0.1:28333"
 )
 
 var (
-	Params = &chaincfg.TestNet3Params
+	Params = &chaincfg.SegNetParams
 	SCon   uspv.SPVCon // global here for now
 )
 
@@ -51,7 +51,7 @@ func shell() {
 	// setup spvCon
 
 	SCon, err = uspv.OpenSPV(
-		SPVHostAdr, headerFileName, dbFileName, &Store, false, false, Params)
+		SPVHostAdr, headerFileName, dbFileName, &Store, true, false, Params)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -61,7 +61,7 @@ func shell() {
 		log.Fatal(err)
 	}
 	if tip == 0 { // DB has never been used, set to birthday
-		tip = 675000 // hardcoded; later base on keyfile date?
+		tip = 1234 // hardcoded; later base on keyfile date?
 		err = SCon.TS.SetDBSyncHeight(tip)
 		if err != nil {
 			log.Fatal(err)
diff --git a/uspv/eight333.go b/uspv/eight333.go
index 3c40f2c23..180d37677 100644
--- a/uspv/eight333.go
+++ b/uspv/eight333.go
@@ -15,7 +15,8 @@ const (
 	headerFileName = "headers.bin"
 
 	// version hardcoded for now, probably ok...?
-	VERSION = 70011
+	// 70012 is for segnet... make this a init var?
+	VERSION = 70012
 )
 
 type SPVCon struct {
@@ -115,7 +116,9 @@ func (s *SPVCon) IngestMerkleBlock(m *wire.MsgMerkleBlock) {
 	// into our SPV header file
 	newMerkBlockSha := m.Header.BlockSha()
 	if !hah.blockhash.IsEqual(&newMerkBlockSha) {
-		log.Printf("merkle block out of order error")
+		log.Printf("merkle block out of order got %s expect %s",
+			m.Header.BlockSha().String(), hah.blockhash.String())
+		log.Printf("has %d hashes", len(m.Hashes))
 		return
 	}
 
diff --git a/uspv/msghandler.go b/uspv/msghandler.go
index 06d10c4a4..c3def1fb2 100644
--- a/uspv/msghandler.go
+++ b/uspv/msghandler.go
@@ -128,7 +128,7 @@ func (s *SPVCon) TxHandler(m *wire.MsgTx) {
 	height, ok := s.TS.OKTxids[m.TxSha()]
 	s.TS.OKMutex.Unlock()
 	if !ok {
-		log.Printf("Tx %s unknown, will not ingest\n")
+		log.Printf("Tx %s unknown, will not ingest\n", m.TxSha().String())
 		return
 	}
 

From ee4dd8aaa084e0f56ad42652f2435f6cbdad1781 Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Sat, 13 Feb 2016 14:03:11 -0800
Subject: [PATCH 02/22] trying to get witness coinbases to validate

(they don't)
---
 shell.go         |  8 ++++----
 uspv/eight333.go | 41 ++++++++++++++++++++++++++++++++++++++---
 uspv/hardmode.go | 48 +++++++++++++-----------------------------------
 uspv/init.go     |  2 ++
 uspv/txstore.go  |  9 ++++++++-
 5 files changed, 65 insertions(+), 43 deletions(-)

diff --git a/shell.go b/shell.go
index 3654cf728..5eb8057b0 100644
--- a/shell.go
+++ b/shell.go
@@ -69,10 +69,10 @@ func shell() {
 	}
 
 	// once we're connected, initiate headers sync
-	err = SCon.AskForHeaders()
-	if err != nil {
-		log.Fatal(err)
-	}
+	//	err = SCon.AskForHeaders()
+	//	if err != nil {
+	//		log.Fatal(err)
+	//	}
 
 	// main shell loop
 	for {
diff --git a/uspv/eight333.go b/uspv/eight333.go
index 180d37677..533b87d65 100644
--- a/uspv/eight333.go
+++ b/uspv/eight333.go
@@ -118,7 +118,8 @@ func (s *SPVCon) IngestMerkleBlock(m *wire.MsgMerkleBlock) {
 	if !hah.blockhash.IsEqual(&newMerkBlockSha) {
 		log.Printf("merkle block out of order got %s expect %s",
 			m.Header.BlockSha().String(), hah.blockhash.String())
-		log.Printf("has %d hashes", len(m.Hashes))
+		log.Printf("has %d hashes %d txs flags: %x",
+			len(m.Hashes), m.Transactions, m.Flags)
 		return
 	}
 
@@ -283,6 +284,40 @@ func (s *SPVCon) AskForHeaders() error {
 	return nil
 }
 
+// AskForOneBlock is for testing only, so you can ask for a specific block height
+// and see what goes wrong
+func (s *SPVCon) AskForOneBlock(h int32) error {
+	var hdr wire.BlockHeader
+	var err error
+
+	dbTip := int32(h)
+	s.headerMutex.Lock() // seek to header we need
+	_, err = s.headerFile.Seek(int64((dbTip)*80), os.SEEK_SET)
+	if err != nil {
+		return err
+	}
+	err = hdr.Deserialize(s.headerFile) // read header, done w/ file for now
+	s.headerMutex.Unlock()              // unlock after reading 1 header
+	if err != nil {
+		log.Printf("header deserialize error!\n")
+		return err
+	}
+
+	bHash := hdr.BlockSha()
+	// create inventory we're asking for
+	iv1 := wire.NewInvVect(wire.InvTypeWitnessBlock, &bHash)
+	gdataMsg := wire.NewMsgGetData()
+	// add inventory
+	err = gdataMsg.AddInvVect(iv1)
+	if err != nil {
+		return err
+	}
+	hah := NewRootAndHeight(bHash, h)
+	s.outMsgQueue <- gdataMsg
+	s.blockQueue <- hah // push height and mroot of requested block on queue
+	return nil
+}
+
 // AskForMerkBlocks requests blocks from current to last
 // right now this asks for 1 block per getData message.
 // Maybe it's faster to ask for many in a each message?
@@ -347,9 +382,9 @@ func (s *SPVCon) AskForBlocks() error {
 		iv1 := new(wire.InvVect)
 		// if hardmode, ask for legit blocks, none of this ralphy stuff
 		if s.HardMode {
-			iv1 = wire.NewInvVect(wire.InvTypeBlock, &bHash)
+			iv1 = wire.NewInvVect(wire.InvTypeWitnessBlock, &bHash)
 		} else { // ah well
-			iv1 = wire.NewInvVect(wire.InvTypeFilteredBlock, &bHash)
+			iv1 = wire.NewInvVect(wire.InvTypeFilteredWitnessBlock, &bHash)
 		}
 		gdataMsg := wire.NewMsgGetData()
 		// add inventory
diff --git a/uspv/hardmode.go b/uspv/hardmode.go
index 7cd62c1e7..c5d99f5f5 100644
--- a/uspv/hardmode.go
+++ b/uspv/hardmode.go
@@ -1,9 +1,9 @@
 package uspv
 
 import (
+	"bytes"
 	"fmt"
 	"log"
-	"os"
 	"sync"
 
 	"github.com/btcsuite/btcd/wire"
@@ -24,6 +24,8 @@ func BlockRootOK(blk wire.MsgBlock) bool {
 	for len(shas) > 1 { // calculate merkle root. Terse, eh?
 		shas = append(shas[2:], MakeMerkleParent(shas[0], shas[1]))
 	} // auto recognizes coinbase-only blocks
+	fmt.Printf("MRs calcd %s given %s\n",
+		shas[0].String(), blk.Header.MerkleRoot.String())
 	return blk.Header.MerkleRoot.IsEqual(shas[0])
 }
 
@@ -31,7 +33,16 @@ func BlockRootOK(blk wire.MsgBlock) bool {
 // different enough that it's better to have 2 separate functions
 func (s *SPVCon) IngestBlock(m *wire.MsgBlock) {
 	var err error
-
+	var buf bytes.Buffer
+	m.SerializeWitness(&buf)
+	fmt.Printf("block hex %x\n", buf.Bytes())
+	for i, tx := range m.Transactions {
+		//		if i > 0 {
+		fmt.Printf("wtxid: %s\n", tx.WTxSha())
+		fmt.Printf("txid: %s\n", tx.TxSha())
+		fmt.Printf("%d %s", i, TxToString(tx))
+		//		}
+	}
 	ok := BlockRootOK(*m) // check block self-consistency
 	if !ok {
 		fmt.Printf("block %s not OK!!11\n", m.BlockSha().String())
@@ -59,7 +70,6 @@ func (s *SPVCon) IngestBlock(m *wire.MsgBlock) {
 	var wg sync.WaitGroup
 	wg.Add(len(m.Transactions))
 	for i, tx := range m.Transactions {
-
 		go func() {
 			hits, err := s.TS.Ingest(tx, hah.height)
 			if err != nil {
@@ -99,35 +109,3 @@ func (s *SPVCon) IngestBlock(m *wire.MsgBlock) {
 	}
 	return
 }
-
-func (s *SPVCon) AskForOneBlock(h int32) error {
-	var hdr wire.BlockHeader
-	var err error
-
-	dbTip := int32(h)
-	s.headerMutex.Lock() // seek to header we need
-	_, err = s.headerFile.Seek(int64((dbTip)*80), os.SEEK_SET)
-	if err != nil {
-		return err
-	}
-	err = hdr.Deserialize(s.headerFile) // read header, done w/ file for now
-	s.headerMutex.Unlock()              // unlock after reading 1 header
-	if err != nil {
-		log.Printf("header deserialize error!\n")
-		return err
-	}
-
-	bHash := hdr.BlockSha()
-	// create inventory we're asking for
-	iv1 := wire.NewInvVect(wire.InvTypeBlock, &bHash)
-	gdataMsg := wire.NewMsgGetData()
-	// add inventory
-	err = gdataMsg.AddInvVect(iv1)
-	if err != nil {
-		return err
-	}
-
-	s.outMsgQueue <- gdataMsg
-
-	return nil
-}
diff --git a/uspv/init.go b/uspv/init.go
index 4dc845562..4fb3844a1 100644
--- a/uspv/init.go
+++ b/uspv/init.go
@@ -49,6 +49,8 @@ func OpenSPV(remoteNode string, hfn, dbfn string,
 	}
 	// must set this to enable SPV stuff
 	myMsgVer.AddService(wire.SFNodeBloom)
+	// set this to enable segWit
+	myMsgVer.AddService(wire.SFNodeWitness)
 	// this actually sends
 	n, err := wire.WriteMessageN(s.con, myMsgVer, s.localVersion, s.TS.Param.Net)
 	if err != nil {
diff --git a/uspv/txstore.go b/uspv/txstore.go
index 17282bfb9..ed6b555e8 100644
--- a/uspv/txstore.go
+++ b/uspv/txstore.go
@@ -133,7 +133,8 @@ func CheckDoubleSpends(
 
 // TxToString prints out some info about a transaction. for testing / debugging
 func TxToString(tx *wire.MsgTx) string {
-	str := fmt.Sprintf("\t - Tx %s\n", tx.TxSha().String())
+	str := fmt.Sprintf("\t size %d wsize %d Tx %s\n",
+		tx.SerializeSize(), tx.SerializeSizeWitness(), tx.TxSha().String())
 	for i, in := range tx.TxIn {
 		str += fmt.Sprintf("Input %d: %s\n", i, in.PreviousOutPoint.String())
 		str += fmt.Sprintf("SigScript for input %d: %x\n", i, in.SignatureScript)
@@ -146,6 +147,12 @@ func TxToString(tx *wire.MsgTx) string {
 			str += fmt.Sprintf("output %d nil (WARNING)\n", i)
 		}
 	}
+	for i, wit := range tx.TxWitness {
+		if wit.ScriptWitness != nil {
+			str += fmt.Sprintf("Witness %d: %x\n", i, wit.ScriptWitness)
+		}
+	}
+
 	return str
 }
 

From a955a428a7763135e4c500988110c36011273184 Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Sun, 14 Feb 2016 12:50:20 -0800
Subject: [PATCH 03/22] fix nil witness crash; sync's now on segnet

---
 shell.go         | 10 +++++-----
 uspv/eight333.go |  2 +-
 uspv/hardmode.go |  2 +-
 uspv/txstore.go  |  7 ++++---
 4 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/shell.go b/shell.go
index 5eb8057b0..365b0ac2f 100644
--- a/shell.go
+++ b/shell.go
@@ -68,11 +68,11 @@ func shell() {
 		}
 	}
 
-	// once we're connected, initiate headers sync
-	//	err = SCon.AskForHeaders()
-	//	if err != nil {
-	//		log.Fatal(err)
-	//	}
+	//	 once we're connected, initiate headers sync
+	err = SCon.AskForHeaders()
+	if err != nil {
+		log.Fatal(err)
+	}
 
 	// main shell loop
 	for {
diff --git a/uspv/eight333.go b/uspv/eight333.go
index 533b87d65..ca88e18f7 100644
--- a/uspv/eight333.go
+++ b/uspv/eight333.go
@@ -349,7 +349,7 @@ func (s *SPVCon) AskForBlocks() error {
 		return nil
 	}
 
-	fmt.Printf("will request merkleblocks %d to %d\n", dbTip, headerTip)
+	fmt.Printf("will request blocks %d to %d\n", dbTip, headerTip)
 
 	if !s.HardMode { // don't send this in hardmode! that's the whole point
 		// create initial filter
diff --git a/uspv/hardmode.go b/uspv/hardmode.go
index c5d99f5f5..3e3bcdbac 100644
--- a/uspv/hardmode.go
+++ b/uspv/hardmode.go
@@ -39,7 +39,7 @@ func (s *SPVCon) IngestBlock(m *wire.MsgBlock) {
 	for i, tx := range m.Transactions {
 		//		if i > 0 {
 		fmt.Printf("wtxid: %s\n", tx.WTxSha())
-		fmt.Printf("txid: %s\n", tx.TxSha())
+		fmt.Printf(" txid: %s\n", tx.TxSha())
 		fmt.Printf("%d %s", i, TxToString(tx))
 		//		}
 	}
diff --git a/uspv/txstore.go b/uspv/txstore.go
index ed6b555e8..3881b485c 100644
--- a/uspv/txstore.go
+++ b/uspv/txstore.go
@@ -133,8 +133,9 @@ func CheckDoubleSpends(
 
 // TxToString prints out some info about a transaction. for testing / debugging
 func TxToString(tx *wire.MsgTx) string {
-	str := fmt.Sprintf("\t size %d wsize %d Tx %s\n",
-		tx.SerializeSize(), tx.SerializeSizeWitness(), tx.TxSha().String())
+	str := fmt.Sprintf("\t size %d vsize %d wsize %d Tx %s\n",
+		tx.SerializeSize(), tx.VirtualSize(), tx.SerializeSizeWitness(),
+		tx.TxSha().String())
 	for i, in := range tx.TxIn {
 		str += fmt.Sprintf("Input %d: %s\n", i, in.PreviousOutPoint.String())
 		str += fmt.Sprintf("SigScript for input %d: %x\n", i, in.SignatureScript)
@@ -148,7 +149,7 @@ func TxToString(tx *wire.MsgTx) string {
 		}
 	}
 	for i, wit := range tx.TxWitness {
-		if wit.ScriptWitness != nil {
+		if wit != nil {
 			str += fmt.Sprintf("Witness %d: %x\n", i, wit.ScriptWitness)
 		}
 	}

From 0d3639435f0edb503b651db0f126e90ff9f3db8f Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Mon, 15 Feb 2016 22:13:17 -0800
Subject: [PATCH 04/22] can sync with segnet in hard mode

lots of changes but they seem to work
---
 lnwallet/channel.go |   2 +-
 lnwallet/wallet.go  |   4 +-
 lnwire/lnwire.go    |  11 ++--
 shell.go            |   2 +-
 uspv/hardmode.go    | 133 ++++++++++++++++++++++++++++++++++++--------
 uspv/msghandler.go  |  13 ++++-
 uspv/txstore.go     |  10 ++--
 7 files changed, 135 insertions(+), 40 deletions(-)

diff --git a/lnwallet/channel.go b/lnwallet/channel.go
index 93ad4938e..8bb4fa62c 100644
--- a/lnwallet/channel.go
+++ b/lnwallet/channel.go
@@ -86,7 +86,7 @@ func newLightningChannel(wallet *LightningWallet, events chainntnfs.ChainNotifie
 		return nil, err
 	}
 	_, multiSigIndex := findScriptOutputIndex(state.FundingTx, fundingPkScript)
-	lc.fundingTxIn = wire.NewTxIn(wire.NewOutPoint(&fundingTxId, multiSigIndex), nil)
+	lc.fundingTxIn = wire.NewTxIn(wire.NewOutPoint(&fundingTxId, multiSigIndex), nil, nil)
 	lc.fundingP2SH = fundingPkScript
 
 	return lc, nil
diff --git a/lnwallet/wallet.go b/lnwallet/wallet.go
index 372b6cf3a..8aaed0acc 100644
--- a/lnwallet/wallet.go
+++ b/lnwallet/wallet.go
@@ -529,7 +529,7 @@ func (l *LightningWallet) handleFundingReserveRequest(req *initFundingReserveMsg
 		// Empty sig script, we'll actually sign if this reservation is
 		// queued up to be completed (the other side accepts).
 		outPoint := wire.NewOutPoint(coin.Hash(), coin.Index())
-		ourContribution.Inputs[i] = wire.NewTxIn(outPoint, nil)
+		ourContribution.Inputs[i] = wire.NewTxIn(outPoint, nil, nil)
 	}
 
 	l.coinSelectMtx.Unlock()
@@ -794,7 +794,7 @@ func (l *LightningWallet) handleContributionMsg(req *addContributionMsg) {
 	// since the outputs are cannonically sorted.
 	fundingNTxid := fundingTx.TxSha() // NOTE: assumes testnet-L
 	_, multiSigIndex := findScriptOutputIndex(fundingTx, multiSigOut.PkScript)
-	fundingTxIn := wire.NewTxIn(wire.NewOutPoint(&fundingNTxid, multiSigIndex), nil)
+	fundingTxIn := wire.NewTxIn(wire.NewOutPoint(&fundingNTxid, multiSigIndex), nil, nil)
 
 	// With the funding tx complete, create both commitment transactions.
 	initialBalance := ourContribution.FundingAmount
diff --git a/lnwire/lnwire.go b/lnwire/lnwire.go
index c9fdb9625..bf07419da 100644
--- a/lnwire/lnwire.go
+++ b/lnwire/lnwire.go
@@ -4,11 +4,12 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
+	"io"
+	"io/ioutil"
+
 	"github.com/btcsuite/btcd/btcec"
 	"github.com/btcsuite/btcd/wire"
 	"github.com/btcsuite/btcutil"
-	"io"
-	"io/ioutil"
 )
 
 var MAX_SLICE_LENGTH = 65535
@@ -21,12 +22,12 @@ type CommitHeight uint64
 
 // Subsatoshi amount (Micro-Satoshi, 1/1000th)
 // Should be a signed int to account for negative fees
-// 
+//
 // "In any science-fiction movie, anywhere in the galaxy, currency is referred
 // to as 'credits.'"
 // 	--Sam Humphries. Ebert, Roger (1999). Ebert's bigger little movie
 // 	glossary. Andrews McMeel. p. 172.
-// 
+//
 // https:// en.wikipedia.org/wiki/List_of_fictional_currencies
 // https:// en.wikipedia.org/wiki/Fictional_currency#Trends_in_the_use_of_fictional_currencies
 // http:// tvtropes.org/pmwiki/pmwiki.php/Main/WeWillSpendCreditsInTheFuture
@@ -557,7 +558,7 @@ func readElement(r io.Reader, element interface{}) error {
 		var txins []*wire.TxIn
 		for i := uint8(0); i < numScripts; i++ {
 			outpoint := new(wire.OutPoint)
-			txin := wire.NewTxIn(outpoint, nil)
+			txin := wire.NewTxIn(outpoint, nil, nil)
 			err = readElement(r, &txin)
 			if err != nil {
 				return err
diff --git a/shell.go b/shell.go
index 365b0ac2f..bc255bb79 100644
--- a/shell.go
+++ b/shell.go
@@ -314,7 +314,7 @@ func SendCoins(s uspv.SPVCon, adr btcutil.Address, sendAmt int64) error {
 			return err
 		}
 		// make new input from this utxo
-		thisInput := wire.NewTxIn(&utxo.Op, prevPKscript)
+		thisInput := wire.NewTxIn(&utxo.Op, prevPKscript, nil)
 		tx.AddTxIn(thisInput)
 		nokori -= utxo.Value
 		if nokori < -10000 { // minimum overage / fee is 1K now
diff --git a/uspv/hardmode.go b/uspv/hardmode.go
index 3e3bcdbac..efa5a6aa7 100644
--- a/uspv/hardmode.go
+++ b/uspv/hardmode.go
@@ -9,41 +9,128 @@ import (
 	"github.com/btcsuite/btcd/wire"
 )
 
-// BlockRootOK checks that all the txs in the block match the merkle root.
-// Only checks merkle root; it doesn't look at txs themselves.
-func BlockRootOK(blk wire.MsgBlock) bool {
-	var shas []*wire.ShaHash
-	for _, tx := range blk.Transactions { // make slice of txids
-		nSha := tx.TxSha()
-		shas = append(shas, &nSha)
+var (
+	WitMagicBytes = []byte{0x6a, 0x24, 0xaa, 0x21, 0xa9, 0xed}
+)
+
+// BlockRootOK checks for block self-consistency.
+// If the block has no wintess txs, and no coinbase witness commitment,
+// it only checks the tx merkle root.  If either a witness commitment or
+// any witnesses are detected, it also checks that as well.
+// Returns false if anything goes wrong, true if everything is fine.
+func BlockOK(blk wire.MsgBlock) bool {
+	var txids, wtxids []*wire.ShaHash // txids and wtxids
+	// witMode true if any tx has a wintess OR coinbase has wit commit
+	var witMode bool
+
+	for _, tx := range blk.Transactions { // make slice of (w)/txids
+		txid := tx.TxSha()
+		wtxid := tx.WTxSha()
+		if !witMode && !txid.IsEqual(&wtxid) {
+			witMode = true
+		}
+		txids = append(txids, &txid)
+		wtxids = append(wtxids, &wtxid)
 	}
-	neededLen := int(nextPowerOfTwo(uint32(len(shas)))) // kindof ugly
-	for len(shas) < neededLen {
-		shas = append(shas, nil) // pad out tx slice to get the full tree base
+
+	var commitBytes []byte
+	// try to extract coinbase witness commitment (even if !witMode)
+	cb := blk.Transactions[0]                 // get coinbase tx
+	for i := len(cb.TxOut) - 1; i >= 0; i-- { // start at the last txout
+		if bytes.HasPrefix(cb.TxOut[i].PkScript, WitMagicBytes) &&
+			len(cb.TxOut[i].PkScript) > 37 {
+			// 38 bytes or more, and starts with WitMagicBytes is a hit
+			commitBytes = cb.TxOut[i].PkScript[6:38]
+			witMode = true // it there is a wit commit it must be valid
+		}
 	}
-	for len(shas) > 1 { // calculate merkle root. Terse, eh?
-		shas = append(shas[2:], MakeMerkleParent(shas[0], shas[1]))
-	} // auto recognizes coinbase-only blocks
-	fmt.Printf("MRs calcd %s given %s\n",
-		shas[0].String(), blk.Header.MerkleRoot.String())
-	return blk.Header.MerkleRoot.IsEqual(shas[0])
+
+	if witMode { // witmode, so check witness tree
+		// first find ways witMode can be disqualified
+		if len(commitBytes) != 32 {
+			// witness in block but didn't find a wintess commitment; fail
+			log.Printf("block %s has witness but no witcommit",
+				blk.BlockSha().String())
+			return false
+		}
+		if len(cb.TxIn) != 1 {
+			log.Printf("block %s coinbase tx has %d txins (must be 1)",
+				blk.BlockSha().String(), len(cb.TxIn))
+			return false
+		}
+		if len(cb.TxIn[0].Witness) != 1 {
+			log.Printf("block %s coinbase has %d witnesses (must be 1)",
+				blk.BlockSha().String(), len(cb.TxIn[0].Witness))
+			return false
+		}
+
+		if len(cb.TxIn[0].Witness[0]) != 32 {
+			log.Printf("block %s coinbase has %d byte witness nonce (not 32)",
+				blk.BlockSha().String(), len(cb.TxIn[0].Witness[0]))
+			return false
+		}
+		// witness nonce is the cb's witness, subject to above constraints
+		witNonce, err := wire.NewShaHash(cb.TxIn[0].Witness[0])
+		if err != nil {
+			log.Printf("Witness nonce error: %s", err.Error())
+			return false // not sure why that'd happen but fail
+		}
+
+		var empty [32]byte
+		wtxids[0].SetBytes(empty[:]) // coinbase wtxid is 0x00...00
+
+		// witness root calculated from wtixds
+		witRoot := calcRoot(wtxids)
+
+		calcWitCommit := wire.DoubleSha256SH(
+			append(witRoot.Bytes(), witNonce.Bytes()...))
+
+		// witness root given in coinbase op_return
+		givenWitCommit, err := wire.NewShaHash(commitBytes)
+		if err != nil {
+			log.Printf("Witness root error: %s", err.Error())
+			return false // not sure why that'd happen but fail
+		}
+		// they should be the same.  If not, fail.
+		if !calcWitCommit.IsEqual(givenWitCommit) {
+			log.Printf("Block %s witRoot error: calc %s given %s",
+				blk.BlockSha().String(),
+				calcWitCommit.String(), givenWitCommit.String())
+			return false
+		}
+	}
+
+	// got through witMode check so that should be OK;
+	// check regular txid merkleroot.  Which is, like, trivial.
+	return blk.Header.MerkleRoot.IsEqual(calcRoot(txids))
+}
+
+// calcRoot calculates the merkle root of a slice of hashes.
+func calcRoot(hashes []*wire.ShaHash) *wire.ShaHash {
+	for len(hashes) < int(nextPowerOfTwo(uint32(len(hashes)))) {
+		hashes = append(hashes, nil) // pad out hash slice to get the full base
+	}
+	for len(hashes) > 1 { // calculate merkle root. Terse, eh?
+		hashes = append(hashes[2:], MakeMerkleParent(hashes[0], hashes[1]))
+	}
+	return hashes[0]
 }
 
 // IngestBlock is like IngestMerkleBlock but aralphic
 // different enough that it's better to have 2 separate functions
 func (s *SPVCon) IngestBlock(m *wire.MsgBlock) {
 	var err error
-	var buf bytes.Buffer
-	m.SerializeWitness(&buf)
-	fmt.Printf("block hex %x\n", buf.Bytes())
-	for i, tx := range m.Transactions {
+	//	var buf bytes.Buffer
+	//	m.SerializeWitness(&buf)
+	//	fmt.Printf("block hex %x\n", buf.Bytes())
+	for _, tx := range m.Transactions {
 		//		if i > 0 {
 		fmt.Printf("wtxid: %s\n", tx.WTxSha())
 		fmt.Printf(" txid: %s\n", tx.TxSha())
-		fmt.Printf("%d %s", i, TxToString(tx))
+		//		fmt.Printf("%d %s", i, TxToString(tx))
 		//		}
 	}
-	ok := BlockRootOK(*m) // check block self-consistency
+	ok := BlockOK(*m) // check block self-consistency
 	if !ok {
 		fmt.Printf("block %s not OK!!11\n", m.BlockSha().String())
 		return
@@ -71,6 +158,7 @@ func (s *SPVCon) IngestBlock(m *wire.MsgBlock) {
 	wg.Add(len(m.Transactions))
 	for i, tx := range m.Transactions {
 		go func() {
+			defer wg.Done()
 			hits, err := s.TS.Ingest(tx, hah.height)
 			if err != nil {
 				log.Printf("Incoming Tx error: %s\n", err.Error())
@@ -80,7 +168,6 @@ func (s *SPVCon) IngestBlock(m *wire.MsgBlock) {
 				log.Printf("block %d tx %d %s ingested and matches %d utxo/adrs.",
 					hah.height, i, tx.TxSha().String(), hits)
 			}
-			wg.Done()
 		}()
 	}
 	wg.Wait()
diff --git a/uspv/msghandler.go b/uspv/msghandler.go
index c3def1fb2..c1ddf99f9 100644
--- a/uspv/msghandler.go
+++ b/uspv/msghandler.go
@@ -88,10 +88,19 @@ func (s *SPVCon) fPositiveHandler() {
 			// send filter
 			s.SendFilter(filt)
 			fmt.Printf("sent filter %x\n", filt.MsgFilterLoad().Filter)
+
 			// clear the channel
-			for len(s.fPositives) != 0 {
-				fpAccumulator += <-s.fPositives
+		finClear:
+			for {
+				select {
+				case x := <-s.fPositives:
+					fpAccumulator += x
+
+				default:
+					break finClear
+				}
 			}
+
 			fmt.Printf("reset %d false positives\n", fpAccumulator)
 			// reset accumulator
 			fpAccumulator = 0
diff --git a/uspv/txstore.go b/uspv/txstore.go
index 3881b485c..0cf6b0285 100644
--- a/uspv/txstore.go
+++ b/uspv/txstore.go
@@ -139,6 +139,10 @@ func TxToString(tx *wire.MsgTx) string {
 	for i, in := range tx.TxIn {
 		str += fmt.Sprintf("Input %d: %s\n", i, in.PreviousOutPoint.String())
 		str += fmt.Sprintf("SigScript for input %d: %x\n", i, in.SignatureScript)
+		for j, wit := range in.Witness {
+			str += fmt.Sprintf("witness %d: %x\t", j, wit)
+		}
+		str += fmt.Sprintf("\n")
 	}
 	for i, out := range tx.TxOut {
 		if out != nil {
@@ -148,12 +152,6 @@ func TxToString(tx *wire.MsgTx) string {
 			str += fmt.Sprintf("output %d nil (WARNING)\n", i)
 		}
 	}
-	for i, wit := range tx.TxWitness {
-		if wit != nil {
-			str += fmt.Sprintf("Witness %d: %x\n", i, wit.ScriptWitness)
-		}
-	}
-
 	return str
 }
 

From a175e098fa7f6d67cd9b9f822b3c0753d53f63ad Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Wed, 17 Feb 2016 23:16:17 -0800
Subject: [PATCH 05/22] remove waitgroups from ingest block

It doesn't work; can ingest out of order within a block,
so if a block has gain then loss, you might read it as loss
then gains, missing the loss.
Also, tryna implement p2wpkh
---
 shell.go             | 31 ++++++++++++++++++++++++-------
 uspv/hardmode.go     | 37 ++++++++++++++-----------------------
 uspv/msghandler.go   | 32 +++++++++++++++++++++++---------
 uspv/sortsignsend.go | 14 +++++++++++++-
 4 files changed, 74 insertions(+), 40 deletions(-)

diff --git a/shell.go b/shell.go
index bc255bb79..c820c7404 100644
--- a/shell.go
+++ b/shell.go
@@ -61,7 +61,7 @@ func shell() {
 		log.Fatal(err)
 	}
 	if tip == 0 { // DB has never been used, set to birthday
-		tip = 1234 // hardcoded; later base on keyfile date?
+		tip = 20000 // hardcoded; later base on keyfile date?
 		err = SCon.TS.SetDBSyncHeight(tip)
 		if err != nil {
 			log.Fatal(err)
@@ -254,7 +254,11 @@ func Send(args []string) error {
 	}
 	// need args, fail
 	if len(args) < 2 {
-		return fmt.Errorf("need args: ssend amount(satoshis) address")
+		return fmt.Errorf("need args: ssend amount(satoshis) address wit?")
+	}
+	var wit bool // whether to send to p2wpkh
+	if len(args) > 2 {
+		wit = true
 	}
 	amt, err := strconv.ParseInt(args[0], 10, 64)
 	if err != nil {
@@ -270,7 +274,8 @@ func Send(args []string) error {
 	}
 	fmt.Printf("send %d to address: %s \n",
 		amt, adr.String())
-	err = SendCoins(SCon, adr, amt)
+
+	err = SendCoins(SCon, adr, amt, wit)
 	if err != nil {
 		return err
 	}
@@ -278,7 +283,10 @@ func Send(args []string) error {
 }
 
 // SendCoins does send coins, but it's very rudimentary
-func SendCoins(s uspv.SPVCon, adr btcutil.Address, sendAmt int64) error {
+// wit makes it into p2wpkh.  Which is not yet spendable.
+func SendCoins(
+	s uspv.SPVCon, adr btcutil.Address, sendAmt int64, wit bool) error {
+
 	var err error
 	var score int64
 	allUtxos, err := s.TS.GetAllUtxos()
@@ -297,10 +305,19 @@ func SendCoins(s uspv.SPVCon, adr btcutil.Address, sendAmt int64) error {
 
 	tx := wire.NewMsgTx() // make new tx
 	// make address script 76a914...88ac
-	adrScript, err := txscript.PayToAddrScript(adr)
-	if err != nil {
-		return err
+	var adrScript []byte
+	if wit {
+		adrScript, err = uspv.P2wpkhScript(adr)
+		if err != nil {
+			return err
+		}
+	} else { // non-wit, use old p2pkh
+		adrScript, err = txscript.PayToAddrScript(adr)
+		if err != nil {
+			return err
+		}
 	}
+
 	// make user specified txout and add to tx
 	txout := wire.NewTxOut(sendAmt, adrScript)
 	tx.AddTxOut(txout)
diff --git a/uspv/hardmode.go b/uspv/hardmode.go
index efa5a6aa7..b54e85264 100644
--- a/uspv/hardmode.go
+++ b/uspv/hardmode.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"fmt"
 	"log"
-	"sync"
 
 	"github.com/btcsuite/btcd/wire"
 )
@@ -123,13 +122,11 @@ func (s *SPVCon) IngestBlock(m *wire.MsgBlock) {
 	//	var buf bytes.Buffer
 	//	m.SerializeWitness(&buf)
 	//	fmt.Printf("block hex %x\n", buf.Bytes())
-	for _, tx := range m.Transactions {
-		//		if i > 0 {
-		fmt.Printf("wtxid: %s\n", tx.WTxSha())
-		fmt.Printf(" txid: %s\n", tx.TxSha())
-		//		fmt.Printf("%d %s", i, TxToString(tx))
-		//		}
-	}
+	//	for _, tx := range m.Transactions {
+	//		fmt.Printf("wtxid: %s\n", tx.WTxSha())
+	//		fmt.Printf(" txid: %s\n", tx.TxSha())
+	//		fmt.Printf("%d %s", i, TxToString(tx))
+	//	}
 	ok := BlockOK(*m) // check block self-consistency
 	if !ok {
 		fmt.Printf("block %s not OK!!11\n", m.BlockSha().String())
@@ -154,23 +151,17 @@ func (s *SPVCon) IngestBlock(m *wire.MsgBlock) {
 	// iterate through all txs in the block, looking for matches.
 	// this is slow and can be sped up by doing in-ram filters client side.
 	// kindof a pain to implement though and it's fast enough for now.
-	var wg sync.WaitGroup
-	wg.Add(len(m.Transactions))
 	for i, tx := range m.Transactions {
-		go func() {
-			defer wg.Done()
-			hits, err := s.TS.Ingest(tx, hah.height)
-			if err != nil {
-				log.Printf("Incoming Tx error: %s\n", err.Error())
-				return
-			}
-			if hits > 0 {
-				log.Printf("block %d tx %d %s ingested and matches %d utxo/adrs.",
-					hah.height, i, tx.TxSha().String(), hits)
-			}
-		}()
+		hits, err := s.TS.Ingest(tx, hah.height)
+		if err != nil {
+			log.Printf("Incoming Tx error: %s\n", err.Error())
+			return
+		}
+		if hits > 0 {
+			log.Printf("block %d tx %d %s ingested and matches %d utxo/adrs.",
+				hah.height, i, tx.TxSha().String(), hits)
+		}
 	}
-	wg.Wait()
 
 	// write to db that we've sync'd to the height indicated in the
 	// merkle block.  This isn't QUITE true since we haven't actually gotten
diff --git a/uspv/msghandler.go b/uspv/msghandler.go
index c1ddf99f9..ecb7c4742 100644
--- a/uspv/msghandler.go
+++ b/uspv/msghandler.go
@@ -95,7 +95,6 @@ func (s *SPVCon) fPositiveHandler() {
 				select {
 				case x := <-s.fPositives:
 					fpAccumulator += x
-
 				default:
 					break finClear
 				}
@@ -181,17 +180,32 @@ func (s *SPVCon) GetDataHandler(m *wire.MsgGetData) {
 	for i, thing := range m.InvList {
 		log.Printf("\t%d)%s : %s",
 			i, thing.Type.String(), thing.Hash.String())
-		if thing.Type != wire.InvTypeTx { // refuse non-tx reqs
-			log.Printf("We only respond to tx requests, ignoring")
+
+		// separate wittx and tx.  needed / combine?
+		// does the same thing right now
+		if thing.Type == wire.InvTypeWitnessTx {
+			tx, err := s.TS.GetTx(&thing.Hash)
+			if err != nil {
+				log.Printf("error getting tx %s: %s",
+					thing.Hash.String(), err.Error())
+			}
+			s.outMsgQueue <- tx
+			sent++
 			continue
 		}
-		tx, err := s.TS.GetTx(&thing.Hash)
-		if err != nil {
-			log.Printf("error getting tx %s: %s",
-				thing.Hash.String(), err.Error())
+		if thing.Type == wire.InvTypeTx {
+			tx, err := s.TS.GetTx(&thing.Hash)
+			if err != nil {
+				log.Printf("error getting tx %s: %s",
+					thing.Hash.String(), err.Error())
+			}
+			s.outMsgQueue <- tx
+			sent++
+			continue
 		}
-		s.outMsgQueue <- tx
-		sent++
+		// didn't match, so it's not something we're responding to
+		log.Printf("We only respond to tx requests, ignoring")
+
 	}
 	log.Printf("sent %d of %d requested items", sent, len(m.InvList))
 }
diff --git a/uspv/sortsignsend.go b/uspv/sortsignsend.go
index ec3e3ce05..d7b91610e 100644
--- a/uspv/sortsignsend.go
+++ b/uspv/sortsignsend.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/btcsuite/btcd/txscript"
 	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
 	"github.com/btcsuite/btcutil/bloom"
 	"github.com/btcsuite/btcutil/hdkeychain"
 	"github.com/btcsuite/btcutil/txsort"
@@ -41,6 +42,17 @@ func (s *SPVCon) Rebroadcast() {
 	return
 }
 
+func P2wpkhScript(adr btcutil.Address) ([]byte, error) {
+	switch adr := adr.(type) {
+	case *btcutil.AddressPubKeyHash:
+		sb := txscript.NewScriptBuilder()
+		sb.AddOp(txscript.OP_0)
+		sb.AddData(adr.ScriptAddress())
+		return sb.Script()
+	}
+	return nil, fmt.Errorf("%s is not pkh address", adr.String())
+}
+
 func (s *SPVCon) NewOutgoingTx(tx *wire.MsgTx) error {
 	txid := tx.TxSha()
 	// assign height of zero for txs we create
@@ -53,7 +65,7 @@ func (s *SPVCon) NewOutgoingTx(tx *wire.MsgTx) error {
 		return err
 	}
 	// make an inv message instead of a tx message to be polite
-	iv1 := wire.NewInvVect(wire.InvTypeTx, &txid)
+	iv1 := wire.NewInvVect(wire.InvTypeWitnessTx, &txid)
 	invMsg := wire.NewMsgInv()
 	err = invMsg.AddInvVect(iv1)
 	if err != nil {

From fbd17846d41e0878cfdf9ec15bc7b412aa2e1d42 Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Fri, 19 Feb 2016 00:32:22 -0800
Subject: [PATCH 06/22] split adrs up into two slices?

here is code to keep track of two sets of addresses -- the
regular ones and the witness ones.  But maybe those should be
merged becuase it's easy for other wallets to switch between them
and send to the same 20 byte pubkey hash but wintessified.
So maybe I'll change this stuff...
---
 shell.go             | 124 +++++++---------------------------
 uspv/eight333.go     |   2 +-
 uspv/sortsignsend.go | 155 ++++++++++++++++++++++++++++++++++++++++---
 uspv/txstore.go      | 104 ++++++++++++++---------------
 uspv/utxodb.go       |  93 ++++++++++++++++++++------
 5 files changed, 293 insertions(+), 185 deletions(-)

diff --git a/shell.go b/shell.go
index c820c7404..bcc84647b 100644
--- a/shell.go
+++ b/shell.go
@@ -2,16 +2,12 @@ package main
 
 import (
 	"bufio"
-	"bytes"
 	"fmt"
 	"log"
 	"os"
 	"strconv"
 	"strings"
 
-	"github.com/btcsuite/btcd/txscript"
-	"github.com/btcsuite/btcd/wire"
-
 	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/btcsuite/btcutil"
 
@@ -205,8 +201,12 @@ func Bal(args []string) error {
 	}
 	var score int64
 	for i, u := range allUtxos {
-		fmt.Printf("\tutxo %d height %d %s key: %d amt %d\n",
+		fmt.Printf("\tutxo %d height %d %s key:%d amt %d",
 			i, u.AtHeight, u.Op.String(), u.KeyIdx, u.Value)
+		if u.IsWit {
+			fmt.Printf(" WIT")
+		}
+		fmt.Printf("\n")
 		score += u.Value
 	}
 	height, _ := SCon.TS.GetDBSyncHeight()
@@ -214,6 +214,9 @@ func Bal(args []string) error {
 	for i, a := range SCon.TS.Adrs {
 		fmt.Printf("address %d %s\n", i, a.PkhAdr.String())
 	}
+	for i, a := range SCon.TS.WitAdrs {
+		fmt.Printf("address %d %s [WIT]\n", i, a.PkhAdr.String())
+	}
 	fmt.Printf("Total known utxos: %d\n", len(allUtxos))
 	fmt.Printf("Total spendable coin: %d\n", score)
 	fmt.Printf("DB sync height: %d\n", height)
@@ -222,12 +225,24 @@ func Bal(args []string) error {
 
 // Adr makes a new address.
 func Adr(args []string) error {
-	a, err := SCon.TS.NewAdr()
+
+	// if there's an arg, make 10 regular adrs
+	if len(args) > 0 {
+		for i := 0; i < 10; i++ {
+			_, err := SCon.TS.NewAdr(false)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// always make one segwit
+	a, err := SCon.TS.NewAdr(true)
 	if err != nil {
 		return err
 	}
-	fmt.Printf("made new address %s, %d addresses total\n",
-		a.String(), len(SCon.TS.Adrs))
+	fmt.Printf("made new address %s\n",
+		a.String())
 
 	return nil
 }
@@ -275,98 +290,7 @@ func Send(args []string) error {
 	fmt.Printf("send %d to address: %s \n",
 		amt, adr.String())
 
-	err = SendCoins(SCon, adr, amt, wit)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// SendCoins does send coins, but it's very rudimentary
-// wit makes it into p2wpkh.  Which is not yet spendable.
-func SendCoins(
-	s uspv.SPVCon, adr btcutil.Address, sendAmt int64, wit bool) error {
-
-	var err error
-	var score int64
-	allUtxos, err := s.TS.GetAllUtxos()
-	if err != nil {
-		return err
-	}
-
-	for _, utxo := range allUtxos {
-		score += utxo.Value
-	}
-	// important rule in bitcoin, output total > input total is invalid.
-	if sendAmt > score {
-		return fmt.Errorf("trying to send %d but %d available.",
-			sendAmt, score)
-	}
-
-	tx := wire.NewMsgTx() // make new tx
-	// make address script 76a914...88ac
-	var adrScript []byte
-	if wit {
-		adrScript, err = uspv.P2wpkhScript(adr)
-		if err != nil {
-			return err
-		}
-	} else { // non-wit, use old p2pkh
-		adrScript, err = txscript.PayToAddrScript(adr)
-		if err != nil {
-			return err
-		}
-	}
-
-	// make user specified txout and add to tx
-	txout := wire.NewTxOut(sendAmt, adrScript)
-	tx.AddTxOut(txout)
-
-	nokori := sendAmt // nokori is how much is needed on input side
-	for _, utxo := range allUtxos {
-		// generate pkscript to sign
-		prevPKscript, err := txscript.PayToAddrScript(
-			s.TS.Adrs[utxo.KeyIdx].PkhAdr)
-		if err != nil {
-			return err
-		}
-		// make new input from this utxo
-		thisInput := wire.NewTxIn(&utxo.Op, prevPKscript, nil)
-		tx.AddTxIn(thisInput)
-		nokori -= utxo.Value
-		if nokori < -10000 { // minimum overage / fee is 1K now
-			break
-		}
-	}
-	// there's enough left to make a change output
-	if nokori < -200000 {
-		change, err := s.TS.NewAdr()
-		if err != nil {
-			return err
-		}
-
-		changeScript, err := txscript.PayToAddrScript(change)
-		if err != nil {
-			return err
-		}
-		changeOut := wire.NewTxOut((-100000)-nokori, changeScript)
-		tx.AddTxOut(changeOut)
-	}
-
-	// use txstore method to sign
-	err = s.TS.SignThis(tx)
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("tx: %s", uspv.TxToString(tx))
-	buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize()))
-	tx.Serialize(buf)
-	fmt.Printf("tx: %x\n", buf.Bytes())
-
-	// send it out on the wire.  hope it gets there.
-	// we should deal with rejects.  Don't yet.
-	err = s.NewOutgoingTx(tx)
+	err = SCon.SendCoins(adr, amt, wit)
 	if err != nil {
 		return err
 	}
diff --git a/uspv/eight333.go b/uspv/eight333.go
index ca88e18f7..183562260 100644
--- a/uspv/eight333.go
+++ b/uspv/eight333.go
@@ -329,7 +329,7 @@ func (s *SPVCon) AskForBlocks() error {
 	s.headerMutex.Unlock() // checked, unlock
 	endPos := stat.Size()
 
-	headerTip := int32(endPos/80) - 1 // move back 1 header length to read
+	headerTip := int32(endPos / 80) // DONT move back 1 header length to read
 
 	dbTip, err := s.TS.GetDBSyncHeight()
 	if err != nil {
diff --git a/uspv/sortsignsend.go b/uspv/sortsignsend.go
index d7b91610e..894978e9e 100644
--- a/uspv/sortsignsend.go
+++ b/uspv/sortsignsend.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"log"
+	"sort"
 
 	"github.com/btcsuite/btcd/txscript"
 	"github.com/btcsuite/btcd/wire"
@@ -42,15 +43,29 @@ func (s *SPVCon) Rebroadcast() {
 	return
 }
 
-func P2wpkhScript(adr btcutil.Address) ([]byte, error) {
-	switch adr := adr.(type) {
-	case *btcutil.AddressPubKeyHash:
-		sb := txscript.NewScriptBuilder()
-		sb.AddOp(txscript.OP_0)
-		sb.AddData(adr.ScriptAddress())
-		return sb.Script()
+// make utxo slices sortable
+type utxoSlice []Utxo
+
+// Sort utxos just like txins -- Len, Less, Swap
+func (s utxoSlice) Len() int      { return len(s) }
+func (s utxoSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+
+// outpoint sort; First input hash (reversed / rpc-style), then index.
+func (s utxoSlice) Less(i, j int) bool {
+	// Input hashes are the same, so compare the index.
+	ihash := s[i].Op.Hash
+	jhash := s[j].Op.Hash
+	if ihash == jhash {
+		return s[i].Op.Index < s[j].Op.Index
 	}
-	return nil, fmt.Errorf("%s is not pkh address", adr.String())
+	// At this point, the hashes are not equal, so reverse them to
+	// big-endian and return the result of the comparison.
+	const hashSize = wire.HashSize
+	for b := 0; b < hashSize/2; b++ {
+		ihash[b], ihash[hashSize-1-b] = ihash[hashSize-1-b], ihash[b]
+		jhash[b], jhash[hashSize-1-b] = jhash[hashSize-1-b], jhash[b]
+	}
+	return bytes.Compare(ihash[:], jhash[:]) == -1
 }
 
 func (s *SPVCon) NewOutgoingTx(tx *wire.MsgTx) error {
@@ -75,6 +90,130 @@ func (s *SPVCon) NewOutgoingTx(tx *wire.MsgTx) error {
 	return nil
 }
 
+// SendCoins does send coins, but it's very rudimentary
+// wit makes it into p2wpkh.  Which is not yet spendable.
+func (s *SPVCon) SendCoins(adr btcutil.Address, sendAmt int64, wit bool) error {
+
+	var err error
+	var score int64
+	allUtxos, err := s.TS.GetAllUtxos()
+	if err != nil {
+		return err
+	}
+
+	for _, utxo := range allUtxos {
+		score += utxo.Value
+	}
+	// important rule in bitcoin, output total > input total is invalid.
+	if sendAmt > score {
+		return fmt.Errorf("trying to send %d but %d available.",
+			sendAmt, score)
+	}
+
+	///////////////////
+	tx := wire.NewMsgTx() // make new tx
+	// make address script 76a914...88ac or 0014...
+	outAdrScript, err := txscript.PayToAddrScript(adr)
+	if err != nil {
+		return err
+	}
+
+	// make user specified txout and add to tx
+	txout := wire.NewTxOut(sendAmt, outAdrScript)
+	tx.AddTxOut(txout)
+	////////////////////////////
+
+	// generate a utxo slice for your inputs
+	nokori := sendAmt // nokori is how much is needed on input side
+	var ins utxoSlice
+	for _, utxo := range allUtxos {
+		// yeah, lets add this utxo!
+		ins = append(ins, *utxo)
+		nokori -= utxo.Value
+		if nokori < -10000 { // minimum overage / fee is 1K now
+			break
+		}
+	}
+	// sort utxos on the input side
+	sort.Sort(ins)
+
+	// add all the utxos as txins
+	for _, in := range ins {
+		var prevPKscript []byte
+
+		prevPKscript, err := txscript.PayToAddrScript(
+			s.TS.Adrs[in.KeyIdx].PkhAdr)
+		if err != nil {
+			return err
+		}
+		tx.AddTxIn(wire.NewTxIn(&in.Op, prevPKscript, nil))
+	}
+
+	// there's enough left to make a change output
+	if nokori < -200000 {
+		change, err := s.TS.NewAdr(true) // change is witnessy
+		if err != nil {
+			return err
+		}
+
+		changeScript, err := txscript.PayToAddrScript(change)
+		if err != nil {
+			return err
+		}
+		changeOut := wire.NewTxOut((-100000)-nokori, changeScript)
+		tx.AddTxOut(changeOut)
+	}
+
+	for _, utxo := range allUtxos {
+		// generate pkscript to sign
+		prevPKscript, err := txscript.PayToAddrScript(
+			s.TS.Adrs[utxo.KeyIdx].PkhAdr)
+		if err != nil {
+			return err
+		}
+		// make new input from this utxo
+		thisInput := wire.NewTxIn(&utxo.Op, prevPKscript, nil)
+		tx.AddTxIn(thisInput)
+		nokori -= utxo.Value
+		if nokori < -10000 { // minimum overage / fee is 1K now
+			break
+		}
+	}
+	// there's enough left to make a change output
+	if nokori < -200000 {
+		change, err := s.TS.NewAdr(true) // change is witnessy
+		if err != nil {
+			return err
+		}
+
+		changeScript, err := txscript.PayToAddrScript(change)
+		if err != nil {
+			return err
+		}
+		changeOut := wire.NewTxOut((-100000)-nokori, changeScript)
+		tx.AddTxOut(changeOut)
+	}
+
+	// use txstore method to sign
+	err = s.TS.SignThis(tx)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("tx: %s", TxToString(tx))
+	buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize()))
+	tx.Serialize(buf)
+	fmt.Printf("tx: %x\n", buf.Bytes())
+
+	// send it out on the wire.  hope it gets there.
+	// we should deal with rejects.  Don't yet.
+	err = s.NewOutgoingTx(tx)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (t *TxStore) SignThis(tx *wire.MsgTx) error {
 	fmt.Printf("-= SignThis =-\n")
 
diff --git a/uspv/txstore.go b/uspv/txstore.go
index 0cf6b0285..ab93f0f3d 100644
--- a/uspv/txstore.go
+++ b/uspv/txstore.go
@@ -21,6 +21,7 @@ type TxStore struct {
 	OKMutex sync.Mutex
 
 	Adrs    []MyAdr  // endeavouring to acquire capital
+	WitAdrs []MyAdr  // separate segwit address slice
 	StateDB *bolt.DB // place to write all this down
 
 	// Params live here, not SCon
@@ -38,7 +39,8 @@ type Utxo struct { // cash money.
 	KeyIdx   uint32 // index for private key needed to sign / spend
 	Value    int64  // higher is better
 
-	//	IsCoinbase bool          // can't spend for a while
+	//		IsCoinbase bool          // can't spend for a while
+	IsWit bool // true if p2wpkh output
 }
 
 // Stxo is a utxo that has moved on.
@@ -77,8 +79,8 @@ func (t *TxStore) AddTxid(txid *wire.ShaHash, height int32) error {
 
 // ... or I'm gonna fade away
 func (t *TxStore) GimmeFilter() (*bloom.Filter, error) {
-	if len(t.Adrs) == 0 {
-		return nil, fmt.Errorf("no addresses to filter for")
+	if len(t.Adrs) == 0 && len(t.WitAdrs) == 9 {
+		return nil, fmt.Errorf("no address to filter for")
 	}
 
 	// get all utxos to add outpoints to filter
@@ -87,9 +89,15 @@ func (t *TxStore) GimmeFilter() (*bloom.Filter, error) {
 		return nil, err
 	}
 
-	elem := uint32(len(t.Adrs) + len(allUtxos))
+	elem := uint32(len(t.Adrs) + len(t.WitAdrs) + len(allUtxos))
 	f := bloom.NewFilter(elem, 0, 0.000001, wire.BloomUpdateAll)
-	for _, a := range t.Adrs {
+
+	// note there could be false positives since we're just looking
+	// for the 20 byte PKH without the opcodes.
+	for _, a := range t.Adrs { // add 20-byte pubkeyhash
+		f.Add(a.PkhAdr.ScriptAddress())
+	}
+	for _, a := range t.WitAdrs { // add witness 20-byte pubkeyhash
 		f.Add(a.PkhAdr.ScriptAddress())
 	}
 
@@ -211,6 +219,16 @@ func (u *Utxo) ToBytes() ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
+	// last byte indicates tx witness flags ( tx[5] from serialized tx)
+	// write a 1 at the end for p2wpkh (same as flags byte)
+	witByte := byte(0x00)
+	if u.IsWit {
+		witByte = 0x01
+	}
+	err = buf.WriteByte(witByte)
+	if err != nil {
+		return nil, err
+	}
 
 	return buf.Bytes(), nil
 }
@@ -224,8 +242,8 @@ func UtxoFromBytes(b []byte) (Utxo, error) {
 		return u, fmt.Errorf("nil input slice")
 	}
 	buf := bytes.NewBuffer(b)
-	if buf.Len() < 52 { // utxos are 52 bytes
-		return u, fmt.Errorf("Got %d bytes for utxo, expect 52", buf.Len())
+	if buf.Len() < 53 { // utxos are 53 bytes
+		return u, fmt.Errorf("Got %d bytes for utxo, expect 53", buf.Len())
 	}
 	// read 32 byte txid
 	err := u.Op.Hash.SetBytes(buf.Next(32))
@@ -252,38 +270,33 @@ func UtxoFromBytes(b []byte) (Utxo, error) {
 	if err != nil {
 		return u, err
 	}
+	// read 1 byte witness flags
+	witByte, err := buf.ReadByte()
+	if err != nil {
+		return u, err
+	}
+	if witByte != 0x00 {
+		u.IsWit = true
+	}
+
 	return u, nil
 }
 
 // ToBytes turns an Stxo into some bytes.
-// outpoint txid, outpoint idx, height, key idx, amt, spendheight, spendtxid
+// prevUtxo serialization, then spendheight [4], spendtxid [32]
 func (s *Stxo) ToBytes() ([]byte, error) {
 	var buf bytes.Buffer
-	// write 32 byte txid of the utxo
-	_, err := buf.Write(s.Op.Hash.Bytes())
+	// first serialize the utxo part
+	uBytes, err := s.Utxo.ToBytes()
 	if err != nil {
 		return nil, err
 	}
-	// write 4 byte outpoint index within the tx to spend
-	err = binary.Write(&buf, binary.BigEndian, s.Op.Index)
-	if err != nil {
-		return nil, err
-	}
-	// write 4 byte height of utxo
-	err = binary.Write(&buf, binary.BigEndian, s.AtHeight)
-	if err != nil {
-		return nil, err
-	}
-	// write 4 byte key index of utxo
-	err = binary.Write(&buf, binary.BigEndian, s.KeyIdx)
-	if err != nil {
-		return nil, err
-	}
-	// write 8 byte amount of money at the utxo
-	err = binary.Write(&buf, binary.BigEndian, s.Value)
+	// write that into the buffer first
+	_, err = buf.Write(uBytes)
 	if err != nil {
 		return nil, err
 	}
+
 	// write 4 byte height where the txo was spent
 	err = binary.Write(&buf, binary.BigEndian, s.SpendHeight)
 	if err != nil {
@@ -298,40 +311,21 @@ func (s *Stxo) ToBytes() ([]byte, error) {
 }
 
 // StxoFromBytes turns bytes into a Stxo.
+// first take the first 53 bytes as a utxo, then the next 36 for how it's spent.
 func StxoFromBytes(b []byte) (Stxo, error) {
 	var s Stxo
-	if b == nil {
-		return s, fmt.Errorf("nil input slice")
+	if len(b) < 89 {
+		return s, fmt.Errorf("Got %d bytes for stxo, expect 89", len(b))
 	}
-	buf := bytes.NewBuffer(b)
-	if buf.Len() < 88 { // stxos are 88 bytes
-		return s, fmt.Errorf("Got %d bytes for stxo, expect 88", buf.Len())
-	}
-	// read 32 byte txid
-	err := s.Op.Hash.SetBytes(buf.Next(32))
-	if err != nil {
-		return s, err
-	}
-	// read 4 byte outpoint index within the tx to spend
-	err = binary.Read(buf, binary.BigEndian, &s.Op.Index)
-	if err != nil {
-		return s, err
-	}
-	// read 4 byte height of utxo
-	err = binary.Read(buf, binary.BigEndian, &s.AtHeight)
-	if err != nil {
-		return s, err
-	}
-	// read 4 byte key index of utxo
-	err = binary.Read(buf, binary.BigEndian, &s.KeyIdx)
-	if err != nil {
-		return s, err
-	}
-	// read 8 byte amount of money at the utxo
-	err = binary.Read(buf, binary.BigEndian, &s.Value)
+
+	u, err := UtxoFromBytes(b[:53])
 	if err != nil {
 		return s, err
 	}
+	s.Utxo = u // assign the utxo
+
+	buf := bytes.NewBuffer(b[53:]) // make buffer for spend data
+
 	// read 4 byte spend height
 	err = binary.Read(buf, binary.BigEndian, &s.SpendHeight)
 	if err != nil {
diff --git a/uspv/utxodb.go b/uspv/utxodb.go
index 8550b50c7..b4010103a 100644
--- a/uspv/utxodb.go
+++ b/uspv/utxodb.go
@@ -22,7 +22,8 @@ var (
 	BKTTxns  = []byte("Txns")      // all txs we care about, for replays
 	BKTState = []byte("MiscState") // last state of DB
 	// these are in the state bucket
-	KEYNumKeys   = []byte("NumKeys")   // number of keys used
+	KEYStdKeys   = []byte("StdKeys")   // number of p2pkh keys used
+	KEYWitKeys   = []byte("WitKeys")   // number of p2wpkh keys used
 	KEYTipHeight = []byte("TipHeight") // height synced to
 )
 
@@ -52,7 +53,7 @@ func (ts *TxStore) OpenDB(filename string) error {
 			return err
 		}
 
-		numKeysBytes := sta.Get(KEYNumKeys)
+		numKeysBytes := sta.Get(KEYStdKeys)
 		if numKeysBytes != nil { // NumKeys exists, read into uint32
 			buf := bytes.NewBuffer(numKeysBytes)
 			err := binary.Read(buf, binary.BigEndian, &numKeys)
@@ -67,7 +68,7 @@ func (ts *TxStore) OpenDB(filename string) error {
 			if err != nil {
 				return err
 			}
-			err = sta.Put(KEYNumKeys, buf.Bytes())
+			err = sta.Put(KEYStdKeys, buf.Bytes())
 			if err != nil {
 				return err
 			}
@@ -82,19 +83,42 @@ func (ts *TxStore) OpenDB(filename string) error {
 
 // NewAdr creates a new, never before seen address, and increments the
 // DB counter as well as putting it in the ram Adrs store, and returns it
-func (ts *TxStore) NewAdr() (*btcutil.AddressPubKeyHash, error) {
+func (ts *TxStore) NewAdr(wit bool) (btcutil.Address, error) {
 	if ts.Param == nil {
-		return nil, fmt.Errorf("nil param")
-	}
-	n := uint32(len(ts.Adrs))
-	priv, err := ts.rootPrivKey.Child(n + hdkeychain.HardenedKeyStart)
-	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("NewAdr error: nil param")
 	}
 
-	newAdr, err := priv.Address(ts.Param)
-	if err != nil {
-		return nil, err
+	priv := new(hdkeychain.ExtendedKey)
+	var err error
+	var n uint32
+	var nAdr btcutil.Address
+
+	if wit {
+		n = uint32(len(ts.WitAdrs))
+
+		// witness keys are another branch down from the rootpriv
+		ephpriv, err := ts.rootPrivKey.Child(1<<30 + hdkeychain.HardenedKeyStart)
+		if err != nil {
+			return nil, err
+		}
+		priv, err = ephpriv.Child(n + hdkeychain.HardenedKeyStart)
+		if err != nil {
+			return nil, err
+		}
+		nAdr, err = priv.WitnessAddress(ts.Param)
+		if err != nil {
+			return nil, err
+		}
+	} else { // regular p2pkh
+		n = uint32(len(ts.Adrs))
+		priv, err = ts.rootPrivKey.Child(n + hdkeychain.HardenedKeyStart)
+		if err != nil {
+			return nil, err
+		}
+		nAdr, err = priv.Address(ts.Param)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	// total number of keys (now +1) into 4 bytes
@@ -107,18 +131,24 @@ func (ts *TxStore) NewAdr() (*btcutil.AddressPubKeyHash, error) {
 	// write to db file
 	err = ts.StateDB.Update(func(btx *bolt.Tx) error {
 		sta := btx.Bucket(BKTState)
-		return sta.Put(KEYNumKeys, buf.Bytes())
+		if wit {
+			return sta.Put(KEYStdKeys, buf.Bytes())
+		}
+		return sta.Put(KEYWitKeys, buf.Bytes())
 	})
 	if err != nil {
 		return nil, err
 	}
 	// add in to ram.
 	var ma MyAdr
-	ma.PkhAdr = newAdr
+	ma.PkhAdr = nAdr
 	ma.KeyIdx = n
-	ts.Adrs = append(ts.Adrs, ma)
-
-	return newAdr, nil
+	if wit {
+		ts.WitAdrs = append(ts.WitAdrs, ma)
+	} else {
+		ts.Adrs = append(ts.Adrs, ma)
+	}
+	return nAdr, nil
 }
 
 // SetDBSyncHeight sets sync height of the db, indicated the latest block
@@ -369,17 +399,38 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 	// also generate PKscripts for all addresses (maybe keep storing these?)
 	for _, adr := range ts.Adrs {
 		// iterate through all our addresses
+
+		// convert regular address to witness address.  (split adrs later)
+		wa, err := btcutil.NewAddressWitnessPubKeyHash(
+			adr.PkhAdr.ScriptAddress(), ts.Param)
+		if err != nil {
+			return hits, err
+		}
+
+		wPKscript, err := txscript.PayToAddrScript(wa)
+		if err != nil {
+			return hits, err
+		}
 		aPKscript, err := txscript.PayToAddrScript(adr.PkhAdr)
 		if err != nil {
 			return hits, err
 		}
-		// iterate through all outputs of this tx
+
+		// iterate through all outputs of this tx, see if we gain
 		for i, out := range tx.TxOut {
-			if bytes.Equal(out.PkScript, aPKscript) { // new utxo for us
-				var newu Utxo
+
+			// detect p2wpkh
+			witBool := false
+			if bytes.Equal(out.PkScript, wPKscript) {
+				witBool = true
+			}
+
+			if bytes.Equal(out.PkScript, aPKscript) || witBool { // new utxo found
+				var newu Utxo // create new utxo and copy into it
 				newu.AtHeight = height
 				newu.KeyIdx = adr.KeyIdx
 				newu.Value = out.Value
+				newu.IsWit = witBool // copy witness version from pkscript
 				var newop wire.OutPoint
 				newop.Hash = tx.TxSha()
 				newop.Index = uint32(i)

From d8e62f689862c06e4892ee5015c72be168abee74 Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Fri, 19 Feb 2016 01:24:23 -0800
Subject: [PATCH 07/22] back to one adr[] slice

all adrs can be converted on the fly and displayed both ways.
in the actual UI this shouldn't happen though.

Also utxos are sorted and signed properly.  utxo selection
is still pretty low tech.
---
 shell.go             |  26 +++++-----
 uspv/sortsignsend.go | 120 ++++++++++++++++++++++++-------------------
 uspv/txstore.go      |   8 +--
 uspv/utxodb.go       |  58 +++++++--------------
 4 files changed, 99 insertions(+), 113 deletions(-)

diff --git a/shell.go b/shell.go
index bcc84647b..81b607f1b 100644
--- a/shell.go
+++ b/shell.go
@@ -212,11 +212,14 @@ func Bal(args []string) error {
 	height, _ := SCon.TS.GetDBSyncHeight()
 
 	for i, a := range SCon.TS.Adrs {
-		fmt.Printf("address %d %s\n", i, a.PkhAdr.String())
-	}
-	for i, a := range SCon.TS.WitAdrs {
-		fmt.Printf("address %d %s [WIT]\n", i, a.PkhAdr.String())
+		wa, err := btcutil.NewAddressWitnessPubKeyHash(
+			a.PkhAdr.ScriptAddress(), Params)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("address %d %s OR %s\n", i, a.PkhAdr.String(), wa.String())
 	}
+
 	fmt.Printf("Total known utxos: %d\n", len(allUtxos))
 	fmt.Printf("Total spendable coin: %d\n", score)
 	fmt.Printf("DB sync height: %d\n", height)
@@ -226,18 +229,18 @@ func Bal(args []string) error {
 // Adr makes a new address.
 func Adr(args []string) error {
 
-	// if there's an arg, make 10 regular adrs
+	// if there's an arg, make 10 adrs
 	if len(args) > 0 {
 		for i := 0; i < 10; i++ {
-			_, err := SCon.TS.NewAdr(false)
+			_, err := SCon.TS.NewAdr()
 			if err != nil {
 				return err
 			}
 		}
 	}
 
-	// always make one segwit
-	a, err := SCon.TS.NewAdr(true)
+	// always make one
+	a, err := SCon.TS.NewAdr()
 	if err != nil {
 		return err
 	}
@@ -271,10 +274,7 @@ func Send(args []string) error {
 	if len(args) < 2 {
 		return fmt.Errorf("need args: ssend amount(satoshis) address wit?")
 	}
-	var wit bool // whether to send to p2wpkh
-	if len(args) > 2 {
-		wit = true
-	}
+
 	amt, err := strconv.ParseInt(args[0], 10, 64)
 	if err != nil {
 		return err
@@ -290,7 +290,7 @@ func Send(args []string) error {
 	fmt.Printf("send %d to address: %s \n",
 		amt, adr.String())
 
-	err = SCon.SendCoins(adr, amt, wit)
+	err = SCon.SendCoins(adr, amt)
 	if err != nil {
 		return err
 	}
diff --git a/uspv/sortsignsend.go b/uspv/sortsignsend.go
index 894978e9e..93813a236 100644
--- a/uspv/sortsignsend.go
+++ b/uspv/sortsignsend.go
@@ -92,7 +92,7 @@ func (s *SPVCon) NewOutgoingTx(tx *wire.MsgTx) error {
 
 // SendCoins does send coins, but it's very rudimentary
 // wit makes it into p2wpkh.  Which is not yet spendable.
-func (s *SPVCon) SendCoins(adr btcutil.Address, sendAmt int64, wit bool) error {
+func (s *SPVCon) SendCoins(adr btcutil.Address, sendAmt int64) error {
 
 	var err error
 	var score int64
@@ -118,86 +118,98 @@ func (s *SPVCon) SendCoins(adr btcutil.Address, sendAmt int64, wit bool) error {
 		return err
 	}
 
-	// make user specified txout and add to tx
-	txout := wire.NewTxOut(sendAmt, outAdrScript)
-	tx.AddTxOut(txout)
 	////////////////////////////
 
 	// generate a utxo slice for your inputs
-	nokori := sendAmt // nokori is how much is needed on input side
 	var ins utxoSlice
+
+	// add utxos until we've had enough
+	nokori := sendAmt // nokori is how much is needed on input side
 	for _, utxo := range allUtxos {
 		// yeah, lets add this utxo!
 		ins = append(ins, *utxo)
 		nokori -= utxo.Value
-		if nokori < -10000 { // minimum overage / fee is 1K now
+		if nokori < -10000 { // minimum overage / fee is 10K now
 			break
 		}
 	}
+
 	// sort utxos on the input side
 	sort.Sort(ins)
 
-	// add all the utxos as txins
+	// make user specified txout and add to tx
+	txout := wire.NewTxOut(sendAmt, outAdrScript)
+	tx.AddTxOut(txout)
+	// see if there's enough left to also add a change output
+	if nokori < -200000 {
+		changeOld, err := s.TS.NewAdr() // change is witnessy
+		if err != nil {
+			return err
+		}
+		changeAdr, err := btcutil.NewAddressWitnessPubKeyHash(
+			changeOld.ScriptAddress(), s.TS.Param)
+		if err != nil {
+			return err
+		}
+
+		changeScript, err := txscript.PayToAddrScript(changeAdr)
+		if err != nil {
+			return err
+		}
+		changeOut := wire.NewTxOut((-100000)-nokori, changeScript)
+		tx.AddTxOut(changeOut)
+	}
+
+	// generate previous pkscripts for all the (now sorted) utxos
+	// then make txins with the utxo and prevpk, and insert them into the tx
 	for _, in := range ins {
-		var prevPKscript []byte
+		var prevPKs []byte
 
-		prevPKscript, err := txscript.PayToAddrScript(
-			s.TS.Adrs[in.KeyIdx].PkhAdr)
-		if err != nil {
-			return err
+		// if wit utxo, convert address to generate pkscript
+		if in.IsWit {
+			wa, err := btcutil.NewAddressWitnessPubKeyHash(
+				s.TS.Adrs[in.KeyIdx].PkhAdr.ScriptAddress(), s.TS.Param)
+			prevPKs, err = txscript.PayToAddrScript(wa)
+			if err != nil {
+				return err
+			}
+		} else { // otherwise generate directly
+			prevPKs, err = txscript.PayToAddrScript(
+				s.TS.Adrs[in.KeyIdx].PkhAdr)
+			if err != nil {
+				return err
+			}
 		}
-		tx.AddTxIn(wire.NewTxIn(&in.Op, prevPKscript, nil))
+		tx.AddTxIn(wire.NewTxIn(&in.Op, prevPKs, nil))
 	}
+	// sort tx -- this only will change txouts since inputs are already sorted
+	txsort.InPlaceSort(tx)
 
-	// there's enough left to make a change output
-	if nokori < -200000 {
-		change, err := s.TS.NewAdr(true) // change is witnessy
+	// tx is ready for signing,
+	sigStash := make([][]byte, len(ins))
+	for i, txin := range tx.TxIn {
+		// pick key
+		child, err := s.TS.rootPrivKey.Child(
+			ins[i].KeyIdx + hdkeychain.HardenedKeyStart)
+		if err != nil {
+			return err
+		}
+		priv, err := child.ECPrivKey()
 		if err != nil {
 			return err
 		}
 
-		changeScript, err := txscript.PayToAddrScript(change)
+		// This is where witness based sighash types need to happen
+		// sign into stash
+		sigStash[i], err = txscript.SignatureScript(
+			tx, i, txin.SignatureScript, txscript.SigHashAll, priv, true)
 		if err != nil {
 			return err
 		}
-		changeOut := wire.NewTxOut((-100000)-nokori, changeScript)
-		tx.AddTxOut(changeOut)
-	}
-
-	for _, utxo := range allUtxos {
-		// generate pkscript to sign
-		prevPKscript, err := txscript.PayToAddrScript(
-			s.TS.Adrs[utxo.KeyIdx].PkhAdr)
-		if err != nil {
-			return err
-		}
-		// make new input from this utxo
-		thisInput := wire.NewTxIn(&utxo.Op, prevPKscript, nil)
-		tx.AddTxIn(thisInput)
-		nokori -= utxo.Value
-		if nokori < -10000 { // minimum overage / fee is 1K now
-			break
-		}
 	}
-	// there's enough left to make a change output
-	if nokori < -200000 {
-		change, err := s.TS.NewAdr(true) // change is witnessy
-		if err != nil {
-			return err
-		}
-
-		changeScript, err := txscript.PayToAddrScript(change)
-		if err != nil {
-			return err
-		}
-		changeOut := wire.NewTxOut((-100000)-nokori, changeScript)
-		tx.AddTxOut(changeOut)
-	}
-
-	// use txstore method to sign
-	err = s.TS.SignThis(tx)
-	if err != nil {
-		return err
+	// swap sigs into sigScripts in txins
+	for i, txin := range tx.TxIn {
+		txin.SignatureScript = sigStash[i]
 	}
 
 	fmt.Printf("tx: %s", TxToString(tx))
diff --git a/uspv/txstore.go b/uspv/txstore.go
index ab93f0f3d..89b83d5fa 100644
--- a/uspv/txstore.go
+++ b/uspv/txstore.go
@@ -21,7 +21,6 @@ type TxStore struct {
 	OKMutex sync.Mutex
 
 	Adrs    []MyAdr  // endeavouring to acquire capital
-	WitAdrs []MyAdr  // separate segwit address slice
 	StateDB *bolt.DB // place to write all this down
 
 	// Params live here, not SCon
@@ -79,7 +78,7 @@ func (t *TxStore) AddTxid(txid *wire.ShaHash, height int32) error {
 
 // ... or I'm gonna fade away
 func (t *TxStore) GimmeFilter() (*bloom.Filter, error) {
-	if len(t.Adrs) == 0 && len(t.WitAdrs) == 9 {
+	if len(t.Adrs) == 0 {
 		return nil, fmt.Errorf("no address to filter for")
 	}
 
@@ -89,7 +88,7 @@ func (t *TxStore) GimmeFilter() (*bloom.Filter, error) {
 		return nil, err
 	}
 
-	elem := uint32(len(t.Adrs) + len(t.WitAdrs) + len(allUtxos))
+	elem := uint32(len(t.Adrs) + len(allUtxos))
 	f := bloom.NewFilter(elem, 0, 0.000001, wire.BloomUpdateAll)
 
 	// note there could be false positives since we're just looking
@@ -97,9 +96,6 @@ func (t *TxStore) GimmeFilter() (*bloom.Filter, error) {
 	for _, a := range t.Adrs { // add 20-byte pubkeyhash
 		f.Add(a.PkhAdr.ScriptAddress())
 	}
-	for _, a := range t.WitAdrs { // add witness 20-byte pubkeyhash
-		f.Add(a.PkhAdr.ScriptAddress())
-	}
 
 	for _, u := range allUtxos {
 		f.AddOutPoint(&u.Op)
diff --git a/uspv/utxodb.go b/uspv/utxodb.go
index b4010103a..e1013c865 100644
--- a/uspv/utxodb.go
+++ b/uspv/utxodb.go
@@ -22,8 +22,7 @@ var (
 	BKTTxns  = []byte("Txns")      // all txs we care about, for replays
 	BKTState = []byte("MiscState") // last state of DB
 	// these are in the state bucket
-	KEYStdKeys   = []byte("StdKeys")   // number of p2pkh keys used
-	KEYWitKeys   = []byte("WitKeys")   // number of p2wpkh keys used
+	KEYNumKeys   = []byte("NumKeys")   // number of p2pkh keys used
 	KEYTipHeight = []byte("TipHeight") // height synced to
 )
 
@@ -53,7 +52,7 @@ func (ts *TxStore) OpenDB(filename string) error {
 			return err
 		}
 
-		numKeysBytes := sta.Get(KEYStdKeys)
+		numKeysBytes := sta.Get(KEYNumKeys)
 		if numKeysBytes != nil { // NumKeys exists, read into uint32
 			buf := bytes.NewBuffer(numKeysBytes)
 			err := binary.Read(buf, binary.BigEndian, &numKeys)
@@ -68,7 +67,7 @@ func (ts *TxStore) OpenDB(filename string) error {
 			if err != nil {
 				return err
 			}
-			err = sta.Put(KEYStdKeys, buf.Bytes())
+			err = sta.Put(KEYNumKeys, buf.Bytes())
 			if err != nil {
 				return err
 			}
@@ -83,7 +82,7 @@ func (ts *TxStore) OpenDB(filename string) error {
 
 // NewAdr creates a new, never before seen address, and increments the
 // DB counter as well as putting it in the ram Adrs store, and returns it
-func (ts *TxStore) NewAdr(wit bool) (btcutil.Address, error) {
+func (ts *TxStore) NewAdr() (btcutil.Address, error) {
 	if ts.Param == nil {
 		return nil, fmt.Errorf("NewAdr error: nil param")
 	}
@@ -93,32 +92,14 @@ func (ts *TxStore) NewAdr(wit bool) (btcutil.Address, error) {
 	var n uint32
 	var nAdr btcutil.Address
 
-	if wit {
-		n = uint32(len(ts.WitAdrs))
-
-		// witness keys are another branch down from the rootpriv
-		ephpriv, err := ts.rootPrivKey.Child(1<<30 + hdkeychain.HardenedKeyStart)
-		if err != nil {
-			return nil, err
-		}
-		priv, err = ephpriv.Child(n + hdkeychain.HardenedKeyStart)
-		if err != nil {
-			return nil, err
-		}
-		nAdr, err = priv.WitnessAddress(ts.Param)
-		if err != nil {
-			return nil, err
-		}
-	} else { // regular p2pkh
-		n = uint32(len(ts.Adrs))
-		priv, err = ts.rootPrivKey.Child(n + hdkeychain.HardenedKeyStart)
-		if err != nil {
-			return nil, err
-		}
-		nAdr, err = priv.Address(ts.Param)
-		if err != nil {
-			return nil, err
-		}
+	n = uint32(len(ts.Adrs))
+	priv, err = ts.rootPrivKey.Child(n + hdkeychain.HardenedKeyStart)
+	if err != nil {
+		return nil, err
+	}
+	nAdr, err = priv.Address(ts.Param)
+	if err != nil {
+		return nil, err
 	}
 
 	// total number of keys (now +1) into 4 bytes
@@ -131,10 +112,9 @@ func (ts *TxStore) NewAdr(wit bool) (btcutil.Address, error) {
 	// write to db file
 	err = ts.StateDB.Update(func(btx *bolt.Tx) error {
 		sta := btx.Bucket(BKTState)
-		if wit {
-			return sta.Put(KEYStdKeys, buf.Bytes())
-		}
-		return sta.Put(KEYWitKeys, buf.Bytes())
+
+		return sta.Put(KEYNumKeys, buf.Bytes())
+
 	})
 	if err != nil {
 		return nil, err
@@ -143,11 +123,9 @@ func (ts *TxStore) NewAdr(wit bool) (btcutil.Address, error) {
 	var ma MyAdr
 	ma.PkhAdr = nAdr
 	ma.KeyIdx = n
-	if wit {
-		ts.WitAdrs = append(ts.WitAdrs, ma)
-	} else {
-		ts.Adrs = append(ts.Adrs, ma)
-	}
+
+	ts.Adrs = append(ts.Adrs, ma)
+
 	return nAdr, nil
 }
 

From 233ec9538c765b284ef826a61942250ec8ffbf24 Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Fri, 19 Feb 2016 14:49:04 -0800
Subject: [PATCH 08/22] add testing sighash folder

---
 .gitignore               |  3 ++
 sighash143/sighash143.go | 91 ++++++++++++++++++++++++++++++++++++++++
 uspv/sortsignsend.go     |  1 +
 uspv/txstore.go          | 25 +++++++++++
 4 files changed, 120 insertions(+)
 create mode 100644 sighash143/sighash143.go

diff --git a/.gitignore b/.gitignore
index b55095c70..2d4d343af 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,6 +33,9 @@ cmd/cmd
 cmd/lncli/lncli
 cmd/lnshell/lnshell
 
+sighash143/sighash143
+sighash143/*.hex
+
 test_wal/*
 
 # vim
diff --git a/sighash143/sighash143.go b/sighash143/sighash143.go
new file mode 100644
index 000000000..7d93206ab
--- /dev/null
+++ b/sighash143/sighash143.go
@@ -0,0 +1,91 @@
+package main
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"strings"
+
+	"github.com/btcsuite/btcd/txscript"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/lightningnetwork/lnd/uspv"
+)
+
+const (
+	inspk0 = "2103c9f4836b9a4f77fc0d81f7bcb01b7f1b35916864b9476c241ce9fc198bd25432ac"
+	inamt0 = uint64(625000000)
+
+	inspk1 = "00141d0f172a0ecb48aee1be1f2687d2963ae33f71a1"
+	inamt1 = uint64(600000000)
+)
+
+func outpointBytesLil(op wire.OutPoint) []byte {
+	var buf bytes.Buffer
+	// ignore errors because.. whatever
+	_ = binary.Write(&buf, binary.LittleEndian, op.Index)
+
+	b := op.Hash[:]
+	return append(b, buf.Bytes()...)
+}
+
+func calcSignatureHash(
+	hashType txscript.SigHashType, tx *wire.MsgTx, idx int) []byte {
+
+	return nil
+}
+
+// if sighash_ALL, hash of all txin outpoints, sequentially.
+// if other sighash type, 0x00 * 32
+func calcHashPrevOuts(tx *wire.MsgTx) [32]byte {
+
+	for _, in := range tx.TxIn {
+		//		in.PreviousOutPoint.Hash
+	}
+
+	return wire.DoubleSha256SH(tx.TxIn[0].SignatureScript)
+}
+
+func main() {
+	fmt.Printf("sighash 143\n")
+
+	// get previous pkscripts for inputs 0 and 1 from hex
+	in0spk, err := hex.DecodeString(string(inspk0))
+	if err != nil {
+		log.Fatal(err)
+	}
+	in1spk, err := hex.DecodeString(string(inspk1))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// load tx skeleton from local file
+	fmt.Printf("loading tx from file tx.hex\n")
+	txhex, err := ioutil.ReadFile("tx.hex")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	txhex = []byte(strings.TrimSpace(string(txhex)))
+	txbytes, err := hex.DecodeString(string(txhex))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Printf("loaded %d byte tx %x\n", len(txbytes), txbytes)
+
+	// 	make tx
+	ttx := wire.NewMsgTx()
+
+	// deserialize into tx
+	buf := bytes.NewBuffer(txbytes)
+	ttx.Deserialize(buf)
+
+	ttx.TxIn[0].SignatureScript = in0spk
+	ttx.TxIn[1].SignatureScript = in1spk
+
+	fmt.Printf(uspv.TxToString(ttx))
+
+}
diff --git a/uspv/sortsignsend.go b/uspv/sortsignsend.go
index 93813a236..f1ef36474 100644
--- a/uspv/sortsignsend.go
+++ b/uspv/sortsignsend.go
@@ -226,6 +226,7 @@ func (s *SPVCon) SendCoins(adr btcutil.Address, sendAmt int64) error {
 	return nil
 }
 
+// SignThis isn't used anymore...
 func (t *TxStore) SignThis(tx *wire.MsgTx) error {
 	fmt.Printf("-= SignThis =-\n")
 
diff --git a/uspv/txstore.go b/uspv/txstore.go
index 89b83d5fa..2fe582f35 100644
--- a/uspv/txstore.go
+++ b/uspv/txstore.go
@@ -185,6 +185,20 @@ func outPointToBytes(op *wire.OutPoint) ([]byte, error) {
 	return buf.Bytes(), nil
 }
 
+/*----- serialization for utxos ------- */
+/* Utxos serialization:
+byte length   desc   at offset
+
+32	txid		0
+4	idx		32
+4	height	36
+4	keyidx	40
+8	amt		44
+1	flag		52
+
+end len 	53
+*/
+
 // ToBytes turns a Utxo into some bytes.
 // note that the txid is the first 36 bytes and in our use cases will be stripped
 // off, but is left here for other applications
@@ -278,6 +292,17 @@ func UtxoFromBytes(b []byte) (Utxo, error) {
 	return u, nil
 }
 
+/*----- serialization for stxos ------- */
+/* Stxo serialization:
+byte length   desc   at offset
+
+53	utxo		0
+4	sheight	53
+32	stxid	57
+
+end len 	89
+*/
+
 // ToBytes turns an Stxo into some bytes.
 // prevUtxo serialization, then spendheight [4], spendtxid [32]
 func (s *Stxo) ToBytes() ([]byte, error) {

From f6cf4b87f064a54272246789ca31a67a3b0eecf0 Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Sat, 20 Feb 2016 00:42:07 -0800
Subject: [PATCH 09/22] pretty ugly.  But got the hash from the bip143 example.

---
 sighash143/sighash143.go | 206 +++++++++++++++++++++++++++++++++++----
 1 file changed, 185 insertions(+), 21 deletions(-)

diff --git a/sighash143/sighash143.go b/sighash143/sighash143.go
index 7d93206ab..086125844 100644
--- a/sighash143/sighash143.go
+++ b/sighash143/sighash143.go
@@ -5,8 +5,10 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
+	"math"
 	"strings"
 
 	"github.com/btcsuite/btcd/txscript"
@@ -16,36 +18,130 @@ import (
 
 const (
 	inspk0 = "2103c9f4836b9a4f77fc0d81f7bcb01b7f1b35916864b9476c241ce9fc198bd25432ac"
-	inamt0 = uint64(625000000)
+	inamt0 = int64(625000000)
 
 	inspk1 = "00141d0f172a0ecb48aee1be1f2687d2963ae33f71a1"
-	inamt1 = uint64(600000000)
+	inamt1 = int64(600000000)
 )
 
-func outpointBytesLil(op wire.OutPoint) []byte {
-	var buf bytes.Buffer
-	// ignore errors because.. whatever
-	_ = binary.Write(&buf, binary.LittleEndian, op.Index)
+// calcWitnessSignatureHash is the witnessified version of calcSignatureHash
+func calcWitnessSignatureHash(
+	hashType txscript.SigHashType, tx *wire.MsgTx, idx int, amt int64) []byte {
+	// in the script.go calcSignatureHash(), idx is assumed safe, so I guess
+	// that's OK here too...
 
-	b := op.Hash[:]
-	return append(b, buf.Bytes()...)
+	// first get hashPrevOuts, hashSequence, and hashOutputs
+	hashPrevOuts := calcHashPrevOuts(tx, hashType)
+	hashSequence := calcHashSequence(tx, hashType)
+	hashOutputs := calcHashOutputs(tx, idx, hashType)
+
+	var buf4 [4]byte // buffer for 4-byte stuff
+	var buf8 [8]byte // buffer for 8-byte stuff
+	var pre []byte   // the pre-image we're generating
+
+	binary.LittleEndian.PutUint32(buf4[:], uint32(tx.Version))
+	pre = append(pre, buf4[:]...)
+
+	pre = append(pre, hashPrevOuts.Bytes()...)
+	pre = append(pre, hashSequence.Bytes()...)
+
+	// outpoint being spent
+	pre = append(pre, tx.TxIn[idx].PreviousOutPoint.Hash.Bytes()...)
+	binary.LittleEndian.PutUint32(buf4[:], tx.TxIn[idx].PreviousOutPoint.Index)
+	pre = append(pre, buf4[:]...)
+
+	// scriptCode which is some new thing
+	sCode := []byte{0x19, 0x76, 0xa9, 0x14}
+	sCode = append(sCode, tx.TxIn[idx].SignatureScript[2:22]...)
+	sCode = append(sCode, []byte{0x88, 0xac}...)
+	pre = append(pre, sCode...)
+
+	// amount being signed off
+	binary.LittleEndian.PutUint64(buf8[:], uint64(amt))
+	pre = append(pre, buf8[:]...)
+
+	// nsequence of input
+	binary.LittleEndian.PutUint32(buf4[:], tx.TxIn[idx].Sequence)
+	pre = append(pre, buf4[:]...)
+
+	pre = append(pre, hashOutputs.Bytes()...)
+
+	// locktime
+	binary.LittleEndian.PutUint32(buf4[:], tx.LockTime)
+	pre = append(pre, buf4[:]...)
+
+	// hashType... in 4 bytes, instead of 1, because reasons.
+	binary.LittleEndian.PutUint32(buf4[:], uint32(hashType))
+	pre = append(pre, buf4[:]...)
+
+	fmt.Printf("pre: %x\n", pre)
+	hsh := wire.DoubleSha256SH(pre)
+	return hsh.Bytes()
 }
 
-func calcSignatureHash(
-	hashType txscript.SigHashType, tx *wire.MsgTx, idx int) []byte {
-
-	return nil
-}
-
-// if sighash_ALL, hash of all txin outpoints, sequentially.
-// if other sighash type, 0x00 * 32
-func calcHashPrevOuts(tx *wire.MsgTx) [32]byte {
-
-	for _, in := range tx.TxIn {
-		//		in.PreviousOutPoint.Hash
+// calcHashPrevOuts makes a single hash of all previous outputs in the tx
+func calcHashPrevOuts(tx *wire.MsgTx, hType txscript.SigHashType) wire.ShaHash {
+	// skip this (0x00) for anyonecanpay
+	if hType == txscript.SigHashAnyOneCanPay {
+		var empty [32]byte
+		return empty
 	}
 
-	return wire.DoubleSha256SH(tx.TxIn[0].SignatureScript)
+	var pre []byte
+	for _, in := range tx.TxIn {
+		// first append 32 byte hash
+		pre = append(pre, in.PreviousOutPoint.Hash.Bytes()...)
+		// then make a buffer, put 4 byte index in lil' endian and append that
+		var buf [4]byte
+		binary.LittleEndian.PutUint32(buf[:], in.PreviousOutPoint.Index)
+		pre = append(pre, buf[:]...)
+	}
+	fmt.Printf("pre: %x\n", pre)
+	return wire.DoubleSha256SH(pre)
+}
+
+// calcHashSequence is hash of txins' seq numbers, lil' endian, stuck together
+func calcHashSequence(tx *wire.MsgTx, hType txscript.SigHashType) wire.ShaHash {
+	// skip (0x00) for single, none, anyonecanpay
+	if hType == txscript.SigHashSingle || hType == txscript.SigHashNone ||
+		hType == txscript.SigHashAnyOneCanPay {
+		var empty [32]byte
+		return empty
+	}
+
+	var pre []byte
+	for _, in := range tx.TxIn {
+		var buf [4]byte
+		binary.LittleEndian.PutUint32(buf[:], in.Sequence)
+		pre = append(pre, buf[:]...)
+	}
+	fmt.Printf("pre: %x\n", pre)
+	return wire.DoubleSha256SH(pre)
+}
+
+// calcHashOutputs also wants a input index, which it only uses for
+// sighash single.  If it's not sighash single, just put a 0 or whatever.
+func calcHashOutputs(
+	tx *wire.MsgTx, inIndex int, hType txscript.SigHashType) wire.ShaHash {
+	if hType == txscript.SigHashNone ||
+		(hType == txscript.SigHashSingle && inIndex <= len(tx.TxOut)) {
+		var empty [32]byte
+		return empty
+	}
+	if hType == txscript.SigHashSingle {
+		var buf bytes.Buffer
+		writeTxOut(&buf, 0, 0, tx.TxOut[inIndex])
+		return wire.DoubleSha256SH(buf.Bytes())
+	}
+
+	var pre []byte
+	for _, out := range tx.TxOut {
+		var buf bytes.Buffer
+		writeTxOut(&buf, 0, 0, out)
+		pre = append(pre, buf.Bytes()...)
+	}
+	fmt.Printf("pre: %x\n", pre)
+	return wire.DoubleSha256SH(pre)
 }
 
 func main() {
@@ -88,4 +184,72 @@ func main() {
 
 	fmt.Printf(uspv.TxToString(ttx))
 
+	hxh := calcWitnessSignatureHash(txscript.SigHashAll, ttx, 1, inamt1)
+
+	fmt.Printf("got sigHash %x\n", hxh)
+}
+
+// pver can be 0, doesn't do anything in these.  Same for msg.Version
+// writeVarInt serializes val to w using a variable number of bytes depending
+// on its value.
+func writeVarInt(w io.Writer, pver uint32, val uint64) error {
+	if val < 0xfd {
+		_, err := w.Write([]byte{uint8(val)})
+		return err
+	}
+
+	if val <= math.MaxUint16 {
+		var buf [3]byte
+		buf[0] = 0xfd
+		binary.LittleEndian.PutUint16(buf[1:], uint16(val))
+		_, err := w.Write(buf[:])
+		return err
+	}
+
+	if val <= math.MaxUint32 {
+		var buf [5]byte
+		buf[0] = 0xfe
+		binary.LittleEndian.PutUint32(buf[1:], uint32(val))
+		_, err := w.Write(buf[:])
+		return err
+	}
+
+	var buf [9]byte
+	buf[0] = 0xff
+	binary.LittleEndian.PutUint64(buf[1:], val)
+	_, err := w.Write(buf[:])
+	return err
+}
+
+// writeVarBytes serializes a variable length byte array to w as a varInt
+// containing the number of bytes, followed by the bytes themselves.
+func writeVarBytes(w io.Writer, pver uint32, bytes []byte) error {
+	slen := uint64(len(bytes))
+	err := writeVarInt(w, pver, slen)
+	if err != nil {
+		return err
+	}
+
+	_, err = w.Write(bytes)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// writeTxOut encodes to into the bitcoin protocol encoding for a transaction
+// output (TxOut) to w.
+func writeTxOut(w io.Writer, pver uint32, version int32, to *wire.TxOut) error {
+	var buf [8]byte
+	binary.LittleEndian.PutUint64(buf[:], uint64(to.Value))
+	_, err := w.Write(buf[:])
+	if err != nil {
+		return err
+	}
+
+	err = writeVarBytes(w, pver, to.PkScript)
+	if err != nil {
+		return err
+	}
+	return nil
 }

From 05590279b2ba585d74f7099d7627a065a8573096 Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Sun, 21 Feb 2016 13:49:04 -0800
Subject: [PATCH 10/22] deal with p2wsh?  maybe..?

---
 sighash143/sighash143.go | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/sighash143/sighash143.go b/sighash143/sighash143.go
index 086125844..2fa27f2f1 100644
--- a/sighash143/sighash143.go
+++ b/sighash143/sighash143.go
@@ -25,10 +25,16 @@ const (
 )
 
 // calcWitnessSignatureHash is the witnessified version of calcSignatureHash
+// put your pkscripts in the sigscript slot before handing the tx to this
+// function.  Also you're clearly supposed to cache the 3 sub-hashes generated
+// here, because they apply to the tx, not a txin.  But this doesn't yet.
 func calcWitnessSignatureHash(
 	hashType txscript.SigHashType, tx *wire.MsgTx, idx int, amt int64) []byte {
 	// in the script.go calcSignatureHash(), idx is assumed safe, so I guess
-	// that's OK here too...
+	// that's OK here too...?  Nah I'm gonna check
+	if idx > len(tx.TxIn)-1 {
+		return nil
+	}
 
 	// first get hashPrevOuts, hashSequence, and hashOutputs
 	hashPrevOuts := calcHashPrevOuts(tx, hashType)
@@ -51,10 +57,27 @@ func calcWitnessSignatureHash(
 	pre = append(pre, buf4[:]...)
 
 	// scriptCode which is some new thing
-	sCode := []byte{0x19, 0x76, 0xa9, 0x14}
-	sCode = append(sCode, tx.TxIn[idx].SignatureScript[2:22]...)
-	sCode = append(sCode, []byte{0x88, 0xac}...)
-	pre = append(pre, sCode...)
+
+	// detect wpkh mode
+	if len(tx.TxIn[idx].SignatureScript) == 22 &&
+		tx.TxIn[idx].SignatureScript[0] == 0x00 &&
+		tx.TxIn[idx].SignatureScript[1] == 0x14 {
+		// wpkh mode .... recreate op_dup codes here
+		sCode := []byte{0x19, 0x76, 0xa9, 0x14}
+		sCode = append(sCode, tx.TxIn[idx].SignatureScript[2:22]...)
+		sCode = append(sCode, []byte{0x88, 0xac}...)
+		pre = append(pre, sCode...)
+	} else if len(tx.TxIn[idx].SignatureScript) == 34 &&
+		tx.TxIn[idx].SignatureScript[0] == 0x00 &&
+		tx.TxIn[idx].SignatureScript[1] == 0x20 {
+		// whs mode- need to remove codeseparators.  this doesn't yet.
+		var buf bytes.Buffer
+		writeVarBytes(&buf, 0, tx.TxIn[idx].Witness[0])
+		pre = append(pre, buf.Bytes()...)
+	} else {
+		// ?? this is not witness tx! fail
+		return nil
+	}
 
 	// amount being signed off
 	binary.LittleEndian.PutUint64(buf8[:], uint64(amt))

From cb3b20ad6caa4c129f17752c35cd585c549b6c3a Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Sun, 21 Feb 2016 14:28:47 -0800
Subject: [PATCH 11/22] use WitnessSignatureScript

---
 uspv/sortsignsend.go | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/uspv/sortsignsend.go b/uspv/sortsignsend.go
index f1ef36474..0061f9e7f 100644
--- a/uspv/sortsignsend.go
+++ b/uspv/sortsignsend.go
@@ -201,10 +201,20 @@ func (s *SPVCon) SendCoins(adr btcutil.Address, sendAmt int64) error {
 
 		// This is where witness based sighash types need to happen
 		// sign into stash
-		sigStash[i], err = txscript.SignatureScript(
-			tx, i, txin.SignatureScript, txscript.SigHashAll, priv, true)
-		if err != nil {
-			return err
+		if ins[i].IsWit {
+			sigStash[i], err = txscript.WitnessSignatureScript(
+				tx, i, ins[i].Value, txin.SignatureScript,
+				txscript.SigHashAll, priv, true)
+			if err != nil {
+				return err
+			}
+		} else {
+			sigStash[i], err = txscript.SignatureScript(
+				tx, i, txin.SignatureScript,
+				txscript.SigHashAll, priv, true)
+			if err != nil {
+				return err
+			}
 		}
 	}
 	// swap sigs into sigScripts in txins

From 57a34f07534fb79b5150eb86db40c337d2f4ae4f Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Sun, 21 Feb 2016 17:42:30 -0800
Subject: [PATCH 12/22] spending witness inputs still doesn't work...

---
 sighash143/sighash143.go | 25 +++++++++++++++++++++----
 uspv/sortsignsend.go     | 14 ++++++++++++--
 2 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/sighash143/sighash143.go b/sighash143/sighash143.go
index 2fa27f2f1..62eb45c87 100644
--- a/sighash143/sighash143.go
+++ b/sighash143/sighash143.go
@@ -11,6 +11,7 @@ import (
 	"math"
 	"strings"
 
+	"github.com/btcsuite/btcd/btcec"
 	"github.com/btcsuite/btcd/txscript"
 	"github.com/btcsuite/btcd/wire"
 	"github.com/lightningnetwork/lnd/uspv"
@@ -22,6 +23,8 @@ const (
 
 	inspk1 = "00141d0f172a0ecb48aee1be1f2687d2963ae33f71a1"
 	inamt1 = int64(600000000)
+
+	xpecthash = "c37af31116d1b27caf68aae9e3ac82f1477929014d5b917657d0eb49478cb670"
 )
 
 // calcWitnessSignatureHash is the witnessified version of calcSignatureHash
@@ -97,7 +100,7 @@ func calcWitnessSignatureHash(
 	binary.LittleEndian.PutUint32(buf4[:], uint32(hashType))
 	pre = append(pre, buf4[:]...)
 
-	fmt.Printf("pre: %x\n", pre)
+	fmt.Printf("calcWitnessSignatureHash pre: %x\n", pre)
 	hsh := wire.DoubleSha256SH(pre)
 	return hsh.Bytes()
 }
@@ -119,7 +122,7 @@ func calcHashPrevOuts(tx *wire.MsgTx, hType txscript.SigHashType) wire.ShaHash {
 		binary.LittleEndian.PutUint32(buf[:], in.PreviousOutPoint.Index)
 		pre = append(pre, buf[:]...)
 	}
-	fmt.Printf("pre: %x\n", pre)
+	fmt.Printf("calcHashPrevOuts pre: %x\n", pre)
 	return wire.DoubleSha256SH(pre)
 }
 
@@ -138,7 +141,7 @@ func calcHashSequence(tx *wire.MsgTx, hType txscript.SigHashType) wire.ShaHash {
 		binary.LittleEndian.PutUint32(buf[:], in.Sequence)
 		pre = append(pre, buf[:]...)
 	}
-	fmt.Printf("pre: %x\n", pre)
+	fmt.Printf("calcHashSequence pre: %x\n", pre)
 	return wire.DoubleSha256SH(pre)
 }
 
@@ -163,7 +166,7 @@ func calcHashOutputs(
 		writeTxOut(&buf, 0, 0, out)
 		pre = append(pre, buf.Bytes()...)
 	}
-	fmt.Printf("pre: %x\n", pre)
+	fmt.Printf("calcHashOutputs pre: %x\n", pre)
 	return wire.DoubleSha256SH(pre)
 }
 
@@ -179,6 +182,10 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
+	xpkt, err := hex.DecodeString(string(xpecthash))
+	if err != nil {
+		log.Fatal(err)
+	}
 
 	// load tx skeleton from local file
 	fmt.Printf("loading tx from file tx.hex\n")
@@ -207,9 +214,19 @@ func main() {
 
 	fmt.Printf(uspv.TxToString(ttx))
 
+	priv, err := btcec.NewPrivateKey(btcec.S256())
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	sig, err := txscript.WitnessScript(ttx, 1, inamt1, in1spk,
+		txscript.SigHashAll, priv, true)
+
+	fmt.Printf("got sig %x\n", sig)
 	hxh := calcWitnessSignatureHash(txscript.SigHashAll, ttx, 1, inamt1)
 
 	fmt.Printf("got sigHash %x\n", hxh)
+	fmt.Printf("expect hash %x\n ", xpkt)
 }
 
 // pver can be 0, doesn't do anything in these.  Same for msg.Version
diff --git a/uspv/sortsignsend.go b/uspv/sortsignsend.go
index 0061f9e7f..c6aa3ebf3 100644
--- a/uspv/sortsignsend.go
+++ b/uspv/sortsignsend.go
@@ -167,6 +167,8 @@ func (s *SPVCon) SendCoins(adr btcutil.Address, sendAmt int64) error {
 
 		// if wit utxo, convert address to generate pkscript
 		if in.IsWit {
+			// adding a witness input, so set the tx as witness tx
+			tx.Flags = 0x01
 			wa, err := btcutil.NewAddressWitnessPubKeyHash(
 				s.TS.Adrs[in.KeyIdx].PkhAdr.ScriptAddress(), s.TS.Param)
 			prevPKs, err = txscript.PayToAddrScript(wa)
@@ -187,6 +189,7 @@ func (s *SPVCon) SendCoins(adr btcutil.Address, sendAmt int64) error {
 
 	// tx is ready for signing,
 	sigStash := make([][]byte, len(ins))
+	witStash := make([][][]byte, len(ins))
 	for i, txin := range tx.TxIn {
 		// pick key
 		child, err := s.TS.rootPrivKey.Child(
@@ -202,7 +205,7 @@ func (s *SPVCon) SendCoins(adr btcutil.Address, sendAmt int64) error {
 		// This is where witness based sighash types need to happen
 		// sign into stash
 		if ins[i].IsWit {
-			sigStash[i], err = txscript.WitnessSignatureScript(
+			witStash[i], err = txscript.WitnessScript(
 				tx, i, ins[i].Value, txin.SignatureScript,
 				txscript.SigHashAll, priv, true)
 			if err != nil {
@@ -219,7 +222,14 @@ func (s *SPVCon) SendCoins(adr btcutil.Address, sendAmt int64) error {
 	}
 	// swap sigs into sigScripts in txins
 	for i, txin := range tx.TxIn {
-		txin.SignatureScript = sigStash[i]
+		if sigStash[i] != nil {
+			txin.SignatureScript = sigStash[i]
+		}
+		if witStash[i] != nil {
+			txin.Witness = witStash[i]
+			txin.SignatureScript = nil
+		}
+
 	}
 
 	fmt.Printf("tx: %s", TxToString(tx))

From 2b808724fb35295d4f4e52b96c7d0e2e1176c2f3 Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Mon, 22 Feb 2016 01:41:40 -0800
Subject: [PATCH 13/22] spend from wpkh works

---
 shell.go                 |  2 +-
 sighash143/sighash143.go | 54 ++++++++++++++++++++++++++++------------
 uspv/msghandler.go       |  2 ++
 uspv/sortsignsend.go     |  2 +-
 uspv/txstore.go          | 13 +++++-----
 uspv/utxodb.go           |  2 +-
 6 files changed, 49 insertions(+), 26 deletions(-)

diff --git a/shell.go b/shell.go
index 81b607f1b..8e2da8866 100644
--- a/shell.go
+++ b/shell.go
@@ -57,7 +57,7 @@ func shell() {
 		log.Fatal(err)
 	}
 	if tip == 0 { // DB has never been used, set to birthday
-		tip = 20000 // hardcoded; later base on keyfile date?
+		tip = 21900 // hardcoded; later base on keyfile date?
 		err = SCon.TS.SetDBSyncHeight(tip)
 		if err != nil {
 			log.Fatal(err)
diff --git a/sighash143/sighash143.go b/sighash143/sighash143.go
index 62eb45c87..419bf54e9 100644
--- a/sighash143/sighash143.go
+++ b/sighash143/sighash143.go
@@ -31,11 +31,13 @@ const (
 // put your pkscripts in the sigscript slot before handing the tx to this
 // function.  Also you're clearly supposed to cache the 3 sub-hashes generated
 // here, because they apply to the tx, not a txin.  But this doesn't yet.
-func calcWitnessSignatureHash(
+func calcWitnessSignatureHash(sigscript []byte,
 	hashType txscript.SigHashType, tx *wire.MsgTx, idx int, amt int64) []byte {
 	// in the script.go calcSignatureHash(), idx is assumed safe, so I guess
 	// that's OK here too...?  Nah I'm gonna check
 	if idx > len(tx.TxIn)-1 {
+		fmt.Printf("calcWitnessSignatureHash error: idx %d but %d txins",
+			idx, len(tx.TxIn))
 		return nil
 	}
 
@@ -62,23 +64,22 @@ func calcWitnessSignatureHash(
 	// scriptCode which is some new thing
 
 	// detect wpkh mode
-	if len(tx.TxIn[idx].SignatureScript) == 22 &&
-		tx.TxIn[idx].SignatureScript[0] == 0x00 &&
-		tx.TxIn[idx].SignatureScript[1] == 0x14 {
+	fmt.Printf("txin %d sscript len %d\n", idx, len(tx.TxIn[idx].SignatureScript))
+	if len(sigscript) == 22 && sigscript[0] == 0x00 && sigscript[1] == 0x14 {
 		// wpkh mode .... recreate op_dup codes here
 		sCode := []byte{0x19, 0x76, 0xa9, 0x14}
-		sCode = append(sCode, tx.TxIn[idx].SignatureScript[2:22]...)
+		sCode = append(sCode, sigscript[2:22]...)
 		sCode = append(sCode, []byte{0x88, 0xac}...)
 		pre = append(pre, sCode...)
-	} else if len(tx.TxIn[idx].SignatureScript) == 34 &&
-		tx.TxIn[idx].SignatureScript[0] == 0x00 &&
-		tx.TxIn[idx].SignatureScript[1] == 0x20 {
+	} else if len(sigscript) == 34 &&
+		sigscript[0] == 0x00 && sigscript[1] == 0x20 {
 		// whs mode- need to remove codeseparators.  this doesn't yet.
 		var buf bytes.Buffer
-		writeVarBytes(&buf, 0, tx.TxIn[idx].Witness[0])
+		writeVarBytes(&buf, 0, sigscript)
 		pre = append(pre, buf.Bytes()...)
 	} else {
 		// ?? this is not witness tx! fail
+		fmt.Printf("Non witness error ")
 		return nil
 	}
 
@@ -204,13 +205,13 @@ func main() {
 
 	// 	make tx
 	ttx := wire.NewMsgTx()
-
 	// deserialize into tx
 	buf := bytes.NewBuffer(txbytes)
 	ttx.Deserialize(buf)
 
+	fmt.Printf("=====tx locktime: %x\n", ttx.LockTime)
 	ttx.TxIn[0].SignatureScript = in0spk
-	ttx.TxIn[1].SignatureScript = in1spk
+	//	ttx.TxIn[1].SignatureScript = in1spk
 
 	fmt.Printf(uspv.TxToString(ttx))
 
@@ -218,15 +219,36 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
-
-	sig, err := txscript.WitnessScript(ttx, 1, inamt1, in1spk,
+	// assert flag before writing witness
+	ttx.Flags = 0x01
+	ttx.TxIn[1].Witness, err = txscript.WitnessScript(ttx, 1, inamt1, in1spk,
 		txscript.SigHashAll, priv, true)
+	//	ttx.TxIn[1].SignatureScript = nil
 
-	fmt.Printf("got sig %x\n", sig)
-	hxh := calcWitnessSignatureHash(txscript.SigHashAll, ttx, 1, inamt1)
+	hxh := calcWitnessSignatureHash(in1spk, txscript.SigHashAll, ttx, 1, inamt1)
 
-	fmt.Printf("got sigHash %x\n", hxh)
+	fmt.Printf("got sigHash %x NON LIB\n", hxh)
 	fmt.Printf("expect hash %x\n ", xpkt)
+	fmt.Printf("\n%s\n", uspv.TxToString(ttx))
+
+	fmt.Printf("loading tx from file xtx.hex\n")
+	txhex, err = ioutil.ReadFile("xtx.hex")
+	if err != nil {
+		log.Fatal(err)
+	}
+	txhex = []byte(strings.TrimSpace(string(txhex)))
+	txbytes, err = hex.DecodeString(string(txhex))
+	if err != nil {
+		log.Fatal(err)
+	}
+	// 	make tx
+	xtx := wire.NewMsgTx()
+	// deserialize into tx
+	buf = bytes.NewBuffer(txbytes)
+	xtx.Deserialize(buf)
+
+	//	ttx.Flags = 0x01
+	fmt.Printf("\n%s\n", uspv.TxToString(xtx))
 }
 
 // pver can be 0, doesn't do anything in these.  Same for msg.Version
diff --git a/uspv/msghandler.go b/uspv/msghandler.go
index ecb7c4742..f7fcf5828 100644
--- a/uspv/msghandler.go
+++ b/uspv/msghandler.go
@@ -189,6 +189,7 @@ func (s *SPVCon) GetDataHandler(m *wire.MsgGetData) {
 				log.Printf("error getting tx %s: %s",
 					thing.Hash.String(), err.Error())
 			}
+			fmt.Printf(TxToString(tx))
 			s.outMsgQueue <- tx
 			sent++
 			continue
@@ -199,6 +200,7 @@ func (s *SPVCon) GetDataHandler(m *wire.MsgGetData) {
 				log.Printf("error getting tx %s: %s",
 					thing.Hash.String(), err.Error())
 			}
+			tx.Flags = 0x00 // dewitnessify
 			s.outMsgQueue <- tx
 			sent++
 			continue
diff --git a/uspv/sortsignsend.go b/uspv/sortsignsend.go
index c6aa3ebf3..b5c04416e 100644
--- a/uspv/sortsignsend.go
+++ b/uspv/sortsignsend.go
@@ -234,7 +234,7 @@ func (s *SPVCon) SendCoins(adr btcutil.Address, sendAmt int64) error {
 
 	fmt.Printf("tx: %s", TxToString(tx))
 	buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize()))
-	tx.Serialize(buf)
+	tx.SerializeWitness(buf)
 	fmt.Printf("tx: %x\n", buf.Bytes())
 
 	// send it out on the wire.  hope it gets there.
diff --git a/uspv/txstore.go b/uspv/txstore.go
index 2fe582f35..a4b8aadd2 100644
--- a/uspv/txstore.go
+++ b/uspv/txstore.go
@@ -137,20 +137,19 @@ func CheckDoubleSpends(
 
 // TxToString prints out some info about a transaction. for testing / debugging
 func TxToString(tx *wire.MsgTx) string {
-	str := fmt.Sprintf("\t size %d vsize %d wsize %d Tx %s\n",
+	str := fmt.Sprintf("size %d vsize %d wsize %d locktime %d flag %x txid %s\n",
 		tx.SerializeSize(), tx.VirtualSize(), tx.SerializeSizeWitness(),
-		tx.TxSha().String())
+		tx.LockTime, tx.Flags, tx.TxSha().String())
 	for i, in := range tx.TxIn {
-		str += fmt.Sprintf("Input %d: %s\n", i, in.PreviousOutPoint.String())
-		str += fmt.Sprintf("SigScript for input %d: %x\n", i, in.SignatureScript)
+		str += fmt.Sprintf("Input %d spends %s\n", i, in.PreviousOutPoint.String())
+		str += fmt.Sprintf("\tSigScript: %x\n", in.SignatureScript)
 		for j, wit := range in.Witness {
-			str += fmt.Sprintf("witness %d: %x\t", j, wit)
+			str += fmt.Sprintf("\twitness %d: %x\n", j, wit)
 		}
-		str += fmt.Sprintf("\n")
 	}
 	for i, out := range tx.TxOut {
 		if out != nil {
-			str += fmt.Sprintf("\toutput %d script: %x amt: %d\n",
+			str += fmt.Sprintf("output %d script: %x amt: %d\n",
 				i, out.PkScript, out.Value)
 		} else {
 			str += fmt.Sprintf("output %d nil (WARNING)\n", i)
diff --git a/uspv/utxodb.go b/uspv/utxodb.go
index e1013c865..7c18626db 100644
--- a/uspv/utxodb.go
+++ b/uspv/utxodb.go
@@ -470,7 +470,7 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 					// store this relevant tx
 					sha := tx.TxSha()
 					var buf bytes.Buffer
-					tx.Serialize(&buf)
+					tx.SerializeWitness(&buf) // always store witness version
 					err = txns.Put(sha.Bytes(), buf.Bytes())
 					if err != nil {
 						return err

From bc4cf5ba809c967fed309a517171f8de42dd60df Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Mon, 22 Feb 2016 14:35:01 -0800
Subject: [PATCH 14/22] add per tx fee estimation

seems to work fine.  Can add monitoring of avg fees later
---
 shell.go             |   7 ++-
 uspv/eight333.go     |   4 +-
 uspv/sortsignsend.go | 141 +++++++++++++++++++++++++++++--------------
 3 files changed, 104 insertions(+), 48 deletions(-)

diff --git a/shell.go b/shell.go
index 8e2da8866..58bdc6a45 100644
--- a/shell.go
+++ b/shell.go
@@ -290,7 +290,12 @@ func Send(args []string) error {
 	fmt.Printf("send %d to address: %s \n",
 		amt, adr.String())
 
-	err = SCon.SendCoins(adr, amt)
+	var adrs []btcutil.Address
+	var amts []int64
+
+	adrs = append(adrs, adr)
+	amts = append(amts, amt)
+	err = SCon.SendCoins(adrs, amts)
 	if err != nil {
 		return err
 	}
diff --git a/uspv/eight333.go b/uspv/eight333.go
index 183562260..548860369 100644
--- a/uspv/eight333.go
+++ b/uspv/eight333.go
@@ -329,7 +329,7 @@ func (s *SPVCon) AskForBlocks() error {
 	s.headerMutex.Unlock() // checked, unlock
 	endPos := stat.Size()
 
-	headerTip := int32(endPos / 80) // DONT move back 1 header length to read
+	headerTip := int32(endPos/80) - 1 // move back 1 header length to read
 
 	dbTip, err := s.TS.GetDBSyncHeight()
 	if err != nil {
@@ -366,7 +366,7 @@ func (s *SPVCon) AskForBlocks() error {
 		// load header from file
 
 		s.headerMutex.Lock() // seek to header we need
-		_, err = s.headerFile.Seek(int64((dbTip-1)*80), os.SEEK_SET)
+		_, err = s.headerFile.Seek(int64((dbTip)*80), os.SEEK_SET)
 		if err != nil {
 			return err
 		}
diff --git a/uspv/sortsignsend.go b/uspv/sortsignsend.go
index b5c04416e..69e81933b 100644
--- a/uspv/sortsignsend.go
+++ b/uspv/sortsignsend.go
@@ -92,10 +92,14 @@ func (s *SPVCon) NewOutgoingTx(tx *wire.MsgTx) error {
 
 // SendCoins does send coins, but it's very rudimentary
 // wit makes it into p2wpkh.  Which is not yet spendable.
-func (s *SPVCon) SendCoins(adr btcutil.Address, sendAmt int64) error {
-
+func (s *SPVCon) SendCoins(adrs []btcutil.Address, sendAmts []int64) error {
+	if len(adrs) != len(sendAmts) {
+		return fmt.Errorf("%d addresses and %d amounts", len(adrs), len(sendAmts))
+	}
 	var err error
-	var score int64
+	var score, totalSend, fee int64
+	dustCutoff := int64(20000) // below this amount, just give to miners
+	satPerByte := int64(30)    // satoshis per byte fee; have as arg later
 	allUtxos, err := s.TS.GetAllUtxos()
 	if err != nil {
 		return err
@@ -104,44 +108,66 @@ func (s *SPVCon) SendCoins(adr btcutil.Address, sendAmt int64) error {
 	for _, utxo := range allUtxos {
 		score += utxo.Value
 	}
+	for _, amt := range sendAmts {
+		totalSend += amt
+	}
 	// important rule in bitcoin, output total > input total is invalid.
-	if sendAmt > score {
+	if totalSend > score {
 		return fmt.Errorf("trying to send %d but %d available.",
-			sendAmt, score)
+			totalSend, score)
 	}
 
-	///////////////////
 	tx := wire.NewMsgTx() // make new tx
-	// make address script 76a914...88ac or 0014...
-	outAdrScript, err := txscript.PayToAddrScript(adr)
-	if err != nil {
-		return err
+	// add non-change (arg) outputs
+	for i, adr := range adrs {
+		// make address script 76a914...88ac or 0014...
+		outAdrScript, err := txscript.PayToAddrScript(adr)
+		if err != nil {
+			return err
+		}
+		// make user specified txout and add to tx
+		txout := wire.NewTxOut(sendAmts[i], outAdrScript)
+		tx.AddTxOut(txout)
 	}
 
-	////////////////////////////
-
 	// generate a utxo slice for your inputs
 	var ins utxoSlice
 
 	// add utxos until we've had enough
-	nokori := sendAmt // nokori is how much is needed on input side
+	nokori := totalSend // nokori is how much is needed on input side
 	for _, utxo := range allUtxos {
 		// yeah, lets add this utxo!
 		ins = append(ins, *utxo)
+		// as we add utxos, fill in sigscripts
+		// generate previous pkscripts (subscritpt?) for all utxos
+		// then make txins with the utxo and prevpk, and insert them into the tx
+		// these are all zeroed out during signing but it's an easy way to keep track
+		var prevPKs []byte
+		if utxo.IsWit {
+			tx.Flags = 0x01
+			wa, err := btcutil.NewAddressWitnessPubKeyHash(
+				s.TS.Adrs[utxo.KeyIdx].PkhAdr.ScriptAddress(), s.TS.Param)
+			prevPKs, err = txscript.PayToAddrScript(wa)
+			if err != nil {
+				return err
+			}
+		} else { // otherwise generate directly
+			prevPKs, err = txscript.PayToAddrScript(
+				s.TS.Adrs[utxo.KeyIdx].PkhAdr)
+			if err != nil {
+				return err
+			}
+		}
+		tx.AddTxIn(wire.NewTxIn(&utxo.Op, prevPKs, nil))
 		nokori -= utxo.Value
-		if nokori < -10000 { // minimum overage / fee is 10K now
+		fee = EstFee(tx, satPerByte)
+		if nokori < -fee { // done adding utxos: nokori below negative est. fee
 			break
 		}
 	}
 
-	// sort utxos on the input side
-	sort.Sort(ins)
-
-	// make user specified txout and add to tx
-	txout := wire.NewTxOut(sendAmt, outAdrScript)
-	tx.AddTxOut(txout)
 	// see if there's enough left to also add a change output
-	if nokori < -200000 {
+	if nokori < -dustCutoff {
 		changeOld, err := s.TS.NewAdr() // change is witnessy
 		if err != nil {
 			return err
@@ -156,34 +182,17 @@ func (s *SPVCon) SendCoins(adr btcutil.Address, sendAmt int64) error {
 		if err != nil {
 			return err
 		}
-		changeOut := wire.NewTxOut((-100000)-nokori, changeScript)
+
+		changeOut := wire.NewTxOut(0, changeScript)
 		tx.AddTxOut(changeOut)
+		fee = EstFee(tx, satPerByte)
+		changeOut.Value = -(nokori + fee)
 	}
 
-	// generate previous pkscripts for all the (now sorted) utxos
-	// then make txins with the utxo and prevpk, and insert them into the tx
-	for _, in := range ins {
-		var prevPKs []byte
+	// sort utxos on the input side.  use this instead of txsort
+	// because we want to remember which keys are associated with which inputs
+	sort.Sort(ins)
 
-		// if wit utxo, convert address to generate pkscript
-		if in.IsWit {
-			// adding a witness input, so set the tx as witness tx
-			tx.Flags = 0x01
-			wa, err := btcutil.NewAddressWitnessPubKeyHash(
-				s.TS.Adrs[in.KeyIdx].PkhAdr.ScriptAddress(), s.TS.Param)
-			prevPKs, err = txscript.PayToAddrScript(wa)
-			if err != nil {
-				return err
-			}
-		} else { // otherwise generate directly
-			prevPKs, err = txscript.PayToAddrScript(
-				s.TS.Adrs[in.KeyIdx].PkhAdr)
-			if err != nil {
-				return err
-			}
-		}
-		tx.AddTxIn(wire.NewTxIn(&in.Op, prevPKs, nil))
-	}
 	// sort tx -- this only will change txouts since inputs are already sorted
 	txsort.InPlaceSort(tx)
 
@@ -246,6 +255,48 @@ func (s *SPVCon) SendCoins(adr btcutil.Address, sendAmt int64) error {
 	return nil
 }
 
+// EstFee gives a fee estimate based on a tx and a sat/Byte target.
+// The TX should have all outputs, including the change address already
+// populated (with potentially 0 amount.  Also it should have all inputs
+// populated, but inputs don't need to have sigscripts or witnesses
+// (it'll guess the sizes of sigs/wits that arent' filled in).
+func EstFee(otx *wire.MsgTx, spB int64) int64 {
+	mtsig := make([]byte, 72)
+	mtpub := make([]byte, 33)
+
+	tx := otx.Copy()
+
+	// iterate through txins, replacing subscript sigscripts with noise
+	// sigs or witnesses
+	for _, txin := range tx.TxIn {
+		// check wpkh
+		if len(txin.SignatureScript) == 22 &&
+			txin.SignatureScript[0] == 0x00 && txin.SignatureScript[1] == 0x14 {
+			txin.SignatureScript = nil
+			txin.Witness = make([][]byte, 2)
+			txin.Witness[0] = mtsig
+			txin.Witness[1] = mtpub
+		} else if len(txin.SignatureScript) == 34 &&
+			txin.SignatureScript[0] == 0x00 && txin.SignatureScript[1] == 0x20 {
+			// p2wsh -- sig lenght is a total guess!
+			txin.SignatureScript = nil
+			txin.Witness = make([][]byte, 3)
+			// 3 sigs? totally guessing here
+			txin.Witness[0] = mtsig
+			txin.Witness[1] = mtsig
+			txin.Witness[2] = mtsig
+		} else {
+			// assume everything else is p2pkh.  Even though it's not
+			txin.Witness = nil
+			txin.SignatureScript = make([]byte, 105) // len of p2pkh sigscript
+		}
+	}
+	fmt.Printf(TxToString(tx))
+	size := int64(tx.VirtualSize())
+	fmt.Printf("%d spB, est vsize %d, fee %d\n", spB, size, size*spB)
+	return size * spB
+}
+
 // SignThis isn't used anymore...
 func (t *TxStore) SignThis(tx *wire.MsgTx) error {
 	fmt.Printf("-= SignThis =-\n")

From dd1f2c00fe07e80c9429c9814dd8f34e8658ef9a Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Mon, 22 Feb 2016 16:32:58 -0800
Subject: [PATCH 15/22] add fanout.  fix boltdb weirdness.

again with keys / values within the bolttx acting weird.
it wasn't deleting utxos that had been spent by the ingested tx.
it'd do the first 30 then stop.  Deferred deletion and copied the
serialized utxo.  Not sure which of those fixed it.  Maybe both.
---
 shell.go             | 59 ++++++++++++++++++++++++++++++++++++++++++++
 uspv/sortsignsend.go | 48 +++++++++++++++++++----------------
 uspv/utxodb.go       | 28 +++++++++++++--------
 3 files changed, 104 insertions(+), 31 deletions(-)

diff --git a/shell.go b/shell.go
index 58bdc6a45..0e197e5ab 100644
--- a/shell.go
+++ b/shell.go
@@ -140,6 +140,13 @@ func Shellparse(cmdslice []string) error {
 		}
 		return nil
 	}
+	if cmd == "fan" {
+		err = Fan(args)
+		if err != nil {
+			fmt.Printf("fan error: %s\n", err)
+		}
+		return nil
+	}
 	if cmd == "txs" {
 		err = Txs(args)
 		if err != nil {
@@ -238,6 +245,14 @@ func Adr(args []string) error {
 			}
 		}
 	}
+	if len(args) > 1 {
+		for i := 0; i < 1000; i++ {
+			_, err := SCon.TS.NewAdr()
+			if err != nil {
+				return err
+			}
+		}
+	}
 
 	// always make one
 	a, err := SCon.TS.NewAdr()
@@ -250,6 +265,50 @@ func Adr(args []string) error {
 	return nil
 }
 
+// Fan generates a bunch of fanout.  Only for testing, can be expensive.
+// syntax: fan numOutputs valOutputs witty
+func Fan(args []string) error {
+	if len(args) < 3 {
+		return fmt.Errorf("fan syntax: fan numOutputs valOutputs witty")
+	}
+	numOutputs, err := strconv.ParseInt(args[0], 10, 64)
+	if err != nil {
+		return err
+	}
+	valOutputs, err := strconv.ParseInt(args[1], 10, 64)
+	if err != nil {
+		return err
+	}
+	wittynum, err := strconv.ParseInt(args[2], 10, 64)
+	if err != nil {
+		return err
+	}
+
+	adrs := make([]btcutil.Address, numOutputs)
+	amts := make([]int64, numOutputs)
+
+	oAdr, err := SCon.TS.NewAdr()
+	if err != nil {
+		return err
+	}
+	for i := int64(0); i < numOutputs; i++ {
+		if wittynum != 0 {
+			wAdr, err := btcutil.NewAddressWitnessPubKeyHash(
+				oAdr.ScriptAddress(), SCon.TS.Param)
+			if err != nil {
+				return err
+			}
+			adrs[i] = wAdr
+		} else {
+			adrs[i] = oAdr
+		}
+		amts[i] = valOutputs + i
+	}
+
+	return SCon.SendCoins(adrs, amts)
+
+}
+
 // Send sends coins.
 func Send(args []string) error {
 	if SCon.RBytes == 0 {
diff --git a/uspv/sortsignsend.go b/uspv/sortsignsend.go
index 69e81933b..1d2944832 100644
--- a/uspv/sortsignsend.go
+++ b/uspv/sortsignsend.go
@@ -160,33 +160,39 @@ func (s *SPVCon) SendCoins(adrs []btcutil.Address, sendAmts []int64) error {
 		}
 		tx.AddTxIn(wire.NewTxIn(&utxo.Op, prevPKs, nil))
 		nokori -= utxo.Value
-		fee = EstFee(tx, satPerByte)
-		if nokori < -fee { // done adding utxos: nokori below negative est. fee
-			break
+		// if nokori is positive, don't bother checking fee yet.
+		if nokori < 0 {
+			fee = EstFee(tx, satPerByte)
+			if nokori < -fee { // done adding utxos: nokori below negative est. fee
+				break
+			}
 		}
 	}
 
 	// see if there's enough left to also add a change output
-	if nokori < -dustCutoff {
-		changeOld, err := s.TS.NewAdr() // change is witnessy
-		if err != nil {
-			return err
-		}
-		changeAdr, err := btcutil.NewAddressWitnessPubKeyHash(
-			changeOld.ScriptAddress(), s.TS.Param)
-		if err != nil {
-			return err
-		}
 
-		changeScript, err := txscript.PayToAddrScript(changeAdr)
-		if err != nil {
-			return err
-		}
+	changeOld, err := s.TS.NewAdr() // change is witnessy
+	if err != nil {
+		return err
+	}
+	changeAdr, err := btcutil.NewAddressWitnessPubKeyHash(
+		changeOld.ScriptAddress(), s.TS.Param)
+	if err != nil {
+		return err
+	}
 
-		changeOut := wire.NewTxOut(0, changeScript)
-		tx.AddTxOut(changeOut)
-		fee = EstFee(tx, satPerByte)
-		changeOut.Value = -(nokori + fee)
+	changeScript, err := txscript.PayToAddrScript(changeAdr)
+	if err != nil {
+		return err
+	}
+
+	changeOut := wire.NewTxOut(0, changeScript)
+	tx.AddTxOut(changeOut)
+	fee = EstFee(tx, satPerByte)
+	changeOut.Value = -(nokori + fee)
+	if changeOut.Value < dustCutoff {
+		// remove last output (change) : not worth it
+		tx.TxOut = tx.TxOut[:len(tx.TxOut)-1]
 	}
 
 	// sort utxos on the input side.  use this instead of txsort
diff --git a/uspv/utxodb.go b/uspv/utxodb.go
index 7c18626db..f5cb37e53 100644
--- a/uspv/utxodb.go
+++ b/uspv/utxodb.go
@@ -353,7 +353,6 @@ func (ts *TxStore) PopulateAdrs(lastKey uint32) error {
 func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 	var hits uint32
 	var err error
-	var spentOPs [][]byte
 	var nUtxoBytes [][]byte
 
 	// tx has been OK'd by SPV; check tx sanity
@@ -366,13 +365,14 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 	// note that you can't check signatures; this is SPV.
 	// 0 conf SPV means pretty much nothing.  Anyone can say anything.
 
+	spentOPs := make([][]byte, len(tx.TxIn))
 	// before entering into db, serialize all inputs of the ingested tx
-	for _, txin := range tx.TxIn {
+	for i, txin := range tx.TxIn {
 		nOP, err := outPointToBytes(&txin.PreviousOutPoint)
 		if err != nil {
 			return hits, err
 		}
-		spentOPs = append(spentOPs, nOP)
+		spentOPs[i] = nOP
 	}
 	// also generate PKscripts for all addresses (maybe keep storing these?)
 	for _, adr := range ts.Adrs {
@@ -419,7 +419,7 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 				}
 				nUtxoBytes = append(nUtxoBytes, b)
 				hits++
-				break // only one match
+				// break // keep looking! address re-use in same tx
 			}
 		}
 	}
@@ -437,24 +437,25 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 		// first see if we lose utxos
 		// iterate through duffel bag and look for matches
 		// this makes us lose money, which is regrettable, but we need to know.
+
+		var delOPs [][]byte
+		fmt.Printf("%d nOP to iterate over\n", len(spentOPs))
 		for _, nOP := range spentOPs {
 			duf.ForEach(func(k, v []byte) error {
 				if bytes.Equal(k, nOP) { // matched, we lost utxo
 					// do all this just to figure out value we lost
 					x := make([]byte, len(k)+len(v))
 					copy(x, k)
+					// mark utxos for deletion
+					delOPs = append(delOPs, x)
 					copy(x[len(k):], v)
 					lostTxo, err := UtxoFromBytes(x)
 					if err != nil {
 						return err
 					}
 					hits++
-					// then delete the utxo from duf, save to old
-					err = duf.Delete(k)
-					if err != nil {
-						return err
-					}
-					// after deletion, save stxo to old bucket
+
+					// after marking for deletion, save stxo to old bucket
 					var st Stxo               // generate spent txo
 					st.Utxo = lostTxo         // assign outpoint
 					st.SpendHeight = height   // spent at height
@@ -480,6 +481,13 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 				}
 				return nil // no match
 			})
+			// delete after done with foreach
+			for _, k := range delOPs {
+				err = duf.Delete(k)
+				if err != nil {
+					return err
+				}
+			}
 		} // done losing utxos, next gain utxos
 		// next add all new utxos to db, this is quick as the work is above
 		for _, ub := range nUtxoBytes {

From a14b9a9e70b402f8f3704ec6dd213d5b82222dc0 Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Mon, 22 Feb 2016 23:21:54 -0800
Subject: [PATCH 16/22] fix the boltdb problem

not that I know how I fixed it... just moved stuff around.
Oh, bolt.  Any time I see "})" I know I'm in bizzaro world.
---
 uspv/utxodb.go | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/uspv/utxodb.go b/uspv/utxodb.go
index f5cb37e53..8383fe314 100644
--- a/uspv/utxodb.go
+++ b/uspv/utxodb.go
@@ -439,21 +439,25 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 		// this makes us lose money, which is regrettable, but we need to know.
 
 		var delOPs [][]byte
-		fmt.Printf("%d nOP to iterate over\n", len(spentOPs))
+		//		fmt.Printf("%d nOP to iterate over\n", len(spentOPs))
 		for _, nOP := range spentOPs {
 			duf.ForEach(func(k, v []byte) error {
 				if bytes.Equal(k, nOP) { // matched, we lost utxo
+					fmt.Printf("will delete point %x\n", k)
+					hits++
 					// do all this just to figure out value we lost
 					x := make([]byte, len(k)+len(v))
 					copy(x, k)
-					// mark utxos for deletion
-					delOPs = append(delOPs, x)
+					y := make([]byte, len(k))
+					// mark utxo for deletion
+					copy(y, k)
+					delOPs = append(delOPs, y)
+
 					copy(x[len(k):], v)
 					lostTxo, err := UtxoFromBytes(x)
 					if err != nil {
 						return err
 					}
-					hits++
 
 					// after marking for deletion, save stxo to old bucket
 					var st Stxo               // generate spent txo
@@ -482,13 +486,15 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 				return nil // no match
 			})
 			// delete after done with foreach
-			for _, k := range delOPs {
-				err = duf.Delete(k)
-				if err != nil {
-					return err
-				}
+		}
+		for _, y := range delOPs {
+			fmt.Printf("deleting outpoint %x\n", y)
+			err = duf.Delete(y)
+			if err != nil {
+				return err
 			}
-		} // done losing utxos, next gain utxos
+		}
+		// done losing utxos, next gain utxos
 		// next add all new utxos to db, this is quick as the work is above
 		for _, ub := range nUtxoBytes {
 			err = duf.Put(ub[:36], ub[36:])

From df04a73e1a2b1cbfca33811119d4b47687fae142 Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Wed, 24 Feb 2016 02:37:52 -0800
Subject: [PATCH 17/22] simplify boltdb ingest

---
 shell.go             | 56 +++++++++++---------------
 uspv/sortsignsend.go | 37 ++++++++++++++---
 uspv/utxodb.go       | 94 +++++++++++++++++++-------------------------
 3 files changed, 96 insertions(+), 91 deletions(-)

diff --git a/shell.go b/shell.go
index 0e197e5ab..a1b85d9a9 100644
--- a/shell.go
+++ b/shell.go
@@ -206,7 +206,7 @@ func Bal(args []string) error {
 	if err != nil {
 		return err
 	}
-	var score int64
+	var score, confScore int64
 	for i, u := range allUtxos {
 		fmt.Printf("\tutxo %d height %d %s key:%d amt %d",
 			i, u.AtHeight, u.Op.String(), u.KeyIdx, u.Value)
@@ -215,6 +215,9 @@ func Bal(args []string) error {
 		}
 		fmt.Printf("\n")
 		score += u.Value
+		if u.AtHeight != 0 {
+			confScore += u.Value
+		}
 	}
 	height, _ := SCon.TS.GetDBSyncHeight()
 
@@ -228,7 +231,7 @@ func Bal(args []string) error {
 	}
 
 	fmt.Printf("Total known utxos: %d\n", len(allUtxos))
-	fmt.Printf("Total spendable coin: %d\n", score)
+	fmt.Printf("Total coin: %d confirmed: %d\n", score, confScore)
 	fmt.Printf("DB sync height: %d\n", height)
 	return nil
 }
@@ -266,20 +269,22 @@ func Adr(args []string) error {
 }
 
 // Fan generates a bunch of fanout.  Only for testing, can be expensive.
-// syntax: fan numOutputs valOutputs witty
+// syntax: fan adr numOutputs valOutputs witty
 func Fan(args []string) error {
 	if len(args) < 3 {
-		return fmt.Errorf("fan syntax: fan numOutputs valOutputs witty")
+		return fmt.Errorf("fan syntax: fan adr numOutputs valOutputs")
 	}
-	numOutputs, err := strconv.ParseInt(args[0], 10, 64)
+
+	adr, err := btcutil.DecodeAddress(args[0], SCon.TS.Param)
+	if err != nil {
+		fmt.Printf("error parsing %s as address\t", args[0])
+		return err
+	}
+	numOutputs, err := strconv.ParseInt(args[1], 10, 64)
 	if err != nil {
 		return err
 	}
-	valOutputs, err := strconv.ParseInt(args[1], 10, 64)
-	if err != nil {
-		return err
-	}
-	wittynum, err := strconv.ParseInt(args[2], 10, 64)
+	valOutputs, err := strconv.ParseInt(args[2], 10, 64)
 	if err != nil {
 		return err
 	}
@@ -287,21 +292,8 @@ func Fan(args []string) error {
 	adrs := make([]btcutil.Address, numOutputs)
 	amts := make([]int64, numOutputs)
 
-	oAdr, err := SCon.TS.NewAdr()
-	if err != nil {
-		return err
-	}
 	for i := int64(0); i < numOutputs; i++ {
-		if wittynum != 0 {
-			wAdr, err := btcutil.NewAddressWitnessPubKeyHash(
-				oAdr.ScriptAddress(), SCon.TS.Param)
-			if err != nil {
-				return err
-			}
-			adrs[i] = wAdr
-		} else {
-			adrs[i] = oAdr
-		}
+		adrs[i] = adr
 		amts[i] = valOutputs + i
 	}
 
@@ -331,21 +323,21 @@ func Send(args []string) error {
 	}
 	// need args, fail
 	if len(args) < 2 {
-		return fmt.Errorf("need args: ssend amount(satoshis) address wit?")
+		return fmt.Errorf("need args: ssend address amount(satoshis) wit?")
 	}
-
-	amt, err := strconv.ParseInt(args[0], 10, 64)
+	adr, err := btcutil.DecodeAddress(args[0], SCon.TS.Param)
+	if err != nil {
+		fmt.Printf("error parsing %s as address\t", args[0])
+		return err
+	}
+	amt, err := strconv.ParseInt(args[1], 10, 64)
 	if err != nil {
 		return err
 	}
 	if amt < 1000 {
 		return fmt.Errorf("can't send %d, too small", amt)
 	}
-	adr, err := btcutil.DecodeAddress(args[1], SCon.TS.Param)
-	if err != nil {
-		fmt.Printf("error parsing %s as address\t", args[1])
-		return err
-	}
+
 	fmt.Printf("send %d to address: %s \n",
 		amt, adr.String())
 
diff --git a/uspv/sortsignsend.go b/uspv/sortsignsend.go
index 1d2944832..c41d806d0 100644
--- a/uspv/sortsignsend.go
+++ b/uspv/sortsignsend.go
@@ -43,7 +43,7 @@ func (s *SPVCon) Rebroadcast() {
 	return
 }
 
-// make utxo slices sortable
+// make utxo slices sortable -- same as txsort
 type utxoSlice []Utxo
 
 // Sort utxos just like txins -- Len, Less, Swap
@@ -68,6 +68,13 @@ func (s utxoSlice) Less(i, j int) bool {
 	return bytes.Compare(ihash[:], jhash[:]) == -1
 }
 
+type utxoByAmt []Utxo
+
+// utxoByAmts get sorted by utxo value
+func (s utxoByAmt) Len() int           { return len(s) }
+func (s utxoByAmt) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+func (s utxoByAmt) Less(i, j int) bool { return s[i].Value < s[j].Value }
+
 func (s *SPVCon) NewOutgoingTx(tx *wire.MsgTx) error {
 	txid := tx.TxSha()
 	// assign height of zero for txs we create
@@ -99,14 +106,22 @@ func (s *SPVCon) SendCoins(adrs []btcutil.Address, sendAmts []int64) error {
 	var err error
 	var score, totalSend, fee int64
 	dustCutoff := int64(20000) // below this amount, just give to miners
-	satPerByte := int64(30)    // satoshis per byte fee; have as arg later
-	allUtxos, err := s.TS.GetAllUtxos()
+	satPerByte := int64(80)    // satoshis per byte fee; have as arg later
+	rawUtxos, err := s.TS.GetAllUtxos()
 	if err != nil {
 		return err
 	}
+	var allUtxos utxoByAmt
+	// start with utxos sorted by value.
 
-	for _, utxo := range allUtxos {
+	for _, utxo := range rawUtxos {
 		score += utxo.Value
+		allUtxos = append(allUtxos, *utxo)
+	}
+	sort.Sort(sort.Reverse(allUtxos))
+	//	sort.Reverse(allUtxos)
+	for _, u := range allUtxos {
+		fmt.Printf("%d ", u.Value)
 	}
 	for _, amt := range sendAmts {
 		totalSend += amt
@@ -136,8 +151,13 @@ func (s *SPVCon) SendCoins(adrs []btcutil.Address, sendAmts []int64) error {
 	// add utxos until we've had enough
 	nokori := totalSend // nokori is how much is needed on input side
 	for _, utxo := range allUtxos {
+		// skip unconfirmed.  Or de-prioritize?
+		//		if utxo.AtHeight == 0 {
+		//			continue
+		//		}
+
 		// yeah, lets add this utxo!
-		ins = append(ins, *utxo)
+		ins = append(ins, utxo)
 		// as we add utxos, fill in sigscripts
 		// generate previous pkscripts (subscritpt?) for all utxos
 		// then make txins with the utxo and prevpk, and insert them into the tx
@@ -205,6 +225,11 @@ func (s *SPVCon) SendCoins(adrs []btcutil.Address, sendAmts []int64) error {
 	// tx is ready for signing,
 	sigStash := make([][]byte, len(ins))
 	witStash := make([][][]byte, len(ins))
+
+	// generate tx-wide hashCache for segwit stuff
+	// middle index number doesn't matter for sighashAll.
+	hCache := txscript.CalcHashCache(tx, 0, txscript.SigHashAll)
+
 	for i, txin := range tx.TxIn {
 		// pick key
 		child, err := s.TS.rootPrivKey.Child(
@@ -221,7 +246,7 @@ func (s *SPVCon) SendCoins(adrs []btcutil.Address, sendAmts []int64) error {
 		// sign into stash
 		if ins[i].IsWit {
 			witStash[i], err = txscript.WitnessScript(
-				tx, i, ins[i].Value, txin.SignatureScript,
+				tx, hCache, i, ins[i].Value, txin.SignatureScript,
 				txscript.SigHashAll, priv, true)
 			if err != nil {
 				return err
diff --git a/uspv/utxodb.go b/uspv/utxodb.go
index 8383fe314..4f1c17461 100644
--- a/uspv/utxodb.go
+++ b/uspv/utxodb.go
@@ -368,16 +368,14 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 	spentOPs := make([][]byte, len(tx.TxIn))
 	// before entering into db, serialize all inputs of the ingested tx
 	for i, txin := range tx.TxIn {
-		nOP, err := outPointToBytes(&txin.PreviousOutPoint)
+		spentOPs[i], err = outPointToBytes(&txin.PreviousOutPoint)
 		if err != nil {
 			return hits, err
 		}
-		spentOPs[i] = nOP
 	}
 	// also generate PKscripts for all addresses (maybe keep storing these?)
 	for _, adr := range ts.Adrs {
 		// iterate through all our addresses
-
 		// convert regular address to witness address.  (split adrs later)
 		wa, err := btcutil.NewAddressWitnessPubKeyHash(
 			adr.PkhAdr.ScriptAddress(), ts.Param)
@@ -437,59 +435,49 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 		// first see if we lose utxos
 		// iterate through duffel bag and look for matches
 		// this makes us lose money, which is regrettable, but we need to know.
-
-		var delOPs [][]byte
-		//		fmt.Printf("%d nOP to iterate over\n", len(spentOPs))
+		//		var delOPs [][]byte
 		for _, nOP := range spentOPs {
-			duf.ForEach(func(k, v []byte) error {
-				if bytes.Equal(k, nOP) { // matched, we lost utxo
-					fmt.Printf("will delete point %x\n", k)
-					hits++
-					// do all this just to figure out value we lost
-					x := make([]byte, len(k)+len(v))
-					copy(x, k)
-					y := make([]byte, len(k))
-					// mark utxo for deletion
-					copy(y, k)
-					delOPs = append(delOPs, y)
-
-					copy(x[len(k):], v)
-					lostTxo, err := UtxoFromBytes(x)
-					if err != nil {
-						return err
-					}
-
-					// after marking for deletion, save stxo to old bucket
-					var st Stxo               // generate spent txo
-					st.Utxo = lostTxo         // assign outpoint
-					st.SpendHeight = height   // spent at height
-					st.SpendTxid = tx.TxSha() // spent by txid
-					stxb, err := st.ToBytes() // serialize
-					if err != nil {
-						return err
-					}
-					err = old.Put(k, stxb) // write k:v outpoint:stxo bytes
-					if err != nil {
-						return err
-					}
-					// store this relevant tx
-					sha := tx.TxSha()
-					var buf bytes.Buffer
-					tx.SerializeWitness(&buf) // always store witness version
-					err = txns.Put(sha.Bytes(), buf.Bytes())
-					if err != nil {
-						return err
-					}
-
-					return nil // matched utxo k, won't match another
+			v := duf.Get(nOP)
+			if v != nil {
+				hits++
+				// do all this just to figure out value we lost
+				x := make([]byte, len(nOP)+len(v))
+				copy(x, nOP)
+				copy(x[len(nOP):], v)
+				lostTxo, err := UtxoFromBytes(x)
+				if err != nil {
+					return err
 				}
-				return nil // no match
-			})
-			// delete after done with foreach
+
+				// after marking for deletion, save stxo to old bucket
+				var st Stxo               // generate spent txo
+				st.Utxo = lostTxo         // assign outpoint
+				st.SpendHeight = height   // spent at height
+				st.SpendTxid = tx.TxSha() // spent by txid
+				stxb, err := st.ToBytes() // serialize
+				if err != nil {
+					return err
+				}
+				err = old.Put(nOP, stxb) // write nOP:v outpoint:stxo bytes
+				if err != nil {
+					return err
+				}
+				// store this relevant tx
+				sha := tx.TxSha()
+				var buf bytes.Buffer
+				tx.SerializeWitness(&buf) // always store witness version
+				err = txns.Put(sha.Bytes(), buf.Bytes())
+				if err != nil {
+					return err
+				}
+				// stash for deletion
+				//				delOPs = append(delOPs, nOP)
+			}
 		}
-		for _, y := range delOPs {
-			fmt.Printf("deleting outpoint %x\n", y)
-			err = duf.Delete(y)
+
+		//delete everything even if it doesn't exist!
+		for _, dOP := range spentOPs {
+			err = duf.Delete(dOP)
 			if err != nil {
 				return err
 			}

From f9a740d3923e8b898cbeadfce0322eeff1f0ca78 Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Wed, 24 Feb 2016 17:27:29 -0800
Subject: [PATCH 18/22] more stress testing stuff

---
 shell.go             |  79 +++++++++++++++++++++++++++---
 uspv/msghandler.go   |   1 -
 uspv/sortsignsend.go | 111 ++++++++++++++++++++++++++++++++++++++-----
 uspv/utxodb.go       |  28 +++++++----
 4 files changed, 189 insertions(+), 30 deletions(-)

diff --git a/shell.go b/shell.go
index a1b85d9a9..2ec9a50ba 100644
--- a/shell.go
+++ b/shell.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -147,6 +148,13 @@ func Shellparse(cmdslice []string) error {
 		}
 		return nil
 	}
+	if cmd == "sweep" {
+		err = Sweep(args)
+		if err != nil {
+			fmt.Printf("sweep error: %s\n", err)
+		}
+		return nil
+	}
 	if cmd == "txs" {
 		err = Txs(args)
 		if err != nil {
@@ -202,10 +210,17 @@ func Bal(args []string) error {
 		return fmt.Errorf("Can't get balance, spv connection broken")
 	}
 	fmt.Printf(" ----- Account Balance ----- \n")
-	allUtxos, err := SCon.TS.GetAllUtxos()
+	rawUtxos, err := SCon.TS.GetAllUtxos()
 	if err != nil {
 		return err
 	}
+	var allUtxos uspv.SortableUtxoSlice
+	for _, utxo := range rawUtxos {
+		allUtxos = append(allUtxos, *utxo)
+	}
+	// smallest and unconfirmed last (because it's reversed)
+	sort.Sort(sort.Reverse(allUtxos))
+
 	var score, confScore int64
 	for i, u := range allUtxos {
 		fmt.Printf("\tutxo %d height %d %s key:%d amt %d",
@@ -221,6 +236,10 @@ func Bal(args []string) error {
 	}
 	height, _ := SCon.TS.GetDBSyncHeight()
 
+	atx, err := SCon.TS.GetAllTxs()
+
+	stxos, err := SCon.TS.GetAllStxos()
+
 	for i, a := range SCon.TS.Adrs {
 		wa, err := btcutil.NewAddressWitnessPubKeyHash(
 			a.PkhAdr.ScriptAddress(), Params)
@@ -229,8 +248,9 @@ func Bal(args []string) error {
 		}
 		fmt.Printf("address %d %s OR %s\n", i, a.PkhAdr.String(), wa.String())
 	}
-
-	fmt.Printf("Total known utxos: %d\n", len(allUtxos))
+	fmt.Printf("Total known txs: %d\n", len(atx))
+	fmt.Printf("Known utxos: %d\tPreviously spent txos: %d\n",
+		len(allUtxos), len(stxos))
 	fmt.Printf("Total coin: %d confirmed: %d\n", score, confScore)
 	fmt.Printf("DB sync height: %d\n", height)
 	return nil
@@ -268,13 +288,62 @@ func Adr(args []string) error {
 	return nil
 }
 
+// Sweep sends every confirmed uxto in your wallet to an address.
+// it does them all individually to there are a lot of txs generated.
+// syntax: sweep adr
+func Sweep(args []string) error {
+	if len(args) < 2 {
+		return fmt.Errorf("sweep syntax: sweep adr")
+	}
+
+	adr, err := btcutil.DecodeAddress(args[0], SCon.TS.Param)
+	if err != nil {
+		fmt.Printf("error parsing %s as address\t", args[0])
+		return err
+	}
+	numTxs, err := strconv.ParseInt(args[1], 10, 64)
+	if err != nil {
+		return err
+	}
+	if numTxs < 1 {
+		return fmt.Errorf("can't send %d txs", numTxs)
+	}
+
+	rawUtxos, err := SCon.TS.GetAllUtxos()
+	if err != nil {
+		return err
+	}
+	var allUtxos uspv.SortableUtxoSlice
+	for _, utxo := range rawUtxos {
+		allUtxos = append(allUtxos, *utxo)
+	}
+	// smallest and unconfirmed last (because it's reversed)
+	sort.Sort(sort.Reverse(allUtxos))
+
+	for i, u := range allUtxos {
+		if u.AtHeight != 0 {
+			err = SCon.SendOne(allUtxos[i], adr)
+			if err != nil {
+				return err
+			}
+			numTxs--
+			if numTxs == 0 {
+				return nil
+			}
+		}
+	}
+
+	fmt.Printf("spent all confirmed utxos; not enough by %d\n", numTxs)
+
+	return nil
+}
+
 // Fan generates a bunch of fanout.  Only for testing, can be expensive.
 // syntax: fan adr numOutputs valOutputs witty
 func Fan(args []string) error {
 	if len(args) < 3 {
 		return fmt.Errorf("fan syntax: fan adr numOutputs valOutputs")
 	}
-
 	adr, err := btcutil.DecodeAddress(args[0], SCon.TS.Param)
 	if err != nil {
 		fmt.Printf("error parsing %s as address\t", args[0])
@@ -296,9 +365,7 @@ func Fan(args []string) error {
 		adrs[i] = adr
 		amts[i] = valOutputs + i
 	}
-
 	return SCon.SendCoins(adrs, amts)
-
 }
 
 // Send sends coins.
diff --git a/uspv/msghandler.go b/uspv/msghandler.go
index f7fcf5828..64f7fce0c 100644
--- a/uspv/msghandler.go
+++ b/uspv/msghandler.go
@@ -189,7 +189,6 @@ func (s *SPVCon) GetDataHandler(m *wire.MsgGetData) {
 				log.Printf("error getting tx %s: %s",
 					thing.Hash.String(), err.Error())
 			}
-			fmt.Printf(TxToString(tx))
 			s.outMsgQueue <- tx
 			sent++
 			continue
diff --git a/uspv/sortsignsend.go b/uspv/sortsignsend.go
index c41d806d0..de8e340e4 100644
--- a/uspv/sortsignsend.go
+++ b/uspv/sortsignsend.go
@@ -68,12 +68,22 @@ func (s utxoSlice) Less(i, j int) bool {
 	return bytes.Compare(ihash[:], jhash[:]) == -1
 }
 
-type utxoByAmt []Utxo
+type SortableUtxoSlice []Utxo
 
-// utxoByAmts get sorted by utxo value
-func (s utxoByAmt) Len() int           { return len(s) }
-func (s utxoByAmt) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
-func (s utxoByAmt) Less(i, j int) bool { return s[i].Value < s[j].Value }
+// utxoByAmts get sorted by utxo value. also put unconfirmed last
+func (s SortableUtxoSlice) Len() int      { return len(s) }
+func (s SortableUtxoSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+
+// height 0 means your lesser
+func (s SortableUtxoSlice) Less(i, j int) bool {
+	if s[i].AtHeight == 0 {
+		return true
+	}
+	if s[j].AtHeight == 0 {
+		return false
+	}
+	return s[i].Value < s[j].Value
+}
 
 func (s *SPVCon) NewOutgoingTx(tx *wire.MsgTx) error {
 	txid := tx.TxSha()
@@ -97,6 +107,84 @@ func (s *SPVCon) NewOutgoingTx(tx *wire.MsgTx) error {
 	return nil
 }
 
+func (s *SPVCon) SendOne(u Utxo, adr btcutil.Address) error {
+	// fixed fee
+	fee := int64(5000)
+
+	sendAmt := u.Value - fee
+	tx := wire.NewMsgTx() // make new tx
+	// add single output
+	outAdrScript, err := txscript.PayToAddrScript(adr)
+	if err != nil {
+		return err
+	}
+	// make user specified txout and add to tx
+	txout := wire.NewTxOut(sendAmt, outAdrScript)
+	tx.AddTxOut(txout)
+
+	var prevPKs []byte
+	if u.IsWit {
+		tx.Flags = 0x01
+		wa, err := btcutil.NewAddressWitnessPubKeyHash(
+			s.TS.Adrs[u.KeyIdx].PkhAdr.ScriptAddress(), s.TS.Param)
+		prevPKs, err = txscript.PayToAddrScript(wa)
+		if err != nil {
+			return err
+		}
+	} else { // otherwise generate directly
+		prevPKs, err = txscript.PayToAddrScript(
+			s.TS.Adrs[u.KeyIdx].PkhAdr)
+		if err != nil {
+			return err
+		}
+	}
+
+	tx.AddTxIn(wire.NewTxIn(&u.Op, prevPKs, nil))
+
+	var sig []byte
+	var wit [][]byte
+	hCache := txscript.CalcHashCache(tx, 0, txscript.SigHashAll)
+
+	child, err := s.TS.rootPrivKey.Child(u.KeyIdx + hdkeychain.HardenedKeyStart)
+
+	if err != nil {
+		return err
+	}
+	priv, err := child.ECPrivKey()
+	if err != nil {
+		return err
+	}
+
+	// This is where witness based sighash types need to happen
+	// sign into stash
+	if u.IsWit {
+		wit, err = txscript.WitnessScript(
+			tx, hCache, 0, u.Value, tx.TxIn[0].SignatureScript,
+			txscript.SigHashAll, priv, true)
+		if err != nil {
+			return err
+		}
+	} else {
+		sig, err = txscript.SignatureScript(
+			tx, 0, tx.TxIn[0].SignatureScript,
+			txscript.SigHashAll, priv, true)
+		if err != nil {
+			return err
+		}
+	}
+
+	// swap sigs into sigScripts in txins
+
+	if sig != nil {
+		tx.TxIn[0].SignatureScript = sig
+	}
+	if wit != nil {
+		tx.TxIn[0].Witness = wit
+		tx.TxIn[0].SignatureScript = nil
+	}
+	return s.NewOutgoingTx(tx)
+}
+
 // SendCoins does send coins, but it's very rudimentary
 // wit makes it into p2wpkh.  Which is not yet spendable.
 func (s *SPVCon) SendCoins(adrs []btcutil.Address, sendAmts []int64) error {
@@ -111,18 +199,17 @@ func (s *SPVCon) SendCoins(adrs []btcutil.Address, sendAmts []int64) error {
 	if err != nil {
 		return err
 	}
-	var allUtxos utxoByAmt
+	var allUtxos SortableUtxoSlice
 	// start with utxos sorted by value.
 
 	for _, utxo := range rawUtxos {
 		score += utxo.Value
 		allUtxos = append(allUtxos, *utxo)
 	}
+	// smallest and unconfirmed last (because it's reversed)
 	sort.Sort(sort.Reverse(allUtxos))
+
 	//	sort.Reverse(allUtxos)
-	for _, u := range allUtxos {
-		fmt.Printf("%d ", u.Value)
-	}
 	for _, amt := range sendAmts {
 		totalSend += amt
 	}
@@ -272,10 +359,8 @@ func (s *SPVCon) SendCoins(adrs []btcutil.Address, sendAmts []int64) error {
 
 	}
 
-	fmt.Printf("tx: %s", TxToString(tx))
-	buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize()))
-	tx.SerializeWitness(buf)
-	fmt.Printf("tx: %x\n", buf.Bytes())
+	//	fmt.Printf("tx: %s", TxToString(tx))
+	//	buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize()))
 
 	// send it out on the wire.  hope it gets there.
 	// we should deal with rejects.  Don't yet.
diff --git a/uspv/utxodb.go b/uspv/utxodb.go
index 4f1c17461..265ec3151 100644
--- a/uspv/utxodb.go
+++ b/uspv/utxodb.go
@@ -373,8 +373,14 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 			return hits, err
 		}
 	}
-	// also generate PKscripts for all addresses (maybe keep storing these?)
-	for _, adr := range ts.Adrs {
+
+	// go through txouts, and then go through addresses to match
+
+	// generate PKscripts for all addresses
+	wPKscripts := make([][]byte, len(ts.Adrs))
+	aPKscripts := make([][]byte, len(ts.Adrs))
+
+	for i, adr := range ts.Adrs {
 		// iterate through all our addresses
 		// convert regular address to witness address.  (split adrs later)
 		wa, err := btcutil.NewAddressWitnessPubKeyHash(
@@ -383,28 +389,30 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 			return hits, err
 		}
 
-		wPKscript, err := txscript.PayToAddrScript(wa)
+		wPKscripts[i], err = txscript.PayToAddrScript(wa)
 		if err != nil {
 			return hits, err
 		}
-		aPKscript, err := txscript.PayToAddrScript(adr.PkhAdr)
+		aPKscripts[i], err = txscript.PayToAddrScript(adr.PkhAdr)
 		if err != nil {
 			return hits, err
 		}
+	}
 
-		// iterate through all outputs of this tx, see if we gain
-		for i, out := range tx.TxOut {
+	// iterate through all outputs of this tx, see if we gain
+	for i, out := range tx.TxOut {
+		for j, ascr := range aPKscripts {
 
 			// detect p2wpkh
 			witBool := false
-			if bytes.Equal(out.PkScript, wPKscript) {
+			if bytes.Equal(out.PkScript, wPKscripts[j]) {
 				witBool = true
 			}
 
-			if bytes.Equal(out.PkScript, aPKscript) || witBool { // new utxo found
+			if bytes.Equal(out.PkScript, ascr) || witBool { // new utxo found
 				var newu Utxo // create new utxo and copy into it
 				newu.AtHeight = height
-				newu.KeyIdx = adr.KeyIdx
+				newu.KeyIdx = ts.Adrs[j].KeyIdx
 				newu.Value = out.Value
 				newu.IsWit = witBool // copy witness version from pkscript
 				var newop wire.OutPoint
@@ -417,7 +425,7 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 				}
 				nUtxoBytes = append(nUtxoBytes, b)
 				hits++
-				// break // keep looking! address re-use in same tx
+				break // txos can match only 1 script
 			}
 		}
 	}

From caf56aaff070ac5544598f4d4341dd3768acbf80 Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Thu, 25 Feb 2016 12:08:35 -0800
Subject: [PATCH 19/22] db optimizations

---
 uspv/msghandler.go   | 32 ++++++++++++++++----------------
 uspv/sortsignsend.go |  4 ++--
 uspv/utxodb.go       | 26 +++++++++++++-------------
 3 files changed, 31 insertions(+), 31 deletions(-)

diff --git a/uspv/msghandler.go b/uspv/msghandler.go
index 64f7fce0c..e8506289d 100644
--- a/uspv/msghandler.go
+++ b/uspv/msghandler.go
@@ -141,22 +141,22 @@ func (s *SPVCon) TxHandler(m *wire.MsgTx) {
 	}
 
 	// check for double spends
-	allTxs, err := s.TS.GetAllTxs()
-	if err != nil {
-		log.Printf("Can't get txs from db: %s", err.Error())
-		return
-	}
-	dubs, err := CheckDoubleSpends(m, allTxs)
-	if err != nil {
-		log.Printf("CheckDoubleSpends error: %s", err.Error())
-		return
-	}
-	if len(dubs) > 0 {
-		for i, dub := range dubs {
-			fmt.Printf("dub %d known tx %s and new tx %s are exclusive!!!\n",
-				i, dub.String(), m.TxSha().String())
-		}
-	}
+	//	allTxs, err := s.TS.GetAllTxs()
+	//	if err != nil {
+	//		log.Printf("Can't get txs from db: %s", err.Error())
+	//		return
+	//	}
+	//	dubs, err := CheckDoubleSpends(m, allTxs)
+	//	if err != nil {
+	//		log.Printf("CheckDoubleSpends error: %s", err.Error())
+	//		return
+	//	}
+	//	if len(dubs) > 0 {
+	//		for i, dub := range dubs {
+	//			fmt.Printf("dub %d known tx %s and new tx %s are exclusive!!!\n",
+	//				i, dub.String(), m.TxSha().String())
+	//		}
+	//	}
 	hits, err := s.TS.Ingest(m, height)
 	if err != nil {
 		log.Printf("Incoming Tx error: %s\n", err.Error())
diff --git a/uspv/sortsignsend.go b/uspv/sortsignsend.go
index de8e340e4..bb51218e2 100644
--- a/uspv/sortsignsend.go
+++ b/uspv/sortsignsend.go
@@ -76,10 +76,10 @@ func (s SortableUtxoSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
 
 // height 0 means your lesser
 func (s SortableUtxoSlice) Less(i, j int) bool {
-	if s[i].AtHeight == 0 {
+	if s[i].AtHeight == 0 && s[j].AtHeight > 0 {
 		return true
 	}
-	if s[j].AtHeight == 0 {
+	if s[j].AtHeight == 0 && s[i].AtHeight > 0 {
 		return false
 	}
 	return s[i].Value < s[j].Value
diff --git a/uspv/utxodb.go b/uspv/utxodb.go
index 265ec3151..8fd194899 100644
--- a/uspv/utxodb.go
+++ b/uspv/utxodb.go
@@ -380,11 +380,11 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 	wPKscripts := make([][]byte, len(ts.Adrs))
 	aPKscripts := make([][]byte, len(ts.Adrs))
 
-	for i, adr := range ts.Adrs {
+	for i, _ := range ts.Adrs {
 		// iterate through all our addresses
 		// convert regular address to witness address.  (split adrs later)
 		wa, err := btcutil.NewAddressWitnessPubKeyHash(
-			adr.PkhAdr.ScriptAddress(), ts.Param)
+			ts.Adrs[i].PkhAdr.ScriptAddress(), ts.Param)
 		if err != nil {
 			return hits, err
 		}
@@ -393,7 +393,7 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 		if err != nil {
 			return hits, err
 		}
-		aPKscripts[i], err = txscript.PayToAddrScript(adr.PkhAdr)
+		aPKscripts[i], err = txscript.PayToAddrScript(ts.Adrs[i].PkhAdr)
 		if err != nil {
 			return hits, err
 		}
@@ -440,10 +440,8 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 			return fmt.Errorf("error: db not initialized")
 		}
 
-		// first see if we lose utxos
 		// iterate through duffel bag and look for matches
 		// this makes us lose money, which is regrettable, but we need to know.
-		//		var delOPs [][]byte
 		for _, nOP := range spentOPs {
 			v := duf.Get(nOP)
 			if v != nil {
@@ -478,18 +476,20 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 				if err != nil {
 					return err
 				}
-				// stash for deletion
-				//				delOPs = append(delOPs, nOP)
+				err = duf.Delete(nOP)
+				if err != nil {
+					return err
+				}
 			}
 		}
 
 		//delete everything even if it doesn't exist!
-		for _, dOP := range spentOPs {
-			err = duf.Delete(dOP)
-			if err != nil {
-				return err
-			}
-		}
+		//		for _, dOP := range spentOPs {
+		//			err = duf.Delete(dOP)
+		//			if err != nil {
+		//				return err
+		//			}
+		//		}
 		// done losing utxos, next gain utxos
 		// next add all new utxos to db, this is quick as the work is above
 		for _, ub := range nUtxoBytes {

From 2c1f42d6fdb0ba17f9f6603f77037a5307a3a9a4 Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Thu, 25 Feb 2016 18:19:43 -0800
Subject: [PATCH 20/22] fix block out of order error

---
 uspv/eight333.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/uspv/eight333.go b/uspv/eight333.go
index 548860369..5a6478a3d 100644
--- a/uspv/eight333.go
+++ b/uspv/eight333.go
@@ -396,9 +396,10 @@ func (s *SPVCon) AskForBlocks() error {
 		if dbTip == headerTip { // if this is the last block, indicate finality
 			hah.final = true
 		}
-		s.outMsgQueue <- gdataMsg
 		// waits here most of the time for the queue to empty out
 		s.blockQueue <- hah // push height and mroot of requested block on queue
+		s.outMsgQueue <- gdataMsg
+
 		dbTip++
 	}
 	return nil

From d2d37ce1abe430b73cc3d3fb1becf8a180136123 Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Thu, 25 Feb 2016 21:05:01 -0800
Subject: [PATCH 21/22] make it way faster

there were a lot of dumb things in Ingest() that made it really slow.
Like re-hashing the tx a bunch of times.  And re-saving it to the
db redundantly.

also added local bloom filters.  Maybe some concurrency issues if you
generate an address just as you're getting a tx with that address but
that doesn't seem like a real problem.  Cheap to rescan anyway.

So it's faster and works better.
---
 uspv/hardmode.go   | 62 ++++++++++++++++++++++++++++++++++++++--------
 uspv/init.go       |  7 ++++++
 uspv/msghandler.go | 28 ++++++++++++---------
 uspv/txstore.go    |  2 ++
 uspv/utxodb.go     | 34 +++++++++++--------------
 5 files changed, 91 insertions(+), 42 deletions(-)

diff --git a/uspv/hardmode.go b/uspv/hardmode.go
index b54e85264..445d10296 100644
--- a/uspv/hardmode.go
+++ b/uspv/hardmode.go
@@ -6,6 +6,8 @@ import (
 	"log"
 
 	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+	"github.com/btcsuite/btcutil/bloom"
 )
 
 var (
@@ -115,6 +117,27 @@ func calcRoot(hashes []*wire.ShaHash) *wire.ShaHash {
 	return hashes[0]
 }
 
+func (ts *TxStore) Refilter() error {
+	allUtxos, err := ts.GetAllUtxos()
+	if err != nil {
+		return err
+	}
+	filterElements := uint32(len(allUtxos) + len(ts.Adrs))
+
+	ts.localFilter = bloom.NewFilter(filterElements, 0, 0, wire.BloomUpdateAll)
+
+	for _, u := range allUtxos {
+		ts.localFilter.AddOutPoint(&u.Op)
+	}
+	for _, a := range ts.Adrs {
+		ts.localFilter.Add(a.PkhAdr.ScriptAddress())
+	}
+
+	msg := ts.localFilter.MsgFilterLoad()
+	fmt.Printf("made %d element filter: %x\n", filterElements, msg.Filter)
+	return nil
+}
+
 // IngestBlock is like IngestMerkleBlock but aralphic
 // different enough that it's better to have 2 separate functions
 func (s *SPVCon) IngestBlock(m *wire.MsgBlock) {
@@ -148,21 +171,39 @@ func (s *SPVCon) IngestBlock(m *wire.MsgBlock) {
 		return
 	}
 
+	fPositive := 0 // local filter false positives
+	reFilter := 10 // after that many false positives, regenerate filter.
+	// 10?  Making it up.  False positives have disk i/o cost, and regenning
+	// the filter also has costs.  With a large local filter, false positives
+	// should be rare.
+
 	// iterate through all txs in the block, looking for matches.
-	// this is slow and can be sped up by doing in-ram filters client side.
-	// kindof a pain to implement though and it's fast enough for now.
+	// use a local bloom filter to ignore txs that don't affect us
 	for i, tx := range m.Transactions {
-		hits, err := s.TS.Ingest(tx, hah.height)
-		if err != nil {
-			log.Printf("Incoming Tx error: %s\n", err.Error())
-			return
-		}
-		if hits > 0 {
-			log.Printf("block %d tx %d %s ingested and matches %d utxo/adrs.",
-				hah.height, i, tx.TxSha().String(), hits)
+		utilTx := btcutil.NewTx(tx)
+		if s.TS.localFilter.MatchTxAndUpdate(utilTx) {
+			hits, err := s.TS.Ingest(tx, hah.height)
+			if err != nil {
+				log.Printf("Incoming Tx error: %s\n", err.Error())
+				return
+			}
+			if hits > 0 {
+				log.Printf("block %d tx %d %s ingested and matches %d utxo/adrs.",
+					hah.height, i, tx.TxSha().String(), hits)
+			} else {
+				fPositive++ // matched filter but no hits
+			}
 		}
 	}
 
+	if fPositive > reFilter {
+		fmt.Printf("%d filter false positives in this block\n", fPositive)
+		err = s.TS.Refilter()
+		if err != nil {
+			log.Printf("Refilter error: %s\n", err.Error())
+			return
+		}
+	}
 	// write to db that we've sync'd to the height indicated in the
 	// merkle block.  This isn't QUITE true since we haven't actually gotten
 	// the txs yet but if there are problems with the txs we should backtrack.
@@ -171,6 +212,7 @@ func (s *SPVCon) IngestBlock(m *wire.MsgBlock) {
 		log.Printf("full block sync error: %s\n", err.Error())
 		return
 	}
+
 	fmt.Printf("ingested full block %s height %d OK\n",
 		m.Header.BlockSha().String(), hah.height)
 
diff --git a/uspv/init.go b/uspv/init.go
index 4fb3844a1..0e723daa2 100644
--- a/uspv/init.go
+++ b/uspv/init.go
@@ -91,6 +91,13 @@ func OpenSPV(remoteNode string, hfn, dbfn string,
 	s.inWaitState = make(chan bool, 1)
 	go s.fPositiveHandler()
 
+	if hard {
+		err = s.TS.Refilter()
+		if err != nil {
+			return s, err
+		}
+	}
+
 	return s, nil
 }
 
diff --git a/uspv/msghandler.go b/uspv/msghandler.go
index e8506289d..2365f993a 100644
--- a/uspv/msghandler.go
+++ b/uspv/msghandler.go
@@ -5,6 +5,7 @@ import (
 	"log"
 
 	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
 )
 
 func (s *SPVCon) incomingMessageHandler() {
@@ -157,19 +158,22 @@ func (s *SPVCon) TxHandler(m *wire.MsgTx) {
 	//				i, dub.String(), m.TxSha().String())
 	//		}
 	//	}
-	hits, err := s.TS.Ingest(m, height)
-	if err != nil {
-		log.Printf("Incoming Tx error: %s\n", err.Error())
-		return
+	utilTx := btcutil.NewTx(m)
+	if !s.HardMode || s.TS.localFilter.MatchTxAndUpdate(utilTx) {
+		hits, err := s.TS.Ingest(m, height)
+		if err != nil {
+			log.Printf("Incoming Tx error: %s\n", err.Error())
+			return
+		}
+		if hits == 0 && !s.HardMode {
+			log.Printf("tx %s had no hits, filter false positive.",
+				m.TxSha().String())
+			s.fPositives <- 1 // add one false positive to chan
+			return
+		}
+		log.Printf("tx %s ingested and matches %d utxo/adrs.",
+			m.TxSha().String(), hits)
 	}
-	if hits == 0 && !s.HardMode {
-		log.Printf("tx %s had no hits, filter false positive.",
-			m.TxSha().String())
-		s.fPositives <- 1 // add one false positive to chan
-		return
-	}
-	log.Printf("tx %s ingested and matches %d utxo/adrs.",
-		m.TxSha().String(), hits)
 }
 
 // GetDataHandler responds to requests for tx data, which happen after
diff --git a/uspv/txstore.go b/uspv/txstore.go
index a4b8aadd2..acf74f9f5 100644
--- a/uspv/txstore.go
+++ b/uspv/txstore.go
@@ -23,6 +23,8 @@ type TxStore struct {
 	Adrs    []MyAdr  // endeavouring to acquire capital
 	StateDB *bolt.DB // place to write all this down
 
+	localFilter *bloom.Filter // local bloom filter for hard mode
+
 	// Params live here, not SCon
 	Param *chaincfg.Params // network parameters (testnet3, testnetL)
 
diff --git a/uspv/utxodb.go b/uspv/utxodb.go
index 8fd194899..96f1115a3 100644
--- a/uspv/utxodb.go
+++ b/uspv/utxodb.go
@@ -125,7 +125,7 @@ func (ts *TxStore) NewAdr() (btcutil.Address, error) {
 	ma.KeyIdx = n
 
 	ts.Adrs = append(ts.Adrs, ma)
-
+	ts.localFilter.Add(ma.PkhAdr.ScriptAddress())
 	return nAdr, nil
 }
 
@@ -399,16 +399,15 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 		}
 	}
 
+	cachedSha := tx.TxSha()
 	// iterate through all outputs of this tx, see if we gain
 	for i, out := range tx.TxOut {
 		for j, ascr := range aPKscripts {
-
 			// detect p2wpkh
 			witBool := false
 			if bytes.Equal(out.PkScript, wPKscripts[j]) {
 				witBool = true
 			}
-
 			if bytes.Equal(out.PkScript, ascr) || witBool { // new utxo found
 				var newu Utxo // create new utxo and copy into it
 				newu.AtHeight = height
@@ -416,7 +415,7 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 				newu.Value = out.Value
 				newu.IsWit = witBool // copy witness version from pkscript
 				var newop wire.OutPoint
-				newop.Hash = tx.TxSha()
+				newop.Hash = cachedSha
 				newop.Index = uint32(i)
 				newu.Op = newop
 				b, err := newu.ToBytes()
@@ -459,7 +458,7 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 				var st Stxo               // generate spent txo
 				st.Utxo = lostTxo         // assign outpoint
 				st.SpendHeight = height   // spent at height
-				st.SpendTxid = tx.TxSha() // spent by txid
+				st.SpendTxid = cachedSha  // spent by txid
 				stxb, err := st.ToBytes() // serialize
 				if err != nil {
 					return err
@@ -468,14 +467,7 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 				if err != nil {
 					return err
 				}
-				// store this relevant tx
-				sha := tx.TxSha()
-				var buf bytes.Buffer
-				tx.SerializeWitness(&buf) // always store witness version
-				err = txns.Put(sha.Bytes(), buf.Bytes())
-				if err != nil {
-					return err
-				}
+
 				err = duf.Delete(nOP)
 				if err != nil {
 					return err
@@ -483,13 +475,6 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 			}
 		}
 
-		//delete everything even if it doesn't exist!
-		//		for _, dOP := range spentOPs {
-		//			err = duf.Delete(dOP)
-		//			if err != nil {
-		//				return err
-		//			}
-		//		}
 		// done losing utxos, next gain utxos
 		// next add all new utxos to db, this is quick as the work is above
 		for _, ub := range nUtxoBytes {
@@ -498,6 +483,15 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 				return err
 			}
 		}
+
+		// if hits is nonzero it's a relevant tx and we should store it
+		var buf bytes.Buffer
+		tx.SerializeWitness(&buf) // always store witness version
+		err = txns.Put(cachedSha.Bytes(), buf.Bytes())
+		if err != nil {
+			return err
+		}
+
 		return nil
 	})
 	return hits, err

From 623d67234d579797d0b72a6968bf2007c70e57d3 Mon Sep 17 00:00:00 2001
From: Tadge Dryja <tadge@lightning.network>
Date: Fri, 26 Feb 2016 12:30:15 -0800
Subject: [PATCH 22/22] was requesting an already known block, fixed that

---
 uspv/eight333.go | 10 +++++-----
 uspv/hardmode.go |  6 +++---
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/uspv/eight333.go b/uspv/eight333.go
index 5a6478a3d..4eb525ac2 100644
--- a/uspv/eight333.go
+++ b/uspv/eight333.go
@@ -349,7 +349,7 @@ func (s *SPVCon) AskForBlocks() error {
 		return nil
 	}
 
-	fmt.Printf("will request blocks %d to %d\n", dbTip, headerTip)
+	fmt.Printf("will request blocks %d to %d\n", dbTip+1, headerTip)
 
 	if !s.HardMode { // don't send this in hardmode! that's the whole point
 		// create initial filter
@@ -362,9 +362,10 @@ func (s *SPVCon) AskForBlocks() error {
 		fmt.Printf("sent filter %x\n", filt.MsgFilterLoad().Filter)
 	}
 	// loop through all heights where we want merkleblocks.
-	for dbTip <= headerTip {
-		// load header from file
+	for dbTip < headerTip {
+		dbTip++ // we're requesting the next header
 
+		// load header from file
 		s.headerMutex.Lock() // seek to header we need
 		_, err = s.headerFile.Seek(int64((dbTip)*80), os.SEEK_SET)
 		if err != nil {
@@ -392,6 +393,7 @@ func (s *SPVCon) AskForBlocks() error {
 		if err != nil {
 			return err
 		}
+
 		hah := NewRootAndHeight(hdr.BlockSha(), dbTip)
 		if dbTip == headerTip { // if this is the last block, indicate finality
 			hah.final = true
@@ -399,8 +401,6 @@ func (s *SPVCon) AskForBlocks() error {
 		// waits here most of the time for the queue to empty out
 		s.blockQueue <- hah // push height and mroot of requested block on queue
 		s.outMsgQueue <- gdataMsg
-
-		dbTip++
 	}
 	return nil
 }
diff --git a/uspv/hardmode.go b/uspv/hardmode.go
index 445d10296..bc40e2c8e 100644
--- a/uspv/hardmode.go
+++ b/uspv/hardmode.go
@@ -179,7 +179,7 @@ func (s *SPVCon) IngestBlock(m *wire.MsgBlock) {
 
 	// iterate through all txs in the block, looking for matches.
 	// use a local bloom filter to ignore txs that don't affect us
-	for i, tx := range m.Transactions {
+	for _, tx := range m.Transactions {
 		utilTx := btcutil.NewTx(tx)
 		if s.TS.localFilter.MatchTxAndUpdate(utilTx) {
 			hits, err := s.TS.Ingest(tx, hah.height)
@@ -188,8 +188,8 @@ func (s *SPVCon) IngestBlock(m *wire.MsgBlock) {
 				return
 			}
 			if hits > 0 {
-				log.Printf("block %d tx %d %s ingested and matches %d utxo/adrs.",
-					hah.height, i, tx.TxSha().String(), hits)
+				// log.Printf("block %d tx %d %s ingested and matches %d utxo/adrs.",
+				//	hah.height, i, tx.TxSha().String(), hits)
 			} else {
 				fPositive++ // matched filter but no hits
 			}