From 02d17bb81a695816510ab7ad92ff1548e14426e0 Mon Sep 17 00:00:00 2001 From: yyforyongyu Date: Fri, 15 Apr 2022 02:42:54 +0800 Subject: [PATCH 1/2] tlv: update go mod to include `require` --- tlv/go.mod | 1 + tlv/go.sum | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/tlv/go.mod b/tlv/go.mod index 9257d61d7..df83fa6b4 100644 --- a/tlv/go.mod +++ b/tlv/go.mod @@ -5,6 +5,7 @@ require ( github.com/btcsuite/btcd/btcec/v2 v2.1.0 github.com/davecgh/go-spew v1.1.1 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 + github.com/stretchr/testify v1.7.1 ) go 1.16 diff --git a/tlv/go.sum b/tlv/go.sum index 04a9b110f..204b90e4c 100644 --- a/tlv/go.sum +++ b/tlv/go.sum @@ -17,6 +17,7 @@ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= @@ -49,6 +50,11 @@ github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -79,9 +85,12 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 4ddb5c586bd83751ff4749dae7498a523501fccc Mon Sep 17 00:00:00 2001 From: yyforyongyu Date: Fri, 15 Apr 2022 02:43:24 +0800 Subject: [PATCH 2/2] tlv: add `MakeBigSizeRecord` to save space This commit adds a pair of encoder/decoder to take the advantage of the BigSize format when encoding an uint64 and possibly an uint32. Often the time an uint64 value is not big enough to fill all the 8 bytes, thus using BigSize can save extra bytes when save it to db. And for uint32, if we know most of the values do not exceed 65536, we can also save at least 1 byte using BigSize format. This commit introduces `MakeBigSizeRecord` that can be used optionally where db space is a concern. --- docs/release-notes/release-notes-0.15.0.md | 3 + tlv/primitive.go | 55 +++++++ tlv/record.go | 37 +++++ tlv/stream_test.go | 162 +++++++++++++++++++++ 4 files changed, 257 insertions(+) diff --git a/docs/release-notes/release-notes-0.15.0.md b/docs/release-notes/release-notes-0.15.0.md index b118d975b..7d1136c99 100644 --- a/docs/release-notes/release-notes-0.15.0.md +++ b/docs/release-notes/release-notes-0.15.0.md @@ -175,6 +175,9 @@ then watch it on chain. Taproot script spends are also supported through the * [Announce the keysend feature bit in NodeAnnouncement if `--accept-keysend` is set](https://github.com/lightningnetwork/lnd/pull/6414) +* [Add a new method in `tlv` to encode an uint64/uint32 field using `BigSize` + format.](https://github.com/lightningnetwork/lnd/pull/6421) + ## RPC Server * [Add value to the field diff --git a/tlv/primitive.go b/tlv/primitive.go index 2e81b6765..fbf9a2ca0 100644 --- a/tlv/primitive.go +++ b/tlv/primitive.go @@ -307,3 +307,58 @@ func DVarBytes(r io.Reader, val interface{}, _ *[8]byte, l uint64) error { } return NewTypeForDecodingErr(val, "[]byte", l, l) } + +// EBigSize encodes an uint32 or an uint64 using BigSize format. An error is +// returned if val is not either *uint32 or *uint64. +func EBigSize(w io.Writer, val interface{}, buf *[8]byte) error { + if i, ok := val.(*uint32); ok { + return WriteVarInt(w, uint64(*i), buf) + } + + if i, ok := val.(*uint64); ok { + return WriteVarInt(w, uint64(*i), buf) + } + + return NewTypeForEncodingErr(val, "BigSize") +} + +// DBigSize decodes an uint32 or an uint64 using BigSize format. An error is +// returned if val is not either *uint32 or *uint64. +func DBigSize(r io.Reader, val interface{}, buf *[8]byte, l uint64) error { + if i, ok := val.(*uint32); ok { + v, err := ReadVarInt(r, buf) + if err != nil { + return err + } + *i = uint32(v) + return nil + } + + if i, ok := val.(*uint64); ok { + v, err := ReadVarInt(r, buf) + if err != nil { + return err + } + *i = v + return nil + } + + return NewTypeForDecodingErr(val, "BigSize", l, 8) +} + +// SizeBigSize returns a SizeFunc that can compute the length of BigSize. +func SizeBigSize(val interface{}) SizeFunc { + var size uint64 + + if i, ok := val.(*uint32); ok { + size = VarIntSize(uint64(*i)) + } + + if i, ok := val.(*uint64); ok { + size = VarIntSize(uint64(*i)) + } + + return func() uint64 { + return size + } +} diff --git a/tlv/record.go b/tlv/record.go index 975f68cdf..474908ebf 100644 --- a/tlv/record.go +++ b/tlv/record.go @@ -249,3 +249,40 @@ func SortRecords(records []Record) { return records[i].Type() < records[j].Type() }) } + +// MakeBigSizeRecord creates a tlv record using the BigSize format. The only +// allowed values are uint64 and uint32. +// +// NOTE: for uint32, we would only gain space reduction if the encoded value is +// no greater than 65535, which requires at most 3 bytes to encode. +func MakeBigSizeRecord(typ Type, val interface{}) Record { + var ( + staticSize uint64 + sizeFunc SizeFunc + encoder Encoder + decoder Decoder + ) + switch val.(type) { + case *uint32: + sizeFunc = SizeBigSize(val) + encoder = EBigSize + decoder = DBigSize + + case *uint64: + sizeFunc = SizeBigSize(val) + encoder = EBigSize + decoder = DBigSize + + default: + panic(fmt.Sprintf("unknown supported compact type: %T", val)) + } + + return Record{ + value: val, + typ: typ, + staticSize: staticSize, + sizeFunc: sizeFunc, + encoder: encoder, + decoder: decoder, + } +} diff --git a/tlv/stream_test.go b/tlv/stream_test.go index 8f67a316f..eb732d4db 100644 --- a/tlv/stream_test.go +++ b/tlv/stream_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/lightningnetwork/lnd/tlv" + "github.com/stretchr/testify/require" ) type parsedTypeTest struct { @@ -88,3 +89,164 @@ func testParsedTypes(t *testing.T, test parsedTypeTest) { t.Fatalf("error mismatch on parsed types") } } + +var ( + smallValue = 1 + smallValueBytes = []byte{ + // uint32 tlv, value uses 1 byte. + 0xa, 0x1, 0x1, + // uint64 tlv, value uses 1 byte. + 0xb, 0x1, 0x1, + } + + medianValue = 255 + medianValueBytes = []byte{ + // uint32 tlv, value uses 3 byte. + 0xa, 0x3, 0xfd, 0x0, 0xff, + // uint64 tlv, value uses 3 byte. + 0xb, 0x3, 0xfd, 0x0, 0xff, + } + + largeValue = 65536 + largeValueBytes = []byte{ + // uint32 tlv, value uses 5 byte. + 0xa, 0x5, 0xfe, 0x0, 0x1, 0x0, 0x0, + // uint64 tlv, value uses 5 byte. + 0xb, 0x5, 0xfe, 0x0, 0x1, 0x0, 0x0, + } +) + +// TestEncodeBigSizeFormatTlvStream tests that the bigsize encoder works as +// expected. +func TestEncodeBigSizeFormatTlvStream(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + value int + expectedBytes []byte + }{ + { + // Test encode 1, which saves us space. + name: "encode small value", + value: smallValue, + expectedBytes: smallValueBytes, + }, + { + // Test encode 255, which still saves us space. + name: "encode median value", + value: medianValue, + expectedBytes: medianValueBytes, + }, + { + // Test encode 65536, which takes more space to encode + // an uint32. + name: "encode large value", + value: largeValue, + expectedBytes: largeValueBytes, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + testUint32 := uint32(tc.value) + testUint64 := uint64(tc.value) + ts := makeBigSizeFormatTlvStream( + t, &testUint32, &testUint64, + ) + + // Encode the tlv stream. + buf := bytes.NewBuffer([]byte{}) + require.NoError(t, ts.Encode(buf)) + + // Check the bytes are written as expected. + require.Equal(t, tc.expectedBytes, buf.Bytes()) + }) + } +} + +// TestDecodeBigSizeFormatTlvStream tests that the bigsize decoder works as +// expected. +func TestDecodeBigSizeFormatTlvStream(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + bytes []byte + expectedValue int + }{ + { + // Test decode 1. + name: "decode small value", + bytes: []byte{ + // uint32 tlv, value uses 1 byte. + 0xa, 0x1, 0x1, + // uint64 tlv, value uses 1 byte. + 0xb, 0x1, 0x1, + }, + expectedValue: smallValue, + }, + { + // Test decode 255. + name: "decode median value", + bytes: []byte{ + // uint32 tlv, value uses 3 byte. + 0xa, 0x3, 0xfd, 0x0, 0xff, + // uint64 tlv, value uses 3 byte. + 0xb, 0x3, 0xfd, 0x0, 0xff, + }, + expectedValue: medianValue, + }, + { + // Test decode 65536. + name: "decode value 65536", + bytes: []byte{ + // uint32 tlv, value uses 5 byte. + 0xa, 0x5, 0xfe, 0x0, 0x1, 0x0, 0x0, + // uint64 tlv, value uses 5 byte. + 0xb, 0x5, 0xfe, 0x0, 0x1, 0x0, 0x0, + }, + expectedValue: largeValue, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + var ( + testUint32 uint32 + testUint64 uint64 + ) + ts := makeBigSizeFormatTlvStream( + t, &testUint32, &testUint64, + ) + + // Decode the tlv stream. + buf := bytes.NewBuffer(tc.bytes) + require.NoError(t, ts.Decode(buf)) + + // Check the values are written as expected. + require.EqualValues(t, tc.expectedValue, testUint32) + require.EqualValues(t, tc.expectedValue, testUint64) + }) + } +} + +func makeBigSizeFormatTlvStream(t *testing.T, vUint32 *uint32, + vUint64 *uint64) *tlv.Stream { + + const ( + typeUint32 tlv.Type = 10 + typeUint64 tlv.Type = 11 + ) + + // Create a dummy tlv stream for testing. + ts, err := tlv.NewStream( + tlv.MakeBigSizeRecord(typeUint32, vUint32), + tlv.MakeBigSizeRecord(typeUint64, vUint64), + ) + require.NoError(t, err) + + return ts +}