etcd: redesign commit queue to make it more robust and scalable

This commit builds on the ideas of @cfromknecht in lnd/5153. The
addition is that the design is now simpler and more robust by queueing
up everything, but allowing maximal parallelism where txns don't block.
Furthermore the commit makes CommitQueue.Done() private essentially
removing the need to understand the queue externally.

kvdb/etcd/commit_queue_test.go

@@ -17,7 +17,7 @@ import (
 func TestCommitQueue(t *testing.T) {
 	// The duration of each commit.
 	const commitDuration = time.Millisecond * 500
-	const numCommits = 4
+	const numCommits = 5
 
 	var wg sync.WaitGroup
 	commits := make([]string, numCommits)
@@ -30,25 +30,25 @@ func TestCommitQueue(t *testing.T) {
 			// Update our log of commit order. Avoid blocking
 			// by preallocating the commit log and increasing
 			// the log index atomically.
-			i := atomic.AddInt32(&idx, 1)
-			commits[i] = tag
-
 			if sleep {
 				time.Sleep(commitDuration)
 			}
+
+			i := atomic.AddInt32(&idx, 1)
+			commits[i] = tag
 		}
 	}
 
 	ctx := context.Background()
 	ctx, cancel := context.WithCancel(ctx)
 	q := NewCommitQueue(ctx)
-	defer q.Wait()
+	defer q.Stop()
 	defer cancel()
 
 	wg.Add(numCommits)
 	t1 := time.Now()
 
-	// Tx1: reads: key1, key2, writes: key3, conflict: none
+	// Tx1 (long): reads: key1, key2, writes: key3, conflict: none
 	q.Add(
 		commit("free", true),
 		[]string{"key1", "key2"},
@@ -60,12 +60,18 @@ func TestCommitQueue(t *testing.T) {
 		[]string{"key1", "key2"},
 		[]string{"key3"},
 	)
-	// Tx3: reads: key1, writes: key4, conflict: none
+	// Tx3 (long): reads: key1, writes: key4, conflict: none
 	q.Add(
 		commit("free", true),
 		[]string{"key1", "key2"},
 		[]string{"key4"},
 	)
+	// Tx4 (long): reads: key1, writes: none, conflict: none
+	q.Add(
+		commit("free", true),
+		[]string{"key1", "key2"},
+		[]string{},
+	)
 	// Tx4: reads: key2, writes: key4 conflict: Tx3
 	q.Add(
 		commit("blocked2", false),
@@ -87,7 +93,7 @@ func TestCommitQueue(t *testing.T) {
 	// before the blocking ones, and the blocking ones are executed in
 	// the order of addition.
 	require.Equal(t,
-		[]string{"free", "free", "blocked1", "blocked2"},
+		[]string{"free", "blocked1", "free", "free", "blocked2"},
 		commits,
 	)
 }