etcd: fix dereferncing issue in etcd.CommitQueue causing contention

This commit fixes an issue where subsequent transaction retries may have
changed the read/write sets inside the STM which in turn left junk
references to these keys in the CommitQueue. The left keys potentially
conflicted with subsequent transactions, queueing them up causing
througput degradation.

kvdb/etcd/stm.go

@@ -280,8 +280,30 @@ func runSTM(s *stm, apply func(STM) error) error {
 		return preApplyErr
 	}
 
+	// Make a copy of the read/write set keys here. The reason why we need
+	// to do this is because subsequent applies may change (shrink) these
+	// sets and so when we decrease reference counts in the commit queue in
+	// Done(...) we'd potentially miss removing references which would
+	// result in queueing up transactions and contending DB access.
+	// Copying these strings is cheap due to Go's immutable string which is
+	// always a reference.
+	rkeys := make([]string, len(s.rset))
+	wkeys := make([]string, len(s.wset))
+
+	i := 0
+	for key := range s.rset {
+		rkeys[i] = key
+		i++
+	}
+
+	i = 0
+	for key := range s.wset {
+		wkeys[i] = key
+		i++
+	}
+
 	// Queue up the transaction for execution.
-	s.txQueue.Add(execute, s.rset, s.wset)
+	s.txQueue.Add(execute, rkeys, wkeys)
 
 	// Wait for the transaction to execute, or break if aborted.
 	select {
@@ -289,7 +311,7 @@ func runSTM(s *stm, apply func(STM) error) error {
 	case <-s.options.ctx.Done():
 	}
 
-	s.txQueue.Done(s.rset, s.wset)
+	s.txQueue.Done(rkeys, wkeys)
 
 	if s.options.commitStatsCallback != nil {
 		stats.Retries = retries