From 88bd0cb806faafacc6df4356cd40c916977e2151 Mon Sep 17 00:00:00 2001 From: yyforyongyu Date: Fri, 25 Oct 2024 03:33:37 +0800 Subject: [PATCH] itest: shuffle test cases to even out blocks mined in tranches This commit adds a new flag to shuffle all the test cases before running them so tests which require lots of blocks to be mined are less likely to be run in the same tranch. The other benefit is this approach provides a more efficient way to figure which tests are broken since all the differnet backends are running different tranches in their builds, we can identify more failed tests in one push. --- Makefile | 4 ++-- itest/list_on_test.go | 4 +++- itest/lnd_test.go | 37 +++++++++++++++++++++++++++++++++++++ make/testing_flags.mk | 6 ++++++ scripts/itest_parallel.sh | 7 ++++--- scripts/itest_part.sh | 10 +++++----- 6 files changed, 57 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 6b28bcc49..2f0c8c9ba 100644 --- a/Makefile +++ b/Makefile @@ -220,7 +220,7 @@ clean-itest-logs: itest-only: clean-itest-logs db-instance @$(call print, "Running integration tests with ${backend} backend.") date - EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_part.sh 0 1 $(TEST_FLAGS) $(ITEST_FLAGS) -test.v + EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_part.sh 0 1 $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS) -test.v $(COLLECT_ITEST_COVERAGE) #? itest: Build and run integration tests @@ -233,7 +233,7 @@ itest-race: build-itest-race itest-only itest-parallel: clean-itest-logs build-itest db-instance @$(call print, "Running tests") date - EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(TEST_FLAGS) $(ITEST_FLAGS) + EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS) $(COLLECT_ITEST_COVERAGE) #? itest-clean: Kill all running itest processes diff --git a/itest/list_on_test.go b/itest/list_on_test.go index a192a423f..67915dee8 100644 --- a/itest/list_on_test.go +++ b/itest/list_on_test.go @@ -2,7 +2,9 @@ package itest -import "github.com/lightningnetwork/lnd/lntest" +import ( + "github.com/lightningnetwork/lnd/lntest" +) var allTestCases = []*lntest.TestCase{ { diff --git a/itest/lnd_test.go b/itest/lnd_test.go index cb0871b96..f2a345fa1 100644 --- a/itest/lnd_test.go +++ b/itest/lnd_test.go @@ -20,6 +20,7 @@ import ( "github.com/lightningnetwork/lnd/lntest/port" "github.com/lightningnetwork/lnd/lntest/wait" "github.com/stretchr/testify/require" + "golang.org/x/exp/rand" "google.golang.org/grpc/grpclog" ) @@ -61,6 +62,13 @@ var ( "0-based index specified by the -runtranche flag", ) + // shuffleSeedFlag is the source of randomness used to shuffle the test + // cases. If not specified, the test cases won't be shuffled. + shuffleSeedFlag = flag.Uint64( + "shuffleseed", 0, "if set, shuffles the test cases using this "+ + "as the source of randomness", + ) + // testCasesRunTranche is the 0-based index of the split test cases // tranche to run in the current invocation. testCasesRunTranche = flag.Uint( @@ -160,6 +168,32 @@ func TestLightningNetworkDaemon(t *testing.T) { harnessTest.CurrentHeight()-height) } +// maybeShuffleTestCases shuffles the test cases if the flag `shuffleseed` is +// set and not 0. In parallel tests we want to shuffle the test cases so they +// are executed in a random order. This is done to even out the blocks mined in +// each test tranche so they can run faster. +// +// NOTE: Because the parallel tests are initialized with the same seed (job +// ID), they will always have the same order. +func maybeShuffleTestCases() { + // Exit if not set. + if shuffleSeedFlag == nil { + return + } + + // Exit if set to 0. + if *shuffleSeedFlag == 0 { + return + } + + // Init the seed and shuffle the test cases. + rand.Seed(*shuffleSeedFlag) + rand.Shuffle(len(allTestCases), func(i, j int) { + allTestCases[i], allTestCases[j] = + allTestCases[j], allTestCases[i] + }) +} + // getTestCaseSplitTranche returns the sub slice of the test cases that should // be run as the current split tranche as well as the index and slice offset of // the tranche. @@ -182,6 +216,9 @@ func getTestCaseSplitTranche() ([]*lntest.TestCase, uint, uint) { runTranche = 0 } + // Shuffle the test cases if the `shuffleseed` flag is set. + maybeShuffleTestCases() + numCases := uint(len(allTestCases)) testsPerTranche := numCases / numTranches trancheOffset := runTranche * testsPerTranche diff --git a/make/testing_flags.mk b/make/testing_flags.mk index b2db6861c..ba5b3fb37 100644 --- a/make/testing_flags.mk +++ b/make/testing_flags.mk @@ -10,6 +10,7 @@ COVER_PKG = $$(go list -deps -tags="$(DEV_TAGS)" ./... | grep '$(PKG)' | grep -v NUM_ITEST_TRANCHES = 4 ITEST_PARALLELISM = $(NUM_ITEST_TRANCHES) POSTGRES_START_DELAY = 5 +SHUFFLE_SEED = 0 # If rpc option is set also add all extra RPC tags to DEV_TAGS ifneq ($(with-rpc),) @@ -27,6 +28,11 @@ ifneq ($(parallel),) ITEST_PARALLELISM = $(parallel) endif +# Set the seed for shuffling the test cases. +ifneq ($(shuffleseed),) +SHUFFLE_SEED = $(shuffleseed) +endif + # Windows needs to append a .exe suffix to all executable files, otherwise it # won't run them. ifneq ($(windows),) diff --git a/scripts/itest_parallel.sh b/scripts/itest_parallel.sh index 2ae7f3f53..ab1b3efa2 100755 --- a/scripts/itest_parallel.sh +++ b/scripts/itest_parallel.sh @@ -3,9 +3,10 @@ # Get all the variables. PROCESSES=$1 TRANCHES=$2 +SHUFFLE_SEED=$3 -# Here we also shift 2 times and get the rest of our flags to pass on in $@. -shift 2 +# Here we also shift 3 times and get the rest of our flags to pass on in $@. +shift 3 # Create a variable to hold the final exit code. exit_code=0 @@ -13,7 +14,7 @@ exit_code=0 # Run commands using xargs in parallel and capture their PIDs pids=() for ((i=0; i