Skip to content

Commit 91dff4c

Browse files
alecgrieseralecgrieser
committed
WIP: run tests in parallel
This begins the process of allowing for parallel test execution. A few issues were noticed that required some changes: * Some tests were taking up a bunch of time and benefitted from parallelized execution of tests within the fixture. Those have been updated to CONCURRENT execution mode * A known issue with JUnit (junit-team/junit5#3108) means that if one of the tests involves a future waiting, that can look to the `ForkJoinPool` like the thread is available for work stealing, so too many tests can end up being executed at once. A new test extension was added that has a semaphore, and that appears to be enough to stop extra tests from being executed * The server was running out of batch GRV transactions, which resulted in tests failing with "batch GRV transactions exhausted". This mainly affected indexing tests, and I was able to resolve this by upping the transaction priority, for better or worse There are still some issues: * A number of tests were hitting deadline exceeded exceptions. It looked like some kind of weird concurrency stuff may be going on, because there were things happening like key space path resolution while creating a record store would include stack traces from closing the test key space path manager, which seems like things were sharing objects that shouldn't have been. This affected both the `:fdb-record-layer-core:test` and `:fdb-lucene:test` tasks * The relational layer tests are not structured to allocate unique key spaces for each test, so they immediatetly hit concurrency problems when run in a parallelized manner
1 parent e9db5c0 commit 91dff4c

19 files changed

+208
-158
lines changed

‎fdb-extensions/src/test/java/com/apple/test/LimitConcurrencyExtension.java

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* LimitConcurrencyExtension.java
3+
*
4+
* This source file is part of the FoundationDB open source project
5+
*
6+
* Copyright 2015-2024 Apple Inc. and the FoundationDB project authors
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
21+
package com.apple.test;
22+
23+
import com.google.auto.service.AutoService;
24+
import org.junit.jupiter.api.extension.AfterEachCallback;
25+
import org.junit.jupiter.api.extension.BeforeEachCallback;
26+
import org.junit.jupiter.api.extension.Extension;
27+
import org.junit.jupiter.api.extension.ExtensionContext;
28+
29+
import java.util.concurrent.Semaphore;
30+
31+
@AutoService(Extension.class)
32+
public class LimitConcurrencyExtension implements BeforeEachCallback, AfterEachCallback {
33+
static Semaphore testConcurrency = new Semaphore(Integer.parseInt(System.getProperty("tests.concurrencyLimit", "10")));
34+
35+
@Override
36+
public void beforeEach(final ExtensionContext context) throws InterruptedException {
37+
testConcurrency.acquire();
38+
}
39+
40+
@Override
41+
public void afterEach(final ExtensionContext context) {
42+
testConcurrency.release();
43+
}
44+
}

‎fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreCountRecordsTest.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@
4545
import com.google.protobuf.Message;
4646
import org.junit.jupiter.api.Tag;
4747
import org.junit.jupiter.api.Test;
48+
import org.junit.jupiter.api.parallel.Execution;
49+
import org.junit.jupiter.api.parallel.ExecutionMode;
4850

4951
import javax.annotation.Nonnull;
5052
import java.util.HashMap;
@@ -69,6 +71,7 @@
6971
* Tests related to built in functionality for getting the count of records in a store.
7072
*/
7173
@Tag(Tags.RequiresFDB)
74+
@Execution(ExecutionMode.CONCURRENT)
7275
public class FDBRecordStoreCountRecordsTest extends FDBRecordStoreTestBase {
7376

7477
@Test
@@ -380,7 +383,7 @@ public void addCountIndex() throws Exception {
380383
}
381384

382385
// Build the index
383-
try (OnlineIndexer onlineIndexBuilder = OnlineIndexer.forRecordStoreAndIndex(recordStore, "record_count")) {
386+
try (OnlineIndexer onlineIndexBuilder = newIndexer("record_count")) {
384387
onlineIndexBuilder.buildIndex();
385388
}
386389
try (FDBRecordContext context = openContext()) {

‎fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreIndexTest.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -1239,8 +1239,7 @@ void markReadableTest() throws Exception {
12391239
openSimpleRecordStore(context);
12401240
Index index = recordStore.getRecordMetaData().getIndex(indexName);
12411241
assertThat(recordStore.isIndexReadable(index), is(false));
1242-
try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setRecordStore(recordStore).setIndex(index)
1243-
.build()) {
1242+
try (OnlineIndexer indexBuilder = newIndexer(indexName)) {
12441243
indexBuilder.buildIndex(false);
12451244
}
12461245
commit(context);

‎fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreTestBase.java

+14
Original file line numberDiff line numberDiff line change
@@ -305,4 +305,18 @@ protected FDBStoredRecord<Message> saveAndSplitSimpleRecord(long recno, String s
305305
md.removeIndex("MySimpleRecord$str_value_indexed");
306306
md.setStoreRecordVersions(false);
307307
};
308+
309+
@Nonnull
310+
protected OnlineIndexer.Builder newIndexerBuilder() {
311+
return OnlineIndexer.newBuilder()
312+
.setRecordStore(recordStore)
313+
.setPriority(FDBTransactionPriority.DEFAULT);
314+
}
315+
316+
@Nonnull
317+
protected OnlineIndexer newIndexer(@Nonnull String indexName) {
318+
return newIndexerBuilder()
319+
.setIndex(indexName)
320+
.build();
321+
}
308322
}

‎fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreUniqueIndexTest.java

+6-5
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@
4545
import com.apple.foundationdb.record.metadata.Key;
4646
import com.apple.foundationdb.record.provider.foundationdb.indexes.InvalidIndexEntry;
4747
import com.apple.foundationdb.record.provider.foundationdb.indexes.ValueIndexMaintainer;
48-
import com.apple.foundationdb.record.query.QueryToKeyMatcher;
4948
import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath;
49+
import com.apple.foundationdb.record.query.QueryToKeyMatcher;
5050
import com.apple.foundationdb.record.query.plan.QueryPlanner;
5151
import com.apple.foundationdb.record.test.TestKeySpace;
5252
import com.apple.foundationdb.record.util.pair.Pair;
@@ -59,6 +59,8 @@
5959
import org.hamcrest.Matchers;
6060
import org.junit.jupiter.api.Tag;
6161
import org.junit.jupiter.api.Test;
62+
import org.junit.jupiter.api.parallel.Execution;
63+
import org.junit.jupiter.api.parallel.ExecutionMode;
6264
import org.junit.jupiter.params.ParameterizedTest;
6365
import org.junit.jupiter.params.provider.Arguments;
6466
import org.junit.jupiter.params.provider.MethodSource;
@@ -98,6 +100,7 @@
98100
* Tests of uniqueness checks.
99101
*/
100102
@Tag(Tags.RequiresFDB)
103+
@Execution(ExecutionMode.CONCURRENT)
101104
public class FDBRecordStoreUniqueIndexTest extends FDBRecordStoreTestBase {
102105

103106
private static final String NO_UNIQUE_CLEAR_INDEX_TYPE = "no_unique_clear";
@@ -310,8 +313,7 @@ void multipleStores() throws Exception {
310313
recordStore = createOrOpenRecordStore(context, simpleMetaData(uniqueHook), path).getLeft();
311314
commit(context);
312315

313-
try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder()
314-
.setRecordStore(recordStore)
316+
try (OnlineIndexer indexBuilder = newIndexerBuilder()
315317
.setTargetIndexes(List.of(uniqueIndex))
316318
.setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder()
317319
.allowUniquePendingState(true))
@@ -630,8 +632,7 @@ public void addUniqueIndexViaBuild() {
630632
if (allowReadableUniquePending) {
631633
indexingPolicy.allowUniquePendingState();
632634
}
633-
try (OnlineIndexer indexer = OnlineIndexer.newBuilder()
634-
.setRecordStore(recordStore)
635+
try (OnlineIndexer indexer = newIndexerBuilder()
635636
.setTargetIndexesByName(List.of(uniqueIndex.getName()))
636637
.setIndexingPolicy(indexingPolicy
637638
.build()).build()) {

‎fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerBuildUnnestedIndexTest.java

+1
Original file line numberDiff line numberDiff line change
@@ -679,6 +679,7 @@ void doNotAllowBuildingIndexFromUnnestedIndex() {
679679
try (OnlineIndexer indexer = OnlineIndexer.newBuilder()
680680
.setRecordStoreBuilder(storeBuilder)
681681
.setIndex(targetIndex)
682+
.setPriority(FDBTransactionPriority.DEFAULT)
682683
.setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder()
683684
.setSourceIndex(sourceIndex.getName())
684685
.setIfDisabled(OnlineIndexer.IndexingPolicy.DesiredAction.REBUILD)

‎fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerMergeTest.java

+1
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,7 @@ void testRepartitionTimeout() {
145145
try (OnlineIndexer indexer = OnlineIndexer.newBuilder()
146146
.setRecordStoreBuilder(storeBuilder)
147147
.setTargetIndexesByName(List.of(INDEX_NAME))
148+
.setPriority(FDBTransactionPriority.DEFAULT)
148149
.setMaxAttempts(9)
149150
.build()) {
150151
Assertions.assertThrows(FDBExceptions.FDBStoreTransactionIsTooOldException.class, indexer::mergeIndex);

‎fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexerTest.java

+2
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,7 @@ OnlineIndexer.Builder newIndexerBuilder() {
156156
.setMetaData(metaData)
157157
.setSubspaceProvider(new SubspaceProviderByKeySpacePath(path))
158158
.setIndexMaintenanceFilter(getIndexMaintenanceFilter())
159+
.setPriority(FDBTransactionPriority.DEFAULT)
159160
.setFormatVersion(formatVersion);
160161
}
161162

@@ -181,6 +182,7 @@ OnlineIndexScrubber.Builder newScrubberBuilder() {
181182
.setMetaData(metaData)
182183
.setSubspaceProvider(new SubspaceProviderByKeySpacePath(path))
183184
.setIndexMaintenanceFilter(getIndexMaintenanceFilter())
185+
.setPriority(FDBTransactionPriority.DEFAULT)
184186
.setFormatVersion(formatVersion);
185187
}
186188

‎fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RecordTypeKeyTest.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -602,7 +602,7 @@ public void testOnlineIndexBuilder() throws Exception {
602602
timer.reset();
603603

604604
// Build in this transaction.
605-
try (OnlineIndexer indexBuilder = OnlineIndexer.forRecordStoreAndIndex(recordStore, "newIndex")) {
605+
try (OnlineIndexer indexBuilder = newIndexer("newIndex")) {
606606
indexBuilder.rebuildIndex(recordStore);
607607
}
608608
recordStore.markIndexReadable("newIndex").join();
@@ -629,7 +629,7 @@ public void testOnlineIndexBuilder() throws Exception {
629629
timer.reset();
630630

631631
// Build in multiple transactions.
632-
try (OnlineIndexer indexBuilder = OnlineIndexer.forRecordStoreAndIndex(recordStore, "newIndex")) {
632+
try (OnlineIndexer indexBuilder = newIndexer("newIndex")) {
633633
indexBuilder.buildIndex();
634634
}
635635

@@ -682,7 +682,7 @@ public void testOnlineIndexMultiTargetBuilder() throws Exception {
682682
}
683683

684684
// Build multiple indexes of typed records
685-
try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setRecordStore(recordStore)
685+
try (OnlineIndexer indexBuilder = newIndexerBuilder()
686686
.addTargetIndex("newIndex")
687687
.addTargetIndex("newSumIndex")
688688
.addTargetIndex("newMaxIndex")

‎fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SimpleMultidimensionalIndexTest.java

+54-59
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
package com.apple.foundationdb.record.provider.foundationdb.indexes;
2222

2323
import org.junit.jupiter.api.Test;
24+
import org.junit.jupiter.api.parallel.Execution;
25+
import org.junit.jupiter.api.parallel.ExecutionMode;
2426
import org.junit.jupiter.params.ParameterizedTest;
2527
import org.junit.jupiter.params.provider.Arguments;
2628
import org.junit.jupiter.params.provider.MethodSource;
@@ -35,18 +37,22 @@
3537
/**
3638
* Simple tests for Multidimensional Index.
3739
*/
40+
@Execution(ExecutionMode.CONCURRENT)
3841
class SimpleMultidimensionalIndexTest extends MultidimensionalIndexTestBase {
3942

43+
static Stream<String> storageAdapterArgs() {
44+
return Stream.of(BY_NODE.toString(), BY_SLOT.toString());
45+
}
46+
47+
static Stream<Boolean> booleanArgs() {
48+
return Stream.of(false, true);
49+
}
50+
4051
static Stream<Arguments> argumentsForBasicReads() {
41-
return Stream.of(
42-
Arguments.of(BY_NODE.toString(), false, false),
43-
Arguments.of(BY_NODE.toString(), false, true),
44-
Arguments.of(BY_NODE.toString(), true, false),
45-
Arguments.of(BY_NODE.toString(), true, true),
46-
Arguments.of(BY_SLOT.toString(), false, false),
47-
Arguments.of(BY_SLOT.toString(), false, true),
48-
Arguments.of(BY_SLOT.toString(), true, false),
49-
Arguments.of(BY_SLOT.toString(), true, true));
52+
return storageAdapterArgs().flatMap(storageAdapter ->
53+
booleanArgs().flatMap(storeHilbertValues ->
54+
booleanArgs().map(useNodeSlotIndex ->
55+
Arguments.of(storageAdapter, storeHilbertValues, useNodeSlotIndex))));
5056
}
5157

5258
/**
@@ -57,62 +63,51 @@ static Stream<Arguments> argumentsForBasicReads() {
5763
*
5864
* @return a stream of arguments
5965
*/
66+
@Nonnull
6067
static Stream<Arguments> argumentsForIndexReads() {
6168
final Random random = new Random(System.currentTimeMillis());
62-
return Stream.of(
63-
Arguments.of(random.nextLong(), 100, BY_SLOT.toString(), false, false),
64-
Arguments.of(random.nextLong(), 100, BY_SLOT.toString(), false, true),
65-
Arguments.of(random.nextLong(), 100, BY_SLOT.toString(), true, false),
66-
Arguments.of(random.nextLong(), 100, BY_SLOT.toString(), true, true),
67-
Arguments.of(random.nextLong(), 100, BY_NODE.toString(), false, false),
68-
Arguments.of(random.nextLong(), 100, BY_NODE.toString(), false, true),
69-
Arguments.of(random.nextLong(), 100, BY_NODE.toString(), true, false),
70-
Arguments.of(random.nextLong(), 100, BY_NODE.toString(), true, true),
71-
Arguments.of(random.nextLong(), 500, BY_SLOT.toString(), false, false),
72-
Arguments.of(random.nextLong(), 500, BY_SLOT.toString(), false, true),
73-
Arguments.of(random.nextLong(), 500, BY_SLOT.toString(), true, false),
74-
Arguments.of(random.nextLong(), 500, BY_SLOT.toString(), true, true),
75-
Arguments.of(random.nextLong(), 500, BY_NODE.toString(), false, false),
76-
Arguments.of(random.nextLong(), 500, BY_NODE.toString(), false, true),
77-
Arguments.of(random.nextLong(), 500, BY_NODE.toString(), true, false),
78-
Arguments.of(random.nextLong(), 500, BY_NODE.toString(), true, true),
79-
// large values only for default config
80-
Arguments.of(random.nextLong(), 1000, BY_NODE.toString(), true, false),
81-
Arguments.of(random.nextLong(), 5000, BY_NODE.toString(), true, false)
82-
);
69+
return storageAdapterArgs().flatMap(storageAdapter ->
70+
booleanArgs().flatMap(storeHilbertValue ->
71+
booleanArgs().flatMap(useNodeSlotIndex ->
72+
argumentsForIndexReads(random, storageAdapter, storeHilbertValue, useNodeSlotIndex))));
8373
}
8474

75+
@Nonnull
76+
static Stream<Arguments> argumentsForIndexReads(@Nonnull Random random, @Nonnull String storageAdapter, boolean storeHilbertValue, boolean useNodeSlotIndex) {
77+
Arguments small = Arguments.of(random.nextLong(), 100, storageAdapter, storeHilbertValue, useNodeSlotIndex);
78+
Arguments medium = Arguments.of(random.nextLong(), 500, storageAdapter, storeHilbertValue, useNodeSlotIndex);
79+
// large values only for default config
80+
if (storeHilbertValue && !useNodeSlotIndex && storageAdapter.equals(BY_NODE.toString())) {
81+
Arguments large = Arguments.of(random.nextLong(), 1000, storageAdapter, storeHilbertValue, useNodeSlotIndex);
82+
Arguments extraLarge = Arguments.of(random.nextLong(), 5000, storageAdapter, storeHilbertValue, useNodeSlotIndex);
83+
return Stream.of(small, medium, large, extraLarge);
84+
} else {
85+
return Stream.of(small, medium);
86+
}
87+
}
88+
89+
@Nonnull
8590
static Stream<Arguments> argumentsForIndexReadsAfterDeletes() {
8691
final Random random = new Random(System.currentTimeMillis());
87-
return Stream.of(
88-
Arguments.of(random.nextLong(), 10, random.nextInt(10) + 1, BY_SLOT.toString(), false, false),
89-
Arguments.of(random.nextLong(), 10, random.nextInt(10) + 1, BY_SLOT.toString(), false, true),
90-
Arguments.of(random.nextLong(), 10, random.nextInt(10) + 1, BY_SLOT.toString(), true, false),
91-
Arguments.of(random.nextLong(), 10, random.nextInt(10) + 1, BY_SLOT.toString(), true, true),
92-
Arguments.of(random.nextLong(), 100, random.nextInt(100) + 1, BY_SLOT.toString(), false, false),
93-
Arguments.of(random.nextLong(), 100, random.nextInt(100) + 1, BY_SLOT.toString(), false, true),
94-
Arguments.of(random.nextLong(), 100, random.nextInt(100) + 1, BY_SLOT.toString(), true, false),
95-
Arguments.of(random.nextLong(), 100, random.nextInt(100) + 1, BY_SLOT.toString(), true, true),
96-
Arguments.of(random.nextLong(), 300, random.nextInt(300) + 1, BY_SLOT.toString(), false, false),
97-
Arguments.of(random.nextLong(), 300, random.nextInt(300) + 1, BY_SLOT.toString(), false, true),
98-
Arguments.of(random.nextLong(), 300, random.nextInt(300) + 1, BY_SLOT.toString(), true, false),
99-
Arguments.of(random.nextLong(), 300, random.nextInt(300) + 1, BY_SLOT.toString(), true, true),
100-
Arguments.of(random.nextLong(), 10, random.nextInt(10) + 1, BY_NODE.toString(), false, false),
101-
Arguments.of(random.nextLong(), 10, random.nextInt(10) + 1, BY_NODE.toString(), false, true),
102-
Arguments.of(random.nextLong(), 10, random.nextInt(10) + 1, BY_NODE.toString(), true, false),
103-
Arguments.of(random.nextLong(), 10, random.nextInt(10) + 1, BY_NODE.toString(), true, true),
104-
Arguments.of(random.nextLong(), 100, random.nextInt(100) + 1, BY_NODE.toString(), false, false),
105-
Arguments.of(random.nextLong(), 100, random.nextInt(100) + 1, BY_NODE.toString(), false, true),
106-
Arguments.of(random.nextLong(), 100, random.nextInt(100) + 1, BY_NODE.toString(), true, false),
107-
Arguments.of(random.nextLong(), 100, random.nextInt(100) + 1, BY_NODE.toString(), true, true),
108-
Arguments.of(random.nextLong(), 300, random.nextInt(300) + 1, BY_NODE.toString(), false, false),
109-
Arguments.of(random.nextLong(), 300, random.nextInt(300) + 1, BY_NODE.toString(), false, true),
110-
Arguments.of(random.nextLong(), 300, random.nextInt(300) + 1, BY_NODE.toString(), true, false),
111-
Arguments.of(random.nextLong(), 300, random.nextInt(300) + 1, BY_NODE.toString(), true, true),
112-
// large values only for default config
113-
Arguments.of(random.nextLong(), 1000, random.nextInt(1000) + 1, BY_NODE.toString(), true, false),
114-
Arguments.of(random.nextLong(), 5000, random.nextInt(1000) + 1, BY_NODE.toString(), true, false)
115-
);
92+
return storageAdapterArgs().flatMap(storageAdapter ->
93+
booleanArgs().flatMap(storeHilbertValue ->
94+
booleanArgs().flatMap(useNodeSlotIndex ->
95+
argumentsForIndexReadsAfterDeletes(random, storageAdapter, storeHilbertValue, useNodeSlotIndex))));
96+
}
97+
98+
@Nonnull
99+
static Stream<Arguments> argumentsForIndexReadsAfterDeletes(@Nonnull Random random, @Nonnull String storageAdapter, boolean storeHilbertValue, boolean useNodeSlotIndex) {
100+
Arguments extraSmall = Arguments.of(random.nextLong(), 10, random.nextInt(10) + 1, storageAdapter, storeHilbertValue, useNodeSlotIndex);
101+
Arguments small = Arguments.of(random.nextLong(), 100, random.nextInt(100) + 1, storageAdapter, storeHilbertValue, useNodeSlotIndex);
102+
Arguments medium = Arguments.of(random.nextLong(), 300, random.nextInt(300) + 1, storageAdapter, storeHilbertValue, useNodeSlotIndex);
103+
// large values only for default config
104+
if (storeHilbertValue && !useNodeSlotIndex && storageAdapter.equals(BY_NODE.toString())) {
105+
Arguments large = Arguments.of(random.nextLong(), 1000, random.nextInt(1000) + 1, storageAdapter, storeHilbertValue, useNodeSlotIndex);
106+
Arguments extraLarge = Arguments.of(random.nextLong(), 5000, random.nextInt(5000) + 1, storageAdapter, storeHilbertValue, useNodeSlotIndex);
107+
return Stream.of(extraSmall, small, medium, large, extraLarge);
108+
} else {
109+
return Stream.of(extraSmall, small, medium);
110+
}
116111
}
117112

118113
/**

0 commit comments

Comments
 (0)