Skip to content

Commit a71e410

Browse files
nbbeekennbbeeken
committed
chore(NODE-6705): migrate parallel benchmarks
1 parent 9fc6aca commit a71e410

12 files changed

+247
-15
lines changed

‎test/benchmarks/driver_bench/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"version": "0.0.0",
44
"private": true,
55
"scripts": {
6-
"prestart": "tsc",
6+
"prestart": "rm -rf lib && tsc",
77
"start": "node lib/main.mjs"
88
}
99
}

‎test/benchmarks/driver_bench/src/driver.mts

+5-2
Original file line numberDiff line numberDiff line change
@@ -105,12 +105,15 @@ export function snakeToCamel(name: string) {
105105
import type mongodb from '../../../../mongodb.js';
106106
export type { mongodb };
107107

108-
const { MongoClient, GridFSBucket } = require(path.join(MONGODB_DRIVER_PATH));
108+
const { MongoClient, GridFSBucket, BSON, EJSON } = require(path.join(MONGODB_DRIVER_PATH));
109+
110+
export { BSON, EJSON };
109111

110112
const DB_NAME = 'perftest';
111113
const COLLECTION_NAME = 'corpus';
112114

113-
const SPEC_DIRECTORY = path.resolve(__dirname, '..', '..', 'driverBench', 'spec');
115+
export const SPEC_DIRECTORY = path.resolve(__dirname, '..', '..', 'driverBench', 'spec');
116+
export const PARALLEL_DIRECTORY = path.resolve(SPEC_DIRECTORY, 'parallel');
114117

115118
export type Metric = {
116119
name: 'megabytes_per_second';

‎test/benchmarks/driver_bench/src/main.mts

+12-12
Original file line numberDiff line numberDiff line change
@@ -120,27 +120,27 @@ function calculateCompositeBenchmarks(results: MetricInfo[]) {
120120
'largeDocBulkInsert',
121121
'smallDocBulkInsert'
122122
],
123-
// parallelBench: [
124-
// 'ldjsonMultiFileUpload',
125-
// 'ldjsonMultiFileExport',
126-
// 'gridfsMultiFileUpload',
127-
// 'gridfsMultiFileDownload'
128-
// ],
123+
parallelBench: [
124+
'ldjsonMultiFileUpload',
125+
'ldjsonMultiFileExport',
126+
'gridfsMultiFileUpload',
127+
'gridfsMultiFileDownload'
128+
],
129129
readBench: [
130130
'findOne',
131131
'findManyAndEmptyCursor',
132-
'gridFsDownload'
133-
// 'gridfsMultiFileDownload',
134-
// 'ldjsonMultiFileExport'
132+
'gridFsDownload',
133+
'gridfsMultiFileDownload',
134+
'ldjsonMultiFileExport'
135135
],
136136
writeBench: [
137137
'smallDocInsertOne',
138138
'largeDocInsertOne',
139139
'smallDocBulkInsert',
140140
'largeDocBulkInsert',
141-
'gridFsUpload'
142-
// 'ldjsonMultiFileUpload',
143-
// 'gridfsMultiFileUpload'
141+
'gridFsUpload',
142+
'ldjsonMultiFileUpload',
143+
'gridfsMultiFileUpload'
144144
]
145145
};
146146

‎test/benchmarks/driver_bench/src/suites/multi_bench/aggregate_a_million_documents_and_to_array.mts ‎test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts

File renamed without changes.

‎test/benchmarks/driver_bench/src/suites/multi_bench/aggregate_a_million_tweets_and_to_array.mts ‎test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts

File renamed without changes.

‎test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_to_array.mts ‎test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts

File renamed without changes.

‎test/benchmarks/driver_bench/src/suites/single_bench/ping.mts ‎test/benchmarks/driver_bench/src/suites/node_specific/ping.mts

File renamed without changes.

‎test/benchmarks/driver_bench/src/suites/single_bench/primes.mts ‎test/benchmarks/driver_bench/src/suites/node_specific/primes.mts

File renamed without changes.

‎test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_download.mts

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import { createReadStream, createWriteStream, promises as fs } from 'node:fs';
2+
import path from 'node:path';
3+
import { pipeline } from 'node:stream/promises';
4+
5+
import { driver, type mongodb, PARALLEL_DIRECTORY } from '../../driver.mjs';
6+
7+
export const taskSize = 262.144;
8+
9+
let bucket: mongodb.GridFSBucket;
10+
let temporaryDirectory: string;
11+
12+
export async function before() {
13+
await driver.drop();
14+
await driver.create();
15+
16+
temporaryDirectory = path.resolve(PARALLEL_DIRECTORY, 'downloads');
17+
await fs.rm(temporaryDirectory, { recursive: true, force: true });
18+
await fs.mkdir(temporaryDirectory);
19+
20+
bucket = driver.bucket(driver.client.db(driver.DB_NAME));
21+
22+
await bucket.drop().catch(() => null);
23+
24+
const directory = path.resolve(PARALLEL_DIRECTORY, 'gridfs_multi');
25+
26+
const files = await fs.readdir(directory);
27+
28+
const uploadPromises = files.map(async filename => {
29+
const file = path.resolve(directory, filename);
30+
const fileStream = createReadStream(file);
31+
const uploadStream = bucket.openUploadStream(file);
32+
return await pipeline(fileStream, uploadStream);
33+
});
34+
35+
await Promise.all(uploadPromises);
36+
}
37+
38+
export async function run() {
39+
const files = await bucket
40+
.find()
41+
.map(({ _id }) => ({
42+
path: path.resolve(temporaryDirectory, `${_id}.txt`),
43+
_id
44+
}))
45+
.toArray();
46+
47+
const downloads = files.map(async ({ _id, path }) => {
48+
const fileStream = createWriteStream(path);
49+
const downloadStream = bucket.openDownloadStream(_id);
50+
return pipeline(downloadStream, fileStream);
51+
});
52+
53+
await Promise.all(downloads);
54+
}
55+
56+
export async function afterEach() {
57+
await fs.rm(temporaryDirectory, { recursive: true, force: true });
58+
}
59+
60+
export async function after() {
61+
await driver.drop();
62+
await driver.close();
63+
}

‎test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_upload.mts

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
import { createReadStream, promises as fs } from 'node:fs';
2+
import path from 'node:path';
3+
import { Readable } from 'node:stream';
4+
import { pipeline } from 'node:stream/promises';
5+
6+
import { driver, type mongodb, PARALLEL_DIRECTORY } from '../../driver.mjs';
7+
8+
export const taskSize = 262.144;
9+
10+
let bucket: mongodb.GridFSBucket;
11+
12+
const directory = path.resolve(PARALLEL_DIRECTORY, 'gridfs_multi');
13+
14+
export async function before() {
15+
await driver.drop();
16+
await driver.create();
17+
18+
bucket = driver.bucket(driver.client.db(driver.DB_NAME));
19+
20+
await bucket.drop().catch(() => null);
21+
}
22+
23+
export async function beforeEach() {
24+
// Create the bucket.
25+
const stream = bucket.openUploadStream('setup-file.txt');
26+
const oneByteFile = Readable.from('a');
27+
await pipeline(oneByteFile, stream);
28+
}
29+
30+
export async function run() {
31+
const files = await fs.readdir(directory);
32+
33+
const uploadPromises = files.map(async filename => {
34+
const file = path.resolve(directory, filename);
35+
const fileStream = createReadStream(file);
36+
const uploadStream = bucket.openUploadStream(file);
37+
return await pipeline(fileStream, uploadStream);
38+
});
39+
40+
await Promise.all(uploadPromises);
41+
}
42+
43+
export async function after() {
44+
await driver.drop();
45+
await driver.close();
46+
}

‎test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
import { createReadStream, createWriteStream, promises as fs } from 'node:fs';
2+
import path from 'node:path';
3+
import readline from 'node:readline/promises';
4+
import stream from 'node:stream/promises';
5+
6+
import { driver, EJSON, type mongodb, PARALLEL_DIRECTORY } from '../../driver.mjs';
7+
8+
export const taskSize = 565;
9+
10+
let temporaryDirectory: string;
11+
let collection: mongodb.Collection;
12+
13+
export async function beforeEach() {
14+
await driver.drop();
15+
await driver.create();
16+
17+
temporaryDirectory = path.resolve(PARALLEL_DIRECTORY, 'downloads');
18+
await fs.rm(temporaryDirectory, { recursive: true, force: true });
19+
await fs.mkdir(temporaryDirectory);
20+
21+
collection = driver.client.db(driver.DB_NAME).collection(driver.COLLECTION_NAME);
22+
23+
const directory = path.resolve(PARALLEL_DIRECTORY, 'ldjson_multi');
24+
25+
const files = await fs.readdir(directory);
26+
const uploads = files.map(async file => {
27+
const fileStream = createReadStream(path.resolve(directory, file));
28+
const lineReader = readline.createInterface({
29+
input: fileStream
30+
});
31+
32+
const operations = [];
33+
34+
for await (const line of lineReader) {
35+
operations.push({
36+
insertOne: {
37+
document: JSON.parse(line)
38+
}
39+
});
40+
}
41+
42+
fileStream.close();
43+
lineReader.close();
44+
45+
return await collection.bulkWrite(operations);
46+
});
47+
48+
await Promise.all(uploads);
49+
}
50+
51+
export async function run() {
52+
const skips = Array.from({ length: 100 }, (_, index) => index * 5000);
53+
54+
const promises = skips.map(async skip => {
55+
const documentCursor = collection.find({}, { skip, limit: 5000 });
56+
documentCursor.map(doc => EJSON.stringify(doc));
57+
const outputStream = createWriteStream(path.resolve(temporaryDirectory, `tmp-${skip}.txt`));
58+
return stream.pipeline(documentCursor.stream(), outputStream);
59+
});
60+
61+
await Promise.all(promises);
62+
}
63+
64+
export async function afterEach() {
65+
await fs.rm(temporaryDirectory, { recursive: true, force: true });
66+
}
67+
68+
export async function after() {
69+
await driver.drop();
70+
await driver.close();
71+
}

‎test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_upload.mts

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
import { createReadStream, promises as fs } from 'node:fs';
2+
import path from 'node:path';
3+
import readline from 'node:readline/promises';
4+
5+
import { driver, type mongodb, PARALLEL_DIRECTORY } from '../../driver.mjs';
6+
7+
export const taskSize = 565;
8+
9+
const directory = path.resolve(PARALLEL_DIRECTORY, 'ldjson_multi');
10+
let collection: mongodb.Collection;
11+
12+
export async function beforeEach() {
13+
await driver.drop();
14+
await driver.create();
15+
16+
collection = driver.client.db(driver.DB_NAME).collection(driver.COLLECTION_NAME);
17+
}
18+
19+
export async function run() {
20+
const files = await fs.readdir(directory);
21+
const uploads = files.map(async file => {
22+
const fileStream = createReadStream(path.resolve(directory, file));
23+
const lineReader = readline.createInterface({
24+
input: fileStream
25+
});
26+
27+
const operations = [];
28+
29+
for await (const line of lineReader) {
30+
operations.push({
31+
insertOne: {
32+
document: JSON.parse(line)
33+
}
34+
});
35+
}
36+
37+
fileStream.close();
38+
lineReader.close();
39+
40+
return await collection.bulkWrite(operations);
41+
});
42+
43+
await Promise.all(uploads);
44+
}
45+
46+
export async function after() {
47+
await driver.drop();
48+
await driver.close();
49+
}

0 commit comments

Comments
 (0)