diff --git a/benchmarks/MongoDB.Driver.Benchmarks/BenchmarkResult.cs b/benchmarks/MongoDB.Driver.Benchmarks/BenchmarkResult.cs index 2e6a002acb8..1eacc5f79d0 100644 --- a/benchmarks/MongoDB.Driver.Benchmarks/BenchmarkResult.cs +++ b/benchmarks/MongoDB.Driver.Benchmarks/BenchmarkResult.cs @@ -27,8 +27,10 @@ public sealed class BenchmarkResult public BenchmarkResult(BenchmarkReport benchmarkReport) { + Categories = new HashSet(benchmarkReport.BenchmarkCase.Descriptor.Categories); + int dataSetSize; - if (benchmarkReport.BenchmarkCase.Descriptor.HasCategory(DriverBenchmarkCategory.BsonBench)) + if (Categories.Contains(DriverBenchmarkCategory.BsonBench)) { var bsonBenchmarkData = (BsonBenchmarkData)benchmarkReport.BenchmarkCase.Parameters["BenchmarkData"]; Name = bsonBenchmarkData.DataSetName + benchmarkReport.BenchmarkCase.Descriptor.Type.Name; @@ -36,12 +38,13 @@ public BenchmarkResult(BenchmarkReport benchmarkReport) } else { - Name = benchmarkReport.BenchmarkCase.Descriptor.Type.Name; + Name = Categories.Contains(DriverBenchmarkCategory.BulkWriteBench) + ? benchmarkReport.BenchmarkCase.Descriptor.WorkloadMethod.Name + : benchmarkReport.BenchmarkCase.Descriptor.Type.Name; + dataSetSize = (int)benchmarkReport.BenchmarkCase.Parameters["BenchmarkDataSetSize"]; } - Categories = new HashSet(benchmarkReport.BenchmarkCase.Descriptor.Categories); - // change the median from nanoseconds to seconds for calculating the score. // since dataSetSize is in bytes, divide the score to convert to MB/s Score = (dataSetSize / (benchmarkReport.ResultStatistics.Median / 1_000_000_000D)) / 1_000_000D; diff --git a/benchmarks/MongoDB.Driver.Benchmarks/DriverBenchmarkCategory.cs b/benchmarks/MongoDB.Driver.Benchmarks/DriverBenchmarkCategory.cs index de98524bd73..3bb57045ccc 100644 --- a/benchmarks/MongoDB.Driver.Benchmarks/DriverBenchmarkCategory.cs +++ b/benchmarks/MongoDB.Driver.Benchmarks/DriverBenchmarkCategory.cs @@ -26,6 +26,9 @@ public static class DriverBenchmarkCategory public const string ReadBench = "ReadBench"; public const string SingleBench = "SingleBench"; public const string WriteBench = "WriteBench"; + + // not included in AllCategories as it's not part of the benchmarking spec + public const string BulkWriteBench = "BulkWriteBench"; public static readonly IEnumerable AllCategories = new[] {BsonBench, ReadBench, WriteBench, MultiBench, SingleBench, ParallelBench, DriverBench}; } diff --git a/benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/BulkWriteMixedOpsBenchmark.cs b/benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/BulkWriteMixedOpsBenchmark.cs new file mode 100644 index 00000000000..bdbf157d64b --- /dev/null +++ b/benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/BulkWriteMixedOpsBenchmark.cs @@ -0,0 +1,94 @@ +/* Copyright 2010-present MongoDB Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Attributes; +using MongoDB.Bson; +using MongoDB.Driver; +using static MongoDB.Benchmarks.BenchmarkHelper; + +namespace MongoDB.Benchmarks.MultiDoc +{ + [IterationCount(15)] + [BenchmarkCategory(DriverBenchmarkCategory.BulkWriteBench, DriverBenchmarkCategory.MultiBench, DriverBenchmarkCategory.WriteBench, DriverBenchmarkCategory.DriverBench)] + public class BulkWriteMixedOpsBenchmark + { + private IMongoClient _client; + private IMongoCollection _collection; + private IMongoDatabase _database; + private readonly List _clientBulkWriteMixedOpsModels = []; + private readonly List> _collectionBulkWriteMixedOpsModels = []; + + private static readonly string[] __collectionNamespaces = Enumerable.Range(0, 10) + .Select(i => $"{MongoConfiguration.PerfTestDatabaseName}.{MongoConfiguration.PerfTestCollectionName}_{i}") + .ToArray(); + + [Params(5_500_000)] + public int BenchmarkDataSetSize { get; set; } // used in BenchmarkResult.cs + + [GlobalSetup] + public void Setup() + { + _client = MongoConfiguration.CreateClient(); + + var smallDocument = ReadExtendedJson("single_and_multi_document/small_doc.json"); + for (var i = 0; i < 10000; i++) + { + var collectionName = __collectionNamespaces[i % __collectionNamespaces.Length]; + + _clientBulkWriteMixedOpsModels.Add(new BulkWriteInsertOneModel(collectionName, smallDocument.DeepClone().AsBsonDocument)); + _clientBulkWriteMixedOpsModels.Add(new BulkWriteReplaceOneModel(collectionName, FilterDefinition.Empty, smallDocument.DeepClone().AsBsonDocument)); + _clientBulkWriteMixedOpsModels.Add(new BulkWriteDeleteOneModel(collectionName, FilterDefinition.Empty)); + + _collectionBulkWriteMixedOpsModels.Add(new InsertOneModel(smallDocument.DeepClone().AsBsonDocument)); + _collectionBulkWriteMixedOpsModels.Add(new ReplaceOneModel(FilterDefinition.Empty, smallDocument.DeepClone().AsBsonDocument)); + _collectionBulkWriteMixedOpsModels.Add(new DeleteOneModel(FilterDefinition.Empty)); + } + } + + [IterationSetup] + public void BeforeTask() + { + _client.DropDatabase(MongoConfiguration.PerfTestDatabaseName); + + _database = _client.GetDatabase(MongoConfiguration.PerfTestDatabaseName); + foreach (var collectionName in __collectionNamespaces) + { + _database.CreateCollection(collectionName.Split('.')[1]); + } + + _collection = _database.GetCollection(MongoConfiguration.PerfTestCollectionName); + } + + [Benchmark] + public void SmallDocCollectionBulkWriteMixedOpsBenchmark() + { + _collection.BulkWrite(_collectionBulkWriteMixedOpsModels, new()); + } + + [Benchmark] + public void SmallDocClientBulkWriteMixedOpsBenchmark() + { + _client.BulkWrite(_clientBulkWriteMixedOpsModels, new()); + } + + [GlobalCleanup] + public void Teardown() + { + _client.Dispose(); + } + } +} diff --git a/benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/InsertManyLargeBenchmark.cs b/benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/LargeDocBulkInsertBenchmark.cs similarity index 55% rename from benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/InsertManyLargeBenchmark.cs rename to benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/LargeDocBulkInsertBenchmark.cs index 45c17ce8fcf..4ed26e65226 100644 --- a/benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/InsertManyLargeBenchmark.cs +++ b/benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/LargeDocBulkInsertBenchmark.cs @@ -18,19 +18,23 @@ using BenchmarkDotNet.Attributes; using MongoDB.Bson; using MongoDB.Driver; -using MongoDB.Driver.TestHelpers; using static MongoDB.Benchmarks.BenchmarkHelper; namespace MongoDB.Benchmarks.MultiDoc { [IterationCount(100)] - [BenchmarkCategory(DriverBenchmarkCategory.MultiBench, DriverBenchmarkCategory.WriteBench, DriverBenchmarkCategory.DriverBench)] - public class InsertManyLargeBenchmark + [BenchmarkCategory(DriverBenchmarkCategory.BulkWriteBench, DriverBenchmarkCategory.MultiBench, DriverBenchmarkCategory.WriteBench, DriverBenchmarkCategory.DriverBench)] + public class LargeDocBulkInsertBenchmark { private IMongoClient _client; private IMongoCollection _collection; private IMongoDatabase _database; - private IEnumerable _largeDocuments; + private BsonDocument[] _largeDocuments; + private InsertOneModel[] _collectionBulkWriteInsertModels; + private BulkWriteInsertOneModel[] _clientBulkWriteInsertModels; + + private static readonly CollectionNamespace __collectionNamespace = + CollectionNamespace.FromFullName($"{MongoConfiguration.PerfTestDatabaseName}.{MongoConfiguration.PerfTestCollectionName}"); [Params(27_310_890)] public int BenchmarkDataSetSize { get; set; } // used in BenchmarkResult.cs @@ -42,7 +46,9 @@ public void Setup() _database = _client.GetDatabase(MongoConfiguration.PerfTestDatabaseName); var largeDocument = ReadExtendedJson("single_and_multi_document/large_doc.json"); - _largeDocuments = Enumerable.Range(0, 10).Select(_ => largeDocument.DeepClone().AsBsonDocument); + _largeDocuments = Enumerable.Range(0, 10).Select(_ => largeDocument.DeepClone().AsBsonDocument).ToArray(); + _collectionBulkWriteInsertModels = _largeDocuments.Select(x => new InsertOneModel(x.DeepClone().AsBsonDocument)).ToArray(); + _clientBulkWriteInsertModels = _largeDocuments.Select(x => new BulkWriteInsertOneModel(__collectionNamespace, x.DeepClone().AsBsonDocument)).ToArray(); } [IterationSetup] @@ -53,9 +59,21 @@ public void BeforeTask() } [Benchmark] - public void InsertManyLarge() + public void InsertManyLargeBenchmark() { - _collection.InsertMany(_largeDocuments, new InsertManyOptions()); + _collection.InsertMany(_largeDocuments, new()); + } + + [Benchmark] + public void LargeDocCollectionBulkWriteInsertBenchmark() + { + _collection.BulkWrite(_collectionBulkWriteInsertModels, new()); + } + + [Benchmark] + public void LargeDocClientBulkWriteInsertBenchmark() + { + _client.BulkWrite(_clientBulkWriteInsertModels, new()); } [GlobalCleanup] diff --git a/benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/InsertManySmallBenchmark.cs b/benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/SmallDocBulkInsertBenchmark.cs similarity index 55% rename from benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/InsertManySmallBenchmark.cs rename to benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/SmallDocBulkInsertBenchmark.cs index 2092dbb0a2b..a1d62e26038 100644 --- a/benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/InsertManySmallBenchmark.cs +++ b/benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/SmallDocBulkInsertBenchmark.cs @@ -13,24 +13,27 @@ * limitations under the License. */ -using System.Collections.Generic; using System.Linq; using BenchmarkDotNet.Attributes; using MongoDB.Bson; using MongoDB.Driver; -using MongoDB.Driver.TestHelpers; using static MongoDB.Benchmarks.BenchmarkHelper; namespace MongoDB.Benchmarks.MultiDoc { [IterationCount(100)] - [BenchmarkCategory(DriverBenchmarkCategory.MultiBench, DriverBenchmarkCategory.WriteBench, DriverBenchmarkCategory.DriverBench)] - public class InsertManySmallBenchmark + [BenchmarkCategory(DriverBenchmarkCategory.BulkWriteBench, DriverBenchmarkCategory.MultiBench, DriverBenchmarkCategory.WriteBench, DriverBenchmarkCategory.DriverBench)] + public class SmallDocBulkInsertBenchmark { private IMongoClient _client; private IMongoCollection _collection; private IMongoDatabase _database; - private IEnumerable _smallDocuments; + private BsonDocument[] _smallDocuments; + private InsertOneModel[] _collectionBulkWriteInsertModels; + private BulkWriteInsertOneModel[] _clientBulkWriteInsertModels; + + private static readonly CollectionNamespace __collectionNamespace = + CollectionNamespace.FromFullName($"{MongoConfiguration.PerfTestDatabaseName}.{MongoConfiguration.PerfTestCollectionName}"); [Params(2_750_000)] public int BenchmarkDataSetSize { get; set; } // used in BenchmarkResult.cs @@ -42,7 +45,9 @@ public void Setup() _database = _client.GetDatabase(MongoConfiguration.PerfTestDatabaseName); var smallDocument = ReadExtendedJson("single_and_multi_document/small_doc.json"); - _smallDocuments = Enumerable.Range(0, 10000).Select(_ => smallDocument.DeepClone().AsBsonDocument); + _smallDocuments = Enumerable.Range(0, 10000).Select(_ => smallDocument.DeepClone().AsBsonDocument).ToArray(); + _collectionBulkWriteInsertModels = _smallDocuments.Select(x => new InsertOneModel(x.DeepClone().AsBsonDocument)).ToArray(); + _clientBulkWriteInsertModels = _smallDocuments.Select(x => new BulkWriteInsertOneModel(__collectionNamespace, x.DeepClone().AsBsonDocument)).ToArray(); } [IterationSetup] @@ -53,9 +58,21 @@ public void BeforeTask() } [Benchmark] - public void InsertManySmall() + public void InsertManySmallBenchmark() + { + _collection.InsertMany(_smallDocuments, new()); + } + + [Benchmark] + public void SmallDocCollectionBulkWriteInsertBenchmark() + { + _collection.BulkWrite(_collectionBulkWriteInsertModels, new()); + } + + [Benchmark] + public void SmallDocClientBulkWriteInsertBenchmark() { - _collection.InsertMany(_smallDocuments, new InsertManyOptions()); + _client.BulkWrite(_clientBulkWriteInsertModels, new()); } [GlobalCleanup] diff --git a/benchmarks/MongoDB.Driver.Benchmarks/README.md b/benchmarks/MongoDB.Driver.Benchmarks/README.md index fa07012c742..899a9df0ae6 100644 --- a/benchmarks/MongoDB.Driver.Benchmarks/README.md +++ b/benchmarks/MongoDB.Driver.Benchmarks/README.md @@ -13,7 +13,7 @@ This suite implements the benchmarks described in this [spec](https://github.com (e.g `dotnet run -c Release -- --driverBenchmarks --envVars MONGODB_URI:"ConnectionString"`) You can also select the benchmarks to run directly on the command for running the benchmarks as such -`dotnet run -c Release -- --driverBenchmarks --fitler "*BenchmarkClassName*"`. The benchmarks are also grouped into categories namely: BSONBench, WriteBench +`dotnet run -c Release -- --driverBenchmarks --filter "*BenchmarkClassName*"`. The benchmarks are also grouped into categories namely: BSONBench, WriteBench ReadBench, ParallelBench, SingleBench, MultiBench and DriverBench. So if you wanted to only run the WriteBench benchmarks, you can do so as follows: `dotnet run -c Release -- --driverBenchmarks --anyCategories "WriteBench"`. diff --git a/benchmarks/MongoDB.Driver.Benchmarks/scripts/download-data.sh b/benchmarks/MongoDB.Driver.Benchmarks/scripts/download-data.sh index 50d417662ff..874fbc22a2f 100755 --- a/benchmarks/MongoDB.Driver.Benchmarks/scripts/download-data.sh +++ b/benchmarks/MongoDB.Driver.Benchmarks/scripts/download-data.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # Copyright 2010 MongoDB, Inc. # diff --git a/evergreen/evergreen.yml b/evergreen/evergreen.yml index 990cad7e94a..f517293147d 100644 --- a/evergreen/evergreen.yml +++ b/evergreen/evergreen.yml @@ -1461,11 +1461,11 @@ tasks: - func: bootstrap-mongo-orchestration - func: run-smoke-tests - - name: performance-tests-net80-server-v6.0 + - name: test-csharp-spec-benchmarks commands: - func: bootstrap-mongo-orchestration vars: - VERSION: "v6.0-perf" + VERSION: "v8.0-perf" TOPOLOGY: "server" SSL: "nossl" AUTH: "noauth" @@ -2527,7 +2527,7 @@ buildvariants: run_on: - rhel90-dbx-perf-large tasks: - - name: performance-tests-net80-server-v6.0 + - name: test-csharp-spec-benchmarks # AWS Lambda tests - name: aws-lambda-tests