CSHARP-5203: Benchmark Collection and Client bulkWrite

benchmarks/MongoDB.Driver.Benchmarks/BenchmarkResult.cs

@@ -27,21 +27,24 @@ public sealed class BenchmarkResult
 
         public BenchmarkResult(BenchmarkReport benchmarkReport)
         {
+            Categories = new HashSet<string>(benchmarkReport.BenchmarkCase.Descriptor.Categories);
+
             int dataSetSize;
-            if (benchmarkReport.BenchmarkCase.Descriptor.HasCategory(DriverBenchmarkCategory.BsonBench))
+            if (Categories.Contains(DriverBenchmarkCategory.BsonBench))
             {
                 var bsonBenchmarkData = (BsonBenchmarkData)benchmarkReport.BenchmarkCase.Parameters["BenchmarkData"];
                 Name = bsonBenchmarkData.DataSetName + benchmarkReport.BenchmarkCase.Descriptor.Type.Name;
                 dataSetSize = bsonBenchmarkData.DataSetSize;
             }
             else
             {
-                Name = benchmarkReport.BenchmarkCase.Descriptor.Type.Name;
+                Name = Categories.Contains(DriverBenchmarkCategory.BulkWriteBench)
+                    ? benchmarkReport.BenchmarkCase.Descriptor.WorkloadMethod.Name
+                    : benchmarkReport.BenchmarkCase.Descriptor.Type.Name;
+
                 dataSetSize = (int)benchmarkReport.BenchmarkCase.Parameters["BenchmarkDataSetSize"];
             }
 
-            Categories = new HashSet<string>(benchmarkReport.BenchmarkCase.Descriptor.Categories);
-
             // change the median from nanoseconds to seconds for calculating the score.
             // since dataSetSize is in bytes, divide the score to convert to MB/s
             Score = (dataSetSize / (benchmarkReport.ResultStatistics.Median / 1_000_000_000D)) / 1_000_000D;

benchmarks/MongoDB.Driver.Benchmarks/DriverBenchmarkCategory.cs

@@ -26,6 +26,9 @@ public static class DriverBenchmarkCategory
         public const string ReadBench = "ReadBench";
         public const string SingleBench = "SingleBench";
         public const string WriteBench = "WriteBench";
+
+        // not included in AllCategories as it's not part of the benchmarking spec
+        public const string BulkWriteBench = "BulkWriteBench";
 
         public static readonly IEnumerable<string> AllCategories = new[] {BsonBench, ReadBench, WriteBench, MultiBench, SingleBench, ParallelBench, DriverBench};
     }

benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/BulkWriteMixedOpsBenchmark.cs

@@ -0,0 +1,94 @@
+/* Copyright 2010-present MongoDB Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System.Collections.Generic;
+using System.Linq;
+using BenchmarkDotNet.Attributes;
+using MongoDB.Bson;
+using MongoDB.Driver;
+using static MongoDB.Benchmarks.BenchmarkHelper;
+
+namespace MongoDB.Benchmarks.MultiDoc
+{
+    [IterationCount(15)]
+    [BenchmarkCategory(DriverBenchmarkCategory.BulkWriteBench, DriverBenchmarkCategory.MultiBench, DriverBenchmarkCategory.WriteBench, DriverBenchmarkCategory.DriverBench)]
+    public class BulkWriteMixedOpsBenchmark
+    {
+        private IMongoClient _client;
+        private IMongoCollection<BsonDocument> _collection;
+        private IMongoDatabase _database;
+        private readonly List<BulkWriteModel> _clientBulkWriteMixedOpsModels = [];
+        private readonly List<WriteModel<BsonDocument>> _collectionBulkWriteMixedOpsModels = [];
+
+        private static readonly string[] __collectionNamespaces = Enumerable.Range(0, 10)
+            .Select(i => $"{MongoConfiguration.PerfTestDatabaseName}.{MongoConfiguration.PerfTestCollectionName}_{i}")
+            .ToArray();
+
+        [Params(5_500_000)]
+        public int BenchmarkDataSetSize { get; set; } // used in BenchmarkResult.cs
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            _client = MongoConfiguration.CreateClient();
+
+            var smallDocument = ReadExtendedJson("single_and_multi_document/small_doc.json");
+            for (var i = 0; i < 10000; i++)
+            {
+                var collectionName = __collectionNamespaces[i % __collectionNamespaces.Length];
+
+                _clientBulkWriteMixedOpsModels.Add(new BulkWriteInsertOneModel<BsonDocument>(collectionName, smallDocument.DeepClone().AsBsonDocument));
+                _clientBulkWriteMixedOpsModels.Add(new BulkWriteReplaceOneModel<BsonDocument>(collectionName, FilterDefinition<BsonDocument>.Empty, smallDocument.DeepClone().AsBsonDocument));
+                _clientBulkWriteMixedOpsModels.Add(new BulkWriteDeleteOneModel<BsonDocument>(collectionName, FilterDefinition<BsonDocument>.Empty));
+
+                _collectionBulkWriteMixedOpsModels.Add(new InsertOneModel<BsonDocument>(smallDocument.DeepClone().AsBsonDocument));
+                _collectionBulkWriteMixedOpsModels.Add(new ReplaceOneModel<BsonDocument>(FilterDefinition<BsonDocument>.Empty, smallDocument.DeepClone().AsBsonDocument));
+                _collectionBulkWriteMixedOpsModels.Add(new DeleteOneModel<BsonDocument>(FilterDefinition<BsonDocument>.Empty));
+            }
+        }
+
+        [IterationSetup]
+        public void BeforeTask()
+        {
+            _client.DropDatabase(MongoConfiguration.PerfTestDatabaseName);
+
+            _database = _client.GetDatabase(MongoConfiguration.PerfTestDatabaseName);
+            foreach (var collectionName in __collectionNamespaces)
+            {
+                _database.CreateCollection(collectionName.Split('.')[1]);
+            }
+
+            _collection = _database.GetCollection<BsonDocument>(MongoConfiguration.PerfTestCollectionName);
+        }
+
+        [Benchmark]
+        public void SmallDocCollectionBulkWriteMixedOpsBenchmark()
+        {
+            _collection.BulkWrite(_collectionBulkWriteMixedOpsModels, new());
+        }
+
+        [Benchmark]
+        public void SmallDocClientBulkWriteMixedOpsBenchmark()
+        {
+            _client.BulkWrite(_clientBulkWriteMixedOpsModels, new());
+        }
+
+        [GlobalCleanup]
+        public void Teardown()
+        {
+            _client.Dispose();
+        }
+    }
+}

benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/InsertManyLargeBenchmark.cs → benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/LargeDocBulkInsertBenchmark.cs

@@ -18,19 +18,23 @@
 using BenchmarkDotNet.Attributes;
 using MongoDB.Bson;
 using MongoDB.Driver;
-using MongoDB.Driver.TestHelpers;
 using static MongoDB.Benchmarks.BenchmarkHelper;
 
 namespace MongoDB.Benchmarks.MultiDoc
 {
     [IterationCount(100)]
-    [BenchmarkCategory(DriverBenchmarkCategory.MultiBench, DriverBenchmarkCategory.WriteBench, DriverBenchmarkCategory.DriverBench)]
-    public class InsertManyLargeBenchmark
+    [BenchmarkCategory(DriverBenchmarkCategory.BulkWriteBench, DriverBenchmarkCategory.MultiBench, DriverBenchmarkCategory.WriteBench, DriverBenchmarkCategory.DriverBench)]
+    public class LargeDocBulkInsertBenchmark
     {
         private IMongoClient _client;
         private IMongoCollection<BsonDocument> _collection;
         private IMongoDatabase _database;
-        private IEnumerable<BsonDocument> _largeDocuments;
+        private BsonDocument[] _largeDocuments;
+        private InsertOneModel<BsonDocument>[] _collectionBulkWriteInsertModels;
+        private BulkWriteInsertOneModel<BsonDocument>[] _clientBulkWriteInsertModels;
+
+        private static readonly CollectionNamespace __collectionNamespace =
+            CollectionNamespace.FromFullName($"{MongoConfiguration.PerfTestDatabaseName}.{MongoConfiguration.PerfTestCollectionName}");
 
         [Params(27_310_890)]
         public int BenchmarkDataSetSize { get; set; } // used in BenchmarkResult.cs
@@ -42,7 +46,9 @@ public void Setup()
             _database = _client.GetDatabase(MongoConfiguration.PerfTestDatabaseName);
 
             var largeDocument = ReadExtendedJson("single_and_multi_document/large_doc.json");
-            _largeDocuments = Enumerable.Range(0, 10).Select(_ => largeDocument.DeepClone().AsBsonDocument);
+            _largeDocuments = Enumerable.Range(0, 10).Select(_ => largeDocument.DeepClone().AsBsonDocument).ToArray();
+            _collectionBulkWriteInsertModels = _largeDocuments.Select(x => new InsertOneModel<BsonDocument>(x.DeepClone().AsBsonDocument)).ToArray();
+            _clientBulkWriteInsertModels = _largeDocuments.Select(x => new BulkWriteInsertOneModel<BsonDocument>(__collectionNamespace, x.DeepClone().AsBsonDocument)).ToArray();
         }
 
         [IterationSetup]
@@ -53,9 +59,21 @@ public void BeforeTask()
         }
 
         [Benchmark]
-        public void InsertManyLarge()
+        public void InsertManyLargeBenchmark()
         {
-            _collection.InsertMany(_largeDocuments, new InsertManyOptions());
+            _collection.InsertMany(_largeDocuments, new());
+        }
+
+        [Benchmark]
+        public void LargeDocCollectionBulkWriteInsertBenchmark()
+        {
+            _collection.BulkWrite(_collectionBulkWriteInsertModels, new());
+        }
+
+        [Benchmark]
+        public void LargeDocClientBulkWriteInsertBenchmark()
+        {
+            _client.BulkWrite(_clientBulkWriteInsertModels, new());
         }
 
         [GlobalCleanup]

benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/InsertManySmallBenchmark.cs → benchmarks/MongoDB.Driver.Benchmarks/MultiDoc/SmallDocBulkInsertBenchmark.cs

@@ -13,24 +13,27 @@
  * limitations under the License.
  */
 
-using System.Collections.Generic;
 using System.Linq;
 using BenchmarkDotNet.Attributes;
 using MongoDB.Bson;
 using MongoDB.Driver;
-using MongoDB.Driver.TestHelpers;
 using static MongoDB.Benchmarks.BenchmarkHelper;
 
 namespace MongoDB.Benchmarks.MultiDoc
 {
     [IterationCount(100)]
-    [BenchmarkCategory(DriverBenchmarkCategory.MultiBench, DriverBenchmarkCategory.WriteBench, DriverBenchmarkCategory.DriverBench)]
-    public class InsertManySmallBenchmark
+    [BenchmarkCategory(DriverBenchmarkCategory.BulkWriteBench, DriverBenchmarkCategory.MultiBench, DriverBenchmarkCategory.WriteBench, DriverBenchmarkCategory.DriverBench)]
+    public class SmallDocBulkInsertBenchmark
     {
         private IMongoClient _client;
         private IMongoCollection<BsonDocument> _collection;
         private IMongoDatabase _database;
-        private IEnumerable<BsonDocument> _smallDocuments;
+        private BsonDocument[] _smallDocuments;
+        private InsertOneModel<BsonDocument>[] _collectionBulkWriteInsertModels;
+        private BulkWriteInsertOneModel<BsonDocument>[] _clientBulkWriteInsertModels;
+
+        private static readonly CollectionNamespace __collectionNamespace =
+            CollectionNamespace.FromFullName($"{MongoConfiguration.PerfTestDatabaseName}.{MongoConfiguration.PerfTestCollectionName}");
 
         [Params(2_750_000)]
         public int BenchmarkDataSetSize { get; set; } // used in BenchmarkResult.cs
@@ -42,7 +45,9 @@ public void Setup()
             _database = _client.GetDatabase(MongoConfiguration.PerfTestDatabaseName);
 
             var smallDocument = ReadExtendedJson("single_and_multi_document/small_doc.json");
-            _smallDocuments = Enumerable.Range(0, 10000).Select(_ => smallDocument.DeepClone().AsBsonDocument);
+            _smallDocuments = Enumerable.Range(0, 10000).Select(_ => smallDocument.DeepClone().AsBsonDocument).ToArray();
+            _collectionBulkWriteInsertModels = _smallDocuments.Select(x => new InsertOneModel<BsonDocument>(x.DeepClone().AsBsonDocument)).ToArray();
+            _clientBulkWriteInsertModels = _smallDocuments.Select(x => new BulkWriteInsertOneModel<BsonDocument>(__collectionNamespace, x.DeepClone().AsBsonDocument)).ToArray();
         }
 
         [IterationSetup]
@@ -53,9 +58,21 @@ public void BeforeTask()
         }
 
         [Benchmark]
-        public void InsertManySmall()
+        public void InsertManySmallBenchmark()
+        {
+            _collection.InsertMany(_smallDocuments, new());
+        }
+
+        [Benchmark]
+        public void SmallDocCollectionBulkWriteInsertBenchmark()
+        {
+            _collection.BulkWrite(_collectionBulkWriteInsertModels, new());
+        }
+
+        [Benchmark]
+        public void SmallDocClientBulkWriteInsertBenchmark()
         {
-            _collection.InsertMany(_smallDocuments, new InsertManyOptions());
+            _client.BulkWrite(_clientBulkWriteInsertModels, new());
         }
 
         [GlobalCleanup]

benchmarks/MongoDB.Driver.Benchmarks/README.md

@@ -13,7 +13,7 @@ This suite implements the benchmarks described in this [spec](https://github.com
    (e.g `dotnet run -c Release -- --driverBenchmarks --envVars MONGODB_URI:"ConnectionString"`)
 
 You can also select the benchmarks to run directly on the command for running the benchmarks as such
-`dotnet run -c Release -- --driverBenchmarks --fitler "*BenchmarkClassName*"`. The benchmarks are also grouped into categories namely: BSONBench, WriteBench
+`dotnet run -c Release -- --driverBenchmarks --filter "*BenchmarkClassName*"`. The benchmarks are also grouped into categories namely: BSONBench, WriteBench
 ReadBench, ParallelBench, SingleBench, MultiBench and DriverBench. So if you wanted to only run the WriteBench benchmarks, you can do so
 as follows: `dotnet run -c Release -- --driverBenchmarks --anyCategories "WriteBench"`.
 

benchmarks/MongoDB.Driver.Benchmarks/scripts/download-data.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 # Copyright 2010 MongoDB, Inc.
 #

evergreen/evergreen.yml

@@ -1461,11 +1461,11 @@ tasks:
         - func: bootstrap-mongo-orchestration
         - func: run-smoke-tests
 
-    - name: performance-tests-net80-server-v6.0
+    - name: test-csharp-spec-benchmarks
       commands:
         - func: bootstrap-mongo-orchestration
           vars:
-            VERSION: "v6.0-perf"
+            VERSION: "v8.0-perf"
             TOPOLOGY: "server"
             SSL: "nossl"
             AUTH: "noauth"
@@ -2527,7 +2527,7 @@ buildvariants:
   run_on:
     - rhel90-dbx-perf-large
   tasks:
-    - name: performance-tests-net80-server-v6.0
+    - name: test-csharp-spec-benchmarks
 
 # AWS Lambda tests
 - name: aws-lambda-tests