Allow Initialise with shared throughput (#40)

RiverPhillips · asosMikeGore · commit 48ce9e5d7e06 · 2019-06-12T15:54:34.000+01:00
* Update to use Microsoft.Azure.DocumentDb 2.4.0 * Use Microsoft.Azure.DocumentDB for .net 4.5.2 * Refactor to remove unnescessary awaits and usings * Revert "Refactor to remove unnescessary awaits and usings" This reverts commit e2a974f. * Allow setting the cosmos throughput at a database level Validate Collection and Database requests units are compatible Set throughput of database and collection when StorageEngine is initialised Update readme to reflect provisioning database with shared throughput Update to latest cosmos libraries (#39) * Update to use Microsoft.Azure.DocumentDb 2.4.0 * Use Microsoft.Azure.DocumentDB for .net 4.5.2 * Refactor to remove unnescessary awaits and usings * Revert "Refactor to remove unnescessary awaits and usings" This reverts commit e2a974f. Allowing setting throughput at either the database or the collection level * Remove unnecessary awaits * Add CONTRIBUTING.md
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -0,0 +1,13 @@
+# Contributing
+
+## Building Simple Event Store
+Clone repository
+```sh
+git clone git@github.com:ASOS/SimpleEventStore.git
+```
+
+Enter the repository and build the provider and run the tests using cake
+```sh
+cd build
+./build.ps1
+```
diff --git a/README.md b/README.md
@@ -45,7 +45,11 @@ DocumentClient client; // Set this up as required
 
 // If UseCollection isn't specified, sensible defaults for development are used.
 // If UseSubscriptions isn't supplied the subscription feature is disabled.
+// If UseSharedThroughput isn't supplied the throughput is set only at a collection level
 return await new AzureDocumentDbStorageEngineBuilder(client, databaseName)
+	.UseSharedThroughput(o => {
+		o.DatabaseRequestUnits = 400;
+	})
 	.UseCollection(o =>
    	{
 		o.ConsistencyLevel = consistencyLevelEnum;
@@ -64,10 +68,17 @@ return await new AzureDocumentDbStorageEngineBuilder(client, databaseName)
 	.Build()
 	.Initialise();
 ```
+### Database Options
+Use this only if you want throughput to be set at a database level
+
+Allows you to specify
+- The number of RUs for the database
+
+
 ### CollectionOptions
 Allows you to specify
 - The consistency level of the database
-- The default number of RUs when the collection is created
+- The number of RUs for the collection - if throughput is set at a database level this cannot be greater than database throughput
 - The collection name
 
 Only use one of the following consistency levels
diff --git a/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb.Tests/AzureDocumentDBEventStoreInitializing.cs b/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb.Tests/AzureDocumentDBEventStoreInitializing.cs
@@ -13,24 +13,24 @@ namespace SimpleEventStore.AzureDocumentDb.Tests
     public class AzureDocumentDBEventStoreInitializing
     {
         private const string DatabaseName = "InitializeTests";
+        private readonly Uri databaseUri = UriFactory.CreateDatabaseUri(DatabaseName);
+        private readonly DocumentClient client = DocumentClientFactory.Create();
 
-        [OneTimeTearDown]
-        public async Task TearDownDatabase()
+        [TearDown]
+        public Task TearDownDatabase()
         {
-            var client = DocumentClientFactory.Create(DatabaseName);
-            await client.DeleteDatabaseAsync(UriFactory.CreateDatabaseUri(DatabaseName));
+            return client.DeleteDatabaseAsync(databaseUri);
         }
 
         [Test]
         public async Task when_initializing_all_expected_resources_are_created()
         {
-            var client = DocumentClientFactory.Create(DatabaseName);
             var collectionName = "AllExpectedResourcesAreCreated_" + Guid.NewGuid();
             var storageEngine = await StorageEngineFactory.Create(DatabaseName, o => o.CollectionName = collectionName);
-            
+
             await storageEngine.Initialise();
 
-            var database = (await client.ReadDatabaseAsync(UriFactory.CreateDatabaseUri(DatabaseName))).Resource;
+            var database = (await client.ReadDatabaseAsync(databaseUri)).Resource;
             var collection = (await client.ReadDocumentCollectionAsync(UriFactory.CreateDocumentCollectionUri(DatabaseName, collectionName))).Resource;
             var storedProcedure = (await client.ReadStoredProcedureAsync(UriFactory.CreateStoredProcedureUri(DatabaseName, collectionName, TestConstants.AppendStoredProcedureName))).Resource;
             var offer = client.CreateOfferQuery()
@@ -55,17 +55,91 @@ public async Task when_initializing_with_a_time_to_live_it_is_set()
         {
             var ttl = 60;
             var collectionName = "TimeToLiveIsSet_" + Guid.NewGuid();
-            var client = DocumentClientFactory.Create(DatabaseName);
-            var storageEngine = await StorageEngineFactory.Create(DatabaseName, o => 
+            var storageEngine = await StorageEngineFactory.Create(DatabaseName, o =>
             {
                 o.CollectionName = collectionName;
                 o.DefaultTimeToLive = ttl;
             });
 
             await storageEngine.Initialise();
 
-            var collection = (await client.ReadDocumentCollectionAsync(UriFactory.CreateDocumentCollectionUri(DatabaseName, collectionName))).Resource;
+            var collection =
+                (await client.ReadDocumentCollectionAsync(
+                    UriFactory.CreateDocumentCollectionUri(DatabaseName, collectionName))).Resource;
             Assert.That(collection.DefaultTimeToLive, Is.EqualTo(ttl));
         }
+
+        [Test]
+        public async Task when_using_shared_throughput_it_is_set_at_a_database_level()
+        {
+            const int throughput = 800;
+            var collectionName = "SharedCollection_" + Guid.NewGuid();
+
+            var storageEngine = await StorageEngineFactory.Create(DatabaseName,
+                collectionOptions =>
+                {
+                    collectionOptions.CollectionName = collectionName;
+                    collectionOptions.CollectionRequestUnits = null;
+                },
+                databaseOptions => { databaseOptions.DatabaseRequestUnits = throughput; });
+
+            await storageEngine.Initialise();
+
+            Assert.AreEqual(throughput, await GetDatabaseThroughput());
+        }
+
+        [Test]
+        public async Task when_throughput_is_set_offer_is_updated()
+        {
+            var dbThroughput = 800;
+            var collectionThroughput = 400;
+            var collectionName = "UpdateThroughput_" + Guid.NewGuid();
+
+            await InitialiseStorageEngine(collectionName, collectionThroughput, dbThroughput);
+
+            Assert.AreEqual(dbThroughput, await GetDatabaseThroughput());
+            Assert.AreEqual(collectionThroughput, await GetCollectionThroughput(collectionName));
+
+            dbThroughput = 1600;
+            collectionThroughput = 800;
+
+            await InitialiseStorageEngine(collectionName, collectionThroughput, dbThroughput);
+
+            Assert.AreEqual(dbThroughput, await GetDatabaseThroughput());
+            Assert.AreEqual(collectionThroughput, await GetCollectionThroughput(collectionName));
+        }
+
+        private static async Task InitialiseStorageEngine(string collectionName, int collectionThroughput,
+            int dbThroughput)
+        {
+            var storageEngine = await StorageEngineFactory.Create(DatabaseName,
+                collectionOptions =>
+                {
+                    collectionOptions.CollectionName = collectionName;
+                    collectionOptions.CollectionRequestUnits = collectionThroughput;
+                },
+                databaseOptions => { databaseOptions.DatabaseRequestUnits = dbThroughput; });
+
+            await storageEngine.Initialise();
+        }
+
+        public async Task<int> GetCollectionThroughput(string collectionName)
+        {
+            var collection = await client.ReadDocumentCollectionAsync(UriFactory.CreateDocumentCollectionUri(DatabaseName, collectionName));
+
+            var collectionOffer = client.CreateOfferQuery().Where(x => x.ResourceLink == collection.Resource.SelfLink)
+                .AsEnumerable().First();
+
+            return ((OfferV2)collectionOffer).Content.OfferThroughput;
+        }
+
+        public async Task<int> GetDatabaseThroughput()
+        {
+            var db = await client.ReadDatabaseAsync(databaseUri);
+            var dbOffer = client.CreateOfferQuery().Where(x => x.ResourceLink == db.Resource.SelfLink).AsEnumerable()
+                .First();
+
+            return ((OfferV2)dbOffer).Content.OfferThroughput;
+        }
     }
 }
diff --git a/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb.Tests/AzureDocumentDbEventStoreLogging.cs b/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb.Tests/AzureDocumentDbEventStoreLogging.cs
@@ -21,10 +21,10 @@ public async Task when_a_write_operation_is_successful_the_log_callback_is_calle
             await sut.AppendToStream(streamId, 0, new EventData(Guid.NewGuid(), new OrderCreated("TEST-ORDER")));
 
             Assert.NotNull(response);
-            TestContext.Out.WriteLine($"Charge: {response.RequestCharge}");
-            TestContext.Out.WriteLine($"Quota Usage: {response.CurrentResourceQuotaUsage}");
-            TestContext.Out.WriteLine($"Max Resource Quote: {response.MaxResourceQuota}");
-            TestContext.Out.WriteLine($"Response headers: {response.ResponseHeaders}");
+            await TestContext.Out.WriteLineAsync($"Charge: {response.RequestCharge}");
+            await TestContext.Out.WriteLineAsync($"Quota Usage: {response.CurrentResourceQuotaUsage}");
+            await TestContext.Out.WriteLineAsync($"Max Resource Quote: {response.MaxResourceQuota}");
+            await TestContext.Out.WriteLineAsync($"Response headers: {response.ResponseHeaders}");
         }
 
         [Test]
@@ -40,7 +40,7 @@ public async Task when_a_read_operation_is_successful_the_log_callback_is_called
             Assert.That(logCount, Is.EqualTo(2));
         }
 
-        private static async Task<IStorageEngine> CreateStorageEngine(Action<ResponseInformation> onSuccessCallback, string databaseName = "LoggingTests")
+        private static Task<IStorageEngine> CreateStorageEngine(Action<ResponseInformation> onSuccessCallback, string databaseName = "LoggingTests")
         {
             var config = new ConfigurationBuilder()
                 .AddJsonFile("appsettings.json")
@@ -57,7 +57,7 @@ private static async Task<IStorageEngine> CreateStorageEngine(Action<ResponseInf
 
             DocumentClient client = new DocumentClient(new Uri(documentDbUri), authKey);
 
-            return await new AzureDocumentDbStorageEngineBuilder(client, databaseName)
+            return new AzureDocumentDbStorageEngineBuilder(client, databaseName)
                 .UseCollection(o =>
                 {
                     o.ConsistencyLevel = consistencyLevelEnum;
diff --git a/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb.Tests/AzureDocumentDbEventStoreReadingPartiallyDeletedStreams.cs b/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb.Tests/AzureDocumentDbEventStoreReadingPartiallyDeletedStreams.cs
@@ -17,7 +17,7 @@ public async Task when_reading_a_stream_that_has_deleted_events_the_stream_can_s
             const string databaseName = "ReadingPartialStreamTests";
             const string collectionName = "Commits";
 
-            var client = DocumentClientFactory.Create(databaseName);
+            var client = DocumentClientFactory.Create();
             var storageEngine = await StorageEngineFactory.Create(databaseName, o => o.CollectionName = collectionName);
             var eventStore = new EventStore(storageEngine);
             var streamId = Guid.NewGuid().ToString();
@@ -31,9 +31,9 @@ public async Task when_reading_a_stream_that_has_deleted_events_the_stream_can_s
             Assert.That(stream.First().EventNumber, Is.EqualTo(2));
         }
 
-        private static async Task SimulateTimeToLiveExpiration(string databaseName, string collectionName, DocumentClient client, string streamId)
+        private static Task SimulateTimeToLiveExpiration(string databaseName, string collectionName, DocumentClient client, string streamId)
         {
-            await client.DeleteDocumentAsync(
+            return client.DeleteDocumentAsync(
                 UriFactory.CreateDocumentUri(databaseName, collectionName, $"{streamId}:1"),
                 new RequestOptions() { PartitionKey = new PartitionKey(streamId) });
         }
diff --git a/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb.Tests/AzureDocumentDbStorageEngineBuilderTests.cs b/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb.Tests/AzureDocumentDbStorageEngineBuilderTests.cs
@@ -54,6 +54,26 @@ public void when_setting_the_jsonserializationsettings_it_must_be_supplied()
             Assert.Throws<ArgumentNullException>(() => builder.UseJsonSerializerSettings(null));
         }
 
+        [Test]
+        public void throughput_must_be_set_in_one_location()
+        {
+            var builder = new AzureDocumentDbStorageEngineBuilder(CreateClient(), "Test")
+                .UseSharedThroughput(o => { o.DatabaseRequestUnits = null; })
+                .UseCollection(o => o.CollectionRequestUnits = null);
+
+            Assert.Throws<ArgumentException>(() => builder.Build());
+        }
+
+        [Test]
+        public void collection_throughput_cannot_be_greater_than_database_throughput()
+        {
+            var builder = new AzureDocumentDbStorageEngineBuilder(CreateClient(), "Test")
+                .UseSharedThroughput(o => { o.DatabaseRequestUnits = 400; })
+                .UseCollection(o => o.CollectionRequestUnits = 500);
+
+            Assert.Throws<ArgumentException>(() => builder.Build());
+        }
+
         private static DocumentClient CreateClient()
         {
             var client = new DocumentClient(new Uri("https://localhost:8081/"), "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==");
diff --git a/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb.Tests/DocumentClientFactory.cs b/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb.Tests/DocumentClientFactory.cs
@@ -7,12 +7,12 @@ namespace SimpleEventStore.AzureDocumentDb.Tests
 {
     internal static class DocumentClientFactory
     {
-        internal static DocumentClient Create(string databaseName)
+        internal static DocumentClient Create()
         {
-            return Create(databaseName, new JsonSerializerSettings());
+            return Create(new JsonSerializerSettings());
         }
 
-        internal static DocumentClient Create(string databaseName, JsonSerializerSettings settings)
+        internal static DocumentClient Create(JsonSerializerSettings settings)
         {
             var config = new ConfigurationBuilder()
                 .AddJsonFile("appsettings.json")
diff --git a/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb.Tests/StorageEngineFactory.cs b/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb.Tests/StorageEngineFactory.cs
@@ -10,12 +10,12 @@ namespace SimpleEventStore.AzureDocumentDb.Tests
 {
     internal static class StorageEngineFactory
     {
-        internal static async Task<IStorageEngine> Create(string databaseName, Action<CollectionOptions> collectionOverrides = null)
+        internal static Task<IStorageEngine> Create(string databaseName, Action<CollectionOptions> collectionOverrides = null, Action<DatabaseOptions> databaseOverrides = null)
         {
-            return await Create(databaseName, new JsonSerializerSettings(), collectionOverrides);
+            return Create(databaseName, new JsonSerializerSettings(), collectionOverrides, databaseOverrides);
         }
 
-        internal static async Task<IStorageEngine> Create(string databaseName, JsonSerializerSettings settings, Action<CollectionOptions> collectionOverrides = null)
+        internal static Task<IStorageEngine> Create(string databaseName, JsonSerializerSettings settings, Action<CollectionOptions> collectionOverrides = null, Action<DatabaseOptions> databaseOverrides = null)
         {
             var config = new ConfigurationBuilder()
                 .AddJsonFile("appsettings.json")
@@ -24,19 +24,23 @@ internal static async Task<IStorageEngine> Create(string databaseName, JsonSeria
             var consistencyLevel = config["ConsistencyLevel"];
             ConsistencyLevel consistencyLevelEnum;
 
-            if(!Enum.TryParse(consistencyLevel, true, out consistencyLevelEnum))
+            if (!Enum.TryParse(consistencyLevel, true, out consistencyLevelEnum))
             {
                 throw new Exception($"The ConsistencyLevel value {consistencyLevel} is not supported");
             }
 
-            var client = DocumentClientFactory.Create(databaseName, settings);
+            var client = DocumentClientFactory.Create(settings);
 
-            return await new AzureDocumentDbStorageEngineBuilder(client, databaseName)
+            return new AzureDocumentDbStorageEngineBuilder(client, databaseName)
+                .UseSharedThroughput(o =>
+                {
+                    databaseOverrides?.Invoke(o);
+                })
                 .UseCollection(o =>
                 {
                     o.ConsistencyLevel = consistencyLevelEnum;
                     o.CollectionRequestUnits = TestConstants.RequestUnits;
-                    if(collectionOverrides != null) collectionOverrides(o);
+                    if (collectionOverrides != null) collectionOverrides(o);
                 })
                 .UseTypeMap(new ConfigurableSerializationTypeMap()
                     .RegisterTypes(
diff --git a/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb/AzureDocumentDbStorageEngine.cs b/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb/AzureDocumentDbStorageEngine.cs
diff --git a/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb/AzureDocumentDbStorageEngineBuilder.cs b/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb/AzureDocumentDbStorageEngineBuilder.cs
diff --git a/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb/CollectionOptions.cs b/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb/CollectionOptions.cs
diff --git a/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb/DatabaseOptions.cs b/src/SimpleEventStore/SimpleEventStore.AzureDocumentDb/DatabaseOptions.cs

Original file line number	Diff line number	Diff line change
`@@ -17,7 +17,7 @@ public async Task when_reading_a_stream_that_has_deleted_events_the_stream_can_s`
`17`	`17`	`const string databaseName = "ReadingPartialStreamTests";`
`18`	`18`	`const string collectionName = "Commits";`
`19`	`19`
`20`		`- var client = DocumentClientFactory.Create(databaseName);`
	`20`	`+ var client = DocumentClientFactory.Create();`
`21`	`21`	`var storageEngine = await StorageEngineFactory.Create(databaseName, o => o.CollectionName = collectionName);`
`22`	`22`	`var eventStore = new EventStore(storageEngine);`
`23`	`23`	`var streamId = Guid.NewGuid().ToString();`
`@@ -31,9 +31,9 @@ public async Task when_reading_a_stream_that_has_deleted_events_the_stream_can_s`
`31`	`31`	`Assert.That(stream.First().EventNumber, Is.EqualTo(2));`
`32`	`32`	`}`
`33`	`33`
`34`		`- private static async Task SimulateTimeToLiveExpiration(string databaseName, string collectionName, DocumentClient client, string streamId)`
	`34`	`+ private static Task SimulateTimeToLiveExpiration(string databaseName, string collectionName, DocumentClient client, string streamId)`
`35`	`35`	`{`
`36`		`- await client.DeleteDocumentAsync(`
	`36`	`+ return client.DeleteDocumentAsync(`
`37`	`37`	`UriFactory.CreateDocumentUri(databaseName, collectionName, $"{streamId}:1"),`
`38`	`38`	`new RequestOptions() { PartitionKey = new PartitionKey(streamId) });`
`39`	`39`	`}`
Original file line number	Diff line number	Diff line change
`@@ -7,12 +7,12 @@ namespace SimpleEventStore.AzureDocumentDb.Tests`
`7`	`7`	`{`
`8`	`8`	`internal static class DocumentClientFactory`
`9`	`9`	`{`
`10`		`- internal static DocumentClient Create(string databaseName)`
	`10`	`+ internal static DocumentClient Create()`
`11`	`11`	`{`
`12`		`- return Create(databaseName, new JsonSerializerSettings());`
	`12`	`+ return Create(new JsonSerializerSettings());`
`13`	`13`	`}`
`14`	`14`
`15`		`- internal static DocumentClient Create(string databaseName, JsonSerializerSettings settings)`
	`15`	`+ internal static DocumentClient Create(JsonSerializerSettings settings)`
`16`	`16`	`{`
`17`	`17`	`var config = new ConfigurationBuilder()`
`18`	`18`	`.AddJsonFile("appsettings.json")`
Original file line number	Diff line number	Diff line change
`@@ -10,12 +10,12 @@ namespace SimpleEventStore.AzureDocumentDb.Tests`
`10`	`10`	`{`
`11`	`11`	`internal static class StorageEngineFactory`
`12`	`12`	`{`
`13`		`- internal static async Task<IStorageEngine> Create(string databaseName, Action<CollectionOptions> collectionOverrides = null)`
	`13`	`+ internal static Task<IStorageEngine> Create(string databaseName, Action<CollectionOptions> collectionOverrides = null, Action<DatabaseOptions> databaseOverrides = null)`
`14`	`14`	`{`
`15`		`- return await Create(databaseName, new JsonSerializerSettings(), collectionOverrides);`
	`15`	`+ return Create(databaseName, new JsonSerializerSettings(), collectionOverrides, databaseOverrides);`
`16`	`16`	`}`
`17`	`17`
`18`		`- internal static async Task<IStorageEngine> Create(string databaseName, JsonSerializerSettings settings, Action<CollectionOptions> collectionOverrides = null)`
	`18`	`+ internal static Task<IStorageEngine> Create(string databaseName, JsonSerializerSettings settings, Action<CollectionOptions> collectionOverrides = null, Action<DatabaseOptions> databaseOverrides = null)`
`19`	`19`	`{`
`20`	`20`	`var config = new ConfigurationBuilder()`
`21`	`21`	`.AddJsonFile("appsettings.json")`
`@@ -24,19 +24,23 @@ internal static async Task<IStorageEngine> Create(string databaseName, JsonSeria`
`24`	`24`	`var consistencyLevel = config["ConsistencyLevel"];`
`25`	`25`	`ConsistencyLevel consistencyLevelEnum;`
`26`	`26`
`27`		`- if(!Enum.TryParse(consistencyLevel, true, out consistencyLevelEnum))`
	`27`	`+ if (!Enum.TryParse(consistencyLevel, true, out consistencyLevelEnum))`
`28`	`28`	`{`
`29`	`29`	`throw new Exception($"The ConsistencyLevel value {consistencyLevel} is not supported");`
`30`	`30`	`}`
`31`	`31`
`32`		`- var client = DocumentClientFactory.Create(databaseName, settings);`
	`32`	`+ var client = DocumentClientFactory.Create(settings);`
`33`	`33`
`34`		`- return await new AzureDocumentDbStorageEngineBuilder(client, databaseName)`
	`34`	`+ return new AzureDocumentDbStorageEngineBuilder(client, databaseName)`
	`35`	`+ .UseSharedThroughput(o =>`
	`36`	`+ {`
	`37`	`+ databaseOverrides?.Invoke(o);`
	`38`	`+ })`
`35`	`39`	`.UseCollection(o =>`
`36`	`40`	`{`
`37`	`41`	`o.ConsistencyLevel = consistencyLevelEnum;`
`38`	`42`	`o.CollectionRequestUnits = TestConstants.RequestUnits;`
`39`		`- if(collectionOverrides != null) collectionOverrides(o);`
	`43`	`+ if (collectionOverrides != null) collectionOverrides(o);`
`40`	`44`	`})`
`41`	`45`	`.UseTypeMap(new ConfigurableSerializationTypeMap()`
`42`	`46`	`.RegisterTypes(`