Exchanged redis client library, fixed DNS issue when connecting (will be solved in .NET core 2.0), increasing processors instances, improvement in docs and script variable naming

Author: DenisBiondic

Create-Infrastructure.ps1

@@ -10,7 +10,7 @@ Param(
  $ResourceGroupLocation = "North Europe",
 
  [switch]
- $SkipCluster
+ $SkipClusterInCloud
 )
 
 # stop the script on first error
@@ -46,7 +46,7 @@ $registryTemplateParameters["EnvironmentTag"] = $EnvironmentTag
 
 DeployTemplate -ResourceGroupName $resourceGroupName -TemplateFileFullPath $registryTemplateFile -TemplateParameters $registryTemplateParameters
 
-if (-not $SkipCluster) {
+if (-not $SkipClusterInCloud) {
     $automationKeyVaultName = "ca-automation-$EnvironmentTag"
     $automationKeyVault = Get-AzureRmKeyVault -VaultName $automationKeyVaultName -ErrorAction SilentlyContinue
 

Deploy-Application.ps1

@@ -19,9 +19,11 @@ $registryUrl = ("cadevcache" + $EnvironmentTag + "registry.azurecr.io")
 
 $keyVaultName = "ca-devcache-$EnvironmentTag"
 
+$storageKeyName = ("cadevcache" + $EnvironmentTag + "storage")
+
 $recieveConnectionString = (Get-AzureKeyVaultSecret -VaultName $keyVaultName -SecretName eventHubReceiveConnectionString).SecretValueText
 $sendConnectionString = (Get-AzureKeyVaultSecret -VaultName $keyVaultName -SecretName eventHubSendConnectionString).SecretValueText
-$storageAccountKey = (Get-AzureKeyVaultSecret -VaultName $keyVaultName -SecretName cadevcachegreenstorage).SecretValueText
+$storageAccountKey = (Get-AzureKeyVaultSecret -VaultName $keyVaultName -SecretName $storageKeyName).SecretValueText
 $storageAccountName = ("cadevcache" + $EnvironmentTag + "storage")
 $eventHubReaderPath = "ca-devcache-$EnvironmentTag-hub"
 

DeviceCache.Frontend/Controllers/DataController.cs

@@ -1,37 +1,36 @@
 using System;
 using System.Collections.Generic;
 using Microsoft.AspNetCore.Mvc;
-using ServiceStack.Redis;
+using StackExchange.Redis;
 
 namespace DeviceCache.Frontend.Controllers
 {
     [Route("api/[controller]")]
     public class DataController : Controller
     {
-        private readonly RedisManagerPool _manager;
+        private readonly ConnectionMultiplexer _redis;
 
-        public DataController(RedisManagerPool manager)
+        public DataController(ConnectionMultiplexer redis)
         {
-            _manager = manager;
+            _redis = redis;
         }
 
         [HttpGet]
         public IEnumerable<string> Get()
         {
             var random = new Random();
-            var keys = new List<string>();
+            var keys = new List<RedisKey>();
 
             for (int i = 0; i < 40; i++)
             {
                 var randomKey = random.Next(0, 100000);
                 keys.Add(randomKey.ToString());
             }
 
-            using (var client = _manager.GetClient())
-            {
-                var values = client.GetValues(keys);
-                return values;
-            }
+            IDatabase db = _redis.GetDatabase();
+            
+            var values = db.StringGet(keys.ToArray());
+            return values.ToStringArray();
         }
     }
 }

DeviceCache.Frontend/DeviceCache.Frontend.csproj

@@ -10,7 +10,7 @@
     <PackageReference Include="Microsoft.AspNetCore" Version="1.1.1" />
     <PackageReference Include="Microsoft.AspNetCore.Mvc" Version="1.1.2" />
     <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="1.1.1" />
-    <PackageReference Include="ServiceStack.Redis.Core" Version="1.0.43" />
+    <PackageReference Include="StackExchange.Redis" Version="1.2.6" />
   </ItemGroup>
   <ItemGroup>
     <DotNetCliToolReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Tools" Version="1.0.0" />

DeviceCache.Frontend/Startup.cs

@@ -1,9 +1,11 @@
-using Microsoft.AspNetCore.Builder;
+using System.Linq;
+using System.Net;
+using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
-using ServiceStack.Redis;
+using StackExchange.Redis;
 
 namespace DeviceCache.Frontend
 {
@@ -27,8 +29,17 @@ public void ConfigureServices(IServiceCollection services)
             // Add framework services.
             services.AddMvc();
 
-            var manager = new RedisManagerPool("devicecache-cache:6379");
-            services.AddSingleton(manager);
+            ConfigurationOptions config = ConfigurationOptions.Parse("devicecache-cache:6379");
+
+            DnsEndPoint addressEndpoint = config.EndPoints.First() as DnsEndPoint;
+            int port = addressEndpoint.Port;
+            
+            IPHostEntry ip = Dns.GetHostEntryAsync(addressEndpoint.Host).Result;
+            config.EndPoints.Remove(addressEndpoint);
+            config.EndPoints.Add(ip.AddressList.First(), port);
+
+            var redis = ConnectionMultiplexer.Connect(config);
+            services.AddSingleton(redis);
         }
 
         // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.

DeviceCache.Infrastructure/EventHub.json

@@ -34,7 +34,7 @@
       "sku": {
         "name": "Standard",
         "tier": "Standard",
-        "capacity": 20
+        "capacity": 10
       },
       "resources": [
         {
@@ -55,7 +55,7 @@
               "name": "devcacheConsumers",
               "type": "ConsumerGroups",
               "dependsOn": [
-                "[variables('eventHubName')]"
+                "[concat('Microsoft.EventHub/namespaces/', variables('namespaceName'),'/EventHubs/',variables('eventHubName'))]"
               ]
             },
             {
@@ -69,7 +69,7 @@
               },
               "resources": [],
               "dependsOn": [
-                "[variables('eventHubName')]"
+                "[concat('Microsoft.EventHub/namespaces/', variables('namespaceName'),'/EventHubs/',variables('eventHubName'))]"
               ]
             },
             {
@@ -83,7 +83,8 @@
               },
               "resources": [],
               "dependsOn": [
-                "[variables('eventHubName')]"
+                "[concat('Microsoft.EventHub/namespaces/', variables('namespaceName'),'/EventHubs/',variables('eventHubName'))]",
+                "[concat('Microsoft.EventHub/namespaces/', variables('namespaceName'),'/EventHubs/',variables('eventHubName'), '/authorizationRules/', variables('receiveRuleName'))]"
               ]
             }
           ]
@@ -123,7 +124,7 @@
         "value": "[listkeys(variables('sendAuthRuleResourceId'), variables('eventHubApiVersion')).primaryConnectionString]"
       },
       "dependsOn": [
-        "[concat('Microsoft.EventHub/namespaces/', variables('namespaceName'))]"
+        "[concat('Microsoft.EventHub/namespaces/', variables('namespaceName'),'/EventHubs/',variables('eventHubName'), '/authorizationRules/', variables('sendRuleName'))]"
       ]
     },
     {
@@ -135,7 +136,7 @@
         "value": "[listkeys(variables('receiveAuthRuleResourceId'), variables('eventHubApiVersion')).primaryConnectionString]"
       },
       "dependsOn": [
-        "[concat('Microsoft.EventHub/namespaces/', variables('namespaceName'))]"
+        "[concat('Microsoft.EventHub/namespaces/', variables('namespaceName'),'/EventHubs/',variables('eventHubName'), '/authorizationRules/', variables('receiveRuleName'))]"
       ]
     }
   ],

DeviceCache.Processor/DeviceCache.Processor.csproj

@@ -10,7 +10,7 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Azure.EventHubs.Processor" Version="1.0.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="1.1.2" />
-    <PackageReference Include="ServiceStack.Redis.Core" Version="1.0.43" />
+    <PackageReference Include="StackExchange.Redis" Version="1.2.6" />
   </ItemGroup>
 
   <ItemGroup>

DeviceCache.Processor/Program.cs

@@ -48,7 +48,8 @@ private static async Task MainAsync(string[] args)
 
             // Registers the Event Processor Host and starts receiving messages
             var eventProcessorOptions = EventProcessorOptions.DefaultOptions;
-            eventProcessorOptions.MaxBatchSize = 2000;
+            eventProcessorOptions.MaxBatchSize = 1000;
+            eventProcessorOptions.PrefetchCount = 1000;
             await eventProcessorHost.RegisterEventProcessorAsync<SimpleEventProcessor>(eventProcessorOptions);
             await Task.Delay(Timeout.Infinite);
         }

DeviceCache.Processor/SimpleEventProcessor.cs

@@ -1,18 +1,20 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
+using System.Net;
 using System.Text;
 using System.Threading.Tasks;
 using Microsoft.Azure.EventHubs;
 using Microsoft.Azure.EventHubs.Processor;
 using Newtonsoft.Json;
-using ServiceStack.Redis;
+using StackExchange.Redis;
 
 namespace DeviceCache.Processor
 {
     public class SimpleEventProcessor : IEventProcessor
     { 
         // very ugly code but gets the job done in proof of concept case
-        private static readonly RedisManagerPool Pool = new RedisManagerPool("devicecache-cache:6379");
+        private static ConnectionMultiplexer _multiplexer;
 
         public Task CloseAsync(PartitionContext context, CloseReason reason)
         {
@@ -22,7 +24,17 @@ public Task CloseAsync(PartitionContext context, CloseReason reason)
 
         public Task OpenAsync(PartitionContext context)
         {
-            Console.WriteLine($"SimpleEventProcessor initialized. Partition: '{context.PartitionId}'");
+            ConfigurationOptions config = ConfigurationOptions.Parse("devicecache-cache:6379");
+
+            DnsEndPoint addressEndpoint = config.EndPoints.First() as DnsEndPoint;
+            int port = addressEndpoint.Port;
+
+            IPHostEntry ip = Dns.GetHostEntryAsync(addressEndpoint.Host).Result;
+            config.EndPoints.Remove(addressEndpoint);
+            config.EndPoints.Add(ip.AddressList.First(), port);
+            
+            _multiplexer = ConnectionMultiplexer.Connect(config);
+            Console.WriteLine($"SimpleEventProcessor initialized. Redis connected. Partition: '{context.PartitionId}'");
             return Task.CompletedTask;
         }
 
@@ -34,7 +46,7 @@ public Task ProcessErrorAsync(PartitionContext context, Exception error)
 
         public Task ProcessEventsAsync(PartitionContext context, IEnumerable<EventData> messages)
         {
-            Dictionary<string, string> map = new Dictionary<string, string>();
+            Dictionary<RedisKey, RedisValue> map = new Dictionary<RedisKey, RedisValue>();
 
             foreach (var eventData in messages)
             {
@@ -43,11 +55,9 @@ public Task ProcessEventsAsync(PartitionContext context, IEnumerable<EventData>
                 map[message.Key] = message.Data;
             }
 
-            using (var client = Pool.GetClient())
-            {
-                client.SetAll(map);
-            }
-
+            var client = _multiplexer.GetDatabase();
+            client.StringSet(map.ToArray());
+            
             Console.WriteLine($"Processed { map.Count } at: { DateTime.Now }, Partition: '{context.PartitionId}'");
 
             return context.CheckpointAsync();

devicecache/charts/devicecache-processor/values.yaml

@@ -1 +1 @@
-replicas: 3
+replicas: 10

docs/cloud-prerequisites.md

@@ -1,6 +1,6 @@
 # Prerequisites for cloud cluster deployment
 
-In case you will be also deploying a remote cluster with ACS (Create-Infrastructure.ps1 script without the -SkipCluster flag), you will need to setup some prerequisites for such deployment:
+In case you will be also deploying a remote cluster with ACS (Create-Infrastructure.ps1 script without the -SkipClusterInCloud flag), you will need to setup some prerequisites for such deployment:
 
 - Subscription admin service principal credentials (**you need to provide this**)
 - Public SSH key (**you need to provide this**)
@@ -21,7 +21,20 @@ You could also use some of the tools like **Bitvise** which are capable of creat
 
 ## Running the Create-CloudClusterPrerequisites.ps1 script
 
-First, execute this line to securely enter your credentials for the service principal:
+First, make sure you log in to your Azure Subscription with
+
+```powershell
+Login-AzureRmAccount
+```
+
+and, if necessary, switch to the correct subscription using
+
+```powershell
+# to find out the subscirption id, run Get-AzureRmSubscription
+Select-AzureRmSubscription
+```
+
+Execute this line to securely enter your credentials for the service principal:
 
 ```powershell
 $servicePrincipalCredentials = Get-Credential

docs/setup.md

@@ -56,10 +56,10 @@ Select-AzureRmSubscription
 You can execute the `Create-Infrastructure.ps1` script now. In case if you will be working with a local cluster (e.g. minikube), run the following:
 
 ```powershell
-.\Create-Infrastructure.ps1 -EnvironmentTag <<set_tag_here>> -SkipCluster
+.\Create-Infrastructure.ps1 -EnvironmentTag <<set_tag_here>> -SkipClusterInCloud
 ```
 
-If you are going to be using a cloud cluster (ACS), omit the -SkipCluster flag.
+If you are going to be using a cloud cluster (ACS), omit the -SkipClusterInCloud flag.
 
 ```powershell
 # make sure you execute the Create-CloudClusterPrerequisites.ps1 script first!