+ Bulk Update

Cesar Romero Silva · Cesar Romero Silva · commit 3ccb3cd080e1 · 2018-09-05T13:07:42.000-03:00
diff --git a/App.config b/App.config
@@ -1,6 +1,6 @@
 ﻿<?xml version="1.0" encoding="utf-8" ?>
 <configuration>
-    <startup> 
-        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.2" />
-    </startup>
+  <startup>
+    <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.2" />
+  </startup>
 </configuration>
diff --git a/BulkOperation.cs b/BulkOperation.cs
@@ -5,69 +5,85 @@
 
 namespace BulkOperations
 {
-    /* TODO 
-     * https://docs.microsoft.com/en-us/dotnet/api/system.data.sqlclient.sqlbulkcopy?redirectedfrom=MSDN&view=netframework-4.7.2
-     * https://docs.microsoft.com/en-us/dotnet/api/system.data.sqlclient.sqlbulkcopy.writetoserver?view=netframework-4.7.2&viewFallbackFrom=netcore-2.1
-     * https://docs.microsoft.com/pt-br/dotnet/framework/data/adonet/sql/single-bulk-copy-operations
-     * https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/sql/bulk-copy-example-setup?view=netframework-4.7.2
-     * https://docs.microsoft.com/en-us/dotnet/api/system.data.sqlclient.sqlbulkcopy?f1url=https%3A%2F%2Fmsdn.microsoft.com%2Fquery%2Fdev15.query%3FappId%3DDev15IDEF1%26l%3DEN-US%26k%3Dk(System.Data.SqlClient.SqlBulkCopy);k(SolutionItemsProject);k(TargetFrameworkMoniker-.NETFramework,Version%3Dv4.7);k(DevLang-csharp)%26rd%3Dtrue&view=netframework-4.7.2
-     * https://github.com/mgravell/fast-member
-     *
-     * Update if exist https://stackoverflow.com/questions/12521692/c-sharp-bulk-insert-sqlbulkcopy-update-if-exists/12535726
-     * https://stackoverflow.com/questions/33027246/insert-object-or-update-if-it-already-exists-using-bulkcopy-c-sql
-     * https://www.databasejournal.com/features/mssql/article.php/3739131/UPSERT-Functionality-in-SQL-Server-2008.htm
-     * https://stackoverflow.com/questions/4889123/any-way-to-sqlbulkcopy-insert-or-update-if-exists
-     */
-
     public static class BulkOperation
     {
-        private const string ConnectionString = "Data Source=NT-03087;Initial Catalog=Insumos;Integrated Security=False;User ID=sa;Password=sa;Connect Timeout=15;Encrypt=False;TrustServerCertificate=True;ApplicationIntent=ReadWrite;MultiSubnetFailover=False;MultipleActiveResultSets=true;";
-
-
         public static async Task InsertAsync(CancellationToken cancellationToken = default(CancellationToken))
         {
             using (var connection = new SqlConnection())
             {
-                connection.ConnectionString = ConnectionString;
+                connection.Configure();
                 await connection.OpenAsync(cancellationToken).ConfigureAwait(true);
 
-                using (var bulk = new SqlBulkCopy(connection, SqlBulkCopyOptions.Default, null))
-                {
-                    var customers = Customer.Generate(1_000_000);
+                const int recordsToGenerate = 100_000;
+                Log.Write($"Gerando {recordsToGenerate:n0} registros clientes");
+                var customers = Customer.Generate(recordsToGenerate);
 
+                Log.Write("Inserindo registros no banco de dados");
+                using (var insertBulk = new SqlBulkCopy(connection, SqlBulkCopyOptions.Default, null))
+                {
                     using (var customerReader = new ObjectDataReader<Customer>(customers.GetEnumerator()))
                     {
-                        bulk.MapEntity();
-                        bulk.ConfigureOptions();
-                        bulk.ConfigureLog();
+                        insertBulk.Configure("Customer");
+
+                        await insertBulk.WriteToServerAsync(customerReader, cancellationToken).ConfigureAwait(false);
+                    }
+                }
 
-                        await bulk.WriteToServerAsync(customerReader, cancellationToken).ConfigureAwait(true);
+                using (var command = new SqlCommand("", connection))
+                {
+                    try
+                    {
+                        Log.Write("Criando tabela temporária e copiando registros");
+                        command.CommandText = BulkUpdate.CreateTempTable;
+                        command.ExecuteNonQuery();
+                        try
+                        {
+                            Log.Write("Atualizando registros: Tabela temporária -> Customers");
+                            command.CommandTimeout = 300;
+                            command.CommandText = BulkUpdate.UpdateTable;
+                            command.ExecuteNonQuery();
+                        }
+                        finally
+                        {
+                            Log.Write("Excluindo tabela temporária");
+                            command.CommandText = BulkUpdate.DropTempTable;
+                            command.ExecuteNonQuery();
+                        }
+                    }
+                    catch (Exception e)
+                    {
+                        Log.Write(e.Message);
                     }
                 }
             }
         }
 
-        private static void MapEntity(this SqlBulkCopy bulk)
+        private static void Configure(this SqlBulkCopy bulk, string tableName)
         {
-            // https://stackoverflow.com/questions/40470357/importing-a-csv-using-sqlbulkcopy-with-asp-net-core
-            bulk.DestinationTableName = "Customer";
-            bulk.ColumnMappings.Add(nameof(Customer.Id), "Id"  );
-            bulk.ColumnMappings.Add(nameof(Customer.FirstName  ), "FirstName");
-            bulk.ColumnMappings.Add(nameof(Customer.LastName   ), "LastName");
+            bulk.MapEntity(tableName);
+            bulk.ConfigureOptions();
+            bulk.ConfigureLog();
+        }
+
+        private static void MapEntity(this SqlBulkCopy bulk, string tableName)
+        {
+            bulk.DestinationTableName = tableName;
+            bulk.ColumnMappings.Add(nameof(Customer.Id), "Id");
+            bulk.ColumnMappings.Add(nameof(Customer.FirstName), "FirstName");
+            bulk.ColumnMappings.Add(nameof(Customer.LastName), "LastName");
             bulk.ColumnMappings.Add(nameof(Customer.DateOfBirth), "DateOfBirth");
         }
 
         private static void ConfigureOptions(this SqlBulkCopy bulk)
         {
             bulk.EnableStreaming = true;
-            bulk.BatchSize       = 10_000;
-            bulk.NotifyAfter     =  1_000;
+            bulk.BatchSize = 10_000;
+            bulk.NotifyAfter = 1_000;
         }
 
         private static void ConfigureLog(this SqlBulkCopy bulk)
         {
-            bulk.SqlRowsCopied += (sender, e) => Console.WriteLine($"{DateTime.Now} RowsCopied: {e.RowsCopied:n0}");
+            bulk.SqlRowsCopied += (sender, e) => Log.Write($"{DateTime.Now} RowsCopied: {e.RowsCopied:n0}");
         }
-
     }
-}
+}
diff --git a/BulkOperations.csproj b/BulkOperations.csproj
@@ -34,6 +34,7 @@
   </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />
+    <Reference Include="System.Configuration" />
     <Reference Include="System.Core" />
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />
@@ -44,11 +45,16 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="BulkOperation.cs" />
+    <Compile Include="BulkUpdate.cs" />
+    <Compile Include="Log.cs" />
     <Compile Include="Customer.cs" />
+    <Compile Include="DataTableExtension.cs" />
     <Compile Include="ObjectDataReader.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="SqlConnectionExtension.cs" />
     <Compile Include="StopwatchExtension.cs" />
+    <Compile Include="Todo.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="App.config" />
diff --git a/BulkUpdate.cs b/BulkUpdate.cs
@@ -0,0 +1,75 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Configuration;
+using System.Data;
+using System.Data.SqlClient;
+
+namespace BulkOperations
+{
+    public static class BulkUpdate
+    {
+        public const string TempTableName = "#Customer";
+        public const string TableName = "Customer";
+
+        public const string CreateTempTable =
+            "select * into #Customer from Customer";
+
+        public const string UpdateTable =
+            @"
+                update Customer
+                set
+                    Customer.FirstName   = #Customer.FirstName,
+                    Customer.LastName    = #Customer.LastName,
+                    Customer.DateOfBirth = #Customer.CreatedAt,
+                    Customer.CreatedAt   = #Customer.DateOfBirth
+                from #Customer
+            ";
+
+        public const string DropTempTable =
+            "drop table #Customer;";
+
+        public static void UpdateData<T>(List<T> list)
+        {
+            var dataTable = new DataTable(TempTableName);
+            dataTable.FromList(list);
+
+            ConfigurationManager.OpenExeConfiguration(ConfigurationUserLevel.PerUserRoamingAndLocal);
+
+            using (var connection = new SqlConnection())
+            {
+                connection.Configure();
+
+                using (var command = new SqlCommand("", connection))
+                {
+                    try
+                    {
+                        connection.Open();
+
+                        command.CommandText = CreateTempTable;
+                        command.ExecuteNonQuery();
+
+                        using (var bulkcopy = new SqlBulkCopy(connection))
+                        {
+                            bulkcopy.BulkCopyTimeout = 660;
+                            bulkcopy.DestinationTableName = TableName;
+                            bulkcopy.WriteToServer(dataTable);
+                            bulkcopy.Close();
+                        }
+
+                        command.CommandTimeout = 300;
+                        command.CommandText = $"{UpdateTable}{DropTempTable}";
+                        command.ExecuteNonQuery();
+                    }
+                    catch (Exception e)
+                    {
+                        Console.WriteLine(e.Message);
+                    }
+                    finally
+                    {
+                        connection.Close();
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/Customer.cs b/Customer.cs
@@ -5,10 +5,10 @@
     use Insumos;
 
     create table Customer (
-	    Id uniqueidentifier primary key,
-	    FirstName varchar(100),
-	    LastName varchar(100),
-	    DateOfBirth date,
+        Id uniqueidentifier primary key,
+        FirstName varchar(100),
+        LastName varchar(100),
+        DateOfBirth date,
         CreatedAt  datetime2 default getdate()
     );
 
@@ -39,4 +39,4 @@ public static IEnumerable<Customer> Generate(int count)
             }
         }
     }
-}
+}
diff --git a/DataTableExtension.cs b/DataTableExtension.cs
@@ -0,0 +1,34 @@
+﻿using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Data;
+
+namespace BulkOperations
+{
+    public static class DataTableExtension
+    {
+        public static DataTable FromList<T>(this DataTable dataTable, IList<T> list)
+        {
+            var properties = TypeDescriptor.GetProperties(typeof(T));
+
+            foreach (PropertyDescriptor prop in properties)
+            {
+                dataTable.Columns.Add(prop.Name, Nullable.GetUnderlyingType(prop.PropertyType) ?? prop.PropertyType);
+            }
+
+            foreach (var item in list)
+            {
+                var row = dataTable.NewRow();
+
+                foreach (PropertyDescriptor prop in properties)
+                {
+                    row[prop.Name] = prop.GetValue(item) ?? DBNull.Value;
+                }
+
+                dataTable.Rows.Add(row);
+            }
+
+            return dataTable;
+        }
+    }
+}
diff --git a/Log.cs b/Log.cs
@@ -0,0 +1,12 @@
+﻿using System;
+
+namespace BulkOperations
+{
+    public static class Log
+    {
+        public static void Write(string value)
+        {
+            Console.WriteLine($"{DateTime.Now} {value}");
+        }
+    }
+}
diff --git a/ObjectDataReader.cs b/ObjectDataReader.cs
@@ -41,7 +41,7 @@ private void Initialize()
                 var parameterExpression = Expression.Parameter(typeof(T), "x");
                 var func = (Func<T, object>)Expression.Lambda(
                     Expression.Convert(
-                        Expression.Property(parameterExpression, propertyName), 
+                        Expression.Property(parameterExpression, propertyName),
                         typeof(object)),
                         parameterExpression)
                     .Compile();
@@ -74,7 +74,6 @@ public override object GetValue(int ordinal)
             return func(_iterator.Current);
         }
 
-
         // optional
         public override int GetValues(object[] values)
         {
diff --git a/Program.cs b/Program.cs
@@ -3,24 +3,25 @@
 
 namespace BulkOperations
 {
-    static class Program
+    internal static class Program
     {
-        static void Main(string[] args)
+        private static void Main(string[] args)
         {
             var stopwatch = new Stopwatch();
             stopwatch.Start();
 
-            Console.WriteLine($"{DateTime.Now} Início processamento");
+            Log.Write("Início processamento");
 
             BulkOperation.InsertAsync().Wait();
 
             stopwatch.Stop();
 
-            Console.WriteLine($"{DateTime.Now} Processamento concluído em {stopwatch.ElapsedTimeFmt()}");
+
+            Log.Write($"Processamento concluído em {stopwatch.ElapsedTimeFmt()}");
 
             Console.WriteLine("");
             Console.WriteLine("Tecle <ENTER> para finalizar...");
             Console.ReadLine();
         }
     }
-}
+}
diff --git a/Properties/AssemblyInfo.cs b/Properties/AssemblyInfo.cs
@@ -1,5 +1,4 @@
 ﻿using System.Reflection;
-using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
 // General Information about an assembly is controlled through the following
@@ -33,4 +32,4 @@
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
 [assembly: AssemblyVersion("1.0.0.0")]
-[assembly: AssemblyFileVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
diff --git a/SqlConnectionExtension.cs b/SqlConnectionExtension.cs
@@ -0,0 +1,15 @@
+﻿using System.Data.SqlClient;
+
+namespace BulkOperations
+{
+    public static class SqlConnectionExtension
+    {
+        // TODO : load from configuration file
+        private const string ConnectionString = "Data Source=NT-03087;Initial Catalog=Insumos;Integrated Security=False;User ID=sa;Password=sa;Connect Timeout=15;Encrypt=False;TrustServerCertificate=True;ApplicationIntent=ReadWrite;MultiSubnetFailover=False;MultipleActiveResultSets=true;";
+
+        public static void Configure(this SqlConnection connection)
+        {
+            connection.ConnectionString = ConnectionString;
+        }
+    }
+}
diff --git a/StopwatchExtension.cs b/StopwatchExtension.cs
@@ -10,4 +10,4 @@ public static string ElapsedTimeFmt(this Stopwatch stopwatch)
             return $"{elapsed.Hours:00}:{elapsed.Minutes:00}:{elapsed.Seconds:00}.{elapsed.Milliseconds / 10:00}";
         }
     }
-}
+}
diff --git a/Todo.cs b/Todo.cs

Original file line number	Diff line number	Diff line change
`@@ -5,10 +5,10 @@`
`5`	`5`	`use Insumos;`
`6`	`6`
`7`	`7`	`create table Customer (`
`8`		`- Id uniqueidentifier primary key,`
`9`		`- FirstName varchar(100),`
`10`		`- LastName varchar(100),`
`11`		`- DateOfBirth date,`
	`8`	`+ Id uniqueidentifier primary key,`
	`9`	`+ FirstName varchar(100),`
	`10`	`+ LastName varchar(100),`
	`11`	`+ DateOfBirth date,`
`12`	`12`	`CreatedAt datetime2 default getdate()`
`13`	`13`	`);`
`14`	`14`
`@@ -39,4 +39,4 @@ public static IEnumerable<Customer> Generate(int count)`
`39`	`39`	`}`
`40`	`40`	`}`
`41`	`41`	`}`
`42`		`-}`
	`42`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -3,24 +3,25 @@`
`3`	`3`
`4`	`4`	`namespace BulkOperations`
`5`	`5`	`{`
`6`		`- static class Program`
	`6`	`+ internal static class Program`
`7`	`7`	`{`
`8`		`- static void Main(string[] args)`
	`8`	`+ private static void Main(string[] args)`
`9`	`9`	`{`
`10`	`10`	`var stopwatch = new Stopwatch();`
`11`	`11`	`stopwatch.Start();`
`12`	`12`
`13`		`- Console.WriteLine($"{DateTime.Now} Início processamento");`
	`13`	`+ Log.Write("Início processamento");`
`14`	`14`
`15`	`15`	`BulkOperation.InsertAsync().Wait();`
`16`	`16`
`17`	`17`	`stopwatch.Stop();`
`18`	`18`
`19`		`- Console.WriteLine($"{DateTime.Now} Processamento concluído em {stopwatch.ElapsedTimeFmt()}");`
	`19`	`+`
	`20`	`+ Log.Write($"Processamento concluído em {stopwatch.ElapsedTimeFmt()}");`
`20`	`21`
`21`	`22`	`Console.WriteLine("");`
`22`	`23`	`Console.WriteLine("Tecle <ENTER> para finalizar...");`
`23`	`24`	`Console.ReadLine();`
`24`	`25`	`}`
`25`	`26`	`}`
`26`		`-}`
	`27`	`+}`