Dapper, Ef core, Freesql 插入大量數據性能比較（一）

時間 2021-04-14

標籤 html sql 數據庫 app 性能 spa pwa code server 欄目系統性能简体版

原文原文鏈接

需求：導入9999行數據時Dapper, Ef core, Freesql 誰的性能更優，是如何執行的，級聯增長誰性能更佳。

確認方法：sql server 的 sys.dm_exec_query_stats

SELECT TOP 1000 (select [text] from sys.dm_exec_sql_text(QS.sql_handle)) as '數據庫語句',
    QS.execution_count AS '執行次數',
    QS.total_elapsed_time AS '耗時',
    QS.total_logical_reads AS '邏輯讀取次數',
    QS.total_logical_writes AS '邏輯寫入次數',
    QS.total_physical_reads AS '物理讀取次數',       
    QS.creation_time AS '執行時間',
    *
FROM sys.dm_exec_query_stats QS
WHERE  QS.creation_time > '2021-04-11 09:42:30'

準備：建立表

CREATE TABLE [dbo].[TestAddSortByXXXX](
    [Id] [int] IDENTITY(1,1) NOT NULL,
    [No] [int] NULL,
    [Col1] [nvarchar](50) NULL,
    [Col2] [nvarchar](50) NULL,
    [Col3] [nvarchar](50) NULL,
    [Col4] [nvarchar](50) NULL,
    [Col5] [nvarchar](50) NULL,
    [Col6] [nvarchar](50) NULL,
    [Col7] [nvarchar](50) NULL,
    [Col8] [nvarchar](50) NULL,
    [Col9] [nvarchar](50) NULL,
    [Col10] [nvarchar](50) NULL,
 CONSTRAINT [PK_TestAddSortByXXXX] PRIMARY KEY CLUSTERED 
(
    [Id] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
) ON [PRIMARY]
GO
CREATE TABLE [dbo].[TestAddSortByXXXXSub](
    [Id] [int] IDENTITY(1,1) NOT NULL,
    [Id2] [int] NULL,
    [Col1] [nvarchar](50) NULL,
    [Col2] [nvarchar](50) NULL,
    [Col3] [nvarchar](50) NULL,
    [Col4] [nvarchar](50) NULL,
    [Col5] [nvarchar](50) NULL,
    [Col6] [nvarchar](50) NULL,
    [Col7] [nvarchar](50) NULL,
    [Col8] [nvarchar](50) NULL,
    [Col9] [nvarchar](50) NULL,
    [Col10] [nvarchar](50) NULL,
 CONSTRAINT [PK_TestAddSortByXXXXSub] PRIMARY KEY CLUSTERED 
(
    [Id] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
) ON [PRIMARY]

構建9999行數據html

List<Entity> datas = new List<Entity>();
for (int i = 0; i < 9999; i++)
{
　　var item = new Entity
　　{
　　　　No = i + 1,
　　　　Col1 = Guid.NewGuid().ToString("N"),
　　　　Col2 = Guid.NewGuid().ToString("N"),
　　　　Col3 = Guid.NewGuid().ToString("N"),
　　　　Col4 = Guid.NewGuid().ToString("N"),
　　　　Col5 = Guid.NewGuid().ToString("N"),
　　　　Col6 = Guid.NewGuid().ToString("N"),
　　　　Col7 = Guid.NewGuid().ToString("N"),
　　　　Col8 = Guid.NewGuid().ToString("N"),
　　　　Col9 = Guid.NewGuid().ToString("N"),
　　　　Col10 = Guid.NewGuid().ToString("N"),
　　};
　　datas.Add(item);
}

Dapper：

static void AddDataByDapper(List<Entity> datas)
{
    int r = 0;
    Stopwatch sw = new Stopwatch();
    sw.Start();
    using (var conn = new SqlConnection(connString))
    {
        conn.Open();
        string sql = "insert into TestAddSortByDapper([No], Col1, Col2, Col3, Col4, Col5, Col6, Col7, Col8, Col9, Col10) values(@No, @Col1, @Col2, @Col3, @Col4, @Col5, @Col6, @Col7, @Col8, @Col9, @Col10);";
        r = conn.Execute(sql, datas);
    }
    sw.Stop();
    Console.WriteLine($"經過 Dapper 導入數據{r}行 毫時{sw.ElapsedMilliseconds}");
}

執行結果總結sql

-- 數據庫實際執行數據
(@Col1 nvarchar(4000),@Col10 nvarchar(4000),...)
insert into TestAddSortByDapper([No], Col1, Col2, Col3, Col4, Col5, Col6, Col7, Col8, Col9, Col10) 
values(@No, @Col1, @Col2, @Col3, @Col4, @Col5, @Col6, @Col7, @Col8, @Col9, @Col10);

從結果咱們能夠看到，dapper使用的是 insert into table () values () 方式循環執行9999次，代碼總耗時3-4秒。數據庫

EfCore：

static void AddDataByEfCore(List<Entity> datas)
{
    int r1 = 0;
    Stopwatch sw = new Stopwatch();
    sw.Start();
    using (var db = new TestContext())
    {
        db.Entity.AddRange(datas);
        r1 = db.SaveChanges();
    }
    sw.Stop();
    Console.WriteLine($"經過 EfCore 導入數據{r1}行 毫時{sw.ElapsedMilliseconds}");
}
[Table("TestAddSortByEfCore")]
public class Entity
{
    public int Id { get; set; }
    public int No { get; set; }
    public string Col1 { get; set; }
    public string Col2 { get; set; }
    public string Col3 { get; set; }
    public string Col4 { get; set; }
    public string Col5 { get; set; }
    public string Col6 { get; set; }
    public string Col7 { get; set; }
    public string Col8 { get; set; }
    public string Col9 { get; set; }
    public string Col10 { get; set; }
}

執行結果總結app

(@p0 nvarchar(4000),@p1 nvarchar(4000),...,@p460 nvarchar(4000),@p461 int)
SET NOCOUNT ON;  
DECLARE @inserted0 TABLE ([Id] int, [_Position] [int]);  
MERGE [TestAddSortByEfCore] USING (  
    VALUES (@p0, @p1, @p2, @p3, @p4, @p5, @p6, @p7, @p8, @p9, @p10, 0),..., (@p451, @p452, @p453, @p454, @p455, @p456, @p457, @p458, @p459, @p460, @p461, 41)
) AS i ([Col1], [Col10], [Col2], [Col3], [Col4], [Col5], [Col6], [Col7], [Col8], [Col9], [No], _Position) ON 1=0  
WHEN NOT MATCHED THEN  
    INSERT ([Col1], [Col10], [Col2], [Col3], [Col4], [Col5], [Col6], [Col7], [Col8], [Col9], [No])  
    VALUES (i.[Col1], i.[Col10], i.[Col2], i.[Col3], i.[Col4], i.[Col5], i.[Col6], i.[Col7], i.[Col8], i.[Col9], i.[No])  
    OUTPUT INSERTED.[Id], i._Position  INTO @inserted0;    
SELECT [t].[Id] FROM [TestAddSortByEfCore] t  INNER JOIN @inserted0 i ON ([t].[Id] = [i].[Id]) ORDER BY [i].[_Position];

從結果咱們能夠看到，EfCore使用的是 Merge 方式增長數據，但數據庫變量最多定義462個，因此每次只能增長42行數據，執行了238+3次，但最大的疑問是執行了兩次，並且插入表數據順序錯了（估計是EfCore代碼上使用了Parallel.For方法，有懂的朋友可否解答一下），代碼總耗時4-5秒。性能

Freesql：

static void AddDataByEfCore(List<Entity> datas)
{
    int r1 = 0;
    Stopwatch sw = new Stopwatch();
    sw.Start();
    using (var db = new TestContext())
    {
        db.Entity.AddRange(datas);
        r1 = db.SaveChanges();
    }
    sw.Stop();
    Console.WriteLine($"經過 EfCore 導入數據{r1}行 毫時{sw.ElapsedMilliseconds}");
}
[FreeSql.DataAnnotations.Table(Name = "TestAddSortByFreesql", DisableSyncStructure = true)]
public class Entity
{
    [FreeSql.DataAnnotations.Column(Name = "id", IsPrimary = true, IsIdentity = true)]
    public int Id { get; set; }
    public int No { get; set; }
    public string Col1 { get; set; }
    public string Col2 { get; set; }
    public string Col3 { get; set; }
    public string Col4 { get; set; }
    public string Col5 { get; set; }
    public string Col6 { get; set; }
    public string Col7 { get; set; }
    public string Col8 { get; set; }
    public string Col9 { get; set; }
    public string Col10 { get; set; }
}

執行結果總結ui

(@No_0 int,@Col1_0 nvarchar(32),...,@Col10_173 nvarchar(32))
INSERT INTO [TestAddSortByFreesql]([No], [Col1], [Col2], [Col3], [Col4], [Col5], [Col6], [Col7], [Col8], [Col9], [Col10]) 
VALUES(@No_0, @Col1_0, @Col2_0, @Col3_0, @Col4_0, @Col5_0, @Col6_0, @Col7_0, @Col8_0, @Col9_0, @Col10_0), ..., (@No_173, @Col1_173, @Col2_173, @Col3_173, @Col4_173, @Col5_173, @Col6_173, @Col7_173, @Col8_173, @Col9_173, @Col10_173)

從結果咱們能夠看到，freesql使用的是 insert into table () values (), () 方式循環執行，每次最多增長173行數據，代碼總耗時7-8秒。spa

從目前結果來看，單表增長大量數據，時間上 Dapper > EfCore > Freesql。pwa

ADO.NET SqlBulkCopy 複製（最優方案）

static void AddDataByBulkCopy(List<Entity> datas)
{
    Stopwatch sw = new Stopwatch();
    var dt = new DataTable();
    dt.Columns.Add("No", typeof(int));
    dt.Columns.Add("Col1", typeof(string));
    dt.Columns.Add("Col2", typeof(string));
    dt.Columns.Add("Col3", typeof(string));
    dt.Columns.Add("Col4", typeof(string));
    dt.Columns.Add("Col5", typeof(string));
    dt.Columns.Add("Col6", typeof(string));
    dt.Columns.Add("Col7", typeof(string));
    dt.Columns.Add("Col8", typeof(string));
    dt.Columns.Add("Col9", typeof(string));
    dt.Columns.Add("Col10", typeof(string));
    foreach (var item in datas) 
    {
        var dr = dt.NewRow();
        dr["No"] = item.No;
        dr["Col1"] = item.Col1;
        dr["Col2"] = item.Col2;
        dr["Col3"] = item.Col3;
        dr["Col4"] = item.Col4;
        dr["Col5"] = item.Col5;
        dr["Col6"] = item.Col6;
        dr["Col7"] = item.Col7;
        dr["Col8"] = item.Col8;
        dr["Col9"] = item.Col9;
        dr["Col10"] = item.Col10;
        dt.Rows.Add(dr);
    }
    sw.Start();
    using (SqlConnection cn = new SqlConnection(connString))
    {
        cn.Open();
        using (SqlBulkCopy sqlBulkCopy = new SqlBulkCopy(cn))
        {
            sqlBulkCopy.BatchSize = dt.Rows.Count;
            sqlBulkCopy.BulkCopyTimeout = 1800;
            sqlBulkCopy.DestinationTableName = "TestAddSortByBulkCopy";

            sqlBulkCopy.ColumnMappings.Add("No", "No");
            sqlBulkCopy.ColumnMappings.Add("Col1", "Col1");
            sqlBulkCopy.ColumnMappings.Add("Col2", "Col2");
            sqlBulkCopy.ColumnMappings.Add("Col3", "Col3");
            sqlBulkCopy.ColumnMappings.Add("Col4", "Col4");
            sqlBulkCopy.ColumnMappings.Add("Col5", "Col5");
            sqlBulkCopy.ColumnMappings.Add("Col6", "Col6");
            sqlBulkCopy.ColumnMappings.Add("Col7", "Col7");
            sqlBulkCopy.ColumnMappings.Add("Col8", "Col8");
            sqlBulkCopy.ColumnMappings.Add("Col9", "Col9");
            sqlBulkCopy.ColumnMappings.Add("Col10", "Col10");
            sqlBulkCopy.WriteToServer(dt);
        }
    }
    sw.Stop();
    Console.WriteLine($"經過 BulkCopy 毫時{sw.ElapsedMilliseconds}");
}