.net批量上傳Csv檔資料應用程序開發總結

應用環境：visual studio 2010開發工具，Database為Sql2008以上版本

最近在生產環境中須要開發一款應用程式，上傳電子檔(.csv)資料至Database

最初方案：

以txt方式打開Csv檔案，逐行進行數據上傳處理，代碼見下文。因為須要上傳N臺機器產生的檔案，天天數據量很是龐大，並且在上傳時要進行篩選去重，並保留最新的（以測試時間為準）測試記錄，導致以上方案在執行時，當天的資料無法在當天上傳完成，嚴重影響次日的資料匯總和報表的生成。

改善方案：

因為資料太多太慢被老闆不知道K了多少次。本來想要使用多線程解決，可是考慮到線程間的安全問題，最終還是放棄額。後來思考為什麼不能把整個Csv檔案用放入Table中進行上傳，減少Database的訪問次數，並下降佔用網絡帶寬。

話很少少，先上改善前後的效果圖相差居然有12秒之多.效率提高43倍之多。

   

詳細代碼以下：

獲取檔案並逐行讀取資料代碼

        private void Readlog()

        {

            state = false;

            foreach (string a in Directory.GetFiles(logfile, "*.csv"))

            {                

 

               if (GetDataSet.FileCopy(a, logbak + "\\" + a.Substring(a.LastIndexOf("\\") + 1)) == true)

               {

                   string[] read = File.ReadAllLines(a, Encoding.GetEncoding("GB18030"));

                   for (int i = 1; i < read.Length; i++)

                   {

                       if (DataSave(a.Substring(a.LastIndexOf("\\") + 1), read[i].ToString()) == false)

                       {

                           ErrorWrite(a.Substring(a.LastIndexOf("\\") + 1) + ":LINE(" + i + ")Write Error");

                           continue;

                       }

                   }

                   GetDataSet.DeleteFile(a);

               }

           }

        }

上傳資料代碼

        private bool DataSave(string a, string sr)

        {

                try

                {

                    f = false;

                    SqlConnection con = new SqlConnection();

                    SqlCommand com = new SqlCommand();

                    string sql;

                    con.ConnectionString = "#####";

                    //ArrayList Panel = new ArrayList();

                    string[] Panel;

                    Panel = sr.ToString().Split(',');

                    com.CommandType = CommandType.Text;

                    sql = "Insert Avi_Data " +

                                      "([CreateMachine No],[Machie Model],[Model no],[Panel Barcode],[MCH],[Board ID],[Pin NO],[Bad board result],[Pitch FINGER]" +

                                      ",[Pitch FINGER(Spec)],[Width FINGER],[Width FINGER (Spec)],[FINGER to Board EDGE SMALL],[FINGER to Board EDGE SMALL(Spec)],[FINGER to Board EDGE LARGE],[FINGER to Board EDGE LARGE(Spec)]" +

                                      ",[PCB Width_Finger Area],[PCB Width_Finger Airea (Spec)],[CreateDatetime],[FileName]) values(";

                    for (int i = 0; i < Panel.Length; i++)

                    {

                        sql = sql + "'" + Strings.StrConv(Panel[i].ToString().Trim(), VbStrConv.TraditionalChinese, 0) + "',";

                    }

                    sql = sql + "Getdate(),'" + a + "')";

                    com.CommandText = sql;

                    com.Connection = con;

                    con.Open();

                    com.ExecuteNonQuery();

                    con.Close();

                    AddItem(Panel[3].ToString() + ":" + Panel[5].ToString()+"測試資料已上傳！");

                    return true;

                }

                catch(Exception ex)

                {

                    ErrorWrite(ex.ToString());

                    return false;

                }

            }                

改善後方案實施步驟：

一、 在DataBase中創建table變量。

CREATE TYPE [dbo].[MultiRowsInsert] AS TABLE

（

）

二、 創建對應的數據保存的存儲過程。

CREATE PROCEDURE [dbo].[InsertMultiRow]

  @DataTable dbo.MultiRowsInsert readonly

as

declare @ID int,@MaxID int

set @ID=1

set @MaxID=(Select max(ID) from @DataTable)

while @ID<=@MaxID

begin

  insert Avi_Data

  select[CreateMachine No],[Machie Model],[Model no],[Panel Barcode],[MCH],[Board ID],[Pin NO],[Bad board result]

       ,[Pitch FINGER],[Pitch FINGER(Spec)],[Width FINGER],[Width FINGER (Spec)],[FINGER to Board EDGE SMALL]

       ,[FINGER to Board EDGE SMALL(Spec)],[FINGER to Board EDGE LARGE],[FINGER to Board EDGE LARGE(Spec)]

       ,[PCB Width_Finger Area],[PCB Width_Finger Airea (Spec)],GETDATE(),[FileName]

  from @DataTable

  where ID=@ID

  set @ID=@ID+1

end

三、 修改程式代碼。

如下為讀取Csv檔案代碼

private void Readlog()

        {

            state = false;

            foreach (string a in Directory.GetFiles(logfile, "*.csv"))

            {               

 

               if (GetDataSet.FileCopy(a, logbak + "\\" + a.Substring(a.LastIndexOf("\\") + 1)) == true)

               {

                   string[] read = File.ReadAllLines(a, Encoding.GetEncoding("GB18030"));

                   DataTable TableInsert = new DataTable();

                   TableInsert.Columns.Add("ID", typeof(int));

                   TableInsert.Columns.Add("CreateMachine No", typeof(string));

                   TableInsert.Columns.Add("Machie Model", typeof(string));

                   TableInsert.Columns.Add("Model no", typeof(string));

                   TableInsert.Columns.Add("Panel Barcode", typeof(string));

                   TableInsert.Columns.Add("MCH", typeof(string));

                   TableInsert.Columns.Add("Board ID", typeof(string));

                   TableInsert.Columns.Add("Pin NO", typeof(string));

                   TableInsert.Columns.Add("Bad board result", typeof(string));

                   TableInsert.Columns.Add("Pitch FINGER", typeof(string));

                   TableInsert.Columns.Add("Pitch FINGER(Spec)", typeof(string));

                   TableInsert.Columns.Add("Width FINGER", typeof(string));

                   TableInsert.Columns.Add("Width FINGER (Spec)", typeof(string));

                   TableInsert.Columns.Add("FINGER to Board EDGE SMALL", typeof(string));

                   TableInsert.Columns.Add("FINGER to Board EDGE SMALL(Spec)", typeof(string));

                   TableInsert.Columns.Add("FINGER to Board EDGE LARGE", typeof(string));

                   TableInsert.Columns.Add("FINGER to Board EDGE LARGE(Spec)", typeof(string));

                   TableInsert.Columns.Add("PCB Width_Finger Area", typeof(string));

                   TableInsert.Columns.Add("PCB Width_Finger Airea (Spec)", typeof(string));

                   TableInsert.Columns.Add("FileName", typeof(string));

                   for (int i = 1; i < read.Length; i++)

                   {

                       string[] ReadCloumn;

                       int c=1;

                       ReadCloumn=read[i].Split(',');

                       DataRow dr = TableInsert.NewRow();

                       dr[0] = i;

                       while (c <= ReadCloumn.Length)

                       {

                           dr[c] = ReadCloumn[c-1].Trim();

                           c++;

                       }

                       dr[c] = a.Substring(a.LastIndexOf("\\") + 1);

                       TableInsert.Rows.Add(dr);

                   }

                   if (SaveData(TableInsert) == false)

                   {

                       int sn = a.Split('_').Length - 2;

                       ErrorWrite(a.Split('_')[sn] + ":Write Error");

                       continue;

                   }

                   else

                   {

                       int sn=a.Split('_').Length-2;

                       AddItem( a.Split('_')[sn]+ "測試資料已上傳！");

                       GetDataSet.DeleteFile(a);

                   }

               }

           }

        }

數據保存代碼

        private bool SaveData(DataTable dt)

        {

            try

            {

                SqlConnection con = new SqlConnection();

                SqlCommand com = new SqlCommand();

                con.ConnectionString = "######";

                SqlParameter[] paras = new SqlParameter[]

                      {

                       new SqlParameter("@DataTable",dt)

                      };

               string msg= GetDataSet.RunSqlProdurce("InsertMultiRow", paras, con);

               if (msg == "ok")

                   return true;

               else

               {

                   return false;

               }

            }

            catch (Exception ex)

            {

                return false;

            }

        }

以上為完整代碼，若有更好的方案，還請各方大神不吝賜教……