JDBC 批量插入數據優化, 使用 addBatch 和 executeBatch
轉載請註明出處：http://blog.csdn.net/linglongxin24/article/details/53769957
本文出自【DylanAndroid的博客】 java
#JDBC批量插入數據優化,使用addBatch和executeBatchgit
在以前的玩轉JDBC打造數據庫操做萬能工具類JDBCUtil，加入了高效的數據庫鏈接池，利用了參數綁定有效防止SQL注入
中其實忽略了一點，那就是SQL的批量插入的問題，若是來個for循環，執行上萬次，確定會很慢，那麼，如何去優化呢？github
#一.用 preparedStatement.addBatch()配合preparedStatement.executeBatch()去批量插入sql
/** * 執行數據庫插入操做 * * @param datas 插入數據表中key爲列名和value爲列對應的值的Map對象的List集合 * @param tableName 要插入的數據庫的表名 * @return 影響的行數 * @throws SQLException SQL異常 */
public static int insertAll(String tableName, List<Map<String, Object>> datas) throws SQLException {
/**影響的行數**/
int affectRowCount = -1;
Connection connection = null;
PreparedStatement preparedStatement = null;
try {
/**從數據庫鏈接池中獲取數據庫鏈接**/
connection = DBConnectionPool.getInstance().getConnection();
Map<String, Object> valueMap = datas.get(0);
/**獲取數據庫插入的Map的鍵值對的值**/
Set<String> keySet = valueMap.keySet();
Iterator<String> iterator = keySet.iterator();
/**要插入的字段sql，其實就是用key拼起來的**/
StringBuilder columnSql = new StringBuilder();
/**要插入的字段值，其實就是？**/
StringBuilder unknownMarkSql = new StringBuilder();
Object[] keys = new Object[valueMap.size()];
int i = 0;
while (iterator.hasNext()) {
String key = iterator.next();
keys[i] = key;
columnSql.append(i == 0 ? "" : ",");
columnSql.append(key);
unknownMarkSql.append(i == 0 ? "" : ",");
unknownMarkSql.append("?");
i++;
}
/**開始拼插入的sql語句**/
StringBuilder sql = new StringBuilder();
sql.append("INSERT INTO ");
sql.append(tableName);
sql.append(" (");
sql.append(columnSql);
sql.append(" ) VALUES (");
sql.append(unknownMarkSql);
sql.append(" )");
/**執行SQL預編譯**/
preparedStatement = connection.prepareStatement(sql.toString());
/**設置不自動提交，以便於在出現異常的時候數據庫回滾**/
connection.setAutoCommit(false);
System.out.println(sql.toString());
for (int j = 0; j < datas.size(); j++) {
for (int k = 0; k < keys.length; k++) {
preparedStatement.setObject(k + 1, datas.get(j).get(keys[k]));
}
preparedStatement.addBatch();
}
int[] arr = preparedStatement.executeBatch();
connection.commit();
affectRowCount = arr.length;
System.out.println("成功了插入了" + affectRowCount + "行");
System.out.println();
} catch (Exception e) {
if (connection != null) {
connection.rollback();
}
e.printStackTrace();
throw e;
} finally {
if (preparedStatement != null) {
preparedStatement.close();
}
if (connection != null) {
connection.close();
}
}
return affectRowCount;
}複製代碼
#二.實驗論證數據庫
1.普通的插入方法一次性插入10000條數據所消耗的時間
private static void testAll1() {
long start = System.currentTimeMillis();
try {
for (int i = 0; i < 10000; i++) {
Map<String, Object> map = new HashMap<>();
map.put("emp_id", 1013);
map.put("name", "JDBCUtil測試");
map.put("job", "developer");
map.put("salary", 10000);
map.put("hire_date", new java.sql.Date(System.currentTimeMillis()));
DBUtil.insert("emp_test3", map);
}
System.out.println("共耗時" + (System.currentTimeMillis() - start));
} catch (SQLException e) {
e.printStackTrace();
}
}複製代碼
- 打印結果
共耗時44110數據結構
2.優化後的方法一次性插入10000條數據所消耗的時間
private static void testAll2() {
List<Map<String, Object>> datas = new ArrayList<>();
for (int i = 0; i < 10000; i++) {
Map<String, Object> map = new HashMap<>();
map.put("emp_id", 1013);
map.put("name", "JDBCUtil測試");
map.put("job", "developer");
map.put("salary", 10000);
map.put("hire_date", new java.sql.Date(System.currentTimeMillis()));
datas.add(map);
}
try {
long start = System.currentTimeMillis();
DBUtil.insertAll("emp_test3", datas);
System.out.println("共耗時" + (System.currentTimeMillis() - start));
} catch (SQLException e) {
e.printStackTrace();
}
}複製代碼
- 打印結果
共耗時649app
#3.DBUtil的完整代碼工具
package cn.bluemobi.dylan.util;
import com.sun.istack.internal.Nullable;
import java.sql.*;
import java.util.*;
import java.util.regex.Pattern;
/** * 數據庫JDBC鏈接工具類 * Created by yuandl on 2016-12-16. */
public class DBUtil {
/** * 執行數據庫插入操做 * * @param valueMap 插入數據表中key爲列名和value爲列對應的值的Map對象 * @param tableName 要插入的數據庫的表名 * @return 影響的行數 * @throws SQLException SQL異常 */
public static int insert(String tableName, Map<String, Object> valueMap) throws SQLException {
/**獲取數據庫插入的Map的鍵值對的值**/
Set<String> keySet = valueMap.keySet();
Iterator<String> iterator = keySet.iterator();
/**要插入的字段sql，其實就是用key拼起來的**/
StringBuilder columnSql = new StringBuilder();
/**要插入的字段值，其實就是？**/
StringBuilder unknownMarkSql = new StringBuilder();
Object[] bindArgs = new Object[valueMap.size()];
int i = 0;
while (iterator.hasNext()) {
String key = iterator.next();
columnSql.append(i == 0 ? "" : ",");
columnSql.append(key);
unknownMarkSql.append(i == 0 ? "" : ",");
unknownMarkSql.append("?");
bindArgs[i] = valueMap.get(key);
i++;
}
/**開始拼插入的sql語句**/
StringBuilder sql = new StringBuilder();
sql.append("INSERT INTO ");
sql.append(tableName);
sql.append(" (");
sql.append(columnSql);
sql.append(" ) VALUES (");
sql.append(unknownMarkSql);
sql.append(" )");
return executeUpdate(sql.toString(), bindArgs);
}
/** * 執行數據庫插入操做 * * @param datas 插入數據表中key爲列名和value爲列對應的值的Map對象的List集合 * @param tableName 要插入的數據庫的表名 * @return 影響的行數 * @throws SQLException SQL異常 */
public static int insertAll(String tableName, List<Map<String, Object>> datas) throws SQLException {
/**影響的行數**/
int affectRowCount = -1;
Connection connection = null;
PreparedStatement preparedStatement = null;
try {
/**從數據庫鏈接池中獲取數據庫鏈接**/
connection = DBConnectionPool.getInstance().getConnection();
Map<String, Object> valueMap = datas.get(0);
/**獲取數據庫插入的Map的鍵值對的值**/
Set<String> keySet = valueMap.keySet();
Iterator<String> iterator = keySet.iterator();
/**要插入的字段sql，其實就是用key拼起來的**/
StringBuilder columnSql = new StringBuilder();
/**要插入的字段值，其實就是？**/
StringBuilder unknownMarkSql = new StringBuilder();
Object[] keys = new Object[valueMap.size()];
int i = 0;
while (iterator.hasNext()) {
String key = iterator.next();
keys[i] = key;
columnSql.append(i == 0 ? "" : ",");
columnSql.append(key);
unknownMarkSql.append(i == 0 ? "" : ",");
unknownMarkSql.append("?");
i++;
}
/**開始拼插入的sql語句**/
StringBuilder sql = new StringBuilder();
sql.append("INSERT INTO ");
sql.append(tableName);
sql.append(" (");
sql.append(columnSql);
sql.append(" ) VALUES (");
sql.append(unknownMarkSql);
sql.append(" )");
/**執行SQL預編譯**/
preparedStatement = connection.prepareStatement(sql.toString());
/**設置不自動提交，以便於在出現異常的時候數據庫回滾**/
connection.setAutoCommit(false);
System.out.println(sql.toString());
for (int j = 0; j < datas.size(); j++) {
for (int k = 0; k < keys.length; k++) {
preparedStatement.setObject(k + 1, datas.get(j).get(keys[k]));
}
preparedStatement.addBatch();
}
int[] arr = preparedStatement.executeBatch();
connection.commit();
affectRowCount = arr.length;
System.out.println("成功了插入了" + affectRowCount + "行");
System.out.println();
} catch (Exception e) {
if (connection != null) {
connection.rollback();
}
e.printStackTrace();
throw e;
} finally {
if (preparedStatement != null) {
preparedStatement.close();
}
if (connection != null) {
connection.close();
}
}
return affectRowCount;
}
/** * 執行更新操做 * * @param tableName 表名 * @param valueMap 要更改的值 * @param whereMap 條件 * @return 影響的行數 * @throws SQLException SQL異常 */
public static int update(String tableName, Map<String, Object> valueMap, Map<String, Object> whereMap) throws SQLException {
/**獲取數據庫插入的Map的鍵值對的值**/
Set<String> keySet = valueMap.keySet();
Iterator<String> iterator = keySet.iterator();
/**開始拼插入的sql語句**/
StringBuilder sql = new StringBuilder();
sql.append("UPDATE ");
sql.append(tableName);
sql.append(" SET ");
/**要更改的的字段sql，其實就是用key拼起來的**/
StringBuilder columnSql = new StringBuilder();
int i = 0;
List<Object> objects = new ArrayList<>();
while (iterator.hasNext()) {
String key = iterator.next();
columnSql.append(i == 0 ? "" : ",");
columnSql.append(key + " = ? ");
objects.add(valueMap.get(key));
i++;
}
sql.append(columnSql);
/**更新的條件:要更改的的字段sql，其實就是用key拼起來的**/
StringBuilder whereSql = new StringBuilder();
int j = 0;
if (whereMap != null && whereMap.size() > 0) {
whereSql.append(" WHERE ");
iterator = whereMap.keySet().iterator();
while (iterator.hasNext()) {
String key = iterator.next();
whereSql.append(j == 0 ? "" : " AND ");
whereSql.append(key + " = ? ");
objects.add(whereMap.get(key));
j++;
}
sql.append(whereSql);
}
return executeUpdate(sql.toString(), objects.toArray());
}
/** * 執行刪除操做 * * @param tableName 要刪除的表名 * @param whereMap 刪除的條件 * @return 影響的行數 * @throws SQLException SQL執行異常 */
public static int delete(String tableName, Map<String, Object> whereMap) throws SQLException {
/**準備刪除的sql語句**/
StringBuilder sql = new StringBuilder();
sql.append("DELETE FROM ");
sql.append(tableName);
/**更新的條件:要更改的的字段sql，其實就是用key拼起來的**/
StringBuilder whereSql = new StringBuilder();
Object[] bindArgs = null;
if (whereMap != null && whereMap.size() > 0) {
bindArgs = new Object[whereMap.size()];
whereSql.append(" WHERE ");
/**獲取數據庫插入的Map的鍵值對的值**/
Set<String> keySet = whereMap.keySet();
Iterator<String> iterator = keySet.iterator();
int i = 0;
while (iterator.hasNext()) {
String key = iterator.next();
whereSql.append(i == 0 ? "" : " AND ");
whereSql.append(key + " = ? ");
bindArgs[i] = whereMap.get(key);
i++;
}
sql.append(whereSql);
}
return executeUpdate(sql.toString(), bindArgs);
}
/** * 能夠執行新增，修改，刪除 * * @param sql sql語句 * @param bindArgs 綁定參數 * @return 影響的行數 * @throws SQLException SQL異常 */
public static int executeUpdate(String sql, Object[] bindArgs) throws SQLException {
/**影響的行數**/
int affectRowCount = -1;
Connection connection = null;
PreparedStatement preparedStatement = null;
try {
/**從數據庫鏈接池中獲取數據庫鏈接**/
connection = DBConnectionPool.getInstance().getConnection();
/**執行SQL預編譯**/
preparedStatement = connection.prepareStatement(sql.toString());
/**設置不自動提交，以便於在出現異常的時候數據庫回滾**/
connection.setAutoCommit(false);
System.out.println(getExecSQL(sql, bindArgs));
if (bindArgs != null) {
/**綁定參數設置sql佔位符中的值**/
for (int i = 0; i < bindArgs.length; i++) {
preparedStatement.setObject(i + 1, bindArgs[i]);
}
}
/**執行sql**/
affectRowCount = preparedStatement.executeUpdate();
connection.commit();
String operate;
if (sql.toUpperCase().indexOf("DELETE FROM") != -1) {
operate = "刪除";
} else if (sql.toUpperCase().indexOf("INSERT INTO") != -1) {
operate = "新增";
} else {
operate = "修改";
}
System.out.println("成功" + operate + "了" + affectRowCount + "行");
System.out.println();
} catch (Exception e) {
if (connection != null) {
connection.rollback();
}
e.printStackTrace();
throw e;
} finally {
if (preparedStatement != null) {
preparedStatement.close();
}
if (connection != null) {
connection.close();
}
}
return affectRowCount;
}
/** * 經過sql查詢數據, * 慎用，會有sql注入問題 * * @param sql * @return 查詢的數據集合 * @throws SQLException */
public static List<Map<String, Object>> query(String sql) throws SQLException {
return executeQuery(sql, null);
}
/** * 執行sql經過 Map<String, Object>限定查詢條件查詢 * * @param tableName 表名 * @param whereMap where條件 * @return List<Map<String, Object>> * @throws SQLException */
public static List<Map<String, Object>> query(String tableName,
Map<String, Object> whereMap) throws Exception {
String whereClause = "";
Object[] whereArgs = null;
if (whereMap != null && whereMap.size() > 0) {
Iterator<String> iterator = whereMap.keySet().iterator();
whereArgs = new Object[whereMap.size()];
int i = 0;
while (iterator.hasNext()) {
String key = iterator.next();
whereClause += (i == 0 ? "" : " AND ");
whereClause += (key + " = ? ");
whereArgs[i] = whereMap.get(key);
i++;
}
}
return query(tableName, false, null, whereClause, whereArgs, null, null, null, null);
}
/** * 執行sql條件參數綁定形式的查詢 * * @param tableName 表名 * @param whereClause where條件的sql * @param whereArgs where條件中佔位符中的值 * @return List<Map<String, Object>> * @throws SQLException */
public static List<Map<String, Object>> query(String tableName,
String whereClause,
String[] whereArgs) throws SQLException {
return query(tableName, false, null, whereClause, whereArgs, null, null, null, null);
}
/** * 執行所有結構的sql查詢 * * @param tableName 表名 * @param distinct 去重 * @param columns 要查詢的列名 * @param selection where條件 * @param selectionArgs where條件中佔位符中的值 * @param groupBy 分組 * @param having 篩選 * @param orderBy 排序 * @param limit 分頁 * @return List<Map<String, Object>> * @throws SQLException */
public static List<Map<String, Object>> query(String tableName,
boolean distinct,
String[] columns,
String selection,
Object[] selectionArgs,
String groupBy,
String having,
String orderBy,
String limit) throws SQLException {
String sql = buildQueryString(distinct, tableName, columns, selection, groupBy, having, orderBy, limit);
return executeQuery(sql, selectionArgs);
}
/** * 執行查詢 * * @param sql 要執行的sql語句 * @param bindArgs 綁定的參數 * @return List<Map<String, Object>>結果集對象 * @throws SQLException SQL執行異常 */
public static List<Map<String, Object>> executeQuery(String sql, Object[] bindArgs) throws SQLException {
List<Map<String, Object>> datas = new ArrayList<>();
Connection connection = null;
PreparedStatement preparedStatement = null;
ResultSet resultSet = null;
try {
/**獲取數據庫鏈接池中的鏈接**/
connection = DBConnectionPool.getInstance().getConnection();
preparedStatement = connection.prepareStatement(sql);
if (bindArgs != null) {
/**設置sql佔位符中的值**/
for (int i = 0; i < bindArgs.length; i++) {
preparedStatement.setObject(i + 1, bindArgs[i]);
}
}
System.out.println(getExecSQL(sql, bindArgs));
/**執行sql語句，獲取結果集**/
resultSet = preparedStatement.executeQuery();
getDatas(resultSet);
System.out.println();
} catch (Exception e) {
e.printStackTrace();
throw e;
} finally {
if (resultSet != null) {
resultSet.close();
}
if (preparedStatement != null) {
preparedStatement.close();
}
if (connection != null) {
connection.close();
}
}
return datas;
}
/** * 將結果集對象封裝成List<Map<String, Object>> 對象 * * @param resultSet 結果多想 * @return 結果的封裝 * @throws SQLException */
private static List<Map<String, Object>> getDatas(ResultSet resultSet) throws SQLException {
List<Map<String, Object>> datas = new ArrayList<>();
/**獲取結果集的數據結構對象**/
ResultSetMetaData metaData = resultSet.getMetaData();
while (resultSet.next()) {
Map<String, Object> rowMap = new HashMap<>();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
rowMap.put(metaData.getColumnName(i), resultSet.getObject(i));
}
datas.add(rowMap);
}
System.out.println("成功查詢到了" + datas.size() + "行數據");
for (int i = 0; i < datas.size(); i++) {
Map<String, Object> map = datas.get(i);
System.out.println("第" + (i + 1) + "行：" + map);
}
return datas;
}
/** * Build an SQL query string from the given clauses. * * @param distinct true if you want each row to be unique, false otherwise. * @param tables The table names to compile the query against. * @param columns A list of which columns to return. Passing null will * return all columns, which is discouraged to prevent reading * data from storage that isn't going to be used. * @param where A filter declaring which rows to return, formatted as an SQL * WHERE clause (excluding the WHERE itself). Passing null will * return all rows for the given URL. * @param groupBy A filter declaring how to group rows, formatted as an SQL * GROUP BY clause (excluding the GROUP BY itself). Passing null * will cause the rows to not be grouped. * @param having A filter declare which row groups to include in the cursor, * if row grouping is being used, formatted as an SQL HAVING * clause (excluding the HAVING itself). Passing null will cause * all row groups to be included, and is required when row * grouping is not being used. * @param orderBy How to order the rows, formatted as an SQL ORDER BY clause * (excluding the ORDER BY itself). Passing null will use the * default sort order, which may be unordered. * @param limit Limits the number of rows returned by the query, * formatted as LIMIT clause. Passing null denotes no LIMIT clause. * @return the SQL query string */
private static String buildQueryString( boolean distinct, String tables, String[] columns, String where, String groupBy, String having, String orderBy, String limit) {
if (isEmpty(groupBy) && !isEmpty(having)) {
throw new IllegalArgumentException(
"HAVING clauses are only permitted when using a groupBy clause");
}
if (!isEmpty(limit) && !sLimitPattern.matcher(limit).matches()) {
throw new IllegalArgumentException("invalid LIMIT clauses:" + limit);
}
StringBuilder query = new StringBuilder(120);
query.append("SELECT ");
if (distinct) {
query.append("DISTINCT ");
}
if (columns != null && columns.length != 0) {
appendColumns(query, columns);
} else {
query.append(" * ");
}
query.append("FROM ");
query.append(tables);
appendClause(query, " WHERE ", where);
appendClause(query, " GROUP BY ", groupBy);
appendClause(query, " HAVING ", having);
appendClause(query, " ORDER BY ", orderBy);
appendClause(query, " LIMIT ", limit);
return query.toString();
}
/** * Add the names that are non-null in columns to s, separating * them with commas. */
private static void appendColumns(StringBuilder s, String[] columns) {
int n = columns.length;
for (int i = 0; i < n; i++) {
String column = columns[i];
if (column != null) {
if (i > 0) {
s.append(", ");
}
s.append(column);
}
}
s.append(' ');
}
/** * addClause * * @param s the add StringBuilder * @param name clauseName * @param clause clauseSelection */
private static void appendClause(StringBuilder s, String name, String clause) {
if (!isEmpty(clause)) {
s.append(name);
s.append(clause);
}
}
/** * Returns true if the string is null or 0-length. * * @param str the string to be examined * @return true if str is null or zero length */
private static boolean isEmpty(@Nullable CharSequence str) {
if (str == null || str.length() == 0)
return true;
else
return false;
}
/** * the pattern of limit */
private static final Pattern sLimitPattern =
Pattern.compile("\\s*\\d+\\s*(,\\s*\\d+\\s*)?");
/** * After the execution of the complete SQL statement, not necessarily the actual implementation of the SQL statement * * @param sql SQL statement * @param bindArgs Binding parameters * @return Replace? SQL statement executed after the */
private static String getExecSQL(String sql, Object[] bindArgs) {
StringBuilder sb = new StringBuilder(sql);
if (bindArgs != null && bindArgs.length > 0) {
int index = 0;
for (int i = 0; i < bindArgs.length; i++) {
index = sb.indexOf("?", index);
sb.replace(index, index + 1, String.valueOf(bindArgs[i]));
}
}
return sb.toString();
}
}複製代碼
#四.GitHub測試
相關文章
- 1. JDBC批量插入優化addbatch
- 2. addBatch()和executeBatch()的作用
- 3. 關於jdbc批量操作(addBatch, executeBatch)的測試【轉載】
- 4. 使用JDBC在MySQL數據庫中快速批量插入數據
- 5. JDBC批量插入數據
- 6. 數據庫批量插入
- 7. MySQL 批量插入PreparedStatement addbatch()的用法
- 8. 使用JDBC中的PreparedStatement批量插入
- 9. 第一週：JDBC中批量插入數據問題
- 10. MySQL 批量插入數據優化
- 更多相關文章...
- • PHP MySQL 插入數據 - PHP教程
- • MySQL INSERT：插入數據（添加數據） - MySQL教程
- • TiDB 在摩拜單車在線數據業務的應用和實踐
- • Flink 數據傳輸及反壓詳解
相關標籤/搜索
每日一句
-
每一个你不满意的现在，都有一个你没有努力的曾经。
歡迎關注本站公眾號,獲取更多信息
