100萬數據插入 mysql 性能測試

Mysql服務器信息

8核16G

 

開發臺式機信息

固態硬盤、公司內部網絡

 

示例代碼

1.採用jdk1.8 + springboot 2.1.6 + mybatis 3.0 框架組合而成的工程示例

2.mysql5.7 全部過程採用默認安裝，沒作特殊優化

3.測試表，只有主鍵，ID不自增

mysql.sql

CREATE TABLE `t_result` (
  `id` bigint(8) NOT NULL,
  `content` varchar(50) DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4

pom.xml

<!-- springboot框架包 -->
<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!-- springJPA-->
<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-thymeleaf</artifactId>
</dependency>
<dependency>
    <groupId>org.mybatis.spring.boot</groupId>
    <artifactId>mybatis-spring-boot-starter</artifactId>
    <version>2.1.0</version>
</dependency>
<!-- mysql驅動包-->
<dependency>
    <groupId>mysql</groupId>
    <artifactId>mysql-connector-java</artifactId>
    <version>5.1.20</version>
</dependency>
<!-- 阿里的數據庫鏈接池-->
<dependency>
    <groupId>com.alibaba</groupId>
    <artifactId>druid</artifactId>
    <version>1.0.18</version>
</dependency>

mybatis.cfg.xml

<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE configuration PUBLIC "-//mybatis.org//DTD Config 3.0//EN" "http://mybatis.org/dtd/mybatis-3-config.dtd">
<!-- 全局配置文件 -->
<configuration>
</configuration>

ResultMapper.xml

<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">

<mapper namespace="com.pro.app.result.dao.ResultMapper">
    <!-- 批量插入 -->
    <insert id="insertResultBatch" parameterType="java.util.List">
        insert into t_result (id, content) values
        <foreach collection="list" item="item" index="index" separator=",">
            (#{item.id}, #{item.content})
        </foreach>
    </insert>
    <!-- 單條插入 -->
    <insert id="insert" useGeneratedKeys="true" keyProperty="id" parameterType="com.pro.app.entity.Result">
        insert into t_result(id,content) value(#{id},#{content})
    </insert>
</mapper>

application.properties

#tomcat address port
server.port=8089
server.address=127.0.0.1
server.sessionTimeout=30

spring.application.name=demo-web

#2.0 path
server.servlet.context-path=/api

#採用logback.xml配置日誌輸出模板 
logging.config=classpath:logback.xml

#springframework.web框架的日誌級別，注意與自已代碼中定義的日誌級別不同
logging.level.org.springframework.web:INFO
#是否啓用springboot的debug調試模式，會打印詳細日誌信息
debug=false

# 數據庫訪問配置
# 主數據源，默認的
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://192.168.110.01:3306/demo?useUnicode=true&characterEncoding=utf-8
spring.datasource.username=root
spring.datasource.password=123465

# 實體類映射SQL配置
mybatis.mapper-locations=classpath:mapper/*.xml
# mybatis配置
mybatis.config-location=classpath:mybatis.cfg.xml

Result.java

@Data
public class Result {
    @Id
    private long id;
    @Column(name = "content")
    private String content;
}

ResultMapper.java

@Mapper
@Component(value = "ResultMapper")
public interface ResultMapper {
    /**
     * 批量插入
     * @param resultList
     * @return
     */
    int insertResultBatch(List<Result> resultList);
    /**
     * 增
     */
    int insert(Result result);
}

ResultService.java

@Service
public class ResultService {
    @Autowired
    private ResultMapper resultMapper;
    /**
     * 新增數據
     * @param result
     */
    public void insert(Result result){
        resultMapper.insert(result);
    }
    /**
     * 新增數據
     * @param resultList
     */
    public void insertResultBatch(List<Result> resultList){
        resultMapper.insertResultBatch(resultList);
    }
}

ResultApi.java

@RestController
@RequestMapping("/result")
public class ResultApi {
    @Autowired
    private ResultService resultService;
    /**
     * 批量添加新的對象
     * @return
     */
    @RequestMapping(value = "/batAdd", method = {RequestMethod.GET, RequestMethod.POST})
    @ResponseBody
    public Object batAdd(){
        Result result;        
        long startTime =  System.currentTimeMillis();
        String content = startTime  + "";
        for (int i=1;i<=1000000;i++){
            result = new Result();
            result.setId(i);
            result.setContent(content);
            resultService.insert(result);
        }
        ApiResult result1 = new ApiResult();
        result1.setData(System.currentTimeMillis() - startTime);
        return result1;
    }
    /**
     * 批量添加新的對象
     * @return
     */
    @RequestMapping(value = "/batAddList", method = {RequestMethod.GET, RequestMethod.POST})
    @ResponseBody
    public Object batAddList(){
        Result result ;
        long startTime =  System.currentTimeMillis();
        String content = startTime  + "";
        List<Result> resultList = new ArrayList<>(1000);
        for (int i=1011001;i<=2011000;i++){
            result = new Result();
            result.setId(i);
            result.setContent(content);
            resultList.add(result);
            if (resultList.size() == 1000){
                resultService.insertResultBatch(resultList);
                resultList.clear();
            }
        }
        ApiResult result1 = new ApiResult();
        result1.setData(System.currentTimeMillis() - startTime);
        return result1;
    }
}

注：ApiResult 是內部封裝的向前端響應的實體對象，主要有三個變量：code,msg,data

AppStart.java

@SpringBootApplication
@EnableAutoConfiguration
public class AppStart {
    public static void main(String[] args) {
        SpringApplication.run(AppStart.class,args);
    }
}

 

測試結果

-- 單條插入1萬
{
    "code": "1",
    "data": "27323",
    "msg": ""
}
-- 單條插入100萬
{
    "code": "1",
    "data": "2868335",
    "msg": ""
}

-- 批量插入1000
{
    "code": "1",
    "data": "160",
    "msg": ""
}
-- 批量插入1萬
{
    "code": "1",
    "data": "355",
    "msg": ""
}
-- 批量插入100萬
{
    "code": "1",
    "data": "31673",
    "msg": ""
}

 

數據統計

累計入庫總記錄數

測試庫總大小

 

總結

    100萬條記錄一條一條的插入到數據表中，須要耗時約 47分鐘 =（2868335 / 1000 / 60） ，說實在的比預期要慢不少；

    100萬條記錄每1000條爲一批的方式插入到數據表中，須要耗時約 31秒 = （31673 / 1000），速度就挺快了，結果比較滿意；

    所以涉及到大批量數據插入到數據庫表中，則因採用批量插入方式，操做前要注意數據表的索引是否過多或先取消除主鍵外的其它索引（每新增數據須要維護索引，會加大數據庫插入的開銷），待100萬以上批量數據所有入庫後在重建索引，同時還須要注意，mysql配置中的max_allowed_packet的值大小，以防SQL語句自己超過指定的大小，致使數據庫異常；