SpringBoot模擬插入1000000萬條數據
一、數據庫表準備
CREATE TABLE `student` ( `id` bigint NOT NULL COMMENT '用戶id', `name` varchar(50) COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '姓名', `address` varchar(250) COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '地址', PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
二、持久層代碼實現
###Mapper方法 /** * @Project * @Description * @Author songwp * @Date 2023/9/8 9:29 **/ @Mapper public interface StudentMapper extends BaseMapper<Student> { int insertSplice(List<Student> students); } ###Mapper.xml <?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd"> <mapper namespace="com.songwp.mapper.StudentMapper"> <insert id="insertSplice"> insert into student (id,name,address) values <foreach collection="students" item="entity" separator=","> (#{entity.id}, #{entity.name}, #{entity.address}) </foreach> </insert> </mapper>
三、業務層代碼實現
###批量插入的具體方法 /** * 批量插入方法 * @param list 需要處理的數據 */ public void batchData(List<Student> list) { int count = list.size() - 1; int pageSize = 1000; // 每批次插入的數據量 int threadNum = count / pageSize + 1; // 線程數 CountDownLatch countDownLatch = new CountDownLatch(threadNum); for (int i = 0; i < threadNum; i++) { int startIndex = i * pageSize; int endIndex = Math.min(count, (i + 1) * pageSize); List<Student> subList = list.subList(startIndex, endIndex); threadPoolTaskExecutor.execute(() -> { DefaultTransactionDefinition transactionDefinition = new DefaultTransactionDefinition(); TransactionStatus status = transactionManager.getTransaction(transactionDefinition); try { studentMapper.insertSplice(subList); transactionManager.commit(status); } catch (Exception e) { transactionManager.rollback(status); throw e; } finally { countDownLatch.countDown(); } }); } try { countDownLatch.await(); } catch (InterruptedException e) { e.printStackTrace(); } } ###控制層業務方法 @GetMapping("/batchInsert") @ApiOperation("批量插入大數據量的方法驗證") public void batchInsert2() { List<Student> arrayList = new ArrayList<>(); // 模擬數據 for (int i = 0; i < 1000000; i++){ Student student = new Student(i+1,"張三"+i,"陜西西安"+i); arrayList.add(student); } long startTime = System.currentTimeMillis(); batchData(arrayList); long endTime = System.currentTimeMillis(); System.out.println("模擬插入: "+arrayList.size()+" 條學生數據,總共耗時:" + (endTime - startTime)+"毫秒"); }
四、插入的結果耗時

該測試項目已集成Swagger,通過Swagger調用該方法結果如下:

古今成大事者,不唯有超世之才,必有堅韌不拔之志!

浙公網安備 33010602011771號