当前位置：首页 > news >正文

csv文件生成与读取

news 2026/6/18 4:57:40

1.csv 文件解析入库方法(少量)

private boolean doProcessNew(FileInputStream fileInputStream) { long start = System.currentTimeMillis(); List<JtComplaintPredictionResultEntity> csvFileList = new ArrayList<>(); try (BufferedReader reader = new BufferedReader(new InputStreamReader(fileInputStream, StandardCharsets.UTF_8)); CSVReader csvReader = new CSVReaderBuilder(reader).build()) { log.info("开始解析CSV文件"); Iterator<String[]> iterator = csvReader.iterator(); // 跳过表头 if (iterator.hasNext()) { iterator.next(); log.info("已跳过表头"); } int lineNum = 1; while (iterator.hasNext()) { lineNum++; String[] next = iterator.next(); // 空行跳过 if (next == null || next.length == 0) { log.warn("第{}行：空行，跳过", lineNum); continue; } // 字段不足跳过 if (next.length < 4) { log.warn("第{}行：字段数量不足，跳过", lineNum); continue; } try { JtComplaintPredictionResultEntity entity = new JtComplaintPredictionResultEntity(); entity.setAccount(next[0] != null ? next[0].trim() : ""); entity.setAccountTv(next[1] != null ? next[1].trim() : ""); entity.setPred(next[2] != null ? next[2].trim() : ""); entity.setGreaterThanT(next[3] != null ? next[3].trim() : ""); csvFileList.add(entity); } catch (Exception e) { log.error("第{}行解析异常：{}", lineNum, e.getMessage()); } } // 批量入库 jtService.saveBatch(csvFileList); log.info("解析完成，成功入库：{} 条，总耗时：{} ms", csvFileList.size(), System.currentTimeMillis() - start); return true; } catch (Exception e) { log.error("CSV解析失败", e); return false; } }

2.csv 文件解析入库方法(大量)

全部解析到内存（几十万以内没问题）
分成小批次（1000～3000 条）
线程池异步批量入库（不要无限制开线程）

// 注入线程池（建议用自定义线程池，不要用Executors） @Resource private ThreadPoolTaskExecutor asyncExecutor; private boolean doProcessBigFile(FileInputStream fileInputStream) { long start = System.currentTimeMillis(); List<JtComplaintPredictionResultEntity> totalList = new ArrayList<>(); // ========== 1. 先完整解析 ========== try (CSVReader csvReader = new CSVReaderBuilder( new BufferedReader(new InputStreamReader(fileInputStream, StandardCharsetsets.UTF_8))).build()) { Iterator<String[]> iterator = csvReader.iterator(); if (iterator.hasNext()) iterator.next(); // 跳过表头 int line = 1; while (iterator.hasNext()) { line++; String[] next = iterator.next(); if (next == null || next.length < 4) continue; JtComplaintPredictionResultEntity entity = new JtComplaintPredictionResultEntity(); entity.setAccount(next[0].trim()); entity.setAccountTv(next[1].trim()); entity.setPred(next[2].trim()); entity.setGreaterThanT(next[3].trim()); totalList.add(entity); } } catch (Exception e) { log.error("解析失败", e); return false; } // ========== 2. 分批次 + 多线程入库 ========== int batchSize = 3000; List<List<JtComplaintPredictionResultEntity>> partitions = Lists.partition(totalList, batchSize); log.info("总数据量：{}，分为{}批入库", totalList.size(), partitions.size()); CountDownLatch latch = new CountDownLatch(partitions.size()); for (List<JtComplaintPredictionResultEntity> batch : partitions) { asyncExecutor.execute(() -> { try { jtService.saveBatch(batch); log.info("批量入库成功：{} 条", batch.size()); } catch (Exception e) { log.error("批量入库失败", e); } finally { latch.countDown(); } }); } try { latch.await(); // 等待所有批次完成 } catch (InterruptedException e) { Thread.currentThread().interrupt(); } log.info("全部入库完成，总耗时：{} ms", System.currentTimeMillis() - start); return true; }

3.少量 CSV 生成

String dateNow = DateTimeFormatter.ofPattern("yyyyMMddHHmmss").format(LocalDateTime.now()); Map<String, Object> param = new HashMap<>(); param.put("startrow", 0); param.put("endrow", 50); long startTime = System.currentTimeMillis(); List<Map<String, Object>> dataList = dao.listJtComplaintPrediction(param); log.info("查询完成：{} 条，耗时：{} ms", dataList.size(), System.currentTimeMillis() - startTime); // 写入文件（流式写，不占内存） String fileName = "_LIST_" + dateNow + ".csv"; File csvFile = new File(fileName); try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(csvFile), StandardCharsets.UTF_8))) { // 写入表头 writer.write("account|tv_account|care_num"); writer.newLine(); // 写入行 for (Map<String, Object> map : dataList) { String account = nullToEmpty(map.get("ACCOUNT")); String careNum = nullToEmpty(map.get("CARE_NUM")); // 拼接行（|分隔） writer.write(account + "|" + careNum); writer.newLine(); } } log.info("CSV生成完成：{}", csvFile.getName()); // 工具方法 private String nullToEmpty(Object obj) { return obj == null ? "" : obj.toString().trim(); }

#4.大量 CSV 生成
分页查询
流式逐页写入（一边查一边写，不占内存）
单线程写文件（CSV 必须顺序写）
可多线程分页查询，但必须排队写入

public void generateBigCsv() { String fileName = "BIG_DATA_" + System.currentTimeMillis() + ".csv"; File csvFile = new File(fileName); int pageSize = 5000; // 每页查5000 int pageNum = 1; try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(csvFile), StandardCharsetsets.UTF_8))) { // 写表头 writer.write("account|tv_account|care_num"); writer.newLine(); while (true) { // 分页查询 Map<String, Object> param = new HashMap<>(); param.put("pageNum", pageNum); param.put("pageSize", pageSize); List<Map<String, Object>> dataList = dao.listJtComplaintPredictionPage(param); if (CollUtil.isEmpty(dataList)) { break; } // 流式写入 for (Map<String, Object> map : dataList) { String account = nullToEmpty(map.get("ACCOUNT")); String careNum = nullToEmpty(map.get("CARE_NUM")); writer.write(account + "|" + careNum); writer.newLine(); } log.info("已写入第{}页，{}条", pageNum, dataList.size()); pageNum++; } } catch (Exception e) { log.error("大文件生成失败", e); } log.info("超大CSV生成完成：{}", csvFile.getName()); }