package com.yssh.utils; import java.util.ArrayList; import java.util.List; import com.yssh.entity.AlertConfig; import com.yssh.entity.SuYuan; import com.yssh.mapper.AlertConfigMapper; import com.yssh.service.SuYuanService; import com.github.biyanwen.impl.AbstractCsvFileParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Resource; public class CsvParser extends AbstractCsvFileParser { protected final Logger logger = LoggerFactory.getLogger(this.getClass()); /** * 每隔100000条存储数据库,然后清理list ,方便内存回收 */ public static final int BATCH_COUNT = 100000; /** * 缓存的数据 */ private List cachedData = new ArrayList<>(BATCH_COUNT); private SuYuanService suYuanService; private String time; private double jcyj; public CsvParser(SuYuanService suYuanService, String time) { this.suYuanService = suYuanService; this.time = time; this.jcyj = suYuanService.getJcyj(); } /** * 所有数据解析完成了 会来调用,防止有数据没有被保存 */ protected void doAfterAllAnalysed() { try { saveSuYuanData(); } catch (Exception e) { logger.error("解析保存数据出现异常,异常原因是:" + e.getMessage(), e); e.printStackTrace(); } } protected void invoke(SuYuan t) { t.setId(t.getX() + "_" + t.getY() + "_" + t.getZ()); /*if ("0_0_0".equals(t.getId())) { return; // 解决主键重复 }*/ /*if (t.getZ() > 0 && t.getC() < jcyj) { return; // 只入第0层数据+Voc值大于1 }*/ // 只入值 > =0.05 if (t.getC() < 0.01) { return; } cachedData.add(t); // 达到BATCH_COUNT了,需要去存储一次数据库,防止数据几万条数据在内存,容易OOM if (cachedData.size() >= BATCH_COUNT) { try { saveSuYuanData(); } catch (Exception e) { logger.error("解析保存数据出现异常,异常原因是:" + e.getMessage(), e); e.printStackTrace(); } // 存储完成清理 list cachedData = new ArrayList<>(BATCH_COUNT); } } private void saveSuYuanData() throws Exception { suYuanService.insertSuYuanDatas(cachedData, time); } }