package com.yssh.utils;
|
|
import java.util.ArrayList;
|
import java.util.List;
|
|
import org.slf4j.Logger;
|
import org.slf4j.LoggerFactory;
|
|
import com.github.biyanwen.impl.AbstractCsvFileParser;
|
import com.yssh.entity.SuYuan;
|
import com.yssh.service.ISuYuanService;
|
|
public class CsvParser extends AbstractCsvFileParser<SuYuan>{
|
|
protected final Logger logger = LoggerFactory.getLogger(this.getClass());
|
/**
|
* 每隔3000条存储数据库，然后清理list ，方便内存回收
|
*/
|
public static final int BATCH_COUNT = 100000;
|
|
/**
|
* 缓存的数据
|
*/
|
private List<SuYuan> cachedData = new ArrayList<>(BATCH_COUNT);
|
|
private ISuYuanService suYuanService;
|
private String time;
|
|
public CsvParser(ISuYuanService suYuanService, String time) {
|
this.suYuanService = suYuanService;
|
this.time = time;
|
}
|
|
/**
|
* 所有数据解析完成了 会来调用,防止有数据没有被保存
|
*/
|
@Override
|
protected void doAfterAllAnalysed() {
|
try {
|
saveSuYuanData();
|
} catch (Exception e) {
|
logger.error("解析保存数据出现异常，异常原因是：" + e.getMessage());
|
e.printStackTrace();
|
}
|
}
|
|
@Override
|
protected void invoke(SuYuan t) {
|
t.setId(t.getX() + "_" + t.getY() + "_" + t.getZ());
|
System.err.println(t.getId());
|
cachedData.add(t);
|
// 达到BATCH_COUNT了，需要去存储一次数据库，防止数据几万条数据在内存，容易OOM
|
if (cachedData.size() >= BATCH_COUNT) {
|
try {
|
saveSuYuanData();
|
} catch (Exception e) {
|
logger.error("解析保存数据出现异常，异常原因是：" + e.getMessage());
|
e.printStackTrace();
|
}
|
// 存储完成清理 list
|
cachedData = new ArrayList<>(BATCH_COUNT);
|
}
|
}
|
|
private void saveSuYuanData() throws Exception {
|
suYuanService.insertSuYuanDatas(cachedData, time);
|
}
|
|
}
|
