package com.terra.bigdata.util;
|
|
import com.terra.bigdata.domain.*;
|
import com.terra.bigdata.mapper.Suyuan100Csv2Mapper;
|
import com.terra.bigdata.service.Suyuan100Csv2Service;
|
import com.terra.bigdata.service.Yssh3dreliService;
|
import com.terra.bigdata.service.YsshQxshCsvService;
|
import com.terra.bigdata.service.YsshSuyuan200Service;
|
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.scheduling.annotation.EnableScheduling;
|
import org.springframework.scheduling.annotation.Scheduled;
|
import org.springframework.stereotype.Service;
|
|
import javax.annotation.PostConstruct;
|
import java.io.*;
|
import java.text.DecimalFormat;
|
import java.text.SimpleDateFormat;
|
import java.util.*;
|
|
@Service
|
@EnableScheduling
|
public class ReadData2 {
|
|
|
static HashMap<Integer, Double> zHash = new HashMap<Integer, Double>();
|
|
static List<Suyuan100Csv2> checkPoints = new ArrayList<Suyuan100Csv2>();
|
static List<Suyuan100Csv2> checkPoints2d = new ArrayList<Suyuan100Csv2>();
|
|
|
@Autowired
|
Suyuan100Csv2Service service;
|
@Autowired
|
Suyuan100Csv2Mapper mapper;
|
@Autowired
|
YsshSuyuan200Service service200;
|
|
@Autowired
|
YsshQxshCsvService ysshQxshCsvService;
|
|
@Autowired
|
Yssh3dreliService yssh3dreliService;
|
//fastdfs配置文件所在路径
|
private static String sourceFilePath;
|
|
|
@Value("${spring.saveLog.path}")
|
public void setFilePath(String path) {
|
this.sourceFilePath = path;
|
}
|
|
|
/* public static String scanner(String tip) throws Exception {
|
Scanner scanner = new Scanner(System.in);
|
StringBuilder help = new StringBuilder();
|
help.append(tip + "：");
|
System.out.println(help.toString());
|
if (scanner.hasNext()) {
|
String ipt = scanner.next();
|
if (ipt != null) {
|
return ipt;
|
|
}
|
}
|
throw new Exception("请输入正确的路径！");
|
}*/
|
|
|
@Scheduled(fixedRate = 1000 * 60 * 60 * 24 * 360)
|
public void enter() throws Exception {
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHH");
|
String fName = sdf.format(new Date());
|
System.out.println("fName = " + fName);
|
//sourceFilePath = sourceFilePath + "\\\\"+fName+".csv" ;
|
System.out.println("目录:" + sourceFilePath + "\\\\" + fName + ".csv");
|
File file = new File(sourceFilePath + "\\\\" + fName + ".csv");
|
if (!file.exists()) {
|
System.out.println(sourceFilePath + " is not exist");
|
// return ;
|
}
|
compressCSV(sourceFilePath + "\\\\" + fName + ".csv");
|
// range确定，那么查询范围就确定，对应的list事固定的，可以反复使用，避免消耗过多内存
|
List<Suyuan100Csv2> savelist = new ArrayList<>();
|
List<YsshSuyuan200> savelist200 = new ArrayList<>();
|
for (Suyuan100Csv2 s : checkPoints) {
|
System.out.println("============100============");
|
List<Suyuan100Csv2> list = ReadData2.computeRangeList(s, 10);
|
saveDB(savelist, s, list);
|
System.out.println("============200============");
|
points473d(list, fName);
|
System.out.println("============ 3d结束============");
|
List<YsshSuyuan200> list200 = ReadData2.computeRangeList200(s, 20);
|
saveDB200(savelist200, s, list200);
|
}
|
System.out.println("============入库结束============");
|
points47(fName);
|
System.out.println("============47 结束============");
|
//=====================================================
|
analytical2dData(fName);
|
System.out.println("============2d结束============");
|
|
System.exit(0);
|
}
|
|
private void points473d(List<Suyuan100Csv2> ll, String fName) {
|
DecimalFormat df = new DecimalFormat("##.######");
|
List<Yssh3dreli> list = new ArrayList<Yssh3dreli>();
|
for (Suyuan100Csv2 k : ll) {
|
Yssh3dreli y = new Yssh3dreli();
|
y.setName(k.getName());
|
y.setLon(getLon(k.getX(), k.getY()));
|
y.setLat(getLat(k.getX(), k.getY()));
|
for (int i = 0; i < 10; i++) {
|
int hash = Integer.parseInt(k.getX() + "" + k.getY() + "" + (k.getZ() + i));
|
if (i == 0)
|
y.setHeight0(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash)));
|
if (i == 1)
|
y.setHeight10(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash)));
|
if (i == 2)
|
y.setHeight20(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash)));
|
if (i == 3)
|
y.setHeight30(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash)));
|
if (i == 4)
|
y.setHeight40(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash)));
|
if (i == 5)
|
y.setHeight50(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash)));
|
if (i == 6)
|
y.setHeight60(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash)));
|
if (i == 7)
|
y.setHeight70(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash)));
|
if (i == 8)
|
y.setHeight80(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash)));
|
if (i == 9)
|
y.setHeight90(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash)));
|
|
}
|
y.setTime(fName);
|
list.add(y);
|
}
|
//使用最新的批量新增3D数据，并补充时间字段
|
yssh3dreliService.batchInsert3dreliData(list);
|
//yssh3dreliService.saveOrUpdateBatch(list);
|
}
|
|
/**
|
* 解析2d数据
|
* 1、按照mesh.dat取出4000多个点数据
|
* 2、根据4000多个点数据从csv文件抽取出对应点位的value值
|
* 3、记录抽取的时间
|
*/
|
private void analytical2dData(String fName) {
|
DecimalFormat df = new DecimalFormat("##.######");
|
List<Yssh2dreli> list = new ArrayList<Yssh2dreli>();
|
for (Suyuan100Csv2 k : checkPoints2d) {
|
Yssh2dreli y = new Yssh2dreli();
|
int hash = Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ());
|
y.setLon(getLon(k.getX(), k.getY()));
|
y.setLat(getLat(k.getX(), k.getY()));
|
y.setTime(fName);
|
y.setValue(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash)));
|
list.add(y);
|
}
|
//使用最新的批量新增2D数据，并补充时间字段
|
if (list.size() > 0) {
|
yssh3dreliService.batchInsert2dreliData(list);
|
}
|
}
|
|
private void points47(String time) {
|
int count = 1;
|
DecimalFormat df = new DecimalFormat("##.######");
|
List<YsshQxshCsv> list = new ArrayList<YsshQxshCsv>();
|
for (Suyuan100Csv2 k : checkPoints) {
|
YsshQxshCsv y = new YsshQxshCsv();
|
int hash = Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ());
|
y.setName("AI-" + (count < 10 ? "0" + count : count + ""));
|
|
y.setLon(getLon(k.getX(), k.getY()));
|
y.setLat(getLat(k.getX(), k.getY()));
|
y.setTime(time);
|
y.setValue(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash)));
|
list.add(y);
|
}
|
ysshQxshCsvService.saveBatch(list);
|
}
|
|
private void saveDB(List<Suyuan100Csv2> savelist, Suyuan100Csv2 s, List<Suyuan100Csv2> list) {
|
DecimalFormat df = new DecimalFormat("##.######");
|
for (Suyuan100Csv2 k : list) {
|
if (zHash.get(Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ())) != null) {
|
k.setVocsValue(df.format(zHash.get(Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ()))));
|
} else k.setVocsValue("0");
|
k.setId(Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ()));
|
savelist.add(k);
|
}
|
System.out.print("save batch " + savelist.size());
|
|
if (savelist.size() > 0)
|
if (service.saveOrUpdateBatch(savelist)) {
|
System.out.println(" " + s.getName() + " " + mapper.getCount() + " 成功!");
|
} else {
|
System.err.println(s.getName() + " " + s.getVocsName() + " 失败!");
|
}
|
savelist.clear();
|
}
|
|
private void saveDB200(List<YsshSuyuan200> savelist, Suyuan100Csv2 s, List<YsshSuyuan200> list) {
|
DecimalFormat df = new DecimalFormat("##.######");
|
for (YsshSuyuan200 k : list) {
|
if (zHash.get(Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ())) != null) {
|
k.setVocsValue(df.format(zHash.get(Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ()))));
|
|
} else k.setVocsValue("0");
|
k.setId(Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ()));
|
savelist.add(k);
|
}
|
System.out.print("save batch " + savelist.size());
|
if (savelist.size() > 0)
|
if (service200.saveOrUpdateBatch(savelist)) {
|
System.out.println(s.getName() + " " + " 成功!");
|
} else {
|
System.err.println(s.getName() + " " + s.getVocsName() + " 失败!");
|
}
|
savelist.clear();
|
}
|
|
/**
|
* 读dat,解析并存入list，经纬度，名称提前写好
|
*/
|
@PostConstruct
|
public void init() throws Exception {
|
File file = new File("D:\\bigdata\\dat\\correct_jk_47.dat");
|
BufferedInputStream fis = new BufferedInputStream(new FileInputStream(file));
|
BufferedReader reader = new BufferedReader(new InputStreamReader(fis, "utf-8"), 7 * 1024 * 1024);// 用7M的缓冲读取文本文件
|
|
long count = 1;
|
|
String line = "";
|
|
while ((line = reader.readLine()) != null) {
|
String[] f = line.split(" ");
|
Suyuan100Csv2 suyuan100Csv2 = new Suyuan100Csv2();
|
suyuan100Csv2.setName("AI-" + (count < 10 ? "0" + count : count + ""));
|
suyuan100Csv2.setX(Integer.parseInt(f[0]));
|
suyuan100Csv2.setY(Integer.parseInt(f[1]));
|
suyuan100Csv2.setZ(Integer.parseInt(f[2]));
|
suyuan100Csv2.setLon(getLon(suyuan100Csv2.getX(), suyuan100Csv2.getY()));
|
suyuan100Csv2.setLat(getLat(suyuan100Csv2.getX(), suyuan100Csv2.getY()));
|
checkPoints.add(suyuan100Csv2);
|
count++;
|
}
|
reader.close();
|
|
//====================2D数据读取解析====================
|
File file2d = new File("D:\\bigdata\\dat\\extract_mesh.dat");
|
BufferedInputStream fis2d = new BufferedInputStream(new FileInputStream(file2d));
|
BufferedReader reader2d = new BufferedReader(new InputStreamReader(fis2d, "utf-8"), 7 * 1024 * 1024);// 用7M的缓冲读取文本文件
|
long count2d = 1;
|
String line2d = "";
|
while ((line2d = reader2d.readLine()) != null) {
|
String[] f2d = line2d.split(" ");
|
Suyuan100Csv2 suyuan100Csv2 = new Suyuan100Csv2();
|
suyuan100Csv2.setName("AI-" + (count2d < 10 ? "0" + count2d : count2d + ""));
|
suyuan100Csv2.setX(Integer.parseInt(f2d[0]));
|
suyuan100Csv2.setY(Integer.parseInt(f2d[1]));
|
suyuan100Csv2.setZ(Integer.parseInt(f2d[2]));
|
suyuan100Csv2.setLon(getLon(suyuan100Csv2.getX(), suyuan100Csv2.getY()));
|
suyuan100Csv2.setLat(getLat(suyuan100Csv2.getX(), suyuan100Csv2.getY()));
|
checkPoints2d.add(suyuan100Csv2);
|
count2d++;
|
}
|
reader2d.close();
|
}
|
|
|
private void compressCSV(String path) {
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");//设置日期格式
|
System.out.println("开始时间" + sdf.format(new Date()));// new Date()为获取当前系统时间
|
long timer = System.currentTimeMillis();
|
try {
|
File file = new File(path);
|
BufferedInputStream fis = new BufferedInputStream(new FileInputStream(file));
|
BufferedReader reader = new BufferedReader(new InputStreamReader(fis, "utf-8"), 7 * 1024 * 1024);// 用7M的缓冲读取文本文件
|
long count = 1;
|
String line = "";
|
while ((line = reader.readLine()) != null) {
|
String[] f = line.split(",");
|
if (Double.parseDouble(f[6]) > 0) {
|
zHash.put(Integer.parseInt(f[0] + f[1] + f[2]), Double.parseDouble(f[6]));
|
count++;
|
if (count % 100000 == 0) System.out.print(".");
|
if (count % 10000000 == 0) System.out.println(count);
|
}
|
}
|
System.out.println(count);
|
reader.close();
|
} catch (IOException e) {
|
System.out.println("读写文件失败!");
|
e.printStackTrace();
|
}
|
timer = System.currentTimeMillis() - timer;
|
System.out.println("处理时间：" + timer + " 毫秒");
|
System.out.println("结束时间：" + sdf.format(new Date()));
|
}
|
|
|
/*static public void data2Hash() {
|
String fileName = "e:/2022/yssh/20221122.bin";
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");//设置日期格式
|
System.out.println("HASH开始时间" + sdf.format(new Date()));// new Date()为获取当前系统时间
|
long timer = System.currentTimeMillis();
|
long sum = 0;
|
try {
|
DataInputStream in = new DataInputStream(
|
new BufferedInputStream(
|
new FileInputStream(fileName)));
|
long length = in.available();
|
long count = 0;
|
|
while (length > count) {
|
int k = in.readInt();
|
|
double v = in.readDouble();
|
count = count + 12;
|
sum = count;
|
if (count % 100000 == 0) System.out.print(".");
|
if (count % 10000000 == 0) System.out.println(count / 1024);
|
zHash.put(k, v);
|
}
|
System.out.println("sum=" + sum);
|
timer = System.currentTimeMillis() - timer;
|
System.out.println("HASH处理时间：" + timer + " 毫秒");
|
System.out.println("HASH结束时间：" + sdf.format(new Date()));
|
|
} catch (Exception e) {
|
e.printStackTrace();
|
}
|
}*/
|
|
|
private static List<Suyuan100Csv2> computeRangeList(Suyuan100Csv2 s, int range) {
|
List<Suyuan100Csv2> list = new ArrayList<Suyuan100Csv2>();
|
int x = s.getX();
|
int y = s.getY();
|
int z = s.getZ();
|
int i = 1;
|
for (int k = -range / 2; k <= range / 2; k++) {
|
int xx = x + k;
|
for (int j = -range / 2; j <= range / 2; j++) {
|
//String h = xx + " " + (y + j) + " " + z;
|
Suyuan100Csv2 ss = new Suyuan100Csv2();
|
ss.setName(s.getName());
|
ss.setVocsName(s.getName() + "-" + i);
|
ss.setLon(getLon(xx, y + j));
|
ss.setLat(getLat(xx, y + j));
|
ss.setX(xx);
|
ss.setY(y + j);
|
ss.setZ(z);
|
list.add(ss);
|
i++;
|
}
|
}
|
return list;
|
}
|
|
private static List<YsshSuyuan200> computeRangeList200(Suyuan100Csv2 s, int range) {
|
List<YsshSuyuan200> list = new ArrayList<YsshSuyuan200>();
|
int x = s.getX();
|
int y = s.getY();
|
int z = s.getZ();
|
int i = 1;
|
for (int k = -range / 2; k <= range / 2; k++) {
|
int xx = x + k;
|
for (int j = -range / 2; j <= range / 2; j++) {
|
//String h = xx + " " + (y + j) + " " + z;
|
YsshSuyuan200 ss = new YsshSuyuan200();
|
ss.setName(s.getName());
|
ss.setVocsName(s.getName() + "-" + i);
|
ss.setLon(getLon(xx, y + j));
|
ss.setLat(getLat(xx, y + j));
|
ss.setX(xx);
|
ss.setY(y + j);
|
ss.setZ(z);
|
list.add(ss);
|
i++;
|
}
|
}
|
return list;
|
}
|
|
/*private static List<Integer> computeRange(int x, int y, int z, int range) {
|
List<Integer> list = new ArrayList<Integer>();
|
for (int k = -range / 2; k <= range / 2; k++) {
|
// String s = ((x+k)+" "+y+" "+z);
|
int xx = x + k;
|
// System.out.println( s ) ;
|
for (int j = -range / 2; j <= range / 2; j++) {
|
String s = xx + " " + (y + j) + " " + z;
|
//System.out.println(s);
|
list.add(s.hashCode());
|
}
|
}
|
return list;
|
}*/
|
|
/* lon = 115.9165227 + 0.000116732 * (x-0.5) + 0.00000116862*(y-0.5)
|
lat = 39.77250000 + 0.000001000 * (x-0.5) - 0.00009000000*(y-0.5)*/
|
private static double getLon(int x, int y) {
|
double lon = 115.9165227 + 0.000116732 * (x - 0.5) + 0.00000116862 * (y - 0.5);
|
if (lon < 115 || lon > 116) {
|
System.out.println("lon is invalid");
|
}
|
return lon;
|
}
|
|
private static double getLat(int x, int y) {
|
double lat = 39.77250000 + 0.000001000 * (x - 0.5) - 0.00009000000 * (y - 0.5);
|
if (lat < 39 || lat > 40) {
|
System.out.println("lat is invalid");
|
}
|
return lat;
|
}
|
}
|
