package com.terra.bigdata.util; import com.terra.bigdata.domain.*; import com.terra.bigdata.mapper.Suyuan100Csv2Mapper; import com.terra.bigdata.service.Suyuan100Csv2Service; import com.terra.bigdata.service.Yssh3dreliService; import com.terra.bigdata.service.YsshQxshCsvService; import com.terra.bigdata.service.YsshSuyuan200Service; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; import java.io.*; import java.text.DecimalFormat; import java.text.SimpleDateFormat; import java.util.*; @Service @EnableScheduling public class ReadData2 { static HashMap zHash = new HashMap(); static List checkPoints = new ArrayList(); static List checkPoints2d = new ArrayList(); @Autowired Suyuan100Csv2Service service; @Autowired Suyuan100Csv2Mapper mapper; @Autowired YsshSuyuan200Service service200; @Autowired YsshQxshCsvService ysshQxshCsvService; @Autowired Yssh3dreliService yssh3dreliService; //fastdfs配置文件所在路径 private static String sourceFilePath; @Value("${spring.saveLog.path}") public void setFilePath(String path) { this.sourceFilePath = path; } /* public static String scanner(String tip) throws Exception { Scanner scanner = new Scanner(System.in); StringBuilder help = new StringBuilder(); help.append(tip + ":"); System.out.println(help.toString()); if (scanner.hasNext()) { String ipt = scanner.next(); if (ipt != null) { return ipt; } } throw new Exception("请输入正确的路径!"); }*/ @Scheduled(fixedRate = 1000 * 60 * 60 * 24 * 360) public void enter() throws Exception { SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHH"); String fName = sdf.format(new Date()); System.out.println("fName = " + fName); //sourceFilePath = sourceFilePath + "\\\\"+fName+".csv" ; System.out.println("目录:" + sourceFilePath + "\\\\" + fName + ".csv"); File file = new File(sourceFilePath + "\\\\" + fName + ".csv"); if (!file.exists()) { System.out.println(sourceFilePath + " is not exist"); // return ; } compressCSV(sourceFilePath + "\\\\" + fName + ".csv"); // range确定,那么查询范围就确定,对应的list事固定的,可以反复使用,避免消耗过多内存 List savelist = new ArrayList<>(); List savelist200 = new ArrayList<>(); for (Suyuan100Csv2 s : checkPoints) { System.out.println("============100============"); List list = ReadData2.computeRangeList(s, 10); saveDB(savelist, s, list); System.out.println("============200============"); points473d(list, fName); System.out.println("============ 3d结束============"); List list200 = ReadData2.computeRangeList200(s, 20); saveDB200(savelist200, s, list200); } System.out.println("============入库结束============"); points47(fName); System.out.println("============47 结束============"); //===================================================== analytical2dData(fName); System.out.println("============2d结束============"); System.exit(0); } private void points473d(List ll, String fName) { DecimalFormat df = new DecimalFormat("##.######"); List list = new ArrayList(); for (Suyuan100Csv2 k : ll) { Yssh3dreli y = new Yssh3dreli(); y.setName(k.getName()); y.setLon(getLon(k.getX(), k.getY())); y.setLat(getLat(k.getX(), k.getY())); for (int i = 0; i < 10; i++) { int hash = Integer.parseInt(k.getX() + "" + k.getY() + "" + (k.getZ() + i)); if (i == 0) y.setHeight0(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); if (i == 1) y.setHeight10(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); if (i == 2) y.setHeight20(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); if (i == 3) y.setHeight30(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); if (i == 4) y.setHeight40(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); if (i == 5) y.setHeight50(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); if (i == 6) y.setHeight60(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); if (i == 7) y.setHeight70(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); if (i == 8) y.setHeight80(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); if (i == 9) y.setHeight90(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); } y.setTime(fName); list.add(y); } //使用最新的批量新增3D数据,并补充时间字段 yssh3dreliService.batchInsert3dreliData(list); //yssh3dreliService.saveOrUpdateBatch(list); } /** * 解析2d数据 * 1、按照mesh.dat取出4000多个点数据 * 2、根据4000多个点数据从csv文件抽取出对应点位的value值 * 3、记录抽取的时间 */ private void analytical2dData(String fName) { DecimalFormat df = new DecimalFormat("##.######"); List list = new ArrayList(); for (Suyuan100Csv2 k : checkPoints2d) { Yssh2dreli y = new Yssh2dreli(); int hash = Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ()); y.setLon(getLon(k.getX(), k.getY())); y.setLat(getLat(k.getX(), k.getY())); y.setTime(fName); y.setValue(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); list.add(y); } //使用最新的批量新增2D数据,并补充时间字段 if (list.size() > 0) { yssh3dreliService.batchInsert2dreliData(list); } } private void points47(String time) { int count = 1; DecimalFormat df = new DecimalFormat("##.######"); List list = new ArrayList(); for (Suyuan100Csv2 k : checkPoints) { YsshQxshCsv y = new YsshQxshCsv(); int hash = Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ()); y.setName("AI-" + (count < 10 ? "0" + count : count + "")); y.setLon(getLon(k.getX(), k.getY())); y.setLat(getLat(k.getX(), k.getY())); y.setTime(time); y.setValue(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); list.add(y); } ysshQxshCsvService.saveBatch(list); } private void saveDB(List savelist, Suyuan100Csv2 s, List list) { DecimalFormat df = new DecimalFormat("##.######"); for (Suyuan100Csv2 k : list) { if (zHash.get(Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ())) != null) { k.setVocsValue(df.format(zHash.get(Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ())))); } else k.setVocsValue("0"); k.setId(Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ())); savelist.add(k); } System.out.print("save batch " + savelist.size()); if (savelist.size() > 0) if (service.saveOrUpdateBatch(savelist)) { System.out.println(" " + s.getName() + " " + mapper.getCount() + " 成功!"); } else { System.err.println(s.getName() + " " + s.getVocsName() + " 失败!"); } savelist.clear(); } private void saveDB200(List savelist, Suyuan100Csv2 s, List list) { DecimalFormat df = new DecimalFormat("##.######"); for (YsshSuyuan200 k : list) { if (zHash.get(Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ())) != null) { k.setVocsValue(df.format(zHash.get(Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ())))); } else k.setVocsValue("0"); k.setId(Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ())); savelist.add(k); } System.out.print("save batch " + savelist.size()); if (savelist.size() > 0) if (service200.saveOrUpdateBatch(savelist)) { System.out.println(s.getName() + " " + " 成功!"); } else { System.err.println(s.getName() + " " + s.getVocsName() + " 失败!"); } savelist.clear(); } /** * 读dat,解析并存入list,经纬度,名称提前写好 */ @PostConstruct public void init() throws Exception { File file = new File("D:\\bigdata\\dat\\correct_jk_47.dat"); BufferedInputStream fis = new BufferedInputStream(new FileInputStream(file)); BufferedReader reader = new BufferedReader(new InputStreamReader(fis, "utf-8"), 7 * 1024 * 1024);// 用7M的缓冲读取文本文件 long count = 1; String line = ""; while ((line = reader.readLine()) != null) { String[] f = line.split(" "); Suyuan100Csv2 suyuan100Csv2 = new Suyuan100Csv2(); suyuan100Csv2.setName("AI-" + (count < 10 ? "0" + count : count + "")); suyuan100Csv2.setX(Integer.parseInt(f[0])); suyuan100Csv2.setY(Integer.parseInt(f[1])); suyuan100Csv2.setZ(Integer.parseInt(f[2])); suyuan100Csv2.setLon(getLon(suyuan100Csv2.getX(), suyuan100Csv2.getY())); suyuan100Csv2.setLat(getLat(suyuan100Csv2.getX(), suyuan100Csv2.getY())); checkPoints.add(suyuan100Csv2); count++; } reader.close(); //====================2D数据读取解析==================== File file2d = new File("D:\\bigdata\\dat\\extract_mesh.dat"); BufferedInputStream fis2d = new BufferedInputStream(new FileInputStream(file2d)); BufferedReader reader2d = new BufferedReader(new InputStreamReader(fis2d, "utf-8"), 7 * 1024 * 1024);// 用7M的缓冲读取文本文件 long count2d = 1; String line2d = ""; while ((line2d = reader2d.readLine()) != null) { String[] f2d = line2d.split(" "); Suyuan100Csv2 suyuan100Csv2 = new Suyuan100Csv2(); suyuan100Csv2.setName("AI-" + (count2d < 10 ? "0" + count2d : count2d + "")); suyuan100Csv2.setX(Integer.parseInt(f2d[0])); suyuan100Csv2.setY(Integer.parseInt(f2d[1])); suyuan100Csv2.setZ(Integer.parseInt(f2d[2])); suyuan100Csv2.setLon(getLon(suyuan100Csv2.getX(), suyuan100Csv2.getY())); suyuan100Csv2.setLat(getLat(suyuan100Csv2.getX(), suyuan100Csv2.getY())); checkPoints2d.add(suyuan100Csv2); count2d++; } reader2d.close(); } private void compressCSV(String path) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");//设置日期格式 System.out.println("开始时间" + sdf.format(new Date()));// new Date()为获取当前系统时间 long timer = System.currentTimeMillis(); try { File file = new File(path); BufferedInputStream fis = new BufferedInputStream(new FileInputStream(file)); BufferedReader reader = new BufferedReader(new InputStreamReader(fis, "utf-8"), 7 * 1024 * 1024);// 用7M的缓冲读取文本文件 long count = 1; String line = ""; while ((line = reader.readLine()) != null) { String[] f = line.split(","); if (Double.parseDouble(f[6]) > 0) { zHash.put(Integer.parseInt(f[0] + f[1] + f[2]), Double.parseDouble(f[6])); count++; if (count % 100000 == 0) System.out.print("."); if (count % 10000000 == 0) System.out.println(count); } } System.out.println(count); reader.close(); } catch (IOException e) { System.out.println("读写文件失败!"); e.printStackTrace(); } timer = System.currentTimeMillis() - timer; System.out.println("处理时间:" + timer + " 毫秒"); System.out.println("结束时间:" + sdf.format(new Date())); } /*static public void data2Hash() { String fileName = "e:/2022/yssh/20221122.bin"; SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");//设置日期格式 System.out.println("HASH开始时间" + sdf.format(new Date()));// new Date()为获取当前系统时间 long timer = System.currentTimeMillis(); long sum = 0; try { DataInputStream in = new DataInputStream( new BufferedInputStream( new FileInputStream(fileName))); long length = in.available(); long count = 0; while (length > count) { int k = in.readInt(); double v = in.readDouble(); count = count + 12; sum = count; if (count % 100000 == 0) System.out.print("."); if (count % 10000000 == 0) System.out.println(count / 1024); zHash.put(k, v); } System.out.println("sum=" + sum); timer = System.currentTimeMillis() - timer; System.out.println("HASH处理时间:" + timer + " 毫秒"); System.out.println("HASH结束时间:" + sdf.format(new Date())); } catch (Exception e) { e.printStackTrace(); } }*/ private static List computeRangeList(Suyuan100Csv2 s, int range) { List list = new ArrayList(); int x = s.getX(); int y = s.getY(); int z = s.getZ(); int i = 1; for (int k = -range / 2; k <= range / 2; k++) { int xx = x + k; for (int j = -range / 2; j <= range / 2; j++) { //String h = xx + " " + (y + j) + " " + z; Suyuan100Csv2 ss = new Suyuan100Csv2(); ss.setName(s.getName()); ss.setVocsName(s.getName() + "-" + i); ss.setLon(getLon(xx, y + j)); ss.setLat(getLat(xx, y + j)); ss.setX(xx); ss.setY(y + j); ss.setZ(z); list.add(ss); i++; } } return list; } private static List computeRangeList200(Suyuan100Csv2 s, int range) { List list = new ArrayList(); int x = s.getX(); int y = s.getY(); int z = s.getZ(); int i = 1; for (int k = -range / 2; k <= range / 2; k++) { int xx = x + k; for (int j = -range / 2; j <= range / 2; j++) { //String h = xx + " " + (y + j) + " " + z; YsshSuyuan200 ss = new YsshSuyuan200(); ss.setName(s.getName()); ss.setVocsName(s.getName() + "-" + i); ss.setLon(getLon(xx, y + j)); ss.setLat(getLat(xx, y + j)); ss.setX(xx); ss.setY(y + j); ss.setZ(z); list.add(ss); i++; } } return list; } /*private static List computeRange(int x, int y, int z, int range) { List list = new ArrayList(); for (int k = -range / 2; k <= range / 2; k++) { // String s = ((x+k)+" "+y+" "+z); int xx = x + k; // System.out.println( s ) ; for (int j = -range / 2; j <= range / 2; j++) { String s = xx + " " + (y + j) + " " + z; //System.out.println(s); list.add(s.hashCode()); } } return list; }*/ /* lon = 115.9165227 + 0.000116732 * (x-0.5) + 0.00000116862*(y-0.5) lat = 39.77250000 + 0.000001000 * (x-0.5) - 0.00009000000*(y-0.5)*/ private static double getLon(int x, int y) { double lon = 115.9165227 + 0.000116732 * (x - 0.5) + 0.00000116862 * (y - 0.5); if (lon < 115 || lon > 116) { System.out.println("lon is invalid"); } return lon; } private static double getLat(int x, int y) { double lat = 39.77250000 + 0.000001000 * (x - 0.5) - 0.00009000000 * (y - 0.5); if (lat < 39 || lat > 40) { System.out.println("lat is invalid"); } return lat; } }