package com.yssh.utils; import com.yssh.dao.Suyuan100Csv2Mapper; import com.yssh.entity.*; import com.yssh.service.Suyuan100Csv2Service; import com.yssh.service.Yssh3dreliService; import com.yssh.service.YsshQxCsvService; import com.yssh.service.YsshSy200Service; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import javax.annotation.PostConstruct; import java.io.*; import java.text.DecimalFormat; import java.text.SimpleDateFormat; import java.util.*; //@Service //@EnableScheduling public class ReadProcessingData { static String distFilePath = "D:\\bigdata\\yssh\\test.bin"; static HashMap zHash = new HashMap(); static List checkPoints = new ArrayList(); static List checkPoints2d = new ArrayList(); @Autowired Suyuan100Csv2Service service; @Autowired Suyuan100Csv2Mapper mapper; @Autowired YsshSy200Service service200; @Autowired YsshQxCsvService ysshQxCsvService; @Autowired Yssh3dreliService yssh3dreliService; //fastdfs配置文件所在路径 private static String sourceFilePath; @Value("${spring.csv.path}") public void setFilePath(String path) { sourceFilePath = path; } public static String scanner(String tip) throws Exception { Scanner scanner = new Scanner(System.in); StringBuilder help = new StringBuilder(); help.append(tip + ":"); System.out.println(help.toString()); if (scanner.hasNext()) { String ipt = scanner.next(); if (ipt != null) { return ipt; } } throw new Exception("请输入正确的路径!"); } /** * 输入 * //每小时55分的时候执行 * //@Scheduled(cron = "0 55 * ? * *") * * @throws Exception 异常 */ //@Scheduled(fixedRate = 1000 * 60 * 60 * 24 * 360) public void enter() throws Exception { /* sourceFilePath = ReadProcessingData.scanner("请输入CSV路径"); File file = new File(sourceFilePath); while(! file.exists()){ System.out.println( sourceFilePath +" is not exist"); sourceFilePath = ReadProcessingData.scanner("请输入CSV路径"); file = new File(sourceFilePath); }*/ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHH"); String fName = sdf.format(new Date()); System.out.println("fName = " + fName); //sourceFilePath = sourceFilePath + "\\\\"+fName+".csv" ; System.out.println("目录:" + sourceFilePath + "\\\\" + fName + ".csv"); File file = new File(sourceFilePath + "\\\\" + fName + ".csv"); if (!file.exists()) { System.out.println(sourceFilePath + " is not exist"); // return ; } compressCSV(sourceFilePath + "\\\\" + fName + ".csv"); // range确定,那么查询范围就确定,对应的list事固定的,可以反复使用,避免消耗过多内存 List savelist = new ArrayList<>(); List savelist200 = new ArrayList<>(); /*for (Suyuan100Csv2 s : checkPoints) { System.out.println("============100============"); List list = ReadProcessingData.computeRangeList(s, 10); saveDB(savelist, s, list); System.out.println("============200============"); List list200 = ReadProcessingData.computeRangeList200(s, 20); saveDB200(savelist200, s, list200); }*/ System.out.println("============入库结束============"); //points47(fName); System.out.println("============47 结束============"); points473d(fName); System.out.println("============47 3d结束============"); //===================================================== analytical2dData(fName); System.out.println("============2d结束============"); //System.exit(0); } private void points473d(String fName) { int count = 1; DecimalFormat df = new DecimalFormat("##.######"); List list = new ArrayList(); for (Suyuan100Csv2 k : checkPoints) { Yssh3dreli y = new Yssh3dreli(); y.setName("AI-" + (count < 10 ? "0" + count : count + "")); count++; y.setLon(getLon(k.getX(), k.getY())); y.setLat(getLat(k.getX(), k.getY())); for (int i = 0; i < 10; i++) { int hash = Integer.parseInt(k.getX() + "" + k.getY() + "" + (k.getZ() + i)); if (i == 0) { y.setHeight0(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); } if (i == 1) { y.setHeight10(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); } if (i == 2) { y.setHeight20(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); } if (i == 3) { y.setHeight30(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); } if (i == 4) { y.setHeight40(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); } if (i == 5) { y.setHeight50(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); } if (i == 6) { y.setHeight60(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); } if (i == 7) { y.setHeight70(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); } if (i == 8) { y.setHeight80(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); } if (i == 9) { y.setHeight90(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); } } y.setTime(fName); list.add(y); } //使用最新的批量新增3D数据,并补充时间字段 yssh3dreliService.batchInsert3dreliData(list); //yssh3dreliService.saveOrUpdateBatch(list); } /** * 解析2d数据 * 1、按照mesh.dat取出4000多个点数据 * 2、根据4000多个点数据从csv文件抽取出对应点位的value值 * 3、记录抽取的时间 */ private void analytical2dData(String fName) { DecimalFormat df = new DecimalFormat("##.######"); List list = new ArrayList(); for (Suyuan100Csv2 k : checkPoints2d) { Yssh2dreli y = new Yssh2dreli(); int hash = Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ()); y.setLon((float) getLon(k.getX(), k.getY())); y.setLat((float) getLat(k.getX(), k.getY())); y.setTime(fName); y.setValue(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); list.add(y); } //使用最新的批量新增2D数据,并补充时间字段 if (list.size() > 0) { yssh3dreliService.batchInsert2dreliData(list); } } private void points47(String time) { int count = 1; DecimalFormat df = new DecimalFormat("##.######"); List list = new ArrayList(); for (Suyuan100Csv2 k : checkPoints) { YsshQxshCsv y = new YsshQxshCsv(); int hash = Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ()); Suyuan100Csv2 suyuan100Csv2 = new Suyuan100Csv2(); y.setName("AI-" + (count < 10 ? "0" + count : count + "")); y.setLon(getLon(k.getX(), k.getY())); y.setLat(getLat(k.getX(), k.getY())); y.setTime(time); y.setValue(zHash.get(hash) == null ? "0" : df.format(zHash.get(hash))); list.add(y); } ysshQxCsvService.saveBatch(list); } private void saveDB(List savelist, Suyuan100Csv2 s, List list) { DecimalFormat df = new DecimalFormat("##.######"); for (Suyuan100Csv2 k : list) { if (zHash.get(Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ())) != null) { k.setVocsValue(df.format(zHash.get(Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ())))); } else { k.setVocsValue("0"); } k.setId(Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ())); savelist.add(k); } System.out.print("save batch " + savelist.size()); if (savelist.size() > 0) { if (service.saveOrUpdateBatch(savelist)) { System.out.println(" " + s.getName() + " " + mapper.getCount() + " 成功!"); } else { System.err.println(s.getName() + " " + s.getVocsName() + " 失败!"); } } //saveList 释放内存 savelist.clear(); } private void saveDB200(List savelist, Suyuan100Csv2 s, List list) { DecimalFormat df = new DecimalFormat("##.######"); for (YsshSuyuan200 k : list) { if (zHash.get(Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ())) != null) { k.setVocsValue(df.format(zHash.get(Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ())))); } else { k.setVocsValue("0"); } k.setId(Integer.parseInt(k.getX() + "" + k.getY() + "" + k.getZ())); savelist.add(k); } System.out.print("save batch " + savelist.size()); if (savelist.size() > 0) { if (service200.saveOrUpdateBatch(savelist)) { System.out.println(s.getName() + " " + " 成功!"); } else { System.err.println(s.getName() + " " + s.getVocsName() + " 失败!"); } } //saveList 释放内存 savelist.clear(); } /** * 读dat,解析并存入list,经纬度,名称提前写好 */ @PostConstruct public void init() throws Exception { //File file = new File("./correct_jk_47.dat"); File file = new File("D:\\bigdata\\dat\\correct_jk_47.dat"); BufferedInputStream fis = new BufferedInputStream(new FileInputStream(file)); BufferedReader reader = new BufferedReader(new InputStreamReader(fis, "utf-8"), 7 * 1024 * 1024);// 用7M的缓冲读取文本文件 long count = 1; String line = ""; while ((line = reader.readLine()) != null) { String[] f = line.split(" "); Suyuan100Csv2 suyuan100Csv2 = new Suyuan100Csv2(); suyuan100Csv2.setName("AI-" + (count < 10 ? "0" + count : count + "")); suyuan100Csv2.setX(Integer.parseInt(f[0])); suyuan100Csv2.setY(Integer.parseInt(f[1])); suyuan100Csv2.setZ(Integer.parseInt(f[2])); suyuan100Csv2.setLon(getLon(suyuan100Csv2.getX(), suyuan100Csv2.getY())); suyuan100Csv2.setLat(getLat(suyuan100Csv2.getX(), suyuan100Csv2.getY())); checkPoints.add(suyuan100Csv2); count++; } reader.close(); //====================2D数据读取解析==================== File file2d = new File("D:\\bigdata\\dat\\extract_mesh.dat"); BufferedInputStream fis2d = new BufferedInputStream(new FileInputStream(file2d)); BufferedReader reader2d = new BufferedReader(new InputStreamReader(fis2d, "utf-8"), 7 * 1024 * 1024);// 用7M的缓冲读取文本文件 long count2d = 1; String line2d = ""; while ((line2d = reader2d.readLine()) != null) { String[] f2d = line2d.split(" "); Suyuan100Csv2 suyuan100Csv2 = new Suyuan100Csv2(); suyuan100Csv2.setName("AI-" + (count2d < 10 ? "0" + count2d : count2d + "")); suyuan100Csv2.setX(Integer.parseInt(f2d[0])); suyuan100Csv2.setY(Integer.parseInt(f2d[1])); suyuan100Csv2.setZ(Integer.parseInt(f2d[2])); suyuan100Csv2.setLon(getLon(suyuan100Csv2.getX(), suyuan100Csv2.getY())); suyuan100Csv2.setLat(getLat(suyuan100Csv2.getX(), suyuan100Csv2.getY())); checkPoints2d.add(suyuan100Csv2); count2d++; } reader2d.close(); } private void compressCSV(String path) { //设置日期格式 SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); // new Date()为获取当前系统时间 System.out.println("开始时间" + sdf.format(new Date())); long timer = System.currentTimeMillis(); try { File file = new File(path); BufferedInputStream fis = new BufferedInputStream(new FileInputStream(file)); // 用7M的缓冲读取文本文件 BufferedReader reader = new BufferedReader(new InputStreamReader(fis, "utf-8"), 7 * 1024 * 1024); long count = 1; // DataOutputStream out = new DataOutputStream( new BufferedOutputStream( new FileOutputStream(distFilePath))); String line = ""; while ((line = reader.readLine()) != null) { String[] f = line.split(","); if (Double.parseDouble(f[6]) > 0) { //out.writeInt((f[0] + " " + f[1] + " " + f[2]).hashCode()); //out.writeDouble(Double.parseDouble(f[6])); zHash.put(Integer.parseInt(f[0] + f[1] + f[2]), Double.parseDouble(f[6])); count++; if (count % 100000 == 0) { System.out.print("."); } if (count % 10000000 == 0) { System.out.println(count); } } } System.out.println(count); reader.close(); // out.flush(); // out.close(); } catch (IOException e) { System.out.println("读写文件失败!"); e.printStackTrace(); } timer = System.currentTimeMillis() - timer; System.out.println("处理时间:" + timer + " 毫秒"); System.out.println("结束时间:" + sdf.format(new Date())); } static public void data2Hash() { String fileName = "e:/2022/yssh/20221122.bin"; //设置日期格式 SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); // new Date()为获取当前系统时间 System.out.println("HASH开始时间" + sdf.format(new Date())); long timer = System.currentTimeMillis(); long sum = 0; try { DataInputStream in = new DataInputStream( new BufferedInputStream( new FileInputStream(fileName))); long length = in.available(); long count = 0; while (length > count) { int k = in.readInt(); double v = in.readDouble(); count = count + 12; sum = count; if (count % 100000 == 0) { System.out.print("."); } if (count % 10000000 == 0) { System.out.println(count / 1024); } zHash.put(k, v); } System.out.println("sum=" + sum); timer = System.currentTimeMillis() - timer; System.out.println("HASH处理时间:" + timer + " 毫秒"); System.out.println("HASH结束时间:" + sdf.format(new Date())); } catch (Exception e) { e.printStackTrace(); } } private static List computeRangeList(Suyuan100Csv2 s, int range) { List list = new ArrayList(); int x = s.getX(); int y = s.getY(); int z = s.getZ(); int i = 1; for (int k = -range / 2; k <= range / 2; k++) { // String s = ((x+k)+" "+y+" "+z); int xx = x + k; // System.out.println( s ) ; for (int j = -range / 2; j <= range / 2; j++) { String h = xx + " " + (y + j) + " " + z; Suyuan100Csv2 ss = new Suyuan100Csv2(); ss.setName(s.getName()); ss.setVocsName(s.getName() + "-" + i); ss.setLon(getLon(xx, y + j)); ss.setLat(getLat(xx, y + j)); ss.setX(xx); ss.setY(y + j); ss.setZ(z); list.add(ss); i++; } } return list; } private static List computeRangeList200(Suyuan100Csv2 s, int range) { List list = new ArrayList(); int x = s.getX(); int y = s.getY(); int z = s.getZ(); int i = 1; for (int k = -range / 2; k <= range / 2; k++) { // String s = ((x+k)+" "+y+" "+z); int xx = x + k; // System.out.println( s ) ; for (int j = -range / 2; j <= range / 2; j++) { String h = xx + " " + (y + j) + " " + z; YsshSuyuan200 ss = new YsshSuyuan200(); ss.setName(s.getName()); ss.setVocsName(s.getName() + "-" + i); ss.setLon(getLon(xx, y + j)); ss.setLat(getLat(xx, y + j)); ss.setX(xx); ss.setY(y + j); ss.setZ(z); list.add(ss); i++; } } return list; } private static List computeRange(int x, int y, int z, int range) { List list = new ArrayList(); for (int k = -range / 2; k <= range / 2; k++) { // String s = ((x+k)+" "+y+" "+z); int xx = x + k; // System.out.println( s ) ; for (int j = -range / 2; j <= range / 2; j++) { String s = xx + " " + (y + j) + " " + z; //System.out.println(s); list.add(s.hashCode()); } } return list; } /* lon = 115.9165227 + 0.000116732 * (x-0.5) + 0.00000116862*(y-0.5) lat = 39.77250000 + 0.000001000 * (x-0.5) - 0.00009000000*(y-0.5)*/ private static double getLon(int x, int y) { double lon = 115.9165227 + 0.000116732 * (x - 0.5) + 0.00000116862 * (y - 0.5); if (lon < 115 || lon > 116) { System.out.println("lon is invalid"); } return lon; } private static double getLat(int x, int y) { double lat = 39.77250000 + 0.000001000 * (x - 0.5) - 0.00009000000 * (y - 0.5); if (lat < 39 || lat > 40) { System.out.println("lat is invalid"); } return lat; } }