[add]h5读取改造

张洋洋

2025-01-10 096abd16d07228aba3189302a14b24f56bb63abd

[add]h5读取改造

 pom.xml

@@ -275,6 +275,17 @@
        </repository>
    </repositories>
    <build>
        <resources>
            <resource>
                <directory>src/main/resources</directory>
                <includes>
                    <include>win32-x86-64/blosc.dll</include>
                    <include>**/*.yml</include>
                    <include>**/*.xml</include>
                </includes>
                <filtering>false</filtering>
            </resource>
        </resources>
        <finalName>SimuServer</finalName>
        <plugins>
            <plugin>

 src/main/java/com/se/simu/controller/SimuController.java

@@ -8,13 +8,11 @@
import com.se.simu.domain.dto.GeLayer;
import com.se.simu.domain.po.DataPo;
import com.se.simu.domain.po.SimuPo;
import com.se.simu.domain.vo.CreateFilesSimuVo;
import com.se.simu.domain.vo.CreateSimuVo;
import com.se.simu.domain.vo.R;
import com.se.simu.domain.vo.SimuVo;
import com.se.simu.domain.vo.*;
import com.se.simu.service.*;
import com.se.simu.utils.CsvToSQLiteUtils;
import com.se.simu.utils.ShpToolUtils;
import com.se.simu.utils.ZarrUtils;
import io.swagger.annotations.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@@ -271,13 +269,13 @@
        String stationRainFile = vo.getStationRainFile();
        log.info("站点雨量CSV文件地址：{}", floodFile);
        JSONArray array = new JSONArray();
        // åˆ›å»ºè¡¨å æ—¶é—´æˆ³
        String tableName = "station_rain_" + System.currentTimeMillis();
        try {
            // åˆ¤æ–­åœ°å€ä¸ä¸ºç©º
            if (StringUtils.isEmpty(stationRainFile)) {
                return fail("站点雨量CSV文件地址不能为空", false);
            }
            // åˆ›å»ºè¡¨å æ—¶é—´æˆ³
            String tableName = "station_rain_" + System.currentTimeMillis();
            // 1 è¯»å–CSV æ–‡ä»¶
            array = CsvToSQLiteUtils.readCsvSaveLocal(stationRainFile, tableName);
            // èŽ·å–仿真时间 duration
@@ -341,5 +339,19 @@
        }
    }

    @ApiOperation(value = "tarr文件组装")
    @GetMapping(value = "/testZarr", produces = "application/json; charset=UTF-8")
    public R<Boolean> testZarr(@RequestParam("name") String tableName) throws IOException {
        List<String> list = CsvToSQLiteUtils.getNameList(tableName);
        String basePath="D:\\城市内涝\\sem\\雨量站点数据\\降雨量zarr\\";
        for (String src : list
        ) {
            List<StationRainVo> stationRainVos = CsvToSQLiteUtils.getList(tableName, src);
            ZarrUtils.saveZarrRainfall(basePath+src,stationRainVos);
            ZarrUtils.saveZarrTime(basePath+src,stationRainVos);
            System.out.println(src+"的zarr数据生成=====================");
        }
        return success(true);
    }

}

 src/main/java/com/se/simu/domain/vo/StationRainVo.java

¶Ô±ÈÐÂÎļþ
@@ -0,0 +1,65 @@
package com.se.simu.domain.vo;


public class StationRainVo {

    private Long id;

    private String stationName;

    private Double rainfall;

    private Double longitude;

    private Double latitude;

    private String datetime;

    public Long getId() {
        return id;
    }

    public void setId(Long id) {
        this.id = id;
    }

    public String getStationName() {
        return stationName;
    }

    public void setStationName(String stationName) {
        this.stationName = stationName;
    }

    public Double getRainfall() {
        return rainfall;
    }

    public void setRainfall(Double rainfall) {
        this.rainfall = rainfall;
    }

    public Double getLongitude() {
        return longitude;
    }

    public void setLongitude(Double longitude) {
        this.longitude = longitude;
    }

    public Double getLatitude() {
        return latitude;
    }

    public void setLatitude(Double latitude) {
        this.latitude = latitude;
    }

    public String getDatetime() {
        return datetime;
    }

    public void setDatetime(String datetime) {
        this.datetime = datetime;
    }
}

 src/main/java/com/se/simu/utils/CsvToSQLiteUtils.java

@@ -3,6 +3,7 @@
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.se.simu.domain.vo.StationRainVo;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVRecord;

@@ -12,6 +13,10 @@
import java.nio.file.Files;
import java.nio.file.Paths;
import java.sql.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;

/**
 * CSV åˆ° SQLite å®žç”¨ç¨‹åº
@@ -268,7 +273,6 @@
        } catch (SQLException e) {
            System.err.println("SQLite连接失败: " + e.getMessage());
        }

        return null;
    }

@@ -299,4 +303,73 @@
        return 1440;
    }

    /**
     * åˆ†ç»„查询
     *
     * @param tableName è¡¨åç§°
     * @return {@link Double}
     */
    public static List<String> getNameList(String tableName) {
        // 1. åˆ›å»ºSQLite数据库连接
        try (Connection conn = DriverManager.getConnection("jdbc:sqlite:rainfall.db")) {
            if (conn != null) {
                // 2. æ‰§è¡ŒSQL查询
                String queryDataSql = "SELECT station_name FROM " + tableName + " GROUP BY station_name";
                // 3. å¤„理查询结果
                try (PreparedStatement pstmt = conn.prepareStatement(queryDataSql)) {
                    ResultSet rs = pstmt.executeQuery();
                    List<String> list = new ArrayList<>();
                    while (rs.next()) {
                        // èŽ·å–总和
                        list.add(rs.getString("station_name"));
                    }
                    return list;
                } catch (SQLException e) {
                    System.err.println("查询数据时出错: " + e.getMessage());
                }
            }
        } catch (SQLException e) {
            System.err.println("SQLite连接失败: " + e.getMessage());
        }
        return null;
    }

    /**
     * åˆ†ç»„查询
     *
     * @param tableName è¡¨åç§°
     * @return {@link Double}
     */
    public static List<StationRainVo> getList(String tableName, String name) {
        // 1. åˆ›å»ºSQLite数据库连接
        try (Connection conn = DriverManager.getConnection("jdbc:sqlite:rainfall.db")) {
            if (conn != null) {
                // 2. æ‰§è¡ŒSQL查询
                String queryDataSql = "SELECT * FROM " + tableName + " WHERE station_name='" + name + "' ORDER BY datetime asc";
                // 3. å¤„理查询结果
                try (PreparedStatement pstmt = conn.prepareStatement(queryDataSql)) {
                    ResultSet rs = pstmt.executeQuery();
                    List<StationRainVo> list = new ArrayList<>();
                    StationRainVo vo;
                    while (rs.next()) {
                        vo = new StationRainVo();
                        vo.setId(rs.getLong("id"));
                        vo.setStationName(rs.getString("station_name"));
                        vo.setRainfall(rs.getDouble("rainfall"));
                        vo.setLatitude(rs.getDouble("latitude"));
                        vo.setLongitude(rs.getDouble("longitude"));
                        vo.setDatetime(rs.getString("datetime"));
                        list.add(vo);
                    }
                    return list;
                } catch (SQLException e) {
                    System.err.println("查询数据时出错: " + e.getMessage());
                }
            }
        } catch (SQLException e) {
            System.err.println("SQLite连接失败: " + e.getMessage());
        }
        return null;
    }

}

 src/main/java/com/se/simu/utils/ZarrUtils.java

@@ -4,33 +4,70 @@
import com.bc.zarr.DataType;
import com.bc.zarr.ZarrArray;
import com.bc.zarr.ZarrGroup;
import com.se.simu.domain.vo.StationRainVo;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;

import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;

public class ZarrUtils {
    public static void main(String[] args) throws Exception{
        // è¾“å…¥ CSV æ–‡ä»¶çš„路径
        String csvFilePath = "D:\\城市内涝\\sem\\雨量站点数据\\雨量站包含坐标.csv";
        // è¾“出 Zarr æ–‡ä»¶çš„路径
        String zarrFilePath = "D:\\城市内涝\\sem\\雨量站点数据\\雨量站包含坐标.zarr";
        try (CSVParser csvParser = new CSVParser(new FileReader(csvFilePath), CSVFormat.DEFAULT.withFirstRecordAsHeader())) {
//    public static void main(String[] args) throws Exception {
//        // è¾“å…¥ CSV æ–‡ä»¶çš„路径
//        String csvFilePath = "D:\\城市内涝\\sem\\雨量站点数据\\雨量站包含坐标.csv";
//        // è¾“出 Zarr æ–‡ä»¶çš„路径
//        String zarrFilePath = "D:\\城市内涝\\sem\\雨量站点数据\\大庞村";
//        try (CSVParser csvParser = new CSVParser(new FileReader(csvFilePath), CSVFormat.DEFAULT.withFirstRecordAsHeader())) {
//            // èŽ·å– CSV åˆ—æ•°
//            int numColumns = csvParser.getHeaderMap().size();
//            // è®¡ç®— CSV è¡Œæ•°
//            int numRows = 0;
//            for (CSVRecord record : csvParser) {
//                numRows++;
//            }
//            // é‡æ–°åˆå§‹åŒ– CSV è§£æžå™¨ï¼Œå› ä¸ºä¹‹å‰çš„遍历已经耗尽了迭代器
//            csvParser.close();
//            CSVParser csvRecords = new CSVParser(new FileReader(csvFilePath), CSVFormat.DEFAULT.withFirstRecordAsHeader());
//            // åˆ›å»º Zarr ç»„
//            Path zarrPath = Paths.get(zarrFilePath);
//            ZarrGroup zarrGroup = ZarrGroup.create(zarrPath);
//            // å®šä¹‰ Zarr æ•°ç»„的维度
//            int[] shape = {numRows, numColumns};
//            // å®šä¹‰åˆ†å—大小
//            int[] chunks = {1440};
//            // åˆ›å»º Zarr æ•°ç»„
//            ArrayParams params = new ArrayParams();
//            params.shape(chunks);
//            params.chunks(chunks);
//            params.dataType(DataType.f4);
//            ZarrArray zarrArray = zarrGroup.createArray("rainfall", params);
//            int rowIndex = 0;
//            for (CSVRecord record : csvRecords) {
//                double value = Double.parseDouble(record.get(1));
//                zarrArray.write(value);
//                rowIndex++;
//            }
//            // å…³é—­ Zarr ç»„
//            //zarrGroup.close();
//        } catch (IOException e) {
//            e.printStackTrace();
//        }
//    }

    public static void saveZarrRainfall(String path, List<StationRainVo> stationRainVos){
        try {
            // èŽ·å– CSV åˆ—æ•°
            int numColumns = csvParser.getHeaderMap().size();
            int numColumns = 5;
            // è®¡ç®— CSV è¡Œæ•°
            int numRows = 0;
            for (CSVRecord record : csvParser) {
                numRows++;
            }
            // é‡æ–°åˆå§‹åŒ– CSV è§£æžå™¨ï¼Œå› ä¸ºä¹‹å‰çš„遍历已经耗尽了迭代器
            csvParser.close();
            CSVParser  csvRecords = new CSVParser(new FileReader(csvFilePath), CSVFormat.DEFAULT.withFirstRecordAsHeader());
            int numRows = stationRainVos.size();
            // åˆ›å»º Zarr ç»„
            Path zarrPath = Paths.get(zarrFilePath);
            Path zarrPath = Paths.get(path);
            ZarrGroup zarrGroup = ZarrGroup.create(zarrPath);
            // å®šä¹‰ Zarr æ•°ç»„的维度
            int[] shape = {numRows, numColumns};
@@ -43,24 +80,55 @@
            params.dataType(DataType.f4);
            ZarrArray zarrArray = zarrGroup.createArray("rainfall",params);
            int rowIndex = 0;
            for (CSVRecord record : csvRecords) {
                int[] x={rowIndex};
//                for (int colIndex = 0; colIndex < numColumns; colIndex++) {
//                    //double value = Double.parseDouble(record.get(colIndex));
//                    String value = record.get(colIndex);
//                    int[] y={colIndex};
//                    //zarrArray.write(value);
//                    zarrArray.write(value,x,y);
//                }
                double value = Double.parseDouble(record.get(1));
            for (StationRainVo record : stationRainVos) {
                double value = record.getRainfall();
                zarrArray.write(value);
                rowIndex++;
            }
            // å…³é—­ Zarr ç»„
            //zarrGroup.close();
        } catch (IOException e) {
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public static void saveZarrTime(String path, List<StationRainVo> stationRainVos){
        try {
            // èŽ·å– CSV åˆ—æ•°
            int numColumns = 5;
            // è®¡ç®— CSV è¡Œæ•°
            int numRows = stationRainVos.size();
            // åˆ›å»º Zarr ç»„
            Path zarrPath = Paths.get(path);
            ZarrGroup zarrGroup = ZarrGroup.create(zarrPath);
            // å®šä¹‰ Zarr æ•°ç»„的维度
            int[] shape = {numRows, numColumns};
            // å®šä¹‰åˆ†å—大小
            int[] chunks = {1440};
            // åˆ›å»º Zarr æ•°ç»„
            ArrayParams params = new ArrayParams();
            params.shape(chunks);
            params.chunks(chunks);
            params.dataType(DataType.i4);
            ZarrArray zarrArray = zarrGroup.createArray("time", params);
            int rowIndex = 0;
            SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm");
            for (StationRainVo record : stationRainVos) {
                long value = sdf.parse(record.getDatetime()).getTime();
                zarrArray.write(value);
                rowIndex++;
            }
            // å…³é—­ Zarr ç»„
            //zarrGroup.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public static void main(String[] args) throws Exception{
        System.out.println("2023/7/31 6:30");
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm");
        Date date = sdf.parse("2023/7/31 6:30");
        System.out.println(date.getTime());
    }
}

 src/main/resources/win32-x86-64/blosc.dll

Binary files differ