From f1ea254a27f491b4039cb5a9c1a416b18480451b Mon Sep 17 00:00:00 2001 From: shiyi Date: Wed, 12 Jun 2024 14:33:40 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B0=94=E8=B1=A1=E6=9C=8D=E5=8A=A1=E5=88=9D?= =?UTF-8?q?=E7=89=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tablestore-grid-master/pom.xml | 2 +- weather-service/gfsDataConfig.json | 12 + weather-service/pom.xml | 34 ++- .../java/com/htfp/weather/config/Config.java | 4 +- .../com/htfp/weather/download/FileInfo.java | 1 + .../htfp/weather/download/GfsDataConfig.java | 3 +- .../htfp/weather/download/GfsDownloader.java | 20 +- .../common/GfsVariableHeightEnum.java | 17 +- .../common/GfsVariableIsobaricEnum.java | 10 + .../weather/griddata/common/TableConfig.java | 155 +++++++---- .../griddata/common/TableConfigBean.java | 143 ---------- .../griddata/common/TableConfigStatic.java | 106 +++++++ .../operation/BaseTableOperation.java | 10 +- .../griddata/operation/CreateTable.java | 6 +- .../griddata/operation/DataDeleter.java | 6 +- .../griddata/operation/GfsDataFetcher.java | 11 +- .../griddata/operation/GfsDataImport.java | 122 +++++--- .../weather/griddata/operation/QueryMeta.java | 6 +- .../griddata/operation/UpdateTable.java | 6 +- .../com/htfp/weather/info/GfsLevelsEnum.java | 9 + ...aProcesser.java => GridDataProcessor.java} | 18 +- .../com/htfp/weather/utils/DateTimeUtils.java | 4 +- .../htfp/weather/utils/HttpClientUtils.java | 26 +- .../com/htfp/weather/utils/MeteoUtils.java | 20 +- .../htfp/weather/web/config/LogAspect.java | 9 + .../web/controller/ConfigController.java | 160 +++++++++-- .../weather/web/controller/ThymeleafTest.java | 7 +- .../controller/UpperWeatherController.java | 55 +++- .../htfp/weather/web/exception/ErrorCode.java | 18 +- ...ConfigUpdate.java => GfsConfigUpdate.java} | 5 +- .../web/pojo/request/TableConfigUpdate.java | 20 ++ .../weather/web/pojo/response/FileInfo.java | 15 + .../web/pojo/response/MultiPointResponse.java | 9 + .../web/pojo/response/NowWeatherStatus.java | 16 +- .../weather/web/pojo/response/Result.java | 2 +- .../web/pojo/response/TimeSeriesDataset.java | 2 + .../htfp/weather/web/service/FileService.java | 64 +++++ .../web/service/GfsDataServiceImpl.java | 262 +++++++++++++----- .../service/surfaceapi/CaiYunServiceImpl.java | 6 +- .../main/resources/config/gfsDataConfig.json | 6 +- .../weather/download/GfsDownloaderTest.java | 6 +- .../weather/griddata/utils/GfsUtilsTest.java | 9 +- .../schedule/GridDataProcesserTest.java | 6 +- .../weather/web/service/FileServiceTest.java | 24 ++ weather-service/tableConf.json | 12 + 45 files changed, 1013 insertions(+), 451 deletions(-) create mode 100644 weather-service/gfsDataConfig.json delete mode 100644 weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigBean.java create mode 100644 weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigStatic.java rename weather-service/src/main/java/com/htfp/weather/schedule/{GridDataProcesser.java => GridDataProcessor.java} (87%) create mode 100644 weather-service/src/main/java/com/htfp/weather/web/config/LogAspect.java rename weather-service/src/main/java/com/htfp/weather/web/pojo/request/{ConfigUpdate.java => GfsConfigUpdate.java} (60%) create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/request/TableConfigUpdate.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/response/FileInfo.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/response/MultiPointResponse.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/service/FileService.java create mode 100644 weather-service/src/test/java/com/htfp/weather/web/service/FileServiceTest.java create mode 100644 weather-service/tableConf.json diff --git a/tablestore-grid-master/pom.xml b/tablestore-grid-master/pom.xml index fe13af0..8963410 100644 --- a/tablestore-grid-master/pom.xml +++ b/tablestore-grid-master/pom.xml @@ -6,7 +6,7 @@ com.aliyun.tablestore tablestore-grid - 1.0-SNAPSHOT + 1.1-SNAPSHOT diff --git a/weather-service/gfsDataConfig.json b/weather-service/gfsDataConfig.json new file mode 100644 index 0000000..b331af5 --- /dev/null +++ b/weather-service/gfsDataConfig.json @@ -0,0 +1,12 @@ +{ + "duration" : 30, + "minLon" : 70.0, + "maxLon" : 140.0, + "minLat" : 0.0, + "maxLat" : 55.0, + "resolution" : 0.25, + "variables" : [ "DZDT", "RH", "TMP", "UGRD", "VGRD", "TCDC" ], + "pressureLevels" : [ 400, 450, 500, 550, 600, 650, 700, 750, 800, 850, 900, 925, 950, 975, 1000 ], + "heightLevels" : [ 2, 10, 20, 30, 40, 50, 80, 100 ], + "saveRoot" : "./GFSData" +} \ No newline at end of file diff --git a/weather-service/pom.xml b/weather-service/pom.xml index 4bf56f9..a1f237d 100644 --- a/weather-service/pom.xml +++ b/weather-service/pom.xml @@ -47,15 +47,20 @@ - ai.djl - api - 0.27.0 - - - ai.djl.pytorch - pytorch-engine - 0.27.0 + com.aliyun.tablestore + tablestore-grid + 1.1-SNAPSHOT + + + + + + + + + + commons-io commons-io @@ -88,11 +93,6 @@ - - com.aliyun.tablestore - tablestore-grid - 1.0-SNAPSHOT - org.projectlombok @@ -110,6 +110,10 @@ hibernate-validator 6.0.16.Final + + org.apache.commons + commons-lang3 + @@ -141,8 +145,8 @@ spring-boot-maven-plugin ${spring-boot.version} - com.htfp.weather.WeatherServiceApplication - true + true + JAR diff --git a/weather-service/src/main/java/com/htfp/weather/config/Config.java b/weather-service/src/main/java/com/htfp/weather/config/Config.java index c539435..33979cc 100644 --- a/weather-service/src/main/java/com/htfp/weather/config/Config.java +++ b/weather-service/src/main/java/com/htfp/weather/config/Config.java @@ -1,10 +1,8 @@ package com.htfp.weather.config; import com.htfp.weather.griddata.common.TableConfig; -import com.htfp.weather.griddata.common.TableConfigBean; import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; import javax.annotation.Resource; /** @@ -15,6 +13,6 @@ import javax.annotation.Resource; @Component("config") public class Config { @Resource - TableConfigBean tableConfigBean; + TableConfig tableConfig; } diff --git a/weather-service/src/main/java/com/htfp/weather/download/FileInfo.java b/weather-service/src/main/java/com/htfp/weather/download/FileInfo.java index c8c8d23..98e831c 100644 --- a/weather-service/src/main/java/com/htfp/weather/download/FileInfo.java +++ b/weather-service/src/main/java/com/htfp/weather/download/FileInfo.java @@ -25,6 +25,7 @@ public class FileInfo { private String savePath; + private boolean isDownloadSuccess; public FileInfo() { } diff --git a/weather-service/src/main/java/com/htfp/weather/download/GfsDataConfig.java b/weather-service/src/main/java/com/htfp/weather/download/GfsDataConfig.java index 6fbf1c3..84c0349 100644 --- a/weather-service/src/main/java/com/htfp/weather/download/GfsDataConfig.java +++ b/weather-service/src/main/java/com/htfp/weather/download/GfsDataConfig.java @@ -75,7 +75,8 @@ public class GfsDataConfig { private String saveRoot; @JsonIgnore - private final String configPath = Objects.requireNonNull(this.getClass().getClassLoader().getResource("config")).getPath() + "/gfsDataConfig.json"; + // private final String configPath = Objects.requireNonNull(this.getClass().getClassLoader().getResource("config")).getPath() + "/gfsDataConfig.json"; + private final String configPath = System.getProperty("user.dir") + "/gfsDataConfig.json"; public void valid() { // TODO 2024/4/21: 校验 if (minLon >= maxLon) { diff --git a/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java b/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java index b17297a..8fc6162 100644 --- a/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java +++ b/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java @@ -73,9 +73,6 @@ public class GfsDownloader extends BaseDataDownloader { String url = fileInfo.getUrl(); File destDir = new File(fileInfo.getSavePath()); - if (!destDir.exists() && !destDir.mkdirs()) { - throw new RuntimeException("创建文件夹" + destDir + "失败"); - } File fileOut = new File(destDir, fileInfo.getFileName()); log.info("[GFS Download] 文件下载中，保存至 {}", fileOut); @@ -84,7 +81,9 @@ public class GfsDownloader extends BaseDataDownloader { // if (fileInfo.getForecastHour() == 12 || fileInfo.getForecastHour() == 18 || fileInfo.getForecastHour() == 24) { // throw new RuntimeException(); // } - FileUtils.copyURLToFile(new URL(url), fileOut, 30000, 30000); + // FileUtils.copyURLToFile(new URL(url), fileOut, 30000, 30000); + // FIXME 2024/6/8: 如果连接是https，jar包下载会报错 javax.net.ssl.SSLException: Received fatal alert: internal_error + HttpClientUtils.downloadFileByUrl(url, fileOut.getPath()); log.info("[GFS Download] 文件下载成功: {}", fileOut); fileInfo.setDownloadSuccess(fileValid(fileOut.getAbsolutePath())); } catch (Exception e) { @@ -134,6 +133,7 @@ public class GfsDownloader extends BaseDataDownloader { String levelsStr = null; String variablesStr = null; try { + gfsDataConfig.valid(); lonlatBoxStr = fillUrlForLonLat(); levelsStr = fillUrlForLevels(); variablesStr = fillUrlForVariables(); @@ -141,20 +141,26 @@ public class GfsDownloader extends BaseDataDownloader { log.error("[GFS Download] 文件下载参数生成错误"); throw new RuntimeException(e); } - + String refTimeStr = this.getRefTime().format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING)); + String savePath = gfsDataConfig.getSaveRoot() + "/" + refTimeStr; + File destDir = new File(savePath); + if (!destDir.exists() && !destDir.mkdirs()) { + throw new RuntimeException("创建文件夹" + destDir + "失败"); + } // 预报数据时间分辨率 final int forecastStep = 1; // 分辨率为3h，整数截断；包括目标时刻本身 final int nFiles = 1 + gfsDataConfig.getDuration() / forecastStep; // 存储文件信息 List fileInfoList = new ArrayList<>(nFiles); + for (int i = 0; i < nFiles; i++) { FileInfo fileInfo = new FileInfo(); int forecastHour = i * forecastStep + hourDiff; fileInfo.setForecastHour(forecastHour); String baseURL = getBaseURL(forecastHour, levelsStr, variablesStr, lonlatBoxStr); fileInfo.setUrl(baseURL); - fileInfo.setRefTimeStr(this.getRefTime().format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING))); + fileInfo.setRefTimeStr(refTimeStr); fileInfo.setForecastUTCTimeStr( this.getRefTime().plusHours(fileInfo.getForecastHour()) .format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING)) @@ -164,7 +170,7 @@ public class GfsDownloader extends BaseDataDownloader { .format(DateTimeFormatter.ofPattern(Constant.BJT_TIME_STRING)) ); fileInfo.setFileName(String.format("%s-from-%s+%d.grib2", fileInfo.getForecastBJTimeStr(), fileInfo.getRefTimeStr(), forecastHour)); - fileInfo.setSavePath(gfsDataConfig.getSaveRoot() + "/" + fileInfo.getRefTimeStr()); + fileInfo.setSavePath(savePath); fileInfoList.add(fileInfo); } return fileInfoList; diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableHeightEnum.java b/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableHeightEnum.java index ceff092..c913322 100644 --- a/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableHeightEnum.java +++ b/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableHeightEnum.java @@ -15,13 +15,15 @@ public enum GfsVariableHeightEnum { RH("humidity", "Relative_humidity_cover_height_above_ground"), WIND_SPEED("windSpeed", "Wind_speed_height_above_ground"), WIND360("wind360", "Wind_direction_height_above_ground"), - GUST("gustSpeed", "Wind_speed_gust_surface"), + // GUST("gustSpeed", "Wind_speed_gust_surface"), + PRECIP("precip", "Precipitation_rate_surface"), ; private static final ConcurrentHashMap variableName = new ConcurrentHashMap() {{ put("temp", "Temperature_height_above_ground"); put("humidity", "Relative_humidity_cover_height_above_ground"); put("windSpeed", "Wind_speed_height_above_ground"); put("wind360", "Wind_direction_height_above_ground"); + put("precip", "Precipitation_rate_surface"); }}; String nameInApi; String nameInFile; @@ -41,8 +43,17 @@ public enum GfsVariableHeightEnum { public static String getGfsVariableName(GfsVariableHeightEnum gfsVariableIsobaricEnum) { return variableName.get(gfsVariableIsobaricEnum.nameInApi); } - public static String getGfsVariableName(String tableVariableName) { - return variableName.get(tableVariableName); + public static String getGfsVariableName(String nameInApi) { + return variableName.get(nameInApi); + } + + public static String getVariableNameInApi(String nameInFile) { + for (GfsVariableHeightEnum anEnum : values()) { + if (anEnum.nameInFile.equals(nameInFile)) { + return anEnum.nameInApi; + } + } + return null; } public static List getVariableNamesInApi() { diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableIsobaricEnum.java b/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableIsobaricEnum.java index 154aa2c..ba9fc03 100644 --- a/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableIsobaricEnum.java +++ b/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableIsobaricEnum.java @@ -30,6 +30,7 @@ public enum GfsVariableIsobaricEnum { this.nameInFile = nameInFile; } + public String getNameInApi() { return nameInApi; } @@ -44,6 +45,15 @@ public enum GfsVariableIsobaricEnum { return variableName.get(tableVariableName); } + public static String getVariableNameInApi(String nameInFile) { + for (GfsVariableIsobaricEnum anEnum : values()) { + if (anEnum.nameInFile.equals(nameInFile)) { + return anEnum.nameInApi; + } + } + return null; + } + public static List getVariableNamesInApi() { return new ArrayList<>(variableName.keySet()); } diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfig.java b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfig.java index c7633a2..b6ada8e 100644 --- a/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfig.java +++ b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfig.java @@ -1,108 +1,143 @@ package com.htfp.weather.griddata.common; -import com.google.gson.Gson; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.htfp.weather.download.GfsDataConfig; +import com.htfp.weather.utils.JSONUtils; +import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; -import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import ucar.ma2.DataType; import javax.annotation.PostConstruct; +import javax.annotation.Resource; import java.io.FileInputStream; +import java.io.IOException; import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.*; -import java.util.stream.Collectors; +import java.nio.charset.StandardCharsets; +import java.nio.file.NoSuchFileException; +import java.util.List; -/** - * @Author : shiyi - * @Date : 2024/1/15 19:10 - * @Description : 表格meta配置 - */ -@Slf4j @Component +@Data @Component @Slf4j +@JsonIgnoreProperties(ignoreUnknown = true) public class TableConfig { - @Autowired + @Resource + @JsonIgnore GfsDataConfig dataConfig; - /** - * 表名和index名 - */ - public static String DATA_TABLE_NAME; - public static String META_TABLE_NAME; - public static String DATA_INDEX_NAME; - public static String META_INDEX_NAME; + + @Value("duration") + public String dataTableName; + public String metaTableName; + public String dataIndexName; + public String metaIndexName; /** * 数据属性：ID、本地数据文件名、目标变量名、变量维度大小、变量类型 */ - public static String DATA_DIR; - public static List VARIABLE_LIST; - public static int lonSize; - public static List lonList; - public static int latSize; - public static List latList; - public static int levSize; - public static List levList; - public static int timeSizeMax = 72; - public static List timeList; - public static DataType DATA_TYPE = DataType.FLOAT; + public String dataDir; + public List variableList; + public int lonSize; + public int latSize; + public int levSize; + public int timeToLive; + + @JsonIgnore + public int timeSizeMax = 99; + @JsonIgnore + public double[] lonList; + @JsonIgnore + public double[] latList; + @JsonIgnore + public int[] pressureList; + @JsonIgnore + public int[] heightList; + // @JsonIgnore + // public List forecastTimeList; // 预报时效列表 + @JsonIgnore + public DataType dataType = DataType.FLOAT; - // 数据过期时间，单位为秒 - public static int TIME_TO_LIVE; + // note 修改作用范围在classes目录下，resource文件下不改变 + @JsonIgnore + // private final String configPath = Objects.requireNonNull(this.getClass().getClassLoader().getResource("config/tableConf.json")).getPath(); + private final String configPath = System.getProperty("user.dir") +"/tableConf.json"; @PostConstruct private void initTableConfig() { - initDatasetConfig(); + readConfig(); initLonList(); initLatList(); initLevList(); } - - /** - * 初始化表格配置 - */ - public static void initDatasetConfig() { - log.info("init table config..."); - String path = Objects.requireNonNull(TableConfig.class.getClassLoader().getResource("config/tableConf.json")).getPath(); - try (InputStream f = new FileInputStream(path)){ - Gson gson = new Gson(); - Map map = gson.fromJson(IOUtils.toString(f), Map.class); - DATA_TABLE_NAME = (String) map.get("dataTableName"); - META_TABLE_NAME = (String) map.get("metaTableName"); - DATA_INDEX_NAME = (String) map.get("dataIndexName"); - META_INDEX_NAME = (String) map.get("metaIndexName"); - DATA_DIR = (String) map.get("dataDir"); - VARIABLE_LIST = (ArrayList) map.get("variableList"); - lonSize = ((Double) map.get("lonSize")).intValue(); - latSize = ((Double) map.get("latSize")).intValue(); - levSize = ((Double) map.get("levSize")).intValue(); - TIME_TO_LIVE = (int) (((Double) map.get("timeToLive")) * 24 *3600 ) ; // 配置文件的单位为天 + public void readConfig() { + // String pathSeparator = System.getProperty("file.separator"); + try (InputStream f = new FileInputStream(configPath)){ + String jsonStr = IOUtils.toString(f, StandardCharsets.UTF_8); + TableConfig tableConfig = JSONUtils.json2pojo(jsonStr, TableConfig.class); + this.dataTableName = tableConfig.getDataTableName(); + this.metaTableName = tableConfig.getMetaTableName(); + this.dataIndexName = tableConfig.getDataIndexName(); + this.metaIndexName = tableConfig.getMetaIndexName(); + this.variableList = tableConfig.getVariableList(); + this.dataDir = tableConfig.getDataDir(); + this.lonSize = tableConfig.getLonSize(); + this.latSize = tableConfig.getLatSize(); + this.levSize = dataConfig.getPressureLevels().length; + this.timeToLive = tableConfig.getTimeToLive(); + log.info("[config] 读取 DataTable 配置 : {}", configPath); + } catch (NoSuchFileException e) { + log.error("[config] 配置文件{}不存在", configPath, e); } catch (Exception ex) { throw new RuntimeException(ex); } } + public void writeConfig(TableConfig updateConfig) throws IOException { + writeConfig(updateConfig, configPath); + } + public void writeConfig(TableConfig updateConfig, String filePath) throws IOException { + this.dataTableName = updateConfig.getDataTableName(); + this.metaTableName = updateConfig.getMetaTableName(); + this.dataIndexName = updateConfig.getDataIndexName(); + this.metaIndexName = updateConfig.getMetaIndexName(); + this.variableList = updateConfig.getVariableList(); + this.dataDir = updateConfig.getDataDir(); + this.lonSize = updateConfig.getLonSize(); + this.latSize = updateConfig.getLatSize(); + this.levSize = updateConfig.getLevSize(); + // // FIXME 2024/5/10: 维度信息取决于数据下载，目前需要两个配置一起更新 + this.timeToLive = updateConfig.getTimeToLive(); + JSONUtils.pojo2jsonFile(this, filePath); + log.info("配置文件 {} 更新为: {}", filePath, updateConfig); + } + + public void valid() { + } + + private void initLonList() { double lonStart = dataConfig.getMinLon(); double res = dataConfig.getResolution(); - lonList = new ArrayList<>(); + lonList = new double[lonSize]; for (int i = 0; i < lonSize; i++) { - lonList.add(lonStart + i * res); + lonList[i] = lonStart + i * res; } } private void initLatList() { double latStart = dataConfig.getMinLat(); double res = dataConfig.getResolution(); - latList = new ArrayList<>(); + latList = new double[latSize]; for (int i = 0; i < latSize; i++) { - latList.add(latStart + i * res); + latList[i] = latStart + i * res; } } private void initLevList() { - levList = Arrays.stream(dataConfig.getPressureLevels()).boxed().collect(Collectors.toList()); - } + pressureList = dataConfig.getPressureLevels(); + heightList = dataConfig.getHeightLevels(); + levSize = pressureList.length + heightList.length; + } } diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigBean.java b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigBean.java deleted file mode 100644 index 5d916f0..0000000 --- a/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigBean.java +++ /dev/null @@ -1,143 +0,0 @@ -package com.htfp.weather.griddata.common; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.htfp.weather.download.GfsDataConfig; -import com.htfp.weather.utils.JSONUtils; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.IOUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import ucar.ma2.DataType; - -import javax.annotation.PostConstruct; -import javax.annotation.Resource; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.NoSuchFileException; -import java.util.List; -import java.util.Objects; - -@Data @Component @Slf4j -@JsonIgnoreProperties(ignoreUnknown = true) -public class TableConfigBean { - @Resource - @JsonIgnore - GfsDataConfig dataConfig; - @Value("duration") - public String dataTableName; - public String metaTableName; - public String dataIndexName; - public String metaIndexName; - - /** - * 数据属性：ID、本地数据文件名、目标变量名、变量维度大小、变量类型 - */ - public String dataDir; - public List variableList; - public int lonSize; - public int latSize; - public int levSize; - public int timeToLive; - - @JsonIgnore - public int timeSizeMax = 99; - @JsonIgnore - public double[] lonList; - @JsonIgnore - public double[] latList; - @JsonIgnore - public int[] pressureList; - @JsonIgnore - public int[] heightList; - // @JsonIgnore - // public List forecastTimeList; // 预报时效列表 - @JsonIgnore - public DataType dataType = DataType.FLOAT; - - - // note 修改作用范围在classes目录下，resource文件下不改变 - @JsonIgnore - private final String configPath = Objects.requireNonNull(this.getClass().getClassLoader().getResource("config/tableConf.json")).getPath(); - @PostConstruct - private void initTableConfig() { - readConfig(); - initLonList(); - initLatList(); - initLevList(); - } - - public void readConfig() { - // String pathSeparator = System.getProperty("file.separator"); - try (InputStream f = new FileInputStream(configPath)){ - String jsonStr = IOUtils.toString(f, StandardCharsets.UTF_8); - TableConfigBean tableConfig = JSONUtils.json2pojo(jsonStr, TableConfigBean.class); - this.dataTableName = tableConfig.getDataTableName(); - this.metaTableName = tableConfig.getMetaTableName(); - this.dataIndexName = tableConfig.getDataIndexName(); - this.metaIndexName = tableConfig.getMetaIndexName(); - this.variableList = tableConfig.getVariableList(); - this.dataDir = tableConfig.getDataDir(); - this.lonSize = tableConfig.getLonSize(); - this.latSize = tableConfig.getLatSize(); - this.levSize = dataConfig.getPressureLevels().length; - this.timeToLive = tableConfig.getTimeToLive(); - log.info("[config] 读取 DataTable 配置 : {}", configPath); - } catch (NoSuchFileException e) { - log.error("[config] 配置文件{}不存在", configPath, e); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - - public void writeConfig(TableConfigBean updateConfig) throws IOException { - writeConfig(updateConfig, configPath); - } - public void writeConfig(TableConfigBean updateConfig, String filePath) throws IOException { - this.dataTableName = updateConfig.getDataTableName(); - this.metaTableName = updateConfig.getMetaTableName(); - this.dataIndexName = updateConfig.getDataIndexName(); - this.metaIndexName = updateConfig.getMetaIndexName(); - this.variableList = updateConfig.getVariableList(); - this.dataDir = updateConfig.getDataDir(); - this.lonSize = updateConfig.getLonSize(); - this.latSize = updateConfig.getLatSize(); - this.levSize = updateConfig.getLevSize(); - // // FIXME 2024/5/10: 维度信息取决于数据下载，目前需要两个配置一起更新 - this.timeToLive = updateConfig.getTimeToLive(); - JSONUtils.pojo2jsonFile(this, filePath); - log.info("配置文件 {} 更新为: {}", filePath, updateConfig); - } - - public void valid() { - } - - - private void initLonList() { - double lonStart = dataConfig.getMinLon(); - double res = dataConfig.getResolution(); - lonList = new double[lonSize]; - for (int i = 0; i < lonSize; i++) { - lonList[i] = lonStart + i * res; - } - } - - private void initLatList() { - double latStart = dataConfig.getMinLat(); - double res = dataConfig.getResolution(); - latList = new double[latSize]; - for (int i = 0; i < latSize; i++) { - latList[i] = latStart + i * res; - } - } - - private void initLevList() { - pressureList = dataConfig.getPressureLevels(); - heightList = dataConfig.getHeightLevels(); - - levSize = pressureList.length + heightList.length; - } -} diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigStatic.java b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigStatic.java new file mode 100644 index 0000000..607e605 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigStatic.java @@ -0,0 +1,106 @@ +package com.htfp.weather.griddata.common; + +import com.google.gson.Gson; +import com.htfp.weather.download.GfsDataConfig; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import ucar.ma2.DataType; + +import javax.annotation.PostConstruct; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.*; +import java.util.stream.Collectors; + +/** + * @Author : shiyi + * @Date : 2024/1/15 19:10 + * @Description : 表格meta配置 + */ +@Slf4j @Component +public class TableConfigStatic { + @Autowired + GfsDataConfig dataConfig; + /** + * 表名和index名 + */ + public static String DATA_TABLE_NAME; + public static String META_TABLE_NAME; + public static String DATA_INDEX_NAME; + public static String META_INDEX_NAME; + + /** + * 数据属性：ID、本地数据文件名、目标变量名、变量维度大小、变量类型 + */ + public static String DATA_DIR; + public static List VARIABLE_LIST; + public static int lonSize; + public static List lonList; + public static int latSize; + public static List latList; + public static int levSize; + public static List levList; + public static int timeSizeMax = 72; + public static List timeList; + public static DataType DATA_TYPE = DataType.FLOAT; + + // 数据过期时间，单位为秒 + public static int TIME_TO_LIVE; + @PostConstruct + private void initTableConfig() { + initDatasetConfig(); + initLonList(); + initLatList(); + initLevList(); + } + + + /** + * 初始化表格配置 + */ + public static void initDatasetConfig() { + log.info("init table config..."); + String path = System.getProperty("user.dir") + "./tableConf.json"; + try (InputStream f = new FileInputStream(path)){ + Gson gson = new Gson(); + Map map = gson.fromJson(IOUtils.toString(f), Map.class); + DATA_TABLE_NAME = (String) map.get("dataTableName"); + META_TABLE_NAME = (String) map.get("metaTableName"); + DATA_INDEX_NAME = (String) map.get("dataIndexName"); + META_INDEX_NAME = (String) map.get("metaIndexName"); + DATA_DIR = (String) map.get("dataDir"); + VARIABLE_LIST = (ArrayList) map.get("variableList"); + lonSize = ((Double) map.get("lonSize")).intValue(); + latSize = ((Double) map.get("latSize")).intValue(); + levSize = ((Double) map.get("levSize")).intValue(); + TIME_TO_LIVE = (int) (((Double) map.get("timeToLive")) * 24 *3600 ) ; // 配置文件的单位为天 + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + private void initLonList() { + double lonStart = dataConfig.getMinLon(); + double res = dataConfig.getResolution(); + lonList = new ArrayList<>(); + for (int i = 0; i < lonSize; i++) { + lonList.add(lonStart + i * res); + } + } + + private void initLatList() { + double latStart = dataConfig.getMinLat(); + double res = dataConfig.getResolution(); + latList = new ArrayList<>(); + for (int i = 0; i < latSize; i++) { + latList.add(latStart + i * res); + } + } + + private void initLevList() { + levList = Arrays.stream(dataConfig.getPressureLevels()).boxed().collect(Collectors.toList()); + } + +} diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/BaseTableOperation.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/BaseTableOperation.java index 619692c..0228d41 100644 --- a/weather-service/src/main/java/com/htfp/weather/griddata/operation/BaseTableOperation.java +++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/BaseTableOperation.java @@ -3,14 +3,10 @@ package com.htfp.weather.griddata.operation; import com.aliyun.tablestore.grid.TableStoreGridConfig; import com.aliyun.tablestore.grid.TableStoreGrid; -import com.google.common.collect.Tables; import com.htfp.weather.griddata.common.TableConfig; -import com.htfp.weather.griddata.common.TableConfigBean; import com.htfp.weather.griddata.common.TableStoreConf; -import org.checkerframework.checker.units.qual.A; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.DependsOn; -import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; @@ -20,7 +16,7 @@ public abstract class BaseTableOperation{ protected TableStoreGrid tableStoreGrid; // private String pathSeperator = "/"; @Autowired - private TableConfigBean tableConfigBean; + private TableConfig tableConfig; @Autowired private TableStoreConf tableStoreConf; @@ -31,8 +27,8 @@ public abstract class BaseTableOperation{ config.setAccessId(tableStoreConf.getAccessId()); config.setAccessKey(tableStoreConf.getAccessKey()); config.setTableStoreInstance(tableStoreConf.getInstanceName()); - config.setDataTableName(tableConfigBean.getDataTableName()); - config.setMetaTableName(tableConfigBean.getMetaTableName()); + config.setDataTableName(tableConfig.getDataTableName()); + config.setMetaTableName(tableConfig.getMetaTableName()); tableStoreGrid = new TableStoreGrid(config); } diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/CreateTable.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/CreateTable.java index ca0cf7f..601b0f9 100644 --- a/weather-service/src/main/java/com/htfp/weather/griddata/operation/CreateTable.java +++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/CreateTable.java @@ -4,7 +4,7 @@ import com.alicloud.openservices.tablestore.model.TableOptions; import com.alicloud.openservices.tablestore.model.search.FieldSchema; import com.alicloud.openservices.tablestore.model.search.FieldType; import com.alicloud.openservices.tablestore.model.search.IndexSchema; -import com.htfp.weather.griddata.common.TableConfig; +import com.htfp.weather.griddata.common.TableConfigStatic; import java.util.Arrays; @@ -21,7 +21,7 @@ public class CreateTable extends BaseTableOperation { * @throws Exception */ private void createStore() throws Exception { - TableOptions tableOptions = new TableOptions(TableConfig.TIME_TO_LIVE, 1); + TableOptions tableOptions = new TableOptions(TableConfigStatic.TIME_TO_LIVE, 1); this.tableStoreGrid.createStore(tableOptions); } @@ -38,7 +38,7 @@ public class CreateTable extends BaseTableOperation { new FieldSchema("create_time", FieldType.LONG).setIndex(true).setEnableSortAndAgg(true), new FieldSchema("ref_time", FieldType.LONG).setIndex(true).setEnableSortAndAgg(true) )); - this.tableStoreGrid.createMetaIndex(TableConfig.META_INDEX_NAME, indexSchema); + this.tableStoreGrid.createMetaIndex(TableConfigStatic.META_INDEX_NAME, indexSchema); } public static void main(String[] args) throws Exception { diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/DataDeleter.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/DataDeleter.java index 8f1224e..51fff05 100644 --- a/weather-service/src/main/java/com/htfp/weather/griddata/operation/DataDeleter.java +++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/DataDeleter.java @@ -6,7 +6,7 @@ import com.aliyun.tablestore.grid.GridDataDeleter; import com.aliyun.tablestore.grid.core.RequestBuilder; import com.aliyun.tablestore.grid.model.DeleteDataParam; import com.aliyun.tablestore.grid.model.GridDataSetMeta; -import com.htfp.weather.griddata.common.TableConfig; +import com.htfp.weather.griddata.common.TableConfigStatic; import lombok.extern.slf4j.Slf4j; @@ -58,7 +58,7 @@ public class DataDeleter extends BaseTableOperation { Set refTimes = dataSetMeta.getForecastHours().stream().map(Integer::parseInt).collect(Collectors.toSet()); for (int t=0;t<3;t++) { for (int z=0; z<8; z++) { - DeleteDataParam param = new DeleteDataParam(TableConfig.DATA_TABLE_NAME, "UTC-20230910", "Total_cloud_cover_isobaric", t, z); + DeleteDataParam param = new DeleteDataParam(TableConfigStatic.DATA_TABLE_NAME, "UTC-20230910", "Total_cloud_cover_isobaric", t, z); //添加到batch操作中。 batchWriteRowRequest.addRowChange(RequestBuilder.buildDeleteDataRequest(param).getRowChange()); } @@ -86,7 +86,7 @@ public class DataDeleter extends BaseTableOperation { AsyncClient client = getClient(); //设置数据表名称。 - RangeRowQueryCriteria rangeRowQueryCriteria = new RangeRowQueryCriteria(TableConfig.DATA_TABLE_NAME); + RangeRowQueryCriteria rangeRowQueryCriteria = new RangeRowQueryCriteria(TableConfigStatic.DATA_TABLE_NAME); //设置起始主键。 PrimaryKeyBuilder primaryKeyBuilder = PrimaryKeyBuilder.createPrimaryKeyBuilder(); diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataFetcher.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataFetcher.java index 214f145..7417d87 100644 --- a/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataFetcher.java +++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataFetcher.java @@ -3,14 +3,13 @@ package com.htfp.weather.griddata.operation; import com.aliyun.tablestore.grid.GridDataFetcher; import com.aliyun.tablestore.grid.model.GridDataSet; -import com.htfp.weather.griddata.common.TableConfigBean; +import com.htfp.weather.griddata.common.TableConfig; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.DependsOn; import org.springframework.stereotype.Component; import ucar.ma2.Array; import javax.annotation.Resource; -import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -20,10 +19,10 @@ import java.util.List; * @Description : 根据维度信息获取数据 */ @Slf4j @Component -@DependsOn({"tableStoreConf","tableConfigBean"}) +@DependsOn({"tableStoreConf", "tableConfig"}) public class GfsDataFetcher extends BaseTableOperation { @Resource - TableConfigBean tableConfigBean; + TableConfig tableConfig; /** * 获取指定变量的四维数据(time, lev, lat, lon) @@ -60,7 +59,7 @@ public class GfsDataFetcher extends BaseTableOperation { */ public Array getPlane(String dataSetId, String variable, int iTime, int iLev) throws Exception { int[] origin = new int[] {iTime, iLev, 0, 0}; - int[] shape = new int[] {1, 1, tableConfigBean.latSize, tableConfigBean.lonSize}; + int[] shape = new int[] {1, 1, tableConfig.latSize, tableConfig.lonSize}; GridDataSet gridDataSet = queryByTableStore(dataSetId, Collections.singletonList(variable), origin, shape); Array array = gridDataSet.getVariable(variable).toArray(); @@ -100,7 +99,7 @@ public class GfsDataFetcher extends BaseTableOperation { */ public Array getProfile(String dataSetId, String variable, int iTime, int iLat, int iLon) throws Exception { int[] origin = new int[] {iTime, 0, iLat, iLon}; - int[] shape = new int[] {1, tableConfigBean.levSize, 1, 1}; + int[] shape = new int[] {1, tableConfig.levSize, 1, 1}; GridDataSet gridDataSet = queryByTableStore(dataSetId, Collections.singletonList(variable), origin, shape); Array array = gridDataSet.getVariable(variable).toArray(); diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataImport.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataImport.java index fcb7737..02ddf1c 100644 --- a/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataImport.java +++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataImport.java @@ -7,13 +7,15 @@ import com.aliyun.tablestore.grid.model.grid.Grid2D; import com.htfp.weather.griddata.common.GfsVariableHeightEnum; import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum; -import com.htfp.weather.griddata.common.TableConfigBean; +import com.htfp.weather.griddata.common.TableConfig; import com.htfp.weather.info.Constant; import com.htfp.weather.utils.MeteoUtils; import com.htfp.weather.web.exception.AppExcpetion; import com.htfp.weather.web.exception.ErrorCode; import lombok.Data; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.context.annotation.DependsOn; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -30,17 +32,19 @@ import java.time.format.DateTimeFormatter; import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; /** * @author shi_y */ @Slf4j @Component -@DependsOn("tableConfigBean") +@DependsOn("tableConfig") public class GfsDataImport extends BaseTableOperation { @Resource - TableConfigBean tableConfigBean; + TableConfig tableConfig; + + @Getter + boolean importing = false; private final ExecutorService executorService = Executors.newFixedThreadPool(5); /** @@ -92,7 +96,7 @@ public class GfsDataImport extends BaseTableOperation { } - public void importData(OffsetDateTime refTime) throws Exception { + public List importData(OffsetDateTime refTime) throws Exception { log.info("[tablestore] 数据导入开始, refTime = {}...", refTime); long start = System.currentTimeMillis(); List fileList = getFiles(refTime); @@ -100,32 +104,43 @@ public class GfsDataImport extends BaseTableOperation { throw new AppExcpetion(ErrorCode.NO_NC_OR_GRIB_FILES); } String dataSetId = refTime.format(DateTimeFormatter.ofPattern(Constant.DATA_SET_ID_FORMAT_STRING)); - List fileVariables = getFileVariables(tableConfigBean.variableList); - int[] shape = new int[]{tableConfigBean.timeSizeMax, tableConfigBean.levSize, tableConfigBean.latSize, tableConfigBean.lonSize}; - GridDataSetMeta meta = initMeta(dataSetId, tableConfigBean.dataType, fileVariables, shape); + List fileVariables = getFileVariables(tableConfig.variableList); + int[] shape = new int[]{tableConfig.timeSizeMax, tableConfig.levSize, tableConfig.latSize, tableConfig.lonSize}; + GridDataSetMeta meta = initMeta(dataSetId, tableConfig.dataType, fileVariables, shape); List forecastHours = new ArrayList<>(); // 记录到数据库属性中 // done 2024/5/13: 待优化，用于数据库的索引，必须连续，因此必须保证前一个时刻成功导入后才能导入下一个时刻，数据量大的时候导入时间较长 // done 2024/5/13: 优化方案，使用多线程，并使用Future来获取结果，但是必须保证每个时刻都成功导入 - List> futures = new ArrayList<>(); - for (int i = 0; i < fileList.size(); i++) { - String file = fileList.get(i); - int iTime = i; - futures.add(executorService.submit(() -> importFromNcFile(meta, file, iTime))); - // ImportResult importResult = importFromNcFile(meta, file, iTime); - // forecastHours.add(String.valueOf(importResult.getForcastHour())); - } - for (Future future : futures) { - ImportResult importResult = future.get(); - forecastHours.add(String.valueOf(importResult.getForcastHour())); + List finishedList; + try { + finishedList = new ArrayList<>(); + for (int i = 0; i < fileList.size(); i++) { + // for(int i = 0; i < 200; i++) { + String file = fileList.get(i); + int iTime = i; + int forecastHour = getForecastHourFromFilename(file); + // futures.add(executorService.submit(() -> importFromNcFile(meta, file, iTime))); + ImportResult importResult = importFromNcFile(meta, file, iTime, forecastHour); + finishedList.add(importResult); + // ImportResult importResult = importFromNcFile(meta, file, 0, forecastHour); + if (importResult.isSuccess()) { + forecastHours.add(String.valueOf(importResult.getForecastHour())); + } + } + } finally { + importing = false; } + // for (Future future : futures) { + // ImportResult importResult = future.get(); + // forecastHours.add(String.valueOf(importResult.getForcastHour())); + // } long end = System.currentTimeMillis(); log.info("[tablestore] 数据导入完成, 耗时: {} s, forecastHours: {}", (end - start)/1000., forecastHours); meta.setForecastHours(forecastHours); meta.addAttribute("reference_time", refTime.toInstant().toEpochMilli()); putMeta(meta); - + return finishedList; } /** @@ -135,13 +150,17 @@ public class GfsDataImport extends BaseTableOperation { * @param file netcdf文件路径 * @param iTime 文件序号 */ - public ImportResult importFromNcFile(GridDataSetMeta meta, String file, int iTime) { - int time = 0; - try { + public ImportResult importFromNcFile(GridDataSetMeta meta, String file, int iTime, int forecastHour) { + try (NetcdfFile ncFile = NetcdfFiles.open(file)) { GridDataWriter writer = tableStoreGrid.getDataWriter(meta); - NetcdfFile ncFile = NetcdfFiles.open(file); + // 相对reftime的小时数，文件可能缺失，因此time可能是不连续的，但是iTime是连续的 - time = (int) ncFile.findVariable("time").read().getDouble(0); + if (forecastHour == -1) { + forecastHour = (int) ncFile.findVariable("time").read().getDouble(0); + } + int xsize = meta.getxSize(); + int ysize = meta.getySize(); + for (String variableName : meta.getVariables()) { if ("Wind_speed_gust_surface".equals(variableName) || "Wind_speed_isobaric".equals(variableName) || "Wind_direction_isobaric".equals(variableName) || "Wind_speed_height_above_ground".equals(variableName) || "Wind_direction_height_above_ground".equals(variableName)) { @@ -155,26 +174,33 @@ public class GfsDataImport extends BaseTableOperation { int zSize = variable.getShape(1); for (int z = 0; z < zSize; z++) { // 注意高度索引是递增排序 - Array array = variable.read(new int[]{0, z, 0, 0}, new int[]{1, 1, meta.getxSize(), meta.getySize()}); + Array array = variable.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize}); transferUnit(variableName, array); Grid2D grid2D = new Grid2D(array.getDataAsByteBuffer(), variable.getDataType(), - new int[]{0, 0}, new int[]{meta.getxSize(), meta.getySize()}); + new int[]{0, 0}, new int[]{xsize, ysize}); writer.writeGrid2D(variableName, iTime, z, grid2D); } - } // TODO 2024/5/23: 导入不含高度坐标的数据 + } else if (shapeLength == 3) { + // DONE 2024/5/24: 导入不含高度坐标的数据 + Array array = variable.read(new int[]{0, 0, 0}, new int[]{1,xsize, ysize}); + transferUnit(variableName, array); + Grid2D grid2D = new Grid2D(array.getDataAsByteBuffer(), variable.getDataType(), + new int[]{0, 0}, new int[]{xsize, ysize}); + writer.writeGrid2D(variableName, iTime, 0, grid2D); + } + } else { log.warn("[tablestore] 数据文件 {} 中没有变量 {}", ncFile.getLocation(), variableName); } } // 导入风速风向 - importWind(meta, ncFile, iTime); - ncFile.close(); + importWind(writer, meta, ncFile, iTime); } catch (Exception e) { - log.error("[tablestore] 导入文件数据失败,_t={}: {}", iTime, file, e); - return new ImportResult(false, file, time, iTime); + log.error("[tablestore] 导入文件数据失败,_t={}: {}", forecastHour, file, e); + return new ImportResult(false, file, forecastHour, iTime); } - log.info("[tablestore] 导入文件数据成功,_t={}: {}", iTime, file); - return new ImportResult(true, file, time, iTime); + log.info("[tablestore] 导入文件数据成功,_t={}: {}", forecastHour, file); + return new ImportResult(true, file, forecastHour, iTime); } private void transferUnit(String variableName, Array array) { @@ -184,9 +210,12 @@ public class GfsDataImport extends BaseTableOperation { if (GfsVariableHeightEnum.TEMP.getNameInFile().equals(variableName)) { MeteoUtils.kelvin2Celsius(array); } + if (GfsVariableHeightEnum.PRECIP.getNameInFile().equals(variableName)) { + MeteoUtils.precipRate2mmPerHour(array); + } } - private void importWind(GridDataSetMeta meta, NetcdfFile ncFile, int iTime) throws Exception { + private void importWind(GridDataWriter writer, GridDataSetMeta meta, NetcdfFile ncFile, int iTime) throws Exception { // TODO 2024/5/8: 风速风向需要保存到文件中 for (String suffix : new String[]{"_isobaric", "_height_above_ground"}) { Variable uwnd = ncFile.findVariable("u-component_of_wind" + suffix); @@ -197,7 +226,6 @@ public class GfsDataImport extends BaseTableOperation { int xsize = meta.getxSize(); int ysize = meta.getySize(); int zsize = uwnd.getShape(1); - GridDataWriter writer = tableStoreGrid.getDataWriter(meta); for (int z = 0; z < zsize; z++) { Array uwndArray = uwnd.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize}); Array vwndArray = vwnd.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize}); @@ -214,7 +242,19 @@ public class GfsDataImport extends BaseTableOperation { } } } - + /** 从文件名中获取预报时效 */ + private int getForecastHourFromFilename(String filename) { + try { + String s = filename.split("\\+")[1].split(".grib2")[0]; + if (StringUtils.isEmpty(s)) { + return -1; + } else { + return Integer.parseInt(s); + } + } catch (Exception e) { + return -1; + } + } private List getFileVariables(List variables) { List fileVariables = new ArrayList<>(); for (String variable : variables) { @@ -243,7 +283,7 @@ public class GfsDataImport extends BaseTableOperation { */ private List getFiles(OffsetDateTime refTime) { String dataFolder = refTime.format(DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING)); - File fileDir = new File(tableConfigBean.dataDir + dataFolder); + File fileDir = new File(tableConfig.dataDir, dataFolder); if (!fileDir.exists()) { log.warn("文件夹 {} 不存在", fileDir); return null; @@ -271,13 +311,13 @@ public class GfsDataImport extends BaseTableOperation { public static class ImportResult { private boolean success; private String file; - private int forcastHour; + private int forecastHour; private int iTime; - public ImportResult(boolean success, String file, int forcastHour, int iTime) { + public ImportResult(boolean success, String file, int forecastHour, int iTime) { this.success = success; this.file = file; - this.forcastHour = forcastHour; + this.forecastHour = forecastHour; this.iTime = iTime; } } diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/QueryMeta.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/QueryMeta.java index 46c5fd2..350354b 100644 --- a/weather-service/src/main/java/com/htfp/weather/griddata/operation/QueryMeta.java +++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/QueryMeta.java @@ -7,7 +7,7 @@ import com.aliyun.tablestore.grid.core.QueryBuilder; import com.aliyun.tablestore.grid.model.GridDataSetMeta; import com.aliyun.tablestore.grid.model.QueryGridDataSetResult; import com.aliyun.tablestore.grid.model.QueryParams; -import com.htfp.weather.griddata.common.TableConfigBean; +import com.htfp.weather.griddata.common.TableConfig; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -23,7 +23,7 @@ import java.util.List; @Component public class QueryMeta extends BaseTableOperation { @Resource - TableConfigBean tableConfigBean; + TableConfig tableConfig; /** * 查询最新的数据集信息 * @@ -33,7 +33,7 @@ public class QueryMeta extends BaseTableOperation { public GridDataSetMeta getLastGridDataSetMeta() throws Exception { QueryGridDataSetResult result = tableStoreGrid.queryDataSets( - tableConfigBean.metaIndexName, + tableConfig.metaIndexName, QueryBuilder.and() .equal("status", "DONE") .build(), diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/UpdateTable.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/UpdateTable.java index 243a1e7..eed74e2 100644 --- a/weather-service/src/main/java/com/htfp/weather/griddata/operation/UpdateTable.java +++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/UpdateTable.java @@ -1,7 +1,7 @@ package com.htfp.weather.griddata.operation; import com.alicloud.openservices.tablestore.model.TableOptions; -import com.htfp.weather.griddata.common.TableConfig; +import com.htfp.weather.griddata.common.TableConfigStatic; /** * @Author : shiyi @@ -11,12 +11,12 @@ import com.htfp.weather.griddata.common.TableConfig; public class UpdateTable extends BaseTableOperation { private void update() throws Exception { - TableOptions tableOptions = new TableOptions(TableConfig.TIME_TO_LIVE); + TableOptions tableOptions = new TableOptions(TableConfigStatic.TIME_TO_LIVE); this.tableStoreGrid.updateStoreOption(tableOptions); } public static void main(String[] args) { - TableConfig.initDatasetConfig(); + TableConfigStatic.initDatasetConfig(); UpdateTable example = new UpdateTable(); try { example.update(); diff --git a/weather-service/src/main/java/com/htfp/weather/info/GfsLevelsEnum.java b/weather-service/src/main/java/com/htfp/weather/info/GfsLevelsEnum.java index 423fad0..bdb7243 100644 --- a/weather-service/src/main/java/com/htfp/weather/info/GfsLevelsEnum.java +++ b/weather-service/src/main/java/com/htfp/weather/info/GfsLevelsEnum.java @@ -43,6 +43,15 @@ public enum GfsLevelsEnum { public String getInfo() { return info; } + + public static GfsLevelsEnum getByCode(Integer code) { + for (GfsLevelsEnum c : GfsLevelsEnum.values()) { + if (c.getCode().equals(code)) { + return c; + } + } + return null; + } public static boolean contains(Integer test) { for (GfsLevelsEnum c : GfsLevelsEnum.values()) { if (c.getCode().equals(test)) { diff --git a/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java b/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java similarity index 87% rename from weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java rename to weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java index 87bb980..7592d0c 100644 --- a/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java +++ b/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java @@ -6,16 +6,15 @@ import com.htfp.weather.download.GfsDownloader; import com.htfp.weather.griddata.operation.GfsDataImport; import com.htfp.weather.info.Constant; import com.htfp.weather.web.exception.AppExcpetion; +import com.htfp.weather.web.service.FileService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; -import org.checkerframework.checker.units.qual.A; import org.springframework.context.annotation.DependsOn; import org.springframework.stereotype.Component; import javax.annotation.Resource; import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.time.LocalDateTime; import java.time.OffsetDateTime; import java.time.ZoneOffset; @@ -30,7 +29,7 @@ import java.util.List; @Component @Slf4j @DependsOn({"gfsDataImport", "gfsDownloader"}) -public class GridDataProcesser { +public class GridDataProcessor { @Resource GfsDataImport gfsDataImport; @Resource @@ -38,16 +37,17 @@ public class GridDataProcesser { @Resource GfsDataConfig gfsDataConfig; + public void dailyDataProcess() throws Exception { gfsDownloader.iniTimeSetting(); OffsetDateTime refTime = gfsDownloader.getRefTime(); List fileInfoList = gfsDownloader.getFilesInfo(); // if (allComplete) { - try { - gfsDataImport.importData(refTime); - } catch (AppExcpetion e) { - log.info("导入数据失败"); - // } + try { + gfsDataImport.importData(refTime) + } catch (AppExcpetion e) { + log.info("导入数据失败"); + } } @@ -63,7 +63,7 @@ public class GridDataProcesser { OffsetDateTime refTime = LocalDateTime.parse(subDir.getName(), DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING)).atOffset(ZoneOffset.UTC); if (refTime.isBefore(now.minusDays(3))) { try { - FileUtils.deleteDirectory(subDir); + FileUtils.deleteDirectory(subDir); log.info("删除过期数据文件夹成功： {}", subDir.getName()); } catch (IOException e) { log.info("删除过期数据文件夹失败： {}", subDir.getName()); diff --git a/weather-service/src/main/java/com/htfp/weather/utils/DateTimeUtils.java b/weather-service/src/main/java/com/htfp/weather/utils/DateTimeUtils.java index 8508aee..da11e0a 100644 --- a/weather-service/src/main/java/com/htfp/weather/utils/DateTimeUtils.java +++ b/weather-service/src/main/java/com/htfp/weather/utils/DateTimeUtils.java @@ -49,8 +49,8 @@ public class DateTimeUtils { } - public static OffsetDateTime getUTCDateTime(OffsetDateTime localDateTime) { - return localDateTime.withOffsetSameInstant(ZoneOffset.UTC); + public static OffsetDateTime getUTCDateTime(OffsetDateTime offsetDateTime) { + return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC); } // public static LocalDateTime getUTCDateTime(String timeString, String formatStr) { diff --git a/weather-service/src/main/java/com/htfp/weather/utils/HttpClientUtils.java b/weather-service/src/main/java/com/htfp/weather/utils/HttpClientUtils.java index 7c7c03e..cc84aaf 100644 --- a/weather-service/src/main/java/com/htfp/weather/utils/HttpClientUtils.java +++ b/weather-service/src/main/java/com/htfp/weather/utils/HttpClientUtils.java @@ -9,9 +9,13 @@ import okhttp3.ResponseBody; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; +import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -27,7 +31,8 @@ public class HttpClientUtils { .readTimeout(60, TimeUnit.SECONDS) .writeTimeout(60, TimeUnit.SECONDS) .build(); - + private static final OkHttpClient DONLOAD_OKHTTP_CLIENT = new OkHttpClient.Builder() + .build(); // public static T sendGet(String url, Map params, Class responseClass) { // MultiValueMap queryParams = new LinkedMultiValueMap<>(); // if (params != null) { @@ -83,4 +88,23 @@ public class HttpClientUtils { } } + + public static boolean downloadFileByUrl(String sourceUrl, String localPath) throws IOException { + File file = new File(localPath); + Request request = new Request.Builder() + .url(sourceUrl) + .build(); + + try (Response response = DONLOAD_OKHTTP_CLIENT.newCall(request).execute()) { + if (response.code() == 200) { + InputStream stream = response.body().byteStream(); + Files.copy(stream, file.toPath(), StandardCopyOption.REPLACE_EXISTING); + } else { + return false; + } + } + + return true; + } + } diff --git a/weather-service/src/main/java/com/htfp/weather/utils/MeteoUtils.java b/weather-service/src/main/java/com/htfp/weather/utils/MeteoUtils.java index 2f34446..7836822 100644 --- a/weather-service/src/main/java/com/htfp/weather/utils/MeteoUtils.java +++ b/weather-service/src/main/java/com/htfp/weather/utils/MeteoUtils.java @@ -1,17 +1,8 @@ package com.htfp.weather.utils; -import ai.djl.ndarray.NDArray; -import ai.djl.ndarray.NDManager; -import ai.djl.ndarray.types.DataType; -import ai.djl.ndarray.types.Shape; -import org.checkerframework.checker.units.qual.A; import ucar.ma2.Array; import ucar.ma2.ArrayFloat; -import ucar.ma2.Index; -import ucar.nc2.units.DateType; -import java.nio.ByteBuffer; -import java.text.DecimalFormat; import java.util.Arrays; /** @@ -46,7 +37,16 @@ public class MeteoUtils { // public static double height2Pressure(double pressure) { // // } - + /** + * @param array 降水率 (kg/m^2/s) + * @return mm/h + */ + public static Array precipRate2mmPerHour(Array array) { + for (int i = 0; i < array.getSize(); i++) { + array.setFloat(i, array.getFloat(i)*3600); + } + return array; + } /** * 开尔文转摄氏度 */ diff --git a/weather-service/src/main/java/com/htfp/weather/web/config/LogAspect.java b/weather-service/src/main/java/com/htfp/weather/web/config/LogAspect.java new file mode 100644 index 0000000..4f1dcda --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/config/LogAspect.java @@ -0,0 +1,9 @@ +package com.htfp.weather.web.config; + +/** + * @Author : shiyi + * @Date : 2024/6/12 14:12 + * @Description : 日志切面 + */ +public class LogAspect { +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java b/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java index bb68bf5..36159fe 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java +++ b/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java @@ -3,19 +3,27 @@ package com.htfp.weather.web.controller; import com.htfp.weather.download.FileInfo; import com.htfp.weather.download.GfsDataConfig; import com.htfp.weather.download.GfsDownloader; -import com.htfp.weather.griddata.common.TableConfigBean; +import com.htfp.weather.griddata.common.TableConfig; import com.htfp.weather.griddata.operation.GfsDataImport; +import com.htfp.weather.griddata.operation.GfsDataImport.ImportResult; +import com.htfp.weather.info.Constant; +import com.htfp.weather.utils.DateTimeUtils; import com.htfp.weather.web.exception.AppExcpetion; import com.htfp.weather.web.exception.ErrorCode; +import com.htfp.weather.web.pojo.request.GfsConfigUpdate; +import com.htfp.weather.web.pojo.request.TableConfigUpdate; import com.htfp.weather.web.pojo.response.Result; +import com.htfp.weather.web.service.FileService; +import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; import javax.annotation.Resource; +import java.io.File; import java.io.IOException; import java.time.OffsetDateTime; -import java.util.Collections; +import java.time.format.DateTimeFormatter; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -31,48 +39,66 @@ import java.util.stream.Collectors; @RequestMapping("/htfp/weather/config") public class ConfigController { @Resource - TableConfigBean tableConfigBean; + TableConfig tableConfig; @Resource GfsDataConfig gfsDataConfig; @Resource GfsDownloader gfsDownloader; @Resource GfsDataImport gfsDataImport; - + @Resource + FileService fileService; private final String SECRET = "htfpweather"; private void validSecret(String secret) { - if (!SECRET.equals(secret)) { + if (StringUtils.isEmpty(secret) || !SECRET.equals(secret)) { throw new AppExcpetion(ErrorCode.SECRET_ERROR); } } + + /** + * 查询tablestore数据库配置 + */ @RequestMapping("/queryDatabaseConfig") public Result queryDatabaseConfig() { - return Result.success(tableConfigBean); + return Result.success(tableConfig); } + /** + * 更新tablestore数据库配置 + */ @RequestMapping("/updateDatabaseConfig") - public Result updateDatabaseConfig(@RequestParam String secret, @Validated @RequestBody TableConfigBean updateConfig) { + public Result updateDatabaseConfig(@RequestBody TableConfigUpdate request) { + String secret = request.getSecret(); + TableConfig updateConfig = request.getConfig(); validSecret(secret); if (updateConfig == null) { throw new AppExcpetion(ErrorCode.CONFIG_ERROR, "配置文件不能为空"); } updateConfig.valid(); try { - tableConfigBean.writeConfig(updateConfig); + tableConfig.writeConfig(updateConfig); } catch (IOException e) { throw new AppExcpetion(ErrorCode.CONFIG_ERROR, e); } return Result.success(); } + /** + * 查询GFS下载配置 + */ @RequestMapping("/queryDataSourceConfig") public Result queryDataSourceConfig() { return Result.success(gfsDataConfig); } + /** + * 更新GFS下载配置 + */ @RequestMapping("/updateDataSourceConfig") - public Result updateDataSourceConfig(@RequestParam String secret, @Validated @RequestBody GfsDataConfig updateConfig) { + public Result updateDataSourceConfig(@Validated @RequestBody GfsConfigUpdate request) { + String secret = request.getSecret(); + GfsDataConfig updateConfig = request.getConfig(); validSecret(secret); if (updateConfig == null) { throw new AppExcpetion(ErrorCode.CONFIG_ERROR, "配置文件不能为空"); @@ -86,8 +112,12 @@ public class ConfigController { return Result.success(); } - @RequestMapping("/download") - public Result download(@RequestParam String secret) { + /** + * 下载当前时刻未来24小时的数据文件 + */ + @RequestMapping("/downloadAllFiles") + public Result downloadAllFiles(@RequestBody Map params) { + String secret = params.get("secret"); validSecret(secret); // List allCompleted = false; try { @@ -104,24 +134,120 @@ public class ConfigController { } else { return new Result(false, ErrorCode.DOWNLOAD_ERROR.getCode(), ErrorCode.DOWNLOAD_ERROR.getMsg() + ":下载未全部完成", data); } - } catch (Exception e) { + } catch (AppExcpetion e) { return Result.error(ErrorCode.DOWNLOAD_START_ERROR, e.getMessage()); } } + /** + * 下载指定时刻文件 + */ + @RequestMapping("/downloadSingleFile") + public Result downloadSingleFile(@RequestBody Map params) { + String secret = params.get("secret"); + validSecret(secret); + OffsetDateTime offsetDateTime = OffsetDateTime.parse(params.get("time"), DateTimeFormatter.ofPattern(Constant.API_TIME_STRING)); + String targetUtcStr = DateTimeUtils.getUTCDateTime(offsetDateTime).format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING)); + try { + gfsDownloader.iniTimeSetting(); + List fileInfoList = gfsDownloader.getFilesInfo(); // TODO 2024/6/12: 可以优化，不用生成24小时所有的文件信息 + for (FileInfo fileInfo : fileInfoList) { + if (fileInfo.getForecastUTCTimeStr().equals(targetUtcStr)) { + FileInfo downloadResult = gfsDownloader.download(fileInfo); + if (downloadResult.isDownloadSuccess()) { + return Result.success(downloadResult); + } else { + return Result.error(ErrorCode.DOWNLOAD_ERROR); + } + } + } + return Result.error(ErrorCode.QUERY_TIME_ERROR, ": 请选择当前时间之后24小时范围内的时刻"); + } catch (AppExcpetion e) { + return Result.error(ErrorCode.DOWNLOAD_START_ERROR, e.getMessage()); + } + } - @RequestMapping("/import") + /** + * 根据起报时间将数据导入数据库 + */ + @RequestMapping("/importToTablestoreByReferenceTime") public Result importData(@RequestBody Map params) { String secret = params.get("secret"); validSecret(secret); - OffsetDateTime time = OffsetDateTime.parse(params.get("time")); + OffsetDateTime time = OffsetDateTime.parse(params.get("time"), DateTimeFormatter.ofPattern(Constant.API_TIME_STRING)); try { - gfsDataImport.importData(time); - return Result.success(); + List finishedList = gfsDataImport.importData(time); + List successList = finishedList.stream().filter(result -> result.isSuccess()).collect(Collectors.toList()); + List failedList = finishedList.stream().filter(result -> !result.isSuccess()).collect(Collectors.toList()); + Map> data = new HashMap<>(); + data.put("successList", successList); + data.put("failedList", failedList); + if (CollectionUtils.isEmpty(failedList)) { + return Result.success(data); + } else { + return new Result(false, ErrorCode.TABLESTORE_IMPORT_ERROR.getCode(), ErrorCode.TABLESTORE_IMPORT_ERROR.getMsg() + ": 数据导入未全部完成", data); + } + } catch (AppExcpetion e) { + return Result.error(e.getErrorCode()); } catch (Exception e) { - e.printStackTrace(); return Result.error(e); } } + /** + * 查询指定起报时间文件夹中的所有文件名 + */ + @RequestMapping("/queryAllFileByReferenceTime") + public Result queryAllFileByReferenceTime(@RequestBody Map params) { + String secret = params.get("secret"); + validSecret(secret); + OffsetDateTime time = OffsetDateTime.parse(params.get("time"), DateTimeFormatter.ofPattern(Constant.API_TIME_STRING)); + String timeStr = DateTimeUtils.getUTCDateTime(time).format(DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING)); + File dataDir = new File(gfsDataConfig.getSaveRoot(), timeStr); + + // 查找目录下的文件 + List fileList = fileService.getFileList(dataDir); + return Result.success(fileList); + } + + /** + * 删除指定起报时间对应的文件夹 + */ + @RequestMapping("/deleteDirByReferenceTime") + public Result deleteDirByReferenceTime(@RequestBody Map params) { + String secret = params.get("secret"); + validSecret(secret); + OffsetDateTime time = OffsetDateTime.parse(params.get("time"), DateTimeFormatter.ofPattern(Constant.API_TIME_STRING)); + String timeStr = DateTimeUtils.getUTCDateTime(time).format(DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING)); + File dataDir = new File(gfsDataConfig.getSaveRoot(), timeStr); + if (gfsDataImport.isImporting()) { + return Result.error(ErrorCode.CAN_NOT_DELETE_WHEN_IMPORT_OR_DOWNLOAD); + } + try { + fileService.deleteDir(dataDir); + return Result.success(); + } catch (IOException e) { + return Result.error(e); + } + } + + /** + * 删除指定文件 + */ + @RequestMapping("/deleteTargetFile") + public Result deleteTargetFile(@RequestBody Map params) { + String secret = params.get("secret"); + validSecret(secret); + String filePath = params.get("filePath"); + File file = new File(filePath); + if (gfsDataImport.isImporting()) { + return Result.error(ErrorCode.CAN_NOT_DELETE_WHEN_IMPORT_OR_DOWNLOAD); + } + try { + fileService.deleteFile(file); + return Result.success(); + } catch (IOException e) { + return Result.error(e); + } + } } diff --git a/weather-service/src/main/java/com/htfp/weather/web/controller/ThymeleafTest.java b/weather-service/src/main/java/com/htfp/weather/web/controller/ThymeleafTest.java index 247ab8d..347bd32 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/controller/ThymeleafTest.java +++ b/weather-service/src/main/java/com/htfp/weather/web/controller/ThymeleafTest.java @@ -1,10 +1,9 @@ package com.htfp.weather.web.controller; -import com.htfp.weather.griddata.common.TableConfigBean; +import com.htfp.weather.griddata.common.TableConfig; import org.springframework.stereotype.Controller; import org.springframework.ui.Model; import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RestController; import javax.annotation.Resource; @@ -16,14 +15,14 @@ import javax.annotation.Resource; @Controller public class ThymeleafTest { @Resource - TableConfigBean tableConfigBean; + TableConfig tableConfig; @GetMapping("index")// 页面的url地址 public String getindex(Model model, String secret) { if (!"shiyi".equals(secret)){ return "wrong"; } - model.addAttribute("tableConfig", tableConfigBean); + model.addAttribute("tableConfig", tableConfig); return "index";// 与templates中index.html对应 } diff --git a/weather-service/src/main/java/com/htfp/weather/web/controller/UpperWeatherController.java b/weather-service/src/main/java/com/htfp/weather/web/controller/UpperWeatherController.java index 92e283f..0ed3599 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/controller/UpperWeatherController.java +++ b/weather-service/src/main/java/com/htfp/weather/web/controller/UpperWeatherController.java @@ -1,9 +1,9 @@ package com.htfp.weather.web.controller; import com.htfp.weather.utils.DateTimeUtils; -import com.htfp.weather.web.pojo.request.PlaneRequest; -import com.htfp.weather.web.pojo.request.Position3D; -import com.htfp.weather.web.pojo.request.ProfileRequest; +import com.htfp.weather.web.exception.AppExcpetion; +import com.htfp.weather.web.exception.ErrorCode; +import com.htfp.weather.web.pojo.request.*; import com.htfp.weather.web.pojo.response.*; import com.htfp.weather.web.service.GfsDataServiceImpl; import org.springframework.validation.annotation.Validated; @@ -11,6 +11,8 @@ import org.springframework.web.bind.annotation.*; import javax.annotation.Resource; import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; /** * @Author : shiyi @@ -28,6 +30,7 @@ public class UpperWeatherController { @RequestMapping("/profileByPressure") public Result queryProfile(@Validated @RequestBody ProfileRequest profileRequest) { OffsetDateTime time = OffsetDateTime.parse(profileRequest.getTime()); + OffsetDateTime utcDateTime = DateTimeUtils.getUTCDateTime(time); String variableName = profileRequest.getVariableName(); double latitude = profileRequest.getLatitude(); @@ -36,8 +39,8 @@ public class UpperWeatherController { return Result.success(profileResponse); } - @PostMapping("/forecast") - public Result queryTimeSeries(@Validated @RequestBody Position3D position3D) { + @PostMapping("/queryForecastTimeSeries") + public Result queryForecastTimeSeries(@Validated @RequestBody Position3D position3D) { double latitude = position3D.getLatitude(); double longitude = position3D.getLongitude(); int level = position3D.getLevel(); @@ -60,12 +63,50 @@ public class UpperWeatherController { return Result.success(forecastSeries); } - @PostMapping("/now") - public Result queryNowWeather(@Validated @RequestBody Position3D position3D) { + @PostMapping("/queryUpperNowWeather") + public Result queryUpperNowWeather(@Validated @RequestBody Position3D position3D) { double latitude = position3D.getLatitude(); double longitude = position3D.getLongitude(); int level = position3D.getLevel(); NowWeatherStatus nowWeather = gfsDataService.getNowWeather(latitude, longitude, level); return Result.success(nowWeather); } + + @PostMapping("/queryUpperNowWeatherInMultiPoints") + public Result queryUpperNowWeatherInMultiPoints(@Validated @RequestBody MultiPointsRequest multiPointsRequest) { + List latitudeList = new ArrayList<>(); + List longitudeList = new ArrayList<>(); + for (Position2D position : multiPointsRequest.getPointList()) { + Double latitude = position.getLatitude(); + Double longitude = position.getLongitude(); + if (latitude != null && longitude != null) { + latitudeList.add(latitude); + longitudeList.add(longitude); + } else { + throw new AppExcpetion(ErrorCode.MULTI_INDEX_ERROR); + } + } + int level = multiPointsRequest.getLevel().intValue(); + List nowWeatherByMultiPoint = gfsDataService.getNowWeatherByMultiPoint(latitudeList, longitudeList, level); + return Result.success(nowWeatherByMultiPoint); + } + + @PostMapping("/queryForecastTimeSeriesInMultiPoints") + public Result queryForecastTimeSeriesInMultiPoints(@Validated @RequestBody MultiPointsRequest multiPointsRequest) { + List latitudeList = new ArrayList<>(); + List longitudeList = new ArrayList<>(); + for (Position2D position : multiPointsRequest.getPointList()) { + Double latitude = position.getLatitude(); + Double longitude = position.getLongitude(); + if (latitude != null && longitude != null) { + latitudeList.add(latitude); + longitudeList.add(longitude); + } else { + throw new AppExcpetion(ErrorCode.MULTI_INDEX_ERROR); + } + } + int level = multiPointsRequest.getLevel().intValue(); + // List nowWeatherByMultiPoint = gfsDataService.getNowWeatherByMultiPoint(latitudeList, longitudeList, level); + return Result.success(); + } } diff --git a/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java b/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java index 7efa8e4..f7c8ef4 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java +++ b/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java @@ -8,11 +8,17 @@ package com.htfp.weather.web.exception; public enum ErrorCode { // + OTHER_ERROR(-1, "其他错误"), VALIDATE_ERROR(1001, "参数校验错误 "), CONFIG_ERROR(1002, "配置相关错误 "), SECRET_ERROR(1003, "无权限访问"), DOWNLOAD_START_ERROR(1004, "数据下载启动错误"), - DOWNLOAD_ERROR(1004, "数据下载错误"), + DOWNLOAD_ERROR(1005, "数据下载错误"), + TABLESTORE_IMPORT_ERROR(1006, "数据导入TableStore数据库错误"), + MULTI_INDEX_ERROR(1007, "多点索引不正确(null或不在正确范围内)"), + DATA_FILE_OR_DIR_NOT_EXISTS(1008, "目标文件或文件夹不存在"), + CAN_NOT_DELETE_WHEN_IMPORT_OR_DOWNLOAD(1009, "数据导入或下载中，无法进行删除操作"), + // 数据查询 HE_FENG_THIS_AREA_HAVE_NO_DATA(3001, "查询的数据或地区不存在"), HE_FENG_REQUEST_ERROR(3002, "查询请求错误"), CAI_YUN_REQUEST_ERROR(4002, "查询请求错误"), @@ -23,10 +29,12 @@ public enum ErrorCode { DATA_SET_EMPTY(6000, "TableStore数据库为空，无法查询"), QUERY_TIME_ERROR(6001, "目标时间不在当前数据范围内"), - LONGITUDE_INDEX_ERROR(6002, "经度坐标索引"), - LATITUDE_INDEX_ERROR(6003, "纬度坐标索引"), - LEVEL_INDEX_ERROR(6004, "高度坐标索引"), - + LONGITUDE_INDEX_ERROR(6002, "经度坐标索引错误"), + LATITUDE_INDEX_ERROR(6003, "纬度坐标索引错误"), + LEVEL_INDEX_ERROR(6004, "高度坐标索引错误"), + INDEX_SIZE_ERROR(6005, "坐标长度不一致"), + NO_SUCH_VARIABLE(6006, "找不到变量名映射"), + RESPONSE_DATA_BUILD_ERROR(6007, "查询数据结果构建错误"), ; private final int code; private final String msg; diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ConfigUpdate.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/GfsConfigUpdate.java similarity index 60% rename from weather-service/src/main/java/com/htfp/weather/web/pojo/request/ConfigUpdate.java rename to weather-service/src/main/java/com/htfp/weather/web/pojo/request/GfsConfigUpdate.java index 315fc08..0cac655 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ConfigUpdate.java +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/GfsConfigUpdate.java @@ -1,5 +1,6 @@ package com.htfp.weather.web.pojo.request; +import com.htfp.weather.download.GfsDataConfig; import lombok.Data; /** @@ -8,7 +9,7 @@ import lombok.Data; * @Description : */ @Data -public class ConfigUpdate { +public class GfsConfigUpdate { String secret; - + GfsDataConfig config; } diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/TableConfigUpdate.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/TableConfigUpdate.java new file mode 100644 index 0000000..34ff61d --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/TableConfigUpdate.java @@ -0,0 +1,20 @@ +package com.htfp.weather.web.pojo.request; + +import com.htfp.weather.griddata.common.TableConfig; +import lombok.Data; + +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Pattern; + +/** + * @Author : shiyi + * @Date : 2024/4/21 18:34 + * @Description : + */ +@Data +public class TableConfigUpdate { + @Pattern(regexp = "^htfpweather$", message = "无权限") + String secret; + @NotNull + TableConfig config; +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/response/FileInfo.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/FileInfo.java new file mode 100644 index 0000000..19d4c63 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/FileInfo.java @@ -0,0 +1,15 @@ +package com.htfp.weather.web.pojo.response; + +import lombok.Data; + +/** + * @Author : shiyi + * @Date : 2024/6/12 11:46 + * @Description : 文件信息 + */ +@Data +public class FileInfo { + private String fileName; + private String filePath; + private long fileSize; +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/response/MultiPointResponse.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/MultiPointResponse.java new file mode 100644 index 0000000..8d85b2a --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/MultiPointResponse.java @@ -0,0 +1,9 @@ +package com.htfp.weather.web.pojo.response; + +/** + * @Author : shiyi + * @Date : 2024/6/6 15:24 + * @Description : 航点气象数据响应 + */ +public class MultiPointResponse { +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/response/NowWeatherStatus.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/NowWeatherStatus.java index dc84448..99e96e0 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/pojo/response/NowWeatherStatus.java +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/NowWeatherStatus.java @@ -11,13 +11,15 @@ import lombok.Data; public class NowWeatherStatus { String time; // 数据时间，北京时 String weatherText; // 天气状况 - float temp; // 温度; // 温度 - float windSpeed; // 风速，m/s - float wind360; // 风向360角度 + Double latitude; + Double longitude; + Float temp; // 温度; // 温度 + Float windSpeed; // 风速，m/s + Float wind360; // 风向360角度 // String windDir; // 风向, 文字描述，如“西北风” // String windScale; // 风力等级 - float humidity; // 相对湿度 % - // float pressure; // 大气压强，默认单位：百帕 - float cloud; // 云量 - float precip; //降水量 + Float humidity; // 相对湿度 % + // Float pressure; // 大气压强，默认单位：百帕 + Float cloud; // 云量 + Float precip; //降水量 } diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/response/Result.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/Result.java index 63ef27c..620f5ad 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/pojo/response/Result.java +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/Result.java @@ -41,6 +41,6 @@ public class Result { } public static Result error(Throwable cause) { - return new Result(false, 1001, cause.getMessage(), null); + return new Result(false, null, cause.getMessage(), null); } } \ No newline at end of file diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/response/TimeSeriesDataset.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/TimeSeriesDataset.java index a79954a..f018919 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/pojo/response/TimeSeriesDataset.java +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/TimeSeriesDataset.java @@ -14,6 +14,8 @@ import java.util.List; @Data public class TimeSeriesDataset { List time; + Double latitude; + Double longitude; float[] temp; float[] windSpeed; float[] wind360; diff --git a/weather-service/src/main/java/com/htfp/weather/web/service/FileService.java b/weather-service/src/main/java/com/htfp/weather/web/service/FileService.java new file mode 100644 index 0000000..68e3bfe --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/service/FileService.java @@ -0,0 +1,64 @@ +package com.htfp.weather.web.service; + +import com.htfp.weather.web.exception.AppExcpetion; +import com.htfp.weather.web.exception.ErrorCode; +import com.htfp.weather.web.pojo.response.FileInfo; +import org.apache.commons.io.FileUtils; +import org.springframework.stereotype.Component; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * @Author : shiyi + * @Date : 2024/6/12 11:25 + * @Description : 文件系统管理 + */ +@Component +public class FileService { + + public List getFileList(File dataDir){ + if (!dataDir.exists() || !dataDir.isDirectory()) { + throw new AppExcpetion(ErrorCode.DATA_FILE_OR_DIR_NOT_EXISTS); + } + List fileInfoList = new ArrayList<>(); + File[] files = dataDir.listFiles(); + if (files == null || files.length == 0) { + return null; + } + for (File file : files) { + String fileName = file.getName(); + if (fileName.endsWith("grib2") || fileName.endsWith("nc")) { + FileInfo fileInfo = new FileInfo(); + fileInfo.setFilePath(file.getAbsolutePath()); + fileInfo.setFileName(fileName); + fileInfo.setFileSize(file.length()); + fileInfoList.add(fileInfo); + } + } + return fileInfoList; + } + public void uploadFile(String filePath){ + // 文件上传逻辑 + } + + public void downloadFile(String filePath){ + // 文件下载逻辑 + } + + public void deleteFile(File file) throws IOException { + if (file == null || !file.exists()) { + throw new AppExcpetion(ErrorCode.DATA_FILE_OR_DIR_NOT_EXISTS); + } + FileUtils.delete(file); + } + + public void deleteDir(File dir) throws IOException { + if (!dir.exists() || !dir.isDirectory()) { + throw new AppExcpetion(ErrorCode.DATA_FILE_OR_DIR_NOT_EXISTS); + } + FileUtils.deleteDirectory(dir); + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/service/GfsDataServiceImpl.java b/weather-service/src/main/java/com/htfp/weather/web/service/GfsDataServiceImpl.java index 953632d..834f5bb 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/service/GfsDataServiceImpl.java +++ b/weather-service/src/main/java/com/htfp/weather/web/service/GfsDataServiceImpl.java @@ -6,10 +6,11 @@ import com.aliyun.tablestore.grid.model.GridDataSetMeta; import com.aliyun.tablestore.grid.model.grid.Grid4D; import com.htfp.weather.griddata.common.GfsVariableHeightEnum; import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum; -import com.htfp.weather.griddata.common.TableConfigBean; +import com.htfp.weather.griddata.common.TableConfig; import com.htfp.weather.griddata.operation.GfsDataFetcher; import com.htfp.weather.griddata.operation.QueryMeta; import com.htfp.weather.info.Constant; +import com.htfp.weather.info.GfsLevelsEnum; import com.htfp.weather.utils.DateTimeUtils; import com.htfp.weather.utils.NdArrayUtils; import com.htfp.weather.web.exception.AppExcpetion; @@ -22,8 +23,10 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import ucar.ma2.Array; +import ucar.ma2.Index4D; import javax.annotation.Resource; +import java.lang.reflect.Method; import java.time.Duration; import java.time.Instant; import java.time.OffsetDateTime; @@ -37,7 +40,8 @@ import java.util.stream.Collectors; * @Date : 2024/2/2 15:47 * @Description : 获取tablestore格点数据 */ -@Service("tablestore-gfs") @Slf4j +@Service("tablestore-gfs") +@Slf4j public class GfsDataServiceImpl implements IDataService { private static final int SURFACE_LEVEL = 9999; @Resource @@ -45,9 +49,18 @@ public class GfsDataServiceImpl implements IDataService { @Resource QueryMeta queryMeta; @Resource - TableConfigBean tableConfigBean; - - + TableConfig tableConfig; + + + /** + * 获取变量随高度的变化廓线 + * + * @param targetTime 目标时间 UTC + * @param variableName 数据库中的变量名 + * @param latitude 纬度 + * @param longitude 经度 + * @return {@link ProfileResponse} + */ public ProfileResponse getProfile(OffsetDateTime targetTime, String variableName, double latitude, double longitude) { String targetVariable = GfsVariableIsobaricEnum.getGfsVariableName(variableName); @@ -66,7 +79,7 @@ public class GfsDataServiceImpl implements IDataService { int iLon = getLongitudeIndex(longitude); try { String gridDataSetId = lastGridDataSetMeta.getGridDataSetId(); - int[] pressureLevels = tableConfigBean.getPressureList(); + int[] pressureLevels = tableConfig.getPressureList(); Array array = gfsDataFetcher.getProfile(gridDataSetId, targetVariable, iTime, iLat, iLon); float[] values = (float[]) array.getStorage(); return new ProfileResponse(pressureLevels, values); @@ -76,6 +89,14 @@ public class GfsDataServiceImpl implements IDataService { } + /** + * 获取未来24小时的气象时间序列数据 + * + * @param latitude 纬度 + * @param longitude 经度 + * @param level 高度：9999代表地面，其他代表指定气压高度 + * @return @{@link TimeSeriesDataset} + */ public TimeSeriesDataset getForecastSeries(double latitude, double longitude, int level) { GridDataSetMeta lastGridDataSetMeta = null; try { @@ -95,6 +116,7 @@ public class GfsDataServiceImpl implements IDataService { try { String gridDataSetId = lastGridDataSetMeta.getGridDataSetId(); GridDataSet gridDataSet = gfsDataFetcher.getSeries(gridDataSetId, variableNames, iTimes, iLev, iLat, iLon); + if (level == SURFACE_LEVEL) { return buildTimeSeriesDatasetSurface(timeList, gridDataSet); } else { @@ -106,12 +128,20 @@ public class GfsDataServiceImpl implements IDataService { } } + /** + * 获取当前时刻的天气状况 + * + * @param latitude 纬度 + * @param longitude 经度 + * @param level 高度：9999代表地面，其他代表指定气压高度 + * @return @{@link NowWeatherStatus} + */ public NowWeatherStatus getNowWeather(double latitude, double longitude, int level) { GridDataSetMeta lastGridDataSetMeta = null; try { lastGridDataSetMeta = queryMeta.getLastGridDataSetMeta(); } catch (Exception e) { - throw new AppExcpetion(ErrorCode.DATA_SET_EMPTY, ": e.getMessage()"); + throw new AppExcpetion(ErrorCode.DATA_SET_EMPTY, ": " + e.getMessage()); } int iTime = getTargetTimeIndex(OffsetDateTime.now(), lastGridDataSetMeta); int iLat = getLatitudeIndex(latitude); @@ -122,38 +152,106 @@ public class GfsDataServiceImpl implements IDataService { try { String gridDataSetId = lastGridDataSetMeta.getGridDataSetId(); GridDataSet gridDataSet = gfsDataFetcher.getSinglePoint(gridDataSetId, variableNames, iTime, iLev, iLat, iLon); - if (level == SURFACE_LEVEL) { - return buildNowWeatherSurface(gridDataSet); - } else { - return buildNowWeatherPressure(gridDataSet); - } + // 每个变量都是0维数组 + final GfsLevelsEnum levelFlag = GfsLevelsEnum.getByCode(level); + NowWeatherStatus nowWeatherStatus = buildNowWeatherInTargetPoint(gridDataSet, 0, 0, 0, levelFlag); + nowWeatherStatus.setLatitude(latitude); + nowWeatherStatus.setLongitude(longitude); + return nowWeatherStatus; } catch (Exception e) { throw new RuntimeException(e); } } + /** + * 获得指定点序列上的当前气象信息 + * @param latitude + * @param longitude + * @param level + * @return + */ + public List getNowWeatherByMultiPoint(List latitude, List longitude, int level) { + GridDataSetMeta lastGridDataSetMeta = null; + try { + lastGridDataSetMeta = queryMeta.getLastGridDataSetMeta(); + } catch (Exception e) { + throw new AppExcpetion(ErrorCode.DATA_SET_EMPTY, ": " + e.getMessage()); + } + if (latitude.size() != longitude.size()) { + throw new AppExcpetion(ErrorCode.INDEX_SIZE_ERROR, ": 经纬度数组大小不一致"); + } + int pointSize = latitude.size(); + List variableNames = getVariableNamesInDatabase(level); + int iTime = getTargetTimeIndex(OffsetDateTime.now(), lastGridDataSetMeta); + List latIndex = latitude.stream().map(this::getLatitudeIndex).collect(Collectors.toList()); + List lonIndex = longitude.stream().map(this::getLongitudeIndex).collect(Collectors.toList()); + int iMinLat = Collections.min(latIndex).intValue(); + int iMaxLat = Collections.max(latIndex).intValue(); + int iMinLon = Collections.min(lonIndex).intValue(); + int iMaxLon = Collections.max(lonIndex).intValue(); + int iLev = getLevelIndex(level); + int latLength = iMaxLat - iMinLat + 1; + int lonLength = iMaxLon - iMinLon + 1; + // int levLength = iMaxLev - iMinLev + 1; + try { - private List getVariableNamesInDatabase(int level) { - List variableNames = new ArrayList<>(); - if (level == SURFACE_LEVEL) { - for (String e: tableConfigBean.getVariableList()){ - String gfsVariableName = GfsVariableHeightEnum.getGfsVariableName(e); - if (gfsVariableName != null){ - variableNames.add(gfsVariableName); - } - } - } else { - for (String e: tableConfigBean.getVariableList()){ - String gfsVariableName = GfsVariableIsobaricEnum.getGfsVariableName(e); - if (gfsVariableName != null){ - variableNames.add(gfsVariableName); - } + String gridDataSetId = lastGridDataSetMeta.getGridDataSetId(); + int[] origin = new int[]{iTime, iLev, 0, 0}; + int[] shape = new int[]{1, 1, tableConfig.getLatSize(), tableConfig.getLatSize()}; + GridDataSet gridDataSet = gfsDataFetcher.queryByTableStore(gridDataSetId, variableNames, + origin, shape); + + List nowWeatherStatusList = new ArrayList<>(); + + for (int i = 0; i < pointSize; i++) { + NowWeatherStatus nowWeatherStatus = new NowWeatherStatus(); + // NOTE 2024/6/11: build中传入的索引和数据库的查询索引意义不同 + + final GfsLevelsEnum levelFlag = GfsLevelsEnum.getByCode(level); + nowWeatherStatus = buildNowWeatherInTargetPoint(gridDataSet, 0, latIndex.get(i), lonIndex.get(i), levelFlag); + nowWeatherStatus.setLatitude(latitude.get(i)); + nowWeatherStatus.setLongitude(longitude.get(i)); + nowWeatherStatusList.add(nowWeatherStatus); } + return nowWeatherStatusList; + + } catch (Exception e) { + throw new RuntimeException(e); } - return variableNames; } + + private int[] getMarginIndexByMultiPoint(List latitude, List longitude, List level) { + int minLat = Integer.MIN_VALUE; + int maxLat = Integer.MAX_VALUE; + int minLon = Integer.MIN_VALUE; + int maxLon = Integer.MAX_VALUE; + int minLev = Integer.MIN_VALUE; + int maxLev = Integer.MAX_VALUE; + + + int iMinLat = getLatitudeIndex(minLat); + int iMaxLat = getLatitudeIndex(maxLat); + int iMinLon = getLongitudeIndex(minLon); + int iMaxLon = getLongitudeIndex(maxLon); + + return new int[]{iMinLat, iMaxLat, iMinLon, iMaxLon}; + } + + + /** + * 获取指定变量、时间、高度的二维数组数据 + * + * @param targetTime 目标时间 UTC + * @param variableName 数据库中的变量名 + * @param level 高度 + * @param minLat 最小纬度 + * @param maxLat 最大纬度 + * @param minLon 最小经度 + * @param maxLon 最大经度 + * @return @{@link PlaneResponse} + */ public PlaneResponse getPlane(OffsetDateTime targetTime, String variableName, int level, double minLat, double maxLat, double minLon, double maxLon) { int iLev; String targetVariable; @@ -194,11 +292,32 @@ public class GfsDataServiceImpl implements IDataService { } + + private List getVariableNamesInDatabase(int level) { + List variableNames = new ArrayList<>(); + if (level == SURFACE_LEVEL) { + for (String e : tableConfig.getVariableList()) { + String gfsVariableName = GfsVariableHeightEnum.getGfsVariableName(e); + if (gfsVariableName != null) { + variableNames.add(gfsVariableName); + } + } + } else { + for (String e : tableConfig.getVariableList()) { + String gfsVariableName = GfsVariableIsobaricEnum.getGfsVariableName(e); + if (gfsVariableName != null) { + variableNames.add(gfsVariableName); + } + } + } + return variableNames; + } + private PlaneResponse buildPlane(Array array, int iMinLat, int iMinLon, int latLength, int lonLength) { PlaneResponse planeResponse = new PlaneResponse(); float[][] values = (float[][]) array.reduce().copyToNDJavaArray(); - planeResponse.setLatList(Arrays.copyOfRange(tableConfigBean.getLatList(), iMinLat, iMinLat + latLength)); - planeResponse.setLonList(Arrays.copyOfRange(tableConfigBean.getLonList(), iMinLon, iMinLon + lonLength)); + planeResponse.setLatList(Arrays.copyOfRange(tableConfig.getLatList(), iMinLat, iMinLat + latLength)); + planeResponse.setLonList(Arrays.copyOfRange(tableConfig.getLonList(), iMinLon, iMinLon + lonLength)); planeResponse.setValues(values); return planeResponse; } @@ -245,49 +364,47 @@ public class GfsDataServiceImpl implements IDataService { return timeSeriesDataset; } - private NowWeatherStatus buildNowWeatherPressure(GridDataSet gridDataSet) { - String timeStr = OffsetDateTime.now().withMinute(0).withNano(0).format(DateTimeFormatter.ofPattern(Constant.API_TIME_STRING)); - NowWeatherStatus nowWeatherStatus = new NowWeatherStatus(); - nowWeatherStatus.setTime(timeStr); - Grid4D grid4D; - grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.TEMP)); - if (grid4D != null) { - nowWeatherStatus.setTemp(((float[]) grid4D.toArray().getStorage())[0]); - } - grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.WIND_SPEED)); - if (grid4D != null) { - nowWeatherStatus.setWindSpeed(((float[]) grid4D.toArray().getStorage())[0]); - } - grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.WIND360)); - if (grid4D != null) { - nowWeatherStatus.setWind360(((float[]) grid4D.toArray().getStorage())[0]); - } - grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.CLOUD)); - if (grid4D != null) { - nowWeatherStatus.setCloud(((float[]) grid4D.toArray().getStorage())[0]); - } - return nowWeatherStatus; - } - private NowWeatherStatus buildNowWeatherSurface(GridDataSet gridDataSet) { + /** + * 构建指定位置的当前天气 + * @param gridDataSet + * @param iLevInGrid-目标点在返回的网格中的高度索引 + * @param iLatInGrid-目标点在返回的网格中的高度索引 + * @param iLonInGrid-目标点在返回的网格中的高度索引 + * @return + */ + private NowWeatherStatus buildNowWeatherInTargetPoint(GridDataSet gridDataSet, int iLevInGrid, int iLatInGrid, int iLonInGrid, GfsLevelsEnum levelFlag) { String timeStr = OffsetDateTime.now().withMinute(0).withNano(0).format(DateTimeFormatter.ofPattern(Constant.API_TIME_STRING)); - NowWeatherStatus nowWeatherStatus = new NowWeatherStatus(); - nowWeatherStatus.setTime(timeStr); - Grid4D grid4D; - grid4D = gridDataSet.getVariable(GfsVariableHeightEnum.getGfsVariableName(GfsVariableHeightEnum.TEMP)); - if (grid4D != null) { - nowWeatherStatus.setTemp(((float[]) grid4D.toArray().getStorage())[0]); - } - grid4D = gridDataSet.getVariable(GfsVariableHeightEnum.getGfsVariableName(GfsVariableHeightEnum.WIND_SPEED)); - if (grid4D != null) { - nowWeatherStatus.setWindSpeed(((float[]) grid4D.toArray().getStorage())[0]); - } - grid4D = gridDataSet.getVariable(GfsVariableHeightEnum.getGfsVariableName(GfsVariableHeightEnum.WIND360)); - if (grid4D != null) { - nowWeatherStatus.setWind360(((float[]) grid4D.toArray().getStorage())[0]); + NowWeatherStatus result = new NowWeatherStatus(); + result.setTime(timeStr); + // 利用反射构建数据集 + Class aClass = result.getClass(); + for (Map.Entry entry : gridDataSet.getVariables().entrySet()) { + String variableNameInFile = entry.getKey(); + String variableNameInApi; + if (GfsLevelsEnum.SURFACE.equals(levelFlag)) { + variableNameInApi = GfsVariableHeightEnum.getVariableNameInApi(variableNameInFile); + } else { + variableNameInApi = GfsVariableIsobaricEnum.getVariableNameInApi(variableNameInFile); + } + if (variableNameInApi == null) { + throw new AppExcpetion(ErrorCode.NO_SUCH_VARIABLE); + } + Grid4D grid4D = entry.getValue(); Index4D index = new Index4D(grid4D.getShape()); + try { + String setMethodName = "set" + variableNameInApi.substring(0, 1).toUpperCase() + variableNameInApi.substring(1); + Method method = aClass.getDeclaredMethod(setMethodName, aClass.getDeclaredField(variableNameInApi).getType()); + method.invoke(result, grid4D.toArray().getFloat(index.set(0, iLevInGrid, iLatInGrid, iLonInGrid))); + } catch (ReflectiveOperationException e) { + throw new AppExcpetion(ErrorCode.NO_SUCH_VARIABLE, e); + } catch (Exception e) { + e.printStackTrace(); + throw new AppExcpetion(ErrorCode.RESPONSE_DATA_BUILD_ERROR); + } } - return nowWeatherStatus; + return result; } + private int getTargetTimeIndex(OffsetDateTime targetTime, GridDataSetMeta lastGridDataSetMeta) { long milli = (long) lastGridDataSetMeta.getAttributes().get(AttributionEnum.REFERENCE_TIME.getName()); OffsetDateTime refTime = OffsetDateTime.ofInstant(Instant.ofEpochMilli(milli), ZoneOffset.ofHours(0)); @@ -336,8 +453,9 @@ public class GfsDataServiceImpl implements IDataService { map.put("timeList", timeList); return map; } + private int getLongitudeIndex(double lon) { - double[] lonList = tableConfigBean.getLonList(); + double[] lonList = tableConfig.getLonList(); if (lonList.length == 0) { throw new AppExcpetion(ErrorCode.LONGITUDE_INDEX_ERROR, "经度列表为空， 无法查找索引"); } @@ -354,7 +472,7 @@ public class GfsDataServiceImpl implements IDataService { } private int getLatitudeIndex(double lat) { - double[] latList = tableConfigBean.getLatList(); + double[] latList = tableConfig.getLatList(); if (latList.length == 0) { throw new AppExcpetion(ErrorCode.LATITUDE_INDEX_ERROR, "纬度列表为空， 无法查找索引"); } @@ -372,7 +490,7 @@ public class GfsDataServiceImpl implements IDataService { } private int getPressureIndex(int pressureLevel) { - List levList = Arrays.stream(tableConfigBean.getPressureList()).boxed().collect(Collectors.toList()); + List levList = Arrays.stream(tableConfig.getPressureList()).boxed().collect(Collectors.toList()); if (CollectionUtils.isEmpty(levList)) { throw new AppExcpetion(ErrorCode.LEVEL_INDEX_ERROR, "气压列表为空， 无法查找索引"); } diff --git a/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImpl.java b/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImpl.java index c4cdf66..da00d7a 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImpl.java +++ b/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImpl.java @@ -107,7 +107,11 @@ public class CaiYunServiceImpl implements ISurfaceDataService { HashMap params = new HashMap<>(); params.put("alert", "true"); CaiYunWarningResponse response = caiYunRequest(url, params, CaiYunWarningResponse.class); - return response.getResult().getAlert().getContent(); + CaiYunWarningResponse.AlertResult.Alert alert = response.getResult().getAlert(); + if (alert == null) { + throw new AppExcpetion(ErrorCode.CAI_YUN_REQUEST_ERROR, "：无可用预警数据"); + } + return alert.getContent(); } private NowWeatherStatus buildNowSurfaceWeatherStatus(CaiYunNowResponse nowResponse) { diff --git a/weather-service/src/main/resources/config/gfsDataConfig.json b/weather-service/src/main/resources/config/gfsDataConfig.json index d36675d..964d18b 100644 --- a/weather-service/src/main/resources/config/gfsDataConfig.json +++ b/weather-service/src/main/resources/config/gfsDataConfig.json @@ -5,7 +5,7 @@ "minLat" : 0.0, "maxLat" : 55.0, "resolution" : 0.25, - "variables" : [ "DZDT", "RH", "TMP", "UGRD", "VGRD", "TCDC" ,"GUST"], - "pressureLevels" : [400, 500, 600, 700, 750, 800, 850, 925, 975, 1000], - "saveRoot" : "GFSData" + "variables" : [ "DZDT", "RH", "TMP", "UGRD", "VGRD", "TCDC" ,"GUST", "PRATE"], + "pressureLevels" : [400, 450, 500, 550, 600, 650, 700, 750, 800, 850, 900, 925, 950, 975, 1000], + "saveRoot" : "./GFSData" } \ No newline at end of file diff --git a/weather-service/src/test/java/com/htfp/weather/download/GfsDownloaderTest.java b/weather-service/src/test/java/com/htfp/weather/download/GfsDownloaderTest.java index 5e38a89..a1e0b2b 100644 --- a/weather-service/src/test/java/com/htfp/weather/download/GfsDownloaderTest.java +++ b/weather-service/src/test/java/com/htfp/weather/download/GfsDownloaderTest.java @@ -36,10 +36,8 @@ class GfsDownloaderTest { void downloadAll() { gfsDownloader.iniTimeSetting(); List fileInfoList = gfsDownloader.getFilesInfo(); - boolean isComplete = gfsDownloader.downloadAll(fileInfoList); - if (isComplete) { - System.out.println("下载完成"); - } + gfsDownloader.downloadAll(fileInfoList); + } @Test diff --git a/weather-service/src/test/java/com/htfp/weather/griddata/utils/GfsUtilsTest.java b/weather-service/src/test/java/com/htfp/weather/griddata/utils/GfsUtilsTest.java index 34540a2..a40da91 100644 --- a/weather-service/src/test/java/com/htfp/weather/griddata/utils/GfsUtilsTest.java +++ b/weather-service/src/test/java/com/htfp/weather/griddata/utils/GfsUtilsTest.java @@ -19,9 +19,12 @@ class GfsUtilsTest { @Test void getRefTime() throws IOException { - NetcdfFile ncFile = NetcdfFiles.open("C:/Users/shi_y/Desktop/GFSData/UTC-20230905/BJT-20230905-0900.grib2"); + String filename = "D:\\HTFP\\weather\\GFSData\\UTC-20240607.00\\BJT-20240607.17-from-UTC-20240607.00+9.grib2"; + NetcdfFile ncFile = NetcdfFiles.open(filename); String refTime = GfsUtils.getRefTime(ncFile); - assertEquals("2023-09-05T00:00+00:00", refTime); - Assertions.assertEquals("2023-09-05T08:00+08:00", DateTimeUtils.getBJTDateTimeStringFromUTC(refTime, CoordinateUtils.DATE_TIME_PATTERN)); + String s = filename.split("\\+")[1]; + Integer.parseInt(s.split(".grib2")[0]); + // assertEquals("2023-09-05T00:00+00:00", refTime); + // Assertions.assertEquals("2023-09-05T08:00+08:00", DateTimeUtils.getBJTDateTimeStringFromUTC(refTime, CoordinateUtils.DATE_TIME_PATTERN)); } } \ No newline at end of file diff --git a/weather-service/src/test/java/com/htfp/weather/schedule/GridDataProcesserTest.java b/weather-service/src/test/java/com/htfp/weather/schedule/GridDataProcesserTest.java index 6cb8c50..5c79fbd 100644 --- a/weather-service/src/test/java/com/htfp/weather/schedule/GridDataProcesserTest.java +++ b/weather-service/src/test/java/com/htfp/weather/schedule/GridDataProcesserTest.java @@ -5,8 +5,6 @@ import org.springframework.boot.test.context.SpringBootTest; import javax.annotation.Resource; -import static org.junit.jupiter.api.Assertions.*; - /** * @Author : shiyi * @Date : 2024/5/17 11:44 @@ -15,9 +13,9 @@ import static org.junit.jupiter.api.Assertions.*; @SpringBootTest class GridDataProcesserTest { @Resource - GridDataProcesser gridDataProcesser; + GridDataProcessor gridDataProcessor; @Test void dailyDataProcess() { - gridDataProcesser.clearExpiredData(); + gridDataProcessor.clearExpiredData(); } } \ No newline at end of file diff --git a/weather-service/src/test/java/com/htfp/weather/web/service/FileServiceTest.java b/weather-service/src/test/java/com/htfp/weather/web/service/FileServiceTest.java new file mode 100644 index 0000000..8ff3a22 --- /dev/null +++ b/weather-service/src/test/java/com/htfp/weather/web/service/FileServiceTest.java @@ -0,0 +1,24 @@ +package com.htfp.weather.web.service; + +import com.htfp.weather.web.pojo.response.FileInfo; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @Author : shiyi + * @Date : 2024/6/12 11:52 + * @Description : + */ +class FileServiceTest { + FileService fileService = new FileService(); + @Test + void getFileList() { + File dir = new File("D:\\HTFP\\weather\\weather-service\\GFSData\\UTC-20240611.00"); + List fileList = fileService.getFileList(dir); + System.out.println(fileList); + } +} \ No newline at end of file diff --git a/weather-service/tableConf.json b/weather-service/tableConf.json new file mode 100644 index 0000000..4a090bb --- /dev/null +++ b/weather-service/tableConf.json @@ -0,0 +1,12 @@ +{ + "dataTableName" : "gfs_data_table", + "metaTableName" : "gfs_meta_table", + "dataIndexName" : "gfs_data_index", + "metaIndexName" : "gfs_meta_table_index", + "dataDir" : "./GFSData", + "variableList" : [ "temp", "cloud", "wind360", "windSpeed"], + "lonSize" : 281, + "latSize" : 221, + "levSize" : 10, + "timeToLive" : 2 +} \ No newline at end of file